From 98c643a1a2a65bb89e57577b338f01b5e12798e3 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Tue, 25 Oct 2022 15:58:08 -0400 Subject: [PATCH 01/54] Adding the incremental entity provider backend Signed-off-by: Damon Kaswell --- .changeset/loud-rockets-reply.md | 5 + packages/backend/package.json | 1 + .../incremental-ingestion-backend/README.md | 321 +++++++++++ .../incremental-ingestion-backend/knexfile.js | 26 + .../migrations/20220613125155_init.js | 90 +++ .../20220906161750_add-ingestion-indexes.js | 53 ++ .../package.json | 47 ++ .../IncrementalIngestionDatabaseManager.ts | 540 ++++++++++++++++++ .../src/database/migrations.ts | 28 + .../src/engine/IncrementalIngestionEngine.ts | 218 +++++++ .../src/index.ts | 18 + .../src/routes.ts | 202 +++++++ .../src/service/IncrementalCatalogBuilder.ts | 127 ++++ .../src/types.ts | 221 +++++++ yarn.lock | 20 + 15 files changed, 1917 insertions(+) create mode 100644 .changeset/loud-rockets-reply.md create mode 100644 plugins/incremental-ingestion-backend/README.md create mode 100644 plugins/incremental-ingestion-backend/knexfile.js create mode 100644 plugins/incremental-ingestion-backend/migrations/20220613125155_init.js create mode 100644 plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js create mode 100644 plugins/incremental-ingestion-backend/package.json create mode 100644 plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts create mode 100644 plugins/incremental-ingestion-backend/src/database/migrations.ts create mode 100644 plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts create mode 100644 plugins/incremental-ingestion-backend/src/index.ts create mode 100644 plugins/incremental-ingestion-backend/src/routes.ts create mode 100644 plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts create mode 100644 plugins/incremental-ingestion-backend/src/types.ts diff --git a/.changeset/loud-rockets-reply.md b/.changeset/loud-rockets-reply.md new file mode 100644 index 0000000000..1262e439b4 --- /dev/null +++ b/.changeset/loud-rockets-reply.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-incremental-ingestion-backend': patch +--- + +This change introduces incremental entity providers, which are used for streaming very large data sources into the catalog. diff --git a/packages/backend/package.json b/packages/backend/package.json index e7d2d5d7b7..1a5d971f93 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -44,6 +44,7 @@ "@backstage/plugin-events-backend": "workspace:^", "@backstage/plugin-events-node": "workspace:^", "@backstage/plugin-graphql-backend": "workspace:^", + "@backstage/plugin-incremental-ingestion-backend": "^0.1.0", "@backstage/plugin-jenkins-backend": "workspace:^", "@backstage/plugin-kafka-backend": "workspace:^", "@backstage/plugin-kubernetes-backend": "workspace:^", diff --git a/plugins/incremental-ingestion-backend/README.md b/plugins/incremental-ingestion-backend/README.md new file mode 100644 index 0000000000..2571b178aa --- /dev/null +++ b/plugins/incremental-ingestion-backend/README.md @@ -0,0 +1,321 @@ +# backend-incremental-ingestion + +The Incremental Ingestion Backend plugin provides an Incremental Entity Provider that can be used to ingest data from sources using delta mutations, while retaining the orphan prevention mechanism provided by full mutations. + +## Why did we create it? + +Backstage provides an [Entity Provider mechanism that has two kinds of mutations](https://backstage.io/docs/features/software-catalog/external-integrations#provider-mutations): `delta` and `full`. `delta` mutations tell Backstage Catalog which entities should be added and removed from the catalog. `full` mutation accepts a list of entities and automatically computes which entities must be removed by comparing the provided entities against existing entities to create a diff between the two sets. These two kinds of mutations are convenient for different kinds of data sources. A `delta` mutation can be used with a data source that emits UPDATE and DELETE events for its data. A `full` mutation is useful for APIs that produce fewer entities than can fit in Backstage processes' memory. + +Unfortunately, these two kinds of mutations are insufficient for very large data sources for the following reasons, + +1. Even when the API provides DELETE events, we still need a way to create the initial list of entities. For example, if you ingest all repositories from GitHub into Backstage and you use webhooks, you still need the initial list of entities. +2. A `delta` mutation can not guarantee that mutations will not be missed. For example, if your Backstage portal is down while receiving a DELETE event, you might miss the event which leaves your catalog in an unclear state. How can you replay the missed events? Some data sources, like GitHub, provide an API for replaying missed events, but this increases complexity and is not available on all APIs. +3. Addressing the above two use case with `full` mutation is not an option on very large datasets because a `full` mutation requires that all entities are in memory to create a diff. If your data source has 100k+ records, this can easily cause your processes to run out of memory. +4. In cases when you can use `full` mutation, committing many entities into the processing pipeline fills up the processing queue and delays the processing of entities from other entity providers. + +We created the Incremental Entity Provider to address all of the above issues. The Incremental Entity Provider addresses these issues with a combination of `delta` mutations and a mark-and-sweep mechanism. Instead of doing a single `full` mutation, it performs a series of bursts. At the end of each burst, the Incremental Entity Provider performs the following three operations, + +1. Marks each received entity in the database. +2. Annotates each entity with `backstage/incremental-entity-provider: ` annotation. +3. Commits all of the entities with a `delta` mutation. + +Incremental Entity Providers will wait a configurable interval before proceeding to the next burst. + +Once the source has no more results, Incremental Entity Provider compares all entities annotated with `frontside/incremental-entity-provider: ` against all marked entities to determine which entities commited by same entity provider were not marked during the last ingestion cycle. All unmarked entities are deleted at the end of the cycle. The Incremental Entity Provider rests for a fixed internal before restarting the ingestion process. + +![Diagram of execution of an Incremental Entity Provider](https://user-images.githubusercontent.com/74687/185822734-ee6279c7-64fa-46b9-9aa8-d4092ab73858.png) + +This approach has the following benefits, + +1. Reduced ingestion latency - each burst commits entities which are processed before the entire list is processed. +2. Stable pressure - each period between bursts provides an opportunity for the processing pipeline to settle without overwhelming the pipeline with a large number of unprocessed entities. +3. Built-in retry/backoff - Failed bursts are automatically retried with a built-in backoff interval providing an opportunity for the data source to reset its rate limits before retrying the burst. +4. Prevents orphan entities - Deleted entities are removed as with `full` mutation with a low memory footprint. + +## Requirements + +The Incremental Entity Provider backend is designed for data sources that provide paginated results. Each burst attempts to handle one or more pages of the query. The plugin will attempt to fetch as many pages as it can within a configurable burst length. At every iteration, it expects to receive the next cursor that will be used to query in the next iteration. Each iteration may happen on a different replica. This has several concequences: + +1. The cursor must be serializable to JSON (not an issue for most RESTful or GraphQL based APIs). +2. The client must be stateless - a client is created from scratch for each iteration to allow distributing processing over multiple replicas. + +## Installation + +1. Install `@backstage/plugin-incremental-ingestion-backend` with `yarn add @backstage/plugin-incremental-ingestion-backend` +2. Import `IncrementalCatalogBuilder` from `@backstage/plugin-incremental-ingestion-backend` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`. + ```ts + const builder = CatalogBuilder.create(env); + // incremental builder receives builder because it'll register + // incremental entity providers with the builder + const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); + ``` +3. Last step, add `await incrementBuilder.build()` after `await builder.build()` to ensure that all `CatalogBuider` migration run before running `incrementBuilder.build()` migrations. + ```ts + const { processingEngine, router } = await builder.build(); + + // this has to run after `await builder.build()` to ensure that catalog migrations are completed + // before incremental builder migrations are executed + await incrementalBuilder.build(); + ``` + +The result should look something like this, + +```ts +import { + CatalogBuilder +} from '@backstage/plugin-catalog-backend'; +import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; +import { IncrementalCatalogBuilder } from '@frontside/backstage-plugin-incremental-ingestion-backend'; +import { GithubRepositoryEntityProvider } from '@frontside/backstage-plugin-incremental-ingestion-github'; +import { Router } from 'express'; +import { Duration } from 'luxon'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin( + env: PluginEnvironment, +): Promise { + + const builder = CatalogBuilder.create(env); + // incremental builder receives builder because it'll register + // incremental entity providers with the builder + const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); + + builder.addProcessor(new ScaffolderEntitiesProcessor()); + + const { processingEngine, router } = await builder.build(); + + // this has to run after `await builder.build()` so ensure that catalog migrations are completed + // before incremental builder migrations are executed + await incrementalBuilder.build(); + // If you want to use a suite of administrative routes to control the incremental providers, use + // `const { incrementalAdminRouter } = await incrementalBuilder.build();` + + await processingEngine.start(); + + return router; +} +``` + +## Writing an Incremental Entity Provider + +To create an Incremental Entity Provider, you need to know how to retrieve a single page of the data that you wish to ingest into the Backstage catalog. If the API has pagination and you know how to make a paginated request to that API, you'll be able to implement an Incremental Entity Provider for this API. For more information about compatibility, checkout Compatible data sources section on this page. + +Here is the type definition for an Incremental Entity Provider. + +```ts +interface IncrementalEntityProvider { + /** + * This name must be unique between all of the entity providers + * operating in the catalog. + */ + getProviderName(): string; + + /** + * Do any setup and teardown necessary in order to provide the + * context for fetching pages. This should always invoke `burst` in + * order to fetch the individual pages. + * + * @param burst - a function which performs a series of iterations + */ + around(burst: (context: TContext) => Promise): Promise; + + /** + * Return a single page of entities from a specific point in the + * ingestion. + * + * @param context - anything needed in order to fetch a single page. + * @param cursor - a uniqiue value identifying the page to ingest. + * @returns the entities to be ingested, as well as the cursor of + * the the next page after this one. + */ + next(context: TContext, cursor?: TCursor): Promise>; +} +``` + +For tutorial, we'll write an Incremental Entity Provider that will call an imaginary API. This imaginary API will return a list of imaginary services. This imaginary API has an imaginary API client with the following interface. + +```ts +interface MyApiClient { + getServices(page: number): MyPaginatedResults +} + +interface MyPaginatedResults { + items: T[]; + totalPages: number; +} + +interface Service { + name: string; +} +``` + +These are the only 3 methods that you need to implement. `getProviderName()` is pretty self explanatory and it's exactly same as on Entity Provider. + +```ts +import { IncrementalEntityProvider, EntityIteratorResult } from '@backstage/plugin-incremental-ingestion-backend'; + +// this will include your pagination information, let's say our API accepts a `page` parameter. +// In this case, the cursor will include `page` +interface MyApiCursor { + page: number; +} + +// This interface describes the type of data that will be passed to your burst function. +interface MyContext { + apiClient: MyApiClient +} + +export class MyIncrementalEntityProvider implements IncrementalEntityProvider { + getProviderName() { + return `MyIncrementalEntityProvider`; + } +} +``` + +`around` method is used for setup and teardown. For example, if you need to create a client that will connect to the API, you would do that here. + +```ts +export class MyIncrementalEntityProvider implements IncrementalEntityProvider { + getProviderName() { + return `MyIncrementalEntityProvider`; + } + + async around(burst: (context: MyContext) => Promise): Promise { + + const apiClient = new MyApiClient(); + + await burst({ apiClient }); + + // if you need to do any teardown, you can do it here + } +} +``` + +If you need to pass a token to your API, then you can create a constructor that will receive a token and use the token to setup th client. + +```ts +export class MyIncrementalEntityProvider implements IncrementalEntityProvider { + + token: string; + + construtor(token: string) { + this.token = token; + } + + getProviderName() { + return `MyIncrementalEntityProvider`; + } + + + async around(burst: (context: MyContext) => Promise): Promise { + + const apiClient = new MyApiClient(this.token) + + await burst({ apiClient }) + } +} +``` + +The last step is to implement the actual `next` method that will accept the cursor, call the API, process the result and return the result. + +```ts +export class MyIncrementalEntityProvider implements IncrementalEntityProvider { + + token: string; + + construtor(token: string) { + this.token = token; + } + + getProviderName() { + return `MyIncrementalEntityProvider`; + } + + + async around(burst: (context: MyContext) => Promise): Promise { + + const apiClient = new MyApiClient(this.token) + + await burst({ apiClient }) + } + + async next(context: MyContext, cursor?: MyApiCursor = { page: 1 }): Promise> { + const { apiClient } = context; + const { page } = cursor; + + // call your API with the current page + const data = await apiClient.getServices(page); + + // calculate the next page + const nextPage = page + 1; + + // figure out if there are any more pages to fetch + const done = nextPage > data.totalPages; + + // convert returned items into entities + const entities = data.items.map(item => ({ + entity: { + apiVersion: 'backstage.io/v1beta1', + kind: 'Component', + metadata: { + name: item.name, + annotations: { + // You need to define these, otherwise they'll fail validation + [ANNOTATION_LOCATION]: this.getProviderName(), + [ANNOTATION_ORIGIN_LOCATION]: this.getProviderName(), + } + } + spec: { + type: 'service' + } + } + })); + + // create the next cursor + const nextCursor = { + page: nextPage + }; + + return { + done, + entities, + cursor: nextCursor + } + } +} +``` + +Now that you have your new Incremental Entity Provider, we can connect it to the catalog. + +## Adding an Incremental Entity Provider to the catalog + +We'll assume you followed the Installation instructions. After you create your `incrementalBuilder`, you can instantiate your Entity Provider and pass it to the `addIncrementalEntityProvider` method. + +```ts + const incrementalBuilder = IncrementalCatalogBuilder.create(env, builder); + + // I'm assuming you're going to get your token from config + const token = config.getString('myApiClient.token'); + + const myEntityProvider = new MyIncrementalEntityProvider(token) + + incrementalBuilder.addIncrementalEntityProvider( + myEntityProvider, + { + // how long should it attempt to read pages from the API + // keep this short. Incremental Entity Provider will attempt to + // read as many pages as it can in this time + burstLength: Duration.fromObject({ seconds: 3 }), + // how long should it wait between bursts? + burstInterval: Duration.fromObject({ seconds: 3 }), + // how long should it rest before re-ingesting again? + restLength: Duration.fromObject({ day: 1 }) + } + ) +``` + +That's it!!! + +## Error handling + +If `around` or `next` methods throw an error, the error will show up in logs and it'll trigger the Incremental Entity Provider to try again after a backoff period. It'll keep trying until it reaches the last backoff attempt. You don't need to do anything special to handle the retry logic. diff --git a/plugins/incremental-ingestion-backend/knexfile.js b/plugins/incremental-ingestion-backend/knexfile.js new file mode 100644 index 0000000000..4cf9ef77c3 --- /dev/null +++ b/plugins/incremental-ingestion-backend/knexfile.js @@ -0,0 +1,26 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file makes it possible to run "yarn knex migrate:make some_file_name" +// to assist in making new migrations +module.exports = { + client: 'better-sqlite3', + connection: ':memory:', + useNullAsDefault: true, + migrations: { + directory: './migrations', + }, +}; diff --git a/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js b/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js new file mode 100644 index 0000000000..3e94f93d15 --- /dev/null +++ b/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js @@ -0,0 +1,90 @@ +exports.up = async function (knex) { + await knex.raw(` +CREATE VIEW ingestion.current_entities as SELECT + FORMAT('%s:%s/%s', + LOWER(final_entity::json #>> '{kind}'), + LOWER(final_entity::json #>> '{metadata, namespace}'), + LOWER(final_entity::json #>> '{metadata, name}')) as ref, + final_entity::json #>> '{metadata, annotations, hp.com/provider-name}' as provider_name, + final_entity FROM public.final_entities; +`); + + const schema = () => knex.schema.withSchema('ingestion'); + + await schema().createTable('ingestions', table => { + table.comment('Tracks ingestion streams for very large data sets'); + + table.uuid('id', { primary: true }).notNullable().comment('Auto-generated ID of the ingestion'); + + table.string('provider_name').notNullable().comment('each provider gets its own identifiable name'); + + table + .string('status') + .notNullable() + .comment('One of "interstitial" | "bursting" | "backing off" | "resting" | "complete"'); + + table.string('next_action').notNullable().comment("what will this, 'ingest', 'rest', 'backoff', 'nothing (done)'"); + + table + .timestamp('next_action_at') + .defaultTo(knex.fn.now()) + .comment('the moment in time at which point ingestion can begin again'); + + table.string('last_error').comment('records any error that occured in the previous burst attempt'); + + table.integer('attempts').defaultTo(0).comment('how many attempts have been made to burst without success'); + + table.timestamp('created_at').defaultTo(knex.fn.now()).comment('when did this ingestion actually begin'); + + table.timestamp('ingestion_completed_at').comment('when did the ingestion actually end'); + + table.timestamp('rest_completed_at').comment('when did the rest period actually end'); + }); + + await schema().createTable('ingestion_marks', table => { + table.comment('tracks each step of an iterative ingestion'); + + table.uuid('id', { primary: true }).notNullable().comment('Auto-generated ID of the ingestion mark'); + + table.uuid('ingestion_id').notNullable().comment('The id of the ingestion in which this mark took place'); + + // table + // .foreign('ingestion_id').references('ingestions.id'); + + table + .json('cursor') + .comment('the current data associated with this iteration wherever it is in this moment in time'); + + table.integer('sequence').defaultTo(0).comment('what is the order of this mark'); + + table.timestamp('created_at').defaultTo(knex.fn.now()); + }); + + await schema().createTable('ingestion_mark_entities', table => { + table.comment('tracks the entities recorded in each step of an iterative ingestion'); + + table.uuid('id', { primary: true }).notNullable().comment('Auto-generated ID of the marked entity'); + + table + .uuid('ingestion_mark_id') + .notNullable() + .comment('Every time a mark happens during an ingestion, there are a list of entities marked.'); + + // table + // .foreign('ingestion_mark_id').references('ingestion_marks.id'); + + table.string('ref').notNullable().comment('the entity reference of the marked entity'); + }); +}; + +exports.down = async function (knex) { + const schema = () => knex.schema.withSchema('ingestion'); + + await schema().dropView('current_entities'); + + await schema().dropTable('ingestion_mark_entities'); + + await schema().dropTable('ingestion_marks'); + + await schema().dropTable('ingestions'); +}; diff --git a/plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js b/plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js new file mode 100644 index 0000000000..38282723e4 --- /dev/null +++ b/plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js @@ -0,0 +1,53 @@ +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = async function (knex) { + const schema = () => knex.schema.withSchema('ingestion'); + + await knex.raw( + `CREATE INDEX IF NOT EXISTS increment_ingestion_provider_name_idx ON public.final_entities ((final_entity::json #>> '{metadata, annotations, backstage.io/incremental-provider-name}'));`, + ); + + await knex.raw(`DROP VIEW IF EXISTS ingestion.current_entities`); + + await schema().alterTable('ingestions', t => { + t.primary('id'); + t.index('provider_name', 'ingestion_provider_name_idx'); + }); + + await schema().alterTable('ingestion_marks', t => { + t.primary('id'); + t.index('ingestion_id', 'ingestion_mark_ingestion_id_idx'); + }); + + await schema().alterTable('ingestion_mark_entities', t => { + t.primary('id'); + t.index('ingestion_mark_id', 'ingestion_mark_entity_ingestion_mark_id_idx'); + }); +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function (knex) { + const schema = () => knex.schema.withSchema('ingestion'); + + await schema().alterTable('ingestions', t => { + t.dropIndex('provider_name', 'ingestion_provider_name_idx'); + t.dropPrimary('id'); + }); + + await schema().alterTable('ingestion_marks', t => { + t.dropIndex('ingestion_id', 'ingestion_mark_ingestion_id_idx'); + t.dropPrimary('id'); + }); + + await schema().alterTable('ingestions_mark_entities', t => { + t.dropIndex('ingestion_mark_id', 'ingestion_mark_entity_ingestion_mark_id_idx'); + t.dropPrimary('id'); + }); + + await knex.raw(`DROP INDEX increment_ingestion_provider_name_idx;`); +}; diff --git a/plugins/incremental-ingestion-backend/package.json b/plugins/incremental-ingestion-backend/package.json new file mode 100644 index 0000000000..2d55704a48 --- /dev/null +++ b/plugins/incremental-ingestion-backend/package.json @@ -0,0 +1,47 @@ +{ + "name": "@backstage/plugin-incremental-ingestion-backend", + "version": "0.1.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": true, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "backend-plugin" + }, + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" + }, + "dependencies": { + "@backstage/backend-common": "^0.14.0", + "@backstage/backend-tasks": "^0.3.6", + "@backstage/catalog-model": "^1.1.2", + "@backstage/config": "^1.0.0", + "@backstage/plugin-catalog-backend": "^1.5.0", + "@backstage/plugin-permission-common": "^0.7.0", + "express": "^4.17.1", + "express-promise-router": "^4.1.0", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/cli": "^0.20.0", + "@types/express": "*", + "@types/supertest": "^2.0.8", + "msw": "^0.35.0", + "supertest": "^4.0.2" + }, + "files": [ + "dist", + "migrations/**/*.{js,d.ts}" + ] +} diff --git a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts new file mode 100644 index 0000000000..b736693df8 --- /dev/null +++ b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts @@ -0,0 +1,540 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Knex } from 'knex'; +import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; +import { stringifyEntityRef } from '@backstage/catalog-model'; +import { Duration } from 'luxon'; +import { v4 } from 'uuid'; +import { + INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, + IngestionRecord, + IngestionRecordInsert, + IngestionRecordUpdate, + IngestionUpsertIFace, + MarkRecord, + MarkRecordInsert, +} from '../types'; + +export class IncrementalIngestionDatabaseManager { + private client: Knex; + + constructor(options: { client: Knex }) { + this.client = options.client; + } + + /** + * Performs an update to the ingestion record with matching `id`. + * @param options IngestionRecordUpdate + */ + async updateIngestionRecordById(options: IngestionRecordUpdate) { + await this.client.transaction(async tx => { + const { ingestionId, update } = options; + await tx('ingestion.ingestions').where('id', ingestionId).update(update); + }); + } + + /** + * Performs an update to the ingestion record with matching provider name. Will only update active records. + * @param provider string + * @param update Partial + */ + async updateIngestionRecordByProvider(provider: string, update: Partial) { + await this.client.transaction(async tx => { + await tx('ingestion.ingestions') + .where('provider_name', provider) + .andWhere('rest_completed_at', null) + .update(update); + }); + } + + /** + * Performs an insert into the `ingestion.ingestions` table with the supplied values. + * @param options IngestionRecordInsert + */ + async insertIngestionRecord(options: IngestionRecordInsert) { + await this.client.transaction(async tx => { + const { record } = options; + await tx('ingestion.ingestions').insert(record); + }); + } + + private async deleteMarkEntities(tx: Knex.Transaction, ids: { id: string }[]) { + const chunks: { id: string }[][] = []; + for (let i = 0; i < ids.length; i += 100) { + const chunk = ids.slice(i, i + 100); + chunks.push(chunk); + } + + let deleted = 0; + + for (const chunk of chunks) { + const chunkDeleted = await tx('ingestion.ingestion_mark_entities') + .delete() + .whereIn( + 'id', + chunk.map(entry => entry.id), + ); + deleted += chunkDeleted; + } + + return deleted; + } + + /** + * Finds the current ingestion record for the named provider. + * @param provider string + * @returns IngestionRecord | undefined + */ + async getCurrentIngestionRecord(provider: string) { + return await this.client.transaction(async tx => { + const record = await tx('ingestion.ingestions') + .where('provider_name', provider) + .andWhere('rest_completed_at', null) + .first(); + return record; + }); + } + + /** + * Removes all entries from `ingestion_marks_entities`, `ingestion_marks`, and `ingestions` + * for prior ingestions that completed (i.e., have a value for `rest_completed_at`). + * @param provider string + * @returns A count of deletions for each record type. + */ + async clearFinishedIngestions(provider: string) { + return await this.client.transaction(async tx => { + const markEntitiesDeleted = await tx('ingestion.ingestion_mark_entities') + .delete() + .whereIn( + 'ingestion_mark_id', + tx('ingestion.ingestion_marks') + .select('id') + .whereIn( + 'ingestion_id', + tx('ingestion.ingestions') + .select('id') + .where('provider_name', provider) + .andWhereNot('rest_completed_at', null), + ), + ); + + const marksDeleted = await tx('ingestion.ingestion_marks') + .delete() + .whereIn( + 'ingestion_id', + tx('ingestion.ingestions') + .select('id') + .where('provider_name', provider) + .andWhereNot('rest_completed_at', null), + ); + + const ingestionsDeleted = await tx('ingestion.ingestions') + .delete() + .where('provider_name', provider) + .andWhereNot('rest_completed_at', null); + + return { + deletions: { + markEntitiesDeleted, + marksDeleted, + ingestionsDeleted, + }, + }; + }); + } + + /** + * Automatically cleans up duplicate ingestion records if they were accidentally created. + * Any ingestion record where the `rest_completed_at` is null (meaning it is active) AND + * the ingestionId is incorrect is a duplicate ingestion record. + * @param ingestionId string + * @param provider string + */ + async clearDuplicateIngestions(ingestionId: string, provider: string) { + await this.client.transaction(async tx => { + const invalid = await tx('ingestion.ingestions') + .where('provider_name', provider) + .andWhere('rest_completed_at', null) + .andWhereNot('id', ingestionId); + + if (invalid.length > 0) { + await tx('ingestion.ingestions').delete().whereIn('id', invalid); + await tx('ingestion.ingestion_mark_entities') + .delete() + .whereIn('ingestion_mark_id', tx('ingestion.ingestion_marks').select('id').whereIn('ingestion_id', invalid)); + await tx('ingestion.ingestion_marks').delete().whereIn('ingestion_id', invalid); + } + }); + } + + /** + * This method fully purges and resets all ingestion records for the named provider, and + * leaves it in a paused state. + * @param provider string + * @returns Counts of all deleted ingestion records + */ + async purgeAndResetProvider(provider: string) { + return await this.client.transaction(async tx => { + const ingestionIDs: { id: string }[] = await tx('ingestion.ingestions') + .select('id') + .where('provider_name', provider); + + const markIDs: { id: string }[] = + ingestionIDs.length > 0 + ? await tx('ingestion.ingestion_marks') + .select('id') + .whereIn( + 'ingestion_id', + ingestionIDs.map(entry => entry.id), + ) + : []; + + const markEntityIDs: { id: string }[] = + markIDs.length > 0 + ? await tx('ingestion.ingestion_mark_entities') + .select('id') + .whereIn( + 'ingestion_mark_id', + markIDs.map(entry => entry.id), + ) + : []; + + const markEntitiesDeleted = await this.deleteMarkEntities(tx, markEntityIDs); + + const marksDeleted = + markIDs.length > 0 + ? await tx('ingestion.ingestion_marks') + .delete() + .whereIn( + 'ingestion_id', + ingestionIDs.map(entry => entry.id), + ) + : 0; + + const ingestionsDeleted = await tx('ingestion.ingestions').delete().where('provider_name', provider); + + const next_action_at = new Date(); + next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000); + + await this.insertRecord({ + record: { + id: v4(), + next_action: 'rest', + provider_name: provider, + next_action_at, + ingestion_completed_at: new Date(), + status: 'resting', + }, + }); + + return { provider, ingestionsDeleted, marksDeleted, markEntitiesDeleted }; + }); + } + + /** + * Creates a new ingestion record. + * @param provider string + * @returns A new ingestion record + */ + async createProviderIngestionRecord(provider: string) { + const ingestionId = v4(); + const nextAction = 'ingest'; + await this.insertIngestionRecord({ + record: { + id: ingestionId, + next_action: nextAction, + provider_name: provider, + status: 'bursting', + }, + }); + return { ingestionId, nextAction, attempts: 0, nextActionAt: Date.now() }; + } + + /** + * Computes which entities to remove, if any, at the end of a burst. + * @param provider string + * @param ingestionId string + * @returns All entities to remove for this burst. + */ + async computeRemoved(provider: string, ingestionId: string) { + return await this.client.transaction(async tx => { + const removed: { entity: string; ref: string }[] = await tx('final_entities') + .select(tx.ref('final_entity').as('entity'), tx.ref('refresh_state.entity_ref').as('ref')) + .join('refresh_state', 'refresh_state.entity_id', 'final_entities.entity_id') + .whereRaw( + `((final_entity::json #>> '{metadata, annotations, ${INCREMENTAL_ENTITY_PROVIDER_ANNOTATION}}')) = ?`, + [provider], + ) + .whereNotIn( + 'entity_ref', + tx('ingestion.ingestion_marks') + .join( + 'ingestion.ingestion_mark_entities', + 'ingestion.ingestion_marks.id', + 'ingestion.ingestion_mark_entities.ingestion_mark_id', + ) + .select('ingestion.ingestion_mark_entities.ref') + .where('ingestion.ingestion_marks.ingestion_id', ingestionId), + ); + return removed.map(entity => { + return { entity: JSON.parse(entity.entity) }; + }); + }); + } + + /** + * Performs a lookup of all providers that have duplicate active ingestion records. + * @returns An array of all duplicate active ingestions + */ + async healthcheck() { + return await this.client.transaction(async tx => { + const records = await tx<{ id: string; provider_name: string }>('ingestion.ingestions') + .distinct('id', 'provider_name') + .where('rest_completed_at', null); + return records; + }); + } + + /** + * Skips any wait time for the next action to run. + * @param provider string + */ + async triggerNextProviderAction(provider: string) { + await this.updateIngestionRecordByProvider(provider, { next_action_at: new Date() }); + } + + /** + * Purges the following tables: + * * `ingestion.ingestions` + * * `ingestion.ingestion_marks` + * * `ingestion.ingestion_mark_entities` + * + * This function leaves the ingestions table with all providers in a paused state. + * @returns Results from cleaning up all ingestion tables. + */ + async cleanupProviders() { + const providers = await this.listProviders(); + + const ingestionsDeleted = await this.purgeTable('ingestion.ingestions'); + + const next_action_at = new Date(); + next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000); + + for (const provider of providers) { + await this.insertIngestionRecord({ + record: { + id: v4(), + next_action: 'rest', + provider_name: provider, + next_action_at, + ingestion_completed_at: new Date(), + status: 'resting', + }, + }); + } + + const ingestionMarksDeleted = await this.purgeTable('ingestion.ingestion_marks'); + const markEntitiesDeleted = await this.purgeTable('ingestion.ingestion_mark_entities'); + + return { ingestionsDeleted, ingestionMarksDeleted, markEntitiesDeleted }; + } + + /** + * Configures the current ingestion record to ingest a burst. + * @param ingestionId string + */ + async setProviderIngesting(ingestionId: string) { + await this.updateIngestionRecordById({ ingestionId, update: { next_action: 'ingest' } }); + } + + /** + * Indicates the provider is currently ingesting a burst. + * @param ingestionId string + */ + async setProviderBursting(ingestionId: string) { + await this.updateIngestionRecordById({ ingestionId, update: { status: 'bursting' } }); + } + + /** + * Finalizes the current ingestion record to indicate that the post-ingestion rest period is complete. + * @param ingestionId string + */ + async setProviderComplete(ingestionId: string) { + await this.updateIngestionRecordById({ + ingestionId, + update: { + next_action: 'nothing (done)', + rest_completed_at: new Date(), + status: 'complete', + }, + }); + } + + /** + * Marks ingestion as complete and starts the post-ingestion rest cycle. + * @param ingestionId string + * @param restLength Duration + */ + async setProviderResting(ingestionId: string, restLength: Duration) { + await this.updateIngestionRecordById({ + ingestionId, + update: { + next_action: 'rest', + next_action_at: new Date(Date.now() + restLength.as('milliseconds')), + ingestion_completed_at: new Date(), + status: 'resting', + }, + }); + } + + /** + * Marks ingestion as paused after a burst completes. + * @param ingestionId string + */ + async setProviderInterstitial(ingestionId: string) { + await this.updateIngestionRecordById({ ingestionId, update: { attempts: 0, status: 'interstitial' } }); + } + + /** + * Starts the cancel process for the current ingestion. + * @param ingestionId string + * @param message string (optional) + */ + async setProviderCanceling(ingestionId: string, message?: string) { + const update: Partial = { + next_action: 'cancel', + last_error: message ? message : undefined, + next_action_at: new Date(), + status: 'canceling', + }; + await this.updateIngestionRecordById({ ingestionId, update }); + } + + /** + * Completes the cancel process and triggers a new ingestion. + * @param ingestionId string + */ + async setProviderCanceled(ingestionId: string) { + await this.updateIngestionRecordById({ + ingestionId, + update: { + next_action: 'nothing (canceled)', + rest_completed_at: new Date(), + status: 'complete', + }, + }); + } + + /** + * Configures the current ingestion to wait and retry, due to a data source error. + * @param ingestionId string + * @param attempts number + * @param error Error + * @param backoffLength number + */ + async setProviderBackoff(ingestionId: string, attempts: number, error: Error, backoffLength: number) { + await this.updateIngestionRecordById({ + ingestionId, + update: { + next_action: 'backoff', + attempts: attempts + 1, + last_error: String(error), + next_action_at: new Date(Date.now() + backoffLength), + status: 'backing off', + }, + }); + } + + /** + * Returns the last record from `ingestion.ingestion_marks` for the supplied ingestionId. + * @param ingestionId string + * @returns MarkRecord | undefined + */ + async getLastMark(ingestionId: string) { + return await this.client.transaction(async tx => { + const mark = await tx('ingestion.ingestion_marks') + .where('ingestion_id', ingestionId) + .orderBy('sequence', 'desc') + .first(); + return mark; + }); + } + + async getAllMarks(ingestionId: string) { + return await this.client.transaction(async tx => { + const marks = await tx('ingestion.ingestion_marks') + .where('ingestion_id', ingestionId) + .orderBy('sequence', 'desc'); + return marks; + }); + } + + /** + * Performs an insert into the `ingestion.ingestion_marks` table with the supplied values. + * @param options MarkRecordInsert + */ + async createMark(options: MarkRecordInsert) { + const { record } = options; + await this.client.transaction(async tx => { + await tx('ingestion.ingestion_marks').insert(record); + }); + } + /** + * Performs an upsert to the `ingestion.ingestion_mark_entities` table for all deferred entities. + * @param markId string + * @param entities DeferredEntity[] + */ + async createMarkEntities(markId: string, entities: DeferredEntity[]) { + await this.client.transaction(async tx => { + await tx('ingestion.ingestion_mark_entities').insert( + entities.map(entity => ({ + id: v4(), + ingestion_mark_id: markId, + ref: stringifyEntityRef(entity.entity), + })), + ); + }); + } + + /** + * Deletes the entire content of a table, and returns the number of records deleted. + * @param table string + * @returns number + */ + async purgeTable(table: string) { + return await this.client.transaction(async tx => { + return await tx(table).delete(); + }); + } + + /** + * Returns a list of all providers. + * @returns string[] + */ + async listProviders() { + return await this.client.transaction(async tx => { + const providers = await tx<{ provider_name: string }>('ingestion.ingestions').distinct('provider_name'); + return providers.map(entry => entry.provider_name); + }); + } + + async updateByName(provider: string, update: Partial) { + await this.updateIngestionRecordByProvider(provider, update); + } + async insertRecord(options: IngestionRecordInsert) { + await this.insertIngestionRecord(options); + } +} diff --git a/plugins/incremental-ingestion-backend/src/database/migrations.ts b/plugins/incremental-ingestion-backend/src/database/migrations.ts new file mode 100644 index 0000000000..8ec5d3a7e6 --- /dev/null +++ b/plugins/incremental-ingestion-backend/src/database/migrations.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { resolvePackagePath } from '@backstage/backend-common'; +import { Knex } from 'knex'; + +export async function applyDatabaseMigrations(knex: Knex): Promise { + const migrationsDir = resolvePackagePath('@devex/backend-incremental-ingestion', 'migrations'); + + await knex.raw('CREATE SCHEMA IF NOT EXISTS ingestion;'); + + await knex.migrate.latest({ + schemaName: 'ingestion', + directory: migrationsDir, + }); +} diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts new file mode 100644 index 0000000000..6182ae95c4 --- /dev/null +++ b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts @@ -0,0 +1,218 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; +import { INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, IterationEngine, IterationEngineOptions } from '../types'; +import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; +import type { AbortSignal } from 'node-abort-controller'; + +import { performance } from 'perf_hooks'; +import { Duration, DurationObjectUnits } from 'luxon'; +import { v4 } from 'uuid'; + +export class IncrementalIngestionEngine implements IterationEngine { + restLength: Duration; + backoff: DurationObjectUnits[]; + + private manager: IncrementalIngestionDatabaseManager; + + constructor(private options: IterationEngineOptions) { + this.manager = options.manager; + this.restLength = Duration.fromObject(options.restLength); + this.backoff = options.backoff ?? [{ minutes: 1 }, { minutes: 5 }, { minutes: 30 }, { hours: 3 }]; + } + + async taskFn(signal: AbortSignal) { + try { + this.options.logger.debug('Begin tick'); + await this.handleNextAction(signal); + } catch (error) { + this.options.logger.error(`${error}`); + throw error; + } finally { + this.options.logger.debug('End tick'); + } + } + + async handleNextAction(signal: AbortSignal) { + await this.options.ready; + + const { ingestionId, nextActionAt, nextAction, attempts } = await this.getCurrentAction(); + + switch (nextAction) { + case 'rest': + if (Date.now() > nextActionAt) { + await this.manager.clearFinishedIngestions(this.options.provider.getProviderName()); + this.options.logger.info( + `incremental-engine: Ingestion ${ingestionId} rest period complete. Ingestion will start again`, + ); + + await this.manager.setProviderComplete(ingestionId); + } else { + this.options.logger.info(`incremental-engine: Ingestion '${ingestionId}' rest period continuing`); + } + break; + case 'ingest': + try { + await this.manager.setProviderBursting(ingestionId); + const done = await this.ingestOneBurst(ingestionId, signal); + if (done) { + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' complete, transitioning to rest period of ${this.restLength.toHuman()}`, + ); + await this.manager.setProviderResting(ingestionId, this.restLength); + } else { + await this.manager.setProviderInterstitial(ingestionId); + this.options.logger.info(`incremental-engine: Ingestion '${ingestionId}' continuing`); + } + } catch (error) { + if ((error as Error).message && (error as Error).message === 'CANCEL') { + this.options.logger.info(`incremental-engine: Ingestion '${ingestionId}' canceled`); + await this.manager.setProviderCanceling(ingestionId, (error as Error).message); + } else { + const currentBackoff = Duration.fromObject(this.backoff[Math.min(this.backoff.length - 1, attempts)]); + + const backoffLength = currentBackoff.as('milliseconds'); + this.options.logger.error(error); + + const truncatedError = (error as string).substring(0, 700); + this.options.logger.error( + `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`, + ); + + await this.manager.setProviderBackoff(ingestionId, attempts, error as Error, backoffLength); + } + } + break; + case 'backoff': + if (Date.now() > nextActionAt) { + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' backoff complete, will attempt to resume`, + ); + await this.manager.setProviderIngesting(ingestionId); + } else { + this.options.logger.info(`incremental-engine: Ingestion '${ingestionId}' backoff continuing`); + } + break; + case 'cancel': + this.options.logger.info(`incremental-engine: Ingestion '${ingestionId}' canceling, will restart`); + await this.manager.setProviderCanceled(ingestionId); + break; + default: + this.options.logger.error( + `incremental-engine: Ingestion '${ingestionId}' received unknown action '${nextAction}'`, + ); + } + } + + async getCurrentAction() { + const providerName = this.options.provider.getProviderName(); + const record = await this.manager.getCurrentIngestionRecord(providerName); + if (record) { + this.options.logger.info(`incremental-engine: Ingestion record found: '${record.id}'`); + return { + ingestionId: record.id, + nextAction: record.next_action as 'rest' | 'ingest' | 'backoff', + attempts: record.attempts as number, + nextActionAt: record.next_action_at.valueOf() as number, + }; + } else { + const result = await this.manager.createProviderIngestionRecord(providerName); + this.options.logger.info(`incremental-engine: Ingestion record created: '${result.ingestionId}'`); + return result; + } + } + + async ingestOneBurst(id: string, signal: AbortSignal) { + const lastMark = await this.manager.getLastMark(id); + + const cursor = lastMark ? lastMark.cursor : void 0; + let sequence = lastMark ? lastMark.sequence + 1 : 0; + + const start = performance.now(); + let count = 0; + let done = false; + this.options.logger.info(`incremental-engine: Ingestion '${id}' burst initiated`); + + await this.options.provider.around(async (context: unknown) => { + let next = await this.options.provider.next(context, cursor); + count++; + while (true) { + done = next.done; + await this.mark(id, sequence, next.entities, next.done, next.cursor); + if (signal.aborted || next.done) { + break; + } else { + next = await this.options.provider.next(context, next.cursor); + count++; + sequence++; + } + } + }); + + this.options.logger.info( + `incremental-engine: Ingestion '${id}' burst complete. (${count} batches in ${Math.round( + performance.now() - start, + )}ms).`, + ); + return done; + } + + async mark(id: string, sequence: number, entities: DeferredEntity[], done: boolean, cursor?: unknown) { + this.options.logger.debug( + `incremental-engine: Ingestion '${id}': MARK ${entities.length} entities, cursor: ${JSON.stringify( + cursor, + )}, done: ${done}`, + ); + const markId = v4(); + + await this.manager.createMark({ + record: { + id: markId, + ingestion_id: id, + cursor, + sequence, + }, + }); + + if (entities.length > 0) { + await this.manager.createMarkEntities(markId, entities); + } + + const added = entities.map(deferred => ({ + ...deferred, + entity: { + ...deferred.entity, + metadata: { + ...deferred.entity.metadata, + annotations: { + ...deferred.entity.metadata.annotations, + [INCREMENTAL_ENTITY_PROVIDER_ANNOTATION]: this.options.provider.getProviderName(), + }, + }, + }, + })); + + const removed: DeferredEntity[] = done + ? [] + : await this.manager.computeRemoved(this.options.provider.getProviderName(), id); + + await this.options.connection.applyMutation({ + type: 'delta', + added, + removed, + }); + } +} diff --git a/plugins/incremental-ingestion-backend/src/index.ts b/plugins/incremental-ingestion-backend/src/index.ts new file mode 100644 index 0000000000..46afb24a25 --- /dev/null +++ b/plugins/incremental-ingestion-backend/src/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export * from './service/IncrementalCatalogBuilder'; +export * from './types'; +export * from './database/IncrementalIngestionDatabaseManager'; diff --git a/plugins/incremental-ingestion-backend/src/routes.ts b/plugins/incremental-ingestion-backend/src/routes.ts new file mode 100644 index 0000000000..8e2f3ae8af --- /dev/null +++ b/plugins/incremental-ingestion-backend/src/routes.ts @@ -0,0 +1,202 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { errorHandler } from '@backstage/backend-common'; +import express from 'express'; +import Router from 'express-promise-router'; +import { Logger } from 'winston'; +import { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager'; + +export const createIncrementalProviderRouter = async (manager: IncrementalIngestionDatabaseManager, logger: Logger) => { + const router = Router(); + router.use(express.json()); + + // Get the overall health of all incremental providers + router.get('/health', async (_, res) => { + const records = await manager.healthcheck(); + const providers = records.map(record => record.provider_name); + const duplicates = [...new Set(providers.filter((e, i, a) => a.indexOf(e) !== i))]; + + if (duplicates.length > 0) { + res.json({ healthy: false, duplicateIngestions: duplicates }); + } else { + res.json({ healthy: true }); + } + }); + + // Clean up and pause all providers + router.delete('/cleanup', async (_, res) => { + const result = await manager.cleanupProviders(); + res.json(result); + }); + + // Get basic status of the provider + router.get('/:provider', async (req, res) => { + const { provider } = req.params; + const record = await manager.getCurrentIngestionRecord(provider); + if (record) { + res.json({ + success: true, + status: { + current_action: record.status, + next_action_at: new Date(record.next_action_at), + }, + last_error: record.last_error, + }); + } else { + const providers: string[] = await manager.listProviders(); + if (providers.includes(provider)) { + res.json({ + success: true, + status: { + current_action: 'rest complete, waiting to start', + }, + }); + } else { + logger.error(`${provider} - No ingestion record found in the database!`); + res.status(404).json({ + success: false, + status: {}, + last_error: `Provider '${provider}' not found`, + }); + } + } + }); + + // Trigger the provider's next action + router.put('/:provider', async (req, res) => { + const { provider } = req.params; + const record = await manager.getCurrentIngestionRecord(provider); + if (record) { + await manager.triggerNextProviderAction(provider); + res.json({ success: true, message: `${provider}: Next action triggered.` }); + } else { + const providers: string[] = await manager.listProviders(); + if (providers.includes(provider)) { + logger.debug(`${provider} - Ingestion record found`); + res.json({ + success: true, + message: 'Unable to trigger next action (provider is restarting)', + }); + } else { + res.status(404).json({ success: false, message: `Provider '${provider}' not found` }); + } + } + }); + + // Start a brand-new ingestion cycle for the provider + router.post('/:provider', async (req, res) => { + const { provider } = req.params; + + const record = await manager.getCurrentIngestionRecord(provider); + if (record) { + await manager.updateByName(provider, { + next_action: 'nothing (done)', + ingestion_completed_at: new Date(), + rest_completed_at: new Date(), + status: 'complete', + }); + res.json({ success: true, message: `${provider}: Next cycle triggered.` }); + } else { + const providers: string[] = await manager.listProviders(); + if (providers.includes(provider)) { + logger.debug(`${provider} - Ingestion record found`); + res.json({ + success: true, + message: 'Provider is already restarting', + }); + } else { + res.status(404).json({ success: false, message: `Provider '${provider}' not found` }); + } + } + }); + + // Stop the provider and pause it for 24 hours + router.post('/:provider/cancel', async (req, res) => { + const { provider } = req.params; + const record = await manager.getCurrentIngestionRecord(provider); + if (record) { + const next_action_at = new Date(); + next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000); + await manager.updateByName(provider, { + next_action: 'nothing (done)', + ingestion_completed_at: new Date(), + next_action_at, + status: 'resting', + }); + res.json({ success: true, message: `${provider}: Current ingestion canceled.` }); + } else { + const providers: string[] = await manager.listProviders(); + if (providers.includes(provider)) { + logger.debug(`${provider} - Ingestion record found`); + res.json({ + success: true, + message: 'Provider is currently restarting, please wait.', + }); + } else { + res.status(404).json({ success: false, message: `Provider '${provider}' not found` }); + } + } + }); + + // Wipe out all ingestion records for the provider and pause for 24 hours + router.delete('/:provider', async (req, res) => { + const { provider } = req.params; + const result = await manager.purgeAndResetProvider(provider); + res.json(result); + }); + + // Get the ingestion marks for the current cycle + router.get('/:provider/marks', async (req, res) => { + const { provider } = req.params; + const record = await manager.getCurrentIngestionRecord(provider); + if (record) { + const id = record.id; + const records = await manager.getAllMarks(id); + res.json({ success: true, records }); + } else { + const providers: string[] = await manager.listProviders(); + if (providers.includes(provider)) { + logger.debug(`${provider} - Ingestion record found`); + res.json({ + success: true, + message: 'No records yet (provider is restarting)', + }); + } else { + logger.error(`${provider} - No ingestion record found in the database!`); + res.status(404).json({ + success: false, + status: {}, + last_error: `Provider '${provider}' not found`, + }); + } + } + }); + + router.delete('/:provider/marks', async (req, res) => { + const { provider } = req.params; + const deletions = await manager.clearFinishedIngestions(provider); + + res.json({ + success: true, + message: `Expired marks for provider '${provider}' removed.`, + deletions, + }); + }); + + router.use(errorHandler()); + + return router; +}; diff --git a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts new file mode 100644 index 0000000000..93c7c571d8 --- /dev/null +++ b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts @@ -0,0 +1,127 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { IncrementalEntityProvider, IncrementalEntityProviderOptions, PluginEnvironment } from '../types'; +import { CatalogBuilder as CoreCatalogBuilder } from '@backstage/plugin-catalog-backend'; +import { Duration } from 'luxon'; +import { Knex } from 'knex'; +import { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine'; +import { applyDatabaseMigrations } from '../database/migrations'; +import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; +import { createIncrementalProviderRouter } from '../routes'; + +export class Deferred implements Promise { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + resolve: (value: T) => void; + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + reject: (error: Error) => void; + + then: Promise['then']; + catch: Promise['catch']; + finally: Promise['finally']; + + constructor() { + const promise = new Promise((resolve, reject) => { + this.resolve = resolve; + this.reject = reject; + }); + + this.then = promise.then.bind(promise); + this.catch = promise.catch.bind(promise); + this.finally = promise.finally.bind(promise); + } + + [Symbol.toStringTag]: 'Deferred' = 'Deferred'; +} + +export class IncrementalCatalogBuilder { + /** + * Creates the incremental catalog builder, which extends the regular catalog builder. + * @param env PluginEnvironment + * @param builder CatalogBuilder + * @returns IncrementalCatalogBuilder + */ + static async create(env: PluginEnvironment, builder: CoreCatalogBuilder) { + const client = await env.database.getClient(); + const manager = new IncrementalIngestionDatabaseManager({ client }); + return new IncrementalCatalogBuilder(env, builder, client, manager); + } + + private ready: Deferred; + + private constructor( + private env: PluginEnvironment, + private builder: CoreCatalogBuilder, + private client: Knex, + private manager: IncrementalIngestionDatabaseManager, + ) { + this.ready = new Deferred(); + } + + async build() { + await applyDatabaseMigrations(this.client); + this.ready.resolve(); + + const routerLogger = this.env.logger.child({ router: 'IncrementalProviderAdmin' }); + + const incrementalAdminRouter = await createIncrementalProviderRouter(this.manager, routerLogger); + + return { incrementalAdminRouter, manager: this.manager }; + } + + addIncrementalEntityProvider( + provider: IncrementalEntityProvider, + options: IncrementalEntityProviderOptions, + ) { + const { burstInterval, burstLength, restLength } = options; + const { logger: catalogLogger, database, scheduler } = this.env; + const ready = this.ready; + + this.builder.addEntityProvider({ + getProviderName: provider.getProviderName.bind(provider), + async connect(connection) { + const logger = catalogLogger.child({ entityProvider: provider.getProviderName() }); + + logger.info(`Connecting`); + + const client = await database.getClient(); + + const manager = new IncrementalIngestionDatabaseManager({ client }); + + const engine = new IncrementalIngestionEngine({ + ...options, + ready, + manager, + logger, + provider, + restLength, + connection, + }); + + const frequency = Duration.isDuration(burstInterval) ? burstInterval : Duration.fromObject(burstInterval); + const length = Duration.isDuration(burstLength) ? burstLength : Duration.fromObject(burstLength); + + await scheduler.scheduleTask({ + id: provider.getProviderName(), + fn: engine.taskFn.bind(engine), + frequency, + timeout: length, + }); + }, + }); + } +} diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/incremental-ingestion-backend/src/types.ts new file mode 100644 index 0000000000..4a63391273 --- /dev/null +++ b/plugins/incremental-ingestion-backend/src/types.ts @@ -0,0 +1,221 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import type { PluginDatabaseManager, UrlReader } from '@backstage/backend-common'; +import type { PluginTaskScheduler, TaskFunction } from '@backstage/backend-tasks'; +import type { Config } from '@backstage/config'; +import type { DeferredEntity, EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import type { PermissionAuthorizer } from '@backstage/plugin-permission-common'; +import type { DurationObjectUnits } from 'luxon'; +import type { Logger } from 'winston'; +import { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager'; + +/** + * Entity annotation containing the incremental entity provider. + * + * @public + */ +export const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = 'backstage.io/incremental-provider-name'; + +/** + * Ingest entities into the catalog in bite-sized chunks. + * + * A Normal `EntityProvider` allows you to introduce entities into the + * processing pipeline by calling an `applyMutation()` on the full set + * of entities. However, this is not great when the number of entities + * that you have to keep track of is extremely large because it + * entails having all of them in memory at once. An + * `IncrementalEntityProvider` by contrast allows you to provide + * batches of entities in sequence so that you never need to have more + * than a few hundred in memory at a time. + * + */ +export interface IncrementalEntityProvider { + /** + * This name must be unique between all of the entity providers + * operating in the catalog. + */ + getProviderName(): string; + + /** + * Return a single page of entities from a specific point in the + * ingestion. + * + * @param context - anything needed in order to fetch a single page. + * @param cursor - a uniqiue value identifying the page to ingest. + * @returns the entities to be ingested, as well as the cursor of + * the the next page after this one. + */ + next(context: TContext, cursor?: TCursor): Promise>; + + /** + * Do any setup and teardown necessary in order to provide the + * context for fetching pages. This should always invoke `burst` in + * order to fetch the individual pages. + * + * @param burst - a function which performs a series of iterations + */ + around(burst: (context: TContext) => Promise): Promise; +} + +/** + * Value returned by an @{link IncrementalEntityProvider} to provide a + * single page of entities to ingest. + */ +export interface EntityIteratorResult { + /** + * Indicates whether there are any further pages of entities to + * ingest after this one. + */ + done: boolean; + + /** + * A value that marks the page of entities after this one. It will + * be used to pass into the following invocation of `next()` + */ + cursor: T; + + /** + * The entities to ingest. + */ + entities: DeferredEntity[]; +} + +export interface IncrementalEntityProviderOptions { + /** + * Entities are ingested in bursts. This interval determines how + * much time to wait in between each burst. + */ + burstInterval: DurationObjectUnits; + + /** + * Entities are ingested in bursts. This value determines how long + * to keep ingesting within each burst. + */ + burstLength: DurationObjectUnits; + + /** + * After a successful ingestion, the incremental entity provider + * will rest for this period of time before starting to ingest + * again. + */ + restLength: DurationObjectUnits; + + /** + * In the event of an error during an ingestion burst, the backoff + * determines how soon it will be retried. E.g. + * [{ minutes: 1}, { minutes: 5}, {minutes: 30 }, { hours: 3 }] + */ + backoff?: DurationObjectUnits[]; +} + +export type PluginEnvironment = { + logger: Logger; + database: PluginDatabaseManager; + scheduler: PluginTaskScheduler; + config: Config; + reader: UrlReader; + permissions: PermissionAuthorizer; +}; + +/** + * The core ingestion engine implements this interface + */ +export interface IterationEngine { + taskFn: TaskFunction; +} + +/** + * Options passed to the core ingestion engine during initialization. + */ +export interface IterationEngineOptions { + logger: Logger; + connection: EntityProviderConnection; + manager: IncrementalIngestionDatabaseManager; + provider: IncrementalEntityProvider; + restLength: DurationObjectUnits; + ready: Promise; + backoff?: IncrementalEntityProviderOptions['backoff']; +} + +/** + * The shape of data inserted into or updated in the `ingestion.ingestions` table. + */ +export interface IngestionUpsertIFace { + next_action: 'rest' | 'ingest' | 'backoff' | 'cancel' | 'nothing (done)' | 'nothing (canceled)'; + status: 'complete' | 'bursting' | 'resting' | 'canceling' | 'interstitial' | 'backing off'; + provider_name: string; + next_action_at?: Date; + last_error?: string; + attempts?: number; + ingestion_completed_at?: Date; + rest_completed_at?: Date; +} + +/** + * This interface supplies all potential values that can be inserted into the `ingestion.ingestions` table. + */ +export interface IngestionRecordInsert { + record: IngestionUpsertIFace & { + id: string; + }; +} + +/** + * This interface is for updating an existing ingestion record. + */ +export interface IngestionRecordUpdate { + ingestionId: string; + update: Partial; +} + +/** + * The expected response from the `ingestion.ingestion_marks` table. + */ +export interface MarkRecord { + id: string; + sequence: number; + ingestion_id: string; + cursor: string; + created_at: string; +} + +/** + * The expected response from the `ingestion.ingestions` table. + */ +export interface IngestionRecord { + id: string; + provider_name: string; + status: string; + next_action: string; + next_action_at: Date; + last_error: string | null; + attempts: number; + created_at: string; + rest_completed_at: string | null; + ingestion_completed_at: string | null; +} + +/** + * This interface supplies all the values for adding an ingestion mark. + */ +export interface MarkRecordInsert { + record: { + id: string; + ingestion_id: string; + cursor: unknown; + sequence: number; + }; +} diff --git a/yarn.lock b/yarn.lock index 53a18f77bb..162c56dc48 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6023,6 +6023,25 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-incremental-ingestion-backend@^0.0.0, @backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend": + version: 0.0.0-use.local + resolution: "@backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend" + dependencies: + "@backstage/backend-common": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/config": "workspace:^" + "@types/express": "*" + "@types/supertest": ^2.0.8 + express: ^4.18.1 + express-promise-router: ^4.1.0 + msw: ^0.47.0 + node-fetch: ^2.6.7 + supertest: ^6.2.4 + winston: ^3.2.1 + yn: ^4.0.0 + languageName: unknown + linkType: soft + "@backstage/plugin-jenkins-backend@workspace:^, @backstage/plugin-jenkins-backend@workspace:plugins/jenkins-backend": version: 0.0.0-use.local resolution: "@backstage/plugin-jenkins-backend@workspace:plugins/jenkins-backend" @@ -21350,6 +21369,7 @@ __metadata: "@backstage/plugin-events-backend": "workspace:^" "@backstage/plugin-events-node": "workspace:^" "@backstage/plugin-graphql-backend": "workspace:^" + "@backstage/plugin-incremental-ingestion-backend": ^0.0.0 "@backstage/plugin-jenkins-backend": "workspace:^" "@backstage/plugin-kafka-backend": "workspace:^" "@backstage/plugin-kubernetes-backend": "workspace:^" From 531914c88a8d9a95b22464505f56b2e9b3b13636 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Wed, 26 Oct 2022 14:02:47 -0700 Subject: [PATCH 02/54] Fixed several typos in the README.md Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/incremental-ingestion-backend/README.md b/plugins/incremental-ingestion-backend/README.md index 2571b178aa..1ba7d83f5c 100644 --- a/plugins/incremental-ingestion-backend/README.md +++ b/plugins/incremental-ingestion-backend/README.md @@ -21,7 +21,7 @@ We created the Incremental Entity Provider to address all of the above issues. T Incremental Entity Providers will wait a configurable interval before proceeding to the next burst. -Once the source has no more results, Incremental Entity Provider compares all entities annotated with `frontside/incremental-entity-provider: ` against all marked entities to determine which entities commited by same entity provider were not marked during the last ingestion cycle. All unmarked entities are deleted at the end of the cycle. The Incremental Entity Provider rests for a fixed internal before restarting the ingestion process. +Once the source has no more results, Incremental Entity Provider compares all entities annotated with `frontside/incremental-entity-provider: ` against all marked entities to determine which entities committed by same entity provider were not marked during the last ingestion cycle. All unmarked entities are deleted at the end of the cycle. The Incremental Entity Provider rests for a fixed internal before restarting the ingestion process. ![Diagram of execution of an Incremental Entity Provider](https://user-images.githubusercontent.com/74687/185822734-ee6279c7-64fa-46b9-9aa8-d4092ab73858.png) @@ -34,10 +34,11 @@ This approach has the following benefits, ## Requirements -The Incremental Entity Provider backend is designed for data sources that provide paginated results. Each burst attempts to handle one or more pages of the query. The plugin will attempt to fetch as many pages as it can within a configurable burst length. At every iteration, it expects to receive the next cursor that will be used to query in the next iteration. Each iteration may happen on a different replica. This has several concequences: +The Incremental Entity Provider backend is designed for data sources that provide paginated results. Each burst attempts to handle one or more pages of the query. The plugin will attempt to fetch as many pages as it can within a configurable burst length. At every iteration, it expects to receive the next cursor that will be used to query in the next iteration. Each iteration may happen on a different replica. This has several consequences: 1. The cursor must be serializable to JSON (not an issue for most RESTful or GraphQL based APIs). 2. The client must be stateless - a client is created from scratch for each iteration to allow distributing processing over multiple replicas. +3. There must be sufficient storage in Postgres to handle the additional data. ## Installation @@ -191,7 +192,7 @@ export class MyIncrementalEntityProvider implements IncrementalEntityProvider { From 5e79895af4cb1fd9eeea6c65e5c73a4837450d76 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Wed, 26 Oct 2022 15:27:37 -0700 Subject: [PATCH 03/54] Correct a few more typos and fix errors in example code Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/plugins/incremental-ingestion-backend/README.md b/plugins/incremental-ingestion-backend/README.md index 1ba7d83f5c..11bff92bf3 100644 --- a/plugins/incremental-ingestion-backend/README.md +++ b/plugins/incremental-ingestion-backend/README.md @@ -199,7 +199,7 @@ export class MyIncrementalEntityProvider implements IncrementalEntityProvider> { const { apiClient } = context; - const { page } = cursor; - // call your API with the current page - const data = await apiClient.getServices(page); + // call your API with the current cursor + const data = await apiClient.getServices(cursor); // calculate the next page const nextPage = page + 1; @@ -293,7 +292,7 @@ Now that you have your new Incremental Entity Provider, we can connect it to the We'll assume you followed the Installation instructions. After you create your `incrementalBuilder`, you can instantiate your Entity Provider and pass it to the `addIncrementalEntityProvider` method. ```ts - const incrementalBuilder = IncrementalCatalogBuilder.create(env, builder); + const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); // I'm assuming you're going to get your token from config const token = config.getString('myApiClient.token'); @@ -319,4 +318,4 @@ That's it!!! ## Error handling -If `around` or `next` methods throw an error, the error will show up in logs and it'll trigger the Incremental Entity Provider to try again after a backoff period. It'll keep trying until it reaches the last backoff attempt. You don't need to do anything special to handle the retry logic. +If `around` or `next` methods throw an error, the error will show up in logs and it'll trigger the Incremental Entity Provider to try again after a backoff period. It'll keep trying until it reaches the last backoff attempt, at which point it will cancel the current ingestion and start over. You don't need to do anything special to handle the retry logic. From 9891b7655f96f344e9e602514ca9d819dc003966 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Wed, 26 Oct 2022 17:52:15 -0700 Subject: [PATCH 04/54] Include backoff configuration and use 'back-off' in non-code Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/plugins/incremental-ingestion-backend/README.md b/plugins/incremental-ingestion-backend/README.md index 11bff92bf3..6cc5cecfb1 100644 --- a/plugins/incremental-ingestion-backend/README.md +++ b/plugins/incremental-ingestion-backend/README.md @@ -29,7 +29,7 @@ This approach has the following benefits, 1. Reduced ingestion latency - each burst commits entities which are processed before the entire list is processed. 2. Stable pressure - each period between bursts provides an opportunity for the processing pipeline to settle without overwhelming the pipeline with a large number of unprocessed entities. -3. Built-in retry/backoff - Failed bursts are automatically retried with a built-in backoff interval providing an opportunity for the data source to reset its rate limits before retrying the burst. +3. Built-in retry / back-off - Failed bursts are automatically retried with a built-in backoff interval providing an opportunity for the data source to reset its rate limits before retrying the burst. 4. Prevents orphan entities - Deleted entities are removed as with `full` mutation with a low memory footprint. ## Requirements @@ -310,6 +310,13 @@ We'll assume you followed the Installation instructi burstInterval: Duration.fromObject({ seconds: 3 }), // how long should it rest before re-ingesting again? restLength: Duration.fromObject({ day: 1 }) + // optional back-off configuration - how long should it wait to retry? + backoff: [ + Duration.fromObject({ seconds: 5 }), + Duration.fromObject({ seconds: 30 }), + Duration.fromObject({ minutes: 10 }), + Duration.fromObject({ hours: 3 }) + ] } ) ``` @@ -318,4 +325,4 @@ That's it!!! ## Error handling -If `around` or `next` methods throw an error, the error will show up in logs and it'll trigger the Incremental Entity Provider to try again after a backoff period. It'll keep trying until it reaches the last backoff attempt, at which point it will cancel the current ingestion and start over. You don't need to do anything special to handle the retry logic. +If `around` or `next` methods throw an error, the error will show up in logs and it'll trigger the Incremental Entity Provider to try again after a back-off period. It'll keep trying until it reaches the last back-off attempt, at which point it will cancel the current ingestion and start over. You don't need to do anything special to handle the retry logic. From fab23eed9380efd3066e42f04df57776e710e68f Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Wed, 26 Oct 2022 18:01:34 -0700 Subject: [PATCH 05/54] Include default Backstage headers on migration files Signed-off-by: Damon Kaswell --- .../migrations/20220613125155_init.js | 16 ++++++++++++++++ .../20220906161750_add-ingestion-indexes.js | 18 +++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js b/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js index 3e94f93d15..a17f71d119 100644 --- a/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js +++ b/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js @@ -1,3 +1,19 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + exports.up = async function (knex) { await knex.raw(` CREATE VIEW ingestion.current_entities as SELECT diff --git a/plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js b/plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js index 38282723e4..78c6afe796 100644 --- a/plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js +++ b/plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js @@ -1,5 +1,21 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** - * @param { import("knex").Knex } knex + * @param { import('knex').Knex } knex * @returns { Promise } */ exports.up = async function (knex) { From faed48221ef13a07ce44136ed327b3b37ea8b043 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Wed, 26 Oct 2022 18:59:32 -0700 Subject: [PATCH 06/54] Fix incorrect version in yarn.lock, and more spell-checking in README.md Signed-off-by: Damon Kaswell --- .../incremental-ingestion-backend/README.md | 4 +- yarn.lock | 1401 ++++++++++++++++- 2 files changed, 1353 insertions(+), 52 deletions(-) diff --git a/plugins/incremental-ingestion-backend/README.md b/plugins/incremental-ingestion-backend/README.md index 6cc5cecfb1..92ed56862d 100644 --- a/plugins/incremental-ingestion-backend/README.md +++ b/plugins/incremental-ingestion-backend/README.md @@ -29,7 +29,7 @@ This approach has the following benefits, 1. Reduced ingestion latency - each burst commits entities which are processed before the entire list is processed. 2. Stable pressure - each period between bursts provides an opportunity for the processing pipeline to settle without overwhelming the pipeline with a large number of unprocessed entities. -3. Built-in retry / back-off - Failed bursts are automatically retried with a built-in backoff interval providing an opportunity for the data source to reset its rate limits before retrying the burst. +3. Built-in retry / back-off - Failed bursts are automatically retried with a built-in back-off interval providing an opportunity for the data source to reset its rate limits before retrying the burst. 4. Prevents orphan entities - Deleted entities are removed as with `full` mutation with a low memory footprint. ## Requirements @@ -173,7 +173,7 @@ export class MyIncrementalEntityProvider implements IncrementalEntityProvider { diff --git a/yarn.lock b/yarn.lock index 162c56dc48..4629a23941 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1419,6 +1419,13 @@ __metadata: languageName: node linkType: hard +"@babel/compat-data@npm:^7.20.0": + version: 7.20.1 + resolution: "@babel/compat-data@npm:7.20.1" + checksum: 989b9b7a6fe43c547bb8329241bd0ba6983488b83d29cc59de35536272ee6bb4cc7487ba6c8a4bceebb3a57f8c5fea1434f80bbbe75202bc79bc1110f955ff25 + languageName: node + linkType: hard + "@babel/core@npm:^7.11.6, @babel/core@npm:^7.12.3, @babel/core@npm:^7.13.16, @babel/core@npm:^7.14.0, @babel/core@npm:^7.18.5": version: 7.19.1 resolution: "@babel/core@npm:7.19.1" @@ -1442,6 +1449,29 @@ __metadata: languageName: node linkType: hard +"@babel/core@npm:^7.19.6": + version: 7.20.2 + resolution: "@babel/core@npm:7.20.2" + dependencies: + "@ampproject/remapping": ^2.1.0 + "@babel/code-frame": ^7.18.6 + "@babel/generator": ^7.20.2 + "@babel/helper-compilation-targets": ^7.20.0 + "@babel/helper-module-transforms": ^7.20.2 + "@babel/helpers": ^7.20.1 + "@babel/parser": ^7.20.2 + "@babel/template": ^7.18.10 + "@babel/traverse": ^7.20.1 + "@babel/types": ^7.20.2 + convert-source-map: ^1.7.0 + debug: ^4.1.0 + gensync: ^1.0.0-beta.2 + json5: ^2.2.1 + semver: ^6.3.0 + checksum: 98faaaef26103a276a30a141b951a93bc8418d100d1f668bf7a69d12f3e25df57958e8b6b9100d95663f720db62da85ade736f6629a5ebb1e640251a1b43c0e4 + languageName: node + linkType: hard + "@babel/generator@npm:^7.14.0, @babel/generator@npm:^7.18.13, @babel/generator@npm:^7.19.0, @babel/generator@npm:^7.7.2": version: 7.19.3 resolution: "@babel/generator@npm:7.19.3" @@ -1453,6 +1483,17 @@ __metadata: languageName: node linkType: hard +"@babel/generator@npm:^7.20.1, @babel/generator@npm:^7.20.2": + version: 7.20.4 + resolution: "@babel/generator@npm:7.20.4" + dependencies: + "@babel/types": ^7.20.2 + "@jridgewell/gen-mapping": ^0.3.2 + jsesc: ^2.5.1 + checksum: 967b59f18e5ce999e5a741825bcecb2be4bbfc1824a92c21b47d0b5694e0eb09314a70f8b9142e9591c149c7fb83d51f73ae8fbd96d30a42666425889e51ceb1 + languageName: node + linkType: hard + "@babel/helper-annotate-as-pure@npm:^7.16.0, @babel/helper-annotate-as-pure@npm:^7.18.6": version: 7.18.6 resolution: "@babel/helper-annotate-as-pure@npm:7.18.6" @@ -1486,6 +1527,20 @@ __metadata: languageName: node linkType: hard +"@babel/helper-compilation-targets@npm:^7.20.0": + version: 7.20.0 + resolution: "@babel/helper-compilation-targets@npm:7.20.0" + dependencies: + "@babel/compat-data": ^7.20.0 + "@babel/helper-validator-option": ^7.18.6 + browserslist: ^4.21.3 + semver: ^6.3.0 + peerDependencies: + "@babel/core": ^7.0.0 + checksum: bc183f2109648849c8fde0b3c5cf08adf2f7ad6dc617b546fd20f34c8ef574ee5ee293c8d1bd0ed0221212e8f5907cdc2c42097870f1dcc769a654107d82c95b + languageName: node + linkType: hard + "@babel/helper-create-class-features-plugin@npm:^7.18.6": version: 7.18.9 resolution: "@babel/helper-create-class-features-plugin@npm:7.18.9" @@ -1602,6 +1657,22 @@ __metadata: languageName: node linkType: hard +"@babel/helper-module-transforms@npm:^7.20.2": + version: 7.20.2 + resolution: "@babel/helper-module-transforms@npm:7.20.2" + dependencies: + "@babel/helper-environment-visitor": ^7.18.9 + "@babel/helper-module-imports": ^7.18.6 + "@babel/helper-simple-access": ^7.20.2 + "@babel/helper-split-export-declaration": ^7.18.6 + "@babel/helper-validator-identifier": ^7.19.1 + "@babel/template": ^7.18.10 + "@babel/traverse": ^7.20.1 + "@babel/types": ^7.20.2 + checksum: 33a60ca115f6fce2c9d98e2a2e5649498aa7b23e2ae3c18745d7a021487708fc311458c33542f299387a0da168afccba94116e077f2cce49ae9e5ab83399e8a2 + languageName: node + linkType: hard + "@babel/helper-optimise-call-expression@npm:^7.18.6": version: 7.18.6 resolution: "@babel/helper-optimise-call-expression@npm:7.18.6" @@ -1654,6 +1725,15 @@ __metadata: languageName: node linkType: hard +"@babel/helper-simple-access@npm:^7.20.2": + version: 7.20.2 + resolution: "@babel/helper-simple-access@npm:7.20.2" + dependencies: + "@babel/types": ^7.20.2 + checksum: ad1e96ee2e5f654ffee2369a586e5e8d2722bf2d8b028a121b4c33ebae47253f64d420157b9f0a8927aea3a9e0f18c0103e74fdd531815cf3650a0a4adca11a1 + languageName: node + linkType: hard + "@babel/helper-skip-transparent-expression-wrappers@npm:^7.18.9": version: 7.18.9 resolution: "@babel/helper-skip-transparent-expression-wrappers@npm:7.18.9" @@ -1679,6 +1759,13 @@ __metadata: languageName: node linkType: hard +"@babel/helper-string-parser@npm:^7.19.4": + version: 7.19.4 + resolution: "@babel/helper-string-parser@npm:7.19.4" + checksum: b2f8a3920b30dfac81ec282ac4ad9598ea170648f8254b10f475abe6d944808fb006aab325d3eb5a8ad3bea8dfa888cfa6ef471050dae5748497c110ec060943 + languageName: node + linkType: hard + "@babel/helper-validator-identifier@npm:^7.18.6, @babel/helper-validator-identifier@npm:^7.19.1": version: 7.19.1 resolution: "@babel/helper-validator-identifier@npm:7.19.1" @@ -1716,6 +1803,17 @@ __metadata: languageName: node linkType: hard +"@babel/helpers@npm:^7.20.1": + version: 7.20.1 + resolution: "@babel/helpers@npm:7.20.1" + dependencies: + "@babel/template": ^7.18.10 + "@babel/traverse": ^7.20.1 + "@babel/types": ^7.20.0 + checksum: be35f78666bdab895775ed94dbeb098f7b4fa08ce4cfb0c3a9e69b7220cce56960dcdc2b14f5df9d3b80388d4bf7df155c97f6cf6768c0138f4e6931d0f44955 + languageName: node + linkType: hard + "@babel/highlight@npm:^7.0.0, @babel/highlight@npm:^7.18.6": version: 7.18.6 resolution: "@babel/highlight@npm:7.18.6" @@ -1736,6 +1834,15 @@ __metadata: languageName: node linkType: hard +"@babel/parser@npm:^7.20.1, @babel/parser@npm:^7.20.2": + version: 7.20.3 + resolution: "@babel/parser@npm:7.20.3" + bin: + parser: ./bin/babel-parser.js + checksum: 33bcdb45de65a3cf27ed376cb34f32be3c3485a10e3252f8d0126f6a034efc3145c0d219e57fcd5a8956361552008bc30b9bae4a723823fb3633027071be8a45 + languageName: node + linkType: hard + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@npm:^7.18.6": version: 7.18.6 resolution: "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@npm:7.18.6" @@ -2839,6 +2946,24 @@ __metadata: languageName: node linkType: hard +"@babel/traverse@npm:^7.20.1": + version: 7.20.1 + resolution: "@babel/traverse@npm:7.20.1" + dependencies: + "@babel/code-frame": ^7.18.6 + "@babel/generator": ^7.20.1 + "@babel/helper-environment-visitor": ^7.18.9 + "@babel/helper-function-name": ^7.19.0 + "@babel/helper-hoist-variables": ^7.18.6 + "@babel/helper-split-export-declaration": ^7.18.6 + "@babel/parser": ^7.20.1 + "@babel/types": ^7.20.0 + debug: ^4.1.0 + globals: ^11.1.0 + checksum: 6696176d574b7ff93466848010bc7e94b250169379ec2a84f1b10da46a7cc2018ea5e3a520c3078487db51e3a4afab9ecff48f25d1dbad8c1319362f4148fb4b + languageName: node + linkType: hard + "@babel/types@npm:^7.0.0, @babel/types@npm:^7.16.8, @babel/types@npm:^7.18.10, @babel/types@npm:^7.18.13, @babel/types@npm:^7.18.4, @babel/types@npm:^7.18.6, @babel/types@npm:^7.18.9, @babel/types@npm:^7.19.0, @babel/types@npm:^7.19.3, @babel/types@npm:^7.3.0, @babel/types@npm:^7.3.3, @babel/types@npm:^7.4.4, @babel/types@npm:^7.8.3": version: 7.19.3 resolution: "@babel/types@npm:7.19.3" @@ -2850,6 +2975,17 @@ __metadata: languageName: node linkType: hard +"@babel/types@npm:^7.20.0, @babel/types@npm:^7.20.2": + version: 7.20.2 + resolution: "@babel/types@npm:7.20.2" + dependencies: + "@babel/helper-string-parser": ^7.19.4 + "@babel/helper-validator-identifier": ^7.19.1 + to-fast-properties: ^2.0.0 + checksum: 57e76e5f21876135f481bfd4010c87f2d38196bb0a2bc60a28d6e55e3afa90cdd9accf164e4cb71bdfb620517fa0a0cb5600cdce36c21d59fdaccfbb899c024c + languageName: node + linkType: hard + "@backstage/app-defaults@workspace:^, @backstage/app-defaults@workspace:packages/app-defaults": version: 0.0.0-use.local resolution: "@backstage/app-defaults@workspace:packages/app-defaults" @@ -2891,6 +3027,130 @@ __metadata: languageName: unknown linkType: soft +"@backstage/backend-common@npm:^0.14.0": + version: 0.14.1 + resolution: "@backstage/backend-common@npm:0.14.1" + dependencies: + "@backstage/cli-common": ^0.1.9 + "@backstage/config": ^1.0.1 + "@backstage/config-loader": ^1.1.3 + "@backstage/errors": ^1.1.0 + "@backstage/integration": ^1.2.2 + "@backstage/types": ^1.0.0 + "@google-cloud/storage": ^6.0.0 + "@keyv/redis": ^2.2.3 + "@manypkg/get-packages": ^1.1.3 + "@octokit/rest": ^19.0.3 + "@types/cors": ^2.8.6 + "@types/dockerode": ^3.3.0 + "@types/express": ^4.17.6 + "@types/luxon": ^2.0.4 + "@types/webpack-env": ^1.15.2 + archiver: ^5.0.2 + aws-sdk: ^2.840.0 + base64-stream: ^1.0.0 + compression: ^1.7.4 + concat-stream: ^2.0.0 + cors: ^2.8.5 + dockerode: ^3.3.1 + express: ^4.17.1 + express-promise-router: ^4.1.0 + fs-extra: 10.1.0 + git-url-parse: ^12.0.0 + helmet: ^5.0.2 + isomorphic-git: ^1.8.0 + jose: ^4.6.0 + keyv: ^4.0.3 + keyv-memcache: ^1.2.5 + knex: ^2.0.0 + lodash: ^4.17.21 + logform: ^2.3.2 + luxon: ^3.0.0 + minimatch: ^5.0.0 + minimist: ^1.2.5 + morgan: ^1.10.0 + node-abort-controller: ^3.0.1 + node-fetch: ^2.6.7 + raw-body: ^2.4.1 + selfsigned: ^2.0.0 + stoppable: ^1.1.0 + tar: ^6.1.2 + unzipper: ^0.10.11 + winston: ^3.2.1 + yn: ^4.0.0 + peerDependencies: + pg-connection-string: ^2.3.0 + peerDependenciesMeta: + pg-connection-string: + optional: true + checksum: 888e42a543bcf8ffcb3802301ff904f34f520393cc0fc1adfa203b4e77928132ce6d587f5a155c338e9c8c51c55fd11a5e330ed444e0d4a8281680928313c81f + languageName: node + linkType: hard + +"@backstage/backend-common@npm:^0.16.0": + version: 0.16.0 + resolution: "@backstage/backend-common@npm:0.16.0" + dependencies: + "@backstage/cli-common": ^0.1.10 + "@backstage/config": ^1.0.4 + "@backstage/config-loader": ^1.1.6 + "@backstage/errors": ^1.1.3 + "@backstage/integration": ^1.4.0 + "@backstage/types": ^1.0.1 + "@google-cloud/storage": ^6.0.0 + "@keyv/memcache": ^1.3.5 + "@keyv/redis": ^2.5.3 + "@kubernetes/client-node": 0.17.0 + "@manypkg/get-packages": ^1.1.3 + "@octokit/rest": ^19.0.3 + "@types/cors": ^2.8.6 + "@types/dockerode": ^3.3.0 + "@types/express": ^4.17.6 + "@types/luxon": ^3.0.0 + "@types/webpack-env": ^1.15.2 + archiver: ^5.0.2 + aws-sdk: ^2.840.0 + base64-stream: ^1.0.0 + compression: ^1.7.4 + concat-stream: ^2.0.0 + cors: ^2.8.5 + dockerode: ^3.3.1 + express: ^4.17.1 + express-promise-router: ^4.1.0 + fs-extra: 10.1.0 + git-url-parse: ^13.0.0 + helmet: ^6.0.0 + isomorphic-git: ^1.8.0 + jose: ^4.6.0 + keyv: ^4.5.2 + knex: ^2.0.0 + lodash: ^4.17.21 + logform: ^2.3.2 + luxon: ^3.0.0 + minimatch: ^5.0.0 + minimist: ^1.2.5 + morgan: ^1.10.0 + node-abort-controller: ^3.0.1 + node-fetch: ^2.6.7 + node-forge: ^1.3.1 + raw-body: ^2.4.1 + request: ^2.88.2 + selfsigned: ^2.0.0 + stoppable: ^1.1.0 + tar: ^6.1.12 + uuid: ^8.3.2 + winston: ^3.2.1 + yauzl: ^2.10.0 + yn: ^4.0.0 + peerDependencies: + pg-connection-string: ^2.3.0 + peerDependenciesMeta: + pg-connection-string: + optional: true + checksum: 92d599323d188b22e1824490158a5e7d8fea3ee3cf5cd2bc50841e9a4a37d3cc02d046a25d7ded0050e9cbcba5849aed13f5535e329691572ea8d74d92a38623 + languageName: node + linkType: hard + "@backstage/backend-common@workspace:^, @backstage/backend-common@workspace:packages/backend-common": version: 0.0.0-use.local resolution: "@backstage/backend-common@workspace:packages/backend-common" @@ -2989,6 +3249,22 @@ __metadata: languageName: unknown linkType: soft +"@backstage/backend-plugin-api@npm:^0.1.4": + version: 0.1.4 + resolution: "@backstage/backend-plugin-api@npm:0.1.4" + dependencies: + "@backstage/backend-common": ^0.16.0 + "@backstage/backend-tasks": ^0.3.7 + "@backstage/config": ^1.0.4 + "@backstage/plugin-permission-common": ^0.7.1 + "@types/express": ^4.17.6 + express: ^4.17.1 + winston: ^3.2.1 + winston-transport: ^4.5.0 + checksum: 991176c0047cc4125b9be9551f71e675e86f51826a90e2ce6161410ea6bed680e68abc659aef2c6c4e679202e89af941d7302aba3d478d177e694a7d7e50870c + languageName: node + linkType: hard + "@backstage/backend-plugin-api@workspace:^, @backstage/backend-plugin-api@workspace:packages/backend-plugin-api": version: 0.0.0-use.local resolution: "@backstage/backend-plugin-api@workspace:packages/backend-plugin-api" @@ -3005,6 +3281,27 @@ __metadata: languageName: unknown linkType: soft +"@backstage/backend-tasks@npm:^0.3.6, @backstage/backend-tasks@npm:^0.3.7": + version: 0.3.7 + resolution: "@backstage/backend-tasks@npm:0.3.7" + dependencies: + "@backstage/backend-common": ^0.16.0 + "@backstage/config": ^1.0.4 + "@backstage/errors": ^1.1.3 + "@backstage/types": ^1.0.1 + "@types/luxon": ^3.0.0 + cron: ^2.0.0 + knex: ^2.0.0 + lodash: ^4.17.21 + luxon: ^3.0.0 + node-abort-controller: ^3.0.1 + uuid: ^8.0.0 + winston: ^3.2.1 + zod: ^3.9.5 + checksum: becfac6867c4301c1c92b39b8a254259133bb5c4684e4fc98256e0dd3739891c7e17ae04c8266ee9b56dda340fd6acf3aa294482391e5678495a3b0a7066fab4 + languageName: node + linkType: hard + "@backstage/backend-tasks@workspace:^, @backstage/backend-tasks@workspace:packages/backend-tasks": version: 0.0.0-use.local resolution: "@backstage/backend-tasks@workspace:packages/backend-tasks" @@ -3104,7 +3401,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/cli-common@workspace:^, @backstage/cli-common@workspace:packages/cli-common": +"@backstage/cli-common@^0.1.10, @backstage/cli-common@^0.1.9, @backstage/cli-common@workspace:^, @backstage/cli-common@workspace:packages/cli-common": version: 0.0.0-use.local resolution: "@backstage/cli-common@workspace:packages/cli-common" dependencies: @@ -3113,6 +3410,118 @@ __metadata: languageName: unknown linkType: soft +"@backstage/cli@npm:^0.20.0": + version: 0.20.0 + resolution: "@backstage/cli@npm:0.20.0" + dependencies: + "@backstage/cli-common": ^0.1.10 + "@backstage/config": ^1.0.3 + "@backstage/config-loader": ^1.1.5 + "@backstage/errors": ^1.1.2 + "@backstage/release-manifests": ^0.0.6 + "@backstage/types": ^1.0.0 + "@manypkg/get-packages": ^1.1.3 + "@octokit/request": ^6.0.0 + "@pmmmwh/react-refresh-webpack-plugin": ^0.5.7 + "@rollup/plugin-commonjs": ^22.0.0 + "@rollup/plugin-json": ^4.1.0 + "@rollup/plugin-node-resolve": ^13.0.6 + "@rollup/plugin-yaml": ^3.1.0 + "@spotify/eslint-config-base": ^14.0.0 + "@spotify/eslint-config-react": ^14.0.0 + "@spotify/eslint-config-typescript": ^14.0.0 + "@sucrase/jest-plugin": ^2.1.1 + "@sucrase/webpack-loader": ^2.0.0 + "@svgr/plugin-jsx": 6.3.x + "@svgr/plugin-svgo": 6.3.x + "@svgr/rollup": 6.3.x + "@svgr/webpack": 6.3.x + "@swc/core": ^1.2.239 + "@swc/helpers": ^0.4.7 + "@swc/jest": ^0.2.22 + "@types/jest": ^29.0.0 + "@types/webpack-env": ^1.15.2 + "@typescript-eslint/eslint-plugin": ^5.9.0 + "@typescript-eslint/parser": ^5.9.0 + "@yarnpkg/lockfile": ^1.1.0 + "@yarnpkg/parsers": ^3.0.0-rc.4 + bfj: ^7.0.2 + buffer: ^6.0.3 + chalk: ^4.0.0 + chokidar: ^3.3.1 + commander: ^9.1.0 + css-loader: ^6.5.1 + diff: ^5.0.0 + esbuild: ^0.14.10 + esbuild-loader: ^2.18.0 + eslint: ^8.6.0 + eslint-config-prettier: ^8.3.0 + eslint-formatter-friendly: ^7.0.0 + eslint-plugin-deprecation: ^1.3.2 + eslint-plugin-import: ^2.25.4 + eslint-plugin-jest: ^27.0.0 + eslint-plugin-jsx-a11y: ^6.5.1 + eslint-plugin-monorepo: ^0.3.2 + eslint-plugin-react: ^7.28.0 + eslint-plugin-react-hooks: ^4.3.0 + eslint-webpack-plugin: ^3.1.1 + express: ^4.17.1 + fork-ts-checker-webpack-plugin: ^7.0.0-alpha.8 + fs-extra: 10.1.0 + glob: ^7.1.7 + global-agent: ^3.0.0 + handlebars: ^4.7.3 + html-webpack-plugin: ^5.3.1 + inquirer: ^8.2.0 + jest: ^29.0.2 + jest-css-modules: ^2.1.0 + jest-environment-jsdom: ^29.0.2 + jest-runtime: ^29.0.2 + json-schema: ^0.4.0 + lodash: ^4.17.21 + mini-css-extract-plugin: ^2.4.2 + minimatch: 5.1.0 + node-fetch: ^2.6.7 + node-libs-browser: ^2.2.1 + npm-packlist: ^5.0.0 + ora: ^5.3.0 + postcss: ^8.1.0 + process: ^0.11.10 + react-dev-utils: ^12.0.0-next.60 + react-refresh: ^0.14.0 + recursive-readdir: ^2.2.2 + replace-in-file: ^6.0.0 + rollup: ^2.60.2 + rollup-plugin-dts: ^4.0.1 + rollup-plugin-esbuild: ^4.7.2 + rollup-plugin-postcss: ^4.0.0 + rollup-pluginutils: ^2.8.2 + run-script-webpack-plugin: ^0.1.0 + semver: ^7.3.2 + style-loader: ^3.3.1 + sucrase: ^3.20.2 + swc-loader: ^0.2.3 + tar: ^6.1.2 + terser-webpack-plugin: ^5.1.3 + util: ^0.12.3 + webpack: ^5.70.0 + webpack-dev-server: ^4.7.3 + webpack-node-externals: ^3.0.0 + yaml: ^2.0.0 + yml-loader: ^2.1.0 + yn: ^4.0.0 + zod: ^3.11.6 + peerDependencies: + "@microsoft/api-extractor": ^7.21.2 + peerDependenciesMeta: + "@microsoft/api-extractor": + optional: true + bin: + backstage-cli: bin/backstage-cli + checksum: b6072f130fd487bbd6947895d4809b67c991554b7c669b93aacc36845f461a8aa548be31d8c325f04b07a8512e9754bdf662cb0cdd4c986c883fa25c00d2f405 + languageName: node + linkType: hard + "@backstage/cli@workspace:*, @backstage/cli@workspace:^, @backstage/cli@workspace:packages/cli": version: 0.0.0-use.local resolution: "@backstage/cli@workspace:packages/cli" @@ -3271,6 +3680,29 @@ __metadata: languageName: unknown linkType: soft +"@backstage/config-loader@npm:^1.1.3, @backstage/config-loader@npm:^1.1.5, @backstage/config-loader@npm:^1.1.6": + version: 1.1.6 + resolution: "@backstage/config-loader@npm:1.1.6" + dependencies: + "@backstage/cli-common": ^0.1.10 + "@backstage/config": ^1.0.4 + "@backstage/errors": ^1.1.3 + "@backstage/types": ^1.0.1 + "@types/json-schema": ^7.0.6 + ajv: ^8.10.0 + chokidar: ^3.5.2 + fs-extra: 10.1.0 + json-schema: ^0.4.0 + json-schema-merge-allof: ^0.8.1 + json-schema-traverse: ^1.0.0 + node-fetch: ^2.6.7 + typescript-json-schema: ^0.54.0 + yaml: ^2.0.0 + yup: ^0.32.9 + checksum: a5347574e9006f2d84dd690f92dc9a43501771152e331258dcd60f07027de293948256d0afeea12443ded088c5147bf1cccc12437fd4df495d73862855ff8c48 + languageName: node + linkType: hard + "@backstage/config-loader@workspace:^, @backstage/config-loader@workspace:packages/config-loader": version: 0.0.0-use.local resolution: "@backstage/config-loader@workspace:packages/config-loader" @@ -3300,7 +3732,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/config@npm:^1.0.3, @backstage/config@npm:^1.0.4": +"@backstage/config@npm:^1.0.0, @backstage/config@npm:^1.0.1, @backstage/config@npm:^1.0.3, @backstage/config@npm:^1.0.4": version: 1.0.4 resolution: "@backstage/config@npm:1.0.4" dependencies: @@ -3634,7 +4066,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/errors@npm:^1.1.2, @backstage/errors@npm:^1.1.3": +"@backstage/errors@npm:^1.1.0, @backstage/errors@npm:^1.1.2, @backstage/errors@npm:^1.1.3": version: 1.1.3 resolution: "@backstage/errors@npm:1.1.3" dependencies: @@ -3702,7 +4134,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/integration@npm:^1.4.0": +"@backstage/integration@npm:^1.2.2, @backstage/integration@npm:^1.4.0": version: 1.4.0 resolution: "@backstage/integration@npm:1.4.0" dependencies: @@ -4122,6 +4554,22 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-auth-node@npm:^0.2.7": + version: 0.2.7 + resolution: "@backstage/plugin-auth-node@npm:0.2.7" + dependencies: + "@backstage/backend-common": ^0.16.0 + "@backstage/config": ^1.0.4 + "@backstage/errors": ^1.1.3 + "@types/express": "*" + express: ^4.17.1 + jose: ^4.6.0 + node-fetch: ^2.6.7 + winston: ^3.2.1 + checksum: 453c6d44edef8fcf4e23948fc1fe77e7fad7a0007fc1ccdab1d80c61ddc96746f2913a90e86e3cb4bb0f895e396e189b6c6797393b792fd497cc5831f76529da + languageName: node + linkType: hard + "@backstage/plugin-auth-node@workspace:^, @backstage/plugin-auth-node@workspace:plugins/auth-node": version: 0.0.0-use.local resolution: "@backstage/plugin-auth-node@workspace:plugins/auth-node" @@ -4687,6 +5135,48 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-catalog-backend@npm:^1.5.0": + version: 1.5.1 + resolution: "@backstage/plugin-catalog-backend@npm:1.5.1" + dependencies: + "@backstage/backend-common": ^0.16.0 + "@backstage/backend-plugin-api": ^0.1.4 + "@backstage/catalog-client": ^1.1.2 + "@backstage/catalog-model": ^1.1.3 + "@backstage/config": ^1.0.4 + "@backstage/errors": ^1.1.3 + "@backstage/integration": ^1.4.0 + "@backstage/plugin-catalog-common": ^1.0.8 + "@backstage/plugin-catalog-node": ^1.2.1 + "@backstage/plugin-permission-common": ^0.7.1 + "@backstage/plugin-permission-node": ^0.7.1 + "@backstage/plugin-scaffolder-common": ^1.2.2 + "@backstage/plugin-search-common": ^1.1.1 + "@backstage/types": ^1.0.1 + "@types/express": ^4.17.6 + codeowners-utils: ^1.0.2 + core-js: ^3.6.5 + express: ^4.17.1 + express-promise-router: ^4.1.0 + fast-json-stable-stringify: ^2.1.0 + fs-extra: 10.1.0 + git-url-parse: ^13.0.0 + glob: ^7.1.6 + knex: ^2.0.0 + lodash: ^4.17.21 + luxon: ^3.0.0 + node-fetch: ^2.6.7 + p-limit: ^3.0.2 + prom-client: ^14.0.1 + uuid: ^8.0.0 + winston: ^3.2.1 + yaml: ^2.0.0 + yn: ^4.0.0 + zod: ^3.11.6 + checksum: fd041132dc53757aeed2eb9fe74e85aca61083839e744c507360c84e7ba63d7ddf517b1d17b26cbd17328fa5554664ebddba868d15e15eff133dc861fceecc01 + languageName: node + linkType: hard + "@backstage/plugin-catalog-backend@workspace:^, @backstage/plugin-catalog-backend@workspace:plugins/catalog-backend": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend@workspace:plugins/catalog-backend" @@ -4863,6 +5353,20 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-catalog-node@npm:^1.2.1": + version: 1.2.1 + resolution: "@backstage/plugin-catalog-node@npm:1.2.1" + dependencies: + "@backstage/backend-plugin-api": ^0.1.4 + "@backstage/catalog-client": ^1.1.2 + "@backstage/catalog-model": ^1.1.3 + "@backstage/errors": ^1.1.3 + "@backstage/plugin-catalog-common": ^1.0.8 + "@backstage/types": ^1.0.1 + checksum: 7cefbdcd071f30da2dec1f04c5ba1df977ccaccdad4aa53dc16e0a35bdcc6d2db6437d97d15ec65d77f579dede3d94796cd4dc4f28586bcf81416a2771b48c1a + languageName: node + linkType: hard + "@backstage/plugin-catalog-node@workspace:^, @backstage/plugin-catalog-node@workspace:plugins/catalog-node": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-node@workspace:plugins/catalog-node" @@ -6023,22 +6527,24 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-incremental-ingestion-backend@^0.0.0, @backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend": +"@backstage/plugin-incremental-ingestion-backend@^0.1.0, @backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend": version: 0.0.0-use.local resolution: "@backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend" dependencies: - "@backstage/backend-common": "workspace:^" - "@backstage/cli": "workspace:^" - "@backstage/config": "workspace:^" + "@backstage/backend-common": ^0.14.0 + "@backstage/backend-tasks": ^0.3.6 + "@backstage/catalog-model": ^1.1.2 + "@backstage/cli": ^0.20.0 + "@backstage/config": ^1.0.0 + "@backstage/plugin-catalog-backend": ^1.5.0 + "@backstage/plugin-permission-common": ^0.7.0 "@types/express": "*" "@types/supertest": ^2.0.8 - express: ^4.18.1 + express: ^4.17.1 express-promise-router: ^4.1.0 - msw: ^0.47.0 - node-fetch: ^2.6.7 - supertest: ^6.2.4 + msw: ^0.35.0 + supertest: ^4.0.2 winston: ^3.2.1 - yn: ^4.0.0 languageName: unknown linkType: soft @@ -6513,7 +7019,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-permission-common@npm:^0.7.1": +"@backstage/plugin-permission-common@npm:^0.7.0, @backstage/plugin-permission-common@npm:^0.7.1": version: 0.7.1 resolution: "@backstage/plugin-permission-common@npm:0.7.1" dependencies: @@ -6542,6 +7048,24 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-permission-node@npm:^0.7.1": + version: 0.7.1 + resolution: "@backstage/plugin-permission-node@npm:0.7.1" + dependencies: + "@backstage/backend-common": ^0.16.0 + "@backstage/config": ^1.0.4 + "@backstage/errors": ^1.1.3 + "@backstage/plugin-auth-node": ^0.2.7 + "@backstage/plugin-permission-common": ^0.7.1 + "@types/express": ^4.17.6 + express: ^4.17.1 + express-promise-router: ^4.1.0 + zod: ^3.11.6 + zod-to-json-schema: ^3.18.1 + checksum: c274a25185066b99e94f262def3169830c1d2ead15a56b558d1c9fd202eef656e9d7278b75ce75415cc46183da7b3ff7b2b5551ca84db5285f37dbc01edbca24 + languageName: node + linkType: hard + "@backstage/plugin-permission-node@workspace:^, @backstage/plugin-permission-node@workspace:plugins/permission-node": version: 0.0.0-use.local resolution: "@backstage/plugin-permission-node@workspace:plugins/permission-node" @@ -6894,6 +7418,16 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-scaffolder-common@npm:^1.2.2": + version: 1.2.2 + resolution: "@backstage/plugin-scaffolder-common@npm:1.2.2" + dependencies: + "@backstage/catalog-model": ^1.1.3 + "@backstage/types": ^1.0.1 + checksum: d6e8d053a57bdb9a8cc3afe4db7e62f3a24f3a7a7fe340e50ec40b23a6bbd1e7e8d9aaa9cc6af28457bdd877c71c6dbc53a70371887664be098edb69f53a8487 + languageName: node + linkType: hard + "@backstage/plugin-scaffolder-common@workspace:^, @backstage/plugin-scaffolder-common@workspace:plugins/scaffolder-common": version: 0.0.0-use.local resolution: "@backstage/plugin-scaffolder-common@workspace:plugins/scaffolder-common" @@ -7922,6 +8456,15 @@ __metadata: languageName: unknown linkType: soft +"@backstage/release-manifests@npm:^0.0.6": + version: 0.0.6 + resolution: "@backstage/release-manifests@npm:0.0.6" + dependencies: + cross-fetch: ^3.1.5 + checksum: 9d6f0a2d5d6004f7f2c08938b84d55e0b54f45e83b1d93a3293b5702ad3709ad490794b46c10951a407f42525977ae1665db2eacd2fae8cf58a991b5e76996b2 + languageName: node + linkType: hard + "@backstage/release-manifests@workspace:^, @backstage/release-manifests@workspace:packages/release-manifests": version: 0.0.0-use.local resolution: "@backstage/release-manifests@workspace:packages/release-manifests" @@ -8561,6 +9104,13 @@ __metadata: languageName: node linkType: hard +"@esbuild/linux-loong64@npm:0.14.54": + version: 0.14.54 + resolution: "@esbuild/linux-loong64@npm:0.14.54" + conditions: os=linux & cpu=loong64 + languageName: node + linkType: hard + "@esbuild/linux-loong64@npm:0.15.15": version: 0.15.15 resolution: "@esbuild/linux-loong64@npm:0.15.15" @@ -10362,7 +10912,7 @@ __metadata: languageName: node linkType: hard -"@keyv/redis@npm:^2.5.3": +"@keyv/redis@npm:^2.2.3, @keyv/redis@npm:^2.5.3": version: 2.5.3 resolution: "@keyv/redis@npm:2.5.3" dependencies: @@ -10802,6 +11352,16 @@ __metadata: languageName: node linkType: hard +"@mswjs/cookies@npm:^0.1.6": + version: 0.1.7 + resolution: "@mswjs/cookies@npm:0.1.7" + dependencies: + "@types/set-cookie-parser": ^2.4.0 + set-cookie-parser: ^2.4.6 + checksum: d9b152dfdeba08b282a236485610bcfe992626e3c638fe51ebc5c7b5273d41d74e5447ae556a6c76460a8d3c7a7de6f544c681232cb50ae05b6b1e112bb77286 + languageName: node + linkType: hard + "@mswjs/cookies@npm:^0.2.0, @mswjs/cookies@npm:^0.2.2": version: 0.2.2 resolution: "@mswjs/cookies@npm:0.2.2" @@ -10812,6 +11372,20 @@ __metadata: languageName: node linkType: hard +"@mswjs/interceptors@npm:^0.12.6": + version: 0.12.7 + resolution: "@mswjs/interceptors@npm:0.12.7" + dependencies: + "@open-draft/until": ^1.0.3 + "@xmldom/xmldom": ^0.7.2 + debug: ^4.3.2 + headers-utils: ^3.0.2 + outvariant: ^1.2.0 + strict-event-emitter: ^0.2.0 + checksum: 426e9a27f13f0bd1f5b8dbf5f3605c65709f0aa73cee5a4a8278bc43859968a11d72aef42266a9c07030f1949f539ac1865a1da7edecc31b1ba40406211918f5 + languageName: node + linkType: hard + "@mswjs/interceptors@npm:^0.15.1": version: 0.15.3 resolution: "@mswjs/interceptors@npm:0.15.3" @@ -12354,7 +12928,16 @@ __metadata: languageName: node linkType: hard -"@svgr/babel-plugin-remove-jsx-attribute@npm:^6.5.0": +"@svgr/babel-plugin-add-jsx-attribute@npm:^6.5.1": + version: 6.5.1 + resolution: "@svgr/babel-plugin-add-jsx-attribute@npm:6.5.1" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: cab83832830a57735329ed68f67c03b57ca21fa037b0134847b0c5c0ef4beca89956d7dacfbf7b2a10fd901e7009e877512086db2ee918b8c69aee7742ae32c0 + languageName: node + linkType: hard + +"@svgr/babel-plugin-remove-jsx-attribute@npm:*, @svgr/babel-plugin-remove-jsx-attribute@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/babel-plugin-remove-jsx-attribute@npm:6.5.0" peerDependencies: @@ -12363,7 +12946,7 @@ __metadata: languageName: node linkType: hard -"@svgr/babel-plugin-remove-jsx-empty-expression@npm:^6.5.0": +"@svgr/babel-plugin-remove-jsx-empty-expression@npm:*, @svgr/babel-plugin-remove-jsx-empty-expression@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/babel-plugin-remove-jsx-empty-expression@npm:6.5.0" peerDependencies: @@ -12381,6 +12964,15 @@ __metadata: languageName: node linkType: hard +"@svgr/babel-plugin-replace-jsx-attribute-value@npm:^6.5.1": + version: 6.5.1 + resolution: "@svgr/babel-plugin-replace-jsx-attribute-value@npm:6.5.1" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: b7d2125758e766e1ebd14b92216b800bdc976959bc696dbfa1e28682919147c1df4bb8b1b5fd037d7a83026e27e681fea3b8d3741af8d3cf4c9dfa3d412125df + languageName: node + linkType: hard + "@svgr/babel-plugin-svg-dynamic-title@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/babel-plugin-svg-dynamic-title@npm:6.5.0" @@ -12390,6 +12982,15 @@ __metadata: languageName: node linkType: hard +"@svgr/babel-plugin-svg-dynamic-title@npm:^6.5.1": + version: 6.5.1 + resolution: "@svgr/babel-plugin-svg-dynamic-title@npm:6.5.1" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 0fd42ebf127ae9163ef341e84972daa99bdcb9e6ed3f83aabd95ee173fddc43e40e02fa847fbc0a1058cf5549f72b7960a2c5e22c3e4ac18f7e3ac81277852ae + languageName: node + linkType: hard + "@svgr/babel-plugin-svg-em-dimensions@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/babel-plugin-svg-em-dimensions@npm:6.5.0" @@ -12399,6 +13000,15 @@ __metadata: languageName: node linkType: hard +"@svgr/babel-plugin-svg-em-dimensions@npm:^6.5.1": + version: 6.5.1 + resolution: "@svgr/babel-plugin-svg-em-dimensions@npm:6.5.1" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: c1550ee9f548526fa66fd171e3ffb5696bfc4e4cd108a631d39db492c7410dc10bba4eb5a190e9df824bf806130ccc586ae7d2e43c547e6a4f93bbb29a18f344 + languageName: node + linkType: hard + "@svgr/babel-plugin-transform-react-native-svg@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/babel-plugin-transform-react-native-svg@npm:6.5.0" @@ -12408,6 +13018,15 @@ __metadata: languageName: node linkType: hard +"@svgr/babel-plugin-transform-react-native-svg@npm:^6.5.1": + version: 6.5.1 + resolution: "@svgr/babel-plugin-transform-react-native-svg@npm:6.5.1" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 4c924af22b948b812629e80efb90ad1ec8faae26a232d8ca8a06b46b53e966a2c415a57806a3ff0ea806a622612e546422719b69ec6839717a7755dac19171d9 + languageName: node + linkType: hard + "@svgr/babel-plugin-transform-svg-component@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/babel-plugin-transform-svg-component@npm:6.5.0" @@ -12417,6 +13036,33 @@ __metadata: languageName: node linkType: hard +"@svgr/babel-plugin-transform-svg-component@npm:^6.5.1": + version: 6.5.1 + resolution: "@svgr/babel-plugin-transform-svg-component@npm:6.5.1" + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: e496bb5ee871feb6bcab250b6e067322da7dd5c9c2b530b41e5586fe090f86611339b49d0a909c334d9b24cbca0fa755c949a2526c6ad03c6b5885666874cf5f + languageName: node + linkType: hard + +"@svgr/babel-preset@npm:^6.3.1, @svgr/babel-preset@npm:^6.5.1": + version: 6.5.1 + resolution: "@svgr/babel-preset@npm:6.5.1" + dependencies: + "@svgr/babel-plugin-add-jsx-attribute": ^6.5.1 + "@svgr/babel-plugin-remove-jsx-attribute": "*" + "@svgr/babel-plugin-remove-jsx-empty-expression": "*" + "@svgr/babel-plugin-replace-jsx-attribute-value": ^6.5.1 + "@svgr/babel-plugin-svg-dynamic-title": ^6.5.1 + "@svgr/babel-plugin-svg-em-dimensions": ^6.5.1 + "@svgr/babel-plugin-transform-react-native-svg": ^6.5.1 + "@svgr/babel-plugin-transform-svg-component": ^6.5.1 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 9f124be39a8e64f909162f925b3a63ddaa5a342a5e24fc0b7f7d9d4d7f7e3b916596c754fb557dc259928399cad5366a27cb231627a0d2dcc4b13ac521cf05af + languageName: node + linkType: hard + "@svgr/babel-preset@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/babel-preset@npm:6.5.0" @@ -12435,6 +13081,19 @@ __metadata: languageName: node linkType: hard +"@svgr/core@npm:^6.3.1": + version: 6.5.1 + resolution: "@svgr/core@npm:6.5.1" + dependencies: + "@babel/core": ^7.19.6 + "@svgr/babel-preset": ^6.5.1 + "@svgr/plugin-jsx": ^6.5.1 + camelcase: ^6.2.0 + cosmiconfig: ^7.0.1 + checksum: fd6d6d5da5aeb956703310480b626c1fb3e3973ad9fe8025efc1dcf3d895f857b70d100c63cf32cebb20eb83c9607bafa464c9436e18fe6fe4fafdc73ed6b1a5 + languageName: node + linkType: hard + "@svgr/core@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/core@npm:6.5.0" @@ -12448,6 +13107,16 @@ __metadata: languageName: node linkType: hard +"@svgr/hast-util-to-babel-ast@npm:^6.3.1, @svgr/hast-util-to-babel-ast@npm:^6.5.1": + version: 6.5.1 + resolution: "@svgr/hast-util-to-babel-ast@npm:6.5.1" + dependencies: + "@babel/types": ^7.20.0 + entities: ^4.4.0 + checksum: 37923cce1b3f4e2039077b0c570b6edbabe37d1cf1a6ee35e71e0fe00f9cffac450eec45e9720b1010418131a999cb0047331ba1b6d1d2c69af1b92ac785aacf + languageName: node + linkType: hard + "@svgr/hast-util-to-babel-ast@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/hast-util-to-babel-ast@npm:6.5.0" @@ -12458,6 +13127,20 @@ __metadata: languageName: node linkType: hard +"@svgr/plugin-jsx@npm:6.3.x": + version: 6.3.1 + resolution: "@svgr/plugin-jsx@npm:6.3.1" + dependencies: + "@babel/core": ^7.18.5 + "@svgr/babel-preset": ^6.3.1 + "@svgr/hast-util-to-babel-ast": ^6.3.1 + svg-parser: ^2.0.4 + peerDependencies: + "@svgr/core": ^6.0.0 + checksum: a2e487dc28d2b69b94b7d96e5cb2593857e559e64c8cb4f818035b8a43ba84e5ddb67f966d15f173b544e807e48a1cda1065da8f9064b94b8d62bbe8cb8c4d73 + languageName: node + linkType: hard + "@svgr/plugin-jsx@npm:6.5.x, @svgr/plugin-jsx@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/plugin-jsx@npm:6.5.0" @@ -12472,6 +13155,33 @@ __metadata: languageName: node linkType: hard +"@svgr/plugin-jsx@npm:^6.3.1, @svgr/plugin-jsx@npm:^6.5.1": + version: 6.5.1 + resolution: "@svgr/plugin-jsx@npm:6.5.1" + dependencies: + "@babel/core": ^7.19.6 + "@svgr/babel-preset": ^6.5.1 + "@svgr/hast-util-to-babel-ast": ^6.5.1 + svg-parser: ^2.0.4 + peerDependencies: + "@svgr/core": ^6.0.0 + checksum: 42f22847a6bdf930514d7bedd3c5e1fd8d53eb3594779f9db16cb94c762425907c375cd8ec789114e100a4d38068aca6c7ab5efea4c612fba63f0630c44cc859 + languageName: node + linkType: hard + +"@svgr/plugin-svgo@npm:6.3.x": + version: 6.3.1 + resolution: "@svgr/plugin-svgo@npm:6.3.1" + dependencies: + cosmiconfig: ^7.0.1 + deepmerge: ^4.2.2 + svgo: ^2.8.0 + peerDependencies: + "@svgr/core": ^6.0.0 + checksum: 037d6f91ba7f362764527408661f7fc4a4a296e9dc142a5f2e33fc88dc63dafd305452caae3091e9adb63adf029e0ce20c604d5af787b968f98aad261a834679 + languageName: node + linkType: hard + "@svgr/plugin-svgo@npm:6.5.x, @svgr/plugin-svgo@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/plugin-svgo@npm:6.5.0" @@ -12485,6 +13195,36 @@ __metadata: languageName: node linkType: hard +"@svgr/plugin-svgo@npm:^6.3.1": + version: 6.5.1 + resolution: "@svgr/plugin-svgo@npm:6.5.1" + dependencies: + cosmiconfig: ^7.0.1 + deepmerge: ^4.2.2 + svgo: ^2.8.0 + peerDependencies: + "@svgr/core": "*" + checksum: cd2833530ac0485221adc2146fd992ab20d79f4b12eebcd45fa859721dd779483158e11dfd9a534858fe468416b9412416e25cbe07ac7932c44ed5fa2021c72e + languageName: node + linkType: hard + +"@svgr/rollup@npm:6.3.x": + version: 6.3.1 + resolution: "@svgr/rollup@npm:6.3.1" + dependencies: + "@babel/core": ^7.18.5 + "@babel/plugin-transform-react-constant-elements": ^7.17.12 + "@babel/preset-env": ^7.18.2 + "@babel/preset-react": ^7.17.12 + "@babel/preset-typescript": ^7.17.12 + "@rollup/pluginutils": ^4.2.1 + "@svgr/core": ^6.3.1 + "@svgr/plugin-jsx": ^6.3.1 + "@svgr/plugin-svgo": ^6.3.1 + checksum: 8d96f95a4c89d96a14bc0095469a4ceb39a2ae4ea3517d08573ed8ca11b05416a505723ef09d00c3cee056adc2d95ddcea6c8055a116bbb461ba33d601bfced8 + languageName: node + linkType: hard + "@svgr/rollup@npm:6.5.x": version: 6.5.0 resolution: "@svgr/rollup@npm:6.5.0" @@ -12502,6 +13242,22 @@ __metadata: languageName: node linkType: hard +"@svgr/webpack@npm:6.3.x": + version: 6.3.1 + resolution: "@svgr/webpack@npm:6.3.1" + dependencies: + "@babel/core": ^7.18.5 + "@babel/plugin-transform-react-constant-elements": ^7.17.12 + "@babel/preset-env": ^7.18.2 + "@babel/preset-react": ^7.17.12 + "@babel/preset-typescript": ^7.17.12 + "@svgr/core": ^6.3.1 + "@svgr/plugin-jsx": ^6.3.1 + "@svgr/plugin-svgo": ^6.3.1 + checksum: 36784eacf80601462ede7eab66347423a8635e68aa9f152308c81878b071807adee152a28eed2cce9c72faaf6553dd500f68f00601062ec6821ec0a3a77f4e13 + languageName: node + linkType: hard + "@svgr/webpack@npm:6.5.x": version: 6.5.0 resolution: "@svgr/webpack@npm:6.5.0" @@ -12588,7 +13344,7 @@ __metadata: languageName: node linkType: hard -"@swc/core@npm:^1.3.9": +"@swc/core@npm:^1.2.239, @swc/core@npm:^1.3.9": version: 1.3.19 resolution: "@swc/core@npm:1.3.19" dependencies: @@ -13572,6 +14328,16 @@ __metadata: languageName: node linkType: hard +"@types/inquirer@npm:^7.3.3": + version: 7.3.3 + resolution: "@types/inquirer@npm:7.3.3" + dependencies: + "@types/through": "*" + rxjs: ^6.4.0 + checksum: 49b21d883ab533dbb84b400fa1aeab2638c37b87978d16f15636316c8d9f70d93a185479cf32081d9013fe2b362db05a83bdc3725771cc93d8bdab9182a96ab9 + languageName: node + linkType: hard + "@types/inquirer@npm:^8.1.3": version: 8.2.5 resolution: "@types/inquirer@npm:8.2.5" @@ -13659,7 +14425,7 @@ __metadata: languageName: node linkType: hard -"@types/js-levenshtein@npm:^1.1.1": +"@types/js-levenshtein@npm:^1.1.0, @types/js-levenshtein@npm:^1.1.1": version: 1.1.1 resolution: "@types/js-levenshtein@npm:1.1.1" checksum: 1d1ff1ee2ad551909e47f3ce19fcf85b64dc5146d3b531c8d26fc775492d36e380b32cf5ef68ff301e812c3b00282f37aac579ebb44498b94baff0ace7509769 @@ -13802,6 +14568,13 @@ __metadata: languageName: node linkType: hard +"@types/luxon@npm:^2.0.4": + version: 2.4.0 + resolution: "@types/luxon@npm:2.4.0" + checksum: eeb16a1bfe5440464c1a9635700d103cd18d3cd8da6063a1938478e435cfba6ab8e893aa80c95a407e541187c1e997c3e4481322726bc1258551cb8606d0e5ad + languageName: node + linkType: hard + "@types/markdown-it@npm:^12.2.3": version: 12.2.3 resolution: "@types/markdown-it@npm:12.2.3" @@ -15227,6 +16000,13 @@ __metadata: languageName: node linkType: hard +"@xmldom/xmldom@npm:^0.7.2": + version: 0.7.9 + resolution: "@xmldom/xmldom@npm:0.7.9" + checksum: 66e37b7800132f891b885b2eceeeebc53f60b69789da10276f1584256b963d79a28c7ae2071bc53a9cd842d9b03554c761b2701fe8036d6052f26bcd0ae8f2bb + languageName: node + linkType: hard + "@xobotyi/scrollbar-width@npm:^1.9.5": version: 1.9.5 resolution: "@xobotyi/scrollbar-width@npm:1.9.5" @@ -16662,6 +17442,13 @@ __metadata: languageName: node linkType: hard +"big-integer@npm:^1.6.17": + version: 1.6.51 + resolution: "big-integer@npm:1.6.51" + checksum: 3d444173d1b2e20747e2c175568bedeebd8315b0637ea95d75fd27830d3b8e8ba36c6af40374f36bdaea7b5de376dcada1b07587cb2a79a928fccdb6e6e3c518 + languageName: node + linkType: hard + "big.js@npm:^5.2.2": version: 5.2.2 resolution: "big.js@npm:5.2.2" @@ -16711,6 +17498,16 @@ __metadata: languageName: node linkType: hard +"binary@npm:~0.3.0": + version: 0.3.0 + resolution: "binary@npm:0.3.0" + dependencies: + buffers: ~0.1.1 + chainsaw: ~0.1.0 + checksum: b4699fda9e2c2981e74a46b0115cf0d472eda9b68c0e9d229ef494e92f29ce81acf0a834415094cffcc340dfee7c4ef8ce5d048c65c18067a7ed850323f777af + languageName: node + linkType: hard + "binaryextensions@npm:^4.15.0, binaryextensions@npm:^4.16.0": version: 4.18.0 resolution: "binaryextensions@npm:4.18.0" @@ -16766,6 +17563,13 @@ __metadata: languageName: node linkType: hard +"bluebird@npm:~3.4.1": + version: 3.4.7 + resolution: "bluebird@npm:3.4.7" + checksum: bffa9dee7d3a41ab15c4f3f24687b49959b4e64e55c058a062176feb8ccefc2163414fb4e1a0f3053bf187600936509660c3ebd168fd9f0e48c7eba23b019466 + languageName: node + linkType: hard + "bmp-js@npm:^0.1.0": version: 0.1.0 resolution: "bmp-js@npm:0.1.0" @@ -17027,6 +17831,13 @@ __metadata: languageName: node linkType: hard +"buffer-indexof-polyfill@npm:~1.0.0": + version: 1.0.2 + resolution: "buffer-indexof-polyfill@npm:1.0.2" + checksum: fbfb2d69c6bb2df235683126f9dc140150c08ac3630da149913a9971947b667df816a913b6993bc48f4d611999cb99a1589914d34c02dccd2234afda5cb75bbc + languageName: node + linkType: hard + "buffer-writer@npm:2.0.0": version: 2.0.0 resolution: "buffer-writer@npm:2.0.0" @@ -17072,6 +17883,13 @@ __metadata: languageName: node linkType: hard +"buffers@npm:~0.1.1": + version: 0.1.1 + resolution: "buffers@npm:0.1.1" + checksum: ad6f8e483efab39cefd92bdc04edbff6805e4211b002f4d1cfb70c6c472a61cc89fb18c37bcdfdd4ee416ca096e9ff606286698a7d41a18b539bac12fd76d4d5 + languageName: node + linkType: hard + "builtin-modules@npm:^3.0.0": version: 3.2.0 resolution: "builtin-modules@npm:3.2.0" @@ -17361,6 +18179,15 @@ __metadata: languageName: node linkType: hard +"chainsaw@npm:~0.1.0": + version: 0.1.0 + resolution: "chainsaw@npm:0.1.0" + dependencies: + traverse: ">=0.3.0 <0.4" + checksum: 22a96b9fb0cd9fb20813607c0869e61817d1acc81b5d455cc6456b5e460ea1dd52630e0f76b291cf8294bfb6c1fc42e299afb52104af9096242699d6d3aa6d3e + languageName: node + linkType: hard + "chalk@npm:2.4.2, chalk@npm:^2.0.0, chalk@npm:^2.1.0, chalk@npm:^2.3.2": version: 2.4.2 resolution: "chalk@npm:2.4.2" @@ -17729,6 +18556,17 @@ __metadata: languageName: node linkType: hard +"cliui@npm:^8.0.1": + version: 8.0.1 + resolution: "cliui@npm:8.0.1" + dependencies: + string-width: ^4.2.0 + strip-ansi: ^6.0.1 + wrap-ansi: ^7.0.0 + checksum: 79648b3b0045f2e285b76fb2e24e207c6db44323581e421c3acbd0e86454cba1b37aea976ab50195a49e7384b871e6dfb2247ad7dec53c02454ac6497394cb56 + languageName: node + linkType: hard + "clone-buffer@npm:^1.0.0": version: 1.0.0 resolution: "clone-buffer@npm:1.0.0" @@ -18153,7 +18991,7 @@ __metadata: languageName: node linkType: hard -"component-emitter@npm:^1.3.0, component-emitter@npm:~1.3.0": +"component-emitter@npm:^1.2.0, component-emitter@npm:^1.3.0, component-emitter@npm:~1.3.0": version: 1.3.0 resolution: "component-emitter@npm:1.3.0" checksum: b3c46de38ffd35c57d1c02488355be9f218e582aec72d72d1b8bbec95a3ac1b38c96cd6e03ff015577e68f550fbb361a3bfdbd9bb248be9390b7b3745691be6b @@ -18403,7 +19241,7 @@ __metadata: languageName: node linkType: hard -"cookie@npm:0.4.2, cookie@npm:^0.4.2, cookie@npm:~0.4.1": +"cookie@npm:0.4.2, cookie@npm:^0.4.1, cookie@npm:^0.4.2, cookie@npm:~0.4.1": version: 0.4.2 resolution: "cookie@npm:0.4.2" checksum: a00833c998bedf8e787b4c342defe5fa419abd96b32f4464f718b91022586b8f1bafbddd499288e75c037642493c83083da426c6a9080d309e3bd90fd11baa9b @@ -18417,7 +19255,7 @@ __metadata: languageName: node linkType: hard -"cookiejar@npm:^2.1.3": +"cookiejar@npm:^2.1.0, cookiejar@npm:^2.1.3": version: 2.1.3 resolution: "cookiejar@npm:2.1.3" checksum: 88259983ebc52ceb23cdacfa48762b6a518a57872eff1c7ed01d214fff5cf492e2660d7d5c04700a28f1787a76811df39e8639f8e17670b3cf94ecd86e161f07 @@ -20016,6 +20854,15 @@ __metadata: languageName: node linkType: hard +"duplexer2@npm:~0.1.4": + version: 0.1.4 + resolution: "duplexer2@npm:0.1.4" + dependencies: + readable-stream: ^2.0.2 + checksum: 744961f03c7f54313f90555ac20284a3fb7bf22fdff6538f041a86c22499560eb6eac9d30ab5768054137cb40e6b18b40f621094e0261d7d8c35a37b7a5ad241 + languageName: node + linkType: hard + "duplexer@npm:^0.1.2, duplexer@npm:~0.1.1": version: 0.1.2 resolution: "duplexer@npm:0.1.2" @@ -20428,6 +21275,13 @@ __metadata: languageName: node linkType: hard +"esbuild-android-64@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-android-64@npm:0.14.54" + conditions: os=android & cpu=x64 + languageName: node + linkType: hard + "esbuild-android-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-android-64@npm:0.15.15" @@ -20435,6 +21289,13 @@ __metadata: languageName: node linkType: hard +"esbuild-android-arm64@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-android-arm64@npm:0.14.54" + conditions: os=android & cpu=arm64 + languageName: node + linkType: hard + "esbuild-android-arm64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-android-arm64@npm:0.15.15" @@ -20442,6 +21303,13 @@ __metadata: languageName: node linkType: hard +"esbuild-darwin-64@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-darwin-64@npm:0.14.54" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + "esbuild-darwin-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-darwin-64@npm:0.15.15" @@ -20449,6 +21317,13 @@ __metadata: languageName: node linkType: hard +"esbuild-darwin-arm64@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-darwin-arm64@npm:0.14.54" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + "esbuild-darwin-arm64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-darwin-arm64@npm:0.15.15" @@ -20456,6 +21331,13 @@ __metadata: languageName: node linkType: hard +"esbuild-freebsd-64@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-freebsd-64@npm:0.14.54" + conditions: os=freebsd & cpu=x64 + languageName: node + linkType: hard + "esbuild-freebsd-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-freebsd-64@npm:0.15.15" @@ -20463,6 +21345,13 @@ __metadata: languageName: node linkType: hard +"esbuild-freebsd-arm64@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-freebsd-arm64@npm:0.14.54" + conditions: os=freebsd & cpu=arm64 + languageName: node + linkType: hard + "esbuild-freebsd-arm64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-freebsd-arm64@npm:0.15.15" @@ -20470,6 +21359,13 @@ __metadata: languageName: node linkType: hard +"esbuild-linux-32@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-linux-32@npm:0.14.54" + conditions: os=linux & cpu=ia32 + languageName: node + linkType: hard + "esbuild-linux-32@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-32@npm:0.15.15" @@ -20477,6 +21373,13 @@ __metadata: languageName: node linkType: hard +"esbuild-linux-64@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-linux-64@npm:0.14.54" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + "esbuild-linux-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-64@npm:0.15.15" @@ -20484,6 +21387,13 @@ __metadata: languageName: node linkType: hard +"esbuild-linux-arm64@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-linux-arm64@npm:0.14.54" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + "esbuild-linux-arm64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-arm64@npm:0.15.15" @@ -20491,6 +21401,13 @@ __metadata: languageName: node linkType: hard +"esbuild-linux-arm@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-linux-arm@npm:0.14.54" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + "esbuild-linux-arm@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-arm@npm:0.15.15" @@ -20498,6 +21415,13 @@ __metadata: languageName: node linkType: hard +"esbuild-linux-mips64le@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-linux-mips64le@npm:0.14.54" + conditions: os=linux & cpu=mips64el + languageName: node + linkType: hard + "esbuild-linux-mips64le@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-mips64le@npm:0.15.15" @@ -20505,6 +21429,13 @@ __metadata: languageName: node linkType: hard +"esbuild-linux-ppc64le@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-linux-ppc64le@npm:0.14.54" + conditions: os=linux & cpu=ppc64 + languageName: node + linkType: hard + "esbuild-linux-ppc64le@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-ppc64le@npm:0.15.15" @@ -20512,6 +21443,13 @@ __metadata: languageName: node linkType: hard +"esbuild-linux-riscv64@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-linux-riscv64@npm:0.14.54" + conditions: os=linux & cpu=riscv64 + languageName: node + linkType: hard + "esbuild-linux-riscv64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-riscv64@npm:0.15.15" @@ -20519,6 +21457,13 @@ __metadata: languageName: node linkType: hard +"esbuild-linux-s390x@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-linux-s390x@npm:0.14.54" + conditions: os=linux & cpu=s390x + languageName: node + linkType: hard + "esbuild-linux-s390x@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-s390x@npm:0.15.15" @@ -20542,6 +21487,13 @@ __metadata: languageName: node linkType: hard +"esbuild-netbsd-64@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-netbsd-64@npm:0.14.54" + conditions: os=netbsd & cpu=x64 + languageName: node + linkType: hard + "esbuild-netbsd-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-netbsd-64@npm:0.15.15" @@ -20549,6 +21501,13 @@ __metadata: languageName: node linkType: hard +"esbuild-openbsd-64@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-openbsd-64@npm:0.14.54" + conditions: os=openbsd & cpu=x64 + languageName: node + linkType: hard + "esbuild-openbsd-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-openbsd-64@npm:0.15.15" @@ -20556,6 +21515,13 @@ __metadata: languageName: node linkType: hard +"esbuild-sunos-64@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-sunos-64@npm:0.14.54" + conditions: os=sunos & cpu=x64 + languageName: node + linkType: hard + "esbuild-sunos-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-sunos-64@npm:0.15.15" @@ -20563,6 +21529,13 @@ __metadata: languageName: node linkType: hard +"esbuild-windows-32@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-windows-32@npm:0.14.54" + conditions: os=win32 & cpu=ia32 + languageName: node + linkType: hard + "esbuild-windows-32@npm:0.15.15": version: 0.15.15 resolution: "esbuild-windows-32@npm:0.15.15" @@ -20570,6 +21543,13 @@ __metadata: languageName: node linkType: hard +"esbuild-windows-64@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-windows-64@npm:0.14.54" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + "esbuild-windows-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-windows-64@npm:0.15.15" @@ -20577,6 +21557,13 @@ __metadata: languageName: node linkType: hard +"esbuild-windows-arm64@npm:0.14.54": + version: 0.14.54 + resolution: "esbuild-windows-arm64@npm:0.14.54" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + "esbuild-windows-arm64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-windows-arm64@npm:0.15.15" @@ -20584,6 +21571,80 @@ __metadata: languageName: node linkType: hard +"esbuild@npm:^0.14.10": + version: 0.14.54 + resolution: "esbuild@npm:0.14.54" + dependencies: + "@esbuild/linux-loong64": 0.14.54 + esbuild-android-64: 0.14.54 + esbuild-android-arm64: 0.14.54 + esbuild-darwin-64: 0.14.54 + esbuild-darwin-arm64: 0.14.54 + esbuild-freebsd-64: 0.14.54 + esbuild-freebsd-arm64: 0.14.54 + esbuild-linux-32: 0.14.54 + esbuild-linux-64: 0.14.54 + esbuild-linux-arm: 0.14.54 + esbuild-linux-arm64: 0.14.54 + esbuild-linux-mips64le: 0.14.54 + esbuild-linux-ppc64le: 0.14.54 + esbuild-linux-riscv64: 0.14.54 + esbuild-linux-s390x: 0.14.54 + esbuild-netbsd-64: 0.14.54 + esbuild-openbsd-64: 0.14.54 + esbuild-sunos-64: 0.14.54 + esbuild-windows-32: 0.14.54 + esbuild-windows-64: 0.14.54 + esbuild-windows-arm64: 0.14.54 + dependenciesMeta: + "@esbuild/linux-loong64": + optional: true + esbuild-android-64: + optional: true + esbuild-android-arm64: + optional: true + esbuild-darwin-64: + optional: true + esbuild-darwin-arm64: + optional: true + esbuild-freebsd-64: + optional: true + esbuild-freebsd-arm64: + optional: true + esbuild-linux-32: + optional: true + esbuild-linux-64: + optional: true + esbuild-linux-arm: + optional: true + esbuild-linux-arm64: + optional: true + esbuild-linux-mips64le: + optional: true + esbuild-linux-ppc64le: + optional: true + esbuild-linux-riscv64: + optional: true + esbuild-linux-s390x: + optional: true + esbuild-netbsd-64: + optional: true + esbuild-openbsd-64: + optional: true + esbuild-sunos-64: + optional: true + esbuild-windows-32: + optional: true + esbuild-windows-64: + optional: true + esbuild-windows-arm64: + optional: true + bin: + esbuild: bin/esbuild + checksum: 49e360b1185c797f5ca3a7f5f0a75121494d97ddf691f65ed1796e6257d318f928342a97f559bb8eced6a90cf604dd22db4a30e0dbbf15edd9dbf22459b639af + languageName: node + linkType: hard + "esbuild@npm:^0.15.0, esbuild@npm:^0.15.6": version: 0.15.15 resolution: "esbuild@npm:0.15.15" @@ -21369,7 +22430,7 @@ __metadata: "@backstage/plugin-events-backend": "workspace:^" "@backstage/plugin-events-node": "workspace:^" "@backstage/plugin-graphql-backend": "workspace:^" - "@backstage/plugin-incremental-ingestion-backend": ^0.0.0 + "@backstage/plugin-incremental-ingestion-backend": ^0.1.0 "@backstage/plugin-jenkins-backend": "workspace:^" "@backstage/plugin-kafka-backend": "workspace:^" "@backstage/plugin-kubernetes-backend": "workspace:^" @@ -22220,7 +23281,7 @@ __metadata: languageName: node linkType: hard -"form-data@npm:^2.3.2, form-data@npm:^2.5.0": +"form-data@npm:^2.3.1, form-data@npm:^2.3.2, form-data@npm:^2.5.0": version: 2.5.1 resolution: "form-data@npm:2.5.1" dependencies: @@ -22281,6 +23342,13 @@ __metadata: languageName: node linkType: hard +"formidable@npm:^1.2.0": + version: 1.2.6 + resolution: "formidable@npm:1.2.6" + checksum: 2b68ed07ba88302b9c63f8eda94f19a460cef6017bfda48348f09f41d2a36660c9353137991618e0e4c3db115b41e4b8f6fa63bc973b7a7c91dec66acdd02a56 + languageName: node + linkType: hard + "formidable@npm:^2.0.1": version: 2.0.1 resolution: "formidable@npm:2.0.1" @@ -22436,6 +23504,18 @@ __metadata: languageName: node linkType: hard +"fstream@npm:^1.0.12": + version: 1.0.12 + resolution: "fstream@npm:1.0.12" + dependencies: + graceful-fs: ^4.1.2 + inherits: ~2.0.0 + mkdirp: ">=0.5 0" + rimraf: 2 + checksum: e6998651aeb85fd0f0a8a68cec4d05a3ada685ecc4e3f56e0d063d0564a4fc39ad11a856f9020f926daf869fc67f7a90e891def5d48e4cadab875dc313094536 + languageName: node + linkType: hard + "function-bind@npm:^1.1.1": version: 1.1.1 resolution: "function-bind@npm:1.1.1" @@ -22690,6 +23770,16 @@ __metadata: languageName: node linkType: hard +"git-up@npm:^6.0.0": + version: 6.0.0 + resolution: "git-up@npm:6.0.0" + dependencies: + is-ssh: ^1.4.0 + parse-url: ^7.0.2 + checksum: 145a1f546d7a078cdfc2616556e518e634d134e34a31c6bf2ed89e44158659cb525dbd451c338121f7107f55cef066d0b37a7bbf178555befc9304b3940b435e + languageName: node + linkType: hard + "git-up@npm:^7.0.0": version: 7.0.0 resolution: "git-up@npm:7.0.0" @@ -22700,6 +23790,15 @@ __metadata: languageName: node linkType: hard +"git-url-parse@npm:^12.0.0": + version: 12.0.0 + resolution: "git-url-parse@npm:12.0.0" + dependencies: + git-up: ^6.0.0 + checksum: b4c8530b816202ecf9d4dabf755f785a314a096b56145018385b3d7171e862f9d0d9b38cce620c0af354b269750fe7b2d9aa95815c7150922090a11dac4ab1e6 + languageName: node + linkType: hard + "git-url-parse@npm:^13.0.0": version: 13.1.0 resolution: "git-url-parse@npm:13.1.0" @@ -22967,7 +24066,7 @@ __metadata: languageName: node linkType: hard -"graceful-fs@npm:^4.1.11, graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.5, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.1.9, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.4, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9": +"graceful-fs@npm:^4.1.11, graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.5, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.1.9, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.2, graceful-fs@npm:^4.2.4, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9": version: 4.2.10 resolution: "graceful-fs@npm:4.2.10" checksum: 3f109d70ae123951905d85032ebeae3c2a5a7a997430df00ea30df0e3a6c60cf6689b109654d6fdacd28810a053348c4d14642da1d075049e6be1ba5216218da @@ -23107,6 +24206,13 @@ __metadata: languageName: node linkType: hard +"graphql@npm:^15.5.1": + version: 15.8.0 + resolution: "graphql@npm:15.8.0" + checksum: 423325271db8858428641b9aca01699283d1fe5b40ef6d4ac622569ecca927019fce8196208b91dd1d8eb8114f00263fe661d241d0eb40c10e5bfd650f86ec5e + languageName: node + linkType: hard + "grouped-queue@npm:^2.0.0": version: 2.0.0 resolution: "grouped-queue@npm:2.0.0" @@ -23397,6 +24503,20 @@ __metadata: languageName: node linkType: hard +"headers-utils@npm:^3.0.2": + version: 3.0.2 + resolution: "headers-utils@npm:3.0.2" + checksum: 210fe65756d6de8a96afe68617463fb6faf675a24d864e849b17bddf051c4a24d621a510a1bb80fd9d4763b932eb44b5d8fd6fc4f14fa62fb211603456a57b4f + languageName: node + linkType: hard + +"helmet@npm:^5.0.2": + version: 5.1.1 + resolution: "helmet@npm:5.1.1" + checksum: b72ba26cc431804ad3b8ecdc18db95409a492cbb7a7e825efc27fc502b9433fec39fc083f2aad4fe7ed1a89a4287560b59f4435f9689eebbae6a2b61a1ec1b7d + languageName: node + linkType: hard + "helmet@npm:^6.0.0": version: 6.0.0 resolution: "helmet@npm:6.0.0" @@ -24017,7 +25137,7 @@ __metadata: languageName: node linkType: hard -"inherits@npm:2, inherits@npm:2.0.4, inherits@npm:^2.0.1, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.1, inherits@npm:~2.0.3": +"inherits@npm:2, inherits@npm:2.0.4, inherits@npm:^2.0.1, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.0, inherits@npm:~2.0.1, inherits@npm:~2.0.3": version: 2.0.4 resolution: "inherits@npm:2.0.4" checksum: 4a48a733847879d6cf6691860a6b1e3f0f4754176e4d71494c41f3475553768b10f84b5ce1d40fbd0e34e6bfbb864ee35858ad4dd2cf31e02fc4a154b724d7f1 @@ -24090,7 +25210,7 @@ __metadata: languageName: node linkType: hard -"inquirer@npm:^8.0.0, inquirer@npm:^8.2.0": +"inquirer@npm:^8.0.0, inquirer@npm:^8.1.1, inquirer@npm:^8.2.0": version: 8.2.5 resolution: "inquirer@npm:8.2.5" dependencies: @@ -26383,7 +27503,17 @@ __metadata: languageName: node linkType: hard -"keyv@npm:^4.0.0, keyv@npm:^4.5.2": +"keyv-memcache@npm:^1.2.5": + version: 1.3.3 + resolution: "keyv-memcache@npm:1.3.3" + dependencies: + json-buffer: ^3.0.1 + memjs: ^1.3.0 + checksum: ddef602c027a325fc1d18917788799e79cf73f46cbbfefeeda893ca585531544aa4b50de6601c1c5e0427a94bb52ad64e9144c919105b26aac0b173ebcdabc28 + languageName: node + linkType: hard + +"keyv@npm:^4.0.0, keyv@npm:^4.0.3, keyv@npm:^4.5.2": version: 4.5.2 resolution: "keyv@npm:4.5.2" dependencies: @@ -26659,6 +27789,13 @@ __metadata: languageName: node linkType: hard +"listenercount@npm:~1.0.1": + version: 1.0.1 + resolution: "listenercount@npm:1.0.1" + checksum: 0f1c9077cdaf2ebc16473c7d72eb7de6d983898ca42500f03da63c3914b6b312dd5f7a90d2657691ea25adf3fe0ac5a43226e8b2c673fd73415ed038041f4757 + languageName: node + linkType: hard + "listr2@npm:^3.8.3": version: 3.14.0 resolution: "listr2@npm:3.14.0" @@ -27745,7 +28882,7 @@ __metadata: languageName: node linkType: hard -"methods@npm:^1.0.0, methods@npm:^1.1.2, methods@npm:~1.1.2": +"methods@npm:^1.0.0, methods@npm:^1.1.1, methods@npm:^1.1.2, methods@npm:~1.1.2": version: 1.1.2 resolution: "methods@npm:1.1.2" checksum: 0917ff4041fa8e2f2fda5425a955fe16ca411591fbd123c0d722fcf02b73971ed6f764d85f0a6f547ce49ee0221ce2c19a5fa692157931cecb422984f1dcd13a @@ -28134,7 +29271,7 @@ __metadata: languageName: node linkType: hard -"mime@npm:1.6.0, mime@npm:^1.3.4": +"mime@npm:1.6.0, mime@npm:^1.3.4, mime@npm:^1.4.1": version: 1.6.0 resolution: "mime@npm:1.6.0" bin: @@ -28418,6 +29555,17 @@ __metadata: languageName: node linkType: hard +"mkdirp@npm:>=0.5 0": + version: 0.5.6 + resolution: "mkdirp@npm:0.5.6" + dependencies: + minimist: ^1.2.6 + bin: + mkdirp: bin/cmd.js + checksum: 0c91b721bb12c3f9af4b77ebf73604baf350e64d80df91754dc509491ae93bf238581e59c7188360cec7cb62fc4100959245a42cfe01834efedc5e9d068376c2 + languageName: node + linkType: hard + "mkdirp@npm:^0.5.1": version: 0.5.5 resolution: "mkdirp@npm:0.5.5" @@ -28500,6 +29648,36 @@ __metadata: languageName: node linkType: hard +"msw@npm:^0.35.0": + version: 0.35.0 + resolution: "msw@npm:0.35.0" + dependencies: + "@mswjs/cookies": ^0.1.6 + "@mswjs/interceptors": ^0.12.6 + "@open-draft/until": ^1.0.3 + "@types/cookie": ^0.4.1 + "@types/inquirer": ^7.3.3 + "@types/js-levenshtein": ^1.1.0 + chalk: ^4.1.1 + chokidar: ^3.4.2 + cookie: ^0.4.1 + graphql: ^15.5.1 + headers-utils: ^3.0.2 + inquirer: ^8.1.1 + is-node-process: ^1.0.1 + js-levenshtein: ^1.1.6 + node-fetch: ^2.6.1 + node-match-path: ^0.6.3 + statuses: ^2.0.0 + strict-event-emitter: ^0.2.0 + type-fest: ^1.2.2 + yargs: ^17.0.1 + bin: + msw: cli/index.js + checksum: cc5e85573e85779a95b1d5bfc306d4d0bcb4de660ac08c640e316c00fe4a0ea24dfe1decf6a46d10538550dfc7f7559836e1fda3d6e1f60ee7e99916bb3db3ff + languageName: node + linkType: hard + "msw@npm:^0.39.2": version: 0.39.2 resolution: "msw@npm:0.39.2" @@ -28908,6 +30086,13 @@ __metadata: languageName: node linkType: hard +"node-match-path@npm:^0.6.3": + version: 0.6.3 + resolution: "node-match-path@npm:0.6.3" + checksum: d515bc069f293688109c058ee02567528fdaa856290d362b80a2254734975014e4eefcdcc5164a8adfd5560aa870e277c97fe8be648074d5088056cf61553c7c + languageName: node + linkType: hard + "node-releases@npm:^2.0.6": version: 2.0.6 resolution: "node-releases@npm:2.0.6" @@ -28996,7 +30181,7 @@ __metadata: languageName: node linkType: hard -"normalize-url@npm:^6.0.1": +"normalize-url@npm:^6.0.1, normalize-url@npm:^6.1.0": version: 6.1.0 resolution: "normalize-url@npm:6.1.0" checksum: 4a4944631173e7d521d6b80e4c85ccaeceb2870f315584fa30121f505a6dfd86439c5e3fdd8cd9e0e291290c41d0c3599f0cb12ab356722ed242584c30348e50 @@ -29554,7 +30739,7 @@ __metadata: languageName: node linkType: hard -"outvariant@npm:^1.2.1, outvariant@npm:^1.3.0": +"outvariant@npm:^1.2.0, outvariant@npm:^1.2.1, outvariant@npm:^1.3.0": version: 1.3.0 resolution: "outvariant@npm:1.3.0" checksum: ac76ca375c1c642989e1c74f0e9ebac84c05bc9fdc8f28be949c16fae1658e9f1f2fb1133fe3cc1e98afabef78fe4298fe9360b5734baf8e6ad440c182680848 @@ -29934,6 +31119,15 @@ __metadata: languageName: node linkType: hard +"parse-path@npm:^5.0.0": + version: 5.0.0 + resolution: "parse-path@npm:5.0.0" + dependencies: + protocols: ^2.0.0 + checksum: e9f670559cd8e535f39f548bf5d41ad96a220190ea98df33d0babd9dfaa7c3c70ee2e55394078517d5e7e93c6a39c8eac1261ed3f9e68033656614fc954262e8 + languageName: node + linkType: hard + "parse-path@npm:^7.0.0": version: 7.0.0 resolution: "parse-path@npm:7.0.0" @@ -29943,6 +31137,18 @@ __metadata: languageName: node linkType: hard +"parse-url@npm:^7.0.2": + version: 7.0.2 + resolution: "parse-url@npm:7.0.2" + dependencies: + is-ssh: ^1.4.0 + normalize-url: ^6.1.0 + parse-path: ^5.0.0 + protocols: ^2.0.1 + checksum: 3e26852706bebe9fac409909316716dee52883d2fb5c82d65577effba1507abb7bc42bb59ce0ba6c8659168fb99acf89000bd8fe096ed3ad7124fa85227436d7 + languageName: node + linkType: hard + "parse-url@npm:^8.1.0": version: 8.1.0 resolution: "parse-url@npm:8.1.0" @@ -31625,6 +32831,15 @@ __metadata: languageName: node linkType: hard +"qs@npm:^6.5.1": + version: 6.11.0 + resolution: "qs@npm:6.11.0" + dependencies: + side-channel: ^1.0.4 + checksum: 6e1f29dd5385f7488ec74ac7b6c92f4d09a90408882d0c208414a34dd33badc1a621019d4c799a3df15ab9b1d0292f97c1dd71dc7c045e69f81a8064e5af7297 + languageName: node + linkType: hard + "qs@npm:~6.5.2": version: 6.5.2 resolution: "qs@npm:6.5.2" @@ -32430,7 +33645,7 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:^2.0.0, readable-stream@npm:^2.0.1, readable-stream@npm:^2.0.2, readable-stream@npm:^2.0.5, readable-stream@npm:^2.0.6, readable-stream@npm:^2.3.3, readable-stream@npm:^2.3.5, readable-stream@npm:^2.3.6": +"readable-stream@npm:^2.0.0, readable-stream@npm:^2.0.1, readable-stream@npm:^2.0.2, readable-stream@npm:^2.0.5, readable-stream@npm:^2.0.6, readable-stream@npm:^2.3.3, readable-stream@npm:^2.3.5, readable-stream@npm:^2.3.6, readable-stream@npm:~2.3.6": version: 2.3.7 resolution: "readable-stream@npm:2.3.7" dependencies: @@ -33168,6 +34383,17 @@ __metadata: languageName: node linkType: hard +"rimraf@npm:2, rimraf@npm:^2.6.3": + version: 2.7.1 + resolution: "rimraf@npm:2.7.1" + dependencies: + glob: ^7.1.3 + bin: + rimraf: ./bin.js + checksum: cdc7f6eacb17927f2a075117a823e1c5951792c6498ebcce81ca8203454a811d4cf8900314154d3259bb8f0b42ab17f67396a8694a54cae3283326e57ad250cd + languageName: node + linkType: hard + "rimraf@npm:3.0.2, rimraf@npm:^3.0.0, rimraf@npm:^3.0.2": version: 3.0.2 resolution: "rimraf@npm:3.0.2" @@ -33179,17 +34405,6 @@ __metadata: languageName: node linkType: hard -"rimraf@npm:^2.6.3": - version: 2.7.1 - resolution: "rimraf@npm:2.7.1" - dependencies: - glob: ^7.1.3 - bin: - rimraf: ./bin.js - checksum: cdc7f6eacb17927f2a075117a823e1c5951792c6498ebcce81ca8203454a811d4cf8900314154d3259bb8f0b42ab17f67396a8694a54cae3283326e57ad250cd - languageName: node - linkType: hard - "rimraf@npm:~2.6.2": version: 2.6.3 resolution: "rimraf@npm:2.6.3" @@ -33419,7 +34634,7 @@ __metadata: languageName: node linkType: hard -"rxjs@npm:^6.6.3": +"rxjs@npm:^6.4.0, rxjs@npm:^6.6.3": version: 6.6.7 resolution: "rxjs@npm:6.6.7" dependencies: @@ -33832,7 +35047,7 @@ __metadata: languageName: node linkType: hard -"setimmediate@npm:^1.0.4, setimmediate@npm:^1.0.5": +"setimmediate@npm:^1.0.4, setimmediate@npm:^1.0.5, setimmediate@npm:~1.0.4": version: 1.0.5 resolution: "setimmediate@npm:1.0.5" checksum: c9a6f2c5b51a2dabdc0247db9c46460152ffc62ee139f3157440bd48e7c59425093f42719ac1d7931f054f153e2d26cf37dfeb8da17a794a58198a2705e527fd @@ -35102,6 +36317,24 @@ __metadata: languageName: node linkType: hard +"superagent@npm:^3.8.3": + version: 3.8.3 + resolution: "superagent@npm:3.8.3" + dependencies: + component-emitter: ^1.2.0 + cookiejar: ^2.1.0 + debug: ^3.1.0 + extend: ^3.0.0 + form-data: ^2.3.1 + formidable: ^1.2.0 + methods: ^1.1.1 + mime: ^1.4.1 + qs: ^6.5.1 + readable-stream: ^2.3.5 + checksum: b13d0303259d76c9180bd40d97d9f0713760f5ced1aef089bdb2fcdf69cfaef89004cd6e986416d59bd9a2f0f9933d72521b5171fa26f89b781a2c3460c516fe + languageName: node + linkType: hard + "superagent@npm:^8.0.0": version: 8.0.0 resolution: "superagent@npm:8.0.0" @@ -35121,6 +36354,16 @@ __metadata: languageName: node linkType: hard +"supertest@npm:^4.0.2": + version: 4.0.2 + resolution: "supertest@npm:4.0.2" + dependencies: + methods: ^1.1.2 + superagent: ^3.8.3 + checksum: ec848088c5d0f6743a0829331c274f67ab4c2e5e6ba724f2fdf4965a73afa22d5091448323e28ae5b69137e42fe3ac67235dfc5b0a0acda3c088bdd193313319 + languageName: node + linkType: hard + "supertest@npm:^6.1.3, supertest@npm:^6.1.6, supertest@npm:^6.2.4": version: 6.2.4 resolution: "supertest@npm:6.2.4" @@ -35864,6 +37107,13 @@ __metadata: languageName: node linkType: hard +"traverse@npm:>=0.3.0 <0.4": + version: 0.3.9 + resolution: "traverse@npm:0.3.9" + checksum: 982982e4e249e9bbf063732a41fe5595939892758524bbef5d547c67cdf371b13af72b5434c6a61d88d4bb4351d6dabc6e22d832e0d16bc1bc684ef97a1cc59e + languageName: node + linkType: hard + "traverse@npm:^0.6.6, traverse@npm:~0.6.6": version: 0.6.6 resolution: "traverse@npm:0.6.6" @@ -35955,7 +37205,7 @@ __metadata: languageName: node linkType: hard -"ts-node@npm:^10.0.0, ts-node@npm:^10.4.0, ts-node@npm:^10.8.1, ts-node@npm:^10.9.1": +"ts-node@npm:^10.0.0, ts-node@npm:^10.2.1, ts-node@npm:^10.4.0, ts-node@npm:^10.8.1, ts-node@npm:^10.9.1": version: 10.9.1 resolution: "ts-node@npm:10.9.1" dependencies: @@ -36193,6 +37443,24 @@ __metadata: languageName: node linkType: hard +"typescript-json-schema@npm:^0.54.0": + version: 0.54.0 + resolution: "typescript-json-schema@npm:0.54.0" + dependencies: + "@types/json-schema": ^7.0.9 + "@types/node": ^16.9.2 + glob: ^7.1.7 + path-equal: ^1.1.2 + safe-stable-stringify: ^2.2.0 + ts-node: ^10.2.1 + typescript: ~4.6.0 + yargs: ^17.1.1 + bin: + typescript-json-schema: bin/typescript-json-schema + checksum: 49e03bd2612f79fe3ee9e9afcea34ae563da9aa799a8b4cf12b73feb60eb62a0786300eedb30261c69c482ed7e545acf1e5617d59861e6deee4f6570a658de88 + languageName: node + linkType: hard + "typescript-json-schema@npm:^0.55.0": version: 0.55.0 resolution: "typescript-json-schema@npm:0.55.0" @@ -36211,7 +37479,7 @@ __metadata: languageName: node linkType: hard -"typescript@npm:~4.6.3": +"typescript@npm:~4.6.0, typescript@npm:~4.6.3": version: 4.6.4 resolution: "typescript@npm:4.6.4" bin: @@ -36241,7 +37509,7 @@ __metadata: languageName: node linkType: hard -"typescript@patch:typescript@~4.6.3#~builtin": +"typescript@patch:typescript@~4.6.0#~builtin, typescript@patch:typescript@~4.6.3#~builtin": version: 4.6.4 resolution: "typescript@patch:typescript@npm%3A4.6.4#~builtin::version=4.6.4&hash=a1c5e5" bin: @@ -36575,6 +37843,24 @@ __metadata: languageName: node linkType: hard +"unzipper@npm:^0.10.11": + version: 0.10.11 + resolution: "unzipper@npm:0.10.11" + dependencies: + big-integer: ^1.6.17 + binary: ~0.3.0 + bluebird: ~3.4.1 + buffer-indexof-polyfill: ~1.0.0 + duplexer2: ~0.1.4 + fstream: ^1.0.12 + graceful-fs: ^4.2.2 + listenercount: ~1.0.1 + readable-stream: ~2.3.6 + setimmediate: ~1.0.4 + checksum: 006cd43ec4d6df47d86aa6b15044a606f50cdcd6a3d6f96f64f54ca0b663c09abb221f76edca0e9592511036d37ea094b1d76ce92c5bf10d7c6eb56f0be678f8 + languageName: node + linkType: hard + "update-browserslist-db@npm:^1.0.9": version: 1.0.9 resolution: "update-browserslist-db@npm:1.0.9" @@ -37885,7 +39171,7 @@ __metadata: languageName: node linkType: hard -"yargs-parser@npm:^21.0.0": +"yargs-parser@npm:^21.0.0, yargs-parser@npm:^21.1.1": version: 21.1.1 resolution: "yargs-parser@npm:21.1.1" checksum: ed2d96a616a9e3e1cc7d204c62ecc61f7aaab633dcbfab2c6df50f7f87b393993fe6640d017759fe112d0cb1e0119f2b4150a87305cc873fd90831c6a58ccf1c @@ -37951,6 +39237,21 @@ __metadata: languageName: node linkType: hard +"yargs@npm:^17.0.1": + version: 17.6.2 + resolution: "yargs@npm:17.6.2" + dependencies: + cliui: ^8.0.1 + escalade: ^3.1.1 + get-caller-file: ^2.0.5 + require-directory: ^2.1.1 + string-width: ^4.2.3 + y18n: ^5.0.5 + yargs-parser: ^21.1.1 + checksum: 47da1b0d854fa16d45a3ded57b716b013b2179022352a5f7467409da5a04a1eef5b3b3d97a2dfc13e8bbe5f2ffc0afe3bc6a4a72f8254e60f5a4bd7947138643 + languageName: node + linkType: hard + "yargs@npm:^5.0.0": version: 5.0.0 resolution: "yargs@npm:5.0.0" From 39c36c7caa4302641a0566827d5426b2b667f066 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Wed, 26 Oct 2022 22:25:08 -0700 Subject: [PATCH 07/54] Correction for README.md and removing company-specific annotation Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/README.md | 13 ++++++------- .../migrations/20220613125155_init.js | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/plugins/incremental-ingestion-backend/README.md b/plugins/incremental-ingestion-backend/README.md index 92ed56862d..d06334f396 100644 --- a/plugins/incremental-ingestion-backend/README.md +++ b/plugins/incremental-ingestion-backend/README.md @@ -21,7 +21,7 @@ We created the Incremental Entity Provider to address all of the above issues. T Incremental Entity Providers will wait a configurable interval before proceeding to the next burst. -Once the source has no more results, Incremental Entity Provider compares all entities annotated with `frontside/incremental-entity-provider: ` against all marked entities to determine which entities committed by same entity provider were not marked during the last ingestion cycle. All unmarked entities are deleted at the end of the cycle. The Incremental Entity Provider rests for a fixed internal before restarting the ingestion process. +Once the source has no more results, Incremental Entity Provider compares all entities annotated with `@backstage/incremental-entity-provider: ` against all marked entities to determine which entities committed by same entity provider were not marked during the last ingestion cycle. All unmarked entities are deleted at the end of the cycle. The Incremental Entity Provider rests for a fixed internal before restarting the ingestion process. ![Diagram of execution of an Incremental Entity Provider](https://user-images.githubusercontent.com/74687/185822734-ee6279c7-64fa-46b9-9aa8-d4092ab73858.png) @@ -38,7 +38,7 @@ The Incremental Entity Provider backend is designed for data sources that provid 1. The cursor must be serializable to JSON (not an issue for most RESTful or GraphQL based APIs). 2. The client must be stateless - a client is created from scratch for each iteration to allow distributing processing over multiple replicas. -3. There must be sufficient storage in Postgres to handle the additional data. +3. There must be sufficient storage in Postgres to handle the additional data. (Presumably, this is also true of sqlite, but it has only been tested with Postgres.) ## Installation @@ -66,8 +66,7 @@ import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; -import { IncrementalCatalogBuilder } from '@frontside/backstage-plugin-incremental-ingestion-backend'; -import { GithubRepositoryEntityProvider } from '@frontside/backstage-plugin-incremental-ingestion-github'; +import { IncrementalCatalogBuilder } from '@backstage/plugin-incremental-ingestion-backend'; import { Router } from 'express'; import { Duration } from 'luxon'; import { PluginEnvironment } from '../types'; @@ -87,9 +86,9 @@ export default async function createPlugin( // this has to run after `await builder.build()` so ensure that catalog migrations are completed // before incremental builder migrations are executed - await incrementalBuilder.build(); - // If you want to use a suite of administrative routes to control the incremental providers, use - // `const { incrementalAdminRouter } = await incrementalBuilder.build();` + const { incrementalAdminRouter } = await incrementalBuilder.build(); + + router.use('/incremental', incrementalAdminRouter); await processingEngine.start(); diff --git a/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js b/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js index a17f71d119..b228da2f28 100644 --- a/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js +++ b/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js @@ -21,7 +21,7 @@ CREATE VIEW ingestion.current_entities as SELECT LOWER(final_entity::json #>> '{kind}'), LOWER(final_entity::json #>> '{metadata, namespace}'), LOWER(final_entity::json #>> '{metadata, name}')) as ref, - final_entity::json #>> '{metadata, annotations, hp.com/provider-name}' as provider_name, + final_entity::json #>> '{metadata, annotations, backstage.io/incremental-provider-name}' as provider_name, final_entity FROM public.final_entities; `); From 29b8eb0507167e04804adf4a63b05657cca39ef6 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 27 Oct 2022 08:09:30 -0700 Subject: [PATCH 08/54] Use workspace dependencies Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/package.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/incremental-ingestion-backend/package.json b/plugins/incremental-ingestion-backend/package.json index 2d55704a48..e3a8495484 100644 --- a/plugins/incremental-ingestion-backend/package.json +++ b/plugins/incremental-ingestion-backend/package.json @@ -23,18 +23,18 @@ "postpack": "backstage-cli package postpack" }, "dependencies": { - "@backstage/backend-common": "^0.14.0", - "@backstage/backend-tasks": "^0.3.6", - "@backstage/catalog-model": "^1.1.2", - "@backstage/config": "^1.0.0", - "@backstage/plugin-catalog-backend": "^1.5.0", - "@backstage/plugin-permission-common": "^0.7.0", + "@backstage/backend-common": "workspace:^", + "@backstage/backend-tasks": "workspace:^", + "@backstage/catalog-model": "workspace:^", + "@backstage/config": "workspace:^", + "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-permission-common": "workspace:^", "express": "^4.17.1", "express-promise-router": "^4.1.0", "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.20.0", + "@backstage/cli": "workspace:^", "@types/express": "*", "@types/supertest": "^2.0.8", "msw": "^0.35.0", From ad1beb072a71b8d0e70c6ef556ef5dab788b39e6 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 27 Oct 2022 08:33:29 -0700 Subject: [PATCH 09/54] Remove currently unused dependencies Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/package.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/plugins/incremental-ingestion-backend/package.json b/plugins/incremental-ingestion-backend/package.json index e3a8495484..e103448d69 100644 --- a/plugins/incremental-ingestion-backend/package.json +++ b/plugins/incremental-ingestion-backend/package.json @@ -35,10 +35,7 @@ }, "devDependencies": { "@backstage/cli": "workspace:^", - "@types/express": "*", - "@types/supertest": "^2.0.8", - "msw": "^0.35.0", - "supertest": "^4.0.2" + "@types/express": "*" }, "files": [ "dist", From 484be3318ea9b0e50a3afca165d68c56c4508ecd Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 27 Oct 2022 08:33:51 -0700 Subject: [PATCH 10/54] Add missing migration file Signed-off-by: Damon Kaswell --- ...21019897897_add-unique-active-ingestion.js | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 plugins/incremental-ingestion-backend/migrations/20221019897897_add-unique-active-ingestion.js diff --git a/plugins/incremental-ingestion-backend/migrations/20221019897897_add-unique-active-ingestion.js b/plugins/incremental-ingestion-backend/migrations/20221019897897_add-unique-active-ingestion.js new file mode 100644 index 0000000000..0fa80ad34a --- /dev/null +++ b/plugins/incremental-ingestion-backend/migrations/20221019897897_add-unique-active-ingestion.js @@ -0,0 +1,86 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const { v4: uuidv4 } = require('uuid'); + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = async function (knex) { + const schema = () => knex.schema.withSchema('ingestion'); + + await knex.transaction(async tx => { + const providers = await tx('ingestion.ingestions').distinct('provider_name'); + for (const provider of providers) { + const valid = await tx('ingestion.ingestions') + .where('provider_name', provider) + .andWhere('rest_completed_at', null) + .first(); + const invalid = []; + + const rows = await tx('ingestion.ingestions') + .where('provider_name', provider) + .andWhere('rest_completed_at', null); + for (const row of rows) { + if (row.id !== valid.id) { + invalid.push(row.id); + } + } + + if (invalid.length > 0) { + await tx('ingestion.ingestions').delete().whereIn('id', invalid); + await tx('ingestion.ingestion_mark_entities') + .delete() + .whereIn('ingestion_mark_id', tx('ingestion.ingestion_marks').select('id').whereIn('ingestion_id', invalid)); + await tx('ingestion.ingestion_marks').delete().whereIn('ingestion_id', invalid); + } + } + }); + + await schema().alterTable('ingestions', t => { + t.string('completion_ticket'); + }); + + await knex.transaction(async tx => { + await tx('ingestion.ingestions').update('completion_ticket', 'open').where('rest_completed_at', null); + + const rows = await tx('ingestion.ingestions').whereNot('rest_completed_at', null); + for (const row of rows) { + await tx('ingestion.ingestions').update('completion_ticket', uuidv4()).where('id', row.id); + } + }); + + await schema().alterTable('ingestions', t => { + t.string('completion_ticket').notNullable().alter(); + t.unique(['provider_name', 'completion_ticket'], { + indexName: 'ingestion_composite_index', + deferrable: 'deferred', + }); + }); +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function (knex) { + const schema = () => knex.schema.withSchema('ingestion'); + + await schema().alterTable('ingestions', t => { + t.dropUnique(['provider_name', 'completion_ticket']); + }); +}; From 9087451fb1f2c83e58415ea86d23e07ddda9f15d Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 27 Oct 2022 11:04:22 -0700 Subject: [PATCH 11/54] Sync dependency versions and apply formatting to migrations Signed-off-by: Damon Kaswell --- .../migrations/20220613125155_init.js | 98 +- .../20220906161750_add-ingestion-indexes.js | 13 +- ...21019897897_add-unique-active-ingestion.js | 6 +- .../package.json | 2 +- yarn.lock | 1398 +---------------- 5 files changed, 127 insertions(+), 1390 deletions(-) diff --git a/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js b/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js index b228da2f28..b31bc09cf0 100644 --- a/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js +++ b/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js @@ -14,7 +14,10 @@ * limitations under the License. */ -exports.up = async function (knex) { +/** + * @param { import("knex").Knex } knex + */ +exports.up = async function up(knex) { await knex.raw(` CREATE VIEW ingestion.current_entities as SELECT FORMAT('%s:%s/%s', @@ -30,77 +33,114 @@ CREATE VIEW ingestion.current_entities as SELECT await schema().createTable('ingestions', table => { table.comment('Tracks ingestion streams for very large data sets'); - table.uuid('id', { primary: true }).notNullable().comment('Auto-generated ID of the ingestion'); + table + .uuid('id', { primary: true }) + .notNullable() + .comment('Auto-generated ID of the ingestion'); - table.string('provider_name').notNullable().comment('each provider gets its own identifiable name'); + table + .string('provider_name') + .notNullable() + .comment('each provider gets its own identifiable name'); table .string('status') .notNullable() - .comment('One of "interstitial" | "bursting" | "backing off" | "resting" | "complete"'); + .comment( + 'One of "interstitial" | "bursting" | "backing off" | "resting" | "complete"', + ); - table.string('next_action').notNullable().comment("what will this, 'ingest', 'rest', 'backoff', 'nothing (done)'"); + table + .string('next_action') + .notNullable() + .comment("what will this, 'ingest', 'rest', 'backoff', 'nothing (done)'"); table .timestamp('next_action_at') .defaultTo(knex.fn.now()) .comment('the moment in time at which point ingestion can begin again'); - table.string('last_error').comment('records any error that occured in the previous burst attempt'); + table + .string('last_error') + .comment('records any error that occured in the previous burst attempt'); - table.integer('attempts').defaultTo(0).comment('how many attempts have been made to burst without success'); + table + .integer('attempts') + .defaultTo(0) + .comment('how many attempts have been made to burst without success'); - table.timestamp('created_at').defaultTo(knex.fn.now()).comment('when did this ingestion actually begin'); + table + .timestamp('created_at') + .defaultTo(knex.fn.now()) + .comment('when did this ingestion actually begin'); - table.timestamp('ingestion_completed_at').comment('when did the ingestion actually end'); + table + .timestamp('ingestion_completed_at') + .comment('when did the ingestion actually end'); - table.timestamp('rest_completed_at').comment('when did the rest period actually end'); + table + .timestamp('rest_completed_at') + .comment('when did the rest period actually end'); }); await schema().createTable('ingestion_marks', table => { table.comment('tracks each step of an iterative ingestion'); - table.uuid('id', { primary: true }).notNullable().comment('Auto-generated ID of the ingestion mark'); + table + .uuid('id', { primary: true }) + .notNullable() + .comment('Auto-generated ID of the ingestion mark'); - table.uuid('ingestion_id').notNullable().comment('The id of the ingestion in which this mark took place'); - - // table - // .foreign('ingestion_id').references('ingestions.id'); + table + .uuid('ingestion_id') + .notNullable() + .comment('The id of the ingestion in which this mark took place'); table .json('cursor') - .comment('the current data associated with this iteration wherever it is in this moment in time'); + .comment( + 'the current data associated with this iteration wherever it is in this moment in time', + ); - table.integer('sequence').defaultTo(0).comment('what is the order of this mark'); + table + .integer('sequence') + .defaultTo(0) + .comment('what is the order of this mark'); table.timestamp('created_at').defaultTo(knex.fn.now()); }); await schema().createTable('ingestion_mark_entities', table => { - table.comment('tracks the entities recorded in each step of an iterative ingestion'); + table.comment( + 'tracks the entities recorded in each step of an iterative ingestion', + ); - table.uuid('id', { primary: true }).notNullable().comment('Auto-generated ID of the marked entity'); + table + .uuid('id', { primary: true }) + .notNullable() + .comment('Auto-generated ID of the marked entity'); table .uuid('ingestion_mark_id') .notNullable() - .comment('Every time a mark happens during an ingestion, there are a list of entities marked.'); + .comment( + 'Every time a mark happens during an ingestion, there are a list of entities marked.', + ); - // table - // .foreign('ingestion_mark_id').references('ingestion_marks.id'); - - table.string('ref').notNullable().comment('the entity reference of the marked entity'); + table + .string('ref') + .notNullable() + .comment('the entity reference of the marked entity'); }); }; -exports.down = async function (knex) { +/** + * @param { import("knex").Knex } knex + */ +exports.down = async function down(knex) { const schema = () => knex.schema.withSchema('ingestion'); - await schema().dropView('current_entities'); - await schema().dropTable('ingestion_mark_entities'); - await schema().dropTable('ingestion_marks'); - await schema().dropTable('ingestions'); }; diff --git a/plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js b/plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js index 78c6afe796..779aa2241b 100644 --- a/plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js +++ b/plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js @@ -15,10 +15,9 @@ */ /** - * @param { import('knex').Knex } knex - * @returns { Promise } + * @param { import("knex").Knex } knex */ -exports.up = async function (knex) { +exports.up = async function up(knex) { const schema = () => knex.schema.withSchema('ingestion'); await knex.raw( @@ -45,9 +44,8 @@ exports.up = async function (knex) { /** * @param { import("knex").Knex } knex - * @returns { Promise } */ -exports.down = async function (knex) { +exports.down = async function down(knex) { const schema = () => knex.schema.withSchema('ingestion'); await schema().alterTable('ingestions', t => { @@ -61,7 +59,10 @@ exports.down = async function (knex) { }); await schema().alterTable('ingestions_mark_entities', t => { - t.dropIndex('ingestion_mark_id', 'ingestion_mark_entity_ingestion_mark_id_idx'); + t.dropIndex( + 'ingestion_mark_id', + 'ingestion_mark_entity_ingestion_mark_id_idx', + ); t.dropPrimary('id'); }); diff --git a/plugins/incremental-ingestion-backend/migrations/20221019897897_add-unique-active-ingestion.js b/plugins/incremental-ingestion-backend/migrations/20221019897897_add-unique-active-ingestion.js index 0fa80ad34a..b5ce148efd 100644 --- a/plugins/incremental-ingestion-backend/migrations/20221019897897_add-unique-active-ingestion.js +++ b/plugins/incremental-ingestion-backend/migrations/20221019897897_add-unique-active-ingestion.js @@ -18,9 +18,8 @@ const { v4: uuidv4 } = require('uuid'); /** * @param { import("knex").Knex } knex - * @returns { Promise } */ -exports.up = async function (knex) { +exports.up = async function up(knex) { const schema = () => knex.schema.withSchema('ingestion'); await knex.transaction(async tx => { @@ -75,9 +74,8 @@ exports.up = async function (knex) { /** * @param { import("knex").Knex } knex - * @returns { Promise } */ -exports.down = async function (knex) { +exports.down = async function down(knex) { const schema = () => knex.schema.withSchema('ingestion'); await schema().alterTable('ingestions', t => { diff --git a/plugins/incremental-ingestion-backend/package.json b/plugins/incremental-ingestion-backend/package.json index e103448d69..199694ba63 100644 --- a/plugins/incremental-ingestion-backend/package.json +++ b/plugins/incremental-ingestion-backend/package.json @@ -35,7 +35,7 @@ }, "devDependencies": { "@backstage/cli": "workspace:^", - "@types/express": "*" + "@types/express": "^4.17.6" }, "files": [ "dist", diff --git a/yarn.lock b/yarn.lock index 4629a23941..fe0d4c4db0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1419,13 +1419,6 @@ __metadata: languageName: node linkType: hard -"@babel/compat-data@npm:^7.20.0": - version: 7.20.1 - resolution: "@babel/compat-data@npm:7.20.1" - checksum: 989b9b7a6fe43c547bb8329241bd0ba6983488b83d29cc59de35536272ee6bb4cc7487ba6c8a4bceebb3a57f8c5fea1434f80bbbe75202bc79bc1110f955ff25 - languageName: node - linkType: hard - "@babel/core@npm:^7.11.6, @babel/core@npm:^7.12.3, @babel/core@npm:^7.13.16, @babel/core@npm:^7.14.0, @babel/core@npm:^7.18.5": version: 7.19.1 resolution: "@babel/core@npm:7.19.1" @@ -1449,29 +1442,6 @@ __metadata: languageName: node linkType: hard -"@babel/core@npm:^7.19.6": - version: 7.20.2 - resolution: "@babel/core@npm:7.20.2" - dependencies: - "@ampproject/remapping": ^2.1.0 - "@babel/code-frame": ^7.18.6 - "@babel/generator": ^7.20.2 - "@babel/helper-compilation-targets": ^7.20.0 - "@babel/helper-module-transforms": ^7.20.2 - "@babel/helpers": ^7.20.1 - "@babel/parser": ^7.20.2 - "@babel/template": ^7.18.10 - "@babel/traverse": ^7.20.1 - "@babel/types": ^7.20.2 - convert-source-map: ^1.7.0 - debug: ^4.1.0 - gensync: ^1.0.0-beta.2 - json5: ^2.2.1 - semver: ^6.3.0 - checksum: 98faaaef26103a276a30a141b951a93bc8418d100d1f668bf7a69d12f3e25df57958e8b6b9100d95663f720db62da85ade736f6629a5ebb1e640251a1b43c0e4 - languageName: node - linkType: hard - "@babel/generator@npm:^7.14.0, @babel/generator@npm:^7.18.13, @babel/generator@npm:^7.19.0, @babel/generator@npm:^7.7.2": version: 7.19.3 resolution: "@babel/generator@npm:7.19.3" @@ -1483,17 +1453,6 @@ __metadata: languageName: node linkType: hard -"@babel/generator@npm:^7.20.1, @babel/generator@npm:^7.20.2": - version: 7.20.4 - resolution: "@babel/generator@npm:7.20.4" - dependencies: - "@babel/types": ^7.20.2 - "@jridgewell/gen-mapping": ^0.3.2 - jsesc: ^2.5.1 - checksum: 967b59f18e5ce999e5a741825bcecb2be4bbfc1824a92c21b47d0b5694e0eb09314a70f8b9142e9591c149c7fb83d51f73ae8fbd96d30a42666425889e51ceb1 - languageName: node - linkType: hard - "@babel/helper-annotate-as-pure@npm:^7.16.0, @babel/helper-annotate-as-pure@npm:^7.18.6": version: 7.18.6 resolution: "@babel/helper-annotate-as-pure@npm:7.18.6" @@ -1527,20 +1486,6 @@ __metadata: languageName: node linkType: hard -"@babel/helper-compilation-targets@npm:^7.20.0": - version: 7.20.0 - resolution: "@babel/helper-compilation-targets@npm:7.20.0" - dependencies: - "@babel/compat-data": ^7.20.0 - "@babel/helper-validator-option": ^7.18.6 - browserslist: ^4.21.3 - semver: ^6.3.0 - peerDependencies: - "@babel/core": ^7.0.0 - checksum: bc183f2109648849c8fde0b3c5cf08adf2f7ad6dc617b546fd20f34c8ef574ee5ee293c8d1bd0ed0221212e8f5907cdc2c42097870f1dcc769a654107d82c95b - languageName: node - linkType: hard - "@babel/helper-create-class-features-plugin@npm:^7.18.6": version: 7.18.9 resolution: "@babel/helper-create-class-features-plugin@npm:7.18.9" @@ -1657,22 +1602,6 @@ __metadata: languageName: node linkType: hard -"@babel/helper-module-transforms@npm:^7.20.2": - version: 7.20.2 - resolution: "@babel/helper-module-transforms@npm:7.20.2" - dependencies: - "@babel/helper-environment-visitor": ^7.18.9 - "@babel/helper-module-imports": ^7.18.6 - "@babel/helper-simple-access": ^7.20.2 - "@babel/helper-split-export-declaration": ^7.18.6 - "@babel/helper-validator-identifier": ^7.19.1 - "@babel/template": ^7.18.10 - "@babel/traverse": ^7.20.1 - "@babel/types": ^7.20.2 - checksum: 33a60ca115f6fce2c9d98e2a2e5649498aa7b23e2ae3c18745d7a021487708fc311458c33542f299387a0da168afccba94116e077f2cce49ae9e5ab83399e8a2 - languageName: node - linkType: hard - "@babel/helper-optimise-call-expression@npm:^7.18.6": version: 7.18.6 resolution: "@babel/helper-optimise-call-expression@npm:7.18.6" @@ -1725,15 +1654,6 @@ __metadata: languageName: node linkType: hard -"@babel/helper-simple-access@npm:^7.20.2": - version: 7.20.2 - resolution: "@babel/helper-simple-access@npm:7.20.2" - dependencies: - "@babel/types": ^7.20.2 - checksum: ad1e96ee2e5f654ffee2369a586e5e8d2722bf2d8b028a121b4c33ebae47253f64d420157b9f0a8927aea3a9e0f18c0103e74fdd531815cf3650a0a4adca11a1 - languageName: node - linkType: hard - "@babel/helper-skip-transparent-expression-wrappers@npm:^7.18.9": version: 7.18.9 resolution: "@babel/helper-skip-transparent-expression-wrappers@npm:7.18.9" @@ -1759,13 +1679,6 @@ __metadata: languageName: node linkType: hard -"@babel/helper-string-parser@npm:^7.19.4": - version: 7.19.4 - resolution: "@babel/helper-string-parser@npm:7.19.4" - checksum: b2f8a3920b30dfac81ec282ac4ad9598ea170648f8254b10f475abe6d944808fb006aab325d3eb5a8ad3bea8dfa888cfa6ef471050dae5748497c110ec060943 - languageName: node - linkType: hard - "@babel/helper-validator-identifier@npm:^7.18.6, @babel/helper-validator-identifier@npm:^7.19.1": version: 7.19.1 resolution: "@babel/helper-validator-identifier@npm:7.19.1" @@ -1803,17 +1716,6 @@ __metadata: languageName: node linkType: hard -"@babel/helpers@npm:^7.20.1": - version: 7.20.1 - resolution: "@babel/helpers@npm:7.20.1" - dependencies: - "@babel/template": ^7.18.10 - "@babel/traverse": ^7.20.1 - "@babel/types": ^7.20.0 - checksum: be35f78666bdab895775ed94dbeb098f7b4fa08ce4cfb0c3a9e69b7220cce56960dcdc2b14f5df9d3b80388d4bf7df155c97f6cf6768c0138f4e6931d0f44955 - languageName: node - linkType: hard - "@babel/highlight@npm:^7.0.0, @babel/highlight@npm:^7.18.6": version: 7.18.6 resolution: "@babel/highlight@npm:7.18.6" @@ -1834,15 +1736,6 @@ __metadata: languageName: node linkType: hard -"@babel/parser@npm:^7.20.1, @babel/parser@npm:^7.20.2": - version: 7.20.3 - resolution: "@babel/parser@npm:7.20.3" - bin: - parser: ./bin/babel-parser.js - checksum: 33bcdb45de65a3cf27ed376cb34f32be3c3485a10e3252f8d0126f6a034efc3145c0d219e57fcd5a8956361552008bc30b9bae4a723823fb3633027071be8a45 - languageName: node - linkType: hard - "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@npm:^7.18.6": version: 7.18.6 resolution: "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@npm:7.18.6" @@ -2946,24 +2839,6 @@ __metadata: languageName: node linkType: hard -"@babel/traverse@npm:^7.20.1": - version: 7.20.1 - resolution: "@babel/traverse@npm:7.20.1" - dependencies: - "@babel/code-frame": ^7.18.6 - "@babel/generator": ^7.20.1 - "@babel/helper-environment-visitor": ^7.18.9 - "@babel/helper-function-name": ^7.19.0 - "@babel/helper-hoist-variables": ^7.18.6 - "@babel/helper-split-export-declaration": ^7.18.6 - "@babel/parser": ^7.20.1 - "@babel/types": ^7.20.0 - debug: ^4.1.0 - globals: ^11.1.0 - checksum: 6696176d574b7ff93466848010bc7e94b250169379ec2a84f1b10da46a7cc2018ea5e3a520c3078487db51e3a4afab9ecff48f25d1dbad8c1319362f4148fb4b - languageName: node - linkType: hard - "@babel/types@npm:^7.0.0, @babel/types@npm:^7.16.8, @babel/types@npm:^7.18.10, @babel/types@npm:^7.18.13, @babel/types@npm:^7.18.4, @babel/types@npm:^7.18.6, @babel/types@npm:^7.18.9, @babel/types@npm:^7.19.0, @babel/types@npm:^7.19.3, @babel/types@npm:^7.3.0, @babel/types@npm:^7.3.3, @babel/types@npm:^7.4.4, @babel/types@npm:^7.8.3": version: 7.19.3 resolution: "@babel/types@npm:7.19.3" @@ -2975,17 +2850,6 @@ __metadata: languageName: node linkType: hard -"@babel/types@npm:^7.20.0, @babel/types@npm:^7.20.2": - version: 7.20.2 - resolution: "@babel/types@npm:7.20.2" - dependencies: - "@babel/helper-string-parser": ^7.19.4 - "@babel/helper-validator-identifier": ^7.19.1 - to-fast-properties: ^2.0.0 - checksum: 57e76e5f21876135f481bfd4010c87f2d38196bb0a2bc60a28d6e55e3afa90cdd9accf164e4cb71bdfb620517fa0a0cb5600cdce36c21d59fdaccfbb899c024c - languageName: node - linkType: hard - "@backstage/app-defaults@workspace:^, @backstage/app-defaults@workspace:packages/app-defaults": version: 0.0.0-use.local resolution: "@backstage/app-defaults@workspace:packages/app-defaults" @@ -3027,130 +2891,6 @@ __metadata: languageName: unknown linkType: soft -"@backstage/backend-common@npm:^0.14.0": - version: 0.14.1 - resolution: "@backstage/backend-common@npm:0.14.1" - dependencies: - "@backstage/cli-common": ^0.1.9 - "@backstage/config": ^1.0.1 - "@backstage/config-loader": ^1.1.3 - "@backstage/errors": ^1.1.0 - "@backstage/integration": ^1.2.2 - "@backstage/types": ^1.0.0 - "@google-cloud/storage": ^6.0.0 - "@keyv/redis": ^2.2.3 - "@manypkg/get-packages": ^1.1.3 - "@octokit/rest": ^19.0.3 - "@types/cors": ^2.8.6 - "@types/dockerode": ^3.3.0 - "@types/express": ^4.17.6 - "@types/luxon": ^2.0.4 - "@types/webpack-env": ^1.15.2 - archiver: ^5.0.2 - aws-sdk: ^2.840.0 - base64-stream: ^1.0.0 - compression: ^1.7.4 - concat-stream: ^2.0.0 - cors: ^2.8.5 - dockerode: ^3.3.1 - express: ^4.17.1 - express-promise-router: ^4.1.0 - fs-extra: 10.1.0 - git-url-parse: ^12.0.0 - helmet: ^5.0.2 - isomorphic-git: ^1.8.0 - jose: ^4.6.0 - keyv: ^4.0.3 - keyv-memcache: ^1.2.5 - knex: ^2.0.0 - lodash: ^4.17.21 - logform: ^2.3.2 - luxon: ^3.0.0 - minimatch: ^5.0.0 - minimist: ^1.2.5 - morgan: ^1.10.0 - node-abort-controller: ^3.0.1 - node-fetch: ^2.6.7 - raw-body: ^2.4.1 - selfsigned: ^2.0.0 - stoppable: ^1.1.0 - tar: ^6.1.2 - unzipper: ^0.10.11 - winston: ^3.2.1 - yn: ^4.0.0 - peerDependencies: - pg-connection-string: ^2.3.0 - peerDependenciesMeta: - pg-connection-string: - optional: true - checksum: 888e42a543bcf8ffcb3802301ff904f34f520393cc0fc1adfa203b4e77928132ce6d587f5a155c338e9c8c51c55fd11a5e330ed444e0d4a8281680928313c81f - languageName: node - linkType: hard - -"@backstage/backend-common@npm:^0.16.0": - version: 0.16.0 - resolution: "@backstage/backend-common@npm:0.16.0" - dependencies: - "@backstage/cli-common": ^0.1.10 - "@backstage/config": ^1.0.4 - "@backstage/config-loader": ^1.1.6 - "@backstage/errors": ^1.1.3 - "@backstage/integration": ^1.4.0 - "@backstage/types": ^1.0.1 - "@google-cloud/storage": ^6.0.0 - "@keyv/memcache": ^1.3.5 - "@keyv/redis": ^2.5.3 - "@kubernetes/client-node": 0.17.0 - "@manypkg/get-packages": ^1.1.3 - "@octokit/rest": ^19.0.3 - "@types/cors": ^2.8.6 - "@types/dockerode": ^3.3.0 - "@types/express": ^4.17.6 - "@types/luxon": ^3.0.0 - "@types/webpack-env": ^1.15.2 - archiver: ^5.0.2 - aws-sdk: ^2.840.0 - base64-stream: ^1.0.0 - compression: ^1.7.4 - concat-stream: ^2.0.0 - cors: ^2.8.5 - dockerode: ^3.3.1 - express: ^4.17.1 - express-promise-router: ^4.1.0 - fs-extra: 10.1.0 - git-url-parse: ^13.0.0 - helmet: ^6.0.0 - isomorphic-git: ^1.8.0 - jose: ^4.6.0 - keyv: ^4.5.2 - knex: ^2.0.0 - lodash: ^4.17.21 - logform: ^2.3.2 - luxon: ^3.0.0 - minimatch: ^5.0.0 - minimist: ^1.2.5 - morgan: ^1.10.0 - node-abort-controller: ^3.0.1 - node-fetch: ^2.6.7 - node-forge: ^1.3.1 - raw-body: ^2.4.1 - request: ^2.88.2 - selfsigned: ^2.0.0 - stoppable: ^1.1.0 - tar: ^6.1.12 - uuid: ^8.3.2 - winston: ^3.2.1 - yauzl: ^2.10.0 - yn: ^4.0.0 - peerDependencies: - pg-connection-string: ^2.3.0 - peerDependenciesMeta: - pg-connection-string: - optional: true - checksum: 92d599323d188b22e1824490158a5e7d8fea3ee3cf5cd2bc50841e9a4a37d3cc02d046a25d7ded0050e9cbcba5849aed13f5535e329691572ea8d74d92a38623 - languageName: node - linkType: hard - "@backstage/backend-common@workspace:^, @backstage/backend-common@workspace:packages/backend-common": version: 0.0.0-use.local resolution: "@backstage/backend-common@workspace:packages/backend-common" @@ -3249,22 +2989,6 @@ __metadata: languageName: unknown linkType: soft -"@backstage/backend-plugin-api@npm:^0.1.4": - version: 0.1.4 - resolution: "@backstage/backend-plugin-api@npm:0.1.4" - dependencies: - "@backstage/backend-common": ^0.16.0 - "@backstage/backend-tasks": ^0.3.7 - "@backstage/config": ^1.0.4 - "@backstage/plugin-permission-common": ^0.7.1 - "@types/express": ^4.17.6 - express: ^4.17.1 - winston: ^3.2.1 - winston-transport: ^4.5.0 - checksum: 991176c0047cc4125b9be9551f71e675e86f51826a90e2ce6161410ea6bed680e68abc659aef2c6c4e679202e89af941d7302aba3d478d177e694a7d7e50870c - languageName: node - linkType: hard - "@backstage/backend-plugin-api@workspace:^, @backstage/backend-plugin-api@workspace:packages/backend-plugin-api": version: 0.0.0-use.local resolution: "@backstage/backend-plugin-api@workspace:packages/backend-plugin-api" @@ -3281,27 +3005,6 @@ __metadata: languageName: unknown linkType: soft -"@backstage/backend-tasks@npm:^0.3.6, @backstage/backend-tasks@npm:^0.3.7": - version: 0.3.7 - resolution: "@backstage/backend-tasks@npm:0.3.7" - dependencies: - "@backstage/backend-common": ^0.16.0 - "@backstage/config": ^1.0.4 - "@backstage/errors": ^1.1.3 - "@backstage/types": ^1.0.1 - "@types/luxon": ^3.0.0 - cron: ^2.0.0 - knex: ^2.0.0 - lodash: ^4.17.21 - luxon: ^3.0.0 - node-abort-controller: ^3.0.1 - uuid: ^8.0.0 - winston: ^3.2.1 - zod: ^3.9.5 - checksum: becfac6867c4301c1c92b39b8a254259133bb5c4684e4fc98256e0dd3739891c7e17ae04c8266ee9b56dda340fd6acf3aa294482391e5678495a3b0a7066fab4 - languageName: node - linkType: hard - "@backstage/backend-tasks@workspace:^, @backstage/backend-tasks@workspace:packages/backend-tasks": version: 0.0.0-use.local resolution: "@backstage/backend-tasks@workspace:packages/backend-tasks" @@ -3401,7 +3104,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/cli-common@^0.1.10, @backstage/cli-common@^0.1.9, @backstage/cli-common@workspace:^, @backstage/cli-common@workspace:packages/cli-common": +"@backstage/cli-common@workspace:^, @backstage/cli-common@workspace:packages/cli-common": version: 0.0.0-use.local resolution: "@backstage/cli-common@workspace:packages/cli-common" dependencies: @@ -3410,118 +3113,6 @@ __metadata: languageName: unknown linkType: soft -"@backstage/cli@npm:^0.20.0": - version: 0.20.0 - resolution: "@backstage/cli@npm:0.20.0" - dependencies: - "@backstage/cli-common": ^0.1.10 - "@backstage/config": ^1.0.3 - "@backstage/config-loader": ^1.1.5 - "@backstage/errors": ^1.1.2 - "@backstage/release-manifests": ^0.0.6 - "@backstage/types": ^1.0.0 - "@manypkg/get-packages": ^1.1.3 - "@octokit/request": ^6.0.0 - "@pmmmwh/react-refresh-webpack-plugin": ^0.5.7 - "@rollup/plugin-commonjs": ^22.0.0 - "@rollup/plugin-json": ^4.1.0 - "@rollup/plugin-node-resolve": ^13.0.6 - "@rollup/plugin-yaml": ^3.1.0 - "@spotify/eslint-config-base": ^14.0.0 - "@spotify/eslint-config-react": ^14.0.0 - "@spotify/eslint-config-typescript": ^14.0.0 - "@sucrase/jest-plugin": ^2.1.1 - "@sucrase/webpack-loader": ^2.0.0 - "@svgr/plugin-jsx": 6.3.x - "@svgr/plugin-svgo": 6.3.x - "@svgr/rollup": 6.3.x - "@svgr/webpack": 6.3.x - "@swc/core": ^1.2.239 - "@swc/helpers": ^0.4.7 - "@swc/jest": ^0.2.22 - "@types/jest": ^29.0.0 - "@types/webpack-env": ^1.15.2 - "@typescript-eslint/eslint-plugin": ^5.9.0 - "@typescript-eslint/parser": ^5.9.0 - "@yarnpkg/lockfile": ^1.1.0 - "@yarnpkg/parsers": ^3.0.0-rc.4 - bfj: ^7.0.2 - buffer: ^6.0.3 - chalk: ^4.0.0 - chokidar: ^3.3.1 - commander: ^9.1.0 - css-loader: ^6.5.1 - diff: ^5.0.0 - esbuild: ^0.14.10 - esbuild-loader: ^2.18.0 - eslint: ^8.6.0 - eslint-config-prettier: ^8.3.0 - eslint-formatter-friendly: ^7.0.0 - eslint-plugin-deprecation: ^1.3.2 - eslint-plugin-import: ^2.25.4 - eslint-plugin-jest: ^27.0.0 - eslint-plugin-jsx-a11y: ^6.5.1 - eslint-plugin-monorepo: ^0.3.2 - eslint-plugin-react: ^7.28.0 - eslint-plugin-react-hooks: ^4.3.0 - eslint-webpack-plugin: ^3.1.1 - express: ^4.17.1 - fork-ts-checker-webpack-plugin: ^7.0.0-alpha.8 - fs-extra: 10.1.0 - glob: ^7.1.7 - global-agent: ^3.0.0 - handlebars: ^4.7.3 - html-webpack-plugin: ^5.3.1 - inquirer: ^8.2.0 - jest: ^29.0.2 - jest-css-modules: ^2.1.0 - jest-environment-jsdom: ^29.0.2 - jest-runtime: ^29.0.2 - json-schema: ^0.4.0 - lodash: ^4.17.21 - mini-css-extract-plugin: ^2.4.2 - minimatch: 5.1.0 - node-fetch: ^2.6.7 - node-libs-browser: ^2.2.1 - npm-packlist: ^5.0.0 - ora: ^5.3.0 - postcss: ^8.1.0 - process: ^0.11.10 - react-dev-utils: ^12.0.0-next.60 - react-refresh: ^0.14.0 - recursive-readdir: ^2.2.2 - replace-in-file: ^6.0.0 - rollup: ^2.60.2 - rollup-plugin-dts: ^4.0.1 - rollup-plugin-esbuild: ^4.7.2 - rollup-plugin-postcss: ^4.0.0 - rollup-pluginutils: ^2.8.2 - run-script-webpack-plugin: ^0.1.0 - semver: ^7.3.2 - style-loader: ^3.3.1 - sucrase: ^3.20.2 - swc-loader: ^0.2.3 - tar: ^6.1.2 - terser-webpack-plugin: ^5.1.3 - util: ^0.12.3 - webpack: ^5.70.0 - webpack-dev-server: ^4.7.3 - webpack-node-externals: ^3.0.0 - yaml: ^2.0.0 - yml-loader: ^2.1.0 - yn: ^4.0.0 - zod: ^3.11.6 - peerDependencies: - "@microsoft/api-extractor": ^7.21.2 - peerDependenciesMeta: - "@microsoft/api-extractor": - optional: true - bin: - backstage-cli: bin/backstage-cli - checksum: b6072f130fd487bbd6947895d4809b67c991554b7c669b93aacc36845f461a8aa548be31d8c325f04b07a8512e9754bdf662cb0cdd4c986c883fa25c00d2f405 - languageName: node - linkType: hard - "@backstage/cli@workspace:*, @backstage/cli@workspace:^, @backstage/cli@workspace:packages/cli": version: 0.0.0-use.local resolution: "@backstage/cli@workspace:packages/cli" @@ -3680,29 +3271,6 @@ __metadata: languageName: unknown linkType: soft -"@backstage/config-loader@npm:^1.1.3, @backstage/config-loader@npm:^1.1.5, @backstage/config-loader@npm:^1.1.6": - version: 1.1.6 - resolution: "@backstage/config-loader@npm:1.1.6" - dependencies: - "@backstage/cli-common": ^0.1.10 - "@backstage/config": ^1.0.4 - "@backstage/errors": ^1.1.3 - "@backstage/types": ^1.0.1 - "@types/json-schema": ^7.0.6 - ajv: ^8.10.0 - chokidar: ^3.5.2 - fs-extra: 10.1.0 - json-schema: ^0.4.0 - json-schema-merge-allof: ^0.8.1 - json-schema-traverse: ^1.0.0 - node-fetch: ^2.6.7 - typescript-json-schema: ^0.54.0 - yaml: ^2.0.0 - yup: ^0.32.9 - checksum: a5347574e9006f2d84dd690f92dc9a43501771152e331258dcd60f07027de293948256d0afeea12443ded088c5147bf1cccc12437fd4df495d73862855ff8c48 - languageName: node - linkType: hard - "@backstage/config-loader@workspace:^, @backstage/config-loader@workspace:packages/config-loader": version: 0.0.0-use.local resolution: "@backstage/config-loader@workspace:packages/config-loader" @@ -3732,7 +3300,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/config@npm:^1.0.0, @backstage/config@npm:^1.0.1, @backstage/config@npm:^1.0.3, @backstage/config@npm:^1.0.4": +"@backstage/config@npm:^1.0.3, @backstage/config@npm:^1.0.4": version: 1.0.4 resolution: "@backstage/config@npm:1.0.4" dependencies: @@ -4066,7 +3634,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/errors@npm:^1.1.0, @backstage/errors@npm:^1.1.2, @backstage/errors@npm:^1.1.3": +"@backstage/errors@npm:^1.1.2, @backstage/errors@npm:^1.1.3": version: 1.1.3 resolution: "@backstage/errors@npm:1.1.3" dependencies: @@ -4134,7 +3702,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/integration@npm:^1.2.2, @backstage/integration@npm:^1.4.0": +"@backstage/integration@npm:^1.4.0": version: 1.4.0 resolution: "@backstage/integration@npm:1.4.0" dependencies: @@ -4554,22 +4122,6 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-auth-node@npm:^0.2.7": - version: 0.2.7 - resolution: "@backstage/plugin-auth-node@npm:0.2.7" - dependencies: - "@backstage/backend-common": ^0.16.0 - "@backstage/config": ^1.0.4 - "@backstage/errors": ^1.1.3 - "@types/express": "*" - express: ^4.17.1 - jose: ^4.6.0 - node-fetch: ^2.6.7 - winston: ^3.2.1 - checksum: 453c6d44edef8fcf4e23948fc1fe77e7fad7a0007fc1ccdab1d80c61ddc96746f2913a90e86e3cb4bb0f895e396e189b6c6797393b792fd497cc5831f76529da - languageName: node - linkType: hard - "@backstage/plugin-auth-node@workspace:^, @backstage/plugin-auth-node@workspace:plugins/auth-node": version: 0.0.0-use.local resolution: "@backstage/plugin-auth-node@workspace:plugins/auth-node" @@ -5135,48 +4687,6 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend@npm:^1.5.0": - version: 1.5.1 - resolution: "@backstage/plugin-catalog-backend@npm:1.5.1" - dependencies: - "@backstage/backend-common": ^0.16.0 - "@backstage/backend-plugin-api": ^0.1.4 - "@backstage/catalog-client": ^1.1.2 - "@backstage/catalog-model": ^1.1.3 - "@backstage/config": ^1.0.4 - "@backstage/errors": ^1.1.3 - "@backstage/integration": ^1.4.0 - "@backstage/plugin-catalog-common": ^1.0.8 - "@backstage/plugin-catalog-node": ^1.2.1 - "@backstage/plugin-permission-common": ^0.7.1 - "@backstage/plugin-permission-node": ^0.7.1 - "@backstage/plugin-scaffolder-common": ^1.2.2 - "@backstage/plugin-search-common": ^1.1.1 - "@backstage/types": ^1.0.1 - "@types/express": ^4.17.6 - codeowners-utils: ^1.0.2 - core-js: ^3.6.5 - express: ^4.17.1 - express-promise-router: ^4.1.0 - fast-json-stable-stringify: ^2.1.0 - fs-extra: 10.1.0 - git-url-parse: ^13.0.0 - glob: ^7.1.6 - knex: ^2.0.0 - lodash: ^4.17.21 - luxon: ^3.0.0 - node-fetch: ^2.6.7 - p-limit: ^3.0.2 - prom-client: ^14.0.1 - uuid: ^8.0.0 - winston: ^3.2.1 - yaml: ^2.0.0 - yn: ^4.0.0 - zod: ^3.11.6 - checksum: fd041132dc53757aeed2eb9fe74e85aca61083839e744c507360c84e7ba63d7ddf517b1d17b26cbd17328fa5554664ebddba868d15e15eff133dc861fceecc01 - languageName: node - linkType: hard - "@backstage/plugin-catalog-backend@workspace:^, @backstage/plugin-catalog-backend@workspace:plugins/catalog-backend": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend@workspace:plugins/catalog-backend" @@ -5353,20 +4863,6 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-node@npm:^1.2.1": - version: 1.2.1 - resolution: "@backstage/plugin-catalog-node@npm:1.2.1" - dependencies: - "@backstage/backend-plugin-api": ^0.1.4 - "@backstage/catalog-client": ^1.1.2 - "@backstage/catalog-model": ^1.1.3 - "@backstage/errors": ^1.1.3 - "@backstage/plugin-catalog-common": ^1.0.8 - "@backstage/types": ^1.0.1 - checksum: 7cefbdcd071f30da2dec1f04c5ba1df977ccaccdad4aa53dc16e0a35bdcc6d2db6437d97d15ec65d77f579dede3d94796cd4dc4f28586bcf81416a2771b48c1a - languageName: node - linkType: hard - "@backstage/plugin-catalog-node@workspace:^, @backstage/plugin-catalog-node@workspace:plugins/catalog-node": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-node@workspace:plugins/catalog-node" @@ -6531,19 +6027,16 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend" dependencies: - "@backstage/backend-common": ^0.14.0 - "@backstage/backend-tasks": ^0.3.6 - "@backstage/catalog-model": ^1.1.2 - "@backstage/cli": ^0.20.0 - "@backstage/config": ^1.0.0 - "@backstage/plugin-catalog-backend": ^1.5.0 - "@backstage/plugin-permission-common": ^0.7.0 - "@types/express": "*" - "@types/supertest": ^2.0.8 + "@backstage/backend-common": "workspace:^" + "@backstage/backend-tasks": "workspace:^" + "@backstage/catalog-model": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/config": "workspace:^" + "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-permission-common": "workspace:^" + "@types/express": ^4.17.6 express: ^4.17.1 express-promise-router: ^4.1.0 - msw: ^0.35.0 - supertest: ^4.0.2 winston: ^3.2.1 languageName: unknown linkType: soft @@ -7019,7 +6512,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-permission-common@npm:^0.7.0, @backstage/plugin-permission-common@npm:^0.7.1": +"@backstage/plugin-permission-common@npm:^0.7.1": version: 0.7.1 resolution: "@backstage/plugin-permission-common@npm:0.7.1" dependencies: @@ -7048,24 +6541,6 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-permission-node@npm:^0.7.1": - version: 0.7.1 - resolution: "@backstage/plugin-permission-node@npm:0.7.1" - dependencies: - "@backstage/backend-common": ^0.16.0 - "@backstage/config": ^1.0.4 - "@backstage/errors": ^1.1.3 - "@backstage/plugin-auth-node": ^0.2.7 - "@backstage/plugin-permission-common": ^0.7.1 - "@types/express": ^4.17.6 - express: ^4.17.1 - express-promise-router: ^4.1.0 - zod: ^3.11.6 - zod-to-json-schema: ^3.18.1 - checksum: c274a25185066b99e94f262def3169830c1d2ead15a56b558d1c9fd202eef656e9d7278b75ce75415cc46183da7b3ff7b2b5551ca84db5285f37dbc01edbca24 - languageName: node - linkType: hard - "@backstage/plugin-permission-node@workspace:^, @backstage/plugin-permission-node@workspace:plugins/permission-node": version: 0.0.0-use.local resolution: "@backstage/plugin-permission-node@workspace:plugins/permission-node" @@ -7418,16 +6893,6 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-scaffolder-common@npm:^1.2.2": - version: 1.2.2 - resolution: "@backstage/plugin-scaffolder-common@npm:1.2.2" - dependencies: - "@backstage/catalog-model": ^1.1.3 - "@backstage/types": ^1.0.1 - checksum: d6e8d053a57bdb9a8cc3afe4db7e62f3a24f3a7a7fe340e50ec40b23a6bbd1e7e8d9aaa9cc6af28457bdd877c71c6dbc53a70371887664be098edb69f53a8487 - languageName: node - linkType: hard - "@backstage/plugin-scaffolder-common@workspace:^, @backstage/plugin-scaffolder-common@workspace:plugins/scaffolder-common": version: 0.0.0-use.local resolution: "@backstage/plugin-scaffolder-common@workspace:plugins/scaffolder-common" @@ -8456,15 +7921,6 @@ __metadata: languageName: unknown linkType: soft -"@backstage/release-manifests@npm:^0.0.6": - version: 0.0.6 - resolution: "@backstage/release-manifests@npm:0.0.6" - dependencies: - cross-fetch: ^3.1.5 - checksum: 9d6f0a2d5d6004f7f2c08938b84d55e0b54f45e83b1d93a3293b5702ad3709ad490794b46c10951a407f42525977ae1665db2eacd2fae8cf58a991b5e76996b2 - languageName: node - linkType: hard - "@backstage/release-manifests@workspace:^, @backstage/release-manifests@workspace:packages/release-manifests": version: 0.0.0-use.local resolution: "@backstage/release-manifests@workspace:packages/release-manifests" @@ -9104,13 +8560,6 @@ __metadata: languageName: node linkType: hard -"@esbuild/linux-loong64@npm:0.14.54": - version: 0.14.54 - resolution: "@esbuild/linux-loong64@npm:0.14.54" - conditions: os=linux & cpu=loong64 - languageName: node - linkType: hard - "@esbuild/linux-loong64@npm:0.15.15": version: 0.15.15 resolution: "@esbuild/linux-loong64@npm:0.15.15" @@ -10912,7 +10361,7 @@ __metadata: languageName: node linkType: hard -"@keyv/redis@npm:^2.2.3, @keyv/redis@npm:^2.5.3": +"@keyv/redis@npm:^2.5.3": version: 2.5.3 resolution: "@keyv/redis@npm:2.5.3" dependencies: @@ -11352,16 +10801,6 @@ __metadata: languageName: node linkType: hard -"@mswjs/cookies@npm:^0.1.6": - version: 0.1.7 - resolution: "@mswjs/cookies@npm:0.1.7" - dependencies: - "@types/set-cookie-parser": ^2.4.0 - set-cookie-parser: ^2.4.6 - checksum: d9b152dfdeba08b282a236485610bcfe992626e3c638fe51ebc5c7b5273d41d74e5447ae556a6c76460a8d3c7a7de6f544c681232cb50ae05b6b1e112bb77286 - languageName: node - linkType: hard - "@mswjs/cookies@npm:^0.2.0, @mswjs/cookies@npm:^0.2.2": version: 0.2.2 resolution: "@mswjs/cookies@npm:0.2.2" @@ -11372,20 +10811,6 @@ __metadata: languageName: node linkType: hard -"@mswjs/interceptors@npm:^0.12.6": - version: 0.12.7 - resolution: "@mswjs/interceptors@npm:0.12.7" - dependencies: - "@open-draft/until": ^1.0.3 - "@xmldom/xmldom": ^0.7.2 - debug: ^4.3.2 - headers-utils: ^3.0.2 - outvariant: ^1.2.0 - strict-event-emitter: ^0.2.0 - checksum: 426e9a27f13f0bd1f5b8dbf5f3605c65709f0aa73cee5a4a8278bc43859968a11d72aef42266a9c07030f1949f539ac1865a1da7edecc31b1ba40406211918f5 - languageName: node - linkType: hard - "@mswjs/interceptors@npm:^0.15.1": version: 0.15.3 resolution: "@mswjs/interceptors@npm:0.15.3" @@ -12928,16 +12353,7 @@ __metadata: languageName: node linkType: hard -"@svgr/babel-plugin-add-jsx-attribute@npm:^6.5.1": - version: 6.5.1 - resolution: "@svgr/babel-plugin-add-jsx-attribute@npm:6.5.1" - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: cab83832830a57735329ed68f67c03b57ca21fa037b0134847b0c5c0ef4beca89956d7dacfbf7b2a10fd901e7009e877512086db2ee918b8c69aee7742ae32c0 - languageName: node - linkType: hard - -"@svgr/babel-plugin-remove-jsx-attribute@npm:*, @svgr/babel-plugin-remove-jsx-attribute@npm:^6.5.0": +"@svgr/babel-plugin-remove-jsx-attribute@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/babel-plugin-remove-jsx-attribute@npm:6.5.0" peerDependencies: @@ -12946,7 +12362,7 @@ __metadata: languageName: node linkType: hard -"@svgr/babel-plugin-remove-jsx-empty-expression@npm:*, @svgr/babel-plugin-remove-jsx-empty-expression@npm:^6.5.0": +"@svgr/babel-plugin-remove-jsx-empty-expression@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/babel-plugin-remove-jsx-empty-expression@npm:6.5.0" peerDependencies: @@ -12964,15 +12380,6 @@ __metadata: languageName: node linkType: hard -"@svgr/babel-plugin-replace-jsx-attribute-value@npm:^6.5.1": - version: 6.5.1 - resolution: "@svgr/babel-plugin-replace-jsx-attribute-value@npm:6.5.1" - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: b7d2125758e766e1ebd14b92216b800bdc976959bc696dbfa1e28682919147c1df4bb8b1b5fd037d7a83026e27e681fea3b8d3741af8d3cf4c9dfa3d412125df - languageName: node - linkType: hard - "@svgr/babel-plugin-svg-dynamic-title@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/babel-plugin-svg-dynamic-title@npm:6.5.0" @@ -12982,15 +12389,6 @@ __metadata: languageName: node linkType: hard -"@svgr/babel-plugin-svg-dynamic-title@npm:^6.5.1": - version: 6.5.1 - resolution: "@svgr/babel-plugin-svg-dynamic-title@npm:6.5.1" - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 0fd42ebf127ae9163ef341e84972daa99bdcb9e6ed3f83aabd95ee173fddc43e40e02fa847fbc0a1058cf5549f72b7960a2c5e22c3e4ac18f7e3ac81277852ae - languageName: node - linkType: hard - "@svgr/babel-plugin-svg-em-dimensions@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/babel-plugin-svg-em-dimensions@npm:6.5.0" @@ -13000,15 +12398,6 @@ __metadata: languageName: node linkType: hard -"@svgr/babel-plugin-svg-em-dimensions@npm:^6.5.1": - version: 6.5.1 - resolution: "@svgr/babel-plugin-svg-em-dimensions@npm:6.5.1" - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: c1550ee9f548526fa66fd171e3ffb5696bfc4e4cd108a631d39db492c7410dc10bba4eb5a190e9df824bf806130ccc586ae7d2e43c547e6a4f93bbb29a18f344 - languageName: node - linkType: hard - "@svgr/babel-plugin-transform-react-native-svg@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/babel-plugin-transform-react-native-svg@npm:6.5.0" @@ -13018,15 +12407,6 @@ __metadata: languageName: node linkType: hard -"@svgr/babel-plugin-transform-react-native-svg@npm:^6.5.1": - version: 6.5.1 - resolution: "@svgr/babel-plugin-transform-react-native-svg@npm:6.5.1" - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 4c924af22b948b812629e80efb90ad1ec8faae26a232d8ca8a06b46b53e966a2c415a57806a3ff0ea806a622612e546422719b69ec6839717a7755dac19171d9 - languageName: node - linkType: hard - "@svgr/babel-plugin-transform-svg-component@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/babel-plugin-transform-svg-component@npm:6.5.0" @@ -13036,33 +12416,6 @@ __metadata: languageName: node linkType: hard -"@svgr/babel-plugin-transform-svg-component@npm:^6.5.1": - version: 6.5.1 - resolution: "@svgr/babel-plugin-transform-svg-component@npm:6.5.1" - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: e496bb5ee871feb6bcab250b6e067322da7dd5c9c2b530b41e5586fe090f86611339b49d0a909c334d9b24cbca0fa755c949a2526c6ad03c6b5885666874cf5f - languageName: node - linkType: hard - -"@svgr/babel-preset@npm:^6.3.1, @svgr/babel-preset@npm:^6.5.1": - version: 6.5.1 - resolution: "@svgr/babel-preset@npm:6.5.1" - dependencies: - "@svgr/babel-plugin-add-jsx-attribute": ^6.5.1 - "@svgr/babel-plugin-remove-jsx-attribute": "*" - "@svgr/babel-plugin-remove-jsx-empty-expression": "*" - "@svgr/babel-plugin-replace-jsx-attribute-value": ^6.5.1 - "@svgr/babel-plugin-svg-dynamic-title": ^6.5.1 - "@svgr/babel-plugin-svg-em-dimensions": ^6.5.1 - "@svgr/babel-plugin-transform-react-native-svg": ^6.5.1 - "@svgr/babel-plugin-transform-svg-component": ^6.5.1 - peerDependencies: - "@babel/core": ^7.0.0-0 - checksum: 9f124be39a8e64f909162f925b3a63ddaa5a342a5e24fc0b7f7d9d4d7f7e3b916596c754fb557dc259928399cad5366a27cb231627a0d2dcc4b13ac521cf05af - languageName: node - linkType: hard - "@svgr/babel-preset@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/babel-preset@npm:6.5.0" @@ -13081,19 +12434,6 @@ __metadata: languageName: node linkType: hard -"@svgr/core@npm:^6.3.1": - version: 6.5.1 - resolution: "@svgr/core@npm:6.5.1" - dependencies: - "@babel/core": ^7.19.6 - "@svgr/babel-preset": ^6.5.1 - "@svgr/plugin-jsx": ^6.5.1 - camelcase: ^6.2.0 - cosmiconfig: ^7.0.1 - checksum: fd6d6d5da5aeb956703310480b626c1fb3e3973ad9fe8025efc1dcf3d895f857b70d100c63cf32cebb20eb83c9607bafa464c9436e18fe6fe4fafdc73ed6b1a5 - languageName: node - linkType: hard - "@svgr/core@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/core@npm:6.5.0" @@ -13107,16 +12447,6 @@ __metadata: languageName: node linkType: hard -"@svgr/hast-util-to-babel-ast@npm:^6.3.1, @svgr/hast-util-to-babel-ast@npm:^6.5.1": - version: 6.5.1 - resolution: "@svgr/hast-util-to-babel-ast@npm:6.5.1" - dependencies: - "@babel/types": ^7.20.0 - entities: ^4.4.0 - checksum: 37923cce1b3f4e2039077b0c570b6edbabe37d1cf1a6ee35e71e0fe00f9cffac450eec45e9720b1010418131a999cb0047331ba1b6d1d2c69af1b92ac785aacf - languageName: node - linkType: hard - "@svgr/hast-util-to-babel-ast@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/hast-util-to-babel-ast@npm:6.5.0" @@ -13127,20 +12457,6 @@ __metadata: languageName: node linkType: hard -"@svgr/plugin-jsx@npm:6.3.x": - version: 6.3.1 - resolution: "@svgr/plugin-jsx@npm:6.3.1" - dependencies: - "@babel/core": ^7.18.5 - "@svgr/babel-preset": ^6.3.1 - "@svgr/hast-util-to-babel-ast": ^6.3.1 - svg-parser: ^2.0.4 - peerDependencies: - "@svgr/core": ^6.0.0 - checksum: a2e487dc28d2b69b94b7d96e5cb2593857e559e64c8cb4f818035b8a43ba84e5ddb67f966d15f173b544e807e48a1cda1065da8f9064b94b8d62bbe8cb8c4d73 - languageName: node - linkType: hard - "@svgr/plugin-jsx@npm:6.5.x, @svgr/plugin-jsx@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/plugin-jsx@npm:6.5.0" @@ -13155,33 +12471,6 @@ __metadata: languageName: node linkType: hard -"@svgr/plugin-jsx@npm:^6.3.1, @svgr/plugin-jsx@npm:^6.5.1": - version: 6.5.1 - resolution: "@svgr/plugin-jsx@npm:6.5.1" - dependencies: - "@babel/core": ^7.19.6 - "@svgr/babel-preset": ^6.5.1 - "@svgr/hast-util-to-babel-ast": ^6.5.1 - svg-parser: ^2.0.4 - peerDependencies: - "@svgr/core": ^6.0.0 - checksum: 42f22847a6bdf930514d7bedd3c5e1fd8d53eb3594779f9db16cb94c762425907c375cd8ec789114e100a4d38068aca6c7ab5efea4c612fba63f0630c44cc859 - languageName: node - linkType: hard - -"@svgr/plugin-svgo@npm:6.3.x": - version: 6.3.1 - resolution: "@svgr/plugin-svgo@npm:6.3.1" - dependencies: - cosmiconfig: ^7.0.1 - deepmerge: ^4.2.2 - svgo: ^2.8.0 - peerDependencies: - "@svgr/core": ^6.0.0 - checksum: 037d6f91ba7f362764527408661f7fc4a4a296e9dc142a5f2e33fc88dc63dafd305452caae3091e9adb63adf029e0ce20c604d5af787b968f98aad261a834679 - languageName: node - linkType: hard - "@svgr/plugin-svgo@npm:6.5.x, @svgr/plugin-svgo@npm:^6.5.0": version: 6.5.0 resolution: "@svgr/plugin-svgo@npm:6.5.0" @@ -13195,36 +12484,6 @@ __metadata: languageName: node linkType: hard -"@svgr/plugin-svgo@npm:^6.3.1": - version: 6.5.1 - resolution: "@svgr/plugin-svgo@npm:6.5.1" - dependencies: - cosmiconfig: ^7.0.1 - deepmerge: ^4.2.2 - svgo: ^2.8.0 - peerDependencies: - "@svgr/core": "*" - checksum: cd2833530ac0485221adc2146fd992ab20d79f4b12eebcd45fa859721dd779483158e11dfd9a534858fe468416b9412416e25cbe07ac7932c44ed5fa2021c72e - languageName: node - linkType: hard - -"@svgr/rollup@npm:6.3.x": - version: 6.3.1 - resolution: "@svgr/rollup@npm:6.3.1" - dependencies: - "@babel/core": ^7.18.5 - "@babel/plugin-transform-react-constant-elements": ^7.17.12 - "@babel/preset-env": ^7.18.2 - "@babel/preset-react": ^7.17.12 - "@babel/preset-typescript": ^7.17.12 - "@rollup/pluginutils": ^4.2.1 - "@svgr/core": ^6.3.1 - "@svgr/plugin-jsx": ^6.3.1 - "@svgr/plugin-svgo": ^6.3.1 - checksum: 8d96f95a4c89d96a14bc0095469a4ceb39a2ae4ea3517d08573ed8ca11b05416a505723ef09d00c3cee056adc2d95ddcea6c8055a116bbb461ba33d601bfced8 - languageName: node - linkType: hard - "@svgr/rollup@npm:6.5.x": version: 6.5.0 resolution: "@svgr/rollup@npm:6.5.0" @@ -13242,22 +12501,6 @@ __metadata: languageName: node linkType: hard -"@svgr/webpack@npm:6.3.x": - version: 6.3.1 - resolution: "@svgr/webpack@npm:6.3.1" - dependencies: - "@babel/core": ^7.18.5 - "@babel/plugin-transform-react-constant-elements": ^7.17.12 - "@babel/preset-env": ^7.18.2 - "@babel/preset-react": ^7.17.12 - "@babel/preset-typescript": ^7.17.12 - "@svgr/core": ^6.3.1 - "@svgr/plugin-jsx": ^6.3.1 - "@svgr/plugin-svgo": ^6.3.1 - checksum: 36784eacf80601462ede7eab66347423a8635e68aa9f152308c81878b071807adee152a28eed2cce9c72faaf6553dd500f68f00601062ec6821ec0a3a77f4e13 - languageName: node - linkType: hard - "@svgr/webpack@npm:6.5.x": version: 6.5.0 resolution: "@svgr/webpack@npm:6.5.0" @@ -13344,7 +12587,7 @@ __metadata: languageName: node linkType: hard -"@swc/core@npm:^1.2.239, @swc/core@npm:^1.3.9": +"@swc/core@npm:^1.3.9": version: 1.3.19 resolution: "@swc/core@npm:1.3.19" dependencies: @@ -14328,16 +13571,6 @@ __metadata: languageName: node linkType: hard -"@types/inquirer@npm:^7.3.3": - version: 7.3.3 - resolution: "@types/inquirer@npm:7.3.3" - dependencies: - "@types/through": "*" - rxjs: ^6.4.0 - checksum: 49b21d883ab533dbb84b400fa1aeab2638c37b87978d16f15636316c8d9f70d93a185479cf32081d9013fe2b362db05a83bdc3725771cc93d8bdab9182a96ab9 - languageName: node - linkType: hard - "@types/inquirer@npm:^8.1.3": version: 8.2.5 resolution: "@types/inquirer@npm:8.2.5" @@ -14425,7 +13658,7 @@ __metadata: languageName: node linkType: hard -"@types/js-levenshtein@npm:^1.1.0, @types/js-levenshtein@npm:^1.1.1": +"@types/js-levenshtein@npm:^1.1.1": version: 1.1.1 resolution: "@types/js-levenshtein@npm:1.1.1" checksum: 1d1ff1ee2ad551909e47f3ce19fcf85b64dc5146d3b531c8d26fc775492d36e380b32cf5ef68ff301e812c3b00282f37aac579ebb44498b94baff0ace7509769 @@ -14568,13 +13801,6 @@ __metadata: languageName: node linkType: hard -"@types/luxon@npm:^2.0.4": - version: 2.4.0 - resolution: "@types/luxon@npm:2.4.0" - checksum: eeb16a1bfe5440464c1a9635700d103cd18d3cd8da6063a1938478e435cfba6ab8e893aa80c95a407e541187c1e997c3e4481322726bc1258551cb8606d0e5ad - languageName: node - linkType: hard - "@types/markdown-it@npm:^12.2.3": version: 12.2.3 resolution: "@types/markdown-it@npm:12.2.3" @@ -16000,13 +15226,6 @@ __metadata: languageName: node linkType: hard -"@xmldom/xmldom@npm:^0.7.2": - version: 0.7.9 - resolution: "@xmldom/xmldom@npm:0.7.9" - checksum: 66e37b7800132f891b885b2eceeeebc53f60b69789da10276f1584256b963d79a28c7ae2071bc53a9cd842d9b03554c761b2701fe8036d6052f26bcd0ae8f2bb - languageName: node - linkType: hard - "@xobotyi/scrollbar-width@npm:^1.9.5": version: 1.9.5 resolution: "@xobotyi/scrollbar-width@npm:1.9.5" @@ -17442,13 +16661,6 @@ __metadata: languageName: node linkType: hard -"big-integer@npm:^1.6.17": - version: 1.6.51 - resolution: "big-integer@npm:1.6.51" - checksum: 3d444173d1b2e20747e2c175568bedeebd8315b0637ea95d75fd27830d3b8e8ba36c6af40374f36bdaea7b5de376dcada1b07587cb2a79a928fccdb6e6e3c518 - languageName: node - linkType: hard - "big.js@npm:^5.2.2": version: 5.2.2 resolution: "big.js@npm:5.2.2" @@ -17498,16 +16710,6 @@ __metadata: languageName: node linkType: hard -"binary@npm:~0.3.0": - version: 0.3.0 - resolution: "binary@npm:0.3.0" - dependencies: - buffers: ~0.1.1 - chainsaw: ~0.1.0 - checksum: b4699fda9e2c2981e74a46b0115cf0d472eda9b68c0e9d229ef494e92f29ce81acf0a834415094cffcc340dfee7c4ef8ce5d048c65c18067a7ed850323f777af - languageName: node - linkType: hard - "binaryextensions@npm:^4.15.0, binaryextensions@npm:^4.16.0": version: 4.18.0 resolution: "binaryextensions@npm:4.18.0" @@ -17563,13 +16765,6 @@ __metadata: languageName: node linkType: hard -"bluebird@npm:~3.4.1": - version: 3.4.7 - resolution: "bluebird@npm:3.4.7" - checksum: bffa9dee7d3a41ab15c4f3f24687b49959b4e64e55c058a062176feb8ccefc2163414fb4e1a0f3053bf187600936509660c3ebd168fd9f0e48c7eba23b019466 - languageName: node - linkType: hard - "bmp-js@npm:^0.1.0": version: 0.1.0 resolution: "bmp-js@npm:0.1.0" @@ -17831,13 +17026,6 @@ __metadata: languageName: node linkType: hard -"buffer-indexof-polyfill@npm:~1.0.0": - version: 1.0.2 - resolution: "buffer-indexof-polyfill@npm:1.0.2" - checksum: fbfb2d69c6bb2df235683126f9dc140150c08ac3630da149913a9971947b667df816a913b6993bc48f4d611999cb99a1589914d34c02dccd2234afda5cb75bbc - languageName: node - linkType: hard - "buffer-writer@npm:2.0.0": version: 2.0.0 resolution: "buffer-writer@npm:2.0.0" @@ -17883,13 +17071,6 @@ __metadata: languageName: node linkType: hard -"buffers@npm:~0.1.1": - version: 0.1.1 - resolution: "buffers@npm:0.1.1" - checksum: ad6f8e483efab39cefd92bdc04edbff6805e4211b002f4d1cfb70c6c472a61cc89fb18c37bcdfdd4ee416ca096e9ff606286698a7d41a18b539bac12fd76d4d5 - languageName: node - linkType: hard - "builtin-modules@npm:^3.0.0": version: 3.2.0 resolution: "builtin-modules@npm:3.2.0" @@ -18179,15 +17360,6 @@ __metadata: languageName: node linkType: hard -"chainsaw@npm:~0.1.0": - version: 0.1.0 - resolution: "chainsaw@npm:0.1.0" - dependencies: - traverse: ">=0.3.0 <0.4" - checksum: 22a96b9fb0cd9fb20813607c0869e61817d1acc81b5d455cc6456b5e460ea1dd52630e0f76b291cf8294bfb6c1fc42e299afb52104af9096242699d6d3aa6d3e - languageName: node - linkType: hard - "chalk@npm:2.4.2, chalk@npm:^2.0.0, chalk@npm:^2.1.0, chalk@npm:^2.3.2": version: 2.4.2 resolution: "chalk@npm:2.4.2" @@ -18556,17 +17728,6 @@ __metadata: languageName: node linkType: hard -"cliui@npm:^8.0.1": - version: 8.0.1 - resolution: "cliui@npm:8.0.1" - dependencies: - string-width: ^4.2.0 - strip-ansi: ^6.0.1 - wrap-ansi: ^7.0.0 - checksum: 79648b3b0045f2e285b76fb2e24e207c6db44323581e421c3acbd0e86454cba1b37aea976ab50195a49e7384b871e6dfb2247ad7dec53c02454ac6497394cb56 - languageName: node - linkType: hard - "clone-buffer@npm:^1.0.0": version: 1.0.0 resolution: "clone-buffer@npm:1.0.0" @@ -18991,7 +18152,7 @@ __metadata: languageName: node linkType: hard -"component-emitter@npm:^1.2.0, component-emitter@npm:^1.3.0, component-emitter@npm:~1.3.0": +"component-emitter@npm:^1.3.0, component-emitter@npm:~1.3.0": version: 1.3.0 resolution: "component-emitter@npm:1.3.0" checksum: b3c46de38ffd35c57d1c02488355be9f218e582aec72d72d1b8bbec95a3ac1b38c96cd6e03ff015577e68f550fbb361a3bfdbd9bb248be9390b7b3745691be6b @@ -19241,7 +18402,7 @@ __metadata: languageName: node linkType: hard -"cookie@npm:0.4.2, cookie@npm:^0.4.1, cookie@npm:^0.4.2, cookie@npm:~0.4.1": +"cookie@npm:0.4.2, cookie@npm:^0.4.2, cookie@npm:~0.4.1": version: 0.4.2 resolution: "cookie@npm:0.4.2" checksum: a00833c998bedf8e787b4c342defe5fa419abd96b32f4464f718b91022586b8f1bafbddd499288e75c037642493c83083da426c6a9080d309e3bd90fd11baa9b @@ -19255,7 +18416,7 @@ __metadata: languageName: node linkType: hard -"cookiejar@npm:^2.1.0, cookiejar@npm:^2.1.3": +"cookiejar@npm:^2.1.3": version: 2.1.3 resolution: "cookiejar@npm:2.1.3" checksum: 88259983ebc52ceb23cdacfa48762b6a518a57872eff1c7ed01d214fff5cf492e2660d7d5c04700a28f1787a76811df39e8639f8e17670b3cf94ecd86e161f07 @@ -20854,15 +20015,6 @@ __metadata: languageName: node linkType: hard -"duplexer2@npm:~0.1.4": - version: 0.1.4 - resolution: "duplexer2@npm:0.1.4" - dependencies: - readable-stream: ^2.0.2 - checksum: 744961f03c7f54313f90555ac20284a3fb7bf22fdff6538f041a86c22499560eb6eac9d30ab5768054137cb40e6b18b40f621094e0261d7d8c35a37b7a5ad241 - languageName: node - linkType: hard - "duplexer@npm:^0.1.2, duplexer@npm:~0.1.1": version: 0.1.2 resolution: "duplexer@npm:0.1.2" @@ -21275,13 +20427,6 @@ __metadata: languageName: node linkType: hard -"esbuild-android-64@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-android-64@npm:0.14.54" - conditions: os=android & cpu=x64 - languageName: node - linkType: hard - "esbuild-android-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-android-64@npm:0.15.15" @@ -21289,13 +20434,6 @@ __metadata: languageName: node linkType: hard -"esbuild-android-arm64@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-android-arm64@npm:0.14.54" - conditions: os=android & cpu=arm64 - languageName: node - linkType: hard - "esbuild-android-arm64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-android-arm64@npm:0.15.15" @@ -21303,13 +20441,6 @@ __metadata: languageName: node linkType: hard -"esbuild-darwin-64@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-darwin-64@npm:0.14.54" - conditions: os=darwin & cpu=x64 - languageName: node - linkType: hard - "esbuild-darwin-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-darwin-64@npm:0.15.15" @@ -21317,13 +20448,6 @@ __metadata: languageName: node linkType: hard -"esbuild-darwin-arm64@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-darwin-arm64@npm:0.14.54" - conditions: os=darwin & cpu=arm64 - languageName: node - linkType: hard - "esbuild-darwin-arm64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-darwin-arm64@npm:0.15.15" @@ -21331,13 +20455,6 @@ __metadata: languageName: node linkType: hard -"esbuild-freebsd-64@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-freebsd-64@npm:0.14.54" - conditions: os=freebsd & cpu=x64 - languageName: node - linkType: hard - "esbuild-freebsd-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-freebsd-64@npm:0.15.15" @@ -21345,13 +20462,6 @@ __metadata: languageName: node linkType: hard -"esbuild-freebsd-arm64@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-freebsd-arm64@npm:0.14.54" - conditions: os=freebsd & cpu=arm64 - languageName: node - linkType: hard - "esbuild-freebsd-arm64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-freebsd-arm64@npm:0.15.15" @@ -21359,13 +20469,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-32@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-linux-32@npm:0.14.54" - conditions: os=linux & cpu=ia32 - languageName: node - linkType: hard - "esbuild-linux-32@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-32@npm:0.15.15" @@ -21373,13 +20476,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-64@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-linux-64@npm:0.14.54" - conditions: os=linux & cpu=x64 - languageName: node - linkType: hard - "esbuild-linux-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-64@npm:0.15.15" @@ -21387,13 +20483,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-arm64@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-linux-arm64@npm:0.14.54" - conditions: os=linux & cpu=arm64 - languageName: node - linkType: hard - "esbuild-linux-arm64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-arm64@npm:0.15.15" @@ -21401,13 +20490,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-arm@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-linux-arm@npm:0.14.54" - conditions: os=linux & cpu=arm - languageName: node - linkType: hard - "esbuild-linux-arm@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-arm@npm:0.15.15" @@ -21415,13 +20497,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-mips64le@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-linux-mips64le@npm:0.14.54" - conditions: os=linux & cpu=mips64el - languageName: node - linkType: hard - "esbuild-linux-mips64le@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-mips64le@npm:0.15.15" @@ -21429,13 +20504,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-ppc64le@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-linux-ppc64le@npm:0.14.54" - conditions: os=linux & cpu=ppc64 - languageName: node - linkType: hard - "esbuild-linux-ppc64le@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-ppc64le@npm:0.15.15" @@ -21443,13 +20511,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-riscv64@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-linux-riscv64@npm:0.14.54" - conditions: os=linux & cpu=riscv64 - languageName: node - linkType: hard - "esbuild-linux-riscv64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-riscv64@npm:0.15.15" @@ -21457,13 +20518,6 @@ __metadata: languageName: node linkType: hard -"esbuild-linux-s390x@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-linux-s390x@npm:0.14.54" - conditions: os=linux & cpu=s390x - languageName: node - linkType: hard - "esbuild-linux-s390x@npm:0.15.15": version: 0.15.15 resolution: "esbuild-linux-s390x@npm:0.15.15" @@ -21487,13 +20541,6 @@ __metadata: languageName: node linkType: hard -"esbuild-netbsd-64@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-netbsd-64@npm:0.14.54" - conditions: os=netbsd & cpu=x64 - languageName: node - linkType: hard - "esbuild-netbsd-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-netbsd-64@npm:0.15.15" @@ -21501,13 +20548,6 @@ __metadata: languageName: node linkType: hard -"esbuild-openbsd-64@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-openbsd-64@npm:0.14.54" - conditions: os=openbsd & cpu=x64 - languageName: node - linkType: hard - "esbuild-openbsd-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-openbsd-64@npm:0.15.15" @@ -21515,13 +20555,6 @@ __metadata: languageName: node linkType: hard -"esbuild-sunos-64@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-sunos-64@npm:0.14.54" - conditions: os=sunos & cpu=x64 - languageName: node - linkType: hard - "esbuild-sunos-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-sunos-64@npm:0.15.15" @@ -21529,13 +20562,6 @@ __metadata: languageName: node linkType: hard -"esbuild-windows-32@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-windows-32@npm:0.14.54" - conditions: os=win32 & cpu=ia32 - languageName: node - linkType: hard - "esbuild-windows-32@npm:0.15.15": version: 0.15.15 resolution: "esbuild-windows-32@npm:0.15.15" @@ -21543,13 +20569,6 @@ __metadata: languageName: node linkType: hard -"esbuild-windows-64@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-windows-64@npm:0.14.54" - conditions: os=win32 & cpu=x64 - languageName: node - linkType: hard - "esbuild-windows-64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-windows-64@npm:0.15.15" @@ -21557,13 +20576,6 @@ __metadata: languageName: node linkType: hard -"esbuild-windows-arm64@npm:0.14.54": - version: 0.14.54 - resolution: "esbuild-windows-arm64@npm:0.14.54" - conditions: os=win32 & cpu=arm64 - languageName: node - linkType: hard - "esbuild-windows-arm64@npm:0.15.15": version: 0.15.15 resolution: "esbuild-windows-arm64@npm:0.15.15" @@ -21571,80 +20583,6 @@ __metadata: languageName: node linkType: hard -"esbuild@npm:^0.14.10": - version: 0.14.54 - resolution: "esbuild@npm:0.14.54" - dependencies: - "@esbuild/linux-loong64": 0.14.54 - esbuild-android-64: 0.14.54 - esbuild-android-arm64: 0.14.54 - esbuild-darwin-64: 0.14.54 - esbuild-darwin-arm64: 0.14.54 - esbuild-freebsd-64: 0.14.54 - esbuild-freebsd-arm64: 0.14.54 - esbuild-linux-32: 0.14.54 - esbuild-linux-64: 0.14.54 - esbuild-linux-arm: 0.14.54 - esbuild-linux-arm64: 0.14.54 - esbuild-linux-mips64le: 0.14.54 - esbuild-linux-ppc64le: 0.14.54 - esbuild-linux-riscv64: 0.14.54 - esbuild-linux-s390x: 0.14.54 - esbuild-netbsd-64: 0.14.54 - esbuild-openbsd-64: 0.14.54 - esbuild-sunos-64: 0.14.54 - esbuild-windows-32: 0.14.54 - esbuild-windows-64: 0.14.54 - esbuild-windows-arm64: 0.14.54 - dependenciesMeta: - "@esbuild/linux-loong64": - optional: true - esbuild-android-64: - optional: true - esbuild-android-arm64: - optional: true - esbuild-darwin-64: - optional: true - esbuild-darwin-arm64: - optional: true - esbuild-freebsd-64: - optional: true - esbuild-freebsd-arm64: - optional: true - esbuild-linux-32: - optional: true - esbuild-linux-64: - optional: true - esbuild-linux-arm: - optional: true - esbuild-linux-arm64: - optional: true - esbuild-linux-mips64le: - optional: true - esbuild-linux-ppc64le: - optional: true - esbuild-linux-riscv64: - optional: true - esbuild-linux-s390x: - optional: true - esbuild-netbsd-64: - optional: true - esbuild-openbsd-64: - optional: true - esbuild-sunos-64: - optional: true - esbuild-windows-32: - optional: true - esbuild-windows-64: - optional: true - esbuild-windows-arm64: - optional: true - bin: - esbuild: bin/esbuild - checksum: 49e360b1185c797f5ca3a7f5f0a75121494d97ddf691f65ed1796e6257d318f928342a97f559bb8eced6a90cf604dd22db4a30e0dbbf15edd9dbf22459b639af - languageName: node - linkType: hard - "esbuild@npm:^0.15.0, esbuild@npm:^0.15.6": version: 0.15.15 resolution: "esbuild@npm:0.15.15" @@ -23281,7 +22219,7 @@ __metadata: languageName: node linkType: hard -"form-data@npm:^2.3.1, form-data@npm:^2.3.2, form-data@npm:^2.5.0": +"form-data@npm:^2.3.2, form-data@npm:^2.5.0": version: 2.5.1 resolution: "form-data@npm:2.5.1" dependencies: @@ -23342,13 +22280,6 @@ __metadata: languageName: node linkType: hard -"formidable@npm:^1.2.0": - version: 1.2.6 - resolution: "formidable@npm:1.2.6" - checksum: 2b68ed07ba88302b9c63f8eda94f19a460cef6017bfda48348f09f41d2a36660c9353137991618e0e4c3db115b41e4b8f6fa63bc973b7a7c91dec66acdd02a56 - languageName: node - linkType: hard - "formidable@npm:^2.0.1": version: 2.0.1 resolution: "formidable@npm:2.0.1" @@ -23504,18 +22435,6 @@ __metadata: languageName: node linkType: hard -"fstream@npm:^1.0.12": - version: 1.0.12 - resolution: "fstream@npm:1.0.12" - dependencies: - graceful-fs: ^4.1.2 - inherits: ~2.0.0 - mkdirp: ">=0.5 0" - rimraf: 2 - checksum: e6998651aeb85fd0f0a8a68cec4d05a3ada685ecc4e3f56e0d063d0564a4fc39ad11a856f9020f926daf869fc67f7a90e891def5d48e4cadab875dc313094536 - languageName: node - linkType: hard - "function-bind@npm:^1.1.1": version: 1.1.1 resolution: "function-bind@npm:1.1.1" @@ -23770,16 +22689,6 @@ __metadata: languageName: node linkType: hard -"git-up@npm:^6.0.0": - version: 6.0.0 - resolution: "git-up@npm:6.0.0" - dependencies: - is-ssh: ^1.4.0 - parse-url: ^7.0.2 - checksum: 145a1f546d7a078cdfc2616556e518e634d134e34a31c6bf2ed89e44158659cb525dbd451c338121f7107f55cef066d0b37a7bbf178555befc9304b3940b435e - languageName: node - linkType: hard - "git-up@npm:^7.0.0": version: 7.0.0 resolution: "git-up@npm:7.0.0" @@ -23790,15 +22699,6 @@ __metadata: languageName: node linkType: hard -"git-url-parse@npm:^12.0.0": - version: 12.0.0 - resolution: "git-url-parse@npm:12.0.0" - dependencies: - git-up: ^6.0.0 - checksum: b4c8530b816202ecf9d4dabf755f785a314a096b56145018385b3d7171e862f9d0d9b38cce620c0af354b269750fe7b2d9aa95815c7150922090a11dac4ab1e6 - languageName: node - linkType: hard - "git-url-parse@npm:^13.0.0": version: 13.1.0 resolution: "git-url-parse@npm:13.1.0" @@ -24066,7 +22966,7 @@ __metadata: languageName: node linkType: hard -"graceful-fs@npm:^4.1.11, graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.5, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.1.9, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.2, graceful-fs@npm:^4.2.4, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9": +"graceful-fs@npm:^4.1.11, graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.5, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.1.9, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.4, graceful-fs@npm:^4.2.6, graceful-fs@npm:^4.2.9": version: 4.2.10 resolution: "graceful-fs@npm:4.2.10" checksum: 3f109d70ae123951905d85032ebeae3c2a5a7a997430df00ea30df0e3a6c60cf6689b109654d6fdacd28810a053348c4d14642da1d075049e6be1ba5216218da @@ -24206,13 +23106,6 @@ __metadata: languageName: node linkType: hard -"graphql@npm:^15.5.1": - version: 15.8.0 - resolution: "graphql@npm:15.8.0" - checksum: 423325271db8858428641b9aca01699283d1fe5b40ef6d4ac622569ecca927019fce8196208b91dd1d8eb8114f00263fe661d241d0eb40c10e5bfd650f86ec5e - languageName: node - linkType: hard - "grouped-queue@npm:^2.0.0": version: 2.0.0 resolution: "grouped-queue@npm:2.0.0" @@ -24503,20 +23396,6 @@ __metadata: languageName: node linkType: hard -"headers-utils@npm:^3.0.2": - version: 3.0.2 - resolution: "headers-utils@npm:3.0.2" - checksum: 210fe65756d6de8a96afe68617463fb6faf675a24d864e849b17bddf051c4a24d621a510a1bb80fd9d4763b932eb44b5d8fd6fc4f14fa62fb211603456a57b4f - languageName: node - linkType: hard - -"helmet@npm:^5.0.2": - version: 5.1.1 - resolution: "helmet@npm:5.1.1" - checksum: b72ba26cc431804ad3b8ecdc18db95409a492cbb7a7e825efc27fc502b9433fec39fc083f2aad4fe7ed1a89a4287560b59f4435f9689eebbae6a2b61a1ec1b7d - languageName: node - linkType: hard - "helmet@npm:^6.0.0": version: 6.0.0 resolution: "helmet@npm:6.0.0" @@ -25137,7 +24016,7 @@ __metadata: languageName: node linkType: hard -"inherits@npm:2, inherits@npm:2.0.4, inherits@npm:^2.0.1, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.0, inherits@npm:~2.0.1, inherits@npm:~2.0.3": +"inherits@npm:2, inherits@npm:2.0.4, inherits@npm:^2.0.1, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.1, inherits@npm:~2.0.3": version: 2.0.4 resolution: "inherits@npm:2.0.4" checksum: 4a48a733847879d6cf6691860a6b1e3f0f4754176e4d71494c41f3475553768b10f84b5ce1d40fbd0e34e6bfbb864ee35858ad4dd2cf31e02fc4a154b724d7f1 @@ -25210,7 +24089,7 @@ __metadata: languageName: node linkType: hard -"inquirer@npm:^8.0.0, inquirer@npm:^8.1.1, inquirer@npm:^8.2.0": +"inquirer@npm:^8.0.0, inquirer@npm:^8.2.0": version: 8.2.5 resolution: "inquirer@npm:8.2.5" dependencies: @@ -27503,17 +26382,7 @@ __metadata: languageName: node linkType: hard -"keyv-memcache@npm:^1.2.5": - version: 1.3.3 - resolution: "keyv-memcache@npm:1.3.3" - dependencies: - json-buffer: ^3.0.1 - memjs: ^1.3.0 - checksum: ddef602c027a325fc1d18917788799e79cf73f46cbbfefeeda893ca585531544aa4b50de6601c1c5e0427a94bb52ad64e9144c919105b26aac0b173ebcdabc28 - languageName: node - linkType: hard - -"keyv@npm:^4.0.0, keyv@npm:^4.0.3, keyv@npm:^4.5.2": +"keyv@npm:^4.0.0, keyv@npm:^4.5.2": version: 4.5.2 resolution: "keyv@npm:4.5.2" dependencies: @@ -27789,13 +26658,6 @@ __metadata: languageName: node linkType: hard -"listenercount@npm:~1.0.1": - version: 1.0.1 - resolution: "listenercount@npm:1.0.1" - checksum: 0f1c9077cdaf2ebc16473c7d72eb7de6d983898ca42500f03da63c3914b6b312dd5f7a90d2657691ea25adf3fe0ac5a43226e8b2c673fd73415ed038041f4757 - languageName: node - linkType: hard - "listr2@npm:^3.8.3": version: 3.14.0 resolution: "listr2@npm:3.14.0" @@ -28882,7 +27744,7 @@ __metadata: languageName: node linkType: hard -"methods@npm:^1.0.0, methods@npm:^1.1.1, methods@npm:^1.1.2, methods@npm:~1.1.2": +"methods@npm:^1.0.0, methods@npm:^1.1.2, methods@npm:~1.1.2": version: 1.1.2 resolution: "methods@npm:1.1.2" checksum: 0917ff4041fa8e2f2fda5425a955fe16ca411591fbd123c0d722fcf02b73971ed6f764d85f0a6f547ce49ee0221ce2c19a5fa692157931cecb422984f1dcd13a @@ -29271,7 +28133,7 @@ __metadata: languageName: node linkType: hard -"mime@npm:1.6.0, mime@npm:^1.3.4, mime@npm:^1.4.1": +"mime@npm:1.6.0, mime@npm:^1.3.4": version: 1.6.0 resolution: "mime@npm:1.6.0" bin: @@ -29555,17 +28417,6 @@ __metadata: languageName: node linkType: hard -"mkdirp@npm:>=0.5 0": - version: 0.5.6 - resolution: "mkdirp@npm:0.5.6" - dependencies: - minimist: ^1.2.6 - bin: - mkdirp: bin/cmd.js - checksum: 0c91b721bb12c3f9af4b77ebf73604baf350e64d80df91754dc509491ae93bf238581e59c7188360cec7cb62fc4100959245a42cfe01834efedc5e9d068376c2 - languageName: node - linkType: hard - "mkdirp@npm:^0.5.1": version: 0.5.5 resolution: "mkdirp@npm:0.5.5" @@ -29648,36 +28499,6 @@ __metadata: languageName: node linkType: hard -"msw@npm:^0.35.0": - version: 0.35.0 - resolution: "msw@npm:0.35.0" - dependencies: - "@mswjs/cookies": ^0.1.6 - "@mswjs/interceptors": ^0.12.6 - "@open-draft/until": ^1.0.3 - "@types/cookie": ^0.4.1 - "@types/inquirer": ^7.3.3 - "@types/js-levenshtein": ^1.1.0 - chalk: ^4.1.1 - chokidar: ^3.4.2 - cookie: ^0.4.1 - graphql: ^15.5.1 - headers-utils: ^3.0.2 - inquirer: ^8.1.1 - is-node-process: ^1.0.1 - js-levenshtein: ^1.1.6 - node-fetch: ^2.6.1 - node-match-path: ^0.6.3 - statuses: ^2.0.0 - strict-event-emitter: ^0.2.0 - type-fest: ^1.2.2 - yargs: ^17.0.1 - bin: - msw: cli/index.js - checksum: cc5e85573e85779a95b1d5bfc306d4d0bcb4de660ac08c640e316c00fe4a0ea24dfe1decf6a46d10538550dfc7f7559836e1fda3d6e1f60ee7e99916bb3db3ff - languageName: node - linkType: hard - "msw@npm:^0.39.2": version: 0.39.2 resolution: "msw@npm:0.39.2" @@ -30086,13 +28907,6 @@ __metadata: languageName: node linkType: hard -"node-match-path@npm:^0.6.3": - version: 0.6.3 - resolution: "node-match-path@npm:0.6.3" - checksum: d515bc069f293688109c058ee02567528fdaa856290d362b80a2254734975014e4eefcdcc5164a8adfd5560aa870e277c97fe8be648074d5088056cf61553c7c - languageName: node - linkType: hard - "node-releases@npm:^2.0.6": version: 2.0.6 resolution: "node-releases@npm:2.0.6" @@ -30181,7 +28995,7 @@ __metadata: languageName: node linkType: hard -"normalize-url@npm:^6.0.1, normalize-url@npm:^6.1.0": +"normalize-url@npm:^6.0.1": version: 6.1.0 resolution: "normalize-url@npm:6.1.0" checksum: 4a4944631173e7d521d6b80e4c85ccaeceb2870f315584fa30121f505a6dfd86439c5e3fdd8cd9e0e291290c41d0c3599f0cb12ab356722ed242584c30348e50 @@ -30739,7 +29553,7 @@ __metadata: languageName: node linkType: hard -"outvariant@npm:^1.2.0, outvariant@npm:^1.2.1, outvariant@npm:^1.3.0": +"outvariant@npm:^1.2.1, outvariant@npm:^1.3.0": version: 1.3.0 resolution: "outvariant@npm:1.3.0" checksum: ac76ca375c1c642989e1c74f0e9ebac84c05bc9fdc8f28be949c16fae1658e9f1f2fb1133fe3cc1e98afabef78fe4298fe9360b5734baf8e6ad440c182680848 @@ -31119,15 +29933,6 @@ __metadata: languageName: node linkType: hard -"parse-path@npm:^5.0.0": - version: 5.0.0 - resolution: "parse-path@npm:5.0.0" - dependencies: - protocols: ^2.0.0 - checksum: e9f670559cd8e535f39f548bf5d41ad96a220190ea98df33d0babd9dfaa7c3c70ee2e55394078517d5e7e93c6a39c8eac1261ed3f9e68033656614fc954262e8 - languageName: node - linkType: hard - "parse-path@npm:^7.0.0": version: 7.0.0 resolution: "parse-path@npm:7.0.0" @@ -31137,18 +29942,6 @@ __metadata: languageName: node linkType: hard -"parse-url@npm:^7.0.2": - version: 7.0.2 - resolution: "parse-url@npm:7.0.2" - dependencies: - is-ssh: ^1.4.0 - normalize-url: ^6.1.0 - parse-path: ^5.0.0 - protocols: ^2.0.1 - checksum: 3e26852706bebe9fac409909316716dee52883d2fb5c82d65577effba1507abb7bc42bb59ce0ba6c8659168fb99acf89000bd8fe096ed3ad7124fa85227436d7 - languageName: node - linkType: hard - "parse-url@npm:^8.1.0": version: 8.1.0 resolution: "parse-url@npm:8.1.0" @@ -32831,15 +31624,6 @@ __metadata: languageName: node linkType: hard -"qs@npm:^6.5.1": - version: 6.11.0 - resolution: "qs@npm:6.11.0" - dependencies: - side-channel: ^1.0.4 - checksum: 6e1f29dd5385f7488ec74ac7b6c92f4d09a90408882d0c208414a34dd33badc1a621019d4c799a3df15ab9b1d0292f97c1dd71dc7c045e69f81a8064e5af7297 - languageName: node - linkType: hard - "qs@npm:~6.5.2": version: 6.5.2 resolution: "qs@npm:6.5.2" @@ -33645,7 +32429,7 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:^2.0.0, readable-stream@npm:^2.0.1, readable-stream@npm:^2.0.2, readable-stream@npm:^2.0.5, readable-stream@npm:^2.0.6, readable-stream@npm:^2.3.3, readable-stream@npm:^2.3.5, readable-stream@npm:^2.3.6, readable-stream@npm:~2.3.6": +"readable-stream@npm:^2.0.0, readable-stream@npm:^2.0.1, readable-stream@npm:^2.0.2, readable-stream@npm:^2.0.5, readable-stream@npm:^2.0.6, readable-stream@npm:^2.3.3, readable-stream@npm:^2.3.5, readable-stream@npm:^2.3.6": version: 2.3.7 resolution: "readable-stream@npm:2.3.7" dependencies: @@ -34383,17 +33167,6 @@ __metadata: languageName: node linkType: hard -"rimraf@npm:2, rimraf@npm:^2.6.3": - version: 2.7.1 - resolution: "rimraf@npm:2.7.1" - dependencies: - glob: ^7.1.3 - bin: - rimraf: ./bin.js - checksum: cdc7f6eacb17927f2a075117a823e1c5951792c6498ebcce81ca8203454a811d4cf8900314154d3259bb8f0b42ab17f67396a8694a54cae3283326e57ad250cd - languageName: node - linkType: hard - "rimraf@npm:3.0.2, rimraf@npm:^3.0.0, rimraf@npm:^3.0.2": version: 3.0.2 resolution: "rimraf@npm:3.0.2" @@ -34405,6 +33178,17 @@ __metadata: languageName: node linkType: hard +"rimraf@npm:^2.6.3": + version: 2.7.1 + resolution: "rimraf@npm:2.7.1" + dependencies: + glob: ^7.1.3 + bin: + rimraf: ./bin.js + checksum: cdc7f6eacb17927f2a075117a823e1c5951792c6498ebcce81ca8203454a811d4cf8900314154d3259bb8f0b42ab17f67396a8694a54cae3283326e57ad250cd + languageName: node + linkType: hard + "rimraf@npm:~2.6.2": version: 2.6.3 resolution: "rimraf@npm:2.6.3" @@ -34634,7 +33418,7 @@ __metadata: languageName: node linkType: hard -"rxjs@npm:^6.4.0, rxjs@npm:^6.6.3": +"rxjs@npm:^6.6.3": version: 6.6.7 resolution: "rxjs@npm:6.6.7" dependencies: @@ -35047,7 +33831,7 @@ __metadata: languageName: node linkType: hard -"setimmediate@npm:^1.0.4, setimmediate@npm:^1.0.5, setimmediate@npm:~1.0.4": +"setimmediate@npm:^1.0.4, setimmediate@npm:^1.0.5": version: 1.0.5 resolution: "setimmediate@npm:1.0.5" checksum: c9a6f2c5b51a2dabdc0247db9c46460152ffc62ee139f3157440bd48e7c59425093f42719ac1d7931f054f153e2d26cf37dfeb8da17a794a58198a2705e527fd @@ -36317,24 +35101,6 @@ __metadata: languageName: node linkType: hard -"superagent@npm:^3.8.3": - version: 3.8.3 - resolution: "superagent@npm:3.8.3" - dependencies: - component-emitter: ^1.2.0 - cookiejar: ^2.1.0 - debug: ^3.1.0 - extend: ^3.0.0 - form-data: ^2.3.1 - formidable: ^1.2.0 - methods: ^1.1.1 - mime: ^1.4.1 - qs: ^6.5.1 - readable-stream: ^2.3.5 - checksum: b13d0303259d76c9180bd40d97d9f0713760f5ced1aef089bdb2fcdf69cfaef89004cd6e986416d59bd9a2f0f9933d72521b5171fa26f89b781a2c3460c516fe - languageName: node - linkType: hard - "superagent@npm:^8.0.0": version: 8.0.0 resolution: "superagent@npm:8.0.0" @@ -36354,16 +35120,6 @@ __metadata: languageName: node linkType: hard -"supertest@npm:^4.0.2": - version: 4.0.2 - resolution: "supertest@npm:4.0.2" - dependencies: - methods: ^1.1.2 - superagent: ^3.8.3 - checksum: ec848088c5d0f6743a0829331c274f67ab4c2e5e6ba724f2fdf4965a73afa22d5091448323e28ae5b69137e42fe3ac67235dfc5b0a0acda3c088bdd193313319 - languageName: node - linkType: hard - "supertest@npm:^6.1.3, supertest@npm:^6.1.6, supertest@npm:^6.2.4": version: 6.2.4 resolution: "supertest@npm:6.2.4" @@ -37107,13 +35863,6 @@ __metadata: languageName: node linkType: hard -"traverse@npm:>=0.3.0 <0.4": - version: 0.3.9 - resolution: "traverse@npm:0.3.9" - checksum: 982982e4e249e9bbf063732a41fe5595939892758524bbef5d547c67cdf371b13af72b5434c6a61d88d4bb4351d6dabc6e22d832e0d16bc1bc684ef97a1cc59e - languageName: node - linkType: hard - "traverse@npm:^0.6.6, traverse@npm:~0.6.6": version: 0.6.6 resolution: "traverse@npm:0.6.6" @@ -37205,7 +35954,7 @@ __metadata: languageName: node linkType: hard -"ts-node@npm:^10.0.0, ts-node@npm:^10.2.1, ts-node@npm:^10.4.0, ts-node@npm:^10.8.1, ts-node@npm:^10.9.1": +"ts-node@npm:^10.0.0, ts-node@npm:^10.4.0, ts-node@npm:^10.8.1, ts-node@npm:^10.9.1": version: 10.9.1 resolution: "ts-node@npm:10.9.1" dependencies: @@ -37443,24 +36192,6 @@ __metadata: languageName: node linkType: hard -"typescript-json-schema@npm:^0.54.0": - version: 0.54.0 - resolution: "typescript-json-schema@npm:0.54.0" - dependencies: - "@types/json-schema": ^7.0.9 - "@types/node": ^16.9.2 - glob: ^7.1.7 - path-equal: ^1.1.2 - safe-stable-stringify: ^2.2.0 - ts-node: ^10.2.1 - typescript: ~4.6.0 - yargs: ^17.1.1 - bin: - typescript-json-schema: bin/typescript-json-schema - checksum: 49e03bd2612f79fe3ee9e9afcea34ae563da9aa799a8b4cf12b73feb60eb62a0786300eedb30261c69c482ed7e545acf1e5617d59861e6deee4f6570a658de88 - languageName: node - linkType: hard - "typescript-json-schema@npm:^0.55.0": version: 0.55.0 resolution: "typescript-json-schema@npm:0.55.0" @@ -37479,7 +36210,7 @@ __metadata: languageName: node linkType: hard -"typescript@npm:~4.6.0, typescript@npm:~4.6.3": +"typescript@npm:~4.6.3": version: 4.6.4 resolution: "typescript@npm:4.6.4" bin: @@ -37509,7 +36240,7 @@ __metadata: languageName: node linkType: hard -"typescript@patch:typescript@~4.6.0#~builtin, typescript@patch:typescript@~4.6.3#~builtin": +"typescript@patch:typescript@~4.6.3#~builtin": version: 4.6.4 resolution: "typescript@patch:typescript@npm%3A4.6.4#~builtin::version=4.6.4&hash=a1c5e5" bin: @@ -37843,24 +36574,6 @@ __metadata: languageName: node linkType: hard -"unzipper@npm:^0.10.11": - version: 0.10.11 - resolution: "unzipper@npm:0.10.11" - dependencies: - big-integer: ^1.6.17 - binary: ~0.3.0 - bluebird: ~3.4.1 - buffer-indexof-polyfill: ~1.0.0 - duplexer2: ~0.1.4 - fstream: ^1.0.12 - graceful-fs: ^4.2.2 - listenercount: ~1.0.1 - readable-stream: ~2.3.6 - setimmediate: ~1.0.4 - checksum: 006cd43ec4d6df47d86aa6b15044a606f50cdcd6a3d6f96f64f54ca0b663c09abb221f76edca0e9592511036d37ea094b1d76ce92c5bf10d7c6eb56f0be678f8 - languageName: node - linkType: hard - "update-browserslist-db@npm:^1.0.9": version: 1.0.9 resolution: "update-browserslist-db@npm:1.0.9" @@ -39171,7 +37884,7 @@ __metadata: languageName: node linkType: hard -"yargs-parser@npm:^21.0.0, yargs-parser@npm:^21.1.1": +"yargs-parser@npm:^21.0.0": version: 21.1.1 resolution: "yargs-parser@npm:21.1.1" checksum: ed2d96a616a9e3e1cc7d204c62ecc61f7aaab633dcbfab2c6df50f7f87b393993fe6640d017759fe112d0cb1e0119f2b4150a87305cc873fd90831c6a58ccf1c @@ -39237,21 +37950,6 @@ __metadata: languageName: node linkType: hard -"yargs@npm:^17.0.1": - version: 17.6.2 - resolution: "yargs@npm:17.6.2" - dependencies: - cliui: ^8.0.1 - escalade: ^3.1.1 - get-caller-file: ^2.0.5 - require-directory: ^2.1.1 - string-width: ^4.2.3 - y18n: ^5.0.5 - yargs-parser: ^21.1.1 - checksum: 47da1b0d854fa16d45a3ded57b716b013b2179022352a5f7467409da5a04a1eef5b3b3d97a2dfc13e8bbe5f2ffc0afe3bc6a4a72f8254e60f5a4bd7947138643 - languageName: node - linkType: hard - "yargs@npm:^5.0.0": version: 5.0.0 resolution: "yargs@npm:5.0.0" From 98cc32359ace45783b158d542a91aa04abc2136a Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 27 Oct 2022 12:58:31 -0700 Subject: [PATCH 12/54] Include eslint config and UUID dependency Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/incremental-ingestion-backend/package.json b/plugins/incremental-ingestion-backend/package.json index 199694ba63..f407cf03bb 100644 --- a/plugins/incremental-ingestion-backend/package.json +++ b/plugins/incremental-ingestion-backend/package.json @@ -21,6 +21,7 @@ "clean": "backstage-cli package clean", "prepack": "backstage-cli package prepack", "postpack": "backstage-cli package postpack" + }, "dependencies": { "@backstage/backend-common": "workspace:^", @@ -31,6 +32,7 @@ "@backstage/plugin-permission-common": "workspace:^", "express": "^4.17.1", "express-promise-router": "^4.1.0", + "uuid": "^8.3.2", "winston": "^3.2.1" }, "devDependencies": { From 0f73743b05844fbebc64e982f0496e37503d769e Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 27 Oct 2022 13:06:56 -0700 Subject: [PATCH 13/54] Actually include eslint Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/.eslintrc.js | 1 + 1 file changed, 1 insertion(+) create mode 100644 plugins/incremental-ingestion-backend/.eslintrc.js diff --git a/plugins/incremental-ingestion-backend/.eslintrc.js b/plugins/incremental-ingestion-backend/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/incremental-ingestion-backend/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); From 79ce0de43f12a17d2c4544649cddc903faf54444 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 27 Oct 2022 15:04:44 -0700 Subject: [PATCH 14/54] Install missing dependencies Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/package.json | 3 ++- yarn.lock | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/incremental-ingestion-backend/package.json b/plugins/incremental-ingestion-backend/package.json index f407cf03bb..0e02e46959 100644 --- a/plugins/incremental-ingestion-backend/package.json +++ b/plugins/incremental-ingestion-backend/package.json @@ -21,7 +21,6 @@ "clean": "backstage-cli package clean", "prepack": "backstage-cli package prepack", "postpack": "backstage-cli package postpack" - }, "dependencies": { "@backstage/backend-common": "workspace:^", @@ -32,6 +31,8 @@ "@backstage/plugin-permission-common": "workspace:^", "express": "^4.17.1", "express-promise-router": "^4.1.0", + "knex": "^2.0.0", + "luxon": "^3.0.0", "uuid": "^8.3.2", "winston": "^3.2.1" }, diff --git a/yarn.lock b/yarn.lock index fe0d4c4db0..57269476f1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6037,6 +6037,9 @@ __metadata: "@types/express": ^4.17.6 express: ^4.17.1 express-promise-router: ^4.1.0 + knex: ^2.0.0 + luxon: ^3.0.0 + uuid: ^8.3.2 winston: ^3.2.1 languageName: unknown linkType: soft From e8856d5ef9912a9dabe676a2b79377f08d0f84bf Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 27 Oct 2022 15:05:17 -0700 Subject: [PATCH 15/54] Use ESLint-enforced code styling Signed-off-by: Damon Kaswell --- .../src/engine/IncrementalIngestionEngine.ts | 10 +++++----- .../src/service/IncrementalCatalogBuilder.ts | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts index 6182ae95c4..bc347310a8 100644 --- a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts @@ -128,11 +128,10 @@ export class IncrementalIngestionEngine implements IterationEngine { attempts: record.attempts as number, nextActionAt: record.next_action_at.valueOf() as number, }; - } else { - const result = await this.manager.createProviderIngestionRecord(providerName); - this.options.logger.info(`incremental-engine: Ingestion record created: '${result.ingestionId}'`); - return result; - } + } + const result = await this.manager.createProviderIngestionRecord(providerName); + this.options.logger.info(`incremental-engine: Ingestion record created: '${result.ingestionId}'`); + return result; } async ingestOneBurst(id: string, signal: AbortSignal) { @@ -149,6 +148,7 @@ export class IncrementalIngestionEngine implements IterationEngine { await this.options.provider.around(async (context: unknown) => { let next = await this.options.provider.next(context, cursor); count++; + // eslint-disable-next-line no-constant-condition while (true) { done = next.done; await this.mark(id, sequence, next.entities, next.done, next.cursor); diff --git a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts index 93c7c571d8..30bdae9746 100644 --- a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts @@ -24,10 +24,10 @@ import { createIncrementalProviderRouter } from '../routes'; export class Deferred implements Promise { // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore + // @ts-ignore resolve: (value: T) => void; // eslint-disable-next-line @typescript-eslint/ban-ts-comment - //@ts-ignore + // @ts-ignore reject: (error: Error) => void; then: Promise['then']; From 2491dfeef5ab89d285453db92a95286a4e8a9c78 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 27 Oct 2022 15:46:34 -0700 Subject: [PATCH 16/54] Use workspace dependency in backend package Signed-off-by: Damon Kaswell --- packages/backend/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 1a5d971f93..ad1734e45a 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -44,7 +44,7 @@ "@backstage/plugin-events-backend": "workspace:^", "@backstage/plugin-events-node": "workspace:^", "@backstage/plugin-graphql-backend": "workspace:^", - "@backstage/plugin-incremental-ingestion-backend": "^0.1.0", + "@backstage/plugin-incremental-ingestion-backend": "workspace:^", "@backstage/plugin-jenkins-backend": "workspace:^", "@backstage/plugin-kafka-backend": "workspace:^", "@backstage/plugin-kubernetes-backend": "workspace:^", From 2389b4d521e3b8de8a267f05c5aae0a179c0ac6c Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 27 Oct 2022 15:50:20 -0700 Subject: [PATCH 17/54] Update yarn.lock with workspace deps Signed-off-by: Damon Kaswell --- yarn.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn.lock b/yarn.lock index 57269476f1..c3c513a481 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6023,7 +6023,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-incremental-ingestion-backend@^0.1.0, @backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend": +"@backstage/plugin-incremental-ingestion-backend@workspace:^, @backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend": version: 0.0.0-use.local resolution: "@backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend" dependencies: @@ -21371,7 +21371,7 @@ __metadata: "@backstage/plugin-events-backend": "workspace:^" "@backstage/plugin-events-node": "workspace:^" "@backstage/plugin-graphql-backend": "workspace:^" - "@backstage/plugin-incremental-ingestion-backend": ^0.1.0 + "@backstage/plugin-incremental-ingestion-backend": "workspace:^" "@backstage/plugin-jenkins-backend": "workspace:^" "@backstage/plugin-kafka-backend": "workspace:^" "@backstage/plugin-kubernetes-backend": "workspace:^" From 2ef75976b1976a170377506f6a5ebadc96721ff9 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 27 Oct 2022 16:25:17 -0700 Subject: [PATCH 18/54] Run prettier on sources Signed-off-by: Damon Kaswell --- .../incremental-ingestion-backend/README.md | 87 ++++++++------- ...21019897897_add-unique-active-ingestion.js | 28 +++-- .../IncrementalIngestionDatabaseManager.ts | 87 +++++++++++---- .../src/database/migrations.ts | 5 +- .../src/engine/IncrementalIngestionEngine.ts | 101 +++++++++++++----- .../src/routes.ts | 53 +++++++-- .../src/service/IncrementalCatalogBuilder.ts | 27 +++-- .../src/types.ts | 41 +++++-- 8 files changed, 318 insertions(+), 111 deletions(-) diff --git a/plugins/incremental-ingestion-backend/README.md b/plugins/incremental-ingestion-backend/README.md index d06334f396..4e9d4fe65b 100644 --- a/plugins/incremental-ingestion-backend/README.md +++ b/plugins/incremental-ingestion-backend/README.md @@ -44,27 +44,28 @@ The Incremental Entity Provider backend is designed for data sources that provid 1. Install `@backstage/plugin-incremental-ingestion-backend` with `yarn add @backstage/plugin-incremental-ingestion-backend` 2. Import `IncrementalCatalogBuilder` from `@backstage/plugin-incremental-ingestion-backend` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`. - ```ts - const builder = CatalogBuilder.create(env); - // incremental builder receives builder because it'll register - // incremental entity providers with the builder - const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); - ``` -3. Last step, add `await incrementBuilder.build()` after `await builder.build()` to ensure that all `CatalogBuider` migration run before running `incrementBuilder.build()` migrations. - ```ts - const { processingEngine, router } = await builder.build(); - // this has to run after `await builder.build()` to ensure that catalog migrations are completed - // before incremental builder migrations are executed - await incrementalBuilder.build(); - ``` +```ts +const builder = CatalogBuilder.create(env); +// incremental builder receives builder because it'll register +// incremental entity providers with the builder +const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); +``` + +3. Last step, add `await incrementBuilder.build()` after `await builder.build()` to ensure that all `CatalogBuider` migration run before running `incrementBuilder.build()` migrations. + +```ts +const { processingEngine, router } = await builder.build(); + +// this has to run after `await builder.build()` to ensure that catalog migrations are completed +// before incremental builder migrations are executed +await incrementalBuilder.build(); +``` The result should look something like this, ```ts -import { - CatalogBuilder -} from '@backstage/plugin-catalog-backend'; +import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; import { IncrementalCatalogBuilder } from '@backstage/plugin-incremental-ingestion-backend'; import { Router } from 'express'; @@ -74,18 +75,20 @@ import { PluginEnvironment } from '../types'; export default async function createPlugin( env: PluginEnvironment, ): Promise { - const builder = CatalogBuilder.create(env); // incremental builder receives builder because it'll register - // incremental entity providers with the builder - const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); + // incremental entity providers with the builder + const incrementalBuilder = await IncrementalCatalogBuilder.create( + env, + builder, + ); builder.addProcessor(new ScaffolderEntitiesProcessor()); const { processingEngine, router } = await builder.build(); - // this has to run after `await builder.build()` so ensure that catalog migrations are completed - // before incremental builder migrations are executed + // this has to run after `await builder.build()` so ensure that catalog migrations are completed + // before incremental builder migrations are executed const { incrementalAdminRouter } = await incrementalBuilder.build(); router.use('/incremental', incrementalAdminRouter); @@ -128,7 +131,10 @@ interface IncrementalEntityProvider { * @returns the entities to be ingested, as well as the cursor of * the the next page after this one. */ - next(context: TContext, cursor?: TCursor): Promise>; + next( + context: TContext, + cursor?: TCursor, + ): Promise>; } ``` @@ -136,7 +142,7 @@ For tutorial, we'll write an Incremental Entity Provider that will call an imagi ```ts interface MyApiClient { - getServices(page: number): MyPaginatedResults + getServices(page: number): MyPaginatedResults; } interface MyPaginatedResults { @@ -152,7 +158,10 @@ interface Service { These are the only 3 methods that you need to implement. `getProviderName()` is pretty self explanatory and it's exactly same as on Entity Provider. ```ts -import { IncrementalEntityProvider, EntityIteratorResult } from '@backstage/plugin-incremental-ingestion-backend'; +import { + IncrementalEntityProvider, + EntityIteratorResult, +} from '@backstage/plugin-incremental-ingestion-backend'; // this will include your pagination information, let's say our API accepts a `page` parameter. // In this case, the cursor will include `page` @@ -162,10 +171,12 @@ interface MyApiCursor { // This interface describes the type of data that will be passed to your burst function. interface MyContext { - apiClient: MyApiClient + apiClient: MyApiClient; } -export class MyIncrementalEntityProvider implements IncrementalEntityProvider { +export class MyIncrementalEntityProvider + implements IncrementalEntityProvider +{ getProviderName() { return `MyIncrementalEntityProvider`; } @@ -175,13 +186,14 @@ export class MyIncrementalEntityProvider implements IncrementalEntityProvider { +export class MyIncrementalEntityProvider + implements IncrementalEntityProvider +{ getProviderName() { return `MyIncrementalEntityProvider`; } async around(burst: (context: MyContext) => Promise): Promise { - const apiClient = new MyApiClient(); await burst({ apiClient }); @@ -194,10 +206,11 @@ export class MyIncrementalEntityProvider implements IncrementalEntityProvider { - +export class MyIncrementalEntityProvider + implements IncrementalEntityProvider +{ token: string; - + constructor(token: string) { this.token = token; } @@ -206,12 +219,10 @@ export class MyIncrementalEntityProvider implements IncrementalEntityProvider Promise): Promise { + const apiClient = new MyApiClient(this.token); - const apiClient = new MyApiClient(this.token) - - await burst({ apiClient }) + await burst({ apiClient }); } } ``` @@ -220,9 +231,9 @@ The last step is to implement the actual `next` method that will accept the curs ```ts export class MyIncrementalEntityProvider implements IncrementalEntityProvider { - + token: string; - + constructor(token: string) { this.token = token; } @@ -284,7 +295,7 @@ export class MyIncrementalEntityProvider implements IncrementalEntityProvider knex.schema.withSchema('ingestion'); await knex.transaction(async tx => { - const providers = await tx('ingestion.ingestions').distinct('provider_name'); + const providers = await tx('ingestion.ingestions').distinct( + 'provider_name', + ); for (const provider of providers) { const valid = await tx('ingestion.ingestions') .where('provider_name', provider) @@ -44,8 +46,15 @@ exports.up = async function up(knex) { await tx('ingestion.ingestions').delete().whereIn('id', invalid); await tx('ingestion.ingestion_mark_entities') .delete() - .whereIn('ingestion_mark_id', tx('ingestion.ingestion_marks').select('id').whereIn('ingestion_id', invalid)); - await tx('ingestion.ingestion_marks').delete().whereIn('ingestion_id', invalid); + .whereIn( + 'ingestion_mark_id', + tx('ingestion.ingestion_marks') + .select('id') + .whereIn('ingestion_id', invalid), + ); + await tx('ingestion.ingestion_marks') + .delete() + .whereIn('ingestion_id', invalid); } } }); @@ -55,11 +64,18 @@ exports.up = async function up(knex) { }); await knex.transaction(async tx => { - await tx('ingestion.ingestions').update('completion_ticket', 'open').where('rest_completed_at', null); + await tx('ingestion.ingestions') + .update('completion_ticket', 'open') + .where('rest_completed_at', null); - const rows = await tx('ingestion.ingestions').whereNot('rest_completed_at', null); + const rows = await tx('ingestion.ingestions').whereNot( + 'rest_completed_at', + null, + ); for (const row of rows) { - await tx('ingestion.ingestions').update('completion_ticket', uuidv4()).where('id', row.id); + await tx('ingestion.ingestions') + .update('completion_ticket', uuidv4()) + .where('id', row.id); } }); diff --git a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts index b736693df8..d593bf4292 100644 --- a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts +++ b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts @@ -51,7 +51,10 @@ export class IncrementalIngestionDatabaseManager { * @param provider string * @param update Partial */ - async updateIngestionRecordByProvider(provider: string, update: Partial) { + async updateIngestionRecordByProvider( + provider: string, + update: Partial, + ) { await this.client.transaction(async tx => { await tx('ingestion.ingestions') .where('provider_name', provider) @@ -71,7 +74,10 @@ export class IncrementalIngestionDatabaseManager { }); } - private async deleteMarkEntities(tx: Knex.Transaction, ids: { id: string }[]) { + private async deleteMarkEntities( + tx: Knex.Transaction, + ids: { id: string }[], + ) { const chunks: { id: string }[][] = []; for (let i = 0; i < ids.length; i += 100) { const chunk = ids.slice(i, i + 100); @@ -174,8 +180,15 @@ export class IncrementalIngestionDatabaseManager { await tx('ingestion.ingestions').delete().whereIn('id', invalid); await tx('ingestion.ingestion_mark_entities') .delete() - .whereIn('ingestion_mark_id', tx('ingestion.ingestion_marks').select('id').whereIn('ingestion_id', invalid)); - await tx('ingestion.ingestion_marks').delete().whereIn('ingestion_id', invalid); + .whereIn( + 'ingestion_mark_id', + tx('ingestion.ingestion_marks') + .select('id') + .whereIn('ingestion_id', invalid), + ); + await tx('ingestion.ingestion_marks') + .delete() + .whereIn('ingestion_id', invalid); } }); } @@ -212,7 +225,10 @@ export class IncrementalIngestionDatabaseManager { ) : []; - const markEntitiesDeleted = await this.deleteMarkEntities(tx, markEntityIDs); + const markEntitiesDeleted = await this.deleteMarkEntities( + tx, + markEntityIDs, + ); const marksDeleted = markIDs.length > 0 @@ -224,7 +240,9 @@ export class IncrementalIngestionDatabaseManager { ) : 0; - const ingestionsDeleted = await tx('ingestion.ingestions').delete().where('provider_name', provider); + const ingestionsDeleted = await tx('ingestion.ingestions') + .delete() + .where('provider_name', provider); const next_action_at = new Date(); next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000); @@ -271,9 +289,18 @@ export class IncrementalIngestionDatabaseManager { */ async computeRemoved(provider: string, ingestionId: string) { return await this.client.transaction(async tx => { - const removed: { entity: string; ref: string }[] = await tx('final_entities') - .select(tx.ref('final_entity').as('entity'), tx.ref('refresh_state.entity_ref').as('ref')) - .join('refresh_state', 'refresh_state.entity_id', 'final_entities.entity_id') + const removed: { entity: string; ref: string }[] = await tx( + 'final_entities', + ) + .select( + tx.ref('final_entity').as('entity'), + tx.ref('refresh_state.entity_ref').as('ref'), + ) + .join( + 'refresh_state', + 'refresh_state.entity_id', + 'final_entities.entity_id', + ) .whereRaw( `((final_entity::json #>> '{metadata, annotations, ${INCREMENTAL_ENTITY_PROVIDER_ANNOTATION}}')) = ?`, [provider], @@ -301,7 +328,9 @@ export class IncrementalIngestionDatabaseManager { */ async healthcheck() { return await this.client.transaction(async tx => { - const records = await tx<{ id: string; provider_name: string }>('ingestion.ingestions') + const records = await tx<{ id: string; provider_name: string }>( + 'ingestion.ingestions', + ) .distinct('id', 'provider_name') .where('rest_completed_at', null); return records; @@ -313,7 +342,9 @@ export class IncrementalIngestionDatabaseManager { * @param provider string */ async triggerNextProviderAction(provider: string) { - await this.updateIngestionRecordByProvider(provider, { next_action_at: new Date() }); + await this.updateIngestionRecordByProvider(provider, { + next_action_at: new Date(), + }); } /** @@ -346,8 +377,12 @@ export class IncrementalIngestionDatabaseManager { }); } - const ingestionMarksDeleted = await this.purgeTable('ingestion.ingestion_marks'); - const markEntitiesDeleted = await this.purgeTable('ingestion.ingestion_mark_entities'); + const ingestionMarksDeleted = await this.purgeTable( + 'ingestion.ingestion_marks', + ); + const markEntitiesDeleted = await this.purgeTable( + 'ingestion.ingestion_mark_entities', + ); return { ingestionsDeleted, ingestionMarksDeleted, markEntitiesDeleted }; } @@ -357,7 +392,10 @@ export class IncrementalIngestionDatabaseManager { * @param ingestionId string */ async setProviderIngesting(ingestionId: string) { - await this.updateIngestionRecordById({ ingestionId, update: { next_action: 'ingest' } }); + await this.updateIngestionRecordById({ + ingestionId, + update: { next_action: 'ingest' }, + }); } /** @@ -365,7 +403,10 @@ export class IncrementalIngestionDatabaseManager { * @param ingestionId string */ async setProviderBursting(ingestionId: string) { - await this.updateIngestionRecordById({ ingestionId, update: { status: 'bursting' } }); + await this.updateIngestionRecordById({ + ingestionId, + update: { status: 'bursting' }, + }); } /** @@ -405,7 +446,10 @@ export class IncrementalIngestionDatabaseManager { * @param ingestionId string */ async setProviderInterstitial(ingestionId: string) { - await this.updateIngestionRecordById({ ingestionId, update: { attempts: 0, status: 'interstitial' } }); + await this.updateIngestionRecordById({ + ingestionId, + update: { attempts: 0, status: 'interstitial' }, + }); } /** @@ -445,7 +489,12 @@ export class IncrementalIngestionDatabaseManager { * @param error Error * @param backoffLength number */ - async setProviderBackoff(ingestionId: string, attempts: number, error: Error, backoffLength: number) { + async setProviderBackoff( + ingestionId: string, + attempts: number, + error: Error, + backoffLength: number, + ) { await this.updateIngestionRecordById({ ingestionId, update: { @@ -526,7 +575,9 @@ export class IncrementalIngestionDatabaseManager { */ async listProviders() { return await this.client.transaction(async tx => { - const providers = await tx<{ provider_name: string }>('ingestion.ingestions').distinct('provider_name'); + const providers = await tx<{ provider_name: string }>( + 'ingestion.ingestions', + ).distinct('provider_name'); return providers.map(entry => entry.provider_name); }); } diff --git a/plugins/incremental-ingestion-backend/src/database/migrations.ts b/plugins/incremental-ingestion-backend/src/database/migrations.ts index 8ec5d3a7e6..991a709732 100644 --- a/plugins/incremental-ingestion-backend/src/database/migrations.ts +++ b/plugins/incremental-ingestion-backend/src/database/migrations.ts @@ -17,7 +17,10 @@ import { resolvePackagePath } from '@backstage/backend-common'; import { Knex } from 'knex'; export async function applyDatabaseMigrations(knex: Knex): Promise { - const migrationsDir = resolvePackagePath('@devex/backend-incremental-ingestion', 'migrations'); + const migrationsDir = resolvePackagePath( + '@devex/backend-incremental-ingestion', + 'migrations', + ); await knex.raw('CREATE SCHEMA IF NOT EXISTS ingestion;'); diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts index bc347310a8..655380cec4 100644 --- a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts @@ -14,7 +14,11 @@ * limitations under the License. */ import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; -import { INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, IterationEngine, IterationEngineOptions } from '../types'; +import { + INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, + IterationEngine, + IterationEngineOptions, +} from '../types'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; import type { AbortSignal } from 'node-abort-controller'; @@ -31,7 +35,12 @@ export class IncrementalIngestionEngine implements IterationEngine { constructor(private options: IterationEngineOptions) { this.manager = options.manager; this.restLength = Duration.fromObject(options.restLength); - this.backoff = options.backoff ?? [{ minutes: 1 }, { minutes: 5 }, { minutes: 30 }, { hours: 3 }]; + this.backoff = options.backoff ?? [ + { minutes: 1 }, + { minutes: 5 }, + { minutes: 30 }, + { hours: 3 }, + ]; } async taskFn(signal: AbortSignal) { @@ -49,19 +58,24 @@ export class IncrementalIngestionEngine implements IterationEngine { async handleNextAction(signal: AbortSignal) { await this.options.ready; - const { ingestionId, nextActionAt, nextAction, attempts } = await this.getCurrentAction(); + const { ingestionId, nextActionAt, nextAction, attempts } = + await this.getCurrentAction(); switch (nextAction) { case 'rest': if (Date.now() > nextActionAt) { - await this.manager.clearFinishedIngestions(this.options.provider.getProviderName()); + await this.manager.clearFinishedIngestions( + this.options.provider.getProviderName(), + ); this.options.logger.info( `incremental-engine: Ingestion ${ingestionId} rest period complete. Ingestion will start again`, ); await this.manager.setProviderComplete(ingestionId); } else { - this.options.logger.info(`incremental-engine: Ingestion '${ingestionId}' rest period continuing`); + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' rest period continuing`, + ); } break; case 'ingest': @@ -75,14 +89,26 @@ export class IncrementalIngestionEngine implements IterationEngine { await this.manager.setProviderResting(ingestionId, this.restLength); } else { await this.manager.setProviderInterstitial(ingestionId); - this.options.logger.info(`incremental-engine: Ingestion '${ingestionId}' continuing`); + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' continuing`, + ); } } catch (error) { - if ((error as Error).message && (error as Error).message === 'CANCEL') { - this.options.logger.info(`incremental-engine: Ingestion '${ingestionId}' canceled`); - await this.manager.setProviderCanceling(ingestionId, (error as Error).message); + if ( + (error as Error).message && + (error as Error).message === 'CANCEL' + ) { + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' canceled`, + ); + await this.manager.setProviderCanceling( + ingestionId, + (error as Error).message, + ); } else { - const currentBackoff = Duration.fromObject(this.backoff[Math.min(this.backoff.length - 1, attempts)]); + const currentBackoff = Duration.fromObject( + this.backoff[Math.min(this.backoff.length - 1, attempts)], + ); const backoffLength = currentBackoff.as('milliseconds'); this.options.logger.error(error); @@ -92,7 +118,12 @@ export class IncrementalIngestionEngine implements IterationEngine { `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`, ); - await this.manager.setProviderBackoff(ingestionId, attempts, error as Error, backoffLength); + await this.manager.setProviderBackoff( + ingestionId, + attempts, + error as Error, + backoffLength, + ); } } break; @@ -103,11 +134,15 @@ export class IncrementalIngestionEngine implements IterationEngine { ); await this.manager.setProviderIngesting(ingestionId); } else { - this.options.logger.info(`incremental-engine: Ingestion '${ingestionId}' backoff continuing`); + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' backoff continuing`, + ); } break; case 'cancel': - this.options.logger.info(`incremental-engine: Ingestion '${ingestionId}' canceling, will restart`); + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' canceling, will restart`, + ); await this.manager.setProviderCanceled(ingestionId); break; default: @@ -121,16 +156,22 @@ export class IncrementalIngestionEngine implements IterationEngine { const providerName = this.options.provider.getProviderName(); const record = await this.manager.getCurrentIngestionRecord(providerName); if (record) { - this.options.logger.info(`incremental-engine: Ingestion record found: '${record.id}'`); + this.options.logger.info( + `incremental-engine: Ingestion record found: '${record.id}'`, + ); return { ingestionId: record.id, nextAction: record.next_action as 'rest' | 'ingest' | 'backoff', attempts: record.attempts as number, nextActionAt: record.next_action_at.valueOf() as number, }; - } - const result = await this.manager.createProviderIngestionRecord(providerName); - this.options.logger.info(`incremental-engine: Ingestion record created: '${result.ingestionId}'`); + } + const result = await this.manager.createProviderIngestionRecord( + providerName, + ); + this.options.logger.info( + `incremental-engine: Ingestion record created: '${result.ingestionId}'`, + ); return result; } @@ -143,7 +184,9 @@ export class IncrementalIngestionEngine implements IterationEngine { const start = performance.now(); let count = 0; let done = false; - this.options.logger.info(`incremental-engine: Ingestion '${id}' burst initiated`); + this.options.logger.info( + `incremental-engine: Ingestion '${id}' burst initiated`, + ); await this.options.provider.around(async (context: unknown) => { let next = await this.options.provider.next(context, cursor); @@ -170,11 +213,17 @@ export class IncrementalIngestionEngine implements IterationEngine { return done; } - async mark(id: string, sequence: number, entities: DeferredEntity[], done: boolean, cursor?: unknown) { + async mark( + id: string, + sequence: number, + entities: DeferredEntity[], + done: boolean, + cursor?: unknown, + ) { this.options.logger.debug( - `incremental-engine: Ingestion '${id}': MARK ${entities.length} entities, cursor: ${JSON.stringify( - cursor, - )}, done: ${done}`, + `incremental-engine: Ingestion '${id}': MARK ${ + entities.length + } entities, cursor: ${JSON.stringify(cursor)}, done: ${done}`, ); const markId = v4(); @@ -199,7 +248,8 @@ export class IncrementalIngestionEngine implements IterationEngine { ...deferred.entity.metadata, annotations: { ...deferred.entity.metadata.annotations, - [INCREMENTAL_ENTITY_PROVIDER_ANNOTATION]: this.options.provider.getProviderName(), + [INCREMENTAL_ENTITY_PROVIDER_ANNOTATION]: + this.options.provider.getProviderName(), }, }, }, @@ -207,7 +257,10 @@ export class IncrementalIngestionEngine implements IterationEngine { const removed: DeferredEntity[] = done ? [] - : await this.manager.computeRemoved(this.options.provider.getProviderName(), id); + : await this.manager.computeRemoved( + this.options.provider.getProviderName(), + id, + ); await this.options.connection.applyMutation({ type: 'delta', diff --git a/plugins/incremental-ingestion-backend/src/routes.ts b/plugins/incremental-ingestion-backend/src/routes.ts index 8e2f3ae8af..ca1743113e 100644 --- a/plugins/incremental-ingestion-backend/src/routes.ts +++ b/plugins/incremental-ingestion-backend/src/routes.ts @@ -19,7 +19,10 @@ import Router from 'express-promise-router'; import { Logger } from 'winston'; import { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager'; -export const createIncrementalProviderRouter = async (manager: IncrementalIngestionDatabaseManager, logger: Logger) => { +export const createIncrementalProviderRouter = async ( + manager: IncrementalIngestionDatabaseManager, + logger: Logger, +) => { const router = Router(); router.use(express.json()); @@ -27,7 +30,9 @@ export const createIncrementalProviderRouter = async (manager: IncrementalIngest router.get('/health', async (_, res) => { const records = await manager.healthcheck(); const providers = records.map(record => record.provider_name); - const duplicates = [...new Set(providers.filter((e, i, a) => a.indexOf(e) !== i))]; + const duplicates = [ + ...new Set(providers.filter((e, i, a) => a.indexOf(e) !== i)), + ]; if (duplicates.length > 0) { res.json({ healthy: false, duplicateIngestions: duplicates }); @@ -65,7 +70,9 @@ export const createIncrementalProviderRouter = async (manager: IncrementalIngest }, }); } else { - logger.error(`${provider} - No ingestion record found in the database!`); + logger.error( + `${provider} - No ingestion record found in the database!`, + ); res.status(404).json({ success: false, status: {}, @@ -81,7 +88,10 @@ export const createIncrementalProviderRouter = async (manager: IncrementalIngest const record = await manager.getCurrentIngestionRecord(provider); if (record) { await manager.triggerNextProviderAction(provider); - res.json({ success: true, message: `${provider}: Next action triggered.` }); + res.json({ + success: true, + message: `${provider}: Next action triggered.`, + }); } else { const providers: string[] = await manager.listProviders(); if (providers.includes(provider)) { @@ -91,7 +101,12 @@ export const createIncrementalProviderRouter = async (manager: IncrementalIngest message: 'Unable to trigger next action (provider is restarting)', }); } else { - res.status(404).json({ success: false, message: `Provider '${provider}' not found` }); + res + .status(404) + .json({ + success: false, + message: `Provider '${provider}' not found`, + }); } } }); @@ -108,7 +123,10 @@ export const createIncrementalProviderRouter = async (manager: IncrementalIngest rest_completed_at: new Date(), status: 'complete', }); - res.json({ success: true, message: `${provider}: Next cycle triggered.` }); + res.json({ + success: true, + message: `${provider}: Next cycle triggered.`, + }); } else { const providers: string[] = await manager.listProviders(); if (providers.includes(provider)) { @@ -118,7 +136,12 @@ export const createIncrementalProviderRouter = async (manager: IncrementalIngest message: 'Provider is already restarting', }); } else { - res.status(404).json({ success: false, message: `Provider '${provider}' not found` }); + res + .status(404) + .json({ + success: false, + message: `Provider '${provider}' not found`, + }); } } }); @@ -136,7 +159,10 @@ export const createIncrementalProviderRouter = async (manager: IncrementalIngest next_action_at, status: 'resting', }); - res.json({ success: true, message: `${provider}: Current ingestion canceled.` }); + res.json({ + success: true, + message: `${provider}: Current ingestion canceled.`, + }); } else { const providers: string[] = await manager.listProviders(); if (providers.includes(provider)) { @@ -146,7 +172,12 @@ export const createIncrementalProviderRouter = async (manager: IncrementalIngest message: 'Provider is currently restarting, please wait.', }); } else { - res.status(404).json({ success: false, message: `Provider '${provider}' not found` }); + res + .status(404) + .json({ + success: false, + message: `Provider '${provider}' not found`, + }); } } }); @@ -175,7 +206,9 @@ export const createIncrementalProviderRouter = async (manager: IncrementalIngest message: 'No records yet (provider is restarting)', }); } else { - logger.error(`${provider} - No ingestion record found in the database!`); + logger.error( + `${provider} - No ingestion record found in the database!`, + ); res.status(404).json({ success: false, status: {}, diff --git a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts index 30bdae9746..a9babf17e4 100644 --- a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts @@ -13,7 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { IncrementalEntityProvider, IncrementalEntityProviderOptions, PluginEnvironment } from '../types'; +import { + IncrementalEntityProvider, + IncrementalEntityProviderOptions, + PluginEnvironment, +} from '../types'; import { CatalogBuilder as CoreCatalogBuilder } from '@backstage/plugin-catalog-backend'; import { Duration } from 'luxon'; import { Knex } from 'knex'; @@ -76,9 +80,14 @@ export class IncrementalCatalogBuilder { await applyDatabaseMigrations(this.client); this.ready.resolve(); - const routerLogger = this.env.logger.child({ router: 'IncrementalProviderAdmin' }); + const routerLogger = this.env.logger.child({ + router: 'IncrementalProviderAdmin', + }); - const incrementalAdminRouter = await createIncrementalProviderRouter(this.manager, routerLogger); + const incrementalAdminRouter = await createIncrementalProviderRouter( + this.manager, + routerLogger, + ); return { incrementalAdminRouter, manager: this.manager }; } @@ -94,7 +103,9 @@ export class IncrementalCatalogBuilder { this.builder.addEntityProvider({ getProviderName: provider.getProviderName.bind(provider), async connect(connection) { - const logger = catalogLogger.child({ entityProvider: provider.getProviderName() }); + const logger = catalogLogger.child({ + entityProvider: provider.getProviderName(), + }); logger.info(`Connecting`); @@ -112,8 +123,12 @@ export class IncrementalCatalogBuilder { connection, }); - const frequency = Duration.isDuration(burstInterval) ? burstInterval : Duration.fromObject(burstInterval); - const length = Duration.isDuration(burstLength) ? burstLength : Duration.fromObject(burstLength); + const frequency = Duration.isDuration(burstInterval) + ? burstInterval + : Duration.fromObject(burstInterval); + const length = Duration.isDuration(burstLength) + ? burstLength + : Duration.fromObject(burstLength); await scheduler.scheduleTask({ id: provider.getProviderName(), diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/incremental-ingestion-backend/src/types.ts index 4a63391273..85f184b52e 100644 --- a/plugins/incremental-ingestion-backend/src/types.ts +++ b/plugins/incremental-ingestion-backend/src/types.ts @@ -13,10 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import type { PluginDatabaseManager, UrlReader } from '@backstage/backend-common'; -import type { PluginTaskScheduler, TaskFunction } from '@backstage/backend-tasks'; +import type { + PluginDatabaseManager, + UrlReader, +} from '@backstage/backend-common'; +import type { + PluginTaskScheduler, + TaskFunction, +} from '@backstage/backend-tasks'; import type { Config } from '@backstage/config'; -import type { DeferredEntity, EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import type { + DeferredEntity, + EntityProviderConnection, +} from '@backstage/plugin-catalog-backend'; import type { PermissionAuthorizer } from '@backstage/plugin-permission-common'; import type { DurationObjectUnits } from 'luxon'; import type { Logger } from 'winston'; @@ -27,7 +36,8 @@ import { IncrementalIngestionDatabaseManager } from './database/IncrementalInges * * @public */ -export const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = 'backstage.io/incremental-provider-name'; +export const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = + 'backstage.io/incremental-provider-name'; /** * Ingest entities into the catalog in bite-sized chunks. @@ -58,7 +68,10 @@ export interface IncrementalEntityProvider { * @returns the entities to be ingested, as well as the cursor of * the the next page after this one. */ - next(context: TContext, cursor?: TCursor): Promise>; + next( + context: TContext, + cursor?: TCursor, + ): Promise>; /** * Do any setup and teardown necessary in order to provide the @@ -131,7 +144,7 @@ export type PluginEnvironment = { }; /** - * The core ingestion engine implements this interface + * The core ingestion engine implements this interface */ export interface IterationEngine { taskFn: TaskFunction; @@ -154,8 +167,20 @@ export interface IterationEngineOptions { * The shape of data inserted into or updated in the `ingestion.ingestions` table. */ export interface IngestionUpsertIFace { - next_action: 'rest' | 'ingest' | 'backoff' | 'cancel' | 'nothing (done)' | 'nothing (canceled)'; - status: 'complete' | 'bursting' | 'resting' | 'canceling' | 'interstitial' | 'backing off'; + next_action: + | 'rest' + | 'ingest' + | 'backoff' + | 'cancel' + | 'nothing (done)' + | 'nothing (canceled)'; + status: + | 'complete' + | 'bursting' + | 'resting' + | 'canceling' + | 'interstitial' + | 'backing off'; provider_name: string; next_action_at?: Date; last_error?: string; From 59d7227a263680d910b146ab92014a38b9904e82 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 27 Oct 2022 16:33:50 -0700 Subject: [PATCH 19/54] Apply prettier to routes.ts Signed-off-by: Damon Kaswell --- .../src/routes.ts | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/plugins/incremental-ingestion-backend/src/routes.ts b/plugins/incremental-ingestion-backend/src/routes.ts index ca1743113e..a35f02e98e 100644 --- a/plugins/incremental-ingestion-backend/src/routes.ts +++ b/plugins/incremental-ingestion-backend/src/routes.ts @@ -101,12 +101,10 @@ export const createIncrementalProviderRouter = async ( message: 'Unable to trigger next action (provider is restarting)', }); } else { - res - .status(404) - .json({ - success: false, - message: `Provider '${provider}' not found`, - }); + res.status(404).json({ + success: false, + message: `Provider '${provider}' not found`, + }); } } }); @@ -136,12 +134,10 @@ export const createIncrementalProviderRouter = async ( message: 'Provider is already restarting', }); } else { - res - .status(404) - .json({ - success: false, - message: `Provider '${provider}' not found`, - }); + res.status(404).json({ + success: false, + message: `Provider '${provider}' not found`, + }); } } }); @@ -172,12 +168,10 @@ export const createIncrementalProviderRouter = async ( message: 'Provider is currently restarting, please wait.', }); } else { - res - .status(404) - .json({ - success: false, - message: `Provider '${provider}' not found`, - }); + res.status(404).json({ + success: false, + message: `Provider '${provider}' not found`, + }); } } }); From 981bfe3eebef06d1f19cb6c0451a0ca3b5f33086 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 28 Oct 2022 07:50:15 -0700 Subject: [PATCH 20/54] Include api-report.md Signed-off-by: Damon Kaswell --- .../api-report.md | 378 ++++++++++++++++++ 1 file changed, 378 insertions(+) create mode 100644 plugins/incremental-ingestion-backend/api-report.md diff --git a/plugins/incremental-ingestion-backend/api-report.md b/plugins/incremental-ingestion-backend/api-report.md new file mode 100644 index 0000000000..5443d9522a --- /dev/null +++ b/plugins/incremental-ingestion-backend/api-report.md @@ -0,0 +1,378 @@ +## API Report File for "@backstage/plugin-incremental-ingestion-backend" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; +import type { Config } from '@backstage/config'; +import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; +import { Duration } from 'luxon'; +import type { DurationObjectUnits } from 'luxon'; +import type { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { Knex } from 'knex'; +import type { Logger } from 'winston'; +import type { PermissionAuthorizer } from '@backstage/plugin-permission-common'; +import type { PluginDatabaseManager } from '@backstage/backend-common'; +import type { PluginTaskScheduler } from '@backstage/backend-tasks'; +import { Router } from 'express'; +import type { TaskFunction } from '@backstage/backend-tasks'; +import type { UrlReader } from '@backstage/backend-common'; + +// Warning: (ae-missing-release-tag) "Deferred" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class Deferred implements Promise { + // (undocumented) + [Symbol.toStringTag]: 'Deferred'; + constructor(); + // (undocumented) + catch: Promise['catch']; + // (undocumented) + finally: Promise['finally']; + // (undocumented) + reject: (error: Error) => void; + // (undocumented) + resolve: (value: T) => void; + // (undocumented) + then: Promise['then']; +} + +// Warning: (tsdoc-at-sign-without-tag-name) Expecting a TSDoc tag name after "@"; if it is not a tag, use a backslash to escape this character +// Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag +// Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" +// Warning: (ae-missing-release-tag) "EntityIteratorResult" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export interface EntityIteratorResult { + cursor: T; + done: boolean; + entities: DeferredEntity[]; +} + +// @public +export const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = + 'backstage.io/incremental-provider-name'; + +// Warning: (ae-missing-release-tag) "IncrementalCatalogBuilder" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class IncrementalCatalogBuilder { + // (undocumented) + addIncrementalEntityProvider( + provider: IncrementalEntityProvider, + options: IncrementalEntityProviderOptions, + ): void; + // (undocumented) + build(): Promise<{ + incrementalAdminRouter: Router; + manager: IncrementalIngestionDatabaseManager; + }>; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + static create( + env: PluginEnvironment, + builder: CatalogBuilder, + ): Promise; +} + +// Warning: (ae-missing-release-tag) "IncrementalEntityProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export interface IncrementalEntityProvider { + around(burst: (context: TContext) => Promise): Promise; + getProviderName(): string; + next( + context: TContext, + cursor?: TCursor, + ): Promise>; +} + +// Warning: (ae-missing-release-tag) "IncrementalEntityProviderOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export interface IncrementalEntityProviderOptions { + // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag + // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag + // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag + // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag + // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" + // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" + // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" + // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" + backoff?: DurationObjectUnits[]; + burstInterval: DurationObjectUnits; + burstLength: DurationObjectUnits; + restLength: DurationObjectUnits; +} + +// Warning: (ae-missing-release-tag) "IncrementalIngestionDatabaseManager" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class IncrementalIngestionDatabaseManager { + constructor(options: { client: Knex }); + cleanupProviders(): Promise<{ + ingestionsDeleted: void; + ingestionMarksDeleted: void; + markEntitiesDeleted: void; + }>; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + clearDuplicateIngestions( + ingestionId: string, + provider: string, + ): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + clearFinishedIngestions(provider: string): Promise<{ + deletions: { + markEntitiesDeleted: number; + marksDeleted: number; + ingestionsDeleted: number; + }; + }>; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + computeRemoved( + provider: string, + ingestionId: string, + ): Promise< + { + entity: any; + }[] + >; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + createMark(options: MarkRecordInsert): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + createMarkEntities(markId: string, entities: DeferredEntity[]): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + createProviderIngestionRecord(provider: string): Promise<{ + ingestionId: string; + nextAction: string; + attempts: number; + nextActionAt: number; + }>; + // (undocumented) + getAllMarks(ingestionId: string): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + getCurrentIngestionRecord( + provider: string, + ): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + getLastMark(ingestionId: string): Promise; + healthcheck(): Promise< + Pick< + { + id: string; + provider_name: string; + }, + 'id' | 'provider_name' + >[] + >; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + insertIngestionRecord(options: IngestionRecordInsert): Promise; + // (undocumented) + insertRecord(options: IngestionRecordInsert): Promise; + listProviders(): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + purgeAndResetProvider(provider: string): Promise<{ + provider: string; + ingestionsDeleted: number; + marksDeleted: number; + markEntitiesDeleted: number; + }>; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + purgeTable(table: string): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + setProviderBackoff( + ingestionId: string, + attempts: number, + error: Error, + backoffLength: number, + ): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + setProviderBursting(ingestionId: string): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + setProviderCanceled(ingestionId: string): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + setProviderCanceling(ingestionId: string, message?: string): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + setProviderComplete(ingestionId: string): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + setProviderIngesting(ingestionId: string): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + setProviderInterstitial(ingestionId: string): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + setProviderResting(ingestionId: string, restLength: Duration): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + triggerNextProviderAction(provider: string): Promise; + // (undocumented) + updateByName( + provider: string, + update: Partial, + ): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + updateIngestionRecordById(options: IngestionRecordUpdate): Promise; + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen + updateIngestionRecordByProvider( + provider: string, + update: Partial, + ): Promise; +} + +// Warning: (ae-missing-release-tag) "IngestionRecord" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export interface IngestionRecord { + // (undocumented) + attempts: number; + // (undocumented) + created_at: string; + // (undocumented) + id: string; + // (undocumented) + ingestion_completed_at: string | null; + // (undocumented) + last_error: string | null; + // (undocumented) + next_action: string; + // (undocumented) + next_action_at: Date; + // (undocumented) + provider_name: string; + // (undocumented) + rest_completed_at: string | null; + // (undocumented) + status: string; +} + +// Warning: (ae-missing-release-tag) "IngestionRecordInsert" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export interface IngestionRecordInsert { + // (undocumented) + record: IngestionUpsertIFace & { + id: string; + }; +} + +// Warning: (ae-missing-release-tag) "IngestionRecordUpdate" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export interface IngestionRecordUpdate { + // (undocumented) + ingestionId: string; + // (undocumented) + update: Partial; +} + +// Warning: (ae-missing-release-tag) "IngestionUpsertIFace" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export interface IngestionUpsertIFace { + // (undocumented) + attempts?: number; + // (undocumented) + ingestion_completed_at?: Date; + // (undocumented) + last_error?: string; + // (undocumented) + next_action: + | 'rest' + | 'ingest' + | 'backoff' + | 'cancel' + | 'nothing (done)' + | 'nothing (canceled)'; + // (undocumented) + next_action_at?: Date; + // (undocumented) + provider_name: string; + // (undocumented) + rest_completed_at?: Date; + // (undocumented) + status: + | 'complete' + | 'bursting' + | 'resting' + | 'canceling' + | 'interstitial' + | 'backing off'; +} + +// Warning: (ae-missing-release-tag) "IterationEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export interface IterationEngine { + // (undocumented) + taskFn: TaskFunction; +} + +// Warning: (ae-missing-release-tag) "IterationEngineOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export interface IterationEngineOptions { + // (undocumented) + backoff?: IncrementalEntityProviderOptions['backoff']; + // (undocumented) + connection: EntityProviderConnection; + // (undocumented) + logger: Logger; + // (undocumented) + manager: IncrementalIngestionDatabaseManager; + // (undocumented) + provider: IncrementalEntityProvider; + // (undocumented) + ready: Promise; + // (undocumented) + restLength: DurationObjectUnits; +} + +// Warning: (ae-missing-release-tag) "MarkRecord" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export interface MarkRecord { + // (undocumented) + created_at: string; + // (undocumented) + cursor: string; + // (undocumented) + id: string; + // (undocumented) + ingestion_id: string; + // (undocumented) + sequence: number; +} + +// Warning: (ae-missing-release-tag) "MarkRecordInsert" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export interface MarkRecordInsert { + // (undocumented) + record: { + id: string; + ingestion_id: string; + cursor: unknown; + sequence: number; + }; +} + +// Warning: (ae-missing-release-tag) "PluginEnvironment" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export type PluginEnvironment = { + logger: Logger; + database: PluginDatabaseManager; + scheduler: PluginTaskScheduler; + config: Config; + reader: UrlReader; + permissions: PermissionAuthorizer; +}; + +// (No @packageDocumentation comment for this package) +``` From 2133fd2926594bec0523bc90152947e56f1dffab Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 28 Oct 2022 10:09:02 -0700 Subject: [PATCH 21/54] Apply mandatory ts-doc annotations for API report Signed-off-by: Damon Kaswell --- .../IncrementalIngestionDatabaseManager.ts | 63 ++++++++++--------- .../src/database/migrations.ts | 1 + .../src/engine/IncrementalIngestionEngine.ts | 1 + .../src/routes.ts | 1 + .../src/service/IncrementalCatalogBuilder.ts | 7 ++- .../src/types.ts | 31 ++++++++- 6 files changed, 68 insertions(+), 36 deletions(-) diff --git a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts index d593bf4292..d7dc7dc1de 100644 --- a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts +++ b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts @@ -28,6 +28,7 @@ import { MarkRecordInsert, } from '../types'; +/** @public */ export class IncrementalIngestionDatabaseManager { private client: Knex; @@ -37,7 +38,7 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an update to the ingestion record with matching `id`. - * @param options IngestionRecordUpdate + * @param options - IngestionRecordUpdate */ async updateIngestionRecordById(options: IngestionRecordUpdate) { await this.client.transaction(async tx => { @@ -48,8 +49,8 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an update to the ingestion record with matching provider name. Will only update active records. - * @param provider string - * @param update Partial + * @param provider - string + * @param update - Partial */ async updateIngestionRecordByProvider( provider: string, @@ -65,7 +66,7 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an insert into the `ingestion.ingestions` table with the supplied values. - * @param options IngestionRecordInsert + * @param options - IngestionRecordInsert */ async insertIngestionRecord(options: IngestionRecordInsert) { await this.client.transaction(async tx => { @@ -101,7 +102,7 @@ export class IncrementalIngestionDatabaseManager { /** * Finds the current ingestion record for the named provider. - * @param provider string + * @param provider - string * @returns IngestionRecord | undefined */ async getCurrentIngestionRecord(provider: string) { @@ -117,7 +118,7 @@ export class IncrementalIngestionDatabaseManager { /** * Removes all entries from `ingestion_marks_entities`, `ingestion_marks`, and `ingestions` * for prior ingestions that completed (i.e., have a value for `rest_completed_at`). - * @param provider string + * @param provider - string * @returns A count of deletions for each record type. */ async clearFinishedIngestions(provider: string) { @@ -166,8 +167,8 @@ export class IncrementalIngestionDatabaseManager { * Automatically cleans up duplicate ingestion records if they were accidentally created. * Any ingestion record where the `rest_completed_at` is null (meaning it is active) AND * the ingestionId is incorrect is a duplicate ingestion record. - * @param ingestionId string - * @param provider string + * @param ingestionId - string + * @param provider - string */ async clearDuplicateIngestions(ingestionId: string, provider: string) { await this.client.transaction(async tx => { @@ -196,7 +197,7 @@ export class IncrementalIngestionDatabaseManager { /** * This method fully purges and resets all ingestion records for the named provider, and * leaves it in a paused state. - * @param provider string + * @param provider - string * @returns Counts of all deleted ingestion records */ async purgeAndResetProvider(provider: string) { @@ -264,7 +265,7 @@ export class IncrementalIngestionDatabaseManager { /** * Creates a new ingestion record. - * @param provider string + * @param provider - string * @returns A new ingestion record */ async createProviderIngestionRecord(provider: string) { @@ -283,8 +284,8 @@ export class IncrementalIngestionDatabaseManager { /** * Computes which entities to remove, if any, at the end of a burst. - * @param provider string - * @param ingestionId string + * @param provider - string + * @param ingestionId - string * @returns All entities to remove for this burst. */ async computeRemoved(provider: string, ingestionId: string) { @@ -339,7 +340,7 @@ export class IncrementalIngestionDatabaseManager { /** * Skips any wait time for the next action to run. - * @param provider string + * @param provider - string */ async triggerNextProviderAction(provider: string) { await this.updateIngestionRecordByProvider(provider, { @@ -389,7 +390,7 @@ export class IncrementalIngestionDatabaseManager { /** * Configures the current ingestion record to ingest a burst. - * @param ingestionId string + * @param ingestionId - string */ async setProviderIngesting(ingestionId: string) { await this.updateIngestionRecordById({ @@ -400,7 +401,7 @@ export class IncrementalIngestionDatabaseManager { /** * Indicates the provider is currently ingesting a burst. - * @param ingestionId string + * @param ingestionId - string */ async setProviderBursting(ingestionId: string) { await this.updateIngestionRecordById({ @@ -411,7 +412,7 @@ export class IncrementalIngestionDatabaseManager { /** * Finalizes the current ingestion record to indicate that the post-ingestion rest period is complete. - * @param ingestionId string + * @param ingestionId - string */ async setProviderComplete(ingestionId: string) { await this.updateIngestionRecordById({ @@ -426,8 +427,8 @@ export class IncrementalIngestionDatabaseManager { /** * Marks ingestion as complete and starts the post-ingestion rest cycle. - * @param ingestionId string - * @param restLength Duration + * @param ingestionId - string + * @param restLength - Duration */ async setProviderResting(ingestionId: string, restLength: Duration) { await this.updateIngestionRecordById({ @@ -443,7 +444,7 @@ export class IncrementalIngestionDatabaseManager { /** * Marks ingestion as paused after a burst completes. - * @param ingestionId string + * @param ingestionId - string */ async setProviderInterstitial(ingestionId: string) { await this.updateIngestionRecordById({ @@ -454,8 +455,8 @@ export class IncrementalIngestionDatabaseManager { /** * Starts the cancel process for the current ingestion. - * @param ingestionId string - * @param message string (optional) + * @param ingestionId - string + * @param message - string (optional) */ async setProviderCanceling(ingestionId: string, message?: string) { const update: Partial = { @@ -469,7 +470,7 @@ export class IncrementalIngestionDatabaseManager { /** * Completes the cancel process and triggers a new ingestion. - * @param ingestionId string + * @param ingestionId - string */ async setProviderCanceled(ingestionId: string) { await this.updateIngestionRecordById({ @@ -484,10 +485,10 @@ export class IncrementalIngestionDatabaseManager { /** * Configures the current ingestion to wait and retry, due to a data source error. - * @param ingestionId string - * @param attempts number - * @param error Error - * @param backoffLength number + * @param ingestionId - string + * @param attempts - number + * @param error - Error + * @param backoffLength - number */ async setProviderBackoff( ingestionId: string, @@ -509,7 +510,7 @@ export class IncrementalIngestionDatabaseManager { /** * Returns the last record from `ingestion.ingestion_marks` for the supplied ingestionId. - * @param ingestionId string + * @param ingestionId - string * @returns MarkRecord | undefined */ async getLastMark(ingestionId: string) { @@ -533,7 +534,7 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an insert into the `ingestion.ingestion_marks` table with the supplied values. - * @param options MarkRecordInsert + * @param options - MarkRecordInsert */ async createMark(options: MarkRecordInsert) { const { record } = options; @@ -543,8 +544,8 @@ export class IncrementalIngestionDatabaseManager { } /** * Performs an upsert to the `ingestion.ingestion_mark_entities` table for all deferred entities. - * @param markId string - * @param entities DeferredEntity[] + * @param markId - string + * @param entities - DeferredEntity[] */ async createMarkEntities(markId: string, entities: DeferredEntity[]) { await this.client.transaction(async tx => { @@ -560,7 +561,7 @@ export class IncrementalIngestionDatabaseManager { /** * Deletes the entire content of a table, and returns the number of records deleted. - * @param table string + * @param table - string * @returns number */ async purgeTable(table: string) { diff --git a/plugins/incremental-ingestion-backend/src/database/migrations.ts b/plugins/incremental-ingestion-backend/src/database/migrations.ts index 991a709732..270f07b0d7 100644 --- a/plugins/incremental-ingestion-backend/src/database/migrations.ts +++ b/plugins/incremental-ingestion-backend/src/database/migrations.ts @@ -16,6 +16,7 @@ import { resolvePackagePath } from '@backstage/backend-common'; import { Knex } from 'knex'; +/** @public */ export async function applyDatabaseMigrations(knex: Knex): Promise { const migrationsDir = resolvePackagePath( '@devex/backend-incremental-ingestion', diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts index 655380cec4..574d678e90 100644 --- a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts @@ -26,6 +26,7 @@ import { performance } from 'perf_hooks'; import { Duration, DurationObjectUnits } from 'luxon'; import { v4 } from 'uuid'; +/** @public */ export class IncrementalIngestionEngine implements IterationEngine { restLength: Duration; backoff: DurationObjectUnits[]; diff --git a/plugins/incremental-ingestion-backend/src/routes.ts b/plugins/incremental-ingestion-backend/src/routes.ts index a35f02e98e..81ebc1d7b6 100644 --- a/plugins/incremental-ingestion-backend/src/routes.ts +++ b/plugins/incremental-ingestion-backend/src/routes.ts @@ -19,6 +19,7 @@ import Router from 'express-promise-router'; import { Logger } from 'winston'; import { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager'; +/** @public */ export const createIncrementalProviderRouter = async ( manager: IncrementalIngestionDatabaseManager, logger: Logger, diff --git a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts index a9babf17e4..e6e1e05091 100644 --- a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts @@ -26,7 +26,7 @@ import { applyDatabaseMigrations } from '../database/migrations'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; import { createIncrementalProviderRouter } from '../routes'; -export class Deferred implements Promise { +class Deferred implements Promise { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore resolve: (value: T) => void; @@ -52,11 +52,12 @@ export class Deferred implements Promise { [Symbol.toStringTag]: 'Deferred' = 'Deferred'; } +/** @public */ export class IncrementalCatalogBuilder { /** * Creates the incremental catalog builder, which extends the regular catalog builder. - * @param env PluginEnvironment - * @param builder CatalogBuilder + * @param env - PluginEnvironment + * @param builder - CatalogBuilder * @returns IncrementalCatalogBuilder */ static async create(env: PluginEnvironment, builder: CoreCatalogBuilder) { diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/incremental-ingestion-backend/src/types.ts index 85f184b52e..12c7249aa2 100644 --- a/plugins/incremental-ingestion-backend/src/types.ts +++ b/plugins/incremental-ingestion-backend/src/types.ts @@ -51,6 +51,7 @@ export const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = * batches of entities in sequence so that you never need to have more * than a few hundred in memory at a time. * + * @public */ export interface IncrementalEntityProvider { /** @@ -84,13 +85,17 @@ export interface IncrementalEntityProvider { } /** - * Value returned by an @{link IncrementalEntityProvider} to provide a + * Value returned by an {@link IncrementalEntityProvider} to provide a * single page of entities to ingest. + * + * @public */ export interface EntityIteratorResult { /** * Indicates whether there are any further pages of entities to * ingest after this one. + * + * @public */ done: boolean; @@ -106,6 +111,7 @@ export interface EntityIteratorResult { entities: DeferredEntity[]; } +/** @public */ export interface IncrementalEntityProviderOptions { /** * Entities are ingested in bursts. This interval determines how @@ -129,11 +135,16 @@ export interface IncrementalEntityProviderOptions { /** * In the event of an error during an ingestion burst, the backoff * determines how soon it will be retried. E.g. - * [{ minutes: 1}, { minutes: 5}, {minutes: 30 }, { hours: 3 }] + * `[{ minutes: 1}, { minutes: 5}, {minutes: 30 }, { hours: 3 }]` */ backoff?: DurationObjectUnits[]; } +/** + * Provides the environment used by the incremental entity provider, + * + * @public + */ export type PluginEnvironment = { logger: Logger; database: PluginDatabaseManager; @@ -145,6 +156,8 @@ export type PluginEnvironment = { /** * The core ingestion engine implements this interface + * + * @public */ export interface IterationEngine { taskFn: TaskFunction; @@ -152,6 +165,8 @@ export interface IterationEngine { /** * Options passed to the core ingestion engine during initialization. + * + * @public */ export interface IterationEngineOptions { logger: Logger; @@ -165,6 +180,8 @@ export interface IterationEngineOptions { /** * The shape of data inserted into or updated in the `ingestion.ingestions` table. + * + * @public */ export interface IngestionUpsertIFace { next_action: @@ -191,6 +208,8 @@ export interface IngestionUpsertIFace { /** * This interface supplies all potential values that can be inserted into the `ingestion.ingestions` table. + * + * @public */ export interface IngestionRecordInsert { record: IngestionUpsertIFace & { @@ -200,6 +219,8 @@ export interface IngestionRecordInsert { /** * This interface is for updating an existing ingestion record. + * + * @public */ export interface IngestionRecordUpdate { ingestionId: string; @@ -208,6 +229,8 @@ export interface IngestionRecordUpdate { /** * The expected response from the `ingestion.ingestion_marks` table. + * + * @public */ export interface MarkRecord { id: string; @@ -219,6 +242,8 @@ export interface MarkRecord { /** * The expected response from the `ingestion.ingestions` table. + * + * @public */ export interface IngestionRecord { id: string; @@ -235,6 +260,8 @@ export interface IngestionRecord { /** * This interface supplies all the values for adding an ingestion mark. + * + * @public */ export interface MarkRecordInsert { record: { From 6932d232225a0cff777cdbfa9d8768df43ea6df6 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 28 Oct 2022 10:09:26 -0700 Subject: [PATCH 22/54] Update api-report.md with results from corrections Signed-off-by: Damon Kaswell --- .../api-report.md | 93 +------------------ 1 file changed, 1 insertion(+), 92 deletions(-) diff --git a/plugins/incremental-ingestion-backend/api-report.md b/plugins/incremental-ingestion-backend/api-report.md index 5443d9522a..438d0add76 100644 --- a/plugins/incremental-ingestion-backend/api-report.md +++ b/plugins/incremental-ingestion-backend/api-report.md @@ -18,30 +18,6 @@ import { Router } from 'express'; import type { TaskFunction } from '@backstage/backend-tasks'; import type { UrlReader } from '@backstage/backend-common'; -// Warning: (ae-missing-release-tag) "Deferred" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export class Deferred implements Promise { - // (undocumented) - [Symbol.toStringTag]: 'Deferred'; - constructor(); - // (undocumented) - catch: Promise['catch']; - // (undocumented) - finally: Promise['finally']; - // (undocumented) - reject: (error: Error) => void; - // (undocumented) - resolve: (value: T) => void; - // (undocumented) - then: Promise['then']; -} - -// Warning: (tsdoc-at-sign-without-tag-name) Expecting a TSDoc tag name after "@"; if it is not a tag, use a backslash to escape this character -// Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag -// Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" -// Warning: (ae-missing-release-tag) "EntityIteratorResult" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public export interface EntityIteratorResult { cursor: T; @@ -53,8 +29,6 @@ export interface EntityIteratorResult { export const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = 'backstage.io/incremental-provider-name'; -// Warning: (ae-missing-release-tag) "IncrementalCatalogBuilder" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public (undocumented) export class IncrementalCatalogBuilder { // (undocumented) @@ -67,16 +41,12 @@ export class IncrementalCatalogBuilder { incrementalAdminRouter: Router; manager: IncrementalIngestionDatabaseManager; }>; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen static create( env: PluginEnvironment, builder: CatalogBuilder, ): Promise; } -// Warning: (ae-missing-release-tag) "IncrementalEntityProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public export interface IncrementalEntityProvider { around(burst: (context: TContext) => Promise): Promise; @@ -87,26 +57,14 @@ export interface IncrementalEntityProvider { ): Promise>; } -// Warning: (ae-missing-release-tag) "IncrementalEntityProviderOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public (undocumented) export interface IncrementalEntityProviderOptions { - // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag - // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag - // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag - // Warning: (tsdoc-escape-right-brace) The "}" character should be escaped using a backslash to avoid confusion with a TSDoc inline tag - // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" - // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" - // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" - // Warning: (tsdoc-malformed-inline-tag) Expecting a TSDoc tag starting with "{@" backoff?: DurationObjectUnits[]; burstInterval: DurationObjectUnits; burstLength: DurationObjectUnits; restLength: DurationObjectUnits; } -// Warning: (ae-missing-release-tag) "IncrementalIngestionDatabaseManager" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public (undocumented) export class IncrementalIngestionDatabaseManager { constructor(options: { client: Knex }); @@ -115,13 +73,10 @@ export class IncrementalIngestionDatabaseManager { ingestionMarksDeleted: void; markEntitiesDeleted: void; }>; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen clearDuplicateIngestions( ingestionId: string, provider: string, ): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen clearFinishedIngestions(provider: string): Promise<{ deletions: { markEntitiesDeleted: number; @@ -129,8 +84,6 @@ export class IncrementalIngestionDatabaseManager { ingestionsDeleted: number; }; }>; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen computeRemoved( provider: string, ingestionId: string, @@ -139,12 +92,8 @@ export class IncrementalIngestionDatabaseManager { entity: any; }[] >; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen createMark(options: MarkRecordInsert): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen createMarkEntities(markId: string, entities: DeferredEntity[]): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen createProviderIngestionRecord(provider: string): Promise<{ ingestionId: string; nextAction: string; @@ -153,11 +102,9 @@ export class IncrementalIngestionDatabaseManager { }>; // (undocumented) getAllMarks(ingestionId: string): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen getCurrentIngestionRecord( provider: string, ): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen getLastMark(ingestionId: string): Promise; healthcheck(): Promise< Pick< @@ -168,65 +115,43 @@ export class IncrementalIngestionDatabaseManager { 'id' | 'provider_name' >[] >; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen insertIngestionRecord(options: IngestionRecordInsert): Promise; // (undocumented) insertRecord(options: IngestionRecordInsert): Promise; listProviders(): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen purgeAndResetProvider(provider: string): Promise<{ provider: string; ingestionsDeleted: number; marksDeleted: number; markEntitiesDeleted: number; }>; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen purgeTable(table: string): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen setProviderBackoff( ingestionId: string, attempts: number, error: Error, backoffLength: number, ): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen setProviderBursting(ingestionId: string): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen setProviderCanceled(ingestionId: string): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen setProviderCanceling(ingestionId: string, message?: string): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen setProviderComplete(ingestionId: string): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen setProviderIngesting(ingestionId: string): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen setProviderInterstitial(ingestionId: string): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen setProviderResting(ingestionId: string, restLength: Duration): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen triggerNextProviderAction(provider: string): Promise; // (undocumented) updateByName( provider: string, update: Partial, ): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen updateIngestionRecordById(options: IngestionRecordUpdate): Promise; - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen - // Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen updateIngestionRecordByProvider( provider: string, update: Partial, ): Promise; } -// Warning: (ae-missing-release-tag) "IngestionRecord" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public export interface IngestionRecord { // (undocumented) @@ -251,8 +176,6 @@ export interface IngestionRecord { status: string; } -// Warning: (ae-missing-release-tag) "IngestionRecordInsert" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public export interface IngestionRecordInsert { // (undocumented) @@ -261,8 +184,6 @@ export interface IngestionRecordInsert { }; } -// Warning: (ae-missing-release-tag) "IngestionRecordUpdate" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public export interface IngestionRecordUpdate { // (undocumented) @@ -271,8 +192,6 @@ export interface IngestionRecordUpdate { update: Partial; } -// Warning: (ae-missing-release-tag) "IngestionUpsertIFace" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public export interface IngestionUpsertIFace { // (undocumented) @@ -305,16 +224,12 @@ export interface IngestionUpsertIFace { | 'backing off'; } -// Warning: (ae-missing-release-tag) "IterationEngine" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public export interface IterationEngine { // (undocumented) taskFn: TaskFunction; } -// Warning: (ae-missing-release-tag) "IterationEngineOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public export interface IterationEngineOptions { // (undocumented) @@ -333,8 +248,6 @@ export interface IterationEngineOptions { restLength: DurationObjectUnits; } -// Warning: (ae-missing-release-tag) "MarkRecord" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public export interface MarkRecord { // (undocumented) @@ -349,8 +262,6 @@ export interface MarkRecord { sequence: number; } -// Warning: (ae-missing-release-tag) "MarkRecordInsert" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// // @public export interface MarkRecordInsert { // (undocumented) @@ -362,9 +273,7 @@ export interface MarkRecordInsert { }; } -// Warning: (ae-missing-release-tag) "PluginEnvironment" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export type PluginEnvironment = { logger: Logger; database: PluginDatabaseManager; From 1c225f8f45a406d4fb756ee694c173139bc5f313 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 28 Oct 2022 10:25:37 -0700 Subject: [PATCH 23/54] Re-run prettier Signed-off-by: Damon Kaswell --- .../src/types.ts | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/incremental-ingestion-backend/src/types.ts index 12c7249aa2..c9f3f50e1d 100644 --- a/plugins/incremental-ingestion-backend/src/types.ts +++ b/plugins/incremental-ingestion-backend/src/types.ts @@ -87,15 +87,14 @@ export interface IncrementalEntityProvider { /** * Value returned by an {@link IncrementalEntityProvider} to provide a * single page of entities to ingest. - * + * * @public */ export interface EntityIteratorResult { /** * Indicates whether there are any further pages of entities to * ingest after this one. - * - * @public + * */ done: boolean; @@ -142,7 +141,7 @@ export interface IncrementalEntityProviderOptions { /** * Provides the environment used by the incremental entity provider, - * + * * @public */ export type PluginEnvironment = { @@ -156,7 +155,7 @@ export type PluginEnvironment = { /** * The core ingestion engine implements this interface - * + * * @public */ export interface IterationEngine { @@ -165,7 +164,7 @@ export interface IterationEngine { /** * Options passed to the core ingestion engine during initialization. - * + * * @public */ export interface IterationEngineOptions { @@ -180,7 +179,7 @@ export interface IterationEngineOptions { /** * The shape of data inserted into or updated in the `ingestion.ingestions` table. - * + * * @public */ export interface IngestionUpsertIFace { @@ -208,7 +207,7 @@ export interface IngestionUpsertIFace { /** * This interface supplies all potential values that can be inserted into the `ingestion.ingestions` table. - * + * * @public */ export interface IngestionRecordInsert { @@ -219,7 +218,7 @@ export interface IngestionRecordInsert { /** * This interface is for updating an existing ingestion record. - * + * * @public */ export interface IngestionRecordUpdate { @@ -229,7 +228,7 @@ export interface IngestionRecordUpdate { /** * The expected response from the `ingestion.ingestion_marks` table. - * + * * @public */ export interface MarkRecord { @@ -242,7 +241,7 @@ export interface MarkRecord { /** * The expected response from the `ingestion.ingestions` table. - * + * * @public */ export interface IngestionRecord { @@ -260,7 +259,7 @@ export interface IngestionRecord { /** * This interface supplies all the values for adding an ingestion mark. - * + * * @public */ export interface MarkRecordInsert { From b089bc1b0b37e0b8845a009a0d15773cc569a31c Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 28 Oct 2022 11:24:09 -0700 Subject: [PATCH 24/54] Remove manually created api report Signed-off-by: Damon Kaswell --- .../api-report.md | 287 ------------------ 1 file changed, 287 deletions(-) delete mode 100644 plugins/incremental-ingestion-backend/api-report.md diff --git a/plugins/incremental-ingestion-backend/api-report.md b/plugins/incremental-ingestion-backend/api-report.md deleted file mode 100644 index 438d0add76..0000000000 --- a/plugins/incremental-ingestion-backend/api-report.md +++ /dev/null @@ -1,287 +0,0 @@ -## API Report File for "@backstage/plugin-incremental-ingestion-backend" - -> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). - -```ts -import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; -import type { Config } from '@backstage/config'; -import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; -import { Duration } from 'luxon'; -import type { DurationObjectUnits } from 'luxon'; -import type { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; -import { Knex } from 'knex'; -import type { Logger } from 'winston'; -import type { PermissionAuthorizer } from '@backstage/plugin-permission-common'; -import type { PluginDatabaseManager } from '@backstage/backend-common'; -import type { PluginTaskScheduler } from '@backstage/backend-tasks'; -import { Router } from 'express'; -import type { TaskFunction } from '@backstage/backend-tasks'; -import type { UrlReader } from '@backstage/backend-common'; - -// @public -export interface EntityIteratorResult { - cursor: T; - done: boolean; - entities: DeferredEntity[]; -} - -// @public -export const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = - 'backstage.io/incremental-provider-name'; - -// @public (undocumented) -export class IncrementalCatalogBuilder { - // (undocumented) - addIncrementalEntityProvider( - provider: IncrementalEntityProvider, - options: IncrementalEntityProviderOptions, - ): void; - // (undocumented) - build(): Promise<{ - incrementalAdminRouter: Router; - manager: IncrementalIngestionDatabaseManager; - }>; - static create( - env: PluginEnvironment, - builder: CatalogBuilder, - ): Promise; -} - -// @public -export interface IncrementalEntityProvider { - around(burst: (context: TContext) => Promise): Promise; - getProviderName(): string; - next( - context: TContext, - cursor?: TCursor, - ): Promise>; -} - -// @public (undocumented) -export interface IncrementalEntityProviderOptions { - backoff?: DurationObjectUnits[]; - burstInterval: DurationObjectUnits; - burstLength: DurationObjectUnits; - restLength: DurationObjectUnits; -} - -// @public (undocumented) -export class IncrementalIngestionDatabaseManager { - constructor(options: { client: Knex }); - cleanupProviders(): Promise<{ - ingestionsDeleted: void; - ingestionMarksDeleted: void; - markEntitiesDeleted: void; - }>; - clearDuplicateIngestions( - ingestionId: string, - provider: string, - ): Promise; - clearFinishedIngestions(provider: string): Promise<{ - deletions: { - markEntitiesDeleted: number; - marksDeleted: number; - ingestionsDeleted: number; - }; - }>; - computeRemoved( - provider: string, - ingestionId: string, - ): Promise< - { - entity: any; - }[] - >; - createMark(options: MarkRecordInsert): Promise; - createMarkEntities(markId: string, entities: DeferredEntity[]): Promise; - createProviderIngestionRecord(provider: string): Promise<{ - ingestionId: string; - nextAction: string; - attempts: number; - nextActionAt: number; - }>; - // (undocumented) - getAllMarks(ingestionId: string): Promise; - getCurrentIngestionRecord( - provider: string, - ): Promise; - getLastMark(ingestionId: string): Promise; - healthcheck(): Promise< - Pick< - { - id: string; - provider_name: string; - }, - 'id' | 'provider_name' - >[] - >; - insertIngestionRecord(options: IngestionRecordInsert): Promise; - // (undocumented) - insertRecord(options: IngestionRecordInsert): Promise; - listProviders(): Promise; - purgeAndResetProvider(provider: string): Promise<{ - provider: string; - ingestionsDeleted: number; - marksDeleted: number; - markEntitiesDeleted: number; - }>; - purgeTable(table: string): Promise; - setProviderBackoff( - ingestionId: string, - attempts: number, - error: Error, - backoffLength: number, - ): Promise; - setProviderBursting(ingestionId: string): Promise; - setProviderCanceled(ingestionId: string): Promise; - setProviderCanceling(ingestionId: string, message?: string): Promise; - setProviderComplete(ingestionId: string): Promise; - setProviderIngesting(ingestionId: string): Promise; - setProviderInterstitial(ingestionId: string): Promise; - setProviderResting(ingestionId: string, restLength: Duration): Promise; - triggerNextProviderAction(provider: string): Promise; - // (undocumented) - updateByName( - provider: string, - update: Partial, - ): Promise; - updateIngestionRecordById(options: IngestionRecordUpdate): Promise; - updateIngestionRecordByProvider( - provider: string, - update: Partial, - ): Promise; -} - -// @public -export interface IngestionRecord { - // (undocumented) - attempts: number; - // (undocumented) - created_at: string; - // (undocumented) - id: string; - // (undocumented) - ingestion_completed_at: string | null; - // (undocumented) - last_error: string | null; - // (undocumented) - next_action: string; - // (undocumented) - next_action_at: Date; - // (undocumented) - provider_name: string; - // (undocumented) - rest_completed_at: string | null; - // (undocumented) - status: string; -} - -// @public -export interface IngestionRecordInsert { - // (undocumented) - record: IngestionUpsertIFace & { - id: string; - }; -} - -// @public -export interface IngestionRecordUpdate { - // (undocumented) - ingestionId: string; - // (undocumented) - update: Partial; -} - -// @public -export interface IngestionUpsertIFace { - // (undocumented) - attempts?: number; - // (undocumented) - ingestion_completed_at?: Date; - // (undocumented) - last_error?: string; - // (undocumented) - next_action: - | 'rest' - | 'ingest' - | 'backoff' - | 'cancel' - | 'nothing (done)' - | 'nothing (canceled)'; - // (undocumented) - next_action_at?: Date; - // (undocumented) - provider_name: string; - // (undocumented) - rest_completed_at?: Date; - // (undocumented) - status: - | 'complete' - | 'bursting' - | 'resting' - | 'canceling' - | 'interstitial' - | 'backing off'; -} - -// @public -export interface IterationEngine { - // (undocumented) - taskFn: TaskFunction; -} - -// @public -export interface IterationEngineOptions { - // (undocumented) - backoff?: IncrementalEntityProviderOptions['backoff']; - // (undocumented) - connection: EntityProviderConnection; - // (undocumented) - logger: Logger; - // (undocumented) - manager: IncrementalIngestionDatabaseManager; - // (undocumented) - provider: IncrementalEntityProvider; - // (undocumented) - ready: Promise; - // (undocumented) - restLength: DurationObjectUnits; -} - -// @public -export interface MarkRecord { - // (undocumented) - created_at: string; - // (undocumented) - cursor: string; - // (undocumented) - id: string; - // (undocumented) - ingestion_id: string; - // (undocumented) - sequence: number; -} - -// @public -export interface MarkRecordInsert { - // (undocumented) - record: { - id: string; - ingestion_id: string; - cursor: unknown; - sequence: number; - }; -} - -// @public -export type PluginEnvironment = { - logger: Logger; - database: PluginDatabaseManager; - scheduler: PluginTaskScheduler; - config: Config; - reader: UrlReader; - permissions: PermissionAuthorizer; -}; - -// (No @packageDocumentation comment for this package) -``` From 4582105db852295a17d490999742026df11e8019 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 28 Oct 2022 11:55:30 -0700 Subject: [PATCH 25/54] Re-added api-report.md (corrected) Signed-off-by: Damon Kaswell --- .../api-report.md | 289 ++++++++++++++++++ 1 file changed, 289 insertions(+) create mode 100644 plugins/incremental-ingestion-backend/api-report.md diff --git a/plugins/incremental-ingestion-backend/api-report.md b/plugins/incremental-ingestion-backend/api-report.md new file mode 100644 index 0000000000..6d4d12cc6d --- /dev/null +++ b/plugins/incremental-ingestion-backend/api-report.md @@ -0,0 +1,289 @@ +## API Report File for "@backstage/plugin-incremental-ingestion-backend" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +/// + +import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; +import type { Config } from '@backstage/config'; +import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; +import { Duration } from 'luxon'; +import type { DurationObjectUnits } from 'luxon'; +import type { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; +import { Knex } from 'knex'; +import type { Logger } from 'winston'; +import type { PermissionAuthorizer } from '@backstage/plugin-permission-common'; +import type { PluginDatabaseManager } from '@backstage/backend-common'; +import type { PluginTaskScheduler } from '@backstage/backend-tasks'; +import { Router } from 'express'; +import type { TaskFunction } from '@backstage/backend-tasks'; +import type { UrlReader } from '@backstage/backend-common'; + +// @public +export interface EntityIteratorResult { + cursor: T; + done: boolean; + entities: DeferredEntity[]; +} + +// @public +export const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = + 'backstage.io/incremental-provider-name'; + +// @public (undocumented) +export class IncrementalCatalogBuilder { + // (undocumented) + addIncrementalEntityProvider( + provider: IncrementalEntityProvider, + options: IncrementalEntityProviderOptions, + ): void; + // (undocumented) + build(): Promise<{ + incrementalAdminRouter: Router; + manager: IncrementalIngestionDatabaseManager; + }>; + static create( + env: PluginEnvironment, + builder: CatalogBuilder, + ): Promise; +} + +// @public +export interface IncrementalEntityProvider { + around(burst: (context: TContext) => Promise): Promise; + getProviderName(): string; + next( + context: TContext, + cursor?: TCursor, + ): Promise>; +} + +// @public (undocumented) +export interface IncrementalEntityProviderOptions { + backoff?: DurationObjectUnits[]; + burstInterval: DurationObjectUnits; + burstLength: DurationObjectUnits; + restLength: DurationObjectUnits; +} + +// @public (undocumented) +export class IncrementalIngestionDatabaseManager { + constructor(options: { client: Knex }); + cleanupProviders(): Promise<{ + ingestionsDeleted: void; + ingestionMarksDeleted: void; + markEntitiesDeleted: void; + }>; + clearDuplicateIngestions( + ingestionId: string, + provider: string, + ): Promise; + clearFinishedIngestions(provider: string): Promise<{ + deletions: { + markEntitiesDeleted: number; + marksDeleted: number; + ingestionsDeleted: number; + }; + }>; + computeRemoved( + provider: string, + ingestionId: string, + ): Promise< + { + entity: any; + }[] + >; + createMark(options: MarkRecordInsert): Promise; + createMarkEntities(markId: string, entities: DeferredEntity[]): Promise; + createProviderIngestionRecord(provider: string): Promise<{ + ingestionId: string; + nextAction: string; + attempts: number; + nextActionAt: number; + }>; + // (undocumented) + getAllMarks(ingestionId: string): Promise; + getCurrentIngestionRecord( + provider: string, + ): Promise; + getLastMark(ingestionId: string): Promise; + healthcheck(): Promise< + Pick< + { + id: string; + provider_name: string; + }, + 'id' | 'provider_name' + >[] + >; + insertIngestionRecord(options: IngestionRecordInsert): Promise; + // (undocumented) + insertRecord(options: IngestionRecordInsert): Promise; + listProviders(): Promise; + purgeAndResetProvider(provider: string): Promise<{ + provider: string; + ingestionsDeleted: number; + marksDeleted: number; + markEntitiesDeleted: number; + }>; + purgeTable(table: string): Promise; + setProviderBackoff( + ingestionId: string, + attempts: number, + error: Error, + backoffLength: number, + ): Promise; + setProviderBursting(ingestionId: string): Promise; + setProviderCanceled(ingestionId: string): Promise; + setProviderCanceling(ingestionId: string, message?: string): Promise; + setProviderComplete(ingestionId: string): Promise; + setProviderIngesting(ingestionId: string): Promise; + setProviderInterstitial(ingestionId: string): Promise; + setProviderResting(ingestionId: string, restLength: Duration): Promise; + triggerNextProviderAction(provider: string): Promise; + // (undocumented) + updateByName( + provider: string, + update: Partial, + ): Promise; + updateIngestionRecordById(options: IngestionRecordUpdate): Promise; + updateIngestionRecordByProvider( + provider: string, + update: Partial, + ): Promise; +} + +// @public +export interface IngestionRecord { + // (undocumented) + attempts: number; + // (undocumented) + created_at: string; + // (undocumented) + id: string; + // (undocumented) + ingestion_completed_at: string | null; + // (undocumented) + last_error: string | null; + // (undocumented) + next_action: string; + // (undocumented) + next_action_at: Date; + // (undocumented) + provider_name: string; + // (undocumented) + rest_completed_at: string | null; + // (undocumented) + status: string; +} + +// @public +export interface IngestionRecordInsert { + // (undocumented) + record: IngestionUpsertIFace & { + id: string; + }; +} + +// @public +export interface IngestionRecordUpdate { + // (undocumented) + ingestionId: string; + // (undocumented) + update: Partial; +} + +// @public +export interface IngestionUpsertIFace { + // (undocumented) + attempts?: number; + // (undocumented) + ingestion_completed_at?: Date; + // (undocumented) + last_error?: string; + // (undocumented) + next_action: + | 'rest' + | 'ingest' + | 'backoff' + | 'cancel' + | 'nothing (done)' + | 'nothing (canceled)'; + // (undocumented) + next_action_at?: Date; + // (undocumented) + provider_name: string; + // (undocumented) + rest_completed_at?: Date; + // (undocumented) + status: + | 'complete' + | 'bursting' + | 'resting' + | 'canceling' + | 'interstitial' + | 'backing off'; +} + +// @public +export interface IterationEngine { + // (undocumented) + taskFn: TaskFunction; +} + +// @public +export interface IterationEngineOptions { + // (undocumented) + backoff?: IncrementalEntityProviderOptions['backoff']; + // (undocumented) + connection: EntityProviderConnection; + // (undocumented) + logger: Logger; + // (undocumented) + manager: IncrementalIngestionDatabaseManager; + // (undocumented) + provider: IncrementalEntityProvider; + // (undocumented) + ready: Promise; + // (undocumented) + restLength: DurationObjectUnits; +} + +// @public +export interface MarkRecord { + // (undocumented) + created_at: string; + // (undocumented) + cursor: string; + // (undocumented) + id: string; + // (undocumented) + ingestion_id: string; + // (undocumented) + sequence: number; +} + +// @public +export interface MarkRecordInsert { + // (undocumented) + record: { + id: string; + ingestion_id: string; + cursor: unknown; + sequence: number; + }; +} + +// @public +export type PluginEnvironment = { + logger: Logger; + database: PluginDatabaseManager; + scheduler: PluginTaskScheduler; + config: Config; + reader: UrlReader; + permissions: PermissionAuthorizer; +}; + +// (No @packageDocumentation comment for this package) +``` From aec2382f8083309f00ee0670b23c2be5639e86d3 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 28 Oct 2022 12:22:09 -0700 Subject: [PATCH 26/54] Correct type dependencies Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/incremental-ingestion-backend/package.json b/plugins/incremental-ingestion-backend/package.json index 0e02e46959..d1fa296474 100644 --- a/plugins/incremental-ingestion-backend/package.json +++ b/plugins/incremental-ingestion-backend/package.json @@ -29,6 +29,7 @@ "@backstage/config": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", + "@types/express": "^4.17.6", "express": "^4.17.1", "express-promise-router": "^4.1.0", "knex": "^2.0.0", @@ -38,7 +39,7 @@ }, "devDependencies": { "@backstage/cli": "workspace:^", - "@types/express": "^4.17.6" + "@types/luxon": "3.0.0" }, "files": [ "dist", From 9868f66466043677d80428076450ca74caf1f075 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 28 Oct 2022 12:29:24 -0700 Subject: [PATCH 27/54] Update yarn.lock with luxon types Signed-off-by: Damon Kaswell --- yarn.lock | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/yarn.lock b/yarn.lock index c3c513a481..4cc281f7dd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6035,6 +6035,7 @@ __metadata: "@backstage/plugin-catalog-backend": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" "@types/express": ^4.17.6 + "@types/luxon": 3.0.0 express: ^4.17.1 express-promise-router: ^4.1.0 knex: ^2.0.0 @@ -13804,6 +13805,13 @@ __metadata: languageName: node linkType: hard +"@types/luxon@npm:3.0.0": + version: 3.0.0 + resolution: "@types/luxon@npm:3.0.0" + checksum: 7738d3f4b91097a8139eca966ab53c6b40bb417b2cc2014c3bbd0aee033d6ff54af14c62046c2eb042434923b6bf796874f24b2af90e07ef422e6d841463d4b2 + languageName: node + linkType: hard + "@types/markdown-it@npm:^12.2.3": version: 12.2.3 resolution: "@types/markdown-it@npm:12.2.3" From 36629e916cda4376b915983437b937c151e95a43 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 28 Oct 2022 13:30:34 -0700 Subject: [PATCH 28/54] Moved luxon types to dependencies Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/incremental-ingestion-backend/package.json b/plugins/incremental-ingestion-backend/package.json index d1fa296474..6fdcdbb138 100644 --- a/plugins/incremental-ingestion-backend/package.json +++ b/plugins/incremental-ingestion-backend/package.json @@ -30,6 +30,7 @@ "@backstage/plugin-catalog-backend": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", "@types/express": "^4.17.6", + "@types/luxon": "3.0.0", "express": "^4.17.1", "express-promise-router": "^4.1.0", "knex": "^2.0.0", @@ -38,8 +39,7 @@ "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "workspace:^", - "@types/luxon": "3.0.0" + "@backstage/cli": "workspace:^" }, "files": [ "dist", From 90c1da927776c9769c133e9b92bed5ed7f22b4e4 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 3 Nov 2022 08:12:45 -0700 Subject: [PATCH 29/54] Fix migrations directory Signed-off-by: Damon Kaswell --- .../incremental-ingestion-backend/src/database/migrations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/incremental-ingestion-backend/src/database/migrations.ts b/plugins/incremental-ingestion-backend/src/database/migrations.ts index 270f07b0d7..4866e2dc98 100644 --- a/plugins/incremental-ingestion-backend/src/database/migrations.ts +++ b/plugins/incremental-ingestion-backend/src/database/migrations.ts @@ -19,7 +19,7 @@ import { Knex } from 'knex'; /** @public */ export async function applyDatabaseMigrations(knex: Knex): Promise { const migrationsDir = resolvePackagePath( - '@devex/backend-incremental-ingestion', + '@backstage/plugin-incremental-ingestion-backend', 'migrations', ); From cfe3612e085423ab16473fd57c67378722147765 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 3 Nov 2022 08:25:25 -0700 Subject: [PATCH 30/54] Fix divergent branch issue Signed-off-by: Damon Kaswell --- .../IncrementalIngestionDatabaseManager.ts | 131 +++++++++--------- .../src/types.ts | 106 +++++++------- 2 files changed, 114 insertions(+), 123 deletions(-) diff --git a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts index d7dc7dc1de..6a59757ca1 100644 --- a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts +++ b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { Knex } from 'knex'; import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; import { stringifyEntityRef } from '@backstage/catalog-model'; @@ -21,14 +22,12 @@ import { v4 } from 'uuid'; import { INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, IngestionRecord, - IngestionRecordInsert, IngestionRecordUpdate, IngestionUpsertIFace, MarkRecord, MarkRecordInsert, } from '../types'; -/** @public */ export class IncrementalIngestionDatabaseManager { private client: Knex; @@ -38,7 +37,7 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an update to the ingestion record with matching `id`. - * @param options - IngestionRecordUpdate + * @param options IngestionRecordUpdate */ async updateIngestionRecordById(options: IngestionRecordUpdate) { await this.client.transaction(async tx => { @@ -49,8 +48,8 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an update to the ingestion record with matching provider name. Will only update active records. - * @param provider - string - * @param update - Partial + * @param provider string + * @param update Partial */ async updateIngestionRecordByProvider( provider: string, @@ -59,18 +58,17 @@ export class IncrementalIngestionDatabaseManager { await this.client.transaction(async tx => { await tx('ingestion.ingestions') .where('provider_name', provider) - .andWhere('rest_completed_at', null) + .andWhere('completion_ticket', 'open') .update(update); }); } /** * Performs an insert into the `ingestion.ingestions` table with the supplied values. - * @param options - IngestionRecordInsert + * @param record IngestionUpsertIFace */ - async insertIngestionRecord(options: IngestionRecordInsert) { + async insertIngestionRecord(record: IngestionUpsertIFace) { await this.client.transaction(async tx => { - const { record } = options; await tx('ingestion.ingestions').insert(record); }); } @@ -102,14 +100,14 @@ export class IncrementalIngestionDatabaseManager { /** * Finds the current ingestion record for the named provider. - * @param provider - string + * @param provider string * @returns IngestionRecord | undefined */ async getCurrentIngestionRecord(provider: string) { return await this.client.transaction(async tx => { const record = await tx('ingestion.ingestions') .where('provider_name', provider) - .andWhere('rest_completed_at', null) + .andWhere('completion_ticket', 'open') .first(); return record; }); @@ -117,8 +115,8 @@ export class IncrementalIngestionDatabaseManager { /** * Removes all entries from `ingestion_marks_entities`, `ingestion_marks`, and `ingestions` - * for prior ingestions that completed (i.e., have a value for `rest_completed_at`). - * @param provider - string + * for prior ingestions that completed (i.e., have a `completion_ticket` value other than 'open'). + * @param provider string * @returns A count of deletions for each record type. */ async clearFinishedIngestions(provider: string) { @@ -134,7 +132,7 @@ export class IncrementalIngestionDatabaseManager { tx('ingestion.ingestions') .select('id') .where('provider_name', provider) - .andWhereNot('rest_completed_at', null), + .andWhereNot('completion_ticket', 'open'), ), ); @@ -145,13 +143,13 @@ export class IncrementalIngestionDatabaseManager { tx('ingestion.ingestions') .select('id') .where('provider_name', provider) - .andWhereNot('rest_completed_at', null), + .andWhereNot('completion_ticket', 'open'), ); const ingestionsDeleted = await tx('ingestion.ingestions') .delete() .where('provider_name', provider) - .andWhereNot('rest_completed_at', null); + .andWhereNot('completion_ticket', 'open'); return { deletions: { @@ -167,8 +165,8 @@ export class IncrementalIngestionDatabaseManager { * Automatically cleans up duplicate ingestion records if they were accidentally created. * Any ingestion record where the `rest_completed_at` is null (meaning it is active) AND * the ingestionId is incorrect is a duplicate ingestion record. - * @param ingestionId - string - * @param provider - string + * @param ingestionId string + * @param provider string */ async clearDuplicateIngestions(ingestionId: string, provider: string) { await this.client.transaction(async tx => { @@ -197,7 +195,7 @@ export class IncrementalIngestionDatabaseManager { /** * This method fully purges and resets all ingestion records for the named provider, and * leaves it in a paused state. - * @param provider - string + * @param provider string * @returns Counts of all deleted ingestion records */ async purgeAndResetProvider(provider: string) { @@ -248,15 +246,14 @@ export class IncrementalIngestionDatabaseManager { const next_action_at = new Date(); next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000); - await this.insertRecord({ - record: { - id: v4(), - next_action: 'rest', - provider_name: provider, - next_action_at, - ingestion_completed_at: new Date(), - status: 'resting', - }, + await this.insertIngestionRecord({ + id: v4(), + next_action: 'rest', + provider_name: provider, + next_action_at, + ingestion_completed_at: new Date(), + status: 'resting', + completion_ticket: v4(), }); return { provider, ingestionsDeleted, marksDeleted, markEntitiesDeleted }; @@ -265,27 +262,31 @@ export class IncrementalIngestionDatabaseManager { /** * Creates a new ingestion record. - * @param provider - string + * @param provider string * @returns A new ingestion record */ async createProviderIngestionRecord(provider: string) { const ingestionId = v4(); const nextAction = 'ingest'; - await this.insertIngestionRecord({ - record: { + try { + await this.insertIngestionRecord({ id: ingestionId, next_action: nextAction, provider_name: provider, status: 'bursting', - }, - }); - return { ingestionId, nextAction, attempts: 0, nextActionAt: Date.now() }; + completion_ticket: 'open', + }); + return { ingestionId, nextAction, attempts: 0, nextActionAt: Date.now() }; + } catch (_e) { + // Creating the ingestion record failed. Return undefined. + return undefined; + } } /** * Computes which entities to remove, if any, at the end of a burst. - * @param provider - string - * @param ingestionId - string + * @param provider string + * @param ingestionId string * @returns All entities to remove for this burst. */ async computeRemoved(provider: string, ingestionId: string) { @@ -340,7 +341,7 @@ export class IncrementalIngestionDatabaseManager { /** * Skips any wait time for the next action to run. - * @param provider - string + * @param provider string */ async triggerNextProviderAction(provider: string) { await this.updateIngestionRecordByProvider(provider, { @@ -367,14 +368,13 @@ export class IncrementalIngestionDatabaseManager { for (const provider of providers) { await this.insertIngestionRecord({ - record: { - id: v4(), - next_action: 'rest', - provider_name: provider, - next_action_at, - ingestion_completed_at: new Date(), - status: 'resting', - }, + id: v4(), + next_action: 'rest', + provider_name: provider, + next_action_at, + ingestion_completed_at: new Date(), + status: 'resting', + completion_ticket: v4(), }); } @@ -390,7 +390,7 @@ export class IncrementalIngestionDatabaseManager { /** * Configures the current ingestion record to ingest a burst. - * @param ingestionId - string + * @param ingestionId string */ async setProviderIngesting(ingestionId: string) { await this.updateIngestionRecordById({ @@ -401,7 +401,7 @@ export class IncrementalIngestionDatabaseManager { /** * Indicates the provider is currently ingesting a burst. - * @param ingestionId - string + * @param ingestionId string */ async setProviderBursting(ingestionId: string) { await this.updateIngestionRecordById({ @@ -412,7 +412,7 @@ export class IncrementalIngestionDatabaseManager { /** * Finalizes the current ingestion record to indicate that the post-ingestion rest period is complete. - * @param ingestionId - string + * @param ingestionId string */ async setProviderComplete(ingestionId: string) { await this.updateIngestionRecordById({ @@ -421,14 +421,15 @@ export class IncrementalIngestionDatabaseManager { next_action: 'nothing (done)', rest_completed_at: new Date(), status: 'complete', + completion_ticket: v4(), }, }); } /** * Marks ingestion as complete and starts the post-ingestion rest cycle. - * @param ingestionId - string - * @param restLength - Duration + * @param ingestionId string + * @param restLength Duration */ async setProviderResting(ingestionId: string, restLength: Duration) { await this.updateIngestionRecordById({ @@ -444,7 +445,7 @@ export class IncrementalIngestionDatabaseManager { /** * Marks ingestion as paused after a burst completes. - * @param ingestionId - string + * @param ingestionId string */ async setProviderInterstitial(ingestionId: string) { await this.updateIngestionRecordById({ @@ -455,8 +456,8 @@ export class IncrementalIngestionDatabaseManager { /** * Starts the cancel process for the current ingestion. - * @param ingestionId - string - * @param message - string (optional) + * @param ingestionId string + * @param message string (optional) */ async setProviderCanceling(ingestionId: string, message?: string) { const update: Partial = { @@ -470,7 +471,7 @@ export class IncrementalIngestionDatabaseManager { /** * Completes the cancel process and triggers a new ingestion. - * @param ingestionId - string + * @param ingestionId string */ async setProviderCanceled(ingestionId: string) { await this.updateIngestionRecordById({ @@ -479,16 +480,17 @@ export class IncrementalIngestionDatabaseManager { next_action: 'nothing (canceled)', rest_completed_at: new Date(), status: 'complete', + completion_ticket: v4(), }, }); } /** * Configures the current ingestion to wait and retry, due to a data source error. - * @param ingestionId - string - * @param attempts - number - * @param error - Error - * @param backoffLength - number + * @param ingestionId string + * @param attempts number + * @param error Error + * @param backoffLength number */ async setProviderBackoff( ingestionId: string, @@ -510,7 +512,7 @@ export class IncrementalIngestionDatabaseManager { /** * Returns the last record from `ingestion.ingestion_marks` for the supplied ingestionId. - * @param ingestionId - string + * @param ingestionId string * @returns MarkRecord | undefined */ async getLastMark(ingestionId: string) { @@ -534,7 +536,7 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an insert into the `ingestion.ingestion_marks` table with the supplied values. - * @param options - MarkRecordInsert + * @param options MarkRecordInsert */ async createMark(options: MarkRecordInsert) { const { record } = options; @@ -544,8 +546,8 @@ export class IncrementalIngestionDatabaseManager { } /** * Performs an upsert to the `ingestion.ingestion_mark_entities` table for all deferred entities. - * @param markId - string - * @param entities - DeferredEntity[] + * @param markId string + * @param entities DeferredEntity[] */ async createMarkEntities(markId: string, entities: DeferredEntity[]) { await this.client.transaction(async tx => { @@ -561,7 +563,7 @@ export class IncrementalIngestionDatabaseManager { /** * Deletes the entire content of a table, and returns the number of records deleted. - * @param table - string + * @param table string * @returns number */ async purgeTable(table: string) { @@ -586,7 +588,4 @@ export class IncrementalIngestionDatabaseManager { async updateByName(provider: string, update: Partial) { await this.updateIngestionRecordByProvider(provider, update); } - async insertRecord(options: IngestionRecordInsert) { - await this.insertIngestionRecord(options); - } } diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/incremental-ingestion-backend/src/types.ts index c9f3f50e1d..5cc0a5072d 100644 --- a/plugins/incremental-ingestion-backend/src/types.ts +++ b/plugins/incremental-ingestion-backend/src/types.ts @@ -30,7 +30,6 @@ import type { PermissionAuthorizer } from '@backstage/plugin-permission-common'; import type { DurationObjectUnits } from 'luxon'; import type { Logger } from 'winston'; import { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager'; - /** * Entity annotation containing the incremental entity provider. * @@ -51,7 +50,6 @@ export const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = * batches of entities in sequence so that you never need to have more * than a few hundred in memory at a time. * - * @public */ export interface IncrementalEntityProvider { /** @@ -60,14 +58,20 @@ export interface IncrementalEntityProvider { */ getProviderName(): string; + /** + * Return a function to register the end of the metric that was + * initialized before the fisrt burst call. + */ + prometheusRegister?: { markIngestionCompleted: (result: string) => void }; + /** * Return a single page of entities from a specific point in the * ingestion. * * @param context - anything needed in order to fetch a single page. * @param cursor - a uniqiue value identifying the page to ingest. - * @returns the entities to be ingested, as well as the cursor of - * the the next page after this one. + * @returns The entities to be ingested, as well as the cursor of + * the next page after this one. */ next( context: TContext, @@ -85,16 +89,13 @@ export interface IncrementalEntityProvider { } /** - * Value returned by an {@link IncrementalEntityProvider} to provide a + * Value returned by an @{link IncrementalEntityProvider} to provide a * single page of entities to ingest. - * - * @public */ export interface EntityIteratorResult { /** * Indicates whether there are any further pages of entities to * ingest after this one. - * */ done: boolean; @@ -110,7 +111,6 @@ export interface EntityIteratorResult { entities: DeferredEntity[]; } -/** @public */ export interface IncrementalEntityProviderOptions { /** * Entities are ingested in bursts. This interval determines how @@ -134,16 +134,11 @@ export interface IncrementalEntityProviderOptions { /** * In the event of an error during an ingestion burst, the backoff * determines how soon it will be retried. E.g. - * `[{ minutes: 1}, { minutes: 5}, {minutes: 30 }, { hours: 3 }]` + * [{ minutes: 1}, { minutes: 5}, {minutes: 30 }, { hours: 3 }] */ backoff?: DurationObjectUnits[]; } -/** - * Provides the environment used by the incremental entity provider, - * - * @public - */ export type PluginEnvironment = { logger: Logger; database: PluginDatabaseManager; @@ -153,20 +148,10 @@ export type PluginEnvironment = { permissions: PermissionAuthorizer; }; -/** - * The core ingestion engine implements this interface - * - * @public - */ export interface IterationEngine { taskFn: TaskFunction; } -/** - * Options passed to the core ingestion engine during initialization. - * - * @public - */ export interface IterationEngineOptions { logger: Logger; connection: EntityProviderConnection; @@ -179,10 +164,15 @@ export interface IterationEngineOptions { /** * The shape of data inserted into or updated in the `ingestion.ingestions` table. - * - * @public */ export interface IngestionUpsertIFace { + /** + * The ingestion record id. + */ + id?: string; + /** + * The next action the incremental entity provider will take. + */ next_action: | 'rest' | 'ingest' @@ -190,6 +180,9 @@ export interface IngestionUpsertIFace { | 'cancel' | 'nothing (done)' | 'nothing (canceled)'; + /** + * Current status of the incremental entity provider. + */ status: | 'complete' | 'bursting' @@ -197,29 +190,38 @@ export interface IngestionUpsertIFace { | 'canceling' | 'interstitial' | 'backing off'; + /** + * The name of the incremental entity provider being updated. + */ provider_name: string; + /** + * Date/time stamp for when the next action will trigger. + */ next_action_at?: Date; - last_error?: string; + /** + * A record of the last error generated by the incremental entity provider. + */ + last_error?: string | null; + /** + * The number of attempts the provider has attempted during the current cycle. + */ attempts?: number; - ingestion_completed_at?: Date; - rest_completed_at?: Date; -} - -/** - * This interface supplies all potential values that can be inserted into the `ingestion.ingestions` table. - * - * @public - */ -export interface IngestionRecordInsert { - record: IngestionUpsertIFace & { - id: string; - }; + /** + * Date/time stamp for the completion of ingestion. + */ + ingestion_completed_at?: Date | string | null; + /** + * Date/time stamp for the end of the rest cycle before the next ingestion. + */ + rest_completed_at?: Date | string | null; + /** + * A record of the finalized status of the ingestion record. Values are either 'open' or a uuid. + */ + completion_ticket: string; } /** * This interface is for updating an existing ingestion record. - * - * @public */ export interface IngestionRecordUpdate { ingestionId: string; @@ -228,8 +230,6 @@ export interface IngestionRecordUpdate { /** * The expected response from the `ingestion.ingestion_marks` table. - * - * @public */ export interface MarkRecord { id: string; @@ -241,26 +241,18 @@ export interface MarkRecord { /** * The expected response from the `ingestion.ingestions` table. - * - * @public */ -export interface IngestionRecord { +export interface IngestionRecord extends IngestionUpsertIFace { id: string; - provider_name: string; - status: string; - next_action: string; next_action_at: Date; - last_error: string | null; - attempts: number; + /** + * The date/time the ingestion record was created. + */ created_at: string; - rest_completed_at: string | null; - ingestion_completed_at: string | null; } /** * This interface supplies all the values for adding an ingestion mark. - * - * @public */ export interface MarkRecordInsert { record: { From 5b94fdbaf59fe8add3eaa08d283ed5d5c1bd3b72 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 3 Nov 2022 08:48:55 -0700 Subject: [PATCH 31/54] Fix more divergent code Signed-off-by: Damon Kaswell --- .../IncrementalIngestionDatabaseManager.ts | 63 +++---- .../src/engine/IncrementalIngestionEngine.ts | 175 ++++++++++-------- .../src/types.ts | 25 ++- 3 files changed, 143 insertions(+), 120 deletions(-) diff --git a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts index 6a59757ca1..cc791a6f88 100644 --- a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts +++ b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts @@ -28,6 +28,7 @@ import { MarkRecordInsert, } from '../types'; +/** @public */ export class IncrementalIngestionDatabaseManager { private client: Knex; @@ -37,7 +38,7 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an update to the ingestion record with matching `id`. - * @param options IngestionRecordUpdate + * @param options - IngestionRecordUpdate */ async updateIngestionRecordById(options: IngestionRecordUpdate) { await this.client.transaction(async tx => { @@ -48,8 +49,8 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an update to the ingestion record with matching provider name. Will only update active records. - * @param provider string - * @param update Partial + * @param provider - string + * @param update - Partial */ async updateIngestionRecordByProvider( provider: string, @@ -65,7 +66,7 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an insert into the `ingestion.ingestions` table with the supplied values. - * @param record IngestionUpsertIFace + * @param record - IngestionUpsertIFace */ async insertIngestionRecord(record: IngestionUpsertIFace) { await this.client.transaction(async tx => { @@ -100,7 +101,7 @@ export class IncrementalIngestionDatabaseManager { /** * Finds the current ingestion record for the named provider. - * @param provider string + * @param provider - string * @returns IngestionRecord | undefined */ async getCurrentIngestionRecord(provider: string) { @@ -116,7 +117,7 @@ export class IncrementalIngestionDatabaseManager { /** * Removes all entries from `ingestion_marks_entities`, `ingestion_marks`, and `ingestions` * for prior ingestions that completed (i.e., have a `completion_ticket` value other than 'open'). - * @param provider string + * @param provider - string * @returns A count of deletions for each record type. */ async clearFinishedIngestions(provider: string) { @@ -165,8 +166,8 @@ export class IncrementalIngestionDatabaseManager { * Automatically cleans up duplicate ingestion records if they were accidentally created. * Any ingestion record where the `rest_completed_at` is null (meaning it is active) AND * the ingestionId is incorrect is a duplicate ingestion record. - * @param ingestionId string - * @param provider string + * @param ingestionId - string + * @param provider - string */ async clearDuplicateIngestions(ingestionId: string, provider: string) { await this.client.transaction(async tx => { @@ -195,7 +196,7 @@ export class IncrementalIngestionDatabaseManager { /** * This method fully purges and resets all ingestion records for the named provider, and * leaves it in a paused state. - * @param provider string + * @param provider - string * @returns Counts of all deleted ingestion records */ async purgeAndResetProvider(provider: string) { @@ -262,7 +263,7 @@ export class IncrementalIngestionDatabaseManager { /** * Creates a new ingestion record. - * @param provider string + * @param provider - string * @returns A new ingestion record */ async createProviderIngestionRecord(provider: string) { @@ -285,8 +286,8 @@ export class IncrementalIngestionDatabaseManager { /** * Computes which entities to remove, if any, at the end of a burst. - * @param provider string - * @param ingestionId string + * @param provider - string + * @param ingestionId - string * @returns All entities to remove for this burst. */ async computeRemoved(provider: string, ingestionId: string) { @@ -341,7 +342,7 @@ export class IncrementalIngestionDatabaseManager { /** * Skips any wait time for the next action to run. - * @param provider string + * @param provider - string */ async triggerNextProviderAction(provider: string) { await this.updateIngestionRecordByProvider(provider, { @@ -390,7 +391,7 @@ export class IncrementalIngestionDatabaseManager { /** * Configures the current ingestion record to ingest a burst. - * @param ingestionId string + * @param ingestionId - string */ async setProviderIngesting(ingestionId: string) { await this.updateIngestionRecordById({ @@ -401,7 +402,7 @@ export class IncrementalIngestionDatabaseManager { /** * Indicates the provider is currently ingesting a burst. - * @param ingestionId string + * @param ingestionId - string */ async setProviderBursting(ingestionId: string) { await this.updateIngestionRecordById({ @@ -412,7 +413,7 @@ export class IncrementalIngestionDatabaseManager { /** * Finalizes the current ingestion record to indicate that the post-ingestion rest period is complete. - * @param ingestionId string + * @param ingestionId - string */ async setProviderComplete(ingestionId: string) { await this.updateIngestionRecordById({ @@ -428,8 +429,8 @@ export class IncrementalIngestionDatabaseManager { /** * Marks ingestion as complete and starts the post-ingestion rest cycle. - * @param ingestionId string - * @param restLength Duration + * @param ingestionId - string + * @param restLength - Duration */ async setProviderResting(ingestionId: string, restLength: Duration) { await this.updateIngestionRecordById({ @@ -445,7 +446,7 @@ export class IncrementalIngestionDatabaseManager { /** * Marks ingestion as paused after a burst completes. - * @param ingestionId string + * @param ingestionId - string */ async setProviderInterstitial(ingestionId: string) { await this.updateIngestionRecordById({ @@ -456,8 +457,8 @@ export class IncrementalIngestionDatabaseManager { /** * Starts the cancel process for the current ingestion. - * @param ingestionId string - * @param message string (optional) + * @param ingestionId - string + * @param message - string (optional) */ async setProviderCanceling(ingestionId: string, message?: string) { const update: Partial = { @@ -471,7 +472,7 @@ export class IncrementalIngestionDatabaseManager { /** * Completes the cancel process and triggers a new ingestion. - * @param ingestionId string + * @param ingestionId - string */ async setProviderCanceled(ingestionId: string) { await this.updateIngestionRecordById({ @@ -487,10 +488,10 @@ export class IncrementalIngestionDatabaseManager { /** * Configures the current ingestion to wait and retry, due to a data source error. - * @param ingestionId string - * @param attempts number - * @param error Error - * @param backoffLength number + * @param ingestionId - string + * @param attempts - number + * @param error - Error + * @param backoffLength - number */ async setProviderBackoff( ingestionId: string, @@ -512,7 +513,7 @@ export class IncrementalIngestionDatabaseManager { /** * Returns the last record from `ingestion.ingestion_marks` for the supplied ingestionId. - * @param ingestionId string + * @param ingestionId - string * @returns MarkRecord | undefined */ async getLastMark(ingestionId: string) { @@ -536,7 +537,7 @@ export class IncrementalIngestionDatabaseManager { /** * Performs an insert into the `ingestion.ingestion_marks` table with the supplied values. - * @param options MarkRecordInsert + * @param options - MarkRecordInsert */ async createMark(options: MarkRecordInsert) { const { record } = options; @@ -546,8 +547,8 @@ export class IncrementalIngestionDatabaseManager { } /** * Performs an upsert to the `ingestion.ingestion_mark_entities` table for all deferred entities. - * @param markId string - * @param entities DeferredEntity[] + * @param markId - string + * @param entities - DeferredEntity[] */ async createMarkEntities(markId: string, entities: DeferredEntity[]) { await this.client.transaction(async tx => { @@ -563,7 +564,7 @@ export class IncrementalIngestionDatabaseManager { /** * Deletes the entire content of a table, and returns the number of records deleted. - * @param table string + * @param table - string * @returns number */ async purgeTable(table: string) { diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts index 574d678e90..790e252dbf 100644 --- a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; import { INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, @@ -26,7 +27,6 @@ import { performance } from 'perf_hooks'; import { Duration, DurationObjectUnits } from 'luxon'; import { v4 } from 'uuid'; -/** @public */ export class IncrementalIngestionEngine implements IterationEngine { restLength: Duration; backoff: DurationObjectUnits[]; @@ -59,97 +59,106 @@ export class IncrementalIngestionEngine implements IterationEngine { async handleNextAction(signal: AbortSignal) { await this.options.ready; - const { ingestionId, nextActionAt, nextAction, attempts } = - await this.getCurrentAction(); + const result = await this.getCurrentAction(); + if (result) { + const { ingestionId, nextActionAt, nextAction, attempts } = result; - switch (nextAction) { - case 'rest': - if (Date.now() > nextActionAt) { - await this.manager.clearFinishedIngestions( - this.options.provider.getProviderName(), - ); - this.options.logger.info( - `incremental-engine: Ingestion ${ingestionId} rest period complete. Ingestion will start again`, - ); - - await this.manager.setProviderComplete(ingestionId); - } else { - this.options.logger.info( - `incremental-engine: Ingestion '${ingestionId}' rest period continuing`, - ); - } - break; - case 'ingest': - try { - await this.manager.setProviderBursting(ingestionId); - const done = await this.ingestOneBurst(ingestionId, signal); - if (done) { - this.options.logger.info( - `incremental-engine: Ingestion '${ingestionId}' complete, transitioning to rest period of ${this.restLength.toHuman()}`, + switch (nextAction) { + case 'rest': + if (Date.now() > nextActionAt) { + await this.manager.clearFinishedIngestions( + this.options.provider.getProviderName(), ); - await this.manager.setProviderResting(ingestionId, this.restLength); - } else { - await this.manager.setProviderInterstitial(ingestionId); this.options.logger.info( - `incremental-engine: Ingestion '${ingestionId}' continuing`, + `incremental-engine: Ingestion ${ingestionId} rest period complete. Ingestion will start again`, + ); + + await this.manager.setProviderComplete(ingestionId); + } else { + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' rest period continuing`, ); } - } catch (error) { - if ( - (error as Error).message && - (error as Error).message === 'CANCEL' - ) { + break; + case 'ingest': + try { + await this.manager.setProviderBursting(ingestionId); + const done = await this.ingestOneBurst(ingestionId, signal); + if (done) { + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' complete, transitioning to rest period of ${this.restLength.toHuman()}`, + ); + await this.manager.setProviderResting( + ingestionId, + this.restLength, + ); + } else { + await this.manager.setProviderInterstitial(ingestionId); + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' continuing`, + ); + } + } catch (error) { + if ( + (error as Error).message && + (error as Error).message === 'CANCEL' + ) { + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' canceled`, + ); + await this.manager.setProviderCanceling( + ingestionId, + (error as Error).message, + ); + } else { + const currentBackoff = Duration.fromObject( + this.backoff[Math.min(this.backoff.length - 1, attempts)], + ); + + const backoffLength = currentBackoff.as('milliseconds'); + this.options.logger.error(error); + + const truncatedError = (error as string).substring(0, 700); + this.options.logger.error( + `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`, + ); + + await this.manager.setProviderBackoff( + ingestionId, + attempts, + error as Error, + backoffLength, + ); + } + } + break; + case 'backoff': + if (Date.now() > nextActionAt) { this.options.logger.info( - `incremental-engine: Ingestion '${ingestionId}' canceled`, - ); - await this.manager.setProviderCanceling( - ingestionId, - (error as Error).message, + `incremental-engine: Ingestion '${ingestionId}' backoff complete, will attempt to resume`, ); + await this.manager.setProviderIngesting(ingestionId); } else { - const currentBackoff = Duration.fromObject( - this.backoff[Math.min(this.backoff.length - 1, attempts)], - ); - - const backoffLength = currentBackoff.as('milliseconds'); - this.options.logger.error(error); - - const truncatedError = (error as string).substring(0, 700); - this.options.logger.error( - `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`, - ); - - await this.manager.setProviderBackoff( - ingestionId, - attempts, - error as Error, - backoffLength, + this.options.logger.info( + `incremental-engine: Ingestion '${ingestionId}' backoff continuing`, ); } - } - break; - case 'backoff': - if (Date.now() > nextActionAt) { + break; + case 'cancel': this.options.logger.info( - `incremental-engine: Ingestion '${ingestionId}' backoff complete, will attempt to resume`, + `incremental-engine: Ingestion '${ingestionId}' canceling, will restart`, ); - await this.manager.setProviderIngesting(ingestionId); - } else { - this.options.logger.info( - `incremental-engine: Ingestion '${ingestionId}' backoff continuing`, + await this.manager.setProviderCanceled(ingestionId); + break; + default: + this.options.logger.error( + `incremental-engine: Ingestion '${ingestionId}' received unknown action '${nextAction}'`, ); - } - break; - case 'cancel': - this.options.logger.info( - `incremental-engine: Ingestion '${ingestionId}' canceling, will restart`, - ); - await this.manager.setProviderCanceled(ingestionId); - break; - default: - this.options.logger.error( - `incremental-engine: Ingestion '${ingestionId}' received unknown action '${nextAction}'`, - ); + } + } else { + this.options.logger.error( + `incremental-engine: Engine tried to create duplicate ingestion record for provider '${this.options.provider.getProviderName()}'.`, + ); } } @@ -170,9 +179,11 @@ export class IncrementalIngestionEngine implements IterationEngine { const result = await this.manager.createProviderIngestionRecord( providerName, ); - this.options.logger.info( - `incremental-engine: Ingestion record created: '${result.ingestionId}'`, - ); + if (result) { + this.options.logger.info( + `incremental-engine: Ingestion record created: '${result.ingestionId}'`, + ); + } return result; } diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/incremental-ingestion-backend/src/types.ts index 5cc0a5072d..7032b13a0c 100644 --- a/plugins/incremental-ingestion-backend/src/types.ts +++ b/plugins/incremental-ingestion-backend/src/types.ts @@ -50,6 +50,7 @@ export const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = * batches of entities in sequence so that you never need to have more * than a few hundred in memory at a time. * + * @public */ export interface IncrementalEntityProvider { /** @@ -58,12 +59,6 @@ export interface IncrementalEntityProvider { */ getProviderName(): string; - /** - * Return a function to register the end of the metric that was - * initialized before the fisrt burst call. - */ - prometheusRegister?: { markIngestionCompleted: (result: string) => void }; - /** * Return a single page of entities from a specific point in the * ingestion. @@ -89,8 +84,10 @@ export interface IncrementalEntityProvider { } /** - * Value returned by an @{link IncrementalEntityProvider} to provide a + * Value returned by an {@link IncrementalEntityProvider} to provide a * single page of entities to ingest. + * + * @public */ export interface EntityIteratorResult { /** @@ -111,6 +108,7 @@ export interface EntityIteratorResult { entities: DeferredEntity[]; } +/** @public */ export interface IncrementalEntityProviderOptions { /** * Entities are ingested in bursts. This interval determines how @@ -139,6 +137,7 @@ export interface IncrementalEntityProviderOptions { backoff?: DurationObjectUnits[]; } +/** @public */ export type PluginEnvironment = { logger: Logger; database: PluginDatabaseManager; @@ -148,10 +147,12 @@ export type PluginEnvironment = { permissions: PermissionAuthorizer; }; +/** @public */ export interface IterationEngine { taskFn: TaskFunction; } +/** @public */ export interface IterationEngineOptions { logger: Logger; connection: EntityProviderConnection; @@ -164,6 +165,8 @@ export interface IterationEngineOptions { /** * The shape of data inserted into or updated in the `ingestion.ingestions` table. + * + * @public */ export interface IngestionUpsertIFace { /** @@ -222,6 +225,8 @@ export interface IngestionUpsertIFace { /** * This interface is for updating an existing ingestion record. + * + * @public */ export interface IngestionRecordUpdate { ingestionId: string; @@ -230,6 +235,8 @@ export interface IngestionRecordUpdate { /** * The expected response from the `ingestion.ingestion_marks` table. + * + * @public */ export interface MarkRecord { id: string; @@ -241,6 +248,8 @@ export interface MarkRecord { /** * The expected response from the `ingestion.ingestions` table. + * + * @public */ export interface IngestionRecord extends IngestionUpsertIFace { id: string; @@ -253,6 +262,8 @@ export interface IngestionRecord extends IngestionUpsertIFace { /** * This interface supplies all the values for adding an ingestion mark. + * + * @public */ export interface MarkRecordInsert { record: { From c689afc5e9829a013e004d688610e6557fd462ef Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 3 Nov 2022 08:53:21 -0700 Subject: [PATCH 32/54] Fix for api report Signed-off-by: Damon Kaswell --- .../api-report.md | 68 ++++++------------- .../src/types.ts | 14 ++-- 2 files changed, 26 insertions(+), 56 deletions(-) diff --git a/plugins/incremental-ingestion-backend/api-report.md b/plugins/incremental-ingestion-backend/api-report.md index 6d4d12cc6d..8aabaa428f 100644 --- a/plugins/incremental-ingestion-backend/api-report.md +++ b/plugins/incremental-ingestion-backend/api-report.md @@ -3,8 +3,6 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -/// - import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; import type { Config } from '@backstage/config'; import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; @@ -96,12 +94,15 @@ export class IncrementalIngestionDatabaseManager { >; createMark(options: MarkRecordInsert): Promise; createMarkEntities(markId: string, entities: DeferredEntity[]): Promise; - createProviderIngestionRecord(provider: string): Promise<{ - ingestionId: string; - nextAction: string; - attempts: number; - nextActionAt: number; - }>; + createProviderIngestionRecord(provider: string): Promise< + | { + ingestionId: string; + nextAction: string; + attempts: number; + nextActionAt: number; + } + | undefined + >; // (undocumented) getAllMarks(ingestionId: string): Promise; getCurrentIngestionRecord( @@ -117,9 +118,7 @@ export class IncrementalIngestionDatabaseManager { 'id' | 'provider_name' >[] >; - insertIngestionRecord(options: IngestionRecordInsert): Promise; - // (undocumented) - insertRecord(options: IngestionRecordInsert): Promise; + insertIngestionRecord(record: IngestionUpsertIFace): Promise; listProviders(): Promise; purgeAndResetProvider(provider: string): Promise<{ provider: string; @@ -155,35 +154,12 @@ export class IncrementalIngestionDatabaseManager { } // @public -export interface IngestionRecord { - // (undocumented) - attempts: number; - // (undocumented) +export interface IngestionRecord extends IngestionUpsertIFace { created_at: string; // (undocumented) id: string; // (undocumented) - ingestion_completed_at: string | null; - // (undocumented) - last_error: string | null; - // (undocumented) - next_action: string; - // (undocumented) next_action_at: Date; - // (undocumented) - provider_name: string; - // (undocumented) - rest_completed_at: string | null; - // (undocumented) - status: string; -} - -// @public -export interface IngestionRecordInsert { - // (undocumented) - record: IngestionUpsertIFace & { - id: string; - }; } // @public @@ -196,13 +172,11 @@ export interface IngestionRecordUpdate { // @public export interface IngestionUpsertIFace { - // (undocumented) attempts?: number; - // (undocumented) - ingestion_completed_at?: Date; - // (undocumented) - last_error?: string; - // (undocumented) + completion_ticket: string; + id?: string; + ingestion_completed_at?: Date | string | null; + last_error?: string | null; next_action: | 'rest' | 'ingest' @@ -210,13 +184,9 @@ export interface IngestionUpsertIFace { | 'cancel' | 'nothing (done)' | 'nothing (canceled)'; - // (undocumented) next_action_at?: Date; - // (undocumented) provider_name: string; - // (undocumented) - rest_completed_at?: Date; - // (undocumented) + rest_completed_at?: Date | string | null; status: | 'complete' | 'bursting' @@ -226,13 +196,13 @@ export interface IngestionUpsertIFace { | 'backing off'; } -// @public +// @public (undocumented) export interface IterationEngine { // (undocumented) taskFn: TaskFunction; } -// @public +// @public (undocumented) export interface IterationEngineOptions { // (undocumented) backoff?: IncrementalEntityProviderOptions['backoff']; @@ -275,7 +245,7 @@ export interface MarkRecordInsert { }; } -// @public +// @public (undocumented) export type PluginEnvironment = { logger: Logger; database: PluginDatabaseManager; diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/incremental-ingestion-backend/src/types.ts index 7032b13a0c..8a3233e3b7 100644 --- a/plugins/incremental-ingestion-backend/src/types.ts +++ b/plugins/incremental-ingestion-backend/src/types.ts @@ -86,7 +86,7 @@ export interface IncrementalEntityProvider { /** * Value returned by an {@link IncrementalEntityProvider} to provide a * single page of entities to ingest. - * + * * @public */ export interface EntityIteratorResult { @@ -132,7 +132,7 @@ export interface IncrementalEntityProviderOptions { /** * In the event of an error during an ingestion burst, the backoff * determines how soon it will be retried. E.g. - * [{ minutes: 1}, { minutes: 5}, {minutes: 30 }, { hours: 3 }] + * `[{ minutes: 1}, { minutes: 5}, {minutes: 30 }, { hours: 3 }]` */ backoff?: DurationObjectUnits[]; } @@ -165,7 +165,7 @@ export interface IterationEngineOptions { /** * The shape of data inserted into or updated in the `ingestion.ingestions` table. - * + * * @public */ export interface IngestionUpsertIFace { @@ -225,7 +225,7 @@ export interface IngestionUpsertIFace { /** * This interface is for updating an existing ingestion record. - * + * * @public */ export interface IngestionRecordUpdate { @@ -235,7 +235,7 @@ export interface IngestionRecordUpdate { /** * The expected response from the `ingestion.ingestion_marks` table. - * + * * @public */ export interface MarkRecord { @@ -248,7 +248,7 @@ export interface MarkRecord { /** * The expected response from the `ingestion.ingestions` table. - * + * * @public */ export interface IngestionRecord extends IngestionUpsertIFace { @@ -262,7 +262,7 @@ export interface IngestionRecord extends IngestionUpsertIFace { /** * This interface supplies all the values for adding an ingestion mark. - * + * * @public */ export interface MarkRecordInsert { From 6f42b641abb562ecbde40f4895524bc792eb111f Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 3 Nov 2022 15:50:50 -0700 Subject: [PATCH 33/54] Update api-report.md for real Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/api-report.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/incremental-ingestion-backend/api-report.md b/plugins/incremental-ingestion-backend/api-report.md index 8aabaa428f..375eedeae0 100644 --- a/plugins/incremental-ingestion-backend/api-report.md +++ b/plugins/incremental-ingestion-backend/api-report.md @@ -3,6 +3,8 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +/// + import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; import type { Config } from '@backstage/config'; import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; From 21f0ae79d6a972b023e42e37ad85b4a6e9791e02 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 4 Nov 2022 09:35:22 -0700 Subject: [PATCH 34/54] Handle error edge cases Signed-off-by: Damon Kaswell --- .../src/engine/IncrementalIngestionEngine.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts index 790e252dbf..4b8588cdfe 100644 --- a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts @@ -118,7 +118,10 @@ export class IncrementalIngestionEngine implements IterationEngine { const backoffLength = currentBackoff.as('milliseconds'); this.options.logger.error(error); - const truncatedError = (error as string).substring(0, 700); + const truncatedError = + typeof error === 'string' + ? error.substring(0, 700) + : `${error}`; this.options.logger.error( `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`, ); From 261e73cd52e48bd98e1c50e8d1fa5c5a72f01f5d Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Wed, 9 Nov 2022 13:27:24 -0800 Subject: [PATCH 35/54] Remove from package.json Signed-off-by: Damon Kaswell --- packages/backend/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index ad1734e45a..e7d2d5d7b7 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -44,7 +44,6 @@ "@backstage/plugin-events-backend": "workspace:^", "@backstage/plugin-events-node": "workspace:^", "@backstage/plugin-graphql-backend": "workspace:^", - "@backstage/plugin-incremental-ingestion-backend": "workspace:^", "@backstage/plugin-jenkins-backend": "workspace:^", "@backstage/plugin-kafka-backend": "workspace:^", "@backstage/plugin-kubernetes-backend": "workspace:^", From d0e85ef9aff735322068a24e16d36d10610f731e Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Wed, 9 Nov 2022 16:26:20 -0800 Subject: [PATCH 36/54] Code hygiene and cleanup Signed-off-by: Damon Kaswell --- .../incremental-ingestion-backend/README.md | 2 +- .../api-report.md | 30 +-------- .../package.json | 4 +- .../src/engine/IncrementalIngestionEngine.ts | 62 +++++++++++-------- .../src/index.ts | 17 ++++- .../src/routes.ts | 2 +- .../src/service/IncrementalCatalogBuilder.ts | 23 +++---- .../src/types.ts | 14 ++--- yarn.lock | 12 +--- 9 files changed, 75 insertions(+), 91 deletions(-) diff --git a/plugins/incremental-ingestion-backend/README.md b/plugins/incremental-ingestion-backend/README.md index 4e9d4fe65b..0e0a4e56e6 100644 --- a/plugins/incremental-ingestion-backend/README.md +++ b/plugins/incremental-ingestion-backend/README.md @@ -42,7 +42,7 @@ The Incremental Entity Provider backend is designed for data sources that provid ## Installation -1. Install `@backstage/plugin-incremental-ingestion-backend` with `yarn add @backstage/plugin-incremental-ingestion-backend` +1. Install `@backstage/plugin-incremental-ingestion-backend` with `yarn workspace backend add @backstage/plugin-incremental-ingestion-backend` 2. Import `IncrementalCatalogBuilder` from `@backstage/plugin-incremental-ingestion-backend` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`. ```ts diff --git a/plugins/incremental-ingestion-backend/api-report.md b/plugins/incremental-ingestion-backend/api-report.md index 375eedeae0..d843aad1a3 100644 --- a/plugins/incremental-ingestion-backend/api-report.md +++ b/plugins/incremental-ingestion-backend/api-report.md @@ -10,21 +10,19 @@ import type { Config } from '@backstage/config'; import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; import { Duration } from 'luxon'; import type { DurationObjectUnits } from 'luxon'; -import type { EntityProviderConnection } from '@backstage/plugin-catalog-backend'; import { Knex } from 'knex'; import type { Logger } from 'winston'; import type { PermissionAuthorizer } from '@backstage/plugin-permission-common'; import type { PluginDatabaseManager } from '@backstage/backend-common'; import type { PluginTaskScheduler } from '@backstage/backend-tasks'; import { Router } from 'express'; -import type { TaskFunction } from '@backstage/backend-tasks'; import type { UrlReader } from '@backstage/backend-common'; // @public export interface EntityIteratorResult { - cursor: T; + cursor?: T; done: boolean; - entities: DeferredEntity[]; + entities?: DeferredEntity[]; } // @public @@ -198,30 +196,6 @@ export interface IngestionUpsertIFace { | 'backing off'; } -// @public (undocumented) -export interface IterationEngine { - // (undocumented) - taskFn: TaskFunction; -} - -// @public (undocumented) -export interface IterationEngineOptions { - // (undocumented) - backoff?: IncrementalEntityProviderOptions['backoff']; - // (undocumented) - connection: EntityProviderConnection; - // (undocumented) - logger: Logger; - // (undocumented) - manager: IncrementalIngestionDatabaseManager; - // (undocumented) - provider: IncrementalEntityProvider; - // (undocumented) - ready: Promise; - // (undocumented) - restLength: DurationObjectUnits; -} - // @public export interface MarkRecord { // (undocumented) diff --git a/plugins/incremental-ingestion-backend/package.json b/plugins/incremental-ingestion-backend/package.json index 6fdcdbb138..195adab7ea 100644 --- a/plugins/incremental-ingestion-backend/package.json +++ b/plugins/incremental-ingestion-backend/package.json @@ -1,10 +1,10 @@ { "name": "@backstage/plugin-incremental-ingestion-backend", + "description": "An entity provider for streaming large asset sources into the catalog", "version": "0.1.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", - "private": true, "publishConfig": { "access": "public", "main": "dist/index.cjs.js", @@ -30,7 +30,7 @@ "@backstage/plugin-catalog-backend": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", "@types/express": "^4.17.6", - "@types/luxon": "3.0.0", + "@types/luxon": "^3.0.0", "express": "^4.17.1", "express-promise-router": "^4.1.0", "knex": "^2.0.0", diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts index 4b8588cdfe..6d35f2db4b 100644 --- a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts @@ -27,6 +27,7 @@ import { performance } from 'perf_hooks'; import { Duration, DurationObjectUnits } from 'luxon'; import { v4 } from 'uuid'; +/** @public */ export class IncrementalIngestionEngine implements IterationEngine { restLength: Duration; backoff: DurationObjectUnits[]; @@ -193,7 +194,7 @@ export class IncrementalIngestionEngine implements IterationEngine { async ingestOneBurst(id: string, signal: AbortSignal) { const lastMark = await this.manager.getLastMark(id); - const cursor = lastMark ? lastMark.cursor : void 0; + const cursor = lastMark ? lastMark.cursor : undefined; let sequence = lastMark ? lastMark.sequence + 1 : 0; const start = performance.now(); @@ -206,10 +207,15 @@ export class IncrementalIngestionEngine implements IterationEngine { await this.options.provider.around(async (context: unknown) => { let next = await this.options.provider.next(context, cursor); count++; - // eslint-disable-next-line no-constant-condition - while (true) { + for (;;) { done = next.done; - await this.mark(id, sequence, next.entities, next.done, next.cursor); + await this.mark({ + id, + sequence, + entities: next?.entities, + done: next.done, + cursor: next?.cursor, + }); if (signal.aborted || next.done) { break; } else { @@ -228,17 +234,20 @@ export class IncrementalIngestionEngine implements IterationEngine { return done; } - async mark( - id: string, - sequence: number, - entities: DeferredEntity[], - done: boolean, - cursor?: unknown, - ) { + async mark(options: { + id: string; + sequence: number; + entities?: DeferredEntity[]; + done: boolean; + cursor?: unknown; + }) { + const { id, sequence, entities, done, cursor } = options; this.options.logger.debug( `incremental-engine: Ingestion '${id}': MARK ${ - entities.length - } entities, cursor: ${JSON.stringify(cursor)}, done: ${done}`, + entities ? entities.length : 0 + } entities, cursor: ${ + cursor ? JSON.stringify(cursor) : 'none' + }, done: ${done}`, ); const markId = v4(); @@ -251,24 +260,25 @@ export class IncrementalIngestionEngine implements IterationEngine { }, }); - if (entities.length > 0) { + if (entities && entities.length > 0) { await this.manager.createMarkEntities(markId, entities); } - const added = entities.map(deferred => ({ - ...deferred, - entity: { - ...deferred.entity, - metadata: { - ...deferred.entity.metadata, - annotations: { - ...deferred.entity.metadata.annotations, - [INCREMENTAL_ENTITY_PROVIDER_ANNOTATION]: - this.options.provider.getProviderName(), + const added = + entities?.map(deferred => ({ + ...deferred, + entity: { + ...deferred.entity, + metadata: { + ...deferred.entity.metadata, + annotations: { + ...deferred.entity.metadata.annotations, + [INCREMENTAL_ENTITY_PROVIDER_ANNOTATION]: + this.options.provider.getProviderName(), + }, }, }, - }, - })); + })) ?? []; const removed: DeferredEntity[] = done ? [] diff --git a/plugins/incremental-ingestion-backend/src/index.ts b/plugins/incremental-ingestion-backend/src/index.ts index 46afb24a25..da50c6b37f 100644 --- a/plugins/incremental-ingestion-backend/src/index.ts +++ b/plugins/incremental-ingestion-backend/src/index.ts @@ -13,6 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export * from './service/IncrementalCatalogBuilder'; -export * from './types'; -export * from './database/IncrementalIngestionDatabaseManager'; +export { IncrementalCatalogBuilder } from './service/IncrementalCatalogBuilder'; +export type { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager'; +export type { + EntityIteratorResult, + IncrementalEntityProvider, + IncrementalEntityProviderOptions, + IngestionRecord, + IngestionRecordUpdate, + IngestionUpsertIFace, + MarkRecord, + MarkRecordInsert, + INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, + PluginEnvironment, +} from './types'; diff --git a/plugins/incremental-ingestion-backend/src/routes.ts b/plugins/incremental-ingestion-backend/src/routes.ts index 81ebc1d7b6..2f51ef6499 100644 --- a/plugins/incremental-ingestion-backend/src/routes.ts +++ b/plugins/incremental-ingestion-backend/src/routes.ts @@ -17,7 +17,7 @@ import { errorHandler } from '@backstage/backend-common'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; -import { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager'; +import { IncrementalIngestionDatabaseManager } from './'; /** @public */ export const createIncrementalProviderRouter = async ( diff --git a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts index e6e1e05091..7bf39e160f 100644 --- a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts @@ -27,12 +27,11 @@ import { IncrementalIngestionDatabaseManager } from '../database/IncrementalInge import { createIncrementalProviderRouter } from '../routes'; class Deferred implements Promise { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - resolve: (value: T) => void; - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - reject: (error: Error) => void; + #resolve?: (value: T) => void; + #reject?: (error: Error) => void; + + get resolve() { return this.#resolve!; } + get reject() { return this.#reject!; } then: Promise['then']; catch: Promise['catch']; @@ -40,8 +39,8 @@ class Deferred implements Promise { constructor() { const promise = new Promise((resolve, reject) => { - this.resolve = resolve; - this.reject = reject; + this.#resolve = resolve; + this.#reject = reject; }); this.then = promise.then.bind(promise); @@ -98,9 +97,11 @@ export class IncrementalCatalogBuilder { options: IncrementalEntityProviderOptions, ) { const { burstInterval, burstLength, restLength } = options; - const { logger: catalogLogger, database, scheduler } = this.env; + const { logger: catalogLogger, scheduler } = this.env; const ready = this.ready; + const manager = this.manager; + this.builder.addEntityProvider({ getProviderName: provider.getProviderName.bind(provider), async connect(connection) { @@ -110,10 +111,6 @@ export class IncrementalCatalogBuilder { logger.info(`Connecting`); - const client = await database.getClient(); - - const manager = new IncrementalIngestionDatabaseManager({ client }); - const engine = new IncrementalIngestionEngine({ ...options, ready, diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/incremental-ingestion-backend/src/types.ts index 8a3233e3b7..2671752ad6 100644 --- a/plugins/incremental-ingestion-backend/src/types.ts +++ b/plugins/incremental-ingestion-backend/src/types.ts @@ -29,7 +29,7 @@ import type { import type { PermissionAuthorizer } from '@backstage/plugin-permission-common'; import type { DurationObjectUnits } from 'luxon'; import type { Logger } from 'winston'; -import { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager'; +import { IncrementalIngestionDatabaseManager } from './'; /** * Entity annotation containing the incremental entity provider. * @@ -100,12 +100,12 @@ export interface EntityIteratorResult { * A value that marks the page of entities after this one. It will * be used to pass into the following invocation of `next()` */ - cursor: T; + cursor?: T; /** * The entities to ingest. */ - entities: DeferredEntity[]; + entities?: DeferredEntity[]; } /** @public */ @@ -225,7 +225,7 @@ export interface IngestionUpsertIFace { /** * This interface is for updating an existing ingestion record. - * + * * @public */ export interface IngestionRecordUpdate { @@ -235,7 +235,7 @@ export interface IngestionRecordUpdate { /** * The expected response from the `ingestion.ingestion_marks` table. - * + * * @public */ export interface MarkRecord { @@ -248,7 +248,7 @@ export interface MarkRecord { /** * The expected response from the `ingestion.ingestions` table. - * + * * @public */ export interface IngestionRecord extends IngestionUpsertIFace { @@ -262,7 +262,7 @@ export interface IngestionRecord extends IngestionUpsertIFace { /** * This interface supplies all the values for adding an ingestion mark. - * + * * @public */ export interface MarkRecordInsert { diff --git a/yarn.lock b/yarn.lock index 4cc281f7dd..17e002800a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6023,7 +6023,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-incremental-ingestion-backend@workspace:^, @backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend": +"@backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend": version: 0.0.0-use.local resolution: "@backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend" dependencies: @@ -6035,7 +6035,7 @@ __metadata: "@backstage/plugin-catalog-backend": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" "@types/express": ^4.17.6 - "@types/luxon": 3.0.0 + "@types/luxon": ^3.0.0 express: ^4.17.1 express-promise-router: ^4.1.0 knex: ^2.0.0 @@ -13805,13 +13805,6 @@ __metadata: languageName: node linkType: hard -"@types/luxon@npm:3.0.0": - version: 3.0.0 - resolution: "@types/luxon@npm:3.0.0" - checksum: 7738d3f4b91097a8139eca966ab53c6b40bb417b2cc2014c3bbd0aee033d6ff54af14c62046c2eb042434923b6bf796874f24b2af90e07ef422e6d841463d4b2 - languageName: node - linkType: hard - "@types/markdown-it@npm:^12.2.3": version: 12.2.3 resolution: "@types/markdown-it@npm:12.2.3" @@ -21379,7 +21372,6 @@ __metadata: "@backstage/plugin-events-backend": "workspace:^" "@backstage/plugin-events-node": "workspace:^" "@backstage/plugin-graphql-backend": "workspace:^" - "@backstage/plugin-incremental-ingestion-backend": "workspace:^" "@backstage/plugin-jenkins-backend": "workspace:^" "@backstage/plugin-kafka-backend": "workspace:^" "@backstage/plugin-kubernetes-backend": "workspace:^" From 8a235456e9d420931e93afd80392c6f1b8c7c25d Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 17 Nov 2022 08:13:50 -0800 Subject: [PATCH 37/54] Fix defect in DB manager identified internally Signed-off-by: Damon Kaswell --- .../src/database/IncrementalIngestionDatabaseManager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts index cc791a6f88..3620c067b8 100644 --- a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts +++ b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts @@ -254,7 +254,7 @@ export class IncrementalIngestionDatabaseManager { next_action_at, ingestion_completed_at: new Date(), status: 'resting', - completion_ticket: v4(), + completion_ticket: 'open', }); return { provider, ingestionsDeleted, marksDeleted, markEntitiesDeleted }; @@ -375,7 +375,7 @@ export class IncrementalIngestionDatabaseManager { next_action_at, ingestion_completed_at: new Date(), status: 'resting', - completion_ticket: v4(), + completion_ticket: 'open', }); } From 6608e0831742bbd8e3f3bed329e74cdc2fd9da15 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 17 Nov 2022 10:54:05 -0800 Subject: [PATCH 38/54] Consolidate migrations and use default schema Signed-off-by: Damon Kaswell --- .../20220906161750_add-ingestion-indexes.js | 70 ------------ ...21019897897_add-unique-active-ingestion.js | 100 ------------------ ...3125155_init.js => 20221116073152_init.js} | 58 +++++++--- .../IncrementalIngestionDatabaseManager.ts | 92 ++++++++-------- .../src/database/migrations.ts | 5 +- .../src/types.ts | 6 +- 6 files changed, 91 insertions(+), 240 deletions(-) delete mode 100644 plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js delete mode 100644 plugins/incremental-ingestion-backend/migrations/20221019897897_add-unique-active-ingestion.js rename plugins/incremental-ingestion-backend/migrations/{20220613125155_init.js => 20221116073152_init.js} (71%) diff --git a/plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js b/plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js deleted file mode 100644 index 779aa2241b..0000000000 --- a/plugins/incremental-ingestion-backend/migrations/20220906161750_add-ingestion-indexes.js +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * @param { import("knex").Knex } knex - */ -exports.up = async function up(knex) { - const schema = () => knex.schema.withSchema('ingestion'); - - await knex.raw( - `CREATE INDEX IF NOT EXISTS increment_ingestion_provider_name_idx ON public.final_entities ((final_entity::json #>> '{metadata, annotations, backstage.io/incremental-provider-name}'));`, - ); - - await knex.raw(`DROP VIEW IF EXISTS ingestion.current_entities`); - - await schema().alterTable('ingestions', t => { - t.primary('id'); - t.index('provider_name', 'ingestion_provider_name_idx'); - }); - - await schema().alterTable('ingestion_marks', t => { - t.primary('id'); - t.index('ingestion_id', 'ingestion_mark_ingestion_id_idx'); - }); - - await schema().alterTable('ingestion_mark_entities', t => { - t.primary('id'); - t.index('ingestion_mark_id', 'ingestion_mark_entity_ingestion_mark_id_idx'); - }); -}; - -/** - * @param { import("knex").Knex } knex - */ -exports.down = async function down(knex) { - const schema = () => knex.schema.withSchema('ingestion'); - - await schema().alterTable('ingestions', t => { - t.dropIndex('provider_name', 'ingestion_provider_name_idx'); - t.dropPrimary('id'); - }); - - await schema().alterTable('ingestion_marks', t => { - t.dropIndex('ingestion_id', 'ingestion_mark_ingestion_id_idx'); - t.dropPrimary('id'); - }); - - await schema().alterTable('ingestions_mark_entities', t => { - t.dropIndex( - 'ingestion_mark_id', - 'ingestion_mark_entity_ingestion_mark_id_idx', - ); - t.dropPrimary('id'); - }); - - await knex.raw(`DROP INDEX increment_ingestion_provider_name_idx;`); -}; diff --git a/plugins/incremental-ingestion-backend/migrations/20221019897897_add-unique-active-ingestion.js b/plugins/incremental-ingestion-backend/migrations/20221019897897_add-unique-active-ingestion.js deleted file mode 100644 index e2a55e323f..0000000000 --- a/plugins/incremental-ingestion-backend/migrations/20221019897897_add-unique-active-ingestion.js +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const { v4: uuidv4 } = require('uuid'); - -/** - * @param { import("knex").Knex } knex - */ -exports.up = async function up(knex) { - const schema = () => knex.schema.withSchema('ingestion'); - - await knex.transaction(async tx => { - const providers = await tx('ingestion.ingestions').distinct( - 'provider_name', - ); - for (const provider of providers) { - const valid = await tx('ingestion.ingestions') - .where('provider_name', provider) - .andWhere('rest_completed_at', null) - .first(); - const invalid = []; - - const rows = await tx('ingestion.ingestions') - .where('provider_name', provider) - .andWhere('rest_completed_at', null); - for (const row of rows) { - if (row.id !== valid.id) { - invalid.push(row.id); - } - } - - if (invalid.length > 0) { - await tx('ingestion.ingestions').delete().whereIn('id', invalid); - await tx('ingestion.ingestion_mark_entities') - .delete() - .whereIn( - 'ingestion_mark_id', - tx('ingestion.ingestion_marks') - .select('id') - .whereIn('ingestion_id', invalid), - ); - await tx('ingestion.ingestion_marks') - .delete() - .whereIn('ingestion_id', invalid); - } - } - }); - - await schema().alterTable('ingestions', t => { - t.string('completion_ticket'); - }); - - await knex.transaction(async tx => { - await tx('ingestion.ingestions') - .update('completion_ticket', 'open') - .where('rest_completed_at', null); - - const rows = await tx('ingestion.ingestions').whereNot( - 'rest_completed_at', - null, - ); - for (const row of rows) { - await tx('ingestion.ingestions') - .update('completion_ticket', uuidv4()) - .where('id', row.id); - } - }); - - await schema().alterTable('ingestions', t => { - t.string('completion_ticket').notNullable().alter(); - t.unique(['provider_name', 'completion_ticket'], { - indexName: 'ingestion_composite_index', - deferrable: 'deferred', - }); - }); -}; - -/** - * @param { import("knex").Knex } knex - */ -exports.down = async function down(knex) { - const schema = () => knex.schema.withSchema('ingestion'); - - await schema().alterTable('ingestions', t => { - t.dropUnique(['provider_name', 'completion_ticket']); - }); -}; diff --git a/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js b/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js similarity index 71% rename from plugins/incremental-ingestion-backend/migrations/20220613125155_init.js rename to plugins/incremental-ingestion-backend/migrations/20221116073152_init.js index b31bc09cf0..1c79fd5a84 100644 --- a/plugins/incremental-ingestion-backend/migrations/20220613125155_init.js +++ b/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js @@ -18,19 +18,19 @@ * @param { import("knex").Knex } knex */ exports.up = async function up(knex) { - await knex.raw(` -CREATE VIEW ingestion.current_entities as SELECT - FORMAT('%s:%s/%s', - LOWER(final_entity::json #>> '{kind}'), - LOWER(final_entity::json #>> '{metadata, namespace}'), - LOWER(final_entity::json #>> '{metadata, name}')) as ref, - final_entity::json #>> '{metadata, annotations, backstage.io/incremental-provider-name}' as provider_name, - final_entity FROM public.final_entities; -`); + /** + * Create an index on the final_entities table for the provider name + * annotation + */ - const schema = () => knex.schema.withSchema('ingestion'); + await knex.raw( + `CREATE INDEX IF NOT EXISTS increment_ingestion_provider_name_idx ON public.final_entities ((final_entity::json #>> '{metadata, annotations, backstage.io/incremental-provider-name}'));`, + ); - await schema().createTable('ingestions', table => { + /** + * Sets up the ingestions table + */ + await knex.schema.createTable('ingestions', table => { table.comment('Tracks ingestion streams for very large data sets'); table @@ -81,9 +81,26 @@ CREATE VIEW ingestion.current_entities as SELECT table .timestamp('rest_completed_at') .comment('when did the rest period actually end'); + + table + .string('completion_ticket') + .notNullable() + .comment('indicates whether the ticket is still open or stamped complete') }); - await schema().createTable('ingestion_marks', table => { + await knex.schema.alterTable('ingestions', t => { + t.primary('id'); + t.index('provider_name', 'ingestion_provider_name_idx'); + t.unique(['provider_name', 'completion_ticket'], { + indexName: 'ingestion_composite_index', + deferrable: 'deferred', + }); + }); + + /** + * Sets up the ingestion_marks table + */ + await knex.schema.createTable('ingestion_marks', table => { table.comment('tracks each step of an iterative ingestion'); table @@ -110,6 +127,14 @@ CREATE VIEW ingestion.current_entities as SELECT table.timestamp('created_at').defaultTo(knex.fn.now()); }); + await schema().alterTable('ingestion_marks', t => { + t.primary('id'); + t.index('ingestion_id', 'ingestion_mark_ingestion_id_idx'); + }); + + /** + * Set up the ingestion_mark_entities table + */ await schema().createTable('ingestion_mark_entities', table => { table.comment( 'tracks the entities recorded in each step of an iterative ingestion', @@ -132,15 +157,20 @@ CREATE VIEW ingestion.current_entities as SELECT .notNullable() .comment('the entity reference of the marked entity'); }); + + await schema().alterTable('ingestion_mark_entities', t => { + t.primary('id'); + t.index('ingestion_mark_id', 'ingestion_mark_entity_ingestion_mark_id_idx'); + }); }; /** * @param { import("knex").Knex } knex */ exports.down = async function down(knex) { - const schema = () => knex.schema.withSchema('ingestion'); - await schema().dropView('current_entities'); + const schema = () => knex.schema.withSchema('public'); await schema().dropTable('ingestion_mark_entities'); await schema().dropTable('ingestion_marks'); await schema().dropTable('ingestions'); + await knex.raw(`DROP INDEX increment_ingestion_provider_name_idx;`); }; diff --git a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts index 3620c067b8..1a7a880901 100644 --- a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts +++ b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts @@ -43,7 +43,7 @@ export class IncrementalIngestionDatabaseManager { async updateIngestionRecordById(options: IngestionRecordUpdate) { await this.client.transaction(async tx => { const { ingestionId, update } = options; - await tx('ingestion.ingestions').where('id', ingestionId).update(update); + await tx('ingestions').where('id', ingestionId).update(update); }); } @@ -57,7 +57,7 @@ export class IncrementalIngestionDatabaseManager { update: Partial, ) { await this.client.transaction(async tx => { - await tx('ingestion.ingestions') + await tx('ingestions') .where('provider_name', provider) .andWhere('completion_ticket', 'open') .update(update); @@ -65,12 +65,12 @@ export class IncrementalIngestionDatabaseManager { } /** - * Performs an insert into the `ingestion.ingestions` table with the supplied values. + * Performs an insert into the `ingestions` table with the supplied values. * @param record - IngestionUpsertIFace */ async insertIngestionRecord(record: IngestionUpsertIFace) { await this.client.transaction(async tx => { - await tx('ingestion.ingestions').insert(record); + await tx('ingestions').insert(record); }); } @@ -87,7 +87,7 @@ export class IncrementalIngestionDatabaseManager { let deleted = 0; for (const chunk of chunks) { - const chunkDeleted = await tx('ingestion.ingestion_mark_entities') + const chunkDeleted = await tx('ingestion_mark_entities') .delete() .whereIn( 'id', @@ -106,7 +106,7 @@ export class IncrementalIngestionDatabaseManager { */ async getCurrentIngestionRecord(provider: string) { return await this.client.transaction(async tx => { - const record = await tx('ingestion.ingestions') + const record = await tx('ingestions') .where('provider_name', provider) .andWhere('completion_ticket', 'open') .first(); @@ -122,32 +122,32 @@ export class IncrementalIngestionDatabaseManager { */ async clearFinishedIngestions(provider: string) { return await this.client.transaction(async tx => { - const markEntitiesDeleted = await tx('ingestion.ingestion_mark_entities') + const markEntitiesDeleted = await tx('ingestion_mark_entities') .delete() .whereIn( 'ingestion_mark_id', - tx('ingestion.ingestion_marks') + tx('ingestion_marks') .select('id') .whereIn( 'ingestion_id', - tx('ingestion.ingestions') + tx('ingestions') .select('id') .where('provider_name', provider) .andWhereNot('completion_ticket', 'open'), ), ); - const marksDeleted = await tx('ingestion.ingestion_marks') + const marksDeleted = await tx('ingestion_marks') .delete() .whereIn( 'ingestion_id', - tx('ingestion.ingestions') + tx('ingestions') .select('id') .where('provider_name', provider) .andWhereNot('completion_ticket', 'open'), ); - const ingestionsDeleted = await tx('ingestion.ingestions') + const ingestionsDeleted = await tx('ingestions') .delete() .where('provider_name', provider) .andWhereNot('completion_ticket', 'open'); @@ -171,24 +171,20 @@ export class IncrementalIngestionDatabaseManager { */ async clearDuplicateIngestions(ingestionId: string, provider: string) { await this.client.transaction(async tx => { - const invalid = await tx('ingestion.ingestions') + const invalid = await tx('ingestions') .where('provider_name', provider) .andWhere('rest_completed_at', null) .andWhereNot('id', ingestionId); if (invalid.length > 0) { - await tx('ingestion.ingestions').delete().whereIn('id', invalid); - await tx('ingestion.ingestion_mark_entities') + await tx('ingestions').delete().whereIn('id', invalid); + await tx('ingestion_mark_entities') .delete() .whereIn( 'ingestion_mark_id', - tx('ingestion.ingestion_marks') - .select('id') - .whereIn('ingestion_id', invalid), + tx('ingestion_marks').select('id').whereIn('ingestion_id', invalid), ); - await tx('ingestion.ingestion_marks') - .delete() - .whereIn('ingestion_id', invalid); + await tx('ingestion_marks').delete().whereIn('ingestion_id', invalid); } }); } @@ -201,13 +197,13 @@ export class IncrementalIngestionDatabaseManager { */ async purgeAndResetProvider(provider: string) { return await this.client.transaction(async tx => { - const ingestionIDs: { id: string }[] = await tx('ingestion.ingestions') + const ingestionIDs: { id: string }[] = await tx('ingestions') .select('id') .where('provider_name', provider); const markIDs: { id: string }[] = ingestionIDs.length > 0 - ? await tx('ingestion.ingestion_marks') + ? await tx('ingestion_marks') .select('id') .whereIn( 'ingestion_id', @@ -217,7 +213,7 @@ export class IncrementalIngestionDatabaseManager { const markEntityIDs: { id: string }[] = markIDs.length > 0 - ? await tx('ingestion.ingestion_mark_entities') + ? await tx('ingestion_mark_entities') .select('id') .whereIn( 'ingestion_mark_id', @@ -232,7 +228,7 @@ export class IncrementalIngestionDatabaseManager { const marksDeleted = markIDs.length > 0 - ? await tx('ingestion.ingestion_marks') + ? await tx('ingestion_marks') .delete() .whereIn( 'ingestion_id', @@ -240,7 +236,7 @@ export class IncrementalIngestionDatabaseManager { ) : 0; - const ingestionsDeleted = await tx('ingestion.ingestions') + const ingestionsDeleted = await tx('ingestions') .delete() .where('provider_name', provider); @@ -310,14 +306,14 @@ export class IncrementalIngestionDatabaseManager { ) .whereNotIn( 'entity_ref', - tx('ingestion.ingestion_marks') + tx('ingestion_marks') .join( - 'ingestion.ingestion_mark_entities', - 'ingestion.ingestion_marks.id', - 'ingestion.ingestion_mark_entities.ingestion_mark_id', + 'ingestion_mark_entities', + 'ingestion_marks.id', + 'ingestion_mark_entities.ingestion_mark_id', ) - .select('ingestion.ingestion_mark_entities.ref') - .where('ingestion.ingestion_marks.ingestion_id', ingestionId), + .select('ingestion_mark_entities.ref') + .where('ingestion_marks.ingestion_id', ingestionId), ); return removed.map(entity => { return { entity: JSON.parse(entity.entity) }; @@ -332,7 +328,7 @@ export class IncrementalIngestionDatabaseManager { async healthcheck() { return await this.client.transaction(async tx => { const records = await tx<{ id: string; provider_name: string }>( - 'ingestion.ingestions', + 'ingestions', ) .distinct('id', 'provider_name') .where('rest_completed_at', null); @@ -352,9 +348,9 @@ export class IncrementalIngestionDatabaseManager { /** * Purges the following tables: - * * `ingestion.ingestions` - * * `ingestion.ingestion_marks` - * * `ingestion.ingestion_mark_entities` + * * `ingestions` + * * `ingestion_marks` + * * `ingestion_mark_entities` * * This function leaves the ingestions table with all providers in a paused state. * @returns Results from cleaning up all ingestion tables. @@ -362,7 +358,7 @@ export class IncrementalIngestionDatabaseManager { async cleanupProviders() { const providers = await this.listProviders(); - const ingestionsDeleted = await this.purgeTable('ingestion.ingestions'); + const ingestionsDeleted = await this.purgeTable('ingestions'); const next_action_at = new Date(); next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000); @@ -379,11 +375,9 @@ export class IncrementalIngestionDatabaseManager { }); } - const ingestionMarksDeleted = await this.purgeTable( - 'ingestion.ingestion_marks', - ); + const ingestionMarksDeleted = await this.purgeTable('ingestion_marks'); const markEntitiesDeleted = await this.purgeTable( - 'ingestion.ingestion_mark_entities', + 'ingestion_mark_entities', ); return { ingestionsDeleted, ingestionMarksDeleted, markEntitiesDeleted }; @@ -512,13 +506,13 @@ export class IncrementalIngestionDatabaseManager { } /** - * Returns the last record from `ingestion.ingestion_marks` for the supplied ingestionId. + * Returns the last record from `ingestion_marks` for the supplied ingestionId. * @param ingestionId - string * @returns MarkRecord | undefined */ async getLastMark(ingestionId: string) { return await this.client.transaction(async tx => { - const mark = await tx('ingestion.ingestion_marks') + const mark = await tx('ingestion_marks') .where('ingestion_id', ingestionId) .orderBy('sequence', 'desc') .first(); @@ -528,7 +522,7 @@ export class IncrementalIngestionDatabaseManager { async getAllMarks(ingestionId: string) { return await this.client.transaction(async tx => { - const marks = await tx('ingestion.ingestion_marks') + const marks = await tx('ingestion_marks') .where('ingestion_id', ingestionId) .orderBy('sequence', 'desc'); return marks; @@ -536,23 +530,23 @@ export class IncrementalIngestionDatabaseManager { } /** - * Performs an insert into the `ingestion.ingestion_marks` table with the supplied values. + * Performs an insert into the `ingestion_marks` table with the supplied values. * @param options - MarkRecordInsert */ async createMark(options: MarkRecordInsert) { const { record } = options; await this.client.transaction(async tx => { - await tx('ingestion.ingestion_marks').insert(record); + await tx('ingestion_marks').insert(record); }); } /** - * Performs an upsert to the `ingestion.ingestion_mark_entities` table for all deferred entities. + * Performs an upsert to the `ingestion_mark_entities` table for all deferred entities. * @param markId - string * @param entities - DeferredEntity[] */ async createMarkEntities(markId: string, entities: DeferredEntity[]) { await this.client.transaction(async tx => { - await tx('ingestion.ingestion_mark_entities').insert( + await tx('ingestion_mark_entities').insert( entities.map(entity => ({ id: v4(), ingestion_mark_id: markId, @@ -580,7 +574,7 @@ export class IncrementalIngestionDatabaseManager { async listProviders() { return await this.client.transaction(async tx => { const providers = await tx<{ provider_name: string }>( - 'ingestion.ingestions', + 'ingestions', ).distinct('provider_name'); return providers.map(entry => entry.provider_name); }); diff --git a/plugins/incremental-ingestion-backend/src/database/migrations.ts b/plugins/incremental-ingestion-backend/src/database/migrations.ts index 4866e2dc98..45c8964054 100644 --- a/plugins/incremental-ingestion-backend/src/database/migrations.ts +++ b/plugins/incremental-ingestion-backend/src/database/migrations.ts @@ -13,20 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { resolvePackagePath } from '@backstage/backend-common'; import { Knex } from 'knex'; -/** @public */ export async function applyDatabaseMigrations(knex: Knex): Promise { const migrationsDir = resolvePackagePath( '@backstage/plugin-incremental-ingestion-backend', 'migrations', ); - await knex.raw('CREATE SCHEMA IF NOT EXISTS ingestion;'); - await knex.migrate.latest({ - schemaName: 'ingestion', directory: migrationsDir, }); } diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/incremental-ingestion-backend/src/types.ts index 2671752ad6..3f1971e97d 100644 --- a/plugins/incremental-ingestion-backend/src/types.ts +++ b/plugins/incremental-ingestion-backend/src/types.ts @@ -164,7 +164,7 @@ export interface IterationEngineOptions { } /** - * The shape of data inserted into or updated in the `ingestion.ingestions` table. + * The shape of data inserted into or updated in the `ingestions` table. * * @public */ @@ -234,7 +234,7 @@ export interface IngestionRecordUpdate { } /** - * The expected response from the `ingestion.ingestion_marks` table. + * The expected response from the `ingestion_marks` table. * * @public */ @@ -247,7 +247,7 @@ export interface MarkRecord { } /** - * The expected response from the `ingestion.ingestions` table. + * The expected response from the `ingestions` table. * * @public */ From 58b627cbecaad783b6281930176f69b18286b4a0 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 17 Nov 2022 10:56:26 -0800 Subject: [PATCH 39/54] Fix migrations Signed-off-by: Damon Kaswell --- .../migrations/20221116073152_init.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js b/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js index 1c79fd5a84..9e05d0a163 100644 --- a/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js +++ b/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js @@ -127,7 +127,7 @@ exports.up = async function up(knex) { table.timestamp('created_at').defaultTo(knex.fn.now()); }); - await schema().alterTable('ingestion_marks', t => { + await knex.schema.alterTable('ingestion_marks', t => { t.primary('id'); t.index('ingestion_id', 'ingestion_mark_ingestion_id_idx'); }); @@ -135,7 +135,7 @@ exports.up = async function up(knex) { /** * Set up the ingestion_mark_entities table */ - await schema().createTable('ingestion_mark_entities', table => { + await knex.schema.createTable('ingestion_mark_entities', table => { table.comment( 'tracks the entities recorded in each step of an iterative ingestion', ); @@ -158,7 +158,7 @@ exports.up = async function up(knex) { .comment('the entity reference of the marked entity'); }); - await schema().alterTable('ingestion_mark_entities', t => { + await knex.schema.alterTable('ingestion_mark_entities', t => { t.primary('id'); t.index('ingestion_mark_id', 'ingestion_mark_entity_ingestion_mark_id_idx'); }); @@ -168,9 +168,8 @@ exports.up = async function up(knex) { * @param { import("knex").Knex } knex */ exports.down = async function down(knex) { - const schema = () => knex.schema.withSchema('public'); - await schema().dropTable('ingestion_mark_entities'); - await schema().dropTable('ingestion_marks'); - await schema().dropTable('ingestions'); + await knex.schema.dropTable('ingestion_mark_entities'); + await knex.schema.dropTable('ingestion_marks'); + await knex.schema.dropTable('ingestions'); await knex.raw(`DROP INDEX increment_ingestion_provider_name_idx;`); }; From 5b850ac0229545e1cd3b67f465fc99d0686303b9 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 17 Nov 2022 11:20:20 -0800 Subject: [PATCH 40/54] Some type cleanup Signed-off-by: Damon Kaswell --- .../IncrementalIngestionDatabaseManager.ts | 10 +++---- .../src/index.ts | 2 +- .../src/types.ts | 30 +++++++------------ 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts index 1a7a880901..6a8a69b886 100644 --- a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts +++ b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts @@ -23,7 +23,7 @@ import { INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, IngestionRecord, IngestionRecordUpdate, - IngestionUpsertIFace, + IngestionUpsert, MarkRecord, MarkRecordInsert, } from '../types'; @@ -54,7 +54,7 @@ export class IncrementalIngestionDatabaseManager { */ async updateIngestionRecordByProvider( provider: string, - update: Partial, + update: Partial, ) { await this.client.transaction(async tx => { await tx('ingestions') @@ -68,7 +68,7 @@ export class IncrementalIngestionDatabaseManager { * Performs an insert into the `ingestions` table with the supplied values. * @param record - IngestionUpsertIFace */ - async insertIngestionRecord(record: IngestionUpsertIFace) { + async insertIngestionRecord(record: IngestionUpsert) { await this.client.transaction(async tx => { await tx('ingestions').insert(record); }); @@ -455,7 +455,7 @@ export class IncrementalIngestionDatabaseManager { * @param message - string (optional) */ async setProviderCanceling(ingestionId: string, message?: string) { - const update: Partial = { + const update: Partial = { next_action: 'cancel', last_error: message ? message : undefined, next_action_at: new Date(), @@ -580,7 +580,7 @@ export class IncrementalIngestionDatabaseManager { }); } - async updateByName(provider: string, update: Partial) { + async updateByName(provider: string, update: Partial) { await this.updateIngestionRecordByProvider(provider, update); } } diff --git a/plugins/incremental-ingestion-backend/src/index.ts b/plugins/incremental-ingestion-backend/src/index.ts index da50c6b37f..142b6ba836 100644 --- a/plugins/incremental-ingestion-backend/src/index.ts +++ b/plugins/incremental-ingestion-backend/src/index.ts @@ -21,7 +21,7 @@ export type { IncrementalEntityProviderOptions, IngestionRecord, IngestionRecordUpdate, - IngestionUpsertIFace, + IngestionUpsert as IngestionUpsertIFace, MarkRecord, MarkRecordInsert, INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/incremental-ingestion-backend/src/types.ts index 3f1971e97d..138b9ba0ee 100644 --- a/plugins/incremental-ingestion-backend/src/types.ts +++ b/plugins/incremental-ingestion-backend/src/types.ts @@ -89,23 +89,15 @@ export interface IncrementalEntityProvider { * * @public */ -export interface EntityIteratorResult { - /** - * Indicates whether there are any further pages of entities to - * ingest after this one. - */ - done: boolean; - - /** - * A value that marks the page of entities after this one. It will - * be used to pass into the following invocation of `next()` - */ - cursor?: T; - - /** - * The entities to ingest. - */ +export type EntityIteratorResult = +| { + done: false; + entities: DeferredEntity[]; + cursor: T; +} | { + done: true; entities?: DeferredEntity[]; + cursor?: T; } /** @public */ @@ -168,7 +160,7 @@ export interface IterationEngineOptions { * * @public */ -export interface IngestionUpsertIFace { +export interface IngestionUpsert { /** * The ingestion record id. */ @@ -230,7 +222,7 @@ export interface IngestionUpsertIFace { */ export interface IngestionRecordUpdate { ingestionId: string; - update: Partial; + update: Partial; } /** @@ -251,7 +243,7 @@ export interface MarkRecord { * * @public */ -export interface IngestionRecord extends IngestionUpsertIFace { +export interface IngestionRecord extends IngestionUpsert { id: string; next_action_at: Date; /** From 25da47e68d8583046f449a41328dcb56c655b5e1 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 17 Nov 2022 11:47:10 -0800 Subject: [PATCH 41/54] Updated API report Signed-off-by: Damon Kaswell --- .../incremental-ingestion-backend/api-report.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/plugins/incremental-ingestion-backend/api-report.md b/plugins/incremental-ingestion-backend/api-report.md index d843aad1a3..5d2f66ac19 100644 --- a/plugins/incremental-ingestion-backend/api-report.md +++ b/plugins/incremental-ingestion-backend/api-report.md @@ -19,11 +19,17 @@ import { Router } from 'express'; import type { UrlReader } from '@backstage/backend-common'; // @public -export interface EntityIteratorResult { - cursor?: T; - done: boolean; - entities?: DeferredEntity[]; -} +export type EntityIteratorResult = + | { + done: false; + entities: DeferredEntity[]; + cursor: T; + } + | { + done: true; + entities?: DeferredEntity[]; + cursor?: T; + }; // @public export const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = From b6d0c26902c97a5a18063e6bc5ce5d7d61ffe058 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 17 Nov 2022 13:23:52 -0800 Subject: [PATCH 42/54] Use foreign key constraints Signed-off-by: Damon Kaswell --- .../migrations/20221116073152_init.js | 6 +++++ .../src/engine/IncrementalIngestionEngine.ts | 24 ++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js b/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js index 9e05d0a163..edbe041afe 100644 --- a/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js +++ b/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js @@ -111,6 +111,9 @@ exports.up = async function up(knex) { table .uuid('ingestion_id') .notNullable() + .references('id') + .inTable('ingestions') + .onDelete('CASCADE') .comment('The id of the ingestion in which this mark took place'); table @@ -148,6 +151,9 @@ exports.up = async function up(knex) { table .uuid('ingestion_mark_id') .notNullable() + .references('id') + .inTable('ingestion_marks') + .onDelete('CASCADE') .comment( 'Every time a mark happens during an ingestion, there are a list of entities marked.', ); diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts index 6d35f2db4b..c370390fd6 100644 --- a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts @@ -280,12 +280,24 @@ export class IncrementalIngestionEngine implements IterationEngine { }, })) ?? []; - const removed: DeferredEntity[] = done - ? [] - : await this.manager.computeRemoved( - this.options.provider.getProviderName(), - id, - ); + const removed: DeferredEntity[] = []; + + let doComputeRemoved = false; + + if (!done) { + doComputeRemoved = true; + } else { + if (entities && entities.length > 0) { + doComputeRemoved = true; + } + } + + if(doComputeRemoved) { + removed.push(...await this.manager.computeRemoved( + this.options.provider.getProviderName(), + id, + )); + } await this.options.connection.applyMutation({ type: 'delta', From d6dde4664f38a5a9cd43f7cd65023d4edeb8534d Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Mon, 21 Nov 2022 14:45:56 -0800 Subject: [PATCH 43/54] Fixes for migrations and routes Signed-off-by: Damon Kaswell --- .../migrations/20221116073152_init.js | 19 +++++++---------- .../IncrementalIngestionDatabaseManager.ts | 12 ++++++----- .../src/database/migrations.ts | 2 ++ .../src/database/tables.ts | 17 +++++++++++++++ .../src/router/paths.ts | 19 +++++++++++++++++ .../src/{ => router}/routes.ts | 21 ++++++++++--------- .../src/service/IncrementalCatalogBuilder.ts | 2 +- 7 files changed, 65 insertions(+), 27 deletions(-) create mode 100644 plugins/incremental-ingestion-backend/src/database/tables.ts create mode 100644 plugins/incremental-ingestion-backend/src/router/paths.ts rename plugins/incremental-ingestion-backend/src/{ => router}/routes.ts (90%) diff --git a/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js b/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js index edbe041afe..7fab18b186 100644 --- a/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js +++ b/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js @@ -18,15 +18,6 @@ * @param { import("knex").Knex } knex */ exports.up = async function up(knex) { - /** - * Create an index on the final_entities table for the provider name - * annotation - */ - - await knex.raw( - `CREATE INDEX IF NOT EXISTS increment_ingestion_provider_name_idx ON public.final_entities ((final_entity::json #>> '{metadata, annotations, backstage.io/incremental-provider-name}'));`, - ); - /** * Sets up the ingestions table */ @@ -57,6 +48,7 @@ exports.up = async function up(knex) { table .timestamp('next_action_at') + .notNullable() .defaultTo(knex.fn.now()) .comment('the moment in time at which point ingestion can begin again'); @@ -66,11 +58,13 @@ exports.up = async function up(knex) { table .integer('attempts') + .notNullable() .defaultTo(0) .comment('how many attempts have been made to burst without success'); table .timestamp('created_at') + .notNullable() .defaultTo(knex.fn.now()) .comment('when did this ingestion actually begin'); @@ -124,10 +118,14 @@ exports.up = async function up(knex) { table .integer('sequence') + .notNullable() .defaultTo(0) .comment('what is the order of this mark'); - table.timestamp('created_at').defaultTo(knex.fn.now()); + table + .timestamp('created_at') + .notNullable() + .defaultTo(knex.fn.now()); }); await knex.schema.alterTable('ingestion_marks', t => { @@ -177,5 +175,4 @@ exports.down = async function down(knex) { await knex.schema.dropTable('ingestion_mark_entities'); await knex.schema.dropTable('ingestion_marks'); await knex.schema.dropTable('ingestions'); - await knex.raw(`DROP INDEX increment_ingestion_provider_name_idx;`); }; diff --git a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts index 6a8a69b886..891f94b537 100644 --- a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts +++ b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts @@ -300,10 +300,7 @@ export class IncrementalIngestionDatabaseManager { 'refresh_state.entity_id', 'final_entities.entity_id', ) - .whereRaw( - `((final_entity::json #>> '{metadata, annotations, ${INCREMENTAL_ENTITY_PROVIDER_ANNOTATION}}')) = ?`, - [provider], - ) + .join('search', 'search.entity_id', 'final_entities.entity_id') .whereNotIn( 'entity_ref', tx('ingestion_marks') @@ -314,7 +311,12 @@ export class IncrementalIngestionDatabaseManager { ) .select('ingestion_mark_entities.ref') .where('ingestion_marks.ingestion_id', ingestionId), - ); + ) + .andWhere( + 'search.key', + `metadata.annotations.${INCREMENTAL_ENTITY_PROVIDER_ANNOTATION}`, + ) + .andWhere('search.value', provider); return removed.map(entity => { return { entity: JSON.parse(entity.entity) }; }); diff --git a/plugins/incremental-ingestion-backend/src/database/migrations.ts b/plugins/incremental-ingestion-backend/src/database/migrations.ts index 45c8964054..58ccd7c8c5 100644 --- a/plugins/incremental-ingestion-backend/src/database/migrations.ts +++ b/plugins/incremental-ingestion-backend/src/database/migrations.ts @@ -16,6 +16,7 @@ import { resolvePackagePath } from '@backstage/backend-common'; import { Knex } from 'knex'; +import { DB_MIGRATIONS_TABLE } from './tables'; export async function applyDatabaseMigrations(knex: Knex): Promise { const migrationsDir = resolvePackagePath( @@ -25,5 +26,6 @@ export async function applyDatabaseMigrations(knex: Knex): Promise { await knex.migrate.latest({ directory: migrationsDir, + tableName: DB_MIGRATIONS_TABLE, }); } diff --git a/plugins/incremental-ingestion-backend/src/database/tables.ts b/plugins/incremental-ingestion-backend/src/database/tables.ts new file mode 100644 index 0000000000..73542abb6b --- /dev/null +++ b/plugins/incremental-ingestion-backend/src/database/tables.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const DB_MIGRATIONS_TABLE = 'plugin_incremental_ingestion_backend__knex_migrations'; diff --git a/plugins/incremental-ingestion-backend/src/router/paths.ts b/plugins/incremental-ingestion-backend/src/router/paths.ts new file mode 100644 index 0000000000..1be540fc10 --- /dev/null +++ b/plugins/incremental-ingestion-backend/src/router/paths.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const PROVIDER_CLEANUP = '/providers/cleanup'; +export const PROVIDER_HEALTH = '/providers/health'; +export const PROVIDER_BASE_PATH = '/providers/:provider'; diff --git a/plugins/incremental-ingestion-backend/src/routes.ts b/plugins/incremental-ingestion-backend/src/router/routes.ts similarity index 90% rename from plugins/incremental-ingestion-backend/src/routes.ts rename to plugins/incremental-ingestion-backend/src/router/routes.ts index 2f51ef6499..8607a4d302 100644 --- a/plugins/incremental-ingestion-backend/src/routes.ts +++ b/plugins/incremental-ingestion-backend/src/router/routes.ts @@ -17,7 +17,8 @@ import { errorHandler } from '@backstage/backend-common'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; -import { IncrementalIngestionDatabaseManager } from './'; +import { IncrementalIngestionDatabaseManager } from '..'; +import { PROVIDER_BASE_PATH, PROVIDER_CLEANUP, PROVIDER_HEALTH } from './paths'; /** @public */ export const createIncrementalProviderRouter = async ( @@ -28,7 +29,7 @@ export const createIncrementalProviderRouter = async ( router.use(express.json()); // Get the overall health of all incremental providers - router.get('/health', async (_, res) => { + router.get(PROVIDER_HEALTH, async (_, res) => { const records = await manager.healthcheck(); const providers = records.map(record => record.provider_name); const duplicates = [ @@ -43,13 +44,13 @@ export const createIncrementalProviderRouter = async ( }); // Clean up and pause all providers - router.delete('/cleanup', async (_, res) => { + router.post(PROVIDER_CLEANUP, async (_, res) => { const result = await manager.cleanupProviders(); res.json(result); }); // Get basic status of the provider - router.get('/:provider', async (req, res) => { + router.get(PROVIDER_BASE_PATH, async (req, res) => { const { provider } = req.params; const record = await manager.getCurrentIngestionRecord(provider); if (record) { @@ -84,7 +85,7 @@ export const createIncrementalProviderRouter = async ( }); // Trigger the provider's next action - router.put('/:provider', async (req, res) => { + router.put(PROVIDER_BASE_PATH, async (req, res) => { const { provider } = req.params; const record = await manager.getCurrentIngestionRecord(provider); if (record) { @@ -111,7 +112,7 @@ export const createIncrementalProviderRouter = async ( }); // Start a brand-new ingestion cycle for the provider - router.post('/:provider', async (req, res) => { + router.post(PROVIDER_BASE_PATH, async (req, res) => { const { provider } = req.params; const record = await manager.getCurrentIngestionRecord(provider); @@ -144,7 +145,7 @@ export const createIncrementalProviderRouter = async ( }); // Stop the provider and pause it for 24 hours - router.post('/:provider/cancel', async (req, res) => { + router.post(`${PROVIDER_BASE_PATH}/cancel`, async (req, res) => { const { provider } = req.params; const record = await manager.getCurrentIngestionRecord(provider); if (record) { @@ -178,14 +179,14 @@ export const createIncrementalProviderRouter = async ( }); // Wipe out all ingestion records for the provider and pause for 24 hours - router.delete('/:provider', async (req, res) => { + router.delete(PROVIDER_BASE_PATH, async (req, res) => { const { provider } = req.params; const result = await manager.purgeAndResetProvider(provider); res.json(result); }); // Get the ingestion marks for the current cycle - router.get('/:provider/marks', async (req, res) => { + router.get(`${PROVIDER_BASE_PATH}/marks`, async (req, res) => { const { provider } = req.params; const record = await manager.getCurrentIngestionRecord(provider); if (record) { @@ -213,7 +214,7 @@ export const createIncrementalProviderRouter = async ( } }); - router.delete('/:provider/marks', async (req, res) => { + router.delete(`${PROVIDER_BASE_PATH}/marks`, async (req, res) => { const { provider } = req.params; const deletions = await manager.clearFinishedIngestions(provider); diff --git a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts index 7bf39e160f..587c1d5834 100644 --- a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts @@ -24,7 +24,7 @@ import { Knex } from 'knex'; import { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine'; import { applyDatabaseMigrations } from '../database/migrations'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; -import { createIncrementalProviderRouter } from '../routes'; +import { createIncrementalProviderRouter } from '../router/routes'; class Deferred implements Promise { #resolve?: (value: T) => void; From 1a3ae5af1f9fad7e3a3c4e2c15dc6d1d646f076b Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Mon, 21 Nov 2022 17:06:16 -0800 Subject: [PATCH 44/54] Refine routes Signed-off-by: Damon Kaswell --- .../src/router/routes.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/plugins/incremental-ingestion-backend/src/router/routes.ts b/plugins/incremental-ingestion-backend/src/router/routes.ts index 8607a4d302..d26ace1de3 100644 --- a/plugins/incremental-ingestion-backend/src/router/routes.ts +++ b/plugins/incremental-ingestion-backend/src/router/routes.ts @@ -85,7 +85,7 @@ export const createIncrementalProviderRouter = async ( }); // Trigger the provider's next action - router.put(PROVIDER_BASE_PATH, async (req, res) => { + router.post(`${PROVIDER_BASE_PATH}/trigger`, async (req, res) => { const { provider } = req.params; const record = await manager.getCurrentIngestionRecord(provider); if (record) { @@ -111,18 +111,19 @@ export const createIncrementalProviderRouter = async ( } }); - // Start a brand-new ingestion cycle for the provider - router.post(PROVIDER_BASE_PATH, async (req, res) => { + // Start a brand-new ingestion cycle for the provider. + // (Cancel's the current run if active, or marks it complete if resting) + router.post(`${PROVIDER_BASE_PATH}/start`, async (req, res) => { const { provider } = req.params; const record = await manager.getCurrentIngestionRecord(provider); if (record) { - await manager.updateByName(provider, { - next_action: 'nothing (done)', - ingestion_completed_at: new Date(), - rest_completed_at: new Date(), - status: 'complete', - }); + const ingestionId = record.id; + if (record.status === 'resting') { + await manager.setProviderComplete(ingestionId); + } else { + await manager.setProviderCanceling(ingestionId); + } res.json({ success: true, message: `${provider}: Next cycle triggered.`, From 6404ad16aad16a5e759afa9bb0d2466a157e12aa Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Mon, 21 Nov 2022 18:02:15 -0800 Subject: [PATCH 45/54] Prettier formatting Signed-off-by: Damon Kaswell --- .../migrations/20221116073152_init.js | 9 +++--- .../src/database/tables.ts | 3 +- .../src/engine/IncrementalIngestionEngine.ts | 14 +++++---- .../src/service/IncrementalCatalogBuilder.ts | 8 +++-- .../src/types.ts | 29 ++++++++++--------- 5 files changed, 35 insertions(+), 28 deletions(-) diff --git a/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js b/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js index 7fab18b186..11357c50fd 100644 --- a/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js +++ b/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js @@ -79,7 +79,9 @@ exports.up = async function up(knex) { table .string('completion_ticket') .notNullable() - .comment('indicates whether the ticket is still open or stamped complete') + .comment( + 'indicates whether the ticket is still open or stamped complete', + ); }); await knex.schema.alterTable('ingestions', t => { @@ -122,10 +124,7 @@ exports.up = async function up(knex) { .defaultTo(0) .comment('what is the order of this mark'); - table - .timestamp('created_at') - .notNullable() - .defaultTo(knex.fn.now()); + table.timestamp('created_at').notNullable().defaultTo(knex.fn.now()); }); await knex.schema.alterTable('ingestion_marks', t => { diff --git a/plugins/incremental-ingestion-backend/src/database/tables.ts b/plugins/incremental-ingestion-backend/src/database/tables.ts index 73542abb6b..ee6cfe0ade 100644 --- a/plugins/incremental-ingestion-backend/src/database/tables.ts +++ b/plugins/incremental-ingestion-backend/src/database/tables.ts @@ -14,4 +14,5 @@ * limitations under the License. */ -export const DB_MIGRATIONS_TABLE = 'plugin_incremental_ingestion_backend__knex_migrations'; +export const DB_MIGRATIONS_TABLE = + 'plugin_incremental_ingestion_backend__knex_migrations'; diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts index c370390fd6..9b94c04c72 100644 --- a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts @@ -291,12 +291,14 @@ export class IncrementalIngestionEngine implements IterationEngine { doComputeRemoved = true; } } - - if(doComputeRemoved) { - removed.push(...await this.manager.computeRemoved( - this.options.provider.getProviderName(), - id, - )); + + if (doComputeRemoved) { + removed.push( + ...(await this.manager.computeRemoved( + this.options.provider.getProviderName(), + id, + )), + ); } await this.options.connection.applyMutation({ diff --git a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts index 587c1d5834..eea2c8c56d 100644 --- a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts @@ -30,8 +30,12 @@ class Deferred implements Promise { #resolve?: (value: T) => void; #reject?: (error: Error) => void; - get resolve() { return this.#resolve!; } - get reject() { return this.#reject!; } + get resolve() { + return this.#resolve!; + } + get reject() { + return this.#reject!; + } then: Promise['then']; catch: Promise['catch']; diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/incremental-ingestion-backend/src/types.ts index 138b9ba0ee..90f9a20d46 100644 --- a/plugins/incremental-ingestion-backend/src/types.ts +++ b/plugins/incremental-ingestion-backend/src/types.ts @@ -89,16 +89,17 @@ export interface IncrementalEntityProvider { * * @public */ -export type EntityIteratorResult = -| { - done: false; - entities: DeferredEntity[]; - cursor: T; -} | { - done: true; - entities?: DeferredEntity[]; - cursor?: T; -} +export type EntityIteratorResult = + | { + done: false; + entities: DeferredEntity[]; + cursor: T; + } + | { + done: true; + entities?: DeferredEntity[]; + cursor?: T; + }; /** @public */ export interface IncrementalEntityProviderOptions { @@ -217,7 +218,7 @@ export interface IngestionUpsert { /** * This interface is for updating an existing ingestion record. - * + * * @public */ export interface IngestionRecordUpdate { @@ -227,7 +228,7 @@ export interface IngestionRecordUpdate { /** * The expected response from the `ingestion_marks` table. - * + * * @public */ export interface MarkRecord { @@ -240,7 +241,7 @@ export interface MarkRecord { /** * The expected response from the `ingestions` table. - * + * * @public */ export interface IngestionRecord extends IngestionUpsert { @@ -254,7 +255,7 @@ export interface IngestionRecord extends IngestionUpsert { /** * This interface supplies all the values for adding an ingestion mark. - * + * * @public */ export interface MarkRecordInsert { From 35455be41413b773f1b1f39fa4525d347dabee22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 22 Nov 2022 11:30:35 +0100 Subject: [PATCH 46/54] shave down the api report, some final fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/loud-rockets-reply.md | 4 +- .../incremental-ingestion-backend/README.md | 145 +++++++-------- .../api-report.md | 169 +----------------- .../migrations/20221116073152_init.js | 2 +- .../package.json | 13 +- .../IncrementalIngestionDatabaseManager.ts | 1 - .../src/database/tables.ts | 3 +- .../src/engine/IncrementalIngestionEngine.ts | 1 - .../src/index.ts | 13 +- .../src/router/routes.ts | 4 +- .../src/service/IncrementalCatalogBuilder.ts | 6 +- .../src/types.ts | 21 +-- yarn.lock | 2 + 13 files changed, 113 insertions(+), 271 deletions(-) diff --git a/.changeset/loud-rockets-reply.md b/.changeset/loud-rockets-reply.md index 1262e439b4..803521c97b 100644 --- a/.changeset/loud-rockets-reply.md +++ b/.changeset/loud-rockets-reply.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-incremental-ingestion-backend': patch +'@backstage/plugin-incremental-ingestion-backend': minor --- -This change introduces incremental entity providers, which are used for streaming very large data sources into the catalog. +Introduces incremental entity providers, which are used for streaming very large data sources into the catalog. diff --git a/plugins/incremental-ingestion-backend/README.md b/plugins/incremental-ingestion-backend/README.md index 0e0a4e56e6..5df35684d5 100644 --- a/plugins/incremental-ingestion-backend/README.md +++ b/plugins/incremental-ingestion-backend/README.md @@ -1,4 +1,4 @@ -# backend-incremental-ingestion +# `@backstage/plugin-incremental-ingestion-backend` The Incremental Ingestion Backend plugin provides an Incremental Entity Provider that can be used to ingest data from sources using delta mutations, while retaining the orphan prevention mechanism provided by full mutations. @@ -45,63 +45,66 @@ The Incremental Entity Provider backend is designed for data sources that provid 1. Install `@backstage/plugin-incremental-ingestion-backend` with `yarn workspace backend add @backstage/plugin-incremental-ingestion-backend` 2. Import `IncrementalCatalogBuilder` from `@backstage/plugin-incremental-ingestion-backend` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`. -```ts -const builder = CatalogBuilder.create(env); -// incremental builder receives builder because it'll register -// incremental entity providers with the builder -const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); -``` + ```ts + const builder = CatalogBuilder.create(env); + // incremental builder receives builder because it'll register + // incremental entity providers with the builder + const incrementalBuilder = await IncrementalCatalogBuilder.create( + env, + builder, + ); + ``` -3. Last step, add `await incrementBuilder.build()` after `await builder.build()` to ensure that all `CatalogBuider` migration run before running `incrementBuilder.build()` migrations. +3. Last step, add `await incrementBuilder.build()` after `await builder.build()` to ensure that all `CatalogBuilder` migration run before running `incrementBuilder.build()` migrations. -```ts -const { processingEngine, router } = await builder.build(); + ```ts + const { processingEngine, router } = await builder.build(); -// this has to run after `await builder.build()` to ensure that catalog migrations are completed -// before incremental builder migrations are executed -await incrementalBuilder.build(); -``` + // this has to run after `await builder.build()` to ensure that catalog migrations are completed + // before incremental builder migrations are executed + await incrementalBuilder.build(); + ``` -The result should look something like this, + The result should look something like this, -```ts -import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; -import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; -import { IncrementalCatalogBuilder } from '@backstage/plugin-incremental-ingestion-backend'; -import { Router } from 'express'; -import { Duration } from 'luxon'; -import { PluginEnvironment } from '../types'; + ```ts + import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; + import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; + import { IncrementalCatalogBuilder } from '@backstage/plugin-incremental-ingestion-backend'; + import { Router } from 'express'; + import { Duration } from 'luxon'; + import { PluginEnvironment } from '../types'; -export default async function createPlugin( - env: PluginEnvironment, -): Promise { - const builder = CatalogBuilder.create(env); - // incremental builder receives builder because it'll register - // incremental entity providers with the builder - const incrementalBuilder = await IncrementalCatalogBuilder.create( - env, - builder, - ); + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = CatalogBuilder.create(env); + // incremental builder receives builder because it'll register + // incremental entity providers with the builder + const incrementalBuilder = await IncrementalCatalogBuilder.create( + env, + builder, + ); - builder.addProcessor(new ScaffolderEntitiesProcessor()); + builder.addProcessor(new ScaffolderEntitiesProcessor()); - const { processingEngine, router } = await builder.build(); + const { processingEngine, router } = await builder.build(); - // this has to run after `await builder.build()` so ensure that catalog migrations are completed - // before incremental builder migrations are executed - const { incrementalAdminRouter } = await incrementalBuilder.build(); + // this has to run after `await builder.build()` so ensure that catalog migrations are completed + // before incremental builder migrations are executed + const { incrementalAdminRouter } = await incrementalBuilder.build(); - router.use('/incremental', incrementalAdminRouter); + router.use('/incremental', incrementalAdminRouter); - await processingEngine.start(); + await processingEngine.start(); - return router; -} -``` + return router; + } + ``` ## Writing an Incremental Entity Provider -To create an Incremental Entity Provider, you need to know how to retrieve a single page of the data that you wish to ingest into the Backstage catalog. If the API has pagination and you know how to make a paginated request to that API, you'll be able to implement an Incremental Entity Provider for this API. For more information about compatibility, checkout Compatible data sources section on this page. +To create an Incremental Entity Provider, you need to know how to retrieve a single page of the data that you wish to ingest into the Backstage catalog. If the API has pagination and you know how to make a paginated request to that API, you'll be able to implement an Incremental Entity Provider for this API. For more information about compatibility, checkout Compatible data sources section on this page. Here is the type definition for an Incremental Entity Provider. @@ -127,7 +130,7 @@ interface IncrementalEntityProvider { * ingestion. * * @param context - anything needed in order to fetch a single page. - * @param cursor - a uniqiue value identifying the page to ingest. + * @param cursor - a unique value identifying the page to ingest. * @returns the entities to be ingested, as well as the cursor of * the the next page after this one. */ @@ -230,7 +233,7 @@ export class MyIncrementalEntityProvider The last step is to implement the actual `next` method that will accept the cursor, call the API, process the result and return the result. ```ts -export class MyIncrementalEntityProvider implements IncrementalEntityProvider { +export class MyIncrementalEntityProvider implements IncrementalEntityProvider { token: string; @@ -250,7 +253,7 @@ export class MyIncrementalEntityProvider implements IncrementalEntityProvider> { + async next(context: MyContext, cursor?: MyApiCursor = { page: 1 }): Promise> { const { apiClient } = context; // call your API with the current cursor @@ -299,36 +302,36 @@ Now that you have your new Incremental Entity Provider, we can connect it to the ## Adding an Incremental Entity Provider to the catalog -We'll assume you followed the Installation instructions. After you create your `incrementalBuilder`, you can instantiate your Entity Provider and pass it to the `addIncrementalEntityProvider` method. +We'll assume you followed the Installation instructions. After you create your `incrementalBuilder`, you can instantiate your Entity Provider and pass it to the `addIncrementalEntityProvider` method. ```ts - const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); +const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder); - // I'm assuming you're going to get your token from config - const token = config.getString('myApiClient.token'); +// I'm assuming you're going to get your token from config +const token = config.getString('myApiClient.token'); - const myEntityProvider = new MyIncrementalEntityProvider(token) +const myEntityProvider = new MyIncrementalEntityProvider(token) - incrementalBuilder.addIncrementalEntityProvider( - myEntityProvider, - { - // how long should it attempt to read pages from the API - // keep this short. Incremental Entity Provider will attempt to - // read as many pages as it can in this time - burstLength: Duration.fromObject({ seconds: 3 }), - // how long should it wait between bursts? - burstInterval: Duration.fromObject({ seconds: 3 }), - // how long should it rest before re-ingesting again? - restLength: Duration.fromObject({ day: 1 }) - // optional back-off configuration - how long should it wait to retry? - backoff: [ - Duration.fromObject({ seconds: 5 }), - Duration.fromObject({ seconds: 30 }), - Duration.fromObject({ minutes: 10 }), - Duration.fromObject({ hours: 3 }) - ] - } - ) +incrementalBuilder.addIncrementalEntityProvider( + myEntityProvider, + { + // how long should it attempt to read pages from the API + // keep this short. Incremental Entity Provider will attempt to + // read as many pages as it can in this time + burstLength: Duration.fromObject({ seconds: 3 }), + // how long should it wait between bursts? + burstInterval: Duration.fromObject({ seconds: 3 }), + // how long should it rest before re-ingesting again? + restLength: Duration.fromObject({ day: 1 }) + // optional back-off configuration - how long should it wait to retry? + backoff: [ + Duration.fromObject({ seconds: 5 }), + Duration.fromObject({ seconds: 30 }), + Duration.fromObject({ minutes: 10 }), + Duration.fromObject({ hours: 3 }) + ] + } +) ``` That's it!!! diff --git a/plugins/incremental-ingestion-backend/api-report.md b/plugins/incremental-ingestion-backend/api-report.md index 5d2f66ac19..ff635871aa 100644 --- a/plugins/incremental-ingestion-backend/api-report.md +++ b/plugins/incremental-ingestion-backend/api-report.md @@ -8,11 +8,9 @@ import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; import type { Config } from '@backstage/config'; import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; -import { Duration } from 'luxon'; import type { DurationObjectUnits } from 'luxon'; -import { Knex } from 'knex'; import type { Logger } from 'winston'; -import type { PermissionAuthorizer } from '@backstage/plugin-permission-common'; +import type { PermissionEvaluator } from '@backstage/plugin-permission-common'; import type { PluginDatabaseManager } from '@backstage/backend-common'; import type { PluginTaskScheduler } from '@backstage/backend-tasks'; import { Router } from 'express'; @@ -38,14 +36,13 @@ export const INCREMENTAL_ENTITY_PROVIDER_ANNOTATION = // @public (undocumented) export class IncrementalCatalogBuilder { // (undocumented) - addIncrementalEntityProvider( - provider: IncrementalEntityProvider, + addIncrementalEntityProvider( + provider: IncrementalEntityProvider, options: IncrementalEntityProviderOptions, ): void; // (undocumented) build(): Promise<{ incrementalAdminRouter: Router; - manager: IncrementalIngestionDatabaseManager; }>; static create( env: PluginEnvironment, @@ -71,162 +68,6 @@ export interface IncrementalEntityProviderOptions { restLength: DurationObjectUnits; } -// @public (undocumented) -export class IncrementalIngestionDatabaseManager { - constructor(options: { client: Knex }); - cleanupProviders(): Promise<{ - ingestionsDeleted: void; - ingestionMarksDeleted: void; - markEntitiesDeleted: void; - }>; - clearDuplicateIngestions( - ingestionId: string, - provider: string, - ): Promise; - clearFinishedIngestions(provider: string): Promise<{ - deletions: { - markEntitiesDeleted: number; - marksDeleted: number; - ingestionsDeleted: number; - }; - }>; - computeRemoved( - provider: string, - ingestionId: string, - ): Promise< - { - entity: any; - }[] - >; - createMark(options: MarkRecordInsert): Promise; - createMarkEntities(markId: string, entities: DeferredEntity[]): Promise; - createProviderIngestionRecord(provider: string): Promise< - | { - ingestionId: string; - nextAction: string; - attempts: number; - nextActionAt: number; - } - | undefined - >; - // (undocumented) - getAllMarks(ingestionId: string): Promise; - getCurrentIngestionRecord( - provider: string, - ): Promise; - getLastMark(ingestionId: string): Promise; - healthcheck(): Promise< - Pick< - { - id: string; - provider_name: string; - }, - 'id' | 'provider_name' - >[] - >; - insertIngestionRecord(record: IngestionUpsertIFace): Promise; - listProviders(): Promise; - purgeAndResetProvider(provider: string): Promise<{ - provider: string; - ingestionsDeleted: number; - marksDeleted: number; - markEntitiesDeleted: number; - }>; - purgeTable(table: string): Promise; - setProviderBackoff( - ingestionId: string, - attempts: number, - error: Error, - backoffLength: number, - ): Promise; - setProviderBursting(ingestionId: string): Promise; - setProviderCanceled(ingestionId: string): Promise; - setProviderCanceling(ingestionId: string, message?: string): Promise; - setProviderComplete(ingestionId: string): Promise; - setProviderIngesting(ingestionId: string): Promise; - setProviderInterstitial(ingestionId: string): Promise; - setProviderResting(ingestionId: string, restLength: Duration): Promise; - triggerNextProviderAction(provider: string): Promise; - // (undocumented) - updateByName( - provider: string, - update: Partial, - ): Promise; - updateIngestionRecordById(options: IngestionRecordUpdate): Promise; - updateIngestionRecordByProvider( - provider: string, - update: Partial, - ): Promise; -} - -// @public -export interface IngestionRecord extends IngestionUpsertIFace { - created_at: string; - // (undocumented) - id: string; - // (undocumented) - next_action_at: Date; -} - -// @public -export interface IngestionRecordUpdate { - // (undocumented) - ingestionId: string; - // (undocumented) - update: Partial; -} - -// @public -export interface IngestionUpsertIFace { - attempts?: number; - completion_ticket: string; - id?: string; - ingestion_completed_at?: Date | string | null; - last_error?: string | null; - next_action: - | 'rest' - | 'ingest' - | 'backoff' - | 'cancel' - | 'nothing (done)' - | 'nothing (canceled)'; - next_action_at?: Date; - provider_name: string; - rest_completed_at?: Date | string | null; - status: - | 'complete' - | 'bursting' - | 'resting' - | 'canceling' - | 'interstitial' - | 'backing off'; -} - -// @public -export interface MarkRecord { - // (undocumented) - created_at: string; - // (undocumented) - cursor: string; - // (undocumented) - id: string; - // (undocumented) - ingestion_id: string; - // (undocumented) - sequence: number; -} - -// @public -export interface MarkRecordInsert { - // (undocumented) - record: { - id: string; - ingestion_id: string; - cursor: unknown; - sequence: number; - }; -} - // @public (undocumented) export type PluginEnvironment = { logger: Logger; @@ -234,8 +75,6 @@ export type PluginEnvironment = { scheduler: PluginTaskScheduler; config: Config; reader: UrlReader; - permissions: PermissionAuthorizer; + permissions: PermissionEvaluator; }; - -// (No @packageDocumentation comment for this package) ``` diff --git a/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js b/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js index 11357c50fd..dc77e0bc31 100644 --- a/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js +++ b/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js @@ -54,7 +54,7 @@ exports.up = async function up(knex) { table .string('last_error') - .comment('records any error that occured in the previous burst attempt'); + .comment('records any error that occurred in the previous burst attempt'); table .integer('attempts') diff --git a/plugins/incremental-ingestion-backend/package.json b/plugins/incremental-ingestion-backend/package.json index 195adab7ea..8d47121c8d 100644 --- a/plugins/incremental-ingestion-backend/package.json +++ b/plugins/incremental-ingestion-backend/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/plugin-incremental-ingestion-backend", "description": "An entity provider for streaming large asset sources into the catalog", - "version": "0.1.0", + "version": "0.0.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", @@ -13,6 +13,15 @@ "backstage": { "role": "backend-plugin" }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/incremental-ingestion-backend" + }, + "keywords": [ + "backstage" + ], "scripts": { "start": "backstage-cli package start", "build": "backstage-cli package build", @@ -24,10 +33,12 @@ }, "dependencies": { "@backstage/backend-common": "workspace:^", + "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", "@types/express": "^4.17.6", "@types/luxon": "^3.0.0", diff --git a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts index 891f94b537..7f4ac4eb7c 100644 --- a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts +++ b/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts @@ -28,7 +28,6 @@ import { MarkRecordInsert, } from '../types'; -/** @public */ export class IncrementalIngestionDatabaseManager { private client: Knex; diff --git a/plugins/incremental-ingestion-backend/src/database/tables.ts b/plugins/incremental-ingestion-backend/src/database/tables.ts index ee6cfe0ade..6eee7e6dc5 100644 --- a/plugins/incremental-ingestion-backend/src/database/tables.ts +++ b/plugins/incremental-ingestion-backend/src/database/tables.ts @@ -14,5 +14,4 @@ * limitations under the License. */ -export const DB_MIGRATIONS_TABLE = - 'plugin_incremental_ingestion_backend__knex_migrations'; +export const DB_MIGRATIONS_TABLE = 'incremental_ingestion__knex_migrations'; diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts index 9b94c04c72..8510881c25 100644 --- a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts @@ -27,7 +27,6 @@ import { performance } from 'perf_hooks'; import { Duration, DurationObjectUnits } from 'luxon'; import { v4 } from 'uuid'; -/** @public */ export class IncrementalIngestionEngine implements IterationEngine { restLength: Duration; backoff: DurationObjectUnits[]; diff --git a/plugins/incremental-ingestion-backend/src/index.ts b/plugins/incremental-ingestion-backend/src/index.ts index 142b6ba836..7d661c2b98 100644 --- a/plugins/incremental-ingestion-backend/src/index.ts +++ b/plugins/incremental-ingestion-backend/src/index.ts @@ -13,17 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +/** + * Provides efficient incremental ingestion of entities into the catalog. + * + * @packageDocumentation + */ + export { IncrementalCatalogBuilder } from './service/IncrementalCatalogBuilder'; -export type { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager'; export type { EntityIteratorResult, IncrementalEntityProvider, IncrementalEntityProviderOptions, - IngestionRecord, - IngestionRecordUpdate, - IngestionUpsert as IngestionUpsertIFace, - MarkRecord, - MarkRecordInsert, INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, PluginEnvironment, } from './types'; diff --git a/plugins/incremental-ingestion-backend/src/router/routes.ts b/plugins/incremental-ingestion-backend/src/router/routes.ts index d26ace1de3..9f480e879e 100644 --- a/plugins/incremental-ingestion-backend/src/router/routes.ts +++ b/plugins/incremental-ingestion-backend/src/router/routes.ts @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { errorHandler } from '@backstage/backend-common'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; -import { IncrementalIngestionDatabaseManager } from '..'; +import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; import { PROVIDER_BASE_PATH, PROVIDER_CLEANUP, PROVIDER_HEALTH } from './paths'; -/** @public */ export const createIncrementalProviderRouter = async ( manager: IncrementalIngestionDatabaseManager, logger: Logger, diff --git a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts index eea2c8c56d..680c76011e 100644 --- a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts @@ -93,11 +93,11 @@ export class IncrementalCatalogBuilder { routerLogger, ); - return { incrementalAdminRouter, manager: this.manager }; + return { incrementalAdminRouter }; } - addIncrementalEntityProvider( - provider: IncrementalEntityProvider, + addIncrementalEntityProvider( + provider: IncrementalEntityProvider, options: IncrementalEntityProviderOptions, ) { const { burstInterval, burstLength, restLength } = options; diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/incremental-ingestion-backend/src/types.ts index 90f9a20d46..47314626ed 100644 --- a/plugins/incremental-ingestion-backend/src/types.ts +++ b/plugins/incremental-ingestion-backend/src/types.ts @@ -26,10 +26,11 @@ import type { DeferredEntity, EntityProviderConnection, } from '@backstage/plugin-catalog-backend'; -import type { PermissionAuthorizer } from '@backstage/plugin-permission-common'; +import type { PermissionEvaluator } from '@backstage/plugin-permission-common'; import type { DurationObjectUnits } from 'luxon'; import type { Logger } from 'winston'; -import { IncrementalIngestionDatabaseManager } from './'; +import { IncrementalIngestionDatabaseManager } from './database/IncrementalIngestionDatabaseManager'; + /** * Entity annotation containing the incremental entity provider. * @@ -64,7 +65,7 @@ export interface IncrementalEntityProvider { * ingestion. * * @param context - anything needed in order to fetch a single page. - * @param cursor - a uniqiue value identifying the page to ingest. + * @param cursor - a unique value identifying the page to ingest. * @returns The entities to be ingested, as well as the cursor of * the next page after this one. */ @@ -137,15 +138,13 @@ export type PluginEnvironment = { scheduler: PluginTaskScheduler; config: Config; reader: UrlReader; - permissions: PermissionAuthorizer; + permissions: PermissionEvaluator; }; -/** @public */ export interface IterationEngine { taskFn: TaskFunction; } -/** @public */ export interface IterationEngineOptions { logger: Logger; connection: EntityProviderConnection; @@ -158,8 +157,6 @@ export interface IterationEngineOptions { /** * The shape of data inserted into or updated in the `ingestions` table. - * - * @public */ export interface IngestionUpsert { /** @@ -218,8 +215,6 @@ export interface IngestionUpsert { /** * This interface is for updating an existing ingestion record. - * - * @public */ export interface IngestionRecordUpdate { ingestionId: string; @@ -228,8 +223,6 @@ export interface IngestionRecordUpdate { /** * The expected response from the `ingestion_marks` table. - * - * @public */ export interface MarkRecord { id: string; @@ -241,8 +234,6 @@ export interface MarkRecord { /** * The expected response from the `ingestions` table. - * - * @public */ export interface IngestionRecord extends IngestionUpsert { id: string; @@ -255,8 +246,6 @@ export interface IngestionRecord extends IngestionUpsert { /** * This interface supplies all the values for adding an ingestion mark. - * - * @public */ export interface MarkRecordInsert { record: { diff --git a/yarn.lock b/yarn.lock index 17e002800a..dc41ad39a8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6028,11 +6028,13 @@ __metadata: resolution: "@backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend" dependencies: "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" "@types/express": ^4.17.6 "@types/luxon": ^3.0.0 From 87cabfed8c5386566b2206f3bf2a31312ed60d95 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Tue, 22 Nov 2022 07:28:41 -0800 Subject: [PATCH 47/54] Use stringifyerror Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/package.json | 1 + .../src/engine/IncrementalIngestionEngine.ts | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/incremental-ingestion-backend/package.json b/plugins/incremental-ingestion-backend/package.json index 8d47121c8d..cd332f7986 100644 --- a/plugins/incremental-ingestion-backend/package.json +++ b/plugins/incremental-ingestion-backend/package.json @@ -33,6 +33,7 @@ }, "dependencies": { "@backstage/backend-common": "workspace:^", + "@backstage/errors": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", "@backstage/catalog-model": "workspace:^", diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts index 8510881c25..d88efdfbdb 100644 --- a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts @@ -26,6 +26,7 @@ import type { AbortSignal } from 'node-abort-controller'; import { performance } from 'perf_hooks'; import { Duration, DurationObjectUnits } from 'luxon'; import { v4 } from 'uuid'; +import { stringifyError } from '@backstage/errors'; export class IncrementalIngestionEngine implements IterationEngine { restLength: Duration; @@ -118,10 +119,7 @@ export class IncrementalIngestionEngine implements IterationEngine { const backoffLength = currentBackoff.as('milliseconds'); this.options.logger.error(error); - const truncatedError = - typeof error === 'string' - ? error.substring(0, 700) - : `${error}`; + const truncatedError = stringifyError(error).substring(0, 700); this.options.logger.error( `incremental-engine: Ingestion '${ingestionId}' threw an error during ingestion burst. Ingestion will backoff for ${currentBackoff.toHuman()} (${truncatedError})`, ); From e426002501435e933a97d7f46af8db93091d373e Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Tue, 22 Nov 2022 07:40:32 -0800 Subject: [PATCH 48/54] Update yarn.lock Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/package.json | 2 +- yarn.lock | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/incremental-ingestion-backend/package.json b/plugins/incremental-ingestion-backend/package.json index cd332f7986..f8ba9bf1f5 100644 --- a/plugins/incremental-ingestion-backend/package.json +++ b/plugins/incremental-ingestion-backend/package.json @@ -33,11 +33,11 @@ }, "dependencies": { "@backstage/backend-common": "workspace:^", - "@backstage/errors": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", diff --git a/yarn.lock b/yarn.lock index dc41ad39a8..14274d6639 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6033,6 +6033,7 @@ __metadata: "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/errors": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" From de022e51c9b7e55a5147ac9b42735b924f7f1d12 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Tue, 22 Nov 2022 09:38:52 -0800 Subject: [PATCH 49/54] Group router paths in a less risky way Signed-off-by: Damon Kaswell --- plugins/incremental-ingestion-backend/src/router/paths.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/incremental-ingestion-backend/src/router/paths.ts b/plugins/incremental-ingestion-backend/src/router/paths.ts index 1be540fc10..2118177821 100644 --- a/plugins/incremental-ingestion-backend/src/router/paths.ts +++ b/plugins/incremental-ingestion-backend/src/router/paths.ts @@ -14,6 +14,6 @@ * limitations under the License. */ -export const PROVIDER_CLEANUP = '/providers/cleanup'; -export const PROVIDER_HEALTH = '/providers/health'; -export const PROVIDER_BASE_PATH = '/providers/:provider'; +export const PROVIDER_CLEANUP = '/incremental/cleanup'; +export const PROVIDER_HEALTH = '/incremental/health'; +export const PROVIDER_BASE_PATH = '/incremental/providers/:provider'; From 7deebb90654a8111c58883926e6224d60a561402 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Tue, 22 Nov 2022 12:02:41 -0800 Subject: [PATCH 50/54] Fix removal logic Signed-off-by: Damon Kaswell --- .../src/engine/IncrementalIngestionEngine.ts | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts index d88efdfbdb..922f181503 100644 --- a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts @@ -279,17 +279,7 @@ export class IncrementalIngestionEngine implements IterationEngine { const removed: DeferredEntity[] = []; - let doComputeRemoved = false; - - if (!done) { - doComputeRemoved = true; - } else { - if (entities && entities.length > 0) { - doComputeRemoved = true; - } - } - - if (doComputeRemoved) { + if (done) { removed.push( ...(await this.manager.computeRemoved( this.options.provider.getProviderName(), From 3f5d620ce72b3f7770dfe3529c58a3f5ad4048b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 23 Nov 2022 15:02:13 +0100 Subject: [PATCH 51/54] rename to be a catalog backend module, implement backend system module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/loud-rockets-reply.md | 2 +- .../.eslintrc.js | 0 .../README.md | 12 +- .../api-report.md | 11 +- .../knexfile.js | 0 .../migrations/20221116073152_init.js | 0 .../package.json | 11 +- .../IncrementalIngestionDatabaseManager.ts | 4 +- .../src/database/migrations.ts | 2 +- .../src/database/tables.ts | 118 +++++++++++++++ .../src/engine/IncrementalIngestionEngine.ts | 1 - .../src/index.ts | 13 +- .../src/module/WrapperProviders.ts | 134 ++++++++++++++++++ ...talIngestionEntityProviderCatalogModule.ts | 67 +++++++++ .../src/module/index.ts | 17 +++ .../src/router/paths.ts | 0 .../src/router/routes.ts | 0 .../src/service/IncrementalCatalogBuilder.ts | 30 +--- .../src/service/index.ts} | 4 +- .../src/types.ts | 102 +------------ .../src/util.ts | 44 ++++++ yarn.lock | 51 +++---- 22 files changed, 444 insertions(+), 179 deletions(-) rename plugins/{incremental-ingestion-backend => catalog-backend-module-incremental-ingestion}/.eslintrc.js (100%) rename plugins/{incremental-ingestion-backend => catalog-backend-module-incremental-ingestion}/README.md (93%) rename plugins/{incremental-ingestion-backend => catalog-backend-module-incremental-ingestion}/api-report.md (85%) rename plugins/{incremental-ingestion-backend => catalog-backend-module-incremental-ingestion}/knexfile.js (100%) rename plugins/{incremental-ingestion-backend => catalog-backend-module-incremental-ingestion}/migrations/20221116073152_init.js (100%) rename plugins/{incremental-ingestion-backend => catalog-backend-module-incremental-ingestion}/package.json (82%) rename plugins/{incremental-ingestion-backend => catalog-backend-module-incremental-ingestion}/src/database/IncrementalIngestionDatabaseManager.ts (99%) rename plugins/{incremental-ingestion-backend => catalog-backend-module-incremental-ingestion}/src/database/migrations.ts (93%) create mode 100644 plugins/catalog-backend-module-incremental-ingestion/src/database/tables.ts rename plugins/{incremental-ingestion-backend => catalog-backend-module-incremental-ingestion}/src/engine/IncrementalIngestionEngine.ts (99%) rename plugins/{incremental-ingestion-backend => catalog-backend-module-incremental-ingestion}/src/index.ts (79%) create mode 100644 plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts create mode 100644 plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts create mode 100644 plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts rename plugins/{incremental-ingestion-backend => catalog-backend-module-incremental-ingestion}/src/router/paths.ts (100%) rename plugins/{incremental-ingestion-backend => catalog-backend-module-incremental-ingestion}/src/router/routes.ts (100%) rename plugins/{incremental-ingestion-backend => catalog-backend-module-incremental-ingestion}/src/service/IncrementalCatalogBuilder.ts (85%) rename plugins/{incremental-ingestion-backend/src/database/tables.ts => catalog-backend-module-incremental-ingestion/src/service/index.ts} (82%) rename plugins/{incremental-ingestion-backend => catalog-backend-module-incremental-ingestion}/src/types.ts (67%) create mode 100644 plugins/catalog-backend-module-incremental-ingestion/src/util.ts diff --git a/.changeset/loud-rockets-reply.md b/.changeset/loud-rockets-reply.md index 803521c97b..85ed77f51f 100644 --- a/.changeset/loud-rockets-reply.md +++ b/.changeset/loud-rockets-reply.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-incremental-ingestion-backend': minor +'@backstage/plugin-catalog-backend-module-incremental-ingestion': minor --- Introduces incremental entity providers, which are used for streaming very large data sources into the catalog. diff --git a/plugins/incremental-ingestion-backend/.eslintrc.js b/plugins/catalog-backend-module-incremental-ingestion/.eslintrc.js similarity index 100% rename from plugins/incremental-ingestion-backend/.eslintrc.js rename to plugins/catalog-backend-module-incremental-ingestion/.eslintrc.js diff --git a/plugins/incremental-ingestion-backend/README.md b/plugins/catalog-backend-module-incremental-ingestion/README.md similarity index 93% rename from plugins/incremental-ingestion-backend/README.md rename to plugins/catalog-backend-module-incremental-ingestion/README.md index 5df35684d5..d14e33ea53 100644 --- a/plugins/incremental-ingestion-backend/README.md +++ b/plugins/catalog-backend-module-incremental-ingestion/README.md @@ -1,6 +1,6 @@ -# `@backstage/plugin-incremental-ingestion-backend` +# `@backstage/plugin-catalog-backend-module-incremental-ingestion` -The Incremental Ingestion Backend plugin provides an Incremental Entity Provider that can be used to ingest data from sources using delta mutations, while retaining the orphan prevention mechanism provided by full mutations. +The Incremental Ingestion catalog backend module provides an Incremental Entity Provider that can be used to ingest data from sources using delta mutations, while retaining the orphan prevention mechanism provided by full mutations. ## Why did we create it? @@ -42,8 +42,8 @@ The Incremental Entity Provider backend is designed for data sources that provid ## Installation -1. Install `@backstage/plugin-incremental-ingestion-backend` with `yarn workspace backend add @backstage/plugin-incremental-ingestion-backend` -2. Import `IncrementalCatalogBuilder` from `@backstage/plugin-incremental-ingestion-backend` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`. +1. Install `@backstage/plugin-catalog-backend-module-incremental-ingestion` with `yarn workspace backend add @backstage/plugin-catalog-backend-module-incremental-ingestion` +2. Import `IncrementalCatalogBuilder` from `@backstage/plugin-catalog-backend-module-incremental-ingestion` and instantiate it with `await IncrementalCatalogBuilder.create(env, builder)`. You have to pass `builder` into `IncrementalCatalogBuilder.create` function because `IncrementalCatalogBuilder` will convert an `IncrementalEntityProvider` into an `EntityProvider` and call `builder.addEntityProvider`. ```ts const builder = CatalogBuilder.create(env); @@ -70,7 +70,7 @@ The Incremental Entity Provider backend is designed for data sources that provid ```ts import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend'; - import { IncrementalCatalogBuilder } from '@backstage/plugin-incremental-ingestion-backend'; + import { IncrementalCatalogBuilder } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; import { Router } from 'express'; import { Duration } from 'luxon'; import { PluginEnvironment } from '../types'; @@ -164,7 +164,7 @@ These are the only 3 methods that you need to implement. `getProviderName()` is import { IncrementalEntityProvider, EntityIteratorResult, -} from '@backstage/plugin-incremental-ingestion-backend'; +} from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; // this will include your pagination information, let's say our API accepts a `page` parameter. // In this case, the cursor will include `page` diff --git a/plugins/incremental-ingestion-backend/api-report.md b/plugins/catalog-backend-module-incremental-ingestion/api-report.md similarity index 85% rename from plugins/incremental-ingestion-backend/api-report.md rename to plugins/catalog-backend-module-incremental-ingestion/api-report.md index ff635871aa..49eb143a4d 100644 --- a/plugins/incremental-ingestion-backend/api-report.md +++ b/plugins/catalog-backend-module-incremental-ingestion/api-report.md @@ -1,10 +1,11 @@ -## API Report File for "@backstage/plugin-incremental-ingestion-backend" +## API Report File for "@backstage/plugin-catalog-backend-module-incremental-ingestion" > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts /// +import { BackendFeature } from '@backstage/backend-plugin-api'; import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; import type { Config } from '@backstage/config'; import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; @@ -68,6 +69,14 @@ export interface IncrementalEntityProviderOptions { restLength: DurationObjectUnits; } +// @alpha +export const incrementalIngestionEntityProviderCatalogModule: (options: { + providers: { + provider: IncrementalEntityProvider; + options: IncrementalEntityProviderOptions; + }[]; +}) => BackendFeature; + // @public (undocumented) export type PluginEnvironment = { logger: Logger; diff --git a/plugins/incremental-ingestion-backend/knexfile.js b/plugins/catalog-backend-module-incremental-ingestion/knexfile.js similarity index 100% rename from plugins/incremental-ingestion-backend/knexfile.js rename to plugins/catalog-backend-module-incremental-ingestion/knexfile.js diff --git a/plugins/incremental-ingestion-backend/migrations/20221116073152_init.js b/plugins/catalog-backend-module-incremental-ingestion/migrations/20221116073152_init.js similarity index 100% rename from plugins/incremental-ingestion-backend/migrations/20221116073152_init.js rename to plugins/catalog-backend-module-incremental-ingestion/migrations/20221116073152_init.js diff --git a/plugins/incremental-ingestion-backend/package.json b/plugins/catalog-backend-module-incremental-ingestion/package.json similarity index 82% rename from plugins/incremental-ingestion-backend/package.json rename to plugins/catalog-backend-module-incremental-ingestion/package.json index f8ba9bf1f5..74ed0c8e5c 100644 --- a/plugins/incremental-ingestion-backend/package.json +++ b/plugins/catalog-backend-module-incremental-ingestion/package.json @@ -1,5 +1,5 @@ { - "name": "@backstage/plugin-incremental-ingestion-backend", + "name": "@backstage/plugin-catalog-backend-module-incremental-ingestion", "description": "An entity provider for streaming large asset sources into the catalog", "version": "0.0.0", "main": "src/index.ts", @@ -7,24 +7,25 @@ "license": "Apache-2.0", "publishConfig": { "access": "public", + "alphaTypes": "dist/index.alpha.d.ts", "main": "dist/index.cjs.js", "types": "dist/index.d.ts" }, "backstage": { - "role": "backend-plugin" + "role": "backend-plugin-module" }, "homepage": "https://backstage.io", "repository": { "type": "git", "url": "https://github.com/backstage/backstage", - "directory": "plugins/incremental-ingestion-backend" + "directory": "plugins/catalog-backend-module-incremental-ingestion" }, "keywords": [ "backstage" ], "scripts": { "start": "backstage-cli package start", - "build": "backstage-cli package build", + "build": "backstage-cli package build --experimental-type-build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", "clean": "backstage-cli package clean", @@ -46,6 +47,7 @@ "express": "^4.17.1", "express-promise-router": "^4.1.0", "knex": "^2.0.0", + "lodash": "^4.17.21", "luxon": "^3.0.0", "uuid": "^8.3.2", "winston": "^3.2.1" @@ -54,6 +56,7 @@ "@backstage/cli": "workspace:^" }, "files": [ + "alpha", "dist", "migrations/**/*.{js,d.ts}" ] diff --git a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts b/plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.ts similarity index 99% rename from plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.ts index 7f4ac4eb7c..3c91b24e2c 100644 --- a/plugins/incremental-ingestion-backend/src/database/IncrementalIngestionDatabaseManager.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.ts @@ -19,14 +19,14 @@ import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; import { stringifyEntityRef } from '@backstage/catalog-model'; import { Duration } from 'luxon'; import { v4 } from 'uuid'; +import { INCREMENTAL_ENTITY_PROVIDER_ANNOTATION } from '../types'; import { - INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, IngestionRecord, IngestionRecordUpdate, IngestionUpsert, MarkRecord, MarkRecordInsert, -} from '../types'; +} from './tables'; export class IncrementalIngestionDatabaseManager { private client: Knex; diff --git a/plugins/incremental-ingestion-backend/src/database/migrations.ts b/plugins/catalog-backend-module-incremental-ingestion/src/database/migrations.ts similarity index 93% rename from plugins/incremental-ingestion-backend/src/database/migrations.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/database/migrations.ts index 58ccd7c8c5..940a82107b 100644 --- a/plugins/incremental-ingestion-backend/src/database/migrations.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/database/migrations.ts @@ -20,7 +20,7 @@ import { DB_MIGRATIONS_TABLE } from './tables'; export async function applyDatabaseMigrations(knex: Knex): Promise { const migrationsDir = resolvePackagePath( - '@backstage/plugin-incremental-ingestion-backend', + '@backstage/plugin-catalog-backend-module-incremental-ingestion', 'migrations', ); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/database/tables.ts b/plugins/catalog-backend-module-incremental-ingestion/src/database/tables.ts new file mode 100644 index 0000000000..00d40fb4a1 --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/src/database/tables.ts @@ -0,0 +1,118 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const DB_MIGRATIONS_TABLE = 'incremental_ingestion__knex_migrations'; + +/** + * The shape of data inserted into or updated in the `ingestions` table. + */ +export interface IngestionUpsert { + /** + * The ingestion record id. + */ + id?: string; + /** + * The next action the incremental entity provider will take. + */ + next_action: + | 'rest' + | 'ingest' + | 'backoff' + | 'cancel' + | 'nothing (done)' + | 'nothing (canceled)'; + /** + * Current status of the incremental entity provider. + */ + status: + | 'complete' + | 'bursting' + | 'resting' + | 'canceling' + | 'interstitial' + | 'backing off'; + /** + * The name of the incremental entity provider being updated. + */ + provider_name: string; + /** + * Date/time stamp for when the next action will trigger. + */ + next_action_at?: Date; + /** + * A record of the last error generated by the incremental entity provider. + */ + last_error?: string | null; + /** + * The number of attempts the provider has attempted during the current cycle. + */ + attempts?: number; + /** + * Date/time stamp for the completion of ingestion. + */ + ingestion_completed_at?: Date | string | null; + /** + * Date/time stamp for the end of the rest cycle before the next ingestion. + */ + rest_completed_at?: Date | string | null; + /** + * A record of the finalized status of the ingestion record. Values are either 'open' or a uuid. + */ + completion_ticket: string; +} + +/** + * This interface is for updating an existing ingestion record. + */ +export interface IngestionRecordUpdate { + ingestionId: string; + update: Partial; +} + +/** + * The expected response from the `ingestion_marks` table. + */ +export interface MarkRecord { + id: string; + sequence: number; + ingestion_id: string; + cursor: string; + created_at: string; +} + +/** + * The expected response from the `ingestions` table. + */ +export interface IngestionRecord extends IngestionUpsert { + id: string; + next_action_at: Date; + /** + * The date/time the ingestion record was created. + */ + created_at: string; +} + +/** + * This interface supplies all the values for adding an ingestion mark. + */ +export interface MarkRecordInsert { + record: { + id: string; + ingestion_id: string; + cursor: unknown; + sequence: number; + }; +} diff --git a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts similarity index 99% rename from plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts index 922f181503..07768c08b7 100644 --- a/plugins/incremental-ingestion-backend/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts @@ -22,7 +22,6 @@ import { } from '../types'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; import type { AbortSignal } from 'node-abort-controller'; - import { performance } from 'perf_hooks'; import { Duration, DurationObjectUnits } from 'luxon'; import { v4 } from 'uuid'; diff --git a/plugins/incremental-ingestion-backend/src/index.ts b/plugins/catalog-backend-module-incremental-ingestion/src/index.ts similarity index 79% rename from plugins/incremental-ingestion-backend/src/index.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/index.ts index 7d661c2b98..84d5fcf741 100644 --- a/plugins/incremental-ingestion-backend/src/index.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/index.ts @@ -20,11 +20,12 @@ * @packageDocumentation */ -export { IncrementalCatalogBuilder } from './service/IncrementalCatalogBuilder'; -export type { - EntityIteratorResult, - IncrementalEntityProvider, - IncrementalEntityProviderOptions, +export * from './module'; +export * from './service'; +export { INCREMENTAL_ENTITY_PROVIDER_ANNOTATION, - PluginEnvironment, + type EntityIteratorResult, + type IncrementalEntityProvider, + type IncrementalEntityProviderOptions, + type PluginEnvironment, } from './types'; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts new file mode 100644 index 0000000000..986a8fbf62 --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts @@ -0,0 +1,134 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PluginDatabaseManager } from '@backstage/backend-common'; +import { Logger, loggerToWinstonLogger } from '@backstage/backend-plugin-api'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; +import { Config } from '@backstage/config'; +import { stringifyError } from '@backstage/errors'; +import { + EntityProvider, + EntityProviderConnection, +} from '@backstage/plugin-catalog-node'; +import once from 'lodash/once'; +import { Duration } from 'luxon'; +import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; +import { applyDatabaseMigrations } from '../database/migrations'; +import { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine'; +import { + IncrementalEntityProvider, + IncrementalEntityProviderOptions, +} from '../types'; +import { Deferred } from '../util'; + +const applyDatabaseMigrationsOnce = once(applyDatabaseMigrations); + +/** + * Helps in the creation of the catalog entity providers that wrap the + * incremental ones. + */ +export class WrapperProviders { + private numberOfProvidersToConnect = 0; + private readonly readySignal = new Deferred(); + + constructor( + private readonly deps: { + config: Config; + logger: Logger; + database: PluginDatabaseManager; + scheduler: PluginTaskScheduler; + }, + ) {} + + wrap( + provider: IncrementalEntityProvider, + options: IncrementalEntityProviderOptions, + ): EntityProvider { + this.numberOfProvidersToConnect += 1; + + const wrapper: EntityProvider = { + getProviderName: () => provider.getProviderName(), + connect: async connection => { + this.readySignal.then(() => + this.startProvider(provider, options, connection), + ); + + this.numberOfProvidersToConnect -= 1; + + if (this.numberOfProvidersToConnect === 0) { + this.readySignal.resolve(); + } + }, + }; + + return wrapper; + } + + private async startProvider( + provider: IncrementalEntityProvider, + options: IncrementalEntityProviderOptions, + connection: EntityProviderConnection, + ) { + const logger = loggerToWinstonLogger( + this.deps.logger.child({ + entityProvider: provider.getProviderName(), + }), + ); + + try { + const client = await this.deps.database.getClient(); + await applyDatabaseMigrationsOnce(client); + + const { burstInterval, burstLength, restLength } = options; + + logger.info(`Connecting`); + + const manager = new IncrementalIngestionDatabaseManager({ + client, + }); + const engine = new IncrementalIngestionEngine({ + ...options, + ready: this.readySignal, + manager, + logger, + provider, + restLength, + connection, + }); + + const frequency = Duration.isDuration(burstInterval) + ? burstInterval + : Duration.fromObject(burstInterval); + const length = Duration.isDuration(burstLength) + ? burstLength + : Duration.fromObject(burstLength); + + await this.deps.scheduler.scheduleTask({ + id: provider.getProviderName(), + fn: engine.taskFn.bind(engine), + frequency, + timeout: length, + }); + } catch (error) { + logger.warn( + `Failed to initialize incremental ingestion provider ${provider.getProviderName()}, ${stringifyError( + error, + )}`, + ); + throw error; + } + } +} diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts new file mode 100644 index 0000000000..80ac169f21 --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts @@ -0,0 +1,67 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + configServiceRef, + createBackendModule, + databaseServiceRef, + loggerServiceRef, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { + IncrementalEntityProvider, + IncrementalEntityProviderOptions, +} from '../types'; +import { WrapperProviders } from './WrapperProviders'; + +/** + * Registers the incremental entity provider with the catalog processing extension point. + * + * @alpha + */ +export const incrementalIngestionEntityProviderCatalogModule = + createBackendModule({ + pluginId: 'catalog', + moduleId: 'incrementalIngestionEntityProvider', + register( + env, + options: { + providers: Array<{ + provider: IncrementalEntityProvider; + options: IncrementalEntityProviderOptions; + }>; + }, + ) { + env.registerInit({ + deps: { + catalog: catalogProcessingExtensionPoint, + config: configServiceRef, + logger: loggerServiceRef, + database: databaseServiceRef, + scheduler: schedulerServiceRef, + }, + async init({ catalog, ...otherDeps }) { + const providers = new WrapperProviders(otherDeps); + for (const entry of options.providers) { + catalog.addEntityProvider( + providers.wrap(entry.provider, entry.options), + ); + } + }, + }); + }, + }); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts new file mode 100644 index 0000000000..577cf193e2 --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { incrementalIngestionEntityProviderCatalogModule } from './incrementalIngestionEntityProviderCatalogModule'; diff --git a/plugins/incremental-ingestion-backend/src/router/paths.ts b/plugins/catalog-backend-module-incremental-ingestion/src/router/paths.ts similarity index 100% rename from plugins/incremental-ingestion-backend/src/router/paths.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/router/paths.ts diff --git a/plugins/incremental-ingestion-backend/src/router/routes.ts b/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts similarity index 100% rename from plugins/incremental-ingestion-backend/src/router/routes.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts diff --git a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts similarity index 85% rename from plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts index 680c76011e..d6bc5549aa 100644 --- a/plugins/incremental-ingestion-backend/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts @@ -25,35 +25,7 @@ import { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine import { applyDatabaseMigrations } from '../database/migrations'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; import { createIncrementalProviderRouter } from '../router/routes'; - -class Deferred implements Promise { - #resolve?: (value: T) => void; - #reject?: (error: Error) => void; - - get resolve() { - return this.#resolve!; - } - get reject() { - return this.#reject!; - } - - then: Promise['then']; - catch: Promise['catch']; - finally: Promise['finally']; - - constructor() { - const promise = new Promise((resolve, reject) => { - this.#resolve = resolve; - this.#reject = reject; - }); - - this.then = promise.then.bind(promise); - this.catch = promise.catch.bind(promise); - this.finally = promise.finally.bind(promise); - } - - [Symbol.toStringTag]: 'Deferred' = 'Deferred'; -} +import { Deferred } from '../util'; /** @public */ export class IncrementalCatalogBuilder { diff --git a/plugins/incremental-ingestion-backend/src/database/tables.ts b/plugins/catalog-backend-module-incremental-ingestion/src/service/index.ts similarity index 82% rename from plugins/incremental-ingestion-backend/src/database/tables.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/service/index.ts index 6eee7e6dc5..5df92feed5 100644 --- a/plugins/incremental-ingestion-backend/src/database/tables.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/service/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2021 The Backstage Authors + * Copyright 2022 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,4 +14,4 @@ * limitations under the License. */ -export const DB_MIGRATIONS_TABLE = 'incremental_ingestion__knex_migrations'; +export { IncrementalCatalogBuilder } from './IncrementalCatalogBuilder'; diff --git a/plugins/incremental-ingestion-backend/src/types.ts b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts similarity index 67% rename from plugins/incremental-ingestion-backend/src/types.ts rename to plugins/catalog-backend-module-incremental-ingestion/src/types.ts index 47314626ed..c2603eea4e 100644 --- a/plugins/incremental-ingestion-backend/src/types.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import type { PluginDatabaseManager, UrlReader, @@ -154,104 +155,3 @@ export interface IterationEngineOptions { ready: Promise; backoff?: IncrementalEntityProviderOptions['backoff']; } - -/** - * The shape of data inserted into or updated in the `ingestions` table. - */ -export interface IngestionUpsert { - /** - * The ingestion record id. - */ - id?: string; - /** - * The next action the incremental entity provider will take. - */ - next_action: - | 'rest' - | 'ingest' - | 'backoff' - | 'cancel' - | 'nothing (done)' - | 'nothing (canceled)'; - /** - * Current status of the incremental entity provider. - */ - status: - | 'complete' - | 'bursting' - | 'resting' - | 'canceling' - | 'interstitial' - | 'backing off'; - /** - * The name of the incremental entity provider being updated. - */ - provider_name: string; - /** - * Date/time stamp for when the next action will trigger. - */ - next_action_at?: Date; - /** - * A record of the last error generated by the incremental entity provider. - */ - last_error?: string | null; - /** - * The number of attempts the provider has attempted during the current cycle. - */ - attempts?: number; - /** - * Date/time stamp for the completion of ingestion. - */ - ingestion_completed_at?: Date | string | null; - /** - * Date/time stamp for the end of the rest cycle before the next ingestion. - */ - rest_completed_at?: Date | string | null; - /** - * A record of the finalized status of the ingestion record. Values are either 'open' or a uuid. - */ - completion_ticket: string; -} - -/** - * This interface is for updating an existing ingestion record. - */ -export interface IngestionRecordUpdate { - ingestionId: string; - update: Partial; -} - -/** - * The expected response from the `ingestion_marks` table. - */ -export interface MarkRecord { - id: string; - sequence: number; - ingestion_id: string; - cursor: string; - created_at: string; -} - -/** - * The expected response from the `ingestions` table. - */ -export interface IngestionRecord extends IngestionUpsert { - id: string; - next_action_at: Date; - /** - * The date/time the ingestion record was created. - */ - created_at: string; -} - -/** - * This interface supplies all the values for adding an ingestion mark. - */ -export interface MarkRecordInsert { - record: { - id: string; - ingestion_id: string; - cursor: unknown; - sequence: number; - }; -} diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/util.ts b/plugins/catalog-backend-module-incremental-ingestion/src/util.ts new file mode 100644 index 0000000000..d9e820f826 --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/src/util.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export class Deferred implements Promise { + #resolve?: (value: T) => void; + #reject?: (error: Error) => void; + + public get resolve() { + return this.#resolve!; + } + public get reject() { + return this.#reject!; + } + + public then: Promise['then']; + public catch: Promise['catch']; + public finally: Promise['finally']; + + public constructor() { + const promise = new Promise((resolve, reject) => { + this.#resolve = resolve; + this.#reject = reject; + }); + + this.then = promise.then.bind(promise); + this.catch = promise.catch.bind(promise); + this.finally = promise.finally.bind(promise); + } + + [Symbol.toStringTag]: 'Deferred' = 'Deferred'; +} diff --git a/yarn.lock b/yarn.lock index 14274d6639..43cb63fa3c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4619,6 +4619,32 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:plugins/catalog-backend-module-incremental-ingestion": + version: 0.0.0-use.local + resolution: "@backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:plugins/catalog-backend-module-incremental-ingestion" + dependencies: + "@backstage/backend-common": "workspace:^" + "@backstage/backend-plugin-api": "workspace:^" + "@backstage/backend-tasks": "workspace:^" + "@backstage/catalog-model": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/config": "workspace:^" + "@backstage/errors": "workspace:^" + "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-permission-common": "workspace:^" + "@types/express": ^4.17.6 + "@types/luxon": ^3.0.0 + express: ^4.17.1 + express-promise-router: ^4.1.0 + knex: ^2.0.0 + lodash: ^4.17.21 + luxon: ^3.0.0 + uuid: ^8.3.2 + winston: ^3.2.1 + languageName: unknown + linkType: soft + "@backstage/plugin-catalog-backend-module-ldap@workspace:plugins/catalog-backend-module-ldap": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-ldap@workspace:plugins/catalog-backend-module-ldap" @@ -6023,31 +6049,6 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend": - version: 0.0.0-use.local - resolution: "@backstage/plugin-incremental-ingestion-backend@workspace:plugins/incremental-ingestion-backend" - dependencies: - "@backstage/backend-common": "workspace:^" - "@backstage/backend-plugin-api": "workspace:^" - "@backstage/backend-tasks": "workspace:^" - "@backstage/catalog-model": "workspace:^" - "@backstage/cli": "workspace:^" - "@backstage/config": "workspace:^" - "@backstage/errors": "workspace:^" - "@backstage/plugin-catalog-backend": "workspace:^" - "@backstage/plugin-catalog-node": "workspace:^" - "@backstage/plugin-permission-common": "workspace:^" - "@types/express": ^4.17.6 - "@types/luxon": ^3.0.0 - express: ^4.17.1 - express-promise-router: ^4.1.0 - knex: ^2.0.0 - luxon: ^3.0.0 - uuid: ^8.3.2 - winston: ^3.2.1 - languageName: unknown - linkType: soft - "@backstage/plugin-jenkins-backend@workspace:^, @backstage/plugin-jenkins-backend@workspace:plugins/jenkins-backend": version: 0.0.0-use.local resolution: "@backstage/plugin-jenkins-backend@workspace:plugins/jenkins-backend" From ed48b42925e113b33339857256758b5a21d64979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 23 Nov 2022 18:14:50 +0100 Subject: [PATCH 52/54] add module tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../package.json | 1 + .../src/module/WrapperProviders.test.ts | 124 ++++++++++++++++++ .../src/module/WrapperProviders.ts | 32 ++--- ...gestionEntityProviderCatalogModule.test.ts | 80 +++++++++++ ...talIngestionEntityProviderCatalogModule.ts | 14 +- yarn.lock | 1 + 6 files changed, 227 insertions(+), 25 deletions(-) create mode 100644 plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts create mode 100644 plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts diff --git a/plugins/catalog-backend-module-incremental-ingestion/package.json b/plugins/catalog-backend-module-incremental-ingestion/package.json index 74ed0c8e5c..46e64465be 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/package.json +++ b/plugins/catalog-backend-module-incremental-ingestion/package.json @@ -36,6 +36,7 @@ "@backstage/backend-common": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/backend-tasks": "workspace:^", + "@backstage/backend-test-utils": "workspace:^", "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts new file mode 100644 index 0000000000..5314ac769a --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts @@ -0,0 +1,124 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { PluginTaskScheduler } from '@backstage/backend-tasks'; +import { TestDatabases } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import { Knex } from 'knex'; +import '../database/migrations'; +import { IncrementalEntityProvider } from '../types'; +import { WrapperProviders } from './WrapperProviders'; + +const applyDatabaseMigrations = jest.fn(); +jest.mock('../database/migrations', () => { + const actual = jest.requireActual('../database/migrations'); + return { + applyDatabaseMigrations: (knex: Knex) => { + applyDatabaseMigrations(knex); + return actual.applyDatabaseMigrations(knex); + }, + }; +}); + +describe('WrapperProviders', () => { + const databases = TestDatabases.create({ + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + }); + const config = new ConfigReader({}); + const logger = getVoidLogger(); + const scheduler = { + scheduleTask: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it.each(databases.eachSupportedId())( + 'should initialize the providers in order, %p', + async databaseId => { + const client = await databases.init(databaseId); + + const provider1: IncrementalEntityProvider<{}, number> = { + getProviderName: () => 'provider1', + around: burst => burst(0), + next: async (cursor, _context) => { + return cursor === 0 + ? { done: false, entities: [], cursor: 1 } + : { done: true }; + }, + }; + + const provider2: IncrementalEntityProvider<{}, number> = { + getProviderName: () => 'provider2', + around: burst => burst(0), + next: async (cursor, _context) => { + return cursor === 0 + ? { done: false, entities: [], cursor: 1 } + : { done: true }; + }, + }; + + const providers = new WrapperProviders({ + config, + logger, + client, + scheduler: + scheduler as Partial as PluginTaskScheduler, + }); + const wrapped1 = providers.wrap(provider1, { + burstInterval: { seconds: 1 }, + burstLength: { seconds: 1 }, + restLength: { seconds: 1 }, + }); + const wrapped2 = providers.wrap(provider2, { + burstInterval: { seconds: 1 }, + burstLength: { seconds: 1 }, + restLength: { seconds: 1 }, + }); + + let resolved = false; + (providers as any).readySignal.then(() => { + resolved = true; + }); + + expect(applyDatabaseMigrations).toHaveBeenCalledTimes(0); + expect(resolved).toBe(false); + expect(scheduler.scheduleTask).not.toHaveBeenCalled(); + + await wrapped1.connect({} as any); // simulates the catalog engine + + expect(resolved).toBe(false); + expect(applyDatabaseMigrations).toHaveBeenCalledTimes(1); + expect(scheduler.scheduleTask).toHaveBeenLastCalledWith( + expect.objectContaining({ + id: 'provider1', + }), + ); + + await wrapped2.connect({} as any); + + expect(resolved).toBe(true); + expect(applyDatabaseMigrations).toHaveBeenCalledTimes(1); + expect(scheduler.scheduleTask).toHaveBeenLastCalledWith( + expect.objectContaining({ + id: 'provider2', + }), + ); + }, + ); +}); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts index 986a8fbf62..71baaae7cb 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import { PluginDatabaseManager } from '@backstage/backend-common'; import { Logger, loggerToWinstonLogger } from '@backstage/backend-plugin-api'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { Config } from '@backstage/config'; @@ -23,6 +22,7 @@ import { EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-node'; +import { Knex } from 'knex'; import once from 'lodash/once'; import { Duration } from 'luxon'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; @@ -45,10 +45,10 @@ export class WrapperProviders { private readonly readySignal = new Deferred(); constructor( - private readonly deps: { + private readonly options: { config: Config; logger: Logger; - database: PluginDatabaseManager; + client: Knex; scheduler: PluginTaskScheduler; }, ) {} @@ -58,49 +58,41 @@ export class WrapperProviders { options: IncrementalEntityProviderOptions, ): EntityProvider { this.numberOfProvidersToConnect += 1; - - const wrapper: EntityProvider = { + return { getProviderName: () => provider.getProviderName(), connect: async connection => { - this.readySignal.then(() => - this.startProvider(provider, options, connection), - ); - + await this.startProvider(provider, options, connection); this.numberOfProvidersToConnect -= 1; - if (this.numberOfProvidersToConnect === 0) { this.readySignal.resolve(); } }, }; - - return wrapper; } private async startProvider( provider: IncrementalEntityProvider, - options: IncrementalEntityProviderOptions, + providerOptions: IncrementalEntityProviderOptions, connection: EntityProviderConnection, ) { const logger = loggerToWinstonLogger( - this.deps.logger.child({ + this.options.logger.child({ entityProvider: provider.getProviderName(), }), ); try { - const client = await this.deps.database.getClient(); - await applyDatabaseMigrationsOnce(client); + await applyDatabaseMigrationsOnce(this.options.client); - const { burstInterval, burstLength, restLength } = options; + const { burstInterval, burstLength, restLength } = providerOptions; logger.info(`Connecting`); const manager = new IncrementalIngestionDatabaseManager({ - client, + client: this.options.client, }); const engine = new IncrementalIngestionEngine({ - ...options, + ...providerOptions, ready: this.readySignal, manager, logger, @@ -116,7 +108,7 @@ export class WrapperProviders { ? burstLength : Duration.fromObject(burstLength); - await this.deps.scheduler.scheduleTask({ + await this.options.scheduler.scheduleTask({ id: provider.getProviderName(), fn: engine.taskFn.bind(engine), frequency, diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts new file mode 100644 index 0000000000..d14a89e11a --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { + configServiceRef, + databaseServiceRef, + loggerServiceRef, + schedulerServiceRef, +} from '@backstage/backend-plugin-api'; +import { startTestBackend } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { IncrementalEntityProvider } from '../types'; +import { incrementalIngestionEntityProviderCatalogModule } from './incrementalIngestionEntityProviderCatalogModule'; + +describe('bitbucketServerEntityProviderCatalogModule', () => { + it('should register provider at the catalog extension point', async () => { + const provider1: IncrementalEntityProvider<{}, number> = { + getProviderName: () => 'provider1', + around: burst => burst(0), + next: async (cursor, _context) => { + return cursor === 0 + ? { done: false, entities: [], cursor: 1 } + : { done: true }; + }, + }; + + const addEntityProvider = jest.fn(); + + const scheduler = {}; + const database = { + getClient: jest.fn(), + }; + + await startTestBackend({ + extensionPoints: [ + [catalogProcessingExtensionPoint, { addEntityProvider }], + ], + services: [ + [configServiceRef, new ConfigReader({})], + [loggerServiceRef, getVoidLogger()], + [schedulerServiceRef, scheduler], + [databaseServiceRef, database], + ], + features: [ + incrementalIngestionEntityProviderCatalogModule({ + providers: [ + { + provider: provider1, + options: { + burstInterval: { seconds: 1 }, + burstLength: { seconds: 1 }, + restLength: { seconds: 1 }, + }, + }, + ], + }), + ], + }); + + expect(addEntityProvider).toHaveBeenCalledTimes(1); + expect(addEntityProvider.mock.calls[0][0].getProviderName()).toBe( + 'provider1', + ); + }); +}); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts index 80ac169f21..1ba75b3ed7 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts @@ -54,12 +54,16 @@ export const incrementalIngestionEntityProviderCatalogModule = database: databaseServiceRef, scheduler: schedulerServiceRef, }, - async init({ catalog, ...otherDeps }) { - const providers = new WrapperProviders(otherDeps); + async init({ catalog, config, logger, database, scheduler }) { + const providers = new WrapperProviders({ + config, + logger, + client: await database.getClient(), + scheduler, + }); for (const entry of options.providers) { - catalog.addEntityProvider( - providers.wrap(entry.provider, entry.options), - ); + const wrapped = providers.wrap(entry.provider, entry.options); + catalog.addEntityProvider(wrapped); } }, }); diff --git a/yarn.lock b/yarn.lock index 43cb63fa3c..6383165776 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4626,6 +4626,7 @@ __metadata: "@backstage/backend-common": "workspace:^" "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" From d7209f1a97f8b5ac01e377f2a6c5a2bca69ce06d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 24 Nov 2022 11:31:31 +0100 Subject: [PATCH 53/54] add dev app, more testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../package.json | 5 +- .../src/engine/IncrementalIngestionEngine.ts | 4 +- .../src/module/WrapperProviders.test.ts | 6 +- .../src/module/WrapperProviders.ts | 9 ++ ...gestionEntityProviderCatalogModule.test.ts | 13 ++- ...talIngestionEntityProviderCatalogModule.ts | 20 +++- .../src/run.ts | 99 +++++++++++++++++++ yarn.lock | 2 + 8 files changed, 146 insertions(+), 12 deletions(-) create mode 100644 plugins/catalog-backend-module-incremental-ingestion/src/run.ts diff --git a/plugins/catalog-backend-module-incremental-ingestion/package.json b/plugins/catalog-backend-module-incremental-ingestion/package.json index 46e64465be..eb68439901 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/package.json +++ b/plugins/catalog-backend-module-incremental-ingestion/package.json @@ -54,7 +54,10 @@ "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "workspace:^" + "@backstage/backend-app-api": "workspace:^", + "@backstage/cli": "workspace:^", + "@backstage/plugin-catalog-backend": "workspace:^", + "get-port": "^6.1.2" }, "files": [ "alpha", diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts index 07768c08b7..a742f63045 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts @@ -28,8 +28,8 @@ import { v4 } from 'uuid'; import { stringifyError } from '@backstage/errors'; export class IncrementalIngestionEngine implements IterationEngine { - restLength: Duration; - backoff: DurationObjectUnits[]; + private readonly restLength: Duration; + private readonly backoff: DurationObjectUnits[]; private manager: IncrementalIngestionDatabaseManager; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts index 5314ac769a..1aa0bfe2eb 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts @@ -53,17 +53,17 @@ describe('WrapperProviders', () => { async databaseId => { const client = await databases.init(databaseId); - const provider1: IncrementalEntityProvider<{}, number> = { + const provider1: IncrementalEntityProvider = { getProviderName: () => 'provider1', around: burst => burst(0), next: async (cursor, _context) => { - return cursor === 0 + return !cursor ? { done: false, entities: [], cursor: 1 } : { done: true }; }, }; - const provider2: IncrementalEntityProvider<{}, number> = { + const provider2: IncrementalEntityProvider = { getProviderName: () => 'provider2', around: burst => burst(0), next: async (cursor, _context) => { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts index 71baaae7cb..7b68051c6b 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts @@ -22,12 +22,14 @@ import { EntityProvider, EntityProviderConnection, } from '@backstage/plugin-catalog-node'; +import express from 'express'; import { Knex } from 'knex'; import once from 'lodash/once'; import { Duration } from 'luxon'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; import { applyDatabaseMigrations } from '../database/migrations'; import { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine'; +import { createIncrementalProviderRouter } from '../router/routes'; import { IncrementalEntityProvider, IncrementalEntityProviderOptions, @@ -70,6 +72,13 @@ export class WrapperProviders { }; } + async adminRouter(): Promise { + return createIncrementalProviderRouter( + new IncrementalIngestionDatabaseManager({ client: this.options.client }), + loggerToWinstonLogger(this.options.logger), + ); + } + private async startProvider( provider: IncrementalEntityProvider, providerOptions: IncrementalEntityProviderOptions, diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts index d14a89e11a..95452c9caf 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts @@ -18,6 +18,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { configServiceRef, databaseServiceRef, + httpRouterServiceRef, loggerServiceRef, schedulerServiceRef, } from '@backstage/backend-plugin-api'; @@ -29,22 +30,26 @@ import { incrementalIngestionEntityProviderCatalogModule } from './incrementalIn describe('bitbucketServerEntityProviderCatalogModule', () => { it('should register provider at the catalog extension point', async () => { - const provider1: IncrementalEntityProvider<{}, number> = { + const provider1: IncrementalEntityProvider = { getProviderName: () => 'provider1', around: burst => burst(0), next: async (cursor, _context) => { - return cursor === 0 + return !cursor ? { done: false, entities: [], cursor: 1 } : { done: true }; }, }; const addEntityProvider = jest.fn(); + const httpRouterUse = jest.fn(); const scheduler = {}; const database = { getClient: jest.fn(), }; + const httpRouter = { + use: httpRouterUse, + }; await startTestBackend({ extensionPoints: [ @@ -52,9 +57,10 @@ describe('bitbucketServerEntityProviderCatalogModule', () => { ], services: [ [configServiceRef, new ConfigReader({})], + [databaseServiceRef, database], + [httpRouterServiceRef, httpRouter], [loggerServiceRef, getVoidLogger()], [schedulerServiceRef, scheduler], - [databaseServiceRef, database], ], features: [ incrementalIngestionEntityProviderCatalogModule({ @@ -76,5 +82,6 @@ describe('bitbucketServerEntityProviderCatalogModule', () => { expect(addEntityProvider.mock.calls[0][0].getProviderName()).toBe( 'provider1', ); + expect(httpRouterUse).toHaveBeenCalledTimes(1); }); }); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts index 1ba75b3ed7..08e9653688 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts @@ -18,6 +18,7 @@ import { configServiceRef, createBackendModule, databaseServiceRef, + httpRouterServiceRef, loggerServiceRef, schedulerServiceRef, } from '@backstage/backend-plugin-api'; @@ -50,21 +51,34 @@ export const incrementalIngestionEntityProviderCatalogModule = deps: { catalog: catalogProcessingExtensionPoint, config: configServiceRef, - logger: loggerServiceRef, database: databaseServiceRef, + httpRouter: httpRouterServiceRef, + logger: loggerServiceRef, scheduler: schedulerServiceRef, }, - async init({ catalog, config, logger, database, scheduler }) { + async init({ + catalog, + config, + database, + httpRouter, + logger, + scheduler, + }) { + const client = await database.getClient(); + const providers = new WrapperProviders({ config, logger, - client: await database.getClient(), + client, scheduler, }); + for (const entry of options.providers) { const wrapped = providers.wrap(entry.provider, entry.options); catalog.addEntityProvider(wrapped); } + + httpRouter.use(await providers.adminRouter()); }, }); }, diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts new file mode 100644 index 0000000000..22dc0c97a2 --- /dev/null +++ b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts @@ -0,0 +1,99 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// eslint-disable-next-line import/no-extraneous-dependencies +import { + databaseFactory, + discoveryFactory, + httpRouterFactory, + lifecycleFactory, + loggerFactory, + permissionsFactory, + rootLoggerFactory, + schedulerFactory, + tokenManagerFactory, + urlReaderFactory, +} from '@backstage/backend-app-api'; +import { configServiceRef } from '@backstage/backend-plugin-api'; +import { startTestBackend } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import { catalogPlugin } from '@backstage/plugin-catalog-backend'; +import { + IncrementalEntityProvider, + incrementalIngestionEntityProviderCatalogModule, +} from '.'; + +const provider: IncrementalEntityProvider = { + getProviderName: () => 'test-provider', + around: burst => burst(0), + next: async (_context, cursor) => { + await new Promise(resolve => setTimeout(resolve, 500)); + if (cursor === undefined || cursor < 3) { + console.log(`### Returning batch #${cursor}`); + return { done: false, entities: [], cursor: (cursor ?? 0) + 1 }; + } + + console.log('### Last batch reached, stopping'); + return { done: true }; + }, +}; + +async function main() { + const config = { + backend: { + baseUrl: 'http://localhost:7007', + listen: ':7007', + database: { client: 'better-sqlite3', connection: ':memory:' }, + }, + }; + + await startTestBackend({ + services: [ + [configServiceRef, new ConfigReader(config)], + databaseFactory(), + discoveryFactory(), + httpRouterFactory(), + lifecycleFactory(), + loggerFactory(), + permissionsFactory(), + rootLoggerFactory(), + schedulerFactory(), + tokenManagerFactory(), + urlReaderFactory(), + ], + extensionPoints: [], + features: [ + catalogPlugin(), + incrementalIngestionEntityProviderCatalogModule({ + providers: [ + { + provider: provider, + options: { + burstInterval: { seconds: 1 }, + burstLength: { seconds: 10 }, + restLength: { seconds: 10 }, + }, + }, + ], + }), + ], + }); +} + +main().catch(error => { + console.error(error.stack); + process.exit(1); +}); diff --git a/yarn.lock b/yarn.lock index 6383165776..9c987ad139 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4623,6 +4623,7 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:plugins/catalog-backend-module-incremental-ingestion" dependencies: + "@backstage/backend-app-api": "workspace:^" "@backstage/backend-common": "workspace:^" "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-tasks": "workspace:^" @@ -4638,6 +4639,7 @@ __metadata: "@types/luxon": ^3.0.0 express: ^4.17.1 express-promise-router: ^4.1.0 + get-port: ^6.1.2 knex: ^2.0.0 lodash: ^4.17.21 luxon: ^3.0.0 From 2b8d1d37f041d40ef81cfbb13f5692c33663882f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 24 Nov 2022 16:22:17 +0100 Subject: [PATCH 54/54] fix racy test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .../src/module/WrapperProviders.test.ts | 23 +++++-------------- .../src/module/WrapperProviders.ts | 15 ++++++++---- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts index 1aa0bfe2eb..0d910f664d 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts @@ -18,23 +18,11 @@ import { getVoidLogger } from '@backstage/backend-common'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { TestDatabases } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; -import { Knex } from 'knex'; -import '../database/migrations'; import { IncrementalEntityProvider } from '../types'; import { WrapperProviders } from './WrapperProviders'; -const applyDatabaseMigrations = jest.fn(); -jest.mock('../database/migrations', () => { - const actual = jest.requireActual('../database/migrations'); - return { - applyDatabaseMigrations: (knex: Knex) => { - applyDatabaseMigrations(knex); - return actual.applyDatabaseMigrations(knex); - }, - }; -}); - describe('WrapperProviders', () => { + const applyDatabaseMigrations = jest.fn(); const databases = TestDatabases.create({ ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); @@ -45,7 +33,7 @@ describe('WrapperProviders', () => { }; beforeEach(() => { - jest.clearAllMocks(); + jest.resetAllMocks(); }); it.each(databases.eachSupportedId())( @@ -56,7 +44,7 @@ describe('WrapperProviders', () => { const provider1: IncrementalEntityProvider = { getProviderName: () => 'provider1', around: burst => burst(0), - next: async (cursor, _context) => { + next: async (_context, cursor) => { return !cursor ? { done: false, entities: [], cursor: 1 } : { done: true }; @@ -66,8 +54,8 @@ describe('WrapperProviders', () => { const provider2: IncrementalEntityProvider = { getProviderName: () => 'provider2', around: burst => burst(0), - next: async (cursor, _context) => { - return cursor === 0 + next: async (_context, cursor) => { + return !cursor ? { done: false, entities: [], cursor: 1 } : { done: true }; }, @@ -79,6 +67,7 @@ describe('WrapperProviders', () => { client, scheduler: scheduler as Partial as PluginTaskScheduler, + applyDatabaseMigrations, }); const wrapped1 = providers.wrap(provider1, { burstInterval: { seconds: 1 }, diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts index 7b68051c6b..44da5067fd 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts @@ -24,7 +24,6 @@ import { } from '@backstage/plugin-catalog-node'; import express from 'express'; import { Knex } from 'knex'; -import once from 'lodash/once'; import { Duration } from 'luxon'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; import { applyDatabaseMigrations } from '../database/migrations'; @@ -36,13 +35,12 @@ import { } from '../types'; import { Deferred } from '../util'; -const applyDatabaseMigrationsOnce = once(applyDatabaseMigrations); - /** * Helps in the creation of the catalog entity providers that wrap the * incremental ones. */ export class WrapperProviders { + private migrate: Promise | undefined; private numberOfProvidersToConnect = 0; private readonly readySignal = new Deferred(); @@ -52,6 +50,7 @@ export class WrapperProviders { logger: Logger; client: Knex; scheduler: PluginTaskScheduler; + applyDatabaseMigrations?: typeof applyDatabaseMigrations; }, ) {} @@ -91,7 +90,15 @@ export class WrapperProviders { ); try { - await applyDatabaseMigrationsOnce(this.options.client); + if (!this.migrate) { + this.migrate = Promise.resolve().then(async () => { + const apply = + this.options.applyDatabaseMigrations ?? applyDatabaseMigrations; + await apply(this.options.client); + }); + } + + await this.migrate; const { burstInterval, burstLength, restLength } = providerOptions;