From 85f46cf1ff21714d650e5e0785aad69f844ffd73 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Fri, 30 Jun 2023 14:16:20 -0700 Subject: [PATCH 1/3] Update README.md Correct README to include correct catalog API endpoint and example code Signed-off-by: Taras Mankovski --- .../README.md | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/README.md b/plugins/catalog-backend-module-incremental-ingestion/README.md index 6ba3e7c3ca..1172980102 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/README.md +++ b/plugins/catalog-backend-module-incremental-ingestion/README.md @@ -101,16 +101,16 @@ If you want to manage your incremental entity providers via REST endpoints, the | Method | Path | Description | | ------ | ------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | -| GET | `/incremental/health` | Checks the health of all incremental providers. Returns array of any unhealthy ones. | -| GET | `/incremental/providers` | Get a list of all known incremental entity providers | -| GET | `/incremental/providers/:provider` | Checks the status of an incremental provider (resting, interstitial, etc). | -| POST | `/incremental/providers/:provider/trigger` | Triggers a provider's next action immediately. E.g., if it's currently interstitial, it will trigger the next burst. | -| POST | `/incremental/providers/:provider/start` | Stop the current ingestion cycle and start a new one immediately. | -| POST | `/incremental/providers/:provider/cancel` | Stop the current ingestion cycle and start a new one in 24 hours. | -| DELETE | `/incremental/providers/:provider` | Completely remove all records for the provider and schedule it to start again in 24 hours. | -| GET | `/incremental/providers/:provider/marks` | Retrieve a list of all ingestion marks for the current ingestion cycle. | -| DELETE | `/incremental/providers/:provider/marks` | Remove all ingestion marks for the current ingestion cycle. | -| POST | `/incremental/cleanup` | Completely remove all records for ALL providers and schedule them to start again in 24 hours. (CAUTION! Can cause orphans!) | +| GET | `/api/catalog/incremental/health` | Checks the health of all incremental providers. Returns array of any unhealthy ones. | +| GET | `/api/catalog/incremental/providers` | Get a list of all known incremental entity providers | +| GET | `/api/catalog/incremental/providers/:provider` | Checks the status of an incremental provider (resting, interstitial, etc). | +| POST | `/api/catalog/incremental/providers/:provider/trigger` | Triggers a provider's next action immediately. E.g., if it's currently interstitial, it will trigger the next burst. | +| POST | `/api/catalog/incremental/providers/:provider/start` | Stop the current ingestion cycle and start a new one immediately. | +| POST | `/api/catalog/incremental/providers/:provider/cancel` | Stop the current ingestion cycle and start a new one in 24 hours. | +| DELETE | `/api/catalog/incremental/providers/:provider` | Completely remove all records for the provider and schedule it to start again in 24 hours. | +| GET | `/api/catalog/incremental/providers/:provider/marks` | Retrieve a list of all ingestion marks for the current ingestion cycle. | +| DELETE | `/api/catalog/incremental/providers/:provider/marks` | Remove all ingestion marks for the current ingestion cycle. | +| POST | `/api/catalog/incremental/cleanup` | Completely remove all records for ALL providers and schedule them to start again in 24 hours. (CAUTION! Can cause orphans!) | In all cases, `:provider` is the name of the incremental entity provider. @@ -323,21 +323,21 @@ incrementalBuilder.addIncrementalEntityProvider(myEntityProvider, { // How long should it attempt to read pages from the API in a // single burst? Keep this short. The Incremental Entity Provider // will attempt to read as many pages as it can in this time - burstLength: Duration.fromObject({ seconds: 3 }), + burstLength: { seconds: 3 }, // How long should it wait between bursts? - burstInterval: Duration.fromObject({ seconds: 3 }), + burstInterval: { seconds: 3 }, // How long should it rest before re-ingesting again? - restLength: Duration.fromObject({ day: 1 }), + restLength: { day: 1 }, // Optional back-off configuration - how long should it wait to retry // in the event of an error? backoff: [ - Duration.fromObject({ seconds: 5 }), - Duration.fromObject({ seconds: 30 }), - Duration.fromObject({ minutes: 10 }), - Duration.fromObject({ hours: 3 }), + { seconds: 5 }, + { seconds: 30 }, + { minutes: 10 }, + { hours: 3 }, ], // Optional. Use this to prevent removal of entities above a given From e1d615757f486cdafcde3ad7e544bca34f193e62 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 4 Jul 2023 11:02:04 +0200 Subject: [PATCH 2/3] chore: add changeset Signed-off-by: blam --- .changeset/pretty-apes-know.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/pretty-apes-know.md diff --git a/.changeset/pretty-apes-know.md b/.changeset/pretty-apes-know.md new file mode 100644 index 0000000000..983c78a592 --- /dev/null +++ b/.changeset/pretty-apes-know.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-incremental-ingestion': patch +--- + +Update readme and instructions From 2acc8ee4e4dfc9a9fed4cbdec6cdb139e2978c06 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 4 Jul 2023 11:21:45 +0200 Subject: [PATCH 3/3] chore: make pretty Signed-off-by: blam --- .../README.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/README.md b/plugins/catalog-backend-module-incremental-ingestion/README.md index 1172980102..3728121540 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/README.md +++ b/plugins/catalog-backend-module-incremental-ingestion/README.md @@ -99,8 +99,8 @@ export default async function createPlugin( If you want to manage your incremental entity providers via REST endpoints, the following endpoints are available: -| Method | Path | Description | -| ------ | ------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | +| Method | Path | Description | +| ------ | ------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | | GET | `/api/catalog/incremental/health` | Checks the health of all incremental providers. Returns array of any unhealthy ones. | | GET | `/api/catalog/incremental/providers` | Get a list of all known incremental entity providers | | GET | `/api/catalog/incremental/providers/:provider` | Checks the status of an incremental provider (resting, interstitial, etc). | @@ -333,12 +333,7 @@ incrementalBuilder.addIncrementalEntityProvider(myEntityProvider, { // Optional back-off configuration - how long should it wait to retry // in the event of an error? - backoff: [ - { seconds: 5 }, - { seconds: 30 }, - { minutes: 10 }, - { hours: 3 }, - ], + backoff: [{ seconds: 5 }, { seconds: 30 }, { minutes: 10 }, { hours: 3 }], // Optional. Use this to prevent removal of entities above a given // percentage. This can be helpful if a data source is flaky and