From 0ee83a57c75c01ef1ed4274952cec8050ff5ffcc Mon Sep 17 00:00:00 2001 From: Taras Date: Thu, 23 Feb 2023 20:04:34 -0500 Subject: [PATCH 1/6] Removed paths file Signed-off-by: Taras --- .../src/router/paths.ts | 19 ----- .../src/router/routes.ts | 79 ++++++++++--------- 2 files changed, 42 insertions(+), 56 deletions(-) delete mode 100644 plugins/catalog-backend-module-incremental-ingestion/src/router/paths.ts diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/router/paths.ts b/plugins/catalog-backend-module-incremental-ingestion/src/router/paths.ts deleted file mode 100644 index 2118177821..0000000000 --- a/plugins/catalog-backend-module-incremental-ingestion/src/router/paths.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export const PROVIDER_CLEANUP = '/incremental/cleanup'; -export const PROVIDER_HEALTH = '/incremental/health'; -export const PROVIDER_BASE_PATH = '/incremental/providers/:provider'; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts b/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts index fc26abdf32..76fce4af75 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts @@ -19,7 +19,6 @@ import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; -import { PROVIDER_BASE_PATH, PROVIDER_CLEANUP, PROVIDER_HEALTH } from './paths'; export class IncrementalProviderRouter { private manager: IncrementalIngestionDatabaseManager; @@ -35,7 +34,7 @@ export class IncrementalProviderRouter { router.use(express.json()); // Get the overall health of all incremental providers - router.get(PROVIDER_HEALTH, async (_, res) => { + router.get('/incremental/health', async (_, res) => { const records = await this.manager.healthcheck(); const providers = records.map(record => record.provider_name); const duplicates = [ @@ -50,13 +49,13 @@ export class IncrementalProviderRouter { }); // Clean up and pause all providers - router.post(PROVIDER_CLEANUP, async (_, res) => { + router.post('/incremental/health', async (_, res) => { const result = await this.manager.cleanupProviders(); res.json(result); }); // Get basic status of the provider - router.get(PROVIDER_BASE_PATH, async (req, res) => { + router.get('/incremental/providers/:provider', async (req, res) => { const { provider } = req.params; const record = await this.manager.getCurrentIngestionRecord(provider); if (record) { @@ -91,35 +90,38 @@ export class IncrementalProviderRouter { }); // Trigger the provider's next action - router.post(`${PROVIDER_BASE_PATH}/trigger`, async (req, res) => { - const { provider } = req.params; - const record = await this.manager.getCurrentIngestionRecord(provider); - if (record) { - await this.manager.triggerNextProviderAction(provider); - res.json({ - success: true, - message: `${provider}: Next action triggered.`, - }); - } else { - const providers: string[] = await this.manager.listProviders(); - if (providers.includes(provider)) { - this.logger.debug(`${provider} - Ingestion record found`); + router.post( + `/incremental/providers/:provider/trigger`, + async (req, res) => { + const { provider } = req.params; + const record = await this.manager.getCurrentIngestionRecord(provider); + if (record) { + await this.manager.triggerNextProviderAction(provider); res.json({ success: true, - message: 'Unable to trigger next action (provider is restarting)', + message: `${provider}: Next action triggered.`, }); } else { - res.status(404).json({ - success: false, - message: `Provider '${provider}' not found`, - }); + const providers: string[] = await this.manager.listProviders(); + if (providers.includes(provider)) { + this.logger.debug(`${provider} - Ingestion record found`); + res.json({ + success: true, + message: 'Unable to trigger next action (provider is restarting)', + }); + } else { + res.status(404).json({ + success: false, + message: `Provider '${provider}' not found`, + }); + } } - } - }); + }, + ); // Start a brand-new ingestion cycle for the provider. // (Cancel's the current run if active, or marks it complete if resting) - router.post(`${PROVIDER_BASE_PATH}/start`, async (req, res) => { + router.post(`/incremental/providers/:provider/start`, async (req, res) => { const { provider } = req.params; const record = await this.manager.getCurrentIngestionRecord(provider); @@ -152,7 +154,7 @@ export class IncrementalProviderRouter { }); // Stop the provider and pause it for 24 hours - router.post(`${PROVIDER_BASE_PATH}/cancel`, async (req, res) => { + router.post(`/incremental/providers/:provider/cancel`, async (req, res) => { const { provider } = req.params; const record = await this.manager.getCurrentIngestionRecord(provider); if (record) { @@ -186,14 +188,14 @@ export class IncrementalProviderRouter { }); // Wipe out all ingestion records for the provider and pause for 24 hours - router.delete(PROVIDER_BASE_PATH, async (req, res) => { + router.delete('/incremental/providers/:provider', async (req, res) => { const { provider } = req.params; const result = await this.manager.purgeAndResetProvider(provider); res.json(result); }); // Get the ingestion marks for the current cycle - router.get(`${PROVIDER_BASE_PATH}/marks`, async (req, res) => { + router.get(`$/incremental/providers/:provider/marks`, async (req, res) => { const { provider } = req.params; const record = await this.manager.getCurrentIngestionRecord(provider); if (record) { @@ -221,16 +223,19 @@ export class IncrementalProviderRouter { } }); - router.delete(`${PROVIDER_BASE_PATH}/marks`, async (req, res) => { - const { provider } = req.params; - const deletions = await this.manager.clearFinishedIngestions(provider); + router.delete( + `/incremental/providers/:provider/marks`, + async (req, res) => { + const { provider } = req.params; + const deletions = await this.manager.clearFinishedIngestions(provider); - res.json({ - success: true, - message: `Expired marks for provider '${provider}' removed.`, - deletions, - }); - }); + res.json({ + success: true, + message: `Expired marks for provider '${provider}' removed.`, + deletions, + }); + }, + ); router.use(errorHandler()); From 655feeceb95ca7ff65842b29965905c28839b867 Mon Sep 17 00:00:00 2001 From: Taras Date: Thu, 23 Feb 2023 20:14:03 -0500 Subject: [PATCH 2/6] Added GET providers endpoint Signed-off-by: Taras --- .../src/router/routes.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts b/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts index 76fce4af75..7fd40db674 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts @@ -153,6 +153,15 @@ export class IncrementalProviderRouter { } }); + router.get(`/incremental/providers/`, async (_req, res) => { + const providers = await this.manager.listProviders(); + + res.json({ + success: true, + providers, + }); + }); + // Stop the provider and pause it for 24 hours router.post(`/incremental/providers/:provider/cancel`, async (req, res) => { const { provider } = req.params; From a811bd246c4b8f0fba4a218d3a2d8e01679c8043 Mon Sep 17 00:00:00 2001 From: Taras Date: Thu, 23 Feb 2023 20:15:58 -0500 Subject: [PATCH 3/6] Added changset Signed-off-by: Taras --- .changeset/forty-snails-clap.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/forty-snails-clap.md diff --git a/.changeset/forty-snails-clap.md b/.changeset/forty-snails-clap.md new file mode 100644 index 0000000000..f5aef12d9d --- /dev/null +++ b/.changeset/forty-snails-clap.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-incremental-ingestion': minor +--- + +Added endpoint to get a list of known incremental entity providers From 754da685fb13f21cd8e49a6266654fa691c2526a Mon Sep 17 00:00:00 2001 From: Taras Date: Thu, 23 Feb 2023 20:19:44 -0500 Subject: [PATCH 4/6] Updated README Signed-off-by: Taras --- plugins/catalog-backend-module-incremental-ingestion/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-backend-module-incremental-ingestion/README.md b/plugins/catalog-backend-module-incremental-ingestion/README.md index 7fe36eeec7..f85321e613 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/README.md +++ b/plugins/catalog-backend-module-incremental-ingestion/README.md @@ -102,6 +102,7 @@ If you want to manage your incremental entity providers via REST endpoints, the | Method | Path | Description | | ------ | ------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | | GET | `/incremental/health` | Checks the health of all incremental providers. Returns array of any unhealthy ones. | +| GET | `/incremental/providers` | Get a list of all known incremental entity providers | | GET | `/incremental/providers/:provider` | Checks the status of an incremental provider (resting, interstitial, etc). | | POST | `/incremental/providers/:provider/trigger` | Triggers a provider's next action immediately. E.g., if it's currently interstitial, it will trigger the next burst. | | POST | `/incremental/providers/:provider/start` | Stop the current ingestion cycle and start a new one immediately. | From fb0a7577cbad6a2eb131a234333ab9e3c0d13fcb Mon Sep 17 00:00:00 2001 From: Taras Date: Thu, 23 Feb 2023 20:21:35 -0500 Subject: [PATCH 5/6] Correct paths Signed-off-by: Taras --- .../src/router/routes.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts b/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts index 7fd40db674..0dd89468bd 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts @@ -49,7 +49,7 @@ export class IncrementalProviderRouter { }); // Clean up and pause all providers - router.post('/incremental/health', async (_, res) => { + router.post('/incremental/cleanup', async (_, res) => { const result = await this.manager.cleanupProviders(); res.json(result); }); @@ -204,7 +204,7 @@ export class IncrementalProviderRouter { }); // Get the ingestion marks for the current cycle - router.get(`$/incremental/providers/:provider/marks`, async (req, res) => { + router.get(`/incremental/providers/:provider/marks`, async (req, res) => { const { provider } = req.params; const record = await this.manager.getCurrentIngestionRecord(provider); if (record) { From fb3d0b3af9144c6cece8ec23496885857a266039 Mon Sep 17 00:00:00 2001 From: Taras Mankovski Date: Fri, 24 Feb 2023 09:21:29 -0500 Subject: [PATCH 6/6] Update plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: Taras Mankovski --- .../src/router/routes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts b/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts index 0dd89468bd..4a89467a93 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts @@ -153,7 +153,7 @@ export class IncrementalProviderRouter { } }); - router.get(`/incremental/providers/`, async (_req, res) => { + router.get(`/incremental/providers`, async (_req, res) => { const providers = await this.manager.listProviders(); res.json({