paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 24 Packages Projects Releases Wiki Activity

Merge pull request #16578 from thefrontside/tm/add-ingestion-entity-provides-route

Browse Source 
Add GET providers endpoint to Incremental Entity Provider admin routes
This commit is contained in:
Fredrik Adelöw
2023-02-27 14:18:26 +01:00
committed by GitHub
parent babbaa9391 fb3d0b3af9
commit d820a763ab
4 changed files with 57 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/forty-snails-clap.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-incremental-ingestion': minor
---
Added endpoint to get a list of known incremental entity providers
plugins/catalog-backend-module-incremental-ingestion/README.md
+1
View File
@@ -102,6 +102,7 @@ If you want to manage your incremental entity providers via REST endpoints, the
| Method | Path | Description |
| ------ | ------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- |
| GET | `/incremental/health` | Checks the health of all incremental providers. Returns array of any unhealthy ones. |
| GET | `/incremental/providers` | Get a list of all known incremental entity providers |
| GET | `/incremental/providers/:provider` | Checks the status of an incremental provider (resting, interstitial, etc). |
| POST | `/incremental/providers/:provider/trigger` | Triggers a provider's next action immediately. E.g., if it's currently interstitial, it will trigger the next burst. |
| POST | `/incremental/providers/:provider/start` | Stop the current ingestion cycle and start a new one immediately. |
plugins/catalog-backend-module-incremental-ingestion/src/router/paths.ts
-19
View File
@@ -1,19 +0,0 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export const PROVIDER_CLEANUP = '/incremental/cleanup';
export const PROVIDER_HEALTH = '/incremental/health';
export const PROVIDER_BASE_PATH = '/incremental/providers/:provider';
plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts
+51 -37
View File
@@ -19,7 +19,6 @@ import express from 'express';
import Router from 'express-promise-router';
import { Logger } from 'winston';
import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager';
import { PROVIDER_BASE_PATH, PROVIDER_CLEANUP, PROVIDER_HEALTH } from './paths';
export class IncrementalProviderRouter {
private manager: IncrementalIngestionDatabaseManager;
@@ -35,7 +34,7 @@ export class IncrementalProviderRouter {
router.use(express.json());
// Get the overall health of all incremental providers
router.get(PROVIDER_HEALTH, async (_, res) => {
router.get('/incremental/health', async (_, res) => {
const records = await this.manager.healthcheck();
const providers = records.map(record => record.provider_name);
const duplicates = [
@@ -50,13 +49,13 @@ export class IncrementalProviderRouter {
});
// Clean up and pause all providers
router.post(PROVIDER_CLEANUP, async (_, res) => {
router.post('/incremental/cleanup', async (_, res) => {
const result = await this.manager.cleanupProviders();
res.json(result);
});
// Get basic status of the provider
router.get(PROVIDER_BASE_PATH, async (req, res) => {
router.get('/incremental/providers/:provider', async (req, res) => {
const { provider } = req.params;
const record = await this.manager.getCurrentIngestionRecord(provider);
if (record) {
@@ -91,35 +90,38 @@ export class IncrementalProviderRouter {
});
// Trigger the provider's next action
router.post(`${PROVIDER_BASE_PATH}/trigger`, async (req, res) => {
const { provider } = req.params;
const record = await this.manager.getCurrentIngestionRecord(provider);
if (record) {
await this.manager.triggerNextProviderAction(provider);
res.json({
success: true,
message: `${provider}: Next action triggered.`,
});
} else {
const providers: string[] = await this.manager.listProviders();
if (providers.includes(provider)) {
this.logger.debug(`${provider} - Ingestion record found`);
router.post(
`/incremental/providers/:provider/trigger`,
async (req, res) => {
const { provider } = req.params;
const record = await this.manager.getCurrentIngestionRecord(provider);
if (record) {
await this.manager.triggerNextProviderAction(provider);
res.json({
success: true,
message: 'Unable to trigger next action (provider is restarting)',
message: `${provider}: Next action triggered.`,
});
} else {
res.status(404).json({
success: false,
message: `Provider '${provider}' not found`,
});
const providers: string[] = await this.manager.listProviders();
if (providers.includes(provider)) {
this.logger.debug(`${provider} - Ingestion record found`);
res.json({
success: true,
message: 'Unable to trigger next action (provider is restarting)',
});
} else {
res.status(404).json({
success: false,
message: `Provider '${provider}' not found`,
});
}
}
}
});
},
);
// Start a brand-new ingestion cycle for the provider.
// (Cancel's the current run if active, or marks it complete if resting)
router.post(`${PROVIDER_BASE_PATH}/start`, async (req, res) => {
router.post(`/incremental/providers/:provider/start`, async (req, res) => {
const { provider } = req.params;
const record = await this.manager.getCurrentIngestionRecord(provider);
@@ -151,8 +153,17 @@ export class IncrementalProviderRouter {
}
});
router.get(`/incremental/providers`, async (_req, res) => {
const providers = await this.manager.listProviders();
res.json({
success: true,
providers,
});
});
// Stop the provider and pause it for 24 hours
router.post(`${PROVIDER_BASE_PATH}/cancel`, async (req, res) => {
router.post(`/incremental/providers/:provider/cancel`, async (req, res) => {
const { provider } = req.params;
const record = await this.manager.getCurrentIngestionRecord(provider);
if (record) {
@@ -186,14 +197,14 @@ export class IncrementalProviderRouter {
});
// Wipe out all ingestion records for the provider and pause for 24 hours
router.delete(PROVIDER_BASE_PATH, async (req, res) => {
router.delete('/incremental/providers/:provider', async (req, res) => {
const { provider } = req.params;
const result = await this.manager.purgeAndResetProvider(provider);
res.json(result);
});
// Get the ingestion marks for the current cycle
router.get(`${PROVIDER_BASE_PATH}/marks`, async (req, res) => {
router.get(`/incremental/providers/:provider/marks`, async (req, res) => {
const { provider } = req.params;
const record = await this.manager.getCurrentIngestionRecord(provider);
if (record) {
@@ -221,16 +232,19 @@ export class IncrementalProviderRouter {
}
});
router.delete(`${PROVIDER_BASE_PATH}/marks`, async (req, res) => {
const { provider } = req.params;
const deletions = await this.manager.clearFinishedIngestions(provider);
router.delete(
`/incremental/providers/:provider/marks`,
async (req, res) => {
const { provider } = req.params;
const deletions = await this.manager.clearFinishedIngestions(provider);
res.json({
success: true,
message: `Expired marks for provider '${provider}' removed.`,
deletions,
});
});
res.json({
success: true,
message: `Expired marks for provider '${provider}' removed.`,
deletions,
});
},
);
router.use(errorHandler());