Merge pull request #22667 from kurtaking/unprocessed-entities-api

feat: delete unprocessed entity from refresh_state table

.changeset/warm-buses-do.md

@@ -0,0 +1,28 @@
+---
+'@backstage/plugin-catalog-backend-module-unprocessed': minor
+'@backstage/plugin-catalog-unprocessed-entities': minor
+'@backstage/plugin-catalog-unprocessed-entities-common': patch
+---
+
+**BREAKING**- the `@backstage/plugin-catalog-backend-module-unprocessed` constructor is now private, and have been moved to using the static `.create` method instead which now requires a `PermissionService` and `DiscoveryService`.
+
+If you're using this module in the old backend system you'll need to migrate to using the `.create` method and pass in the new required parameters in `packages/backend/src/plugins/catalog.ts`.
+
+No changes should be required if you're using the new backend system.
+
+```diff
+-  const unprocessed = new UnprocessedEntitiesModule(
+-    await env.database.getClient(),
+-    router,
+-  );
++ const unprocessed = UnprocessedEntitiesModule.create({
++    database: await env.database.getClient(),
++    router,
++    permissions: env.permissions,
++    discovery: env.discovery,
++  });
+
+  unprocessed.registerRoutes();
+```
+
+Adds the ability to delete an unprocessed entity from the `refresh_state` table. This change requires enabling permissions for your Backstage instance.

packages/backend/src/plugins/catalog.ts

@@ -37,11 +37,15 @@ export default async function createPlugin(
 
   const { processingEngine, router } = await builder.build();
 
-  const unprocessed = new UnprocessedEntitiesModule(
-    await env.database.getClient(),
+  const unprocessed = UnprocessedEntitiesModule.create({
+    database: await env.database.getClient(),
     router,
-  );
+    permissions: env.permissions,
+    discovery: env.discovery,
+  });
+
   unprocessed.registerRoutes();
+
   await processingEngine.start();
   return router;
 }

plugins/catalog-backend-module-unprocessed/api-report.md

@@ -4,8 +4,11 @@
 
 ```ts
 import { BackendFeature } from '@backstage/backend-plugin-api';
+import { DiscoveryService } from '@backstage/backend-plugin-api';
+import { HttpAuthService } from '@backstage/backend-plugin-api';
 import { HttpRouterService } from '@backstage/backend-plugin-api';
 import { Knex } from 'knex';
+import { PermissionsService } from '@backstage/backend-plugin-api';
 
 // @public
 const catalogModuleUnprocessedEntities: () => BackendFeature;
@@ -13,7 +16,14 @@ export default catalogModuleUnprocessedEntities;
 
 // @public
 export class UnprocessedEntitiesModule {
-  constructor(database: Knex, router: Pick<HttpRouterService, 'use'>);
+  // (undocumented)
+  static create(options: {
+    router: Pick<HttpRouterService, 'use'>;
+    database: Knex;
+    discovery: DiscoveryService;
+    permissions: PermissionsService;
+    httpAuth?: HttpAuthService;
+  }): UnprocessedEntitiesModule;
   // (undocumented)
   registerRoutes(): void;
 }

plugins/catalog-backend-module-unprocessed/package.json

@@ -28,14 +28,21 @@
     "postpack": "backstage-cli package postpack"
   },
   "devDependencies": {
-    "@backstage/cli": "workspace:^"
+    "@backstage/cli": "workspace:^",
+    "@types/express": "^4.17.6"
   },
   "files": [
     "dist"
   ],
   "dependencies": {
+    "@backstage/backend-common": "workspace:^",
     "@backstage/backend-plugin-api": "workspace:^",
     "@backstage/catalog-model": "workspace:^",
+    "@backstage/errors": "workspace:^",
+    "@backstage/plugin-auth-node": "workspace:^",
+    "@backstage/plugin-catalog-unprocessed-entities-common": "workspace:^",
+    "@backstage/plugin-permission-common": "workspace:^",
+    "@backstage/plugin-permission-node": "workspace:^",
     "express-promise-router": "^4.1.1",
     "knex": "^3.0.0"
   }

plugins/catalog-backend-module-unprocessed/src/UnprocessedEntitiesModule.ts

@@ -21,8 +21,23 @@ import {
   UnprocessedEntitiesResponse,
 } from './types';
 import { Knex } from 'knex';
-import { HttpRouterService } from '@backstage/backend-plugin-api';
+import {
+  DiscoveryService,
+  HttpAuthService,
+  HttpRouterService,
+  PermissionsService,
+} from '@backstage/backend-plugin-api';
 import Router from 'express-promise-router';
+import type { Request } from 'express';
+
+import {
+  AuthorizeResult,
+  BasicPermission,
+} from '@backstage/plugin-permission-common';
+import { createPermissionIntegrationRouter } from '@backstage/plugin-permission-node';
+import { unprocessedEntitiesDeletePermission } from '@backstage/plugin-catalog-unprocessed-entities-common';
+import { NotAllowedError } from '@backstage/errors';
+import { createLegacyAuthAdapters } from '@backstage/backend-common';
 
 /**
  * Module providing Unprocessed Entities API endpoints
@@ -32,12 +47,38 @@ import Router from 'express-promise-router';
 export class UnprocessedEntitiesModule {
   private readonly moduleRouter;
 
-  constructor(
+  private readonly httpAuth: HttpAuthService;
+
+  private constructor(
     private readonly database: Knex,
     private readonly router: Pick<HttpRouterService, 'use'>,
+    private readonly permissions: PermissionsService,
+    discovery: DiscoveryService,
+    httpAuth?: HttpAuthService,
   ) {
     this.moduleRouter = Router();
     this.router.use(this.moduleRouter);
+
+    this.httpAuth = createLegacyAuthAdapters({
+      discovery,
+      httpAuth,
+    }).httpAuth;
+  }
+
+  static create(options: {
+    router: Pick<HttpRouterService, 'use'>;
+    database: Knex;
+    discovery: DiscoveryService;
+    permissions: PermissionsService;
+    httpAuth?: HttpAuthService;
+  }) {
+    return new UnprocessedEntitiesModule(
+      options.database,
+      options.router,
+      options.permissions,
+      options.discovery,
+      options.httpAuth,
+    );
   }
 
   private async unprocessed(
@@ -104,12 +145,31 @@ export class UnprocessedEntitiesModule {
   }
 
   registerRoutes() {
+    const permissionIntegrationRouter = createPermissionIntegrationRouter({
+      permissions: [unprocessedEntitiesDeletePermission],
+    });
+
+    const isRequestAuthorized = async (
+      req: Request,
+      permission: BasicPermission,
+    ): Promise<boolean> => {
+      const decision = (
+        await this.permissions.authorize([{ permission }], {
+          credentials: await this.httpAuth.credentials(req),
+        })
+      )[0];
+
+      return decision.result !== AuthorizeResult.DENY;
+    };
+
+    this.router.use(permissionIntegrationRouter);
+
     this.moduleRouter
       .get('/entities/unprocessed/failed', async (req, res) => {
         return res.json(
           await this.unprocessed({
             reason: 'failed',
-            owner: req.query.owner as string,
+            owner: String(req.query.owner),
           }),
         );
       })
@@ -117,9 +177,28 @@ export class UnprocessedEntitiesModule {
         return res.json(
           await this.unprocessed({
             reason: 'pending',
-            owner: req.query.owner as string,
+            owner: String(req.query.owner),
           }),
         );
-      });
+      })
+      .delete(
+        '/entities/unprocessed/delete/:entity_id',
+        async (request, response) => {
+          const authorized = await isRequestAuthorized(
+            request,
+            unprocessedEntitiesDeletePermission,
+          );
+
+          if (!authorized) {
+            throw new NotAllowedError('Unauthorized');
+          }
+
+          await this.database('refresh_state')
+            .where({ entity_id: request.params.entity_id })
+            .delete();
+
+          response.status(204).send();
+        },
+      );
   }
 }

plugins/catalog-backend-module-unprocessed/src/module.ts

@@ -34,14 +34,28 @@ export const catalogModuleUnprocessedEntities = createBackendModule({
         database: coreServices.database,
         router: coreServices.httpRouter,
         logger: coreServices.logger,
+        httpAuth: coreServices.httpAuth,
+        discovery: coreServices.discovery,
+        permissions: coreServices.permissions,
       },
-      async init({ database, router, logger }) {
-        const module = new UnprocessedEntitiesModule(
-          await database.getClient(),
+      async init({
+        database,
+        router,
+        logger,
+        permissions,
+        httpAuth,
+        discovery,
+      }) {
+        const module = UnprocessedEntitiesModule.create({
+          database: await database.getClient(),
           router,
-        );
+          permissions,
+          discovery,
+          httpAuth,
+        });
 
         module.registerRoutes();
+
         logger.info(
           'registered additional routes for catalogModuleUnprocessedEntities',
         );

plugins/catalog-unprocessed-entities-common/.eslintrc.js

@@ -0,0 +1 @@
+module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);

plugins/catalog-unprocessed-entities-common/README.md

@@ -0,0 +1,5 @@
+# @backstage/plugin-catalog-unprocessed-entities-common
+
+Welcome to the common package for the catalog-unprocessed-entities plugin!
+
+_This plugin was created through the Backstage CLI_

plugins/catalog-unprocessed-entities-common/api-report.md

@@ -0,0 +1,15 @@
+## API Report File for "@backstage/plugin-catalog-unprocessed-entities-common"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+import { BasicPermission } from '@backstage/plugin-permission-common';
+
+// @public
+export const unprocessedEntitiesDeletePermission: BasicPermission;
+
+// @public
+export const unprocessedEntitiesPermissions: {
+  unprocessedEntitiesDeletePermission: BasicPermission;
+};
+```

plugins/catalog-unprocessed-entities-common/catalog-info.yaml

@@ -0,0 +1,9 @@
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  name: backstage-plugin-catalog-unprocessed-entities-common
+  title: '@backstage/plugin-catalog-unprocessed-entities-common'
+spec:
+  lifecycle: experimental
+  type: backstage-common-library
+  owner: catalog-maintainers

plugins/catalog-unprocessed-entities-common/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@backstage/plugin-catalog-unprocessed-entities-common",
+  "version": "0.0.0",
+  "description": "Common functionalities for the catalog-unprocessed-entities plugin",
+  "backstage": {
+    "role": "common-library"
+  },
+  "publishConfig": {
+    "access": "public",
+    "main": "dist/index.cjs.js",
+    "module": "dist/index.esm.js",
+    "types": "dist/index.d.ts"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/backstage/backstage",
+    "directory": "plugins/catalog-unprocessed-entities-common"
+  },
+  "license": "Apache-2.0",
+  "sideEffects": false,
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "backstage-cli package build",
+    "clean": "backstage-cli package clean",
+    "lint": "backstage-cli package lint",
+    "prepack": "backstage-cli package prepack",
+    "postpack": "backstage-cli package postpack",
+    "test": "backstage-cli package test"
+  },
+  "dependencies": {
+    "@backstage/plugin-permission-common": "workspace:^"
+  },
+  "devDependencies": {
+    "@backstage/cli": "workspace:^"
+  }
+}

plugins/catalog-unprocessed-entities-common/src/index.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Common functionalities for the catalog-unprocessed-entities plugin.
+ *
+ * @packageDocumentation
+ */
+
+export * from './permissions';

plugins/catalog-unprocessed-entities-common/src/permissions.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { createPermission } from '@backstage/plugin-permission-common';
+
+/**
+ * This permission is used to designate actions that involve removing an
+ * unprocessed entity record from the refresh_state table.
+ * @public
+ */
+export const unprocessedEntitiesDeletePermission = createPermission({
+  name: 'catalog.entities.unprocessed.delete',
+  attributes: { action: 'delete' },
+});
+
+/**
+ * List of all unprocessed entity permissions
+ * @public
+ */
+export const unprocessedEntitiesPermissions = {
+  unprocessedEntitiesDeletePermission,
+};

plugins/catalog-unprocessed-entities-common/src/setupTests.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export {};

plugins/catalog-unprocessed-entities/api-report.md

@@ -14,6 +14,7 @@ import { RouteRef } from '@backstage/core-plugin-api';
 
 // @public
 export interface CatalogUnprocessedEntitiesApi {
+  delete(entityId: string): Promise<void>;
   failed(): Promise<CatalogUnprocessedEntitiesApiResponse>;
   pending(): Promise<CatalogUnprocessedEntitiesApiResponse>;
 }

plugins/catalog-unprocessed-entities/src/api/index.ts

@@ -54,6 +54,10 @@ export interface CatalogUnprocessedEntitiesApi {
    * Returns a list of entities with state 'failed'
    */
   failed(): Promise<CatalogUnprocessedEntitiesApiResponse>;
+  /**
+   * Deletes an entity from the refresh_state table
+   */
+  delete(entityId: string): Promise<void>;
 }
 
 /**
@@ -69,11 +73,12 @@ export class CatalogUnprocessedEntitiesClient
   private async fetch<T>(path: string, init?: RequestInit): Promise<T> {
     const url = await this.discovery.getBaseUrl('catalog');
     const resp = await this.fetchApi.fetch(`${url}/${path}`, init);
+
     if (!resp.ok) {
       throw await ResponseError.fromResponse(resp);
     }
 
-    return await resp.json();
+    return resp.status === 204 ? (resp as T) : await resp.json();
   }
 
   async pending(): Promise<CatalogUnprocessedEntitiesApiResponse> {
@@ -83,4 +88,10 @@ export class CatalogUnprocessedEntitiesClient
   async failed(): Promise<CatalogUnprocessedEntitiesApiResponse> {
     return await this.fetch('entities/unprocessed/failed');
   }
+
+  async delete(entityId: string): Promise<void> {
+    await this.fetch(`entities/unprocessed/delete/${entityId}`, {
+      method: 'DELETE',
+    });
+  }
 }

plugins/catalog-unprocessed-entities/src/components/FailedEntities.tsx

@@ -22,15 +22,18 @@ import {
   Table,
   TableColumn,
 } from '@backstage/core-components';
-import { useApi } from '@backstage/core-plugin-api';
+
 import Box from '@material-ui/core/Box';
 import Typography from '@material-ui/core/Typography';
+import IconButton from '@material-ui/core/IconButton';
 import { Theme, makeStyles } from '@material-ui/core/styles';
+import { alertApiRef, useApi } from '@backstage/core-plugin-api';
 
 import { UnprocessedEntity } from '../types';
 import { EntityDialog } from './EntityDialog';
 import { catalogUnprocessedEntitiesApiRef } from '../api';
 import useAsync from 'react-use/lib/useAsync';
+import DeleteIcon from '@material-ui/icons/Delete';
 
 const useStyles = makeStyles((theme: Theme) => ({
   errorBox: {
@@ -94,6 +97,8 @@ export const FailedEntities = () => {
     value: data,
   } = useAsync(async () => await unprocessedApi.failed());
   const [, setSelectedSearchTerm] = useState<string>('');
+  const unprocessedEntityApi = useApi(catalogUnprocessedEntitiesApiRef);
+  const alertApi = useApi(alertApiRef);
 
   if (loading) {
     return <Progress />;
@@ -102,6 +107,27 @@ export const FailedEntities = () => {
     return <ErrorPanel error={error} />;
   }
 
+  const handleDelete = async ({
+    entityId,
+    entityRef,
+  }: {
+    entityId: string;
+    entityRef: string;
+  }) => {
+    try {
+      await unprocessedEntityApi.delete(entityId);
+      alertApi.post({
+        message: `Entity ${entityRef} has been deleted`,
+        severity: 'success',
+      });
+    } catch (e) {
+      alertApi.post({
+        message: `Ran into an issue when deleting ${entityRef}. Please try again later.`,
+        severity: 'error',
+      });
+    }
+  };
+
   const columns: TableColumn[] = [
     {
       title: <Typography>entityRef</Typography>,
@@ -136,43 +162,60 @@ export const FailedEntities = () => {
         <EntityDialog entity={rowData as UnprocessedEntity} />
       ),
     },
+    {
+      title: <Typography>Actions</Typography>,
+      render: (rowData: UnprocessedEntity | {}) => {
+        const { entity_id, entity_ref } = rowData as UnprocessedEntity;
+
+        return (
+          <IconButton
+            aria-label="delete"
+            onClick={async () =>
+              await handleDelete({
+                entityId: entity_id,
+                entityRef: entity_ref,
+              })
+            }
+          >
+            <DeleteIcon fontSize="small" data-testid="delete-icon" />
+          </IconButton>
+        );
+      },
+    },
   ];
+
   return (
-    <>
-      <Table
-        options={{ pageSize: 20, search: true }}
-        columns={columns}
-        data={data?.entities || []}
-        emptyContent={
-          <Typography className={classes.successMessage}>
-            No failed entities found
-          </Typography>
-        }
-        onSearchChange={(searchTerm: string) =>
-          setSelectedSearchTerm(searchTerm)
-        }
-        detailPanel={({ rowData }) => {
-          const errors = (rowData as UnprocessedEntity).errors;
-          return (
-            <>
-              {errors?.map(e => {
-                return (
-                  <Box className={classes.errorBox}>
-                    <Typography className={classes.errorTitle}>
-                      {e.name}
-                    </Typography>
-                    <MarkdownContent content={e.message} />
-                    <RenderErrorContext
-                      error={e}
-                      rowData={rowData as UnprocessedEntity}
-                    />
-                  </Box>
-                );
-              })}
-            </>
-          );
-        }}
-      />
-    </>
+    <Table
+      options={{ pageSize: 20, search: true }}
+      columns={columns}
+      data={data?.entities ?? []}
+      emptyContent={
+        <Typography className={classes.successMessage}>
+          No failed entities found
+        </Typography>
+      }
+      onSearchChange={(searchTerm: string) => setSelectedSearchTerm(searchTerm)}
+      detailPanel={({ rowData }) => {
+        const errors = (rowData as UnprocessedEntity).errors;
+        return (
+          <>
+            {errors?.map((e, idx) => {
+              return (
+                <Box key={idx} className={classes.errorBox}>
+                  <Typography className={classes.errorTitle}>
+                    {e.name}
+                  </Typography>
+                  <MarkdownContent content={e.message} />
+                  <RenderErrorContext
+                    error={e}
+                    rowData={rowData as UnprocessedEntity}
+                  />
+                </Box>
+              );
+            })}
+          </>
+        );
+      }}
+    />
   );
 };

yarn.lock

@@ -5772,9 +5772,16 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@backstage/plugin-catalog-backend-module-unprocessed@workspace:plugins/catalog-backend-module-unprocessed"
   dependencies:
+    "@backstage/backend-common": "workspace:^"
     "@backstage/backend-plugin-api": "workspace:^"
     "@backstage/catalog-model": "workspace:^"
     "@backstage/cli": "workspace:^"
+    "@backstage/errors": "workspace:^"
+    "@backstage/plugin-auth-node": "workspace:^"
+    "@backstage/plugin-catalog-unprocessed-entities-common": "workspace:^"
+    "@backstage/plugin-permission-common": "workspace:^"
+    "@backstage/plugin-permission-node": "workspace:^"
+    "@types/express": ^4.17.6
     express-promise-router: ^4.1.1
     knex: ^3.0.0
   languageName: unknown
@@ -6037,6 +6044,15 @@ __metadata:
   languageName: unknown
   linkType: soft
 
+"@backstage/plugin-catalog-unprocessed-entities-common@workspace:^, @backstage/plugin-catalog-unprocessed-entities-common@workspace:plugins/catalog-unprocessed-entities-common":
+  version: 0.0.0-use.local
+  resolution: "@backstage/plugin-catalog-unprocessed-entities-common@workspace:plugins/catalog-unprocessed-entities-common"
+  dependencies:
+    "@backstage/cli": "workspace:^"
+    "@backstage/plugin-permission-common": "workspace:^"
+  languageName: unknown
+  linkType: soft
+
 "@backstage/plugin-catalog-unprocessed-entities@workspace:^, @backstage/plugin-catalog-unprocessed-entities@workspace:plugins/catalog-unprocessed-entities":
   version: 0.0.0-use.local
   resolution: "@backstage/plugin-catalog-unprocessed-entities@workspace:plugins/catalog-unprocessed-entities"