diff --git a/.changeset/grumpy-carrots-sin.md b/.changeset/grumpy-carrots-sin.md new file mode 100644 index 0000000000..8f2843ebd0 --- /dev/null +++ b/.changeset/grumpy-carrots-sin.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog-unprocessed-entities': minor +'@backstage/plugin-catalog-backend-module-unprocessed': minor +--- + +Added a new plugin to expose entities which are unprocessed or have errors processing diff --git a/packages/app/package.json b/packages/app/package.json index b7c75009fe..dd13ae8f1d 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -26,6 +26,7 @@ "@backstage/plugin-catalog-graph": "workspace:^", "@backstage/plugin-catalog-import": "workspace:^", "@backstage/plugin-catalog-react": "workspace:^", + "@backstage/plugin-catalog-unprocessed-entities": "workspace:^", "@backstage/plugin-circleci": "workspace:^", "@backstage/plugin-cloudbuild": "workspace:^", "@backstage/plugin-code-coverage": "workspace:^", diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index d46774a757..df54f37dc6 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -111,6 +111,7 @@ import { StackstormPage } from '@backstage/plugin-stackstorm'; import { PuppetDbPage } from '@backstage/plugin-puppetdb'; import { DevToolsPage } from '@backstage/plugin-devtools'; import { customDevToolsPage } from './components/devtools/CustomDevToolsPage'; +import { CatalogUnprocessedEntitiesPage } from '@backstage/plugin-catalog-unprocessed-entities'; const app = createApp({ apis, @@ -171,6 +172,10 @@ const routes = ( > {entityPage} + } + /> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; +import { HttpRouterService } from '@backstage/backend-plugin-api'; +import { Knex } from 'knex'; + +// @public +export const catalogModuleUnprocessedEntities: () => BackendFeature; + +// @public +export class UnprocessedEntitesModule { + constructor(database: Knex, router: HttpRouterService); + // (undocumented) + registerRoutes(): void; +} +``` diff --git a/plugins/catalog-backend-module-unprocessed/package.json b/plugins/catalog-backend-module-unprocessed/package.json new file mode 100644 index 0000000000..55228d5ab5 --- /dev/null +++ b/plugins/catalog-backend-module-unprocessed/package.json @@ -0,0 +1,37 @@ +{ + "name": "@backstage/plugin-catalog-backend-module-unprocessed", + "description": "Backstage Catalog module to view unprocessed entities", + "version": "0.0.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "backend-plugin-module" + }, + "scripts": { + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" + }, + "devDependencies": { + "@backstage/cli": "workspace:^" + }, + "files": [ + "dist" + ], + "dependencies": { + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/catalog-model": "workspace:^", + "@backstage/plugin-auth-node": "workspace:^", + "express-promise-router": "^4.1.1", + "knex": "^2.4.2" + } +} diff --git a/plugins/catalog-backend-module-unprocessed/src/UnprocessedEntitiesModule.ts b/plugins/catalog-backend-module-unprocessed/src/UnprocessedEntitiesModule.ts new file mode 100644 index 0000000000..6ea184ce83 --- /dev/null +++ b/plugins/catalog-backend-module-unprocessed/src/UnprocessedEntitiesModule.ts @@ -0,0 +1,132 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + HydratedRefreshState, + RefreshState, + UnprocessedEntitiesRequest, + UnprocessedEntitiesResponse, +} from './types'; +import { Knex } from 'knex'; +import { HttpRouterService } from '@backstage/backend-plugin-api'; +import Router from 'express-promise-router'; +import { getBearerTokenFromAuthorizationHeader } from '@backstage/plugin-auth-node'; + +/** + * Module providing Unprocessed Entities API endpoints + * + * @public + */ +export class UnprocessedEntitesModule { + private readonly moduleRouter; + + constructor( + private readonly database: Knex, + private readonly router: HttpRouterService, + ) { + this.moduleRouter = Router(); + this.router.use(this.moduleRouter); + } + + private async unprocessed( + request: UnprocessedEntitiesRequest, + ): Promise { + if (request.reason === 'pending') { + return { + type: 'pending', + entities: await this.pending(request.owner), + }; + } + return { + type: 'failed', + entities: await this.failed(request.owner), + }; + } + + private hydrateRefreshState(r: RefreshState): HydratedRefreshState { + return { + ...r, + unprocessed_entity: JSON.parse(r.unprocessed_entity), + ...(r.processed_entity && { + processed_entity: JSON.parse(r.processed_entity), + }), + ...(r.errors && { errors: JSON.parse(r.errors) }), + ...(r.cache && { cache: JSON.parse(r.cache) }), + }; + } + + private async pending(owner?: string): Promise { + const res = ( + await this.database('refresh_state.*') + .from('refresh_state') + .leftJoin( + 'final_entities', + 'final_entities.entity_id', + 'refresh_state.entity_id', + ) + .whereNull('final_entities.entity_id') + ).map(this.hydrateRefreshState); + if (owner) { + return res.filter(r => r.unprocessed_entity.spec?.owner === owner); + } + + return res; + } + + private async failed(owner?: string): Promise { + const res = ( + await this.database('refresh_state.*') + .from('refresh_state') + .rightJoin( + 'final_entities', + 'final_entities.entity_id', + 'refresh_state.entity_id', + ) + .whereNull('final_entities.final_entity') + ).map(this.hydrateRefreshState); + if (owner) { + return res.filter(r => r.unprocessed_entity.spec?.owner === owner); + } + + return res; + } + + registerRoutes() { + this.moduleRouter + .get('/entities/unprocessed/failed', async (req, res) => { + return res.json( + await this.unprocessed({ + reason: 'failed', + owner: req.query.owner as string, + authorizationToken: getBearerTokenFromAuthorizationHeader( + req.header('authorization'), + ), + }), + ); + }) + .get('/entities/unprocessed/pending', async (req, res) => { + return res.json( + await this.unprocessed({ + reason: 'pending', + owner: req.query.owner as string, + authorizationToken: getBearerTokenFromAuthorizationHeader( + req.header('authorization'), + ), + }), + ); + }); + } +} diff --git a/plugins/catalog-backend-module-unprocessed/src/index.ts b/plugins/catalog-backend-module-unprocessed/src/index.ts new file mode 100644 index 0000000000..21cb2dbcd6 --- /dev/null +++ b/plugins/catalog-backend-module-unprocessed/src/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Node.js library for the catalog-backend-module-unprocessed plugin. + * + * @packageDocumentation + */ + +export * from './module'; +export * from './UnprocessedEntitiesModule'; diff --git a/plugins/catalog-backend-module-unprocessed/src/module.ts b/plugins/catalog-backend-module-unprocessed/src/module.ts new file mode 100644 index 0000000000..1c4b6c658f --- /dev/null +++ b/plugins/catalog-backend-module-unprocessed/src/module.ts @@ -0,0 +1,51 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; +import { UnprocessedEntitesModule } from './UnprocessedEntitiesModule'; + +/** + * Catalog Module for Unprocessed Entities + * + * @public + */ +export const catalogModuleUnprocessedEntities = createBackendModule({ + pluginId: 'catalog', + moduleId: 'catalogModuleUnprocessedEntities', + register(env) { + env.registerInit({ + deps: { + database: coreServices.database, + router: coreServices.httpRouter, + logger: coreServices.logger, + }, + async init({ database, router, logger }) { + const module = new UnprocessedEntitesModule( + await database.getClient(), + router, + ); + + module.registerRoutes(); + logger.info( + 'registered additional routes for catalogModuleUnprocessedEntities', + ); + }, + }); + }, +}); diff --git a/plugins/catalog-backend-module-unprocessed/src/setupTests.ts b/plugins/catalog-backend-module-unprocessed/src/setupTests.ts new file mode 100644 index 0000000000..4b9026cde5 --- /dev/null +++ b/plugins/catalog-backend-module-unprocessed/src/setupTests.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export {}; diff --git a/plugins/catalog-backend-module-unprocessed/src/types.ts b/plugins/catalog-backend-module-unprocessed/src/types.ts new file mode 100644 index 0000000000..8d70345682 --- /dev/null +++ b/plugins/catalog-backend-module-unprocessed/src/types.ts @@ -0,0 +1,69 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Entity } from '@backstage/catalog-model'; + +export type HydratedRefreshState = { + entity_id: string; + entity_ref: string; + unprocessed_entity: Entity; + unprocessed_hash?: string; + processed_entity?: Entity; + result_hash?: string; + cache?: RefreshStateCache; + next_update_at: string | Date; + last_discovery_at: string | Date; // remove? + errors?: RefreshStateError[]; + location_key?: string; +}; + +export type RefreshState = { + entity_id: string; + entity_ref: string; + unprocessed_entity: string; + unprocessed_hash?: string; + processed_entity?: string; + result_hash?: string; + cache?: string; + next_update_at: string | Date; + last_discovery_at: string | Date; // remove? + errors?: string; + location_key?: string; +}; +export type RefreshStateCache = { + ttl: number; + cache: object; +}; + +export type RefreshStateError = { + name: string; + message: string; + cause: { + name: string; + message: string; + stack: string; + }; +}; + +export interface UnprocessedEntitiesRequest { + reason: 'failed' | 'pending'; + owner?: string; + authorizationToken?: string; +} + +export interface UnprocessedEntitiesResponse { + type: 'pending' | 'failed'; + entities: HydratedRefreshState[]; +} diff --git a/plugins/catalog-unprocessed-entities/.eslintrc.js b/plugins/catalog-unprocessed-entities/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/catalog-unprocessed-entities/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/catalog-unprocessed-entities/README.md b/plugins/catalog-unprocessed-entities/README.md new file mode 100644 index 0000000000..fd824bec94 --- /dev/null +++ b/plugins/catalog-unprocessed-entities/README.md @@ -0,0 +1,29 @@ +# catalog-unprocessed-entities + +Frontend plugin to view unprocessed entities. + +## Requirements + +Requires the `@backstage/plugin-catalog-backend-module-unprocessed` module to be installed. + +## Installation + +Import into your App.tsx and include into the `` component: + +```tsx +import { CatalogUnprocessedEntitiesPage } from '@backstage/plugin-catalog-unprocessed-entities'; +//... + +} +/>; +``` + +## Getting started + +Your plugin has been added to the example app in this repository, meaning you'll be able to access it by running `yarn start` in the root directory, and then navigating to [/catalog-unprocessed-entities](http://localhost:3000/catalog-unprocessed-entities). + +You can also serve the plugin in isolation by running `yarn start` in the plugin directory. +This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads. +It is only meant for local development, and the setup for it can be found inside the [/dev](./dev) directory. diff --git a/plugins/catalog-unprocessed-entities/api-report.md b/plugins/catalog-unprocessed-entities/api-report.md new file mode 100644 index 0000000000..b1e0d6f53d --- /dev/null +++ b/plugins/catalog-unprocessed-entities/api-report.md @@ -0,0 +1,24 @@ +## API Report File for "@backstage/plugin-catalog-unprocessed-entities" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +/// + +import { BackstagePlugin } from '@backstage/core-plugin-api'; +import { RouteRef } from '@backstage/core-plugin-api'; + +// @public +export const CatalogUnprocessedEntitiesPage: () => JSX.Element; + +// @public +export const catalogUnprocessedEntitiesPlugin: BackstagePlugin< + { + root: RouteRef; + }, + {}, + {} +>; + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/catalog-unprocessed-entities/dev/index.tsx b/plugins/catalog-unprocessed-entities/dev/index.tsx new file mode 100644 index 0000000000..ab11cb554d --- /dev/null +++ b/plugins/catalog-unprocessed-entities/dev/index.tsx @@ -0,0 +1,30 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React from 'react'; +import { createDevApp } from '@backstage/dev-utils'; +import { + catalogUnprocessedEntitiesPlugin, + CatalogUnprocessedEntitiesPage, +} from '../src/plugin'; + +createDevApp() + .registerPlugin(catalogUnprocessedEntitiesPlugin) + .addPage({ + element: , + title: 'Root Page', + path: '/catalog-unprocessed-entities', + }) + .render(); diff --git a/plugins/catalog-unprocessed-entities/package.json b/plugins/catalog-unprocessed-entities/package.json new file mode 100644 index 0000000000..9a05c7e8cb --- /dev/null +++ b/plugins/catalog-unprocessed-entities/package.json @@ -0,0 +1,53 @@ +{ + "name": "@backstage/plugin-catalog-unprocessed-entities", + "version": "0.0.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "main": "dist/index.esm.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "frontend-plugin" + }, + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" + }, + "dependencies": { + "@backstage/catalog-model": "workspace:^", + "@backstage/core-components": "workspace:^", + "@backstage/core-plugin-api": "workspace:^", + "@backstage/errors": "workspace:^", + "@backstage/theme": "workspace:^", + "@material-ui/core": "^4.9.13", + "@material-ui/icons": "^4.9.1", + "@material-ui/lab": "^4.0.0-alpha.60", + "react-use": "^17.2.4" + }, + "peerDependencies": { + "react": "^16.13.1 || ^17.0.0" + }, + "devDependencies": { + "@backstage/cli": "workspace:^", + "@backstage/core-app-api": "workspace:^", + "@backstage/dev-utils": "workspace:^", + "@backstage/test-utils": "workspace:^", + "@testing-library/jest-dom": "^5.10.1", + "@testing-library/react": "^12.1.3", + "@testing-library/user-event": "^14.0.0", + "@types/node": "*", + "cross-fetch": "^3.1.5", + "msw": "^1.0.0" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/catalog-unprocessed-entities/src/api/index.ts b/plugins/catalog-unprocessed-entities/src/api/index.ts new file mode 100644 index 0000000000..a4f2294aac --- /dev/null +++ b/plugins/catalog-unprocessed-entities/src/api/index.ts @@ -0,0 +1,59 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { DiscoveryApi, createApiRef } from '@backstage/core-plugin-api'; +import { ResponseError } from '@backstage/errors'; +import { UnprocessedEntity } from '../types'; + +/** + * {@link @backstage/core-plugin-api#ApiRef} for the {@link CatalogUnprocessedEntitiesApi} + * + * @public + */ +export const catalogUnprocessedEntitiesApiRef = + createApiRef({ + id: 'plugin.catalog-unprocessed-entities.service', + }); + +/** + * API client for the Catalog Unprocessed Entities plugin + * + * @public + */ +export class CatalogUnprocessedEntitiesApi { + url: string = ''; + + constructor(public discovery: DiscoveryApi) {} + + private async fetch(path: string, init?: RequestInit): Promise { + if (!this.url) { + this.url = await this.discovery.getBaseUrl('catalog'); + } + const resp = await fetch(`${this.url}/${path}`, init); + if (!resp.ok) { + throw await ResponseError.fromResponse(resp); + } + + return await resp.json(); + } + + async pending(): Promise<{ entities: UnprocessedEntity[] }> { + return await this.fetch('entities/unprocessed/pending'); + } + + async failed(): Promise<{ entities: UnprocessedEntity[] }> { + return await this.fetch('entities/unprocessed/failed'); + } +} diff --git a/plugins/catalog-unprocessed-entities/src/components/EntityDialog.tsx b/plugins/catalog-unprocessed-entities/src/components/EntityDialog.tsx new file mode 100644 index 0000000000..a02d8419d4 --- /dev/null +++ b/plugins/catalog-unprocessed-entities/src/components/EntityDialog.tsx @@ -0,0 +1,89 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { useState } from 'react'; + +import Dialog from '@material-ui/core/Dialog'; +import DialogContent from '@material-ui/core/DialogContent'; +import DialogTitle from '@material-ui/core/DialogTitle'; +import IconButton from '@material-ui/core/IconButton'; +import { makeStyles, createStyles, Theme } from '@material-ui/core/styles'; +import CloseIcon from '@material-ui/icons/Close'; +import DescriptionIcon from '@material-ui/icons/Description'; + +import { UnprocessedEntity } from './../types'; +import { CodeSnippet } from '@backstage/core-components'; + +const useStyles = makeStyles((theme: Theme) => + createStyles({ + closeButton: { + position: 'absolute', + right: theme.spacing(1), + top: theme.spacing(1), + color: theme.palette.grey[500], + }, + entity: { + overflow: 'scroll', + width: '100%', + }, + codeBox: { + border: '1px solid black', + padding: '1em', + }, + }), +); + +export const EntityDialog = ({ entity }: { entity: UnprocessedEntity }) => { + const [open, setOpen] = useState(false); + const classes = useStyles(); + + const openDialog = () => { + setOpen(true); + }; + + const closeDialog = () => { + setOpen(false); + }; + + const dialogContent = () => { + return ( + + ); + }; + + return ( + <> + + + + + + + + + + {dialogContent()} + + + ); +}; diff --git a/plugins/catalog-unprocessed-entities/src/components/FailedEntities.tsx b/plugins/catalog-unprocessed-entities/src/components/FailedEntities.tsx new file mode 100644 index 0000000000..44a146f027 --- /dev/null +++ b/plugins/catalog-unprocessed-entities/src/components/FailedEntities.tsx @@ -0,0 +1,161 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React from 'react'; + +import { + ErrorPanel, + MarkdownContent, + Progress, + Table, + TableColumn, +} from '@backstage/core-components'; +import { useApi } from '@backstage/core-plugin-api'; +import { BackstageThemeOptions } from '@backstage/theme'; +import { Box, Typography, makeStyles } from '@material-ui/core'; + +import { UnprocessedEntity } from '../types'; +import { EntityDialog } from './EntityDialog'; +import { catalogUnprocessedEntitiesApiRef } from '../api'; +import useAsync from 'react-use/lib/useAsync'; + +const useStyles = makeStyles((theme: BackstageThemeOptions) => ({ + errorBox: { + color: theme.palette.status.error, + backgroundColor: theme.palette.errorBackground, + padding: '1em', + margin: '1em', + border: `1px solid ${theme.palette.status.error}`, + }, + errorTitle: { + width: '100%', + fontWeight: 'bold', + }, + successMessage: { + background: theme.palette.infoBackground, + color: theme.palette.infoText, + }, +})); + +const RenderErrorContext = ({ + error, + rowData, +}: { + error: { message: string }; + rowData: UnprocessedEntity; +}) => { + if (error.message.includes('tags.')) { + return ( + <> + Tags +
    + {rowData.unprocessed_entity.metadata.tags?.map(t => ( +
  • {t}
    • + ))} +
+ + ); + } + + if (error.message.includes('metadata.name')) { + return ( + <> + Name + + {rowData.unprocessed_entity.metadata.name} + + + ); + } + + return null; +}; + +export const FailedEntities = () => { + const classes = useStyles(); + const unprocessedApi = useApi(catalogUnprocessedEntitiesApiRef); + const { + loading, + error, + value: data, + } = useAsync(async () => await unprocessedApi.failed()); + + if (loading) { + return ; + } + if (error) { + return ; + } + + const columns: TableColumn[] = [ + { + title: entityRef, + render: (rowData: UnprocessedEntity | {}) => + (rowData as UnprocessedEntity).entity_ref, + }, + { + title: Kind, + render: (rowData: UnprocessedEntity | {}) => + (rowData as UnprocessedEntity).unprocessed_entity.kind, + }, + { + title: Owner, + render: (rowData: UnprocessedEntity | {}) => + (rowData as UnprocessedEntity).unprocessed_entity.spec?.owner || + 'unknown', + }, + { + title: Raw, + render: (rowData: UnprocessedEntity | {}) => ( + + ), + }, + ]; + return ( + <> + + No failed entities found + + } + detailPanel={({ rowData }) => { + const errors = (rowData as UnprocessedEntity).errors; + return ( + <> + {errors?.map(e => { + return ( + + + {e.name} + + + + + ); + })} + + ); + }} + /> + + ); +}; diff --git a/plugins/catalog-unprocessed-entities/src/components/PendingEntities.tsx b/plugins/catalog-unprocessed-entities/src/components/PendingEntities.tsx new file mode 100644 index 0000000000..88ace1cfb0 --- /dev/null +++ b/plugins/catalog-unprocessed-entities/src/components/PendingEntities.tsx @@ -0,0 +1,96 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React from 'react'; + +import { + ErrorPanel, + Progress, + TableColumn, + Table, +} from '@backstage/core-components'; +import { Typography, makeStyles } from '@material-ui/core'; + +import { UnprocessedEntity } from '../types'; + +import { EntityDialog } from './EntityDialog'; +import { useApi } from '@backstage/core-plugin-api'; +import useAsync from 'react-use/lib/useAsync'; +import { catalogUnprocessedEntitiesApiRef } from '../api'; +import { BackstageTheme } from '@backstage/theme'; + +const useStyles = makeStyles((theme: BackstageTheme) => ({ + successMessage: { + background: theme.palette.infoBackground, + color: theme.palette.infoText, + padding: theme.spacing(2), + }, +})); + +export const PendingEntities = () => { + const classes = useStyles(); + const unprocessedApi = useApi(catalogUnprocessedEntitiesApiRef); + const { + loading, + error, + value: data, + } = useAsync(async () => await unprocessedApi.pending()); + + if (loading) { + return ; + } + if (error) { + return ; + } + + const columns: TableColumn[] = [ + { + title: entityRef, + render: (rowData: UnprocessedEntity | {}) => + (rowData as UnprocessedEntity).entity_ref, + }, + { + title: Kind, + render: (rowData: UnprocessedEntity | {}) => + (rowData as UnprocessedEntity).unprocessed_entity.kind, + }, + { + title: Owner, + render: (rowData: UnprocessedEntity | {}) => + (rowData as UnprocessedEntity).unprocessed_entity.spec?.owner || + 'unknown', + }, + { + title: Raw, + render: (rowData: UnprocessedEntity | {}) => ( + + ), + }, + ]; + return ( + <> +
+ No pending entities found + + } + /> + + ); +}; diff --git a/plugins/catalog-unprocessed-entities/src/components/UnprocessedEntities.tsx b/plugins/catalog-unprocessed-entities/src/components/UnprocessedEntities.tsx new file mode 100644 index 0000000000..ba186888ec --- /dev/null +++ b/plugins/catalog-unprocessed-entities/src/components/UnprocessedEntities.tsx @@ -0,0 +1,56 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { useState } from 'react'; + +import { Page, Header, Content } from '@backstage/core-components'; +import { Tab } from '@material-ui/core'; +import { TabContext, TabList, TabPanel } from '@material-ui/lab'; + +import { FailedEntities } from './FailedEntities'; +import { PendingEntities } from './PendingEntities'; + +export const UnprocessedEntitiesContent = () => { + const [tab, setTab] = useState('failed'); + const handleChange = (_event: React.ChangeEvent<{}>, tabValue: string) => { + setTab(tabValue); + }; + + return ( + + + + + + + + + + + + + + + ); +}; + +export const UnprocessedEntities = () => { + return ( + +
+ + + ); +}; diff --git a/plugins/catalog-unprocessed-entities/src/index.ts b/plugins/catalog-unprocessed-entities/src/index.ts new file mode 100644 index 0000000000..a65283926e --- /dev/null +++ b/plugins/catalog-unprocessed-entities/src/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { + catalogUnprocessedEntitiesPlugin, + CatalogUnprocessedEntitiesPage, +} from './plugin'; diff --git a/plugins/catalog-unprocessed-entities/src/plugin.test.ts b/plugins/catalog-unprocessed-entities/src/plugin.test.ts new file mode 100644 index 0000000000..add3d677f0 --- /dev/null +++ b/plugins/catalog-unprocessed-entities/src/plugin.test.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { catalogUnprocessedEntitiesPlugin } from './plugin'; + +describe('catalog-unprocessed-entities', () => { + it('should export plugin', () => { + expect(catalogUnprocessedEntitiesPlugin).toBeDefined(); + }); +}); diff --git a/plugins/catalog-unprocessed-entities/src/plugin.ts b/plugins/catalog-unprocessed-entities/src/plugin.ts new file mode 100644 index 0000000000..88d3ed45ea --- /dev/null +++ b/plugins/catalog-unprocessed-entities/src/plugin.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + createApiFactory, + createPlugin, + createRoutableExtension, + discoveryApiRef, +} from '@backstage/core-plugin-api'; + +import { rootRouteRef } from './routes'; +import { + CatalogUnprocessedEntitiesApi, + catalogUnprocessedEntitiesApiRef, +} from './api'; + +/** + * Plugin entry point + * + * @public + */ +export const catalogUnprocessedEntitiesPlugin = createPlugin({ + id: 'catalog-unprocessed-entities', + routes: { + root: rootRouteRef, + }, + apis: [ + createApiFactory({ + api: catalogUnprocessedEntitiesApiRef, + deps: { discoveryApi: discoveryApiRef }, + factory: ({ discoveryApi }) => + new CatalogUnprocessedEntitiesApi(discoveryApi), + }), + ], +}); + +/** + * Tool page for the Catalog Unprocessed Entities Plugin + * + * @public + */ +export const CatalogUnprocessedEntitiesPage = + catalogUnprocessedEntitiesPlugin.provide( + createRoutableExtension({ + name: 'CatalogUnprocessedEntitiesPage', + component: () => + import('./components/UnprocessedEntities').then( + m => m.UnprocessedEntities, + ), + mountPoint: rootRouteRef, + }), + ); diff --git a/plugins/catalog-unprocessed-entities/src/routes.ts b/plugins/catalog-unprocessed-entities/src/routes.ts new file mode 100644 index 0000000000..1bd268c392 --- /dev/null +++ b/plugins/catalog-unprocessed-entities/src/routes.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { createRouteRef } from '@backstage/core-plugin-api'; + +export const rootRouteRef = createRouteRef({ + id: 'catalog-unprocessed-entities', +}); diff --git a/plugins/catalog-unprocessed-entities/src/setupTests.ts b/plugins/catalog-unprocessed-entities/src/setupTests.ts new file mode 100644 index 0000000000..73dd8dce47 --- /dev/null +++ b/plugins/catalog-unprocessed-entities/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import '@testing-library/jest-dom'; +import 'cross-fetch/polyfill'; diff --git a/plugins/catalog-unprocessed-entities/src/types.ts b/plugins/catalog-unprocessed-entities/src/types.ts new file mode 100644 index 0000000000..490027b57a --- /dev/null +++ b/plugins/catalog-unprocessed-entities/src/types.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Entity } from '@backstage/catalog-model'; + +export type UnprocessedEntity = { + entity_id: string; + entity_ref: string; + unprocessed_entity: Entity; + unprocessed_hash?: string; + processed_entity?: Entity; + result_hash?: string; + cache?: UnprocessedEntityCache; + next_update_at: string | Date; + last_discovery_at: string | Date; // remove? + errors?: UnprocessedEntityError[]; + location_key?: string; +}; + +export type UnprocessedEntityCache = { + ttl: number; + cache: object; +}; + +export type UnprocessedEntityError = { + name: string; + message: string; + cause: { + name: string; + message: string; + stack: string; + }; +}; diff --git a/yarn.lock b/yarn.lock index a6a300e49b..87399900be 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5609,6 +5609,19 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-catalog-backend-module-unprocessed@workspace:^, @backstage/plugin-catalog-backend-module-unprocessed@workspace:plugins/catalog-backend-module-unprocessed": + version: 0.0.0-use.local + resolution: "@backstage/plugin-catalog-backend-module-unprocessed@workspace:plugins/catalog-backend-module-unprocessed" + dependencies: + "@backstage/backend-plugin-api": "workspace:^" + "@backstage/catalog-model": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/plugin-auth-node": "workspace:^" + express-promise-router: ^4.1.1 + knex: ^2.4.2 + languageName: unknown + linkType: soft + "@backstage/plugin-catalog-backend@workspace:^, @backstage/plugin-catalog-backend@workspace:plugins/catalog-backend": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend@workspace:plugins/catalog-backend" @@ -5891,6 +5904,34 @@ __metadata: languageName: unknown linkType: soft +"@backstage/plugin-catalog-unprocessed-entities@workspace:^, @backstage/plugin-catalog-unprocessed-entities@workspace:plugins/catalog-unprocessed-entities": + version: 0.0.0-use.local + resolution: "@backstage/plugin-catalog-unprocessed-entities@workspace:plugins/catalog-unprocessed-entities" + dependencies: + "@backstage/catalog-model": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/core-app-api": "workspace:^" + "@backstage/core-components": "workspace:^" + "@backstage/core-plugin-api": "workspace:^" + "@backstage/dev-utils": "workspace:^" + "@backstage/errors": "workspace:^" + "@backstage/test-utils": "workspace:^" + "@backstage/theme": "workspace:^" + "@material-ui/core": ^4.9.13 + "@material-ui/icons": ^4.9.1 + "@material-ui/lab": ^4.0.0-alpha.60 + "@testing-library/jest-dom": ^5.10.1 + "@testing-library/react": ^12.1.3 + "@testing-library/user-event": ^14.0.0 + "@types/node": "*" + cross-fetch: ^3.1.5 + msw: ^1.0.0 + react-use: ^17.2.4 + peerDependencies: + react: ^16.13.1 || ^17.0.0 + languageName: unknown + linkType: soft + "@backstage/plugin-catalog@workspace:^, @backstage/plugin-catalog@workspace:plugins/catalog": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog@workspace:plugins/catalog" @@ -24341,6 +24382,7 @@ __metadata: "@backstage/plugin-catalog-graph": "workspace:^" "@backstage/plugin-catalog-import": "workspace:^" "@backstage/plugin-catalog-react": "workspace:^" + "@backstage/plugin-catalog-unprocessed-entities": "workspace:^" "@backstage/plugin-circleci": "workspace:^" "@backstage/plugin-cloudbuild": "workspace:^" "@backstage/plugin-code-coverage": "workspace:^" @@ -24433,6 +24475,7 @@ __metadata: "@backstage/plugin-app-backend": "workspace:^" "@backstage/plugin-auth-node": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^" "@backstage/plugin-kubernetes-backend": "workspace:^" "@backstage/plugin-permission-backend": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" @@ -24467,6 +24510,7 @@ __metadata: "@backstage/plugin-azure-sites-backend": "workspace:^" "@backstage/plugin-badges-backend": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" "@backstage/plugin-code-coverage-backend": "workspace:^" "@backstage/plugin-devtools-backend": "workspace:^" @@ -24642,7 +24686,7 @@ __metadata: languageName: node linkType: hard -"express-promise-router@npm:^4.1.0": +"express-promise-router@npm:^4.1.0, express-promise-router@npm:^4.1.1": version: 4.1.1 resolution: "express-promise-router@npm:4.1.1" dependencies: