diff --git a/.changeset/chatty-cooks-begin.md b/.changeset/chatty-cooks-begin.md new file mode 100644 index 0000000000..e32d22d532 --- /dev/null +++ b/.changeset/chatty-cooks-begin.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-kubernetes': patch +'@backstage/plugin-kubernetes-backend': patch +--- + +Add AWS auth provider for Kubernetes diff --git a/.changeset/fast-flowers-tickle.md b/.changeset/fast-flowers-tickle.md new file mode 100644 index 0000000000..3eacb6f1a0 --- /dev/null +++ b/.changeset/fast-flowers-tickle.md @@ -0,0 +1,31 @@ +--- +'@backstage/plugin-kafka': minor +'@backstage/plugin-kafka-backend': minor +--- + +Added support for multiple Kafka clusters and multiple consumers per component. +Note that this introduces several breaking changes. + +1. Configuration in `app-config.yaml` has changed to support the ability to configure multiple clusters. This means you are required to update the configs in the following way: + +```diff +kafka: + clientId: backstage +- brokers: +- - localhost:9092 ++ clusters: ++ - name: prod ++ brokers: ++ - localhost:9092 +``` + +2. Configuration of services has changed as well to support multiple clusters: + +```diff + annotations: +- kafka.apache.org/consumer-groups: consumer ++ kafka.apache.org/consumer-groups: prod/consumer +``` + +3. Kafka Backend API has changed, so querying offsets of a consumer group is now done with the following query path: + `/consumers/${clusterId}/${consumerGroup}/offsets` diff --git a/.changeset/forty-jobs-occur.md b/.changeset/forty-jobs-occur.md new file mode 100644 index 0000000000..b1620a7535 --- /dev/null +++ b/.changeset/forty-jobs-occur.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-catalog': patch +--- + +Add `EntityRefLinks` that shows one or multiple entity ref links. + +Change the about card and catalog table to use `EntityRefLinks` due to the +nature of relations to support multiple relations per type. diff --git a/.changeset/green-boats-attend.md b/.changeset/green-boats-attend.md new file mode 100644 index 0000000000..ae2324291e --- /dev/null +++ b/.changeset/green-boats-attend.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Support supplying a custom catalog descriptor file parser diff --git a/.changeset/loud-terms-kiss.md b/.changeset/loud-terms-kiss.md new file mode 100644 index 0000000000..da50e56e3a --- /dev/null +++ b/.changeset/loud-terms-kiss.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-sonarqube': patch +--- + +Ask the SonarQube server for all support metrics prior to querying them for a project. diff --git a/app-config.yaml b/app-config.yaml index 02afb8ac41..c150ef5902 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -99,6 +99,13 @@ kubernetes: - 'config' clusters: [] +kafka: + clientId: backstage + clusters: + - name: cluster + brokers: + - localhost:9092 + integrations: github: - host: github.com @@ -372,8 +379,3 @@ homepage: timezone: 'Asia/Tokyo' pagerduty: eventsBaseUrl: 'https://events.pagerduty.com/v2' - -kafka: - clientId: backstage - brokers: - - localhost:9092 diff --git a/docs/architecture-decisions/adr000-template.md b/docs/architecture-decisions/adr000-template.md index 538afccf87..5e6edd8dba 100644 --- a/docs/architecture-decisions/adr000-template.md +++ b/docs/architecture-decisions/adr000-template.md @@ -4,8 +4,6 @@ title: ADR000: [TITLE] description: Architecture Decision Record (ADR) for [TITLE] [DESCRIPTION] --- -# ADR000: [title] - ## Context diff --git a/docs/architecture-decisions/adr010-luxon-date-library.md b/docs/architecture-decisions/adr010-luxon-date-library.md index b4c1ddedba..43f18f0f69 100644 --- a/docs/architecture-decisions/adr010-luxon-date-library.md +++ b/docs/architecture-decisions/adr010-luxon-date-library.md @@ -4,8 +4,6 @@ title: ADR010: Use the Luxon Date Library description: Architecture Decision Record (ADR) for Luxon Date Library --- -# ADR010: Use the Luxon Date Library - ## Context Date formatting (e.g. `a day ago`) and calculations are common within Backstage. diff --git a/docs/assets/features/kubernetes/backstage-k8s-2-deployments.png b/docs/assets/features/kubernetes/backstage-k8s-2-deployments.png new file mode 100644 index 0000000000..4a9f2b06fa Binary files /dev/null and b/docs/assets/features/kubernetes/backstage-k8s-2-deployments.png differ diff --git a/docs/features/kubernetes/configuration.md b/docs/features/kubernetes/configuration.md new file mode 100644 index 0000000000..3e138a7a0b --- /dev/null +++ b/docs/features/kubernetes/configuration.md @@ -0,0 +1,130 @@ +--- +id: configuration +title: Configuring Kubernetes integration +sidebar_label: Configuration +# prettier-ignore +description: Configuring the Kubernetes integration for Backstage expose your entity's objects +--- + +Configuring the Backstage Kubernetes integration involves two steps: + +1. Enabling the backend to collect objects from your Kubernetes cluster(s). +2. Surfacing your Kubernetes objects in catalog entities + +## Configuring Kubernetes Clusters + +The following is a full example entry in `app-config.yaml`: + +```yaml +kubernetes: + serviceLocatorMethod: 'multiTenant' + clusterLocatorMethods: + - 'config' + clusters: + - url: http://127.0.0.1:9999 + name: minikube + authProvider: 'serviceAccount' + serviceAccountToken: + $env: K8S_MINIKUBE_TOKEN + - url: http://127.0.0.2:9999 + name: gke-cluster-1 + authProvider: 'google' +``` + +### `serviceLocatorMethod` + +This configures how to determine which clusters a component is running in. + +Currently, the only valid value is: + +- `multiTenant` - This configuration assumes that all components run on all the + provided clusters. + +### `clusterLocatorMethods` + +This is an array used to determine where to retrieve cluster configuration from. + +Currently, the only valid cluster locator method is: + +- `config` - This cluster locator method will read cluster information from your + app-config (see below). + +### `clusters` + +Used by the `config` cluster locator method to construct Kubernetes clients. + +### `clusters.\*.url` + +The base URL to the Kubernetes control plane. Can be found by using the +"Kubernetes master" result from running the `kubectl cluster-info` command. + +### `clusters.\*.name` + +A name to represent this cluster, this must be unique within the `clusters` +array. Users will see this value in the Service Catalog Kubernetes plugin. + +### `clusters.\*.authProvider` + +This determines how the Kubernetes client authenticates with the Kubernetes +cluster. Valid values are: + +| Value | Description | +| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `serviceAccount` | This will use a Kubernetes [service account](https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/) to access the Kubernetes API. When this is used the `serviceAccountToken` field should also be set. | +| `google` | This will use a user's Google auth token from the [Google auth plugin](https://backstage.io/docs/auth/) to access the Kubernetes API. | + +### `clusters.\*.serviceAccount` (optional) + +The service account token to be used when using the `serviceAccount` auth +provider. + +### Role Based Access Control + +The current RBAC permissions required are read-only cluster wide, for the +following objects: + +- pods +- services +- configmaps +- deployments +- replicasets +- horizontalpodautoscalers +- ingresses + +## Surfacing your Kubernetes components as part of an entity + +There are two ways to surface your Kubernetes components as part of an entity. +The label selector takes precedence over the annotation/service id. + +### Common `backstage.io/kubernetes-id` label + +#### Adding the entity annotation + +In order for Backstage to detect that an entity has Kubernetes components, the +following annotation should be added to the entity's `catalog-info.yaml`: + +```yaml +annotations: + 'backstage.io/kubernetes-id': dice-roller +``` + +#### Labeling Kubernetes components + +In order for Kubernetes components to show up in the service catalog as a part +of an entity, Kubernetes components themselves can have the following label: + +```yaml +'backstage.io/kubernetes-id': +``` + +### Label selector query annotation + +You can write your own custom label selector query that Backstage will use to +lookup the objects (similar to `kubectl --selector="your query here"`). Review +the +[labels and selectors Kubernetes documentation](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) +for more info. + +```yaml +'backstage.io/kubernetes-label-selector': 'app=my-app,component=front-end' +``` diff --git a/docs/features/kubernetes/index.md b/docs/features/kubernetes/index.md index c468fdb116..26794eca10 100644 --- a/docs/features/kubernetes/index.md +++ b/docs/features/kubernetes/index.md @@ -5,123 +5,26 @@ sidebar_label: Overview description: Monitoring Kubernetes based services with the service catalog --- -Kubernetes in Backstage is a way to monitor your service's current status when -it is deployed on Kubernetes. +Kubernetes in Backstage is a tool that's designed around the needs of service +owners, not cluster admins. Now developers can easily check the health of their +services no matter how or where those services are deployed — whether it's on a +local host for testing or in production on dozens of clusters around the world. -## Configuration +It will elevate the visibility of errors where identified, and provide drill +down about the deployments, pods, and other objects for a service. -Example: +![Kubernetes plugin screenshot](../../assets/features/kubernetes/backstage-k8s-2-deployments.png) -```yaml -kubernetes: - serviceLocatorMethod: 'multiTenant' - clusterLocatorMethods: - - 'config' - clusters: - - url: http://127.0.0.1:9999 - name: minikube - authProvider: 'serviceAccount' - serviceAccountToken: - $env: K8S_MINIKUBE_TOKEN - - url: http://127.0.0.2:9999 - name: gke-cluster-1 - authProvider: 'google' -``` +The feature is made up of two plugins: +[`@backstage/plugin-kubernetes`](https://github.com/backstage/backstage/tree/master/plugins/kubernetes) +and +[`@backstage/plugin-kubernetes-backend`](https://github.com/backstage/backstage/tree/master/plugins/kubernetes-backend). -### serviceLocatorMethod +The frontend plugin exposes information to the end user in a digestible way, +while the backend wraps the mechanics to connect to Kubernetes clusters to +collect the relevant information. -This configures how to determine which clusters a component is running in. +## Let's use it! -Currently, the only valid value is: - -- `multiTenant` - This configuration assumes that all components run on all the - provided clusters. - -### clusterLocatorMethods - -This is an array used to determine where to retrieve cluster configuration from. - -Currently, the only valid cluster locator method is: - -- `config` - This cluster locator method will read cluster information from your - app-config (see below). - -### clusters - -Used by the `config` cluster locator method to construct Kubernetes clients. - -### clusters.\*.url - -The base URL to the Kubernetes control plane. Can be found by using the -"Kubernetes master" result from running the `kubectl cluster-info` command. - -### clusters.\*.name - -A name to represent this cluster, this must be unique within the `clusters` -array. Users will see this value in the Service Catalog Kubernetes plugin. - -### clusters.\*.authProvider - -This determines how the Kubernetes client authenticates with the Kubernetes -cluster. Valid values are: - -| Value | Description | -| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `serviceAccount` | This will use a Kubernetes [service account](https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/) to access the Kubernetes API. When this is used the `serviceAccountToken` field should also be set. | -| `google` | This will use a user's Google auth token from the [Google auth plugin](https://backstage.io/docs/auth/) to access the Kubernetes API. | - -### clusters.\*.serviceAccount (optional) - -The service account token to be used when using the `serviceAccount` auth -provider. - -## Role Based Access Control - -The current RBAC permissions required are read-only cluster wide, for the -following objects: - -- pods -- services -- configmaps -- deployments -- replicasets -- horizontalpodautoscalers -- ingresses - -## Surfacing your Kubernetes components as part of an entity - -There are two ways to surface your Kubernetes components as part of an entity. -The label selector takes precedence over the annotation/service id. - -### Common `backstage.io/kubernetes-id` label - -#### Adding the entity annotation - -In order for Backstage to detect that an entity has Kubernetes components, the -following annotation should be added to the entity's `catalog-info.yaml`: - -```yaml -annotations: - 'backstage.io/kubernetes-id': dice-roller -``` - -#### Labeling Kubernetes components - -In order for Kubernetes components to show up in the service catalog as a part -of an entity, Kubernetes components themselves can have the following label: - -```yaml -'backstage.io/kubernetes-id': -``` - -### Label selector query annotation - -You can write your own custom label selector query that Backstage will use to -lookup the objects (similar to `kubectl --selector="your query here"`). Review -the -[labels and selectors Kubernetes documentation](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) -for more info. - -```yaml -'backstage.io/kubernetes-label-selector': 'app=my-app,component=front-end' -``` +To get started, first you must [install the Kubernetes plugins](installation.md) +and then [configure them](configuration.md). diff --git a/docs/features/kubernetes/installation.md b/docs/features/kubernetes/installation.md new file mode 100644 index 0000000000..9e531de655 --- /dev/null +++ b/docs/features/kubernetes/installation.md @@ -0,0 +1,119 @@ +--- +id: installation +title: Installation +description: Installing Kubernetes plugin into Backstage +--- + +The Kubernetes feature is a plugin to Backstage, and it is exposed as a tab when +viewing entities in the software catalog. + +If you haven't setup Backstage already, start +[here](../../getting-started/index.md). + +## Adding the Kubernetes frontend plugin + +The first step is to add the frontend Kubernetes plugin to your Backstage +application. Navigate to your new Backstage application directory. And then to +your `packages/app` directory, and install the `@backstage/plugin-kubernetes` +package. + +```bash +cd my-backstage-app/ +cd packages/app +yarn add @backstage/plugin-kubernetes +``` + +Once the package has been installed, you need to import the plugin in your app. +Add the following to `packages/app/src/plugins.ts`: + +`plugins.ts`: + +```typescript +export { plugin as Kubernetes } from '@backstage/plugin-kubernetes'; +``` + +Now, add the "Kubernetes" tab to the catalog entity page. In +`packages/app/src/components/catalog/EntityPage.tsx`, you'll add a router to get +to the tab, and add the tab itself. + +`EntityPage.tsx`: + +```tsx +import { Router as KubernetesRouter } from '@backstage/plugin-kubernetes'; + +// ... + +const ServiceEntityPage = ({ entity }: { entity: Entity }) => ( + + // ... + } + /> + // ... + +); +``` + +That's it! But now, we need the Kubernetes Backend plugin for the frontend to +work. + +## Adding Kubernetes Backend plugin + +Navigate to `packages/backend` of your Backstage app, and install the +`@backstage/plugin-kubernetes-backend` package. + +```bash +cd my-backstage-app/ +cd packages/backend +yarn add @backstage/plugin-kubernetes-backend +``` + +Create a file called `kubernetes.ts` inside `packages/backend/src/plugins/` and +add the following + +`kubernetes.ts`: + +```typescript +import { createRouter } from '@backstage/plugin-kubernetes-backend'; +import { PluginEnvironment } from '../types'; + +export default async function createPlugin({ + logger, + config, +}: PluginEnvironment) { + return await createRouter({ logger, config }); +} +``` + +And import the plugin to `packages/backend/src/index.ts`. There are three lines +of code you'll need to add, and they should be added near similar code in your +existing Backstage backend. + +`index.ts`: + +```typescript +import kubernetes from './plugins/kubernetes'; + +// ... + +const kubernetesEnv = useHotMemoize(module, () => createEnv('kubernetes')); + +// ... + +apiRouter.use('/kubernetes', await kubernetes(kubernetesEnv)); +``` + +That's it! The Kubernetes frontend and backend have now been added to your +Backstage app. + +## Running Backstage locally + +Start the frontend and the backend app by +[running backstage locally](../../getting-started/running-backstage-locally.md). + +## Configuration + +After installing the plugins in the code, you'll need to then +[configure them](configuration.md). diff --git a/microsite/sidebars.json b/microsite/sidebars.json index c2ccea8740..e8d3ab1409 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -39,7 +39,11 @@ { "type": "subcategory", "label": "Kubernetes", - "ids": ["features/kubernetes/overview"] + "ids": [ + "features/kubernetes/overview", + "features/kubernetes/installation", + "features/kubernetes/configuration" + ] }, { "type": "subcategory", diff --git a/packages/backend-common/src/config.ts b/packages/backend-common/src/config.ts index 86beb65805..6989f3567b 100644 --- a/packages/backend-common/src/config.ts +++ b/packages/backend-common/src/config.ts @@ -37,7 +37,6 @@ export async function loadBackendConfig(options: Options): Promise { /* eslint-disable-next-line no-restricted-syntax */ const paths = findPaths(__dirname); const configs = await loadConfig({ - env: process.env.APP_ENV ?? process.env.NODE_ENV ?? 'development', configRoot: paths.targetRoot, configPaths: configOpts.map(opt => resolvePath(opt)), }); diff --git a/packages/catalog-model/examples/acme/backstage-group.yaml b/packages/catalog-model/examples/acme/backstage-group.yaml index 3aecfa0023..f83587532c 100644 --- a/packages/catalog-model/examples/acme/backstage-group.yaml +++ b/packages/catalog-model/examples/acme/backstage-group.yaml @@ -8,6 +8,6 @@ spec: profile: displayName: Backstage email: backstage@example.com - picture: https://example.com/groups/backstage.jpeg + picture: https://avatars.dicebear.com/api/identicon/backstage@example.com.svg?background=%23fff&margin=25 parent: infrastructure children: [team-a, team-b] diff --git a/packages/catalog-model/examples/acme/org.yaml b/packages/catalog-model/examples/acme/org.yaml index 2b44183bc5..05afc265c0 100644 --- a/packages/catalog-model/examples/acme/org.yaml +++ b/packages/catalog-model/examples/acme/org.yaml @@ -8,7 +8,7 @@ spec: profile: displayName: ACME Corp email: info@example.com - picture: https://example.com/logo.jpeg + picture: https://avatars.dicebear.com/api/identicon/info@example.com.svg?background=%23fff&margin=25 children: [infrastructure] --- apiVersion: backstage.io/v1alpha1 diff --git a/packages/catalog-model/examples/acme/team-a-group.yaml b/packages/catalog-model/examples/acme/team-a-group.yaml index ed736dce50..e343209d5f 100644 --- a/packages/catalog-model/examples/acme/team-a-group.yaml +++ b/packages/catalog-model/examples/acme/team-a-group.yaml @@ -8,7 +8,7 @@ spec: profile: # Intentional no displayName for testing email: team-a@example.com - picture: https://example.com/groups/team-a.jpeg + picture: https://avatars.dicebear.com/api/identicon/team-a@example.com.svg?background=%23fff&margin=25 parent: backstage children: [] --- @@ -20,7 +20,7 @@ spec: profile: # Intentional no displayName for testing email: breanna-davison@example.com - picture: https://example.com/staff/breanna.jpeg + picture: https://avatars.dicebear.com/api/avataaars/breanna-davison@example.com.svg?background=%23fff memberOf: [team-a] --- apiVersion: backstage.io/v1alpha1 @@ -31,7 +31,7 @@ spec: profile: displayName: Janelle Dawe email: janelle-dawe@example.com - picture: https://example.com/staff/janelle.jpeg + picture: https://avatars.dicebear.com/api/avataaars/janelle-dawe@example.com.svg?background=%23fff memberOf: [team-a] --- apiVersion: backstage.io/v1alpha1 @@ -42,7 +42,7 @@ spec: profile: displayName: Nigel Manning email: nigel-manning@example.com - picture: https://example.com/staff/nigel.jpeg + picture: https://avatars.dicebear.com/api/avataaars/nigel-manning@example.com.svg?background=%23fff memberOf: [team-a] --- # This user is added as an example, to make it more easy for the "Guest" @@ -56,5 +56,5 @@ spec: profile: displayName: Guest User email: guest@example.com - picture: https://example.com/staff/the-ceos-dog.jpeg + picture: https://avatars.dicebear.com/api/avataaars/guest@example.com.svg?background=%23fff memberOf: [team-a] diff --git a/packages/catalog-model/examples/acme/team-b-group.yaml b/packages/catalog-model/examples/acme/team-b-group.yaml index 9ed50b04c2..20ab8721ea 100644 --- a/packages/catalog-model/examples/acme/team-b-group.yaml +++ b/packages/catalog-model/examples/acme/team-b-group.yaml @@ -8,7 +8,7 @@ spec: profile: displayName: Team B email: team-b@example.com - picture: https://example.com/groups/team-b.jpeg + picture: https://avatars.dicebear.com/api/identicon/team-b@example.com.svg?background=%23fff&margin=25 parent: backstage children: [] --- @@ -20,7 +20,7 @@ spec: profile: displayName: Amelia Park email: amelia-park@example.com - picture: https://example.com/staff/amelia.jpeg + picture: https://avatars.dicebear.com/api/avataaars/amelia-park@example.com.svg?background=%23fff memberOf: [team-b] --- apiVersion: backstage.io/v1alpha1 @@ -31,7 +31,7 @@ spec: profile: displayName: Colette Brock email: colette-brock@example.com - picture: https://example.com/staff/colette.jpeg + picture: https://avatars.dicebear.com/api/avataaars/colette-brock@example.com.svg?background=%23fff memberOf: [team-b] --- apiVersion: backstage.io/v1alpha1 @@ -42,7 +42,7 @@ spec: profile: displayName: Jenny Doe email: jenny-doe@example.com - picture: https://example.com/staff/jenny.jpeg + picture: https://avatars.dicebear.com/api/avataaars/jenny-doe@example.com.svg?background=%23fff memberOf: [team-b] --- apiVersion: backstage.io/v1alpha1 @@ -53,7 +53,7 @@ spec: profile: displayName: Jonathon Page email: jonathon-page@example.com - picture: https://example.com/staff/jonathon.jpeg + picture: https://avatars.dicebear.com/api/avataaars/jonathon-page@example.com.svg?background=%23fff memberOf: [team-b] --- apiVersion: backstage.io/v1alpha1 @@ -64,5 +64,5 @@ spec: profile: displayName: Justine Barrow email: justine-barrow@example.com - picture: https://example.com/staff/justine.jpeg + picture: https://avatars.dicebear.com/api/avataaars/justine-barrow@example.com.svg?background=%23fff memberOf: [team-b] diff --git a/packages/catalog-model/examples/acme/team-c-group.yaml b/packages/catalog-model/examples/acme/team-c-group.yaml index 7f151dbb1e..639b591d61 100644 --- a/packages/catalog-model/examples/acme/team-c-group.yaml +++ b/packages/catalog-model/examples/acme/team-c-group.yaml @@ -8,7 +8,7 @@ spec: profile: displayName: Team C email: team-c@example.com - picture: https://example.com/groups/team-c.jpeg + picture: https://avatars.dicebear.com/api/identicon/team-c@example.com.svg?background=%23fff&margin=25 parent: boxoffice children: [] --- @@ -20,7 +20,7 @@ spec: profile: displayName: Calum Leavy email: calum-leavy@example.com - picture: https://example.com/staff/calum.jpeg + picture: https://avatars.dicebear.com/api/avataaars/calum-leavy@example.com.svg?background=%23fff memberOf: [team-c] --- apiVersion: backstage.io/v1alpha1 @@ -31,7 +31,7 @@ spec: profile: displayName: Frank Tiernan email: frank-tiernan@example.com - picture: https://example.com/staff/frank.jpeg + picture: https://avatars.dicebear.com/api/avataaars/frank-tiernan@example.com.svg?background=%23fff memberOf: [team-c] --- apiVersion: backstage.io/v1alpha1 @@ -42,7 +42,7 @@ spec: profile: displayName: Peadar MacMahon email: peadar-macmahon@example.com - picture: https://example.com/staff/peadar.jpeg + picture: https://avatars.dicebear.com/api/avataaars/peadar-macmahon@example.com.svg?background=%23fff memberOf: [team-c] --- apiVersion: backstage.io/v1alpha1 @@ -53,7 +53,7 @@ spec: profile: displayName: Sarah Gilroy email: sarah-gilroy@example.com - picture: https://example.com/staff/sarah.jpeg + picture: https://avatars.dicebear.com/api/avataaars/sarah-gilroy@example.com.svg?background=%23fff memberOf: [team-c] --- apiVersion: backstage.io/v1alpha1 @@ -64,5 +64,5 @@ spec: profile: displayName: Tara MacGovern email: tara-macgovern@example.com - picture: https://example.com/staff/tara.jpeg + picture: https://avatars.dicebear.com/api/avataaars/tara-macgovern@example.com.svg?background=%23fff memberOf: [team-c] diff --git a/packages/catalog-model/examples/acme/team-d-group.yaml b/packages/catalog-model/examples/acme/team-d-group.yaml index a0d3c2351a..898ed8be6c 100644 --- a/packages/catalog-model/examples/acme/team-d-group.yaml +++ b/packages/catalog-model/examples/acme/team-d-group.yaml @@ -8,7 +8,7 @@ spec: profile: displayName: Team D email: team-d@example.com - picture: https://example.com/groups/team-d.jpeg + picture: https://avatars.dicebear.com/api/identicon/team-d@example.com.svg?background=%23fff&margin=25 parent: boxoffice children: [] --- @@ -20,7 +20,7 @@ spec: profile: displayName: Eva MacDowell email: eva-macdowell@example.com - picture: https://example.com/staff/eva.jpeg + picture: https://avatars.dicebear.com/api/avataaars/eva-macdowell@example.com.svg?background=%23fff memberOf: [team-d] --- apiVersion: backstage.io/v1alpha1 @@ -31,5 +31,5 @@ spec: profile: displayName: Lucy Sheehan email: lucy-sheehan@example.com - picture: https://example.com/staff/lucy.jpeg + picture: https://avatars.dicebear.com/api/avataaars/lucy-sheehan@example.com.svg?background=%23fff memberOf: [team-d] diff --git a/packages/cli/package.json b/packages/cli/package.json index 0d1605f752..a98534ecb6 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -125,7 +125,7 @@ "@types/html-webpack-plugin": "^3.2.2", "@types/http-proxy": "^1.17.4", "@types/inquirer": "^7.3.1", - "@types/mini-css-extract-plugin": "^0.9.1", + "@types/mini-css-extract-plugin": "^1.2.2", "@types/mock-fs": "^4.13.0", "@types/node": "^13.7.2", "@types/react-dev-utils": "^9.0.4", diff --git a/packages/config-loader/src/lib/transform/include.test.ts b/packages/config-loader/src/lib/transform/include.test.ts index 564326aefc..1724159e56 100644 --- a/packages/config-loader/src/lib/transform/include.test.ts +++ b/packages/config-loader/src/lib/transform/include.test.ts @@ -14,8 +14,12 @@ * limitations under the License. */ +import * as os from 'os'; +import { resolve as resolvePath } from 'path'; import { createIncludeTransform } from './include'; +const root = os.platform() === 'win32' ? 'C:\\' : '/'; + const env = jest.fn(async (name: string) => { return ({ SECRET: 'my-secret', @@ -24,11 +28,11 @@ const env = jest.fn(async (name: string) => { const readFile = jest.fn(async (path: string) => { const content = ({ - '/my-secret': 'secret', - '/my-data.json': '{"a":{"b":{"c":42}}}', - '/my-data.yaml': 'some:\n yaml:\n key: 7', - '/my-data.yml': 'different: { key: hello }', - '/invalid.yaml': 'foo: [}', + [resolvePath(root, 'my-secret')]: 'secret', + [resolvePath(root, 'my-data.json')]: '{"a":{"b":{"c":42}}}', + [resolvePath(root, 'my-data.yaml')]: 'some:\n yaml:\n key: 7', + [resolvePath(root, 'my-data.yml')]: 'different: { key: hello }', + [resolvePath(root, 'invalid.yaml')]: 'foo: [}', } as { [key: string]: string })[path]; if (!content) { @@ -41,56 +45,58 @@ const includeTransform = createIncludeTransform(env, readFile); describe('includeTransform', () => { it('should not transform unknown values', async () => { - await expect(includeTransform('foo', '/')).resolves.toEqual({ + await expect(includeTransform('foo', root)).resolves.toEqual({ applied: false, }); - await expect(includeTransform([1], '/')).resolves.toEqual({ + await expect(includeTransform([1], root)).resolves.toEqual({ applied: false, }); - await expect(includeTransform(1, '/')).resolves.toEqual({ applied: false }); - await expect(includeTransform({ x: 'y' }, '/')).resolves.toEqual({ + await expect(includeTransform(1, root)).resolves.toEqual({ applied: false, }); - await expect(includeTransform(null, '/')).resolves.toEqual({ + await expect(includeTransform({ x: 'y' }, root)).resolves.toEqual({ + applied: false, + }); + await expect(includeTransform(null, root)).resolves.toEqual({ applied: false, }); }); it('should include text files', async () => { await expect( - includeTransform({ $file: 'my-secret' }, '/'), + includeTransform({ $file: 'my-secret' }, root), ).resolves.toEqual({ applied: true, value: 'secret' }); - await expect(includeTransform({ $file: 'no-secret' }, '/')).rejects.toThrow( - 'File not found!', - ); + await expect( + includeTransform({ $file: 'no-secret' }, root), + ).rejects.toThrow('File not found!'); }); it('should include env vars', async () => { - await expect(includeTransform({ $env: 'SECRET' }, '/')).resolves.toEqual({ + await expect(includeTransform({ $env: 'SECRET' }, root)).resolves.toEqual({ applied: true, value: 'my-secret', }); - await expect(includeTransform({ $env: 'NO_SECRET' }, '/')).resolves.toEqual( - { - applied: true, - value: undefined, - }, - ); + await expect( + includeTransform({ $env: 'NO_SECRET' }, root), + ).resolves.toEqual({ + applied: true, + value: undefined, + }); }); it('should include config files', async () => { // New format with path in fragment await expect( - includeTransform({ $include: 'my-data.json#a.b.c' }, '/'), + includeTransform({ $include: 'my-data.json#a.b.c' }, root), ).resolves.toEqual({ applied: true, value: 42 }); await expect( - includeTransform({ $include: 'my-data.json#a.b' }, '/'), + includeTransform({ $include: 'my-data.json#a.b' }, root), ).resolves.toEqual({ applied: true, value: { c: 42 } }); await expect( - includeTransform({ $include: 'my-data.yaml#some.yaml.key' }, '/'), + includeTransform({ $include: 'my-data.yaml#some.yaml.key' }, root), ).resolves.toEqual({ applied: true, value: 7 }); await expect( - includeTransform({ $include: 'my-data.yaml' }, '/'), + includeTransform({ $include: 'my-data.yaml' }, root), ).resolves.toEqual({ applied: true, value: { @@ -98,7 +104,7 @@ describe('includeTransform', () => { }, }); await expect( - includeTransform({ $include: 'my-data.yaml#' }, '/'), + includeTransform({ $include: 'my-data.yaml#' }, root), ).resolves.toEqual({ applied: true, value: { @@ -106,26 +112,26 @@ describe('includeTransform', () => { }, }); await expect( - includeTransform({ $include: 'my-data.yml#different.key' }, '/'), + includeTransform({ $include: 'my-data.yml#different.key' }, root), ).resolves.toEqual({ applied: true, value: 'hello' }); }); it('should reject invalid includes', async () => { await expect( - includeTransform({ $include: 'no-parser.js' }, '/'), + includeTransform({ $include: 'no-parser.js' }, root), ).rejects.toThrow( 'no configuration parser available for included file no-parser.js', ); await expect( - includeTransform({ $include: 'no-data.yml#different.key' }, '/'), + includeTransform({ $include: 'no-data.yml#different.key' }, root), ).rejects.toThrow('File not found!'); await expect( - includeTransform({ $include: 'my-data.yml#missing.key' }, '/'), + includeTransform({ $include: 'my-data.yml#missing.key' }, root), ).rejects.toThrow( "value at 'missing' in included file my-data.yml is not an object", ); await expect( - includeTransform({ $include: 'invalid.yaml' }, '/'), + includeTransform({ $include: 'invalid.yaml' }, root), ).rejects.toThrow( 'failed to parse included file invalid.yaml, YAMLSyntaxError: Flow sequence contains an unexpected }', ); diff --git a/packages/core/src/components/ProgressBars/Gauge.stories.tsx b/packages/core/src/components/ProgressBars/Gauge.stories.tsx index 7882714529..ab9c263f05 100644 --- a/packages/core/src/components/ProgressBars/Gauge.stories.tsx +++ b/packages/core/src/components/ProgressBars/Gauge.stories.tsx @@ -20,7 +20,7 @@ import { Gauge } from './Gauge'; const containerStyle = { width: 300 }; export default { - title: 'Feedback/Gauge', + title: 'Data Display/Gauge', component: Gauge, }; diff --git a/packages/core/src/components/ProgressBars/LinearGauge.stories.tsx b/packages/core/src/components/ProgressBars/LinearGauge.stories.tsx index a745d210b4..fa3c7c00f0 100644 --- a/packages/core/src/components/ProgressBars/LinearGauge.stories.tsx +++ b/packages/core/src/components/ProgressBars/LinearGauge.stories.tsx @@ -20,7 +20,7 @@ import { LinearGauge } from './LinearGauge'; const containerStyle = { width: 300 }; export default { - title: 'Feedback/LinearGauge', + title: 'Data Display/LinearGauge', component: LinearGauge, }; diff --git a/packages/core/src/components/Status/Status.stories.tsx b/packages/core/src/components/Status/Status.stories.tsx index 560d07ad87..205645e1ec 100644 --- a/packages/core/src/components/Status/Status.stories.tsx +++ b/packages/core/src/components/Status/Status.stories.tsx @@ -27,7 +27,7 @@ import { Table } from '../Table'; import { InfoCard } from '../../layout/InfoCard'; export default { - title: 'Feedback/Status', + title: 'Data Display/Status', component: StatusOK, }; diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index cb703b8e15..046b6bdf76 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -32,6 +32,7 @@ import { CatalogProcessorEntityResult, CatalogProcessorErrorResult, CatalogProcessorLocationResult, + CatalogProcessorParser, CatalogProcessorResult, } from './processors/types'; import { LocationReader, ReadLocationResult } from './types'; @@ -41,6 +42,7 @@ const MAX_DEPTH = 10; type Options = { reader: UrlReader; + parser: CatalogProcessorParser; logger: Logger; config: Config; processors: CatalogProcessor[]; @@ -137,7 +139,6 @@ export class LocationReaders implements LocationReader { if (emitResult.type === 'relation') { throw new Error('readLocation may not emit entity relations'); } - emit(emitResult); }; @@ -149,6 +150,7 @@ export class LocationReaders implements LocationReader { item.location, item.optional, validatedEmit, + this.options.parser, ) ) { return; diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts index aac4235e3a..49b6fe71e0 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts @@ -25,6 +25,7 @@ import { CatalogProcessorErrorResult, CatalogProcessorResult, } from './types'; +import { defaultEntityDataParser } from './util/parse'; describe('UrlReaderProcessor', () => { const mockApiOrigin = 'http://localhost'; @@ -52,7 +53,7 @@ describe('UrlReaderProcessor', () => { ); const generated = (await new Promise(emit => - processor.readLocation(spec, false, emit), + processor.readLocation(spec, false, emit, defaultEntityDataParser), )) as CatalogProcessorEntityResult; expect(generated.type).toBe('entity'); @@ -81,7 +82,7 @@ describe('UrlReaderProcessor', () => { ); const generated = (await new Promise(emit => - processor.readLocation(spec, false, emit), + processor.readLocation(spec, false, emit, defaultEntityDataParser), )) as CatalogProcessorErrorResult; expect(generated.type).toBe('error'); diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index 04f5ee738a..9daae29d3b 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -18,8 +18,11 @@ import { UrlReader } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; import { Logger } from 'winston'; import * as result from './results'; -import { CatalogProcessor, CatalogProcessorEmit } from './types'; -import { parseEntityYaml } from './util/parse'; +import { + CatalogProcessor, + CatalogProcessorEmit, + CatalogProcessorParser, +} from './types'; // TODO(Rugvip): Added for backwards compatibility when moving to UrlReader, this // can be removed in a bit @@ -43,6 +46,7 @@ export class UrlReaderProcessor implements CatalogProcessor { location: LocationSpec, optional: boolean, emit: CatalogProcessorEmit, + parser: CatalogProcessorParser, ): Promise { if (deprecatedTypes.includes(location.type)) { // TODO(Rugvip): Remove this warning a month or two into 2021, and remove support for the deprecated types. @@ -57,7 +61,7 @@ export class UrlReaderProcessor implements CatalogProcessor { try { const data = await this.options.reader.read(location.target); - for (const parseResult of parseEntityYaml(data, location)) { + for await (const parseResult of parser({ data, location })) { emit(parseResult); } } catch (error) { diff --git a/plugins/catalog-backend/src/ingestion/processors/types.ts b/plugins/catalog-backend/src/ingestion/processors/types.ts index 1bf30567bc..f7e11d5616 100644 --- a/plugins/catalog-backend/src/ingestion/processors/types.ts +++ b/plugins/catalog-backend/src/ingestion/processors/types.ts @@ -27,12 +27,15 @@ export type CatalogProcessor = { * @param location The location to read * @param optional Whether a missing target should trigger an error * @param emit A sink for items resulting from the read + * @param parser A parser, that is able to take the raw catalog descriptor + * data and turn it into the actual result pieces. * @returns True if handled by this processor, false otherwise */ readLocation?( location: LocationSpec, optional: boolean, emit: CatalogProcessorEmit, + parser: CatalogProcessorParser, ): Promise; /** @@ -100,6 +103,16 @@ export type CatalogProcessor = { ): Promise; }; +/** + * A parser, that is able to take the raw catalog descriptor data and turn it + * into the actual result pieces. The default implementation performs a YAML + * document parsing. + */ +export type CatalogProcessorParser = (options: { + data: Buffer; + location: LocationSpec; +}) => AsyncIterable; + export type CatalogProcessorEmit = (generated: CatalogProcessorResult) => void; export type CatalogProcessorLocationResult = { diff --git a/plugins/catalog-backend/src/ingestion/processors/util/parse.ts b/plugins/catalog-backend/src/ingestion/processors/util/parse.ts index c3dcd42d62..aa24968d6d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/parse.ts +++ b/plugins/catalog-backend/src/ingestion/processors/util/parse.ts @@ -18,7 +18,7 @@ import { Entity, LocationSpec } from '@backstage/catalog-model'; import lodash from 'lodash'; import yaml from 'yaml'; import * as result from '../results'; -import { CatalogProcessorResult } from '../types'; +import { CatalogProcessorParser, CatalogProcessorResult } from '../types'; export function* parseEntityYaml( data: Buffer, @@ -50,3 +50,12 @@ export function* parseEntityYaml( } } } + +export const defaultEntityDataParser: CatalogProcessorParser = async function* defaultEntityDataParser({ + data, + location, +}) { + for (const e of parseEntityYaml(data, location)) { + yield e; + } +}; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts index 37337f76f1..695902fec9 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts @@ -20,6 +20,7 @@ import { ConfigReader } from '@backstage/config'; import Knex from 'knex'; import yaml from 'yaml'; import { DatabaseManager } from '../database'; +import { CatalogProcessorParser } from '../ingestion'; import * as result from '../ingestion/processors/results'; import { CatalogBuilder, CatalogEnvironment } from './CatalogBuilder'; @@ -209,4 +210,26 @@ describe('CatalogBuilder', () => { }), ]); }); + + it('setEntityDataParser works', async () => { + const mockParser: CatalogProcessorParser = jest + .fn() + .mockImplementation(() => {}); + + const builder = new CatalogBuilder(env) + .setEntityDataParser(mockParser) + .replaceProcessors([ + { + async readLocation(_location, _optional, _emit, parser) { + expect(parser).toBe(mockParser); + return true; + }, + }, + ]); + + const { higherOrderOperation } = await builder.build(); + await higherOrderOperation.addLocation({ type: 'x', target: 'y' }); + + expect.assertions(1); + }); }); diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index f6170135fd..938a406e56 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -39,6 +39,7 @@ import { AnnotateLocationEntityProcessor, BuiltinKindsEntityProcessor, CatalogProcessor, + CatalogProcessorParser, CodeOwnersProcessor, FileReaderProcessor, GithubOrgReaderProcessor, @@ -60,6 +61,7 @@ import { textPlaceholderResolver, yamlPlaceholderResolver, } from '../ingestion/processors/PlaceholderProcessor'; +import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; import { LocationAnalyzer } from '../ingestion/types'; export type CatalogEnvironment = { @@ -96,6 +98,7 @@ export class CatalogBuilder { private fieldFormatValidators: Partial; private processors: CatalogProcessor[]; private processorsReplace: boolean; + private parser: CatalogProcessorParser | undefined; constructor(env: CatalogEnvironment) { this.env = env; @@ -105,6 +108,7 @@ export class CatalogBuilder { this.fieldFormatValidators = {}; this.processors = []; this.processorsReplace = false; + this.parser = undefined; } /** @@ -197,6 +201,20 @@ export class CatalogBuilder { return this; } + /** + * Sets up the catalog to use a custom parser for entity data. + * + * This is the function that gets called immediately after some raw entity + * specification data has been read from a remote source, and needs to be + * parsed and emitted as structured data. + * + * @param parser The custom parser + */ + setEntityDataParser(parser: CatalogProcessorParser): CatalogBuilder { + this.parser = parser; + return this; + } + /** * Wires up and returns all of the component parts of the catalog */ @@ -211,9 +229,11 @@ export class CatalogBuilder { const policy = this.buildEntityPolicy(); const processors = this.buildProcessors(); const rulesEnforcer = CatalogRulesEnforcer.fromConfig(config); + const parser = this.parser || defaultEntityDataParser; const locationReader = new LocationReaders({ ...this.env, + parser, processors, rulesEnforcer, policy, diff --git a/plugins/catalog/src/components/AboutCard/AboutContent.tsx b/plugins/catalog/src/components/AboutCard/AboutContent.tsx index b2d029c927..82652b3986 100644 --- a/plugins/catalog/src/components/AboutCard/AboutContent.tsx +++ b/plugins/catalog/src/components/AboutCard/AboutContent.tsx @@ -21,7 +21,7 @@ import { } from '@backstage/catalog-model'; import { Chip, Grid, makeStyles, Typography } from '@material-ui/core'; import React from 'react'; -import { EntityRefLink } from '../EntityRefLink'; +import { EntityRefLinks } from '../EntityRefLink'; import { getEntityRelations } from '../getEntityRelations'; import { AboutField } from './AboutField'; @@ -41,17 +41,17 @@ export const AboutContent = ({ entity }: Props) => { const isDomain = entity.kind.toLowerCase() === 'domain'; const isResource = entity.kind.toLowerCase() === 'resource'; const isComponent = entity.kind.toLowerCase() === 'component'; - const [partOfSystemRelation] = getEntityRelations(entity, RELATION_PART_OF, { + const partOfSystemRelations = getEntityRelations(entity, RELATION_PART_OF, { kind: 'system', }); - const [partOfComponentRelation] = getEntityRelations( + const partOfComponentRelations = getEntityRelations( entity, RELATION_PART_OF, { kind: 'component', }, ); - const [partOfDomainRelation] = getEntityRelations(entity, RELATION_PART_OF, { + const partOfDomainRelations = getEntityRelations(entity, RELATION_PART_OF, { kind: 'domain', }); const ownedByRelations = getEntityRelations(entity, RELATION_OWNED_BY); @@ -64,12 +64,7 @@ export const AboutContent = ({ entity }: Props) => { - {ownedByRelations.map((t, i) => ( - - {i > 0 && ', '} - - - ))} + {isSystem && ( { value="No Domain" gridSizes={{ xs: 12, sm: 6, lg: 4 }} > - {partOfDomainRelation && ( - - )} + )} {!isSystem && !isDomain && ( @@ -91,22 +84,20 @@ export const AboutContent = ({ entity }: Props) => { value="No System" gridSizes={{ xs: 12, sm: 6, lg: 4 }} > - {partOfSystemRelation && ( - - )} + )} - {isComponent && partOfComponentRelation && ( + {isComponent && partOfComponentRelations.length > 0 && ( - diff --git a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx index 8741cb6157..18c61c2a04 100644 --- a/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx +++ b/plugins/catalog/src/components/CatalogTable/CatalogTable.tsx @@ -20,17 +20,19 @@ import { RELATION_PART_OF, } from '@backstage/catalog-model'; import { Table, TableColumn, TableProps } from '@backstage/core'; -import { Chip, Link } from '@material-ui/core'; +import { Chip } from '@material-ui/core'; import Edit from '@material-ui/icons/Edit'; import OpenInNew from '@material-ui/icons/OpenInNew'; import { Alert } from '@material-ui/lab'; import React from 'react'; -import { generatePath, Link as RouterLink } from 'react-router-dom'; import { findLocationForEntityMeta } from '../../data/utils'; import { useStarredEntities } from '../../hooks/useStarredEntities'; -import { entityRoute, entityRouteParams } from '../../routes'; import { createEditLink } from '../createEditLink'; -import { EntityRefLink, formatEntityRefTitle } from '../EntityRefLink'; +import { + EntityRefLink, + EntityRefLinks, + formatEntityRefTitle, +} from '../EntityRefLink'; import { favouriteEntityIcon, favouriteEntityTooltip, @@ -40,7 +42,7 @@ import { getEntityRelations } from '../getEntityRelations'; type EntityRow = Entity & { row: { partOfSystemRelationTitle?: string; - partOfSystemRelation?: EntityName; + partOfSystemRelations: EntityName[]; ownedByRelationsTitle?: string; ownedByRelations: EntityName[]; }; @@ -52,43 +54,27 @@ const columns: TableColumn[] = [ field: 'metadata.name', highlight: true, render: entity => ( - - {entity.metadata.name} - + {entity.metadata.name} ), }, { title: 'System', field: 'row.partOfSystemRelationTitle', render: entity => ( - <> - {entity.row.partOfSystemRelation && ( - - )} - + ), }, { title: 'Owner', field: 'row.ownedByRelationsTitle', render: entity => ( - <> - {entity.row.ownedByRelations.map((t, i) => ( - - {i > 0 && ', '} - - - ))} - + ), }, { @@ -182,7 +168,7 @@ export const CatalogTable = ({ ]; const rows = entities.map(e => { - const [partOfSystemRelation] = getEntityRelations(e, RELATION_PART_OF, { + const partOfSystemRelations = getEntityRelations(e, RELATION_PART_OF, { kind: 'system', }); const ownedByRelations = getEntityRelations(e, RELATION_OWNED_BY); @@ -194,12 +180,13 @@ export const CatalogTable = ({ .map(r => formatEntityRefTitle(r, { defaultKind: 'group' })) .join(', '), ownedByRelations, - partOfSystemRelationTitle: partOfSystemRelation - ? formatEntityRefTitle(partOfSystemRelation, { - defaultKind: 'system', - }) - : undefined, - partOfSystemRelation, + partOfSystemRelationTitle: + partOfSystemRelations.length > 0 + ? formatEntityRefTitle(partOfSystemRelations[0], { + defaultKind: 'system', + }) + : undefined, + partOfSystemRelations, }, }; }); diff --git a/plugins/catalog/src/components/EntityRefLink/EntityRefLinks.test.tsx b/plugins/catalog/src/components/EntityRefLink/EntityRefLinks.test.tsx new file mode 100644 index 0000000000..9bc26b0954 --- /dev/null +++ b/plugins/catalog/src/components/EntityRefLink/EntityRefLinks.test.tsx @@ -0,0 +1,66 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { render } from '@testing-library/react'; +import React from 'react'; +import { MemoryRouter } from 'react-router'; +import { EntityRefLinks } from './EntityRefLinks'; + +describe('', () => { + it('renders a single link', () => { + const entityNames = [ + { + kind: 'Component', + namespace: 'default', + name: 'software', + }, + ]; + const { getByText } = render(, { + wrapper: MemoryRouter, + }); + expect(getByText('component:software')).toHaveAttribute( + 'href', + '/catalog/default/component/software', + ); + }); + + it('renders multiple links', () => { + const entityNames = [ + { + kind: 'Component', + namespace: 'default', + name: 'software', + }, + { + kind: 'API', + namespace: 'default', + name: 'interface', + }, + ]; + const { getByText } = render(, { + wrapper: MemoryRouter, + }); + expect(getByText(',')).toBeInTheDocument(); + expect(getByText('component:software')).toHaveAttribute( + 'href', + '/catalog/default/component/software', + ); + expect(getByText('api:interface')).toHaveAttribute( + 'href', + '/catalog/default/api/interface', + ); + }); +}); diff --git a/plugins/catalog/src/components/EntityRefLink/EntityRefLinks.tsx b/plugins/catalog/src/components/EntityRefLink/EntityRefLinks.tsx new file mode 100644 index 0000000000..3c8beaec6f --- /dev/null +++ b/plugins/catalog/src/components/EntityRefLink/EntityRefLinks.tsx @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Entity, EntityName } from '@backstage/catalog-model'; +import React from 'react'; +import { EntityRefLink } from './EntityRefLink'; + +type EntityRefLinksProps = { + entityRefs: (Entity | EntityName)[]; + defaultKind?: string; +}; + +// TODO: Move into a shared helper package +export const EntityRefLinks = ({ + entityRefs, + defaultKind, +}: EntityRefLinksProps) => { + return ( + <> + {entityRefs.map((r, i) => ( + + {i > 0 && ', '} + + + ))} + + ); +}; diff --git a/plugins/catalog/src/components/EntityRefLink/index.ts b/plugins/catalog/src/components/EntityRefLink/index.ts index 76a7da38c5..9e6e440514 100644 --- a/plugins/catalog/src/components/EntityRefLink/index.ts +++ b/plugins/catalog/src/components/EntityRefLink/index.ts @@ -14,4 +14,5 @@ * limitations under the License. */ export { EntityRefLink } from './EntityRefLink'; +export { EntityRefLinks } from './EntityRefLinks'; export { formatEntityRefTitle } from './format'; diff --git a/plugins/kafka-backend/README.md b/plugins/kafka-backend/README.md index b2271a2911..13078ea816 100644 --- a/plugins/kafka-backend/README.md +++ b/plugins/kafka-backend/README.md @@ -23,6 +23,8 @@ Example: ```yaml kafka: clientId: backstage - brokers: - - localhost:9092 + clusters: + name: prod + brokers: + - localhost:9092 ``` diff --git a/plugins/kafka-backend/config.d.ts b/plugins/kafka-backend/config.d.ts index c2ab5ea777..87254b152c 100644 --- a/plugins/kafka-backend/config.d.ts +++ b/plugins/kafka-backend/config.d.ts @@ -19,20 +19,22 @@ export interface Config { * Client ID used to Backstage uses to identify when connecting to the Kafka cluster. */ clientId: string; - /** - * List of brokers in the Kafka cluster to connect to. - */ - brokers: string[]; - - /** - * Optional SSL connection parameters to connect to the cluster. Passed directly to Node tls.connect. - * See https://nodejs.org/dist/latest-v8.x/docs/api/tls.html#tls_tls_createsecurecontext_options - */ - ssl?: { - ca: string[]; - /** @visibility secret */ - key: string; - cert: string; - }; + clusters: { + name: string; + /** + * List of brokers in the Kafka cluster to connect to. + */ + brokers: string[]; + /** + * Optional SSL connection parameters to connect to the cluster. Passed directly to Node tls.connect. + * See https://nodejs.org/dist/latest-v8.x/docs/api/tls.html#tls_tls_createsecurecontext_options + */ + ssl?: { + ca: string[]; + /** @visibility secret */ + key: string; + cert: string; + }; + }[]; }; } diff --git a/plugins/kafka-backend/src/config/ClusterReader.test.ts b/plugins/kafka-backend/src/config/ClusterReader.test.ts new file mode 100644 index 0000000000..0529684425 --- /dev/null +++ b/plugins/kafka-backend/src/config/ClusterReader.test.ts @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { getClusterDetails } from './ClusterReader'; + +describe('getClusterDetails', () => { + it('empty clusters return empty cluster details', async () => { + const config = new ConfigReader({ clusters: [] }); + + const result = getClusterDetails(config.getConfigArray('clusters')); + + expect(result).toStrictEqual([]); + }); + + it('two clusters return two cluster details', async () => { + const config = new ConfigReader({ + clusters: [ + { name: 'cluster1', brokers: ['a', 'b'] }, + { name: 'cluster2', brokers: ['d'] }, + ], + }); + + const result = getClusterDetails(config.getConfigArray('clusters')); + + expect(result).toStrictEqual([ + { name: 'cluster1', brokers: ['a', 'b'] }, + { name: 'cluster2', brokers: ['d'] }, + ]); + }); +}); diff --git a/plugins/kafka-backend/src/config/ClusterReader.ts b/plugins/kafka-backend/src/config/ClusterReader.ts new file mode 100644 index 0000000000..a4e6a4e4fe --- /dev/null +++ b/plugins/kafka-backend/src/config/ClusterReader.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; +import { ConnectionOptions } from 'tls'; +import { ClusterDetails } from '../types/types'; + +export function getClusterDetails(config: Config[]): ClusterDetails[] { + return config.map(clusterConfig => { + const clusterDetails = { + name: clusterConfig.getString('name'), + brokers: clusterConfig.getStringArray('brokers'), + }; + const sslConfig = clusterConfig.getOptional('kafka.ssl'); + if (sslConfig) { + return { + ...clusterDetails, + ssl: sslConfig as ConnectionOptions, + }; + } + + return clusterDetails; + }); +} diff --git a/plugins/kafka-backend/src/service/router.test.ts b/plugins/kafka-backend/src/service/router.test.ts index 64c4cf0916..42cd92ddfb 100644 --- a/plugins/kafka-backend/src/service/router.test.ts +++ b/plugins/kafka-backend/src/service/router.test.ts @@ -16,22 +16,34 @@ import request from 'supertest'; import express from 'express'; -import { makeRouter } from './router'; +import { makeRouter, ClusterApi } from './router'; import { getVoidLogger } from '@backstage/backend-common'; import { KafkaApi } from './KafkaApi'; import { when } from 'jest-when'; describe('router', () => { let app: express.Express; - let kafkaApi: jest.Mocked; + let apis: ClusterApi[]; + let devKafkaApi: jest.Mocked; + let prodKafkaApi: jest.Mocked; beforeAll(async () => { - kafkaApi = { + devKafkaApi = { fetchTopicOffsets: jest.fn(), fetchGroupOffsets: jest.fn(), }; - const router = makeRouter(getVoidLogger(), kafkaApi); + prodKafkaApi = { + fetchTopicOffsets: jest.fn(), + fetchGroupOffsets: jest.fn(), + }; + + apis = [ + { name: 'dev', api: devKafkaApi }, + { name: 'prod', api: prodKafkaApi }, + ]; + + const router = makeRouter(getVoidLogger(), apis); app = express().use(router); }); @@ -39,7 +51,7 @@ describe('router', () => { jest.resetAllMocks(); }); - describe('get /consumer/:consumerId/offsets', () => { + describe('get /consumers/clusterId/:consumerId/offsets', () => { it('returns topic and group offsets', async () => { const topic1Offsets = [ { id: 1, offset: '500' }, @@ -60,15 +72,17 @@ describe('router', () => { partitions: [{ id: 1, offset: '456' }], }, ]; - when(kafkaApi.fetchTopicOffsets) + when(prodKafkaApi.fetchTopicOffsets) .calledWith('topic1') .mockResolvedValue(topic1Offsets); - when(kafkaApi.fetchTopicOffsets) + when(prodKafkaApi.fetchTopicOffsets) .calledWith('topic2') .mockResolvedValue(topic2Offsets); - kafkaApi.fetchGroupOffsets.mockResolvedValue(groupOffsets); + when(prodKafkaApi.fetchGroupOffsets) + .calledWith('hey') + .mockResolvedValue(groupOffsets); - const response = await request(app).get('/consumer/hey/offsets'); + const response = await request(app).get('/consumers/prod/hey/offsets'); expect(response.status).toEqual(200); expect(response.body.consumerId).toEqual('hey'); @@ -98,11 +112,70 @@ describe('router', () => { }); it('handles internal error correctly', async () => { - kafkaApi.fetchGroupOffsets.mockRejectedValue(Error('oh no')); + prodKafkaApi.fetchGroupOffsets.mockRejectedValue(Error('oh no')); - const response = await request(app).get('/consumer/hey/offsets'); + const response = await request(app).get('/consumers/prod/hey/offsets'); expect(response.status).toEqual(500); }); + + it('uses correct kafka cluster', async () => { + const topic1ProdOffsets = [{ id: 1, offset: '500' }]; + const topic1DevOffsets = [{ id: 1, offset: '1234' }]; + const groupProdOffsets = [ + { + topic: 'topic1', + partitions: [{ id: 1, offset: '100' }], + }, + ]; + const groupDevOffsets = [ + { + topic: 'topic1', + partitions: [{ id: 1, offset: '567' }], + }, + ]; + when(prodKafkaApi.fetchTopicOffsets) + .calledWith('topic1') + .mockResolvedValue(topic1ProdOffsets); + when(prodKafkaApi.fetchGroupOffsets) + .calledWith('hey') + .mockResolvedValue(groupProdOffsets); + when(devKafkaApi.fetchTopicOffsets) + .calledWith('topic1') + .mockResolvedValue(topic1DevOffsets); + when(devKafkaApi.fetchGroupOffsets) + .calledWith('hey') + .mockResolvedValue(groupDevOffsets); + + const prodResponse = await request(app).get( + '/consumers/prod/hey/offsets', + ); + const devResponse = await request(app).get('/consumers/dev/hey/offsets'); + + expect(prodResponse.status).toEqual(200); + expect(prodResponse.body.consumerId).toEqual('hey'); + expect(new Set(prodResponse.body.offsets)).toStrictEqual( + new Set([ + { + topic: 'topic1', + partitionId: 1, + groupOffset: '100', + topicOffset: '500', + }, + ]), + ); + expect(devResponse.status).toEqual(200); + expect(devResponse.body.consumerId).toEqual('hey'); + expect(new Set(devResponse.body.offsets)).toStrictEqual( + new Set([ + { + topic: 'topic1', + partitionId: 1, + groupOffset: '567', + topicOffset: '1234', + }, + ]), + ); + }); }); }); diff --git a/plugins/kafka-backend/src/service/router.ts b/plugins/kafka-backend/src/service/router.ts index 2bbc877fe2..465347ed8f 100644 --- a/plugins/kafka-backend/src/service/router.ts +++ b/plugins/kafka-backend/src/service/router.ts @@ -20,31 +20,43 @@ import { Logger } from 'winston'; import { Config } from '@backstage/config'; import { KafkaApi, KafkaJsApiImpl } from './KafkaApi'; import _ from 'lodash'; -import { ConnectionOptions } from 'tls'; +import { getClusterDetails } from '../config/ClusterReader'; export interface RouterOptions { logger: Logger; config: Config; } +export interface ClusterApi { + name: string; + api: KafkaApi; +} + export const makeRouter = ( logger: Logger, - kafkaApi: KafkaApi, + kafkaApis: ClusterApi[], ): express.Router => { const router = Router(); router.use(express.json()); - router.get('/consumer/:consumerId/offsets', async (req, res) => { + const kafkaApiByClusterName = _.keyBy(kafkaApis, item => item.name); + + router.get('/consumers/:clusterId/:consumerId/offsets', async (req, res) => { + const clusterId = req.params.clusterId; const consumerId = req.params.consumerId; - logger.debug(`Fetch consumer group ${consumerId} offsets`); + logger.info( + `Fetch consumer group ${consumerId} offsets from cluster ${clusterId}`, + ); - const groupOffsets = await kafkaApi.fetchGroupOffsets(consumerId); + const kafkaApi = kafkaApiByClusterName[clusterId]; + + const groupOffsets = await kafkaApi.api.fetchGroupOffsets(consumerId); const groupWithTopicOffsets = await Promise.all( groupOffsets.map(async ({ topic, partitions }) => { const topicOffsets = _.keyBy( - await kafkaApi.fetchTopicOffsets(topic), + await kafkaApi.api.fetchTopicOffsets(topic), partition => partition.id, ); @@ -70,12 +82,15 @@ export async function createRouter( logger.info('Initializing Kafka backend'); const clientId = options.config.getString('kafka.clientId'); - const brokers = options.config.getStringArray('kafka.brokers'); - const sslConfig = options.config.getOptional('kafka.ssl'); - const ssl = sslConfig ? (sslConfig as ConnectionOptions) : undefined; + const clusters = getClusterDetails( + options.config.getConfigArray('kafka.clusters'), + ); - const kafkaApi = new KafkaJsApiImpl({ clientId, brokers, logger, ssl }); + const kafkaApis = clusters.map(cluster => ({ + name: cluster.name, + api: new KafkaJsApiImpl({ clientId, logger, ...cluster }), + })); - return makeRouter(logger, kafkaApi); + return makeRouter(logger, kafkaApis); } diff --git a/plugins/kafka-backend/src/types/types.ts b/plugins/kafka-backend/src/types/types.ts new file mode 100644 index 0000000000..5f70998dd8 --- /dev/null +++ b/plugins/kafka-backend/src/types/types.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConnectionOptions } from 'tls'; + +export interface ClusterDetails { + name: string; + brokers: string[]; + ssl?: ConnectionOptions; +} diff --git a/plugins/kafka/README.md b/plugins/kafka/README.md index 7d947ae41b..c41c328741 100644 --- a/plugins/kafka/README.md +++ b/plugins/kafka/README.md @@ -65,8 +65,10 @@ import { Router as KafkaRouter } from '@backstage/plugin-kafka'; ```yaml kafka: clientId: backstage - brokers: - - localhost:9092 + clusters: + - name: cluster-name + brokers: + - localhost:9092 ``` 6. Add `kafka.apache.org/consumer-groups` annotation to your services: @@ -77,7 +79,7 @@ kind: Component metadata: # ... annotations: - kafka.apache.org/consumer-groups: consumer-group-name + kafka.apache.org/consumer-groups: cluster-name/consumer-group-name spec: type: service ``` diff --git a/plugins/kafka/src/api/KafkaBackendClient.ts b/plugins/kafka/src/api/KafkaBackendClient.ts index 4b41323098..8166da4601 100644 --- a/plugins/kafka/src/api/KafkaBackendClient.ts +++ b/plugins/kafka/src/api/KafkaBackendClient.ts @@ -43,8 +43,11 @@ export class KafkaBackendClient implements KafkaApi { } async getConsumerGroupOffsets( + clusterId: string, consumerGroup: string, ): Promise { - return await this.internalGet(`/consumer/${consumerGroup}/offsets`); + return await this.internalGet( + `/consumers/${clusterId}/${consumerGroup}/offsets`, + ); } } diff --git a/plugins/kafka/src/api/types.ts b/plugins/kafka/src/api/types.ts index 9c2239156b..c4307d5c1f 100644 --- a/plugins/kafka/src/api/types.ts +++ b/plugins/kafka/src/api/types.ts @@ -34,6 +34,7 @@ export type ConsumerGroupOffsetsResponse = { export interface KafkaApi { getConsumerGroupOffsets( + clusterId: string, consumerGroup: string, ): Promise; } diff --git a/plugins/kafka/src/components/ConsumerGroupOffsets/ConsumerGroupOffsets.test.tsx b/plugins/kafka/src/components/ConsumerGroupOffsets/ConsumerGroupOffsets.test.tsx index ea0f7e1ddb..9cedc5e135 100644 --- a/plugins/kafka/src/components/ConsumerGroupOffsets/ConsumerGroupOffsets.test.tsx +++ b/plugins/kafka/src/components/ConsumerGroupOffsets/ConsumerGroupOffsets.test.tsx @@ -27,6 +27,7 @@ describe('ConsumerGroupOffsets', () => { const rendered = await renderInTestApp( {}} diff --git a/plugins/kafka/src/components/ConsumerGroupOffsets/ConsumerGroupOffsets.tsx b/plugins/kafka/src/components/ConsumerGroupOffsets/ConsumerGroupOffsets.tsx index 448d420bc0..7928393526 100644 --- a/plugins/kafka/src/components/ConsumerGroupOffsets/ConsumerGroupOffsets.tsx +++ b/plugins/kafka/src/components/ConsumerGroupOffsets/ConsumerGroupOffsets.tsx @@ -15,7 +15,7 @@ */ import { Table, TableColumn } from '@backstage/core'; -import { Box, Typography } from '@material-ui/core'; +import { Box, Grid, Typography } from '@material-ui/core'; import RetryIcon from '@material-ui/icons/Replay'; import React from 'react'; import { useConsumerGroupsOffsetsForEntity } from './useConsumerGroupsOffsetsForEntity'; @@ -62,6 +62,7 @@ const generatedColumns: TableColumn[] = [ type Props = { loading: boolean; retry: () => void; + clusterId: string; consumerGroup: string; topics?: TopicPartitionInfo[]; }; @@ -69,6 +70,7 @@ type Props = { export const ConsumerGroupOffsets = ({ loading, topics, + clusterId, consumerGroup, retry, }: Props) => { @@ -87,7 +89,7 @@ export const ConsumerGroupOffsets = ({ title={ - Consumed Topics for {consumerGroup} + Consumed Topics for {consumerGroup} ({clusterId}) } @@ -98,6 +100,15 @@ export const ConsumerGroupOffsets = ({ export const KafkaTopicsForConsumer = () => { const [tableProps, { retry }] = useConsumerGroupsOffsetsForEntity(); - - return ; + return ( + + {tableProps.consumerGroupsTopics?.map(consumerGroup => ( + + ))} + + ); }; diff --git a/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsForEntity.test.tsx b/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsForEntity.test.tsx index 44693d5bbc..3bee4e2e88 100644 --- a/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsForEntity.test.tsx +++ b/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsForEntity.test.tsx @@ -20,21 +20,7 @@ import { EntityContext } from '@backstage/plugin-catalog'; import { Entity } from '@backstage/catalog-model'; describe('useConsumerGroupOffsets', () => { - const entity: Entity = { - apiVersion: 'v1', - kind: 'Component', - metadata: { - name: 'test', - annotations: { - 'kafka.apache.org/consumer-groups': 'consumer', - }, - }, - spec: { - owner: 'guest', - type: 'Website', - lifecycle: 'development', - }, - }; + let entity: Entity; const wrapper = ({ children }: PropsWithChildren<{}>) => { return ( @@ -46,8 +32,107 @@ describe('useConsumerGroupOffsets', () => { const subject = () => renderHook(useConsumerGroupsForEntity, { wrapper }); - it('returns correct consumer group for annotation', async () => { + it('returns correct cluster and consumer group for annotation', async () => { + entity = { + apiVersion: 'v1', + kind: 'Component', + metadata: { + name: 'test', + annotations: { + 'kafka.apache.org/consumer-groups': 'prod/consumer', + }, + }, + spec: { + owner: 'guest', + type: 'Website', + lifecycle: 'development', + }, + }; const { result } = subject(); - expect(result.current).toBe('consumer'); + expect(result.current).toStrictEqual([ + { + clusterId: 'prod', + consumerGroup: 'consumer', + }, + ]); + }); + + it('returns correct cluster and consumer group for multiple consumers', async () => { + entity = { + apiVersion: 'v1', + kind: 'Component', + metadata: { + name: 'test', + annotations: { + 'kafka.apache.org/consumer-groups': + 'prod/consumer,dev/another-consumer', + }, + }, + spec: { + owner: 'guest', + type: 'Website', + lifecycle: 'development', + }, + }; + const { result } = subject(); + expect(result.current).toStrictEqual([ + { clusterId: 'prod', consumerGroup: 'consumer' }, + { + clusterId: 'dev', + consumerGroup: 'another-consumer', + }, + ]); + }); + + it('returns correct cluster and consumer group for annotation with extra spaces', async () => { + entity = { + apiVersion: 'v1', + kind: 'Component', + metadata: { + name: 'test', + annotations: { + 'kafka.apache.org/consumer-groups': + ' prod/consumer , dev/another-consumer ', + }, + }, + spec: { + owner: 'guest', + type: 'Website', + lifecycle: 'development', + }, + }; + const { result } = subject(); + expect(result.current).toStrictEqual([ + { clusterId: 'prod', consumerGroup: 'consumer' }, + { + clusterId: 'dev', + consumerGroup: 'another-consumer', + }, + ]); + }); + + it('fails on missing cluster', async () => { + entity = { + apiVersion: 'v1', + kind: 'Component', + metadata: { + name: 'test', + annotations: { + 'kafka.apache.org/consumer-groups': 'dev/another,consumer', + }, + }, + spec: { + owner: 'guest', + type: 'Website', + lifecycle: 'development', + }, + }; + const { result } = subject(); + expect(() => result.current).toThrowError(); + expect(result.error).toStrictEqual( + new Error( + `Failed to parse kafka consumer group annotation: got "dev/another,consumer"`, + ), + ); }); }); diff --git a/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsForEntity.ts b/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsForEntity.ts index 5003c2f5b5..522a6273fe 100644 --- a/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsForEntity.ts +++ b/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsForEntity.ts @@ -15,10 +15,29 @@ */ import { useEntity } from '@backstage/plugin-catalog'; +import { useMemo } from 'react'; import { KAFKA_CONSUMER_GROUP_ANNOTATION } from '../../constants'; export const useConsumerGroupsForEntity = () => { const { entity } = useEntity(); + const annotation = + entity.metadata.annotations?.[KAFKA_CONSUMER_GROUP_ANNOTATION] ?? ''; - return entity.metadata.annotations?.[KAFKA_CONSUMER_GROUP_ANNOTATION] ?? ''; + const consumerList = useMemo(() => { + return annotation.split(',').map(consumer => { + const [clusterId, consumerGroup] = consumer.split('/'); + + if (!clusterId || !consumerGroup) { + throw new Error( + `Failed to parse kafka consumer group annotation: got "${annotation}"`, + ); + } + return { + clusterId: clusterId.trim(), + consumerGroup: consumerGroup.trim(), + }; + }); + }, [annotation]); + + return consumerList; }; diff --git a/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsOffsetsForEntity.test.tsx b/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsOffsetsForEntity.test.tsx index 28f0ff7c28..b7e3a29e80 100644 --- a/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsOffsetsForEntity.test.tsx +++ b/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsOffsetsForEntity.test.tsx @@ -45,7 +45,7 @@ describe('useConsumerGroupOffsets', () => { metadata: { name: 'test', annotations: { - 'kafka.apache.org/consumer-groups': consumerGroupOffsets.consumerId, + 'kafka.apache.org/consumer-groups': `prod/${consumerGroupOffsets.consumerId}`, }, }, spec: { @@ -74,16 +74,24 @@ describe('useConsumerGroupOffsets', () => { renderHook(useConsumerGroupsOffsetsForEntity, { wrapper }); it('returns correct consumer group for annotation', async () => { + mockKafkaApi.getConsumerGroupOffsets.mockResolvedValue( + consumerGroupOffsets, + ); when(mockKafkaApi.getConsumerGroupOffsets) - .calledWith(consumerGroupOffsets.consumerId) + .calledWith('prod', consumerGroupOffsets.consumerId) .mockResolvedValue(consumerGroupOffsets); const { result, waitForNextUpdate } = subject(); await waitForNextUpdate(); const [tableProps] = result.current; - expect(tableProps.consumerGroup).toBe(consumerGroupOffsets.consumerId); - expect(tableProps.topics).toBe(consumerGroupOffsets.offsets); + expect(tableProps.consumerGroupsTopics).toStrictEqual([ + { + clusterId: 'prod', + consumerGroup: consumerGroupOffsets.consumerId, + topics: consumerGroupOffsets.offsets, + }, + ]); }); it('posts an error to the error api', async () => { diff --git a/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsOffsetsForEntity.ts b/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsOffsetsForEntity.ts index eb7e105989..8cb30040e9 100644 --- a/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsOffsetsForEntity.ts +++ b/plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsOffsetsForEntity.ts @@ -20,25 +20,35 @@ import { kafkaApiRef } from '../../api/types'; import { useConsumerGroupsForEntity } from './useConsumerGroupsForEntity'; export const useConsumerGroupsOffsetsForEntity = () => { - const consumerGroup = useConsumerGroupsForEntity(); + const consumers = useConsumerGroupsForEntity(); const api = useApi(kafkaApiRef); const errorApi = useApi(errorApiRef); - const { loading, value: topics, retry } = useAsyncRetry(async () => { + const { + loading, + value: consumerGroupsTopics, + retry, + } = useAsyncRetry(async () => { try { - const response = await api.getConsumerGroupOffsets(consumerGroup); - return response.offsets; + return await Promise.all( + consumers.map(async ({ clusterId, consumerGroup }) => { + const response = await api.getConsumerGroupOffsets( + clusterId, + consumerGroup, + ); + return { clusterId, consumerGroup, topics: response.offsets }; + }), + ); } catch (e) { errorApi.post(e); throw e; } - }, [api, errorApi, consumerGroup]); + }, [consumers, api, errorApi]); return [ { loading, - consumerGroup, - topics, + consumerGroupsTopics, }, { retry, diff --git a/plugins/kubernetes-backend/README.md b/plugins/kubernetes-backend/README.md index a9fdc14707..b724babca3 100644 --- a/plugins/kubernetes-backend/README.md +++ b/plugins/kubernetes-backend/README.md @@ -1,83 +1,19 @@ # Kubernetes Backend -WORK IN PROGRESS +This is the backend part of the Kubernetes plugin for Backstage. It is called by and responds to requests from the frontend [`@backstage/plugin-kubernetes`](https://github.com/backstage/backstage/tree/master/plugins/kubernetes) plugin. -This is the backend part of the Kubernetes plugin. +It directly interfaces with the Kubernetes API control plane to obtain information about objects that will then be presented at the front end. -It responds to Kubernetes requests from the frontend. +## Introduction -## Configuration +See our announcement blog post [New Backstage feature: Kubernetes for Service Owners](https://backstage.io/blog/2021/01/12/new-backstage-feature-kubernetes-for-service-owners) to learn more about the motivation behind developing the plugin. -### serviceLocatorMethod +## Setup & Configuration -This configures how to determine which clusters a component is running in. +This plugin must be explicitly added to a Backstage app, along with it's peer frontend plugin. -Currently, the only valid serviceLocatorMethod is: +The plugin requires configuration in the Backstage `app-config.yaml` to connect to a Kubernetes API control plane. -#### multiTenant +In addition, configuration of an entity's `catalog-info.yaml` helps identify which specific Kubernetes object(s) should be presented on a specific entity catalog page. -This configuration assumes that all components run on all the provided clusters. - -### clusterLocatorMethods - -This is used to determine where to retrieve cluster configuration from. - -Currently, the only valid serviceLocatorMethod is: - -#### config - -This clusterLocatorMethod will read cluster information in from config - -Example: - -```yaml -kubernetes: - serviceLocatorMethod: 'multiTenant' - clusterLocatorMethods: - - 'config' - clusters: - - url: http://127.0.0.1:9999 - name: minikube - serviceAccountToken: - authProvider: 'serviceAccount' - - url: http://127.0.0.2:9999 - name: gke-cluster-1 - authProvider: 'google' -``` - -##### clusters - -Used by the `config` `clusterLocatorMethods` to construct Kubernetes clients. - -###### url - -The base url to the Kubernetes control plane. Can be found by using the `Kubernetes master` result from running the `kubectl cluster-info` command. - -###### name - -A name to represent this cluster, this must be unique within the `clusters` array. Users will see this value in the Service Catalog Kubernetes plugin. - -###### authProvider - -This determines how the Kubernetes client authenticate with the Kubernetes cluster. Valid values are: - -| Value | Description | -| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `serviceAccount` | This will use a Kubernetes [service account](https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/) to access the Kubernetes API. When this is used the `serviceAccountToken` field should also be set. | -| `google` | This will use a user's google auth token from the [google auth plugin](https://backstage.io/docs/auth/) to access the Kubernetes API. | - -###### serviceAccount (optional) - -The service account token to be used when using the `authProvider`, `serviceAccount`. - -## RBAC - -The current RBAC permissions required are read-only cluster wide, for the following objects: - -- pods -- services -- configmaps -- deployments -- replicasets -- horizontalpodautoscalers -- ingresses +For more information, see the [formal documentation about the Kubernetes feature in Backstage](https://backstage.io/docs/features/kubernetes/overview). diff --git a/plugins/kubernetes-backend/package.json b/plugins/kubernetes-backend/package.json index ec0869d522..eeb91de777 100644 --- a/plugins/kubernetes-backend/package.json +++ b/plugins/kubernetes-backend/package.json @@ -31,11 +31,13 @@ "clean": "backstage-cli clean" }, "dependencies": { + "@aws-sdk/credential-provider-node": "^3.3.0", "@backstage/backend-common": "^0.5.0", "@backstage/catalog-model": "^0.7.0", "@backstage/config": "^0.1.2", "@kubernetes/client-node": "^0.13.2", "@types/express": "^4.17.6", + "aws4": "^1.11.0", "compression": "^1.7.4", "cors": "^2.8.5", "express": "^4.17.1", @@ -50,6 +52,7 @@ }, "devDependencies": { "@backstage/cli": "^0.4.7", + "@types/aws4": "^1.5.1", "supertest": "^4.0.2" }, "files": [ diff --git a/plugins/kubernetes-backend/schema.d.ts b/plugins/kubernetes-backend/schema.d.ts index 862cb6d2ce..31b6a2022b 100644 --- a/plugins/kubernetes-backend/schema.d.ts +++ b/plugins/kubernetes-backend/schema.d.ts @@ -20,8 +20,8 @@ export interface Config { clusters: { url: string; name: string; - serviceAccountToken: string; - authProvider: 'serviceAccount'; + serviceAccountToken: string | undefined; + authProvider: 'aws' | 'google' | 'serviceAccount'; }[]; }; } diff --git a/plugins/kubernetes-backend/src/kubernetes-auth-translator/AwsIamKubernetesAuthTranslator.test.ts b/plugins/kubernetes-backend/src/kubernetes-auth-translator/AwsIamKubernetesAuthTranslator.test.ts new file mode 100644 index 0000000000..4411658ee6 --- /dev/null +++ b/plugins/kubernetes-backend/src/kubernetes-auth-translator/AwsIamKubernetesAuthTranslator.test.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +const mockCredentialProvider = jest.fn(); +jest.mock('@aws-sdk/credential-provider-node', () => { + return { + defaultProvider: () => mockCredentialProvider, + }; +}); + +import { AwsIamKubernetesAuthTranslator } from './AwsIamKubernetesAuthTranslator'; + +describe('AwsIamKubernetesAuthTranslator tests', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + it('returns a signed url for aws credentials', async () => { + const authTranslator = new AwsIamKubernetesAuthTranslator(); + + mockCredentialProvider.mockImplementation(async () => { + // These credentials are not real. + // Pulled from example in docs: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html + return { + accessKeyId: 'AKIAIOSFODNN7EXAMPLE', + secretKeyId: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', + }; + }); + + const clusterDetails = await authTranslator.decorateClusterDetailsWithAuth({ + name: 'test-cluster', + url: '', + authProvider: 'aws', + }); + expect(clusterDetails.serviceAccountToken).toBeDefined(); + }); + + it('throws when unable to get aws credentials', async () => { + const authTranslator = new AwsIamKubernetesAuthTranslator(); + + mockCredentialProvider.mockImplementation(async () => { + throw new Error('not implemented'); + }); + + const promise = authTranslator.decorateClusterDetailsWithAuth({ + name: 'test-cluster', + url: '', + authProvider: 'aws', + }); + await expect(promise).rejects.toThrow('not implemented'); + }); +}); diff --git a/plugins/kubernetes-backend/src/kubernetes-auth-translator/AwsIamKubernetesAuthTranslator.ts b/plugins/kubernetes-backend/src/kubernetes-auth-translator/AwsIamKubernetesAuthTranslator.ts new file mode 100644 index 0000000000..3b5ea59f18 --- /dev/null +++ b/plugins/kubernetes-backend/src/kubernetes-auth-translator/AwsIamKubernetesAuthTranslator.ts @@ -0,0 +1,73 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { defaultProvider } from '@aws-sdk/credential-provider-node'; +import { sign } from 'aws4'; +import { KubernetesAuthTranslator } from './types'; +import { ClusterDetails } from '..'; + +const base64 = (str: string) => + Buffer.from(str.toString(), 'binary').toString('base64'); +const prepend = (prep: string) => (str: string) => prep + str; +const replace = (search: string | RegExp, substitution: string) => ( + str: string, +) => str.replace(search, substitution); +const pipe = (fns: ReadonlyArray) => (thing: string): string => + fns.reduce((val, fn) => fn(val), thing); +const removePadding = replace(/=+$/, ''); +const makeUrlSafe = pipe([replace('+', '-'), replace('/', '_')]); + +export class AwsIamKubernetesAuthTranslator + implements KubernetesAuthTranslator { + async getBearerToken(clusterName: string): Promise { + const credentialProvider = defaultProvider(); + const credentials = await credentialProvider(); + const request = { + host: `sts.amazonaws.com`, + path: `/?Action=GetCallerIdentity&Version=2011-06-15&X-Amz-Expires=60`, + headers: { + 'x-k8s-aws-id': clusterName, + }, + signQuery: true, + }; + const signedRequest = sign(request, { + accessKeyId: credentials.accessKeyId, + secretAccessKey: credentials.secretAccessKey, + sessionToken: credentials.sessionToken, + }); + + return pipe([ + (signed: any) => `https://${signed.host}${signed.path}`, + base64, + removePadding, + makeUrlSafe, + prepend('k8s-aws-v1.'), + ])(signedRequest); + } + + async decorateClusterDetailsWithAuth( + clusterDetails: ClusterDetails, + ): Promise { + const clusterDetailsWithAuthToken: ClusterDetails = Object.assign( + {}, + clusterDetails, + ); + + clusterDetailsWithAuthToken.serviceAccountToken = await this.getBearerToken( + clusterDetails.name, + ); + return clusterDetailsWithAuthToken; + } +} diff --git a/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.test.ts b/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.test.ts index 3b27eb012a..ec2f6ef120 100644 --- a/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.test.ts +++ b/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.test.ts @@ -18,6 +18,7 @@ import { KubernetesAuthTranslator } from './types'; import { GoogleKubernetesAuthTranslator } from './GoogleKubernetesAuthTranslator'; import { KubernetesAuthTranslatorGenerator } from './KubernetesAuthTranslatorGenerator'; import { ServiceAccountKubernetesAuthTranslator } from './ServiceAccountKubernetesAuthTranslator'; +import { AwsIamKubernetesAuthTranslator } from './AwsIamKubernetesAuthTranslator'; describe('getKubernetesAuthTranslatorInstance', () => { const sut = KubernetesAuthTranslatorGenerator; @@ -29,6 +30,13 @@ describe('getKubernetesAuthTranslatorInstance', () => { expect(authTranslator instanceof GoogleKubernetesAuthTranslator).toBe(true); }); + it('can return an auth translator for aws auth', () => { + const authTranslator: KubernetesAuthTranslator = sut.getKubernetesAuthTranslatorInstance( + 'aws', + ); + expect(authTranslator instanceof AwsIamKubernetesAuthTranslator).toBe(true); + }); + it('can return an auth translator for serviceAccount auth', () => { const authTranslator: KubernetesAuthTranslator = sut.getKubernetesAuthTranslatorInstance( 'serviceAccount', diff --git a/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts b/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts index f7cfc92112..4ae2aff35a 100644 --- a/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts +++ b/plugins/kubernetes-backend/src/kubernetes-auth-translator/KubernetesAuthTranslatorGenerator.ts @@ -17,6 +17,7 @@ import { KubernetesAuthTranslator } from './types'; import { GoogleKubernetesAuthTranslator } from './GoogleKubernetesAuthTranslator'; import { ServiceAccountKubernetesAuthTranslator } from './ServiceAccountKubernetesAuthTranslator'; +import { AwsIamKubernetesAuthTranslator } from './AwsIamKubernetesAuthTranslator'; export class KubernetesAuthTranslatorGenerator { static getKubernetesAuthTranslatorInstance( @@ -26,6 +27,9 @@ export class KubernetesAuthTranslatorGenerator { case 'google': { return new GoogleKubernetesAuthTranslator(); } + case 'aws': { + return new AwsIamKubernetesAuthTranslator(); + } case 'serviceAccount': { return new ServiceAccountKubernetesAuthTranslator(); } diff --git a/plugins/kubernetes-backend/src/types/types.ts b/plugins/kubernetes-backend/src/types/types.ts index cf6576153b..86191c6d72 100644 --- a/plugins/kubernetes-backend/src/types/types.ts +++ b/plugins/kubernetes-backend/src/types/types.ts @@ -148,4 +148,4 @@ export interface KubernetesFetchError { export type ServiceLocatorMethod = 'multiTenant' | 'http'; // TODO implement http export type ClusterLocatorMethod = 'config'; -export type AuthProviderType = 'google' | 'serviceAccount'; +export type AuthProviderType = 'google' | 'serviceAccount' | 'aws'; diff --git a/plugins/kubernetes/README.md b/plugins/kubernetes/README.md index a343764b70..15dfd16b17 100644 --- a/plugins/kubernetes/README.md +++ b/plugins/kubernetes/README.md @@ -1,9 +1,29 @@ -# kubernetes +# Kubernetes -Welcome to the kubernetes plugin! +Welcome to the Backstage Kubernetes frontend plugin! + +This plugin exposes information about your entity-specific Kubernetes objects with a desire to provide value to the service owner, rather than just a Kubernetes cluster administrator. + +It will elevate the visibility of errors where identified, and provide drill down about the deployments, pods, and other objects for a service. + +It directly interfaces with the [Kubernetes Backend Plugin (`@backstage-plugin-kubernetes-backend`)](https://github.com/backstage/backstage/tree/master/plugins/kubernetes-backend). _This plugin was created through the Backstage CLI_ +## Introduction + +See our announcement blog post [New Backstage feature: Kubernetes for Service Owners](https://backstage.io/blog/2021/01/12/new-backstage-feature-kubernetes-for-service-owners) to learn more about the motivation behind developing the plugin. + +## Setup & Configuration + +This plugin must be explicitly added to a Backstage app, along with it's peer backend plugin. + +It requires configuration in the Backstage `app-config.yaml` to connect to a Kubernetes API control plane. + +In addition, configuration of an entity's `catalog-info.yaml` helps identify which specific Kubernetes object(s) should be presented on a specific entity catalog page. + +For more information, see the [formal documentation about the Kubernetes feature in Backstage](https://backstage.io/docs/features/kubernetes/overview). + ## Getting started Your plugin has been added to the example app in this repository, meaning you'll be able to access it by running `yarn start` in the root directory, and then navigating to [/kubernetes](http://localhost:3000/kubernetes). @@ -11,40 +31,3 @@ Your plugin has been added to the example app in this repository, meaning you'll You can also serve the plugin in isolation by running `yarn start` in the plugin directory. This method of serving the plugin provides quicker iteration speed and a faster startup and hot reloads. It is only meant for local development, and the setup for it can be found inside the [/dev](./dev) directory. - -## Surfacing your Kubernetes components as part of an entity - -There are two ways to surface your kubernetes components as part of an entity. -The label selector takes precedence over the annotation/service id. - -### Common `backstage.io/kubernetes-id` label - -#### Adding the entity annotation - -In order for Backstage to detect that an entity has Kubernetes components, -the following annotation should be added to the entity. - -```yaml -annotations: - 'backstage.io/kubernetes-id': dice-roller -``` - -#### Labeling Kubernetes components - -In order for Kubernetes components to show up in the service catalog -as a part of an entity, Kubernetes components must be labeled with the following label: - -```yaml -'backstage.io/kubernetes-id': -``` - -### label selector query annotation - -#### Adding a label selector query annotation - -You can write your own custom label selector query that backstage will use to lookup the objects (similar to `kubectl --selector="your query here"`) -review the documentation [here](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) for more info - -```yaml -'backstage.io/kubernetes-label-selector': 'app=my-app,component=front-end' -``` diff --git a/plugins/kubernetes/schema.d.ts b/plugins/kubernetes/schema.d.ts index f3f3c98f8c..39598edb39 100644 --- a/plugins/kubernetes/schema.d.ts +++ b/plugins/kubernetes/schema.d.ts @@ -35,11 +35,11 @@ export interface Config { /** * @visibility secret */ - serviceAccountToken: string; + serviceAccountToken: string | undefined; /** * @visibility frontend */ - authProvider: 'serviceAccount'; + authProvider: 'aws' | 'google' | 'serviceAccount'; }[]; }; } diff --git a/plugins/sonarqube/src/api/SonarQubeClient.test.ts b/plugins/sonarqube/src/api/SonarQubeClient.test.ts index 1b3b5c5709..e5cbc2ccdf 100644 --- a/plugins/sonarqube/src/api/SonarQubeClient.test.ts +++ b/plugins/sonarqube/src/api/SonarQubeClient.test.ts @@ -29,7 +29,31 @@ describe('SonarQubeClient', () => { const mockBaseUrl = 'http://backstage:9191/api/proxy'; const discoveryApi = UrlPatternDiscovery.compile(mockBaseUrl); - const setupHandlers = () => { + const setupHandlers = ( + metricKeys = [ + 'alert_status', + 'bugs', + 'reliability_rating', + 'vulnerabilities', + 'security_rating', + 'security_hotspots_reviewed', + 'security_review_rating', + 'code_smells', + 'sqale_rating', + 'coverage', + 'duplicated_lines_density', + ], + ) => { + server.use( + rest.get(`${mockBaseUrl}/sonarqube/metrics/search`, (_, res, ctx) => { + return res( + ctx.json({ + metrics: metricKeys.map(k => ({ key: k })), + }), + ); + }), + ); + server.use( rest.get(`${mockBaseUrl}/sonarqube/components/show`, (req, res, ctx) => { expect(req.url.searchParams.toString()).toBe('component=our-service'); @@ -46,8 +70,9 @@ describe('SonarQubeClient', () => { server.use( rest.get(`${mockBaseUrl}/sonarqube/measures/search`, (req, res, ctx) => { expect(req.url.searchParams.toString()).toBe( - 'projectKeys=our-service&metricKeys=alert_status%2Cbugs%2Creliability_rating%2Cvulnerabilities%2Csecurity_rating%2Csecurity_hotspots_reviewed%2Csecurity_review_rating%2Ccode_smells%2Csqale_rating%2Ccoverage%2Cduplicated_lines_density', + `projectKeys=our-service&metricKeys=${metricKeys.join('%2C')}`, ); + return res( ctx.json({ measures: [ @@ -111,7 +136,7 @@ describe('SonarQubeClient', () => { value: '1.0', component: 'our-service', }, - ], + ].filter(m => metricKeys.includes(m.metric)), } as MeasuresWrapper), ); }), @@ -187,4 +212,32 @@ describe('SonarQubeClient', () => { 'http://a.instance.local/component_measures?id=our-service&metric=coverage&resolved=false&view=list', ); }); + + it('should only request selected metrics', async () => { + setupHandlers(['alert_status', 'bugs']); + + const client = new SonarQubeClient({ + discoveryApi, + baseUrl: 'http://a.instance.local', + }); + + const summary = await client.getFindingSummary('our-service'); + + expect(summary).toEqual( + expect.objectContaining({ + lastAnalysis: '2020-01-01T00:00:00Z', + metrics: { + alert_status: 'OK', + bugs: '2', + }, + projectUrl: 'http://a.instance.local/dashboard?id=our-service', + }) as FindingSummary, + ); + expect(summary?.getIssuesUrl('CODE_SMELL')).toEqual( + 'http://a.instance.local/project/issues?id=our-service&types=CODE_SMELL&resolved=false', + ); + expect(summary?.getComponentMeasuresUrl('COVERAGE')).toEqual( + 'http://a.instance.local/component_measures?id=our-service&metric=coverage&resolved=false&view=list', + ); + }); }); diff --git a/plugins/sonarqube/src/api/SonarQubeClient.ts b/plugins/sonarqube/src/api/SonarQubeClient.ts index 061919dd41..35035b33c5 100644 --- a/plugins/sonarqube/src/api/SonarQubeClient.ts +++ b/plugins/sonarqube/src/api/SonarQubeClient.ts @@ -43,6 +43,13 @@ export class SonarQubeClient implements SonarQubeApi { return undefined; } + private async getSupportedMetrics(): Promise { + const result = await this.callApi<{ metrics: Array<{ key: string }> }>( + 'metrics/search', + ); + return result?.metrics?.map(m => m.key) ?? []; + } + async getFindingSummary( componentKey?: string, ): Promise { @@ -71,10 +78,16 @@ export class SonarQubeClient implements SonarQubeApi { duplicated_lines_density: undefined, }; + // select the metrics that are supported by the SonarQube instance + const supportedMetrics = await this.getSupportedMetrics(); + const metricKeys = Object.keys(metrics).filter(m => + supportedMetrics.includes(m), + ); + const measures = await this.callApi( - `measures/search?projectKeys=${componentKey}&metricKeys=${Object.keys( - metrics, - ).join(',')}`, + `measures/search?projectKeys=${componentKey}&metricKeys=${metricKeys.join( + ',', + )}`, ); if (!measures) { return undefined; diff --git a/yarn.lock b/yarn.lock index 28602e734f..7c743bc5b3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -278,6 +278,15 @@ "@aws-sdk/property-provider" "3.1.0" tslib "^1.8.0" +"@aws-sdk/credential-provider-env@3.3.0": + version "3.3.0" + resolved "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.3.0.tgz#7930e504a7a79ab98a9fd34befc5c84b8c4df679" + integrity sha512-kyqZMlGdH/05IhuXLBUXtj5+hhRfYiHFcJLc3ts/uiwCixswVHPAYHgyWm9ajFkmWtpz6ih+0LoYryhPbYu01A== + dependencies: + "@aws-sdk/property-provider" "3.3.0" + "@aws-sdk/types" "3.1.0" + tslib "^1.8.0" + "@aws-sdk/credential-provider-imds@3.1.0": version "3.1.0" resolved "https://registry.npmjs.org/@aws-sdk/credential-provider-imds/-/credential-provider-imds-3.1.0.tgz#33d48753dc00bddce79d2aa8076a7cb5bf8562df" @@ -286,6 +295,15 @@ "@aws-sdk/property-provider" "3.1.0" tslib "^1.8.0" +"@aws-sdk/credential-provider-imds@3.3.0": + version "3.3.0" + resolved "https://registry.npmjs.org/@aws-sdk/credential-provider-imds/-/credential-provider-imds-3.3.0.tgz#ff0cf5489853c16d23fc99d7bae425587e836c40" + integrity sha512-Cx0YMnO/ScGQVDns006bLbqOxNURGN2Xm21bCY0l0ZUJCdJ2va1/9q1rljDyw2KvdzZNQVRQII3uUgj/Oq/K+g== + dependencies: + "@aws-sdk/property-provider" "3.3.0" + "@aws-sdk/types" "3.1.0" + tslib "^1.8.0" + "@aws-sdk/credential-provider-ini@3.1.0": version "3.1.0" resolved "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.1.0.tgz#1a6cf9ab9fa1450d4472b9e371099b0c0283349b" @@ -295,6 +313,16 @@ "@aws-sdk/shared-ini-file-loader" "3.1.0" tslib "^1.8.0" +"@aws-sdk/credential-provider-ini@3.3.0": + version "3.3.0" + resolved "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.3.0.tgz#55fe8f391b72d30e650ba8bc680e82bbeacbbfe5" + integrity sha512-zawNFJoasXiaV5n0H3/KNOi7mAZ7mHpG1+nBEkoWhZ31lIUM9+heGPcxKCbf/pMQjiOebUqL1OpWe4uSWxIVMw== + dependencies: + "@aws-sdk/property-provider" "3.3.0" + "@aws-sdk/shared-ini-file-loader" "3.1.0" + "@aws-sdk/types" "3.1.0" + tslib "^1.8.0" + "@aws-sdk/credential-provider-node@3.1.0": version "3.1.0" resolved "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.1.0.tgz#89bc8803752a3e580c6f2410306c7edad6be7fa2" @@ -307,6 +335,19 @@ "@aws-sdk/property-provider" "3.1.0" tslib "^1.8.0" +"@aws-sdk/credential-provider-node@^3.3.0": + version "3.3.0" + resolved "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.3.0.tgz#5c97323fa7b23590070d06aa7b1be8d93b2bf4be" + integrity sha512-PPBNzPq8fHk9dEQTTE4iJi6ZWtmo057Lc+I8Rlzmvz6NthK9iKiU819tfaxVBb6ZR7bLP0BuDiCi4G1lD+rQnQ== + dependencies: + "@aws-sdk/credential-provider-env" "3.3.0" + "@aws-sdk/credential-provider-imds" "3.3.0" + "@aws-sdk/credential-provider-ini" "3.3.0" + "@aws-sdk/credential-provider-process" "3.3.0" + "@aws-sdk/property-provider" "3.3.0" + "@aws-sdk/types" "3.1.0" + tslib "^1.8.0" + "@aws-sdk/credential-provider-process@3.1.0": version "3.1.0" resolved "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.1.0.tgz#ff817b29a9760c463b77be3ce49375eaeb753ef3" @@ -317,6 +358,17 @@ "@aws-sdk/shared-ini-file-loader" "3.1.0" tslib "^1.8.0" +"@aws-sdk/credential-provider-process@3.3.0": + version "3.3.0" + resolved "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.3.0.tgz#9de0984bd6dd0f5e40cff3672d7dd19e8cd43074" + integrity sha512-7oOF1j6ydUq43P3SsasiIpbxMKCmT0C+XwggHTGiVxNtX+QZiH1vdMf8otA7puLEey0iY5wTAIEcZhC6HenojA== + dependencies: + "@aws-sdk/credential-provider-ini" "3.3.0" + "@aws-sdk/property-provider" "3.3.0" + "@aws-sdk/shared-ini-file-loader" "3.1.0" + "@aws-sdk/types" "3.1.0" + tslib "^1.8.0" + "@aws-sdk/eventstream-marshaller@3.1.0": version "3.1.0" resolved "https://registry.npmjs.org/@aws-sdk/eventstream-marshaller/-/eventstream-marshaller-3.1.0.tgz#65a217e37abcaa162276ccb1d4487d42431d1534" @@ -649,6 +701,14 @@ dependencies: tslib "^1.8.0" +"@aws-sdk/property-provider@3.3.0": + version "3.3.0" + resolved "https://registry.npmjs.org/@aws-sdk/property-provider/-/property-provider-3.3.0.tgz#49979cb1a3e5562d51807c7403c5fd48cb9f2cdc" + integrity sha512-JTyjtXVNhFczL9IfgwXD55F6DqXL50PhfZxFW92t5dDj5VtWpOL74BbuxHQxHBgnQv1FKLr6N9cr7gfXWexDug== + dependencies: + "@aws-sdk/types" "3.1.0" + tslib "^1.8.0" + "@aws-sdk/protocol-http@3.1.0": version "3.1.0" resolved "https://registry.npmjs.org/@aws-sdk/protocol-http/-/protocol-http-3.1.0.tgz#7da0ebcf02a40a8300f3bd52f9206f25fdf1ca7f" @@ -4774,9 +4834,9 @@ once "^1.4.0" "@octokit/request@^5.2.0", "@octokit/request@^5.3.0", "@octokit/request@^5.4.11", "@octokit/request@^5.4.12": - version "5.4.12" - resolved "https://registry.npmjs.org/@octokit/request/-/request-5.4.12.tgz#b04826fa934670c56b135a81447be2c1723a2ffc" - integrity sha512-MvWYdxengUWTGFpfpefBBpVmmEYfkwMoxonIB3sUGp5rhdgwjXL1ejo6JbgzG/QD9B/NYt/9cJX1pxXeSIUCkg== + version "5.4.13" + resolved "https://registry.npmjs.org/@octokit/request/-/request-5.4.13.tgz#eec5987b3e96f984fc5f41967e001170c6d23a18" + integrity sha512-WcNRH5XPPtg7i1g9Da5U9dvZ6YbTffw9BN2rVezYiE7couoSyaRsw0e+Tl8uk1fArHE7Dn14U7YqUDy59WaqEw== dependencies: "@octokit/endpoint" "^6.0.1" "@octokit/request-error" "^2.0.0" @@ -6183,6 +6243,11 @@ resolved "https://registry.npmjs.org/@types/aria-query/-/aria-query-4.2.0.tgz#14264692a9d6e2fa4db3df5e56e94b5e25647ac0" integrity sha512-iIgQNzCm0v7QMhhe4Jjn9uRh+I6GoPmt03CbEtwx3ao8/EfoQcmgtqH4vQ5Db/lxiIGaWDv6nwvunuh0RyX0+A== +"@types/aws4@^1.5.1": + version "1.5.1" + resolved "https://registry.npmjs.org/@types/aws4/-/aws4-1.5.1.tgz#361fadab198a030ab398269183ae3fa86e958ed9" + integrity sha1-Nh+tqxmKAwqzmCaRg64/qG6Vjtk= + "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.7": version "7.1.9" resolved "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.9.tgz#77e59d438522a6fb898fa43dc3455c6e72f3963d" @@ -6828,10 +6893,10 @@ resolved "https://registry.npmjs.org/@types/mime/-/mime-2.0.1.tgz#dc488842312a7f075149312905b5e3c0b054c79d" integrity sha512-FwI9gX75FgVBJ7ywgnq/P7tw+/o1GUbtP0KzbtusLigAOgIgNISRK0ZPl4qertvXSIE8YbsVJueQ90cDt9YYyw== -"@types/mini-css-extract-plugin@^0.9.1": - version "0.9.1" - resolved "https://registry.npmjs.org/@types/mini-css-extract-plugin/-/mini-css-extract-plugin-0.9.1.tgz#d4bdde5197326fca039d418f4bdda03dc74dc451" - integrity sha512-+mN04Oszdz9tGjUP/c1ReVwJXxSniLd7lF++sv+8dkABxVNthg6uccei+4ssKxRHGoMmPxdn7uBdJWONSJGTGQ== +"@types/mini-css-extract-plugin@^1.2.2": + version "1.2.2" + resolved "https://registry.npmjs.org/@types/mini-css-extract-plugin/-/mini-css-extract-plugin-1.2.2.tgz#e6031da8d60777b3da3f5b4daf285437d7b6580b" + integrity sha512-EoHBJ4rcrd5j7weAFE4yU1gxedx53EFCWKso03G7DW0h2YvtwjKYz/NnuFHudcQDI1HpTLqoQFTwEgfJxygYCw== dependencies: "@types/webpack" "*" @@ -8602,6 +8667,11 @@ aws-sign2@~0.7.0: resolved "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg= +aws4@^1.11.0: + version "1.11.0" + resolved "https://registry.npmjs.org/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" + integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== + aws4@^1.8.0: version "1.9.1" resolved "https://registry.npmjs.org/aws4/-/aws4-1.9.1.tgz#7e33d8f7d449b3f673cd72deb9abdc552dbe528e"