feat: add dashboard URL feature and fix minor styling issues

Signed-off-by: David Weber <david.weber@w3tec.ch>

.changeset/odd-adults-smash.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-kafka': patch
+---
+
+Add dashboard URL feature and fix minor styling issues.

app-config.yaml

@@ -161,6 +161,7 @@ kafka:
   clientId: backstage
   clusters:
     - name: cluster
+      dashboardUrl: https://akhq.io/
       brokers:
         - localhost:9092
 

plugins/kafka/README.md

@@ -73,6 +73,8 @@ kafka:
 
 5. Add the `kafka.apache.org/consumer-groups` annotation to your services:
 
+Can be a comma separated list.
+
 ```yaml
 apiVersion: backstage.io/v1alpha1
 kind: Component
@@ -84,6 +86,34 @@ spec:
   type: service
 ```
 
+6. Configure dashboard urls:
+
+You have two options.  
+Either configure it with an annotation called `kafka.apache.org/dashboard-urls`
+
+```yaml
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+  # ...
+  annotations:
+    kafka.apache.org/dashboard-urls: cluster-name/consumer-group-name/dashboard-url
+spec:
+  type: service
+```
+
+> The consumer-group-name is optional.
+
+or with configs in `app-config.yaml`
+
+```yaml
+kafka:
+  # ...
+  clusters:
+    - name: cluster-name
+      dashboardUrl: https://dashboard.com
+```
+
 ## Features
 
 - List topics offsets and consumer group offsets for configured services.

plugins/kafka/config.d.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export interface Config {
+  kafka?: {
+    clusters: Array<{
+      /**
+       * Cluster name
+       * @visibility frontend
+       */
+      name: string;
+      /**
+       * Cluster dashboard url
+       * @visibility frontend
+       */
+      dashboardUrl?: string;
+    }>;
+  };
+}

plugins/kafka/package.json

@@ -13,6 +13,7 @@
   "backstage": {
     "role": "frontend-plugin"
   },
+  "configSchema": "config.d.ts",
   "scripts": {
     "build": "backstage-cli package build",
     "start": "backstage-cli package start",
@@ -29,6 +30,7 @@
     "@backstage/core-plugin-api": "^1.0.4",
     "@backstage/plugin-catalog-react": "^1.1.2",
     "@backstage/theme": "^0.2.16",
+    "@backstage/config": "^1.0.1",
     "@material-ui/core": "^4.12.2",
     "@material-ui/icons": "^4.9.1",
     "@material-ui/lab": "4.0.0-alpha.57",
@@ -54,6 +56,7 @@
     "msw": "^0.44.0"
   },
   "files": [
-    "dist"
+    "dist",
+    "config.d.ts"
   ]
 }

plugins/kafka/src/api/KafkaDashboardClient.test.ts

@@ -0,0 +1,138 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Entity } from '@backstage/catalog-model';
+import { KafkaDashboardClient } from './KafkaDashboardClient';
+import { ConfigApi, configApiRef } from '@backstage/core-plugin-api';
+import { KAFKA_DASHBOARD_URL } from '../constants';
+
+const mockConfigApi: jest.Mocked<Partial<typeof configApiRef.T>> = {
+  getConfigArray: jest.fn(_ => []),
+};
+
+describe('KafkaDashboardClient', () => {
+  let kafkaDashboardClient: KafkaDashboardClient;
+
+  beforeEach(() => {
+    kafkaDashboardClient = new KafkaDashboardClient({
+      configApi: mockConfigApi as ConfigApi,
+    });
+  });
+
+  it('Should return undefined on empty annotation', async () => {
+    const mockEntity = {
+      metadata: { annotations: { [KAFKA_DASHBOARD_URL]: '' } },
+    } as unknown as Entity;
+
+    expect(
+      kafkaDashboardClient.getDashboardUrl('', '', mockEntity).url,
+    ).toBeUndefined();
+  });
+
+  it('Should return consumer group and cluster based dashboard url', async () => {
+    const mockEntity = {
+      metadata: {
+        annotations: {
+          [KAFKA_DASHBOARD_URL]: 'cluster1/consumerGroup1/https://example.com',
+        },
+      },
+    } as unknown as Entity;
+
+    expect(
+      kafkaDashboardClient.getDashboardUrl(
+        'cluster1',
+        'consumerGroup1',
+        mockEntity,
+      ).url,
+    ).toEqual('https://example.com');
+  });
+
+  it('Should return cluster based dashboard url', async () => {
+    const mockEntity = {
+      metadata: {
+        annotations: { [KAFKA_DASHBOARD_URL]: 'cluster1/https://example.com' },
+      },
+    } as unknown as Entity;
+
+    expect(
+      kafkaDashboardClient.getDashboardUrl(
+        'cluster1',
+        'consumerGroup1',
+        mockEntity,
+      ).url,
+    ).toEqual('https://example.com');
+  });
+
+  it('Should return one dashboard url for list of dashboards', async () => {
+    const mockEntity = {
+      metadata: {
+        annotations: {
+          [KAFKA_DASHBOARD_URL]:
+            'cluster1/https://example.com,cluster2/https://example2.com',
+        },
+      },
+    } as unknown as Entity;
+
+    expect(
+      kafkaDashboardClient.getDashboardUrl('cluster2', '', mockEntity).url,
+    ).toEqual('https://example2.com');
+  });
+
+  it('Should return most specific dashboard url for list of dashboards', async () => {
+    const mockEntity = {
+      metadata: {
+        annotations: {
+          [KAFKA_DASHBOARD_URL]:
+            'cluster1/https://example.com,cluster1/consumerGroup1/https://example2.com',
+        },
+      },
+    } as unknown as Entity;
+
+    expect(
+      kafkaDashboardClient.getDashboardUrl(
+        'cluster1',
+        'consumerGroup1',
+        mockEntity,
+      ).url,
+    ).toEqual('https://example2.com');
+  });
+
+  it('Should return dashboard url with query parameters', async () => {
+    const mockEntity = {
+      metadata: {
+        annotations: {
+          [KAFKA_DASHBOARD_URL]:
+            'cluster1/https://example.com?consumer-group=consumergroup1',
+        },
+      },
+    } as unknown as Entity;
+
+    expect(
+      kafkaDashboardClient.getDashboardUrl('cluster1', '', mockEntity).url,
+    ).toEqual('https://example.com?consumer-group=consumergroup1');
+  });
+
+  it('Should return should fallback to config', async () => {
+    const mockEntity = {
+      metadata: {
+        annotations: { [KAFKA_DASHBOARD_URL]: 'cluster1/https://example.com' },
+      },
+    } as unknown as Entity;
+
+    kafkaDashboardClient.getDashboardUrl('cluster2', '', mockEntity);
+    expect(mockConfigApi.getConfigArray).toBeCalled();
+  });
+});

plugins/kafka/src/api/KafkaDashboardClient.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { KafkaDashboardApi } from './types';
+import { Entity } from '@backstage/catalog-model';
+import { ConfigApi } from '@backstage/core-plugin-api';
+import { KAFKA_DASHBOARD_URL } from '../constants';
+
+export class KafkaDashboardClient implements KafkaDashboardApi {
+  private readonly configApi: ConfigApi;
+  private readonly regexPattern =
+    /^([a-z0-9._-]+)\/([a-z0-9._-]+)?\/?(https?.*)$/i;
+
+  constructor(options: { configApi: ConfigApi }) {
+    this.configApi = options.configApi;
+  }
+
+  getDashboardUrl(
+    clusterId: string,
+    consumerGroup: string,
+    entity: Entity,
+  ): { url?: string } {
+    const annotation = entity.metadata.annotations?.[KAFKA_DASHBOARD_URL] ?? '';
+
+    const dashboardList = annotation
+      .split(',')
+      .filter(value => value !== undefined && value !== '')
+      .map(value => value.match(this.regexPattern) as string[])
+      .filter(
+        value =>
+          value[1] === clusterId &&
+          (value[2] === undefined || value[2] === consumerGroup),
+      )
+      .sort((a, b) => {
+        if (a[2] === b[2]) return 0;
+        if (a[2] !== undefined) return -1;
+        return 1;
+      });
+
+    if (dashboardList.length > 0) {
+      return { url: dashboardList[0][3] };
+    }
+
+    return {
+      url:
+        this.configApi
+          .getConfigArray('kafka.clusters')
+          .filter(value => value.getString('name') === clusterId)
+          .map(value => value.getOptionalString('dashboardUrl'))[0] ||
+        undefined,
+    };
+  }
+}

plugins/kafka/src/api/types.ts

@@ -15,11 +15,16 @@
  */
 
 import { createApiRef } from '@backstage/core-plugin-api';
+import { Entity } from '@backstage/catalog-model';
 
 export const kafkaApiRef = createApiRef<KafkaApi>({
   id: 'plugin.kafka.service',
 });
 
+export const kafkaDashboardApiRef = createApiRef<KafkaDashboardApi>({
+  id: 'plugin.kafka.dashboard',
+});
+
 export type ConsumerGroupOffsetsResponse = {
   consumerId: string;
   offsets: {
@@ -36,3 +41,11 @@ export interface KafkaApi {
     consumerGroup: string,
   ): Promise<ConsumerGroupOffsetsResponse>;
 }
+
+export interface KafkaDashboardApi {
+  getDashboardUrl(
+    clusterId: string,
+    consumerGroup: string,
+    entity: Entity,
+  ): { url?: string };
+}

plugins/kafka/src/components/ConsumerGroupOffsets/ConsumerGroupOffsets.tsx

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { Box, Grid, Typography } from '@material-ui/core';
+import { Box, Grid, Typography, Link } from '@material-ui/core';
 import RetryIcon from '@material-ui/icons/Replay';
 import React from 'react';
 import { useConsumerGroupsOffsetsForEntity } from './useConsumerGroupsOffsetsForEntity';
@@ -74,6 +74,7 @@ type Props = {
   loading: boolean;
   retry: () => void;
   clusterId: string;
+  dashboardUrl?: string;
   consumerGroup: string;
   topics?: TopicPartitionInfo[];
 };
@@ -82,6 +83,7 @@ export const ConsumerGroupOffsets = ({
   loading,
   topics,
   clusterId,
+  dashboardUrl,
   consumerGroup,
   retry,
 }: Props) => {
@@ -100,7 +102,14 @@ export const ConsumerGroupOffsets = ({
       title={
         <Box display="flex" alignItems="center">
           <Typography variant="h6">
-            Consumed Topics for {consumerGroup} ({clusterId})
+            Consumed Topics for {consumerGroup} (
+            {(dashboardUrl && (
+              <Link href={dashboardUrl} target="_blank">
+                {clusterId}
+              </Link>
+            )) ||
+              clusterId}
+            )
           </Typography>
         </Box>
       }
@@ -112,13 +121,15 @@ export const ConsumerGroupOffsets = ({
 export const KafkaTopicsForConsumer = () => {
   const [tableProps, { retry }] = useConsumerGroupsOffsetsForEntity();
   return (
-    <Grid>
+    <Grid container spacing={3}>
       {tableProps.consumerGroupsTopics?.map(consumerGroup => (
-        <ConsumerGroupOffsets
-          {...consumerGroup}
-          loading={tableProps.loading}
-          retry={retry}
-        />
+        <Grid item xs={12} key={consumerGroup.clusterId}>
+          <ConsumerGroupOffsets
+            {...consumerGroup}
+            loading={tableProps.loading}
+            retry={retry}
+          />
+        </Grid>
       ))}
     </Grid>
   );

plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsForEntity.test.tsx

@@ -18,12 +18,22 @@ import { EntityProvider } from '@backstage/plugin-catalog-react';
 import { renderHook } from '@testing-library/react-hooks';
 import React, { PropsWithChildren } from 'react';
 import { useConsumerGroupsForEntity } from './useConsumerGroupsForEntity';
+import { TestApiProvider } from '@backstage/test-utils';
+import { configApiRef } from '@backstage/core-plugin-api';
+
+const mockConfigApi: jest.Mocked<Partial<typeof configApiRef.T>> = {
+  getConfigArray: jest.fn(_ => []),
+};
 
 describe('useConsumerGroupOffsets', () => {
   let entity: Entity;
 
   const wrapper = ({ children }: PropsWithChildren<{}>) => {
-    return <EntityProvider entity={entity}>{children}</EntityProvider>;
+    return (
+      <TestApiProvider apis={[[configApiRef, mockConfigApi]]}>
+        <EntityProvider entity={entity}>{children}</EntityProvider>
+      </TestApiProvider>
+    );
   };
 
   const subject = () => renderHook(useConsumerGroupsForEntity, { wrapper });
@@ -53,6 +63,32 @@ describe('useConsumerGroupOffsets', () => {
     ]);
   });
 
+  it('returns correct dashboard url for cluster for annotation', async () => {
+    entity = {
+      apiVersion: 'v1',
+      kind: 'Component',
+      metadata: {
+        name: 'test',
+        annotations: {
+          'kafka.apache.org/consumer-groups': 'prod/consumer',
+          'kafka.apache.org/dashboard-urls': 'prod/https://akhq.io',
+        },
+      },
+      spec: {
+        owner: 'guest',
+        type: 'Website',
+        lifecycle: 'development',
+      },
+    };
+    const { result } = subject();
+    expect(result.current).toStrictEqual([
+      {
+        clusterId: 'prod',
+        consumerGroup: 'consumer',
+      },
+    ]);
+  });
+
   it('returns correct cluster and consumer group for multiple consumers', async () => {
     entity = {
       apiVersion: 'v1',
@@ -72,7 +108,10 @@ describe('useConsumerGroupOffsets', () => {
     };
     const { result } = subject();
     expect(result.current).toStrictEqual([
-      { clusterId: 'prod', consumerGroup: 'consumer' },
+      {
+        clusterId: 'prod',
+        consumerGroup: 'consumer',
+      },
       {
         clusterId: 'dev',
         consumerGroup: 'another-consumer',
@@ -99,7 +138,10 @@ describe('useConsumerGroupOffsets', () => {
     };
     const { result } = subject();
     expect(result.current).toStrictEqual([
-      { clusterId: 'prod', consumerGroup: 'consumer' },
+      {
+        clusterId: 'prod',
+        consumerGroup: 'consumer',
+      },
       {
         clusterId: 'dev',
         consumerGroup: 'another-consumer',

plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsForEntity.ts

@@ -23,7 +23,7 @@ export const useConsumerGroupsForEntity = () => {
   const annotation =
     entity.metadata.annotations?.[KAFKA_CONSUMER_GROUP_ANNOTATION] ?? '';
 
-  const consumerList = useMemo(() => {
+  return useMemo(() => {
     return annotation.split(',').map(consumer => {
       const [clusterId, consumerGroup] = consumer.split('/');
 
@@ -32,12 +32,11 @@ export const useConsumerGroupsForEntity = () => {
           `Failed to parse kafka consumer group annotation: got "${annotation}"`,
         );
       }
+
       return {
         clusterId: clusterId.trim(),
         consumerGroup: consumerGroup.trim(),
       };
     });
   }, [annotation]);
-
-  return consumerList;
 };

plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsOffsetsForEntity.test.tsx

@@ -22,11 +22,14 @@ import {
   ConsumerGroupOffsetsResponse,
   KafkaApi,
   kafkaApiRef,
+  KafkaDashboardApi,
+  kafkaDashboardApiRef,
 } from '../../api/types';
 import { useConsumerGroupsOffsetsForEntity } from './useConsumerGroupsOffsetsForEntity';
 import * as data from './__fixtures__/consumer-group-offsets.json';
 
 import { errorApiRef } from '@backstage/core-plugin-api';
+import { configApiRef } from '@backstage/core-plugin-api';
 import { TestApiProvider } from '@backstage/test-utils';
 
 const consumerGroupOffsets = data as ConsumerGroupOffsetsResponse;
@@ -40,6 +43,15 @@ const mockKafkaApi: jest.Mocked<KafkaApi> = {
   getConsumerGroupOffsets: jest.fn(),
 };
 
+const mockKafkaDashboardApi: jest.Mocked<KafkaDashboardApi> = {
+  getDashboardUrl: jest.fn(),
+};
+
+// @ts-ignore
+const mockConfigApi: jest.Mocked<typeof configApiRef.T> = {
+  getConfigArray: jest.fn(_ => []),
+};
+
 describe('useConsumerGroupOffsets', () => {
   const entity: Entity = {
     apiVersion: 'v1',
@@ -63,6 +75,8 @@ describe('useConsumerGroupOffsets', () => {
         apis={[
           [errorApiRef, mockErrorApi],
           [kafkaApiRef, mockKafkaApi],
+          [kafkaDashboardApiRef, mockKafkaDashboardApi],
+          [configApiRef, mockConfigApi],
         ]}
       >
         <EntityProvider entity={entity}>{children}</EntityProvider>
@@ -80,6 +94,7 @@ describe('useConsumerGroupOffsets', () => {
     when(mockKafkaApi.getConsumerGroupOffsets)
       .calledWith('prod', consumerGroupOffsets.consumerId)
       .mockResolvedValue(consumerGroupOffsets);
+    when(mockKafkaDashboardApi.getDashboardUrl).mockReturnValue({});
 
     const { result, waitForNextUpdate } = subject();
     await waitForNextUpdate();
@@ -89,6 +104,7 @@ describe('useConsumerGroupOffsets', () => {
       {
         clusterId: 'prod',
         consumerGroup: consumerGroupOffsets.consumerId,
+        dashboardUrl: undefined,
         topics: consumerGroupOffsets.offsets,
       },
     ]);

plugins/kafka/src/components/ConsumerGroupOffsets/useConsumerGroupsOffsetsForEntity.ts

@@ -15,13 +15,16 @@
  */
 
 import useAsyncRetry from 'react-use/lib/useAsyncRetry';
-import { kafkaApiRef } from '../../api/types';
+import { kafkaApiRef, kafkaDashboardApiRef } from '../../api/types';
 import { useConsumerGroupsForEntity } from './useConsumerGroupsForEntity';
 import { errorApiRef, useApi } from '@backstage/core-plugin-api';
+import { useEntity } from '@backstage/plugin-catalog-react';
 
 export const useConsumerGroupsOffsetsForEntity = () => {
   const consumers = useConsumerGroupsForEntity();
+  const { entity } = useEntity();
   const api = useApi(kafkaApiRef);
+  const apiDashboard = useApi(kafkaDashboardApiRef);
   const errorApi = useApi(errorApiRef);
 
   const {
@@ -36,14 +39,23 @@ export const useConsumerGroupsOffsetsForEntity = () => {
             clusterId,
             consumerGroup,
           );
-          return { clusterId, consumerGroup, topics: response.offsets };
+          return {
+            clusterId,
+            dashboardUrl: apiDashboard.getDashboardUrl(
+              clusterId,
+              consumerGroup,
+              entity,
+            ).url,
+            consumerGroup,
+            topics: response.offsets,
+          };
         }),
       );
     } catch (e) {
       errorApi.post(e);
       throw e;
     }
-  }, [consumers, api, errorApi]);
+  }, [consumers, api, apiDashboard, errorApi, entity]);
 
   return [
     {

plugins/kafka/src/constants.ts

@@ -15,3 +15,4 @@
  */
 export const KAFKA_CONSUMER_GROUP_ANNOTATION =
   'kafka.apache.org/consumer-groups';
+export const KAFKA_DASHBOARD_URL = 'kafka.apache.org/dashboard-urls';

plugins/kafka/src/plugin.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 import { KafkaBackendClient } from './api/KafkaBackendClient';
-import { kafkaApiRef } from './api/types';
+import { kafkaApiRef, kafkaDashboardApiRef } from './api/types';
 import {
   createApiFactory,
   createPlugin,
@@ -22,7 +22,9 @@ import {
   createRouteRef,
   discoveryApiRef,
   identityApiRef,
+  configApiRef,
 } from '@backstage/core-plugin-api';
+import { KafkaDashboardClient } from './api/KafkaDashboardClient';
 
 export const rootCatalogKafkaRouteRef = createRouteRef({
   id: 'kafka',
@@ -37,6 +39,11 @@ export const kafkaPlugin = createPlugin({
       factory: ({ discoveryApi, identityApi }) =>
         new KafkaBackendClient({ discoveryApi, identityApi }),
     }),
+    createApiFactory({
+      api: kafkaDashboardApiRef,
+      deps: { configApi: configApiRef },
+      factory: ({ configApi }) => new KafkaDashboardClient({ configApi }),
+    }),
   ],
   routes: {
     entityContent: rootCatalogKafkaRouteRef,