Handle the error when fetching pod metrics

Include the `PodStatus` into the `FetchResponse` union type. Now the method `fetchPodMetricsByNamespaces` accepts a set of namespaces and return a `FetchResponseWrapper`, which is consistent with other public fetch methods.

The Kubernetes errors happened when fetching pod metrics is now captured and returned to the frontend.

Signed-off-by: Mengnan Gong <namco1992@gmail.com>

.changeset/wet-cameras-call.md

@@ -0,0 +1,8 @@
+---
+'@backstage/plugin-kubernetes-backend': minor
+'@backstage/plugin-kubernetes-common': minor
+---
+
+The Kubernetes errors when fetching pod metrics are now captured and returned to the frontend.
+
+Include the `PodStatusFetchResponse` into the `FetchResponse` union type. Now the method `fetchPodMetricsByNamespaces` accepts a set of namespaces and returns a `FetchResponseWrapper`, just like other public fetch methods in the fetcher.

plugins/kubernetes-backend/api-report.md

@@ -22,7 +22,6 @@ import { Logger } from 'winston';
 import { Metrics } from '@kubernetes/client-node';
 import type { ObjectsByEntityResponse } from '@backstage/plugin-kubernetes-common';
 import { PluginEndpointDiscovery } from '@backstage/backend-common';
-import { PodStatus } from '@kubernetes/client-node/dist/top';
 import { TokenCredential } from '@azure/identity';
 
 // @alpha (undocumented)
@@ -267,10 +266,10 @@ export interface KubernetesFetcher {
     params: ObjectFetchParams,
   ): Promise<FetchResponseWrapper>;
   // (undocumented)
-  fetchPodMetricsByNamespace(
+  fetchPodMetricsByNamespaces(
     clusterDetails: ClusterDetails,
-    namespace: string,
-  ): Promise<PodStatus[]>;
+    namespaces: Set<string>,
+  ): Promise<FetchResponseWrapper>;
 }
 
 // @alpha (undocumented)

plugins/kubernetes-backend/src/service/KubernetesBuilder.test.ts

@@ -30,7 +30,6 @@ import {
 } from '../types/types';
 import { KubernetesBuilder } from './KubernetesBuilder';
 import { KubernetesFanOutHandler } from './KubernetesFanOutHandler';
-import { PodStatus } from '@kubernetes/client-node';
 import { CatalogApi } from '@backstage/catalog-client';
 
 describe('KubernetesBuilder', () => {
@@ -222,11 +221,11 @@ describe('KubernetesBuilder', () => {
       };
 
       const fetcher: KubernetesFetcher = {
-        fetchPodMetricsByNamespace(
+        fetchPodMetricsByNamespaces(
           _clusterDetails: ClusterDetails,
-          _namespace: string,
-        ): Promise<PodStatus[]> {
-          return Promise.resolve([]);
+          _namespaces: Set<string>,
+        ): Promise<FetchResponseWrapper> {
+          return Promise.resolve({ errors: [], responses: [] });
         },
         fetchObjectsForService(
           _params: ObjectFetchParams,

plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts

@@ -18,13 +18,13 @@ import { getVoidLogger } from '@backstage/backend-common';
 import {
   ClusterDetails,
   CustomResource,
+  FetchResponseWrapper,
   ObjectFetchParams,
 } from '../types/types';
 import { KubernetesFanOutHandler } from './KubernetesFanOutHandler';
-import { PodStatus } from '@kubernetes/client-node/dist/top';
 
 const fetchObjectsForService = jest.fn();
-const fetchPodMetricsByNamespace = jest.fn();
+const fetchPodMetricsByNamespaces = jest.fn();
 
 const getClustersByEntity = jest.fn();
 
@@ -55,8 +55,9 @@ const mockFetch = (mock: jest.Mock) => {
 };
 
 const mockMetrics = (mock: jest.Mock) => {
-  mock.mockImplementation((clusterDetails: ClusterDetails, namespace: string) =>
-    Promise.resolve(generatePodStatus(clusterDetails.name, namespace)),
+  mock.mockImplementation(
+    (clusterDetails: ClusterDetails, namespaces: Set<string>) =>
+      Promise.resolve(generatePodStatus(clusterDetails.name, namespaces)),
   );
 };
 
@@ -143,7 +144,7 @@ function mockFetchAndGetKubernetesFanOutHandler(
   customResources: CustomResource[],
 ) {
   mockFetch(fetchObjectsForService);
-  mockMetrics(fetchPodMetricsByNamespace);
+  mockMetrics(fetchPodMetricsByNamespaces);
 
   return getKubernetesFanOutHandler(customResources);
 }
@@ -153,7 +154,7 @@ function getKubernetesFanOutHandler(customResources: CustomResource[]) {
     logger: getVoidLogger(),
     fetcher: {
       fetchObjectsForService,
-      fetchPodMetricsByNamespace,
+      fetchPodMetricsByNamespaces,
     },
     serviceLocator: {
       getClustersByEntity,
@@ -164,24 +165,32 @@ function getKubernetesFanOutHandler(customResources: CustomResource[]) {
 
 function generatePodStatus(
   _clusterName: string,
-  _namespace: string,
-): PodStatus[] {
-  return [
-    {
-      Pod: {},
-      CPU: {
-        CurrentUsage: 100,
-        RequestTotal: 101,
-        LimitTotal: 102,
-      },
-      Memory: {
-        CurrentUsage: BigInt('1000'),
-        RequestTotal: BigInt('1001'),
-        LimitTotal: BigInt('1002'),
-      },
-      Containers: [],
-    },
-  ] as any;
+  _namespaces: Set<string>,
+): FetchResponseWrapper {
+  return {
+    errors: [],
+    responses: Array.from(_namespaces).map(() => {
+      return {
+        type: 'podstatus',
+        resources: [
+          {
+            Pod: {},
+            CPU: {
+              CurrentUsage: 100,
+              RequestTotal: 101,
+              LimitTotal: 102,
+            },
+            Memory: {
+              CurrentUsage: BigInt('1000'),
+              RequestTotal: BigInt('1001'),
+              LimitTotal: BigInt('1002'),
+            },
+            Containers: [],
+          },
+        ],
+      };
+    }),
+  };
 }
 
 function generateMockResourcesAndErrors(
@@ -292,9 +301,9 @@ describe('getKubernetesObjectsByEntity', () => {
 
     expect(getClustersByEntity.mock.calls.length).toBe(1);
     expect(fetchObjectsForService.mock.calls.length).toBe(1);
-    expect(fetchPodMetricsByNamespace.mock.calls.length).toBe(1);
-    expect(fetchPodMetricsByNamespace.mock.calls[0][1]).toBe(
-      'ns-test-component-test-cluster',
+    expect(fetchPodMetricsByNamespaces.mock.calls.length).toBe(1);
+    expect(fetchPodMetricsByNamespaces.mock.calls[0][1]).toStrictEqual(
+      new Set(['ns-test-component-test-cluster']),
     );
 
     expect(result).toStrictEqual({
@@ -433,7 +442,7 @@ describe('getKubernetesObjectsByEntity', () => {
       }),
     );
 
-    mockMetrics(fetchPodMetricsByNamespace);
+    mockMetrics(fetchPodMetricsByNamespaces);
 
     const sut = getKubernetesFanOutHandler([]);
 
@@ -444,9 +453,10 @@ describe('getKubernetesObjectsByEntity', () => {
 
     expect(getClustersByEntity.mock.calls.length).toBe(1);
     expect(fetchObjectsForService.mock.calls.length).toBe(1);
-    expect(fetchPodMetricsByNamespace.mock.calls.length).toBe(2);
-    expect(fetchPodMetricsByNamespace.mock.calls[0][1]).toBe('ns-a');
-    expect(fetchPodMetricsByNamespace.mock.calls[1][1]).toBe('ns-b');
+    expect(fetchPodMetricsByNamespaces.mock.calls.length).toBe(1);
+    expect(fetchPodMetricsByNamespaces.mock.calls[0][1]).toStrictEqual(
+      new Set(['ns-a', 'ns-b']),
+    );
 
     expect(result).toStrictEqual({
       items: [

plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.ts

@@ -40,6 +40,7 @@ import {
   PodFetchResponse,
   KubernetesRequestAuth,
   CustomResourceMatcher,
+  PodStatusFetchResponse,
 } from '@backstage/plugin-kubernetes-common';
 import {
   ContainerStatus,
@@ -162,19 +163,22 @@ const toClientSafeContainer = (
 };
 
 const toClientSafePodMetrics = (
-  podMetrics: PodStatus[][],
+  podMetrics: PodStatusFetchResponse[],
 ): ClientPodStatus[] => {
-  return podMetrics.flat().map((pd: PodStatus): ClientPodStatus => {
-    return {
-      pod: pd.Pod,
-      memory: toClientSafeResource(pd.Memory),
-      cpu: toClientSafeResource(pd.CPU),
-      containers: pd.Containers.map(toClientSafeContainer),
-    };
-  });
+  return podMetrics
+    .map(r => r.resources)
+    .flat()
+    .map((pd: PodStatus): ClientPodStatus => {
+      return {
+        pod: pd.Pod,
+        memory: toClientSafeResource(pd.Memory),
+        cpu: toClientSafeResource(pd.CPU),
+        containers: pd.Containers.map(toClientSafeContainer),
+      };
+    });
 };
 
-type responseWithMetrics = [FetchResponseWrapper, PodStatus[][]];
+type responseWithMetrics = [FetchResponseWrapper, PodStatusFetchResponse[]];
 
 export class KubernetesFanOutHandler {
   private readonly logger: Logger;
@@ -345,11 +349,17 @@ export class KubernetesFanOutHandler {
         .filter(isString),
     );
 
-    const podMetrics = Array.from(namespaces).map(ns =>
-      this.fetcher.fetchPodMetricsByNamespace(clusterDetails, ns),
+    if (namespaces.size === 0) {
+      return [result, []];
+    }
+
+    const podMetrics = await this.fetcher.fetchPodMetricsByNamespaces(
+      clusterDetails,
+      namespaces,
     );
 
-    return Promise.all([result, Promise.all(podMetrics)]);
+    result.errors.concat(podMetrics.errors);
+    return [result, podMetrics.responses as PodStatusFetchResponse[]];
   }
 
   private getAuthTranslator(provider: string): KubernetesAuthTranslator {

plugins/kubernetes-backend/src/service/KubernetesFetcher.ts

@@ -29,9 +29,9 @@ import {
   FetchResponse,
   KubernetesFetchError,
   KubernetesErrorTypes,
+  PodStatusFetchResponse,
 } from '@backstage/plugin-kubernetes-common';
 import { KubernetesClientProvider } from './KubernetesClientProvider';
-import { PodStatus } from '@kubernetes/client-node/dist/top';
 
 export interface Clients {
   core: CoreV1Api;
@@ -104,10 +104,10 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher {
     return Promise.all(fetchResults).then(fetchResultsToResponseWrapper);
   }
 
-  fetchPodMetricsByNamespace(
+  fetchPodMetricsByNamespaces(
     clusterDetails: ClusterDetails,
-    namespace: string,
-  ): Promise<PodStatus[]> {
+    namespaces: Set<string>,
+  ): Promise<FetchResponseWrapper> {
     const metricsClient =
       this.kubernetesClientProvider.getMetricsClient(clusterDetails);
     const coreApi =
@@ -115,7 +115,18 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher {
         clusterDetails,
       );
 
-    return topPods(coreApi, metricsClient, namespace);
+    const fetchResults = Array.from(namespaces).map(ns =>
+      topPods(coreApi, metricsClient, ns)
+        .then(r => {
+          return {
+            type: 'podstatus',
+            resources: r,
+          } as PodStatusFetchResponse;
+        })
+        .catch(this.captureKubernetesErrorsRethrowOthers.bind(this)),
+    );
+
+    return Promise.all(fetchResults).then(fetchResultsToResponseWrapper);
   }
 
   private captureKubernetesErrorsRethrowOthers(e: any): KubernetesFetchError {

plugins/kubernetes-backend/src/types/types.ts

@@ -25,7 +25,6 @@ import type {
   KubernetesRequestBody,
   ObjectsByEntityResponse,
 } from '@backstage/plugin-kubernetes-common';
-import { PodStatus } from '@kubernetes/client-node/dist/top';
 
 /**
  *
@@ -53,10 +52,10 @@ export interface KubernetesFetcher {
   fetchObjectsForService(
     params: ObjectFetchParams,
   ): Promise<FetchResponseWrapper>;
-  fetchPodMetricsByNamespace(
+  fetchPodMetricsByNamespaces(
     clusterDetails: ClusterDetails,
-    namespace: string,
-  ): Promise<PodStatus[]>;
+    namespaces: Set<string>,
+  ): Promise<FetchResponseWrapper>;
 }
 
 /**

plugins/kubernetes-common/api-report.md

@@ -5,6 +5,7 @@
 ```ts
 import { Entity } from '@backstage/catalog-model';
 import type { JsonObject } from '@backstage/types';
+import { PodStatus } from '@kubernetes/client-node';
 import { V1ConfigMap } from '@kubernetes/client-node';
 import { V1CronJob } from '@kubernetes/client-node';
 import { V1DaemonSet } from '@kubernetes/client-node';
@@ -147,7 +148,8 @@ export type FetchResponse =
   | IngressesFetchResponse
   | CustomResourceFetchResponse
   | StatefulSetsFetchResponse
-  | DaemonSetsFetchResponse;
+  | DaemonSetsFetchResponse
+  | PodStatusFetchResponse;
 
 // @public (undocumented)
 export interface HorizontalPodAutoscalersFetchResponse {
@@ -230,6 +232,14 @@ export interface PodFetchResponse {
   type: 'pods';
 }
 
+// @public (undocumented)
+export interface PodStatusFetchResponse {
+  // (undocumented)
+  resources: Array<PodStatus>;
+  // (undocumented)
+  type: 'podstatus';
+}
+
 // @public (undocumented)
 export interface ReplicaSetsFetchResponse {
   // (undocumented)

plugins/kubernetes-common/src/types.ts

@@ -16,6 +16,7 @@
 
 import type { JsonObject } from '@backstage/types';
 import {
+  PodStatus,
   V1ConfigMap,
   V1CronJob,
   V1DaemonSet,
@@ -134,7 +135,8 @@ export type FetchResponse =
   | IngressesFetchResponse
   | CustomResourceFetchResponse
   | StatefulSetsFetchResponse
-  | DaemonSetsFetchResponse;
+  | DaemonSetsFetchResponse
+  | PodStatusFetchResponse;
 
 /** @public */
 export interface PodFetchResponse {
@@ -214,6 +216,12 @@ export interface DaemonSetsFetchResponse {
   resources: Array<V1DaemonSet>;
 }
 
+/** @public */
+export interface PodStatusFetchResponse {
+  type: 'podstatus';
+  resources: Array<PodStatus>;
+}
+
 /** @public */
 export interface KubernetesFetchError {
   errorType: KubernetesErrorTypes;