From cbf5d11fdfeebbfdb0baa8ad7d29ab50316897d3 Mon Sep 17 00:00:00 2001 From: Mengnan Gong Date: Mon, 3 Oct 2022 14:42:33 +0800 Subject: [PATCH] Handle the error when fetching pod metrics Include the `PodStatus` into the `FetchResponse` union type. Now the method `fetchPodMetricsByNamespaces` accepts a set of namespaces and return a `FetchResponseWrapper`, which is consistent with other public fetch methods. The Kubernetes errors happened when fetching pod metrics is now captured and returned to the frontend. Signed-off-by: Mengnan Gong --- .changeset/wet-cameras-call.md | 8 +++ plugins/kubernetes-backend/api-report.md | 7 +- .../src/service/KubernetesBuilder.test.ts | 9 ++- .../service/KubernetesFanOutHandler.test.ts | 72 +++++++++++-------- .../src/service/KubernetesFanOutHandler.ts | 36 ++++++---- .../src/service/KubernetesFetcher.ts | 21 ++++-- plugins/kubernetes-backend/src/types/types.ts | 7 +- plugins/kubernetes-common/api-report.md | 12 +++- plugins/kubernetes-common/src/types.ts | 10 ++- 9 files changed, 118 insertions(+), 64 deletions(-) create mode 100644 .changeset/wet-cameras-call.md diff --git a/.changeset/wet-cameras-call.md b/.changeset/wet-cameras-call.md new file mode 100644 index 0000000000..5524e23d15 --- /dev/null +++ b/.changeset/wet-cameras-call.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-kubernetes-backend': minor +'@backstage/plugin-kubernetes-common': minor +--- + +The Kubernetes errors when fetching pod metrics are now captured and returned to the frontend. + +Include the `PodStatusFetchResponse` into the `FetchResponse` union type. Now the method `fetchPodMetricsByNamespaces` accepts a set of namespaces and returns a `FetchResponseWrapper`, just like other public fetch methods in the fetcher. diff --git a/plugins/kubernetes-backend/api-report.md b/plugins/kubernetes-backend/api-report.md index 4219387819..91a95bc078 100644 --- a/plugins/kubernetes-backend/api-report.md +++ b/plugins/kubernetes-backend/api-report.md @@ -22,7 +22,6 @@ import { Logger } from 'winston'; import { Metrics } from '@kubernetes/client-node'; import type { ObjectsByEntityResponse } from '@backstage/plugin-kubernetes-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; -import { PodStatus } from '@kubernetes/client-node/dist/top'; import { TokenCredential } from '@azure/identity'; // @alpha (undocumented) @@ -267,10 +266,10 @@ export interface KubernetesFetcher { params: ObjectFetchParams, ): Promise; // (undocumented) - fetchPodMetricsByNamespace( + fetchPodMetricsByNamespaces( clusterDetails: ClusterDetails, - namespace: string, - ): Promise; + namespaces: Set, + ): Promise; } // @alpha (undocumented) diff --git a/plugins/kubernetes-backend/src/service/KubernetesBuilder.test.ts b/plugins/kubernetes-backend/src/service/KubernetesBuilder.test.ts index ed56c2da8a..2766f9abfc 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesBuilder.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesBuilder.test.ts @@ -30,7 +30,6 @@ import { } from '../types/types'; import { KubernetesBuilder } from './KubernetesBuilder'; import { KubernetesFanOutHandler } from './KubernetesFanOutHandler'; -import { PodStatus } from '@kubernetes/client-node'; import { CatalogApi } from '@backstage/catalog-client'; describe('KubernetesBuilder', () => { @@ -222,11 +221,11 @@ describe('KubernetesBuilder', () => { }; const fetcher: KubernetesFetcher = { - fetchPodMetricsByNamespace( + fetchPodMetricsByNamespaces( _clusterDetails: ClusterDetails, - _namespace: string, - ): Promise { - return Promise.resolve([]); + _namespaces: Set, + ): Promise { + return Promise.resolve({ errors: [], responses: [] }); }, fetchObjectsForService( _params: ObjectFetchParams, diff --git a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts index 04256ed3dc..8627795a7f 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts @@ -18,13 +18,13 @@ import { getVoidLogger } from '@backstage/backend-common'; import { ClusterDetails, CustomResource, + FetchResponseWrapper, ObjectFetchParams, } from '../types/types'; import { KubernetesFanOutHandler } from './KubernetesFanOutHandler'; -import { PodStatus } from '@kubernetes/client-node/dist/top'; const fetchObjectsForService = jest.fn(); -const fetchPodMetricsByNamespace = jest.fn(); +const fetchPodMetricsByNamespaces = jest.fn(); const getClustersByEntity = jest.fn(); @@ -55,8 +55,9 @@ const mockFetch = (mock: jest.Mock) => { }; const mockMetrics = (mock: jest.Mock) => { - mock.mockImplementation((clusterDetails: ClusterDetails, namespace: string) => - Promise.resolve(generatePodStatus(clusterDetails.name, namespace)), + mock.mockImplementation( + (clusterDetails: ClusterDetails, namespaces: Set) => + Promise.resolve(generatePodStatus(clusterDetails.name, namespaces)), ); }; @@ -143,7 +144,7 @@ function mockFetchAndGetKubernetesFanOutHandler( customResources: CustomResource[], ) { mockFetch(fetchObjectsForService); - mockMetrics(fetchPodMetricsByNamespace); + mockMetrics(fetchPodMetricsByNamespaces); return getKubernetesFanOutHandler(customResources); } @@ -153,7 +154,7 @@ function getKubernetesFanOutHandler(customResources: CustomResource[]) { logger: getVoidLogger(), fetcher: { fetchObjectsForService, - fetchPodMetricsByNamespace, + fetchPodMetricsByNamespaces, }, serviceLocator: { getClustersByEntity, @@ -164,24 +165,32 @@ function getKubernetesFanOutHandler(customResources: CustomResource[]) { function generatePodStatus( _clusterName: string, - _namespace: string, -): PodStatus[] { - return [ - { - Pod: {}, - CPU: { - CurrentUsage: 100, - RequestTotal: 101, - LimitTotal: 102, - }, - Memory: { - CurrentUsage: BigInt('1000'), - RequestTotal: BigInt('1001'), - LimitTotal: BigInt('1002'), - }, - Containers: [], - }, - ] as any; + _namespaces: Set, +): FetchResponseWrapper { + return { + errors: [], + responses: Array.from(_namespaces).map(() => { + return { + type: 'podstatus', + resources: [ + { + Pod: {}, + CPU: { + CurrentUsage: 100, + RequestTotal: 101, + LimitTotal: 102, + }, + Memory: { + CurrentUsage: BigInt('1000'), + RequestTotal: BigInt('1001'), + LimitTotal: BigInt('1002'), + }, + Containers: [], + }, + ], + }; + }), + }; } function generateMockResourcesAndErrors( @@ -292,9 +301,9 @@ describe('getKubernetesObjectsByEntity', () => { expect(getClustersByEntity.mock.calls.length).toBe(1); expect(fetchObjectsForService.mock.calls.length).toBe(1); - expect(fetchPodMetricsByNamespace.mock.calls.length).toBe(1); - expect(fetchPodMetricsByNamespace.mock.calls[0][1]).toBe( - 'ns-test-component-test-cluster', + expect(fetchPodMetricsByNamespaces.mock.calls.length).toBe(1); + expect(fetchPodMetricsByNamespaces.mock.calls[0][1]).toStrictEqual( + new Set(['ns-test-component-test-cluster']), ); expect(result).toStrictEqual({ @@ -433,7 +442,7 @@ describe('getKubernetesObjectsByEntity', () => { }), ); - mockMetrics(fetchPodMetricsByNamespace); + mockMetrics(fetchPodMetricsByNamespaces); const sut = getKubernetesFanOutHandler([]); @@ -444,9 +453,10 @@ describe('getKubernetesObjectsByEntity', () => { expect(getClustersByEntity.mock.calls.length).toBe(1); expect(fetchObjectsForService.mock.calls.length).toBe(1); - expect(fetchPodMetricsByNamespace.mock.calls.length).toBe(2); - expect(fetchPodMetricsByNamespace.mock.calls[0][1]).toBe('ns-a'); - expect(fetchPodMetricsByNamespace.mock.calls[1][1]).toBe('ns-b'); + expect(fetchPodMetricsByNamespaces.mock.calls.length).toBe(1); + expect(fetchPodMetricsByNamespaces.mock.calls[0][1]).toStrictEqual( + new Set(['ns-a', 'ns-b']), + ); expect(result).toStrictEqual({ items: [ diff --git a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.ts b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.ts index 23163d1004..7de3e70249 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.ts @@ -40,6 +40,7 @@ import { PodFetchResponse, KubernetesRequestAuth, CustomResourceMatcher, + PodStatusFetchResponse, } from '@backstage/plugin-kubernetes-common'; import { ContainerStatus, @@ -162,19 +163,22 @@ const toClientSafeContainer = ( }; const toClientSafePodMetrics = ( - podMetrics: PodStatus[][], + podMetrics: PodStatusFetchResponse[], ): ClientPodStatus[] => { - return podMetrics.flat().map((pd: PodStatus): ClientPodStatus => { - return { - pod: pd.Pod, - memory: toClientSafeResource(pd.Memory), - cpu: toClientSafeResource(pd.CPU), - containers: pd.Containers.map(toClientSafeContainer), - }; - }); + return podMetrics + .map(r => r.resources) + .flat() + .map((pd: PodStatus): ClientPodStatus => { + return { + pod: pd.Pod, + memory: toClientSafeResource(pd.Memory), + cpu: toClientSafeResource(pd.CPU), + containers: pd.Containers.map(toClientSafeContainer), + }; + }); }; -type responseWithMetrics = [FetchResponseWrapper, PodStatus[][]]; +type responseWithMetrics = [FetchResponseWrapper, PodStatusFetchResponse[]]; export class KubernetesFanOutHandler { private readonly logger: Logger; @@ -345,11 +349,17 @@ export class KubernetesFanOutHandler { .filter(isString), ); - const podMetrics = Array.from(namespaces).map(ns => - this.fetcher.fetchPodMetricsByNamespace(clusterDetails, ns), + if (namespaces.size === 0) { + return [result, []]; + } + + const podMetrics = await this.fetcher.fetchPodMetricsByNamespaces( + clusterDetails, + namespaces, ); - return Promise.all([result, Promise.all(podMetrics)]); + result.errors.concat(podMetrics.errors); + return [result, podMetrics.responses as PodStatusFetchResponse[]]; } private getAuthTranslator(provider: string): KubernetesAuthTranslator { diff --git a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts index 1189dfa390..6c61f10700 100644 --- a/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts +++ b/plugins/kubernetes-backend/src/service/KubernetesFetcher.ts @@ -29,9 +29,9 @@ import { FetchResponse, KubernetesFetchError, KubernetesErrorTypes, + PodStatusFetchResponse, } from '@backstage/plugin-kubernetes-common'; import { KubernetesClientProvider } from './KubernetesClientProvider'; -import { PodStatus } from '@kubernetes/client-node/dist/top'; export interface Clients { core: CoreV1Api; @@ -104,10 +104,10 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { return Promise.all(fetchResults).then(fetchResultsToResponseWrapper); } - fetchPodMetricsByNamespace( + fetchPodMetricsByNamespaces( clusterDetails: ClusterDetails, - namespace: string, - ): Promise { + namespaces: Set, + ): Promise { const metricsClient = this.kubernetesClientProvider.getMetricsClient(clusterDetails); const coreApi = @@ -115,7 +115,18 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher { clusterDetails, ); - return topPods(coreApi, metricsClient, namespace); + const fetchResults = Array.from(namespaces).map(ns => + topPods(coreApi, metricsClient, ns) + .then(r => { + return { + type: 'podstatus', + resources: r, + } as PodStatusFetchResponse; + }) + .catch(this.captureKubernetesErrorsRethrowOthers.bind(this)), + ); + + return Promise.all(fetchResults).then(fetchResultsToResponseWrapper); } private captureKubernetesErrorsRethrowOthers(e: any): KubernetesFetchError { diff --git a/plugins/kubernetes-backend/src/types/types.ts b/plugins/kubernetes-backend/src/types/types.ts index 5ddfe7367d..b7393ce7a3 100644 --- a/plugins/kubernetes-backend/src/types/types.ts +++ b/plugins/kubernetes-backend/src/types/types.ts @@ -25,7 +25,6 @@ import type { KubernetesRequestBody, ObjectsByEntityResponse, } from '@backstage/plugin-kubernetes-common'; -import { PodStatus } from '@kubernetes/client-node/dist/top'; /** * @@ -53,10 +52,10 @@ export interface KubernetesFetcher { fetchObjectsForService( params: ObjectFetchParams, ): Promise; - fetchPodMetricsByNamespace( + fetchPodMetricsByNamespaces( clusterDetails: ClusterDetails, - namespace: string, - ): Promise; + namespaces: Set, + ): Promise; } /** diff --git a/plugins/kubernetes-common/api-report.md b/plugins/kubernetes-common/api-report.md index 85015662c6..3ed89d2a56 100644 --- a/plugins/kubernetes-common/api-report.md +++ b/plugins/kubernetes-common/api-report.md @@ -5,6 +5,7 @@ ```ts import { Entity } from '@backstage/catalog-model'; import type { JsonObject } from '@backstage/types'; +import { PodStatus } from '@kubernetes/client-node'; import { V1ConfigMap } from '@kubernetes/client-node'; import { V1CronJob } from '@kubernetes/client-node'; import { V1DaemonSet } from '@kubernetes/client-node'; @@ -147,7 +148,8 @@ export type FetchResponse = | IngressesFetchResponse | CustomResourceFetchResponse | StatefulSetsFetchResponse - | DaemonSetsFetchResponse; + | DaemonSetsFetchResponse + | PodStatusFetchResponse; // @public (undocumented) export interface HorizontalPodAutoscalersFetchResponse { @@ -230,6 +232,14 @@ export interface PodFetchResponse { type: 'pods'; } +// @public (undocumented) +export interface PodStatusFetchResponse { + // (undocumented) + resources: Array; + // (undocumented) + type: 'podstatus'; +} + // @public (undocumented) export interface ReplicaSetsFetchResponse { // (undocumented) diff --git a/plugins/kubernetes-common/src/types.ts b/plugins/kubernetes-common/src/types.ts index 149caadb15..58fc81f3ff 100644 --- a/plugins/kubernetes-common/src/types.ts +++ b/plugins/kubernetes-common/src/types.ts @@ -16,6 +16,7 @@ import type { JsonObject } from '@backstage/types'; import { + PodStatus, V1ConfigMap, V1CronJob, V1DaemonSet, @@ -134,7 +135,8 @@ export type FetchResponse = | IngressesFetchResponse | CustomResourceFetchResponse | StatefulSetsFetchResponse - | DaemonSetsFetchResponse; + | DaemonSetsFetchResponse + | PodStatusFetchResponse; /** @public */ export interface PodFetchResponse { @@ -214,6 +216,12 @@ export interface DaemonSetsFetchResponse { resources: Array; } +/** @public */ +export interface PodStatusFetchResponse { + type: 'podstatus'; + resources: Array; +} + /** @public */ export interface KubernetesFetchError { errorType: KubernetesErrorTypes;