paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 19 Packages Projects Releases Wiki Activity

Merge pull request #13966 from namco1992/pod-metrics-error-handling

Browse Source 
[K8s plugin] Handle the error when fetching pod metrics
This commit is contained in:
Johan Haals
2022-10-19 10:55:47 +02:00
committed by GitHub
parent 78b8448b21 cc9749a88e
commit ba64de0372
10 changed files with 712 additions and 468 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/wet-cameras-call.md
+10
View File
@@ -0,0 +1,10 @@
---
'@backstage/plugin-kubernetes-backend': minor
'@backstage/plugin-kubernetes-common': patch
---
The Kubernetes errors when fetching pod metrics are now captured and returned to the frontend.
- **BREAKING** The method `fetchPodMetricsByNamespace` in the interface `KubernetesFetcher` is changed to `fetchPodMetricsByNamespaces`. It now accepts a set of namespace strings and returns `Promise<FetchResponseWrapper>`.
- Add the `PodStatusFetchResponse` to the `FetchResponse` union type.
- Add `NOT_FOUND` to the `KubernetesErrorTypes` union type, the HTTP error with status code 404 will be mapped to this error.
plugins/kubernetes-backend/api-report.md
+3 -4
View File
@@ -22,7 +22,6 @@ import { Logger } from 'winston';
import { Metrics } from '@kubernetes/client-node';
import type { ObjectsByEntityResponse } from '@backstage/plugin-kubernetes-common';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { PodStatus } from '@kubernetes/client-node/dist/top';
import { TokenCredential } from '@azure/identity';
// @alpha (undocumented)
@@ -267,10 +266,10 @@ export interface KubernetesFetcher {
params: ObjectFetchParams,
): Promise<FetchResponseWrapper>;
// (undocumented)
fetchPodMetricsByNamespace(
fetchPodMetricsByNamespaces(
clusterDetails: ClusterDetails,
namespace: string,
): Promise<PodStatus[]>;
namespaces: Set<string>,
): Promise<FetchResponseWrapper>;
}
// @alpha (undocumented)
plugins/kubernetes-backend/src/service/KubernetesBuilder.test.ts
+4 -5
View File
@@ -30,7 +30,6 @@ import {
} from '../types/types';
import { KubernetesBuilder } from './KubernetesBuilder';
import { KubernetesFanOutHandler } from './KubernetesFanOutHandler';
import { PodStatus } from '@kubernetes/client-node';
import { CatalogApi } from '@backstage/catalog-client';
describe('KubernetesBuilder', () => {
@@ -222,11 +221,11 @@ describe('KubernetesBuilder', () => {
};
const fetcher: KubernetesFetcher = {
fetchPodMetricsByNamespace(
fetchPodMetricsByNamespaces(
_clusterDetails: ClusterDetails,
_namespace: string,
): Promise<PodStatus[]> {
return Promise.resolve([]);
_namespaces: Set<string>,
): Promise<FetchResponseWrapper> {
return Promise.resolve({ errors: [], responses: [] });
},
fetchObjectsForService(
_params: ObjectFetchParams,
plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.test.ts
+117 -31
View File
@@ -18,13 +18,13 @@ import { getVoidLogger } from '@backstage/backend-common';
import {
ClusterDetails,
CustomResource,
FetchResponseWrapper,
ObjectFetchParams,
} from '../types/types';
import { KubernetesFanOutHandler } from './KubernetesFanOutHandler';
import { PodStatus } from '@kubernetes/client-node/dist/top';
const fetchObjectsForService = jest.fn();
const fetchPodMetricsByNamespace = jest.fn();
const fetchPodMetricsByNamespaces = jest.fn();
const getClustersByEntity = jest.fn();
@@ -55,8 +55,9 @@ const mockFetch = (mock: jest.Mock) => {
};
const mockMetrics = (mock: jest.Mock) => {
mock.mockImplementation((clusterDetails: ClusterDetails, namespace: string) =>
Promise.resolve(generatePodStatus(clusterDetails.name, namespace)),
mock.mockImplementation(
(clusterDetails: ClusterDetails, namespaces: Set<string>) =>
Promise.resolve(generatePodStatus(clusterDetails.name, namespaces)),
);
};
@@ -143,7 +144,7 @@ function mockFetchAndGetKubernetesFanOutHandler(
customResources: CustomResource[],
) {
mockFetch(fetchObjectsForService);
mockMetrics(fetchPodMetricsByNamespace);
mockMetrics(fetchPodMetricsByNamespaces);
return getKubernetesFanOutHandler(customResources);
}
@@ -153,7 +154,7 @@ function getKubernetesFanOutHandler(customResources: CustomResource[]) {
logger: getVoidLogger(),
fetcher: {
fetchObjectsForService,
fetchPodMetricsByNamespace,
fetchPodMetricsByNamespaces,
},
serviceLocator: {
getClustersByEntity,
@@ -164,24 +165,32 @@ function getKubernetesFanOutHandler(customResources: CustomResource[]) {
function generatePodStatus(
_clusterName: string,
_namespace: string,
): PodStatus[] {
return [
{
Pod: {},
CPU: {
CurrentUsage: 100,
RequestTotal: 101,
LimitTotal: 102,
},
Memory: {
CurrentUsage: BigInt('1000'),
RequestTotal: BigInt('1001'),
LimitTotal: BigInt('1002'),
},
Containers: [],
},
] as any;
_namespaces: Set<string>,
): FetchResponseWrapper {
return {
errors: [],
responses: Array.from(_namespaces).map(() => {
return {
type: 'podstatus',
resources: [
{
Pod: {},
CPU: {
CurrentUsage: 100,
RequestTotal: 101,
LimitTotal: 102,
},
Memory: {
CurrentUsage: BigInt('1000'),
RequestTotal: BigInt('1001'),
LimitTotal: BigInt('1002'),
},
Containers: [],
},
],
};
}),
};
}
function generateMockResourcesAndErrors(
@@ -292,9 +301,9 @@ describe('getKubernetesObjectsByEntity', () => {
expect(getClustersByEntity.mock.calls.length).toBe(1);
expect(fetchObjectsForService.mock.calls.length).toBe(1);
expect(fetchPodMetricsByNamespace.mock.calls.length).toBe(1);
expect(fetchPodMetricsByNamespace.mock.calls[0][1]).toBe(
'ns-test-component-test-cluster',
expect(fetchPodMetricsByNamespaces.mock.calls.length).toBe(1);
expect(fetchPodMetricsByNamespaces.mock.calls[0][1]).toStrictEqual(
new Set(['ns-test-component-test-cluster']),
);
expect(result).toStrictEqual({
@@ -433,7 +442,7 @@ describe('getKubernetesObjectsByEntity', () => {
}),
);
mockMetrics(fetchPodMetricsByNamespace);
mockMetrics(fetchPodMetricsByNamespaces);
const sut = getKubernetesFanOutHandler([]);
@@ -444,9 +453,10 @@ describe('getKubernetesObjectsByEntity', () => {
expect(getClustersByEntity.mock.calls.length).toBe(1);
expect(fetchObjectsForService.mock.calls.length).toBe(1);
expect(fetchPodMetricsByNamespace.mock.calls.length).toBe(2);
expect(fetchPodMetricsByNamespace.mock.calls[0][1]).toBe('ns-a');
expect(fetchPodMetricsByNamespace.mock.calls[1][1]).toBe('ns-b');
expect(fetchPodMetricsByNamespaces.mock.calls.length).toBe(1);
expect(fetchPodMetricsByNamespaces.mock.calls[0][1]).toStrictEqual(
new Set(['ns-a', 'ns-b']),
);
expect(result).toStrictEqual({
items: [
@@ -665,6 +675,82 @@ describe('getKubernetesObjectsByEntity', () => {
],
});
});
it('retrieve objects for two clusters, one fails to fetch pod metrics', async () => {
getClustersByEntity.mockImplementation(() =>
Promise.resolve({
clusters: [
{
name: 'test-cluster',
authProvider: 'serviceAccount',
dashboardUrl: 'https://k8s.foo.coom',
},
{
name: 'other-cluster',
authProvider: 'google',
},
],
}),
);
mockFetch(fetchObjectsForService);
// To simulate the partial failure, return a valid response for the first call,
// and an error for the second call.
fetchPodMetricsByNamespaces
.mockImplementationOnce(
(clusterDetails: ClusterDetails, namespaces: Set<string>) =>
Promise.resolve(generatePodStatus(clusterDetails.name, namespaces)),
)
.mockResolvedValueOnce({
errors: [
{
errorType: 'NOT_FOUND',
resourcePath: '/some/path',
statusCode: 404,
},
],
responses: [],
});
const sut = getKubernetesFanOutHandler([]);
const result = await sut.getKubernetesObjectsByEntity({
entity,
auth: {
google: 'google_token_123',
},
});
expect(getClustersByEntity.mock.calls.length).toBe(1);
expect(fetchObjectsForService.mock.calls.length).toBe(2);
expect(result).toStrictEqual({
items: [
{
cluster: {
dashboardUrl: 'https://k8s.foo.coom',
name: 'test-cluster',
},
errors: [],
podMetrics: [POD_METRICS_FIXTURE],
resources: resourcesByCluster('test-cluster'),
},
{
cluster: {
name: 'other-cluster',
},
errors: [
{
errorType: 'NOT_FOUND',
resourcePath: '/some/path',
statusCode: 404,
},
],
podMetrics: [],
resources: resourcesByCluster('other-cluster'),
},
],
});
});
});
describe('getCustomResourcesByEntity', () => {
plugins/kubernetes-backend/src/service/KubernetesFanOutHandler.ts
+23 -13
View File
@@ -41,6 +41,7 @@ import {
PodFetchResponse,
KubernetesRequestAuth,
CustomResourceMatcher,
PodStatusFetchResponse,
} from '@backstage/plugin-kubernetes-common';
import {
ContainerStatus,
@@ -163,19 +164,22 @@ const toClientSafeContainer = (
};
const toClientSafePodMetrics = (
podMetrics: PodStatus[][],
podMetrics: PodStatusFetchResponse[],
): ClientPodStatus[] => {
return podMetrics.flat().map((pd: PodStatus): ClientPodStatus => {
return {
pod: pd.Pod,
memory: toClientSafeResource(pd.Memory),
cpu: toClientSafeResource(pd.CPU),
containers: pd.Containers.map(toClientSafeContainer),
};
});
return podMetrics
.map(r => r.resources)
.flat()
.map((pd: PodStatus): ClientPodStatus => {
return {
pod: pd.Pod,
memory: toClientSafeResource(pd.Memory),
cpu: toClientSafeResource(pd.CPU),
containers: pd.Containers.map(toClientSafeContainer),
};
});
};
type responseWithMetrics = [FetchResponseWrapper, PodStatus[][]];
type responseWithMetrics = [FetchResponseWrapper, PodStatusFetchResponse[]];
export class KubernetesFanOutHandler {
private readonly logger: Logger;
@@ -350,11 +354,17 @@ export class KubernetesFanOutHandler {
.filter(isString),
);
const podMetrics = Array.from(namespaces).map(ns =>
this.fetcher.fetchPodMetricsByNamespace(clusterDetails, ns),
if (namespaces.size === 0) {
return [result, []];
}
const podMetrics = await this.fetcher.fetchPodMetricsByNamespaces(
clusterDetails,
namespaces,
);
return Promise.all([result, Promise.all(podMetrics)]);
result.errors.push(...podMetrics.errors);
return [result, podMetrics.responses as PodStatusFetchResponse[]];
}
private getAuthTranslator(provider: string): KubernetesAuthTranslator {
plugins/kubernetes-backend/src/service/KubernetesFetcher.test.ts
+511 -399
View File
File diff suppressed because it is too large Load Diff
plugins/kubernetes-backend/src/service/KubernetesFetcher.ts
+19 -10
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { CoreV1Api, topPods } from '@kubernetes/client-node';
import { topPods } from '@kubernetes/client-node';
import lodash, { Dictionary } from 'lodash';
import { Logger } from 'winston';
import {
@@ -29,13 +29,9 @@ import {
FetchResponse,
KubernetesFetchError,
KubernetesErrorTypes,
PodStatusFetchResponse,
} from '@backstage/plugin-kubernetes-common';
import { KubernetesClientProvider } from './KubernetesClientProvider';
import { PodStatus } from '@kubernetes/client-node/dist/top';
export interface Clients {
core: CoreV1Api;
}
export interface KubernetesClientBasedFetcherOptions {
kubernetesClientProvider: KubernetesClientProvider;
@@ -66,6 +62,8 @@ const statusCodeToErrorType = (statusCode: number): KubernetesErrorTypes => {
return 'BAD_REQUEST';
case 401:
return 'UNAUTHORIZED_ERROR';
case 404:
return 'NOT_FOUND';
case 500:
return 'SYSTEM_ERROR';
default:
@@ -104,10 +102,10 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher {
return Promise.all(fetchResults).then(fetchResultsToResponseWrapper);
}
fetchPodMetricsByNamespace(
fetchPodMetricsByNamespaces(
clusterDetails: ClusterDetails,
namespace: string,
): Promise<PodStatus[]> {
namespaces: Set<string>,
): Promise<FetchResponseWrapper> {
const metricsClient =
this.kubernetesClientProvider.getMetricsClient(clusterDetails);
const coreApi =
@@ -115,7 +113,18 @@ export class KubernetesClientBasedFetcher implements KubernetesFetcher {
clusterDetails,
);
return topPods(coreApi, metricsClient, namespace);
const fetchResults = Array.from(namespaces).map(ns =>
topPods(coreApi, metricsClient, ns)
.then(r => {
return {
type: 'podstatus',
resources: r,
} as PodStatusFetchResponse;
})
.catch(this.captureKubernetesErrorsRethrowOthers.bind(this)),
);
return Promise.all(fetchResults).then(fetchResultsToResponseWrapper);
}
private captureKubernetesErrorsRethrowOthers(e: any): KubernetesFetchError {
plugins/kubernetes-backend/src/types/types.ts
+3 -4
View File
@@ -25,7 +25,6 @@ import type {
KubernetesRequestBody,
ObjectsByEntityResponse,
} from '@backstage/plugin-kubernetes-common';
import { PodStatus } from '@kubernetes/client-node/dist/top';
/**
*
@@ -53,10 +52,10 @@ export interface KubernetesFetcher {
fetchObjectsForService(
params: ObjectFetchParams,
): Promise<FetchResponseWrapper>;
fetchPodMetricsByNamespace(
fetchPodMetricsByNamespaces(
clusterDetails: ClusterDetails,
namespace: string,
): Promise<PodStatus[]>;
namespaces: Set<string>,
): Promise<FetchResponseWrapper>;
}
/**
plugins/kubernetes-common/api-report.md
+12 -1
View File
@@ -5,6 +5,7 @@
```ts
import { Entity } from '@backstage/catalog-model';
import type { JsonObject } from '@backstage/types';
import { PodStatus } from '@kubernetes/client-node';
import { V1ConfigMap } from '@kubernetes/client-node';
import { V1CronJob } from '@kubernetes/client-node';
import { V1DaemonSet } from '@kubernetes/client-node';
@@ -147,7 +148,8 @@ export type FetchResponse =
| IngressesFetchResponse
| CustomResourceFetchResponse
| StatefulSetsFetchResponse
| DaemonSetsFetchResponse;
| DaemonSetsFetchResponse
| PodStatusFetchResponse;
// @public (undocumented)
export interface HorizontalPodAutoscalersFetchResponse {
@@ -177,6 +179,7 @@ export interface JobsFetchResponse {
export type KubernetesErrorTypes =
| 'BAD_REQUEST'
| 'UNAUTHORIZED_ERROR'
| 'NOT_FOUND'
| 'SYSTEM_ERROR'
| 'UNKNOWN_ERROR';
@@ -230,6 +233,14 @@ export interface PodFetchResponse {
type: 'pods';
}
// @public (undocumented)
export interface PodStatusFetchResponse {
// (undocumented)
resources: Array<PodStatus>;
// (undocumented)
type: 'podstatus';
}
// @public (undocumented)
export interface ReplicaSetsFetchResponse {
// (undocumented)
plugins/kubernetes-common/src/types.ts
+10 -1
View File
@@ -16,6 +16,7 @@
import type { JsonObject } from '@backstage/types';
import {
PodStatus,
V1ConfigMap,
V1CronJob,
V1DaemonSet,
@@ -134,7 +135,8 @@ export type FetchResponse =
| IngressesFetchResponse
| CustomResourceFetchResponse
| StatefulSetsFetchResponse
| DaemonSetsFetchResponse;
| DaemonSetsFetchResponse
| PodStatusFetchResponse;
/** @public */
export interface PodFetchResponse {
@@ -214,6 +216,12 @@ export interface DaemonSetsFetchResponse {
resources: Array<V1DaemonSet>;
}
/** @public */
export interface PodStatusFetchResponse {
type: 'podstatus';
resources: Array<PodStatus>;
}
/** @public */
export interface KubernetesFetchError {
errorType: KubernetesErrorTypes;
@@ -225,6 +233,7 @@ export interface KubernetesFetchError {
export type KubernetesErrorTypes =
| 'BAD_REQUEST'
| 'UNAUTHORIZED_ERROR'
| 'NOT_FOUND'
| 'SYSTEM_ERROR'
| 'UNKNOWN_ERROR';