refactor: Refactors the Kubernetes plugin in line with ADR11 (#19446)
* refactor: k8s plugins Signed-off-by: Matthew Clarke <mclarke@spotify.com> * chore: move more files Signed-off-by: Matthew Clarke <mclarke@spotify.com> * chore: fix Signed-off-by: Matthew Clarke <mclarke@spotify.com> * fix: tsc Signed-off-by: Matthew Clarke <mclarke@spotify.com> * chore: changeset Signed-off-by: Matthew Clarke <mclarke@spotify.com> * chore: api report Signed-off-by: Matthew Clarke <mclarke@spotify.com> * fix: catalog Signed-off-by: Matthew Clarke <mclarke@spotify.com> * fix: types Signed-off-by: Matthew Clarke <mclarke@spotify.com> * fix: update lock Signed-off-by: Matthew Clarke <mclarke@spotify.com> --------- Signed-off-by: Matthew Clarke <mclarke@spotify.com>
This commit is contained in:
Matthew Clarke
@@ -5,7 +5,9 @@
|
||||
```ts
|
||||
import { BasicPermission } from '@backstage/plugin-permission-common';
|
||||
import { Entity } from '@backstage/catalog-model';
|
||||
import { FetchResponse as FetchResponse_2 } from '@backstage/plugin-kubernetes-common';
|
||||
import type { JsonObject } from '@backstage/types';
|
||||
import { ObjectsByEntityResponse as ObjectsByEntityResponse_2 } from '@backstage/plugin-kubernetes-common';
|
||||
import { PodStatus } from '@kubernetes/client-node';
|
||||
import { V1ConfigMap } from '@kubernetes/client-node';
|
||||
import { V1CronJob } from '@kubernetes/client-node';
|
||||
@@ -174,6 +176,67 @@ export interface DeploymentFetchResponse {
|
||||
type: 'deployments';
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface DeploymentResources {
|
||||
// (undocumented)
|
||||
deployments: V1Deployment[];
|
||||
// (undocumented)
|
||||
horizontalPodAutoscalers: V1HorizontalPodAutoscaler[];
|
||||
// (undocumented)
|
||||
pods: V1Pod[];
|
||||
// (undocumented)
|
||||
replicaSets: V1ReplicaSet[];
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface DetectedError {
|
||||
// (undocumented)
|
||||
message: string;
|
||||
// (undocumented)
|
||||
occurrenceCount: number;
|
||||
// (undocumented)
|
||||
proposedFix?: ProposedFix;
|
||||
// (undocumented)
|
||||
severity: ErrorSeverity;
|
||||
// (undocumented)
|
||||
sourceRef: ResourceRef;
|
||||
// (undocumented)
|
||||
type: string;
|
||||
}
|
||||
|
||||
// @public
|
||||
export type DetectedErrorsByCluster = Map<string, DetectedError[]>;
|
||||
|
||||
// @public
|
||||
export const detectErrors: (
|
||||
objects: ObjectsByEntityResponse_2,
|
||||
) => DetectedErrorsByCluster;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface DocsSolution extends ProposedFixBase {
|
||||
// (undocumented)
|
||||
docsLink: string;
|
||||
// (undocumented)
|
||||
type: 'docs';
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface ErrorMapper<T> {
|
||||
// (undocumented)
|
||||
detectErrors: (resource: T) => DetectedError[];
|
||||
}
|
||||
|
||||
// @public
|
||||
export type ErrorSeverity = 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface EventsSolution extends ProposedFixBase {
|
||||
// (undocumented)
|
||||
podName: string;
|
||||
// (undocumented)
|
||||
type: 'events';
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type FetchResponse =
|
||||
| PodFetchResponse
|
||||
@@ -191,6 +254,29 @@ export type FetchResponse =
|
||||
| DaemonSetsFetchResponse
|
||||
| PodStatusFetchResponse;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface GroupedResponses extends DeploymentResources {
|
||||
// (undocumented)
|
||||
configMaps: V1ConfigMap[];
|
||||
// (undocumented)
|
||||
cronJobs: V1CronJob[];
|
||||
// (undocumented)
|
||||
customResources: any[];
|
||||
// (undocumented)
|
||||
ingresses: V1Ingress[];
|
||||
// (undocumented)
|
||||
jobs: V1Job[];
|
||||
// (undocumented)
|
||||
services: V1Service[];
|
||||
// (undocumented)
|
||||
statefulsets: V1StatefulSet[];
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export const groupResponses: (
|
||||
fetchResponse: FetchResponse_2[],
|
||||
) => GroupedResponses;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface HorizontalPodAutoscalersFetchResponse {
|
||||
// (undocumented)
|
||||
@@ -251,6 +337,14 @@ export interface LimitRangeFetchResponse {
|
||||
type: 'limitranges';
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface LogSolution extends ProposedFixBase {
|
||||
// (undocumented)
|
||||
container: string;
|
||||
// (undocumented)
|
||||
type: 'logs';
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface ObjectsByEntityResponse {
|
||||
// (undocumented)
|
||||
@@ -273,6 +367,19 @@ export interface PodStatusFetchResponse {
|
||||
type: 'podstatus';
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type ProposedFix = LogSolution | DocsSolution | EventsSolution;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface ProposedFixBase {
|
||||
// (undocumented)
|
||||
actions: string[];
|
||||
// (undocumented)
|
||||
errorType: string;
|
||||
// (undocumented)
|
||||
rootCauseExplanation: string;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface RawFetchError {
|
||||
// (undocumented)
|
||||
@@ -289,6 +396,18 @@ export interface ReplicaSetsFetchResponse {
|
||||
type: 'replicasets';
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface ResourceRef {
|
||||
// (undocumented)
|
||||
apiGroup: string;
|
||||
// (undocumented)
|
||||
kind: string;
|
||||
// (undocumented)
|
||||
name: string;
|
||||
// (undocumented)
|
||||
namespace: string;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface ServiceFetchResponse {
|
||||
// (undocumented)
|
||||
|
||||
@@ -40,11 +40,20 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@backstage/catalog-model": "workspace:^",
|
||||
"@backstage/core-plugin-api": "workspace:^",
|
||||
"@backstage/errors": "workspace:^",
|
||||
"@backstage/plugin-permission-common": "workspace:^",
|
||||
"@kubernetes/client-node": "0.18.1"
|
||||
"@backstage/types": "workspace:^",
|
||||
"@kubernetes/client-node": "0.18.1",
|
||||
"kubernetes-models": "^4.3.1",
|
||||
"lodash": "^4.17.21",
|
||||
"luxon": "^3.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@backstage/cli": "workspace:^"
|
||||
"@backstage/cli": "workspace:^",
|
||||
"@backstage/core-app-api": "workspace:^",
|
||||
"@backstage/test-utils": "workspace:^",
|
||||
"msw": "^1.3.1"
|
||||
},
|
||||
"jest": {
|
||||
"roots": [
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
{
|
||||
"metadata": {
|
||||
"annotations": {
|
||||
"deployment.kubernetes.io/revision": "3",
|
||||
"kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apps/v1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"backstage.io/kubernetes-id\":\"dice-roller\"},\"name\":\"dice-roller-canary\",\"namespace\":\"default\"},\"spec\":{\"replicas\":2,\"selector\":{\"matchLabels\":{\"app\":\"dice-roller-canary\"}},\"template\":{\"metadata\":{\"labels\":{\"app\":\"dice-roller-canary\",\"backstage.io/kubernetes-id\":\"dice-roller\"}},\"spec\":{\"containers\":[{\"image\":\"nginx:1.14.2\",\"name\":\"nginx\",\"ports\":[{\"containerPort\":80}]},{\"image\":\"nginx:1.14.2\",\"name\":\"side-car\",\"ports\":[{\"containerPort\":81}]},{\"image\":\"nginx:1.14.2\",\"name\":\"other-side-car\",\"ports\":[{\"containerPort\":82}]}]}}}}\n"
|
||||
},
|
||||
"creationTimestamp": "2020-09-25T09:02:53.000Z",
|
||||
"generation": 3,
|
||||
"labels": {
|
||||
"backstage.io/kubernetes-id": "dice-roller"
|
||||
},
|
||||
"name": "dice-roller-canary",
|
||||
"namespace": "default",
|
||||
"resourceVersion": "620480",
|
||||
"selfLink": "/apis/apps/v1/namespaces/default/deployments/dice-roller-canary",
|
||||
"uid": "0b6ae80f-999b-40e9-b116-ea925f0ed07b"
|
||||
},
|
||||
"spec": {
|
||||
"progressDeadlineSeconds": 600,
|
||||
"replicas": 2,
|
||||
"revisionHistoryLimit": 10,
|
||||
"selector": {
|
||||
"matchLabels": {
|
||||
"app": "dice-roller-canary"
|
||||
}
|
||||
},
|
||||
"strategy": {
|
||||
"rollingUpdate": {
|
||||
"maxSurge": "25%",
|
||||
"maxUnavailable": "25%"
|
||||
},
|
||||
"type": "RollingUpdate"
|
||||
},
|
||||
"template": {
|
||||
"metadata": {
|
||||
"creationTimestamp": null,
|
||||
"labels": {
|
||||
"app": "dice-roller-canary",
|
||||
"backstage.io/kubernetes-id": "dice-roller"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "nginx:1.14.2",
|
||||
"imagePullPolicy": "IfNotPresent",
|
||||
"name": "nginx",
|
||||
"ports": [
|
||||
{
|
||||
"containerPort": 80,
|
||||
"protocol": "TCP"
|
||||
}
|
||||
],
|
||||
"resources": {},
|
||||
"terminationMessagePath": "/dev/termination-log",
|
||||
"terminationMessagePolicy": "File"
|
||||
},
|
||||
{
|
||||
"image": "nginx:1.14.2",
|
||||
"imagePullPolicy": "IfNotPresent",
|
||||
"name": "side-car",
|
||||
"ports": [
|
||||
{
|
||||
"containerPort": 81,
|
||||
"protocol": "TCP"
|
||||
}
|
||||
],
|
||||
"resources": {},
|
||||
"terminationMessagePath": "/dev/termination-log",
|
||||
"terminationMessagePolicy": "File"
|
||||
},
|
||||
{
|
||||
"image": "nginx:1.14.2",
|
||||
"imagePullPolicy": "IfNotPresent",
|
||||
"name": "other-side-car",
|
||||
"ports": [
|
||||
{
|
||||
"containerPort": 82,
|
||||
"protocol": "TCP"
|
||||
}
|
||||
],
|
||||
"resources": {},
|
||||
"terminationMessagePath": "/dev/termination-log",
|
||||
"terminationMessagePolicy": "File"
|
||||
}
|
||||
],
|
||||
"dnsPolicy": "ClusterFirst",
|
||||
"restartPolicy": "Always",
|
||||
"schedulerName": "default-scheduler",
|
||||
"securityContext": {},
|
||||
"terminationGracePeriodSeconds": 30
|
||||
}
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"conditions": [
|
||||
{
|
||||
"lastTransitionTime": "2020-09-25T09:02:53.000Z",
|
||||
"lastUpdateTime": "2020-09-25T10:34:04.000Z",
|
||||
"message": "ReplicaSet \"dice-roller-canary-7d64cd756c\" has successfully progressed.",
|
||||
"reason": "NewReplicaSetAvailable",
|
||||
"status": "True",
|
||||
"type": "Progressing"
|
||||
},
|
||||
{
|
||||
"lastTransitionTime": "2020-09-25T13:48:06.000Z",
|
||||
"lastUpdateTime": "2020-09-25T13:48:06.000Z",
|
||||
"message": "Deployment does not have minimum availability.",
|
||||
"reason": "MinimumReplicasUnavailable",
|
||||
"status": "False",
|
||||
"type": "Available"
|
||||
}
|
||||
],
|
||||
"observedGeneration": 3,
|
||||
"replicas": 2,
|
||||
"unavailableReplicas": 2,
|
||||
"updatedReplicas": 2
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
{
|
||||
"metadata": {
|
||||
"annotations": {
|
||||
"deployment.kubernetes.io/revision": "2",
|
||||
"kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"apps/v1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"backstage.io/kubernetes-id\":\"dice-roller\"},\"name\":\"dice-roller\",\"namespace\":\"default\"},\"spec\":{\"replicas\":10,\"selector\":{\"matchLabels\":{\"app\":\"dice-roller\"}},\"template\":{\"metadata\":{\"labels\":{\"app\":\"dice-roller\",\"backstage.io/kubernetes-id\":\"dice-roller\"}},\"spec\":{\"containers\":[{\"image\":\"nginx:1.14.2\",\"name\":\"nginx\",\"ports\":[{\"containerPort\":80}]}]}}}}\n"
|
||||
},
|
||||
"creationTimestamp": "2020-09-23T12:00:55.000Z",
|
||||
"generation": 3,
|
||||
"labels": {
|
||||
"backstage.io/kubernetes-id": "dice-roller"
|
||||
},
|
||||
"name": "dice-roller",
|
||||
"namespace": "default",
|
||||
"resourceVersion": "593230",
|
||||
"selfLink": "/apis/apps/v1/namespaces/default/deployments/dice-roller",
|
||||
"uid": "7551e949-42d1-4061-83c5-9da107186e47"
|
||||
},
|
||||
"spec": {
|
||||
"progressDeadlineSeconds": 600,
|
||||
"replicas": 10,
|
||||
"revisionHistoryLimit": 10,
|
||||
"selector": {
|
||||
"matchLabels": {
|
||||
"app": "dice-roller"
|
||||
}
|
||||
},
|
||||
"strategy": {
|
||||
"rollingUpdate": {
|
||||
"maxSurge": "25%",
|
||||
"maxUnavailable": "25%"
|
||||
},
|
||||
"type": "RollingUpdate"
|
||||
},
|
||||
"template": {
|
||||
"metadata": {
|
||||
"creationTimestamp": null,
|
||||
"labels": {
|
||||
"app": "dice-roller",
|
||||
"backstage.io/kubernetes-id": "dice-roller"
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "nginx:1.14.2",
|
||||
"imagePullPolicy": "IfNotPresent",
|
||||
"name": "nginx",
|
||||
"ports": [
|
||||
{
|
||||
"containerPort": 80,
|
||||
"protocol": "TCP"
|
||||
}
|
||||
],
|
||||
"resources": {},
|
||||
"terminationMessagePath": "/dev/termination-log",
|
||||
"terminationMessagePolicy": "File"
|
||||
}
|
||||
],
|
||||
"dnsPolicy": "ClusterFirst",
|
||||
"restartPolicy": "Always",
|
||||
"schedulerName": "default-scheduler",
|
||||
"securityContext": {},
|
||||
"terminationGracePeriodSeconds": 30
|
||||
}
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"availableReplicas": 10,
|
||||
"conditions": [
|
||||
{
|
||||
"lastTransitionTime": "2020-09-23T12:00:55.000Z",
|
||||
"lastUpdateTime": "2020-09-24T11:39:28.000Z",
|
||||
"message": "ReplicaSet \"dice-roller-6c8646bfd\" has successfully progressed.",
|
||||
"reason": "NewReplicaSetAvailable",
|
||||
"status": "True",
|
||||
"type": "Progressing"
|
||||
},
|
||||
{
|
||||
"lastTransitionTime": "2020-09-25T09:58:55.000Z",
|
||||
"lastUpdateTime": "2020-09-25T09:58:55.000Z",
|
||||
"message": "Deployment has minimum availability.",
|
||||
"reason": "MinimumReplicasAvailable",
|
||||
"status": "True",
|
||||
"type": "Available"
|
||||
}
|
||||
],
|
||||
"observedGeneration": 3,
|
||||
"readyReplicas": 10,
|
||||
"replicas": 10,
|
||||
"updatedReplicas": 10
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"metadata": {
|
||||
"annotations": {
|
||||
"autoscaling.alpha.kubernetes.io/conditions": "[{\"type\":\"AbleToScale\",\"status\":\"True\",\"lastTransitionTime\":\"2020-09-28T13:28:15Z\",\"reason\":\"SucceededGetScale\",\"message\":\"the HPA controller was able to get the target's current scale\"},{\"type\":\"ScalingActive\",\"status\":\"False\",\"lastTransitionTime\":\"2020-09-28T13:28:15Z\",\"reason\":\"FailedGetResourceMetric\",\"message\":\"the HPA was unable to compute the replica count: unable to get metrics for resource cpu: unable to fetch metrics from resource metrics API: the server could not find the requested resource (get pods.metrics.k8s.io)\"}]",
|
||||
"kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"autoscaling/v1\",\"kind\":\"HorizontalPodAutoscaler\",\"metadata\":{\"annotations\":{},\"labels\":{\"backstage.io/kubernetes-id\":\"dice-roller\"},\"name\":\"dice-roller\",\"namespace\":\"default\"},\"spec\":{\"maxReplicas\":15,\"minReplicas\":10,\"scaleTargetRef\":{\"apiVersion\":\"apps/v1\",\"kind\":\"Deployment\",\"name\":\"dice-roller\"},\"targetCPUUtilizationPercentage\":50}}\n"
|
||||
},
|
||||
"creationTimestamp": "2020-09-28T13:28:00.000Z",
|
||||
"labels": {
|
||||
"backstage.io/kubernetes-id": "dice-roller"
|
||||
},
|
||||
"name": "dice-roller",
|
||||
"namespace": "default",
|
||||
"resourceVersion": "698957",
|
||||
"selfLink": "/apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers/dice-roller",
|
||||
"uid": "a70c8a90-5605-4d7d-adea-05cfb8d9d446"
|
||||
},
|
||||
"spec": {
|
||||
"maxReplicas": 15,
|
||||
"minReplicas": 10,
|
||||
"scaleTargetRef": {
|
||||
"apiVersion": "apps/v1",
|
||||
"kind": "Deployment",
|
||||
"name": "dice-roller"
|
||||
},
|
||||
"targetCPUUtilizationPercentage": 50
|
||||
},
|
||||
"status": {
|
||||
"currentReplicas": 13,
|
||||
"desiredReplicas": 14,
|
||||
"currentCPUUtilizationPercentage": 30
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"metadata": {
|
||||
"annotations": {
|
||||
"autoscaling.alpha.kubernetes.io/conditions": "[{\"type\":\"AbleToScale\",\"status\":\"True\",\"lastTransitionTime\":\"2020-09-28T13:28:15Z\",\"reason\":\"SucceededGetScale\",\"message\":\"the HPA controller was able to get the target's current scale\"},{\"type\":\"ScalingActive\",\"status\":\"False\",\"lastTransitionTime\":\"2020-09-28T13:28:15Z\",\"reason\":\"FailedGetResourceMetric\",\"message\":\"the HPA was unable to compute the replica count: unable to get metrics for resource cpu: unable to fetch metrics from resource metrics API: the server could not find the requested resource (get pods.metrics.k8s.io)\"}]",
|
||||
"kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"autoscaling/v1\",\"kind\":\"HorizontalPodAutoscaler\",\"metadata\":{\"annotations\":{},\"labels\":{\"backstage.io/kubernetes-id\":\"dice-roller\"},\"name\":\"dice-roller\",\"namespace\":\"default\"},\"spec\":{\"maxReplicas\":15,\"minReplicas\":10,\"scaleTargetRef\":{\"apiVersion\":\"apps/v1\",\"kind\":\"Deployment\",\"name\":\"dice-roller\"},\"targetCPUUtilizationPercentage\":50}}\n"
|
||||
},
|
||||
"creationTimestamp": "2020-09-28T13:28:00.000Z",
|
||||
"labels": {
|
||||
"backstage.io/kubernetes-id": "dice-roller"
|
||||
},
|
||||
"name": "dice-roller",
|
||||
"namespace": "default",
|
||||
"resourceVersion": "698957",
|
||||
"selfLink": "/apis/autoscaling/v1/namespaces/default/horizontalpodautoscalers/dice-roller",
|
||||
"uid": "a70c8a90-5605-4d7d-adea-05cfb8d9d446"
|
||||
},
|
||||
"spec": {
|
||||
"maxReplicas": 10,
|
||||
"minReplicas": 5,
|
||||
"scaleTargetRef": {
|
||||
"apiVersion": "apps/v1",
|
||||
"kind": "Deployment",
|
||||
"name": "dice-roller"
|
||||
},
|
||||
"targetCPUUtilizationPercentage": 70
|
||||
},
|
||||
"status": {
|
||||
"currentReplicas": 10,
|
||||
"desiredReplicas": 10,
|
||||
"currentCPUUtilizationPercentage": 100
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
{
|
||||
"metadata": {
|
||||
"creationTimestamp": "2020-09-25T10:34:01.000Z",
|
||||
"generateName": "dice-roller-canary-7d64cd756c-",
|
||||
"labels": {
|
||||
"app": "dice-roller-canary",
|
||||
"backstage.io/kubernetes-id": "dice-roller",
|
||||
"pod-template-hash": "7d64cd756c"
|
||||
},
|
||||
"name": "dice-roller-canary-7d64cd756c-55rfq",
|
||||
"namespace": "default",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"apiVersion": "apps/v1",
|
||||
"blockOwnerDeletion": true,
|
||||
"controller": true,
|
||||
"kind": "ReplicaSet",
|
||||
"name": "dice-roller-canary-7d64cd756c",
|
||||
"uid": "9208395b-a9a7-4e46-b881-6a189f7fbdb0"
|
||||
}
|
||||
],
|
||||
"resourceVersion": "620452",
|
||||
"selfLink": "/api/v1/namespaces/default/pods/dice-roller-canary-7d64cd756c-55rfq",
|
||||
"uid": "65ad28e3-5d51-4b4b-9bf8-4cb069803034"
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "nginx:1.14.2",
|
||||
"imagePullPolicy": "IfNotPresent",
|
||||
"name": "nginx",
|
||||
"ports": [
|
||||
{
|
||||
"containerPort": 80,
|
||||
"protocol": "TCP"
|
||||
}
|
||||
],
|
||||
"resources": {},
|
||||
"terminationMessagePath": "/dev/termination-log",
|
||||
"terminationMessagePolicy": "File",
|
||||
"volumeMounts": [
|
||||
{
|
||||
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
|
||||
"name": "default-token-5gctn",
|
||||
"readOnly": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"image": "nginx:1.14.2",
|
||||
"imagePullPolicy": "IfNotPresent",
|
||||
"name": "side-car",
|
||||
"ports": [
|
||||
{
|
||||
"containerPort": 81,
|
||||
"protocol": "TCP"
|
||||
}
|
||||
],
|
||||
"resources": {},
|
||||
"terminationMessagePath": "/dev/termination-log",
|
||||
"terminationMessagePolicy": "File",
|
||||
"volumeMounts": [
|
||||
{
|
||||
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
|
||||
"name": "default-token-5gctn",
|
||||
"readOnly": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"image": "nginx:1.14.2",
|
||||
"imagePullPolicy": "IfNotPresent",
|
||||
"name": "other-side-car",
|
||||
"ports": [
|
||||
{
|
||||
"containerPort": 82,
|
||||
"protocol": "TCP"
|
||||
}
|
||||
],
|
||||
"resources": {},
|
||||
"terminationMessagePath": "/dev/termination-log",
|
||||
"terminationMessagePolicy": "File",
|
||||
"volumeMounts": [
|
||||
{
|
||||
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
|
||||
"name": "default-token-5gctn",
|
||||
"readOnly": true
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"dnsPolicy": "ClusterFirst",
|
||||
"enableServiceLinks": true,
|
||||
"nodeName": "minikube",
|
||||
"priority": 0,
|
||||
"restartPolicy": "Always",
|
||||
"schedulerName": "default-scheduler",
|
||||
"securityContext": {},
|
||||
"serviceAccount": "default",
|
||||
"serviceAccountName": "default",
|
||||
"terminationGracePeriodSeconds": 30,
|
||||
"tolerations": [
|
||||
{
|
||||
"effect": "NoExecute",
|
||||
"key": "node.kubernetes.io/not-ready",
|
||||
"operator": "Exists",
|
||||
"tolerationSeconds": 300
|
||||
},
|
||||
{
|
||||
"effect": "NoExecute",
|
||||
"key": "node.kubernetes.io/unreachable",
|
||||
"operator": "Exists",
|
||||
"tolerationSeconds": 300
|
||||
}
|
||||
],
|
||||
"volumes": [
|
||||
{
|
||||
"name": "default-token-5gctn",
|
||||
"secret": {
|
||||
"defaultMode": 420,
|
||||
"secretName": "default-token-5gctn"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"conditions": [
|
||||
{
|
||||
"lastProbeTime": null,
|
||||
"lastTransitionTime": "2020-09-25T10:34:01.000Z",
|
||||
"status": "True",
|
||||
"type": "Initialized"
|
||||
},
|
||||
{
|
||||
"lastProbeTime": null,
|
||||
"lastTransitionTime": "2020-09-25T14:18:53.000Z",
|
||||
"message": "containers with unready status: [side-car other-side-car]",
|
||||
"reason": "ContainersNotReady",
|
||||
"status": "False",
|
||||
"type": "Ready"
|
||||
},
|
||||
{
|
||||
"lastProbeTime": null,
|
||||
"lastTransitionTime": "2020-09-25T14:18:53.000Z",
|
||||
"message": "containers with unready status: [side-car other-side-car]",
|
||||
"reason": "ContainersNotReady",
|
||||
"status": "False",
|
||||
"type": "ContainersReady"
|
||||
},
|
||||
{
|
||||
"lastProbeTime": null,
|
||||
"lastTransitionTime": "2020-09-25T10:34:01.000Z",
|
||||
"status": "True",
|
||||
"type": "PodScheduled"
|
||||
}
|
||||
],
|
||||
"containerStatuses": [
|
||||
{
|
||||
"containerID": "docker://6ce15178d114a85f3d2e832de45c3355ab5b71ed5f4d4d225ee1c83bf07f69d9",
|
||||
"image": "nginx:1.14.2",
|
||||
"imageID": "docker-pullable://nginx@sha256:f7988fb6c02e0ce69257d9bd9cf37ae20a60f1df7563c3a2a6abe24160306b8d",
|
||||
"lastState": {},
|
||||
"name": "nginx",
|
||||
"ready": true,
|
||||
"restartCount": 0,
|
||||
"started": true,
|
||||
"state": {
|
||||
"running": {
|
||||
"startedAt": "2020-09-25T10:34:01.000Z"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"containerID": "docker://b3ce93d7f90bfe22558c61d2505b8473580574accdebb5fa4e51c0729c3511f4",
|
||||
"image": "nginx:1.14.2",
|
||||
"imageID": "docker-pullable://nginx@sha256:f7988fb6c02e0ce69257d9bd9cf37ae20a60f1df7563c3a2a6abe24160306b8d",
|
||||
"lastState": {
|
||||
"terminated": {
|
||||
"containerID": "docker://b3ce93d7f90bfe22558c61d2505b8473580574accdebb5fa4e51c0729c3511f4",
|
||||
"exitCode": 1,
|
||||
"finishedAt": "2020-09-25T14:18:52.000Z",
|
||||
"reason": "Error",
|
||||
"startedAt": "2020-09-25T14:18:50.000Z"
|
||||
}
|
||||
},
|
||||
"name": "other-side-car",
|
||||
"ready": false,
|
||||
"restartCount": 123,
|
||||
"started": false,
|
||||
"state": {
|
||||
"waiting": {
|
||||
"message": "back-off 5m0s restarting failed container=other-side-car pod=dice-roller-canary-7d64cd756c-55rfq_default(65ad28e3-5d51-4b4b-9bf8-4cb069803034)",
|
||||
"reason": "CrashLoopBackOff"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"containerID": "docker://b7f0e65a2b8ab48c5f234616cfe8286aa96b55c3ef09c5cfbc4cdbe67a96f8cb",
|
||||
"image": "nginx:1.14.2",
|
||||
"imageID": "docker-pullable://nginx@sha256:f7988fb6c02e0ce69257d9bd9cf37ae20a60f1df7563c3a2a6abe24160306b8d",
|
||||
"lastState": {
|
||||
"terminated": {
|
||||
"containerID": "docker://b7f0e65a2b8ab48c5f234616cfe8286aa96b55c3ef09c5cfbc4cdbe67a96f8cb",
|
||||
"exitCode": 1,
|
||||
"finishedAt": "2020-09-25T14:18:52.000Z",
|
||||
"reason": "Error",
|
||||
"startedAt": "2020-09-25T14:18:50.000Z"
|
||||
}
|
||||
},
|
||||
"name": "side-car",
|
||||
"ready": false,
|
||||
"restartCount": 38,
|
||||
"started": false,
|
||||
"state": {
|
||||
"waiting": {
|
||||
"message": "back-off 5m0s restarting failed container=side-car pod=dice-roller-canary-7d64cd756c-55rfq_default(65ad28e3-5d51-4b4b-9bf8-4cb069803034)",
|
||||
"reason": "CrashLoopBackOff"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"hostIP": "192.168.64.2",
|
||||
"phase": "Running",
|
||||
"podIP": "172.17.0.16",
|
||||
"podIPs": [
|
||||
{
|
||||
"ip": "172.17.0.16"
|
||||
}
|
||||
],
|
||||
"qosClass": "BestEffort",
|
||||
"startTime": "2020-09-25T10:34:01.000Z"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
{
|
||||
"apiVersion": "v1",
|
||||
"kind": "Pod",
|
||||
"metadata": {
|
||||
"creationTimestamp": "2021-01-06T14:33:54Z",
|
||||
"generateName": "dice-roller-bad-cm-855bf85464-",
|
||||
"labels": {
|
||||
"app": "dice-roller-bad-cm",
|
||||
"backstage.io/kubernetes-id": "dice-roller",
|
||||
"pod-template-hash": "855bf85464"
|
||||
},
|
||||
"name": "dice-roller-bad-cm-855bf85464-mg6xb",
|
||||
"namespace": "default",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"apiVersion": "apps/v1",
|
||||
"blockOwnerDeletion": true,
|
||||
"controller": true,
|
||||
"kind": "ReplicaSet",
|
||||
"name": "dice-roller-bad-cm-855bf85464",
|
||||
"uid": "9bc2a418-60eb-4dfc-9748-78cf49ea9863"
|
||||
}
|
||||
],
|
||||
"resourceVersion": "2457755284",
|
||||
"selfLink": "/api/v1/namespaces/default/pods/dice-roller-bad-cm-855bf85464-mg6xb",
|
||||
"uid": "5f257d3c-a16d-4ef1-9dc9-d11e321a640a"
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"env": [
|
||||
{
|
||||
"name": "SOME_ENV_VAR",
|
||||
"valueFrom": {
|
||||
"configMapKeyRef": {
|
||||
"key": "some-key",
|
||||
"name": "some-cm"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"image": "nginx:1.14.2",
|
||||
"imagePullPolicy": "IfNotPresent",
|
||||
"name": "nginx",
|
||||
"ports": [
|
||||
{
|
||||
"containerPort": 80,
|
||||
"protocol": "TCP"
|
||||
}
|
||||
],
|
||||
"resources": {
|
||||
"limits": {
|
||||
"cpu": "500m",
|
||||
"memory": "128Mi"
|
||||
},
|
||||
"requests": {
|
||||
"cpu": "50m",
|
||||
"memory": "64Mi"
|
||||
}
|
||||
},
|
||||
"securityContext": {
|
||||
"allowPrivilegeEscalation": false
|
||||
},
|
||||
"terminationMessagePath": "/dev/termination-log",
|
||||
"terminationMessagePolicy": "File",
|
||||
"volumeMounts": [
|
||||
{
|
||||
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
|
||||
"name": "default-token-dbkkb",
|
||||
"readOnly": true
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"dnsPolicy": "ClusterFirst",
|
||||
"enableServiceLinks": true,
|
||||
"nodeName": "node1",
|
||||
"priority": 0,
|
||||
"restartPolicy": "Always",
|
||||
"schedulerName": "default-scheduler",
|
||||
"securityContext": {},
|
||||
"serviceAccount": "default",
|
||||
"serviceAccountName": "default",
|
||||
"terminationGracePeriodSeconds": 30,
|
||||
"tolerations": [
|
||||
{
|
||||
"effect": "NoExecute",
|
||||
"key": "node.kubernetes.io/not-ready",
|
||||
"operator": "Exists",
|
||||
"tolerationSeconds": 300
|
||||
},
|
||||
{
|
||||
"effect": "NoExecute",
|
||||
"key": "node.kubernetes.io/unreachable",
|
||||
"operator": "Exists",
|
||||
"tolerationSeconds": 300
|
||||
}
|
||||
],
|
||||
"volumes": [
|
||||
{
|
||||
"name": "default-token-dbkkb",
|
||||
"secret": {
|
||||
"defaultMode": 420,
|
||||
"secretName": "default-token-dbkkb"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"conditions": [
|
||||
{
|
||||
"lastProbeTime": null,
|
||||
"lastTransitionTime": "2021-01-06T14:33:54Z",
|
||||
"status": "True",
|
||||
"type": "Initialized"
|
||||
},
|
||||
{
|
||||
"lastProbeTime": null,
|
||||
"lastTransitionTime": "2021-01-06T14:33:54Z",
|
||||
"message": "containers with unready status: [nginx]",
|
||||
"reason": "ContainersNotReady",
|
||||
"status": "False",
|
||||
"type": "Ready"
|
||||
},
|
||||
{
|
||||
"lastProbeTime": null,
|
||||
"lastTransitionTime": "2021-01-06T14:33:54Z",
|
||||
"message": "containers with unready status: [nginx]",
|
||||
"reason": "ContainersNotReady",
|
||||
"status": "False",
|
||||
"type": "ContainersReady"
|
||||
},
|
||||
{
|
||||
"lastProbeTime": null,
|
||||
"lastTransitionTime": "2021-01-06T14:33:54Z",
|
||||
"status": "True",
|
||||
"type": "PodScheduled"
|
||||
}
|
||||
],
|
||||
"containerStatuses": [
|
||||
{
|
||||
"image": "nginx:1.14.2",
|
||||
"imageID": "",
|
||||
"lastState": {},
|
||||
"name": "nginx",
|
||||
"ready": false,
|
||||
"restartCount": 0,
|
||||
"started": false,
|
||||
"state": {
|
||||
"waiting": {
|
||||
"message": "configmap \"some-cm\" not found",
|
||||
"reason": "CreateContainerConfigError"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"hostIP": "10.1.33.73",
|
||||
"phase": "Pending",
|
||||
"podIP": "10.1.115.15",
|
||||
"podIPs": [
|
||||
{
|
||||
"ip": "10.1.115.15"
|
||||
}
|
||||
],
|
||||
"qosClass": "Burstable",
|
||||
"startTime": "2021-01-06T14:33:54Z"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
{
|
||||
"metadata": {
|
||||
"creationTimestamp": "2020-09-25T09:58:50.000Z",
|
||||
"generateName": "dice-roller-6c8646bfd-",
|
||||
"labels": {
|
||||
"app": "dice-roller",
|
||||
"backstage.io/kubernetes-id": "dice-roller",
|
||||
"pod-template-hash": "6c8646bfd"
|
||||
},
|
||||
"name": "dice-roller-6c8646bfd-2m5hv",
|
||||
"namespace": "default",
|
||||
"ownerReferences": [
|
||||
{
|
||||
"apiVersion": "apps/v1",
|
||||
"blockOwnerDeletion": true,
|
||||
"controller": true,
|
||||
"kind": "ReplicaSet",
|
||||
"name": "dice-roller-6c8646bfd",
|
||||
"uid": "5126c354-4310-4e23-a9e4-c9b87cb69792"
|
||||
}
|
||||
],
|
||||
"resourceVersion": "593216",
|
||||
"selfLink": "/api/v1/namespaces/default/pods/dice-roller-6c8646bfd-2m5hv",
|
||||
"uid": "aadb71c0-36fa-43e3-b38a-162f134d4359"
|
||||
},
|
||||
"spec": {
|
||||
"containers": [
|
||||
{
|
||||
"image": "nginx:1.14.2",
|
||||
"imagePullPolicy": "IfNotPresent",
|
||||
"name": "nginx",
|
||||
"ports": [
|
||||
{
|
||||
"containerPort": 80,
|
||||
"protocol": "TCP"
|
||||
}
|
||||
],
|
||||
"resources": {},
|
||||
"terminationMessagePath": "/dev/termination-log",
|
||||
"terminationMessagePolicy": "File",
|
||||
"volumeMounts": [
|
||||
{
|
||||
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount",
|
||||
"name": "default-token-5gctn",
|
||||
"readOnly": true
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"dnsPolicy": "ClusterFirst",
|
||||
"enableServiceLinks": true,
|
||||
"nodeName": "minikube",
|
||||
"priority": 0,
|
||||
"restartPolicy": "Always",
|
||||
"schedulerName": "default-scheduler",
|
||||
"securityContext": {},
|
||||
"serviceAccount": "default",
|
||||
"serviceAccountName": "default",
|
||||
"terminationGracePeriodSeconds": 30,
|
||||
"tolerations": [
|
||||
{
|
||||
"effect": "NoExecute",
|
||||
"key": "node.kubernetes.io/not-ready",
|
||||
"operator": "Exists",
|
||||
"tolerationSeconds": 300
|
||||
},
|
||||
{
|
||||
"effect": "NoExecute",
|
||||
"key": "node.kubernetes.io/unreachable",
|
||||
"operator": "Exists",
|
||||
"tolerationSeconds": 300
|
||||
}
|
||||
],
|
||||
"volumes": [
|
||||
{
|
||||
"name": "default-token-5gctn",
|
||||
"secret": {
|
||||
"defaultMode": 420,
|
||||
"secretName": "default-token-5gctn"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": {
|
||||
"conditions": [
|
||||
{
|
||||
"lastProbeTime": null,
|
||||
"lastTransitionTime": "2020-09-25T09:58:50.000Z",
|
||||
"status": "True",
|
||||
"type": "Initialized"
|
||||
},
|
||||
{
|
||||
"lastProbeTime": null,
|
||||
"lastTransitionTime": "2020-09-25T09:58:55.000Z",
|
||||
"status": "True",
|
||||
"type": "Ready"
|
||||
},
|
||||
{
|
||||
"lastProbeTime": null,
|
||||
"lastTransitionTime": "2020-09-25T09:58:55.000Z",
|
||||
"status": "True",
|
||||
"type": "ContainersReady"
|
||||
},
|
||||
{
|
||||
"lastProbeTime": null,
|
||||
"lastTransitionTime": "2020-09-25T09:58:50.000Z",
|
||||
"status": "True",
|
||||
"type": "PodScheduled"
|
||||
}
|
||||
],
|
||||
"containerStatuses": [
|
||||
{
|
||||
"containerID": "docker://aa4489297c34c48bb33c18474a8d2b33854a82ed42155680b259f635f556ce70",
|
||||
"image": "nginx:1.14.2",
|
||||
"imageID": "docker-pullable://nginx@sha256:f7988fb6c02e0ce69257d9bd9cf37ae20a60f1df7563c3a2a6abe24160306b8d",
|
||||
"lastState": {},
|
||||
"name": "nginx",
|
||||
"ready": true,
|
||||
"restartCount": 0,
|
||||
"started": true,
|
||||
"state": {
|
||||
"running": {
|
||||
"startedAt": "2020-09-25T09:58:53.000Z"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"hostIP": "192.168.64.2",
|
||||
"phase": "Running",
|
||||
"podIP": "172.17.0.11",
|
||||
"podIPs": [
|
||||
{
|
||||
"ip": "172.17.0.11"
|
||||
}
|
||||
],
|
||||
"qosClass": "BestEffort",
|
||||
"startTime": "2020-09-25T09:58:50.000Z"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { DetectedError, ErrorMapper } from './types';
|
||||
|
||||
// Run through the each error mapper for each object
|
||||
// returning a deduplicated (mostly) result
|
||||
export const detectErrorsInObjects = <T>(
|
||||
objects: T[],
|
||||
errorMappers: ErrorMapper<T>[],
|
||||
): DetectedError[] => {
|
||||
return objects.flatMap(o => {
|
||||
return errorMappers.flatMap(em => em.detectErrors(o));
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { DetectedError, ErrorMapper } from './types';
|
||||
import { Deployment } from 'kubernetes-models/apps/v1';
|
||||
import { detectErrorsInObjects } from './common';
|
||||
|
||||
const deploymentErrorMappers: ErrorMapper<Deployment>[] = [
|
||||
{
|
||||
detectErrors: deployment => {
|
||||
return (deployment.status?.conditions ?? [])
|
||||
.filter(c => c.status === 'False')
|
||||
.filter(c => c.message !== undefined)
|
||||
.map(c => ({
|
||||
type: 'condition-message-present',
|
||||
message: c.message ?? '',
|
||||
severity: 6,
|
||||
sourceRef: {
|
||||
name: deployment.metadata?.name ?? 'unknown hpa',
|
||||
namespace: deployment.metadata?.namespace ?? 'unknown namespace',
|
||||
kind: 'Deployment',
|
||||
apiGroup: 'apps/v1',
|
||||
},
|
||||
occurrenceCount: 1,
|
||||
}));
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
export const detectErrorsInDeployments = (
|
||||
deployments: Deployment[],
|
||||
): DetectedError[] =>
|
||||
detectErrorsInObjects(deployments, deploymentErrorMappers);
|
||||
@@ -0,0 +1,494 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
V1Pod,
|
||||
V1Deployment,
|
||||
V1HorizontalPodAutoscaler,
|
||||
} from '@kubernetes/client-node';
|
||||
import { detectErrors } from './error-detection';
|
||||
import * as healthyPod from './__fixtures__/pod.json';
|
||||
import * as podMissingCm from './__fixtures__/pod-missing-cm.json';
|
||||
import * as crashingPod from './__fixtures__/pod-crashing.json';
|
||||
import * as healthyDeploy from './__fixtures__/deploy-healthy.json';
|
||||
import * as failingDeploy from './__fixtures__/deploy-bad.json';
|
||||
import * as healthyHpa from './__fixtures__/hpa-healthy.json';
|
||||
import * as maxedOutHpa from './__fixtures__/hpa-maxed-out.json';
|
||||
import {
|
||||
FetchResponse,
|
||||
ObjectsByEntityResponse,
|
||||
} from '@backstage/plugin-kubernetes-common';
|
||||
|
||||
const CLUSTER_NAME = 'cluster-a';
|
||||
|
||||
const oneItem = (value: FetchResponse): ObjectsByEntityResponse => {
|
||||
return {
|
||||
items: [
|
||||
{
|
||||
cluster: { name: CLUSTER_NAME },
|
||||
errors: [],
|
||||
podMetrics: [],
|
||||
resources: [value],
|
||||
},
|
||||
],
|
||||
};
|
||||
};
|
||||
|
||||
const onePod = (pod: V1Pod): ObjectsByEntityResponse => {
|
||||
return oneItem({
|
||||
type: 'pods',
|
||||
resources: [pod],
|
||||
});
|
||||
};
|
||||
|
||||
const oneDeployment = (deployment: V1Deployment): ObjectsByEntityResponse => {
|
||||
return oneItem({
|
||||
type: 'deployments',
|
||||
resources: [deployment],
|
||||
});
|
||||
};
|
||||
|
||||
const oneHpa = (hpa: V1HorizontalPodAutoscaler): ObjectsByEntityResponse => {
|
||||
return oneItem({
|
||||
type: 'horizontalpodautoscalers',
|
||||
resources: [hpa],
|
||||
});
|
||||
};
|
||||
|
||||
describe('detectErrors', () => {
|
||||
it('should return errors from different clusters', () => {
|
||||
const result = detectErrors({
|
||||
items: [
|
||||
{
|
||||
cluster: { name: 'cluster-a' },
|
||||
errors: [],
|
||||
podMetrics: [],
|
||||
resources: [
|
||||
{
|
||||
type: 'pods',
|
||||
resources: [crashingPod as any],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
cluster: { name: 'cluster-b' },
|
||||
errors: [],
|
||||
podMetrics: [],
|
||||
resources: [
|
||||
{
|
||||
type: 'horizontalpodautoscalers',
|
||||
resources: [maxedOutHpa as any],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
cluster: { name: 'cluster-c' },
|
||||
errors: [],
|
||||
podMetrics: [],
|
||||
resources: [
|
||||
{
|
||||
type: 'deployments',
|
||||
resources: [healthyDeploy as any],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.size).toBe(3);
|
||||
|
||||
const errorsFromClusterA = result.get('cluster-a');
|
||||
const errorsFromClusterB = result.get('cluster-b');
|
||||
const errorsFromClusterC = result.get('cluster-c');
|
||||
|
||||
expect(errorsFromClusterA).toBeDefined();
|
||||
expect(errorsFromClusterA).toHaveLength(4);
|
||||
|
||||
expect(errorsFromClusterB).toBeDefined();
|
||||
expect(errorsFromClusterB).toHaveLength(1);
|
||||
|
||||
expect(errorsFromClusterC).toBeDefined();
|
||||
expect(errorsFromClusterC).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should detect no errors in healthy pod', () => {
|
||||
const result = detectErrors(onePod(healthyPod as any));
|
||||
|
||||
expect(result.size).toBe(1);
|
||||
|
||||
const errors = result.get(CLUSTER_NAME);
|
||||
|
||||
expect(errors).toBeDefined();
|
||||
expect(errors).toHaveLength(0);
|
||||
});
|
||||
it('should detect errors in crashing pod', () => {
|
||||
const result = detectErrors(onePod(crashingPod as any));
|
||||
|
||||
expect(result.size).toBe(1);
|
||||
|
||||
const errors = result.get(CLUSTER_NAME);
|
||||
|
||||
expect(errors).toBeDefined();
|
||||
expect(errors).toHaveLength(4);
|
||||
|
||||
const [err1, err2, err3, err4] = errors ?? [];
|
||||
|
||||
expect(err1).toStrictEqual({
|
||||
sourceRef: {
|
||||
apiGroup: 'v1',
|
||||
kind: 'Pod',
|
||||
name: 'dice-roller-canary-7d64cd756c-55rfq',
|
||||
namespace: 'default',
|
||||
},
|
||||
message:
|
||||
'back-off 5m0s restarting failed container=other-side-car pod=dice-roller-canary-7d64cd756c-55rfq_default(65ad28e3-5d51-4b4b-9bf8-4cb069803034)',
|
||||
severity: 4,
|
||||
proposedFix: {
|
||||
container: 'other-side-car',
|
||||
errorType: 'CrashLoopBackOff',
|
||||
actions: ['Check the crash logs for stacktraces'],
|
||||
rootCauseExplanation:
|
||||
'The container other-side-car has crashed many times, it will be exponentially restarted until it stops crashing',
|
||||
type: 'logs',
|
||||
},
|
||||
occurrenceCount: 1,
|
||||
type: 'container-waiting',
|
||||
});
|
||||
|
||||
expect(err2).toStrictEqual({
|
||||
sourceRef: {
|
||||
apiGroup: 'v1',
|
||||
kind: 'Pod',
|
||||
name: 'dice-roller-canary-7d64cd756c-55rfq',
|
||||
namespace: 'default',
|
||||
},
|
||||
proposedFix: {
|
||||
container: 'other-side-car',
|
||||
errorType: 'CrashLoopBackOff',
|
||||
actions: ['Check the crash logs for stacktraces'],
|
||||
rootCauseExplanation:
|
||||
'The container other-side-car has crashed many times, it will be exponentially restarted until it stops crashing',
|
||||
type: 'logs',
|
||||
},
|
||||
message:
|
||||
'back-off 5m0s restarting failed container=side-car pod=dice-roller-canary-7d64cd756c-55rfq_default(65ad28e3-5d51-4b4b-9bf8-4cb069803034)',
|
||||
severity: 4,
|
||||
occurrenceCount: 1,
|
||||
type: 'container-waiting',
|
||||
});
|
||||
|
||||
expect(err3).toStrictEqual({
|
||||
sourceRef: {
|
||||
apiGroup: 'v1',
|
||||
kind: 'Pod',
|
||||
name: 'dice-roller-canary-7d64cd756c-55rfq',
|
||||
namespace: 'default',
|
||||
},
|
||||
message: 'container=other-side-car restarted 123 times',
|
||||
severity: 4,
|
||||
occurrenceCount: 123,
|
||||
proposedFix: {
|
||||
container: 'other-side-car',
|
||||
errorType: 'Error',
|
||||
actions: ['Check the crash logs for stacktraces'],
|
||||
rootCauseExplanation:
|
||||
'This container has exited with a non-zero exit code (1)',
|
||||
type: 'logs',
|
||||
},
|
||||
type: 'containers-restarting',
|
||||
});
|
||||
|
||||
expect(err4).toStrictEqual({
|
||||
sourceRef: {
|
||||
apiGroup: 'v1',
|
||||
kind: 'Pod',
|
||||
name: 'dice-roller-canary-7d64cd756c-55rfq',
|
||||
namespace: 'default',
|
||||
},
|
||||
proposedFix: {
|
||||
container: 'other-side-car',
|
||||
errorType: 'Error',
|
||||
actions: ['Check the crash logs for stacktraces'],
|
||||
rootCauseExplanation:
|
||||
'This container has exited with a non-zero exit code (1)',
|
||||
type: 'logs',
|
||||
},
|
||||
message: 'container=side-car restarted 38 times',
|
||||
severity: 4,
|
||||
occurrenceCount: 38,
|
||||
type: 'containers-restarting',
|
||||
});
|
||||
});
|
||||
it('should detect errors in pod with missing Config Map', () => {
|
||||
const result = detectErrors(onePod(podMissingCm as any));
|
||||
|
||||
expect(result.size).toBe(1);
|
||||
|
||||
const errors = result.get(CLUSTER_NAME);
|
||||
|
||||
expect(errors).toBeDefined();
|
||||
expect(errors).toHaveLength(1);
|
||||
|
||||
const [err1] = errors ?? [];
|
||||
|
||||
expect(err1).toStrictEqual({
|
||||
message: 'configmap "some-cm" not found',
|
||||
occurrenceCount: 1,
|
||||
proposedFix: {
|
||||
docsLink:
|
||||
'https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/',
|
||||
errorType: 'CreateContainerConfigError',
|
||||
actions: [
|
||||
'Ensure ConfigMaps references in the Deployment manifest are correct and the keys exist',
|
||||
'Ensure Secrets references in the Deployment manifest are correct and the keys exist',
|
||||
],
|
||||
rootCauseExplanation:
|
||||
'There is missing or mismatching configuration required to start the container',
|
||||
type: 'docs',
|
||||
},
|
||||
severity: 4,
|
||||
sourceRef: {
|
||||
apiGroup: 'v1',
|
||||
kind: 'Pod',
|
||||
name: 'dice-roller-bad-cm-855bf85464-mg6xb',
|
||||
namespace: 'default',
|
||||
},
|
||||
type: 'container-waiting',
|
||||
});
|
||||
});
|
||||
it('should detect no errors in healthy deployment', () => {
|
||||
const result = detectErrors(oneDeployment(healthyDeploy as any));
|
||||
|
||||
expect(result.size).toBe(1);
|
||||
|
||||
const errors = result.get(CLUSTER_NAME);
|
||||
|
||||
expect(errors).toBeDefined();
|
||||
expect(errors).toHaveLength(0);
|
||||
});
|
||||
it('should detect in deployment which cannot progress', () => {
|
||||
const result = detectErrors(oneDeployment(failingDeploy as any));
|
||||
|
||||
expect(result.size).toBe(1);
|
||||
|
||||
const errors = result.get(CLUSTER_NAME);
|
||||
|
||||
expect(errors).toBeDefined();
|
||||
expect(errors).toHaveLength(1);
|
||||
|
||||
const [err1] = errors ?? [];
|
||||
|
||||
expect(err1).toStrictEqual({
|
||||
sourceRef: {
|
||||
apiGroup: 'apps/v1',
|
||||
kind: 'Deployment',
|
||||
name: 'dice-roller-canary',
|
||||
namespace: 'default',
|
||||
},
|
||||
message: 'Deployment does not have minimum availability.',
|
||||
severity: 6,
|
||||
occurrenceCount: 1,
|
||||
type: 'condition-message-present',
|
||||
});
|
||||
});
|
||||
it('should detect no errors in healthy hpa', () => {
|
||||
const result = detectErrors(oneHpa(healthyHpa as any));
|
||||
|
||||
expect(result.size).toBe(1);
|
||||
|
||||
const errors = result.get(CLUSTER_NAME);
|
||||
|
||||
expect(errors).toBeDefined();
|
||||
expect(errors).toHaveLength(0);
|
||||
});
|
||||
it('should detect in maxed out hpa', () => {
|
||||
const result = detectErrors(oneHpa(maxedOutHpa as any));
|
||||
|
||||
expect(result.size).toBe(1);
|
||||
|
||||
const errors = result.get(CLUSTER_NAME);
|
||||
|
||||
expect(errors).toBeDefined();
|
||||
expect(errors).toHaveLength(1);
|
||||
|
||||
const [err1] = errors ?? [];
|
||||
|
||||
expect(err1).toStrictEqual({
|
||||
sourceRef: {
|
||||
apiGroup: 'autoscaling/v1',
|
||||
kind: 'HorizontalPodAutoscaler',
|
||||
name: 'dice-roller',
|
||||
namespace: 'default',
|
||||
},
|
||||
message:
|
||||
'Current number of replicas (10) is equal to the configured max number of replicas (10)',
|
||||
severity: 8,
|
||||
occurrenceCount: 1,
|
||||
type: 'hpa-max-current-replicas',
|
||||
});
|
||||
});
|
||||
it('pending pod is not an error', async () => {
|
||||
const expiredReadiness = new Date();
|
||||
expiredReadiness.setFullYear(expiredReadiness.getFullYear() - 1);
|
||||
const result = await detectErrors(
|
||||
onePod({
|
||||
spec: {
|
||||
containers: [
|
||||
{
|
||||
name: 'some-container',
|
||||
readinessProbe: {
|
||||
initialDelaySeconds: 20000,
|
||||
failureThreshold: 5,
|
||||
periodSeconds: 5,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
status: {
|
||||
containerStatuses: [
|
||||
{
|
||||
name: 'some-container',
|
||||
image: 'some-image',
|
||||
imageID: 'some-image-id',
|
||||
restartCount: 0,
|
||||
containerID: 'running-container',
|
||||
ready: false,
|
||||
state: {
|
||||
running: {
|
||||
startedAt: new Date().toISOString() as any,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
message: 'Container running',
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const errors = result.get(CLUSTER_NAME);
|
||||
|
||||
expect(errors).toBeDefined();
|
||||
expect(errors).toHaveLength(0);
|
||||
});
|
||||
it('no probe pod has no errors', async () => {
|
||||
const expiredReadiness = new Date();
|
||||
expiredReadiness.setFullYear(expiredReadiness.getFullYear() - 1);
|
||||
const result = await detectErrors(
|
||||
onePod({
|
||||
spec: {
|
||||
containers: [
|
||||
{
|
||||
name: 'some-container',
|
||||
},
|
||||
],
|
||||
},
|
||||
status: {
|
||||
containerStatuses: [
|
||||
{
|
||||
name: 'some-container',
|
||||
image: 'some-image',
|
||||
imageID: 'some-image-id',
|
||||
restartCount: 0,
|
||||
containerID: 'running-container',
|
||||
ready: false,
|
||||
state: {
|
||||
running: {
|
||||
startedAt: new Date().toISOString() as any,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
message: 'Container running',
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const errors = result.get(CLUSTER_NAME);
|
||||
|
||||
expect(errors).toBeDefined();
|
||||
expect(errors).toHaveLength(0);
|
||||
});
|
||||
it('readiness probe failure results in error', async () => {
|
||||
const expiredReadiness = new Date();
|
||||
expiredReadiness.setFullYear(expiredReadiness.getFullYear() - 1);
|
||||
const result = await detectErrors(
|
||||
onePod({
|
||||
spec: {
|
||||
containers: [
|
||||
{
|
||||
name: 'some-container',
|
||||
readinessProbe: {
|
||||
initialDelaySeconds: 20,
|
||||
failureThreshold: 5,
|
||||
periodSeconds: 5,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
status: {
|
||||
containerStatuses: [
|
||||
{
|
||||
name: 'some-container',
|
||||
image: 'some-image',
|
||||
imageID: 'some-image-id',
|
||||
restartCount: 0,
|
||||
containerID: 'running-container',
|
||||
ready: false,
|
||||
state: {
|
||||
running: {
|
||||
startedAt: expiredReadiness.toISOString() as any,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
message: 'Container running',
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const errors = result.get(CLUSTER_NAME);
|
||||
|
||||
expect(errors).toBeDefined();
|
||||
expect(errors).toHaveLength(1);
|
||||
|
||||
const [err1] = errors ?? [];
|
||||
|
||||
expect(err1).toStrictEqual({
|
||||
message:
|
||||
'The container some-container failed to start properly, but is not crashing',
|
||||
occurrenceCount: 1,
|
||||
proposedFix: {
|
||||
errorType: 'ReadinessProbeFailed',
|
||||
podName: '',
|
||||
actions: [
|
||||
'Ensure that the container starts correctly locally',
|
||||
"Check the container's logs looking for error during startup",
|
||||
],
|
||||
rootCauseExplanation:
|
||||
'The container some-container failed to start properly, but is not crashing',
|
||||
type: 'events',
|
||||
},
|
||||
severity: 4,
|
||||
sourceRef: {
|
||||
apiGroup: 'v1',
|
||||
kind: 'Pod',
|
||||
name: 'unknown pod',
|
||||
namespace: 'unknown namespace',
|
||||
},
|
||||
type: 'readiness-probe-taking-too-long',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { DetectedError, DetectedErrorsByCluster } from './types';
|
||||
import { ObjectsByEntityResponse } from '@backstage/plugin-kubernetes-common';
|
||||
import { groupResponses } from '../util';
|
||||
import { detectErrorsInPods } from './pods';
|
||||
import { detectErrorsInDeployments } from './deployments';
|
||||
import { detectErrorsInHpa } from './hpas';
|
||||
import { Deployment } from 'kubernetes-models/apps/v1';
|
||||
import { HorizontalPodAutoscaler } from 'kubernetes-models/autoscaling/v1';
|
||||
import { Pod } from 'kubernetes-models/v1';
|
||||
|
||||
/**
|
||||
* For each cluster try to find errors in each of the object types provided
|
||||
* returning a map of cluster names to errors in that cluster
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export const detectErrors = (
|
||||
objects: ObjectsByEntityResponse,
|
||||
): DetectedErrorsByCluster => {
|
||||
const errors: DetectedErrorsByCluster = new Map<string, DetectedError[]>();
|
||||
|
||||
for (const clusterResponse of objects.items) {
|
||||
let clusterErrors: DetectedError[] = [];
|
||||
|
||||
const groupedResponses = groupResponses(clusterResponse.resources);
|
||||
|
||||
clusterErrors = clusterErrors.concat(
|
||||
detectErrorsInPods(groupedResponses.pods as Pod[]),
|
||||
);
|
||||
|
||||
clusterErrors = clusterErrors.concat(
|
||||
detectErrorsInDeployments(groupedResponses.deployments as Deployment[]),
|
||||
);
|
||||
|
||||
clusterErrors = clusterErrors.concat(
|
||||
detectErrorsInHpa(
|
||||
groupedResponses.horizontalPodAutoscalers as HorizontalPodAutoscaler[],
|
||||
),
|
||||
);
|
||||
|
||||
errors.set(clusterResponse.cluster.name, clusterErrors);
|
||||
}
|
||||
|
||||
return errors;
|
||||
};
|
||||
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { HorizontalPodAutoscaler } from 'kubernetes-models/autoscaling/v1';
|
||||
import { DetectedError, ErrorMapper } from './types';
|
||||
import { detectErrorsInObjects } from './common';
|
||||
|
||||
const hpaErrorMappers: ErrorMapper<HorizontalPodAutoscaler>[] = [
|
||||
{
|
||||
detectErrors: hpa => {
|
||||
if ((hpa.spec?.maxReplicas ?? -1) === hpa.status?.currentReplicas) {
|
||||
return [
|
||||
{
|
||||
type: 'hpa-max-current-replicas',
|
||||
message: `Current number of replicas (${
|
||||
hpa.status?.currentReplicas
|
||||
}) is equal to the configured max number of replicas (${
|
||||
hpa.spec?.maxReplicas ?? -1
|
||||
})`,
|
||||
severity: 8,
|
||||
sourceRef: {
|
||||
name: hpa.metadata?.name ?? 'unknown hpa',
|
||||
namespace: hpa.metadata?.namespace ?? 'unknown namespace',
|
||||
kind: 'HorizontalPodAutoscaler',
|
||||
apiGroup: 'autoscaling/v1',
|
||||
},
|
||||
occurrenceCount: 1,
|
||||
},
|
||||
];
|
||||
}
|
||||
return [];
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
export const detectErrorsInHpa = (
|
||||
hpas: HorizontalPodAutoscaler[],
|
||||
): DetectedError[] => detectErrorsInObjects(hpas, hpaErrorMappers);
|
||||
@@ -0,0 +1,18 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export * from './types';
|
||||
export { detectErrors } from './error-detection';
|
||||
@@ -0,0 +1,257 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Pod, IContainerStatus, IContainer } from 'kubernetes-models/v1';
|
||||
import { DetectedError, ErrorMapper, ProposedFix } from './types';
|
||||
import { detectErrorsInObjects } from './common';
|
||||
import lodash from 'lodash';
|
||||
import { DateTime } from 'luxon';
|
||||
|
||||
function isPodReadinessProbeUnready({
|
||||
container,
|
||||
containerStatus,
|
||||
}: ContainerSpecAndStatus): boolean {
|
||||
if (
|
||||
containerStatus.ready ||
|
||||
containerStatus.state?.running?.startedAt === undefined ||
|
||||
!container.readinessProbe
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const startDateTime = DateTime.fromISO(
|
||||
containerStatus.state?.running?.startedAt,
|
||||
)
|
||||
// Add initial delay
|
||||
.plus({
|
||||
seconds: container.readinessProbe?.initialDelaySeconds ?? 0,
|
||||
})
|
||||
// Add failure threshold
|
||||
.plus({
|
||||
seconds:
|
||||
(container.readinessProbe?.periodSeconds ?? 0) *
|
||||
(container.readinessProbe?.failureThreshold ?? 0),
|
||||
});
|
||||
return startDateTime < DateTime.now();
|
||||
}
|
||||
|
||||
interface ContainerSpecAndStatus {
|
||||
container: IContainer;
|
||||
containerStatus: IContainerStatus;
|
||||
}
|
||||
|
||||
const podToContainerSpecsAndStatuses = (pod: Pod): ContainerSpecAndStatus[] => {
|
||||
const specs = lodash.groupBy(pod.spec?.containers ?? [], value => value.name);
|
||||
|
||||
const result: ContainerSpecAndStatus[] = [];
|
||||
|
||||
for (const cs of pod.status?.containerStatuses ?? []) {
|
||||
const spec = specs[cs.name];
|
||||
if (spec.length > 0) {
|
||||
result.push({
|
||||
container: spec[0],
|
||||
containerStatus: cs,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
const readinessProbeProposedFixes = (pod: Pod): ProposedFix | undefined => {
|
||||
const firstUnreadyContainerStatus = pod.status?.containerStatuses?.find(
|
||||
cs => {
|
||||
return cs.ready === false;
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
errorType: 'ReadinessProbeFailed',
|
||||
rootCauseExplanation: `The container ${firstUnreadyContainerStatus?.name} failed to start properly, but is not crashing`,
|
||||
actions: [
|
||||
'Ensure that the container starts correctly locally',
|
||||
"Check the container's logs looking for error during startup",
|
||||
],
|
||||
type: 'events',
|
||||
podName: pod.metadata?.name ?? '',
|
||||
};
|
||||
};
|
||||
|
||||
const restartingPodProposedFixes = (pod: Pod): ProposedFix | undefined => {
|
||||
const lastTerminatedCs = (pod.status?.containerStatuses ?? []).find(
|
||||
cs => cs.lastState?.terminated !== undefined,
|
||||
);
|
||||
|
||||
const lastTerminated = lastTerminatedCs?.lastState?.terminated;
|
||||
|
||||
if (!lastTerminated) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
switch (lastTerminated?.reason) {
|
||||
case 'Unknown':
|
||||
return {
|
||||
// TODO check this one, it's more likely a cluster issue
|
||||
errorType: 'Unknown',
|
||||
rootCauseExplanation: `This container has exited with a non-zero exit code (${lastTerminated.exitCode})`,
|
||||
actions: ['Check the crash logs for stacktraces'],
|
||||
container: lastTerminatedCs.name,
|
||||
type: 'logs',
|
||||
};
|
||||
case 'Error':
|
||||
return {
|
||||
errorType: 'Error',
|
||||
rootCauseExplanation: `This container has exited with a non-zero exit code (${lastTerminated.exitCode})`,
|
||||
actions: ['Check the crash logs for stacktraces'],
|
||||
container: lastTerminatedCs.name,
|
||||
type: 'logs',
|
||||
};
|
||||
case 'OOMKilled':
|
||||
return {
|
||||
errorType: 'OOMKilled',
|
||||
rootCauseExplanation: `The container "${lastTerminatedCs.name}" has crashed because it has tried to use more memory that it has been allocated`,
|
||||
actions: [
|
||||
`Increase the amount of memory assigned to the container`,
|
||||
'Ensure the application is memory bounded and is not trying to consume too much memory',
|
||||
],
|
||||
docsLink:
|
||||
'https://kubernetes.io/docs/tasks/configure-pod-container/assign-memory-resource/#exceed-a-container-s-memory-limit',
|
||||
type: 'docs',
|
||||
};
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
const waitingProposedFix = (pod: Pod): ProposedFix | undefined => {
|
||||
const waitingCs = (pod.status?.containerStatuses ?? []).find(
|
||||
cs => cs.state?.waiting !== undefined,
|
||||
);
|
||||
|
||||
const waiting = (pod.status?.containerStatuses ?? [])
|
||||
.map(cs => cs.state?.waiting)
|
||||
.find(w => w?.reason !== undefined);
|
||||
|
||||
switch (waiting?.reason) {
|
||||
case 'InvalidImageName':
|
||||
return {
|
||||
errorType: 'InvalidImageName',
|
||||
rootCauseExplanation: 'The image in the pod is invalid',
|
||||
actions: ['Ensure the image name is correct and valid image name'],
|
||||
type: 'docs',
|
||||
docsLink:
|
||||
'https://docs.docker.com/engine/reference/commandline/tag/#extended-description',
|
||||
};
|
||||
case 'ImagePullBackOff':
|
||||
return {
|
||||
errorType: 'ImagePullBackOff',
|
||||
rootCauseExplanation:
|
||||
'The image either could not be found or Kubernetes does not have permission to pull it',
|
||||
actions: [
|
||||
'Ensure the image name is correct',
|
||||
'Ensure Kubernetes has permission to pull this image',
|
||||
],
|
||||
type: 'docs',
|
||||
docsLink:
|
||||
'https://kubernetes.io/docs/concepts/containers/images/#imagepullbackoff',
|
||||
};
|
||||
case 'CrashLoopBackOff':
|
||||
return {
|
||||
errorType: 'CrashLoopBackOff',
|
||||
rootCauseExplanation: `The container ${waitingCs?.name} has crashed many times, it will be exponentially restarted until it stops crashing`,
|
||||
actions: ['Check the crash logs for stacktraces'],
|
||||
type: 'logs',
|
||||
container: waitingCs?.name ?? 'unknown',
|
||||
};
|
||||
case 'CreateContainerConfigError':
|
||||
return {
|
||||
errorType: 'CreateContainerConfigError',
|
||||
rootCauseExplanation:
|
||||
'There is missing or mismatching configuration required to start the container',
|
||||
actions: [
|
||||
'Ensure ConfigMaps references in the Deployment manifest are correct and the keys exist',
|
||||
'Ensure Secrets references in the Deployment manifest are correct and the keys exist',
|
||||
],
|
||||
type: 'docs',
|
||||
docsLink:
|
||||
'https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/',
|
||||
};
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
const podErrorMappers: ErrorMapper<Pod>[] = [
|
||||
{
|
||||
detectErrors: pod => {
|
||||
return podToContainerSpecsAndStatuses(pod)
|
||||
.filter(isPodReadinessProbeUnready)
|
||||
.map(cs => ({
|
||||
type: 'readiness-probe-taking-too-long',
|
||||
message: `The container ${cs.container.name} failed to start properly, but is not crashing`,
|
||||
severity: 4,
|
||||
proposedFix: readinessProbeProposedFixes(pod),
|
||||
sourceRef: {
|
||||
name: pod.metadata?.name ?? 'unknown pod',
|
||||
namespace: pod.metadata?.namespace ?? 'unknown namespace',
|
||||
kind: 'Pod',
|
||||
apiGroup: 'v1',
|
||||
},
|
||||
occurrenceCount: 1,
|
||||
}));
|
||||
},
|
||||
},
|
||||
{
|
||||
detectErrors: pod => {
|
||||
return (pod.status?.containerStatuses ?? [])
|
||||
.filter(cs => cs.state?.waiting?.message !== undefined)
|
||||
.map(cs => ({
|
||||
type: 'container-waiting',
|
||||
message: cs.state?.waiting?.message ?? 'container waiting',
|
||||
severity: 4,
|
||||
proposedFix: waitingProposedFix(pod),
|
||||
sourceRef: {
|
||||
name: pod.metadata?.name ?? 'unknown pod',
|
||||
namespace: pod.metadata?.namespace ?? 'unknown namespace',
|
||||
kind: 'Pod',
|
||||
apiGroup: 'v1',
|
||||
},
|
||||
occurrenceCount: 1,
|
||||
}));
|
||||
},
|
||||
},
|
||||
{
|
||||
detectErrors: pod => {
|
||||
return (pod.status?.containerStatuses ?? [])
|
||||
.filter(cs => cs.restartCount > 0)
|
||||
.map(cs => ({
|
||||
type: 'containers-restarting',
|
||||
message: `container=${cs.name} restarted ${cs.restartCount} times`,
|
||||
severity: 4,
|
||||
proposedFix: restartingPodProposedFixes(pod),
|
||||
sourceRef: {
|
||||
name: pod.metadata?.name ?? 'unknown pod',
|
||||
namespace: pod.metadata?.namespace ?? 'unknown namespace',
|
||||
kind: 'Pod',
|
||||
apiGroup: 'v1',
|
||||
},
|
||||
occurrenceCount: cs.restartCount,
|
||||
}));
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
export const detectErrorsInPods = (pods: Pod[]): DetectedError[] =>
|
||||
detectErrorsInObjects(pods, podErrorMappers);
|
||||
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Severity of the error, where 10 is critical and 0 is very low.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type ErrorSeverity = 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10;
|
||||
|
||||
/**
|
||||
* A list of errors keyed by Cluster name
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type DetectedErrorsByCluster = Map<string, DetectedError[]>;
|
||||
|
||||
/**
|
||||
* A reference to a Kubernetes object
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export interface ResourceRef {
|
||||
name: string;
|
||||
namespace: string;
|
||||
kind: string;
|
||||
apiGroup: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents an error found on a Kubernetes object
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export interface DetectedError {
|
||||
type: string;
|
||||
severity: ErrorSeverity;
|
||||
message: string;
|
||||
proposedFix?: ProposedFix;
|
||||
sourceRef: ResourceRef;
|
||||
occurrenceCount: number;
|
||||
}
|
||||
|
||||
/** @public */
|
||||
export type ProposedFix = LogSolution | DocsSolution | EventsSolution;
|
||||
|
||||
/** @public */
|
||||
export interface ProposedFixBase {
|
||||
errorType: string;
|
||||
rootCauseExplanation: string;
|
||||
actions: string[];
|
||||
}
|
||||
|
||||
/** @public */
|
||||
export interface LogSolution extends ProposedFixBase {
|
||||
type: 'logs';
|
||||
container: string;
|
||||
}
|
||||
|
||||
/** @public */
|
||||
export interface DocsSolution extends ProposedFixBase {
|
||||
type: 'docs';
|
||||
docsLink: string;
|
||||
}
|
||||
|
||||
/** @public */
|
||||
export interface EventsSolution extends ProposedFixBase {
|
||||
type: 'events';
|
||||
podName: string;
|
||||
}
|
||||
|
||||
/** @public */
|
||||
export interface ErrorMapper<T> {
|
||||
detectErrors: (resource: T) => DetectedError[];
|
||||
}
|
||||
@@ -26,3 +26,5 @@ export {
|
||||
kubernetesProxyPermission,
|
||||
kubernetesPermissions,
|
||||
} from './permissions';
|
||||
export * from './error-detection';
|
||||
export * from './util';
|
||||
|
||||
@@ -262,3 +262,22 @@ export interface ClientPodStatus {
|
||||
memory: ClientCurrentResourceUsage;
|
||||
containers: ClientContainerStatus[];
|
||||
}
|
||||
|
||||
/** @public */
|
||||
export interface DeploymentResources {
|
||||
pods: V1Pod[];
|
||||
replicaSets: V1ReplicaSet[];
|
||||
deployments: V1Deployment[];
|
||||
horizontalPodAutoscalers: V1HorizontalPodAutoscaler[];
|
||||
}
|
||||
|
||||
/** @public */
|
||||
export interface GroupedResponses extends DeploymentResources {
|
||||
services: V1Service[];
|
||||
configMaps: V1ConfigMap[];
|
||||
ingresses: V1Ingress[];
|
||||
jobs: V1Job[];
|
||||
cronJobs: V1CronJob[];
|
||||
customResources: any[];
|
||||
statefulsets: V1StatefulSet[];
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export * from './response';
|
||||
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* Copyright 2020 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { FetchResponse } from '@backstage/plugin-kubernetes-common';
|
||||
import { GroupedResponses } from '../types';
|
||||
|
||||
/** @public */
|
||||
export const groupResponses = (
|
||||
fetchResponse: FetchResponse[],
|
||||
): GroupedResponses => {
|
||||
// TODO this could probably be a lodash groupBy
|
||||
return fetchResponse.reduce(
|
||||
(prev, next) => {
|
||||
switch (next.type) {
|
||||
case 'deployments':
|
||||
prev.deployments.push(...next.resources);
|
||||
break;
|
||||
case 'pods':
|
||||
prev.pods.push(...next.resources);
|
||||
break;
|
||||
case 'replicasets':
|
||||
prev.replicaSets.push(...next.resources);
|
||||
break;
|
||||
case 'services':
|
||||
prev.services.push(...next.resources);
|
||||
break;
|
||||
case 'configmaps':
|
||||
prev.configMaps.push(...next.resources);
|
||||
break;
|
||||
case 'horizontalpodautoscalers':
|
||||
prev.horizontalPodAutoscalers.push(...next.resources);
|
||||
break;
|
||||
case 'ingresses':
|
||||
prev.ingresses.push(...next.resources);
|
||||
break;
|
||||
case 'jobs':
|
||||
prev.jobs.push(...next.resources);
|
||||
break;
|
||||
case 'cronjobs':
|
||||
prev.cronJobs.push(...next.resources);
|
||||
break;
|
||||
case 'customresources':
|
||||
prev.customResources.push(...next.resources);
|
||||
break;
|
||||
case 'statefulsets':
|
||||
prev.statefulsets.push(...next.resources);
|
||||
break;
|
||||
default:
|
||||
}
|
||||
return prev;
|
||||
},
|
||||
{
|
||||
pods: [],
|
||||
replicaSets: [],
|
||||
deployments: [],
|
||||
services: [],
|
||||
configMaps: [],
|
||||
horizontalPodAutoscalers: [],
|
||||
ingresses: [],
|
||||
jobs: [],
|
||||
cronJobs: [],
|
||||
customResources: [],
|
||||
statefulsets: [],
|
||||
} as GroupedResponses,
|
||||
);
|
||||
};
|
||||
Reference in New Issue
Block a user