refactor: kubernetes error detection (#17135)

* feat: refactor kubenetes error detection to make way for proposed solutions Signed-off-by: Matthew Clarke <mclarke@spotify.com> * docs: api-docs Signed-off-by: Matthew Clarke <mclarke@spotify.com> * chore: changeset Signed-off-by: Matthew Clarke <mclarke@spotify.com> * refactor: use luxon Signed-off-by: Matthew Clarke <mclarke@spotify.com> --------- Signed-off-by: Matthew Clarke <mclarke@spotify.com>
2023-04-05 04:27:00 -04:00
parent 4bea0f1d1b
commit 754be7c510
15 changed files with 504 additions and 318 deletions
@@ -15,84 +15,64 @@
 */
 import * as React from 'react';
 import { DetectedError, DetectedErrorsByCluster } from '../../error-detection';
-import { Chip } from '@material-ui/core';
 import { Table, TableColumn } from '@backstage/core-components';

 type ErrorReportingProps = {
  detectedErrors: DetectedErrorsByCluster;
 };

-const columns: TableColumn<DetectedError>[] = [
+const columns: TableColumn<Row>[] = [
  {
    title: 'cluster',
    width: '10%',
-    render: (detectedError: DetectedError) => detectedError.cluster,
+    render: (row: Row) => row.clusterName,
  },
  {
    title: 'namespace',
    width: '10%',
-    render: (detectedError: DetectedError) => detectedError.namespace,
+    render: (row: Row) => row.error.sourceRef.namespace,
  },
  {
    title: 'kind',
    width: '10%',
-    render: (detectedError: DetectedError) => detectedError.kind,
+    render: (row: Row) => row.error.sourceRef.kind,
  },
  {
    title: 'name',
    width: '30%',
-    render: (detectedError: DetectedError) => {
-      const errorCount = detectedError.names.length;
-
-      if (errorCount === 0) {
-        // This shouldn't happen
-        return null;
-      }
-
-      const displayName = detectedError.names[0];
-
-      const otherErrorCount = errorCount - 1;
-
-      return (
-        <>
-          {displayName}{' '}
-          {otherErrorCount > 0 && (
-            <Chip
-              label={`+ ${otherErrorCount} other${
-                otherErrorCount > 1 ? 's' : ''
-              }`}
-              size="small"
-            />
-          )}
-        </>
-      );
+    render: (row: Row) => {
+      return <>{row.error.sourceRef.name} </>;
    },
  },
  {
    title: 'messages',
    width: '40%',
-    render: (detectedError: DetectedError) => (
-      <>
-        {detectedError.message.map((m, i) => (
-          <div key={i}>{m}</div>
-        ))}
-      </>
-    ),
+    render: (row: Row) => row.error.message,
  },
 ];

-const sortBySeverity = (a: DetectedError, b: DetectedError) => {
-  if (a.severity < b.severity) {
+interface Row {
+  clusterName: string;
+  error: DetectedError;
+}
+
+const sortBySeverity = (a: Row, b: Row) => {
+  if (a.error.severity < b.error.severity) {
    return 1;
-  } else if (b.severity < a.severity) {
+  } else if (b.error.severity < a.error.severity) {
    return -1;
  }
  return 0;
 };

 export const ErrorReporting = ({ detectedErrors }: ErrorReportingProps) => {
-  const errors = Array.from(detectedErrors.values())
-    .flat()
+  const errors = Array.from(detectedErrors.entries())
+    .flatMap(([clusterName, resourceErrors]) => {
+      return resourceErrors.map(e => ({
+        clusterName,
+        error: e,
+      }));
+    })
    .sort(sortBySeverity);

  return (
@@ -114,9 +114,8 @@ export const KubernetesContent = ({
                  const podsWithErrors = new Set<string>(
                    detectedErrors
                      .get(item.cluster.name)
-                      ?.filter(de => de.kind === 'Pod')
-                      .map(de => de.names)
-                      .flat() ?? [],
+                      ?.filter(de => de.sourceRef.kind === 'Pod')
+                      .map(de => de.sourceRef.name),
                  );

                  return (
@@ -185,7 +185,7 @@
        },
        "name": "other-side-car",
        "ready": false,
-        "restartCount": 38,
+        "restartCount": 123,
        "started": false,
        "state": {
          "waiting": {
@@ -14,59 +14,15 @@
 * limitations under the License.
 */

-import {
-  DetectedError,
-  ErrorDetectable,
-  ErrorDetectableKind,
-  ErrorMapper,
-} from './types';
+import { DetectedError, ErrorMapper } from './types';

 // Run through the each error mapper for each object
 // returning a deduplicated (mostly) result
-export const detectErrorsInObjects = <T extends ErrorDetectable>(
+export const detectErrorsInObjects = <T>(
  objects: T[],
-  kind: ErrorDetectableKind,
-  clusterName: string,
  errorMappers: ErrorMapper<T>[],
 ): DetectedError[] => {
-  // Build up a map of errors
-  // key: the joined message produced by an error
-  // value: the error
-  const errors = new Map<string, DetectedError>();
-
-  for (const object of objects) {
-    for (const errorMapper of errorMappers) {
-      if (errorMapper.errorExists(object)) {
-        const message = errorMapper.messageAccessor(object);
-
-        // TODO This is not perfect as errors with uuid/hashes/date/times will not be caught by this
-        const dedupKey = message.join('');
-
-        const value = errors.get(dedupKey);
-
-        const name = object.metadata?.name ?? 'unknown';
-        const namespace = object.metadata?.namespace ?? 'unknown';
-
-        if (value !== undefined) {
-          // This gets translated into the Chip "+5 others"
-          // in the ErrorReporting component
-          // but we need to keep the names so we can easily
-          // find which objects owns the error later
-          value.names.push(name);
-          errors.set(dedupKey, value);
-        } else {
-          errors.set(dedupKey, {
-            cluster: clusterName,
-            kind: kind,
-            names: [name],
-            message: message,
-            severity: errorMapper.severity,
-            namespace,
-          });
-        }
-      }
-    }
-  }
-
-  return Array.from(errors.values());
+  return objects.flatMap(o => {
+    return errorMappers.flatMap(em => em.detectErrors(o));
+  });
 };
@@ -15,35 +15,33 @@
 */

 import { DetectedError, ErrorMapper } from './types';
-import { V1Deployment } from '@kubernetes/client-node';
+import { Deployment } from 'kubernetes-models/apps/v1';
 import { detectErrorsInObjects } from './common';

-const deploymentErrorMappers: ErrorMapper<V1Deployment>[] = [
+const deploymentErrorMappers: ErrorMapper<Deployment>[] = [
  {
-    // this is probably important
-    severity: 6,
-    errorExplanation: 'condition-message-present',
-    errorExists: deployment => {
-      return (deployment.status?.conditions ?? [])
-        .filter(c => c.status === 'False')
-        .some(c => c.message !== undefined);
-    },
-    messageAccessor: deployment => {
+    detectErrors: deployment => {
      return (deployment.status?.conditions ?? [])
        .filter(c => c.status === 'False')
        .filter(c => c.message !== undefined)
-        .map(c => c.message ?? '');
+        .map(c => ({
+          type: 'condition-message-present',
+          message: c.message ?? '',
+          severity: 6,
+          proposedFix: [], // TODO next PR
+          sourceRef: {
+            name: deployment.metadata?.name ?? 'unknown hpa',
+            namespace: deployment.metadata?.namespace ?? 'unknown namespace',
+            kind: 'Deployment',
+            apiGroup: 'apps/v1',
+          },
+          occuranceCount: 1,
+        }));
    },
  },
 ];

 export const detectErrorsInDeployments = (
-  deployments: V1Deployment[],
-  clusterName: string,
+  deployments: Deployment[],
 ): DetectedError[] =>
-  detectErrorsInObjects(
-    deployments,
-    'Deployment',
-    clusterName,
-    deploymentErrorMappers,
-  );
+  detectErrorsInObjects(deployments, deploymentErrorMappers);
@@ -147,51 +147,61 @@ describe('detectErrors', () => {
    const [err1, err2, err3, err4] = errors ?? [];

    expect(err1).toStrictEqual({
-      cluster: 'cluster-a',
-      kind: 'Pod',
-      message: [
-        'container=other-side-car restarted 38 times',
-        'container=side-car restarted 38 times',
-      ],
-      names: ['dice-roller-canary-7d64cd756c-55rfq'],
-      namespace: 'default',
+      sourceRef: {
+        apiGroup: 'v1',
+        kind: 'Pod',
+        name: 'dice-roller-canary-7d64cd756c-55rfq',
+        namespace: 'default',
+      },
+      message:
+        'back-off 5m0s restarting failed container=other-side-car pod=dice-roller-canary-7d64cd756c-55rfq_default(65ad28e3-5d51-4b4b-9bf8-4cb069803034)',
      severity: 4,
+      occuranceCount: 1,
+      proposedFix: [],
+      type: 'container-waiting',
    });

    expect(err2).toStrictEqual({
-      cluster: 'cluster-a',
-      kind: 'Pod',
-      message: [
-        'containers with unready status: [side-car other-side-car]',
-        'containers with unready status: [side-car other-side-car]',
-      ],
-      names: ['dice-roller-canary-7d64cd756c-55rfq'],
-      namespace: 'default',
-      severity: 5,
+      sourceRef: {
+        apiGroup: 'v1',
+        kind: 'Pod',
+        name: 'dice-roller-canary-7d64cd756c-55rfq',
+        namespace: 'default',
+      },
+      message:
+        'back-off 5m0s restarting failed container=side-car pod=dice-roller-canary-7d64cd756c-55rfq_default(65ad28e3-5d51-4b4b-9bf8-4cb069803034)',
+      severity: 4,
+      occuranceCount: 1,
+      proposedFix: [],
+      type: 'container-waiting',
    });

    expect(err3).toStrictEqual({
-      cluster: 'cluster-a',
-      kind: 'Pod',
-      message: [
-        'back-off 5m0s restarting failed container=other-side-car pod=dice-roller-canary-7d64cd756c-55rfq_default(65ad28e3-5d51-4b4b-9bf8-4cb069803034)',
-        'back-off 5m0s restarting failed container=side-car pod=dice-roller-canary-7d64cd756c-55rfq_default(65ad28e3-5d51-4b4b-9bf8-4cb069803034)',
-      ],
-      names: ['dice-roller-canary-7d64cd756c-55rfq'],
-      namespace: 'default',
-      severity: 6,
+      sourceRef: {
+        apiGroup: 'v1',
+        kind: 'Pod',
+        name: 'dice-roller-canary-7d64cd756c-55rfq',
+        namespace: 'default',
+      },
+      message: 'container=other-side-car restarted 123 times',
+      severity: 4,
+      occuranceCount: 123,
+      proposedFix: [],
+      type: 'containers-restarting',
    });

    expect(err4).toStrictEqual({
-      cluster: 'cluster-a',
-      kind: 'Pod',
-      message: [
-        'container=other-side-car exited with error code (1)',
-        'container=side-car exited with error code (1)',
-      ],
-      names: ['dice-roller-canary-7d64cd756c-55rfq'],
-      namespace: 'default',
+      sourceRef: {
+        apiGroup: 'v1',
+        kind: 'Pod',
+        name: 'dice-roller-canary-7d64cd756c-55rfq',
+        namespace: 'default',
+      },
+      message: 'container=side-car restarted 38 times',
      severity: 4,
+      occuranceCount: 38,
+      proposedFix: [],
+      type: 'containers-restarting',
    });
  });
  it('should detect errors in pod with missing Config Map', () => {
@@ -202,29 +212,22 @@ describe('detectErrors', () => {
    const errors = result.get(CLUSTER_NAME);

    expect(errors).toBeDefined();
-    expect(errors).toHaveLength(2);
+    expect(errors).toHaveLength(1);

-    const [err1, err2] = errors ?? [];
+    const [err1] = errors ?? [];

    expect(err1).toStrictEqual({
-      cluster: 'cluster-a',
-      kind: 'Pod',
-      message: [
-        'containers with unready status: [nginx]',
-        'containers with unready status: [nginx]',
-      ],
-      names: ['dice-roller-bad-cm-855bf85464-mg6xb'],
-      namespace: 'default',
-      severity: 5,
-    });
-
-    expect(err2).toStrictEqual({
-      cluster: 'cluster-a',
-      kind: 'Pod',
-      message: ['configmap "some-cm" not found'],
-      names: ['dice-roller-bad-cm-855bf85464-mg6xb'],
-      namespace: 'default',
-      severity: 6,
+      message: 'configmap "some-cm" not found',
+      occuranceCount: 1,
+      proposedFix: [],
+      severity: 4,
+      sourceRef: {
+        apiGroup: 'v1',
+        kind: 'Pod',
+        name: 'dice-roller-bad-cm-855bf85464-mg6xb',
+        namespace: 'default',
+      },
+      type: 'container-waiting',
    });
  });
  it('should detect no errors in healthy deployment', () => {
@@ -250,12 +253,17 @@ describe('detectErrors', () => {
    const [err1] = errors ?? [];

    expect(err1).toStrictEqual({
-      cluster: 'cluster-a',
-      kind: 'Deployment',
-      message: ['Deployment does not have minimum availability.'],
-      names: ['dice-roller-canary'],
-      namespace: 'default',
+      sourceRef: {
+        apiGroup: 'apps/v1',
+        kind: 'Deployment',
+        name: 'dice-roller-canary',
+        namespace: 'default',
+      },
+      message: 'Deployment does not have minimum availability.',
      severity: 6,
+      occuranceCount: 1,
+      proposedFix: [],
+      type: 'condition-message-present',
    });
  });
  it('should detect no errors in healthy hpa', () => {
@@ -281,14 +289,159 @@ describe('detectErrors', () => {
    const [err1] = errors ?? [];

    expect(err1).toStrictEqual({
-      cluster: 'cluster-a',
-      kind: 'HorizontalPodAutoscaler',
-      message: [
+      sourceRef: {
+        apiGroup: 'autoscaling/v1',
+        kind: 'HorizontalPodAutoscaler',
+        name: 'dice-roller',
+        namespace: 'default',
+      },
+      message:
        'Current number of replicas (10) is equal to the configured max number of replicas (10)',
-      ],
-      names: ['dice-roller'],
-      namespace: 'default',
      severity: 8,
+      occuranceCount: 1,
+      proposedFix: [],
+      type: 'hpa-max-current-replicas',
+    });
+  });
+  it('pending pod is not an error', async () => {
+    const expiredReadiness = new Date();
+    expiredReadiness.setFullYear(expiredReadiness.getFullYear() - 1);
+    const result = await detectErrors(
+      onePod({
+        spec: {
+          containers: [
+            {
+              name: 'some-container',
+              readinessProbe: {
+                initialDelaySeconds: 20000,
+                failureThreshold: 5,
+                periodSeconds: 5,
+              },
+            },
+          ],
+        },
+        status: {
+          containerStatuses: [
+            {
+              name: 'some-container',
+              image: 'some-image',
+              imageID: 'some-image-id',
+              restartCount: 0,
+              containerID: 'running-container',
+              ready: false,
+              state: {
+                running: {
+                  startedAt: new Date().toISOString() as any,
+                },
+              },
+            },
+          ],
+          message: 'Container running',
+        },
+      }),
+    );
+
+    const errors = result.get(CLUSTER_NAME);
+
+    expect(errors).toBeDefined();
+    expect(errors).toHaveLength(0);
+  });
+  it('no probe pod has no errors', async () => {
+    const expiredReadiness = new Date();
+    expiredReadiness.setFullYear(expiredReadiness.getFullYear() - 1);
+    const result = await detectErrors(
+      onePod({
+        spec: {
+          containers: [
+            {
+              name: 'some-container',
+            },
+          ],
+        },
+        status: {
+          containerStatuses: [
+            {
+              name: 'some-container',
+              image: 'some-image',
+              imageID: 'some-image-id',
+              restartCount: 0,
+              containerID: 'running-container',
+              ready: false,
+              state: {
+                running: {
+                  startedAt: new Date().toISOString() as any,
+                },
+              },
+            },
+          ],
+          message: 'Container running',
+        },
+      }),
+    );
+
+    const errors = result.get(CLUSTER_NAME);
+
+    expect(errors).toBeDefined();
+    expect(errors).toHaveLength(0);
+  });
+  it('readiness probe failure results in error', async () => {
+    const expiredReadiness = new Date();
+    expiredReadiness.setFullYear(expiredReadiness.getFullYear() - 1);
+    const result = await detectErrors(
+      onePod({
+        spec: {
+          containers: [
+            {
+              name: 'some-container',
+              readinessProbe: {
+                initialDelaySeconds: 20,
+                failureThreshold: 5,
+                periodSeconds: 5,
+              },
+            },
+          ],
+        },
+        status: {
+          containerStatuses: [
+            {
+              name: 'some-container',
+              image: 'some-image',
+              imageID: 'some-image-id',
+              restartCount: 0,
+              containerID: 'running-container',
+              ready: false,
+              state: {
+                running: {
+                  startedAt: expiredReadiness.toISOString() as any,
+                },
+              },
+            },
+          ],
+          message: 'Container running',
+        },
+      }),
+    );
+
+    const errors = result.get(CLUSTER_NAME);
+
+    expect(errors).toBeDefined();
+    expect(errors).toHaveLength(1);
+
+    const [err1] = errors ?? [];
+
+    expect(err1).toStrictEqual({
+      message:
+        'The container some-container failed to start properly, but is not crashing',
+      occuranceCount: 1,
+      proposedFix: [],
+      severity: 4,
+      sourceRef: {
+        apiGroup: 'v1',
+        kind: 'Pod',
+        name: 'unknown pod',
+        namespace: 'unknown namespace',
+      },
+      type: 'readiness-probe-taking-too-long',
    });
  });
 });
@@ -20,6 +20,9 @@ import { groupResponses } from '../utils/response';
 import { detectErrorsInPods } from './pods';
 import { detectErrorsInDeployments } from './deployments';
 import { detectErrorsInHpa } from './hpas';
+import { Deployment } from 'kubernetes-models/apps/v1';
+import { HorizontalPodAutoscaler } from 'kubernetes-models/autoscaling/v1';
+import { Pod } from 'kubernetes-models/v1';

 /**
 * For each cluster try to find errors in each of the object types provided
@@ -38,20 +41,16 @@ export const detectErrors = (
    const groupedResponses = groupResponses(clusterResponse.resources);

    clusterErrors = clusterErrors.concat(
-      detectErrorsInPods(groupedResponses.pods, clusterResponse.cluster.name),
+      detectErrorsInPods(groupedResponses.pods as Pod[]),
    );

    clusterErrors = clusterErrors.concat(
-      detectErrorsInDeployments(
-        groupedResponses.deployments,
-        clusterResponse.cluster.name,
-      ),
+      detectErrorsInDeployments(groupedResponses.deployments as Deployment[]),
    );

    clusterErrors = clusterErrors.concat(
      detectErrorsInHpa(
-        groupedResponses.horizontalPodAutoscalers,
-        clusterResponse.cluster.name,
+        groupedResponses.horizontalPodAutoscalers as HorizontalPodAutoscaler[],
      ),
    );

@@ -14,37 +14,39 @@
 * limitations under the License.
 */

-import { V1HorizontalPodAutoscaler } from '@kubernetes/client-node';
+import { HorizontalPodAutoscaler } from 'kubernetes-models/autoscaling/v1';
 import { DetectedError, ErrorMapper } from './types';
 import { detectErrorsInObjects } from './common';

-const hpaErrorMappers: ErrorMapper<V1HorizontalPodAutoscaler>[] = [
+const hpaErrorMappers: ErrorMapper<HorizontalPodAutoscaler>[] = [
  {
-    // this is probably important
-    severity: 8,
-    errorExplanation: 'hpa-max-current-replicas',
-    errorExists: hpa => {
-      return (hpa.spec?.maxReplicas ?? -1) === hpa.status?.currentReplicas;
-    },
-    messageAccessor: hpa => {
-      return [
-        `Current number of replicas (${
-          hpa.status?.currentReplicas
-        }) is equal to the configured max number of replicas (${
-          hpa.spec?.maxReplicas ?? -1
-        })`,
-      ];
+    detectErrors: hpa => {
+      if ((hpa.spec?.maxReplicas ?? -1) === hpa.status?.currentReplicas) {
+        return [
+          {
+            type: 'hpa-max-current-replicas',
+            message: `Current number of replicas (${
+              hpa.status?.currentReplicas
+            }) is equal to the configured max number of replicas (${
+              hpa.spec?.maxReplicas ?? -1
+            })`,
+            severity: 8,
+            proposedFix: [], // TODO next PR
+            sourceRef: {
+              name: hpa.metadata?.name ?? 'unknown hpa',
+              namespace: hpa.metadata?.namespace ?? 'unknown namespace',
+              kind: 'HorizontalPodAutoscaler',
+              apiGroup: 'autoscaling/v1',
+            },
+            occuranceCount: 1,
+          },
+        ];
+      }
+      return [];
    },
  },
 ];

 export const detectErrorsInHpa = (
-  hpas: V1HorizontalPodAutoscaler[],
-  clusterName: string,
-): DetectedError[] =>
-  detectErrorsInObjects(
-    hpas,
-    'HorizontalPodAutoscaler',
-    clusterName,
-    hpaErrorMappers,
-  );
+  hpas: HorizontalPodAutoscaler[],
+): DetectedError[] => detectErrorsInObjects(hpas, hpaErrorMappers);
@@ -15,7 +15,6 @@
 */

 export type {
-  ErrorDetectableKind,
  DetectedError,
  DetectedErrorsByCluster,
  ErrorSeverity,
@@ -14,82 +14,121 @@
 * limitations under the License.
 */

-import { V1Pod } from '@kubernetes/client-node';
-import { totalRestarts } from '../utils/pod';
+import { Pod, IContainerStatus, IContainer } from 'kubernetes-models/v1';
 import { DetectedError, ErrorMapper } from './types';
 import { detectErrorsInObjects } from './common';
+import lodash from 'lodash';
+import { DateTime } from 'luxon';

-const podErrorMappers: ErrorMapper<V1Pod>[] = [
+function isPodReadinessProbeUnready({
+  container,
+  containerStatus,
+}: ContainerSpecAndStatus): boolean {
+  if (
+    containerStatus.ready ||
+    containerStatus.state?.running?.startedAt === undefined ||
+    !container.readinessProbe
+  ) {
+    return false;
+  }
+  const startDateTime = DateTime.fromISO(
+    containerStatus.state?.running?.startedAt,
+  )
+    // Add initial delay
+    .plus({
+      seconds: container.readinessProbe?.initialDelaySeconds ?? 0,
+    })
+    // Add failure threshold
+    .plus({
+      seconds:
+        (container.readinessProbe?.periodSeconds ?? 0) *
+        (container.readinessProbe?.failureThreshold ?? 0),
+    });
+  return startDateTime < DateTime.now();
+}
+
+interface ContainerSpecAndStatus {
+  container: IContainer;
+  containerStatus: IContainerStatus;
+}
+
+const podToContainerSpecsAndStatuses = (pod: Pod): ContainerSpecAndStatus[] => {
+  const specs = lodash.groupBy(pod.spec?.containers ?? [], value => value.name);
+
+  const result: ContainerSpecAndStatus[] = [];
+
+  for (const cs of pod.status?.containerStatuses ?? []) {
+    const spec = specs[cs.name];
+    if (spec.length > 0) {
+      result.push({
+        container: spec[0],
+        containerStatus: cs,
+      });
+    }
+  }
+
+  return result;
+};
+
+const podErrorMappers: ErrorMapper<Pod>[] = [
  {
-    severity: 5,
-    errorExplanation: 'status-message',
-    errorExists: pod => {
-      return pod.status?.message !== undefined;
-    },
-    messageAccessor: pod => {
-      return [pod.status?.message ?? ''];
+    detectErrors: pod => {
+      return podToContainerSpecsAndStatuses(pod)
+        .filter(isPodReadinessProbeUnready)
+        .map(cs => ({
+          type: 'readiness-probe-taking-too-long',
+          message: `The container ${cs.container.name} failed to start properly, but is not crashing`,
+          severity: 4,
+          proposedFix: [], // TODO next PR
+          sourceRef: {
+            name: pod.metadata?.name ?? 'unknown pod',
+            namespace: pod.metadata?.namespace ?? 'unknown namespace',
+            kind: 'Pod',
+            apiGroup: 'v1',
+          },
+          occuranceCount: 1,
+        }));
    },
  },
  {
-    severity: 4,
-    errorExplanation: 'containers-restarting',
-    errorExists: pod => {
-      // TODO magic number
-      return totalRestarts(pod) > 3;
-    },
-    messageAccessor: pod => {
-      return (pod.status?.containerStatuses ?? [])
-        .filter(cs => cs.restartCount > 0)
-        .map(cs => `container=${cs.name} restarted ${cs.restartCount} times`);
-    },
-  },
-  {
-    severity: 5,
-    errorExplanation: 'condition-message-present',
-    errorExists: pod => {
-      return (pod.status?.conditions ?? []).some(c => c.message !== undefined);
-    },
-    messageAccessor: pod => {
-      return (pod.status?.conditions ?? [])
-        .filter(c => c.message !== undefined)
-        .map(c => c.message ?? '');
-    },
-  },
-  {
-    severity: 6,
-    errorExplanation: 'container-waiting',
-    errorExists: pod => {
-      return (pod.status?.containerStatuses ?? []).some(
-        cs => cs.state?.waiting?.message !== undefined,
-      );
-    },
-    messageAccessor: pod => {
+    detectErrors: pod => {
      return (pod.status?.containerStatuses ?? [])
        .filter(cs => cs.state?.waiting?.message !== undefined)
-        .map(cs => cs.state?.waiting?.message ?? '');
+        .map(cs => ({
+          type: 'container-waiting',
+          message: cs.state?.waiting?.message ?? 'container waiting',
+          severity: 4,
+          proposedFix: [], // TODO next PR
+          sourceRef: {
+            name: pod.metadata?.name ?? 'unknown pod',
+            namespace: pod.metadata?.namespace ?? 'unknown namespace',
+            kind: 'Pod',
+            apiGroup: 'v1',
+          },
+          occuranceCount: 1,
+        }));
    },
  },
  {
-    severity: 4,
-    errorExplanation: 'container-last-state-error',
-    errorExists: pod => {
-      return (pod.status?.containerStatuses ?? []).some(
-        cs => (cs.lastState?.terminated?.reason ?? '') === 'Error',
-      );
-    },
-    messageAccessor: pod => {
+    detectErrors: pod => {
      return (pod.status?.containerStatuses ?? [])
-        .filter(cs => (cs.lastState?.terminated?.reason ?? '') === 'Error')
-        .map(
-          cs =>
-            `container=${cs.name} exited with error code (${cs.lastState?.terminated?.exitCode})`,
-        );
+        .filter(cs => cs.restartCount > 0)
+        .map(cs => ({
+          type: 'containers-restarting',
+          message: `container=${cs.name} restarted ${cs.restartCount} times`,
+          severity: 4,
+          proposedFix: [], // TODO next PR
+          sourceRef: {
+            name: pod.metadata?.name ?? 'unknown pod',
+            namespace: pod.metadata?.namespace ?? 'unknown namespace',
+            kind: 'Pod',
+            apiGroup: 'v1',
+          },
+          occuranceCount: cs.restartCount,
+        }));
    },
  },
 ];

-export const detectErrorsInPods = (
-  pods: V1Pod[],
-  clusterName: string,
-): DetectedError[] =>
-  detectErrorsInObjects(pods, 'Pod', clusterName, podErrorMappers);
+export const detectErrorsInPods = (pods: Pod[]): DetectedError[] =>
+  detectErrorsInObjects(pods, podErrorMappers);
@@ -14,13 +14,6 @@
 * limitations under the License.
 */

-// Higher is more sever, but it's relative
-import {
-  V1Deployment,
-  V1HorizontalPodAutoscaler,
-  V1Pod,
-} from '@kubernetes/client-node';
-
 /**
 * Severity of the error, where 10 is critical and 0 is very low.
 *
@@ -28,18 +21,6 @@ import {
 */
 export type ErrorSeverity = 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10;

-export type ErrorDetectable = V1Pod | V1Deployment | V1HorizontalPodAutoscaler;
-
-/**
- * Kubernetes kinds that errors might be reported by the plugin
- *
- * @public
- */
-export type ErrorDetectableKind =
-  | 'Pod'
-  | 'Deployment'
-  | 'HorizontalPodAutoscaler';
-
 /**
 * A list of errors keyed by Cluster name
 *
@@ -47,23 +28,51 @@ export type ErrorDetectableKind =
 */
 export type DetectedErrorsByCluster = Map<string, DetectedError[]>;

+export interface ResourceRef {
+  name: string;
+  namespace: string;
+  kind: string;
+  apiGroup: string;
+}
+
 /**
 * Represents an error found on a Kubernetes object
 *
 * @public
 */
 export interface DetectedError {
+  type: string;
  severity: ErrorSeverity;
-  cluster: string;
-  namespace: string;
-  kind: ErrorDetectableKind;
-  names: string[];
-  message: string[];
+  message: string;
+  proposedFix: ProposedFix[];
+  sourceRef: ResourceRef;
+  occuranceCount: number;
 }

-export interface ErrorMapper<T extends ErrorDetectable> {
-  severity: ErrorSeverity;
-  errorExplanation: string;
-  errorExists: (object: T) => boolean;
-  messageAccessor: (object: T) => string[];
+type ProposedFix = LogSolution | DocsSolution | EventsSolution;
+
+interface ProposedFixBase {
+  errorType: string;
+  rootCauseExplanation: string;
+  possibleFixes: string[];
+}
+
+export interface LogSolution extends ProposedFixBase {
+  type: 'logs';
+  container: string;
+}
+
+export interface DocsSolution extends ProposedFixBase {
+  type: 'docs';
+  docsLink: string;
+}
+
+export interface EventsSolution extends ProposedFixBase {
+  type: 'events';
+  docsLink: string;
+  podName: string;
+}
+
+export interface ErrorMapper<T> {
+  detectErrors: (resource: T) => DetectedError[];
 }