v0.11.1: repeated restarts and OOM kills stay on the map for a day
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
4c5eb288cb
commit
8ed76cd01a
@@ -55,10 +55,17 @@ type Problem struct {
|
||||
var (
|
||||
now = time.Now
|
||||
restartWindow = 30 * time.Minute
|
||||
eventWindow = 30 * time.Minute
|
||||
pendingAfter = 5 * time.Minute
|
||||
notReadyAfter = 2 * time.Minute // a pod starting or restarting is not yet a problem
|
||||
maxPerCard = 20
|
||||
// A container that keeps restarting stays marked much longer: one
|
||||
// restart an hour ago is history, the fifth OOM kill in a day is a
|
||||
// memory limit that is too low. Found when a panel was OOM-killed five
|
||||
// times in 23 hours and the map showed nothing 40 minutes after the
|
||||
// last one.
|
||||
repeatedRestarts int32 = 3
|
||||
repeatedWindow = 24 * time.Hour
|
||||
eventWindow = 30 * time.Minute
|
||||
pendingAfter = 5 * time.Minute
|
||||
notReadyAfter = 2 * time.Minute // a pod starting or restarting is not yet a problem
|
||||
maxPerCard = 20
|
||||
// Warning events that mark a card even when seen once. Any other reason
|
||||
// (a failed readiness probe during a restart, say) needs eventRepeats
|
||||
// occurrences in the window; below that it is listed, not badged.
|
||||
@@ -257,7 +264,7 @@ func podProblems(s *problemSet, p corev1.Pod) {
|
||||
s.add(id, Problem{Severity: Critical, Source: "state", Reason: w.Reason, Message: w.Message, Object: obj, Count: int(cs.RestartCount)})
|
||||
continue
|
||||
}
|
||||
if t := cs.LastTerminationState.Terminated; t != nil && now().Sub(t.FinishedAt.Time) < restartWindow {
|
||||
if t := cs.LastTerminationState.Terminated; t != nil && recentRestart(t.FinishedAt.Time, cs.RestartCount) {
|
||||
reason := "Restarted"
|
||||
if t.Reason == "OOMKilled" {
|
||||
reason = "OOMKilled"
|
||||
@@ -571,3 +578,10 @@ func alertProblem(s *problemSet, a *unstructured.Unstructured, podOwner map[stri
|
||||
since, _, _ := unstructured.NestedString(a.Object, "status", "startsAt")
|
||||
s.add(AlertTargetID(labels, podOwner), Problem{Severity: AlertSeverity(level), Source: "alert", Reason: name, Message: strings.TrimSpace(summary), Since: since})
|
||||
}
|
||||
|
||||
// recentRestart: the last restart is recent, or the container restarts
|
||||
// again and again and the last one was within a day.
|
||||
func recentRestart(finished time.Time, restarts int32) bool {
|
||||
age := now().Sub(finished)
|
||||
return age < restartWindow || (restarts >= repeatedRestarts && age < repeatedWindow)
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ func TestProblemsFromState(t *testing.T) {
|
||||
withNow(t)
|
||||
ready := []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}
|
||||
g := &Graph{Nodes: []Node{
|
||||
{ID: "ns/shop"}, {ID: "wl/shop/deployment/web", Kind: "deployment", Facts: map[string]string{"health": "warn"}}, {ID: "service/shop/web"}, {ID: "service/shop/idle"},
|
||||
{ID: "ns/shop"}, {ID: "wl/shop/deployment/web", Kind: "deployment", Facts: map[string]string{"health": "warn"}}, {ID: "wl/shop/deployment/api", Kind: "deployment"}, {ID: "service/shop/web"}, {ID: "service/shop/idle"},
|
||||
{ID: "ingress/shop/web"}, {ID: "wl/shop/cronjob/backup"}, {ID: "node/node-a"},
|
||||
}}
|
||||
in := problemInputs{
|
||||
@@ -78,6 +78,13 @@ func TestProblemsFromState(t *testing.T) {
|
||||
Name: "app", LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "OOMKilled", FinishedAt: metav1.NewTime(t0.Add(-3 * time.Hour))}},
|
||||
}}},
|
||||
},
|
||||
{ // OOM-killed five times, the last 70 minutes ago: still a problem
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "api-7c-a", Namespace: "shop", Labels: map[string]string{"app": "api"}, OwnerReferences: owned("ReplicaSet", "api-7c")},
|
||||
Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: ready, ContainerStatuses: []corev1.ContainerStatus{{
|
||||
Name: "app", RestartCount: 5,
|
||||
LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "OOMKilled", ExitCode: 137, FinishedAt: metav1.NewTime(t0.Add(-70 * time.Minute))}},
|
||||
}}},
|
||||
},
|
||||
{ // bare pod pending for 10 minutes: badge on the namespace
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "debug", Namespace: "shop", CreationTimestamp: metav1.NewTime(t0.Add(-10 * time.Minute))},
|
||||
Status: corev1.PodStatus{Phase: corev1.PodPending, Conditions: []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Message: "0/3 nodes are available"}}},
|
||||
@@ -113,6 +120,9 @@ func TestProblemsFromState(t *testing.T) {
|
||||
if _, ok := web["OOMKilled"]; ok {
|
||||
t.Error("an OOM kill three hours ago must not mark the card")
|
||||
}
|
||||
if p := problemsOf(g, "wl/shop/deployment/api")["OOMKilled"]; p.Severity != Warning || p.Count != 5 {
|
||||
t.Errorf("repeated OOM kills 70 minutes ago: %+v", problemsOf(g, "wl/shop/deployment/api"))
|
||||
}
|
||||
if _, ok := problemsOf(g, "service/shop/web")["NoEndpoints"]; ok {
|
||||
t.Error("service with a ready pod reported without endpoints")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user