v0.11.1: repeated restarts and OOM kills stay on the map for a day

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Виталий Чефанов
2026-09-25 22:50:08 +03:00
co-authored by Claude Opus 5.5
parent 4c5eb288cb
commit 8ed76cd01a
5 changed files with 35 additions and 8 deletions
@@ -55,10 +55,17 @@ type Problem struct {
var (
now = time.Now
restartWindow = 30 * time.Minute
eventWindow = 30 * time.Minute
pendingAfter = 5 * time.Minute
notReadyAfter = 2 * time.Minute // a pod starting or restarting is not yet a problem
maxPerCard = 20
// A container that keeps restarting stays marked much longer: one
// restart an hour ago is history, the fifth OOM kill in a day is a
// memory limit that is too low. Found when a panel was OOM-killed five
// times in 23 hours and the map showed nothing 40 minutes after the
// last one.
repeatedRestarts int32 = 3
repeatedWindow = 24 * time.Hour
eventWindow = 30 * time.Minute
pendingAfter = 5 * time.Minute
notReadyAfter = 2 * time.Minute // a pod starting or restarting is not yet a problem
maxPerCard = 20
// Warning events that mark a card even when seen once. Any other reason
// (a failed readiness probe during a restart, say) needs eventRepeats
// occurrences in the window; below that it is listed, not badged.
@@ -257,7 +264,7 @@ func podProblems(s *problemSet, p corev1.Pod) {
s.add(id, Problem{Severity: Critical, Source: "state", Reason: w.Reason, Message: w.Message, Object: obj, Count: int(cs.RestartCount)})
continue
}
if t := cs.LastTerminationState.Terminated; t != nil && now().Sub(t.FinishedAt.Time) < restartWindow {
if t := cs.LastTerminationState.Terminated; t != nil && recentRestart(t.FinishedAt.Time, cs.RestartCount) {
reason := "Restarted"
if t.Reason == "OOMKilled" {
reason = "OOMKilled"
@@ -571,3 +578,10 @@ func alertProblem(s *problemSet, a *unstructured.Unstructured, podOwner map[stri
since, _, _ := unstructured.NestedString(a.Object, "status", "startsAt")
s.add(AlertTargetID(labels, podOwner), Problem{Severity: AlertSeverity(level), Source: "alert", Reason: name, Message: strings.TrimSpace(summary), Since: since})
}
// recentRestart: the last restart is recent, or the container restarts
// again and again and the last one was within a day.
func recentRestart(finished time.Time, restarts int32) bool {
age := now().Sub(finished)
return age < restartWindow || (restarts >= repeatedRestarts && age < repeatedWindow)
}
@@ -59,7 +59,7 @@ func TestProblemsFromState(t *testing.T) {
withNow(t)
ready := []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}
g := &Graph{Nodes: []Node{
{ID: "ns/shop"}, {ID: "wl/shop/deployment/web", Kind: "deployment", Facts: map[string]string{"health": "warn"}}, {ID: "service/shop/web"}, {ID: "service/shop/idle"},
{ID: "ns/shop"}, {ID: "wl/shop/deployment/web", Kind: "deployment", Facts: map[string]string{"health": "warn"}}, {ID: "wl/shop/deployment/api", Kind: "deployment"}, {ID: "service/shop/web"}, {ID: "service/shop/idle"},
{ID: "ingress/shop/web"}, {ID: "wl/shop/cronjob/backup"}, {ID: "node/node-a"},
}}
in := problemInputs{
@@ -78,6 +78,13 @@ func TestProblemsFromState(t *testing.T) {
Name: "app", LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "OOMKilled", FinishedAt: metav1.NewTime(t0.Add(-3 * time.Hour))}},
}}},
},
{ // OOM-killed five times, the last 70 minutes ago: still a problem
ObjectMeta: metav1.ObjectMeta{Name: "api-7c-a", Namespace: "shop", Labels: map[string]string{"app": "api"}, OwnerReferences: owned("ReplicaSet", "api-7c")},
Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: ready, ContainerStatuses: []corev1.ContainerStatus{{
Name: "app", RestartCount: 5,
LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "OOMKilled", ExitCode: 137, FinishedAt: metav1.NewTime(t0.Add(-70 * time.Minute))}},
}}},
},
{ // bare pod pending for 10 minutes: badge on the namespace
ObjectMeta: metav1.ObjectMeta{Name: "debug", Namespace: "shop", CreationTimestamp: metav1.NewTime(t0.Add(-10 * time.Minute))},
Status: corev1.PodStatus{Phase: corev1.PodPending, Conditions: []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Message: "0/3 nodes are available"}}},
@@ -113,6 +120,9 @@ func TestProblemsFromState(t *testing.T) {
if _, ok := web["OOMKilled"]; ok {
t.Error("an OOM kill three hours ago must not mark the card")
}
if p := problemsOf(g, "wl/shop/deployment/api")["OOMKilled"]; p.Severity != Warning || p.Count != 5 {
t.Errorf("repeated OOM kills 70 minutes ago: %+v", problemsOf(g, "wl/shop/deployment/api"))
}
if _, ok := problemsOf(g, "service/shop/web")["NoEndpoints"]; ok {
t.Error("service with a ready pod reported without endpoints")
}