v0.11.2: events of finished or since-scheduled pods are history

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Виталий Чефанов
2026-09-26 14:22:43 +03:00
co-authored by Claude Opus 5.5
parent 8ed76cd01a
commit 4a63f57674
3 changed files with 62 additions and 4 deletions
+3
View File
@@ -1,3 +1,6 @@
v0.11.2:
fixes:
- "Warning events of a pod that has since finished, and FailedScheduling of a pod that has since been scheduled, are listed for information and no longer badge the card: a backup CronJob that waited all night for its node and then ran stayed yellow for half an hour."
v0.11.1: v0.11.1:
fixes: fixes:
- "A container restarted three times or more stays marked for 24 hours after its last restart or OOM kill, not 30 minutes: a pod OOM-killed five times in a day was no longer shown 40 minutes after the last kill. A single restart still clears after 30 minutes." - "A container restarted three times or more stays marked for 24 hours after its last restart or OOM kill, not 30 minutes: a pod OOM-killed five times in a day was no longer shown 40 minutes after the last kill. A single restart still clears after 30 minutes."
@@ -178,7 +178,7 @@ func attachProblems(g *Graph, in problemInputs) {
for _, n := range in.nodes { for _, n := range in.nodes {
nodeProblems(s, n) nodeProblems(s, n)
} }
eventProblems(s, in.events, podOwner, podsLive(in.pods)) eventProblems(s, in.events, podOwner, podsLive(in.pods), podsScheduled(in.pods))
for _, n := range g.Nodes { for _, n := range g.Nodes {
if !syntheticProbe(n.Namespace, n.Label) { if !syntheticProbe(n.Namespace, n.Label) {
readinessProblem(s, n, notReadySince) readinessProblem(s, n, notReadySince)
@@ -396,7 +396,7 @@ func eventsInWindow(e corev1.Event, last time.Time) int {
// eventProblems folds recent Warning events into one problem per card and // eventProblems folds recent Warning events into one problem per card and
// reason, keeping the latest message. // reason, keeping the latest message.
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live map[string]bool) { func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live, scheduled map[string]bool) {
type key struct{ id, reason string } type key struct{ id, reason string }
folded := map[key]*Problem{} folded := map[key]*Problem{}
var order []key var order []key
@@ -424,7 +424,13 @@ func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]str
p.Severity = Info p.Severity = Info
} }
// the pod the latest event was about is gone: history, not state // the pod the latest event was about is gone: history, not state
if p.Object != "" && !live[namespaceOfID(k.id)+"/"+p.Object] { pod := namespaceOfID(k.id) + "/" + p.Object
if p.Object != "" && !live[pod] {
p.Severity = Info
}
// it could not be scheduled, but has been since: a CronJob pod
// that waited all night for its node to come back, then ran
if p.Reason == "FailedScheduling" && scheduled[pod] {
p.Severity = Info p.Severity = Info
} }
s.add(k.id, *p) s.add(k.id, *p)
@@ -470,11 +476,25 @@ func redirects(ing networkingv1.Ingress) bool {
var redirectSnippet = regexp.MustCompile(`(?m)\brewrite\s+\S+\s+\S+\s+(permanent|redirect)\s*;|\breturn\s+30[1278]\b`) var redirectSnippet = regexp.MustCompile(`(?m)\brewrite\s+\S+\s+\S+\s+(permanent|redirect)\s*;|\breturn\s+30[1278]\b`)
// podsLive: pods whose warnings still describe the present. A pod that
// finished successfully is history, like one that is gone.
func podsLive(pods []corev1.Pod) map[string]bool { func podsLive(pods []corev1.Pod) map[string]bool {
out := make(map[string]bool, len(pods)) out := make(map[string]bool, len(pods))
for _, p := range pods { for _, p := range pods {
if p.Status.Phase != corev1.PodSucceeded {
out[p.Namespace+"/"+p.Name] = true out[p.Namespace+"/"+p.Name] = true
} }
}
return out
}
func podsScheduled(pods []corev1.Pod) map[string]bool {
out := make(map[string]bool, len(pods))
for _, p := range pods {
if p.Spec.NodeName != "" {
out[p.Namespace+"/"+p.Name] = true
}
}
return out return out
} }
@@ -306,3 +306,38 @@ func TestAlertSeverity(t *testing.T) {
} }
} }
} }
func TestScheduledAndFinishedPodEvents(t *testing.T) {
withNow(t)
pod := func(name string, node string, phase corev1.PodPhase) corev1.Pod {
return corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "games", OwnerReferences: owned("Job", name[:len(name)-2])},
Spec: corev1.PodSpec{NodeName: node}, Status: corev1.PodStatus{Phase: phase}}
}
warn := func(reason, podName string) corev1.Event {
return corev1.Event{Type: corev1.EventTypeWarning, Reason: reason, Count: 28, LastTimestamp: metav1.NewTime(t0),
InvolvedObject: corev1.ObjectReference{Kind: "Pod", Namespace: "games", Name: podName}}
}
g := &Graph{Nodes: []Node{{ID: "ns/games"}}}
attachProblems(g, problemInputs{
pods: []corev1.Pod{
pod("waited-a", "games-vm", corev1.PodSucceeded), // waited for its node overnight, then ran
pod("stuck-a", "", corev1.PodPending), // still cannot be placed
pod("done-a", "xeon", corev1.PodSucceeded),
},
events: []corev1.Event{warn("FailedScheduling", "waited-a"), warn("FailedScheduling", "stuck-a"), warn("FailedMount", "done-a")},
})
var stuck, waited, done bool
for _, p := range g.Nodes[0].Problems {
switch p.Object {
case "stuck-a":
stuck = p.Severity == Warning
case "waited-a":
waited = p.Severity == Info
case "done-a":
done = p.Severity == Info
}
}
if !stuck || !waited || !done {
t.Errorf("pending pod must badge, scheduled or finished must not (stuck=%v waited=%v done=%v): %+v", stuck, waited, done, g.Nodes[0].Problems)
}
}