v0.11.2: events of finished or since-scheduled pods are history
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
8ed76cd01a
commit
4a63f57674
@@ -1,3 +1,6 @@
|
||||
v0.11.2:
|
||||
fixes:
|
||||
- "Warning events of a pod that has since finished, and FailedScheduling of a pod that has since been scheduled, are listed for information and no longer badge the card: a backup CronJob that waited all night for its node and then ran stayed yellow for half an hour."
|
||||
v0.11.1:
|
||||
fixes:
|
||||
- "A container restarted three times or more stays marked for 24 hours after its last restart or OOM kill, not 30 minutes: a pod OOM-killed five times in a day was no longer shown 40 minutes after the last kill. A single restart still clears after 30 minutes."
|
||||
|
||||
@@ -178,7 +178,7 @@ func attachProblems(g *Graph, in problemInputs) {
|
||||
for _, n := range in.nodes {
|
||||
nodeProblems(s, n)
|
||||
}
|
||||
eventProblems(s, in.events, podOwner, podsLive(in.pods))
|
||||
eventProblems(s, in.events, podOwner, podsLive(in.pods), podsScheduled(in.pods))
|
||||
for _, n := range g.Nodes {
|
||||
if !syntheticProbe(n.Namespace, n.Label) {
|
||||
readinessProblem(s, n, notReadySince)
|
||||
@@ -396,7 +396,7 @@ func eventsInWindow(e corev1.Event, last time.Time) int {
|
||||
|
||||
// eventProblems folds recent Warning events into one problem per card and
|
||||
// reason, keeping the latest message.
|
||||
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live map[string]bool) {
|
||||
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live, scheduled map[string]bool) {
|
||||
type key struct{ id, reason string }
|
||||
folded := map[key]*Problem{}
|
||||
var order []key
|
||||
@@ -424,7 +424,13 @@ func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]str
|
||||
p.Severity = Info
|
||||
}
|
||||
// the pod the latest event was about is gone: history, not state
|
||||
if p.Object != "" && !live[namespaceOfID(k.id)+"/"+p.Object] {
|
||||
pod := namespaceOfID(k.id) + "/" + p.Object
|
||||
if p.Object != "" && !live[pod] {
|
||||
p.Severity = Info
|
||||
}
|
||||
// it could not be scheduled, but has been since: a CronJob pod
|
||||
// that waited all night for its node to come back, then ran
|
||||
if p.Reason == "FailedScheduling" && scheduled[pod] {
|
||||
p.Severity = Info
|
||||
}
|
||||
s.add(k.id, *p)
|
||||
@@ -470,11 +476,25 @@ func redirects(ing networkingv1.Ingress) bool {
|
||||
|
||||
var redirectSnippet = regexp.MustCompile(`(?m)\brewrite\s+\S+\s+\S+\s+(permanent|redirect)\s*;|\breturn\s+30[1278]\b`)
|
||||
|
||||
// podsLive: pods whose warnings still describe the present. A pod that
|
||||
// finished successfully is history, like one that is gone.
|
||||
func podsLive(pods []corev1.Pod) map[string]bool {
|
||||
out := make(map[string]bool, len(pods))
|
||||
for _, p := range pods {
|
||||
if p.Status.Phase != corev1.PodSucceeded {
|
||||
out[p.Namespace+"/"+p.Name] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func podsScheduled(pods []corev1.Pod) map[string]bool {
|
||||
out := make(map[string]bool, len(pods))
|
||||
for _, p := range pods {
|
||||
if p.Spec.NodeName != "" {
|
||||
out[p.Namespace+"/"+p.Name] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -306,3 +306,38 @@ func TestAlertSeverity(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduledAndFinishedPodEvents(t *testing.T) {
|
||||
withNow(t)
|
||||
pod := func(name string, node string, phase corev1.PodPhase) corev1.Pod {
|
||||
return corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "games", OwnerReferences: owned("Job", name[:len(name)-2])},
|
||||
Spec: corev1.PodSpec{NodeName: node}, Status: corev1.PodStatus{Phase: phase}}
|
||||
}
|
||||
warn := func(reason, podName string) corev1.Event {
|
||||
return corev1.Event{Type: corev1.EventTypeWarning, Reason: reason, Count: 28, LastTimestamp: metav1.NewTime(t0),
|
||||
InvolvedObject: corev1.ObjectReference{Kind: "Pod", Namespace: "games", Name: podName}}
|
||||
}
|
||||
g := &Graph{Nodes: []Node{{ID: "ns/games"}}}
|
||||
attachProblems(g, problemInputs{
|
||||
pods: []corev1.Pod{
|
||||
pod("waited-a", "games-vm", corev1.PodSucceeded), // waited for its node overnight, then ran
|
||||
pod("stuck-a", "", corev1.PodPending), // still cannot be placed
|
||||
pod("done-a", "xeon", corev1.PodSucceeded),
|
||||
},
|
||||
events: []corev1.Event{warn("FailedScheduling", "waited-a"), warn("FailedScheduling", "stuck-a"), warn("FailedMount", "done-a")},
|
||||
})
|
||||
var stuck, waited, done bool
|
||||
for _, p := range g.Nodes[0].Problems {
|
||||
switch p.Object {
|
||||
case "stuck-a":
|
||||
stuck = p.Severity == Warning
|
||||
case "waited-a":
|
||||
waited = p.Severity == Info
|
||||
case "done-a":
|
||||
done = p.Severity == Info
|
||||
}
|
||||
}
|
||||
if !stuck || !waited || !done {
|
||||
t.Errorf("pending pod must badge, scheduled or finished must not (stuck=%v waited=%v done=%v): %+v", stuck, waited, done, g.Nodes[0].Problems)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user