v0.11.2: events of finished or since-scheduled pods are history
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
8ed76cd01a
commit
4a63f57674
@@ -1,3 +1,6 @@
|
|||||||
|
v0.11.2:
|
||||||
|
fixes:
|
||||||
|
- "Warning events of a pod that has since finished, and FailedScheduling of a pod that has since been scheduled, are listed for information and no longer badge the card: a backup CronJob that waited all night for its node and then ran stayed yellow for half an hour."
|
||||||
v0.11.1:
|
v0.11.1:
|
||||||
fixes:
|
fixes:
|
||||||
- "A container restarted three times or more stays marked for 24 hours after its last restart or OOM kill, not 30 minutes: a pod OOM-killed five times in a day was no longer shown 40 minutes after the last kill. A single restart still clears after 30 minutes."
|
- "A container restarted three times or more stays marked for 24 hours after its last restart or OOM kill, not 30 minutes: a pod OOM-killed five times in a day was no longer shown 40 minutes after the last kill. A single restart still clears after 30 minutes."
|
||||||
|
|||||||
@@ -178,7 +178,7 @@ func attachProblems(g *Graph, in problemInputs) {
|
|||||||
for _, n := range in.nodes {
|
for _, n := range in.nodes {
|
||||||
nodeProblems(s, n)
|
nodeProblems(s, n)
|
||||||
}
|
}
|
||||||
eventProblems(s, in.events, podOwner, podsLive(in.pods))
|
eventProblems(s, in.events, podOwner, podsLive(in.pods), podsScheduled(in.pods))
|
||||||
for _, n := range g.Nodes {
|
for _, n := range g.Nodes {
|
||||||
if !syntheticProbe(n.Namespace, n.Label) {
|
if !syntheticProbe(n.Namespace, n.Label) {
|
||||||
readinessProblem(s, n, notReadySince)
|
readinessProblem(s, n, notReadySince)
|
||||||
@@ -396,7 +396,7 @@ func eventsInWindow(e corev1.Event, last time.Time) int {
|
|||||||
|
|
||||||
// eventProblems folds recent Warning events into one problem per card and
|
// eventProblems folds recent Warning events into one problem per card and
|
||||||
// reason, keeping the latest message.
|
// reason, keeping the latest message.
|
||||||
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live map[string]bool) {
|
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live, scheduled map[string]bool) {
|
||||||
type key struct{ id, reason string }
|
type key struct{ id, reason string }
|
||||||
folded := map[key]*Problem{}
|
folded := map[key]*Problem{}
|
||||||
var order []key
|
var order []key
|
||||||
@@ -424,7 +424,13 @@ func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]str
|
|||||||
p.Severity = Info
|
p.Severity = Info
|
||||||
}
|
}
|
||||||
// the pod the latest event was about is gone: history, not state
|
// the pod the latest event was about is gone: history, not state
|
||||||
if p.Object != "" && !live[namespaceOfID(k.id)+"/"+p.Object] {
|
pod := namespaceOfID(k.id) + "/" + p.Object
|
||||||
|
if p.Object != "" && !live[pod] {
|
||||||
|
p.Severity = Info
|
||||||
|
}
|
||||||
|
// it could not be scheduled, but has been since: a CronJob pod
|
||||||
|
// that waited all night for its node to come back, then ran
|
||||||
|
if p.Reason == "FailedScheduling" && scheduled[pod] {
|
||||||
p.Severity = Info
|
p.Severity = Info
|
||||||
}
|
}
|
||||||
s.add(k.id, *p)
|
s.add(k.id, *p)
|
||||||
@@ -470,10 +476,24 @@ func redirects(ing networkingv1.Ingress) bool {
|
|||||||
|
|
||||||
var redirectSnippet = regexp.MustCompile(`(?m)\brewrite\s+\S+\s+\S+\s+(permanent|redirect)\s*;|\breturn\s+30[1278]\b`)
|
var redirectSnippet = regexp.MustCompile(`(?m)\brewrite\s+\S+\s+\S+\s+(permanent|redirect)\s*;|\breturn\s+30[1278]\b`)
|
||||||
|
|
||||||
|
// podsLive: pods whose warnings still describe the present. A pod that
|
||||||
|
// finished successfully is history, like one that is gone.
|
||||||
func podsLive(pods []corev1.Pod) map[string]bool {
|
func podsLive(pods []corev1.Pod) map[string]bool {
|
||||||
out := make(map[string]bool, len(pods))
|
out := make(map[string]bool, len(pods))
|
||||||
for _, p := range pods {
|
for _, p := range pods {
|
||||||
out[p.Namespace+"/"+p.Name] = true
|
if p.Status.Phase != corev1.PodSucceeded {
|
||||||
|
out[p.Namespace+"/"+p.Name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func podsScheduled(pods []corev1.Pod) map[string]bool {
|
||||||
|
out := make(map[string]bool, len(pods))
|
||||||
|
for _, p := range pods {
|
||||||
|
if p.Spec.NodeName != "" {
|
||||||
|
out[p.Namespace+"/"+p.Name] = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -306,3 +306,38 @@ func TestAlertSeverity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestScheduledAndFinishedPodEvents(t *testing.T) {
|
||||||
|
withNow(t)
|
||||||
|
pod := func(name string, node string, phase corev1.PodPhase) corev1.Pod {
|
||||||
|
return corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "games", OwnerReferences: owned("Job", name[:len(name)-2])},
|
||||||
|
Spec: corev1.PodSpec{NodeName: node}, Status: corev1.PodStatus{Phase: phase}}
|
||||||
|
}
|
||||||
|
warn := func(reason, podName string) corev1.Event {
|
||||||
|
return corev1.Event{Type: corev1.EventTypeWarning, Reason: reason, Count: 28, LastTimestamp: metav1.NewTime(t0),
|
||||||
|
InvolvedObject: corev1.ObjectReference{Kind: "Pod", Namespace: "games", Name: podName}}
|
||||||
|
}
|
||||||
|
g := &Graph{Nodes: []Node{{ID: "ns/games"}}}
|
||||||
|
attachProblems(g, problemInputs{
|
||||||
|
pods: []corev1.Pod{
|
||||||
|
pod("waited-a", "games-vm", corev1.PodSucceeded), // waited for its node overnight, then ran
|
||||||
|
pod("stuck-a", "", corev1.PodPending), // still cannot be placed
|
||||||
|
pod("done-a", "xeon", corev1.PodSucceeded),
|
||||||
|
},
|
||||||
|
events: []corev1.Event{warn("FailedScheduling", "waited-a"), warn("FailedScheduling", "stuck-a"), warn("FailedMount", "done-a")},
|
||||||
|
})
|
||||||
|
var stuck, waited, done bool
|
||||||
|
for _, p := range g.Nodes[0].Problems {
|
||||||
|
switch p.Object {
|
||||||
|
case "stuck-a":
|
||||||
|
stuck = p.Severity == Warning
|
||||||
|
case "waited-a":
|
||||||
|
waited = p.Severity == Info
|
||||||
|
case "done-a":
|
||||||
|
done = p.Severity == Info
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !stuck || !waited || !done {
|
||||||
|
t.Errorf("pending pod must badge, scheduled or finished must not (stuck=%v waited=%v done=%v): %+v", stuck, waited, done, g.Nodes[0].Problems)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user