diff --git a/changelog.yaml b/changelog.yaml index fd381c4..0c8581f 100644 --- a/changelog.yaml +++ b/changelog.yaml @@ -1,3 +1,6 @@ +v0.11.2: + fixes: + - "Warning events of a pod that has since finished, and FailedScheduling of a pod that has since been scheduled, are listed for information and no longer badge the card: a backup CronJob that waited all night for its node and then ran stayed yellow for half an hour." v0.11.1: fixes: - "A container restarted three times or more stays marked for 24 hours after its last restart or OOM kill, not 30 minutes: a pod OOM-killed five times in a day was no longer shown 40 minutes after the last kill. A single restart still clears after 30 minutes." diff --git a/images/cluster-map/src/internal/topology/problems.go b/images/cluster-map/src/internal/topology/problems.go index b4f0f09..75cf15c 100644 --- a/images/cluster-map/src/internal/topology/problems.go +++ b/images/cluster-map/src/internal/topology/problems.go @@ -178,7 +178,7 @@ func attachProblems(g *Graph, in problemInputs) { for _, n := range in.nodes { nodeProblems(s, n) } - eventProblems(s, in.events, podOwner, podsLive(in.pods)) + eventProblems(s, in.events, podOwner, podsLive(in.pods), podsScheduled(in.pods)) for _, n := range g.Nodes { if !syntheticProbe(n.Namespace, n.Label) { readinessProblem(s, n, notReadySince) @@ -396,7 +396,7 @@ func eventsInWindow(e corev1.Event, last time.Time) int { // eventProblems folds recent Warning events into one problem per card and // reason, keeping the latest message. -func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live map[string]bool) { +func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live, scheduled map[string]bool) { type key struct{ id, reason string } folded := map[key]*Problem{} var order []key @@ -424,7 +424,13 @@ func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]str p.Severity = Info } // the pod the latest event was about is gone: history, not state - if p.Object != "" && !live[namespaceOfID(k.id)+"/"+p.Object] { + pod := namespaceOfID(k.id) + "/" + p.Object + if p.Object != "" && !live[pod] { + p.Severity = Info + } + // it could not be scheduled, but has been since: a CronJob pod + // that waited all night for its node to come back, then ran + if p.Reason == "FailedScheduling" && scheduled[pod] { p.Severity = Info } s.add(k.id, *p) @@ -470,10 +476,24 @@ func redirects(ing networkingv1.Ingress) bool { var redirectSnippet = regexp.MustCompile(`(?m)\brewrite\s+\S+\s+\S+\s+(permanent|redirect)\s*;|\breturn\s+30[1278]\b`) +// podsLive: pods whose warnings still describe the present. A pod that +// finished successfully is history, like one that is gone. func podsLive(pods []corev1.Pod) map[string]bool { out := make(map[string]bool, len(pods)) for _, p := range pods { - out[p.Namespace+"/"+p.Name] = true + if p.Status.Phase != corev1.PodSucceeded { + out[p.Namespace+"/"+p.Name] = true + } + } + return out +} + +func podsScheduled(pods []corev1.Pod) map[string]bool { + out := make(map[string]bool, len(pods)) + for _, p := range pods { + if p.Spec.NodeName != "" { + out[p.Namespace+"/"+p.Name] = true + } } return out } diff --git a/images/cluster-map/src/internal/topology/problems_test.go b/images/cluster-map/src/internal/topology/problems_test.go index c592ee6..20302b4 100644 --- a/images/cluster-map/src/internal/topology/problems_test.go +++ b/images/cluster-map/src/internal/topology/problems_test.go @@ -306,3 +306,38 @@ func TestAlertSeverity(t *testing.T) { } } } + +func TestScheduledAndFinishedPodEvents(t *testing.T) { + withNow(t) + pod := func(name string, node string, phase corev1.PodPhase) corev1.Pod { + return corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "games", OwnerReferences: owned("Job", name[:len(name)-2])}, + Spec: corev1.PodSpec{NodeName: node}, Status: corev1.PodStatus{Phase: phase}} + } + warn := func(reason, podName string) corev1.Event { + return corev1.Event{Type: corev1.EventTypeWarning, Reason: reason, Count: 28, LastTimestamp: metav1.NewTime(t0), + InvolvedObject: corev1.ObjectReference{Kind: "Pod", Namespace: "games", Name: podName}} + } + g := &Graph{Nodes: []Node{{ID: "ns/games"}}} + attachProblems(g, problemInputs{ + pods: []corev1.Pod{ + pod("waited-a", "games-vm", corev1.PodSucceeded), // waited for its node overnight, then ran + pod("stuck-a", "", corev1.PodPending), // still cannot be placed + pod("done-a", "xeon", corev1.PodSucceeded), + }, + events: []corev1.Event{warn("FailedScheduling", "waited-a"), warn("FailedScheduling", "stuck-a"), warn("FailedMount", "done-a")}, + }) + var stuck, waited, done bool + for _, p := range g.Nodes[0].Problems { + switch p.Object { + case "stuck-a": + stuck = p.Severity == Warning + case "waited-a": + waited = p.Severity == Info + case "done-a": + done = p.Severity == Info + } + } + if !stuck || !waited || !done { + t.Errorf("pending pod must badge, scheduled or finished must not (stuck=%v waited=%v done=%v): %+v", stuck, waited, done, g.Nodes[0].Problems) + } +}