v0.8.2: no false problems for redirecting Ingresses and deleted pods

Found on the live cluster: d8-monitoring/grafana-v10 routes
prometheus.<domain> to a Service that does not exist, but a snippet
redirects every such request, so the backend is never used. And a
verification pod created and deleted by a backup check kept a warning
badge on its namespace for 30 minutes after it was gone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Виталий Чефанов
2026-09-24 18:25:11 +03:00
co-authored by Claude Opus 5.5
parent 1025c40561
commit b01f884530
4 changed files with 66 additions and 3 deletions
@@ -17,6 +17,7 @@ limitations under the License.
package topology
import (
"regexp"
"sort"
"strconv"
"strings"
@@ -150,6 +151,9 @@ func attachProblems(g *Graph, in problemInputs) {
services[svc.Namespace+"/"+svc.Name] = true
}
for _, ing := range in.ingresses {
if redirects(ing) {
continue
}
for _, name := range IngressBackendServices(ing) {
// Deckhouse's ingress-nginx points its config-reload Ingresses at a
// service that does not exist on purpose
@@ -161,7 +165,7 @@ func attachProblems(g *Graph, in problemInputs) {
for _, n := range in.nodes {
nodeProblems(s, n)
}
eventProblems(s, in.events, podOwner)
eventProblems(s, in.events, podOwner, podsLive(in.pods))
for _, n := range g.Nodes {
if !syntheticProbe(n.Namespace, n.Label) {
readinessProblem(s, n, notReadySince)
@@ -357,7 +361,7 @@ func EventTargetID(ref corev1.ObjectReference, podOwner map[string]string) strin
// eventProblems folds recent Warning events into one problem per card and
// reason, keeping the latest message.
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string) {
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live map[string]bool) {
type key struct{ id, reason string }
folded := map[key]*Problem{}
var order []key
@@ -388,6 +392,10 @@ func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]str
if !eventNotable[p.Reason] && p.Count < eventRepeats {
p.Severity = Info
}
// the pod the latest event was about is gone: history, not state
if p.Object != "" && !live[namespaceOfID(k.id)+"/"+p.Object] {
p.Severity = Info
}
s.add(k.id, *p)
}
}
@@ -412,6 +420,33 @@ func readinessProblem(s *problemSet, n Node, notReadySince map[string]time.Time)
}
}
// redirects reports whether an Ingress answers with a redirect instead of
// proxying, so its backends are never used. Deckhouse's prometheus module
// routes prometheus.<domain> to a Service that does not exist and sends
// every request to grafana.<domain> from a configuration snippet.
func redirects(ing networkingv1.Ingress) bool {
a := ing.Annotations
if a["nginx.ingress.kubernetes.io/permanent-redirect"] != "" || a["nginx.ingress.kubernetes.io/temporal-redirect"] != "" {
return true
}
for _, key := range []string{"nginx.ingress.kubernetes.io/configuration-snippet", "nginx.ingress.kubernetes.io/server-snippet"} {
if redirectSnippet.MatchString(a[key]) {
return true
}
}
return false
}
var redirectSnippet = regexp.MustCompile(`(?m)\brewrite\s+\S+\s+\S+\s+(permanent|redirect)\s*;|\breturn\s+30[1278]\b`)
func podsLive(pods []corev1.Pod) map[string]bool {
out := make(map[string]bool, len(pods))
for _, p := range pods {
out[p.Namespace+"/"+p.Name] = true
}
return out
}
// syntheticProbe: upmeter, Deckhouse's availability monitor, creates short
// lived StatefulSets and pods named upmeter-probe-* that are unschedulable
// on purpose (it measures how fast the control plane reacts). Their
@@ -171,6 +171,30 @@ func TestReadinessGraceAndProbes(t *testing.T) {
}
}
func TestRedirectIngressAndGonePods(t *testing.T) {
withNow(t)
redirect := ingressTo("d8-monitoring", "grafana-v10", "grafana")
redirect.Annotations = map[string]string{"nginx.ingress.kubernetes.io/configuration-snippet": "if ($host = 'prometheus.example.com' ) {\n rewrite ^ https://grafana.example.com$request_uri permanent;\n}\n"}
g := &Graph{Nodes: []Node{{ID: "ns/shop"}, {ID: "ingress/d8-monitoring/grafana-v10"}, {ID: "wl/shop/deployment/web"}}}
attachProblems(g, problemInputs{
ingresses: []networkingv1.Ingress{redirect},
pods: []corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Name: "web-5f-live", Namespace: "shop", OwnerReferences: owned("ReplicaSet", "web-5f")}}},
events: []corev1.Event{
{Type: corev1.EventTypeWarning, Reason: "FailedMount", LastTimestamp: metav1.NewTime(t0), InvolvedObject: corev1.ObjectReference{Kind: "Pod", Namespace: "shop", Name: "web-5f-live"}},
{Type: corev1.EventTypeWarning, Reason: "Failed", Count: 10, LastTimestamp: metav1.NewTime(t0), InvolvedObject: corev1.ObjectReference{Kind: "Pod", Namespace: "shop", Name: "verify-gone"}},
},
})
if len(problemsOf(g, "ingress/d8-monitoring/grafana-v10")) != 0 {
t.Error("a redirecting Ingress reported for its unused backend")
}
if problemsOf(g, "wl/shop/deployment/web")["FailedMount"].Severity != Warning {
t.Error("event of a live pod must badge")
}
if p := problemsOf(g, "ns/shop")["Failed"]; p.Severity != Info {
t.Errorf("event of a deleted pod must be information: %+v", p)
}
}
func alert(name, level string, labels map[string]string) *unstructured.Unstructured {
l := map[string]any{}
for k, v := range labels {