v0.8.2: no false problems for redirecting Ingresses and deleted pods

Found on the live cluster: d8-monitoring/grafana-v10 routes
prometheus.<domain> to a Service that does not exist, but a snippet
redirects every such request, so the backend is never used. And a
verification pod created and deleted by a backup check kept a warning
badge on its namespace for 30 minutes after it was gone.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Виталий Чефанов
2026-09-24 18:25:11 +03:00
co-authored by Claude Opus 5.5
parent 1025c40561
commit b01f884530
4 changed files with 66 additions and 3 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
name: cluster-map name: cluster-map
version: 0.8.1 version: 0.8.2
dependencies: dependencies:
- name: deckhouse_lib_helm - name: deckhouse_lib_helm
version: 1.72.24 version: 1.72.24
+4
View File
@@ -1,3 +1,7 @@
v0.8.2:
fixes:
- An Ingress that answers with a redirect (redirect annotations, or rewrite ... permanent / return 30x in a snippet) is not reported for a backend Service that does not exist; Deckhouse's prometheus Ingress does exactly that.
- Warning events about a pod that no longer exists are listed for information and no longer badge the card.
v0.8.1: v0.8.1:
fixes: fixes:
- "Actions: the DexAuthenticator is annotated allow-access-to-kubernetes, so the user's token carries the kubernetes audience and the API server accepts it. In v0.8.0 the token lacked that audience, the API server rejected it and actions were unavailable." - "Actions: the DexAuthenticator is annotated allow-access-to-kubernetes, so the user's token carries the kubernetes audience and the API server accepts it. In v0.8.0 the token lacked that audience, the API server rejected it and actions were unavailable."
@@ -17,6 +17,7 @@ limitations under the License.
package topology package topology
import ( import (
"regexp"
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
@@ -150,6 +151,9 @@ func attachProblems(g *Graph, in problemInputs) {
services[svc.Namespace+"/"+svc.Name] = true services[svc.Namespace+"/"+svc.Name] = true
} }
for _, ing := range in.ingresses { for _, ing := range in.ingresses {
if redirects(ing) {
continue
}
for _, name := range IngressBackendServices(ing) { for _, name := range IngressBackendServices(ing) {
// Deckhouse's ingress-nginx points its config-reload Ingresses at a // Deckhouse's ingress-nginx points its config-reload Ingresses at a
// service that does not exist on purpose // service that does not exist on purpose
@@ -161,7 +165,7 @@ func attachProblems(g *Graph, in problemInputs) {
for _, n := range in.nodes { for _, n := range in.nodes {
nodeProblems(s, n) nodeProblems(s, n)
} }
eventProblems(s, in.events, podOwner) eventProblems(s, in.events, podOwner, podsLive(in.pods))
for _, n := range g.Nodes { for _, n := range g.Nodes {
if !syntheticProbe(n.Namespace, n.Label) { if !syntheticProbe(n.Namespace, n.Label) {
readinessProblem(s, n, notReadySince) readinessProblem(s, n, notReadySince)
@@ -357,7 +361,7 @@ func EventTargetID(ref corev1.ObjectReference, podOwner map[string]string) strin
// eventProblems folds recent Warning events into one problem per card and // eventProblems folds recent Warning events into one problem per card and
// reason, keeping the latest message. // reason, keeping the latest message.
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string) { func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live map[string]bool) {
type key struct{ id, reason string } type key struct{ id, reason string }
folded := map[key]*Problem{} folded := map[key]*Problem{}
var order []key var order []key
@@ -388,6 +392,10 @@ func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]str
if !eventNotable[p.Reason] && p.Count < eventRepeats { if !eventNotable[p.Reason] && p.Count < eventRepeats {
p.Severity = Info p.Severity = Info
} }
// the pod the latest event was about is gone: history, not state
if p.Object != "" && !live[namespaceOfID(k.id)+"/"+p.Object] {
p.Severity = Info
}
s.add(k.id, *p) s.add(k.id, *p)
} }
} }
@@ -412,6 +420,33 @@ func readinessProblem(s *problemSet, n Node, notReadySince map[string]time.Time)
} }
} }
// redirects reports whether an Ingress answers with a redirect instead of
// proxying, so its backends are never used. Deckhouse's prometheus module
// routes prometheus.<domain> to a Service that does not exist and sends
// every request to grafana.<domain> from a configuration snippet.
func redirects(ing networkingv1.Ingress) bool {
a := ing.Annotations
if a["nginx.ingress.kubernetes.io/permanent-redirect"] != "" || a["nginx.ingress.kubernetes.io/temporal-redirect"] != "" {
return true
}
for _, key := range []string{"nginx.ingress.kubernetes.io/configuration-snippet", "nginx.ingress.kubernetes.io/server-snippet"} {
if redirectSnippet.MatchString(a[key]) {
return true
}
}
return false
}
var redirectSnippet = regexp.MustCompile(`(?m)\brewrite\s+\S+\s+\S+\s+(permanent|redirect)\s*;|\breturn\s+30[1278]\b`)
func podsLive(pods []corev1.Pod) map[string]bool {
out := make(map[string]bool, len(pods))
for _, p := range pods {
out[p.Namespace+"/"+p.Name] = true
}
return out
}
// syntheticProbe: upmeter, Deckhouse's availability monitor, creates short // syntheticProbe: upmeter, Deckhouse's availability monitor, creates short
// lived StatefulSets and pods named upmeter-probe-* that are unschedulable // lived StatefulSets and pods named upmeter-probe-* that are unschedulable
// on purpose (it measures how fast the control plane reacts). Their // on purpose (it measures how fast the control plane reacts). Their
@@ -171,6 +171,30 @@ func TestReadinessGraceAndProbes(t *testing.T) {
} }
} }
func TestRedirectIngressAndGonePods(t *testing.T) {
withNow(t)
redirect := ingressTo("d8-monitoring", "grafana-v10", "grafana")
redirect.Annotations = map[string]string{"nginx.ingress.kubernetes.io/configuration-snippet": "if ($host = 'prometheus.example.com' ) {\n rewrite ^ https://grafana.example.com$request_uri permanent;\n}\n"}
g := &Graph{Nodes: []Node{{ID: "ns/shop"}, {ID: "ingress/d8-monitoring/grafana-v10"}, {ID: "wl/shop/deployment/web"}}}
attachProblems(g, problemInputs{
ingresses: []networkingv1.Ingress{redirect},
pods: []corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Name: "web-5f-live", Namespace: "shop", OwnerReferences: owned("ReplicaSet", "web-5f")}}},
events: []corev1.Event{
{Type: corev1.EventTypeWarning, Reason: "FailedMount", LastTimestamp: metav1.NewTime(t0), InvolvedObject: corev1.ObjectReference{Kind: "Pod", Namespace: "shop", Name: "web-5f-live"}},
{Type: corev1.EventTypeWarning, Reason: "Failed", Count: 10, LastTimestamp: metav1.NewTime(t0), InvolvedObject: corev1.ObjectReference{Kind: "Pod", Namespace: "shop", Name: "verify-gone"}},
},
})
if len(problemsOf(g, "ingress/d8-monitoring/grafana-v10")) != 0 {
t.Error("a redirecting Ingress reported for its unused backend")
}
if problemsOf(g, "wl/shop/deployment/web")["FailedMount"].Severity != Warning {
t.Error("event of a live pod must badge")
}
if p := problemsOf(g, "ns/shop")["Failed"]; p.Severity != Info {
t.Errorf("event of a deleted pod must be information: %+v", p)
}
}
func alert(name, level string, labels map[string]string) *unstructured.Unstructured { func alert(name, level string, labels map[string]string) *unstructured.Unstructured {
l := map[string]any{} l := map[string]any{}
for k, v := range labels { for k, v := range labels {