v0.8.2: no false problems for redirecting Ingresses and deleted pods
Found on the live cluster: d8-monitoring/grafana-v10 routes prometheus.<domain> to a Service that does not exist, but a snippet redirects every such request, so the backend is never used. And a verification pod created and deleted by a backup check kept a warning badge on its namespace for 30 minutes after it was gone. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
1025c40561
commit
b01f884530
+1
-1
@@ -1,5 +1,5 @@
|
||||
name: cluster-map
|
||||
version: 0.8.1
|
||||
version: 0.8.2
|
||||
dependencies:
|
||||
- name: deckhouse_lib_helm
|
||||
version: 1.72.24
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
v0.8.2:
|
||||
fixes:
|
||||
- An Ingress that answers with a redirect (redirect annotations, or rewrite ... permanent / return 30x in a snippet) is not reported for a backend Service that does not exist; Deckhouse's prometheus Ingress does exactly that.
|
||||
- Warning events about a pod that no longer exists are listed for information and no longer badge the card.
|
||||
v0.8.1:
|
||||
fixes:
|
||||
- "Actions: the DexAuthenticator is annotated allow-access-to-kubernetes, so the user's token carries the kubernetes audience and the API server accepts it. In v0.8.0 the token lacked that audience, the API server rejected it and actions were unavailable."
|
||||
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
package topology
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -150,6 +151,9 @@ func attachProblems(g *Graph, in problemInputs) {
|
||||
services[svc.Namespace+"/"+svc.Name] = true
|
||||
}
|
||||
for _, ing := range in.ingresses {
|
||||
if redirects(ing) {
|
||||
continue
|
||||
}
|
||||
for _, name := range IngressBackendServices(ing) {
|
||||
// Deckhouse's ingress-nginx points its config-reload Ingresses at a
|
||||
// service that does not exist on purpose
|
||||
@@ -161,7 +165,7 @@ func attachProblems(g *Graph, in problemInputs) {
|
||||
for _, n := range in.nodes {
|
||||
nodeProblems(s, n)
|
||||
}
|
||||
eventProblems(s, in.events, podOwner)
|
||||
eventProblems(s, in.events, podOwner, podsLive(in.pods))
|
||||
for _, n := range g.Nodes {
|
||||
if !syntheticProbe(n.Namespace, n.Label) {
|
||||
readinessProblem(s, n, notReadySince)
|
||||
@@ -357,7 +361,7 @@ func EventTargetID(ref corev1.ObjectReference, podOwner map[string]string) strin
|
||||
|
||||
// eventProblems folds recent Warning events into one problem per card and
|
||||
// reason, keeping the latest message.
|
||||
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string) {
|
||||
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live map[string]bool) {
|
||||
type key struct{ id, reason string }
|
||||
folded := map[key]*Problem{}
|
||||
var order []key
|
||||
@@ -388,6 +392,10 @@ func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]str
|
||||
if !eventNotable[p.Reason] && p.Count < eventRepeats {
|
||||
p.Severity = Info
|
||||
}
|
||||
// the pod the latest event was about is gone: history, not state
|
||||
if p.Object != "" && !live[namespaceOfID(k.id)+"/"+p.Object] {
|
||||
p.Severity = Info
|
||||
}
|
||||
s.add(k.id, *p)
|
||||
}
|
||||
}
|
||||
@@ -412,6 +420,33 @@ func readinessProblem(s *problemSet, n Node, notReadySince map[string]time.Time)
|
||||
}
|
||||
}
|
||||
|
||||
// redirects reports whether an Ingress answers with a redirect instead of
|
||||
// proxying, so its backends are never used. Deckhouse's prometheus module
|
||||
// routes prometheus.<domain> to a Service that does not exist and sends
|
||||
// every request to grafana.<domain> from a configuration snippet.
|
||||
func redirects(ing networkingv1.Ingress) bool {
|
||||
a := ing.Annotations
|
||||
if a["nginx.ingress.kubernetes.io/permanent-redirect"] != "" || a["nginx.ingress.kubernetes.io/temporal-redirect"] != "" {
|
||||
return true
|
||||
}
|
||||
for _, key := range []string{"nginx.ingress.kubernetes.io/configuration-snippet", "nginx.ingress.kubernetes.io/server-snippet"} {
|
||||
if redirectSnippet.MatchString(a[key]) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var redirectSnippet = regexp.MustCompile(`(?m)\brewrite\s+\S+\s+\S+\s+(permanent|redirect)\s*;|\breturn\s+30[1278]\b`)
|
||||
|
||||
func podsLive(pods []corev1.Pod) map[string]bool {
|
||||
out := make(map[string]bool, len(pods))
|
||||
for _, p := range pods {
|
||||
out[p.Namespace+"/"+p.Name] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// syntheticProbe: upmeter, Deckhouse's availability monitor, creates short
|
||||
// lived StatefulSets and pods named upmeter-probe-* that are unschedulable
|
||||
// on purpose (it measures how fast the control plane reacts). Their
|
||||
|
||||
@@ -171,6 +171,30 @@ func TestReadinessGraceAndProbes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedirectIngressAndGonePods(t *testing.T) {
|
||||
withNow(t)
|
||||
redirect := ingressTo("d8-monitoring", "grafana-v10", "grafana")
|
||||
redirect.Annotations = map[string]string{"nginx.ingress.kubernetes.io/configuration-snippet": "if ($host = 'prometheus.example.com' ) {\n rewrite ^ https://grafana.example.com$request_uri permanent;\n}\n"}
|
||||
g := &Graph{Nodes: []Node{{ID: "ns/shop"}, {ID: "ingress/d8-monitoring/grafana-v10"}, {ID: "wl/shop/deployment/web"}}}
|
||||
attachProblems(g, problemInputs{
|
||||
ingresses: []networkingv1.Ingress{redirect},
|
||||
pods: []corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Name: "web-5f-live", Namespace: "shop", OwnerReferences: owned("ReplicaSet", "web-5f")}}},
|
||||
events: []corev1.Event{
|
||||
{Type: corev1.EventTypeWarning, Reason: "FailedMount", LastTimestamp: metav1.NewTime(t0), InvolvedObject: corev1.ObjectReference{Kind: "Pod", Namespace: "shop", Name: "web-5f-live"}},
|
||||
{Type: corev1.EventTypeWarning, Reason: "Failed", Count: 10, LastTimestamp: metav1.NewTime(t0), InvolvedObject: corev1.ObjectReference{Kind: "Pod", Namespace: "shop", Name: "verify-gone"}},
|
||||
},
|
||||
})
|
||||
if len(problemsOf(g, "ingress/d8-monitoring/grafana-v10")) != 0 {
|
||||
t.Error("a redirecting Ingress reported for its unused backend")
|
||||
}
|
||||
if problemsOf(g, "wl/shop/deployment/web")["FailedMount"].Severity != Warning {
|
||||
t.Error("event of a live pod must badge")
|
||||
}
|
||||
if p := problemsOf(g, "ns/shop")["Failed"]; p.Severity != Info {
|
||||
t.Errorf("event of a deleted pod must be information: %+v", p)
|
||||
}
|
||||
}
|
||||
|
||||
func alert(name, level string, labels map[string]string) *unstructured.Unstructured {
|
||||
l := map[string]any{}
|
||||
for k, v := range labels {
|
||||
|
||||
Reference in New Issue
Block a user