v0.8.2: no false problems for redirecting Ingresses and deleted pods
Found on the live cluster: d8-monitoring/grafana-v10 routes prometheus.<domain> to a Service that does not exist, but a snippet redirects every such request, so the backend is never used. And a verification pod created and deleted by a backup check kept a warning badge on its namespace for 30 minutes after it was gone. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
1025c40561
commit
b01f884530
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
name: cluster-map
|
name: cluster-map
|
||||||
version: 0.8.1
|
version: 0.8.2
|
||||||
dependencies:
|
dependencies:
|
||||||
- name: deckhouse_lib_helm
|
- name: deckhouse_lib_helm
|
||||||
version: 1.72.24
|
version: 1.72.24
|
||||||
|
|||||||
@@ -1,3 +1,7 @@
|
|||||||
|
v0.8.2:
|
||||||
|
fixes:
|
||||||
|
- An Ingress that answers with a redirect (redirect annotations, or rewrite ... permanent / return 30x in a snippet) is not reported for a backend Service that does not exist; Deckhouse's prometheus Ingress does exactly that.
|
||||||
|
- Warning events about a pod that no longer exists are listed for information and no longer badge the card.
|
||||||
v0.8.1:
|
v0.8.1:
|
||||||
fixes:
|
fixes:
|
||||||
- "Actions: the DexAuthenticator is annotated allow-access-to-kubernetes, so the user's token carries the kubernetes audience and the API server accepts it. In v0.8.0 the token lacked that audience, the API server rejected it and actions were unavailable."
|
- "Actions: the DexAuthenticator is annotated allow-access-to-kubernetes, so the user's token carries the kubernetes audience and the API server accepts it. In v0.8.0 the token lacked that audience, the API server rejected it and actions were unavailable."
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ limitations under the License.
|
|||||||
package topology
|
package topology
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"regexp"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -150,6 +151,9 @@ func attachProblems(g *Graph, in problemInputs) {
|
|||||||
services[svc.Namespace+"/"+svc.Name] = true
|
services[svc.Namespace+"/"+svc.Name] = true
|
||||||
}
|
}
|
||||||
for _, ing := range in.ingresses {
|
for _, ing := range in.ingresses {
|
||||||
|
if redirects(ing) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
for _, name := range IngressBackendServices(ing) {
|
for _, name := range IngressBackendServices(ing) {
|
||||||
// Deckhouse's ingress-nginx points its config-reload Ingresses at a
|
// Deckhouse's ingress-nginx points its config-reload Ingresses at a
|
||||||
// service that does not exist on purpose
|
// service that does not exist on purpose
|
||||||
@@ -161,7 +165,7 @@ func attachProblems(g *Graph, in problemInputs) {
|
|||||||
for _, n := range in.nodes {
|
for _, n := range in.nodes {
|
||||||
nodeProblems(s, n)
|
nodeProblems(s, n)
|
||||||
}
|
}
|
||||||
eventProblems(s, in.events, podOwner)
|
eventProblems(s, in.events, podOwner, podsLive(in.pods))
|
||||||
for _, n := range g.Nodes {
|
for _, n := range g.Nodes {
|
||||||
if !syntheticProbe(n.Namespace, n.Label) {
|
if !syntheticProbe(n.Namespace, n.Label) {
|
||||||
readinessProblem(s, n, notReadySince)
|
readinessProblem(s, n, notReadySince)
|
||||||
@@ -357,7 +361,7 @@ func EventTargetID(ref corev1.ObjectReference, podOwner map[string]string) strin
|
|||||||
|
|
||||||
// eventProblems folds recent Warning events into one problem per card and
|
// eventProblems folds recent Warning events into one problem per card and
|
||||||
// reason, keeping the latest message.
|
// reason, keeping the latest message.
|
||||||
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string) {
|
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live map[string]bool) {
|
||||||
type key struct{ id, reason string }
|
type key struct{ id, reason string }
|
||||||
folded := map[key]*Problem{}
|
folded := map[key]*Problem{}
|
||||||
var order []key
|
var order []key
|
||||||
@@ -388,6 +392,10 @@ func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]str
|
|||||||
if !eventNotable[p.Reason] && p.Count < eventRepeats {
|
if !eventNotable[p.Reason] && p.Count < eventRepeats {
|
||||||
p.Severity = Info
|
p.Severity = Info
|
||||||
}
|
}
|
||||||
|
// the pod the latest event was about is gone: history, not state
|
||||||
|
if p.Object != "" && !live[namespaceOfID(k.id)+"/"+p.Object] {
|
||||||
|
p.Severity = Info
|
||||||
|
}
|
||||||
s.add(k.id, *p)
|
s.add(k.id, *p)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -412,6 +420,33 @@ func readinessProblem(s *problemSet, n Node, notReadySince map[string]time.Time)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// redirects reports whether an Ingress answers with a redirect instead of
|
||||||
|
// proxying, so its backends are never used. Deckhouse's prometheus module
|
||||||
|
// routes prometheus.<domain> to a Service that does not exist and sends
|
||||||
|
// every request to grafana.<domain> from a configuration snippet.
|
||||||
|
func redirects(ing networkingv1.Ingress) bool {
|
||||||
|
a := ing.Annotations
|
||||||
|
if a["nginx.ingress.kubernetes.io/permanent-redirect"] != "" || a["nginx.ingress.kubernetes.io/temporal-redirect"] != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, key := range []string{"nginx.ingress.kubernetes.io/configuration-snippet", "nginx.ingress.kubernetes.io/server-snippet"} {
|
||||||
|
if redirectSnippet.MatchString(a[key]) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
var redirectSnippet = regexp.MustCompile(`(?m)\brewrite\s+\S+\s+\S+\s+(permanent|redirect)\s*;|\breturn\s+30[1278]\b`)
|
||||||
|
|
||||||
|
func podsLive(pods []corev1.Pod) map[string]bool {
|
||||||
|
out := make(map[string]bool, len(pods))
|
||||||
|
for _, p := range pods {
|
||||||
|
out[p.Namespace+"/"+p.Name] = true
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// syntheticProbe: upmeter, Deckhouse's availability monitor, creates short
|
// syntheticProbe: upmeter, Deckhouse's availability monitor, creates short
|
||||||
// lived StatefulSets and pods named upmeter-probe-* that are unschedulable
|
// lived StatefulSets and pods named upmeter-probe-* that are unschedulable
|
||||||
// on purpose (it measures how fast the control plane reacts). Their
|
// on purpose (it measures how fast the control plane reacts). Their
|
||||||
|
|||||||
@@ -171,6 +171,30 @@ func TestReadinessGraceAndProbes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRedirectIngressAndGonePods(t *testing.T) {
|
||||||
|
withNow(t)
|
||||||
|
redirect := ingressTo("d8-monitoring", "grafana-v10", "grafana")
|
||||||
|
redirect.Annotations = map[string]string{"nginx.ingress.kubernetes.io/configuration-snippet": "if ($host = 'prometheus.example.com' ) {\n rewrite ^ https://grafana.example.com$request_uri permanent;\n}\n"}
|
||||||
|
g := &Graph{Nodes: []Node{{ID: "ns/shop"}, {ID: "ingress/d8-monitoring/grafana-v10"}, {ID: "wl/shop/deployment/web"}}}
|
||||||
|
attachProblems(g, problemInputs{
|
||||||
|
ingresses: []networkingv1.Ingress{redirect},
|
||||||
|
pods: []corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Name: "web-5f-live", Namespace: "shop", OwnerReferences: owned("ReplicaSet", "web-5f")}}},
|
||||||
|
events: []corev1.Event{
|
||||||
|
{Type: corev1.EventTypeWarning, Reason: "FailedMount", LastTimestamp: metav1.NewTime(t0), InvolvedObject: corev1.ObjectReference{Kind: "Pod", Namespace: "shop", Name: "web-5f-live"}},
|
||||||
|
{Type: corev1.EventTypeWarning, Reason: "Failed", Count: 10, LastTimestamp: metav1.NewTime(t0), InvolvedObject: corev1.ObjectReference{Kind: "Pod", Namespace: "shop", Name: "verify-gone"}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if len(problemsOf(g, "ingress/d8-monitoring/grafana-v10")) != 0 {
|
||||||
|
t.Error("a redirecting Ingress reported for its unused backend")
|
||||||
|
}
|
||||||
|
if problemsOf(g, "wl/shop/deployment/web")["FailedMount"].Severity != Warning {
|
||||||
|
t.Error("event of a live pod must badge")
|
||||||
|
}
|
||||||
|
if p := problemsOf(g, "ns/shop")["Failed"]; p.Severity != Info {
|
||||||
|
t.Errorf("event of a deleted pod must be information: %+v", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func alert(name, level string, labels map[string]string) *unstructured.Unstructured {
|
func alert(name, level string, labels map[string]string) *unstructured.Unstructured {
|
||||||
l := map[string]any{}
|
l := map[string]any{}
|
||||||
for k, v := range labels {
|
for k, v := range labels {
|
||||||
|
|||||||
Reference in New Issue
Block a user