fix: classify healthy topology routing fallback as information

This commit is contained in:
Виталий Чефанов
2026-10-27 10:06:00 +03:00
parent e6dafc746c
commit a1d5fe98ae
5 changed files with 73 additions and 3 deletions
@@ -180,7 +180,19 @@ func attachProblems(g *Graph, in problemInputs) {
for _, n := range in.nodes {
nodeProblems(s, n)
}
eventProblems(s, in.events, podOwner, podsLive(in.pods), podsScheduled(in.pods), podsReady(in.pods))
readyServices := map[string]bool{}
for _, svc := range in.services {
if len(svc.Spec.Selector) == 0 {
continue
}
for _, p := range podsByNs[svc.Namespace] {
if serviceBackendReady(p) && SelectorMatches(svc.Spec.Selector, p.Labels) {
readyServices["service/"+svc.Namespace+"/"+svc.Name] = true
break
}
}
}
eventProblems(s, in.events, podOwner, podsLive(in.pods), podsScheduled(in.pods), podsReady(in.pods), readyServices)
for _, n := range g.Nodes {
if !syntheticProbe(n.Namespace, n.Label) {
readinessProblem(s, n, notReadySince)
@@ -306,7 +318,7 @@ func serviceProblems(s *problemSet, svc corev1.Service, pods []corev1.Pod, in pr
return
}
for _, p := range pods {
if podReady(p) && SelectorMatches(svc.Spec.Selector, p.Labels) {
if serviceBackendReady(p) && SelectorMatches(svc.Spec.Selector, p.Labels) {
return
}
}
@@ -318,6 +330,10 @@ func serviceProblems(s *problemSet, svc corev1.Service, pods []corev1.Pod, in pr
s.add("service/"+svc.Namespace+"/"+svc.Name, p)
}
func serviceBackendReady(p corev1.Pod) bool {
return p.DeletionTimestamp == nil && p.Status.Phase == corev1.PodRunning && podReady(p)
}
// A zero-pod DaemonSet (no eligible nodes), or a scaled-down workload,
// can intentionally leave a Service empty. Unknown/stale state stays WARN.
func serviceControllersIdle(svc corev1.Service, in problemInputs) bool {
@@ -449,7 +465,7 @@ func eventsInWindow(e corev1.Event, last time.Time) int {
// eventProblems folds recent Warning events into one problem per card and
// reason, keeping the latest message.
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live, scheduled, ready map[string]bool) {
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live, scheduled, ready, readyServices map[string]bool) {
type key struct{ id, reason string }
folded := map[key]*Problem{}
var order []key
@@ -482,6 +498,13 @@ func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]str
if eventBenign[p.Reason] || (!notable && p.Count < eventRepeats) {
p.Severity = Info
}
// Kubernetes falls back to cluster-wide endpoint routing when topology
// hints cannot be allocated. Keep that diagnostic visible, but do not
// badge a service with a Ready backend as unavailable. Missing backends
// still retain both this warning and the independent NoEndpoints state.
if p.Reason == "TopologyAwareHintsDisabled" && readyServices[k.id] {
p.Severity = Info
}
// A failed probe of a pod that is ready now: the pod's own state
// already says more. Deckhouse answers /readyz with 500 while it
// works through its queue after a module change, and its probe
@@ -0,0 +1,40 @@
package topology
import (
"testing"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestTopologyHintsFallbackRequiresReadyService(t *testing.T) {
for _, tc := range []struct {
name string
ready bool
terminating bool
want string
}{{"healthy-cluster-wide-routing", true, false, Info}, {"missing-backend", false, false, Warning}, {"terminating-backend", true, true, Warning}} {
t.Run(tc.name, func(t *testing.T) {
labels := map[string]string{"app": "gatekeeper"}
in := problemInputs{
services: []corev1.Service{{ObjectMeta: metav1.ObjectMeta{Name: "webhook", Namespace: "system"}, Spec: corev1.ServiceSpec{Selector: labels}}},
events: []corev1.Event{{Type: corev1.EventTypeWarning, Reason: "TopologyAwareHintsDisabled", Count: 6, LastTimestamp: metav1.NewTime(now()), InvolvedObject: corev1.ObjectReference{Kind: "Service", Namespace: "system", Name: "webhook"}}},
}
if tc.ready {
in.pods = []corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Name: "gatekeeper", Namespace: "system", Labels: labels}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}}}
if tc.terminating {
at := metav1.NewTime(now())
in.pods[0].DeletionTimestamp = &at
}
}
g := &Graph{Nodes: []Node{{ID: "service/system/webhook"}}}
attachProblems(g, in)
if p := problemsOf(g, "service/system/webhook")["TopologyAwareHintsDisabled"]; p.Severity != tc.want {
t.Fatalf("got %+v, want %s", p, tc.want)
}
if tc.want == Warning && problemsOf(g, "service/system/webhook")["NoEndpoints"].Severity != Warning {
t.Fatal("missing backend must remain a warning")
}
})
}
}