fix: classify healthy topology routing fallback as information
This commit is contained in:
@@ -1,3 +1,6 @@
|
|||||||
|
v0.11.6:
|
||||||
|
fixes:
|
||||||
|
- Topology routing fallback is informational only when a Service has a Ready selected backend; missing backends still warn.
|
||||||
v0.11.4:
|
v0.11.4:
|
||||||
fixes:
|
fixes:
|
||||||
- "Failed probes (Unhealthy events) of a pod that is ready now are listed for information: the pod's state already reports NotReady and restarts. Deckhouse answers /readyz with 500 while it works through its queue after a module change and tolerates ten minutes of it; that badged its card after every module release."
|
- "Failed probes (Unhealthy events) of a pod that is ready now are listed for information: the pod's state already reports NotReady and restarts. Deckhouse answers /readyz with 500 while it works through its queue after a module change and tolerates ten minutes of it; that badged its card after every module release."
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ A card with a problem carries a badge: yellow `!` for a warning, red `!` for an
|
|||||||
|
|
||||||
Information-level entries (a cordoned node, a single ordinary warning) are listed and never badge a card. Problems are recomputed on every map request and the open page refreshes them every minute.
|
Information-level entries (a cordoned node, a single ordinary warning) are listed and never badge a card. Problems are recomputed on every map request and the open page refreshes them every minute.
|
||||||
|
|
||||||
|
Ordinary restart history expires after 30 minutes regardless of lifetime restart count; repeated OOM kills retain the 24-hour window. `TopologyAwareHintsDisabled` is informational only for a Service with a Ready selected backend: Kubernetes uses cluster-wide routing when hints cannot be allocated. Missing backends still produce warnings. No cluster routing configuration is changed.
|
||||||
|
|
||||||
## Vulnerabilities
|
## Vulnerabilities
|
||||||
|
|
||||||
With operator-trivy installed, a card whose images have CRITICAL CVEs shows a red shield with their number in the bottom-right corner. The details panel lists every CRITICAL and HIGH CVE of the workload's images: the packages it hits, the installed version and the version that fixes it, with a link to the advisory. A CVE counts once per card, however many packages and containers it is found in. Vulnerabilities are shown apart from problems: nearly every image has some, and a badge on each card would hide what is actually broken.
|
With operator-trivy installed, a card whose images have CRITICAL CVEs shows a red shield with their number in the bottom-right corner. The details panel lists every CRITICAL and HIGH CVE of the workload's images: the packages it hits, the installed version and the version that fixes it, with a link to the advisory. A CVE counts once per card, however many packages and containers it is found in. Vulnerabilities are shown apart from problems: nearly every image has some, and a badge on each card would hide what is actually broken.
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ Workload'ы под нестандартными контроллерами то
|
|||||||
|
|
||||||
Записи уровня «для сведения» (нода закрыта для новых подов, одиночное обычное предупреждение) есть в списке, но значка не дают. Проблемы пересчитываются при каждом запросе карты, открытая страница обновляет их раз в минуту.
|
Записи уровня «для сведения» (нода закрыта для новых подов, одиночное обычное предупреждение) есть в списке, но значка не дают. Проблемы пересчитываются при каждом запросе карты, открытая страница обновляет их раз в минуту.
|
||||||
|
|
||||||
|
Обычная история рестартов истекает через 30 минут независимо от накопленного счётчика; для повторных OOM остаётся окно 24 часа. `TopologyAwareHintsDisabled` показывается для сведения только у Service с готовым выбранным backend: Kubernetes использует обычную маршрутизацию по кластеру, когда не может назначить hints. Отсутствующие backend по-прежнему дают предупреждения. Настройки маршрутизации кластера не меняются.
|
||||||
|
|
||||||
## Уязвимости
|
## Уязвимости
|
||||||
|
|
||||||
Если установлен operator-trivy, у карточки, в образах которой есть CVE уровня CRITICAL, в правом нижнем углу красный щит с их числом. В панели деталей — все CVE уровней CRITICAL и HIGH образов workload'а: какие пакеты затронуты, стоящая версия и версия с исправлением, ссылка на описание. Одна CVE считается на карточке один раз, в скольких бы пакетах и контейнерах она ни нашлась. Уязвимости показываются отдельно от проблем: они есть почти в каждом образе, и значок на каждой карточке скрыл бы то, что действительно сломано.
|
Если установлен operator-trivy, у карточки, в образах которой есть CVE уровня CRITICAL, в правом нижнем углу красный щит с их числом. В панели деталей — все CVE уровней CRITICAL и HIGH образов workload'а: какие пакеты затронуты, стоящая версия и версия с исправлением, ссылка на описание. Одна CVE считается на карточке один раз, в скольких бы пакетах и контейнерах она ни нашлась. Уязвимости показываются отдельно от проблем: они есть почти в каждом образе, и значок на каждой карточке скрыл бы то, что действительно сломано.
|
||||||
|
|||||||
@@ -180,7 +180,19 @@ func attachProblems(g *Graph, in problemInputs) {
|
|||||||
for _, n := range in.nodes {
|
for _, n := range in.nodes {
|
||||||
nodeProblems(s, n)
|
nodeProblems(s, n)
|
||||||
}
|
}
|
||||||
eventProblems(s, in.events, podOwner, podsLive(in.pods), podsScheduled(in.pods), podsReady(in.pods))
|
readyServices := map[string]bool{}
|
||||||
|
for _, svc := range in.services {
|
||||||
|
if len(svc.Spec.Selector) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, p := range podsByNs[svc.Namespace] {
|
||||||
|
if serviceBackendReady(p) && SelectorMatches(svc.Spec.Selector, p.Labels) {
|
||||||
|
readyServices["service/"+svc.Namespace+"/"+svc.Name] = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
eventProblems(s, in.events, podOwner, podsLive(in.pods), podsScheduled(in.pods), podsReady(in.pods), readyServices)
|
||||||
for _, n := range g.Nodes {
|
for _, n := range g.Nodes {
|
||||||
if !syntheticProbe(n.Namespace, n.Label) {
|
if !syntheticProbe(n.Namespace, n.Label) {
|
||||||
readinessProblem(s, n, notReadySince)
|
readinessProblem(s, n, notReadySince)
|
||||||
@@ -306,7 +318,7 @@ func serviceProblems(s *problemSet, svc corev1.Service, pods []corev1.Pod, in pr
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
for _, p := range pods {
|
for _, p := range pods {
|
||||||
if podReady(p) && SelectorMatches(svc.Spec.Selector, p.Labels) {
|
if serviceBackendReady(p) && SelectorMatches(svc.Spec.Selector, p.Labels) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -318,6 +330,10 @@ func serviceProblems(s *problemSet, svc corev1.Service, pods []corev1.Pod, in pr
|
|||||||
s.add("service/"+svc.Namespace+"/"+svc.Name, p)
|
s.add("service/"+svc.Namespace+"/"+svc.Name, p)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func serviceBackendReady(p corev1.Pod) bool {
|
||||||
|
return p.DeletionTimestamp == nil && p.Status.Phase == corev1.PodRunning && podReady(p)
|
||||||
|
}
|
||||||
|
|
||||||
// A zero-pod DaemonSet (no eligible nodes), or a scaled-down workload,
|
// A zero-pod DaemonSet (no eligible nodes), or a scaled-down workload,
|
||||||
// can intentionally leave a Service empty. Unknown/stale state stays WARN.
|
// can intentionally leave a Service empty. Unknown/stale state stays WARN.
|
||||||
func serviceControllersIdle(svc corev1.Service, in problemInputs) bool {
|
func serviceControllersIdle(svc corev1.Service, in problemInputs) bool {
|
||||||
@@ -449,7 +465,7 @@ func eventsInWindow(e corev1.Event, last time.Time) int {
|
|||||||
|
|
||||||
// eventProblems folds recent Warning events into one problem per card and
|
// eventProblems folds recent Warning events into one problem per card and
|
||||||
// reason, keeping the latest message.
|
// reason, keeping the latest message.
|
||||||
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live, scheduled, ready map[string]bool) {
|
func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]string, live, scheduled, ready, readyServices map[string]bool) {
|
||||||
type key struct{ id, reason string }
|
type key struct{ id, reason string }
|
||||||
folded := map[key]*Problem{}
|
folded := map[key]*Problem{}
|
||||||
var order []key
|
var order []key
|
||||||
@@ -482,6 +498,13 @@ func eventProblems(s *problemSet, events []corev1.Event, podOwner map[string]str
|
|||||||
if eventBenign[p.Reason] || (!notable && p.Count < eventRepeats) {
|
if eventBenign[p.Reason] || (!notable && p.Count < eventRepeats) {
|
||||||
p.Severity = Info
|
p.Severity = Info
|
||||||
}
|
}
|
||||||
|
// Kubernetes falls back to cluster-wide endpoint routing when topology
|
||||||
|
// hints cannot be allocated. Keep that diagnostic visible, but do not
|
||||||
|
// badge a service with a Ready backend as unavailable. Missing backends
|
||||||
|
// still retain both this warning and the independent NoEndpoints state.
|
||||||
|
if p.Reason == "TopologyAwareHintsDisabled" && readyServices[k.id] {
|
||||||
|
p.Severity = Info
|
||||||
|
}
|
||||||
// A failed probe of a pod that is ready now: the pod's own state
|
// A failed probe of a pod that is ready now: the pod's own state
|
||||||
// already says more. Deckhouse answers /readyz with 500 while it
|
// already says more. Deckhouse answers /readyz with 500 while it
|
||||||
// works through its queue after a module change, and its probe
|
// works through its queue after a module change, and its probe
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
package topology
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
corev1 "k8s.io/api/core/v1"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTopologyHintsFallbackRequiresReadyService(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
ready bool
|
||||||
|
terminating bool
|
||||||
|
want string
|
||||||
|
}{{"healthy-cluster-wide-routing", true, false, Info}, {"missing-backend", false, false, Warning}, {"terminating-backend", true, true, Warning}} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
labels := map[string]string{"app": "gatekeeper"}
|
||||||
|
in := problemInputs{
|
||||||
|
services: []corev1.Service{{ObjectMeta: metav1.ObjectMeta{Name: "webhook", Namespace: "system"}, Spec: corev1.ServiceSpec{Selector: labels}}},
|
||||||
|
events: []corev1.Event{{Type: corev1.EventTypeWarning, Reason: "TopologyAwareHintsDisabled", Count: 6, LastTimestamp: metav1.NewTime(now()), InvolvedObject: corev1.ObjectReference{Kind: "Service", Namespace: "system", Name: "webhook"}}},
|
||||||
|
}
|
||||||
|
if tc.ready {
|
||||||
|
in.pods = []corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Name: "gatekeeper", Namespace: "system", Labels: labels}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}}}
|
||||||
|
if tc.terminating {
|
||||||
|
at := metav1.NewTime(now())
|
||||||
|
in.pods[0].DeletionTimestamp = &at
|
||||||
|
}
|
||||||
|
}
|
||||||
|
g := &Graph{Nodes: []Node{{ID: "service/system/webhook"}}}
|
||||||
|
attachProblems(g, in)
|
||||||
|
if p := problemsOf(g, "service/system/webhook")["TopologyAwareHintsDisabled"]; p.Severity != tc.want {
|
||||||
|
t.Fatalf("got %+v, want %s", p, tc.want)
|
||||||
|
}
|
||||||
|
if tc.want == Warning && problemsOf(g, "service/system/webhook")["NoEndpoints"].Severity != Warning {
|
||||||
|
t.Fatal("missing backend must remain a warning")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user