fix: distinguish recovered restarts and intentionally idle services
This commit is contained in:
@@ -118,3 +118,9 @@ Cilium разбирает HTTP только на портах, упомянут
|
||||
- Видимость — по namespace: пользователь, который может смотреть поды в namespace, видит все его карточки, включая манифесты Deployment и Service, даже если его роль не дала бы прочитать эти объекты по отдельности.
|
||||
- Для алертов нужен модуль `prometheus`, он создаёт объекты `ClusterAlert`. Без него карта показывает проблемы только по состоянию объектов и событиям.
|
||||
- Подробности HTTP — только там, где CiliumNetworkPolicy включает разбор L7, см. [Подробности HTTP](#подробности-http). Остальные протоколы показываются на уровне L3/L4.
|
||||
|
||||
## Уточнение диагностики после перезапуска
|
||||
|
||||
Обычный перезапуск контейнера отмечается в течение 30 минут. Накопленный счётчик перезапусков за всю жизнь pod’а не доказывает повторные сбои за сутки; расширенное окно остаётся для последних завершений OOMKilled. Текущий CrashLoopBackOff и неготовность проверяются независимо.
|
||||
|
||||
Сервис без endpoints остаётся предупреждением, кроме подтверждённого случая, когда все подходящие Deployment/StatefulSet/DaemonSet уже обработали конфигурацию и ожидают ноль pod’ов. Тогда запись информационная. Неизвестное состояние, устаревший статус контроллера, подходящий CronJob или незавершённый pod сохраняют предупреждение.
|
||||
|
||||
@@ -178,7 +178,7 @@ func Build(c *kube.Cache, isSystem func(string) bool) (*Graph, error) {
|
||||
return nil, err
|
||||
}
|
||||
attachProblems(g, problemInputs{
|
||||
pods: pods, deployments: deployments, cronjobs: cronjobs,
|
||||
pods: pods, deployments: deployments, daemonsets: daemonsets, statefulsets: statefulsets, cronjobs: cronjobs,
|
||||
services: services, ingresses: ingresses, nodes: hostNodes, events: events, alerts: c.AlertList(),
|
||||
})
|
||||
g.AlertsAvailable = c.Alerts != nil
|
||||
|
||||
@@ -114,6 +114,8 @@ func (s *problemSet) add(id string, p Problem) {
|
||||
type problemInputs struct {
|
||||
pods []corev1.Pod
|
||||
deployments []appsv1.Deployment
|
||||
daemonsets []appsv1.DaemonSet
|
||||
statefulsets []appsv1.StatefulSet
|
||||
cronjobs []batchv1.CronJob
|
||||
services []corev1.Service
|
||||
ingresses []networkingv1.Ingress
|
||||
@@ -156,7 +158,7 @@ func attachProblems(g *Graph, in problemInputs) {
|
||||
}
|
||||
for _, svc := range in.services {
|
||||
if !syntheticProbe(svc.Namespace, svc.Name) {
|
||||
serviceProblems(s, svc, podsByNs[svc.Namespace])
|
||||
serviceProblems(s, svc, podsByNs[svc.Namespace], in)
|
||||
}
|
||||
}
|
||||
services := map[string]bool{}
|
||||
@@ -264,7 +266,7 @@ func podProblems(s *problemSet, p corev1.Pod) {
|
||||
s.add(id, Problem{Severity: Critical, Source: "state", Reason: w.Reason, Message: w.Message, Object: obj, Count: int(cs.RestartCount)})
|
||||
continue
|
||||
}
|
||||
if t := cs.LastTerminationState.Terminated; t != nil && recentRestart(t.FinishedAt.Time, cs.RestartCount) {
|
||||
if t := cs.LastTerminationState.Terminated; t != nil && recentRestart(t.FinishedAt.Time, cs.RestartCount, t.Reason) {
|
||||
reason := "Restarted"
|
||||
if t.Reason == "OOMKilled" {
|
||||
reason = "OOMKilled"
|
||||
@@ -299,7 +301,7 @@ func cronJobProblems(s *problemSet, cj batchv1.CronJob) {
|
||||
s.add(WorkloadID(cj.Namespace, "cronjob", cj.Name), Problem{Severity: Warning, Source: "state", Reason: "LastRunFailed", Since: stamp(last.Time)})
|
||||
}
|
||||
|
||||
func serviceProblems(s *problemSet, svc corev1.Service, pods []corev1.Pod) {
|
||||
func serviceProblems(s *problemSet, svc corev1.Service, pods []corev1.Pod, in problemInputs) {
|
||||
if len(svc.Spec.Selector) == 0 || svc.Spec.Type == corev1.ServiceTypeExternalName {
|
||||
return
|
||||
}
|
||||
@@ -308,7 +310,58 @@ func serviceProblems(s *problemSet, svc corev1.Service, pods []corev1.Pod) {
|
||||
return
|
||||
}
|
||||
}
|
||||
s.add("service/"+svc.Namespace+"/"+svc.Name, Problem{Severity: Warning, Source: "state", Reason: "NoEndpoints"})
|
||||
p := Problem{Severity: Warning, Source: "state", Reason: "NoEndpoints"}
|
||||
if serviceControllersIdle(svc, in) {
|
||||
p.Severity = Info
|
||||
p.Message = "No endpoints expected: all matching workload controllers have observed their configuration and currently require zero pods."
|
||||
}
|
||||
s.add("service/"+svc.Namespace+"/"+svc.Name, p)
|
||||
}
|
||||
|
||||
// A zero-pod DaemonSet (no eligible nodes), or a scaled-down workload,
|
||||
// can intentionally leave a Service empty. Unknown/stale state stays WARN.
|
||||
func serviceControllersIdle(svc corev1.Service, in problemInputs) bool {
|
||||
found := false
|
||||
// A matching CronJob may be starting a Job whose pod is not listed yet.
|
||||
for _, cj := range in.cronjobs {
|
||||
if cj.Namespace == svc.Namespace && SelectorMatches(svc.Spec.Selector, cj.Spec.JobTemplate.Spec.Template.Labels) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, ds := range in.daemonsets {
|
||||
if ds.Namespace != svc.Namespace || !SelectorMatches(svc.Spec.Selector, ds.Spec.Template.Labels) {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if ds.Status.ObservedGeneration < ds.Generation || ds.Status.DesiredNumberScheduled != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, d := range in.deployments {
|
||||
if d.Namespace != svc.Namespace || !SelectorMatches(svc.Spec.Selector, d.Spec.Template.Labels) {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if d.Status.ObservedGeneration < d.Generation || d.Spec.Replicas == nil || *d.Spec.Replicas != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, st := range in.statefulsets {
|
||||
if st.Namespace != svc.Namespace || !SelectorMatches(svc.Spec.Selector, st.Spec.Template.Labels) {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if st.Status.ObservedGeneration < st.Generation || st.Spec.Replicas == nil || *st.Spec.Replicas != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// An unmanaged matching pod could still be genuinely broken.
|
||||
for _, p := range in.pods {
|
||||
if p.Namespace == svc.Namespace && SelectorMatches(svc.Spec.Selector, p.Labels) && p.DeletionTimestamp == nil && p.Status.Phase != corev1.PodSucceeded {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
func nodeProblems(s *problemSet, n corev1.Node) {
|
||||
@@ -625,9 +678,11 @@ func alertProblem(s *problemSet, a *unstructured.Unstructured, podOwner map[stri
|
||||
s.add(AlertTargetID(labels, podOwner), Problem{Severity: AlertSeverity(level), Source: "alert", Reason: name, Message: strings.TrimSpace(summary), Since: since})
|
||||
}
|
||||
|
||||
// recentRestart: the last restart is recent, or the container restarts
|
||||
// again and again and the last one was within a day.
|
||||
func recentRestart(finished time.Time, restarts int32) bool {
|
||||
// recentRestart includes recent restarts and the extended OOM diagnostic.
|
||||
func recentRestart(finished time.Time, restarts int32, reason string) bool {
|
||||
age := now().Sub(finished)
|
||||
return age < restartWindow || (restarts >= repeatedRestarts && age < repeatedWindow)
|
||||
// The count is lifetime, not a rate. A reboot can increment an old
|
||||
// count without proving repeated recent failures. Keep the extended
|
||||
// memory-pressure diagnostic only for an observed OOM termination.
|
||||
return age < restartWindow || (reason == "OOMKilled" && restarts >= repeatedRestarts && age < repeatedWindow)
|
||||
}
|
||||
|
||||
@@ -30,6 +30,19 @@ import (
|
||||
|
||||
var t0 = time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
func TestLifetimeRestartCountDoesNotExtendOrdinaryRestartWindow(t *testing.T) {
|
||||
withNow(t)
|
||||
for _, reason := range []string{"Unknown", "Error", "Completed"} {
|
||||
t.Run(reason, func(t *testing.T) {
|
||||
g := &Graph{Nodes: []Node{{ID: "ns/system"}}}
|
||||
attachProblems(g, problemInputs{pods: []corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Name: "recovered", Namespace: "system"}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, ContainerStatuses: []corev1.ContainerStatus{{Name: "app", RestartCount: 12, LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: reason, FinishedAt: metav1.NewTime(t0.Add(-40 * time.Minute))}}}}}}}})
|
||||
if p, ok := problemsOf(g, "ns/system")["Restarted"]; ok {
|
||||
t.Fatalf("lifetime count treated as recent recurrence: %+v", p)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func withNow(t *testing.T) {
|
||||
t.Helper()
|
||||
old := now
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package topology
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestServiceWithoutEndpointsExpectedZero(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
desired int32
|
||||
observed int64
|
||||
activeDeployment bool
|
||||
want string
|
||||
}{
|
||||
{"idle-daemonset", 0, 3, false, Info},
|
||||
{"missing-daemonset-pod", 1, 3, false, Warning},
|
||||
{"stale-controller-status", 0, 2, false, Warning},
|
||||
{"another-active-controller", 0, 3, true, Warning},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
labels := map[string]string{"app": "time"}
|
||||
in := problemInputs{services: []corev1.Service{{ObjectMeta: metav1.ObjectMeta{Name: "time", Namespace: "system"}, Spec: corev1.ServiceSpec{Selector: labels}}},
|
||||
daemonsets: []appsv1.DaemonSet{{ObjectMeta: metav1.ObjectMeta{Name: "masters", Namespace: "system", Generation: 3}, Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}}}, Status: appsv1.DaemonSetStatus{DesiredNumberScheduled: tc.desired, ObservedGeneration: tc.observed}}}}
|
||||
if tc.activeDeployment {
|
||||
one := int32(1)
|
||||
in.deployments = []appsv1.Deployment{{ObjectMeta: metav1.ObjectMeta{Namespace: "system"}, Spec: appsv1.DeploymentSpec{Replicas: &one, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}}}}}
|
||||
}
|
||||
g := &Graph{Nodes: []Node{{ID: "service/system/time"}}}
|
||||
attachProblems(g, in)
|
||||
p := problemsOf(g, "service/system/time")["NoEndpoints"]
|
||||
if p.Severity != tc.want {
|
||||
t.Fatalf("got %+v, want %s", p, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdleServiceStillWarnsForCronJobAndPendingPod(t *testing.T) {
|
||||
zero := int32(0)
|
||||
labels := map[string]string{"app": "time"}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cron bool
|
||||
}{{"cronjob-starting", true}, {"pod-pending", false}} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
in := problemInputs{deployments: []appsv1.Deployment{{ObjectMeta: metav1.ObjectMeta{Namespace: "system"}, Spec: appsv1.DeploymentSpec{Replicas: &zero, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}}}}}}
|
||||
if tc.cron {
|
||||
in.cronjobs = []batchv1.CronJob{{ObjectMeta: metav1.ObjectMeta{Namespace: "system"}, Spec: batchv1.CronJobSpec{JobTemplate: batchv1.JobTemplateSpec{Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}}}}}}}
|
||||
} else {
|
||||
in.pods = []corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Namespace: "system", Labels: labels}, Status: corev1.PodStatus{Phase: corev1.PodPending}}}
|
||||
}
|
||||
if serviceControllersIdle(corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: "system"}, Spec: corev1.ServiceSpec{Selector: labels}}, in) {
|
||||
t.Fatal("active controller/pod incorrectly treated as idle")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user