diff --git a/docs/README_RU.md b/docs/README_RU.md index 73194cc..d07a810 100644 --- a/docs/README_RU.md +++ b/docs/README_RU.md @@ -118,3 +118,9 @@ Cilium разбирает HTTP только на портах, упомянут - Видимость — по namespace: пользователь, который может смотреть поды в namespace, видит все его карточки, включая манифесты Deployment и Service, даже если его роль не дала бы прочитать эти объекты по отдельности. - Для алертов нужен модуль `prometheus`, он создаёт объекты `ClusterAlert`. Без него карта показывает проблемы только по состоянию объектов и событиям. - Подробности HTTP — только там, где CiliumNetworkPolicy включает разбор L7, см. [Подробности HTTP](#подробности-http). Остальные протоколы показываются на уровне L3/L4. + +## Уточнение диагностики после перезапуска + +Обычный перезапуск контейнера отмечается в течение 30 минут. Накопленный счётчик перезапусков за всю жизнь pod’а не доказывает повторные сбои за сутки; расширенное окно остаётся для последних завершений OOMKilled. Текущий CrashLoopBackOff и неготовность проверяются независимо. + +Сервис без endpoints остаётся предупреждением, кроме подтверждённого случая, когда все подходящие Deployment/StatefulSet/DaemonSet уже обработали конфигурацию и ожидают ноль pod’ов. Тогда запись информационная. Неизвестное состояние, устаревший статус контроллера, подходящий CronJob или незавершённый pod сохраняют предупреждение. diff --git a/images/cluster-map/src/internal/topology/graph.go b/images/cluster-map/src/internal/topology/graph.go index b6b9dce..b491399 100644 --- a/images/cluster-map/src/internal/topology/graph.go +++ b/images/cluster-map/src/internal/topology/graph.go @@ -178,7 +178,7 @@ func Build(c *kube.Cache, isSystem func(string) bool) (*Graph, error) { return nil, err } attachProblems(g, problemInputs{ - pods: pods, deployments: deployments, cronjobs: cronjobs, + pods: pods, deployments: deployments, daemonsets: daemonsets, statefulsets: statefulsets, cronjobs: cronjobs, services: services, ingresses: ingresses, nodes: hostNodes, events: events, alerts: c.AlertList(), }) g.AlertsAvailable = c.Alerts != nil diff --git a/images/cluster-map/src/internal/topology/problems.go b/images/cluster-map/src/internal/topology/problems.go index e46bc7e..506cb5c 100644 --- a/images/cluster-map/src/internal/topology/problems.go +++ b/images/cluster-map/src/internal/topology/problems.go @@ -112,14 +112,16 @@ func (s *problemSet) add(id string, p Problem) { } type problemInputs struct { - pods []corev1.Pod - deployments []appsv1.Deployment - cronjobs []batchv1.CronJob - services []corev1.Service - ingresses []networkingv1.Ingress - nodes []corev1.Node - events []corev1.Event - alerts []*unstructured.Unstructured + pods []corev1.Pod + deployments []appsv1.Deployment + daemonsets []appsv1.DaemonSet + statefulsets []appsv1.StatefulSet + cronjobs []batchv1.CronJob + services []corev1.Service + ingresses []networkingv1.Ingress + nodes []corev1.Node + events []corev1.Event + alerts []*unstructured.Unstructured } // attachProblems fills Node.Problems and Graph.ClusterProblems. Problems @@ -156,7 +158,7 @@ func attachProblems(g *Graph, in problemInputs) { } for _, svc := range in.services { if !syntheticProbe(svc.Namespace, svc.Name) { - serviceProblems(s, svc, podsByNs[svc.Namespace]) + serviceProblems(s, svc, podsByNs[svc.Namespace], in) } } services := map[string]bool{} @@ -264,7 +266,7 @@ func podProblems(s *problemSet, p corev1.Pod) { s.add(id, Problem{Severity: Critical, Source: "state", Reason: w.Reason, Message: w.Message, Object: obj, Count: int(cs.RestartCount)}) continue } - if t := cs.LastTerminationState.Terminated; t != nil && recentRestart(t.FinishedAt.Time, cs.RestartCount) { + if t := cs.LastTerminationState.Terminated; t != nil && recentRestart(t.FinishedAt.Time, cs.RestartCount, t.Reason) { reason := "Restarted" if t.Reason == "OOMKilled" { reason = "OOMKilled" @@ -299,7 +301,7 @@ func cronJobProblems(s *problemSet, cj batchv1.CronJob) { s.add(WorkloadID(cj.Namespace, "cronjob", cj.Name), Problem{Severity: Warning, Source: "state", Reason: "LastRunFailed", Since: stamp(last.Time)}) } -func serviceProblems(s *problemSet, svc corev1.Service, pods []corev1.Pod) { +func serviceProblems(s *problemSet, svc corev1.Service, pods []corev1.Pod, in problemInputs) { if len(svc.Spec.Selector) == 0 || svc.Spec.Type == corev1.ServiceTypeExternalName { return } @@ -308,7 +310,58 @@ func serviceProblems(s *problemSet, svc corev1.Service, pods []corev1.Pod) { return } } - s.add("service/"+svc.Namespace+"/"+svc.Name, Problem{Severity: Warning, Source: "state", Reason: "NoEndpoints"}) + p := Problem{Severity: Warning, Source: "state", Reason: "NoEndpoints"} + if serviceControllersIdle(svc, in) { + p.Severity = Info + p.Message = "No endpoints expected: all matching workload controllers have observed their configuration and currently require zero pods." + } + s.add("service/"+svc.Namespace+"/"+svc.Name, p) +} + +// A zero-pod DaemonSet (no eligible nodes), or a scaled-down workload, +// can intentionally leave a Service empty. Unknown/stale state stays WARN. +func serviceControllersIdle(svc corev1.Service, in problemInputs) bool { + found := false + // A matching CronJob may be starting a Job whose pod is not listed yet. + for _, cj := range in.cronjobs { + if cj.Namespace == svc.Namespace && SelectorMatches(svc.Spec.Selector, cj.Spec.JobTemplate.Spec.Template.Labels) { + return false + } + } + for _, ds := range in.daemonsets { + if ds.Namespace != svc.Namespace || !SelectorMatches(svc.Spec.Selector, ds.Spec.Template.Labels) { + continue + } + found = true + if ds.Status.ObservedGeneration < ds.Generation || ds.Status.DesiredNumberScheduled != 0 { + return false + } + } + for _, d := range in.deployments { + if d.Namespace != svc.Namespace || !SelectorMatches(svc.Spec.Selector, d.Spec.Template.Labels) { + continue + } + found = true + if d.Status.ObservedGeneration < d.Generation || d.Spec.Replicas == nil || *d.Spec.Replicas != 0 { + return false + } + } + for _, st := range in.statefulsets { + if st.Namespace != svc.Namespace || !SelectorMatches(svc.Spec.Selector, st.Spec.Template.Labels) { + continue + } + found = true + if st.Status.ObservedGeneration < st.Generation || st.Spec.Replicas == nil || *st.Spec.Replicas != 0 { + return false + } + } + // An unmanaged matching pod could still be genuinely broken. + for _, p := range in.pods { + if p.Namespace == svc.Namespace && SelectorMatches(svc.Spec.Selector, p.Labels) && p.DeletionTimestamp == nil && p.Status.Phase != corev1.PodSucceeded { + return false + } + } + return found } func nodeProblems(s *problemSet, n corev1.Node) { @@ -625,9 +678,11 @@ func alertProblem(s *problemSet, a *unstructured.Unstructured, podOwner map[stri s.add(AlertTargetID(labels, podOwner), Problem{Severity: AlertSeverity(level), Source: "alert", Reason: name, Message: strings.TrimSpace(summary), Since: since}) } -// recentRestart: the last restart is recent, or the container restarts -// again and again and the last one was within a day. -func recentRestart(finished time.Time, restarts int32) bool { +// recentRestart includes recent restarts and the extended OOM diagnostic. +func recentRestart(finished time.Time, restarts int32, reason string) bool { age := now().Sub(finished) - return age < restartWindow || (restarts >= repeatedRestarts && age < repeatedWindow) + // The count is lifetime, not a rate. A reboot can increment an old + // count without proving repeated recent failures. Keep the extended + // memory-pressure diagnostic only for an observed OOM termination. + return age < restartWindow || (reason == "OOMKilled" && restarts >= repeatedRestarts && age < repeatedWindow) } diff --git a/images/cluster-map/src/internal/topology/problems_test.go b/images/cluster-map/src/internal/topology/problems_test.go index 2c42c37..679fbba 100644 --- a/images/cluster-map/src/internal/topology/problems_test.go +++ b/images/cluster-map/src/internal/topology/problems_test.go @@ -30,6 +30,19 @@ import ( var t0 = time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC) +func TestLifetimeRestartCountDoesNotExtendOrdinaryRestartWindow(t *testing.T) { + withNow(t) + for _, reason := range []string{"Unknown", "Error", "Completed"} { + t.Run(reason, func(t *testing.T) { + g := &Graph{Nodes: []Node{{ID: "ns/system"}}} + attachProblems(g, problemInputs{pods: []corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Name: "recovered", Namespace: "system"}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, ContainerStatuses: []corev1.ContainerStatus{{Name: "app", RestartCount: 12, LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: reason, FinishedAt: metav1.NewTime(t0.Add(-40 * time.Minute))}}}}}}}}) + if p, ok := problemsOf(g, "ns/system")["Restarted"]; ok { + t.Fatalf("lifetime count treated as recent recurrence: %+v", p) + } + }) + } +} + func withNow(t *testing.T) { t.Helper() old := now diff --git a/images/cluster-map/src/internal/topology/service_idle_test.go b/images/cluster-map/src/internal/topology/service_idle_test.go new file mode 100644 index 0000000..34f5064 --- /dev/null +++ b/images/cluster-map/src/internal/topology/service_idle_test.go @@ -0,0 +1,62 @@ +package topology + +import ( + "testing" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestServiceWithoutEndpointsExpectedZero(t *testing.T) { + for _, tc := range []struct { + name string + desired int32 + observed int64 + activeDeployment bool + want string + }{ + {"idle-daemonset", 0, 3, false, Info}, + {"missing-daemonset-pod", 1, 3, false, Warning}, + {"stale-controller-status", 0, 2, false, Warning}, + {"another-active-controller", 0, 3, true, Warning}, + } { + t.Run(tc.name, func(t *testing.T) { + labels := map[string]string{"app": "time"} + in := problemInputs{services: []corev1.Service{{ObjectMeta: metav1.ObjectMeta{Name: "time", Namespace: "system"}, Spec: corev1.ServiceSpec{Selector: labels}}}, + daemonsets: []appsv1.DaemonSet{{ObjectMeta: metav1.ObjectMeta{Name: "masters", Namespace: "system", Generation: 3}, Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}}}, Status: appsv1.DaemonSetStatus{DesiredNumberScheduled: tc.desired, ObservedGeneration: tc.observed}}}} + if tc.activeDeployment { + one := int32(1) + in.deployments = []appsv1.Deployment{{ObjectMeta: metav1.ObjectMeta{Namespace: "system"}, Spec: appsv1.DeploymentSpec{Replicas: &one, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}}}}} + } + g := &Graph{Nodes: []Node{{ID: "service/system/time"}}} + attachProblems(g, in) + p := problemsOf(g, "service/system/time")["NoEndpoints"] + if p.Severity != tc.want { + t.Fatalf("got %+v, want %s", p, tc.want) + } + }) + } +} + +func TestIdleServiceStillWarnsForCronJobAndPendingPod(t *testing.T) { + zero := int32(0) + labels := map[string]string{"app": "time"} + for _, tc := range []struct { + name string + cron bool + }{{"cronjob-starting", true}, {"pod-pending", false}} { + t.Run(tc.name, func(t *testing.T) { + in := problemInputs{deployments: []appsv1.Deployment{{ObjectMeta: metav1.ObjectMeta{Namespace: "system"}, Spec: appsv1.DeploymentSpec{Replicas: &zero, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}}}}}} + if tc.cron { + in.cronjobs = []batchv1.CronJob{{ObjectMeta: metav1.ObjectMeta{Namespace: "system"}, Spec: batchv1.CronJobSpec{JobTemplate: batchv1.JobTemplateSpec{Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}}}}}}} + } else { + in.pods = []corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Namespace: "system", Labels: labels}, Status: corev1.PodStatus{Phase: corev1.PodPending}}} + } + if serviceControllersIdle(corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: "system"}, Spec: corev1.ServiceSpec{Selector: labels}}, in) { + t.Fatal("active controller/pod incorrectly treated as idle") + } + }) + } +}