fix: distinguish recovered restarts and intentionally idle services

This commit is contained in:
Виталий Чефанов
2026-10-27 08:39:47 +03:00
parent 8d9bac1840
commit e6dafc746c
5 changed files with 153 additions and 17 deletions
+6
View File
@@ -118,3 +118,9 @@ Cilium разбирает HTTP только на портах, упомянут
- Видимость — по namespace: пользователь, который может смотреть поды в namespace, видит все его карточки, включая манифесты Deployment и Service, даже если его роль не дала бы прочитать эти объекты по отдельности.
- Для алертов нужен модуль `prometheus`, он создаёт объекты `ClusterAlert`. Без него карта показывает проблемы только по состоянию объектов и событиям.
- Подробности HTTP — только там, где CiliumNetworkPolicy включает разбор L7, см. [Подробности HTTP](#подробности-http). Остальные протоколы показываются на уровне L3/L4.
## Уточнение диагностики после перезапуска
Обычный перезапуск контейнера отмечается в течение 30 минут. Накопленный счётчик перезапусков за всю жизнь pod’а не доказывает повторные сбои за сутки; расширенное окно остаётся для последних завершений OOMKilled. Текущий CrashLoopBackOff и неготовность проверяются независимо.
Сервис без endpoints остаётся предупреждением, кроме подтверждённого случая, когда все подходящие Deployment/StatefulSet/DaemonSet уже обработали конфигурацию и ожидают ноль pod’ов. Тогда запись информационная. Неизвестное состояние, устаревший статус контроллера, подходящий CronJob или незавершённый pod сохраняют предупреждение.
@@ -178,7 +178,7 @@ func Build(c *kube.Cache, isSystem func(string) bool) (*Graph, error) {
return nil, err
}
attachProblems(g, problemInputs{
pods: pods, deployments: deployments, cronjobs: cronjobs,
pods: pods, deployments: deployments, daemonsets: daemonsets, statefulsets: statefulsets, cronjobs: cronjobs,
services: services, ingresses: ingresses, nodes: hostNodes, events: events, alerts: c.AlertList(),
})
g.AlertsAvailable = c.Alerts != nil
@@ -114,6 +114,8 @@ func (s *problemSet) add(id string, p Problem) {
type problemInputs struct {
pods []corev1.Pod
deployments []appsv1.Deployment
daemonsets []appsv1.DaemonSet
statefulsets []appsv1.StatefulSet
cronjobs []batchv1.CronJob
services []corev1.Service
ingresses []networkingv1.Ingress
@@ -156,7 +158,7 @@ func attachProblems(g *Graph, in problemInputs) {
}
for _, svc := range in.services {
if !syntheticProbe(svc.Namespace, svc.Name) {
serviceProblems(s, svc, podsByNs[svc.Namespace])
serviceProblems(s, svc, podsByNs[svc.Namespace], in)
}
}
services := map[string]bool{}
@@ -264,7 +266,7 @@ func podProblems(s *problemSet, p corev1.Pod) {
s.add(id, Problem{Severity: Critical, Source: "state", Reason: w.Reason, Message: w.Message, Object: obj, Count: int(cs.RestartCount)})
continue
}
if t := cs.LastTerminationState.Terminated; t != nil && recentRestart(t.FinishedAt.Time, cs.RestartCount) {
if t := cs.LastTerminationState.Terminated; t != nil && recentRestart(t.FinishedAt.Time, cs.RestartCount, t.Reason) {
reason := "Restarted"
if t.Reason == "OOMKilled" {
reason = "OOMKilled"
@@ -299,7 +301,7 @@ func cronJobProblems(s *problemSet, cj batchv1.CronJob) {
s.add(WorkloadID(cj.Namespace, "cronjob", cj.Name), Problem{Severity: Warning, Source: "state", Reason: "LastRunFailed", Since: stamp(last.Time)})
}
func serviceProblems(s *problemSet, svc corev1.Service, pods []corev1.Pod) {
func serviceProblems(s *problemSet, svc corev1.Service, pods []corev1.Pod, in problemInputs) {
if len(svc.Spec.Selector) == 0 || svc.Spec.Type == corev1.ServiceTypeExternalName {
return
}
@@ -308,7 +310,58 @@ func serviceProblems(s *problemSet, svc corev1.Service, pods []corev1.Pod) {
return
}
}
s.add("service/"+svc.Namespace+"/"+svc.Name, Problem{Severity: Warning, Source: "state", Reason: "NoEndpoints"})
p := Problem{Severity: Warning, Source: "state", Reason: "NoEndpoints"}
if serviceControllersIdle(svc, in) {
p.Severity = Info
p.Message = "No endpoints expected: all matching workload controllers have observed their configuration and currently require zero pods."
}
s.add("service/"+svc.Namespace+"/"+svc.Name, p)
}
// A zero-pod DaemonSet (no eligible nodes), or a scaled-down workload,
// can intentionally leave a Service empty. Unknown/stale state stays WARN.
func serviceControllersIdle(svc corev1.Service, in problemInputs) bool {
found := false
// A matching CronJob may be starting a Job whose pod is not listed yet.
for _, cj := range in.cronjobs {
if cj.Namespace == svc.Namespace && SelectorMatches(svc.Spec.Selector, cj.Spec.JobTemplate.Spec.Template.Labels) {
return false
}
}
for _, ds := range in.daemonsets {
if ds.Namespace != svc.Namespace || !SelectorMatches(svc.Spec.Selector, ds.Spec.Template.Labels) {
continue
}
found = true
if ds.Status.ObservedGeneration < ds.Generation || ds.Status.DesiredNumberScheduled != 0 {
return false
}
}
for _, d := range in.deployments {
if d.Namespace != svc.Namespace || !SelectorMatches(svc.Spec.Selector, d.Spec.Template.Labels) {
continue
}
found = true
if d.Status.ObservedGeneration < d.Generation || d.Spec.Replicas == nil || *d.Spec.Replicas != 0 {
return false
}
}
for _, st := range in.statefulsets {
if st.Namespace != svc.Namespace || !SelectorMatches(svc.Spec.Selector, st.Spec.Template.Labels) {
continue
}
found = true
if st.Status.ObservedGeneration < st.Generation || st.Spec.Replicas == nil || *st.Spec.Replicas != 0 {
return false
}
}
// An unmanaged matching pod could still be genuinely broken.
for _, p := range in.pods {
if p.Namespace == svc.Namespace && SelectorMatches(svc.Spec.Selector, p.Labels) && p.DeletionTimestamp == nil && p.Status.Phase != corev1.PodSucceeded {
return false
}
}
return found
}
func nodeProblems(s *problemSet, n corev1.Node) {
@@ -625,9 +678,11 @@ func alertProblem(s *problemSet, a *unstructured.Unstructured, podOwner map[stri
s.add(AlertTargetID(labels, podOwner), Problem{Severity: AlertSeverity(level), Source: "alert", Reason: name, Message: strings.TrimSpace(summary), Since: since})
}
// recentRestart: the last restart is recent, or the container restarts
// again and again and the last one was within a day.
func recentRestart(finished time.Time, restarts int32) bool {
// recentRestart includes recent restarts and the extended OOM diagnostic.
func recentRestart(finished time.Time, restarts int32, reason string) bool {
age := now().Sub(finished)
return age < restartWindow || (restarts >= repeatedRestarts && age < repeatedWindow)
// The count is lifetime, not a rate. A reboot can increment an old
// count without proving repeated recent failures. Keep the extended
// memory-pressure diagnostic only for an observed OOM termination.
return age < restartWindow || (reason == "OOMKilled" && restarts >= repeatedRestarts && age < repeatedWindow)
}
@@ -30,6 +30,19 @@ import (
var t0 = time.Date(2026, 9, 24, 12, 0, 0, 0, time.UTC)
func TestLifetimeRestartCountDoesNotExtendOrdinaryRestartWindow(t *testing.T) {
withNow(t)
for _, reason := range []string{"Unknown", "Error", "Completed"} {
t.Run(reason, func(t *testing.T) {
g := &Graph{Nodes: []Node{{ID: "ns/system"}}}
attachProblems(g, problemInputs{pods: []corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Name: "recovered", Namespace: "system"}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, ContainerStatuses: []corev1.ContainerStatus{{Name: "app", RestartCount: 12, LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: reason, FinishedAt: metav1.NewTime(t0.Add(-40 * time.Minute))}}}}}}}})
if p, ok := problemsOf(g, "ns/system")["Restarted"]; ok {
t.Fatalf("lifetime count treated as recent recurrence: %+v", p)
}
})
}
}
func withNow(t *testing.T) {
t.Helper()
old := now
@@ -0,0 +1,62 @@
package topology
import (
"testing"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestServiceWithoutEndpointsExpectedZero(t *testing.T) {
for _, tc := range []struct {
name string
desired int32
observed int64
activeDeployment bool
want string
}{
{"idle-daemonset", 0, 3, false, Info},
{"missing-daemonset-pod", 1, 3, false, Warning},
{"stale-controller-status", 0, 2, false, Warning},
{"another-active-controller", 0, 3, true, Warning},
} {
t.Run(tc.name, func(t *testing.T) {
labels := map[string]string{"app": "time"}
in := problemInputs{services: []corev1.Service{{ObjectMeta: metav1.ObjectMeta{Name: "time", Namespace: "system"}, Spec: corev1.ServiceSpec{Selector: labels}}},
daemonsets: []appsv1.DaemonSet{{ObjectMeta: metav1.ObjectMeta{Name: "masters", Namespace: "system", Generation: 3}, Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}}}, Status: appsv1.DaemonSetStatus{DesiredNumberScheduled: tc.desired, ObservedGeneration: tc.observed}}}}
if tc.activeDeployment {
one := int32(1)
in.deployments = []appsv1.Deployment{{ObjectMeta: metav1.ObjectMeta{Namespace: "system"}, Spec: appsv1.DeploymentSpec{Replicas: &one, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}}}}}
}
g := &Graph{Nodes: []Node{{ID: "service/system/time"}}}
attachProblems(g, in)
p := problemsOf(g, "service/system/time")["NoEndpoints"]
if p.Severity != tc.want {
t.Fatalf("got %+v, want %s", p, tc.want)
}
})
}
}
func TestIdleServiceStillWarnsForCronJobAndPendingPod(t *testing.T) {
zero := int32(0)
labels := map[string]string{"app": "time"}
for _, tc := range []struct {
name string
cron bool
}{{"cronjob-starting", true}, {"pod-pending", false}} {
t.Run(tc.name, func(t *testing.T) {
in := problemInputs{deployments: []appsv1.Deployment{{ObjectMeta: metav1.ObjectMeta{Namespace: "system"}, Spec: appsv1.DeploymentSpec{Replicas: &zero, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}}}}}}
if tc.cron {
in.cronjobs = []batchv1.CronJob{{ObjectMeta: metav1.ObjectMeta{Namespace: "system"}, Spec: batchv1.CronJobSpec{JobTemplate: batchv1.JobTemplateSpec{Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}}}}}}}
} else {
in.pods = []corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Namespace: "system", Labels: labels}, Status: corev1.PodStatus{Phase: corev1.PodPending}}}
}
if serviceControllersIdle(corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: "system"}, Spec: corev1.ServiceSpec{Selector: labels}}, in) {
t.Fatal("active controller/pod incorrectly treated as idle")
}
})
}
}