diff --git a/README.md b/README.md index d266ca2..257acdc 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ Topology map of the Project Chimera Deckhouse cluster. → workload by label selector); dashed animated arrows are real traffic from Hubble Relay over the chosen window (5 min / 15 min / 1 h), red when Cilium dropped some of it. Peers from other namespaces are drawn as - dashed "ghost" cards. + dashed "ghost" cards. Traffic refreshes in place every 30 s without + relayout; peers that would need new cards wait for "Обновить". - **Right**: details for the selected object — overview, links (config and traffic, clickable), live YAML, pod logs. @@ -20,18 +21,27 @@ way Hubble names them, so traffic attaches to the same cards. ## Backend -Go, `client-go` in-cluster config. Every `/api/graph` call lists the API -fresh — no cache, no watch. `/api/flows` reads recent flows from -`hubble-relay.d8-cni-cilium.svc:443` (TLS, verified with the public part of -Deckhouse's hubble CA mounted at `/etc/hubble-ca/ca.crt`; relay does not -ask for a client certificate) and aggregates them per source → destination -→ port. Replies are excluded, so each connection counts once. +Go, `client-go` in-cluster config. The graph is built from informer caches +(watch-backed, managedFields stripped), so `/api/graph` costs milliseconds +instead of ~0.9 s of List calls. + +Traffic: each Cilium agent keeps only 4095 flows in its ring buffer — at a +few hundred flows/s that is ~10 s of history, so reading the buffer on +request can't answer "last 15 minutes". A background follow stream from +`hubble-relay.d8-cni-cilium.svc:443` (TLS verified with the public part of +Deckhouse's hubble CA at `/etc/hubble-ca/ca.crt`; relay does not ask for a +client certificate) feeds per-minute buckets for the last hour. Counts are +**distinct connections** (5-tuples), not Hubble events — one connection +produces several events. `new` = opened by TCP SYN inside the window, +`dropped` = tuples with a DROPPED verdict. After a restart the collector +only knows what it has seen since; `coveredSec` says how much of the +window that is. | Endpoint | What | |---|---| | `/api/graph` | nodes + config edges + pod placement | -| `/api/flows?namespaces=a,b&window=300` | aggregated Hubble traffic | -| `/api/resource?kind=&namespace=&name=` | live YAML | +| `/api/flows?namespaces=a,b&window=300` | aggregated traffic touching those namespaces | +| `/api/resource?kind=&namespace=&name=` | live YAML (managedFields stripped) | | `/api/pods?id=wl///&namespace=` | pods of a workload | | `/api/logs?namespace=&pod=&container=` | last 500 log lines | | `/healthz` | liveness/readiness | @@ -46,16 +56,12 @@ CDN. Theme follows `prefers-color-scheme`. ## Build and deploy -The in-cluster registry can't be fed from inside the cluster yet (Kaniko's -image lives on gcr.io, blocked here), so for now: - ``` -docker build -t cluster-map:vX . -docker save cluster-map:vX -o cluster-map.tar -scp cluster-map.tar xeon:/tmp/ -ssh xeon 'sudo /opt/deckhouse/bin/ctr -n k8s.io images import /tmp/cluster-map.tar' +R=git.project-chimera.ru/fanchji/cluster-map/cluster-map-app +docker build -t $R:vX . && docker push $R:vX +docker inspect --format '{{index .RepoDigests 0}}' $R:vX ``` -The Deployment pins `nodeSelector: xeon` + `imagePullPolicy: Never`. -Manifests: `03-kubernetes/apps/cluster-map/` in `project-chimera`, mirrored -to `k8s-manifests` for Argo. +The Deployment pins the image by digest. Anonymous pull works (public Gitea +user), no imagePullSecret. Manifests: `03-kubernetes/apps/cluster-map/` in +`project-chimera`, mirrored to `k8s-manifests` for Argo. diff --git a/flows.go b/flows.go index 674bc49..bfc34f7 100644 --- a/flows.go +++ b/flows.go @@ -7,21 +7,24 @@ import ( "encoding/json" "errors" "fmt" - "io" + "hash/fnv" + "log" "net/http" "os" "sort" "strconv" "strings" + "sync" "time" flowpb "github.com/cilium/cilium/api/v1/flow" observerpb "github.com/cilium/cilium/api/v1/observer" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/fieldmaskpb" "google.golang.org/protobuf/types/known/timestamppb" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // Relay serves a *.hubble-relay.cilium.io certificate signed by Deckhouse's @@ -29,36 +32,13 @@ import ( // that CA is enough. The CA is mounted from a ConfigMap (public part only). const hubbleServerName = "cluster-map.hubble-relay.cilium.io" -var observer observerpb.ObserverClient -var hubbleInitErr error - -func initHubble() { - addr := envOr("HUBBLE_RELAY_ADDR", "hubble-relay.d8-cni-cilium.svc:443") - caPEM, err := os.ReadFile(envOr("HUBBLE_CA_FILE", "/etc/hubble-ca/ca.crt")) - if err != nil { - hubbleInitErr = fmt.Errorf("hubble CA: %w", err) - return - } - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM(caPEM) { - hubbleInitErr = errors.New("hubble CA: no certificate in file") - return - } - creds := credentials.NewTLS(&tls.Config{RootCAs: pool, ServerName: hubbleServerName, MinVersion: tls.VersionTLS12}) - conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(creds)) - if err != nil { - hubbleInitErr = err - return - } - observer = observerpb.NewObserverClient(conn) -} - -func envOr(k, def string) string { - if v := os.Getenv(k); v != "" { - return v - } - return def -} +// Each agent keeps only 4095 flows in its ring buffer, which at a few +// hundred flows/s is ~10 s of history. So traffic is collected continuously +// from a follow stream into per-minute buckets covering the last hour. +const ( + bucketCount = 60 + maxTuplesPerKey = 20000 // per edge per minute; beyond it counts stop growing +) type FlowEndpoint struct { ID string `json:"id"` @@ -68,153 +48,342 @@ type FlowEndpoint struct { } type FlowEdge struct { - Source string `json:"source"` - Target string `json:"target"` - Port string `json:"port"` // "443/TCP" - Count int `json:"count"` - Dropped int `json:"dropped"` - Last string `json:"last"` + Source string `json:"source"` + Target string `json:"target"` + Port string `json:"port"` // "443/TCP" + Connections int `json:"connections"` // distinct 5-tuples seen in the window + New int `json:"new"` // of those, opened (TCP SYN) in the window + Dropped int `json:"dropped"` // distinct 5-tuples with a DROPPED verdict + Last string `json:"last"` } type FlowsResult struct { - WindowSec int `json:"windowSec"` - Scanned int `json:"scanned"` - Endpoints []FlowEndpoint `json:"endpoints"` - Edges []FlowEdge `json:"edges"` + WindowSec int `json:"windowSec"` + CoveredSec int `json:"coveredSec"` // how much of the window the collector has actually seen + Connected bool `json:"connected"` + Error string `json:"error,omitempty"` + Endpoints []FlowEndpoint `json:"endpoints"` + Edges []FlowEdge `json:"edges"` } -const maxFlows = 20000 +type edgeKey struct{ src, dst, port string } -// handleFlows aggregates recent Hubble flows touching the given namespaces -// into workload-to-workload edges. Replies are excluded so each connection -// is counted once, in the direction it was opened. -func handleFlows(w http.ResponseWriter, r *http.Request) { - if observer == nil { - http.Error(w, "hubble недоступен: "+errString(hubbleInitErr), http.StatusServiceUnavailable) +const ( + tupleSYN uint8 = 1 + tupleDrop uint8 = 2 +) + +type bucketEdge struct { + tuples map[uint64]uint8 + last int64 +} + +type bucket struct { + minute int64 + edges map[edgeKey]*bucketEdge +} + +type collector struct { + mu sync.Mutex + buckets [bucketCount]*bucket + endpoints map[string]FlowEndpoint + startedAt time.Time + connected bool + lastErr string + flows uint64 +} + +var flowsCollector = &collector{endpoints: map[string]FlowEndpoint{}} + +func startHubble() { + caPEM, err := os.ReadFile(envOr("HUBBLE_CA_FILE", "/etc/hubble-ca/ca.crt")) + if err != nil { + flowsCollector.setErr(fmt.Errorf("hubble CA: %w", err)) return } - var namespaces []string - for _, ns := range strings.Split(r.URL.Query().Get("namespaces"), ",") { - if ns = strings.TrimSpace(ns); ns != "" { - namespaces = append(namespaces, ns) + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + flowsCollector.setErr(errors.New("hubble CA: no certificate in file")) + return + } + creds := credentials.NewTLS(&tls.Config{RootCAs: pool, ServerName: hubbleServerName, MinVersion: tls.VersionTLS12}) + conn, err := grpc.NewClient(envOr("HUBBLE_RELAY_ADDR", "hubble-relay.d8-cni-cilium.svc:443"), grpc.WithTransportCredentials(creds)) + if err != nil { + flowsCollector.setErr(err) + return + } + go flowsCollector.run(observerpb.NewObserverClient(conn)) +} + +func envOr(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} + +func (c *collector) setErr(err error) { + c.mu.Lock() + defer c.mu.Unlock() + c.connected = false + c.lastErr = err.Error() + log.Printf("hubble: %v", err) +} + +var flowFields = []string{"time", "verdict", "IP", "l4", "source", "destination", "source_names", "destination_names", "node_name"} + +func (c *collector) run(obs observerpb.ObserverClient) { + c.mu.Lock() + c.startedAt = time.Now() + c.mu.Unlock() + useMask := true + backoff := time.Second + for { + req := &observerpb.GetFlowsRequest{ + Follow: true, + // backfill whatever the ring buffers still hold; duplicate + // tuples after a reconnect collapse in the per-minute sets + Since: timestamppb.New(time.Now().Add(-time.Minute)), + Whitelist: []*flowpb.FlowFilter{{Reply: []bool{false}}}, + } + if useMask { + req.FieldMask = &fieldmaskpb.FieldMask{Paths: flowFields} + } + before := c.flowsSeen() + err := c.stream(obs, req) + if status.Code(err) == codes.InvalidArgument && useMask { + useMask = false + log.Printf("hubble: relay rejected field mask, retrying without it: %v", err) + continue + } + c.setErr(err) + if c.flowsSeen() > before { + backoff = time.Second // the stream worked for a while; reconnect quickly + } + time.Sleep(backoff) + backoff = min(backoff*2, 30*time.Second) + } +} + +func (c *collector) flowsSeen() uint64 { + c.mu.Lock() + defer c.mu.Unlock() + return c.flows +} + +func (c *collector) stream(obs observerpb.ObserverClient, req *observerpb.GetFlowsRequest) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s, err := obs.GetFlows(ctx, req) + if err != nil { + return err + } + for { + resp, err := s.Recv() + if err != nil { + return err + } + if f := resp.GetFlow(); f != nil { + c.add(f) } } - if len(namespaces) == 0 || len(namespaces) > 50 { - http.Error(w, "namespaces: от 1 до 50 через запятую", http.StatusBadRequest) +} + +func (c *collector) add(f *flowpb.Flow) { + src := flowEndpoint(f.GetSource(), f.GetSourceNames(), f.GetNodeName()) + dst := flowEndpoint(f.GetDestination(), f.GetDestinationNames(), f.GetNodeName()) + if src.ID == dst.ID { + return + } + port, tuple, syn := l4Info(f) + ts := time.Now() + if t := f.GetTime(); t != nil { + ts = t.AsTime() + } + minute := ts.Unix() / 60 + + c.mu.Lock() + defer c.mu.Unlock() + if !c.connected { + c.connected, c.lastErr = true, "" + } + c.flows++ + b := c.buckets[minute%bucketCount] + if b == nil || b.minute != minute { + if b != nil && b.minute > minute { + return // older than the retained hour + } + b = &bucket{minute: minute, edges: map[edgeKey]*bucketEdge{}} + c.buckets[minute%bucketCount] = b + c.pruneEndpoints() + } + c.endpoints[src.ID] = src + c.endpoints[dst.ID] = dst + k := edgeKey{src.ID, dst.ID, port} + e := b.edges[k] + if e == nil { + e = &bucketEdge{tuples: map[uint64]uint8{}} + b.edges[k] = e + } + h := hashTuple(f.GetIP().GetSource(), f.GetIP().GetDestination(), tuple) + flags, seen := e.tuples[h] + if !seen && len(e.tuples) >= maxTuplesPerKey { + return + } + if syn { + flags |= tupleSYN + } + if f.GetVerdict() == flowpb.Verdict_DROPPED { + flags |= tupleDrop + } + e.tuples[h] = flags + e.last = max(e.last, ts.Unix()) +} + +// pruneEndpoints drops metadata for endpoints no retained bucket mentions. +func (c *collector) pruneEndpoints() { + used := map[string]bool{} + for _, b := range c.buckets { + if b == nil { + continue + } + for k := range b.edges { + used[k.src], used[k.dst] = true, true + } + } + for id := range c.endpoints { + if !used[id] { + delete(c.endpoints, id) + } + } +} + +func hashTuple(srcIP, dstIP, rest string) uint64 { + h := fnv.New64a() + h.Write([]byte(srcIP)) + h.Write([]byte{0}) + h.Write([]byte(dstIP)) + h.Write([]byte{0}) + h.Write([]byte(rest)) + return h.Sum64() +} + +// l4Info returns the display port ("443/TCP"), the port part of the +// connection tuple, and whether this event opens a TCP connection. +func l4Info(f *flowpb.Flow) (port, tuple string, syn bool) { + l4 := f.GetL4() + switch { + case l4.GetTCP() != nil: + t := l4.GetTCP() + fl := t.GetFlags() + return strconv.Itoa(int(t.GetDestinationPort())) + "/TCP", + "tcp:" + strconv.Itoa(int(t.GetSourcePort())) + ":" + strconv.Itoa(int(t.GetDestinationPort())), + fl.GetSYN() && !fl.GetACK() + case l4.GetUDP() != nil: + u := l4.GetUDP() + return strconv.Itoa(int(u.GetDestinationPort())) + "/UDP", + "udp:" + strconv.Itoa(int(u.GetSourcePort())) + ":" + strconv.Itoa(int(u.GetDestinationPort())), false + case l4.GetSCTP() != nil: + s := l4.GetSCTP() + return strconv.Itoa(int(s.GetDestinationPort())) + "/SCTP", + "sctp:" + strconv.Itoa(int(s.GetSourcePort())) + ":" + strconv.Itoa(int(s.GetDestinationPort())), false + case l4.GetICMPv4() != nil || l4.GetICMPv6() != nil: + return "ICMP", "icmp", false + } + return "?", "?", false +} + +func (c *collector) query(namespaces map[string]bool, window int) FlowsResult { + now := time.Now() + fromMinute := now.Add(-time.Duration(window)*time.Second).Unix() / 60 + + c.mu.Lock() + defer c.mu.Unlock() + res := FlowsResult{WindowSec: window, Connected: c.connected, Error: c.lastErr} + if !c.startedAt.IsZero() { + res.CoveredSec = min(window, int(now.Sub(c.startedAt).Seconds())) + } + touches := func(id string) bool { return namespaces[c.endpoints[id].Namespace] } + + type acc struct { + tuples map[uint64]uint8 + last int64 + } + merged := map[edgeKey]*acc{} + for _, b := range c.buckets { + if b == nil || b.minute < fromMinute { + continue + } + for k, e := range b.edges { + if !touches(k.src) && !touches(k.dst) { + continue + } + a := merged[k] + if a == nil { + a = &acc{tuples: make(map[uint64]uint8, len(e.tuples))} + merged[k] = a + } + for h, fl := range e.tuples { + a.tuples[h] |= fl + } + a.last = max(a.last, e.last) + } + } + eps := map[string]bool{} + for k, a := range merged { + fe := FlowEdge{Source: k.src, Target: k.dst, Port: k.port, Connections: len(a.tuples), Last: time.Unix(a.last, 0).UTC().Format(time.RFC3339)} + for _, fl := range a.tuples { + if fl&tupleSYN != 0 { + fe.New++ + } + if fl&tupleDrop != 0 { + fe.Dropped++ + } + } + res.Edges = append(res.Edges, fe) + eps[k.src], eps[k.dst] = true, true + } + for id := range eps { + res.Endpoints = append(res.Endpoints, c.endpoints[id]) + } + sort.Slice(res.Endpoints, func(i, j int) bool { return res.Endpoints[i].ID < res.Endpoints[j].ID }) + sort.Slice(res.Edges, func(i, j int) bool { return res.Edges[i].Connections > res.Edges[j].Connections }) + return res +} + +func handleFlows(w http.ResponseWriter, r *http.Request) { + namespaces := map[string]bool{} + for _, ns := range strings.Split(r.URL.Query().Get("namespaces"), ",") { + if ns = strings.TrimSpace(ns); ns != "" { + namespaces[ns] = true + } + } + if len(namespaces) == 0 || len(namespaces) > 100 { + http.Error(w, "namespaces: от 1 до 100 через запятую", http.StatusBadRequest) return } window, _ := strconv.Atoi(r.URL.Query().Get("window")) if window <= 0 { window = 300 } - if window > 3600 { - window = 3600 - } - - var podPrefixes []string - for _, ns := range namespaces { - podPrefixes = append(podPrefixes, ns+"/") - } - notReply := []bool{false} - req := &observerpb.GetFlowsRequest{ - Number: maxFlows, - Since: timestamppb.New(time.Now().Add(-time.Duration(window) * time.Second)), - Whitelist: []*flowpb.FlowFilter{ - {SourcePod: podPrefixes, Reply: notReply}, - {DestinationPod: podPrefixes, Reply: notReply}, - }, - } - - ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) - defer cancel() - stream, err := observer.GetFlows(ctx, req) - if err != nil { - http.Error(w, "hubble: "+err.Error(), http.StatusBadGateway) - return - } - - // Hubble leaves Endpoint.Workloads empty for some flows (typically the - // half observed on the peer's node), so pods are resolved to their - // workload here, with the same podOwner rules the graph uses. - pods, err := clientset.CoreV1().Pods(corev1.NamespaceAll).List(ctx, metav1.ListOptions{}) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - podWl := make(map[string]FlowEndpoint, len(pods.Items)) - for _, p := range pods.Items { - if kind, name, _ := podOwner(p); kind != "" { - podWl[p.Namespace+"/"+p.Name] = FlowEndpoint{ID: workloadID(p.Namespace, kind, name), Label: name, Kind: kind, Namespace: p.Namespace} - } - } - - res := FlowsResult{WindowSec: window} - endpoints := map[string]FlowEndpoint{} - edges := map[string]*FlowEdge{} - for { - resp, err := stream.Recv() - if err == io.EOF { - break - } - if err != nil { - if res.Scanned == 0 { - http.Error(w, "hubble: "+err.Error(), http.StatusBadGateway) - return - } - break // partial result is still useful - } - f := resp.GetFlow() - if f == nil { - continue - } - res.Scanned++ - src := flowEndpoint(f.GetSource(), f.GetSourceNames(), f.GetNodeName(), podWl) - dst := flowEndpoint(f.GetDestination(), f.GetDestinationNames(), f.GetNodeName(), podWl) - if src.ID == dst.ID { - continue - } - endpoints[src.ID] = src - endpoints[dst.ID] = dst - port := l4Port(f.GetL4()) - key := src.ID + "|" + dst.ID + "|" + port - e := edges[key] - if e == nil { - e = &FlowEdge{Source: src.ID, Target: dst.ID, Port: port} - edges[key] = e - } - e.Count++ - if f.GetVerdict() == flowpb.Verdict_DROPPED { - e.Dropped++ - } - if t := f.GetTime(); t != nil { - if ts := t.AsTime().UTC().Format(time.RFC3339); ts > e.Last { - e.Last = ts - } - } - } - - for _, ep := range endpoints { - res.Endpoints = append(res.Endpoints, ep) - } - for _, e := range edges { - res.Edges = append(res.Edges, *e) - } - sort.Slice(res.Endpoints, func(i, j int) bool { return res.Endpoints[i].ID < res.Endpoints[j].ID }) - sort.Slice(res.Edges, func(i, j int) bool { return res.Edges[i].Count > res.Edges[j].Count }) + window = min(window, bucketCount*60) w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(res) + json.NewEncoder(w).Encode(flowsCollector.query(namespaces, window)) } // flowEndpoint maps a Hubble endpoint to an ID that matches a graph node -// when the peer is one of our workloads, so traffic edges attach to the -// same cards the structural graph already draws. -func flowEndpoint(e *flowpb.Endpoint, names []string, nodeName string, podWl map[string]FlowEndpoint) FlowEndpoint { +// when the peer is one of our workloads. Hubble leaves Endpoint.Workloads +// empty on some flows (typically the half observed on the peer's node), so +// pods are resolved through the informer cache with the graph's own rules, +// and Hubble's workload name is only a fallback for pods already gone. +func flowEndpoint(e *flowpb.Endpoint, names []string, nodeName string) FlowEndpoint { if e.GetPodName() != "" { ns := e.GetNamespace() - if ep, ok := podWl[ns+"/"+e.GetPodName()]; ok { - return ep + if p, err := podLister.Pods(ns).Get(e.GetPodName()); err == nil { + if kind, name, _ := podOwner(*p); kind != "" { + return FlowEndpoint{ID: workloadID(ns, kind, name), Label: name, Kind: kind, Namespace: ns} + } } - // pod already gone: fall back to what Hubble recorded for _, wl := range e.GetWorkloads() { kind, name := strings.ToLower(wl.GetKind()), wl.GetName() if kind == "job" { @@ -222,10 +391,9 @@ func flowEndpoint(e *flowpb.Endpoint, names []string, nodeName string, podWl map kind, name = "cronjob", name[:i] } } - if kind == "" || name == "" { - continue + if kind != "" && name != "" { + return FlowEndpoint{ID: workloadID(ns, kind, name), Label: name, Kind: kind, Namespace: ns} } - return FlowEndpoint{ID: workloadID(ns, kind, name), Label: name, Kind: kind, Namespace: ns} } return FlowEndpoint{ID: "pod/" + ns + "/" + e.GetPodName(), Label: e.GetPodName(), Kind: "pod", Namespace: ns} } @@ -253,24 +421,3 @@ func flowEndpoint(e *flowpb.Endpoint, names []string, nodeName string, podWl map } return FlowEndpoint{ID: "ext/world", Label: "интернет", Kind: "external"} } - -func l4Port(l4 *flowpb.Layer4) string { - switch { - case l4.GetTCP() != nil: - return strconv.Itoa(int(l4.GetTCP().GetDestinationPort())) + "/TCP" - case l4.GetUDP() != nil: - return strconv.Itoa(int(l4.GetUDP().GetDestinationPort())) + "/UDP" - case l4.GetICMPv4() != nil || l4.GetICMPv6() != nil: - return "ICMP" - case l4.GetSCTP() != nil: - return strconv.Itoa(int(l4.GetSCTP().GetDestinationPort())) + "/SCTP" - } - return "?" -} - -func errString(err error) string { - if err == nil { - return "не инициализирован" - } - return err.Error() -} diff --git a/informers.go b/informers.go new file mode 100644 index 0000000..26e3776 --- /dev/null +++ b/informers.go @@ -0,0 +1,87 @@ +package main + +import ( + "fmt" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/informers" + appslisters "k8s.io/client-go/listers/apps/v1" + batchlisters "k8s.io/client-go/listers/batch/v1" + corelisters "k8s.io/client-go/listers/core/v1" + netlisters "k8s.io/client-go/listers/networking/v1" +) + +var ( + nsLister corelisters.NamespaceLister + podLister corelisters.PodLister + svcLister corelisters.ServiceLister + nodeLister corelisters.NodeLister + deployLister appslisters.DeploymentLister + stsLister appslisters.StatefulSetLister + dsLister appslisters.DaemonSetLister + cronLister batchlisters.CronJobLister + ingLister netlisters.IngressLister +) + +func startInformers() error { + f := informers.NewSharedInformerFactoryWithOptions(clientset, 0, + // managedFields is most of a cached object's size and nothing here reads it + informers.WithTransform(func(obj any) (any, error) { + if a, err := meta.Accessor(obj); err == nil { + a.SetManagedFields(nil) + } + return obj, nil + })) + nsLister = f.Core().V1().Namespaces().Lister() + podLister = f.Core().V1().Pods().Lister() + svcLister = f.Core().V1().Services().Lister() + nodeLister = f.Core().V1().Nodes().Lister() + deployLister = f.Apps().V1().Deployments().Lister() + stsLister = f.Apps().V1().StatefulSets().Lister() + dsLister = f.Apps().V1().DaemonSets().Lister() + cronLister = f.Batch().V1().CronJobs().Lister() + ingLister = f.Networking().V1().Ingresses().Lister() + + stop := make(chan struct{}) + f.Start(stop) + done := make(chan map[string]bool, 1) + go func() { + synced := map[string]bool{} + for t, ok := range f.WaitForCacheSync(stop) { + synced[fmt.Sprint(t)] = ok + } + done <- synced + }() + select { + case synced := <-done: + for t, ok := range synced { + if !ok { + return fmt.Errorf("cache for %s did not sync", t) + } + } + return nil + case <-time.After(90 * time.Second): + return fmt.Errorf("caches did not sync in 90s") + } +} + +type listOf[T any] struct{ Items []T } + +// listAll reads a lister into the same shape a typed List() call returns, +// so graph code iterates .Items unchanged. Items are copies: callers may not +// mutate cached objects, and copying the top-level struct is cheap. +func listAll[T any](l interface { + List(labels.Selector) ([]*T, error) +}) (*listOf[T], error) { + ptrs, err := l.List(labels.Everything()) + if err != nil { + return nil, err + } + out := &listOf[T]{Items: make([]T, len(ptrs))} + for i, p := range ptrs { + out.Items[i] = *p + } + return out, nil +} diff --git a/main.go b/main.go index ce779fe..dafdd01 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,10 @@ // cluster-map: a live topology view of a Deckhouse/Kubernetes cluster. // -// Builds a graph from the live k8s API on every request to /api/graph: -// namespaces group workloads (Deployment/StatefulSet/DaemonSet/CronJob), -// which connect to Services (by label-selector match) and Ingresses (by -// backend reference). Nothing is cached or watched -- this is meant to -// be a small, low-traffic internal tool, not a high-QPS dashboard, so a -// fresh List() per request is simpler and correct-by-construction (no -// stale-cache class of bugs to worry about). +// Builds a graph on every request to /api/graph from informer caches: +// namespaces group workloads, which connect to Services (by label-selector +// match) and Ingresses (by backend reference). Informers keep the caches +// in step with the API by watch; a fresh List per request cost ~0.9 s and +// timed out the 1 s probe. package main import ( @@ -15,7 +13,6 @@ import ( "io" "log" "net/http" - "os" "sort" "strings" @@ -63,10 +60,10 @@ func main() { log.Fatalf("building clientset: %v", err) } - initHubble() - if hubbleInitErr != nil { - log.Printf("hubble disabled: %v", hubbleInitErr) + if err := startInformers(); err != nil { + log.Fatalf("informers: %v", err) } + startHubble() mux := http.NewServeMux() mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("ok")) }) @@ -111,7 +108,7 @@ func selectorSubset(sel, labels map[string]string) bool { func buildGraph(ctx context.Context) (*Graph, error) { g := &Graph{} - namespaces, err := clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + namespaces, err := listAll[corev1.Namespace](nsLister) if err != nil { return nil, err } @@ -133,7 +130,7 @@ func buildGraph(ctx context.Context) (*Graph, error) { } var workloads []workloadRef - deployments, err := clientset.AppsV1().Deployments(corev1.NamespaceAll).List(ctx, metav1.ListOptions{}) + deployments, err := listAll[appsv1.Deployment](deployLister) if err != nil { return nil, err } @@ -143,7 +140,7 @@ func buildGraph(ctx context.Context) (*Graph, error) { workloads = append(workloads, workloadRef{id, d.Spec.Template.Labels}) } - statefulsets, err := clientset.AppsV1().StatefulSets(corev1.NamespaceAll).List(ctx, metav1.ListOptions{}) + statefulsets, err := listAll[appsv1.StatefulSet](stsLister) if err != nil { return nil, err } @@ -153,7 +150,7 @@ func buildGraph(ctx context.Context) (*Graph, error) { workloads = append(workloads, workloadRef{id, s.Spec.Template.Labels}) } - daemonsets, err := clientset.AppsV1().DaemonSets(corev1.NamespaceAll).List(ctx, metav1.ListOptions{}) + daemonsets, err := listAll[appsv1.DaemonSet](dsLister) if err != nil { return nil, err } @@ -163,7 +160,7 @@ func buildGraph(ctx context.Context) (*Graph, error) { workloads = append(workloads, workloadRef{id, ds.Spec.Template.Labels}) } - cronjobs, err := clientset.BatchV1().CronJobs(corev1.NamespaceAll).List(ctx, metav1.ListOptions{}) + cronjobs, err := listAll[batchv1.CronJob](cronLister) if err != nil { return nil, err } @@ -177,7 +174,7 @@ func buildGraph(ctx context.Context) (*Graph, error) { // DaemonSet behind ingress-nginx, database operators, bare Jobs) would // otherwise be invisible. They become workload nodes of their own, IDed // the way Hubble names workloads, so traffic attaches to them too. - pods, err := clientset.CoreV1().Pods(corev1.NamespaceAll).List(ctx, metav1.ListOptions{}) + pods, err := listAll[corev1.Pod](podLister) if err != nil { return nil, err } @@ -231,7 +228,7 @@ func buildGraph(ctx context.Context) (*Graph, error) { workloads = append(workloads, workloadRef{id, f.labels}) } - services, err := clientset.CoreV1().Services(corev1.NamespaceAll).List(ctx, metav1.ListOptions{}) + services, err := listAll[corev1.Service](svcLister) if err != nil { return nil, err } @@ -251,7 +248,7 @@ func buildGraph(ctx context.Context) (*Graph, error) { } } - ingresses, err := clientset.NetworkingV1().Ingresses(corev1.NamespaceAll).List(ctx, metav1.ListOptions{}) + ingresses, err := listAll[networkingv1.Ingress](ingLister) if err != nil { return nil, err } @@ -271,7 +268,7 @@ func buildGraph(ctx context.Context) (*Graph, error) { // workload to wherever its pods actually run right now -- this is the // "what's deployed where" question specifically, not covered by the // Service/Ingress graph above. - nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + nodes, err := listAll[corev1.Node](nodeLister) if err != nil { return nil, err } @@ -570,6 +567,9 @@ func handleResource(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusNotFound) return } + if a, ok := obj.(metav1.Object); ok { + a.SetManagedFields(nil) + } out, err := yaml.Marshal(obj) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -584,7 +584,7 @@ func handleResource(w http.ResponseWriter, r *http.Request) { // schedules-on edges -- so "which pods back this node" stays consistent // with what the graph already shows for placement. func podsForWorkload(ctx context.Context, ns, target string) ([]corev1.Pod, error) { - pods, err := clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{}) + pods, err := listAll[corev1.Pod](podLister.Pods(ns)) if err != nil { return nil, err } @@ -657,5 +657,3 @@ func handleLogs(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") io.Copy(w, stream) } - -var _ = os.Stdout // keep the os import if logging grows; avoids an unused-import edit later diff --git a/static/app.js b/static/app.js index 8c61a2e..edce823 100644 --- a/static/app.js +++ b/static/app.js @@ -532,8 +532,9 @@ function aggregateFlows(flows) { for (const e of flows.edges) { const key = e.source + '→' + e.target; let p = pairs.get(key); - if (!p) { p = { source: e.source, target: e.target, count: 0, dropped: 0, ports: [], last: '' }; pairs.set(key, p); } - p.count += e.count; + if (!p) { p = { source: e.source, target: e.target, count: 0, fresh: 0, dropped: 0, ports: [], last: '' }; pairs.set(key, p); } + p.count += e.connections; + p.fresh += e.new; p.dropped += e.dropped; if (!p.ports.includes(e.port)) p.ports.push(e.port); if (e.last > p.last) p.last = e.last; @@ -584,14 +585,63 @@ function buildElements() { } } state.trafficEdges.push(e); - const width = 1.6 + Math.min(4.5, Math.log10(e.count + 1) * 1.7); - const label = `${e.ports.slice(0, 3).join(', ')}${e.ports.length > 3 ? '…' : ''} · ${e.count}${e.dropped ? ` · drop ${e.dropped}` : ''}`; - add({ group: 'edges', classes: 'traffic' + (e.dropped ? ' drop' : ''), data: { id: 'tr:' + e.source + '→' + e.target, source: e.source, target: e.target, width, label, flow: e } }); + add(trafficEdgeElement(e)); } } return els; } +function trafficEdgeElement(e) { + return { + group: 'edges', classes: 'traffic' + (e.dropped ? ' drop' : ''), + data: { id: 'tr:' + e.source + '→' + e.target, source: e.source, target: e.target, ...trafficEdgeData(e) }, + }; +} +function trafficEdgeData(e) { + return { + width: 1.6 + Math.min(4.5, Math.log10(e.count + 1) * 1.7), + label: `${e.ports.slice(0, 3).join(', ')}${e.ports.length > 3 ? '…' : ''} · ${e.count} соед.${e.dropped ? ` · drop ${e.dropped}` : ''}`, + flow: e, + }; +} + +// Refresh traffic on the drawn map without relayout: counts and widths are +// updated, edges between cards already on the map come and go. Peers that +// would need new cards are only counted, so the layout never jumps under +// the cursor; "Обновить" redraws with them. +let hiddenPeers = 0; +async function refreshTraffic() { + if (!state.traffic || state.scope.mode === 'cluster' || document.hidden) return; + const token = renderToken; + let flows; + try { flows = await fetchFlows(scopeNamespaces().slice(0, 100)); } catch { return; } + if (token !== renderToken) return; + state.flows = flows; + state.flowsError = null; + const members = new Set(cy.nodes(':childless').filter((n) => !n.hasClass('ghostcard') && !n.data('external')).map((n) => n.id())); + const next = new Map(); + hiddenPeers = 0; + for (const e of aggregateFlows(flows)) { + if (!state.infra && isInfraEdge(e)) continue; + if (!members.has(e.source) && !members.has(e.target)) continue; + if (cy.getElementById(e.source).empty() || cy.getElementById(e.target).empty()) { hiddenPeers++; continue; } + next.set('tr:' + e.source + '→' + e.target, e); + } + cy.batch(() => { + cy.edges('.traffic').forEach((edge) => { if (!next.has(edge.id())) edge.remove(); }); + for (const [id, e] of next) { + const edge = cy.getElementById(id); + if (edge.nonempty()) { edge.data(trafficEdgeData(e)); edge.toggleClass('drop', e.dropped > 0); } + else cy.add(trafficEdgeElement(e)); + } + cy.edges('.traffic').toggleClass('quiet', next.size > QUIET_AT); + }); + state.trafficEdges = [...next.values()]; + if (detailsId) renderLinks(detailsId); + updateStatus(cy.elements().map((x) => ({ group: x.group(), data: x.data() }))); +} +setInterval(refreshTraffic, 30000); + function buildClusterElements() { const els = []; const groups = [ @@ -727,12 +777,14 @@ async function renderMap() { if (state.traffic && namespaces.length) { setStatus('загрузка трафика…'); try { - state.flows = await fetchFlows(namespaces.slice(0, 50)); + state.flows = await fetchFlows(namespaces.slice(0, 100)); + if (!state.flows.connected && !state.flows.edges?.length) state.flowsError = state.flows.error || 'нет связи с Hubble Relay'; } catch (err) { state.flowsError = err.message; } if (token !== renderToken) return; } + hiddenPeers = 0; const els = buildElements(); cy.batch(() => { cy.elements().remove(); cy.add(els); }); const quiet = cy.edges('.traffic').length > QUIET_AT; @@ -754,13 +806,25 @@ function updateStatus(els) { if (state.flowsError) { setStatus('трафик недоступен: ' + state.flowsError, true); return; } if (state.flows) { const total = state.trafficEdges.reduce((s, e) => s + e.count, 0); - const win = { 300: '5 мин', 900: '15 мин', 3600: '1 час' }[state.flows.windowSec] || state.flows.windowSec + ' с'; + const f = state.flows; + // The collector only knows what it has seen since the pod started. + const win = f.coveredSec < f.windowSec - 30 + ? `${fmtDuration(f.coveredSec)} (из ${fmtDuration(f.windowSec)}: сбор идёт с запуска)` + : fmtDuration(f.windowSec); + if (!f.connected) setStatus(`связь с Hubble потеряна (${f.error || 'нет ответа'}), данные могут устареть`, true); const hint = state.trafficEdges.length > QUIET_AT ? ' — наведите на карточку, чтобы увидеть её трафик' : ''; + if (!f.connected) return; + const more = hiddenPeers ? `; ещё ${plural(hiddenPeers, 'новое направление', 'новых направления', 'новых направлений')} — «Обновить»` : ''; setStatus(state.trafficEdges.length - ? `трафик за ${win}: ${plural(state.trafficEdges.length, 'направление', 'направления', 'направлений')}, ${plural(total, 'соединение', 'соединения', 'соединений')}${hint}` - : `трафика за ${win} не видно`); + ? `трафик за ${win}: ${plural(state.trafficEdges.length, 'направление', 'направления', 'направлений')}, ${plural(total, 'соединение', 'соединения', 'соединений')}${hint}${more}` + : `трафика за ${win} не видно${more}`); } } +function fmtDuration(sec) { + if (sec < 90) return plural(Math.max(1, Math.round(sec)), 'секунду', 'секунды', 'секунд'); + if (sec < 5400) return Math.round(sec / 60) + ' мин'; + return plural(Math.round(sec / 3600), 'час', 'часа', 'часов'); +} function setStatus(text, isError = false) { const s = $('#stage-status'); s.textContent = text; @@ -823,16 +887,24 @@ $('#zoom-in').addEventListener('click', () => cy.zoom({ level: cy.zoom() * 1.25, $('#zoom-out').addEventListener('click', () => cy.zoom({ level: cy.zoom() / 1.25, renderedPosition: { x: cy.width() / 2, y: cy.height() / 2 } })); $('#zoom-fit').addEventListener('click', fitView); -/* animated dashes on traffic edges */ +/* Animated dashes on traffic edges. Any edge style change makes cytoscape + redraw the whole element layer (~15 ms for a 200-element node view), so + the animation is rationed: 12 fps, only highlighted edges in dense views + (no highlight -> no redraw at all), and asleep after a minute without + input. */ +const reduceMotion = matchMedia('(prefers-reduced-motion: reduce)'); let dash = 0; let lastTick = 0; +let lastInput = performance.now(); +for (const ev of ['pointermove', 'wheel', 'keydown', 'pointerdown']) addEventListener(ev, () => { lastInput = performance.now(); }, { passive: true }); function tick(t) { requestAnimationFrame(tick); - if (document.hidden || matchMedia('(prefers-reduced-motion: reduce)').matches || t - lastTick < 50) return; + if (document.hidden || reduceMotion.matches || t - lastTick < 80 || t - lastInput > 60000) return; lastTick = t; - const edges = cy.edges('.traffic'); + let edges = cy.edges('.traffic'); + if (edges.length > QUIET_AT) edges = edges.filter('.hl'); if (!edges.length) return; - dash = (dash + 1.2) % 120; + dash = (dash + 2) % 120; edges.style('line-dash-offset', -dash); } requestAnimationFrame(tick); @@ -985,7 +1057,7 @@ function renderLinks(id) { } const tIn = [], tOut = []; for (const e of state.trafficEdges || []) { - const meta = `${e.ports.join(', ')} · ${e.count}${e.dropped ? ` · drop ${e.dropped}` : ''}`; + const meta = `${e.ports.join(', ')} · ${e.count} соед.${e.fresh ? `, новых ${e.fresh}` : ''}${e.dropped ? ` · drop ${e.dropped}` : ''}`; if (e.target === id) tIn.push(linkButton(e.source, meta, e.dropped)); if (e.source === id) tOut.push(linkButton(e.target, meta, e.dropped)); }