Files
cluster-map/flows.go
T
Vitaliy ChefanovandClaude Opus 5.5 6ead0a0d6d Collect Hubble traffic continuously; graph from informer caches
The ring buffer on each agent holds 4095 flows, ~10 s at the observed
~380 flows/s, so the 5/15/60 min windows were really showing the last
few seconds. A background follow stream now fills per-minute buckets
for the last hour, and counts are distinct connections (5-tuples)
instead of Hubble events, with opened (SYN) and dropped broken out.
/api/flows reports coveredSec so the UI can say how much of the window
the collector has seen since start.

/api/graph reads informer caches instead of listing every kind per
request (~0.9 s before). Pod-to-workload resolution for flows uses the
same cache. YAML view drops managedFields.

Frontend: traffic refreshes in place every 30 s without relayout; dash
animation rationed (12 fps, only highlighted edges in dense views,
paused after a minute idle) since each step redraws the whole element
layer, ~15 ms on a 200-element view.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 20:23:20 +03:00

424 lines
12 KiB
Go

package main
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"hash/fnv"
"log"
"net/http"
"os"
"sort"
"strconv"
"strings"
"sync"
"time"
flowpb "github.com/cilium/cilium/api/v1/flow"
observerpb "github.com/cilium/cilium/api/v1/observer"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/fieldmaskpb"
"google.golang.org/protobuf/types/known/timestamppb"
)
// Relay serves a *.hubble-relay.cilium.io certificate signed by Deckhouse's
// hubble CA and does not ask for a client certificate, so plain TLS with
// that CA is enough. The CA is mounted from a ConfigMap (public part only).
const hubbleServerName = "cluster-map.hubble-relay.cilium.io"
// Each agent keeps only 4095 flows in its ring buffer, which at a few
// hundred flows/s is ~10 s of history. So traffic is collected continuously
// from a follow stream into per-minute buckets covering the last hour.
const (
bucketCount = 60
maxTuplesPerKey = 20000 // per edge per minute; beyond it counts stop growing
)
type FlowEndpoint struct {
ID string `json:"id"`
Label string `json:"label"`
Kind string `json:"kind"` // lowercased controller kind, "pod", or "external"
Namespace string `json:"namespace,omitempty"`
}
type FlowEdge struct {
Source string `json:"source"`
Target string `json:"target"`
Port string `json:"port"` // "443/TCP"
Connections int `json:"connections"` // distinct 5-tuples seen in the window
New int `json:"new"` // of those, opened (TCP SYN) in the window
Dropped int `json:"dropped"` // distinct 5-tuples with a DROPPED verdict
Last string `json:"last"`
}
type FlowsResult struct {
WindowSec int `json:"windowSec"`
CoveredSec int `json:"coveredSec"` // how much of the window the collector has actually seen
Connected bool `json:"connected"`
Error string `json:"error,omitempty"`
Endpoints []FlowEndpoint `json:"endpoints"`
Edges []FlowEdge `json:"edges"`
}
type edgeKey struct{ src, dst, port string }
const (
tupleSYN uint8 = 1
tupleDrop uint8 = 2
)
type bucketEdge struct {
tuples map[uint64]uint8
last int64
}
type bucket struct {
minute int64
edges map[edgeKey]*bucketEdge
}
type collector struct {
mu sync.Mutex
buckets [bucketCount]*bucket
endpoints map[string]FlowEndpoint
startedAt time.Time
connected bool
lastErr string
flows uint64
}
var flowsCollector = &collector{endpoints: map[string]FlowEndpoint{}}
func startHubble() {
caPEM, err := os.ReadFile(envOr("HUBBLE_CA_FILE", "/etc/hubble-ca/ca.crt"))
if err != nil {
flowsCollector.setErr(fmt.Errorf("hubble CA: %w", err))
return
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(caPEM) {
flowsCollector.setErr(errors.New("hubble CA: no certificate in file"))
return
}
creds := credentials.NewTLS(&tls.Config{RootCAs: pool, ServerName: hubbleServerName, MinVersion: tls.VersionTLS12})
conn, err := grpc.NewClient(envOr("HUBBLE_RELAY_ADDR", "hubble-relay.d8-cni-cilium.svc:443"), grpc.WithTransportCredentials(creds))
if err != nil {
flowsCollector.setErr(err)
return
}
go flowsCollector.run(observerpb.NewObserverClient(conn))
}
func envOr(k, def string) string {
if v := os.Getenv(k); v != "" {
return v
}
return def
}
func (c *collector) setErr(err error) {
c.mu.Lock()
defer c.mu.Unlock()
c.connected = false
c.lastErr = err.Error()
log.Printf("hubble: %v", err)
}
var flowFields = []string{"time", "verdict", "IP", "l4", "source", "destination", "source_names", "destination_names", "node_name"}
func (c *collector) run(obs observerpb.ObserverClient) {
c.mu.Lock()
c.startedAt = time.Now()
c.mu.Unlock()
useMask := true
backoff := time.Second
for {
req := &observerpb.GetFlowsRequest{
Follow: true,
// backfill whatever the ring buffers still hold; duplicate
// tuples after a reconnect collapse in the per-minute sets
Since: timestamppb.New(time.Now().Add(-time.Minute)),
Whitelist: []*flowpb.FlowFilter{{Reply: []bool{false}}},
}
if useMask {
req.FieldMask = &fieldmaskpb.FieldMask{Paths: flowFields}
}
before := c.flowsSeen()
err := c.stream(obs, req)
if status.Code(err) == codes.InvalidArgument && useMask {
useMask = false
log.Printf("hubble: relay rejected field mask, retrying without it: %v", err)
continue
}
c.setErr(err)
if c.flowsSeen() > before {
backoff = time.Second // the stream worked for a while; reconnect quickly
}
time.Sleep(backoff)
backoff = min(backoff*2, 30*time.Second)
}
}
func (c *collector) flowsSeen() uint64 {
c.mu.Lock()
defer c.mu.Unlock()
return c.flows
}
func (c *collector) stream(obs observerpb.ObserverClient, req *observerpb.GetFlowsRequest) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
s, err := obs.GetFlows(ctx, req)
if err != nil {
return err
}
for {
resp, err := s.Recv()
if err != nil {
return err
}
if f := resp.GetFlow(); f != nil {
c.add(f)
}
}
}
func (c *collector) add(f *flowpb.Flow) {
src := flowEndpoint(f.GetSource(), f.GetSourceNames(), f.GetNodeName())
dst := flowEndpoint(f.GetDestination(), f.GetDestinationNames(), f.GetNodeName())
if src.ID == dst.ID {
return
}
port, tuple, syn := l4Info(f)
ts := time.Now()
if t := f.GetTime(); t != nil {
ts = t.AsTime()
}
minute := ts.Unix() / 60
c.mu.Lock()
defer c.mu.Unlock()
if !c.connected {
c.connected, c.lastErr = true, ""
}
c.flows++
b := c.buckets[minute%bucketCount]
if b == nil || b.minute != minute {
if b != nil && b.minute > minute {
return // older than the retained hour
}
b = &bucket{minute: minute, edges: map[edgeKey]*bucketEdge{}}
c.buckets[minute%bucketCount] = b
c.pruneEndpoints()
}
c.endpoints[src.ID] = src
c.endpoints[dst.ID] = dst
k := edgeKey{src.ID, dst.ID, port}
e := b.edges[k]
if e == nil {
e = &bucketEdge{tuples: map[uint64]uint8{}}
b.edges[k] = e
}
h := hashTuple(f.GetIP().GetSource(), f.GetIP().GetDestination(), tuple)
flags, seen := e.tuples[h]
if !seen && len(e.tuples) >= maxTuplesPerKey {
return
}
if syn {
flags |= tupleSYN
}
if f.GetVerdict() == flowpb.Verdict_DROPPED {
flags |= tupleDrop
}
e.tuples[h] = flags
e.last = max(e.last, ts.Unix())
}
// pruneEndpoints drops metadata for endpoints no retained bucket mentions.
func (c *collector) pruneEndpoints() {
used := map[string]bool{}
for _, b := range c.buckets {
if b == nil {
continue
}
for k := range b.edges {
used[k.src], used[k.dst] = true, true
}
}
for id := range c.endpoints {
if !used[id] {
delete(c.endpoints, id)
}
}
}
func hashTuple(srcIP, dstIP, rest string) uint64 {
h := fnv.New64a()
h.Write([]byte(srcIP))
h.Write([]byte{0})
h.Write([]byte(dstIP))
h.Write([]byte{0})
h.Write([]byte(rest))
return h.Sum64()
}
// l4Info returns the display port ("443/TCP"), the port part of the
// connection tuple, and whether this event opens a TCP connection.
func l4Info(f *flowpb.Flow) (port, tuple string, syn bool) {
l4 := f.GetL4()
switch {
case l4.GetTCP() != nil:
t := l4.GetTCP()
fl := t.GetFlags()
return strconv.Itoa(int(t.GetDestinationPort())) + "/TCP",
"tcp:" + strconv.Itoa(int(t.GetSourcePort())) + ":" + strconv.Itoa(int(t.GetDestinationPort())),
fl.GetSYN() && !fl.GetACK()
case l4.GetUDP() != nil:
u := l4.GetUDP()
return strconv.Itoa(int(u.GetDestinationPort())) + "/UDP",
"udp:" + strconv.Itoa(int(u.GetSourcePort())) + ":" + strconv.Itoa(int(u.GetDestinationPort())), false
case l4.GetSCTP() != nil:
s := l4.GetSCTP()
return strconv.Itoa(int(s.GetDestinationPort())) + "/SCTP",
"sctp:" + strconv.Itoa(int(s.GetSourcePort())) + ":" + strconv.Itoa(int(s.GetDestinationPort())), false
case l4.GetICMPv4() != nil || l4.GetICMPv6() != nil:
return "ICMP", "icmp", false
}
return "?", "?", false
}
func (c *collector) query(namespaces map[string]bool, window int) FlowsResult {
now := time.Now()
fromMinute := now.Add(-time.Duration(window)*time.Second).Unix() / 60
c.mu.Lock()
defer c.mu.Unlock()
res := FlowsResult{WindowSec: window, Connected: c.connected, Error: c.lastErr}
if !c.startedAt.IsZero() {
res.CoveredSec = min(window, int(now.Sub(c.startedAt).Seconds()))
}
touches := func(id string) bool { return namespaces[c.endpoints[id].Namespace] }
type acc struct {
tuples map[uint64]uint8
last int64
}
merged := map[edgeKey]*acc{}
for _, b := range c.buckets {
if b == nil || b.minute < fromMinute {
continue
}
for k, e := range b.edges {
if !touches(k.src) && !touches(k.dst) {
continue
}
a := merged[k]
if a == nil {
a = &acc{tuples: make(map[uint64]uint8, len(e.tuples))}
merged[k] = a
}
for h, fl := range e.tuples {
a.tuples[h] |= fl
}
a.last = max(a.last, e.last)
}
}
eps := map[string]bool{}
for k, a := range merged {
fe := FlowEdge{Source: k.src, Target: k.dst, Port: k.port, Connections: len(a.tuples), Last: time.Unix(a.last, 0).UTC().Format(time.RFC3339)}
for _, fl := range a.tuples {
if fl&tupleSYN != 0 {
fe.New++
}
if fl&tupleDrop != 0 {
fe.Dropped++
}
}
res.Edges = append(res.Edges, fe)
eps[k.src], eps[k.dst] = true, true
}
for id := range eps {
res.Endpoints = append(res.Endpoints, c.endpoints[id])
}
sort.Slice(res.Endpoints, func(i, j int) bool { return res.Endpoints[i].ID < res.Endpoints[j].ID })
sort.Slice(res.Edges, func(i, j int) bool { return res.Edges[i].Connections > res.Edges[j].Connections })
return res
}
func handleFlows(w http.ResponseWriter, r *http.Request) {
namespaces := map[string]bool{}
for _, ns := range strings.Split(r.URL.Query().Get("namespaces"), ",") {
if ns = strings.TrimSpace(ns); ns != "" {
namespaces[ns] = true
}
}
if len(namespaces) == 0 || len(namespaces) > 100 {
http.Error(w, "namespaces: от 1 до 100 через запятую", http.StatusBadRequest)
return
}
window, _ := strconv.Atoi(r.URL.Query().Get("window"))
if window <= 0 {
window = 300
}
window = min(window, bucketCount*60)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(flowsCollector.query(namespaces, window))
}
// flowEndpoint maps a Hubble endpoint to an ID that matches a graph node
// when the peer is one of our workloads. Hubble leaves Endpoint.Workloads
// empty on some flows (typically the half observed on the peer's node), so
// pods are resolved through the informer cache with the graph's own rules,
// and Hubble's workload name is only a fallback for pods already gone.
func flowEndpoint(e *flowpb.Endpoint, names []string, nodeName string) FlowEndpoint {
if e.GetPodName() != "" {
ns := e.GetNamespace()
if p, err := podLister.Pods(ns).Get(e.GetPodName()); err == nil {
if kind, name, _ := podOwner(*p); kind != "" {
return FlowEndpoint{ID: workloadID(ns, kind, name), Label: name, Kind: kind, Namespace: ns}
}
}
for _, wl := range e.GetWorkloads() {
kind, name := strings.ToLower(wl.GetKind()), wl.GetName()
if kind == "job" {
if i := strings.LastIndex(name, "-"); i > 0 && allDigits(name[i+1:]) {
kind, name = "cronjob", name[:i]
}
}
if kind != "" && name != "" {
return FlowEndpoint{ID: workloadID(ns, kind, name), Label: name, Kind: kind, Namespace: ns}
}
}
return FlowEndpoint{ID: "pod/" + ns + "/" + e.GetPodName(), Label: e.GetPodName(), Kind: "pod", Namespace: ns}
}
labels := e.GetLabels()
has := func(l string) bool {
for _, x := range labels {
if x == l {
return true
}
}
return false
}
switch {
case has("reserved:host"):
return FlowEndpoint{ID: "ext/host/" + nodeName, Label: "host " + nodeName, Kind: "external"}
case has("reserved:remote-node"):
return FlowEndpoint{ID: "ext/remote-node", Label: "другие ноды", Kind: "external"}
case has("reserved:kube-apiserver"):
return FlowEndpoint{ID: "ext/kube-apiserver", Label: "kube-apiserver", Kind: "external"}
case has("reserved:health"):
return FlowEndpoint{ID: "ext/health", Label: "cilium health", Kind: "external"}
}
if len(names) > 0 {
return FlowEndpoint{ID: "ext/dns/" + names[0], Label: names[0], Kind: "external"}
}
return FlowEndpoint{ID: "ext/world", Label: "интернет", Kind: "external"}
}