package main import ( "context" "crypto/tls" "crypto/x509" "encoding/json" "errors" "fmt" "hash/fnv" "log" "net/http" "os" "sort" "strconv" "strings" "sync" "time" flowpb "github.com/cilium/cilium/api/v1/flow" observerpb "github.com/cilium/cilium/api/v1/observer" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/fieldmaskpb" "google.golang.org/protobuf/types/known/timestamppb" ) // Relay serves a *.hubble-relay.cilium.io certificate signed by Deckhouse's // hubble CA and does not ask for a client certificate, so plain TLS with // that CA is enough. The CA is mounted from a ConfigMap (public part only). const hubbleServerName = "cluster-map.hubble-relay.cilium.io" // Each agent keeps only 4095 flows in its ring buffer, which at a few // hundred flows/s is ~10 s of history. So traffic is collected continuously // from a follow stream into per-minute buckets covering the last hour. const ( bucketCount = 60 maxTuplesPerKey = 20000 // per edge per minute; beyond it counts stop growing ) type FlowEndpoint struct { ID string `json:"id"` Label string `json:"label"` Kind string `json:"kind"` // lowercased controller kind, "pod", or "external" Namespace string `json:"namespace,omitempty"` } type FlowEdge struct { Source string `json:"source"` Target string `json:"target"` Port string `json:"port"` // "443/TCP" Connections int `json:"connections"` // distinct 5-tuples seen in the window New int `json:"new"` // of those, opened (TCP SYN) in the window Dropped int `json:"dropped"` // distinct 5-tuples with a DROPPED verdict Last string `json:"last"` } type FlowsResult struct { WindowSec int `json:"windowSec"` CoveredSec int `json:"coveredSec"` // how much of the window the collector has actually seen Connected bool `json:"connected"` Error string `json:"error,omitempty"` Endpoints []FlowEndpoint `json:"endpoints"` Edges []FlowEdge `json:"edges"` } type edgeKey struct{ src, dst, port string } const ( tupleSYN uint8 = 1 tupleDrop uint8 = 2 ) type bucketEdge struct { tuples map[uint64]uint8 last int64 } type bucket struct { minute int64 edges map[edgeKey]*bucketEdge } type collector struct { mu sync.Mutex buckets [bucketCount]*bucket endpoints map[string]FlowEndpoint startedAt time.Time connected bool lastErr string flows uint64 } var flowsCollector = &collector{endpoints: map[string]FlowEndpoint{}} func startHubble() { caPEM, err := os.ReadFile(envOr("HUBBLE_CA_FILE", "/etc/hubble-ca/ca.crt")) if err != nil { flowsCollector.setErr(fmt.Errorf("hubble CA: %w", err)) return } pool := x509.NewCertPool() if !pool.AppendCertsFromPEM(caPEM) { flowsCollector.setErr(errors.New("hubble CA: no certificate in file")) return } creds := credentials.NewTLS(&tls.Config{RootCAs: pool, ServerName: hubbleServerName, MinVersion: tls.VersionTLS12}) conn, err := grpc.NewClient(envOr("HUBBLE_RELAY_ADDR", "hubble-relay.d8-cni-cilium.svc:443"), grpc.WithTransportCredentials(creds)) if err != nil { flowsCollector.setErr(err) return } go flowsCollector.run(observerpb.NewObserverClient(conn)) } func envOr(k, def string) string { if v := os.Getenv(k); v != "" { return v } return def } func (c *collector) setErr(err error) { c.mu.Lock() defer c.mu.Unlock() c.connected = false c.lastErr = err.Error() log.Printf("hubble: %v", err) } var flowFields = []string{"time", "verdict", "IP", "l4", "source", "destination", "source_names", "destination_names", "node_name"} func (c *collector) run(obs observerpb.ObserverClient) { c.mu.Lock() c.startedAt = time.Now() c.mu.Unlock() useMask := true backoff := time.Second for { req := &observerpb.GetFlowsRequest{ Follow: true, // backfill whatever the ring buffers still hold; duplicate // tuples after a reconnect collapse in the per-minute sets Since: timestamppb.New(time.Now().Add(-time.Minute)), Whitelist: []*flowpb.FlowFilter{{Reply: []bool{false}}}, } if useMask { req.FieldMask = &fieldmaskpb.FieldMask{Paths: flowFields} } before := c.flowsSeen() err := c.stream(obs, req) if status.Code(err) == codes.InvalidArgument && useMask { useMask = false log.Printf("hubble: relay rejected field mask, retrying without it: %v", err) continue } c.setErr(err) if c.flowsSeen() > before { backoff = time.Second // the stream worked for a while; reconnect quickly } time.Sleep(backoff) backoff = min(backoff*2, 30*time.Second) } } func (c *collector) flowsSeen() uint64 { c.mu.Lock() defer c.mu.Unlock() return c.flows } func (c *collector) stream(obs observerpb.ObserverClient, req *observerpb.GetFlowsRequest) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() s, err := obs.GetFlows(ctx, req) if err != nil { return err } for { resp, err := s.Recv() if err != nil { return err } if f := resp.GetFlow(); f != nil { c.add(f) } } } func (c *collector) add(f *flowpb.Flow) { src := flowEndpoint(f.GetSource(), f.GetSourceNames(), f.GetNodeName()) dst := flowEndpoint(f.GetDestination(), f.GetDestinationNames(), f.GetNodeName()) if src.ID == dst.ID { return } port, tuple, syn := l4Info(f) ts := time.Now() if t := f.GetTime(); t != nil { ts = t.AsTime() } minute := ts.Unix() / 60 c.mu.Lock() defer c.mu.Unlock() if !c.connected { c.connected, c.lastErr = true, "" } c.flows++ b := c.buckets[minute%bucketCount] if b == nil || b.minute != minute { if b != nil && b.minute > minute { return // older than the retained hour } b = &bucket{minute: minute, edges: map[edgeKey]*bucketEdge{}} c.buckets[minute%bucketCount] = b c.pruneEndpoints() } c.endpoints[src.ID] = src c.endpoints[dst.ID] = dst k := edgeKey{src.ID, dst.ID, port} e := b.edges[k] if e == nil { e = &bucketEdge{tuples: map[uint64]uint8{}} b.edges[k] = e } h := hashTuple(f.GetIP().GetSource(), f.GetIP().GetDestination(), tuple) flags, seen := e.tuples[h] if !seen && len(e.tuples) >= maxTuplesPerKey { return } if syn { flags |= tupleSYN } if f.GetVerdict() == flowpb.Verdict_DROPPED { flags |= tupleDrop } e.tuples[h] = flags e.last = max(e.last, ts.Unix()) } // pruneEndpoints drops metadata for endpoints no retained bucket mentions. func (c *collector) pruneEndpoints() { used := map[string]bool{} for _, b := range c.buckets { if b == nil { continue } for k := range b.edges { used[k.src], used[k.dst] = true, true } } for id := range c.endpoints { if !used[id] { delete(c.endpoints, id) } } } func hashTuple(srcIP, dstIP, rest string) uint64 { h := fnv.New64a() h.Write([]byte(srcIP)) h.Write([]byte{0}) h.Write([]byte(dstIP)) h.Write([]byte{0}) h.Write([]byte(rest)) return h.Sum64() } // l4Info returns the display port ("443/TCP"), the port part of the // connection tuple, and whether this event opens a TCP connection. func l4Info(f *flowpb.Flow) (port, tuple string, syn bool) { l4 := f.GetL4() switch { case l4.GetTCP() != nil: t := l4.GetTCP() fl := t.GetFlags() return strconv.Itoa(int(t.GetDestinationPort())) + "/TCP", "tcp:" + strconv.Itoa(int(t.GetSourcePort())) + ":" + strconv.Itoa(int(t.GetDestinationPort())), fl.GetSYN() && !fl.GetACK() case l4.GetUDP() != nil: u := l4.GetUDP() return strconv.Itoa(int(u.GetDestinationPort())) + "/UDP", "udp:" + strconv.Itoa(int(u.GetSourcePort())) + ":" + strconv.Itoa(int(u.GetDestinationPort())), false case l4.GetSCTP() != nil: s := l4.GetSCTP() return strconv.Itoa(int(s.GetDestinationPort())) + "/SCTP", "sctp:" + strconv.Itoa(int(s.GetSourcePort())) + ":" + strconv.Itoa(int(s.GetDestinationPort())), false case l4.GetICMPv4() != nil || l4.GetICMPv6() != nil: return "ICMP", "icmp", false } return "?", "?", false } func (c *collector) query(namespaces map[string]bool, window int) FlowsResult { now := time.Now() fromMinute := now.Add(-time.Duration(window)*time.Second).Unix() / 60 c.mu.Lock() defer c.mu.Unlock() res := FlowsResult{WindowSec: window, Connected: c.connected, Error: c.lastErr} if !c.startedAt.IsZero() { res.CoveredSec = min(window, int(now.Sub(c.startedAt).Seconds())) } touches := func(id string) bool { return namespaces[c.endpoints[id].Namespace] } type acc struct { tuples map[uint64]uint8 last int64 } merged := map[edgeKey]*acc{} for _, b := range c.buckets { if b == nil || b.minute < fromMinute { continue } for k, e := range b.edges { if !touches(k.src) && !touches(k.dst) { continue } a := merged[k] if a == nil { a = &acc{tuples: make(map[uint64]uint8, len(e.tuples))} merged[k] = a } for h, fl := range e.tuples { a.tuples[h] |= fl } a.last = max(a.last, e.last) } } eps := map[string]bool{} for k, a := range merged { fe := FlowEdge{Source: k.src, Target: k.dst, Port: k.port, Connections: len(a.tuples), Last: time.Unix(a.last, 0).UTC().Format(time.RFC3339)} for _, fl := range a.tuples { if fl&tupleSYN != 0 { fe.New++ } if fl&tupleDrop != 0 { fe.Dropped++ } } res.Edges = append(res.Edges, fe) eps[k.src], eps[k.dst] = true, true } for id := range eps { res.Endpoints = append(res.Endpoints, c.endpoints[id]) } sort.Slice(res.Endpoints, func(i, j int) bool { return res.Endpoints[i].ID < res.Endpoints[j].ID }) sort.Slice(res.Edges, func(i, j int) bool { return res.Edges[i].Connections > res.Edges[j].Connections }) return res } func handleFlows(w http.ResponseWriter, r *http.Request) { namespaces := map[string]bool{} for _, ns := range strings.Split(r.URL.Query().Get("namespaces"), ",") { if ns = strings.TrimSpace(ns); ns != "" { namespaces[ns] = true } } if len(namespaces) == 0 || len(namespaces) > 100 { http.Error(w, "namespaces: от 1 до 100 через запятую", http.StatusBadRequest) return } window, _ := strconv.Atoi(r.URL.Query().Get("window")) if window <= 0 { window = 300 } window = min(window, bucketCount*60) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(flowsCollector.query(namespaces, window)) } // flowEndpoint maps a Hubble endpoint to an ID that matches a graph node // when the peer is one of our workloads. Hubble leaves Endpoint.Workloads // empty on some flows (typically the half observed on the peer's node), so // pods are resolved through the informer cache with the graph's own rules, // and Hubble's workload name is only a fallback for pods already gone. func flowEndpoint(e *flowpb.Endpoint, names []string, nodeName string) FlowEndpoint { if e.GetPodName() != "" { ns := e.GetNamespace() if p, err := podLister.Pods(ns).Get(e.GetPodName()); err == nil { if kind, name, _ := podOwner(*p); kind != "" { return FlowEndpoint{ID: workloadID(ns, kind, name), Label: name, Kind: kind, Namespace: ns} } } for _, wl := range e.GetWorkloads() { kind, name := strings.ToLower(wl.GetKind()), wl.GetName() if kind == "job" { if i := strings.LastIndex(name, "-"); i > 0 && allDigits(name[i+1:]) { kind, name = "cronjob", name[:i] } } if kind != "" && name != "" { return FlowEndpoint{ID: workloadID(ns, kind, name), Label: name, Kind: kind, Namespace: ns} } } return FlowEndpoint{ID: "pod/" + ns + "/" + e.GetPodName(), Label: e.GetPodName(), Kind: "pod", Namespace: ns} } labels := e.GetLabels() has := func(l string) bool { for _, x := range labels { if x == l { return true } } return false } switch { case has("reserved:host"): return FlowEndpoint{ID: "ext/host/" + nodeName, Label: "host " + nodeName, Kind: "external"} case has("reserved:remote-node"): return FlowEndpoint{ID: "ext/remote-node", Label: "другие ноды", Kind: "external"} case has("reserved:kube-apiserver"): return FlowEndpoint{ID: "ext/kube-apiserver", Label: "kube-apiserver", Kind: "external"} case has("reserved:health"): return FlowEndpoint{ID: "ext/health", Label: "cilium health", Kind: "external"} } if len(names) > 0 { return FlowEndpoint{ID: "ext/dns/" + names[0], Label: names[0], Kind: "external"} } return FlowEndpoint{ID: "ext/world", Label: "интернет", Kind: "external"} }