The ring buffer on each agent holds 4095 flows, ~10 s at the observed ~380 flows/s, so the 5/15/60 min windows were really showing the last few seconds. A background follow stream now fills per-minute buckets for the last hour, and counts are distinct connections (5-tuples) instead of Hubble events, with opened (SYN) and dropped broken out. /api/flows reports coveredSec so the UI can say how much of the window the collector has seen since start. /api/graph reads informer caches instead of listing every kind per request (~0.9 s before). Pod-to-workload resolution for flows uses the same cache. YAML view drops managedFields. Frontend: traffic refreshes in place every 30 s without relayout; dash animation rationed (12 fps, only highlighted edges in dense views, paused after a minute idle) since each step redraws the whole element layer, ~15 ms on a 200-element view. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
424 lines
12 KiB
Go
424 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"hash/fnv"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
flowpb "github.com/cilium/cilium/api/v1/flow"
|
|
observerpb "github.com/cilium/cilium/api/v1/observer"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/credentials"
|
|
"google.golang.org/grpc/status"
|
|
"google.golang.org/protobuf/types/known/fieldmaskpb"
|
|
"google.golang.org/protobuf/types/known/timestamppb"
|
|
)
|
|
|
|
// Relay serves a *.hubble-relay.cilium.io certificate signed by Deckhouse's
|
|
// hubble CA and does not ask for a client certificate, so plain TLS with
|
|
// that CA is enough. The CA is mounted from a ConfigMap (public part only).
|
|
const hubbleServerName = "cluster-map.hubble-relay.cilium.io"
|
|
|
|
// Each agent keeps only 4095 flows in its ring buffer, which at a few
|
|
// hundred flows/s is ~10 s of history. So traffic is collected continuously
|
|
// from a follow stream into per-minute buckets covering the last hour.
|
|
const (
|
|
bucketCount = 60
|
|
maxTuplesPerKey = 20000 // per edge per minute; beyond it counts stop growing
|
|
)
|
|
|
|
type FlowEndpoint struct {
|
|
ID string `json:"id"`
|
|
Label string `json:"label"`
|
|
Kind string `json:"kind"` // lowercased controller kind, "pod", or "external"
|
|
Namespace string `json:"namespace,omitempty"`
|
|
}
|
|
|
|
type FlowEdge struct {
|
|
Source string `json:"source"`
|
|
Target string `json:"target"`
|
|
Port string `json:"port"` // "443/TCP"
|
|
Connections int `json:"connections"` // distinct 5-tuples seen in the window
|
|
New int `json:"new"` // of those, opened (TCP SYN) in the window
|
|
Dropped int `json:"dropped"` // distinct 5-tuples with a DROPPED verdict
|
|
Last string `json:"last"`
|
|
}
|
|
|
|
type FlowsResult struct {
|
|
WindowSec int `json:"windowSec"`
|
|
CoveredSec int `json:"coveredSec"` // how much of the window the collector has actually seen
|
|
Connected bool `json:"connected"`
|
|
Error string `json:"error,omitempty"`
|
|
Endpoints []FlowEndpoint `json:"endpoints"`
|
|
Edges []FlowEdge `json:"edges"`
|
|
}
|
|
|
|
type edgeKey struct{ src, dst, port string }
|
|
|
|
const (
|
|
tupleSYN uint8 = 1
|
|
tupleDrop uint8 = 2
|
|
)
|
|
|
|
type bucketEdge struct {
|
|
tuples map[uint64]uint8
|
|
last int64
|
|
}
|
|
|
|
type bucket struct {
|
|
minute int64
|
|
edges map[edgeKey]*bucketEdge
|
|
}
|
|
|
|
type collector struct {
|
|
mu sync.Mutex
|
|
buckets [bucketCount]*bucket
|
|
endpoints map[string]FlowEndpoint
|
|
startedAt time.Time
|
|
connected bool
|
|
lastErr string
|
|
flows uint64
|
|
}
|
|
|
|
var flowsCollector = &collector{endpoints: map[string]FlowEndpoint{}}
|
|
|
|
func startHubble() {
|
|
caPEM, err := os.ReadFile(envOr("HUBBLE_CA_FILE", "/etc/hubble-ca/ca.crt"))
|
|
if err != nil {
|
|
flowsCollector.setErr(fmt.Errorf("hubble CA: %w", err))
|
|
return
|
|
}
|
|
pool := x509.NewCertPool()
|
|
if !pool.AppendCertsFromPEM(caPEM) {
|
|
flowsCollector.setErr(errors.New("hubble CA: no certificate in file"))
|
|
return
|
|
}
|
|
creds := credentials.NewTLS(&tls.Config{RootCAs: pool, ServerName: hubbleServerName, MinVersion: tls.VersionTLS12})
|
|
conn, err := grpc.NewClient(envOr("HUBBLE_RELAY_ADDR", "hubble-relay.d8-cni-cilium.svc:443"), grpc.WithTransportCredentials(creds))
|
|
if err != nil {
|
|
flowsCollector.setErr(err)
|
|
return
|
|
}
|
|
go flowsCollector.run(observerpb.NewObserverClient(conn))
|
|
}
|
|
|
|
func envOr(k, def string) string {
|
|
if v := os.Getenv(k); v != "" {
|
|
return v
|
|
}
|
|
return def
|
|
}
|
|
|
|
func (c *collector) setErr(err error) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
c.connected = false
|
|
c.lastErr = err.Error()
|
|
log.Printf("hubble: %v", err)
|
|
}
|
|
|
|
var flowFields = []string{"time", "verdict", "IP", "l4", "source", "destination", "source_names", "destination_names", "node_name"}
|
|
|
|
func (c *collector) run(obs observerpb.ObserverClient) {
|
|
c.mu.Lock()
|
|
c.startedAt = time.Now()
|
|
c.mu.Unlock()
|
|
useMask := true
|
|
backoff := time.Second
|
|
for {
|
|
req := &observerpb.GetFlowsRequest{
|
|
Follow: true,
|
|
// backfill whatever the ring buffers still hold; duplicate
|
|
// tuples after a reconnect collapse in the per-minute sets
|
|
Since: timestamppb.New(time.Now().Add(-time.Minute)),
|
|
Whitelist: []*flowpb.FlowFilter{{Reply: []bool{false}}},
|
|
}
|
|
if useMask {
|
|
req.FieldMask = &fieldmaskpb.FieldMask{Paths: flowFields}
|
|
}
|
|
before := c.flowsSeen()
|
|
err := c.stream(obs, req)
|
|
if status.Code(err) == codes.InvalidArgument && useMask {
|
|
useMask = false
|
|
log.Printf("hubble: relay rejected field mask, retrying without it: %v", err)
|
|
continue
|
|
}
|
|
c.setErr(err)
|
|
if c.flowsSeen() > before {
|
|
backoff = time.Second // the stream worked for a while; reconnect quickly
|
|
}
|
|
time.Sleep(backoff)
|
|
backoff = min(backoff*2, 30*time.Second)
|
|
}
|
|
}
|
|
|
|
func (c *collector) flowsSeen() uint64 {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.flows
|
|
}
|
|
|
|
func (c *collector) stream(obs observerpb.ObserverClient, req *observerpb.GetFlowsRequest) error {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
s, err := obs.GetFlows(ctx, req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for {
|
|
resp, err := s.Recv()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if f := resp.GetFlow(); f != nil {
|
|
c.add(f)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (c *collector) add(f *flowpb.Flow) {
|
|
src := flowEndpoint(f.GetSource(), f.GetSourceNames(), f.GetNodeName())
|
|
dst := flowEndpoint(f.GetDestination(), f.GetDestinationNames(), f.GetNodeName())
|
|
if src.ID == dst.ID {
|
|
return
|
|
}
|
|
port, tuple, syn := l4Info(f)
|
|
ts := time.Now()
|
|
if t := f.GetTime(); t != nil {
|
|
ts = t.AsTime()
|
|
}
|
|
minute := ts.Unix() / 60
|
|
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
if !c.connected {
|
|
c.connected, c.lastErr = true, ""
|
|
}
|
|
c.flows++
|
|
b := c.buckets[minute%bucketCount]
|
|
if b == nil || b.minute != minute {
|
|
if b != nil && b.minute > minute {
|
|
return // older than the retained hour
|
|
}
|
|
b = &bucket{minute: minute, edges: map[edgeKey]*bucketEdge{}}
|
|
c.buckets[minute%bucketCount] = b
|
|
c.pruneEndpoints()
|
|
}
|
|
c.endpoints[src.ID] = src
|
|
c.endpoints[dst.ID] = dst
|
|
k := edgeKey{src.ID, dst.ID, port}
|
|
e := b.edges[k]
|
|
if e == nil {
|
|
e = &bucketEdge{tuples: map[uint64]uint8{}}
|
|
b.edges[k] = e
|
|
}
|
|
h := hashTuple(f.GetIP().GetSource(), f.GetIP().GetDestination(), tuple)
|
|
flags, seen := e.tuples[h]
|
|
if !seen && len(e.tuples) >= maxTuplesPerKey {
|
|
return
|
|
}
|
|
if syn {
|
|
flags |= tupleSYN
|
|
}
|
|
if f.GetVerdict() == flowpb.Verdict_DROPPED {
|
|
flags |= tupleDrop
|
|
}
|
|
e.tuples[h] = flags
|
|
e.last = max(e.last, ts.Unix())
|
|
}
|
|
|
|
// pruneEndpoints drops metadata for endpoints no retained bucket mentions.
|
|
func (c *collector) pruneEndpoints() {
|
|
used := map[string]bool{}
|
|
for _, b := range c.buckets {
|
|
if b == nil {
|
|
continue
|
|
}
|
|
for k := range b.edges {
|
|
used[k.src], used[k.dst] = true, true
|
|
}
|
|
}
|
|
for id := range c.endpoints {
|
|
if !used[id] {
|
|
delete(c.endpoints, id)
|
|
}
|
|
}
|
|
}
|
|
|
|
func hashTuple(srcIP, dstIP, rest string) uint64 {
|
|
h := fnv.New64a()
|
|
h.Write([]byte(srcIP))
|
|
h.Write([]byte{0})
|
|
h.Write([]byte(dstIP))
|
|
h.Write([]byte{0})
|
|
h.Write([]byte(rest))
|
|
return h.Sum64()
|
|
}
|
|
|
|
// l4Info returns the display port ("443/TCP"), the port part of the
|
|
// connection tuple, and whether this event opens a TCP connection.
|
|
func l4Info(f *flowpb.Flow) (port, tuple string, syn bool) {
|
|
l4 := f.GetL4()
|
|
switch {
|
|
case l4.GetTCP() != nil:
|
|
t := l4.GetTCP()
|
|
fl := t.GetFlags()
|
|
return strconv.Itoa(int(t.GetDestinationPort())) + "/TCP",
|
|
"tcp:" + strconv.Itoa(int(t.GetSourcePort())) + ":" + strconv.Itoa(int(t.GetDestinationPort())),
|
|
fl.GetSYN() && !fl.GetACK()
|
|
case l4.GetUDP() != nil:
|
|
u := l4.GetUDP()
|
|
return strconv.Itoa(int(u.GetDestinationPort())) + "/UDP",
|
|
"udp:" + strconv.Itoa(int(u.GetSourcePort())) + ":" + strconv.Itoa(int(u.GetDestinationPort())), false
|
|
case l4.GetSCTP() != nil:
|
|
s := l4.GetSCTP()
|
|
return strconv.Itoa(int(s.GetDestinationPort())) + "/SCTP",
|
|
"sctp:" + strconv.Itoa(int(s.GetSourcePort())) + ":" + strconv.Itoa(int(s.GetDestinationPort())), false
|
|
case l4.GetICMPv4() != nil || l4.GetICMPv6() != nil:
|
|
return "ICMP", "icmp", false
|
|
}
|
|
return "?", "?", false
|
|
}
|
|
|
|
func (c *collector) query(namespaces map[string]bool, window int) FlowsResult {
|
|
now := time.Now()
|
|
fromMinute := now.Add(-time.Duration(window)*time.Second).Unix() / 60
|
|
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
res := FlowsResult{WindowSec: window, Connected: c.connected, Error: c.lastErr}
|
|
if !c.startedAt.IsZero() {
|
|
res.CoveredSec = min(window, int(now.Sub(c.startedAt).Seconds()))
|
|
}
|
|
touches := func(id string) bool { return namespaces[c.endpoints[id].Namespace] }
|
|
|
|
type acc struct {
|
|
tuples map[uint64]uint8
|
|
last int64
|
|
}
|
|
merged := map[edgeKey]*acc{}
|
|
for _, b := range c.buckets {
|
|
if b == nil || b.minute < fromMinute {
|
|
continue
|
|
}
|
|
for k, e := range b.edges {
|
|
if !touches(k.src) && !touches(k.dst) {
|
|
continue
|
|
}
|
|
a := merged[k]
|
|
if a == nil {
|
|
a = &acc{tuples: make(map[uint64]uint8, len(e.tuples))}
|
|
merged[k] = a
|
|
}
|
|
for h, fl := range e.tuples {
|
|
a.tuples[h] |= fl
|
|
}
|
|
a.last = max(a.last, e.last)
|
|
}
|
|
}
|
|
eps := map[string]bool{}
|
|
for k, a := range merged {
|
|
fe := FlowEdge{Source: k.src, Target: k.dst, Port: k.port, Connections: len(a.tuples), Last: time.Unix(a.last, 0).UTC().Format(time.RFC3339)}
|
|
for _, fl := range a.tuples {
|
|
if fl&tupleSYN != 0 {
|
|
fe.New++
|
|
}
|
|
if fl&tupleDrop != 0 {
|
|
fe.Dropped++
|
|
}
|
|
}
|
|
res.Edges = append(res.Edges, fe)
|
|
eps[k.src], eps[k.dst] = true, true
|
|
}
|
|
for id := range eps {
|
|
res.Endpoints = append(res.Endpoints, c.endpoints[id])
|
|
}
|
|
sort.Slice(res.Endpoints, func(i, j int) bool { return res.Endpoints[i].ID < res.Endpoints[j].ID })
|
|
sort.Slice(res.Edges, func(i, j int) bool { return res.Edges[i].Connections > res.Edges[j].Connections })
|
|
return res
|
|
}
|
|
|
|
func handleFlows(w http.ResponseWriter, r *http.Request) {
|
|
namespaces := map[string]bool{}
|
|
for _, ns := range strings.Split(r.URL.Query().Get("namespaces"), ",") {
|
|
if ns = strings.TrimSpace(ns); ns != "" {
|
|
namespaces[ns] = true
|
|
}
|
|
}
|
|
if len(namespaces) == 0 || len(namespaces) > 100 {
|
|
http.Error(w, "namespaces: от 1 до 100 через запятую", http.StatusBadRequest)
|
|
return
|
|
}
|
|
window, _ := strconv.Atoi(r.URL.Query().Get("window"))
|
|
if window <= 0 {
|
|
window = 300
|
|
}
|
|
window = min(window, bucketCount*60)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(flowsCollector.query(namespaces, window))
|
|
}
|
|
|
|
// flowEndpoint maps a Hubble endpoint to an ID that matches a graph node
|
|
// when the peer is one of our workloads. Hubble leaves Endpoint.Workloads
|
|
// empty on some flows (typically the half observed on the peer's node), so
|
|
// pods are resolved through the informer cache with the graph's own rules,
|
|
// and Hubble's workload name is only a fallback for pods already gone.
|
|
func flowEndpoint(e *flowpb.Endpoint, names []string, nodeName string) FlowEndpoint {
|
|
if e.GetPodName() != "" {
|
|
ns := e.GetNamespace()
|
|
if p, err := podLister.Pods(ns).Get(e.GetPodName()); err == nil {
|
|
if kind, name, _ := podOwner(*p); kind != "" {
|
|
return FlowEndpoint{ID: workloadID(ns, kind, name), Label: name, Kind: kind, Namespace: ns}
|
|
}
|
|
}
|
|
for _, wl := range e.GetWorkloads() {
|
|
kind, name := strings.ToLower(wl.GetKind()), wl.GetName()
|
|
if kind == "job" {
|
|
if i := strings.LastIndex(name, "-"); i > 0 && allDigits(name[i+1:]) {
|
|
kind, name = "cronjob", name[:i]
|
|
}
|
|
}
|
|
if kind != "" && name != "" {
|
|
return FlowEndpoint{ID: workloadID(ns, kind, name), Label: name, Kind: kind, Namespace: ns}
|
|
}
|
|
}
|
|
return FlowEndpoint{ID: "pod/" + ns + "/" + e.GetPodName(), Label: e.GetPodName(), Kind: "pod", Namespace: ns}
|
|
}
|
|
labels := e.GetLabels()
|
|
has := func(l string) bool {
|
|
for _, x := range labels {
|
|
if x == l {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
switch {
|
|
case has("reserved:host"):
|
|
return FlowEndpoint{ID: "ext/host/" + nodeName, Label: "host " + nodeName, Kind: "external"}
|
|
case has("reserved:remote-node"):
|
|
return FlowEndpoint{ID: "ext/remote-node", Label: "другие ноды", Kind: "external"}
|
|
case has("reserved:kube-apiserver"):
|
|
return FlowEndpoint{ID: "ext/kube-apiserver", Label: "kube-apiserver", Kind: "external"}
|
|
case has("reserved:health"):
|
|
return FlowEndpoint{ID: "ext/health", Label: "cilium health", Kind: "external"}
|
|
}
|
|
if len(names) > 0 {
|
|
return FlowEndpoint{ID: "ext/dns/" + names[0], Label: names[0], Kind: "external"}
|
|
}
|
|
return FlowEndpoint{ID: "ext/world", Label: "интернет", Kind: "external"}
|
|
}
|