package main import ( "context" "crypto/tls" "crypto/x509" "encoding/json" "errors" "fmt" "io" "net/http" "os" "sort" "strconv" "strings" "time" flowpb "github.com/cilium/cilium/api/v1/flow" observerpb "github.com/cilium/cilium/api/v1/observer" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/protobuf/types/known/timestamppb" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // Relay serves a *.hubble-relay.cilium.io certificate signed by Deckhouse's // hubble CA and does not ask for a client certificate, so plain TLS with // that CA is enough. The CA is mounted from a ConfigMap (public part only). const hubbleServerName = "cluster-map.hubble-relay.cilium.io" var observer observerpb.ObserverClient var hubbleInitErr error func initHubble() { addr := envOr("HUBBLE_RELAY_ADDR", "hubble-relay.d8-cni-cilium.svc:443") caPEM, err := os.ReadFile(envOr("HUBBLE_CA_FILE", "/etc/hubble-ca/ca.crt")) if err != nil { hubbleInitErr = fmt.Errorf("hubble CA: %w", err) return } pool := x509.NewCertPool() if !pool.AppendCertsFromPEM(caPEM) { hubbleInitErr = errors.New("hubble CA: no certificate in file") return } creds := credentials.NewTLS(&tls.Config{RootCAs: pool, ServerName: hubbleServerName, MinVersion: tls.VersionTLS12}) conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(creds)) if err != nil { hubbleInitErr = err return } observer = observerpb.NewObserverClient(conn) } func envOr(k, def string) string { if v := os.Getenv(k); v != "" { return v } return def } type FlowEndpoint struct { ID string `json:"id"` Label string `json:"label"` Kind string `json:"kind"` // lowercased controller kind, "pod", or "external" Namespace string `json:"namespace,omitempty"` } type FlowEdge struct { Source string `json:"source"` Target string `json:"target"` Port string `json:"port"` // "443/TCP" Count int `json:"count"` Dropped int `json:"dropped"` Last string `json:"last"` } type FlowsResult struct { WindowSec int `json:"windowSec"` Scanned int `json:"scanned"` Endpoints []FlowEndpoint `json:"endpoints"` Edges []FlowEdge `json:"edges"` } const maxFlows = 20000 // handleFlows aggregates recent Hubble flows touching the given namespaces // into workload-to-workload edges. Replies are excluded so each connection // is counted once, in the direction it was opened. func handleFlows(w http.ResponseWriter, r *http.Request) { if observer == nil { http.Error(w, "hubble недоступен: "+errString(hubbleInitErr), http.StatusServiceUnavailable) return } var namespaces []string for _, ns := range strings.Split(r.URL.Query().Get("namespaces"), ",") { if ns = strings.TrimSpace(ns); ns != "" { namespaces = append(namespaces, ns) } } if len(namespaces) == 0 || len(namespaces) > 50 { http.Error(w, "namespaces: от 1 до 50 через запятую", http.StatusBadRequest) return } window, _ := strconv.Atoi(r.URL.Query().Get("window")) if window <= 0 { window = 300 } if window > 3600 { window = 3600 } var podPrefixes []string for _, ns := range namespaces { podPrefixes = append(podPrefixes, ns+"/") } notReply := []bool{false} req := &observerpb.GetFlowsRequest{ Number: maxFlows, Since: timestamppb.New(time.Now().Add(-time.Duration(window) * time.Second)), Whitelist: []*flowpb.FlowFilter{ {SourcePod: podPrefixes, Reply: notReply}, {DestinationPod: podPrefixes, Reply: notReply}, }, } ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) defer cancel() stream, err := observer.GetFlows(ctx, req) if err != nil { http.Error(w, "hubble: "+err.Error(), http.StatusBadGateway) return } // Hubble leaves Endpoint.Workloads empty for some flows (typically the // half observed on the peer's node), so pods are resolved to their // workload here, with the same podOwner rules the graph uses. pods, err := clientset.CoreV1().Pods(corev1.NamespaceAll).List(ctx, metav1.ListOptions{}) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } podWl := make(map[string]FlowEndpoint, len(pods.Items)) for _, p := range pods.Items { if kind, name, _ := podOwner(p); kind != "" { podWl[p.Namespace+"/"+p.Name] = FlowEndpoint{ID: workloadID(p.Namespace, kind, name), Label: name, Kind: kind, Namespace: p.Namespace} } } res := FlowsResult{WindowSec: window} endpoints := map[string]FlowEndpoint{} edges := map[string]*FlowEdge{} for { resp, err := stream.Recv() if err == io.EOF { break } if err != nil { if res.Scanned == 0 { http.Error(w, "hubble: "+err.Error(), http.StatusBadGateway) return } break // partial result is still useful } f := resp.GetFlow() if f == nil { continue } res.Scanned++ src := flowEndpoint(f.GetSource(), f.GetSourceNames(), f.GetNodeName(), podWl) dst := flowEndpoint(f.GetDestination(), f.GetDestinationNames(), f.GetNodeName(), podWl) if src.ID == dst.ID { continue } endpoints[src.ID] = src endpoints[dst.ID] = dst port := l4Port(f.GetL4()) key := src.ID + "|" + dst.ID + "|" + port e := edges[key] if e == nil { e = &FlowEdge{Source: src.ID, Target: dst.ID, Port: port} edges[key] = e } e.Count++ if f.GetVerdict() == flowpb.Verdict_DROPPED { e.Dropped++ } if t := f.GetTime(); t != nil { if ts := t.AsTime().UTC().Format(time.RFC3339); ts > e.Last { e.Last = ts } } } for _, ep := range endpoints { res.Endpoints = append(res.Endpoints, ep) } for _, e := range edges { res.Edges = append(res.Edges, *e) } sort.Slice(res.Endpoints, func(i, j int) bool { return res.Endpoints[i].ID < res.Endpoints[j].ID }) sort.Slice(res.Edges, func(i, j int) bool { return res.Edges[i].Count > res.Edges[j].Count }) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(res) } // flowEndpoint maps a Hubble endpoint to an ID that matches a graph node // when the peer is one of our workloads, so traffic edges attach to the // same cards the structural graph already draws. func flowEndpoint(e *flowpb.Endpoint, names []string, nodeName string, podWl map[string]FlowEndpoint) FlowEndpoint { if e.GetPodName() != "" { ns := e.GetNamespace() if ep, ok := podWl[ns+"/"+e.GetPodName()]; ok { return ep } // pod already gone: fall back to what Hubble recorded for _, wl := range e.GetWorkloads() { kind, name := strings.ToLower(wl.GetKind()), wl.GetName() if kind == "job" { if i := strings.LastIndex(name, "-"); i > 0 && allDigits(name[i+1:]) { kind, name = "cronjob", name[:i] } } if kind == "" || name == "" { continue } return FlowEndpoint{ID: workloadID(ns, kind, name), Label: name, Kind: kind, Namespace: ns} } return FlowEndpoint{ID: "pod/" + ns + "/" + e.GetPodName(), Label: e.GetPodName(), Kind: "pod", Namespace: ns} } labels := e.GetLabels() has := func(l string) bool { for _, x := range labels { if x == l { return true } } return false } switch { case has("reserved:host"): return FlowEndpoint{ID: "ext/host/" + nodeName, Label: "host " + nodeName, Kind: "external"} case has("reserved:remote-node"): return FlowEndpoint{ID: "ext/remote-node", Label: "другие ноды", Kind: "external"} case has("reserved:kube-apiserver"): return FlowEndpoint{ID: "ext/kube-apiserver", Label: "kube-apiserver", Kind: "external"} case has("reserved:health"): return FlowEndpoint{ID: "ext/health", Label: "cilium health", Kind: "external"} } if len(names) > 0 { return FlowEndpoint{ID: "ext/dns/" + names[0], Label: names[0], Kind: "external"} } return FlowEndpoint{ID: "ext/world", Label: "интернет", Kind: "external"} } func l4Port(l4 *flowpb.Layer4) string { switch { case l4.GetTCP() != nil: return strconv.Itoa(int(l4.GetTCP().GetDestinationPort())) + "/TCP" case l4.GetUDP() != nil: return strconv.Itoa(int(l4.GetUDP().GetDestinationPort())) + "/UDP" case l4.GetICMPv4() != nil || l4.GetICMPv6() != nil: return "ICMP" case l4.GetSCTP() != nil: return strconv.Itoa(int(l4.GetSCTP().GetDestinationPort())) + "/SCTP" } return "?" } func errString(err error) string { if err == nil { return "не инициализирован" } return err.Error() }