Collect Hubble traffic continuously; graph from informer caches

The ring buffer on each agent holds 4095 flows, ~10 s at the observed
~380 flows/s, so the 5/15/60 min windows were really showing the last
few seconds. A background follow stream now fills per-minute buckets
for the last hour, and counts are distinct connections (5-tuples)
instead of Hubble events, with opened (SYN) and dropped broken out.
/api/flows reports coveredSec so the UI can say how much of the window
the collector has seen since start.

/api/graph reads informer caches instead of listing every kind per
request (~0.9 s before). Pod-to-workload resolution for flows uses the
same cache. YAML view drops managedFields.

Frontend: traffic refreshes in place every 30 s without relayout; dash
animation rationed (12 fps, only highlighted edges in dense views,
paused after a minute idle) since each step redraws the whole element
layer, ~15 ms on a 200-element view.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Vitaliy Chefanov
2026-09-23 20:23:20 +03:00
co-authored by Claude Opus 5.5
parent cf84e9976f
commit 6ead0a0d6d
5 changed files with 547 additions and 237 deletions
+21 -23
View File
@@ -1,12 +1,10 @@
// cluster-map: a live topology view of a Deckhouse/Kubernetes cluster.
//
// Builds a graph from the live k8s API on every request to /api/graph:
// namespaces group workloads (Deployment/StatefulSet/DaemonSet/CronJob),
// which connect to Services (by label-selector match) and Ingresses (by
// backend reference). Nothing is cached or watched -- this is meant to
// be a small, low-traffic internal tool, not a high-QPS dashboard, so a
// fresh List() per request is simpler and correct-by-construction (no
// stale-cache class of bugs to worry about).
// Builds a graph on every request to /api/graph from informer caches:
// namespaces group workloads, which connect to Services (by label-selector
// match) and Ingresses (by backend reference). Informers keep the caches
// in step with the API by watch; a fresh List per request cost ~0.9 s and
// timed out the 1 s probe.
package main
import (
@@ -15,7 +13,6 @@ import (
"io"
"log"
"net/http"
"os"
"sort"
"strings"
@@ -63,10 +60,10 @@ func main() {
log.Fatalf("building clientset: %v", err)
}
initHubble()
if hubbleInitErr != nil {
log.Printf("hubble disabled: %v", hubbleInitErr)
if err := startInformers(); err != nil {
log.Fatalf("informers: %v", err)
}
startHubble()
mux := http.NewServeMux()
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("ok")) })
@@ -111,7 +108,7 @@ func selectorSubset(sel, labels map[string]string) bool {
func buildGraph(ctx context.Context) (*Graph, error) {
g := &Graph{}
namespaces, err := clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
namespaces, err := listAll[corev1.Namespace](nsLister)
if err != nil {
return nil, err
}
@@ -133,7 +130,7 @@ func buildGraph(ctx context.Context) (*Graph, error) {
}
var workloads []workloadRef
deployments, err := clientset.AppsV1().Deployments(corev1.NamespaceAll).List(ctx, metav1.ListOptions{})
deployments, err := listAll[appsv1.Deployment](deployLister)
if err != nil {
return nil, err
}
@@ -143,7 +140,7 @@ func buildGraph(ctx context.Context) (*Graph, error) {
workloads = append(workloads, workloadRef{id, d.Spec.Template.Labels})
}
statefulsets, err := clientset.AppsV1().StatefulSets(corev1.NamespaceAll).List(ctx, metav1.ListOptions{})
statefulsets, err := listAll[appsv1.StatefulSet](stsLister)
if err != nil {
return nil, err
}
@@ -153,7 +150,7 @@ func buildGraph(ctx context.Context) (*Graph, error) {
workloads = append(workloads, workloadRef{id, s.Spec.Template.Labels})
}
daemonsets, err := clientset.AppsV1().DaemonSets(corev1.NamespaceAll).List(ctx, metav1.ListOptions{})
daemonsets, err := listAll[appsv1.DaemonSet](dsLister)
if err != nil {
return nil, err
}
@@ -163,7 +160,7 @@ func buildGraph(ctx context.Context) (*Graph, error) {
workloads = append(workloads, workloadRef{id, ds.Spec.Template.Labels})
}
cronjobs, err := clientset.BatchV1().CronJobs(corev1.NamespaceAll).List(ctx, metav1.ListOptions{})
cronjobs, err := listAll[batchv1.CronJob](cronLister)
if err != nil {
return nil, err
}
@@ -177,7 +174,7 @@ func buildGraph(ctx context.Context) (*Graph, error) {
// DaemonSet behind ingress-nginx, database operators, bare Jobs) would
// otherwise be invisible. They become workload nodes of their own, IDed
// the way Hubble names workloads, so traffic attaches to them too.
pods, err := clientset.CoreV1().Pods(corev1.NamespaceAll).List(ctx, metav1.ListOptions{})
pods, err := listAll[corev1.Pod](podLister)
if err != nil {
return nil, err
}
@@ -231,7 +228,7 @@ func buildGraph(ctx context.Context) (*Graph, error) {
workloads = append(workloads, workloadRef{id, f.labels})
}
services, err := clientset.CoreV1().Services(corev1.NamespaceAll).List(ctx, metav1.ListOptions{})
services, err := listAll[corev1.Service](svcLister)
if err != nil {
return nil, err
}
@@ -251,7 +248,7 @@ func buildGraph(ctx context.Context) (*Graph, error) {
}
}
ingresses, err := clientset.NetworkingV1().Ingresses(corev1.NamespaceAll).List(ctx, metav1.ListOptions{})
ingresses, err := listAll[networkingv1.Ingress](ingLister)
if err != nil {
return nil, err
}
@@ -271,7 +268,7 @@ func buildGraph(ctx context.Context) (*Graph, error) {
// workload to wherever its pods actually run right now -- this is the
// "what's deployed where" question specifically, not covered by the
// Service/Ingress graph above.
nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
nodes, err := listAll[corev1.Node](nodeLister)
if err != nil {
return nil, err
}
@@ -570,6 +567,9 @@ func handleResource(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
if a, ok := obj.(metav1.Object); ok {
a.SetManagedFields(nil)
}
out, err := yaml.Marshal(obj)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -584,7 +584,7 @@ func handleResource(w http.ResponseWriter, r *http.Request) {
// schedules-on edges -- so "which pods back this node" stays consistent
// with what the graph already shows for placement.
func podsForWorkload(ctx context.Context, ns, target string) ([]corev1.Pod, error) {
pods, err := clientset.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{})
pods, err := listAll[corev1.Pod](podLister.Pods(ns))
if err != nil {
return nil, err
}
@@ -657,5 +657,3 @@ func handleLogs(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
io.Copy(w, stream)
}
var _ = os.Stdout // keep the os import if logging grows; avoids an unused-import edit later