Filter infra traffic at the Hubble agents

Host-originated flows, remote-node, cilium health and DNS :53 were ~74%
of the non-reply events on a 12k-event sample and none of them map to a
workload card; streaming them all kept hubble-relay at ~400m CPU. They
are now blacklisted in the GetFlows request, so agents drop them before
the relay. The UI's "Служебный" toggle had nothing left to show and is
gone; the legend says what isn't collected.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Vitaliy Chefanov
2026-09-23 20:26:05 +03:00
co-authored by Claude Opus 5.5
parent 6ead0a0d6d
commit 41f9b47556
4 changed files with 23 additions and 11 deletions
+8 -1
View File
@@ -10,7 +10,8 @@ Topology map of the Project Chimera Deckhouse cluster.
→ workload by label selector); dashed animated arrows are real traffic
from Hubble Relay over the chosen window (5 min / 15 min / 1 h), red when
Cilium dropped some of it. Peers from other namespaces are drawn as
dashed "ghost" cards. Traffic refreshes in place every 30 s without
dashed "ghost" cards. Kubelet probes, DNS, cilium health and node-to-node
traffic are not collected (see Backend). Traffic refreshes in place every 30 s without
relayout; peers that would need new cards wait for "Обновить".
- **Right**: details for the selected object — overview, links (config and
traffic, clickable), live YAML, pod logs.
@@ -37,6 +38,12 @@ produces several events. `new` = opened by TCP SYN inside the window,
only knows what it has seen since; `coveredSec` says how much of the
window that is.
Infra traffic is filtered at the agents (`infraFilters` in `flows.go`):
host-originated flows (kubelet probes, hostNetwork pods), remote-node,
cilium health and DNS :53. On a 12k-event sample they were ~74% of
non-reply events, none of them attach to a workload card, and streaming
them cost hubble-relay ~400m CPU.
| Endpoint | What |
|---|---|
| `/api/graph` | nodes + config edges + pod placement |
+14
View File
@@ -130,6 +130,19 @@ func (c *collector) setErr(err error) {
log.Printf("hubble: %v", err)
}
// Filtered at the agents, so the relay never serialises them. Measured on a
// 12k-event sample: host-originated traffic (kubelet probes, hostNetwork
// pods) 43% of non-reply events, DNS :53 19%, remote-node 11%, cilium
// health 1%. None of it attaches to a workload card, and the relay spent
// ~400m CPU streaming it all.
var infraFilters = []*flowpb.FlowFilter{
{SourceLabel: []string{"reserved:host"}},
{SourceLabel: []string{"reserved:remote-node"}},
{SourceLabel: []string{"reserved:health"}},
{DestinationLabel: []string{"reserved:health"}},
{DestinationPort: []string{"53"}},
}
var flowFields = []string{"time", "verdict", "IP", "l4", "source", "destination", "source_names", "destination_names", "node_name"}
func (c *collector) run(obs observerpb.ObserverClient) {
@@ -145,6 +158,7 @@ func (c *collector) run(obs observerpb.ObserverClient) {
// tuples after a reconnect collapse in the per-minute sets
Since: timestamppb.New(time.Now().Add(-time.Minute)),
Whitelist: []*flowpb.FlowFilter{{Reply: []bool{false}}},
Blacklist: infraFilters,
}
if useMask {
req.FieldMask = &fieldmaskpb.FieldMask{Paths: flowFields}
+1 -9
View File
@@ -39,7 +39,6 @@ const CARD = { w: 196, h: 48 };
const PILL = { w: 164, h: 36 };
const NSCARD = { w: 232, h: 56 };
const QUIET_AT = 30; // above this many traffic edges they stay faint until hovered
const INFRA_PREFIXES = ['ext/host/', 'ext/health', 'ext/remote-node'];
const $ = (sel) => document.querySelector(sel);
const el = (tag, props = {}, ...children) => {
@@ -187,7 +186,6 @@ const state = {
filter: '',
traffic: true,
trafficWindow: 300,
infra: false,
flows: null,
flowsError: null,
};
@@ -521,10 +519,6 @@ function scopeNamespaces() {
return [];
}
function isInfraEdge(e) {
if (INFRA_PREFIXES.some((p) => e.source.startsWith(p) || e.target.startsWith(p))) return true;
return e.ports.every((p) => p === '53/UDP' || p === '53/TCP');
}
// Merge per-port flow edges into one edge per pair, for drawing.
function aggregateFlows(flows) {
@@ -569,7 +563,6 @@ function buildElements() {
state.trafficEdges = [];
if (state.traffic && state.flows) {
for (const e of aggregateFlows(state.flows)) {
if (!state.infra && isInfraEdge(e)) continue;
const touches = memberIds.has(e.source) || memberIds.has(e.target);
if (!touches) continue;
for (const id of [e.source, e.target]) {
@@ -622,7 +615,6 @@ async function refreshTraffic() {
const next = new Map();
hiddenPeers = 0;
for (const e of aggregateFlows(flows)) {
if (!state.infra && isInfraEdge(e)) continue;
if (!members.has(e.source) && !members.has(e.target)) continue;
if (cy.getElementById(e.source).empty() || cy.getElementById(e.target).empty()) { hiddenPeers++; continue; }
next.set('tr:' + e.source + '→' + e.target, e);
@@ -940,6 +932,7 @@ function renderLegend() {
el('span', {}, el('span', { class: 'line traffic' }), 'реальный трафик (Hubble)'),
el('span', {}, el('span', { class: 'line drop' }), 'есть drop'),
el('span', {}, 'пунктирная карточка — из другого namespace'),
el('span', {}, 'не собирается: пробы kubelet и трафик с хостов нод, DNS, cilium health'),
);
}
@@ -1126,7 +1119,6 @@ $('#logs-reload').addEventListener('click', loadLogs);
/* ---------- top bar ---------- */
$('#traffic-on').addEventListener('change', (e) => { state.traffic = e.target.checked; renderMap(); });
$('#traffic-window').addEventListener('change', (e) => { state.trafficWindow = Number(e.target.value); if (state.traffic) renderMap(); });
$('#infra-traffic').addEventListener('change', (e) => { state.infra = e.target.checked; renderMap(); });
$('#reload').addEventListener('click', () => load());
matchMedia('(prefers-color-scheme: dark)').addEventListener('change', () => {
-1
View File
@@ -20,7 +20,6 @@
<option value="900">15 мин</option>
<option value="3600">1 час</option>
</select>
<label class="toggle" title="Проверки kubelet, cilium health, трафик между нодами"><input type="checkbox" id="infra-traffic"> Служебный</label>
<button type="button" id="reload" class="btn">Обновить</button>
<span class="muted small" id="updated"></span>
</div>