Filter infra traffic at the Hubble agents

Host-originated flows, remote-node, cilium health and DNS :53 were ~74%
of the non-reply events on a 12k-event sample and none of them map to a
workload card; streaming them all kept hubble-relay at ~400m CPU. They
are now blacklisted in the GetFlows request, so agents drop them before
the relay. The UI's "Служебный" toggle had nothing left to show and is
gone; the legend says what isn't collected.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Vitaliy Chefanov
2026-09-23 20:26:05 +03:00
co-authored by Claude Opus 5.5
parent 6ead0a0d6d
commit 41f9b47556
4 changed files with 23 additions and 11 deletions
+8 -1
View File
@@ -10,7 +10,8 @@ Topology map of the Project Chimera Deckhouse cluster.
→ workload by label selector); dashed animated arrows are real traffic → workload by label selector); dashed animated arrows are real traffic
from Hubble Relay over the chosen window (5 min / 15 min / 1 h), red when from Hubble Relay over the chosen window (5 min / 15 min / 1 h), red when
Cilium dropped some of it. Peers from other namespaces are drawn as Cilium dropped some of it. Peers from other namespaces are drawn as
dashed "ghost" cards. Traffic refreshes in place every 30 s without dashed "ghost" cards. Kubelet probes, DNS, cilium health and node-to-node
traffic are not collected (see Backend). Traffic refreshes in place every 30 s without
relayout; peers that would need new cards wait for "Обновить". relayout; peers that would need new cards wait for "Обновить".
- **Right**: details for the selected object — overview, links (config and - **Right**: details for the selected object — overview, links (config and
traffic, clickable), live YAML, pod logs. traffic, clickable), live YAML, pod logs.
@@ -37,6 +38,12 @@ produces several events. `new` = opened by TCP SYN inside the window,
only knows what it has seen since; `coveredSec` says how much of the only knows what it has seen since; `coveredSec` says how much of the
window that is. window that is.
Infra traffic is filtered at the agents (`infraFilters` in `flows.go`):
host-originated flows (kubelet probes, hostNetwork pods), remote-node,
cilium health and DNS :53. On a 12k-event sample they were ~74% of
non-reply events, none of them attach to a workload card, and streaming
them cost hubble-relay ~400m CPU.
| Endpoint | What | | Endpoint | What |
|---|---| |---|---|
| `/api/graph` | nodes + config edges + pod placement | | `/api/graph` | nodes + config edges + pod placement |
+14
View File
@@ -130,6 +130,19 @@ func (c *collector) setErr(err error) {
log.Printf("hubble: %v", err) log.Printf("hubble: %v", err)
} }
// Filtered at the agents, so the relay never serialises them. Measured on a
// 12k-event sample: host-originated traffic (kubelet probes, hostNetwork
// pods) 43% of non-reply events, DNS :53 19%, remote-node 11%, cilium
// health 1%. None of it attaches to a workload card, and the relay spent
// ~400m CPU streaming it all.
var infraFilters = []*flowpb.FlowFilter{
{SourceLabel: []string{"reserved:host"}},
{SourceLabel: []string{"reserved:remote-node"}},
{SourceLabel: []string{"reserved:health"}},
{DestinationLabel: []string{"reserved:health"}},
{DestinationPort: []string{"53"}},
}
var flowFields = []string{"time", "verdict", "IP", "l4", "source", "destination", "source_names", "destination_names", "node_name"} var flowFields = []string{"time", "verdict", "IP", "l4", "source", "destination", "source_names", "destination_names", "node_name"}
func (c *collector) run(obs observerpb.ObserverClient) { func (c *collector) run(obs observerpb.ObserverClient) {
@@ -145,6 +158,7 @@ func (c *collector) run(obs observerpb.ObserverClient) {
// tuples after a reconnect collapse in the per-minute sets // tuples after a reconnect collapse in the per-minute sets
Since: timestamppb.New(time.Now().Add(-time.Minute)), Since: timestamppb.New(time.Now().Add(-time.Minute)),
Whitelist: []*flowpb.FlowFilter{{Reply: []bool{false}}}, Whitelist: []*flowpb.FlowFilter{{Reply: []bool{false}}},
Blacklist: infraFilters,
} }
if useMask { if useMask {
req.FieldMask = &fieldmaskpb.FieldMask{Paths: flowFields} req.FieldMask = &fieldmaskpb.FieldMask{Paths: flowFields}
+1 -9
View File
@@ -39,7 +39,6 @@ const CARD = { w: 196, h: 48 };
const PILL = { w: 164, h: 36 }; const PILL = { w: 164, h: 36 };
const NSCARD = { w: 232, h: 56 }; const NSCARD = { w: 232, h: 56 };
const QUIET_AT = 30; // above this many traffic edges they stay faint until hovered const QUIET_AT = 30; // above this many traffic edges they stay faint until hovered
const INFRA_PREFIXES = ['ext/host/', 'ext/health', 'ext/remote-node'];
const $ = (sel) => document.querySelector(sel); const $ = (sel) => document.querySelector(sel);
const el = (tag, props = {}, ...children) => { const el = (tag, props = {}, ...children) => {
@@ -187,7 +186,6 @@ const state = {
filter: '', filter: '',
traffic: true, traffic: true,
trafficWindow: 300, trafficWindow: 300,
infra: false,
flows: null, flows: null,
flowsError: null, flowsError: null,
}; };
@@ -521,10 +519,6 @@ function scopeNamespaces() {
return []; return [];
} }
function isInfraEdge(e) {
if (INFRA_PREFIXES.some((p) => e.source.startsWith(p) || e.target.startsWith(p))) return true;
return e.ports.every((p) => p === '53/UDP' || p === '53/TCP');
}
// Merge per-port flow edges into one edge per pair, for drawing. // Merge per-port flow edges into one edge per pair, for drawing.
function aggregateFlows(flows) { function aggregateFlows(flows) {
@@ -569,7 +563,6 @@ function buildElements() {
state.trafficEdges = []; state.trafficEdges = [];
if (state.traffic && state.flows) { if (state.traffic && state.flows) {
for (const e of aggregateFlows(state.flows)) { for (const e of aggregateFlows(state.flows)) {
if (!state.infra && isInfraEdge(e)) continue;
const touches = memberIds.has(e.source) || memberIds.has(e.target); const touches = memberIds.has(e.source) || memberIds.has(e.target);
if (!touches) continue; if (!touches) continue;
for (const id of [e.source, e.target]) { for (const id of [e.source, e.target]) {
@@ -622,7 +615,6 @@ async function refreshTraffic() {
const next = new Map(); const next = new Map();
hiddenPeers = 0; hiddenPeers = 0;
for (const e of aggregateFlows(flows)) { for (const e of aggregateFlows(flows)) {
if (!state.infra && isInfraEdge(e)) continue;
if (!members.has(e.source) && !members.has(e.target)) continue; if (!members.has(e.source) && !members.has(e.target)) continue;
if (cy.getElementById(e.source).empty() || cy.getElementById(e.target).empty()) { hiddenPeers++; continue; } if (cy.getElementById(e.source).empty() || cy.getElementById(e.target).empty()) { hiddenPeers++; continue; }
next.set('tr:' + e.source + '→' + e.target, e); next.set('tr:' + e.source + '→' + e.target, e);
@@ -940,6 +932,7 @@ function renderLegend() {
el('span', {}, el('span', { class: 'line traffic' }), 'реальный трафик (Hubble)'), el('span', {}, el('span', { class: 'line traffic' }), 'реальный трафик (Hubble)'),
el('span', {}, el('span', { class: 'line drop' }), 'есть drop'), el('span', {}, el('span', { class: 'line drop' }), 'есть drop'),
el('span', {}, 'пунктирная карточка — из другого namespace'), el('span', {}, 'пунктирная карточка — из другого namespace'),
el('span', {}, 'не собирается: пробы kubelet и трафик с хостов нод, DNS, cilium health'),
); );
} }
@@ -1126,7 +1119,6 @@ $('#logs-reload').addEventListener('click', loadLogs);
/* ---------- top bar ---------- */ /* ---------- top bar ---------- */
$('#traffic-on').addEventListener('change', (e) => { state.traffic = e.target.checked; renderMap(); }); $('#traffic-on').addEventListener('change', (e) => { state.traffic = e.target.checked; renderMap(); });
$('#traffic-window').addEventListener('change', (e) => { state.trafficWindow = Number(e.target.value); if (state.traffic) renderMap(); }); $('#traffic-window').addEventListener('change', (e) => { state.trafficWindow = Number(e.target.value); if (state.traffic) renderMap(); });
$('#infra-traffic').addEventListener('change', (e) => { state.infra = e.target.checked; renderMap(); });
$('#reload').addEventListener('click', () => load()); $('#reload').addEventListener('click', () => load());
matchMedia('(prefers-color-scheme: dark)').addEventListener('change', () => { matchMedia('(prefers-color-scheme: dark)').addEventListener('change', () => {
-1
View File
@@ -20,7 +20,6 @@
<option value="900">15 мин</option> <option value="900">15 мин</option>
<option value="3600">1 час</option> <option value="3600">1 час</option>
</select> </select>
<label class="toggle" title="Проверки kubelet, cilium health, трафик между нодами"><input type="checkbox" id="infra-traffic"> Служебный</label>
<button type="button" id="reload" class="btn">Обновить</button> <button type="button" id="reload" class="btn">Обновить</button>
<span class="muted small" id="updated"></span> <span class="muted small" id="updated"></span>
</div> </div>