Filter infra traffic at the Hubble agents
Host-originated flows, remote-node, cilium health and DNS :53 were ~74% of the non-reply events on a 12k-event sample and none of them map to a workload card; streaming them all kept hubble-relay at ~400m CPU. They are now blacklisted in the GetFlows request, so agents drop them before the relay. The UI's "Служебный" toggle had nothing left to show and is gone; the legend says what isn't collected. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
6ead0a0d6d
commit
41f9b47556
@@ -10,7 +10,8 @@ Topology map of the Project Chimera Deckhouse cluster.
|
||||
→ workload by label selector); dashed animated arrows are real traffic
|
||||
from Hubble Relay over the chosen window (5 min / 15 min / 1 h), red when
|
||||
Cilium dropped some of it. Peers from other namespaces are drawn as
|
||||
dashed "ghost" cards. Traffic refreshes in place every 30 s without
|
||||
dashed "ghost" cards. Kubelet probes, DNS, cilium health and node-to-node
|
||||
traffic are not collected (see Backend). Traffic refreshes in place every 30 s without
|
||||
relayout; peers that would need new cards wait for "Обновить".
|
||||
- **Right**: details for the selected object — overview, links (config and
|
||||
traffic, clickable), live YAML, pod logs.
|
||||
@@ -37,6 +38,12 @@ produces several events. `new` = opened by TCP SYN inside the window,
|
||||
only knows what it has seen since; `coveredSec` says how much of the
|
||||
window that is.
|
||||
|
||||
Infra traffic is filtered at the agents (`infraFilters` in `flows.go`):
|
||||
host-originated flows (kubelet probes, hostNetwork pods), remote-node,
|
||||
cilium health and DNS :53. On a 12k-event sample they were ~74% of
|
||||
non-reply events, none of them attach to a workload card, and streaming
|
||||
them cost hubble-relay ~400m CPU.
|
||||
|
||||
| Endpoint | What |
|
||||
|---|---|
|
||||
| `/api/graph` | nodes + config edges + pod placement |
|
||||
|
||||
@@ -130,6 +130,19 @@ func (c *collector) setErr(err error) {
|
||||
log.Printf("hubble: %v", err)
|
||||
}
|
||||
|
||||
// Filtered at the agents, so the relay never serialises them. Measured on a
|
||||
// 12k-event sample: host-originated traffic (kubelet probes, hostNetwork
|
||||
// pods) 43% of non-reply events, DNS :53 19%, remote-node 11%, cilium
|
||||
// health 1%. None of it attaches to a workload card, and the relay spent
|
||||
// ~400m CPU streaming it all.
|
||||
var infraFilters = []*flowpb.FlowFilter{
|
||||
{SourceLabel: []string{"reserved:host"}},
|
||||
{SourceLabel: []string{"reserved:remote-node"}},
|
||||
{SourceLabel: []string{"reserved:health"}},
|
||||
{DestinationLabel: []string{"reserved:health"}},
|
||||
{DestinationPort: []string{"53"}},
|
||||
}
|
||||
|
||||
var flowFields = []string{"time", "verdict", "IP", "l4", "source", "destination", "source_names", "destination_names", "node_name"}
|
||||
|
||||
func (c *collector) run(obs observerpb.ObserverClient) {
|
||||
@@ -145,6 +158,7 @@ func (c *collector) run(obs observerpb.ObserverClient) {
|
||||
// tuples after a reconnect collapse in the per-minute sets
|
||||
Since: timestamppb.New(time.Now().Add(-time.Minute)),
|
||||
Whitelist: []*flowpb.FlowFilter{{Reply: []bool{false}}},
|
||||
Blacklist: infraFilters,
|
||||
}
|
||||
if useMask {
|
||||
req.FieldMask = &fieldmaskpb.FieldMask{Paths: flowFields}
|
||||
|
||||
+1
-9
@@ -39,7 +39,6 @@ const CARD = { w: 196, h: 48 };
|
||||
const PILL = { w: 164, h: 36 };
|
||||
const NSCARD = { w: 232, h: 56 };
|
||||
const QUIET_AT = 30; // above this many traffic edges they stay faint until hovered
|
||||
const INFRA_PREFIXES = ['ext/host/', 'ext/health', 'ext/remote-node'];
|
||||
|
||||
const $ = (sel) => document.querySelector(sel);
|
||||
const el = (tag, props = {}, ...children) => {
|
||||
@@ -187,7 +186,6 @@ const state = {
|
||||
filter: '',
|
||||
traffic: true,
|
||||
trafficWindow: 300,
|
||||
infra: false,
|
||||
flows: null,
|
||||
flowsError: null,
|
||||
};
|
||||
@@ -521,10 +519,6 @@ function scopeNamespaces() {
|
||||
return [];
|
||||
}
|
||||
|
||||
function isInfraEdge(e) {
|
||||
if (INFRA_PREFIXES.some((p) => e.source.startsWith(p) || e.target.startsWith(p))) return true;
|
||||
return e.ports.every((p) => p === '53/UDP' || p === '53/TCP');
|
||||
}
|
||||
|
||||
// Merge per-port flow edges into one edge per pair, for drawing.
|
||||
function aggregateFlows(flows) {
|
||||
@@ -569,7 +563,6 @@ function buildElements() {
|
||||
state.trafficEdges = [];
|
||||
if (state.traffic && state.flows) {
|
||||
for (const e of aggregateFlows(state.flows)) {
|
||||
if (!state.infra && isInfraEdge(e)) continue;
|
||||
const touches = memberIds.has(e.source) || memberIds.has(e.target);
|
||||
if (!touches) continue;
|
||||
for (const id of [e.source, e.target]) {
|
||||
@@ -622,7 +615,6 @@ async function refreshTraffic() {
|
||||
const next = new Map();
|
||||
hiddenPeers = 0;
|
||||
for (const e of aggregateFlows(flows)) {
|
||||
if (!state.infra && isInfraEdge(e)) continue;
|
||||
if (!members.has(e.source) && !members.has(e.target)) continue;
|
||||
if (cy.getElementById(e.source).empty() || cy.getElementById(e.target).empty()) { hiddenPeers++; continue; }
|
||||
next.set('tr:' + e.source + '→' + e.target, e);
|
||||
@@ -940,6 +932,7 @@ function renderLegend() {
|
||||
el('span', {}, el('span', { class: 'line traffic' }), 'реальный трафик (Hubble)'),
|
||||
el('span', {}, el('span', { class: 'line drop' }), 'есть drop'),
|
||||
el('span', {}, 'пунктирная карточка — из другого namespace'),
|
||||
el('span', {}, 'не собирается: пробы kubelet и трафик с хостов нод, DNS, cilium health'),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1126,7 +1119,6 @@ $('#logs-reload').addEventListener('click', loadLogs);
|
||||
/* ---------- top bar ---------- */
|
||||
$('#traffic-on').addEventListener('change', (e) => { state.traffic = e.target.checked; renderMap(); });
|
||||
$('#traffic-window').addEventListener('change', (e) => { state.trafficWindow = Number(e.target.value); if (state.traffic) renderMap(); });
|
||||
$('#infra-traffic').addEventListener('change', (e) => { state.infra = e.target.checked; renderMap(); });
|
||||
$('#reload').addEventListener('click', () => load());
|
||||
|
||||
matchMedia('(prefers-color-scheme: dark)').addEventListener('change', () => {
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
<option value="900">15 мин</option>
|
||||
<option value="3600">1 час</option>
|
||||
</select>
|
||||
<label class="toggle" title="Проверки kubelet, cilium health, трафик между нодами"><input type="checkbox" id="infra-traffic"> Служебный</label>
|
||||
<button type="button" id="reload" class="btn">Обновить</button>
|
||||
<span class="muted small" id="updated"></span>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user