Filter infra traffic at the Hubble agents
Host-originated flows, remote-node, cilium health and DNS :53 were ~74% of the non-reply events on a 12k-event sample and none of them map to a workload card; streaming them all kept hubble-relay at ~400m CPU. They are now blacklisted in the GetFlows request, so agents drop them before the relay. The UI's "Служебный" toggle had nothing left to show and is gone; the legend says what isn't collected. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
6ead0a0d6d
commit
41f9b47556
@@ -10,7 +10,8 @@ Topology map of the Project Chimera Deckhouse cluster.
|
|||||||
→ workload by label selector); dashed animated arrows are real traffic
|
→ workload by label selector); dashed animated arrows are real traffic
|
||||||
from Hubble Relay over the chosen window (5 min / 15 min / 1 h), red when
|
from Hubble Relay over the chosen window (5 min / 15 min / 1 h), red when
|
||||||
Cilium dropped some of it. Peers from other namespaces are drawn as
|
Cilium dropped some of it. Peers from other namespaces are drawn as
|
||||||
dashed "ghost" cards. Traffic refreshes in place every 30 s without
|
dashed "ghost" cards. Kubelet probes, DNS, cilium health and node-to-node
|
||||||
|
traffic are not collected (see Backend). Traffic refreshes in place every 30 s without
|
||||||
relayout; peers that would need new cards wait for "Обновить".
|
relayout; peers that would need new cards wait for "Обновить".
|
||||||
- **Right**: details for the selected object — overview, links (config and
|
- **Right**: details for the selected object — overview, links (config and
|
||||||
traffic, clickable), live YAML, pod logs.
|
traffic, clickable), live YAML, pod logs.
|
||||||
@@ -37,6 +38,12 @@ produces several events. `new` = opened by TCP SYN inside the window,
|
|||||||
only knows what it has seen since; `coveredSec` says how much of the
|
only knows what it has seen since; `coveredSec` says how much of the
|
||||||
window that is.
|
window that is.
|
||||||
|
|
||||||
|
Infra traffic is filtered at the agents (`infraFilters` in `flows.go`):
|
||||||
|
host-originated flows (kubelet probes, hostNetwork pods), remote-node,
|
||||||
|
cilium health and DNS :53. On a 12k-event sample they were ~74% of
|
||||||
|
non-reply events, none of them attach to a workload card, and streaming
|
||||||
|
them cost hubble-relay ~400m CPU.
|
||||||
|
|
||||||
| Endpoint | What |
|
| Endpoint | What |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `/api/graph` | nodes + config edges + pod placement |
|
| `/api/graph` | nodes + config edges + pod placement |
|
||||||
|
|||||||
@@ -130,6 +130,19 @@ func (c *collector) setErr(err error) {
|
|||||||
log.Printf("hubble: %v", err)
|
log.Printf("hubble: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Filtered at the agents, so the relay never serialises them. Measured on a
|
||||||
|
// 12k-event sample: host-originated traffic (kubelet probes, hostNetwork
|
||||||
|
// pods) 43% of non-reply events, DNS :53 19%, remote-node 11%, cilium
|
||||||
|
// health 1%. None of it attaches to a workload card, and the relay spent
|
||||||
|
// ~400m CPU streaming it all.
|
||||||
|
var infraFilters = []*flowpb.FlowFilter{
|
||||||
|
{SourceLabel: []string{"reserved:host"}},
|
||||||
|
{SourceLabel: []string{"reserved:remote-node"}},
|
||||||
|
{SourceLabel: []string{"reserved:health"}},
|
||||||
|
{DestinationLabel: []string{"reserved:health"}},
|
||||||
|
{DestinationPort: []string{"53"}},
|
||||||
|
}
|
||||||
|
|
||||||
var flowFields = []string{"time", "verdict", "IP", "l4", "source", "destination", "source_names", "destination_names", "node_name"}
|
var flowFields = []string{"time", "verdict", "IP", "l4", "source", "destination", "source_names", "destination_names", "node_name"}
|
||||||
|
|
||||||
func (c *collector) run(obs observerpb.ObserverClient) {
|
func (c *collector) run(obs observerpb.ObserverClient) {
|
||||||
@@ -145,6 +158,7 @@ func (c *collector) run(obs observerpb.ObserverClient) {
|
|||||||
// tuples after a reconnect collapse in the per-minute sets
|
// tuples after a reconnect collapse in the per-minute sets
|
||||||
Since: timestamppb.New(time.Now().Add(-time.Minute)),
|
Since: timestamppb.New(time.Now().Add(-time.Minute)),
|
||||||
Whitelist: []*flowpb.FlowFilter{{Reply: []bool{false}}},
|
Whitelist: []*flowpb.FlowFilter{{Reply: []bool{false}}},
|
||||||
|
Blacklist: infraFilters,
|
||||||
}
|
}
|
||||||
if useMask {
|
if useMask {
|
||||||
req.FieldMask = &fieldmaskpb.FieldMask{Paths: flowFields}
|
req.FieldMask = &fieldmaskpb.FieldMask{Paths: flowFields}
|
||||||
|
|||||||
+1
-9
@@ -39,7 +39,6 @@ const CARD = { w: 196, h: 48 };
|
|||||||
const PILL = { w: 164, h: 36 };
|
const PILL = { w: 164, h: 36 };
|
||||||
const NSCARD = { w: 232, h: 56 };
|
const NSCARD = { w: 232, h: 56 };
|
||||||
const QUIET_AT = 30; // above this many traffic edges they stay faint until hovered
|
const QUIET_AT = 30; // above this many traffic edges they stay faint until hovered
|
||||||
const INFRA_PREFIXES = ['ext/host/', 'ext/health', 'ext/remote-node'];
|
|
||||||
|
|
||||||
const $ = (sel) => document.querySelector(sel);
|
const $ = (sel) => document.querySelector(sel);
|
||||||
const el = (tag, props = {}, ...children) => {
|
const el = (tag, props = {}, ...children) => {
|
||||||
@@ -187,7 +186,6 @@ const state = {
|
|||||||
filter: '',
|
filter: '',
|
||||||
traffic: true,
|
traffic: true,
|
||||||
trafficWindow: 300,
|
trafficWindow: 300,
|
||||||
infra: false,
|
|
||||||
flows: null,
|
flows: null,
|
||||||
flowsError: null,
|
flowsError: null,
|
||||||
};
|
};
|
||||||
@@ -521,10 +519,6 @@ function scopeNamespaces() {
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
function isInfraEdge(e) {
|
|
||||||
if (INFRA_PREFIXES.some((p) => e.source.startsWith(p) || e.target.startsWith(p))) return true;
|
|
||||||
return e.ports.every((p) => p === '53/UDP' || p === '53/TCP');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Merge per-port flow edges into one edge per pair, for drawing.
|
// Merge per-port flow edges into one edge per pair, for drawing.
|
||||||
function aggregateFlows(flows) {
|
function aggregateFlows(flows) {
|
||||||
@@ -569,7 +563,6 @@ function buildElements() {
|
|||||||
state.trafficEdges = [];
|
state.trafficEdges = [];
|
||||||
if (state.traffic && state.flows) {
|
if (state.traffic && state.flows) {
|
||||||
for (const e of aggregateFlows(state.flows)) {
|
for (const e of aggregateFlows(state.flows)) {
|
||||||
if (!state.infra && isInfraEdge(e)) continue;
|
|
||||||
const touches = memberIds.has(e.source) || memberIds.has(e.target);
|
const touches = memberIds.has(e.source) || memberIds.has(e.target);
|
||||||
if (!touches) continue;
|
if (!touches) continue;
|
||||||
for (const id of [e.source, e.target]) {
|
for (const id of [e.source, e.target]) {
|
||||||
@@ -622,7 +615,6 @@ async function refreshTraffic() {
|
|||||||
const next = new Map();
|
const next = new Map();
|
||||||
hiddenPeers = 0;
|
hiddenPeers = 0;
|
||||||
for (const e of aggregateFlows(flows)) {
|
for (const e of aggregateFlows(flows)) {
|
||||||
if (!state.infra && isInfraEdge(e)) continue;
|
|
||||||
if (!members.has(e.source) && !members.has(e.target)) continue;
|
if (!members.has(e.source) && !members.has(e.target)) continue;
|
||||||
if (cy.getElementById(e.source).empty() || cy.getElementById(e.target).empty()) { hiddenPeers++; continue; }
|
if (cy.getElementById(e.source).empty() || cy.getElementById(e.target).empty()) { hiddenPeers++; continue; }
|
||||||
next.set('tr:' + e.source + '→' + e.target, e);
|
next.set('tr:' + e.source + '→' + e.target, e);
|
||||||
@@ -940,6 +932,7 @@ function renderLegend() {
|
|||||||
el('span', {}, el('span', { class: 'line traffic' }), 'реальный трафик (Hubble)'),
|
el('span', {}, el('span', { class: 'line traffic' }), 'реальный трафик (Hubble)'),
|
||||||
el('span', {}, el('span', { class: 'line drop' }), 'есть drop'),
|
el('span', {}, el('span', { class: 'line drop' }), 'есть drop'),
|
||||||
el('span', {}, 'пунктирная карточка — из другого namespace'),
|
el('span', {}, 'пунктирная карточка — из другого namespace'),
|
||||||
|
el('span', {}, 'не собирается: пробы kubelet и трафик с хостов нод, DNS, cilium health'),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1126,7 +1119,6 @@ $('#logs-reload').addEventListener('click', loadLogs);
|
|||||||
/* ---------- top bar ---------- */
|
/* ---------- top bar ---------- */
|
||||||
$('#traffic-on').addEventListener('change', (e) => { state.traffic = e.target.checked; renderMap(); });
|
$('#traffic-on').addEventListener('change', (e) => { state.traffic = e.target.checked; renderMap(); });
|
||||||
$('#traffic-window').addEventListener('change', (e) => { state.trafficWindow = Number(e.target.value); if (state.traffic) renderMap(); });
|
$('#traffic-window').addEventListener('change', (e) => { state.trafficWindow = Number(e.target.value); if (state.traffic) renderMap(); });
|
||||||
$('#infra-traffic').addEventListener('change', (e) => { state.infra = e.target.checked; renderMap(); });
|
|
||||||
$('#reload').addEventListener('click', () => load());
|
$('#reload').addEventListener('click', () => load());
|
||||||
|
|
||||||
matchMedia('(prefers-color-scheme: dark)').addEventListener('change', () => {
|
matchMedia('(prefers-color-scheme: dark)').addEventListener('change', () => {
|
||||||
|
|||||||
@@ -20,7 +20,6 @@
|
|||||||
<option value="900">15 мин</option>
|
<option value="900">15 мин</option>
|
||||||
<option value="3600">1 час</option>
|
<option value="3600">1 час</option>
|
||||||
</select>
|
</select>
|
||||||
<label class="toggle" title="Проверки kubelet, cilium health, трафик между нодами"><input type="checkbox" id="infra-traffic"> Служебный</label>
|
|
||||||
<button type="button" id="reload" class="btn">Обновить</button>
|
<button type="button" id="reload" class="btn">Обновить</button>
|
||||||
<span class="muted small" id="updated"></span>
|
<span class="muted small" id="updated"></span>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
Reference in New Issue
Block a user