From a5b36678ed87a919dcbc2f8fa2125f2d5c5c51c7 Mon Sep 17 00:00:00 2001 From: Michal Date: Thu, 13 Aug 2026 00:56:06 +0100 Subject: [PATCH] feat(labsim): live topology view with per-path latency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Grafana heatmap of 1s and 0s said almost nothing, and the state timeline was an unreadable pile of overlapping series labels. Replaced as the primary view with a purpose-built page served by the exporter itself. - Probe now captures ICMP RTT, exposed as labsim_rtt_ms{src,dst}. A path that is up but slow is a different problem from one that is down, and a pass/fail grid cannot show it. - Exporter serves / (topology), /api/matrix (JSON) and /metrics. - topology.html: node per VLAN in a ring, VyOS router in the centre because every inter-VLAN packet really does traverse it, one line per pair coloured green/red with the RTT on it. Hovering gives per-direction state. A node ring goes red if anything to or from it is blocked. Side panels list blocked paths and the slowest links. Refreshes every 5s, no dependencies. Grafana stays for what it is actually good at — history of when a path flipped. Label placement is deliberate: RTT captions sit ~32% along each edge with a perpendicular nudge, because every diagonal of a 6-node mesh crosses the centre and midpoint labels stack on the router node. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DMVzWZgiKW2wquf5z8S1yH --- labsim/README.md | 16 ++++ labsim/labsim-exporter.py | 59 +++++++++++-- labsim/labsim-matrix.py | 19 ++-- labsim/monitoring-up.sh | 3 +- labsim/topology.html | 181 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 264 insertions(+), 14 deletions(-) create mode 100644 labsim/topology.html diff --git a/labsim/README.md b/labsim/README.md index 12ca514..615c93d 100644 --- a/labsim/README.md +++ b/labsim/README.md @@ -62,6 +62,22 @@ Console, when the network is the thing that is broken: sudo virsh console labsim-2-k8s # root / labsim ``` +## Watching it + +```bash +./labsim-matrix.py --watch 2 # terminal grid, changed cells highlighted +./monitoring-up.sh # topology page + Prometheus + Grafana +``` + +- **http://localhost:9101/** — live mesh: a node per VLAN, the router in the + middle, one line per pair coloured green/red with the ICMP RTT on it. Hover a + line for per-direction detail. Refreshes every 5s. This is the one to watch + while changing firewall rules. +- **http://localhost:3000/d/labsim-matrix** — Grafana (anonymous, no login) for + *history*: when did a path flip, and how has latency moved. +- **http://localhost:9101/metrics** — `labsim_reachable{src,dst,proto}` and + `labsim_rtt_ms{src,dst}`. + ## Notes for whoever extends this Things that cost time the first time round, all verified on this image: diff --git a/labsim/labsim-exporter.py b/labsim/labsim-exporter.py index c9accd8..f82ab29 100755 --- a/labsim/labsim-exporter.py +++ b/labsim/labsim-exporter.py @@ -19,6 +19,8 @@ from __future__ import annotations import argparse import http.server +import json +import os import threading import time @@ -48,6 +50,24 @@ class Collector: pass time.sleep(max(1.0, self.interval - (time.time() - started))) + def snapshot(self) -> dict: + """Everything the topology page needs, in one JSON payload.""" + with self.lock: + results, duration = dict(self.results), self.duration + reach = total = 0 + for data in results.values(): + if "__error__" in data: + continue + for protos in data.values(): + for proto, ok in protos.items(): + if proto == "rtt_ms": + continue + total += 1 + if ok: + reach += 1 + return {"vlans": self.vlans, "results": results, "reachable": reach, + "total": total, "sweep_seconds": duration} + def render(self) -> str: with self.lock: results, duration, sweeps = dict(self.results), self.duration, self.sweeps @@ -56,13 +76,22 @@ class Collector: "# HELP labsim_reachable 1 if dst is reachable from src over proto", "# TYPE labsim_reachable gauge", ] + rtts = [] for src, data in results.items(): if "__error__" in data: continue for dst, protos in data.items(): for proto, ok in protos.items(): + if proto == "rtt_ms": + if isinstance(ok, (int, float)): + rtts.append((src, dst, ok)) + continue out.append( f'labsim_reachable{{src="{src}",dst="{dst}",proto="{proto}"}} {1 if ok else 0}') + out += ["# HELP labsim_rtt_ms ICMP round-trip time", + "# TYPE labsim_rtt_ms gauge"] + for src, dst, val in rtts: + out.append(f'labsim_rtt_ms{{src="{src}",dst="{dst}"}} {val}') out += [ "# HELP labsim_sweep_seconds duration of the last sweep", "# TYPE labsim_sweep_seconds gauge", @@ -87,23 +116,39 @@ def main() -> int: collector = Collector(args.interval, args.timeout) threading.Thread(target=collector.loop, daemon=True).start() + here = os.path.dirname(os.path.abspath(__file__)) + class Handler(http.server.BaseHTTPRequestHandler): - def do_GET(self) -> None: # noqa: N802 - stdlib API - if self.path.rstrip("/") not in ("", "/metrics"): - self.send_error(404) - return - body = collector.render().encode() + def _send(self, body: bytes, ctype: str) -> None: self.send_response(200) - self.send_header("Content-Type", "text/plain; version=0.0.4") + self.send_header("Content-Type", ctype) self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") self.end_headers() self.wfile.write(body) + def do_GET(self) -> None: # noqa: N802 - stdlib API + path = self.path.split("?")[0].rstrip("/") + if path in ("", "/topology"): + # Live topology view — the thing you actually watch. + try: + with open(os.path.join(here, "topology.html"), "rb") as fh: + self._send(fh.read(), "text/html; charset=utf-8") + except OSError: + self.send_error(500, "topology.html missing") + elif path == "/api/matrix": + self._send(json.dumps(collector.snapshot()).encode(), "application/json") + elif path == "/metrics": + self._send(collector.render().encode(), "text/plain; version=0.0.4") + else: + self.send_error(404) + def log_message(self, *_args) -> None: # keep the console quiet return srv = http.server.ThreadingHTTPServer(("0.0.0.0", args.port), Handler) - print(f"labsim exporter on :{args.port}/metrics (sweep every {args.interval}s)") + print(f"labsim topology http://localhost:{args.port}/") + print(f"labsim metrics http://localhost:{args.port}/metrics (sweep every {args.interval}s)") srv.serve_forever() return 0 diff --git a/labsim/labsim-matrix.py b/labsim/labsim-matrix.py index 86632f1..425624f 100755 --- a/labsim/labsim-matrix.py +++ b/labsim/labsim-matrix.py @@ -37,18 +37,25 @@ PROTOS = ("icmp", "tcp22", "tcp80") # Runs ON the guest. Keep it stdlib-only and quick — a hung probe delays the # whole sweep, so every check is hard-bounded by a timeout. PROBE = r''' -import json, socket, subprocess, sys +import json, re, socket, subprocess, sys targets = json.load(sys.stdin) out = {} for name, ip in targets.items(): res = {} try: - res["icmp"] = subprocess.run( - ["ping", "-c", "1", "-W", "1", ip], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=4 - ).returncode == 0 + p = subprocess.run(["ping", "-c", "1", "-W", "1", ip], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=4) + res["icmp"] = p.returncode == 0 + # RTT as well as pass/fail: a path that is up but slow is a different + # problem from one that is down, and the grid alone cannot show it. + res["rtt_ms"] = None + if res["icmp"]: + m = re.search(r"time[=<]\s*([0-9.]+)\s*ms", p.stdout.decode("utf-8", "replace")) + if m: + res["rtt_ms"] = float(m.group(1)) except Exception: res["icmp"] = False + res["rtt_ms"] = None for port in (22, 80): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(1.5) @@ -147,7 +154,7 @@ def render(vlans: list[dict], results: dict, prev: dict | None, protos: tuple[st print(row) reach = sum(1 for s in results.values() if "__error__" not in s - for d in s.values() for p in protos if d.get(p)) + for d in s.values() for p in protos if d.get(p) is True) total = sum(1 for s in results.values() if "__error__" not in s for _d in s.values() for _p in protos) print(f"\n reachable: {reach}/{total} " diff --git a/labsim/monitoring-up.sh b/labsim/monitoring-up.sh index 5307fda..273cda2 100755 --- a/labsim/monitoring-up.sh +++ b/labsim/monitoring-up.sh @@ -76,6 +76,7 @@ for _ in $(seq 1 40); do done echo -log "Grafana: http://localhost:${GRAFANA_PORT}/d/labsim-matrix (no login)" +log "Topology: http://localhost:${EXPORTER_PORT}/ <- live mesh, red/green + RTT" +log "Grafana: http://localhost:${GRAFANA_PORT}/d/labsim-matrix (no login, history)" log "Prometheus: http://localhost:${PROM_PORT}" log "Exporter: http://localhost:${EXPORTER_PORT}/metrics" diff --git a/labsim/topology.html b/labsim/topology.html new file mode 100644 index 0000000..f4d910f --- /dev/null +++ b/labsim/topology.html @@ -0,0 +1,181 @@ + + + + +labsim — live VLAN topology + + + +
+

labsim — live VLAN topology

+ … + every path is probed from a VM to every other VM, through the VyOS router + +
+ +
+
+

Mesh — line colour is reachability, label is ICMP RTT

+ +
+ reachable + blocked + hover a line for detail · node ring turns red if anything to/from it is blocked +
+
+ + +
+ + + +