feat(labsim): live topology view with per-path latency

The Grafana heatmap of 1s and 0s said almost nothing, and the state timeline
was an unreadable pile of overlapping series labels. Replaced as the primary
view with a purpose-built page served by the exporter itself.

- Probe now captures ICMP RTT, exposed as labsim_rtt_ms{src,dst}. A path that
  is up but slow is a different problem from one that is down, and a pass/fail
  grid cannot show it.
- Exporter serves / (topology), /api/matrix (JSON) and /metrics.
- topology.html: node per VLAN in a ring, VyOS router in the centre because
  every inter-VLAN packet really does traverse it, one line per pair coloured
  green/red with the RTT on it. Hovering gives per-direction state. A node ring
  goes red if anything to or from it is blocked. Side panels list blocked paths
  and the slowest links. Refreshes every 5s, no dependencies.

Grafana stays for what it is actually good at — history of when a path flipped.

Label placement is deliberate: RTT captions sit ~32% along each edge with a
perpendicular nudge, because every diagonal of a 6-node mesh crosses the centre
and midpoint labels stack on the router node.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DMVzWZgiKW2wquf5z8S1yH

labsim/README.md

@@ -62,6 +62,22 @@ Console, when the network is the thing that is broken:
 sudo virsh console labsim-2-k8s     # root / labsim
 ```
 
+## Watching it
+
+```bash
+./labsim-matrix.py --watch 2     # terminal grid, changed cells highlighted
+./monitoring-up.sh               # topology page + Prometheus + Grafana
+```
+
+- **http://localhost:9101/** — live mesh: a node per VLAN, the router in the
+  middle, one line per pair coloured green/red with the ICMP RTT on it. Hover a
+  line for per-direction detail. Refreshes every 5s. This is the one to watch
+  while changing firewall rules.
+- **http://localhost:3000/d/labsim-matrix** — Grafana (anonymous, no login) for
+  *history*: when did a path flip, and how has latency moved.
+- **http://localhost:9101/metrics** — `labsim_reachable{src,dst,proto}` and
+  `labsim_rtt_ms{src,dst}`.
+
 ## Notes for whoever extends this
 
 Things that cost time the first time round, all verified on this image:

labsim/labsim-exporter.py

@@ -19,6 +19,8 @@ from __future__ import annotations
 
 import argparse
 import http.server
+import json
+import os
 import threading
 import time
 
@@ -48,6 +50,24 @@ class Collector:
                 pass
             time.sleep(max(1.0, self.interval - (time.time() - started)))
 
+    def snapshot(self) -> dict:
+        """Everything the topology page needs, in one JSON payload."""
+        with self.lock:
+            results, duration = dict(self.results), self.duration
+        reach = total = 0
+        for data in results.values():
+            if "__error__" in data:
+                continue
+            for protos in data.values():
+                for proto, ok in protos.items():
+                    if proto == "rtt_ms":
+                        continue
+                    total += 1
+                    if ok:
+                        reach += 1
+        return {"vlans": self.vlans, "results": results, "reachable": reach,
+                "total": total, "sweep_seconds": duration}
+
     def render(self) -> str:
         with self.lock:
             results, duration, sweeps = dict(self.results), self.duration, self.sweeps
@@ -56,13 +76,22 @@ class Collector:
             "# HELP labsim_reachable 1 if dst is reachable from src over proto",
             "# TYPE labsim_reachable gauge",
         ]
+        rtts = []
         for src, data in results.items():
             if "__error__" in data:
                 continue
             for dst, protos in data.items():
                 for proto, ok in protos.items():
+                    if proto == "rtt_ms":
+                        if isinstance(ok, (int, float)):
+                            rtts.append((src, dst, ok))
+                        continue
                     out.append(
                         f'labsim_reachable{{src="{src}",dst="{dst}",proto="{proto}"}} {1 if ok else 0}')
+        out += ["# HELP labsim_rtt_ms ICMP round-trip time",
+                "# TYPE labsim_rtt_ms gauge"]
+        for src, dst, val in rtts:
+            out.append(f'labsim_rtt_ms{{src="{src}",dst="{dst}"}} {val}')
         out += [
             "# HELP labsim_sweep_seconds duration of the last sweep",
             "# TYPE labsim_sweep_seconds gauge",
@@ -87,23 +116,39 @@ def main() -> int:
     collector = Collector(args.interval, args.timeout)
     threading.Thread(target=collector.loop, daemon=True).start()
 
+    here = os.path.dirname(os.path.abspath(__file__))
+
     class Handler(http.server.BaseHTTPRequestHandler):
-        def do_GET(self) -> None:  # noqa: N802 - stdlib API
-            if self.path.rstrip("/") not in ("", "/metrics"):
-                self.send_error(404)
-                return
-            body = collector.render().encode()
+        def _send(self, body: bytes, ctype: str) -> None:
             self.send_response(200)
-            self.send_header("Content-Type", "text/plain; version=0.0.4")
+            self.send_header("Content-Type", ctype)
             self.send_header("Content-Length", str(len(body)))
+            self.send_header("Cache-Control", "no-store")
             self.end_headers()
             self.wfile.write(body)
 
+        def do_GET(self) -> None:  # noqa: N802 - stdlib API
+            path = self.path.split("?")[0].rstrip("/")
+            if path in ("", "/topology"):
+                # Live topology view — the thing you actually watch.
+                try:
+                    with open(os.path.join(here, "topology.html"), "rb") as fh:
+                        self._send(fh.read(), "text/html; charset=utf-8")
+                except OSError:
+                    self.send_error(500, "topology.html missing")
+            elif path == "/api/matrix":
+                self._send(json.dumps(collector.snapshot()).encode(), "application/json")
+            elif path == "/metrics":
+                self._send(collector.render().encode(), "text/plain; version=0.0.4")
+            else:
+                self.send_error(404)
+
         def log_message(self, *_args) -> None:  # keep the console quiet
             return
 
     srv = http.server.ThreadingHTTPServer(("0.0.0.0", args.port), Handler)
-    print(f"labsim exporter on :{args.port}/metrics (sweep every {args.interval}s)")
+    print(f"labsim topology  http://localhost:{args.port}/")
+    print(f"labsim metrics   http://localhost:{args.port}/metrics  (sweep every {args.interval}s)")
     srv.serve_forever()
     return 0
 

labsim/labsim-matrix.py

@@ -37,18 +37,25 @@ PROTOS = ("icmp", "tcp22", "tcp80")
 # Runs ON the guest. Keep it stdlib-only and quick — a hung probe delays the
 # whole sweep, so every check is hard-bounded by a timeout.
 PROBE = r'''
-import json, socket, subprocess, sys
+import json, re, socket, subprocess, sys
 targets = json.load(sys.stdin)
 out = {}
 for name, ip in targets.items():
     res = {}
     try:
-        res["icmp"] = subprocess.run(
-            ["ping", "-c", "1", "-W", "1", ip],
-            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=4
-        ).returncode == 0
+        p = subprocess.run(["ping", "-c", "1", "-W", "1", ip],
+                           stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=4)
+        res["icmp"] = p.returncode == 0
+        # RTT as well as pass/fail: a path that is up but slow is a different
+        # problem from one that is down, and the grid alone cannot show it.
+        res["rtt_ms"] = None
+        if res["icmp"]:
+            m = re.search(r"time[=<]\s*([0-9.]+)\s*ms", p.stdout.decode("utf-8", "replace"))
+            if m:
+                res["rtt_ms"] = float(m.group(1))
     except Exception:
         res["icmp"] = False
+        res["rtt_ms"] = None
     for port in (22, 80):
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         s.settimeout(1.5)
@@ -147,7 +154,7 @@ def render(vlans: list[dict], results: dict, prev: dict | None, protos: tuple[st
             print(row)
 
     reach = sum(1 for s in results.values() if "__error__" not in s
-                for d in s.values() for p in protos if d.get(p))
+                for d in s.values() for p in protos if d.get(p) is True)
     total = sum(1 for s in results.values() if "__error__" not in s
                 for _d in s.values() for _p in protos)
     print(f"\n  reachable: {reach}/{total}   "

labsim/monitoring-up.sh

@@ -76,6 +76,7 @@ for _ in $(seq 1 40); do
 done
 
 echo
-log "Grafana:    http://localhost:${GRAFANA_PORT}/d/labsim-matrix  (no login)"
+log "Topology:   http://localhost:${EXPORTER_PORT}/   <- live mesh, red/green + RTT"
+log "Grafana:    http://localhost:${GRAFANA_PORT}/d/labsim-matrix  (no login, history)"
 log "Prometheus: http://localhost:${PROM_PORT}"
 log "Exporter:   http://localhost:${EXPORTER_PORT}/metrics"

labsim/topology.html

@@ -0,0 +1,181 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>labsim — live VLAN topology</title>
+<style>
+  :root {
+    --bg:#0e1116; --panel:#161b22; --line:#30363d; --text:#e6edf3; --dim:#8b949e;
+    --ok:#3fb950; --bad:#f85149; --warn:#d29922; --router:#58a6ff;
+  }
+  * { box-sizing:border-box; }
+  body { margin:0; background:var(--bg); color:var(--text);
+         font:14px/1.5 ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif; }
+  header { display:flex; align-items:baseline; gap:16px; flex-wrap:wrap;
+           padding:14px 20px; border-bottom:1px solid var(--line); }
+  h1 { font-size:16px; margin:0; font-weight:650; letter-spacing:.2px; }
+  .meta { color:var(--dim); font-size:12px; }
+  .pill { padding:2px 8px; border-radius:999px; font-size:12px; font-weight:600; }
+  .pill.ok { background:rgba(63,185,80,.15); color:var(--ok); }
+  .pill.bad { background:rgba(248,81,73,.15); color:var(--bad); }
+  main { display:grid; grid-template-columns:minmax(0,1.35fr) minmax(320px,.65fr);
+         gap:16px; padding:16px 20px; align-items:start; }
+  @media (max-width:1000px){ main { grid-template-columns:1fr; } }
+  .card { background:var(--panel); border:1px solid var(--line); border-radius:10px; padding:14px; }
+  .card h2 { margin:0 0 10px; font-size:13px; font-weight:600; color:var(--dim);
+             text-transform:uppercase; letter-spacing:.6px; }
+  svg { width:100%; height:auto; display:block; }
+  .edge { stroke-width:2.5; transition:stroke .25s, opacity .25s; }
+  .edge.ok { stroke:var(--ok); opacity:.55; }
+  .edge.bad { stroke:var(--bad); opacity:.95; stroke-dasharray:7 5; }
+  .edge:hover { opacity:1; stroke-width:4; }
+  .node circle { fill:#0d1117; stroke-width:2.5; }
+  .node text { text-anchor:middle; font-size:11px; font-weight:600; fill:var(--text); }
+  .node .sub { font-size:9.5px; font-weight:400; fill:var(--dim); }
+  .rtt { font-size:9px; fill:var(--dim); text-anchor:middle; }
+  table { width:100%; border-collapse:collapse; font-size:12.5px; }
+  th,td { text-align:left; padding:5px 8px; border-bottom:1px solid var(--line); }
+  th { color:var(--dim); font-weight:600; font-size:11px; text-transform:uppercase; }
+  td.n { text-align:right; font-variant-numeric:tabular-nums; }
+  .b-ok { color:var(--ok); } .b-bad { color:var(--bad); }
+  .empty { color:var(--dim); padding:10px 4px; }
+  .legend { display:flex; gap:14px; align-items:center; color:var(--dim);
+            font-size:11.5px; margin-top:10px; flex-wrap:wrap; }
+  .swatch { display:inline-block; width:22px; height:0; border-top:2.5px solid; margin-right:5px;
+            vertical-align:middle; }
+</style>
+</head>
+<body>
+<header>
+  <h1>labsim — live VLAN topology</h1>
+  <span id="summary" class="pill ok">…</span>
+  <span class="meta">every path is probed <em>from</em> a VM <em>to</em> every other VM, through the VyOS router</span>
+  <span class="meta" id="clock" style="margin-left:auto"></span>
+</header>
+
+<main>
+  <section class="card">
+    <h2>Mesh — line colour is reachability, label is ICMP RTT</h2>
+    <svg id="topo" viewBox="0 0 720 560" role="img" aria-label="VLAN topology"></svg>
+    <div class="legend">
+      <span><i class="swatch" style="border-color:var(--ok)"></i>reachable</span>
+      <span><i class="swatch" style="border-color:var(--bad); border-top-style:dashed"></i>blocked</span>
+      <span>hover a line for detail · node ring turns red if anything to/from it is blocked</span>
+    </div>
+  </section>
+
+  <aside style="display:grid; gap:16px">
+    <section class="card">
+      <h2>Blocked paths</h2>
+      <div id="blocked"></div>
+    </section>
+    <section class="card">
+      <h2>Latency (ICMP, ms)</h2>
+      <table><thead><tr><th>path</th><th class="n">rtt</th></tr></thead>
+      <tbody id="lat"></tbody></table>
+    </section>
+  </aside>
+</main>
+
+<script>
+const REFRESH_MS = 5000;
+const CX = 360, CY = 250, R = 185;
+
+function polar(i, n) {
+  const a = (i / n) * Math.PI * 2 - Math.PI / 2;
+  return { x: CX + R * Math.cos(a), y: CY + R * Math.sin(a) };
+}
+
+function render(data) {
+  const vlans = data.vlans, res = data.results;
+  const svg = document.getElementById('topo');
+  const n = vlans.length;
+  const pos = vlans.map((_, i) => polar(i, n));
+  let out = '';
+
+  // Router in the middle — every inter-VLAN packet really does traverse it.
+  out += `<circle cx="${CX}" cy="${CY}" r="40" fill="#0d1117" stroke="var(--router)" stroke-width="2.5"/>`;
+  out += `<text x="${CX}" y="${CY-6}" text-anchor="middle" font-size="12" font-weight="700" fill="var(--router)">VyOS</text>`;
+  out += `<text x="${CX}" y="${CY+9}" text-anchor="middle" font-size="8.5" fill="var(--dim)">bond0</text>`;
+  out += `<text x="${CX}" y="${CY+20}" text-anchor="middle" font-size="8.5" fill="var(--dim)">LACP</text>`;
+
+  const bad = new Set();
+  // One line per unordered pair; a pair is bad if EITHER direction fails.
+  for (let i = 0; i < n; i++) {
+    for (let j = i + 1; j < n; j++) {
+      const a = vlans[i].label, b = vlans[j].label;
+      const ab = (res[a] || {})[b] || {}, ba = (res[b] || {})[a] || {};
+      const okAB = ab.icmp === true, okBA = ba.icmp === true;
+      const ok = okAB && okBA;
+      if (!ok) { bad.add(a); bad.add(b); }
+      const rtts = [ab.rtt_ms, ba.rtt_ms].filter(v => typeof v === 'number');
+      const rtt = rtts.length ? (rtts.reduce((s,v)=>s+v,0)/rtts.length) : null;
+      // Place the label ~32% along the edge, not at the midpoint: diagonals of
+      // a 6-node mesh all cross the centre, so midpoint labels stack on top of
+      // the router node. Plus a small perpendicular nudge off the line itself.
+      const dx = pos[j].x - pos[i].x, dy = pos[j].y - pos[i].y;
+      const len = Math.hypot(dx, dy) || 1;
+      const t = 0.32;
+      const mx = pos[i].x + dx * t + (-dy / len) * 8;
+      const my = pos[i].y + dy * t + ( dx / len) * 8;
+      const tip = `${a} ↔ ${b}\n→ ${okAB ? 'ok' : 'BLOCKED'}   ← ${okBA ? 'ok' : 'BLOCKED'}` +
+                  (rtt !== null ? `\nrtt ${rtt.toFixed(2)} ms` : '');
+      out += `<line class="edge ${ok?'ok':'bad'}" x1="${pos[i].x}" y1="${pos[i].y}" x2="${pos[j].x}" y2="${pos[j].y}"><title>${tip}</title></line>`;
+      if (ok && rtt !== null)
+        out += `<text class="rtt" x="${mx}" y="${my}">${rtt.toFixed(2)}</text>`;
+    }
+  }
+
+  vlans.forEach((v, i) => {
+    const p = pos[i], isBad = bad.has(v.label);
+    out += `<g class="node"><circle cx="${p.x}" cy="${p.y}" r="30" stroke="${isBad?'var(--bad)':'var(--ok)'}"/>` +
+           `<text x="${p.x}" y="${p.y-2}">${v.name}</text>` +
+           `<text class="sub" x="${p.x}" y="${p.y+11}">vlan ${v.vid}</text>` +
+           `<text class="sub" x="${p.x}" y="${p.y+47}">${v.ip}</text></g>`;
+  });
+  svg.innerHTML = out;
+
+  // Blocked list — the thing you actually act on.
+  const rows = [];
+  for (const src of vlans) for (const dst of vlans) {
+    if (src.label === dst.label) continue;
+    const d = (res[src.label] || {})[dst.label] || {};
+    for (const proto of ['icmp','tcp22','tcp80'])
+      if (d[proto] === false) rows.push(`${src.label} → ${dst.label} <span style="color:var(--dim)">(${proto})</span>`);
+  }
+  document.getElementById('blocked').innerHTML = rows.length
+    ? `<table><tbody>${rows.map(r=>`<tr><td class="b-bad">${r}</td></tr>`).join('')}</tbody></table>`
+    : `<div class="empty">none — all ${vlans.length*(vlans.length-1)*3} paths open</div>`;
+
+  // Latency table, slowest first.
+  const lat = [];
+  for (const src of vlans) for (const dst of vlans) {
+    if (src.label === dst.label) continue;
+    const d = (res[src.label] || {})[dst.label] || {};
+    if (typeof d.rtt_ms === 'number') lat.push([`${src.label} → ${dst.label}`, d.rtt_ms]);
+  }
+  lat.sort((a,b) => b[1]-a[1]);
+  document.getElementById('lat').innerHTML = lat.slice(0,12)
+    .map(([k,v]) => `<tr><td>${k}</td><td class="n">${v.toFixed(2)}</td></tr>`).join('')
+    || `<tr><td class="empty" colspan="2">no RTT data</td></tr>`;
+
+  const total = data.total, reach = data.reachable;
+  const pill = document.getElementById('summary');
+  pill.textContent = `${reach}/${total} paths open`;
+  pill.className = 'pill ' + (reach === total ? 'ok' : 'bad');
+  document.getElementById('clock').textContent =
+    `updated ${new Date().toLocaleTimeString()} · sweep ${data.sweep_seconds.toFixed(2)}s · refresh ${REFRESH_MS/1000}s`;
+}
+
+async function tick() {
+  try {
+    const r = await fetch('/api/matrix', {cache:'no-store'});
+    render(await r.json());
+  } catch (e) {
+    document.getElementById('clock').textContent = 'exporter unreachable — ' + e;
+  }
+}
+tick(); setInterval(tick, REFRESH_MS);
+</script>
+</body>
+</html>