diff --git a/labsim/README.md b/labsim/README.md index 12ca514..615c93d 100644 --- a/labsim/README.md +++ b/labsim/README.md @@ -62,6 +62,22 @@ Console, when the network is the thing that is broken: sudo virsh console labsim-2-k8s # root / labsim ``` +## Watching it + +```bash +./labsim-matrix.py --watch 2 # terminal grid, changed cells highlighted +./monitoring-up.sh # topology page + Prometheus + Grafana +``` + +- **http://localhost:9101/** — live mesh: a node per VLAN, the router in the + middle, one line per pair coloured green/red with the ICMP RTT on it. Hover a + line for per-direction detail. Refreshes every 5s. This is the one to watch + while changing firewall rules. +- **http://localhost:3000/d/labsim-matrix** — Grafana (anonymous, no login) for + *history*: when did a path flip, and how has latency moved. +- **http://localhost:9101/metrics** — `labsim_reachable{src,dst,proto}` and + `labsim_rtt_ms{src,dst}`. + ## Notes for whoever extends this Things that cost time the first time round, all verified on this image: diff --git a/labsim/labsim-exporter.py b/labsim/labsim-exporter.py index c9accd8..f82ab29 100755 --- a/labsim/labsim-exporter.py +++ b/labsim/labsim-exporter.py @@ -19,6 +19,8 @@ from __future__ import annotations import argparse import http.server +import json +import os import threading import time @@ -48,6 +50,24 @@ class Collector: pass time.sleep(max(1.0, self.interval - (time.time() - started))) + def snapshot(self) -> dict: + """Everything the topology page needs, in one JSON payload.""" + with self.lock: + results, duration = dict(self.results), self.duration + reach = total = 0 + for data in results.values(): + if "__error__" in data: + continue + for protos in data.values(): + for proto, ok in protos.items(): + if proto == "rtt_ms": + continue + total += 1 + if ok: + reach += 1 + return {"vlans": self.vlans, "results": results, "reachable": reach, + "total": total, "sweep_seconds": duration} + def render(self) -> str: with self.lock: results, duration, sweeps = dict(self.results), self.duration, self.sweeps @@ -56,13 +76,22 @@ class Collector: "# HELP labsim_reachable 1 if dst is reachable from src over proto", "# TYPE labsim_reachable gauge", ] + rtts = [] for src, data in results.items(): if "__error__" in data: continue for dst, protos in data.items(): for proto, ok in protos.items(): + if proto == "rtt_ms": + if isinstance(ok, (int, float)): + rtts.append((src, dst, ok)) + continue out.append( f'labsim_reachable{{src="{src}",dst="{dst}",proto="{proto}"}} {1 if ok else 0}') + out += ["# HELP labsim_rtt_ms ICMP round-trip time", + "# TYPE labsim_rtt_ms gauge"] + for src, dst, val in rtts: + out.append(f'labsim_rtt_ms{{src="{src}",dst="{dst}"}} {val}') out += [ "# HELP labsim_sweep_seconds duration of the last sweep", "# TYPE labsim_sweep_seconds gauge", @@ -87,23 +116,39 @@ def main() -> int: collector = Collector(args.interval, args.timeout) threading.Thread(target=collector.loop, daemon=True).start() + here = os.path.dirname(os.path.abspath(__file__)) + class Handler(http.server.BaseHTTPRequestHandler): - def do_GET(self) -> None: # noqa: N802 - stdlib API - if self.path.rstrip("/") not in ("", "/metrics"): - self.send_error(404) - return - body = collector.render().encode() + def _send(self, body: bytes, ctype: str) -> None: self.send_response(200) - self.send_header("Content-Type", "text/plain; version=0.0.4") + self.send_header("Content-Type", ctype) self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") self.end_headers() self.wfile.write(body) + def do_GET(self) -> None: # noqa: N802 - stdlib API + path = self.path.split("?")[0].rstrip("/") + if path in ("", "/topology"): + # Live topology view — the thing you actually watch. + try: + with open(os.path.join(here, "topology.html"), "rb") as fh: + self._send(fh.read(), "text/html; charset=utf-8") + except OSError: + self.send_error(500, "topology.html missing") + elif path == "/api/matrix": + self._send(json.dumps(collector.snapshot()).encode(), "application/json") + elif path == "/metrics": + self._send(collector.render().encode(), "text/plain; version=0.0.4") + else: + self.send_error(404) + def log_message(self, *_args) -> None: # keep the console quiet return srv = http.server.ThreadingHTTPServer(("0.0.0.0", args.port), Handler) - print(f"labsim exporter on :{args.port}/metrics (sweep every {args.interval}s)") + print(f"labsim topology http://localhost:{args.port}/") + print(f"labsim metrics http://localhost:{args.port}/metrics (sweep every {args.interval}s)") srv.serve_forever() return 0 diff --git a/labsim/labsim-matrix.py b/labsim/labsim-matrix.py index 86632f1..425624f 100755 --- a/labsim/labsim-matrix.py +++ b/labsim/labsim-matrix.py @@ -37,18 +37,25 @@ PROTOS = ("icmp", "tcp22", "tcp80") # Runs ON the guest. Keep it stdlib-only and quick — a hung probe delays the # whole sweep, so every check is hard-bounded by a timeout. PROBE = r''' -import json, socket, subprocess, sys +import json, re, socket, subprocess, sys targets = json.load(sys.stdin) out = {} for name, ip in targets.items(): res = {} try: - res["icmp"] = subprocess.run( - ["ping", "-c", "1", "-W", "1", ip], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=4 - ).returncode == 0 + p = subprocess.run(["ping", "-c", "1", "-W", "1", ip], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=4) + res["icmp"] = p.returncode == 0 + # RTT as well as pass/fail: a path that is up but slow is a different + # problem from one that is down, and the grid alone cannot show it. + res["rtt_ms"] = None + if res["icmp"]: + m = re.search(r"time[=<]\s*([0-9.]+)\s*ms", p.stdout.decode("utf-8", "replace")) + if m: + res["rtt_ms"] = float(m.group(1)) except Exception: res["icmp"] = False + res["rtt_ms"] = None for port in (22, 80): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(1.5) @@ -147,7 +154,7 @@ def render(vlans: list[dict], results: dict, prev: dict | None, protos: tuple[st print(row) reach = sum(1 for s in results.values() if "__error__" not in s - for d in s.values() for p in protos if d.get(p)) + for d in s.values() for p in protos if d.get(p) is True) total = sum(1 for s in results.values() if "__error__" not in s for _d in s.values() for _p in protos) print(f"\n reachable: {reach}/{total} " diff --git a/labsim/monitoring-up.sh b/labsim/monitoring-up.sh index 5307fda..273cda2 100755 --- a/labsim/monitoring-up.sh +++ b/labsim/monitoring-up.sh @@ -76,6 +76,7 @@ for _ in $(seq 1 40); do done echo -log "Grafana: http://localhost:${GRAFANA_PORT}/d/labsim-matrix (no login)" +log "Topology: http://localhost:${EXPORTER_PORT}/ <- live mesh, red/green + RTT" +log "Grafana: http://localhost:${GRAFANA_PORT}/d/labsim-matrix (no login, history)" log "Prometheus: http://localhost:${PROM_PORT}" log "Exporter: http://localhost:${EXPORTER_PORT}/metrics" diff --git a/labsim/topology.html b/labsim/topology.html new file mode 100644 index 0000000..f4d910f --- /dev/null +++ b/labsim/topology.html @@ -0,0 +1,181 @@ + + + + +labsim — live VLAN topology + + + +
+

labsim — live VLAN topology

+ … + every path is probed from a VM to every other VM, through the VyOS router + +
+ +
+
+

Mesh — line colour is reachability, label is ICMP RTT

+ +
+ reachable + blocked + hover a line for detail · node ring turns red if anything to/from it is blocked +
+
+ + +
+ + + +