Files
lab/labsim/labsim-exporter.py

113 lines
4.0 KiB
Python
Raw Normal View History

feat(labsim): libvirt replica of the lab network with LACP + VyOS routing A throwaway copy of the production VLAN topology so routing and firewall changes can be tested before they touch the real network. Same VLAN IDs and roles as UniFi, deliberately different ranges (172.31.<vlan>.0/24) so nothing here can be mistaken for production. - OVS fabric: real 802.1Q. Access port per micro VM, host leg per VLAN (.2, for SSH only — NOT the VMs' default route, so inter-VLAN tests exercise the router rather than the host's routing table), and a trunk portgroup with VLAN 1 declared nativeMode='untagged'. - Six Alpine micro VMs (256MB, copy-on-write overlays on one 176MB image), SSH + a hello-world HTTP page naming the VLAN. - VyOS router installed to disk unattended over the console, with the SAME config shape as the VP2440s: two NICs in an LACP bond carrying the trunk, VLAN 1 native, bond0.<vlan> holding the .1 gateway on each. - labsim-matrix.py: full-mesh ICMP/TCP22/TCP80 probe, ~0.2s, --watch highlights cells that changed since the last sweep. Guest-side probe is python3 (already present via cloud-init) so nothing is installed on VMs that have no internet. - Prometheus + Grafana (anonymous auth, no login) with a provisioned dashboard: heatmap plus a state timeline showing exactly when a path flipped. Verified end to end: one VyOS rule took sum(labsim_reachable) from 90 to 84, blocking precisely kvm<->k8s across all three protocols. Traps found building this, all now encoded in the scripts: - virtio-net breaks 802.3ad: the guest's bonding driver reports slaves "MII Status: down" despite carrier=1 and never sends an LACPDU, so the bond sits in AD_STATE_DEFAULTED. e1000e fixes it with no other change. Matches the netdev thread "bonding (IEEE 802.3ad) not working with qemu/virtio". - OVS defaults bonds to active-backup, which does not speak LACP at all — bond_mode=balance-tcp is required. - LACP deadlock: OVS holds members disabled until negotiation while the partner needs carrier before it will send LACPDUs. lacp-fallback-ab breaks it. - LACPDUs are untagged, so a trunk with no native VLAN has nowhere to put them. - --boot cdrom,hd re-runs the ISO on every restart, so every commit+save went to a live system that evaporated. Install now switches the VM to boot hd. - cloud-init on Alpine: users stay locked without lock_passwd:false, one failing runcmd aborts the rest, busybox here has no httpd applet, and start-stop-daemon --exec /usr/bin/python3 matches cloud-init's own python3. - The user-data heredoc is unquoted, so backticks in a COMMENT were executed by the host shell and their output corrupted the YAML. build_seed now validates with yaml.safe_load before building the ISO. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DMVzWZgiKW2wquf5z8S1yH
2026-08-13 00:42:39 +01:00
#!/usr/bin/env python3
"""Prometheus exporter for the labsim connectivity matrix.
Runs the same sweep as labsim-matrix.py on an interval and exposes it as
metrics, so Grafana can show the mesh as a heatmap and more usefully a
history of exactly when a cell flipped after a firewall change.
labsim_reachable{src,dst,proto} 1 = reachable, 0 = blocked
labsim_sweep_seconds how long the last sweep took
labsim_sweep_total sweeps completed since start
labsim_up 1 while the exporter is alive
Deliberately stdlib-only (http.server + threads): this runs on the workstation
next to libvirt, and adding a dependency to watch a lab network is silly.
./labsim-exporter.py --port 9101 --interval 15
"""
from __future__ import annotations
import argparse
import http.server
import threading
import time
import labsim_matrix_lib as m # thin import shim, see below
class Collector:
def __init__(self, interval: int, timeout: int) -> None:
self.interval = interval
self.timeout = timeout
self.vlans = m.load_vlans()
self.lock = threading.Lock()
self.results: dict = {}
self.duration = 0.0
self.sweeps = 0
def loop(self) -> None:
while True:
started = time.time()
try:
results = m.sweep(self.vlans, self.timeout)
with self.lock:
self.results = results
self.duration = time.time() - started
self.sweeps += 1
except Exception: # noqa: BLE001 - never let the loop die
pass
time.sleep(max(1.0, self.interval - (time.time() - started)))
def render(self) -> str:
with self.lock:
results, duration, sweeps = dict(self.results), self.duration, self.sweeps
out = [
"# HELP labsim_reachable 1 if dst is reachable from src over proto",
"# TYPE labsim_reachable gauge",
]
for src, data in results.items():
if "__error__" in data:
continue
for dst, protos in data.items():
for proto, ok in protos.items():
out.append(
f'labsim_reachable{{src="{src}",dst="{dst}",proto="{proto}"}} {1 if ok else 0}')
out += [
"# HELP labsim_sweep_seconds duration of the last sweep",
"# TYPE labsim_sweep_seconds gauge",
f"labsim_sweep_seconds {duration:.3f}",
"# HELP labsim_sweep_total sweeps completed",
"# TYPE labsim_sweep_total counter",
f"labsim_sweep_total {sweeps}",
"# HELP labsim_up exporter liveness",
"# TYPE labsim_up gauge",
"labsim_up 1",
]
return "\n".join(out) + "\n"
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--port", type=int, default=9101)
ap.add_argument("--interval", type=int, default=15)
ap.add_argument("--timeout", type=int, default=30)
args = ap.parse_args()
collector = Collector(args.interval, args.timeout)
threading.Thread(target=collector.loop, daemon=True).start()
class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self) -> None: # noqa: N802 - stdlib API
if self.path.rstrip("/") not in ("", "/metrics"):
self.send_error(404)
return
body = collector.render().encode()
self.send_response(200)
self.send_header("Content-Type", "text/plain; version=0.0.4")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *_args) -> None: # keep the console quiet
return
srv = http.server.ThreadingHTTPServer(("0.0.0.0", args.port), Handler)
print(f"labsim exporter on :{args.port}/metrics (sweep every {args.interval}s)")
srv.serve_forever()
return 0
if __name__ == "__main__":
raise SystemExit(main())