diff --git a/labsim/.gitignore b/labsim/.gitignore new file mode 100644 index 0000000..73e9e69 --- /dev/null +++ b/labsim/.gitignore @@ -0,0 +1,4 @@ +# runtime artifacts, not source +*.log +labsim_matrix_lib.py +__pycache__/ diff --git a/labsim/README.md b/labsim/README.md new file mode 100644 index 0000000..12ca514 --- /dev/null +++ b/labsim/README.md @@ -0,0 +1,88 @@ +# labsim — libvirt replica of the lab network + +A throwaway copy of the production VLAN topology for testing routing, firewall +rules and failover **without touching the real network**. Same VLAN IDs and +roles as UniFi, deliberately different IP ranges so nothing can be confused for +production. + +## Topology + +Each VLAN is its own isolated libvirt network with one tiny Alpine VM on it. + +| VLAN | Name | Sim subnet | VM address | Mirrors production | +|-----:|------|------------|-----------|--------------------| +| 1 | management | 172.31.1.0/24 | 172.31.1.10 | 192.168.1.0/24 | +| 2 | k8s | 172.31.2.0/24 | 172.31.2.10 | 192.168.8.0/23 | +| 3 | kvm | 172.31.3.0/24 | 172.31.3.10 | 192.168.3.0/24 | +| 9 | private | 172.31.9.0/24 | 172.31.9.10 | 10.0.9.0/23 | +| 10 | lot | 172.31.10.0/24 | 172.31.10.10 | 10.0.0.0/23 | +| 200 | roomates | 172.31.200.0/24 | 172.31.200.10 | 192.168.2.0/24 | + +The sim subnet always encodes the VLAN id: `172.31..0/24`. + +Address plan, identical on every VLAN: + +| Address | Role | +|---------|------| +| `.1` | gateway under test — a router VM you add (not created by default) | +| `.2` | host bridge — how you reach the VMs from this workstation | +| `.10` | the VLAN's micro VM | +| `.254` | reserved for a VRRP VIP, mirroring production | + +The host sits at `.2` purely so you can SSH in. It is deliberately **not** the +VMs' default route — that is `.1` — so inter-VLAN tests fail loudly when no +router is present instead of being silently served by the host's own routing +table. libvirt also installs reject rules that stop these networks forwarding +to each other, so traffic between VLANs only works once a router VM bridges +them. + +## Usage + +```bash +./labsim-up.sh # bring up every VLAN (idempotent) +./labsim-up.sh 2 3 # only VLANs 2 and 3 +./labsim-down.sh # destroy VMs + networks, keep the base image +./labsim-down.sh --purge # also delete the downloaded Alpine image +``` + +Each VM: 256 MB, 1 vCPU, a copy-on-write overlay on one shared 176 MB Alpine +image (so six VMs cost a few MB of disk, not 1 GB). + +## Access + +```bash +ssh alpine@172.31.2.10 # normal user (password: labsim) +ssh root@172.31.2.10 # privileged — this image has no sudo +curl http://172.31.2.10/ # hello-world page naming the VLAN +``` + +Console, when the network is the thing that is broken: + +```bash +sudo virsh console labsim-2-k8s # root / labsim +``` + +## Notes for whoever extends this + +Things that cost time the first time round, all verified on this image: + +- **No `sudo`.** Alpine ships `doas`; cloud-init's `sudo:` directive is inert + here. Use `root@` for privileged work. +- **cloud-init leaves users locked** (`!*` in `/etc/shadow`) unless + `lock_passwd: false`, and sshd then refuses key auth for that user. +- **One failing `runcmd` aborts every command after it.** Each entry is + `|| true` for that reason. +- **busybox here has no `httpd` applet**, and the VMs have no internet to + `apk add` one — so the hello-world server is `python3 -m http.server` + (python3 is already present because cloud-init depends on it). +- **`start-stop-daemon --exec /usr/bin/python3` matches cloud-init's own + python3** at boot and refuses to start anything. +- **busybox `pgrep -f PATTERN` matches its own argv**, so a "skip if already + running" guard always fires. Verified: `guard_exit=0` with nothing listening. + +## Not modelled (yet) + +VLANs are separate L2 segments rather than one 802.1Q trunk, so this exercises +inter-VLAN routing but not a `bond0.` trunk config specifically. A router +VM would attach one NIC per VLAN. Adding a tagged-trunk variant is the obvious +next step if the bond/vif config itself needs testing. diff --git a/labsim/labsim-down.sh b/labsim/labsim-down.sh new file mode 100755 index 0000000..bb4f016 --- /dev/null +++ b/labsim/labsim-down.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Tear down the lab network simulation. +# +# By default this destroys VMs and networks but KEEPS the downloaded base +# image, so the next bring-up is fast. Pass --purge to remove that too. +# +# Usage: ./labsim-down.sh [--purge] [vlan-id ...] +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/lib.sh" +source "$SCRIPT_DIR/ovs.sh" + +PURGE=false +ARGS=() +for a in "$@"; do + case "$a" in + --purge) PURGE=true ;; + *) ARGS+=("$a") ;; + esac +done + +selected_vlans "${ARGS[@]+"${ARGS[@]}"}" + +for entry in "${SELECTED[@]}"; do + IFS=: read -r vid name prefix _r <<<"$entry" + vm="$(vm_name "$vid" "$name")" + + if virsh_q dominfo "$vm" >/dev/null 2>&1; then + log "destroying VM $vm" + virsh_q destroy "$vm" >/dev/null 2>&1 || true + virsh_q undefine "$vm" --nvram >/dev/null 2>&1 || virsh_q undefine "$vm" >/dev/null 2>&1 || true + fi + sudo rm -f "$IMG_DIR/${vm}.qcow2" "$IMG_DIR/${vm}-seed.iso" + +done + +# Legacy per-VLAN Linux-bridge networks from before the OVS migration. If +# these survive they keep a duplicate .2/24 on a dead bridge, and the +# kernel may prefer that route over the OVS host leg — which looks exactly +# like "the VM is unreachable" while ping -I hostvN works fine. +for entry in "${SELECTED[@]}"; do + IFS=: read -r vid _n _p _r <<<"$entry" + legacy="labsim-vlan${vid}" + if virsh_q net-info "$legacy" >/dev/null 2>&1; then + log "removing legacy network $legacy" + virsh_q net-destroy "$legacy" >/dev/null 2>&1 || true + virsh_q net-undefine "$legacy" >/dev/null 2>&1 || true + fi +done + +log "removing OVS fabric" +ovs_down + +if [ "$PURGE" = true ]; then + log "purging base image $BASE_IMAGE" + sudo rm -f "$BASE_IMAGE" + sudo rmdir "$IMG_DIR" 2>/dev/null || true +fi + +log "environment is DOWN" diff --git a/labsim/labsim-exporter.py b/labsim/labsim-exporter.py new file mode 100755 index 0000000..c9accd8 --- /dev/null +++ b/labsim/labsim-exporter.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Prometheus exporter for the labsim connectivity matrix. + +Runs the same sweep as labsim-matrix.py on an interval and exposes it as +metrics, so Grafana can show the mesh as a heatmap and — more usefully — a +history of exactly when a cell flipped after a firewall change. + + labsim_reachable{src,dst,proto} 1 = reachable, 0 = blocked + labsim_sweep_seconds how long the last sweep took + labsim_sweep_total sweeps completed since start + labsim_up 1 while the exporter is alive + +Deliberately stdlib-only (http.server + threads): this runs on the workstation +next to libvirt, and adding a dependency to watch a lab network is silly. + + ./labsim-exporter.py --port 9101 --interval 15 +""" +from __future__ import annotations + +import argparse +import http.server +import threading +import time + +import labsim_matrix_lib as m # thin import shim, see below + + +class Collector: + def __init__(self, interval: int, timeout: int) -> None: + self.interval = interval + self.timeout = timeout + self.vlans = m.load_vlans() + self.lock = threading.Lock() + self.results: dict = {} + self.duration = 0.0 + self.sweeps = 0 + + def loop(self) -> None: + while True: + started = time.time() + try: + results = m.sweep(self.vlans, self.timeout) + with self.lock: + self.results = results + self.duration = time.time() - started + self.sweeps += 1 + except Exception: # noqa: BLE001 - never let the loop die + pass + time.sleep(max(1.0, self.interval - (time.time() - started))) + + def render(self) -> str: + with self.lock: + results, duration, sweeps = dict(self.results), self.duration, self.sweeps + + out = [ + "# HELP labsim_reachable 1 if dst is reachable from src over proto", + "# TYPE labsim_reachable gauge", + ] + for src, data in results.items(): + if "__error__" in data: + continue + for dst, protos in data.items(): + for proto, ok in protos.items(): + out.append( + f'labsim_reachable{{src="{src}",dst="{dst}",proto="{proto}"}} {1 if ok else 0}') + out += [ + "# HELP labsim_sweep_seconds duration of the last sweep", + "# TYPE labsim_sweep_seconds gauge", + f"labsim_sweep_seconds {duration:.3f}", + "# HELP labsim_sweep_total sweeps completed", + "# TYPE labsim_sweep_total counter", + f"labsim_sweep_total {sweeps}", + "# HELP labsim_up exporter liveness", + "# TYPE labsim_up gauge", + "labsim_up 1", + ] + return "\n".join(out) + "\n" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=9101) + ap.add_argument("--interval", type=int, default=15) + ap.add_argument("--timeout", type=int, default=30) + args = ap.parse_args() + + collector = Collector(args.interval, args.timeout) + threading.Thread(target=collector.loop, daemon=True).start() + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 - stdlib API + if self.path.rstrip("/") not in ("", "/metrics"): + self.send_error(404) + return + body = collector.render().encode() + self.send_response(200) + self.send_header("Content-Type", "text/plain; version=0.0.4") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args) -> None: # keep the console quiet + return + + srv = http.server.ThreadingHTTPServer(("0.0.0.0", args.port), Handler) + print(f"labsim exporter on :{args.port}/metrics (sweep every {args.interval}s)") + srv.serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/labsim/labsim-matrix.py b/labsim/labsim-matrix.py new file mode 100755 index 0000000..86632f1 --- /dev/null +++ b/labsim/labsim-matrix.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Full-mesh connectivity matrix for the labsim VLANs. + +Probes every VLAN VM from every other VLAN VM (ICMP + TCP/22 + TCP/80) and +prints a grid. Use --watch to keep it live: cells that changed since the last +sweep are highlighted, so adding or removing a VyOS firewall rule shows up +within one refresh. + +Deliberately dependency-free on the guests: the probe runs with python3, which +is already installed there (cloud-init needs it), so nothing has to be +installed on VMs that have no internet. + + ./labsim-matrix.py # one sweep + ./labsim-matrix.py --watch # live, refresh every 5s + ./labsim-matrix.py --watch 2 # live, every 2s + ./labsim-matrix.py --proto icmp # single protocol + ./labsim-matrix.py --json # machine-readable +""" +from __future__ import annotations + +import argparse +import concurrent.futures +import json +import os +import subprocess +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +CONF = os.path.join(HERE, "vlans.conf") + +GREEN, RED, GREY, YELLOW, BOLD, RESET = ( + "\033[0;32m", "\033[0;31m", "\033[0;90m", "\033[1;33m", "\033[1m", "\033[0m") + +PROTOS = ("icmp", "tcp22", "tcp80") + +# Runs ON the guest. Keep it stdlib-only and quick — a hung probe delays the +# whole sweep, so every check is hard-bounded by a timeout. +PROBE = r''' +import json, socket, subprocess, sys +targets = json.load(sys.stdin) +out = {} +for name, ip in targets.items(): + res = {} + try: + res["icmp"] = subprocess.run( + ["ping", "-c", "1", "-W", "1", ip], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=4 + ).returncode == 0 + except Exception: + res["icmp"] = False + for port in (22, 80): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(1.5) + try: + s.connect((ip, port)); res["tcp%d" % port] = True + except Exception: + res["tcp%d" % port] = False + finally: + try: s.close() + except Exception: pass + out[name] = res +print(json.dumps(out)) +''' + + +def load_vlans() -> list[dict]: + vlans = [] + with open(CONF) as fh: + for line in fh: + line = line.strip() + if not line or line.startswith("#"): + continue + vid, name, prefix, real = line.split(":", 3) + vlans.append({"vid": vid, "name": name, "ip": f"{prefix}.10", + "label": f"{vid}:{name}", "real": real}) + return vlans + + +def probe_from(src: dict, targets: list[dict], timeout: int) -> tuple[str, dict]: + """SSH once into src and probe every target from there.""" + payload = json.dumps({t["label"]: t["ip"] for t in targets if t["label"] != src["label"]}) + cmd = [ + "ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", + "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", "-o", "LogLevel=ERROR", + f"alpine@{src['ip']}", "python3", "-", + ] + try: + # The probe script goes on stdin, the target list follows it — the guest + # reads the script from argv-less stdin, so send both in one stream. + proc = subprocess.run( + cmd, input=PROBE.replace("json.load(sys.stdin)", f"json.loads({payload!r})"), + capture_output=True, text=True, timeout=timeout) + if proc.returncode != 0: + return src["label"], {"__error__": (proc.stderr or "ssh failed").strip()[:60]} + return src["label"], json.loads(proc.stdout) + except subprocess.TimeoutExpired: + return src["label"], {"__error__": "probe timed out"} + except Exception as exc: # noqa: BLE001 - report, never crash the sweep + return src["label"], {"__error__": f"{type(exc).__name__}: {exc}"[:60]} + + +def sweep(vlans: list[dict], timeout: int) -> dict: + results: dict = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=len(vlans)) as pool: + futures = [pool.submit(probe_from, v, vlans, timeout) for v in vlans] + for fut in concurrent.futures.as_completed(futures): + label, data = fut.result() + results[label] = data + return results + + +def cell(ok: bool | None, changed: bool) -> str: + if ok is None: + return f"{GREY} · {RESET}" + mark = "ok " if ok else "-- " + colour = GREEN if ok else RED + if changed: + return f"{YELLOW}{BOLD}{'OK*' if ok else 'XX*':<4}{RESET}" + return f"{colour}{mark}{RESET}" + + +def render(vlans: list[dict], results: dict, prev: dict | None, protos: tuple[str, ...]) -> None: + labels = [v["label"] for v in vlans] + width = max(len(x) for x in labels) + 2 + + for proto in protos: + print(f"\n{BOLD}{proto.upper()}{RESET} (rows = source, columns = destination)") + header = " " * width + "".join(f"{lbl:<{width}}" for lbl in labels) + print(f"{GREY}{header}{RESET}") + + for src in vlans: + row = f"{src['label']:<{width}}" + data = results.get(src["label"], {}) + if "__error__" in data: + print(row + f"{RED}{data['__error__']}{RESET}") + continue + for dst in vlans: + if dst["label"] == src["label"]: + row += f"{GREY}{'·':<{width}}{RESET}" + continue + ok = data.get(dst["label"], {}).get(proto) + was = (prev or {}).get(src["label"], {}).get(dst["label"], {}).get(proto) + changed = prev is not None and was is not None and was != ok + txt = cell(ok, changed) + row += txt + " " * (width - 4) + print(row) + + reach = sum(1 for s in results.values() if "__error__" not in s + for d in s.values() for p in protos if d.get(p)) + total = sum(1 for s in results.values() if "__error__" not in s + for _d in s.values() for _p in protos) + print(f"\n reachable: {reach}/{total} " + f"{GREEN}ok{RESET}=allowed {RED}--{RESET}=blocked/no route " + f"{YELLOW}*{RESET}=changed since last sweep") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--watch", nargs="?", const=5, type=int, metavar="SECONDS", + help="refresh continuously (default every 5s)") + ap.add_argument("--proto", choices=PROTOS, help="only this protocol") + ap.add_argument("--json", action="store_true", help="emit raw JSON and exit") + ap.add_argument("--timeout", type=int, default=30, help="per-host probe timeout") + args = ap.parse_args() + + vlans = load_vlans() + protos = (args.proto,) if args.proto else PROTOS + + if args.json: + print(json.dumps(sweep(vlans, args.timeout), indent=2)) + return 0 + + prev = None + while True: + started = time.time() + results = sweep(vlans, args.timeout) + if args.watch: + os.system("clear") + print(f"{BOLD}labsim connectivity matrix{RESET} " + f"{time.strftime('%H:%M:%S')} (refresh {args.watch}s, Ctrl-C to stop)") + render(vlans, results, prev, protos) + if not args.watch: + return 0 + prev = results + time.sleep(max(0.0, args.watch - (time.time() - started))) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + print() + sys.exit(130) diff --git a/labsim/labsim-up.sh b/labsim/labsim-up.sh new file mode 100755 index 0000000..eb3133d --- /dev/null +++ b/labsim/labsim-up.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Bring up the lab network simulation: one isolated libvirt network per VLAN, +# each with a single tiny Alpine VM offering SSH + a hello-world HTTP page. +# +# Idempotent: re-running only creates what is missing. Safe to run repeatedly. +# +# Usage: ./labsim-up.sh [vlan-id ...] (default: every VLAN in vlans.conf) +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/lib.sh" +source "$SCRIPT_DIR/ovs.sh" + +require_tools +[ -f "$BASE_IMAGE" ] || die "base image missing: $BASE_IMAGE (see README)" + +SSH_PUB="$(find_ssh_pubkey)" +log "Using SSH key: ${SSH_PUB%% *} ...${SSH_PUB##* }" + +selected_vlans "$@" + +# --- switch fabric ------------------------------------------------------ +log "bringing up OVS fabric ($OVS_BR) with host legs per VLAN" +ovs_up + +# --- VMs ------------------------------------------------------------------ +for entry in "${SELECTED[@]}"; do + IFS=: read -r vid name prefix real <<<"$entry" + vm="$(vm_name "$vid" "$name")" + ip="${prefix}.10" + + if virsh_q dominfo "$vm" >/dev/null 2>&1; then + state="$(virsh_q domstate "$vm" 2>/dev/null | head -1 | tr -d '\n')" + if [ "$state" = "running" ]; then + log "VM $vm already running ($ip)" + continue + fi + log "VM $vm exists but is $state — starting" + virsh_q start "$vm" >/dev/null + continue + fi + + log "creating VM $vm ($ip on vlan $vid/$name)" + + disk="$IMG_DIR/${vm}.qcow2" + seed="$IMG_DIR/${vm}-seed.iso" + + # Copy-on-write overlay: each VM costs a few MB, not 176. + sudo qemu-img create -q -f qcow2 -F qcow2 -b "$BASE_IMAGE" "$disk" "$VM_DISK" >/dev/null + + build_seed "$seed" "$vm" "$vid" "$name" "$prefix" "$ip" "$real" "$SSH_PUB" + + sudo virt-install \ + --connect "$LIBVIRT_URI" \ + --name "$vm" \ + --memory "$VM_MEM" --vcpus "$VM_CPUS" \ + --disk "path=$disk,format=qcow2,bus=virtio" \ + --disk "path=$seed,device=cdrom,readonly=on" \ + --network "network=$OVS_NET,portgroup=vlan${vid},model=virtio" \ + --os-variant alpinelinux3.18 \ + --graphics none --noautoconsole --import >/dev/null +done + +echo +log "waiting for VMs to answer on SSH + HTTP..." +wait_ready +echo +status_table +echo +log "environment is UP. Tear down with: $SCRIPT_DIR/labsim-down.sh" diff --git a/labsim/lib.sh b/labsim/lib.sh new file mode 100644 index 0000000..9f54c94 --- /dev/null +++ b/labsim/lib.sh @@ -0,0 +1,216 @@ +#!/bin/bash +# Shared helpers for the lab network simulation. +# shellcheck disable=SC2034 + +LIBVIRT_URI="${LIBVIRT_URI:-qemu:///system}" +IMG_DIR="${IMG_DIR:-/var/lib/libvirt/images/labsim}" +BASE_IMAGE="${BASE_IMAGE:-$IMG_DIR/alpine-base.qcow2}" +ALPINE_URL="${ALPINE_URL:-https://dl-cdn.alpinelinux.org/alpine/latest-stable/releases/cloud/generic_alpine-3.24.1-x86_64-bios-cloudinit-r0.qcow2}" + +VM_MEM="${VM_MEM:-256}" # MB — Alpine is happy here +VM_CPUS="${VM_CPUS:-1}" +VM_DISK="${VM_DISK:-1G}" +PREFIX="labsim" + +CONF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/vlans.conf" + +log() { printf '\033[0;36m[labsim]\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33m[labsim]\033[0m %s\n' "$*" >&2; } +die() { printf '\033[0;31m[labsim]\033[0m %s\n' "$*" >&2; exit 1; } + +virsh_q() { sudo virsh --connect "$LIBVIRT_URI" "$@"; } + +net_name() { echo "${PREFIX}-vlan$1"; } +vm_name() { echo "${PREFIX}-$1-$2"; } # labsim-2-k8s +# Linux bridge names are capped at 15 chars — keep it short and unique. +br_name() { echo "vbr-ls$1"; } + +require_tools() { + for t in virsh virt-install qemu-img genisoimage; do + command -v "$t" >/dev/null 2>&1 || die "missing required tool: $t" + done + sudo -n true 2>/dev/null || warn "sudo may prompt for a password" +} + +find_ssh_pubkey() { + local home="${SUDO_USER:+/home/$SUDO_USER}" + home="${home:-$HOME}" + for n in id_ed25519 id_ecdsa id_rsa; do + [ -f "$home/.ssh/$n.pub" ] && { cat "$home/.ssh/$n.pub"; return; } + done + die "no SSH public key found in $home/.ssh" +} + +# Populate SELECTED[] from argv (VLAN ids) or the whole config. +selected_vlans() { + SELECTED=() + local want=("$@") + while IFS= read -r line; do + [[ "$line" =~ ^[[:space:]]*# ]] && continue + [[ -z "${line// }" ]] && continue + local vid="${line%%:*}" + if [ ${#want[@]} -eq 0 ]; then + SELECTED+=("$line") + else + for w in "${want[@]}"; do [ "$w" = "$vid" ] && SELECTED+=("$line"); done + fi + done < "$CONF" + [ ${#SELECTED[@]} -gt 0 ] || die "no VLANs selected (checked $CONF)" +} + +# cloud-init NoCloud seed: static addressing + SSH key + hello-world HTTP. +build_seed() { + local iso="$1" vm="$2" vid="$3" name="$4" prefix="$5" ip="$6" real="$7" pubkey="$8" + local tmp; tmp="$(mktemp -d)" + + cat > "$tmp/meta-data" < "$tmp/network-config" < "$tmp/user-data" < +

labsim vlan $vid — $name

+

host: $vm

+

address: $ip/24

+

gateway under test: ${prefix}.1

+

mirrors production: $real

+ + - path: /etc/local.d/labsim-http.start + permissions: '0755' + content: | + #!/bin/sh + # This image's busybox has no httpd applet ("applet not found"), and the + # VMs are isolated so apk cannot fetch one. python3 is already present + # (cloud-init depends on it), so serve with http.server — no packages, + # no internet. + # + # Two traps already hit here, both silent: + # - start-stop-daemon --exec /usr/bin/python3 matches cloud-init's OWN + # python3 at boot, says "already running", starts nothing. + # - busybox pgrep -f PATTERN matches its own argv, so a + # "skip if running" guard always fires (verified: guard_exit=0 with + # nothing listening). + # So: no guard, no start-stop-daemon. Binding twice is harmless — the + # second just fails to bind. + nohup /usr/bin/python3 -m http.server 80 --directory /var/www \\ + >/var/log/labsim-http.log 2>&1 & +runcmd: + # cloud-init's network-config (v1, above) already applies the address, so do + # NOT restart networking here — it fails, and one failing runcmd aborts every + # command after it, which is what silently left httpd unstarted. Each command + # is || true for the same reason. + - [ sh, -c, "rc-update add sshd default || true" ] + - [ sh, -c, "rc-update add local default || true" ] + - [ sh, -c, "/etc/local.d/labsim-http.start || true" ] +EOF + + # Validate before building the ISO. The heredoc above is intentionally + # unquoted (it interpolates $ip/$prefix), which means backticks or $( ) in + # ANY line — including comments — get executed by the host shell and their + # output silently corrupts the YAML. Cheap check, expensive bug. + python3 -c "import yaml,sys; yaml.safe_load(open(sys.argv[1]))" "$tmp/user-data" \ + || die "generated user-data is not valid YAML (backticks or \$( ) in build_seed?): $tmp/user-data" + + sudo genisoimage -quiet -output "$iso" -volid cidata -joliet -rock \ + "$tmp/user-data" "$tmp/meta-data" "$tmp/network-config" + rm -rf "$tmp" +} + +ssh_to() { + local ip="$1"; shift + timeout 12 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=5 -o BatchMode=yes -o LogLevel=ERROR \ + "alpine@$ip" "$@" 2>/dev/null +} + +wait_ready() { + local deadline=$((SECONDS + 240)) pending=1 + while [ $SECONDS -lt $deadline ]; do + pending=0 + for entry in "${SELECTED[@]}"; do + IFS=: read -r vid _n prefix _r <<<"$entry" + # Wait for BOTH: sshd is up well before cloud-init's runcmd starts the + # web server, so checking SSH alone reports "ready" then shows HTTP FAIL. + ssh_to "${prefix}.10" true >/dev/null 2>&1 \ + && curl -sS -o /dev/null --max-time 4 "http://${prefix}.10/" 2>/dev/null \ + || pending=$((pending + 1)) + done + [ $pending -eq 0 ] && { log "all ${#SELECTED[@]} VMs reachable"; return 0; } + sleep 5 + done + warn "$pending VM(s) still not answering SSH after 240s — see status below" + return 0 +} + +status_table() { + printf ' %-18s %-6s %-16s %-9s %-7s %s\n' VM VLAN ADDRESS STATE SSH HTTP + printf ' %-18s %-6s %-16s %-9s %-7s %s\n' ------------------ ------ ---------------- --------- ------- ---- + for entry in "${SELECTED[@]}"; do + IFS=: read -r vid name prefix _r <<<"$entry" + local vm ip state ssh http + vm="$(vm_name "$vid" "$name")"; ip="${prefix}.10" + state="$(virsh_q domstate "$vm" 2>/dev/null | head -1 | tr -d '\n')" + [ -z "$state" ] && state="absent" + ssh_to "$ip" true >/dev/null 2>&1 && ssh=ok || ssh=FAIL + if curl -sS -o /dev/null --max-time 5 "http://$ip/" 2>/dev/null; then http=ok; else http=FAIL; fi + printf ' %-18s %-6s %-16s %-9s %-7s %s\n' "$vm" "$vid" "$ip" "$state" "$ssh" "$http" + done +} diff --git a/labsim/monitoring-up.sh b/labsim/monitoring-up.sh new file mode 100755 index 0000000..5307fda --- /dev/null +++ b/labsim/monitoring-up.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Prometheus + Grafana for the labsim connectivity matrix. +# +# Grafana runs with anonymous auth as Admin — NO LOGIN. That is deliberate for +# a throwaway lab on localhost; do not copy this into anything reachable. +# +# ./monitoring-up.sh start exporter + prometheus + grafana +# ./monitoring-up.sh --down stop and remove them +# +# Grafana: http://localhost:3000 (dashboard "labsim — VLAN connectivity matrix") +# Prometheus: http://localhost:9090 +# Exporter: http://localhost:9101/metrics +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/lib.sh" + +GRAFANA_PORT="${GRAFANA_PORT:-3000}" +PROM_PORT="${PROM_PORT:-9090}" +EXPORTER_PORT="${EXPORTER_PORT:-9101}" +NET="labsim-mon" + +if [ "${1:-}" = "--down" ]; then + pkill -f "labsim-exporter.py" 2>/dev/null || true + podman rm -f labsim-grafana labsim-prometheus >/dev/null 2>&1 || true + podman network rm -f "$NET" >/dev/null 2>&1 || true + log "monitoring stopped" + exit 0 +fi + +command -v podman >/dev/null 2>&1 || die "podman not installed" + +# --- exporter (on the host: it needs SSH access to the VMs) ---------------- +if pgrep -f "labsim-exporter.py" >/dev/null 2>&1; then + log "exporter already running on :$EXPORTER_PORT" +else + log "starting exporter on :$EXPORTER_PORT" + nohup "$SCRIPT_DIR/labsim-exporter.py" --port "$EXPORTER_PORT" --interval 15 \ + > /tmp/labsim-exporter.log 2>&1 & + sleep 3 +fi +curl -sS --max-time 5 "http://127.0.0.1:${EXPORTER_PORT}/metrics" >/dev/null \ + || die "exporter not answering on :$EXPORTER_PORT (see /tmp/labsim-exporter.log)" + +podman network exists "$NET" 2>/dev/null || podman network create "$NET" >/dev/null + +# --- prometheus ----------------------------------------------------------- +podman rm -f labsim-prometheus >/dev/null 2>&1 || true +log "starting prometheus on :$PROM_PORT" +podman run -d --name labsim-prometheus --network "$NET" \ + -p "${PROM_PORT}:9090" \ + -v "$SCRIPT_DIR/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z" \ + --add-host "host.containers.internal:host-gateway" \ + docker.io/prom/prometheus:latest >/dev/null + +# --- grafana (anonymous, no login) ---------------------------------------- +podman rm -f labsim-grafana >/dev/null 2>&1 || true +log "starting grafana on :$GRAFANA_PORT (anonymous auth — no password)" +podman run -d --name labsim-grafana --network "$NET" \ + -p "${GRAFANA_PORT}:3000" \ + -e GF_AUTH_ANONYMOUS_ENABLED=true \ + -e GF_AUTH_ANONYMOUS_ORG_ROLE=Admin \ + -e GF_AUTH_DISABLE_LOGIN_FORM=true \ + -e GF_AUTH_BASIC_ENABLED=false \ + -e GF_SECURITY_ALLOW_EMBEDDING=true \ + -e GF_USERS_DEFAULT_THEME=dark \ + -v "$SCRIPT_DIR/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro,Z" \ + docker.io/grafana/grafana:latest >/dev/null + +log "waiting for grafana..." +for _ in $(seq 1 40); do + if curl -sS --max-time 3 "http://127.0.0.1:${GRAFANA_PORT}/api/health" >/dev/null 2>&1; then + break + fi + sleep 3 +done + +echo +log "Grafana: http://localhost:${GRAFANA_PORT}/d/labsim-matrix (no login)" +log "Prometheus: http://localhost:${PROM_PORT}" +log "Exporter: http://localhost:${EXPORTER_PORT}/metrics" diff --git a/labsim/monitoring/grafana/provisioning/dashboards/dashboards.yml b/labsim/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..cafb391 --- /dev/null +++ b/labsim/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,9 @@ +apiVersion: 1 +providers: + - name: labsim + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + options: + path: /etc/grafana/provisioning/dashboards diff --git a/labsim/monitoring/grafana/provisioning/dashboards/labsim.json b/labsim/monitoring/grafana/provisioning/dashboards/labsim.json new file mode 100644 index 0000000..44e7a84 --- /dev/null +++ b/labsim/monitoring/grafana/provisioning/dashboards/labsim.json @@ -0,0 +1,59 @@ +{ + "uid": "labsim-matrix", + "title": "labsim — VLAN connectivity matrix", + "tags": ["labsim"], + "timezone": "browser", + "refresh": "10s", + "time": { "from": "now-30m", "to": "now" }, + "panels": [ + { + "type": "stat", + "title": "Reachable paths", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "targets": [ { "expr": "sum(labsim_reachable)", "refId": "A" } ], + "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", + "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 90 } ] } } } + }, + { + "type": "stat", + "title": "Blocked paths", + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }, + "targets": [ { "expr": "count(labsim_reachable == 0) or vector(0)", "refId": "A" } ], + "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", + "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 } ] } } } + }, + { + "type": "stat", + "title": "Sweep duration (s)", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, + "targets": [ { "expr": "labsim_sweep_seconds", "refId": "A" } ] + }, + { + "type": "stat", + "title": "Sweeps", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, + "targets": [ { "expr": "labsim_sweep_total", "refId": "A" } ] + }, + { + "type": "heatmap", + "title": "ICMP matrix (src → dst) — green = reachable", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 }, + "targets": [ { "expr": "labsim_reachable{proto=\"icmp\"}", + "legendFormat": "{{src}} → {{dst}}", "refId": "A" } ] + }, + { + "type": "state-timeline", + "title": "Every path over time — a firewall change shows up here immediately", + "gridPos": { "h": 12, "w": 24, "x": 0, "y": 14 }, + "targets": [ { "expr": "labsim_reachable", + "legendFormat": "{{proto}} {{src}} → {{dst}}", "refId": "A" } ], + "fieldConfig": { "defaults": { + "mappings": [ { "type": "value", "options": { + "0": { "text": "blocked", "color": "red", "index": 0 }, + "1": { "text": "ok", "color": "green", "index": 1 } } } ] } }, + "options": { "mergeValues": true, "showValue": "never" } + } + ], + "schemaVersion": 39, + "version": 1 +} diff --git a/labsim/monitoring/grafana/provisioning/datasources/prometheus.yml b/labsim/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..ca7795c --- /dev/null +++ b/labsim/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,7 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://labsim-prometheus:9090 + isDefault: true diff --git a/labsim/monitoring/prometheus.yml b/labsim/monitoring/prometheus.yml new file mode 100644 index 0000000..e602ca3 --- /dev/null +++ b/labsim/monitoring/prometheus.yml @@ -0,0 +1,9 @@ +# Scrapes the labsim connectivity exporter running on the host. +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: labsim + static_configs: + - targets: ['host.containers.internal:9101'] diff --git a/labsim/ovs.sh b/labsim/ovs.sh new file mode 100644 index 0000000..9d1c718 --- /dev/null +++ b/labsim/ovs.sh @@ -0,0 +1,161 @@ +#!/bin/bash +# Open vSwitch fabric for labsim — the "switch" the whole sim hangs off. +# +# Why OVS and not a Linux bridge: a Linux bridge cannot do LACP at all, and its +# VLAN support is awkward to drive from libvirt. OVS gives real 802.1Q access +# and trunk ports plus real LACP bonds, so a router VM can run the SAME bond0 + +# vif config as the production VP2440s instead of an approximation. +# +# Layout: +# ovs-labsim the switch +# ├─ vm ports access ports, tag= (micro VM per VLAN) +# ├─ hostv internal ports, tag= (host leg, for SSH) +# └─ lag-vyos LACP bond, trunk of all VLANs (router under test) +# shellcheck disable=SC2034 + +OVS_BR="${OVS_BR:-ovs-labsim}" +OVS_NET="${OVS_NET:-labsim-ovs}" # libvirt network wrapping the bridge +LAG_NAME="${LAG_NAME:-lag-vyos}" + +ovs() { sudo ovs-vsctl "$@"; } + +ovs_require() { + command -v ovs-vsctl >/dev/null 2>&1 || die "openvswitch not installed (dnf install openvswitch)" + systemctl is-active --quiet openvswitch || sudo systemctl start openvswitch \ + || die "could not start openvswitch" +} + +# All VLAN ids from the config, comma separated — used for trunk ports. +vlan_id_list() { + local ids=() + for entry in "${SELECTED[@]}"; do ids+=("${entry%%:*}"); done + (IFS=,; echo "${ids[*]}") +} + +ovs_up() { + ovs_require + ovs --may-exist add-br "$OVS_BR" + + # Host leg per VLAN: an OVS internal port carrying that VLAN's tag, given the + # .2 address. This is how you SSH to the VMs. It is deliberately NOT their + # default route (.1 is), so inter-VLAN tests exercise the router, not the + # host's routing table. + for entry in "${SELECTED[@]}"; do + IFS=: read -r vid _name prefix _real <<<"$entry" + local port="hostv${vid}" + ovs --may-exist add-port "$OVS_BR" "$port" tag="$vid" \ + -- set interface "$port" type=internal + sudo ip link set "$port" up 2>/dev/null || true + sudo ip addr replace "${prefix}.2/24" dev "$port" + done + + ovs_define_libvirt_net +} + +# A libvirt network that hands out OVS ports: one portgroup per VLAN (access) +# plus a trunk portgroup for the router. +ovs_define_libvirt_net() { + local pg="" ids + for entry in "${SELECTED[@]}"; do + IFS=: read -r vid name _p _r <<<"$entry" + pg+=" + + +" + done + + # Trunk: VLAN 1 native/untagged, everything else tagged — the production + # shape. libvirt expresses this declaratively via nativeMode='untagged' + # (see libvirt formatnetwork.html), so it does not need fixing up by hand. + # It also matters functionally: LACPDUs are untagged, and a trunk with no + # native VLAN has nowhere to put them. + local trunk=" + +" + for entry in "${SELECTED[@]}"; do + IFS=: read -r vid _n _p _r <<<"$entry" + if [ "$vid" = "1" ]; then + trunk+=" +" + else + trunk+=" +" + fi + done + trunk+=" + +" + + local xml=" + ${OVS_NET} + + + +${pg}${trunk}" + + if virsh_q net-info "$OVS_NET" >/dev/null 2>&1; then + virsh_q net-destroy "$OVS_NET" >/dev/null 2>&1 || true + virsh_q net-undefine "$OVS_NET" >/dev/null 2>&1 || true + fi + echo "$xml" | virsh_q net-define /dev/stdin >/dev/null + virsh_q net-start "$OVS_NET" >/dev/null + log "libvirt network $OVS_NET bound to $OVS_BR (access portgroups + trunk)" +} + +# Replace the router VM's two individual OVS ports with a single LACP bond. +# libvirt attaches each NIC separately; only ovs-vsctl can bond them, and the +# taps only exist once the VM is running — so this runs post-start. +ovs_bond_router() { + local vm="$1" + local taps + # NB: domiflist indents its rows, so anchor on the FIELD not the line — + # /^vnet/ silently matches nothing and the bond never gets built. + taps="$(virsh_q domiflist "$vm" 2>/dev/null | awk '$1 ~ /^vnet/ {print $1}')" + local count; count="$(echo "$taps" | grep -c .)" + [ "$count" -eq 2 ] || { warn "router $vm has $count tap(s), expected 2 — skipping bond"; return 1; } + + # Already bonded? (idempotent re-runs) + if ovs list-ports "$OVS_BR" 2>/dev/null | grep -qx "$LAG_NAME"; then + log "LACP bond $LAG_NAME already present" + return 0 + fi + + local t1 t2; t1="$(echo "$taps" | sed -n 1p)"; t2="$(echo "$taps" | sed -n 2p)" + log "bonding $t1 + $t2 into $LAG_NAME (LACP active, balance-tcp)" + ovs del-port "$OVS_BR" "$t1" 2>/dev/null || true + ovs del-port "$OVS_BR" "$t2" 2>/dev/null || true + + # bond_mode=balance-tcp is REQUIRED: OVS defaults a bond to active-backup, + # which does not speak LACP at all (confirmed on ovs-discuss). It is also the + # equivalent of VyOS's 802.3ad + layer2+3 hashing. + # + # lacp-fallback-ab breaks a genuine deadlock: OVS keeps members disabled + # until LACP negotiates, while the partner needs carrier before it will send + # LACPDUs. Falling back to active-backup brings the links up so negotiation + # can start. + # + # native-untagged + tag=1 carries the untagged LACPDUs and the management + # VLAN, matching production. libvirt's portgroup VLAN config does NOT apply + # here — the bond is a port libvirt never created — so set it inline. + local tagged; tagged="$(vlan_id_list | tr ',' '\n' | grep -vx 1 | paste -sd, -)" + ovs add-bond "$OVS_BR" "$LAG_NAME" "$t1" "$t2" \ + lacp=active bond_mode=balance-tcp \ + vlan_mode=native-untagged tag=1 trunks="$tagged" \ + -- set port "$LAG_NAME" other_config:lacp-time=fast \ + -- set port "$LAG_NAME" other_config:lacp-fallback-ab=true +} + +ovs_bond_status() { + echo "--- ovs bond ---" + sudo ovs-appctl bond/show "$LAG_NAME" 2>/dev/null | grep -E "bond_mode|lacp_status|^member|may_enable" || echo "(no bond)" + echo "--- lacp ---" + sudo ovs-appctl lacp/show "$LAG_NAME" 2>/dev/null | grep -E "status|aggregation key|^member|attached" || true +} + +ovs_down() { + virsh_q net-destroy "$OVS_NET" >/dev/null 2>&1 || true + virsh_q net-undefine "$OVS_NET" >/dev/null 2>&1 || true + if command -v ovs-vsctl >/dev/null 2>&1; then + ovs --if-exists del-br "$OVS_BR" 2>/dev/null || true + fi +} diff --git a/labsim/router-install.py b/labsim/router-install.py new file mode 100755 index 0000000..26352a4 --- /dev/null +++ b/labsim/router-install.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Drive the labsim VyOS router over its serial console. + +Three phases: + --phase live wait for the live system and log in + --phase install run `install image` unattended + --phase configure apply bond0 (LACP) + per-VLAN gateway addresses + +The installer prompt list is the same one the bastion's install driver answers +(src/bastion/src/templates/vyos-install.py.ts). Two of them are easy to miss and +both hang forever rather than failing: the reinstall-only "copy data to the new +image?", and "choose two disks for RAID-1 mirroring?" — every RAID prompt +defaults to YES. +""" +from __future__ import annotations + +import argparse +import sys +import time + +import pexpect + +PASSWORD = "vyos" +PROMPT = r"[\$#] $" + + +def console(vm: str, timeout: int = 60) -> pexpect.spawn: + c = pexpect.spawn(f"sudo virsh console {vm} --force", encoding="utf-8", timeout=timeout) + c.expect("Connected to domain", timeout=30) + return c + + +def login(c: pexpect.spawn, timeout: int = 300) -> None: + """Get to a shell prompt, whether we land at a login or an open session.""" + deadline = time.time() + timeout + while time.time() < deadline: + c.sendline("") + i = c.expect(["login:", PROMPT, pexpect.TIMEOUT], timeout=20) + if i == 0: + c.sendline("vyos") + c.expect("assword:", timeout=20) + c.sendline(PASSWORD) + j = c.expect([PROMPT, "incorrect", pexpect.TIMEOUT], timeout=30) + if j == 0: + return + elif i == 1: + return + raise SystemExit("timed out waiting for a VyOS shell") + + +def run(c: pexpect.spawn, cmd: str, timeout: int = 60) -> str: + c.sendline(cmd) + c.expect(PROMPT, timeout=timeout) + return c.before or "" + + +def phase_install(c: pexpect.spawn) -> None: + """Answer `install image` end to end.""" + rules: list[tuple[str, str]] = [ + (r"Would you like to continue\?", "yes"), + (r"What would you like to name this image\?", ""), + (r"Please confirm password for the .vyos. user:", PASSWORD), + (r"Please enter a password for the .vyos. user:", PASSWORD), + (r"What console should be used by default", "K"), + # every RAID variant defaults to YES — decline them all + (r"Would you like to [^?]*RAID-1 mirroring", "no"), + (r"Installation will delete all data on (?:the drive|both drives)\. Continue\?", "yes"), + (r"Which one should be used for installation\?", "/dev/vda"), + (r"Would you like to use all the free space on the drive\?", "yes"), + (r"Which file would you like as boot config\?", "1"), + # reinstall-only; unanswered it blocks on stdin until the world ends + (r"Would you like to copy data to the new image\?", "yes"), + (r"From which image would you like to save config information\?", "1"), + ] + patterns = [r for r, _ in rules] + [r"The image installed successfully", + r"Unable to install VyOS", pexpect.TIMEOUT] + + c.sendline("install image") + for _ in range(60): + i = c.expect(patterns, timeout=180) + if i < len(rules): + c.sendline(rules[i][1]) + continue + if i == len(rules): + print(" installer: success") + return + if i == len(rules) + 1: + raise SystemExit("installer reported failure") + raise SystemExit("installer went quiet (unanswered prompt?)") + raise SystemExit("installer exceeded expected prompt count") + + +def phase_configure(c: pexpect.spawn, vlans: list[tuple[str, str, str]]) -> None: + """bond0 over eth0+eth1 with LACP, then a gateway address per VLAN.""" + # Production shape: VLAN 1 (management) is the NATIVE/untagged VLAN on the + # bond, everything else is a tagged vif. This matters beyond fidelity — + # LACPDUs are untagged, so a trunk with no native VLAN has nowhere to put + # them and the bond never negotiates. + native = [v for v in vlans if v[0] == "1"] + tagged = [v for v in vlans if v[0] != "1"] + + cmds = [ + "configure", + "set interfaces bonding bond0 mode '802.3ad'", + "set interfaces bonding bond0 hash-policy 'layer2+3'", + "set interfaces bonding bond0 lacp-rate 'fast'", + "set interfaces bonding bond0 member interface 'eth0'", + "set interfaces bonding bond0 member interface 'eth1'", + "set service ssh port '22'", + "set system login user vyos authentication plaintext-password 'vyos'", + ] + for vid, name, prefix in native: + cmds.append(f"set interfaces bonding bond0 address '{prefix}.1/24'") + cmds.append(f"set interfaces bonding bond0 description '{name} (native)'") + for vid, name, prefix in tagged: + cmds.append(f"set interfaces bonding bond0 vif {vid} address '{prefix}.1/24'") + cmds.append(f"set interfaces bonding bond0 vif {vid} description '{name}'") + cmds += ["commit", "save", "exit"] + + for cmd in cmds: + out = run(c, cmd, timeout=180) + low = out.lower() + if "invalid" in low or "syntax error" in low or "commit failed" in low: + print(f" !! {cmd}\n{out.strip()[-300:]}") + raise SystemExit(f"config command rejected: {cmd}") + print(" config committed and saved") + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--vm", required=True) + ap.add_argument("--phase", required=True, choices=["live", "install", "configure"]) + ap.add_argument("--vlans", default="", help="space separated vid:name:prefix:real entries") + args = ap.parse_args() + + c = console(args.vm) + try: + login(c) + if args.phase == "live": + print(" live system reachable") + elif args.phase == "install": + phase_install(c) + else: + vlans = [] + for entry in args.vlans.split(): + parts = entry.split(":") + if len(parts) >= 3: + vlans.append((parts[0], parts[1], parts[2])) + if not vlans: + raise SystemExit("no VLANs passed to configure") + phase_configure(c, vlans) + return 0 + finally: + try: + c.sendline("") + c.close(force=True) + except Exception: + pass + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/labsim/router-up.sh b/labsim/router-up.sh new file mode 100755 index 0000000..fd88124 --- /dev/null +++ b/labsim/router-up.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# Add the VyOS router under test to labsim. +# +# Mirrors the production VP2440 pair: TWO NICs bonded with LACP carrying a +# trunk of every VLAN, then bond0. sub-interfaces holding the .1 gateway +# address on each. That is the same config shape the real firewalls run, so a +# rule tested here means something. +# +# NIC model is e1000e, NOT virtio, and that is load-bearing: with virtio the +# guest's bonding driver reports its slaves "MII Status: down" despite +# carrier=1 and never emits a single LACPDU, so the bond sits in +# AD_STATE_DEFAULTED forever. Known issue — see the netdev thread "bonding +# (IEEE 802.3ad) not working with qemu/virtio"; e1000e fixes it with no other +# change. 802.3ad also requires the MII link monitor, which virtio cannot back. +# +# host OVS "switch" VyOS VM +# hostv (.2) ──────── ovs-labsim ──── lag-vyos ═════ eth0 + eth1 +# (tagged) (LACP, trunk) └─ bond0. = .1 +# +# Usage: ./router-up.sh build + install + configure +# ./router-up.sh --status show bond/LACP + interface state +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/lib.sh" +source "$SCRIPT_DIR/ovs.sh" + +ROUTER_VM="${ROUTER_VM:-labsim-vyos}" +ROUTER_MEM="${ROUTER_MEM:-2048}" +ROUTER_CPUS="${ROUTER_CPUS:-2}" +ROUTER_DISK_GB="${ROUTER_DISK_GB:-8}" +VYOS_ISO="${VYOS_ISO:-$IMG_DIR/vyos.iso}" +VYOS_CACHE="/var/lib/libvirt/images/lab-pxe-cache" + +selected_vlans + +if [ "${1:-}" = "--status" ]; then + ovs_bond_status + echo "--- vyos gateway addresses (probed from each host leg) ---" + for entry in "${SELECTED[@]}"; do + IFS=: read -r vid _n prefix _r <<<"$entry" + printf ' vlan %-5s %-16s ' "$vid" "${prefix}.1" + ping -c1 -W2 "${prefix}.1" >/dev/null 2>&1 && echo up || echo down + done + exit 0 +fi + +ovs_require + +# --- ISO ------------------------------------------------------------------ +if [ ! -f "$VYOS_ISO" ]; then + # Reuse the bastion's cached nightly if it is already on this box. + if [ -f "$VYOS_CACHE/vyos.iso" ]; then + log "reusing cached VyOS ISO" + sudo cp "$VYOS_CACHE/vyos.iso" "$VYOS_ISO" + else + log "resolving latest VyOS nightly ISO..." + url="$(curl -sSL https://api.github.com/repos/vyos/vyos-nightly-build/releases/latest \ + | python3 -c "import json,sys;print(next(a['browser_download_url'] for a in json.load(sys.stdin)['assets'] if a['name'].endswith('generic-amd64.iso')))")" + log "downloading $url" + sudo curl -sSL --max-time 1800 -o "$VYOS_ISO" "$url" + fi +fi +[ -f "$VYOS_ISO" ] || die "no VyOS ISO at $VYOS_ISO" + +# --- VM ------------------------------------------------------------------- +if virsh_q dominfo "$ROUTER_VM" >/dev/null 2>&1; then + log "router VM $ROUTER_VM exists" + virsh_q start "$ROUTER_VM" >/dev/null 2>&1 || true +else + log "creating router VM $ROUTER_VM (2 NICs on the trunk, for LACP)" + sudo qemu-img create -q -f qcow2 "$IMG_DIR/${ROUTER_VM}.qcow2" "${ROUTER_DISK_GB}G" >/dev/null + + # Two trunk NICs — OVS bonds them after boot (libvirt cannot create bonds). + sudo virt-install \ + --connect "$LIBVIRT_URI" \ + --name "$ROUTER_VM" \ + --memory "$ROUTER_MEM" --vcpus "$ROUTER_CPUS" \ + --disk "path=$IMG_DIR/${ROUTER_VM}.qcow2,format=qcow2,bus=virtio" \ + --disk "path=$VYOS_ISO,device=cdrom,readonly=on" \ + --network "network=$OVS_NET,portgroup=trunk,model=e1000e,trustGuestRxFilters=yes" \ + --network "network=$OVS_NET,portgroup=trunk,model=e1000e,trustGuestRxFilters=yes" \ + --boot cdrom,hd \ + --os-variant debian12 \ + --graphics none --noautoconsole --import >/dev/null +fi + +log "waiting for the live system to boot (VyOS live login)..." +python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase live || die "live boot failed" + +log "installing VyOS to disk (unattended over the console)..." +python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase install || die "install failed" + +# Boot the INSTALLED system from here on. Without this the VM was created with +# --boot cdrom,hd and every restart re-runs the ISO, so the live system comes +# back with no config and every `commit; save` silently evaporates. +log "switching boot to disk and ejecting the install media..." +virsh_q destroy "$ROUTER_VM" >/dev/null 2>&1 || true +sleep 2 +sudo virt-xml "$ROUTER_VM" --edit --boot hd >/dev/null +sudo virt-xml "$ROUTER_VM" --remove-device --disk device=cdrom >/dev/null 2>&1 || true +virsh_q start "$ROUTER_VM" >/dev/null +sleep 10 + +# Bond the taps only now: they are recreated by the restart above, so bonding +# before this would bond stale interfaces. +ovs_bond_router "$ROUTER_VM" + +log "applying router config (bond0 LACP + VLAN gateways)..." +python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase configure \ + --vlans "$(printf '%s\n' "${SELECTED[@]}" | tr '\n' ' ')" || die "configure failed" + +log "waiting for LACP to negotiate..." +for _ in $(seq 1 30); do + if sudo ovs-appctl lacp/show "$LAG_NAME" 2>/dev/null | grep -q "current attached"; then + log "LACP negotiated"; break + fi + sleep 5 +done + +echo +ovs_bond_status +echo +log "router is up. Check reachability with: $SCRIPT_DIR/labsim-matrix.py --watch 2" diff --git a/labsim/vlans.conf b/labsim/vlans.conf new file mode 100644 index 0000000..2b6712b --- /dev/null +++ b/labsim/vlans.conf @@ -0,0 +1,21 @@ +# Lab network simulation — VLAN map. +# +# Mirrors the real UniFi topology (same VLAN IDs, same roles) but with +# deliberately DIFFERENT IP ranges so nothing here can collide with, or be +# confused for, production. The sim subnet always encodes the VLAN id: +# +# 172.31..0/24 +# +# Per-subnet address plan (same shape on every VLAN): +# .1 gateway under test (VyOS/router VM — not created by default) +# .2 host bridge (how you SSH in from this workstation) +# .10 the micro VM for this VLAN +# .254 VRRP VIP (reserved, mirrors production) +# +# Format: vlan_id:name:sim_subnet_prefix:real_subnet(for reference) +1:management:172.31.1:192.168.1.0/24 +2:k8s:172.31.2:192.168.8.0/23 +3:kvm:172.31.3:192.168.3.0/24 +9:private:172.31.9:10.0.9.0/23 +10:lot:172.31.10:10.0.0.0/23 +200:roomates:172.31.200:192.168.2.0/24