#!/bin/bash # Build a fake Hurricane Electric 6in4 endpoint inside labsim. # # WHY THIS EXISTS # The production HE tunnel was never rehearsed. The override that introduced it # says so in its own reason text -- "the sim has no public IPv4 and no HE # endpoint, so there is nothing to tunnel to" -- and that gap is why IPv6 was # the one half of the WAN story with no matrix behind it. Then the WAN became # HA and IPv6 did not follow, which nobody caught, because nothing tests it. # # THE PROBLEM THIS HAD TO SOLVE # The sim's two WANs are isolated islands. Verified: # # 203.0.113.1 from 203.0.113.107 : OK <- 10 gig analogue # 203.0.113.1 from 198.51.100.137 : unreachable # 198.51.100.1 from 198.51.100.137 : OK <- PPPoE analogue # 198.51.100.1 from 203.0.113.107 : unreachable # # A 6in4 tunnel has ONE remote address, and production never changes it -- so an # endpoint reachable over only one WAN could not rehearse the case that matters # most: the WITHIN-box fall back from the 10 gig to PPPoE, where he-tunnel-follow # re-points the tunnel and calls the HE API. That is exactly where the 2026-09-06 # near-miss lived. # # So the sim needs a minimal "internet": both ISP boxes already sit on the # libvirt default network (192.168.122.0/24) and both forward, so that becomes # the backbone, and HE lives on a single address behind it, reachable over # either WAN. No new VMs, no new networks. # # 192.0.2.10 "HE" -- on isp-dhcp, reached from the PPPoE island via # 192.168.122.136, and directly from the 10 gig island # # EVERYTHING HERE IS KERNEL-LEVEL, not VyOS config. The ISP boxes are scaffold, # not the thing under test: `ip` commands leave no config to drift, no commit to # fail, and a reboot cleans up. The ROUTER side is deliberately the opposite -- # it goes through real VyOS config, because "will VyOS commit a tunnel whose # source-address does not exist on this box?" is one of the questions. # # ./labsim-he-endpoint.sh up build it # ./labsim-he-endpoint.sh down tear it down # ./labsim-he-endpoint.sh status what is live # ./labsim-he-endpoint.sh calls how many times the HE API was called # ./labsim-he-endpoint.sh point IP point the endpoint by hand (sim bookkeeping) set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" DHCP_ISP="${DHCP_ISP:-192.168.122.136}" # owns the 10 gig segment, hosts "HE" PPPOE_ISP="${PPPOE_ISP:-192.168.122.63}" # owns the PPPoE segment PW="${VYOS_PW:-vyos}" # TEST-NET-1 for the endpoint, and the documentation prefix for v6. Production # uses 2001:470:187e::/48 from HE; the sim mirrors its SHAPE # (2001:db8:187e:::/64) so the scheme is exercised, not just the tunnel. HE_ADDR="${HE_ADDR:-192.0.2.10}" HE_LINK6="${HE_LINK6:-2001:db8:1f1c:f6::1}" # HE side of the tunnel /64 RT_LINK6="${RT_LINK6:-2001:db8:1f1c:f6::2}" # router side SITE6="${SITE6:-2001:db8:187e::/48}" # routed to the router side TENGIG_NET="${TENGIG_NET:-203.0.113.0/24}" # Any valid address that will never be a router WAN -- see the tunnel creation # below for why this cannot be 0.0.0.0. PLACEHOLDER_REMOTE="${PLACEHOLDER_REMOTE:-203.0.113.1}" PPPOE_NET="${PPPOE_NET:-198.51.100.0/24}" SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password) dhcp_isp() { timeout 40 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$DHCP_ISP" "$@" 2>/dev/null; } pppoe_isp() { timeout 40 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$PPPOE_ISP" "$@" 2>/dev/null; } log() { printf '\033[0;36m[he-sim]\033[0m %s\n' "$*"; } die() { printf '\033[0;31m[he-sim]\033[0m %s\n' "$*" >&2; exit 1; } # --- the stub tunnelbroker API --------------------------------------------- # HE's real endpoint is a dyndns-style updater that re-points the tunnel's remote # address. This is that, in 40 lines, so the sim can exercise the HE-SIDE half of # a failover -- the half production can never safely test. # # It logs every call to /run/he-sim-api.log, which is what lets the matrix assert # the invariant that matters: a ROUTER-level failover must call this ZERO times, # because the 10 gig address follows the cloned MAC to the other box unchanged. API_PY=' import http.server, subprocess, urllib.parse, datetime, sys, re TUN = "he-sim" LOCAL = sys.argv[1] if len(sys.argv) > 1 else "192.0.2.10" V6_LOCAL = sys.argv[2] if len(sys.argv) > 2 else "2001:db8:1f1c:f6::1/64" V6_PEER = sys.argv[3] if len(sys.argv) > 3 else "2001:db8:1f1c:f6::2" SITE6 = sys.argv[4] if len(sys.argv) > 4 else "2001:db8:187e::/48" LOG = "/run/he-sim-api.log" def note(msg): with open(LOG, "a") as f: f.write("%s %s\n" % (datetime.datetime.now().isoformat(timespec="seconds"), msg)) def current_remote(): out = subprocess.run(["ip", "tunnel", "show", TUN], capture_output=True, text=True).stdout m = re.search(r"remote ([0-9.]+)", out) return m.group(1) if m else None class H(http.server.BaseHTTPRequestHandler): def reply(self, body): b = body.encode() self.send_response(200) self.send_header("Content-Type", "text/plain") self.send_header("Content-Length", str(len(b))) self.end_headers() self.wfile.write(b) def handle_update(self, qs): q = urllib.parse.parse_qs(qs) ip = (q.get("myip") or [""])[0] if not ip: note("REFUSED no myip"); return self.reply("nohost") cur = current_remote() if cur == ip: note("nochg %s" % ip); return self.reply("nochg %s" % ip) rc = subprocess.run(["ip", "tunnel", "change", TUN, "mode", "sit", "local", LOCAL, "remote", ip], capture_output=True, text=True) if rc.returncode != 0: # Recreate rather than report a success we did not achieve. This is # the path that a multipoint tunnel takes; keeping it means a stub # that cannot silently no-op. note("change failed (%s) -- recreating" % rc.stderr.strip()) subprocess.run(["ip", "tunnel", "del", TUN], check=False) subprocess.run(["ip", "tunnel", "add", TUN, "mode", "sit", "local", LOCAL, "remote", ip, "ttl", "64"], check=False) subprocess.run(["ip", "link", "set", TUN, "up", "mtu", "1480"], check=False) subprocess.run(["ip", "-6", "addr", "replace", V6_LOCAL, "dev", TUN], check=False) subprocess.run(["ip", "-6", "route", "replace", SITE6, "via", V6_PEER, "dev", TUN], check=False) # Verify rather than trust: read the remote back. got = current_remote() if got != ip: note("FAILED to point %s at %s (reads %s)" % (TUN, ip, got)) return self.reply("dnserr") note("good %s (was %s)" % (ip, cur)) self.reply("good %s" % ip) def do_GET(self): u = urllib.parse.urlparse(self.path) if u.path == "/nic/update": self.handle_update(u.query) else: self.reply("badauth") def do_POST(self): n = int(self.headers.get("Content-Length") or 0) self.handle_update(self.rfile.read(n).decode()) def log_message(self, *a): pass http.server.HTTPServer(("0.0.0.0", 80), H).serve_forever() ' up() { log "backbone: teaching each ISP box how to reach the other island" # isp-dhcp owns HE and must be able to answer a router that arrived over # PPPoE, so it needs a route back to that island via the backbone. dhcp_isp "sudo ip route replace $PPPOE_NET via $PPPOE_ISP" \ || die "could not add the PPPoE-island route on isp-dhcp" # isp-pppoe must forward its clients' traffic for HE across the backbone. pppoe_isp "sudo ip route replace $HE_ADDR/32 via $DHCP_ISP" \ || die "could not add the HE route on isp-pppoe" log "HE endpoint: $HE_ADDR on isp-dhcp" # A dummy interface, not a loopback alias: `ip tunnel` wants a real local # address and a dummy is the honest way to have one that is not tied to # either WAN segment -- which is the point, HE is neither. dhcp_isp "sudo modprobe dummy 2>/dev/null; sudo ip link add he-lo type dummy 2>/dev/null; sudo ip link set he-lo up; sudo ip addr replace $HE_ADDR/32 dev he-lo" log "6in4 tunnel he-sim: local $HE_ADDR, remote set by the API on demand" # A PLACEHOLDER remote, not 0.0.0.0. A sit tunnel created with `remote any` # is multipoint (6rd-shaped), and `ip tunnel change` then refuses to convert # it to point-to-point -- "add tunnel he-sim failed: Invalid argument". The # API's update silently did nothing, so the stub logged "good", production's # he-tunnel-follow logged success, and the tunnel still pointed nowhere. # Created point-to-point from the start, `change` works. dhcp_isp "sudo ip tunnel del he-sim 2>/dev/null; sudo ip tunnel add he-sim mode sit local $HE_ADDR remote $PLACEHOLDER_REMOTE ttl 64; sudo ip link set he-sim up mtu 1480; sudo ip -6 addr replace $HE_LINK6/64 dev he-sim; sudo ip -6 route replace $SITE6 via $RT_LINK6 dev he-sim; sudo sysctl -qw net.ipv6.conf.all.forwarding=1" log "stub tunnelbroker API on $HE_ADDR:80" printf '%s' "$API_PY" | dhcp_isp "cat > /tmp/he-sim-api.py" # Launch from a script FILE, not an inline ssh command. The remote login # shell is vbash, and a multi-line inlined `sudo setsid nohup ... &` through # it silently ran nothing at all: no process, no /run/he-sim-api.out, and a # `pgrep -f he-sim-api.py` status check that reported "running" because the # unbracketed pattern matched its OWN ssh command line. Two self-inflicted # illusions stacked on each other. # # This repo already learned this once -- see isp_session_control() in # labsim-pppoe-ha-test.sh, where driving vbash inline made every iteration # of the T4 matrix test the wrong policy while printing the right one. printf '%s\n' \ '#!/bin/sh' \ '# started detached so it outlives the ssh session that launched it' \ 'pkill -f "he-sim-api[.]py" 2>/dev/null' \ 'rm -f /run/he-sim-api.log /run/he-sim-api.out' \ "exec setsid python3 /tmp/he-sim-api.py $HE_ADDR '$HE_LINK6/64' $RT_LINK6 $SITE6 >/run/he-sim-api.out 2>&1 /tmp/he-sim-start.sh" dhcp_isp "chmod +x /tmp/he-sim-start.sh && sudo /tmp/he-sim-start.sh" >/dev/null # Poll for the bind rather than sleeping a guessed interval. local i probe="" for i in $(seq 1 10); do sleep 1 probe="$(dhcp_isp "curl -sS --max-time 3 'http://$HE_ADDR/nic/update' 2>&1")" [ "$probe" = nohost ] && break done case "$probe" in nohost) log "API answering (returned 'nohost' for a call with no myip -- correct)" ;; *) die "stub API not answering on $HE_ADDR:80 (got: ${probe:-})" ;; esac log "up. Router side is NOT configured by this script -- that is real VyOS" log "config and belongs to the matrix; see labsim-ipv6-ha-test.sh --setup." } down() { log "tearing down" dhcp_isp "sudo pkill -f 'he-sim-api[.]py' 2>/dev/null; sudo ip tunnel del he-sim 2>/dev/null; sudo ip link del he-lo 2>/dev/null; sudo ip route del $PPPOE_NET via $PPPOE_ISP 2>/dev/null" >/dev/null pppoe_isp "sudo ip route del $HE_ADDR/32 via $DHCP_ISP 2>/dev/null" >/dev/null log "down" } status() { printf ' HE address : %s\n' "$(dhcp_isp "ip -4 -br addr show he-lo 2>/dev/null | awk '{print \$3}'" || echo '')" printf ' he-sim : %s\n' "$(dhcp_isp "ip tunnel show he-sim 2>/dev/null" || echo '')" printf ' he-sim v6 : %s\n' "$(dhcp_isp "ip -6 -br addr show he-sim 2>/dev/null | awk '{print \$3}'" || echo '-')" # Bracketed so the pattern cannot match the ssh command line carrying it -- # unbracketed, this reported "running" while nothing was listening at all. printf ' API : %s\n' "$(dhcp_isp "pgrep -f 'he-sim-api[.]py' >/dev/null && echo running || echo stopped")" printf ' API bound : %s\n' "$(dhcp_isp "curl -sS --max-time 3 'http://$HE_ADDR/nic/update' 2>/dev/null" || echo 'NOT ANSWERING')" printf ' API calls : %s\n' "$(dhcp_isp "grep -c . /run/he-sim-api.log 2>/dev/null" || echo 0)" printf ' route back : %s\n' "$(dhcp_isp "ip route show $PPPOE_NET 2>/dev/null" || echo '')" } # Count of endpoint-CHANGING calls. `nochg` does not count: production's # he-tunnel-follow re-sends the same address happily and HE treats it as a # no-op, so only a real move is evidence that something re-pointed the tunnel. calls() { dhcp_isp "grep -c ' good ' /run/he-sim-api.log 2>/dev/null" | tr -d ' \n'; } # Point the endpoint at an address WITHOUT going through the API, and without # counting as an API call. # # Needed because rebuilding the sim endpoint resets its remote, while the # routers' he-tunnel-follow still reads "in sync" and therefore never re-asserts # -- it only calls HE when its own LOCAL source changes, and has no way to learn # that the far end drifted. The real HE does not forget, so this is sim # bookkeeping, not a behaviour production needs. Keeping it out of the call # counter is the point: the matrix asserts on that counter. point() { local ip="$1" [ -n "$ip" ] || die "usage: $0 point " dhcp_isp "sudo ip tunnel change he-sim mode sit local $HE_ADDR remote $ip 2>/dev/null \ || { sudo ip tunnel del he-sim 2>/dev/null; sudo ip tunnel add he-sim mode sit local $HE_ADDR remote $ip ttl 64; sudo ip link set he-sim up mtu 1480; sudo ip -6 addr replace $HE_LINK6/64 dev he-sim; sudo ip -6 route replace $SITE6 via $RT_LINK6 dev he-sim; }" local got got="$(dhcp_isp "ip tunnel show he-sim 2>/dev/null | sed -nE 's/.* remote ([0-9.]+).*/\\1/p'" | tr -d ' \n')" [ "$got" = "$ip" ] || die "endpoint still points at ${got:-nothing}, wanted $ip" log "endpoint now points at $ip" } case "${1:-status}" in up) up ;; down) down ;; status) status ;; calls) calls; echo ;; point) point "${2:-}" ;; *) die "usage: $0 {up|down|status|calls}" ;; esac