#!/bin/bash # Does IPv6 follow VRRP mastership, and does it do so WITHOUT touching HE? # # The WAN became HA on 2026-09-06 and IPv6 did not follow it. Nothing caught # that, because nothing tested it: wan-drill measured IPv4 only, and PPPOE-HA.md # recorded "IPv6 stayed up at 15.5ms" from a reading taken outside the failover # window. This is the matrix that would have caught it. # # TWO INVARIANTS, checked independently of any individual test: # # 1. At most ONE router ever has a live tunnel. An UP tunnel on a box that # does not own the source address is not harmless -- see V4, it is a # BLACKHOLE that will happily attract the v6 default route. # 2. A ROUTER-level failover calls the HE API ZERO times. The 10 gig address # is bound to a cloned MAC and follows the VIP to the other box unchanged, # so there is nothing to tell HE. A non-zero count means something # re-pointed the tunnel at a PPPoE address -- which the ISP re-issues on # every dial, so it would be wrong within minutes. # # KNOWN SIM GAP, 2026-09-06 -- read this before believing a v6_online failure. # The MECHANISM is proven here: tun0 up on the master and down on the backup, # radvd following mastership, he-tunnel-follow's master guard, its hysteresis, # its HE call and the 1480->1472 MTU switch, and VLAN 9 hosts autoconfiguring # from the RA (observed: real SLAAC traffic from 2001:db8:187e:9::/64 arriving # at the endpoint encapsulated). # # What is NOT yet proven is the end-to-end v6 DATAPATH, because the sim's # "internet" is asymmetric: 6in4 packets from the PPPoE island reach the # endpoint with an outer source of 192.168.122.1 -- the libvirt host's NAT -- # so HE's replies go back to the tunnel remote by a path with no NAT state and # are lost. Forward works, return does not. # # That is a topology fault in the scaffold, not in the thing under test. Fixing # it means giving the two ISP islands a real transit path that does not traverse # libvirt NAT. Until then, treat v6_online failures as UNPROVEN rather than as # evidence the design is wrong -- and do not let that ambiguity leak into # production sign-off, which is exactly the mistake wan-drill made by asserting # "IPv6 stayed up" from a reading taken outside the window. # # Requires the fake HE endpoint: ./labsim-he-endpoint.sh up # # ./labsim-ipv6-ha-test.sh --setup configure the router side (once) # ./labsim-ipv6-ha-test.sh --list # ./labsim-ipv6-ha-test.sh V1 # ./labsim-ipv6-ha-test.sh --all set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" R1="${R1:-172.31.1.252}"; R2="${R2:-172.31.1.253}" VIP="${VIP:-172.31.1.1}" LAN9="${LAN9:-172.31.9.10}" # VLAN 9 client, for RA tests HE_ADDR="${HE_ADDR:-192.0.2.10}" HE_LINK6="${HE_LINK6:-2001:db8:1f1c:f6::1}" RT_LINK6="${RT_LINK6:-2001:db8:1f1c:f6::2}" V9_PREFIX="${V9_PREFIX:-2001:db8:187e:9}" # Mirrors production: one MAC, one lease, whichever router holds the VIP. WAN_MAC="${WAN_MAC:-02:9f:c2:12:9b:4f}" PW="${VYOS_PW:-vyos}"; LANPW="${LANPW:-labsim}" EVID="$SCRIPT_DIR/ipv6-ha-evidence" HE="$SCRIPT_DIR/labsim-he-endpoint.sh" SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password) r() { timeout 45 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; } # The Alpine LAN VMs do not offer `password` auth -- reusing the routers' option # set makes ssh exit 255 before running anything, which reads as "the network is # broken". Same trap as lan() in labsim-pppoe-ha-test.sh. LAN_SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=6) lan9() { timeout 30 sshpass -p "$LANPW" ssh "${LAN_SSH[@]}" "root@$LAN9" "$@" 2>/dev/null; } log() { printf '\033[36m==>\033[0m %s\n' "$*"; } pass() { printf ' \033[32mPASS\033[0m %s\n' "$*"; } fail() { printf ' \033[31mFAIL\033[0m %s\n' "$*"; FAILED=$((FAILED+1)); } warn() { printf ' \033[33mWARN\033[0m %s\n' "$*"; } FAILED=0 # --- observations ---------------------------------------------------------- # A destroyed or unreachable router is emphatically NOT holding the tunnel, but # ssh returns an EMPTY string, and `[ "" = 0 ]` is false -- the pppoe matrix hung # on exactly this waiting for a dead box to report zero. Default everything to 0. tun_state() { local v; v="$(r "$1" "ip -br link show tun0 2>/dev/null | awk '{print \$2}'" | tr -d ' \n')"; echo "${v:-absent}"; } tun_src() { r "$1" 'ip tunnel show tun0 2>/dev/null | sed -nE "s/.* local ([0-9.]+).*/\1/p"' | tr -d ' \n'; } # Whichever router currently holds the 10 gig lease -- under the cloned MAC only # one ever does. Read rather than assumed: the address changes when the MAC does. tengig_addr() { local h a; for h in "$R1" "$R2"; do a="$(r "$h" 'ip -4 addr show bond0.53 2>/dev/null | sed -nE "s/.*inet ([0-9.]+).*/\1/p"' | tr -d ' \n')" [ -n "$a" ] && { echo "$a"; return; } done; echo ""; } tun_mtu() { local v; v="$(r "$1" 'cat /sys/class/net/tun0/mtu 2>/dev/null' | tr -d ' \n')"; echo "${v:-0}"; } holder() { for h in "$R1" "$R2"; do [ "$(r "$h" "ip -4 -o addr show | grep -c ' ${VIP}/'" | tr -d ' \n')" != 0 ] \ && { echo "$h"; return; }; done; echo none; } # Ask the ROUTER, not a client: a client can be answered by the wrong path. v6_online() { [ "$(r "$1" "ping -6 -c1 -W3 $HE_LINK6 >/dev/null 2>&1 && echo y" | tr -d ' \n')" = y ]; } radvd_on() { [ "$(r "$1" 'systemctl is-active radvd 2>/dev/null' | tr -d ' \n')" = active ]; } he_calls() { timeout 60 "$HE" calls | tr -d ' \n'; } # How many routers have a tunnel that is UP. Invariant 1's numerator. tun_holders() { local n=0 h; for h in "$R1" "$R2"; do case "$(tun_state "$h")" in UP|UNKNOWN) n=$((n+1)) ;; esac done; echo "$n"; } check_invariants() { local t ok=0 t="$(tun_holders)" [ "${t:-0}" -le 1 ] || { fail "INVARIANT: $t routers have tun0 up"; ok=1; } return $ok } save_evidence() { local name="$1"; local d="$EVID/$name"; mkdir -p "$d" { echo "=== $(date -Is) ===" echo "--- HE endpoint ---"; timeout 60 "$HE" status for h in "$R1" "$R2"; do echo "--- $h ---" r "$h" 'sudo /config/vrrp-wan-reconcile --status 2>/dev/null ip -br link show tun0 2>/dev/null; ip tunnel show tun0 2>/dev/null ip -6 route show default; systemctl is-active radvd sudo journalctl -t vrrp-wan -t he-tunnel-follow -n 10 --no-pager' done; } > "$d/state.txt" 2>&1 log "evidence -> ipv6-ha-evidence/$name/" } # --- setup ------------------------------------------------------------------ # Applied through REAL VyOS config, unlike the ISP-side scaffold, because "will # VyOS accept this?" is one of the questions being asked. vyos_apply() { # host, then set-lines on stdin local h="$1" { printf '#!/bin/vbash\nsource /opt/vyatta/etc/functions/script-template\nconfigure\n' cat printf 'commit\nsave\nexit\n' } | timeout 90 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$h" \ 'cat > /tmp/v6-apply.sh && chmod +x /tmp/v6-apply.sh && sudo /tmp/v6-apply.sh' 2>&1 | tail -3 # `vbash -c` never starts a config session and commit fails to stderr, which # a helper like this discards -- the T4 matrix in labsim-pppoe-ha-test.sh ran # its whole policy sweep against the default while printing the mode it # thought it was testing. Always read the value back. } setup() { log "--setup: router-side IPv6, both routers" # THE CLONED MAC, first and on its own. Production pins f0:9f:c2:12:9b:4f on # vif 53 so the 10 gig lease follows the VIP and the tunnel source is the # SAME address on either box. The sim never had it -- each router took its # own lease -- so the sim could not reproduce the one property the whole # IPv6-HA design leans on, and invariant 2 would have been untestable here. local h pref v9 for h in "$R1" "$R2"; do log " $h: pinning the cloned WAN MAC $WAN_MAC" vyos_apply "$h" < /tmp/he-secrets && sudo install -o root -g vyattacfg -m 0640 /tmp/he-secrets /config/he-secrets' >/dev/null done for h in "$R1" "$R2"; do [ "$h" = "$R1" ] && { pref=high; v9=1; } || { pref=low; v9=2; } log " $h (RA preference $pref, bond0.9 ::${v9})" vyos_apply "$h" < expecting $to (HE calls so far: ${before:-0})" t0=$(date +%s) r "$from" 'sudo mkdir -p /run/vrrp-wan && sudo touch /run/vrrp-wan/force-fault' local took="" for i in $(seq 1 36); do sleep 5 [ "$(holder)" = "$to" ] && v6_online "$to" && { took=$(( $(date +%s) - t0 )); break; } done [ -n "$took" ] && pass "IPv6 reached $to in ${took}s" \ || fail "IPv6 never followed to $to within 180s" case "$(tun_state "$from")" in DOWN|absent) pass "$from released its tunnel" ;; *) fail "$from still has tun0 $(tun_state "$from") -- blackhole risk" ;; esac after="$(he_calls)" # THE invariant this test exists for. [ "${after:-0}" = "${before:-0}" ] \ && pass "HE API not called (${after:-0} total) -- the address followed the MAC" \ || fail "HE API called $(( ${after:-0} - ${before:-0} )) time(s) during a ROUTER failover" check_invariants save_evidence V1-clean-failover r "$from" 'sudo rm -f /run/vrrp-wan/force-fault' sleep 40 } V2() { # 10 gig down on the master: HE must be told, exactly once log "V2 10 gig down: he-tunnel-follow re-points the tunnel and tells HE" local h before after mtu i ok=no h="$(holder)"; before="$(he_calls)" local tengig; tengig="$(tengig_addr)" r "$h" 'sudo ip link set bond0.53 down' # he-tunnel-follow runs on a 1m task-scheduler with a 2-tick hysteresis, so # allow well past 2 minutes before calling it a failure. for i in $(seq 1 30); do sleep 10 [ "$(tun_src "$h")" != "$tengig" ] && { ok=yes; break; } done [ "$ok" = yes ] && pass "tunnel source moved to $(tun_src "$h") after $((i*10))s" \ || fail "tunnel source never left the 10 gig address" mtu="$(tun_mtu "$h")" [ "$mtu" = 1472 ] && pass "MTU dropped to 1472 for the PPPoE path" \ || fail "MTU is $mtu, want 1472 -- large transfers will hang" after="$(he_calls)" [ "$(( ${after:-0} - ${before:-0} ))" -ge 1 ] \ && pass "HE API called $(( ${after:-0} - ${before:-0} )) time(s), as it must be here" \ || fail "HE was never told -- it still points at an address this box no longer has" v6_online "$h" && pass "IPv6 still up over PPPoE" || fail "IPv6 down on the PPPoE path" save_evidence V2-tengig-down r "$h" 'sudo ip link set bond0.53 up' sleep 60 } V3() { # a cold backup must not advertise, dial, or blackhole log "V3 cold backup: no tunnel, no RA, no HE call" local h o before after h="$(holder)"; o=$([ "$h" = "$R1" ] && echo "$R2" || echo "$R1") before="$(he_calls)" r "$o" 'sudo systemctl restart vrrp-wan-reconcile.service' >/dev/null sleep 20 case "$(tun_state "$o")" in DOWN|absent) pass "backup tunnel stays $(tun_state "$o")" ;; *) fail "backup brought tun0 up while not holding the VIP" ;; esac radvd_on "$o" && fail "backup is advertising on VLAN 9" || pass "backup is silent on VLAN 9" after="$(he_calls)" [ "${after:-0}" = "${before:-0}" ] && pass "backup made no HE call" \ || fail "the BACKUP called the HE API -- it would point HE at its own idle line" save_evidence V3-cold-backup } V4() { # the assumption the production override was built on log "V4 a tunnel whose source-address is absent: what does VyOS actually do?" # The production override says such a tunnel "would simply stay down", and # treats that as the reason it was safe to leave IPv6 single-homed. Measured # on the sim backup 2026-09-06: the commit SUCCEEDS and the link comes up # anyway -- it is a blackhole, not an inert node. That is why the runtime # gate is load-bearing rather than a nicety, exactly like the PPPoE gate. local o h; h="$(holder)"; o=$([ "$h" = "$R1" ] && echo "$R2" || echo "$R1") r "$o" 'sudo ip link set tun0 up' >/dev/null; sleep 3 case "$(tun_state "$o")" in UP|UNKNOWN) pass "confirmed: VyOS leaves it UP with no source address (blackhole)" ;; *) warn "this VyOS version keeps it $(tun_state "$o") -- the override's assumption holds here; re-check the production version before relying on it" ;; esac v6_online "$o" && fail "the backup somehow reached HE -- two live tunnels" \ || pass "and it carries nothing, as expected" # Hand it straight back to the reconciler rather than leaving it up. r "$o" 'sudo systemctl restart vrrp-wan-reconcile.service' >/dev/null sleep 15 case "$(tun_state "$o")" in DOWN|absent) pass "the reconciler put it back down" ;; *) fail "the reconciler did NOT re-close the gate -- this is the load-bearing bit" ;; esac save_evidence V4-absent-source-address } V5() { # never two live tunnels, even mid-transition log "V5 both routers momentarily master: never two live tunnels" local from to i worst=0 n from="$(holder)"; to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1") r "$from" 'sudo mkdir -p /run/vrrp-wan && sudo touch /run/vrrp-wan/force-fault' # Sample THROUGH the transition rather than at the ends. The interesting # window is the one where both boxes briefly think they are in charge. for i in $(seq 1 24); do n="$(tun_holders)"; [ "${n:-0}" -gt "$worst" ] && worst="$n" sleep 5 done [ "$worst" -le 1 ] && pass "at most $worst live tunnel throughout the transition" \ || fail "saw $worst live tunnels at once -- HE would receive two claimants" r "$from" 'sudo rm -f /run/vrrp-wan/force-fault' save_evidence V5-transition-invariant sleep 40 } V6() { # RA deprecation: does a VLAN 9 host drop the dead gateway? log "V6 RA deprecation: the client must stop using a demoted router" local h before h="$(holder)" before="$(lan9 'ip -6 route show default 2>/dev/null | head -1')" if [ -z "$before" ]; then warn "VLAN 9 client has no IPv6 default route -- SLAAC may not have run; skipping" return fi log " client default was: $before" r "$h" 'sudo mkdir -p /run/vrrp-wan && sudo touch /run/vrrp-wan/force-fault' sleep 45 local after; after="$(lan9 'ip -6 route show default 2>/dev/null | head -1')" log " client default now: ${after:-}" # radvd emits a final RA with router-lifetime 0 on a graceful stop. Either # the client moved to the new master or it dropped the route entirely; both # are correct. Still pointing at the demoted box is not. if [ "$after" = "$before" ]; then fail "client still points at the demoted router -- the farewell RA did not land" else pass "client stopped using the demoted router" fi r "$h" 'sudo rm -f /run/vrrp-wan/force-fault' save_evidence V6-ra-deprecation sleep 40 } V7() { # replay the 2026-09-06 near-miss, but slowly log "V7 slow WAN restore: does the hysteresis still hold?" # On 2026-09-06 vif53-pin-boot-disable bounced the 10 gig, he-tunnel-follow # ticked once and saw the PPPoE address, and vyos-failover restored the 10 # gig 22 SECONDS before the second tick would have pushed HE at an address # Vodafone reissues on every dial. 22s of margin is not a safety property. # Here the restore is deliberately slower than the hysteresis window. local h before after tengig h="$(holder)"; before="$(he_calls)"; tengig="$(tengig_addr)" r "$h" 'sudo ip link set bond0.53 down' sleep 200 # > 2 ticks of a 1m scheduler r "$h" 'sudo ip link set bond0.53 up' sleep 90 after="$(he_calls)" if [ "$(( ${after:-0} - ${before:-0} ))" -ge 1 ]; then warn "HE was updated $(( ${after:-0} - ${before:-0} )) time(s) and then had to move back -- this is the 2026-09-06 shape, now reproduced deliberately. Either widen HYSTERESIS or make vif53-pin-boot-disable hold he-tunnel-follow off for the bounce." else pass "no HE churn across a slow WAN bounce" fi # Whatever happened, the tunnel must end up back on the 10 gig. local i for i in $(seq 1 24); do [ "$(tun_src "$h")" = "$tengig" ] && break sleep 10 done [ "$(tun_src "$h")" = "$tengig" ] && pass "tunnel returned to the 10 gig address" \ || fail "tunnel stuck on $(tun_src "$h") after the 10 gig came back" save_evidence V7-slow-restore } preflight() { log "preflight" local rc=0 [ "$(timeout 60 "$HE" status | grep -c 'API bound : nohost')" = 1 ] \ || { fail "the fake HE endpoint is not answering -- run ./labsim-he-endpoint.sh up"; rc=1; } local h for h in "$R1" "$R2"; do [ "$(tun_state "$h")" = absent ] \ && { fail "$h has no tun0 -- run --setup first"; rc=1; } [ "$(r "$h" 'systemctl is-active vrrp-wan-reconcile.timer')" = active ] \ || { fail "$h vrrp-wan-reconcile.timer not active"; rc=1; } # The reconciler must be the version that knows about the v6 plane, or # every result below measures the OLD behaviour while printing the new # test names -- the failure mode this repo has already been bitten by. r "$h" 'grep -q v6_take /config/vrrp-wan-reconcile' \ || { fail "$h has a vrrp-wan-reconcile with no IPv6 plane -- run migration/vrrp-wan-install"; rc=1; } done # Line the sim's endpoint up with whoever actually holds the tunnel. The real # HE remembers where it was pointed; a rebuilt sim endpoint does not, and # he-tunnel-follow will not re-assert because from its side nothing changed. # Does NOT count as an API call, so the invariant-2 assertions stay honest. local m src m="$(holder)"; src="$(tun_src "$m")" if [ -n "$src" ]; then timeout 60 "$HE" point "$src" >/dev/null 2>&1 \ || { fail "could not point the sim HE endpoint at $src"; rc=1; } fi [ "$rc" -eq 0 ] && pass "HE endpoint answering and pointed at $src, both routers have tun0 and a v6-aware reconciler" return $rc } case "${1:---all}" in --list) echo "V0 baseline | V1 clean failover | V2 10gig-down | V3 cold backup | V4 absent source-address | V5 transition invariant | V6 RA deprecation | V7 slow restore"; exit 0 ;; --setup) setup; exit 0 ;; --all) preflight || exit 1; V0; V1; V2; V3; V4; V5; V6; V7 ;; *) preflight || exit 1; "$1" ;; esac echo [ "$FAILED" -eq 0 ] && { echo "ALL PASS"; exit 0; } echo "$FAILED check(s) FAILED"; exit 1