431 lines
22 KiB
Bash
431 lines
22 KiB
Bash
|
#!/bin/bash
|
||
|
|
# Does IPv6 follow VRRP mastership, and does it do so WITHOUT touching HE?
|
||
|
|
#
|
||
|
|
# The WAN became HA on 2026-09-06 and IPv6 did not follow it. Nothing caught
|
||
|
|
# that, because nothing tested it: wan-drill measured IPv4 only, and PPPOE-HA.md
|
||
|
|
# recorded "IPv6 stayed up at 15.5ms" from a reading taken outside the failover
|
||
|
|
# window. This is the matrix that would have caught it.
|
||
|
|
#
|
||
|
|
# TWO INVARIANTS, checked independently of any individual test:
|
||
|
|
#
|
||
|
|
# 1. At most ONE router ever has a live tunnel. An UP tunnel on a box that
|
||
|
|
# does not own the source address is not harmless -- see V4, it is a
|
||
|
|
# BLACKHOLE that will happily attract the v6 default route.
|
||
|
|
# 2. A ROUTER-level failover calls the HE API ZERO times. The 10 gig address
|
||
|
|
# is bound to a cloned MAC and follows the VIP to the other box unchanged,
|
||
|
|
# so there is nothing to tell HE. A non-zero count means something
|
||
|
|
# re-pointed the tunnel at a PPPoE address -- which the ISP re-issues on
|
||
|
|
# every dial, so it would be wrong within minutes.
|
||
|
|
#
|
||
|
|
# KNOWN SIM GAP, 2026-09-06 -- read this before believing a v6_online failure.
|
||
|
|
# The MECHANISM is proven here: tun0 up on the master and down on the backup,
|
||
|
|
# radvd following mastership, he-tunnel-follow's master guard, its hysteresis,
|
||
|
|
# its HE call and the 1480->1472 MTU switch, and VLAN 9 hosts autoconfiguring
|
||
|
|
# from the RA (observed: real SLAAC traffic from 2001:db8:187e:9::/64 arriving
|
||
|
|
# at the endpoint encapsulated).
|
||
|
|
#
|
||
|
|
# What is NOT yet proven is the end-to-end v6 DATAPATH, because the sim's
|
||
|
|
# "internet" is asymmetric: 6in4 packets from the PPPoE island reach the
|
||
|
|
# endpoint with an outer source of 192.168.122.1 -- the libvirt host's NAT --
|
||
|
|
# so HE's replies go back to the tunnel remote by a path with no NAT state and
|
||
|
|
# are lost. Forward works, return does not.
|
||
|
|
#
|
||
|
|
# That is a topology fault in the scaffold, not in the thing under test. Fixing
|
||
|
|
# it means giving the two ISP islands a real transit path that does not traverse
|
||
|
|
# libvirt NAT. Until then, treat v6_online failures as UNPROVEN rather than as
|
||
|
|
# evidence the design is wrong -- and do not let that ambiguity leak into
|
||
|
|
# production sign-off, which is exactly the mistake wan-drill made by asserting
|
||
|
|
# "IPv6 stayed up" from a reading taken outside the window.
|
||
|
|
#
|
||
|
|
# Requires the fake HE endpoint: ./labsim-he-endpoint.sh up
|
||
|
|
#
|
||
|
|
# ./labsim-ipv6-ha-test.sh --setup configure the router side (once)
|
||
|
|
# ./labsim-ipv6-ha-test.sh --list
|
||
|
|
# ./labsim-ipv6-ha-test.sh V1
|
||
|
|
# ./labsim-ipv6-ha-test.sh --all
|
||
|
|
set -uo pipefail
|
||
|
|
|
||
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||
|
|
R1="${R1:-172.31.1.252}"; R2="${R2:-172.31.1.253}"
|
||
|
|
VIP="${VIP:-172.31.1.1}"
|
||
|
|
LAN9="${LAN9:-172.31.9.10}" # VLAN 9 client, for RA tests
|
||
|
|
HE_ADDR="${HE_ADDR:-192.0.2.10}"
|
||
|
|
HE_LINK6="${HE_LINK6:-2001:db8:1f1c:f6::1}"
|
||
|
|
RT_LINK6="${RT_LINK6:-2001:db8:1f1c:f6::2}"
|
||
|
|
V9_PREFIX="${V9_PREFIX:-2001:db8:187e:9}"
|
||
|
|
# Mirrors production: one MAC, one lease, whichever router holds the VIP.
|
||
|
|
WAN_MAC="${WAN_MAC:-02:9f:c2:12:9b:4f}"
|
||
|
|
PW="${VYOS_PW:-vyos}"; LANPW="${LANPW:-labsim}"
|
||
|
|
EVID="$SCRIPT_DIR/ipv6-ha-evidence"
|
||
|
|
HE="$SCRIPT_DIR/labsim-he-endpoint.sh"
|
||
|
|
|
||
|
|
SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
||
|
|
-o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password)
|
||
|
|
r() { timeout 45 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; }
|
||
|
|
# The Alpine LAN VMs do not offer `password` auth -- reusing the routers' option
|
||
|
|
# set makes ssh exit 255 before running anything, which reads as "the network is
|
||
|
|
# broken". Same trap as lan() in labsim-pppoe-ha-test.sh.
|
||
|
|
LAN_SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
||
|
|
-o LogLevel=ERROR -o ConnectTimeout=6)
|
||
|
|
lan9() { timeout 30 sshpass -p "$LANPW" ssh "${LAN_SSH[@]}" "root@$LAN9" "$@" 2>/dev/null; }
|
||
|
|
|
||
|
|
log() { printf '\033[36m==>\033[0m %s\n' "$*"; }
|
||
|
|
pass() { printf ' \033[32mPASS\033[0m %s\n' "$*"; }
|
||
|
|
fail() { printf ' \033[31mFAIL\033[0m %s\n' "$*"; FAILED=$((FAILED+1)); }
|
||
|
|
warn() { printf ' \033[33mWARN\033[0m %s\n' "$*"; }
|
||
|
|
FAILED=0
|
||
|
|
|
||
|
|
# --- observations ----------------------------------------------------------
|
||
|
|
# A destroyed or unreachable router is emphatically NOT holding the tunnel, but
|
||
|
|
# ssh returns an EMPTY string, and `[ "" = 0 ]` is false -- the pppoe matrix hung
|
||
|
|
# on exactly this waiting for a dead box to report zero. Default everything to 0.
|
||
|
|
tun_state() { local v; v="$(r "$1" "ip -br link show tun0 2>/dev/null | awk '{print \$2}'" | tr -d ' \n')"; echo "${v:-absent}"; }
|
||
|
|
tun_src() { r "$1" 'ip tunnel show tun0 2>/dev/null | sed -nE "s/.* local ([0-9.]+).*/\1/p"' | tr -d ' \n'; }
|
||
|
|
# Whichever router currently holds the 10 gig lease -- under the cloned MAC only
|
||
|
|
# one ever does. Read rather than assumed: the address changes when the MAC does.
|
||
|
|
tengig_addr() { local h a; for h in "$R1" "$R2"; do
|
||
|
|
a="$(r "$h" 'ip -4 addr show bond0.53 2>/dev/null | sed -nE "s/.*inet ([0-9.]+).*/\1/p"' | tr -d ' \n')"
|
||
|
|
[ -n "$a" ] && { echo "$a"; return; }
|
||
|
|
done; echo ""; }
|
||
|
|
tun_mtu() { local v; v="$(r "$1" 'cat /sys/class/net/tun0/mtu 2>/dev/null' | tr -d ' \n')"; echo "${v:-0}"; }
|
||
|
|
holder() { for h in "$R1" "$R2"; do
|
||
|
|
[ "$(r "$h" "ip -4 -o addr show | grep -c ' ${VIP}/'" | tr -d ' \n')" != 0 ] \
|
||
|
|
&& { echo "$h"; return; }; done; echo none; }
|
||
|
|
# Ask the ROUTER, not a client: a client can be answered by the wrong path.
|
||
|
|
v6_online() { [ "$(r "$1" "ping -6 -c1 -W3 $HE_LINK6 >/dev/null 2>&1 && echo y" | tr -d ' \n')" = y ]; }
|
||
|
|
radvd_on() { [ "$(r "$1" 'systemctl is-active radvd 2>/dev/null' | tr -d ' \n')" = active ]; }
|
||
|
|
he_calls() { timeout 60 "$HE" calls | tr -d ' \n'; }
|
||
|
|
|
||
|
|
# How many routers have a tunnel that is UP. Invariant 1's numerator.
|
||
|
|
tun_holders() { local n=0 h; for h in "$R1" "$R2"; do
|
||
|
|
case "$(tun_state "$h")" in UP|UNKNOWN) n=$((n+1)) ;; esac
|
||
|
|
done; echo "$n"; }
|
||
|
|
|
||
|
|
check_invariants() {
|
||
|
|
local t ok=0
|
||
|
|
t="$(tun_holders)"
|
||
|
|
[ "${t:-0}" -le 1 ] || { fail "INVARIANT: $t routers have tun0 up"; ok=1; }
|
||
|
|
return $ok
|
||
|
|
}
|
||
|
|
|
||
|
|
save_evidence() {
|
||
|
|
local name="$1"; local d="$EVID/$name"; mkdir -p "$d"
|
||
|
|
{ echo "=== $(date -Is) ==="
|
||
|
|
echo "--- HE endpoint ---"; timeout 60 "$HE" status
|
||
|
|
for h in "$R1" "$R2"; do echo "--- $h ---"
|
||
|
|
r "$h" 'sudo /config/vrrp-wan-reconcile --status 2>/dev/null
|
||
|
|
ip -br link show tun0 2>/dev/null; ip tunnel show tun0 2>/dev/null
|
||
|
|
ip -6 route show default; systemctl is-active radvd
|
||
|
|
sudo journalctl -t vrrp-wan -t he-tunnel-follow -n 10 --no-pager'
|
||
|
|
done; } > "$d/state.txt" 2>&1
|
||
|
|
log "evidence -> ipv6-ha-evidence/$name/"
|
||
|
|
}
|
||
|
|
|
||
|
|
# --- setup ------------------------------------------------------------------
|
||
|
|
# Applied through REAL VyOS config, unlike the ISP-side scaffold, because "will
|
||
|
|
# VyOS accept this?" is one of the questions being asked.
|
||
|
|
vyos_apply() { # host, then set-lines on stdin
|
||
|
|
local h="$1"
|
||
|
|
{ printf '#!/bin/vbash\nsource /opt/vyatta/etc/functions/script-template\nconfigure\n'
|
||
|
|
cat
|
||
|
|
printf 'commit\nsave\nexit\n'
|
||
|
|
} | timeout 90 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$h" \
|
||
|
|
'cat > /tmp/v6-apply.sh && chmod +x /tmp/v6-apply.sh && sudo /tmp/v6-apply.sh' 2>&1 | tail -3
|
||
|
|
# `vbash -c` never starts a config session and commit fails to stderr, which
|
||
|
|
# a helper like this discards -- the T4 matrix in labsim-pppoe-ha-test.sh ran
|
||
|
|
# its whole policy sweep against the default while printing the mode it
|
||
|
|
# thought it was testing. Always read the value back.
|
||
|
|
}
|
||
|
|
|
||
|
|
setup() {
|
||
|
|
log "--setup: router-side IPv6, both routers"
|
||
|
|
|
||
|
|
# THE CLONED MAC, first and on its own. Production pins f0:9f:c2:12:9b:4f on
|
||
|
|
# vif 53 so the 10 gig lease follows the VIP and the tunnel source is the
|
||
|
|
# SAME address on either box. The sim never had it -- each router took its
|
||
|
|
# own lease -- so the sim could not reproduce the one property the whole
|
||
|
|
# IPv6-HA design leans on, and invariant 2 would have been untestable here.
|
||
|
|
local h pref v9
|
||
|
|
for h in "$R1" "$R2"; do
|
||
|
|
log " $h: pinning the cloned WAN MAC $WAN_MAC"
|
||
|
|
vyos_apply "$h" <<EOF
|
||
|
|
set interfaces bonding bond0 vif 53 mac '$WAN_MAC'
|
||
|
|
EOF
|
||
|
|
done
|
||
|
|
|
||
|
|
# A new MAC means a NEW lease, so the tunnel source cannot be hardcoded --
|
||
|
|
# discover it. Hardcoding the pre-change address here would have configured
|
||
|
|
# every tunnel with a source neither router owns, i.e. the V4 blackhole, on
|
||
|
|
# both boxes, while the matrix reported setup success.
|
||
|
|
local src="" i
|
||
|
|
for i in $(seq 1 30); do
|
||
|
|
src="$(tengig_addr)"; [ -n "$src" ] && break
|
||
|
|
sleep 5
|
||
|
|
done
|
||
|
|
[ -n "$src" ] || { fail "no router took a 10 gig lease after the MAC change -- cannot set a tunnel source"; return 1; }
|
||
|
|
log " 10 gig lease under the cloned MAC: $src"
|
||
|
|
|
||
|
|
# Credentials pointing at the stub. HE_UPDATE_URL is read AFTER the secrets
|
||
|
|
# file is sourced, so putting it here overrides the production default
|
||
|
|
# without the script needing a sim-specific branch.
|
||
|
|
for h in "$R1" "$R2"; do
|
||
|
|
printf 'HE_USER=sim\nHE_UPDATE_KEY=sim\nHE_TUNNEL_ID=1\nHE_UPDATE_URL=http://%s/nic/update\n' "$HE_ADDR" \
|
||
|
|
| timeout 30 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$h" \
|
||
|
|
'cat > /tmp/he-secrets && sudo install -o root -g vyattacfg -m 0640 /tmp/he-secrets /config/he-secrets' >/dev/null
|
||
|
|
done
|
||
|
|
|
||
|
|
for h in "$R1" "$R2"; do
|
||
|
|
[ "$h" = "$R1" ] && { pref=high; v9=1; } || { pref=low; v9=2; }
|
||
|
|
log " $h (RA preference $pref, bond0.9 ::${v9})"
|
||
|
|
vyos_apply "$h" <<EOF
|
||
|
|
set interfaces tunnel tun0 encapsulation 'sit'
|
||
|
|
set interfaces tunnel tun0 source-address '$src'
|
||
|
|
set interfaces tunnel tun0 remote '$HE_ADDR'
|
||
|
|
set interfaces tunnel tun0 address '$RT_LINK6/64'
|
||
|
|
set interfaces tunnel tun0 mtu '1480'
|
||
|
|
set protocols static route6 ::/0 next-hop '$HE_LINK6'
|
||
|
|
set interfaces bonding bond0 vif 9 address '${V9_PREFIX}::${v9}/64'
|
||
|
|
set service router-advert interface bond0.9 prefix ${V9_PREFIX}::/64 preferred-lifetime '604800'
|
||
|
|
set service router-advert interface bond0.9 link-mtu '1472'
|
||
|
|
set service router-advert interface bond0.9 default-preference '$pref'
|
||
|
|
set system task-scheduler task he-tunnel-follow executable path '/config/he-tunnel-follow'
|
||
|
|
set system task-scheduler task he-tunnel-follow executable arguments 'run'
|
||
|
|
set system task-scheduler task he-tunnel-follow interval '1m'
|
||
|
|
EOF
|
||
|
|
done
|
||
|
|
# RA link-mtu is 1472, the PPPoE figure, on BOTH -- deliberately not 1480.
|
||
|
|
# It cannot be reconciled at runtime (it needs a commit, and the tunnel plane
|
||
|
|
# is commit-free on purpose), so advertise the lower of the two paths and be
|
||
|
|
# correct on either WAN. Production pinned 1480 and was wrong whenever the
|
||
|
|
# WAN fell back.
|
||
|
|
log " setup done -- run V0 to check the baseline"
|
||
|
|
}
|
||
|
|
|
||
|
|
# --- tests ------------------------------------------------------------------
|
||
|
|
V0() { # baseline
|
||
|
|
log "V0 baseline: the VIP holder owns the tunnel, the backup does not"
|
||
|
|
local h o; h="$(holder)"; o=$([ "$h" = "$R1" ] && echo "$R2" || echo "$R1")
|
||
|
|
[ "$h" = none ] && { fail "no VIP holder"; return; }
|
||
|
|
case "$(tun_state "$h")" in UP|UNKNOWN) pass "master $h has tun0 up" ;;
|
||
|
|
*) fail "master $h tun0 is $(tun_state "$h")" ;; esac
|
||
|
|
case "$(tun_state "$o")" in DOWN|absent) pass "backup $o tun0 is $(tun_state "$o")" ;;
|
||
|
|
*) fail "backup $o tun0 is $(tun_state "$o") -- it should be held down" ;; esac
|
||
|
|
v6_online "$h" && pass "master reaches HE over v6" || fail "master has no IPv6"
|
||
|
|
radvd_on "$h" && pass "master is advertising on VLAN 9" || fail "master radvd not running"
|
||
|
|
radvd_on "$o" && fail "backup is ALSO advertising -- two default routers on VLAN 9" \
|
||
|
|
|| pass "backup is not advertising"
|
||
|
|
check_invariants
|
||
|
|
save_evidence V0-baseline
|
||
|
|
}
|
||
|
|
|
||
|
|
V1() { # clean failover: v6 follows, and HE is never called
|
||
|
|
log "V1 clean failover: IPv6 follows, HE API untouched"
|
||
|
|
local from to t0 before after i
|
||
|
|
from="$(holder)"; to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1")
|
||
|
|
before="$(he_calls)"
|
||
|
|
log " master=$from -> expecting $to (HE calls so far: ${before:-0})"
|
||
|
|
t0=$(date +%s)
|
||
|
|
r "$from" 'sudo mkdir -p /run/vrrp-wan && sudo touch /run/vrrp-wan/force-fault'
|
||
|
|
local took=""
|
||
|
|
for i in $(seq 1 36); do
|
||
|
|
sleep 5
|
||
|
|
[ "$(holder)" = "$to" ] && v6_online "$to" && { took=$(( $(date +%s) - t0 )); break; }
|
||
|
|
done
|
||
|
|
[ -n "$took" ] && pass "IPv6 reached $to in ${took}s" \
|
||
|
|
|| fail "IPv6 never followed to $to within 180s"
|
||
|
|
case "$(tun_state "$from")" in DOWN|absent) pass "$from released its tunnel" ;;
|
||
|
|
*) fail "$from still has tun0 $(tun_state "$from") -- blackhole risk" ;; esac
|
||
|
|
after="$(he_calls)"
|
||
|
|
# THE invariant this test exists for.
|
||
|
|
[ "${after:-0}" = "${before:-0}" ] \
|
||
|
|
&& pass "HE API not called (${after:-0} total) -- the address followed the MAC" \
|
||
|
|
|| fail "HE API called $(( ${after:-0} - ${before:-0} )) time(s) during a ROUTER failover"
|
||
|
|
check_invariants
|
||
|
|
save_evidence V1-clean-failover
|
||
|
|
r "$from" 'sudo rm -f /run/vrrp-wan/force-fault'
|
||
|
|
sleep 40
|
||
|
|
}
|
||
|
|
|
||
|
|
V2() { # 10 gig down on the master: HE must be told, exactly once
|
||
|
|
log "V2 10 gig down: he-tunnel-follow re-points the tunnel and tells HE"
|
||
|
|
local h before after mtu i ok=no
|
||
|
|
h="$(holder)"; before="$(he_calls)"
|
||
|
|
local tengig; tengig="$(tengig_addr)"
|
||
|
|
r "$h" 'sudo ip link set bond0.53 down'
|
||
|
|
# he-tunnel-follow runs on a 1m task-scheduler with a 2-tick hysteresis, so
|
||
|
|
# allow well past 2 minutes before calling it a failure.
|
||
|
|
for i in $(seq 1 30); do
|
||
|
|
sleep 10
|
||
|
|
[ "$(tun_src "$h")" != "$tengig" ] && { ok=yes; break; }
|
||
|
|
done
|
||
|
|
[ "$ok" = yes ] && pass "tunnel source moved to $(tun_src "$h") after $((i*10))s" \
|
||
|
|
|| fail "tunnel source never left the 10 gig address"
|
||
|
|
mtu="$(tun_mtu "$h")"
|
||
|
|
[ "$mtu" = 1472 ] && pass "MTU dropped to 1472 for the PPPoE path" \
|
||
|
|
|| fail "MTU is $mtu, want 1472 -- large transfers will hang"
|
||
|
|
after="$(he_calls)"
|
||
|
|
[ "$(( ${after:-0} - ${before:-0} ))" -ge 1 ] \
|
||
|
|
&& pass "HE API called $(( ${after:-0} - ${before:-0} )) time(s), as it must be here" \
|
||
|
|
|| fail "HE was never told -- it still points at an address this box no longer has"
|
||
|
|
v6_online "$h" && pass "IPv6 still up over PPPoE" || fail "IPv6 down on the PPPoE path"
|
||
|
|
save_evidence V2-tengig-down
|
||
|
|
r "$h" 'sudo ip link set bond0.53 up'
|
||
|
|
sleep 60
|
||
|
|
}
|
||
|
|
|
||
|
|
V3() { # a cold backup must not advertise, dial, or blackhole
|
||
|
|
log "V3 cold backup: no tunnel, no RA, no HE call"
|
||
|
|
local h o before after
|
||
|
|
h="$(holder)"; o=$([ "$h" = "$R1" ] && echo "$R2" || echo "$R1")
|
||
|
|
before="$(he_calls)"
|
||
|
|
r "$o" 'sudo systemctl restart vrrp-wan-reconcile.service' >/dev/null
|
||
|
|
sleep 20
|
||
|
|
case "$(tun_state "$o")" in DOWN|absent) pass "backup tunnel stays $(tun_state "$o")" ;;
|
||
|
|
*) fail "backup brought tun0 up while not holding the VIP" ;; esac
|
||
|
|
radvd_on "$o" && fail "backup is advertising on VLAN 9" || pass "backup is silent on VLAN 9"
|
||
|
|
after="$(he_calls)"
|
||
|
|
[ "${after:-0}" = "${before:-0}" ] && pass "backup made no HE call" \
|
||
|
|
|| fail "the BACKUP called the HE API -- it would point HE at its own idle line"
|
||
|
|
save_evidence V3-cold-backup
|
||
|
|
}
|
||
|
|
|
||
|
|
V4() { # the assumption the production override was built on
|
||
|
|
log "V4 a tunnel whose source-address is absent: what does VyOS actually do?"
|
||
|
|
# The production override says such a tunnel "would simply stay down", and
|
||
|
|
# treats that as the reason it was safe to leave IPv6 single-homed. Measured
|
||
|
|
# on the sim backup 2026-09-06: the commit SUCCEEDS and the link comes up
|
||
|
|
# anyway -- it is a blackhole, not an inert node. That is why the runtime
|
||
|
|
# gate is load-bearing rather than a nicety, exactly like the PPPoE gate.
|
||
|
|
local o h; h="$(holder)"; o=$([ "$h" = "$R1" ] && echo "$R2" || echo "$R1")
|
||
|
|
r "$o" 'sudo ip link set tun0 up' >/dev/null; sleep 3
|
||
|
|
case "$(tun_state "$o")" in
|
||
|
|
UP|UNKNOWN) pass "confirmed: VyOS leaves it UP with no source address (blackhole)" ;;
|
||
|
|
*) warn "this VyOS version keeps it $(tun_state "$o") -- the override's assumption holds here; re-check the production version before relying on it" ;;
|
||
|
|
esac
|
||
|
|
v6_online "$o" && fail "the backup somehow reached HE -- two live tunnels" \
|
||
|
|
|| pass "and it carries nothing, as expected"
|
||
|
|
# Hand it straight back to the reconciler rather than leaving it up.
|
||
|
|
r "$o" 'sudo systemctl restart vrrp-wan-reconcile.service' >/dev/null
|
||
|
|
sleep 15
|
||
|
|
case "$(tun_state "$o")" in DOWN|absent) pass "the reconciler put it back down" ;;
|
||
|
|
*) fail "the reconciler did NOT re-close the gate -- this is the load-bearing bit" ;; esac
|
||
|
|
save_evidence V4-absent-source-address
|
||
|
|
}
|
||
|
|
|
||
|
|
V5() { # never two live tunnels, even mid-transition
|
||
|
|
log "V5 both routers momentarily master: never two live tunnels"
|
||
|
|
local from to i worst=0 n
|
||
|
|
from="$(holder)"; to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1")
|
||
|
|
r "$from" 'sudo mkdir -p /run/vrrp-wan && sudo touch /run/vrrp-wan/force-fault'
|
||
|
|
# Sample THROUGH the transition rather than at the ends. The interesting
|
||
|
|
# window is the one where both boxes briefly think they are in charge.
|
||
|
|
for i in $(seq 1 24); do
|
||
|
|
n="$(tun_holders)"; [ "${n:-0}" -gt "$worst" ] && worst="$n"
|
||
|
|
sleep 5
|
||
|
|
done
|
||
|
|
[ "$worst" -le 1 ] && pass "at most $worst live tunnel throughout the transition" \
|
||
|
|
|| fail "saw $worst live tunnels at once -- HE would receive two claimants"
|
||
|
|
r "$from" 'sudo rm -f /run/vrrp-wan/force-fault'
|
||
|
|
save_evidence V5-transition-invariant
|
||
|
|
sleep 40
|
||
|
|
}
|
||
|
|
|
||
|
|
V6() { # RA deprecation: does a VLAN 9 host drop the dead gateway?
|
||
|
|
log "V6 RA deprecation: the client must stop using a demoted router"
|
||
|
|
local h before
|
||
|
|
h="$(holder)"
|
||
|
|
before="$(lan9 'ip -6 route show default 2>/dev/null | head -1')"
|
||
|
|
if [ -z "$before" ]; then
|
||
|
|
warn "VLAN 9 client has no IPv6 default route -- SLAAC may not have run; skipping"
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
log " client default was: $before"
|
||
|
|
r "$h" 'sudo mkdir -p /run/vrrp-wan && sudo touch /run/vrrp-wan/force-fault'
|
||
|
|
sleep 45
|
||
|
|
local after; after="$(lan9 'ip -6 route show default 2>/dev/null | head -1')"
|
||
|
|
log " client default now: ${after:-<none>}"
|
||
|
|
# radvd emits a final RA with router-lifetime 0 on a graceful stop. Either
|
||
|
|
# the client moved to the new master or it dropped the route entirely; both
|
||
|
|
# are correct. Still pointing at the demoted box is not.
|
||
|
|
if [ "$after" = "$before" ]; then
|
||
|
|
fail "client still points at the demoted router -- the farewell RA did not land"
|
||
|
|
else
|
||
|
|
pass "client stopped using the demoted router"
|
||
|
|
fi
|
||
|
|
r "$h" 'sudo rm -f /run/vrrp-wan/force-fault'
|
||
|
|
save_evidence V6-ra-deprecation
|
||
|
|
sleep 40
|
||
|
|
}
|
||
|
|
|
||
|
|
V7() { # replay the 2026-09-06 near-miss, but slowly
|
||
|
|
log "V7 slow WAN restore: does the hysteresis still hold?"
|
||
|
|
# On 2026-09-06 vif53-pin-boot-disable bounced the 10 gig, he-tunnel-follow
|
||
|
|
# ticked once and saw the PPPoE address, and vyos-failover restored the 10
|
||
|
|
# gig 22 SECONDS before the second tick would have pushed HE at an address
|
||
|
|
# Vodafone reissues on every dial. 22s of margin is not a safety property.
|
||
|
|
# Here the restore is deliberately slower than the hysteresis window.
|
||
|
|
local h before after tengig
|
||
|
|
h="$(holder)"; before="$(he_calls)"; tengig="$(tengig_addr)"
|
||
|
|
r "$h" 'sudo ip link set bond0.53 down'
|
||
|
|
sleep 200 # > 2 ticks of a 1m scheduler
|
||
|
|
r "$h" 'sudo ip link set bond0.53 up'
|
||
|
|
sleep 90
|
||
|
|
after="$(he_calls)"
|
||
|
|
if [ "$(( ${after:-0} - ${before:-0} ))" -ge 1 ]; then
|
||
|
|
warn "HE was updated $(( ${after:-0} - ${before:-0} )) time(s) and then had to move back -- this is the 2026-09-06 shape, now reproduced deliberately. Either widen HYSTERESIS or make vif53-pin-boot-disable hold he-tunnel-follow off for the bounce."
|
||
|
|
else
|
||
|
|
pass "no HE churn across a slow WAN bounce"
|
||
|
|
fi
|
||
|
|
# Whatever happened, the tunnel must end up back on the 10 gig.
|
||
|
|
local i
|
||
|
|
for i in $(seq 1 24); do
|
||
|
|
[ "$(tun_src "$h")" = "$tengig" ] && break
|
||
|
|
sleep 10
|
||
|
|
done
|
||
|
|
[ "$(tun_src "$h")" = "$tengig" ] && pass "tunnel returned to the 10 gig address" \
|
||
|
|
|| fail "tunnel stuck on $(tun_src "$h") after the 10 gig came back"
|
||
|
|
save_evidence V7-slow-restore
|
||
|
|
}
|
||
|
|
|
||
|
|
preflight() {
|
||
|
|
log "preflight"
|
||
|
|
local rc=0
|
||
|
|
[ "$(timeout 60 "$HE" status | grep -c 'API bound : nohost')" = 1 ] \
|
||
|
|
|| { fail "the fake HE endpoint is not answering -- run ./labsim-he-endpoint.sh up"; rc=1; }
|
||
|
|
local h
|
||
|
|
for h in "$R1" "$R2"; do
|
||
|
|
[ "$(tun_state "$h")" = absent ] \
|
||
|
|
&& { fail "$h has no tun0 -- run --setup first"; rc=1; }
|
||
|
|
[ "$(r "$h" 'systemctl is-active vrrp-wan-reconcile.timer')" = active ] \
|
||
|
|
|| { fail "$h vrrp-wan-reconcile.timer not active"; rc=1; }
|
||
|
|
# The reconciler must be the version that knows about the v6 plane, or
|
||
|
|
# every result below measures the OLD behaviour while printing the new
|
||
|
|
# test names -- the failure mode this repo has already been bitten by.
|
||
|
|
r "$h" 'grep -q v6_take /config/vrrp-wan-reconcile' \
|
||
|
|
|| { fail "$h has a vrrp-wan-reconcile with no IPv6 plane -- run migration/vrrp-wan-install"; rc=1; }
|
||
|
|
done
|
||
|
|
# Line the sim's endpoint up with whoever actually holds the tunnel. The real
|
||
|
|
# HE remembers where it was pointed; a rebuilt sim endpoint does not, and
|
||
|
|
# he-tunnel-follow will not re-assert because from its side nothing changed.
|
||
|
|
# Does NOT count as an API call, so the invariant-2 assertions stay honest.
|
||
|
|
local m src
|
||
|
|
m="$(holder)"; src="$(tun_src "$m")"
|
||
|
|
if [ -n "$src" ]; then
|
||
|
|
timeout 60 "$HE" point "$src" >/dev/null 2>&1 \
|
||
|
|
|| { fail "could not point the sim HE endpoint at $src"; rc=1; }
|
||
|
|
fi
|
||
|
|
[ "$rc" -eq 0 ] && pass "HE endpoint answering and pointed at $src, both routers have tun0 and a v6-aware reconciler"
|
||
|
|
return $rc
|
||
|
|
}
|
||
|
|
|
||
|
|
case "${1:---all}" in
|
||
|
|
--list) echo "V0 baseline | V1 clean failover | V2 10gig-down | V3 cold backup | V4 absent source-address | V5 transition invariant | V6 RA deprecation | V7 slow restore"; exit 0 ;;
|
||
|
|
--setup) setup; exit 0 ;;
|
||
|
|
--all) preflight || exit 1; V0; V1; V2; V3; V4; V5; V6; V7 ;;
|
||
|
|
*) preflight || exit 1; "$1" ;;
|
||
|
|
esac
|
||
|
|
|
||
|
|
echo
|
||
|
|
[ "$FAILED" -eq 0 ] && { echo "ALL PASS"; exit 0; }
|
||
|
|
echo "$FAILED check(s) FAILED"; exit 1
|