lab/migration/wan-drill

#!/usr/bin/env bash
# Controlled WAN failover drill. Runs from the workstation over the LAN.
#
# It is written as ONE unattended script on purpose. The drill takes the
# household's internet down, so anything driving it step-by-step from off-site
# -- a person on a laptop, or an agent that needs the internet to think --
# stops being able to act at exactly the moment it matters. This script only
# needs the LAN, and it always runs its cleanup.
#
#   ./wan-drill                 run it
#   ./wan-drill --dry           show what it would do, touch nothing
#
# Recovery if this script itself dies: see migration/RECOVERY-CARD-wan-panic.md
# or /config/RECOVERY-CARD.md on either router. Short version, on vyos002:
#     sudo /config/wan-panic
set -uo pipefail

P1=10.0.1.252   # vyos001, normally MASTER
P2=10.0.1.253   # vyos002, normally BACKUP
PW=vyos
SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
     -o LogLevel=ERROR -o ConnectTimeout=5)
TAKEOVER_BUDGET=240    # give vyos002 this long to raise a WAN
FAILBACK_BUDGET=180    # and vyos001 this long to take it back
WATCHDOG_HOLD=150      # vyos002 stands down by itself after this with no WAN

LOG="${LOG:-/tmp/wan-drill-$(date +%H%M%S).log}"
DRY=0; [ "${1:-}" = "--dry" ] && DRY=1

say() { printf '%s %s\n' "$(date +%T)" "$*" | tee -a "$LOG"; }
r()   { timeout 25 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; }

holder()  { for h in "$P1" "$P2"; do
              [ "$(r "$h" 'ip -4 -o addr show | grep -c " 192.168.1.1/"' | tr -d ' \n')" != 0 ] \
                && { echo "$h"; return; }; done; echo none; }
wan_of()  { r "$1" 'for i in bond0.53 pppoe0; do a=$(ip -4 addr show dev $i 2>/dev/null | sed -n "s/.*inet \([0-9.]*\).*/\1/p"); [ -n "$a" ] && printf "%s=%s " $i $a; done'; }
online()  { [ "$(r "$1" 'ping -c1 -W2 9.9.9.9 >/dev/null 2>&1 && echo y' | tr -d ' \n')" = y ]; }

# --- IPv6 -------------------------------------------------------------------
# Until 2026-09-06 this drill measured IPv4 only, and PPPOE-HA.md recorded that
# "IPv6 stayed up" through a failover on the strength of a reading taken outside
# the window. It cannot have: the reconciler disables bond0.53 on the demoted
# box, so 87.192.101.48 moves to the survivor, and inbound protocol 41 from HE
# then lands on whichever router owns the tunnel. Measure it rather than assume.
#
# Quad9 again, so the v6 result is comparable with the v4 one on the line above.
online6() { [ "$(r "$1" 'ping -6 -c1 -W2 2620:fe::fe >/dev/null 2>&1 && echo y' | tr -d ' \n')" = y ]; }
# Tunnel link state and source address. The source is the interesting half: it
# should be IDENTICAL before and after a router-level failover, because the 10
# gig lease follows the cloned MAC. A changed source means something called the
# HE API during the drill, which a router failover must never need to do.
tun_of()  { r "$1" 'ip tunnel show tun0 2>/dev/null | sed -nE "s/.* local ([0-9.]+).*/\1/p"' | tr -d ' \n'; }
he_calls(){ r "$1" 'sudo journalctl -t he-tunnel-follow --since "'"$2"'" --no-pager 2>/dev/null | grep -c "HE endpoint set to"' | tr -d ' \n'; }

cleanup() {
    say "--- cleanup (always runs) ---"
    r "$P2" 'sudo /config/wan-drill-watchdog disarm' >/dev/null
    r "$P1" 'sudo rm -f /run/vrrp-wan/force-fault' >/dev/null
    r "$P2" 'sudo rm -f /run/vrrp-wan/force-fault' >/dev/null
    sleep 20
    say "final holder : $(holder)"
    say "final WAN    : $P1 [$(wan_of $P1)]  $P2 [$(wan_of $P2)]"
    online "$P1" && say "final internet: UP via $P1" || {
        online "$P2" && say "final internet: UP via $P2" \
                     || say "final internet: *** DOWN -- run: sudo /config/wan-panic on $P2 ***"; }
    say "log: $LOG"
}
trap cleanup EXIT

say "=== pre-flight ==="
DRILL_START="$(date '+%Y-%m-%d %H:%M:%S')"
start_holder="$(holder)"
say "holder now   : $start_holder"
say "$P1 WAN      : $(wan_of $P1)"
say "$P2 WAN      : $(wan_of $P2)"
online "$P1" && say "internet     : UP via $P1" || { say "internet ALREADY DOWN -- refusing to drill"; exit 1; }
tun_before="$(tun_of $P1)"; tun2_before="$(tun_of $P2)"
say "$P1 tun0 src : ${tun_before:-<no tunnel>}"
say "$P2 tun0 src : ${tun2_before:-<no tunnel -- IPv6 cannot survive a failover>}"
if online6 "$P1"; then say "IPv6         : UP via $P1"
else say "IPv6         : DOWN on $P1 before we start -- v6 figures below are not meaningful"; fi
[ "$start_holder" = "$P1" ] || { say "expected $P1 to hold the VIP, got $start_holder -- refusing"; exit 1; }

if [ "$DRY" = 1 ]; then
    say "--dry: would arm the watchdog on $P2 (${WATCHDOG_HOLD}s) and force-fault $P1"
    trap - EXIT; exit 0
fi

say "=== arming auto-abort on $P2 ==="
r "$P2" "sudo /config/wan-drill-watchdog arm $WATCHDOG_HOLD" | tee -a "$LOG"

say "=== DRILL: force-faulting $P1 ==="
t0=$(date +%s)
r "$P1" 'sudo touch /run/vrrp-wan/force-fault'

took=""; took6=""
while [ $(( $(date +%s) - t0 )) -lt "$TAKEOVER_BUDGET" ]; do
    sleep 5
    h="$(holder)"; w="$(wan_of $P2)"
    # v6 keeps being probed after v4 comes back, because the two recover
    # independently and the gap between them IS the number this drill exists to
    # produce. Stop only when both are up, or the budget runs out.
    [ -z "$took6" ] && online6 "$P2" && took6=$(( $(date +%s) - t0 ))
    say "  t+$(( $(date +%s) - t0 ))s holder=$h  vyos002_wan=[$w]  v6=$([ -n "$took6" ] && echo up || echo down)"
    if [ -z "$took" ] && [ "$h" = "$P2" ] && [ -n "$w" ] && online "$P2"; then
        took=$(( $(date +%s) - t0 ))
    fi
    [ -n "$took" ] && [ -n "$took6" ] && break
done

if [ -n "$took" ]; then
    say "*** TAKEOVER OK: $P2 held the VIP and reached the internet in ${took}s ***"
else
    say "*** TAKEOVER FAILED within ${TAKEOVER_BUDGET}s -- failing back ***"
fi
if [ -n "$took6" ] && [ -n "$took" ]; then
    say "*** IPv6 followed in ${took6}s (v4 ${took}s, gap $(( took6 - took ))s) ***"
elif [ -n "$took6" ]; then
    say "*** IPv6 followed in ${took6}s, but IPv4 never did ***"
else
    say "*** IPv6 did NOT return within ${TAKEOVER_BUDGET}s on $P2 -- the v6 estate is down for the whole takeover ***"
fi

say "=== failing back to $P1 ==="
t1=$(date +%s)
r "$P2" 'sudo /config/wan-drill-watchdog disarm' >/dev/null
r "$P1" 'sudo rm -f /run/vrrp-wan/force-fault'
r "$P2" 'sudo touch /run/vrrp-wan/force-fault'
back=""; back6=""
while [ $(( $(date +%s) - t1 )) -lt "$FAILBACK_BUDGET" ]; do
    sleep 5
    h="$(holder)"; w="$(wan_of $P1)"
    [ -z "$back6" ] && online6 "$P1" && back6=$(( $(date +%s) - t1 ))
    say "  t+$(( $(date +%s) - t1 ))s holder=$h  vyos001_wan=[$w]  v6=$([ -n "$back6" ] && echo up || echo down)"
    if [ -z "$back" ] && [ "$h" = "$P1" ] && [ -n "$w" ] && online "$P1"; then
        back=$(( $(date +%s) - t1 ))
    fi
    [ -n "$back" ] && [ -n "$back6" ] && break
done
[ -n "$back" ] && say "*** FAILBACK OK in ${back}s ***" \
               || say "*** FAILBACK FAILED -- cleanup will clear both levers ***"
[ -n "$back6" ] && say "*** IPv6 back in ${back6}s ***" \
                || say "*** IPv6 did NOT return within ${FAILBACK_BUDGET}s on $P1 ***"

# The invariant a router-level failover must satisfy: the HE endpoint is never
# touched. 87.192.101.48 follows the cloned MAC to the other box, so the tunnel
# source is the same address on either router and there is nothing to tell HE.
# A non-zero count here means something re-pointed the tunnel at a PPPoE address
# -- which Vodafone re-issues on every dial, so it would be wrong within minutes.
say "=== IPv6 invariants ==="
tun_after="$(tun_of $P1)"
say "tun0 src     : ${tun_before:-none} -> ${tun_after:-none}"
[ "$tun_before" = "$tun_after" ] && say "  OK: tunnel source unchanged across the drill" \
                                 || say "  *** CHANGED -- a router failover should never move the HE endpoint ***"
for h in "$P1" "$P2"; do
    n="$(he_calls "$h" "$DRILL_START")"
    say "HE updates from $h since ${DRILL_START}: ${n:-?}"
    [ "${n:-0}" = 0 ] || say "  *** $h called the HE API during a router failover -- it should not need to ***"
done