wan-drill: run the failover drill unattended, because the operator goes offline

The drill takes the household's internet down, which means anything driving it
step-by-step stops being able to act at exactly the moment it matters -- an
agent needs the internet to think, so it would freeze mid-failover with the
levers half-thrown. This script needs only the LAN, and its cleanup runs from
an EXIT trap so both force-fault levers are cleared on any exit path, including
being killed.

Belt and braces around it: the watchdog on vyos002 stands the box down by
itself after 150s holding the VIP with no WAN, so even if the script dies the
internet comes back. Refuses to start if the internet is already down or if
vyos001 is not the current holder.

migration/wan-drill

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+# Controlled WAN failover drill. Runs from the workstation over the LAN.
+#
+# It is written as ONE unattended script on purpose. The drill takes the
+# household's internet down, so anything driving it step-by-step from off-site
+# -- a person on a laptop, or an agent that needs the internet to think --
+# stops being able to act at exactly the moment it matters. This script only
+# needs the LAN, and it always runs its cleanup.
+#
+#   ./wan-drill                 run it
+#   ./wan-drill --dry           show what it would do, touch nothing
+#
+# Recovery if this script itself dies: see migration/RECOVERY-CARD-wan-panic.md
+# or /config/RECOVERY-CARD.md on either router. Short version, on vyos002:
+#     sudo /config/wan-panic
+set -uo pipefail
+
+P1=10.0.1.252   # vyos001, normally MASTER
+P2=10.0.1.253   # vyos002, normally BACKUP
+PW=vyos
+SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+     -o LogLevel=ERROR -o ConnectTimeout=5)
+TAKEOVER_BUDGET=240    # give vyos002 this long to raise a WAN
+FAILBACK_BUDGET=180    # and vyos001 this long to take it back
+WATCHDOG_HOLD=150      # vyos002 stands down by itself after this with no WAN
+
+LOG="${LOG:-/tmp/wan-drill-$(date +%H%M%S).log}"
+DRY=0; [ "${1:-}" = "--dry" ] && DRY=1
+
+say() { printf '%s %s\n' "$(date +%T)" "$*" | tee -a "$LOG"; }
+r()   { timeout 25 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; }
+
+holder()  { for h in "$P1" "$P2"; do
+              [ "$(r "$h" 'ip -4 -o addr show | grep -c " 192.168.1.1/"' | tr -d ' \n')" != 0 ] \
+                && { echo "$h"; return; }; done; echo none; }
+wan_of()  { r "$1" 'for i in bond0.53 pppoe0; do a=$(ip -4 addr show dev $i 2>/dev/null | sed -n "s/.*inet \([0-9.]*\).*/\1/p"); [ -n "$a" ] && printf "%s=%s " $i $a; done'; }
+online()  { [ "$(r "$1" 'ping -c1 -W2 9.9.9.9 >/dev/null 2>&1 && echo y' | tr -d ' \n')" = y ]; }
+
+cleanup() {
+    say "--- cleanup (always runs) ---"
+    r "$P2" 'sudo /config/wan-drill-watchdog disarm' >/dev/null
+    r "$P1" 'sudo rm -f /run/vrrp-wan/force-fault' >/dev/null
+    r "$P2" 'sudo rm -f /run/vrrp-wan/force-fault' >/dev/null
+    sleep 20
+    say "final holder : $(holder)"
+    say "final WAN    : $P1 [$(wan_of $P1)]  $P2 [$(wan_of $P2)]"
+    online "$P1" && say "final internet: UP via $P1" || {
+        online "$P2" && say "final internet: UP via $P2" \
+                     || say "final internet: *** DOWN -- run: sudo /config/wan-panic on $P2 ***"; }
+    say "log: $LOG"
+}
+trap cleanup EXIT
+
+say "=== pre-flight ==="
+start_holder="$(holder)"
+say "holder now   : $start_holder"
+say "$P1 WAN      : $(wan_of $P1)"
+say "$P2 WAN      : $(wan_of $P2)"
+online "$P1" && say "internet     : UP via $P1" || { say "internet ALREADY DOWN -- refusing to drill"; exit 1; }
+[ "$start_holder" = "$P1" ] || { say "expected $P1 to hold the VIP, got $start_holder -- refusing"; exit 1; }
+
+if [ "$DRY" = 1 ]; then
+    say "--dry: would arm the watchdog on $P2 (${WATCHDOG_HOLD}s) and force-fault $P1"
+    trap - EXIT; exit 0
+fi
+
+say "=== arming auto-abort on $P2 ==="
+r "$P2" "sudo /config/wan-drill-watchdog arm $WATCHDOG_HOLD" | tee -a "$LOG"
+
+say "=== DRILL: force-faulting $P1 ==="
+t0=$(date +%s)
+r "$P1" 'sudo touch /run/vrrp-wan/force-fault'
+
+took=""
+while [ $(( $(date +%s) - t0 )) -lt "$TAKEOVER_BUDGET" ]; do
+    sleep 5
+    h="$(holder)"; w="$(wan_of $P2)"
+    say "  t+$(( $(date +%s) - t0 ))s holder=$h  vyos002_wan=[$w]"
+    if [ "$h" = "$P2" ] && [ -n "$w" ] && online "$P2"; then
+        took=$(( $(date +%s) - t0 )); break
+    fi
+done
+
+if [ -n "$took" ]; then
+    say "*** TAKEOVER OK: $P2 held the VIP and reached the internet in ${took}s ***"
+else
+    say "*** TAKEOVER FAILED within ${TAKEOVER_BUDGET}s -- failing back ***"
+fi
+
+say "=== failing back to $P1 ==="
+t1=$(date +%s)
+r "$P2" 'sudo /config/wan-drill-watchdog disarm' >/dev/null
+r "$P1" 'sudo rm -f /run/vrrp-wan/force-fault'
+r "$P2" 'sudo touch /run/vrrp-wan/force-fault'
+back=""
+while [ $(( $(date +%s) - t1 )) -lt "$FAILBACK_BUDGET" ]; do
+    sleep 5
+    h="$(holder)"; w="$(wan_of $P1)"
+    say "  t+$(( $(date +%s) - t1 ))s holder=$h  vyos001_wan=[$w]"
+    if [ "$h" = "$P1" ] && [ -n "$w" ] && online "$P1"; then
+        back=$(( $(date +%s) - t1 )); break
+    fi
+done
+[ -n "$back" ] && say "*** FAILBACK OK in ${back}s ***" \
+               || say "*** FAILBACK FAILED -- cleanup will clear both levers ***"