diff --git a/migration/wan-drill b/migration/wan-drill new file mode 100755 index 0000000..ab361c0 --- /dev/null +++ b/migration/wan-drill @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Controlled WAN failover drill. Runs from the workstation over the LAN. +# +# It is written as ONE unattended script on purpose. The drill takes the +# household's internet down, so anything driving it step-by-step from off-site +# -- a person on a laptop, or an agent that needs the internet to think -- +# stops being able to act at exactly the moment it matters. This script only +# needs the LAN, and it always runs its cleanup. +# +# ./wan-drill run it +# ./wan-drill --dry show what it would do, touch nothing +# +# Recovery if this script itself dies: see migration/RECOVERY-CARD-wan-panic.md +# or /config/RECOVERY-CARD.md on either router. Short version, on vyos002: +# sudo /config/wan-panic +set -uo pipefail + +P1=10.0.1.252 # vyos001, normally MASTER +P2=10.0.1.253 # vyos002, normally BACKUP +PW=vyos +SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR -o ConnectTimeout=5) +TAKEOVER_BUDGET=240 # give vyos002 this long to raise a WAN +FAILBACK_BUDGET=180 # and vyos001 this long to take it back +WATCHDOG_HOLD=150 # vyos002 stands down by itself after this with no WAN + +LOG="${LOG:-/tmp/wan-drill-$(date +%H%M%S).log}" +DRY=0; [ "${1:-}" = "--dry" ] && DRY=1 + +say() { printf '%s %s\n' "$(date +%T)" "$*" | tee -a "$LOG"; } +r() { timeout 25 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; } + +holder() { for h in "$P1" "$P2"; do + [ "$(r "$h" 'ip -4 -o addr show | grep -c " 192.168.1.1/"' | tr -d ' \n')" != 0 ] \ + && { echo "$h"; return; }; done; echo none; } +wan_of() { r "$1" 'for i in bond0.53 pppoe0; do a=$(ip -4 addr show dev $i 2>/dev/null | sed -n "s/.*inet \([0-9.]*\).*/\1/p"); [ -n "$a" ] && printf "%s=%s " $i $a; done'; } +online() { [ "$(r "$1" 'ping -c1 -W2 9.9.9.9 >/dev/null 2>&1 && echo y' | tr -d ' \n')" = y ]; } + +cleanup() { + say "--- cleanup (always runs) ---" + r "$P2" 'sudo /config/wan-drill-watchdog disarm' >/dev/null + r "$P1" 'sudo rm -f /run/vrrp-wan/force-fault' >/dev/null + r "$P2" 'sudo rm -f /run/vrrp-wan/force-fault' >/dev/null + sleep 20 + say "final holder : $(holder)" + say "final WAN : $P1 [$(wan_of $P1)] $P2 [$(wan_of $P2)]" + online "$P1" && say "final internet: UP via $P1" || { + online "$P2" && say "final internet: UP via $P2" \ + || say "final internet: *** DOWN -- run: sudo /config/wan-panic on $P2 ***"; } + say "log: $LOG" +} +trap cleanup EXIT + +say "=== pre-flight ===" +start_holder="$(holder)" +say "holder now : $start_holder" +say "$P1 WAN : $(wan_of $P1)" +say "$P2 WAN : $(wan_of $P2)" +online "$P1" && say "internet : UP via $P1" || { say "internet ALREADY DOWN -- refusing to drill"; exit 1; } +[ "$start_holder" = "$P1" ] || { say "expected $P1 to hold the VIP, got $start_holder -- refusing"; exit 1; } + +if [ "$DRY" = 1 ]; then + say "--dry: would arm the watchdog on $P2 (${WATCHDOG_HOLD}s) and force-fault $P1" + trap - EXIT; exit 0 +fi + +say "=== arming auto-abort on $P2 ===" +r "$P2" "sudo /config/wan-drill-watchdog arm $WATCHDOG_HOLD" | tee -a "$LOG" + +say "=== DRILL: force-faulting $P1 ===" +t0=$(date +%s) +r "$P1" 'sudo touch /run/vrrp-wan/force-fault' + +took="" +while [ $(( $(date +%s) - t0 )) -lt "$TAKEOVER_BUDGET" ]; do + sleep 5 + h="$(holder)"; w="$(wan_of $P2)" + say " t+$(( $(date +%s) - t0 ))s holder=$h vyos002_wan=[$w]" + if [ "$h" = "$P2" ] && [ -n "$w" ] && online "$P2"; then + took=$(( $(date +%s) - t0 )); break + fi +done + +if [ -n "$took" ]; then + say "*** TAKEOVER OK: $P2 held the VIP and reached the internet in ${took}s ***" +else + say "*** TAKEOVER FAILED within ${TAKEOVER_BUDGET}s -- failing back ***" +fi + +say "=== failing back to $P1 ===" +t1=$(date +%s) +r "$P2" 'sudo /config/wan-drill-watchdog disarm' >/dev/null +r "$P1" 'sudo rm -f /run/vrrp-wan/force-fault' +r "$P2" 'sudo touch /run/vrrp-wan/force-fault' +back="" +while [ $(( $(date +%s) - t1 )) -lt "$FAILBACK_BUDGET" ]; do + sleep 5 + h="$(holder)"; w="$(wan_of $P1)" + say " t+$(( $(date +%s) - t1 ))s holder=$h vyos001_wan=[$w]" + if [ "$h" = "$P1" ] && [ -n "$w" ] && online "$P1"; then + back=$(( $(date +%s) - t1 )); break + fi +done +[ -n "$back" ] && say "*** FAILBACK OK in ${back}s ***" \ + || say "*** FAILBACK FAILED -- cleanup will clear both levers ***"