#!/usr/bin/env bash
# Controlled WAN failover drill. Runs from the workstation over the LAN.
#
# It is written as ONE unattended script on purpose. The drill takes the
# household's internet down, so anything driving it step-by-step from off-site
# -- a person on a laptop, or an agent that needs the internet to think --
# stops being able to act at exactly the moment it matters. This script only
# needs the LAN, and it always runs its cleanup.
#
#   ./wan-drill                 run it
#   ./wan-drill --dry           show what it would do, touch nothing
#
# Recovery if this script itself dies: see migration/RECOVERY-CARD-wan-panic.md
# or /config/RECOVERY-CARD.md on either router. Short version, on vyos002:
#     sudo /config/wan-panic
set -uo pipefail

P1=10.0.1.252   # vyos001, normally MASTER
P2=10.0.1.253   # vyos002, normally BACKUP
PW=vyos
SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
     -o LogLevel=ERROR -o ConnectTimeout=5)
TAKEOVER_BUDGET=240    # give vyos002 this long to raise a WAN
FAILBACK_BUDGET=180    # and vyos001 this long to take it back
WATCHDOG_HOLD=150      # vyos002 stands down by itself after this with no WAN

LOG="${LOG:-/tmp/wan-drill-$(date +%H%M%S).log}"
DRY=0; [ "${1:-}" = "--dry" ] && DRY=1

say() { printf '%s %s\n' "$(date +%T)" "$*" | tee -a "$LOG"; }
r()   { timeout 25 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; }

holder()  { for h in "$P1" "$P2"; do
              [ "$(r "$h" 'ip -4 -o addr show | grep -c " 192.168.1.1/"' | tr -d ' \n')" != 0 ] \
                && { echo "$h"; return; }; done; echo none; }
wan_of()  { r "$1" 'for i in bond0.53 pppoe0; do a=$(ip -4 addr show dev $i 2>/dev/null | sed -n "s/.*inet \([0-9.]*\).*/\1/p"); [ -n "$a" ] && printf "%s=%s " $i $a; done'; }
online()  { [ "$(r "$1" 'ping -c1 -W2 9.9.9.9 >/dev/null 2>&1 && echo y' | tr -d ' \n')" = y ]; }

cleanup() {
    say "--- cleanup (always runs) ---"
    r "$P2" 'sudo /config/wan-drill-watchdog disarm' >/dev/null
    r "$P1" 'sudo rm -f /run/vrrp-wan/force-fault' >/dev/null
    r "$P2" 'sudo rm -f /run/vrrp-wan/force-fault' >/dev/null
    sleep 20
    say "final holder : $(holder)"
    say "final WAN    : $P1 [$(wan_of $P1)]  $P2 [$(wan_of $P2)]"
    online "$P1" && say "final internet: UP via $P1" || {
        online "$P2" && say "final internet: UP via $P2" \
                     || say "final internet: *** DOWN -- run: sudo /config/wan-panic on $P2 ***"; }
    say "log: $LOG"
}
trap cleanup EXIT

say "=== pre-flight ==="
start_holder="$(holder)"
say "holder now   : $start_holder"
say "$P1 WAN      : $(wan_of $P1)"
say "$P2 WAN      : $(wan_of $P2)"
online "$P1" && say "internet     : UP via $P1" || { say "internet ALREADY DOWN -- refusing to drill"; exit 1; }
[ "$start_holder" = "$P1" ] || { say "expected $P1 to hold the VIP, got $start_holder -- refusing"; exit 1; }

if [ "$DRY" = 1 ]; then
    say "--dry: would arm the watchdog on $P2 (${WATCHDOG_HOLD}s) and force-fault $P1"
    trap - EXIT; exit 0
fi

say "=== arming auto-abort on $P2 ==="
r "$P2" "sudo /config/wan-drill-watchdog arm $WATCHDOG_HOLD" | tee -a "$LOG"

say "=== DRILL: force-faulting $P1 ==="
t0=$(date +%s)
r "$P1" 'sudo touch /run/vrrp-wan/force-fault'

took=""
while [ $(( $(date +%s) - t0 )) -lt "$TAKEOVER_BUDGET" ]; do
    sleep 5
    h="$(holder)"; w="$(wan_of $P2)"
    say "  t+$(( $(date +%s) - t0 ))s holder=$h  vyos002_wan=[$w]"
    if [ "$h" = "$P2" ] && [ -n "$w" ] && online "$P2"; then
        took=$(( $(date +%s) - t0 )); break
    fi
done

if [ -n "$took" ]; then
    say "*** TAKEOVER OK: $P2 held the VIP and reached the internet in ${took}s ***"
else
    say "*** TAKEOVER FAILED within ${TAKEOVER_BUDGET}s -- failing back ***"
fi

say "=== failing back to $P1 ==="
t1=$(date +%s)
r "$P2" 'sudo /config/wan-drill-watchdog disarm' >/dev/null
r "$P1" 'sudo rm -f /run/vrrp-wan/force-fault'
r "$P2" 'sudo touch /run/vrrp-wan/force-fault'
back=""
while [ $(( $(date +%s) - t1 )) -lt "$FAILBACK_BUDGET" ]; do
    sleep 5
    h="$(holder)"; w="$(wan_of $P1)"
    say "  t+$(( $(date +%s) - t1 ))s holder=$h  vyos001_wan=[$w]"
    if [ "$h" = "$P1" ] && [ -n "$w" ] && online "$P1"; then
        back=$(( $(date +%s) - t1 )); break
    fi
done
[ -n "$back" ] && say "*** FAILBACK OK in ${back}s ***" \
               || say "*** FAILBACK FAILED -- cleanup will clear both levers ***"