106 lines
4.3 KiB
Plaintext
106 lines
4.3 KiB
Plaintext
|
#!/usr/bin/env bash
|
||
|
|
# Controlled WAN failover drill. Runs from the workstation over the LAN.
|
||
|
|
#
|
||
|
|
# It is written as ONE unattended script on purpose. The drill takes the
|
||
|
|
# household's internet down, so anything driving it step-by-step from off-site
|
||
|
|
# -- a person on a laptop, or an agent that needs the internet to think --
|
||
|
|
# stops being able to act at exactly the moment it matters. This script only
|
||
|
|
# needs the LAN, and it always runs its cleanup.
|
||
|
|
#
|
||
|
|
# ./wan-drill run it
|
||
|
|
# ./wan-drill --dry show what it would do, touch nothing
|
||
|
|
#
|
||
|
|
# Recovery if this script itself dies: see migration/RECOVERY-CARD-wan-panic.md
|
||
|
|
# or /config/RECOVERY-CARD.md on either router. Short version, on vyos002:
|
||
|
|
# sudo /config/wan-panic
|
||
|
|
set -uo pipefail
|
||
|
|
|
||
|
|
P1=10.0.1.252 # vyos001, normally MASTER
|
||
|
|
P2=10.0.1.253 # vyos002, normally BACKUP
|
||
|
|
PW=vyos
|
||
|
|
SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
||
|
|
-o LogLevel=ERROR -o ConnectTimeout=5)
|
||
|
|
TAKEOVER_BUDGET=240 # give vyos002 this long to raise a WAN
|
||
|
|
FAILBACK_BUDGET=180 # and vyos001 this long to take it back
|
||
|
|
WATCHDOG_HOLD=150 # vyos002 stands down by itself after this with no WAN
|
||
|
|
|
||
|
|
LOG="${LOG:-/tmp/wan-drill-$(date +%H%M%S).log}"
|
||
|
|
DRY=0; [ "${1:-}" = "--dry" ] && DRY=1
|
||
|
|
|
||
|
|
say() { printf '%s %s\n' "$(date +%T)" "$*" | tee -a "$LOG"; }
|
||
|
|
r() { timeout 25 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; }
|
||
|
|
|
||
|
|
holder() { for h in "$P1" "$P2"; do
|
||
|
|
[ "$(r "$h" 'ip -4 -o addr show | grep -c " 192.168.1.1/"' | tr -d ' \n')" != 0 ] \
|
||
|
|
&& { echo "$h"; return; }; done; echo none; }
|
||
|
|
wan_of() { r "$1" 'for i in bond0.53 pppoe0; do a=$(ip -4 addr show dev $i 2>/dev/null | sed -n "s/.*inet \([0-9.]*\).*/\1/p"); [ -n "$a" ] && printf "%s=%s " $i $a; done'; }
|
||
|
|
online() { [ "$(r "$1" 'ping -c1 -W2 9.9.9.9 >/dev/null 2>&1 && echo y' | tr -d ' \n')" = y ]; }
|
||
|
|
|
||
|
|
cleanup() {
|
||
|
|
say "--- cleanup (always runs) ---"
|
||
|
|
r "$P2" 'sudo /config/wan-drill-watchdog disarm' >/dev/null
|
||
|
|
r "$P1" 'sudo rm -f /run/vrrp-wan/force-fault' >/dev/null
|
||
|
|
r "$P2" 'sudo rm -f /run/vrrp-wan/force-fault' >/dev/null
|
||
|
|
sleep 20
|
||
|
|
say "final holder : $(holder)"
|
||
|
|
say "final WAN : $P1 [$(wan_of $P1)] $P2 [$(wan_of $P2)]"
|
||
|
|
online "$P1" && say "final internet: UP via $P1" || {
|
||
|
|
online "$P2" && say "final internet: UP via $P2" \
|
||
|
|
|| say "final internet: *** DOWN -- run: sudo /config/wan-panic on $P2 ***"; }
|
||
|
|
say "log: $LOG"
|
||
|
|
}
|
||
|
|
trap cleanup EXIT
|
||
|
|
|
||
|
|
say "=== pre-flight ==="
|
||
|
|
start_holder="$(holder)"
|
||
|
|
say "holder now : $start_holder"
|
||
|
|
say "$P1 WAN : $(wan_of $P1)"
|
||
|
|
say "$P2 WAN : $(wan_of $P2)"
|
||
|
|
online "$P1" && say "internet : UP via $P1" || { say "internet ALREADY DOWN -- refusing to drill"; exit 1; }
|
||
|
|
[ "$start_holder" = "$P1" ] || { say "expected $P1 to hold the VIP, got $start_holder -- refusing"; exit 1; }
|
||
|
|
|
||
|
|
if [ "$DRY" = 1 ]; then
|
||
|
|
say "--dry: would arm the watchdog on $P2 (${WATCHDOG_HOLD}s) and force-fault $P1"
|
||
|
|
trap - EXIT; exit 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
say "=== arming auto-abort on $P2 ==="
|
||
|
|
r "$P2" "sudo /config/wan-drill-watchdog arm $WATCHDOG_HOLD" | tee -a "$LOG"
|
||
|
|
|
||
|
|
say "=== DRILL: force-faulting $P1 ==="
|
||
|
|
t0=$(date +%s)
|
||
|
|
r "$P1" 'sudo touch /run/vrrp-wan/force-fault'
|
||
|
|
|
||
|
|
took=""
|
||
|
|
while [ $(( $(date +%s) - t0 )) -lt "$TAKEOVER_BUDGET" ]; do
|
||
|
|
sleep 5
|
||
|
|
h="$(holder)"; w="$(wan_of $P2)"
|
||
|
|
say " t+$(( $(date +%s) - t0 ))s holder=$h vyos002_wan=[$w]"
|
||
|
|
if [ "$h" = "$P2" ] && [ -n "$w" ] && online "$P2"; then
|
||
|
|
took=$(( $(date +%s) - t0 )); break
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
if [ -n "$took" ]; then
|
||
|
|
say "*** TAKEOVER OK: $P2 held the VIP and reached the internet in ${took}s ***"
|
||
|
|
else
|
||
|
|
say "*** TAKEOVER FAILED within ${TAKEOVER_BUDGET}s -- failing back ***"
|
||
|
|
fi
|
||
|
|
|
||
|
|
say "=== failing back to $P1 ==="
|
||
|
|
t1=$(date +%s)
|
||
|
|
r "$P2" 'sudo /config/wan-drill-watchdog disarm' >/dev/null
|
||
|
|
r "$P1" 'sudo rm -f /run/vrrp-wan/force-fault'
|
||
|
|
r "$P2" 'sudo touch /run/vrrp-wan/force-fault'
|
||
|
|
back=""
|
||
|
|
while [ $(( $(date +%s) - t1 )) -lt "$FAILBACK_BUDGET" ]; do
|
||
|
|
sleep 5
|
||
|
|
h="$(holder)"; w="$(wan_of $P1)"
|
||
|
|
say " t+$(( $(date +%s) - t1 ))s holder=$h vyos001_wan=[$w]"
|
||
|
|
if [ "$h" = "$P1" ] && [ -n "$w" ] && online "$P1"; then
|
||
|
|
back=$(( $(date +%s) - t1 )); break
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
[ -n "$back" ] && say "*** FAILBACK OK in ${back}s ***" \
|
||
|
|
|| say "*** FAILBACK FAILED -- cleanup will clear both levers ***"
|