wan-drill: run the failover drill unattended, because the operator goes offline
Some checks failed
Some checks failed
The drill takes the household's internet down, which means anything driving it step-by-step stops being able to act at exactly the moment it matters -- an agent needs the internet to think, so it would freeze mid-failover with the levers half-thrown. This script needs only the LAN, and its cleanup runs from an EXIT trap so both force-fault levers are cleared on any exit path, including being killed. Belt and braces around it: the watchdog on vyos002 stands the box down by itself after 150s holding the VIP with no WAN, so even if the script dies the internet comes back. Refuses to start if the internet is already down or if vyos001 is not the current holder.
This commit is contained in:
Michal
105
migration/wan-drill
Executable file
105
migration/wan-drill
Executable file
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env bash
|
||||
# Controlled WAN failover drill. Runs from the workstation over the LAN.
|
||||
#
|
||||
# It is written as ONE unattended script on purpose. The drill takes the
|
||||
# household's internet down, so anything driving it step-by-step from off-site
|
||||
# -- a person on a laptop, or an agent that needs the internet to think --
|
||||
# stops being able to act at exactly the moment it matters. This script only
|
||||
# needs the LAN, and it always runs its cleanup.
|
||||
#
|
||||
# ./wan-drill run it
|
||||
# ./wan-drill --dry show what it would do, touch nothing
|
||||
#
|
||||
# Recovery if this script itself dies: see migration/RECOVERY-CARD-wan-panic.md
|
||||
# or /config/RECOVERY-CARD.md on either router. Short version, on vyos002:
|
||||
# sudo /config/wan-panic
|
||||
set -uo pipefail
|
||||
|
||||
P1=10.0.1.252 # vyos001, normally MASTER
|
||||
P2=10.0.1.253 # vyos002, normally BACKUP
|
||||
PW=vyos
|
||||
SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
||||
-o LogLevel=ERROR -o ConnectTimeout=5)
|
||||
TAKEOVER_BUDGET=240 # give vyos002 this long to raise a WAN
|
||||
FAILBACK_BUDGET=180 # and vyos001 this long to take it back
|
||||
WATCHDOG_HOLD=150 # vyos002 stands down by itself after this with no WAN
|
||||
|
||||
LOG="${LOG:-/tmp/wan-drill-$(date +%H%M%S).log}"
|
||||
DRY=0; [ "${1:-}" = "--dry" ] && DRY=1
|
||||
|
||||
say() { printf '%s %s\n' "$(date +%T)" "$*" | tee -a "$LOG"; }
|
||||
r() { timeout 25 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; }
|
||||
|
||||
holder() { for h in "$P1" "$P2"; do
|
||||
[ "$(r "$h" 'ip -4 -o addr show | grep -c " 192.168.1.1/"' | tr -d ' \n')" != 0 ] \
|
||||
&& { echo "$h"; return; }; done; echo none; }
|
||||
wan_of() { r "$1" 'for i in bond0.53 pppoe0; do a=$(ip -4 addr show dev $i 2>/dev/null | sed -n "s/.*inet \([0-9.]*\).*/\1/p"); [ -n "$a" ] && printf "%s=%s " $i $a; done'; }
|
||||
online() { [ "$(r "$1" 'ping -c1 -W2 9.9.9.9 >/dev/null 2>&1 && echo y' | tr -d ' \n')" = y ]; }
|
||||
|
||||
cleanup() {
|
||||
say "--- cleanup (always runs) ---"
|
||||
r "$P2" 'sudo /config/wan-drill-watchdog disarm' >/dev/null
|
||||
r "$P1" 'sudo rm -f /run/vrrp-wan/force-fault' >/dev/null
|
||||
r "$P2" 'sudo rm -f /run/vrrp-wan/force-fault' >/dev/null
|
||||
sleep 20
|
||||
say "final holder : $(holder)"
|
||||
say "final WAN : $P1 [$(wan_of $P1)] $P2 [$(wan_of $P2)]"
|
||||
online "$P1" && say "final internet: UP via $P1" || {
|
||||
online "$P2" && say "final internet: UP via $P2" \
|
||||
|| say "final internet: *** DOWN -- run: sudo /config/wan-panic on $P2 ***"; }
|
||||
say "log: $LOG"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
say "=== pre-flight ==="
|
||||
start_holder="$(holder)"
|
||||
say "holder now : $start_holder"
|
||||
say "$P1 WAN : $(wan_of $P1)"
|
||||
say "$P2 WAN : $(wan_of $P2)"
|
||||
online "$P1" && say "internet : UP via $P1" || { say "internet ALREADY DOWN -- refusing to drill"; exit 1; }
|
||||
[ "$start_holder" = "$P1" ] || { say "expected $P1 to hold the VIP, got $start_holder -- refusing"; exit 1; }
|
||||
|
||||
if [ "$DRY" = 1 ]; then
|
||||
say "--dry: would arm the watchdog on $P2 (${WATCHDOG_HOLD}s) and force-fault $P1"
|
||||
trap - EXIT; exit 0
|
||||
fi
|
||||
|
||||
say "=== arming auto-abort on $P2 ==="
|
||||
r "$P2" "sudo /config/wan-drill-watchdog arm $WATCHDOG_HOLD" | tee -a "$LOG"
|
||||
|
||||
say "=== DRILL: force-faulting $P1 ==="
|
||||
t0=$(date +%s)
|
||||
r "$P1" 'sudo touch /run/vrrp-wan/force-fault'
|
||||
|
||||
took=""
|
||||
while [ $(( $(date +%s) - t0 )) -lt "$TAKEOVER_BUDGET" ]; do
|
||||
sleep 5
|
||||
h="$(holder)"; w="$(wan_of $P2)"
|
||||
say " t+$(( $(date +%s) - t0 ))s holder=$h vyos002_wan=[$w]"
|
||||
if [ "$h" = "$P2" ] && [ -n "$w" ] && online "$P2"; then
|
||||
took=$(( $(date +%s) - t0 )); break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$took" ]; then
|
||||
say "*** TAKEOVER OK: $P2 held the VIP and reached the internet in ${took}s ***"
|
||||
else
|
||||
say "*** TAKEOVER FAILED within ${TAKEOVER_BUDGET}s -- failing back ***"
|
||||
fi
|
||||
|
||||
say "=== failing back to $P1 ==="
|
||||
t1=$(date +%s)
|
||||
r "$P2" 'sudo /config/wan-drill-watchdog disarm' >/dev/null
|
||||
r "$P1" 'sudo rm -f /run/vrrp-wan/force-fault'
|
||||
r "$P2" 'sudo touch /run/vrrp-wan/force-fault'
|
||||
back=""
|
||||
while [ $(( $(date +%s) - t1 )) -lt "$FAILBACK_BUDGET" ]; do
|
||||
sleep 5
|
||||
h="$(holder)"; w="$(wan_of $P1)"
|
||||
say " t+$(( $(date +%s) - t1 ))s holder=$h vyos001_wan=[$w]"
|
||||
if [ "$h" = "$P1" ] && [ -n "$w" ] && online "$P1"; then
|
||||
back=$(( $(date +%s) - t1 )); break
|
||||
fi
|
||||
done
|
||||
[ -n "$back" ] && say "*** FAILBACK OK in ${back}s ***" \
|
||||
|| say "*** FAILBACK FAILED -- cleanup will clear both levers ***"
|
||||
Reference in New Issue
Block a user