diff --git a/migration/wan-drill-watchdog b/migration/wan-drill-watchdog new file mode 100644 index 0000000..8b197d9 --- /dev/null +++ b/migration/wan-drill-watchdog @@ -0,0 +1,83 @@ +#!/bin/sh +# Auto-abort for a WAN failover drill. Armed on the router that is EXPECTED TO +# TAKE OVER, before the drill starts. +# +# The drill takes the internet down for as long as the new master needs to +# raise a WAN. If it never does, whoever is running the drill has no internet +# either -- and if that is an agent, it simply stops responding mid-incident. +# So the abort cannot depend on anyone being there. +# +# Rule: if I hold the VIP and have had NO WAN for HOLD consecutive seconds, +# stand down. force-fault sheds every VIP and the peer -- which is healthy and +# merely non-preempting -- takes them straight back. +# +# This is deliberately SHORTER than GRACE in vrrp-wan.conf (300s). GRACE is +# sized for a real hostile-ISP takeover that is still making progress; this is +# sized for "the drill failed, give the house its internet back". Anything the +# drill proves after two and a half minutes of downtime is not worth the +# downtime. +# +# wan-drill-watchdog arm [seconds] background it, then run the drill +# wan-drill-watchdog disarm cancel it (drill succeeded) + +STATE=/run/vrrp-wan +VIP=192.168.1.1 +[ -r /config/vrrp-wan.conf ] && . /config/vrrp-wan.conf +VIP="${VRRP_WAN_VIP:-$VIP}" +HOLD="${2:-150}" +PIDF=/run/wan-drill-watchdog.pid + +have_vip() { ip -4 -o addr show 2>/dev/null | grep -q " ${VIP}/"; } +have_wan() { + ip -4 addr show dev bond0.53 2>/dev/null | grep -q 'inet ' && return 0 + ip -4 addr show dev pppoe0 2>/dev/null | grep -q 'inet ' && return 0 + return 1 +} + +case "${1:-arm}" in +disarm) + if [ -f "$PIDF" ]; then + kill "$(cat "$PIDF")" 2>/dev/null + rm -f "$PIDF" + echo " watchdog disarmed" + else + echo " no watchdog armed" + fi + exit 0 + ;; +arm) + [ -f "$PIDF" ] && kill "$(cat "$PIDF")" 2>/dev/null + mkdir -p "$STATE" 2>/dev/null + # setsid so it survives the ssh session that armed it going away -- the + # whole point is that it outlives whoever started the drill. + setsid sh -c ' + bad=0 + while :; do + sleep 5 + if ip -4 -o addr show 2>/dev/null | grep -q " '"$VIP"'/"; then + if ip -4 addr show dev bond0.53 2>/dev/null | grep -q "inet " || + ip -4 addr show dev pppoe0 2>/dev/null | grep -q "inet "; then + bad=0 + else + bad=$((bad + 5)) + fi + else + bad=0 + fi + if [ "$bad" -ge '"$HOLD"' ]; then + logger -t wan-drill "ABORT: held the VIP with no WAN for '"$HOLD"'s -- standing down" + touch '"$STATE"'/force-fault + rm -f '"$PIDF"' + exit 0 + fi + done + ' >/dev/null 2>&1 & + echo $! > "$PIDF" + echo " watchdog armed (pid $(cat "$PIDF")): stand down after ${HOLD}s holding the VIP with no WAN" + exit 0 + ;; +*) + echo "usage: wan-drill-watchdog [arm [seconds]|disarm]" >&2 + exit 2 + ;; +esac