84 lines
2.9 KiB
Plaintext
84 lines
2.9 KiB
Plaintext
|
#!/bin/sh
|
||
|
|
# Auto-abort for a WAN failover drill. Armed on the router that is EXPECTED TO
|
||
|
|
# TAKE OVER, before the drill starts.
|
||
|
|
#
|
||
|
|
# The drill takes the internet down for as long as the new master needs to
|
||
|
|
# raise a WAN. If it never does, whoever is running the drill has no internet
|
||
|
|
# either -- and if that is an agent, it simply stops responding mid-incident.
|
||
|
|
# So the abort cannot depend on anyone being there.
|
||
|
|
#
|
||
|
|
# Rule: if I hold the VIP and have had NO WAN for HOLD consecutive seconds,
|
||
|
|
# stand down. force-fault sheds every VIP and the peer -- which is healthy and
|
||
|
|
# merely non-preempting -- takes them straight back.
|
||
|
|
#
|
||
|
|
# This is deliberately SHORTER than GRACE in vrrp-wan.conf (300s). GRACE is
|
||
|
|
# sized for a real hostile-ISP takeover that is still making progress; this is
|
||
|
|
# sized for "the drill failed, give the house its internet back". Anything the
|
||
|
|
# drill proves after two and a half minutes of downtime is not worth the
|
||
|
|
# downtime.
|
||
|
|
#
|
||
|
|
# wan-drill-watchdog arm [seconds] background it, then run the drill
|
||
|
|
# wan-drill-watchdog disarm cancel it (drill succeeded)
|
||
|
|
|
||
|
|
STATE=/run/vrrp-wan
|
||
|
|
VIP=192.168.1.1
|
||
|
|
[ -r /config/vrrp-wan.conf ] && . /config/vrrp-wan.conf
|
||
|
|
VIP="${VRRP_WAN_VIP:-$VIP}"
|
||
|
|
HOLD="${2:-150}"
|
||
|
|
PIDF=/run/wan-drill-watchdog.pid
|
||
|
|
|
||
|
|
have_vip() { ip -4 -o addr show 2>/dev/null | grep -q " ${VIP}/"; }
|
||
|
|
have_wan() {
|
||
|
|
ip -4 addr show dev bond0.53 2>/dev/null | grep -q 'inet ' && return 0
|
||
|
|
ip -4 addr show dev pppoe0 2>/dev/null | grep -q 'inet ' && return 0
|
||
|
|
return 1
|
||
|
|
}
|
||
|
|
|
||
|
|
case "${1:-arm}" in
|
||
|
|
disarm)
|
||
|
|
if [ -f "$PIDF" ]; then
|
||
|
|
kill "$(cat "$PIDF")" 2>/dev/null
|
||
|
|
rm -f "$PIDF"
|
||
|
|
echo " watchdog disarmed"
|
||
|
|
else
|
||
|
|
echo " no watchdog armed"
|
||
|
|
fi
|
||
|
|
exit 0
|
||
|
|
;;
|
||
|
|
arm)
|
||
|
|
[ -f "$PIDF" ] && kill "$(cat "$PIDF")" 2>/dev/null
|
||
|
|
mkdir -p "$STATE" 2>/dev/null
|
||
|
|
# setsid so it survives the ssh session that armed it going away -- the
|
||
|
|
# whole point is that it outlives whoever started the drill.
|
||
|
|
setsid sh -c '
|
||
|
|
bad=0
|
||
|
|
while :; do
|
||
|
|
sleep 5
|
||
|
|
if ip -4 -o addr show 2>/dev/null | grep -q " '"$VIP"'/"; then
|
||
|
|
if ip -4 addr show dev bond0.53 2>/dev/null | grep -q "inet " ||
|
||
|
|
ip -4 addr show dev pppoe0 2>/dev/null | grep -q "inet "; then
|
||
|
|
bad=0
|
||
|
|
else
|
||
|
|
bad=$((bad + 5))
|
||
|
|
fi
|
||
|
|
else
|
||
|
|
bad=0
|
||
|
|
fi
|
||
|
|
if [ "$bad" -ge '"$HOLD"' ]; then
|
||
|
|
logger -t wan-drill "ABORT: held the VIP with no WAN for '"$HOLD"'s -- standing down"
|
||
|
|
touch '"$STATE"'/force-fault
|
||
|
|
rm -f '"$PIDF"'
|
||
|
|
exit 0
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
' >/dev/null 2>&1 &
|
||
|
|
echo $! > "$PIDF"
|
||
|
|
echo " watchdog armed (pid $(cat "$PIDF")): stand down after ${HOLD}s holding the VIP with no WAN"
|
||
|
|
exit 0
|
||
|
|
;;
|
||
|
|
*)
|
||
|
|
echo "usage: wan-drill-watchdog [arm [seconds]|disarm]" >&2
|
||
|
|
exit 2
|
||
|
|
;;
|
||
|
|
esac
|