#!/bin/sh
# Auto-abort for a WAN failover drill. Armed on the router that is EXPECTED TO
# TAKE OVER, before the drill starts.
#
# The drill takes the internet down for as long as the new master needs to
# raise a WAN. If it never does, whoever is running the drill has no internet
# either -- and if that is an agent, it simply stops responding mid-incident.
# So the abort cannot depend on anyone being there.
#
# Rule: if I hold the VIP and have had NO WAN for HOLD consecutive seconds,
# stand down. force-fault sheds every VIP and the peer -- which is healthy and
# merely non-preempting -- takes them straight back.
#
# This is deliberately SHORTER than GRACE in vrrp-wan.conf (300s). GRACE is
# sized for a real hostile-ISP takeover that is still making progress; this is
# sized for "the drill failed, give the house its internet back". Anything the
# drill proves after two and a half minutes of downtime is not worth the
# downtime.
#
#   wan-drill-watchdog arm [seconds]     background it, then run the drill
#   wan-drill-watchdog disarm            cancel it (drill succeeded)

STATE=/run/vrrp-wan
VIP=192.168.1.1
[ -r /config/vrrp-wan.conf ] && . /config/vrrp-wan.conf
VIP="${VRRP_WAN_VIP:-$VIP}"
HOLD="${2:-150}"
PIDF=/run/wan-drill-watchdog.pid

have_vip() { ip -4 -o addr show 2>/dev/null | grep -q " ${VIP}/"; }
have_wan() {
    ip -4 addr show dev bond0.53 2>/dev/null | grep -q 'inet ' && return 0
    ip -4 addr show dev pppoe0   2>/dev/null | grep -q 'inet ' && return 0
    return 1
}

case "${1:-arm}" in
disarm)
    if [ -f "$PIDF" ]; then
        kill "$(cat "$PIDF")" 2>/dev/null
        rm -f "$PIDF"
        echo "  watchdog disarmed"
    else
        echo "  no watchdog armed"
    fi
    exit 0
    ;;
arm)
    [ -f "$PIDF" ] && kill "$(cat "$PIDF")" 2>/dev/null
    mkdir -p "$STATE" 2>/dev/null
    # setsid so it survives the ssh session that armed it going away -- the
    # whole point is that it outlives whoever started the drill.
    setsid sh -c '
        bad=0
        while :; do
            sleep 5
            if ip -4 -o addr show 2>/dev/null | grep -q " '"$VIP"'/"; then
                if ip -4 addr show dev bond0.53 2>/dev/null | grep -q "inet " ||
                   ip -4 addr show dev pppoe0 2>/dev/null | grep -q "inet "; then
                    bad=0
                else
                    bad=$((bad + 5))
                fi
            else
                bad=0
            fi
            if [ "$bad" -ge '"$HOLD"' ]; then
                logger -t wan-drill "ABORT: held the VIP with no WAN for '"$HOLD"'s -- standing down"
                touch '"$STATE"'/force-fault
                rm -f '"$PIDF"'
                exit 0
            fi
        done
    ' >/dev/null 2>&1 &
    echo $! > "$PIDF"
    echo "  watchdog armed (pid $(cat "$PIDF")): stand down after ${HOLD}s holding the VIP with no WAN"
    exit 0
    ;;
*)
    echo "usage: wan-drill-watchdog [arm [seconds]|disarm]" >&2
    exit 2
    ;;
esac