lab/migration/wan-panic

#!/bin/sh
# wan-panic -- get the internet back. No Claude, no internet, no thinking.
#
# Run it on EITHER router. It works out which box it is and does the right
# thing. Safe to run twice, safe to run on both, safe to run when nothing is
# wrong.
#
#     sudo /config/wan-panic          hand the WAN back to vyos001
#     sudo /config/wan-panic status   show who has what, change nothing
#     sudo /config/wan-panic undo     also stop the whole vrrp-wan mechanism
#
# WHY THIS EXISTS: the WAN now follows VRRP mastership. If a failover leaves
# the wrong box holding the VIPs, or the new master cannot raise a WAN, the
# house has no internet -- and whoever is debugging it has no internet either.
# So the recovery path must be a single command that is already on the box.
#
# HOW TO REACH THE ROUTERS WITH THE NETWORK BROKEN: ssh the LoT leg,
#   ssh vyos@10.0.1.252   (vyos001)
#   ssh vyos@10.0.1.253   (vyos002)
# It is L2-direct on bond0.10 and survives Management, VRRP and routing being
# broken. Console via the JetKVMs is the fallback.

STATE=/run/vrrp-wan
HOST="$(cat /etc/hostname 2>/dev/null || hostname)"
VIP=192.168.1.1
[ -r /config/vrrp-wan.conf ] && . /config/vrrp-wan.conf
VIP="${VRRP_WAN_VIP:-$VIP}"

have_vip() { ip -4 -o addr show 2>/dev/null | grep -q " ${VIP}/"; }
wan_list() {
    for i in bond0.53 pppoe0; do
        a=$(ip -4 addr show dev "$i" 2>/dev/null | sed -n 's/.*inet \([0-9.]*\).*/\1/p')
        [ -n "$a" ] && printf '%s=%s ' "$i" "$a"
    done
}

show() {
    printf '  host      : %s\n' "$HOST"
    printf '  holds VIP : %s\n' "$(have_vip && echo YES || echo no)"
    printf '  WAN       : %s\n' "$(wan_list)"
    printf '  route     : %s\n' "$(ip route show default 2>/dev/null | head -1)"
    printf '  force-fault: %s\n' "$([ -f "$STATE/force-fault" ] && echo SET || echo clear)"
}

case "${1:-failback}" in
status)
    show
    exit 0
    ;;

undo)
    # Full stop. Leaves whatever WAN is currently up exactly as it is and
    # stops anything from moving it again. Use when the mechanism itself is
    # suspect. `vyos-known-good restore` is the heavier hammer below.
    systemctl disable --now vrrp-wan-reconcile.timer vrrp-wan-guard.timer 2>/dev/null
    rm -f "$STATE/force-fault"
    echo "  vrrp-wan timers stopped. Nothing will move the WAN now."
    echo "  The 10 gig / PPPoE stay exactly as they are this second."
    echo
    echo "  If the config itself is wrong, the heavier hammer is:"
    echo "      sudo /config/vyos-known-good restore"
    echo "  on BOTH routers (it reboots them onto the pinned config)."
    echo
    show
    exit 0
    ;;

failback)
    # The common case: put vyos001 back in charge.
    #
    # It is done with the force-fault lever, not by restarting keepalived,
    # because failing the health check is the SUPPORTED way to shed
    # mastership -- the sync group goes FAULT, releases every VIP, and the
    # peer takes over by the same path a genuine WAN loss uses. `restart
    # vrrp` is not dependable: with advert_int 1 the peer declares the master
    # dead in ~3.6s and a restart usually finishes inside that window.
    case "$HOST" in
    *002|*2)
        # This is the secondary. Stand down so vyos001 can have it back.
        mkdir -p "$STATE" 2>/dev/null
        touch "$STATE/force-fault"
        echo "  $HOST: standing down (force-fault SET)."
        echo "  vyos001 should take the VIPs and raise the WAN within ~30s."
        echo
        echo "  When the dust settles and you WANT this box eligible again:"
        echo "      sudo rm /run/vrrp-wan/force-fault"
        ;;
    *)
        # This is the primary. Make sure nothing is holding it back.
        rm -f "$STATE/force-fault"
        echo "  $HOST: cleared force-fault -- eligible for MASTER."
        echo "  NOTE: VRRP is no-preempt, so if the peer currently holds the"
        echo "  VIPs it KEEPS them. To actually take them back, run this on"
        echo "  the OTHER box (vyos002 / 10.0.1.253):"
        echo "      sudo /config/wan-panic"
        ;;
    esac
    # Reconcile now rather than waiting up to 30s for the timer.
    [ -x /config/vrrp-wan-reconcile ] && /config/vrrp-wan-reconcile 9>&- 2>/dev/null
    echo
    show
    exit 0
    ;;

*)
    echo "usage: wan-panic [failback|status|undo]" >&2
    exit 2
    ;;
esac