#!/bin/sh # wan-panic -- get the internet back. No Claude, no internet, no thinking. # # Run it on EITHER router. It works out which box it is and does the right # thing. Safe to run twice, safe to run on both, safe to run when nothing is # wrong. # # sudo /config/wan-panic hand the WAN back to vyos001 # sudo /config/wan-panic status show who has what, change nothing # sudo /config/wan-panic undo also stop the whole vrrp-wan mechanism # # WHY THIS EXISTS: the WAN now follows VRRP mastership. If a failover leaves # the wrong box holding the VIPs, or the new master cannot raise a WAN, the # house has no internet -- and whoever is debugging it has no internet either. # So the recovery path must be a single command that is already on the box. # # HOW TO REACH THE ROUTERS WITH THE NETWORK BROKEN: ssh the LoT leg, # ssh vyos@10.0.1.252 (vyos001) # ssh vyos@10.0.1.253 (vyos002) # It is L2-direct on bond0.10 and survives Management, VRRP and routing being # broken. Console via the JetKVMs is the fallback. STATE=/run/vrrp-wan HOST="$(cat /etc/hostname 2>/dev/null || hostname)" VIP=192.168.1.1 [ -r /config/vrrp-wan.conf ] && . /config/vrrp-wan.conf VIP="${VRRP_WAN_VIP:-$VIP}" have_vip() { ip -4 -o addr show 2>/dev/null | grep -q " ${VIP}/"; } wan_list() { for i in bond0.53 pppoe0; do a=$(ip -4 addr show dev "$i" 2>/dev/null | sed -n 's/.*inet \([0-9.]*\).*/\1/p') [ -n "$a" ] && printf '%s=%s ' "$i" "$a" done } show() { printf ' host : %s\n' "$HOST" printf ' holds VIP : %s\n' "$(have_vip && echo YES || echo no)" printf ' WAN : %s\n' "$(wan_list)" printf ' route : %s\n' "$(ip route show default 2>/dev/null | head -1)" printf ' force-fault: %s\n' "$([ -f "$STATE/force-fault" ] && echo SET || echo clear)" } case "${1:-failback}" in status) show exit 0 ;; undo) # Full stop. Leaves whatever WAN is currently up exactly as it is and # stops anything from moving it again. Use when the mechanism itself is # suspect. `vyos-known-good restore` is the heavier hammer below. systemctl disable --now vrrp-wan-reconcile.timer vrrp-wan-guard.timer 2>/dev/null rm -f "$STATE/force-fault" echo " vrrp-wan timers stopped. Nothing will move the WAN now." echo " The 10 gig / PPPoE stay exactly as they are this second." echo echo " If the config itself is wrong, the heavier hammer is:" echo " sudo /config/vyos-known-good restore" echo " on BOTH routers (it reboots them onto the pinned config)." echo show exit 0 ;; failback) # The common case: put vyos001 back in charge. # # It is done with the force-fault lever, not by restarting keepalived, # because failing the health check is the SUPPORTED way to shed # mastership -- the sync group goes FAULT, releases every VIP, and the # peer takes over by the same path a genuine WAN loss uses. `restart # vrrp` is not dependable: with advert_int 1 the peer declares the master # dead in ~3.6s and a restart usually finishes inside that window. case "$HOST" in *002|*2) # This is the secondary. Stand down so vyos001 can have it back. mkdir -p "$STATE" 2>/dev/null touch "$STATE/force-fault" echo " $HOST: standing down (force-fault SET)." echo " vyos001 should take the VIPs and raise the WAN within ~30s." echo echo " When the dust settles and you WANT this box eligible again:" echo " sudo rm /run/vrrp-wan/force-fault" ;; *) # This is the primary. Make sure nothing is holding it back. rm -f "$STATE/force-fault" echo " $HOST: cleared force-fault -- eligible for MASTER." echo " NOTE: VRRP is no-preempt, so if the peer currently holds the" echo " VIPs it KEEPS them. To actually take them back, run this on" echo " the OTHER box (vyos002 / 10.0.1.253):" echo " sudo /config/wan-panic" ;; esac # Reconcile now rather than waiting up to 30s for the timer. [ -x /config/vrrp-wan-reconcile ] && /config/vrrp-wan-reconcile 9>&- 2>/dev/null echo show exit 0 ;; *) echo "usage: wan-panic [failback|status|undo]" >&2 exit 2 ;; esac