lab/migration/vrrp-wan-health

#!/bin/sh
# VRRP health check: may THIS router hold the floating IPs?
#
# It may only if it can actually carry the WAN. Without this, VRRP decides
# mastership purely on whether the peer is still advertising -- so a router with
# no WAN at all happily takes the VIPs and blackholes the entire LAN's internet
# while looking perfectly healthy. That is not hypothetical: it is the outage of
# 2026-09-02, reproduced in labsim.
#
# ---------------------------------------------------------------------------
# The first version of this script asked one question: "do I have an address on
# a WAN interface". That is correct for a pair where both routers hold WAN all
# the time. Ours cannot: the 10 gig lease is bound to a cloned MAC and the
# PPPoE line to a single credential, so the WAN follows mastership (see
# vrrp-wan-take). Against that design the old check DEADLOCKS --
#
#     may I be master?  ->  only if I already have WAN
#     do I have WAN?    ->  only if I am master
#
# -- and the backup sits in FAULT for ever. vyos002 sat exactly there, which
# meant the pair could not fail over at all: the safety check had quietly
# removed the redundancy it was protecting.
#
# So the question is now asked in the right order: enforce "must have WAN" only
# on the router that is actually HOLDING the VIPs, and give a new master time to
# bring the WAN up before judging it.
# ---------------------------------------------------------------------------
#
# exit 0 = eligible for MASTER, non-zero = release and let the peer have it.

STATE=/run/vrrp-wan
GRACE=90          # seconds a new master gets to complete DHCP / PPPoE dial-up
VIP="${VRRP_WAN_VIP:-192.168.1.1}"

# A deliberate hand-over lever.
#
# There is no reliable way to MAKE this pair fail over on demand. VyOS offers
# only `restart vrrp`, and neither that nor `systemctl restart keepalived` is
# dependable: with advert_int 1 the peer declares the master dead after ~3.6s,
# and a restart usually finishes inside that window. Measured in labsim -- the
# same command moved mastership on one run and not on the next three. A
# fail-back procedure you cannot trigger on purpose is not a procedure.
#
# Failing the health check IS the supported way to shed mastership: the sync
# group goes FAULT, releases every VIP, and the peer takes over -- the same path
# a genuine WAN loss takes, so the planned drill exercises the real mechanism
# rather than a special case.
#
#   touch /run/vrrp-wan/force-fault    hand over within failure-count*interval
#   rm    /run/vrrp-wan/force-fault    become eligible again (no-preempt keeps
#                                      it BACKUP until the peer hands back)
#
# It lives in /run deliberately: a reboot clears it, so a forgotten drill cannot
# leave a router permanently ineligible.
[ -f /run/vrrp-wan/force-fault ] && exit 1

# Am I holding the VIPs? Asked of REALITY -- is the management VIP actually on
# this box -- and not of a /run marker.
#
# The marker was the first design and it is unsafe: it is written by the VRRP
# transition script, and in labsim that script silently failed to run on a
# promotion (VyOS's keepalived-fifo.py helper stopped delivering while
# keepalived's own notifies kept working). The router then believed it was
# backup, passed this check, and sat holding every VIP with no WAN -- the exact
# outage this script exists to prevent, re-created by trusting the reporter
# instead of the fact.
if [ -z "$(ip -4 -o addr show 2>/dev/null | grep " ${VIP}/")" ]; then
    # Clear the grace stamp on the way down, HERE, not only in the reconciler.
    # The reconciler runs every 30s; this runs every 5s. A promotion that
    # inherited a stamp from an earlier mastership scored grace = hours, failed
    # immediately, and took the sync group to FAULT ~5s after passing -- with the
    # peer already faulted, that left BOTH routers in FAULT and the LAN with no
    # gateway. The stamp must belong to the CURRENT mastership or it is worse
    # than useless.
    rm -f "$STATE/since" 2>/dev/null
    exit 0
fi

# Start the grace clock HERE, the moment mastership is first observed.
#
# It used to be stamped only by vrrp-wan-reconcile, which runs on a 30s timer --
# so a freshly promoted master reached this check with no stamp, scored grace=0,
# failed, and went FAULT before it had any chance to bring the WAN up. The peer
# then found itself alone with no WAN either and did the same. Observed in
# labsim: BOTH routers in FAULT, nobody holding the VIPs, the LAN with no
# gateway at all. That is worse than the outage this script exists to prevent,
# and it would have hit a REAL failover, not just a drill -- the health check
# runs every 5s and the reconciler had not yet ticked.
mkdir -p "$STATE" 2>/dev/null
[ -f "$STATE/since" ] || date +%s > "$STATE/since"

# Master with an address on a WAN interface: healthy.
#
# Deliberately NOT "can I reach the internet" and NOT "do I have a default
# route". During a real ISP outage the route disappears on BOTH routers; a check
# keyed on that would put both into FAULT, nobody would hold the VIPs, and the
# LAN would lose inter-VLAN routing too -- turning an internet outage into a
# total one. A DHCP lease survives an ISP outage, so an address still
# distinguishes "this box structurally cannot route" from "the internet is down
# right now", which is the distinction that matters.
for ifc in bond0.53 pppoe0; do
    ip -4 addr show dev "$ifc" 2>/dev/null | grep -q 'inet ' && exit 0
done

# Master, no WAN yet, still within the grace window: DHCP negotiation and PPPoE
# dial-up take real time, and the ISP has to accept the cloned MAC arriving on a
# different port. Failing here would demote the new master before it ever had a
# chance, and hand the VIPs straight back -- a flap, not a failover.
since=$(cat "$STATE/since" 2>/dev/null || echo 0)
[ $(( $(date +%s) - since )) -lt "$GRACE" ] && exit 0

# Master, past grace, still no WAN: release. This is the 2026-09-02 case.
exit 1