diff --git a/migration/vrrp-wan-apply b/migration/vrrp-wan-apply index 13529e8..a19cc9f 100644 --- a/migration/vrrp-wan-apply +++ b/migration/vrrp-wan-apply @@ -17,16 +17,32 @@ WAN_VIF=53 cfg() { /opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands 2>/dev/null; } wan_disabled(){ cfg | grep -q "vif ${WAN_VIF} disable"; } ppp_disabled(){ cfg | grep -q "pppoe pppoe0 disable"; } +# Only touch pppoe0 if it is actually configured. `set interfaces pppoe pppoe0 +# disable` on a box that has no pppoe0 CREATES the node with nothing but +# `disable`, and VyOS then refuses the commit with "Physical source-interface +# required for pppoe0!" -- taking the bond0.53 change down with it, because one +# invalid node fails the whole commit. Seen on the labsim secondary, which has +# no PPPoE; production has it on both, so this would have been an untested path +# that only ever ran during a failover. +ppp_exists(){ cfg | grep -q "pppoe pppoe0 source-interface"; } configure if [ "$MODE" = enable ]; then # Guarded: `delete` of an absent node aborts the whole batch with # "Nothing to delete", which left the box detected-but-unfixed. wan_disabled && delete interfaces bonding bond0 vif ${WAN_VIF} disable - ppp_disabled && delete interfaces pppoe pppoe0 disable + ppp_exists && ppp_disabled && delete interfaces pppoe pppoe0 disable else wan_disabled || set interfaces bonding bond0 vif ${WAN_VIF} disable - ppp_disabled || set interfaces pppoe pppoe0 disable + ppp_exists && { ppp_disabled || set interfaces pppoe pppoe0 disable; } +fi +# Report the commit's verdict. The script previously ended on `exit` (a +# script-template function) and returned 0 even after "Commit failed", so the +# reconciler logged a successful release that had not happened -- the worst kind +# of failure for something whose whole job is to keep two routers from holding +# one WAN. +if commit 2>&1 | tee /tmp/vrrp-wan-commit.log | grep -qi "commit failed"; then + logger -t vrrp-wan "COMMIT FAILED applying '$MODE' -- see /tmp/vrrp-wan-commit.log" + exit 1 fi -commit exit diff --git a/migration/vrrp-wan-health b/migration/vrrp-wan-health index 1d3a7f2..1e14a4d 100755 --- a/migration/vrrp-wan-health +++ b/migration/vrrp-wan-health @@ -32,6 +32,28 @@ STATE=/run/vrrp-wan GRACE=90 # seconds a new master gets to complete DHCP / PPPoE dial-up VIP="${VRRP_WAN_VIP:-192.168.1.1}" +# A deliberate hand-over lever. +# +# There is no reliable way to MAKE this pair fail over on demand. VyOS offers +# only `restart vrrp`, and neither that nor `systemctl restart keepalived` is +# dependable: with advert_int 1 the peer declares the master dead after ~3.6s, +# and a restart usually finishes inside that window. Measured in labsim -- the +# same command moved mastership on one run and not on the next three. A +# fail-back procedure you cannot trigger on purpose is not a procedure. +# +# Failing the health check IS the supported way to shed mastership: the sync +# group goes FAULT, releases every VIP, and the peer takes over -- the same path +# a genuine WAN loss takes, so the planned drill exercises the real mechanism +# rather than a special case. +# +# touch /run/vrrp-wan/force-fault hand over within failure-count*interval +# rm /run/vrrp-wan/force-fault become eligible again (no-preempt keeps +# it BACKUP until the peer hands back) +# +# It lives in /run deliberately: a reboot clears it, so a forgotten drill cannot +# leave a router permanently ineligible. +[ -f /run/vrrp-wan/force-fault ] && exit 1 + # Am I holding the VIPs? Asked of REALITY -- is the management VIP actually on # this box -- and not of a /run marker. # @@ -42,7 +64,30 @@ VIP="${VRRP_WAN_VIP:-192.168.1.1}" # backup, passed this check, and sat holding every VIP with no WAN -- the exact # outage this script exists to prevent, re-created by trusting the reporter # instead of the fact. -[ -n "$(ip -4 -o addr show 2>/dev/null | grep " ${VIP}/")" ] || exit 0 +if [ -z "$(ip -4 -o addr show 2>/dev/null | grep " ${VIP}/")" ]; then + # Clear the grace stamp on the way down, HERE, not only in the reconciler. + # The reconciler runs every 30s; this runs every 5s. A promotion that + # inherited a stamp from an earlier mastership scored grace = hours, failed + # immediately, and took the sync group to FAULT ~5s after passing -- with the + # peer already faulted, that left BOTH routers in FAULT and the LAN with no + # gateway. The stamp must belong to the CURRENT mastership or it is worse + # than useless. + rm -f "$STATE/since" 2>/dev/null + exit 0 +fi + +# Start the grace clock HERE, the moment mastership is first observed. +# +# It used to be stamped only by vrrp-wan-reconcile, which runs on a 30s timer -- +# so a freshly promoted master reached this check with no stamp, scored grace=0, +# failed, and went FAULT before it had any chance to bring the WAN up. The peer +# then found itself alone with no WAN either and did the same. Observed in +# labsim: BOTH routers in FAULT, nobody holding the VIPs, the LAN with no +# gateway at all. That is worse than the outage this script exists to prevent, +# and it would have hit a REAL failover, not just a drill -- the health check +# runs every 5s and the reconciler had not yet ticked. +mkdir -p "$STATE" 2>/dev/null +[ -f "$STATE/since" ] || date +%s > "$STATE/since" # Master with an address on a WAN interface: healthy. # diff --git a/migration/vrrp-wan-reconcile b/migration/vrrp-wan-reconcile index 04f0026..352bc43 100644 --- a/migration/vrrp-wan-reconcile +++ b/migration/vrrp-wan-reconcile @@ -51,6 +51,14 @@ fi # One writer. The 30s timer and a VRRP transition can fire together, and two # VyOS commits in flight on one box do not queue -- the second fails outright. +# +# The lock fd MUST be closed for children (`9>&-` on every call below). Entering +# VyOS config mode spawns a long-lived unionfs-fuse for the session, and it +# INHERITS this descriptor and never lets go -- `lsof /run/vrrp-wan.lock` showed +# it held by unionfs-fuse with fd 9w. From the first config change onward every +# later run lost the flock and exited 0 without doing anything, so the +# reconciler looked healthy in the journal ("Finished") while quietly having +# stopped reconciling. It is how a demoted router kept the WAN. exec 9>"$LOCK" flock -n 9 || exit 0 @@ -85,7 +93,7 @@ if holds_vip; then [ -f "$STATE/since" ] || date +%s > "$STATE/since" wan_disabled || ppp_disabled || exit 0 logger -t vrrp-wan "MASTER with WAN disabled -> enabling bond0.${WAN_VIF} + pppoe0" - "$APPLY" enable + "$APPLY" enable 9>&- else echo backup > "$STATE/role" rm -f "$STATE/since" @@ -95,7 +103,7 @@ else # new master, and the switch sends the ISP's replies to whichever port spoke # last -- the WAN-side twin of the eth2 incident. logger -t vrrp-wan "not MASTER but WAN enabled -> releasing bond0.${WAN_VIF} + pppoe0" - "$APPLY" disable + "$APPLY" disable 9>&- fi # Deliberately no `save`. config.boot keeps `disable` on BOTH routers, so a