diff --git a/migration/vrrp-wan-apply b/migration/vrrp-wan-apply new file mode 100644 index 0000000..13529e8 --- /dev/null +++ b/migration/vrrp-wan-apply @@ -0,0 +1,32 @@ +#!/bin/vbash +# Enable or disable the WAN. Split out from vrrp-wan-reconcile for one reason: +# `source /opt/vyatta/etc/functions/script-template` must be the FIRST thing the +# script does. Sourced after a few statements -- an if, an exec, a mkdir -- it +# silently terminated the script; `set -x` showed execution stopping inside the +# source with no error and rc=0, so the reconciler reported success having done +# nothing. Only a single assignment may precede it (the template resets the +# positional parameters, so the mode is captured first), which is the same shape +# /config/vyos-known-good uses. +# +# vrrp-wan-apply enable take the WAN +# vrrp-wan-apply disable release it +MODE="${1:-}" +source /opt/vyatta/etc/functions/script-template + +WAN_VIF=53 +cfg() { /opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands 2>/dev/null; } +wan_disabled(){ cfg | grep -q "vif ${WAN_VIF} disable"; } +ppp_disabled(){ cfg | grep -q "pppoe pppoe0 disable"; } + +configure +if [ "$MODE" = enable ]; then + # Guarded: `delete` of an absent node aborts the whole batch with + # "Nothing to delete", which left the box detected-but-unfixed. + wan_disabled && delete interfaces bonding bond0 vif ${WAN_VIF} disable + ppp_disabled && delete interfaces pppoe pppoe0 disable +else + wan_disabled || set interfaces bonding bond0 vif ${WAN_VIF} disable + ppp_disabled || set interfaces pppoe pppoe0 disable +fi +commit +exit diff --git a/migration/vrrp-wan-health b/migration/vrrp-wan-health index 736aa2b..1d3a7f2 100755 --- a/migration/vrrp-wan-health +++ b/migration/vrrp-wan-health @@ -7,18 +7,62 @@ # while looking perfectly healthy. That is not hypothetical: it is the outage of # 2026-09-02, reproduced in labsim. # -# The test is "do I have an ADDRESS on a WAN interface", deliberately NOT "can I -# reach the internet" and NOT "do I have a default route". During a real ISP -# outage the default route disappears on BOTH routers; a check keyed on that -# would put both into FAULT, nobody would hold the VIPs, and the LAN would lose -# inter-VLAN routing too -- turning an internet outage into a total one. An -# address on a WAN interface distinguishes "this box structurally cannot route" -# (the failure we must prevent) from "the internet happens to be down right now" -# (which the router can do nothing about, and during which it should keep -# serving the LAN). +# --------------------------------------------------------------------------- +# The first version of this script asked one question: "do I have an address on +# a WAN interface". That is correct for a pair where both routers hold WAN all +# the time. Ours cannot: the 10 gig lease is bound to a cloned MAC and the +# PPPoE line to a single credential, so the WAN follows mastership (see +# vrrp-wan-take). Against that design the old check DEADLOCKS -- +# +# may I be master? -> only if I already have WAN +# do I have WAN? -> only if I am master +# +# -- and the backup sits in FAULT for ever. vyos002 sat exactly there, which +# meant the pair could not fail over at all: the safety check had quietly +# removed the redundancy it was protecting. +# +# So the question is now asked in the right order: enforce "must have WAN" only +# on the router that is actually HOLDING the VIPs, and give a new master time to +# bring the WAN up before judging it. +# --------------------------------------------------------------------------- # # exit 0 = eligible for MASTER, non-zero = release and let the peer have it. + +STATE=/run/vrrp-wan +GRACE=90 # seconds a new master gets to complete DHCP / PPPoE dial-up +VIP="${VRRP_WAN_VIP:-192.168.1.1}" + +# Am I holding the VIPs? Asked of REALITY -- is the management VIP actually on +# this box -- and not of a /run marker. +# +# The marker was the first design and it is unsafe: it is written by the VRRP +# transition script, and in labsim that script silently failed to run on a +# promotion (VyOS's keepalived-fifo.py helper stopped delivering while +# keepalived's own notifies kept working). The router then believed it was +# backup, passed this check, and sat holding every VIP with no WAN -- the exact +# outage this script exists to prevent, re-created by trusting the reporter +# instead of the fact. +[ -n "$(ip -4 -o addr show 2>/dev/null | grep " ${VIP}/")" ] || exit 0 + +# Master with an address on a WAN interface: healthy. +# +# Deliberately NOT "can I reach the internet" and NOT "do I have a default +# route". During a real ISP outage the route disappears on BOTH routers; a check +# keyed on that would put both into FAULT, nobody would hold the VIPs, and the +# LAN would lose inter-VLAN routing too -- turning an internet outage into a +# total one. A DHCP lease survives an ISP outage, so an address still +# distinguishes "this box structurally cannot route" from "the internet is down +# right now", which is the distinction that matters. for ifc in bond0.53 pppoe0; do ip -4 addr show dev "$ifc" 2>/dev/null | grep -q 'inet ' && exit 0 done + +# Master, no WAN yet, still within the grace window: DHCP negotiation and PPPoE +# dial-up take real time, and the ISP has to accept the cloned MAC arriving on a +# different port. Failing here would demote the new master before it ever had a +# chance, and hand the VIPs straight back -- a flap, not a failover. +since=$(cat "$STATE/since" 2>/dev/null || echo 0) +[ $(( $(date +%s) - since )) -lt "$GRACE" ] && exit 0 + +# Master, past grace, still no WAN: release. This is the 2026-09-02 case. exit 1 diff --git a/migration/vrrp-wan-reconcile b/migration/vrrp-wan-reconcile new file mode 100644 index 0000000..04f0026 --- /dev/null +++ b/migration/vrrp-wan-reconcile @@ -0,0 +1,106 @@ +#!/bin/sh +# Make the WAN match VRRP mastership. Idempotent; safe to run every 30s and on +# every VRRP transition. +# +# Why a reconciler and not just transition scripts +# ------------------------------------------------ +# VyOS delivers `transition-script` through a helper process, +# /usr/libexec/vyos/system/keepalived-fifo.py, fed by keepalived's notify_fifo. +# Observed in labsim on 2026-09-02: the primary's Keepalived_vrrp logged +# "(native) Entering MASTER STATE" for all six instances and the built-in +# notify_master for conntrack-sync ran -- while the fifo helper logged NOTHING +# and the master transition script never ran. The helper process was still +# alive. The result was a router holding every VIP with no WAN at all: the exact +# 2026-09-02 outage, re-created by the mechanism meant to prevent it. +# +# So transition scripts are kept for speed but nothing is trusted to them: this +# also runs on a timer, and derives everything from ground truth rather than +# from a marker that only exists if the script it depends on ran. +# +# vrrp-wan-reconcile reconcile once +# vrrp-wan-reconcile --status what it thinks, changing nothing +# +# Ground truth for "am I master" is whether the management VIP is really on this +# box. It is what VRRP actually does, it is observable, and it cannot silently +# disagree with reality. + +VIP="${VRRP_WAN_VIP:-192.168.1.1}" # management VIP; sim overrides via env +WAN_VIF=53 # bond0.53, the DHCP WAN +STATE=/run/vrrp-wan +LOCK=/run/vrrp-wan.lock + +cfg() { /opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands 2>/dev/null; } +holds_vip() { ip -4 -o addr show 2>/dev/null | grep -q " ${VIP}/"; } +wan_up() { ip -4 addr show "bond0.${WAN_VIF}" 2>/dev/null | grep -q 'inet '; } +wan_disabled(){ cfg | grep -q "vif ${WAN_VIF} disable"; } +ppp_disabled(){ cfg | grep -q "pppoe pppoe0 disable"; } + +# --status must answer WITHOUT sourcing script-template. The template's `exit` +# is a function that leaves configuration mode, not the shell builtin, so a +# status run that had sourced it opened and closed a config session on every +# call -- which is how a read-only query started colliding with the timer and +# logging "Configuration system temporarily locked due to another commit". +if [ "${1:-}" = "--status" ]; then + printf 'vip=%s holds_vip=%s wan_disabled=%s wan_up=%s role=%s\n' \ + "$VIP" "$(holds_vip && echo yes || echo no)" \ + "$(wan_disabled && echo yes || echo no)" \ + "$(wan_up && echo yes || echo no)" \ + "$(cat "$STATE/role" 2>/dev/null || echo unset)" + exit 0 +fi + +# One writer. The 30s timer and a VRRP transition can fire together, and two +# VyOS commits in flight on one box do not queue -- the second fails outright. +exec 9>"$LOCK" +flock -n 9 || exit 0 + +mkdir -p "$STATE" + +# Reap config sessions whose owning process is gone. VyOS creates +# /opt/vyatta/config/tmp/new_config_ (a unionfs mount) per `configure`, and +# a script that dies inside a session never removes it. One of those holds the +# commit lock, and from then on EVERY commit fails with "Configuration system +# temporarily locked due to another commit in progress" -- including the manual +# one you try in order to fix it. A job on a 30s timer that can leak a session +# per failure will wedge the router's config system on its own, so it cleans up +# before it starts. `umount -l` first: the directory is a mount point and plain +# rm returns "Device or resource busy". +for d in /opt/vyatta/config/tmp/new_config_*; do + [ -d "$d" ] || continue + pid=${d##*_} + kill -0 "$pid" 2>/dev/null && continue + umount -l "$d" 2>/dev/null + rm -rf "$d" 2>/dev/null +done + +# The config edit lives in vrrp-wan-apply, because script-template must be the +# first thing its script does -- sourced any later it terminates the script +# silently with rc=0. See the header there. +APPLY=/config/vrrp-wan-apply + +if holds_vip; then + echo master > "$STATE/role" + # Stamp only on entry to master, so the health check's grace window measures + # time-since-promotion rather than time-since-last-tick. + [ -f "$STATE/since" ] || date +%s > "$STATE/since" + wan_disabled || ppp_disabled || exit 0 + logger -t vrrp-wan "MASTER with WAN disabled -> enabling bond0.${WAN_VIF} + pppoe0" + "$APPLY" enable +else + echo backup > "$STATE/role" + rm -f "$STATE/since" + { wan_disabled && ppp_disabled; } && exit 0 + # Releasing matters more than taking. A demoted router that keeps the WAN up + # holds the cloned MAC f0:9f:c2:12:9b:4f on VLAN 53 at the same time as the + # new master, and the switch sends the ISP's replies to whichever port spoke + # last -- the WAN-side twin of the eth2 incident. + logger -t vrrp-wan "not MASTER but WAN enabled -> releasing bond0.${WAN_VIF} + pppoe0" + "$APPLY" disable +fi + +# Deliberately no `save`. config.boot keeps `disable` on BOTH routers, so a +# reboot in any order comes up unable to claim the shared MAC, and only holding +# the VIP re-enables it. NOTE: any `save` while this box is master (a hand +# commit, or `pulumi up`) WILL persist the enabled state -- observed in labsim. +# The Pulumi model asserts `disable` on both routers so an apply puts it back, +# and vyos:verify reports it as drift if it does not. diff --git a/migration/vrrp-wan-reconcile.service b/migration/vrrp-wan-reconcile.service new file mode 100644 index 0000000..66cdc6f --- /dev/null +++ b/migration/vrrp-wan-reconcile.service @@ -0,0 +1,10 @@ +[Unit] +# Belt to the transition scripts' braces. VyOS's keepalived-fifo.py helper was +# observed dropping a MASTER transition silently, leaving a router holding every +# VIP with no WAN. A timer cannot be dropped the same way. +Description=Reconcile WAN interface state with VRRP mastership +After=keepalived.service + +[Service] +Type=oneshot +ExecStart=/config/vrrp-wan-reconcile diff --git a/migration/vrrp-wan-reconcile.timer b/migration/vrrp-wan-reconcile.timer new file mode 100644 index 0000000..4eb469d --- /dev/null +++ b/migration/vrrp-wan-reconcile.timer @@ -0,0 +1,13 @@ +[Unit] +Description=Reconcile WAN with VRRP mastership every 30s + +[Timer] +# 30s: fast enough that a dropped transition is a blip rather than an outage, +# slow enough that it is never the thing generating load. It only commits when +# state actually disagrees, so a steady-state tick is two `ip` calls and a grep. +OnBootSec=60 +OnUnitActiveSec=30 +AccuracySec=5 + +[Install] +WantedBy=timers.target diff --git a/migration/vrrp-wan-release b/migration/vrrp-wan-release new file mode 100644 index 0000000..c2140dd --- /dev/null +++ b/migration/vrrp-wan-release @@ -0,0 +1,5 @@ +#!/bin/sh +# VRRP transition hook. One code path: the reconciler derives everything from +# ground truth, so take and release are the same operation asked at different +# moments. Speed comes from here; correctness comes from the timer. +exec /config/vrrp-wan-reconcile diff --git a/migration/vrrp-wan-take b/migration/vrrp-wan-take new file mode 100644 index 0000000..c2140dd --- /dev/null +++ b/migration/vrrp-wan-take @@ -0,0 +1,5 @@ +#!/bin/sh +# VRRP transition hook. One code path: the reconciler derives everything from +# ground truth, so take and release are the same operation asked at different +# moments. Speed comes from here; correctness comes from the timer. +exec /config/vrrp-wan-reconcile