diff --git a/labsim/console-apply.py b/labsim/console-apply.py index a15bb90..02173f7 100755 --- a/labsim/console-apply.py +++ b/labsim/console-apply.py @@ -22,6 +22,12 @@ def main() -> int: ap.add_argument("--config", required=True) ap.add_argument("--user", default="vyos") ap.add_argument("--password", default="vyos") + # `save` writes config.boot. For WAN work that is dangerous: the resting + # state must stay `vif 53 disable` on both routers, and saving while a box + # is master persists the ENABLED state -- so a reboot would have it claim + # the cloned MAC. Observed in labsim on 2026-09-02. + ap.add_argument("--no-save", action="store_true", + help="commit without saving (leave config.boot untouched)") args = ap.parse_args() cmds = [l.rstrip() for l in open(args.config) @@ -83,8 +89,9 @@ def main() -> int: c.sendline("commit") c.expect(r"# ", timeout=300) commit_out = c.before or "" - c.sendline("save") - c.expect(r"# ", timeout=120) + if not args.no_save: + c.sendline("save") + c.expect(r"# ", timeout=120) # Accept either prompt on the way out. Insisting on `$ ` here hangs against # a healthy box -- and worse, leaves the console parked in config mode, so # the NEXT run finds a `# ` it was not expecting either. One strict expect diff --git a/labsim/sim-ha-config.py b/labsim/sim-ha-config.py index 0e87ec3..2ff3475 100755 --- a/labsim/sim-ha-config.py +++ b/labsim/sim-ha-config.py @@ -85,6 +85,27 @@ def build(role: str) -> list[str]: ] out += [ + "# --- WAN follows VRRP mastership ---", + # These four hooks and the health check existed on both live sim VMs but + # in NEITHER generator, so `sim-net-apply.sh check` reported "in sync" + # while the mechanism under test was pure undetected drift -- exactly + # the failure mode this file was written to end. + # + # The check goes on the SYNC GROUP, not per group: VyOS rejects a + # per-group check while the group is in a sync group ("Only sync group + # health check will be used"). + "set high-availability vrrp sync-group MAIN health-check script '/config/vrrp-wan-health'", + "set high-availability vrrp sync-group MAIN health-check interval '5'", + "set high-availability vrrp sync-group MAIN health-check failure-count '3'", + # take and release both exec vrrp-wan-reconcile: one code path, asked at + # different moments. `stop` matters as much as `backup` -- a stopped + # keepalived is a demotion too, and without it the box would keep the + # WAN while holding no VIPs. + "set high-availability vrrp sync-group MAIN transition-script master '/config/vrrp-wan-take'", + "set high-availability vrrp sync-group MAIN transition-script backup '/config/vrrp-wan-release'", + "set high-availability vrrp sync-group MAIN transition-script fault '/config/vrrp-wan-release'", + "set high-availability vrrp sync-group MAIN transition-script stop '/config/vrrp-wan-release'", + "", "# --- DHCP high-availability ---", "# The thing under test: active-passive should mean exactly one OFFER.", "set service dhcp-server high-availability mode active-passive", diff --git a/labsim/sim-net-apply.sh b/labsim/sim-net-apply.sh index f4b770f..f6a3fd6 100755 --- a/labsim/sim-net-apply.sh +++ b/labsim/sim-net-apply.sh @@ -17,9 +17,15 @@ ACTION="${1:-check}" WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT # role : vm : address : regex selecting the subtrees this generator owns +# +# primary and secondary now own the SAME subtrees. The secondary's used to omit +# `interfaces pppoe`, `interfaces bonding`, `nat source` and `protocols +# failover|static`, so `check` was blind to precisely the WAN config the +# failover mechanism depends on -- it reported "in sync" for a box that had no +# WAN at all. TARGETS=( "primary:labsim-vyos:172.31.1.252:^set (protocols (bgp|failover|static)|policy (prefix-list|route-map)|nat source rule 1[12]0|interfaces (pppoe|bonding bond0 vif 5[13])|firewall (group interface-group LAN|ipv4|ipv6))" - "secondary:labsim-vyos2:172.31.1.253:^set (protocols bgp|policy (prefix-list|route-map)|firewall (group interface-group LAN|ipv4|ipv6))" + "secondary:labsim-vyos2:172.31.1.253:^set (protocols (bgp|failover|static)|policy (prefix-list|route-map)|nat source rule 1[12]0|interfaces (pppoe|bonding bond0 vif 5[13])|firewall (group interface-group LAN|ipv4|ipv6))" "isp-dhcp:labsim-isp-dhcp:192.168.122.136:^set (interfaces ethernet|nat source|service dhcp-server|firewall ipv4 forward|system host-name)" "isp-pppoe:labsim-isp-pppoe:192.168.122.63:^set (interfaces ethernet|nat source|service pppoe-server|firewall ipv4 forward|system host-name)" ) @@ -29,7 +35,13 @@ SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o PreferredAuthentications=password -o ConnectTimeout=5) live() { timeout 30 sshpass -p vyos ssh "${SSH_OPTS[@]}" "vyos@$1" \ "/opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands" 2>/dev/null; } -norm() { sed "s/'//g" | grep -v 'hw-id\|offload' | sort -u; } +# `vif 53 disable` is RUNTIME state, not config drift. The generators declare it +# on both routers as the safe resting state (only one box may hold the cloned +# MAC), and vrrp-wan-reconcile removes it on whichever box currently holds the +# VIP. Comparing it would therefore report drift on the master for ever, and a +# check that always cries wolf is a check nobody reads. +norm() { sed "s/'//g" | grep -v 'hw-id\|offload' \ + | grep -v 'interfaces bonding bond0 vif 53 disable' | sort -u; } rc=0 for t in "${TARGETS[@]}"; do diff --git a/labsim/sim-net-config.py b/labsim/sim-net-config.py index 12f3215..d41a20a 100755 --- a/labsim/sim-net-config.py +++ b/labsim/sim-net-config.py @@ -80,6 +80,13 @@ WAN_DHCP_DISTANCE = 210 # NOT `no-default-route`, which blanks new_routers PPPOE_DISTANCE = 10 # in the lease file, leaving failover no gateway # to install and silently handing the default # route to the backup line. + +# One MAC, cloned onto BOTH routers' bond0.53, mirroring production's use of the +# retired USG's WAN2 MAC to keep its DHCP lease. The sim did not model a shared +# MAC at all, which is exactly why bond0.53 has to stay on the config plane -- +# only VyOS config can move a MAC between boxes. Locally-administered, sim-only. +WAN_DHCP_MAC = "02:53:10:61:00:53" + SIM_LAN = "172.31.0.0/16" # The sim routers' own libvirt-NAT uplink, from before the ISP VMs existed. It @@ -120,8 +127,27 @@ def bgp(role: str) -> list[str]: return out -def wan(drop_scaffold: bool) -> list[str]: - """Dual WAN + health-checked failover. Primary router only -- see README.""" +def wan(drop_scaffold: bool, role: str = "primary") -> list[str]: + """Dual WAN + health-checked failover. IDENTICAL on both routers. + + It used to be primary-only, on the grounds that "two PPPoE clients sharing + one credential against a single access concentrator is a different failure + mode than anything production has". That was backwards: production has + exactly that, and by omitting it the sim could not test the one thing most + likely to go wrong. The secondary having no WAN is also why it ended up with + zero NAT rules while the primary had ten -- the pair was not comparable. + + Both routers therefore get the same WAN config. What differs is the RESTING + STATE, and only for the DHCP line: + + bond0.53 `disable` on BOTH. Its lease is bound to a cloned MAC, and two + boxes claiming one MAC is the fault this whole design exists to + prevent. vrrp-wan-reconcile removes `disable` on the master. + pppoe0 enabled on BOTH, never `disable`d. `disable` unlinks + /etc/ppp/peers/pppoe0, which is pppd's own options file, so the + promotion path destroyed what it needed. Dialling is gated at + the systemd unit instead -- see migration/ppp-vrrp-gate.conf. + """ out = [ "# --- WAN: DHCP (primary) + PPPoE (backup), health-checked ---", f"set interfaces bonding bond0 vif {WAN_PPPOE_VLAN} description " @@ -131,6 +157,13 @@ def wan(drop_scaffold: bool) -> list[str]: f"'WAN3 10gig-equivalent (sim ISP DHCP)'", f"set interfaces bonding bond0 vif {WAN_DHCP_VLAN} dhcp-options " f"default-route-distance {WAN_DHCP_DISTANCE}", + # The cloned MAC. Production clones the old USG's WAN2 MAC so the ISP + # keeps handing back the same lease; the sim did not model a shared MAC + # at all, which is precisely why bond0.53 must stay on the config plane. + # Modelling it lets the sim prove the lease returns to the new master. + f"set interfaces bonding bond0 vif {WAN_DHCP_VLAN} mac {WAN_DHCP_MAC}", + # Safe at rest on BOTH routers: only the VIP holder enables it. + f"set interfaces bonding bond0 vif {WAN_DHCP_VLAN} disable", f"set interfaces pppoe pppoe0 source-interface bond0.{WAN_PPPOE_VLAN}", f"set interfaces pppoe pppoe0 authentication username {PPPOE_USER}", f"set interfaces pppoe pppoe0 authentication password {PPPOE_PASS}", @@ -364,21 +397,20 @@ def _isp_common(uplink_if: str, customer_net: str, desc: str) -> list[str]: def build(role: str, drop_scaffold: bool, wan_if: str, uplink_if: str) -> list[str]: - if role == "primary": - # WAN lives on the primary only. Production has WAN on both routers; - # the sim does not, because two PPPoE clients sharing one credential - # against a single access concentrator is a different failure mode than - # anything production has. VRRP/conntrack failover is still exercised -- - # see README, "known gaps". + if role in ("primary", "secondary"): + # BOTH routers get the identical WAN. The sim used to give it to the + # primary only, reasoning that two PPPoE clients sharing one credential + # was "a different failure mode than anything production has" -- but + # that IS production, and omitting it meant the failover path was the + # one path the sim could not exercise. It also left the pair + # incomparable: ten NAT rules on one box, none on the other. + # + # Safety comes from resting state, not from asymmetry: bond0.53 is + # `disable`d on both (cloned MAC), pppoe0 is enabled on both but gated + # at the systemd unit. See wan() and migration/ppp-vrrp-gate.conf. return ([f"# labsim routing -- {role}", ""] - + bgp(role) + wan(drop_scaffold) + firewall()) - if role == "secondary": - # The backup has no WAN in the sim, so it has no DHCP client to - # exempt -- but it gets the same policy otherwise, because after a VRRP - # failover it IS the router and a divergent ruleset would only be - # discovered during the failover. - return ([f"# labsim routing -- {role}", ""] - + bgp(role) + firewall(wan_dhcp_if=None)) + + bgp(role) + wan(drop_scaffold, role) + + firewall(wan_dhcp_if=f"bond0.{WAN_DHCP_VLAN}")) if role == "isp-dhcp": return isp_dhcp(wan_if, uplink_if) return isp_pppoe(wan_if, uplink_if) diff --git a/migration/ppp-vrrp-gate.conf b/migration/ppp-vrrp-gate.conf new file mode 100644 index 0000000..5bbea46 --- /dev/null +++ b/migration/ppp-vrrp-gate.conf @@ -0,0 +1,50 @@ +# Installed to /etc/systemd/system/ppp@pppoe0.service.d/10-vrrp-wan-gate.conf +# +# This drop-in is the ONLY thing preventing both routers from dialling the one +# ISP credential at the same time. Do not remove it without reading this. +# +# pppoe0 is configured identically and ENABLED on both routers, because the +# alternative -- `set interfaces pppoe pppoe0 disable` -- unlinks +# /etc/ppp/peers/pppoe0 (interfaces_pppoe.py treats `disable` and `delete` +# identically), and pppd's options file IS that path. A promotion then had to +# re-render it via a full config commit at priority 322, where one unrelated +# invalid node fails the whole commit and takes the 10 gig down with it. It also +# made op-mode `connect interface pppoe0` unusable, since that refuses when the +# peers file is absent. +# +# With the node enabled, interfaces_pppoe.py's apply() does this on EVERY commit +# that touches the pppoe subtree: +# +# if not is_systemd_service_running('ppp@pppoe0.service') or shutdown_required: +# call('systemctl restart ppp@pppoe0.service') +# +# -- i.e. the backup actively tries to dial whenever anything commits. A +# `pulumi up`, a `sim-net-apply.sh apply`, or the boot-time config load are all +# that commit. This gate is what makes that a no-op. +# +# /run is tmpfs, so the gate is shut at boot on both boxes and neither can dial +# before VRRP has decided. ppp@.service is already After=vyos-router.service, so +# no extra ordering is needed. +[Unit] +# Both must hold; multiple ConditionPathExists are ANDed. +# may-dial -- vrrp-wan-reconcile has blessed this box (a renewed lease) +# /etc/ppp/peers -- refuse to start pppd against a missing options file, which +# is what produced a restart loop of 47 and counting on +# 2026-09-05. A failed Condition is NOT a failure: the job +# succeeds, the unit stays inactive, and `systemctl start` +# exits 0 -- so callers must check is-active, never rc. +ConditionPathExists=/run/vrrp-wan/may-dial +ConditionPathExists=/etc/ppp/peers/pppoe0 + +# Belt to that brace. The stock unit is Restart=on-failure/RestartSec=5s against +# systemd's default StartLimitIntervalSec=10s/Burst=5 -- two restarts per window, +# so the limiter can never trip and a doomed pppd retries for ever. +StartLimitIntervalSec=600 +StartLimitBurst=6 + +[Service] +RestartSec=15 +# A hung pppd must be resolved inside the failover budget. The stock 90s means a +# demoted router could still hold the session while the new master is dialling. +# 20s still allows a clean LCP Terminate + PADT in the normal case. +TimeoutStopSec=20 diff --git a/migration/vrrp-wan-apply b/migration/vrrp-wan-apply index a19cc9f..95f96d5 100644 --- a/migration/vrrp-wan-apply +++ b/migration/vrrp-wan-apply @@ -1,46 +1,50 @@ #!/bin/vbash -# Enable or disable the WAN. Split out from vrrp-wan-reconcile for one reason: -# `source /opt/vyatta/etc/functions/script-template` must be the FIRST thing the -# script does. Sourced after a few statements -- an if, an exec, a mkdir -- it -# silently terminated the script; `set -x` showed execution stopping inside the -# source with no error and rc=0, so the reconciler reported success having done -# nothing. Only a single assignment may precede it (the template resets the -# positional parameters, so the mode is captured first), which is the same shape -# /config/vyos-known-good uses. +# Enable or disable the DHCP WAN (bond0.53). The ONLY part of the failover that +# touches VyOS configuration. # -# vrrp-wan-apply enable take the WAN +# PPPoE deliberately does NOT appear here any more, and should not be added back +# for symmetry. `set interfaces pppoe pppoe0 disable` unlinks +# /etc/ppp/peers/pppoe0 -- interfaces_pppoe.py treats `disable` and `delete` +# identically -- and that path is pppd's options file, so the resting state +# destroyed what the promotion path needed and the unit restart-looped (47 times, +# zero sessions at the AC). It also made a pppoe node able to fail this commit +# and take the 10 gig down with it: `interfaces pppoe` is priority 322 and one +# invalid node fails the whole commit. PPPoE is now gated at the systemd unit +# instead; see migration/ppp-vrrp-gate.conf and vrrp-wan-reconcile. +# +# bond0.53 stays here because its lease is bound to a cloned MAC and only VyOS +# config can move a MAC between boxes. +# +# `source /opt/vyatta/etc/functions/script-template` must be the FIRST thing the +# script does. Sourced after an if, an exec and a mkdir it terminated the script +# inside the source, rc=0, no output -- the caller reported success having done +# nothing. Only a single assignment may precede it (the template resets the +# positional parameters), which is the shape /config/vyos-known-good uses. +# +# vrrp-wan-apply enable take the DHCP WAN # vrrp-wan-apply disable release it MODE="${1:-}" source /opt/vyatta/etc/functions/script-template -WAN_VIF=53 +CONF=/config/vrrp-wan.conf +[ -r "$CONF" ] && . "$CONF" +WAN_VIF="${WAN_VIF:-53}" + cfg() { /opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands 2>/dev/null; } wan_disabled(){ cfg | grep -q "vif ${WAN_VIF} disable"; } -ppp_disabled(){ cfg | grep -q "pppoe pppoe0 disable"; } -# Only touch pppoe0 if it is actually configured. `set interfaces pppoe pppoe0 -# disable` on a box that has no pppoe0 CREATES the node with nothing but -# `disable`, and VyOS then refuses the commit with "Physical source-interface -# required for pppoe0!" -- taking the bond0.53 change down with it, because one -# invalid node fails the whole commit. Seen on the labsim secondary, which has -# no PPPoE; production has it on both, so this would have been an untested path -# that only ever ran during a failover. -ppp_exists(){ cfg | grep -q "pppoe pppoe0 source-interface"; } configure if [ "$MODE" = enable ]; then # Guarded: `delete` of an absent node aborts the whole batch with - # "Nothing to delete", which left the box detected-but-unfixed. + # "Nothing to delete", which once left the box detected-but-unfixed. wan_disabled && delete interfaces bonding bond0 vif ${WAN_VIF} disable - ppp_exists && ppp_disabled && delete interfaces pppoe pppoe0 disable else wan_disabled || set interfaces bonding bond0 vif ${WAN_VIF} disable - ppp_exists && { ppp_disabled || set interfaces pppoe pppoe0 disable; } fi -# Report the commit's verdict. The script previously ended on `exit` (a +# Report the commit's verdict. This script previously ended on `exit` (a # script-template function) and returned 0 even after "Commit failed", so the -# reconciler logged a successful release that had not happened -- the worst kind -# of failure for something whose whole job is to keep two routers from holding -# one WAN. +# reconciler logged a release that had not happened -- the worst kind of failure +# for something whose job is to keep two routers from holding one WAN. if commit 2>&1 | tee /tmp/vrrp-wan-commit.log | grep -qi "commit failed"; then logger -t vrrp-wan "COMMIT FAILED applying '$MODE' -- see /tmp/vrrp-wan-commit.log" exit 1 diff --git a/migration/vrrp-wan-guard b/migration/vrrp-wan-guard new file mode 100755 index 0000000..b79535e --- /dev/null +++ b/migration/vrrp-wan-guard @@ -0,0 +1,45 @@ +#!/bin/sh +# Revoke the PPPoE dial lease. Runs every 5s. Only ever takes the WAN AWAY. +# +# vrrp-wan-reconcile is the single writer and runs every 30s; this is the +# watcher, and it exists because `may-dial` must be a LEASE, not a flag. +# ConditionPathExists is evaluated at START only -- it can prevent a dial, it can +# never revoke one. So if the reconciler stops running (timer masked, box wedged, +# someone stops it during maintenance) and the box is then demoted, nothing would +# ever hang up: it would keep the one ISP session while the new master tries to +# take it. +# +# Two conditions revoke, both biased the safe way: +# - this box does not hold the management VIP +# - the lease has not been renewed within LEASE_TTL (the reconciler is dead) +# +# Deliberately tiny: no config mode, no flock, no commit. It cannot wedge the +# router's configuration system, which is what earns it a 5s timer. Running at +# 5s rather than the reconciler's 30s is also what shrinks the double-dial +# window on a demotion from up to 30s down to about 5. + +CONF=/config/vrrp-wan.conf +[ -r "$CONF" ] && . "$CONF" +VIP="${VRRP_WAN_VIP:-192.168.1.1}" +LEASE_TTL="${LEASE_TTL:-75}" +STATE=/run/vrrp-wan + +revoke() { + rm -f "$STATE/may-dial" + systemctl is-active --quiet ppp@pppoe0 2>/dev/null || return 0 + logger -t vrrp-wan "GUARD: $1 -- hanging up pppoe0" + systemctl stop ppp@pppoe0 2>/dev/null +} + +if ! ip -4 -o addr show 2>/dev/null | grep -q " ${VIP}/"; then + revoke "does not hold ${VIP}" + exit 0 +fi + +# Holds the VIP, so it is entitled to dial -- but only while something is +# actively renewing the lease on its behalf. +if [ -f "$STATE/may-dial" ]; then + age=$(( $(date +%s) - $(stat -c %Y "$STATE/may-dial" 2>/dev/null || echo 0) )) + [ "$age" -gt "$LEASE_TTL" ] && revoke "lease stale (${age}s > ${LEASE_TTL}s; is vrrp-wan-reconcile.timer running?)" +fi +exit 0 diff --git a/migration/vrrp-wan-guard.service b/migration/vrrp-wan-guard.service new file mode 100644 index 0000000..cf27307 --- /dev/null +++ b/migration/vrrp-wan-guard.service @@ -0,0 +1,10 @@ +[Unit] +# Revokes the PPPoE dial lease. Only ever takes the WAN away, never grants it, +# which is what makes a 5s cadence safe: it touches no VyOS configuration and +# cannot wedge the commit lock. +Description=Revoke the PPPoE dial lease when this router is not master +After=vyos-router.service + +[Service] +Type=oneshot +ExecStart=/config/vrrp-wan-guard diff --git a/migration/vrrp-wan-guard.timer b/migration/vrrp-wan-guard.timer new file mode 100644 index 0000000..7991551 --- /dev/null +++ b/migration/vrrp-wan-guard.timer @@ -0,0 +1,13 @@ +[Unit] +Description=Revoke the PPPoE dial lease every 5s + +[Timer] +# 5s, against the reconciler's 30s. A demotion must hang up fast -- the window +# between "no longer master" and "stopped dialling" is the window in which two +# routers can hold one ISP session. +OnBootSec=10 +OnUnitActiveSec=5 +AccuracySec=1 + +[Install] +WantedBy=timers.target diff --git a/migration/vrrp-wan-health b/migration/vrrp-wan-health index 1e14a4d..e34ef23 100755 --- a/migration/vrrp-wan-health +++ b/migration/vrrp-wan-health @@ -28,10 +28,20 @@ # # exit 0 = eligible for MASTER, non-zero = release and let the peer have it. +CONF=/config/vrrp-wan.conf +[ -r "$CONF" ] && . "$CONF" STATE=/run/vrrp-wan -GRACE=90 # seconds a new master gets to complete DHCP / PPPoE dial-up VIP="${VRRP_WAN_VIP:-192.168.1.1}" +# Seconds a new master may go without any WAN. Sourced from vrrp-wan.conf; the +# fallback is deliberately NOT the old 90. accel-ppp's dead-peer budget is +# lcp-echo-interval(30) x lcp-echo-failure(3) = 90s, so a hard failover into an +# access concentrator that does not replace the stale session lands exactly on +# the boundary: the new master fails its own check, sheds the VIPs, and the peer +# -- in the same position -- does likewise. Both end in FAULT, which is worse +# than the outage this check exists to prevent. +GRACE="${GRACE:-180}" + # A deliberate hand-over lever. # # There is no reliable way to MAKE this pair fail over on demand. VyOS offers @@ -98,9 +108,17 @@ mkdir -p "$STATE" 2>/dev/null # total one. A DHCP lease survives an ISP outage, so an address still # distinguishes "this box structurally cannot route" from "the internet is down # right now", which is the distinction that matters. +# Any WAN counts. Requiring the 10 gig specifically would fault a healthy master +# during a genuine 10 gig outage and turn a degraded state into a total one -- +# the same reasoning as the default-route note above. Which one satisfied it is +# recorded for the operator and the test harness, but does not affect the verdict. for ifc in bond0.53 pppoe0; do - ip -4 addr show dev "$ifc" 2>/dev/null | grep -q 'inet ' && exit 0 + if ip -4 addr show dev "$ifc" 2>/dev/null | grep -q 'inet '; then + echo "$ifc" > "$STATE/wan" 2>/dev/null + exit 0 + fi done +rm -f "$STATE/wan" 2>/dev/null # Master, no WAN yet, still within the grace window: DHCP negotiation and PPPoE # dial-up take real time, and the ISP has to accept the cloned MAC arriving on a diff --git a/migration/vrrp-wan-install b/migration/vrrp-wan-install new file mode 100755 index 0000000..5f83587 --- /dev/null +++ b/migration/vrrp-wan-install @@ -0,0 +1,103 @@ +#!/bin/bash +# Install (or verify) the WAN-follows-VRRP mechanism on a VyOS router. +# +# This exists because on 2026-09-05 the sim's failover proof was obtained from +# scripts that had been hand-`sed`-ed in place: /config/vrrp-wan-health and +# -reconcile differed from git by an edited VIP, so the tested behaviour was not +# the committed behaviour and any reinstall would have silently reverted it. +# `--check` makes that class of drift a hard failure instead of a discovery. +# +# It installs ONLY the mechanism -- scripts, units, drop-in, settings. It never +# touches VyOS configuration: the `interfaces pppoe` node, `vif 53 disable` and +# the VRRP sync-group hooks are config and belong in the config model +# (labsim/sim-*.py for the sim, infra/vyos/subtrees/overrides.json for +# production), not in an installer. +# +# vrrp-wan-install --vip 192.168.1.1 [--host vyos@10.0.1.253] +# vrrp-wan-install --check [--host ...] # exits non-zero on any drift +# +# With no --host it operates on the local machine, so it can be scp'd to a +# router and run there. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VIP=""; HOST=""; MODE=install; PW="${VYOS_PW:-vyos}" + +while [ $# -gt 0 ]; do + case "$1" in + --vip) VIP="$2"; shift 2 ;; + --host) HOST="$2"; shift 2 ;; + --check) MODE=check; shift ;; + *) echo "usage: $0 [--vip A.B.C.D] [--host user@ip] [--check]" >&2; exit 2 ;; + esac +done + +SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR -o ConnectTimeout=8 -o PreferredAuthentications=password) +run() { # run a command on the target + if [ -n "$HOST" ]; then timeout 60 sshpass -p "$PW" ssh "${SSH_OPTS[@]}" "$HOST" "$@" + else bash -c "$*"; fi +} +put() { # copy a file to the target + if [ -n "$HOST" ]; then timeout 60 sshpass -p "$PW" scp "${SSH_OPTS[@]}" "$1" "$HOST:$2" >/dev/null + else cp "$1" "$2"; fi +} + +# script -> destination. take/release are hooks keepalived calls; both exec the +# reconciler, so there is one code path. +SCRIPTS="vrrp-wan-reconcile vrrp-wan-apply vrrp-wan-health vrrp-wan-guard vrrp-wan-take vrrp-wan-release" +UNITS="vrrp-wan-reconcile.service vrrp-wan-reconcile.timer vrrp-wan-guard.service vrrp-wan-guard.timer" +GATE_DIR=/etc/systemd/system/ppp@pppoe0.service.d +GATE=$GATE_DIR/10-vrrp-wan-gate.conf + +if [ "$MODE" = check ]; then + rc=0 + for f in $SCRIPTS; do + local_sum=$(md5sum "$HERE/$f" | cut -d' ' -f1) + remote_sum=$(run "md5sum /config/$f 2>/dev/null | cut -d' ' -f1") + [ "$local_sum" = "$remote_sum" ] || { echo " DRIFT /config/$f"; rc=1; } + done + for f in $UNITS; do + local_sum=$(md5sum "$HERE/$f" | cut -d' ' -f1) + remote_sum=$(run "md5sum /etc/systemd/system/$f 2>/dev/null | cut -d' ' -f1") + [ "$local_sum" = "$remote_sum" ] || { echo " DRIFT /etc/systemd/system/$f"; rc=1; } + done + gate_sum=$(md5sum "$HERE/ppp-vrrp-gate.conf" | cut -d' ' -f1) + remote_gate=$(run "md5sum $GATE 2>/dev/null | cut -d' ' -f1") + [ "$gate_sum" = "$remote_gate" ] || { echo " DRIFT $GATE (a VyOS upgrade wipes /etc -- both routers would dial)"; rc=1; } + run "[ -r /config/vrrp-wan.conf ]" || { echo " MISSING /config/vrrp-wan.conf"; rc=1; } + for t in vrrp-wan-reconcile.timer vrrp-wan-guard.timer; do + [ "$(run "systemctl is-enabled $t 2>/dev/null")" = enabled ] || { echo " NOT ENABLED $t"; rc=1; } + done + [ "$rc" -eq 0 ] && echo " vrrp-wan in sync" + exit "$rc" +fi + +[ -n "$VIP" ] || { echo "--vip is required to install" >&2; exit 2; } + +for f in $SCRIPTS; do + put "$HERE/$f" "/tmp/$f" + # root:vyattacfg 0775 -- vrrp-wan-apply enters config mode, which requires + # membership of vyattacfg. + run "sudo install -o root -g vyattacfg -m 0775 /tmp/$f /config/$f" +done + +# Settings, with the VIP substituted. One file, read by BOTH the reconciler and +# the health check -- keepalived invokes the latter with no environment at all, +# so an Environment= line in the unit would be read by one and not the other. +sed "s|^VRRP_WAN_VIP=.*|VRRP_WAN_VIP=${VIP}|" "$HERE/vrrp-wan.conf" > /tmp/vrrp-wan.conf.gen +put /tmp/vrrp-wan.conf.gen /tmp/vrrp-wan.conf.gen +run "sudo install -o root -g vyattacfg -m 0664 /tmp/vrrp-wan.conf.gen /config/vrrp-wan.conf" + +for f in $UNITS; do + put "$HERE/$f" "/tmp/$f" + run "sudo install -m 0644 /tmp/$f /etc/systemd/system/$f" +done + +put "$HERE/ppp-vrrp-gate.conf" /tmp/ppp-vrrp-gate.conf +run "sudo mkdir -p $GATE_DIR && sudo install -m 0644 /tmp/ppp-vrrp-gate.conf $GATE" + +run "sudo systemctl daemon-reload && sudo systemctl enable --now vrrp-wan-reconcile.timer vrrp-wan-guard.timer" >/dev/null 2>&1 + +echo " installed (vip=$VIP)" +run "sudo /config/vrrp-wan-reconcile --status" diff --git a/migration/vrrp-wan-reconcile b/migration/vrrp-wan-reconcile index 352bc43..61b1ea9 100644 --- a/migration/vrrp-wan-reconcile +++ b/migration/vrrp-wan-reconcile @@ -2,77 +2,87 @@ # Make the WAN match VRRP mastership. Idempotent; safe to run every 30s and on # every VRRP transition. # -# Why a reconciler and not just transition scripts -# ------------------------------------------------ -# VyOS delivers `transition-script` through a helper process, -# /usr/libexec/vyos/system/keepalived-fifo.py, fed by keepalived's notify_fifo. -# Observed in labsim on 2026-09-02: the primary's Keepalived_vrrp logged -# "(native) Entering MASTER STATE" for all six instances and the built-in -# notify_master for conntrack-sync ran -- while the fifo helper logged NOTHING -# and the master transition script never ran. The helper process was still -# alive. The result was a router holding every VIP with no WAN at all: the exact -# 2026-09-02 outage, re-created by the mechanism meant to prevent it. +# Two WANs, two different control planes, for a reason: # -# So transition scripts are kept for speed but nothing is trusted to them: this -# also runs on a timer, and derives everything from ground truth rather than -# from a marker that only exists if the script it depends on ran. +# bond0.53 (10 gig, DHCP) -- CONFIG plane. Its lease is bound to a cloned MAC +# (f0:9f:c2:12:9b:4f, the old USG's), and only VyOS +# config can move a MAC. One commit per failover. +# pppoe0 (Vodafone) -- SYSTEMD plane. Gated by a drop-in on +# ppp@pppoe0; see migration/ppp-vrrp-gate.conf. +# No commit, no config lock, no `save`. +# +# PPPoE used to be on the config plane too, via `set interfaces pppoe pppoe0 +# disable`. That could not work: `disable` unlinks /etc/ppp/peers/pppoe0, which +# is pppd's options file, so the promotion path deleted the very thing it needed +# and left the unit restart-looping (observed: 47 restarts, zero sessions at the +# access concentrator). +# +# Why a reconciler and not just transition scripts: VyOS delivers +# `transition-script` through keepalived-fifo.py, and on 2026-09-02 that helper +# logged NOTHING for a promotion while Keepalived_vrrp logged all six instances +# entering MASTER. A router held every VIP with no WAN -- the outage, recreated +# by the mechanism meant to prevent it. Scripts give speed; the timer gives +# correctness. # # vrrp-wan-reconcile reconcile once # vrrp-wan-reconcile --status what it thinks, changing nothing -# -# Ground truth for "am I master" is whether the management VIP is really on this -# box. It is what VRRP actually does, it is observable, and it cannot silently -# disagree with reality. -VIP="${VRRP_WAN_VIP:-192.168.1.1}" # management VIP; sim overrides via env -WAN_VIF=53 # bond0.53, the DHCP WAN +CONF=/config/vrrp-wan.conf +[ -r "$CONF" ] && . "$CONF" +VIP="${VRRP_WAN_VIP:-192.168.1.1}" +WAN_VIF="${WAN_VIF:-53}" +FLAP_MAX="${FLAP_MAX:-6}" +FLAP_WINDOW="${FLAP_WINDOW:-600}" +FLAP_HOLDOFF="${FLAP_HOLDOFF:-900}" + STATE=/run/vrrp-wan LOCK=/run/vrrp-wan.lock +APPLY=/config/vrrp-wan-apply +DROPIN=/etc/systemd/system/ppp@pppoe0.service.d/10-vrrp-wan-gate.conf cfg() { /opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands 2>/dev/null; } holds_vip() { ip -4 -o addr show 2>/dev/null | grep -q " ${VIP}/"; } wan_up() { ip -4 addr show "bond0.${WAN_VIF}" 2>/dev/null | grep -q 'inet '; } -wan_disabled(){ cfg | grep -q "vif ${WAN_VIF} disable"; } -ppp_disabled(){ cfg | grep -q "pppoe pppoe0 disable"; } +ppp_up() { ip -4 addr show pppoe0 2>/dev/null | grep -q 'inet '; } +ppp_active() { systemctl is-active --quiet ppp@pppoe0 2>/dev/null; } +lease_age() { s=$(stat -c %Y "$STATE/may-dial" 2>/dev/null) || return 1 + echo $(( $(date +%s) - s )); } + +# NOTE: there is deliberately no ppp_disabled(). pppoe0 is now ENABLED in config +# on both routers, so such a test would be permanently false and the backup +# early-exit below would never fire -- entering config mode every 30s for ever, +# committing nothing. That exact shape was already live on the sim secondary, +# whose /tmp/vrrp-wan-commit.log read "No configuration changes to commit" while +# the script reported success. +wan_disabled(){ cfg | grep -q "vif ${WAN_VIF} disable"; } -# --status must answer WITHOUT sourcing script-template. The template's `exit` -# is a function that leaves configuration mode, not the shell builtin, so a -# status run that had sourced it opened and closed a config session on every -# call -- which is how a read-only query started colliding with the timer and -# logging "Configuration system temporarily locked due to another commit". if [ "${1:-}" = "--status" ]; then - printf 'vip=%s holds_vip=%s wan_disabled=%s wan_up=%s role=%s\n' \ + printf 'vip=%s holds_vip=%s wan_disabled=%s wan_up=%s ppp_up=%s ppp_active=%s may_dial=%s lease_age=%s dropin=%s role=%s\n' \ "$VIP" "$(holds_vip && echo yes || echo no)" \ "$(wan_disabled && echo yes || echo no)" \ "$(wan_up && echo yes || echo no)" \ + "$(ppp_up && echo yes || echo no)" \ + "$(ppp_active && echo yes || echo no)" \ + "$([ -f "$STATE/may-dial" ] && echo yes || echo no)" \ + "$(lease_age 2>/dev/null || echo -)" \ + "$([ -f "$DROPIN" ] && echo yes || echo MISSING)" \ "$(cat "$STATE/role" 2>/dev/null || echo unset)" exit 0 fi -# One writer. The 30s timer and a VRRP transition can fire together, and two -# VyOS commits in flight on one box do not queue -- the second fails outright. -# -# The lock fd MUST be closed for children (`9>&-` on every call below). Entering -# VyOS config mode spawns a long-lived unionfs-fuse for the session, and it -# INHERITS this descriptor and never lets go -- `lsof /run/vrrp-wan.lock` showed -# it held by unionfs-fuse with fd 9w. From the first config change onward every -# later run lost the flock and exited 0 without doing anything, so the -# reconciler looked healthy in the journal ("Finished") while quietly having -# stopped reconciling. It is how a demoted router kept the WAN. +# One writer. The lock fd MUST be closed for children (`9>&-` on every call): +# entering VyOS config mode spawns a long-lived unionfs-fuse for the session +# which INHERITS the descriptor and never releases it, so from the first commit +# onward every later run lost the flock and exited 0 having done nothing. That +# is how a demoted router kept the WAN. exec 9>"$LOCK" flock -n 9 || exit 0 mkdir -p "$STATE" -# Reap config sessions whose owning process is gone. VyOS creates -# /opt/vyatta/config/tmp/new_config_ (a unionfs mount) per `configure`, and -# a script that dies inside a session never removes it. One of those holds the -# commit lock, and from then on EVERY commit fails with "Configuration system -# temporarily locked due to another commit in progress" -- including the manual -# one you try in order to fix it. A job on a 30s timer that can leak a session -# per failure will wedge the router's config system on its own, so it cleans up -# before it starts. `umount -l` first: the directory is a mount point and plain -# rm returns "Device or resource busy". +# Reap config sessions whose owning process is gone. VyOS leaves a unionfs mount +# per `configure`, one of them holds the commit lock, and after that EVERY commit +# fails -- including the manual one you try in order to fix it. for d in /opt/vyatta/config/tmp/new_config_*; do [ -d "$d" ] || continue pid=${d##*_} @@ -81,34 +91,84 @@ for d in /opt/vyatta/config/tmp/new_config_*; do rm -rf "$d" 2>/dev/null done -# The config edit lives in vrrp-wan-apply, because script-template must be the -# first thing its script does -- sourced any later it terminates the script -# silently with rc=0. See the header there. -APPLY=/config/vrrp-wan-apply - -if holds_vip; then - echo master > "$STATE/role" - # Stamp only on entry to master, so the health check's grace window measures - # time-since-promotion rather than time-since-last-tick. - [ -f "$STATE/since" ] || date +%s > "$STATE/since" - wan_disabled || ppp_disabled || exit 0 - logger -t vrrp-wan "MASTER with WAN disabled -> enabling bond0.${WAN_VIF} + pppoe0" - "$APPLY" enable 9>&- -else - echo backup > "$STATE/role" - rm -f "$STATE/since" - { wan_disabled && ppp_disabled; } && exit 0 - # Releasing matters more than taking. A demoted router that keeps the WAN up - # holds the cloned MAC f0:9f:c2:12:9b:4f on VLAN 53 at the same time as the - # new master, and the switch sends the ISP's replies to whichever port spoke - # last -- the WAN-side twin of the eth2 incident. - logger -t vrrp-wan "not MASTER but WAN enabled -> releasing bond0.${WAN_VIF} + pppoe0" - "$APPLY" disable 9>&- +# `show configuration commands` has been observed returning EMPTY transiently +# under commit-lock contention. Every grep against it then reads false, which on +# the master path looks like "the WAN is disabled" and triggers a pointless +# commit -- one such spurious "releasing" was logged on a box where both WANs +# were already in the right state. A real config is ~500 lines; refuse to act on +# a suspiciously short one. +if [ "$(cfg | wc -l)" -lt 50 ]; then + logger -t vrrp-wan "config read returned <50 lines; skipping this tick" + exit 0 fi -# Deliberately no `save`. config.boot keeps `disable` on BOTH routers, so a -# reboot in any order comes up unable to claim the shared MAC, and only holding -# the VIP re-enables it. NOTE: any `save` while this box is master (a hand -# commit, or `pulumi up`) WILL persist the enabled state -- observed in labsim. -# The Pulumi model asserts `disable` on both routers so an apply puts it back, -# and vyos:verify reports it as drift if it does not. +# --- PPPoE: the systemd plane --------------------------------------------- +ppp_dial() { + # Refuse to bless a box whose gate is missing. /etc is per-image, so a VyOS + # upgrade silently drops the drop-in -- and without it BOTH routers dial on + # the next commit that touches the pppoe subtree. Failing closed turns a + # silent loss of protection into "PPPoE never comes up", the safe direction. + if [ ! -f "$DROPIN" ]; then + logger -t vrrp-wan "REFUSING to dial: gate drop-in $DROPIN is missing (VyOS upgrade?)" + return 1 + fi + now=$(date +%s) + if [ -f "$STATE/holdoff" ] && [ "$now" -lt "$(cat "$STATE/holdoff" 2>/dev/null || echo 0)" ]; then + return 1 + fi + touch "$STATE/may-dial" # renew the lease every tick + ppp_active && return 0 + # Trim the dial log to the window, then decide. + if [ -f "$STATE/dials" ]; then + awk -v c="$((now - FLAP_WINDOW))" '$1 > c' "$STATE/dials" > "$STATE/dials.new" 2>/dev/null + mv "$STATE/dials.new" "$STATE/dials" 2>/dev/null + fi + if [ "$(wc -l < "$STATE/dials" 2>/dev/null || echo 0)" -ge "$FLAP_MAX" ]; then + echo $((now + FLAP_HOLDOFF)) > "$STATE/holdoff" + logger -t vrrp-wan "DIAL FLAP: >=${FLAP_MAX} attempts in ${FLAP_WINDOW}s -- holding off ${FLAP_HOLDOFF}s" + return 1 + fi + echo "$now" >> "$STATE/dials" + logger -t vrrp-wan "MASTER: dialling pppoe0" + systemctl reset-failed ppp@pppoe0 2>/dev/null + # `systemctl start` exits 0 even when a Condition blocks the start, so its + # return code proves nothing. is-active is the only honest answer. + systemctl start ppp@pppoe0 2>/dev/null +} + +ppp_release() { + # Order matters: revoke the lease FIRST, then stop. The file's absence blocks + # any NEW start (including one a concurrent VyOS commit would trigger); the + # stop kills the process that already exists. Stopping first leaves a window + # in which a commit re-dials a box that is being demoted. + rm -f "$STATE/may-dial" + ppp_active || return 0 + logger -t vrrp-wan "not MASTER: hanging up pppoe0" + systemctl stop ppp@pppoe0 2>/dev/null +} + +# --- decide ---------------------------------------------------------------- +if holds_vip; then + echo master > "$STATE/role" + [ -f "$STATE/since" ] || date +%s > "$STATE/since" + ppp_dial + wan_disabled || exit 0 + logger -t vrrp-wan "MASTER with bond0.${WAN_VIF} disabled -> enabling" + t0=$(date +%s) + "$APPLY" enable 9>&- + logger -t vrrp-wan "bond0.${WAN_VIF} enable commit took $(( $(date +%s) - t0 ))s" +else + echo backup > "$STATE/role" + rm -f "$STATE/since" "$STATE/holdoff" + ppp_release + wan_disabled && exit 0 + logger -t vrrp-wan "not MASTER but bond0.${WAN_VIF} enabled -> releasing" + t0=$(date +%s) + "$APPLY" disable 9>&- + logger -t vrrp-wan "bond0.${WAN_VIF} disable commit took $(( $(date +%s) - t0 ))s" +fi + +# No `save`, deliberately. config.boot keeps `vif 53 disable` on BOTH routers, so +# a reboot in any order comes up unable to claim the cloned MAC. PPPoE needs no +# such convention any more: with the gate, config.boot is safe by construction +# and a stray `save` cannot make both boxes dial. diff --git a/migration/vrrp-wan-reconcile.service b/migration/vrrp-wan-reconcile.service index 66cdc6f..770a3a4 100644 --- a/migration/vrrp-wan-reconcile.service +++ b/migration/vrrp-wan-reconcile.service @@ -3,7 +3,9 @@ # observed dropping a MASTER transition silently, leaving a router holding every # VIP with no WAN. A timer cannot be dropped the same way. Description=Reconcile WAN interface state with VRRP mastership -After=keepalived.service +# vyos-router loads config at boot and its pppoe handler will try to dial; the +# gate drop-in blocks that, but ordering after it keeps the logs readable. +After=keepalived.service vyos-router.service [Service] Type=oneshot diff --git a/migration/vrrp-wan.conf b/migration/vrrp-wan.conf new file mode 100644 index 0000000..3607b51 --- /dev/null +++ b/migration/vrrp-wan.conf @@ -0,0 +1,42 @@ +# Settings for the vrrp-wan scripts. Installed to /config/vrrp-wan.conf. +# +# Why a file and not systemd Environment=: keepalived invokes vrrp-wan-health +# with NO environment at all, so an Environment= line in the .service would be +# read by the reconciler and ignored by the health check -- two sources of truth +# for the one value that decides who is master. It is also what stops a repeat +# of 2026-09-05, when the sim's proof was obtained from scripts hand-`sed`-ed in +# place: /config/vrrp-wan-health differed from git, and a reinstall would have +# silently reverted the tested behaviour. + +# The management VIP. "Do I hold this address" IS the definition of master here +# -- ground truth, not a marker written by a script that may not have run. +VRRP_WAN_VIP=192.168.1.1 + +# The DHCP WAN sub-interface. Stays on the config plane because its lease is +# bound to a cloned MAC, which only VyOS config can move. +WAN_VIF=53 + +# Seconds a new master may go without any WAN before the health check fails it. +# +# Must exceed the ISP's stale-session hold-down, or a hard failover blows the +# window and BOTH routers end up in FAULT -- worse than the outage the check +# exists to prevent. accel-ppp's default dead-peer budget is +# lcp-echo-interval(30) x lcp-echo-failure(3) = 90s, so 90 sat exactly on the +# boundary. Measured worst case: see labsim/wan-failover-evidence/. +GRACE=180 + +# may-dial is a LEASE, not a flag. vrrp-wan-reconcile renews its mtime every +# tick; vrrp-wan-guard revokes it once it goes stale. A plain flag survives the +# reconciler dying, and a router that stops reconciling while demoted would keep +# dialling for ever. +LEASE_TTL=75 + +# Flap damper. Two routers that both believe they hold the VIP (a VRRP +# partition) will both dial; with the AC set to `replace` each dial kills the +# other's session, the loser's pppd exits non-zero, systemd redials in 5s, and +# the pair hammers the access concentrator indefinitely. Against a real ISP that +# is how an account gets rate-limited. More than FLAP_MAX dials in FLAP_WINDOW +# puts this box in hold-off and logs loudly. +FLAP_MAX=6 +FLAP_WINDOW=600 +FLAP_HOLDOFF=900