diff --git a/labsim/labsim-pppoe-ha-test.sh b/labsim/labsim-pppoe-ha-test.sh index 87669b1..6efef57 100755 --- a/labsim/labsim-pppoe-ha-test.sh +++ b/labsim/labsim-pppoe-ha-test.sh @@ -34,6 +34,30 @@ SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password) r() { timeout 45 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; } isp() { timeout 30 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$ISP" "$@" 2>/dev/null; } + +# Set the AC's session policy, and PROVE it landed. +# +# `vbash -c 'source script-template; configure; ...; commit'` does NOT work: the +# config session never starts and commit dies with "Invalid command: [commit]", +# on stderr, which the isp() helper discards. The whole T4 matrix therefore ran +# all three iterations against the accel-ppp DEFAULT while printing +# "--- session-control=deny ---" -- it reported coverage it did not have, which +# is worse than reporting a failure. Drive it from a real script FILE, then read +# the value back and abort the run if it disagrees. +isp_session_control() { + local mode="$1" + printf '#!/bin/vbash\nsource /opt/vyatta/etc/functions/script-template\nconfigure\nset service pppoe-server session-control %s\ncommit\nsave\nexit\n' "$mode" \ + | timeout 30 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$ISP" 'cat > /tmp/set-sc.sh && chmod +x /tmp/set-sc.sh && sudo /tmp/set-sc.sh' >/dev/null 2>&1 + local got + got="$(isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands' \ + | sed -n "s/.*session-control '\\(.*\\)'/\\1/p")" + if [ "$got" = "$mode" ]; then + log " AC session-control=$mode (verified)" + return 0 + fi + fail "could not set AC session-control=$mode (reads '${got:-unset}') -- results would be fiction" + return 1 +} # The LAN VMs are Alpine and their sshd offers keyboard-interactive, not # `password`. Reusing the routers' option set here made ssh exit 255 BEFORE # running anything, and T5 read that as "the LAN lost the internet" while a @@ -264,7 +288,8 @@ T4() { # hard failover across all three AC session-control policies local mode from to vm t0 t1 for mode in replace deny disable; do log " --- session-control=$mode ---" - isp "vbash -c 'source /opt/vyatta/etc/functions/script-template; configure; set service pppoe-server session-control $mode; commit; save; exit'" >/dev/null 2>&1 + # Skip the iteration rather than measure the wrong policy. + isp_session_control "$mode" || continue sleep 5 from="$(holder)"; to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1") vm=$([ "$from" = "$R1" ] && echo labsim-vyos || echo labsim-vyos2) @@ -296,7 +321,7 @@ T4() { # hard failover across all three AC session-control policies [ "$(ppp_on "$from")" = 0 ] && pass "$mode: $from did not dial on reboot" \ || fail "$mode: $from dialled on reboot (gate failed)" done - isp "vbash -c 'source /opt/vyatta/etc/functions/script-template; configure; set service pppoe-server session-control replace; commit; save; exit'" >/dev/null 2>&1 + isp_session_control replace >/dev/null log " AC restored to session-control=replace" } diff --git a/labsim/wan-failover-evidence/T4-hard-failover-deny/state.txt b/labsim/wan-failover-evidence/T4-hard-failover-deny/state.txt new file mode 100644 index 0000000..38dd919 --- /dev/null +++ b/labsim/wan-failover-evidence/T4-hard-failover-deny/state.txt @@ -0,0 +1,19 @@ +=== 2026-09-06T00:13:35+01:00 === +--- AC sessions --- + ifname | username | ip | ip6 | ip6-dp | calling-sid | rate-limit | state | uptime | rx-bytes | tx-bytes +--------+----------+----------------+-----+--------+-------------------+------------+--------+----------+----------+---------- + ppp0 | simdsl | 198.51.100.128 | | | 52:54:00:e5:95:a2 | | active | 00:00:31 | 514 B | 204 B +--- 172.31.1.252 --- +vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes lease_age=0 dropin=yes role=master +pppoe0 UNKNOWN 198.51.100.128 peer 198.51.100.1/32 +default via 203.0.113.1 dev bond0.53 proto failover metric 1 +Sep 05 23:06:18 apitest vrrp-wan[18704]: MASTER with bond0.53 disabled -> enabling +Sep 05 23:06:22 apitest vrrp-wan[19398]: bond0.53 enable commit took 4s +Sep 05 23:07:35 apitest vrrp-wan[22111]: GUARD: lease stale (206s > 75s; is vrrp-wan-reconcile.timer running?) -- hanging up pppoe0 +Sep 05 23:07:43 apitest vrrp-wan[22667]: MASTER: dialling pppoe0 +Sep 05 23:08:14 apitest vrrp-wan[24120]: MASTER: dialling pppoe0 +-- Boot c5f23399239b468c8c8b752a4305c515 -- +Sep 05 23:13:04 apitest vrrp-wan[6261]: MASTER: dialling pppoe0 +Sep 05 23:13:04 apitest vrrp-wan[6413]: MASTER with bond0.53 disabled -> enabling +Sep 05 23:13:08 apitest vrrp-wan[7122]: bond0.53 enable commit took 4s +--- 172.31.1.253 --- diff --git a/migration/pulumi-override-pppoe-gated.json b/migration/pulumi-override-pppoe-gated.json new file mode 100644 index 0000000..5fb7efe --- /dev/null +++ b/migration/pulumi-override-pppoe-gated.json @@ -0,0 +1,55 @@ +{ + "_comment": [ + "STAGED, NOT APPLIED. Paste these two objects into", + "kubernetes-deployment infra/vyos/subtrees/overrides.json -- but ONLY after", + "migration/vrrp-wan-install has run on BOTH production routers.", + "", + "Ordering is not a nicety. Another agent runs `pulumi up` on that repo, so", + "merging this file IS a production change, made by someone else, at a time", + "you do not choose. Removing `interfaces pppoe pppoe0 disable` from vyos002", + "before the gate exists there lets vyos002 dial the moment anything commits,", + "and Vodafone is a single-session account: it would take the live session off", + "vyos001 and drop the household's internet.", + "", + "Install the gate first. Verify `systemctl show ppp@pppoe0 -p ConditionResult`", + "reads `no` on vyos002. Only then merge.", + "", + "Note there is deliberately NO override asserting `vif 53 disable`.", + "applyTree is delete-then-set per subtree, so the model is authoritative and", + "omission means deletion -- but the 10 gig resting state is RUNTIME state", + "owned by vrrp-wan-reconcile, keyed on who holds the management VIP. Pinning", + "it in the model would fight the reconciler on every apply. The rule is", + "`npm run vyos:export && npm run vyos:render` immediately before any", + "`pulumi up`, so the model follows whichever router actually holds the WAN." + ], + "overrides": [ + { + "id": "pppoe-gated-not-config-disabled", + "routers": ["vyos002"], + "reason": "PPPoE cannot live on the VyOS config plane. interfaces_pppoe.py treats `disable` and `delete` identically: both unlink /etc/ppp/peers/pppoe0, which is pppd's own options file (ExecStart=/usr/sbin/pppd call %I), call PPPoEIf.remove() to withdraw the FRR default route, and stop the unit. So the resting state destroyed exactly what the promotion path needed, and ppp@pppoe0 restart-looped against the missing file -- 47 restarts observed, zero sessions at the access concentrator -- without ever tripping systemd's limiter, because RestartSec=5s against the default 10s/5-burst window is only two restarts per interval. It also made op-mode `connect interface pppoe0` unusable and put every failover behind a priority-322 commit where one unrelated invalid node fails the whole thing. pppoe0 is now ENABLED on both routers so the peers file always exists, and dialling is gated by ppp@pppoe0.service.d/10-vrrp-wan-gate.conf on /run/vrrp-wan/may-dial, a lease renewed by vrrp-wan-reconcile and revoked by vrrp-wan-guard. /run is tmpfs, so the gate is shut at boot and neither box dials before VRRP has decided. This override encodes must-never-come-back: a re-run of migration/vyos-mode-delta.py or a stale import must not re-pin `disable`. PREREQUISITE: the gate drop-in must already be installed on vyos002, or removing `disable` lets it dial and steal the single ISP session. Proven in labsim across session-control replace/deny/disable; see lab migration/PPPOE-HA.md.", + "remove": [["interfaces", "pppoe", "pppoe0", "disable"]] + }, + { + "id": "vrrp-transition-scripts-wan-follows-master", + "reason": "Gives the WAN a fast path on top of the 30s reconcile timer. Both hooks exec the same reconciler -- one code path, asked at different moments -- so a lost or duplicated transition cannot desynchronise anything; the timer remains the correctness guarantee and the scripts are only latency. `stop` and `fault` matter as much as `backup`: a stopped keepalived is a demotion too, and without those a box would keep the WAN while holding no VIPs, which is the 2026-09-02 outage shape. They go on the SYNC GROUP because VyOS refuses a per-group script while the group is in a sync group. Do not rely on these alone: on 2026-09-02 keepalived-fifo.py logged NOTHING for a promotion while Keepalived_vrrp logged all six instances entering MASTER, which is precisely why the reconcile timer exists.", + "set": [ + { + "path": ["high-availability", "vrrp", "sync-group", "MAIN", "transition-script", "master"], + "value": "/config/vrrp-wan-take" + }, + { + "path": ["high-availability", "vrrp", "sync-group", "MAIN", "transition-script", "backup"], + "value": "/config/vrrp-wan-release" + }, + { + "path": ["high-availability", "vrrp", "sync-group", "MAIN", "transition-script", "fault"], + "value": "/config/vrrp-wan-release" + }, + { + "path": ["high-availability", "vrrp", "sync-group", "MAIN", "transition-script", "stop"], + "value": "/config/vrrp-wan-release" + } + ] + } + ] +}