diff --git a/labsim/labsim-pppoe-ha-test.sh b/labsim/labsim-pppoe-ha-test.sh index 2bce68c..fe63916 100755 --- a/labsim/labsim-pppoe-ha-test.sh +++ b/labsim/labsim-pppoe-ha-test.sh @@ -56,7 +56,12 @@ FAILED=0 ac_sessions() { isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show pppoe-server sessions' \ | grep -c ' simdsl ' || true; } ac_detail() { isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show pppoe-server sessions'; } -ppp_on() { r "$1" 'ip -4 addr show pppoe0 2>/dev/null | grep -c inet' | tr -d ' \n'; } +# A destroyed or unreachable router is emphatically NOT holding pppoe0, but ssh +# returns an EMPTY string rather than 0 -- and `[ "" = 0 ]` is false, so a +# hard-failover test waited for the dead box to "report" zero and hung until its +# timeout, long after the survivor had taken over correctly. Default to 0. +ppp_on() { local v; v="$(r "$1" 'ip -4 addr show pppoe0 2>/dev/null | grep -c inet' | tr -d ' \n')" + echo "${v:-0}"; } holder() { for h in "$R1" "$R2"; do [ "$(r "$h" "ip -4 -o addr show | grep -c ' ${VIP}/'" | tr -d ' \n')" != 0 ] \ && { echo "$h"; return; }; done; echo none; } @@ -219,9 +224,57 @@ T8() { # lease expiry: the guard must hang up a demoted-but-unreconciled box settle 24 >/dev/null } + +T4() { # hard failover across all three AC session-control policies + log "T4 hard failover (destroy the master) x session-control" + # Vodafone's policy is unknowable from here, so prove the design survives + # every one VyOS can express. `replace` is the accel-ppp default and the + # friendly case; `deny` is the hostile one, where the AC refuses the second + # session until its own dead-peer timer (lcp-echo-interval 30 x failure 3 = + # 90s) frees the first -- which is exactly why GRACE is no longer 90. + local mode from to vm t0 t1 + for mode in replace deny disable; do + log " --- session-control=$mode ---" + isp "vbash -c 'source /opt/vyatta/etc/functions/script-template; configure; set service pppoe-server session-control $mode; commit; save; exit'" >/dev/null 2>&1 + sleep 5 + from="$(holder)"; to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1") + vm=$([ "$from" = "$R1" ] && echo labsim-vyos || echo labsim-vyos2) + [ "$from" = none ] && { fail "no master before $mode run"; continue; } + log " destroying $vm (master=$from), expecting $to" + t0=$(date +%s) + sudo virsh destroy "$vm" >/dev/null 2>&1 + if settle_on "$to" 48; then + t1=$(date +%s) + pass "$mode: pppoe0 reached $to in $((t1-t0))s" + else + fail "$mode: $to never dialled within 240s" + fi + check_invariant + save_evidence "T4-hard-failover-$mode" + sudo virsh start "$vm" >/dev/null 2>&1 + # Re-bond. A VM restart recreates its taps under NEW names, and the OVS + # bond keeps the old ones -- lacp dies, VLAN 1 goes with it, and the box + # comes back reachable on some VLANs but not others. ovs_bond_router + # detects the stale membership and rebuilds, but nothing runs it + # automatically, so a destroy/start test must do it or the survivor + # looks like a failover failure. + ( source "$SCRIPT_DIR/lib.sh"; source "$SCRIPT_DIR/ovs.sh"; selected_vlans + LAG_NAME=$([ "$vm" = labsim-vyos ] && echo lag-vyos || echo lag-vyos2) + ovs_bond_router "$vm" ) >/dev/null 2>&1 + # Give the returning box time to boot and settle as BACKUP before the + # next iteration; it must NOT dial on the way up. + sleep 90 + [ "$(ppp_on "$from")" = 0 ] && pass "$mode: $from did not dial on reboot" \ + || fail "$mode: $from dialled on reboot (gate failed)" + done + isp "vbash -c 'source /opt/vyatta/etc/functions/script-template; configure; set service pppoe-server session-control replace; commit; save; exit'" >/dev/null 2>&1 + log " AC restored to session-control=replace" +} + case "${1:---all}" in - --list) echo "T0 baseline | T3 clean failover | T5 10gig-down | T8 lease expiry | T11 no-peers-file"; exit 0 ;; + --list) echo "T0 baseline | T3 clean failover | T4 hard failover x policy | T5 10gig-down | T8 lease expiry | T11 no-peers-file"; exit 0 ;; --all) preflight || exit 1; T0; T3; T5; T8; T11 ;; + --hard) preflight || exit 1; T4 ;; *) preflight || exit 1; "$1" ;; esac diff --git a/labsim/wan-failover-evidence/T4-hard-failover-replace/state.txt b/labsim/wan-failover-evidence/T4-hard-failover-replace/state.txt new file mode 100644 index 0000000..14ca0e8 --- /dev/null +++ b/labsim/wan-failover-evidence/T4-hard-failover-replace/state.txt @@ -0,0 +1,17 @@ +=== hard failover, session-control=replace, 2026-09-05T19:20:56+01:00 === +master labsim-vyos2 (172.31.1.253) destroyed with virsh destroy -- no PADT sent. + +-- survivor took over and dialled -- + 172.31.1.252 holds_vip=yes wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes role=master + +-- AC: exactly one session throughout, from the survivor's MAC -- + ppp0 simdsl 198.51.100.121 calling-sid 52:54:00:e5:95:a2 active 00:07:01 + +-- returning router did NOT dial on boot (the gate) -- + ConditionResult=no ActiveState=inactive NRestarts=0 pppoe0 absent may_dial=no + AC session count after its return: 1 + +-- caveat -- + A VM restart recreates its taps, leaving the OVS bond holding stale members + (lacp dead, VLAN 1 unreachable). Re-bonded via ovs_bond_router; the harness + now does this automatically after virsh start. diff --git a/migration/PPPOE-HA.md b/migration/PPPOE-HA.md new file mode 100644 index 0000000..d170434 --- /dev/null +++ b/migration/PPPOE-HA.md @@ -0,0 +1,116 @@ +# PPPoE high availability + +Proven in labsim. **Not applied to production.** + +## What it does + +One consumer ISP account, two routers. The 10 gig lease is bound to a cloned MAC +(`f0:9f:c2:12:9b:4f`, the retired USG's) and the Vodafone line to a single +credential, so neither may be live on both boxes. The WAN follows VRRP +mastership — but the two halves use different control planes, and that is the +whole design: + +| | plane | why | +|---|---|---| +| `bond0.53` (10 gig) | VyOS **config** (`disable`) | only config can move a MAC | +| `pppoe0` (Vodafone) | **systemd** unit gate | see below | + +## Why PPPoE cannot live on the config plane + +`interfaces_pppoe.py` treats `disable` and `delete` identically: both **unlink +`/etc/ppp/peers/pppoe0`**, call `PPPoEIf.remove()` (withdrawing the FRR default +route) and stop the unit. That path is pppd's own options file +(`ExecStart=/usr/sbin/pppd call %I`), so the resting state destroyed exactly what +the promotion path needed. `ppp@pppoe0` then restart-looped against the missing +file — 47 restarts observed, zero sessions at the access concentrator — and +never tripped systemd's limiter, because `RestartSec=5s` against the default +10s/5-burst window is only two restarts per interval. + +It also made op-mode `connect interface pppoe0` unusable (it refuses without the +peers file), and put every failover behind a priority-322 commit where one +unrelated invalid node fails the whole thing. + +## The gate + +`pppoe0` is configured identically and **enabled on both** routers, so the peers +file always exists. Dialling is gated by +`/etc/systemd/system/ppp@pppoe0.service.d/10-vrrp-wan-gate.conf`: + +```ini +ConditionPathExists=/run/vrrp-wan/may-dial +ConditionPathExists=/etc/ppp/peers/pppoe0 +StartLimitIntervalSec=600 +StartLimitBurst=6 +``` + +`/run` is tmpfs, so the gate is shut at boot and neither box can dial before VRRP +has decided. **This is load-bearing, not a nicety:** with the node enabled, +`interfaces_pppoe.py` restarts ppp on *every* commit touching the pppoe subtree +when the daemon is not running — so the backup actively tries to dial whenever +anything commits (`pulumi up`, a hand commit, the boot-time config load). The +gate is the only thing making that a no-op, which is why `vrrp-wan-reconcile` +**refuses to bless a box whose drop-in is missing**: `/etc` is per-image, so a +VyOS upgrade silently removes the protection, and failing closed turns that into +"PPPoE never dials" rather than "both routers dial". + +`may-dial` is a **lease, not a flag**. `ConditionPathExists` is evaluated at +start only — it can prevent a dial, never revoke one. `vrrp-wan-reconcile` +renews it every 30s; `vrrp-wan-guard` runs every 5s and only ever revokes, on +either "I do not hold the VIP" or "the lease is stale". + +## Measured in labsim + +| | | +|---|---| +| Clean failover (`force-fault`) | `pppoe0` moves in **26s**, reproducible | +| 10 gig down → PPPoE | route falls to `pppoe0`; LAN back online in **5s** | +| Stale lease | guard hangs up within ~5s | +| Missing peers file | `NRestarts=0` — no loop | +| Invariant | AC never showed two `simdsl` sessions | + +## Deploying (not yet done) + +1. `sudo /config/vyos-known-good save` on both. +2. `migration/vrrp-wan-install --vip 192.168.1.1 --host vyos@10.0.1.253` then the + same for `.252`. Then `--check` on both. **No config change yet** — verify + nothing dials. +3. Confirm `/config/wan-secrets` is present and identical on both. +4. **vyos002 first** (the non-master), on its own commit — `interfaces pppoe` is + priority 322 and one bad node fails everything: + `delete interfaces pppoe pppoe0 disable`, `commit-confirm 10`. +5. Verify vyos002 did **not** dial — check on the wire, not from state: + `sudo tcpdump -i bond0.51 -nn pppoed` should show no PADI. Then `confirm`; `save`. +6. vyos001: nothing to change; it already has `pppoe0` enabled. +7. Confirm `vif 53 disable` is in **both** `config.boot`s. +8. Add the drop-in re-install to the VyOS image-upgrade runbook. + +**Rollback**, from either box: `set interfaces pppoe pppoe0 disable` on both and +`rm /run/vrrp-wan/may-dial`. That restores today's behaviour exactly. + +## What the sim cannot prove + +- **Vodafone's `session-control`.** The sim's accel-ppp defaults to `replace`, so + a new auth kills the old session immediately. A real BRAS may `deny` and hold + the session for its own dead-peer timer. The matrix runs all three modes to + bracket the risk; it cannot tell you which one you will meet, and account + rate-limiting or lockout on repeated dials has no sim analogue at all. The + flap damper (6 dials / 600s → 15 min hold-off) exists for that. +- **Whether Vodafone honours our LCP Terminate / PADT** on a graceful stop. +- **The cloned-MAC lease** — whether the 10 gig ISP re-issues `87.192.101.48` to + `f0:9f:c2:12:9b:4f` arriving on a different switch port. That risk belongs to + `bond0.53`, not PPPoE, and is the largest untested item in the failover. +- **Real dial time and MTU/MSS under load.** PPPoE was proven on the *USG*; + VyOS dialling Vodafone has never been done. +- **Timing under load.** The sim routers are idle 2-vCPU VMs; commit latency on + the VP2440s under kea + BGP + conntrack will be worse, and commit latency is + the dominant term in the `bond0.53` half of a failover. + +## A model hazard to fix before applying + +The imported baseline records the **running** state, not the safe one: vyos001 +has no `vif 53 disable` (it is master), vyos002 does. So an apply performed while +vyos002 held the VIP would enable vyos001's WAN as well, putting the cloned MAC +on both boxes. An override must assert `vif 53 disable` on **both** routers, with +the reconciler re-enabling whichever holds the VIP. Note the consequence: an +apply then briefly disables the current master's 10 gig until the reconciler +restores it (≤30s).