diff --git a/labsim/labsim-pppoe-ha-test.sh b/labsim/labsim-pppoe-ha-test.sh index 6efef57..9c85789 100755 --- a/labsim/labsim-pppoe-ha-test.sh +++ b/labsim/labsim-pppoe-ha-test.sh @@ -95,11 +95,35 @@ status() { r "$1" 'sudo /config/vrrp-wan-reconcile --status'; } ppp_holders() { n=0; for h in "$R1" "$R2"; do [ "$(ppp_on "$h")" != 0 ] && n=$((n+1)); done; echo "$n"; } +warn() { printf ' \033[33mWARN\033[0m %s\n' "$*"; } + check_invariant() { local s p ok=0 s="$(ac_sessions)"; p="$(ppp_holders)" - [ "${s:-0}" -le 1 ] || { fail "INVARIANT: AC reports $s simdsl sessions"; ok=1; } + + # THE invariant. Two of OUR routers dialled at once is the failure that + # matters: one ISP account, and against a real ISP that is how you get + # rate-limited or locked out. [ "${p:-0}" -le 1 ] || { fail "INVARIANT: $p routers hold pppoe0"; ok=1; } + + # The AC's session count is a PROXY for the above, and only a valid one + # while the AC enforces single-session. Under session-control=disable it + # does not, so a destroyed router's session simply stays in the table and + # the count reads 2 while exactly one live router is dialled -- which is AC + # bookkeeping, not a double dial. Attribute it rather than failing blind: + # only call it a violation when more than one router is ACTUALLY dialled. + # + # Do not silence it either. An orphaned session still occupies the single + # slot at a real ISP, and that is exactly what made session-control=deny + # take 148s while the survivor's dial attempts were refused. + if [ "${s:-0}" -gt 1 ]; then + if [ "${p:-0}" -gt 1 ]; then + fail "INVARIANT: AC reports $s simdsl sessions AND $p routers are dialled" + ok=1 + else + warn "AC reports $s simdsl sessions but only ${p:-0} router is dialled -- stale session from the destroyed peer (expected where the AC does not enforce single-session; it is what a hostile AC holds against the survivor)" + fi + fi return $ok } diff --git a/labsim/wan-failover-evidence/T4-hard-failover-deny/state.txt b/labsim/wan-failover-evidence/T4-hard-failover-deny/state.txt index 38dd919..8a40ab1 100644 --- a/labsim/wan-failover-evidence/T4-hard-failover-deny/state.txt +++ b/labsim/wan-failover-evidence/T4-hard-failover-deny/state.txt @@ -1,19 +1,20 @@ -=== 2026-09-06T00:13:35+01:00 === +=== 2026-09-06T00:21:00+01:00 === --- AC sessions --- ifname | username | ip | ip6 | ip6-dp | calling-sid | rate-limit | state | uptime | rx-bytes | tx-bytes --------+----------+----------------+-----+--------+-------------------+------------+--------+----------+----------+---------- - ppp0 | simdsl | 198.51.100.128 | | | 52:54:00:e5:95:a2 | | active | 00:00:31 | 514 B | 204 B + ppp0 | simdsl | 198.51.100.130 | | | 52:54:00:e5:95:a2 | | active | 00:00:18 | 438 B | 204 B --- 172.31.1.252 --- -vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes lease_age=0 dropin=yes role=master -pppoe0 UNKNOWN 198.51.100.128 peer 198.51.100.1/32 +vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes lease_age=30 dropin=yes role=master +pppoe0 UNKNOWN 198.51.100.130 peer 198.51.100.1/32 default via 203.0.113.1 dev bond0.53 proto failover metric 1 -Sep 05 23:06:18 apitest vrrp-wan[18704]: MASTER with bond0.53 disabled -> enabling -Sep 05 23:06:22 apitest vrrp-wan[19398]: bond0.53 enable commit took 4s -Sep 05 23:07:35 apitest vrrp-wan[22111]: GUARD: lease stale (206s > 75s; is vrrp-wan-reconcile.timer running?) -- hanging up pppoe0 Sep 05 23:07:43 apitest vrrp-wan[22667]: MASTER: dialling pppoe0 Sep 05 23:08:14 apitest vrrp-wan[24120]: MASTER: dialling pppoe0 -- Boot c5f23399239b468c8c8b752a4305c515 -- Sep 05 23:13:04 apitest vrrp-wan[6261]: MASTER: dialling pppoe0 Sep 05 23:13:04 apitest vrrp-wan[6413]: MASTER with bond0.53 disabled -> enabling Sep 05 23:13:08 apitest vrrp-wan[7122]: bond0.53 enable commit took 4s +-- Boot 46727d256db344d6ad4b37c142272071 -- +Sep 05 23:18:31 apitest vrrp-wan[6258]: MASTER: dialling pppoe0 +Sep 05 23:18:31 apitest vrrp-wan[6411]: MASTER with bond0.53 disabled -> enabling +Sep 05 23:18:36 apitest vrrp-wan[7038]: bond0.53 enable commit took 5s --- 172.31.1.253 --- diff --git a/labsim/wan-failover-evidence/T4-hard-failover-disable/state.txt b/labsim/wan-failover-evidence/T4-hard-failover-disable/state.txt new file mode 100644 index 0000000..9e6fb79 --- /dev/null +++ b/labsim/wan-failover-evidence/T4-hard-failover-disable/state.txt @@ -0,0 +1,21 @@ +=== 2026-09-06T00:23:19+01:00 === +--- AC sessions --- + ifname | username | ip | ip6 | ip6-dp | calling-sid | rate-limit | state | uptime | rx-bytes | tx-bytes +--------+----------+----------------+-----+--------+-------------------+------------+--------+----------+----------+---------- + ppp0 | simdsl | 198.51.100.130 | | | 52:54:00:e5:95:a2 | | active | 00:02:37 | 438 B | 204 B + ppp1 | simdsl | 198.51.100.131 | | | 52:54:00:4e:0b:56 | | active | 00:00:27 | 1.1 KiB | 204 B +--- 172.31.1.252 --- +--- 172.31.1.253 --- +vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes lease_age=15 dropin=yes role=master +pppoe0 UNKNOWN 198.51.100.131 peer 198.51.100.1/32 +default nhid 57 dev pppoe0 proto static metric 20 +Sep 05 23:10:44 vyos vrrp-wan[221212]: MASTER with bond0.53 disabled -> enabling +Sep 05 23:10:48 vyos vrrp-wan[221995]: bond0.53 enable commit took 4s +-- Boot 1f635fc6900c4661b3ac6d016d63d293 -- +Sep 05 23:16:10 vyos vrrp-wan[7037]: MASTER: dialling pppoe0 +Sep 05 23:16:11 vyos vrrp-wan[7190]: MASTER with bond0.53 disabled -> enabling +Sep 05 23:16:15 vyos vrrp-wan[7979]: bond0.53 enable commit took 4s +-- Boot c0e38585d7354ad6ab018800c7f3f6be -- +Sep 05 23:22:54 vyos vrrp-wan[5970]: MASTER: dialling pppoe0 +Sep 05 23:22:54 vyos vrrp-wan[6126]: MASTER with bond0.53 disabled -> enabling +Sep 05 23:22:59 vyos vrrp-wan[6915]: bond0.53 enable commit took 5s diff --git a/labsim/wan-failover-evidence/T4-hard-failover-replace/state.txt b/labsim/wan-failover-evidence/T4-hard-failover-replace/state.txt index 8a2fb10..a675a26 100644 --- a/labsim/wan-failover-evidence/T4-hard-failover-replace/state.txt +++ b/labsim/wan-failover-evidence/T4-hard-failover-replace/state.txt @@ -1,18 +1,19 @@ -=== 2026-09-06T00:11:13+01:00 === +=== 2026-09-06T00:16:41+01:00 === --- AC sessions --- ifname | username | ip | ip6 | ip6-dp | calling-sid | rate-limit | state | uptime | rx-bytes | tx-bytes --------+----------+----------------+-----+--------+-------------------+------------+--------+----------+----------+---------- - ppp0 | simdsl | 198.51.100.127 | | | 52:54:00:4e:0b:56 | | active | 00:00:30 | 514 B | 204 B + ppp0 | simdsl | 198.51.100.129 | | | 52:54:00:4e:0b:56 | | active | 00:00:31 | 438 B | 204 B --- 172.31.1.252 --- --- 172.31.1.253 --- -vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes lease_age=10 dropin=yes role=master -pppoe0 UNKNOWN 198.51.100.127 peer 198.51.100.1/32 -default nhid 110 dev pppoe0 proto static metric 20 -Sep 05 22:57:40 vyos vrrp-wan[199948]: GUARD: lease stale (81s > 75s; is vrrp-wan-reconcile.timer running?) -- hanging up pppoe0 -Sep 05 23:02:41 vyos vrrp-wan[206596]: MASTER: dialling pppoe0 -Sep 05 23:06:17 vyos vrrp-wan[213698]: not MASTER: hanging up pppoe0 +vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes lease_age=22 dropin=yes role=master +pppoe0 UNKNOWN 198.51.100.129 peer 198.51.100.1/32 +default nhid 62 dev pppoe0 proto static metric 20 Sep 05 23:06:18 vyos vrrp-wan[213859]: not MASTER but bond0.53 enabled -> releasing Sep 05 23:06:23 vyos vrrp-wan[214503]: bond0.53 disable commit took 5s Sep 05 23:10:43 vyos vrrp-wan[221064]: MASTER: dialling pppoe0 Sep 05 23:10:44 vyos vrrp-wan[221212]: MASTER with bond0.53 disabled -> enabling Sep 05 23:10:48 vyos vrrp-wan[221995]: bond0.53 enable commit took 4s +-- Boot 1f635fc6900c4661b3ac6d016d63d293 -- +Sep 05 23:16:10 vyos vrrp-wan[7037]: MASTER: dialling pppoe0 +Sep 05 23:16:11 vyos vrrp-wan[7190]: MASTER with bond0.53 disabled -> enabling +Sep 05 23:16:15 vyos vrrp-wan[7979]: bond0.53 enable commit took 4s diff --git a/migration/PPPOE-HA.md b/migration/PPPOE-HA.md index c772a57..05f755a 100644 --- a/migration/PPPOE-HA.md +++ b/migration/PPPOE-HA.md @@ -115,19 +115,60 @@ standby that can never take over. `sudo tcpdump -i bond0.51 -nn pppoed` should show no PADI. Then `confirm`; `save`. 6. vyos001: nothing to change; it already has `pppoe0` enabled. 7. Confirm `vif 53 disable` is in **both** `config.boot`s. -8. Add the drop-in re-install to the VyOS image-upgrade runbook. +8. **Only now** merge `migration/pulumi-override-pppoe-gated.json` into + `kubernetes-deployment` `infra/vyos/subtrees/overrides.json`. It is staged + here, unapplied, on purpose: another agent runs `pulumi up` on that repo, so + merging it *is* a production change made by someone else at a time you do not + choose. Removing `pppoe0 disable` from vyos002 before the gate exists there + lets it dial on the next commit and take the single Vodafone session off + vyos001. Run `npm run vyos:export && npm run vyos:render` first so the model + follows whichever router actually holds the WAN. +9. Add the drop-in re-install to the VyOS image-upgrade runbook. + +Step 4 is the one that matters most and is worth stopping on. vyos002 has been +in **FAULT on all six groups for over three days** — verified again while +writing this, alongside vyos001 holding `192.168.1.1` on `bond0.1` and +`pppoe0` up on `83.106.5.72`. Until vyos002 reaches BACKUP there is no standby +at all: if vyos001 died today nothing would pick up the gateway VIPs. The +existing `vrrp-health-check-wan-present` override predicted exactly this in its +own reason text — *"with the primary genuinely dead the secondary stays FAULT +and nothing holds the gateway. The fix for that is WAN-follows-master, which is +a separate change."* This is that change. **Rollback**, from either box: `set interfaces pppoe pppoe0 disable` on both and `rm /run/vrrp-wan/may-dial`. That restores today's behaviour exactly. ## What the sim cannot prove -- **Vodafone's `session-control`.** The sim's accel-ppp defaults to `replace`, so - a new auth kills the old session immediately. A real BRAS may `deny` and hold - the session for its own dead-peer timer. The matrix runs all three modes to - bracket the risk; it cannot tell you which one you will meet, and account - rate-limiting or lockout on repeated dials has no sim analogue at all. The - flap damper (6 dials / 600s → 15 min hold-off) exists for that. +- **Vodafone's `session-control`.** The matrix now brackets it properly. + Destroying the master and timing the survivor's session: + + | policy | takeover | + |---|---| + | `replace` (accel-ppp default) | 26s | + | **`deny`** (hostile) | **148s** | + | `disable` | 21s | + + `deny` is the sizing case: the AC refuses the survivor until its own + dead-peer timer frees the dead session, and the poller caught two dial + attempts being rejected before one succeeded. `GRACE` is set from that — see + `migration/vrrp-wan.conf`. This still cannot tell you which policy Vodafone + runs, and account rate-limiting or lockout on repeated dials has no sim + analogue at all; the flap damper (6 dials / 600s → 15 min hold-off) exists + for that. + + Treat the 148s as a floor rather than a worst case. These are idle 2-vCPU + VMs, and the AC shares an OVS bridge with the routers, so `virsh destroy` + removes the port and accel-ppp sees the peer physically vanish. A real BRAS + reached over DSL does not learn that our router died — it waits out its own + timers, which are longer and not ours to know. + + Worth knowing how close this came to being missed: until 2026-09-06 the + matrix set the policy with `vbash -c 'source script-template; configure; + ...; commit'`, which never starts a config session. `commit` failed to + stderr, the helper discarded it, and all three iterations ran against the + default while printing the mode they were supposedly testing. It reported + `deny` at 25s. The real figure is 148s. - **Whether Vodafone honours our LCP Terminate / PADT** on a graceful stop. - **The cloned-MAC lease** — whether the 10 gig ISP re-issues `87.192.101.48` to `f0:9f:c2:12:9b:4f` arriving on a different switch port. That risk belongs to diff --git a/migration/vrrp-wan-health b/migration/vrrp-wan-health index f7b1706..650d453 100755 --- a/migration/vrrp-wan-health +++ b/migration/vrrp-wan-health @@ -40,7 +40,14 @@ VIP="${VRRP_WAN_VIP:-192.168.1.1}" # the boundary: the new master fails its own check, sheds the VIPs, and the peer # -- in the same position -- does likewise. Both end in FAULT, which is worse # than the outage this check exists to prevent. -GRACE="${GRACE:-180}" +# +# The fallback tracks vrrp-wan.conf, where the reasoning and the measurements +# live. Short version: labsim T4 timed a destroyed master's takeover at 26s +# (replace), 148s (deny) and 21s (disable); `deny` is the sizing case and 180 +# left only 32s over it. Keep the two in step -- keepalived runs this script +# with no environment, so if vrrp-wan.conf is ever missing THIS number is the +# one that decides mastership. +GRACE="${GRACE:-300}" # A deliberate hand-over lever. # diff --git a/migration/vrrp-wan.conf b/migration/vrrp-wan.conf index 3607b51..fc35b58 100644 --- a/migration/vrrp-wan.conf +++ b/migration/vrrp-wan.conf @@ -20,10 +20,38 @@ WAN_VIF=53 # # Must exceed the ISP's stale-session hold-down, or a hard failover blows the # window and BOTH routers end up in FAULT -- worse than the outage the check -# exists to prevent. accel-ppp's default dead-peer budget is -# lcp-echo-interval(30) x lcp-echo-failure(3) = 90s, so 90 sat exactly on the -# boundary. Measured worst case: see labsim/wan-failover-evidence/. -GRACE=180 +# exists to prevent. +# +# MEASURED, labsim T4, master destroyed with `virsh destroy`, time until the +# survivor held a PPPoE session (labsim/wan-failover-evidence/T4-*): +# +# session-control=replace 26s +# session-control=deny 148s <-- worst +# session-control=disable 21s +# +# `deny` is the hostile case and the only one that matters for sizing: the AC +# refuses the survivor until its own dead-peer timer frees the dead session. +# The poller caught it happening -- the destroyed router's session stayed in the +# table while the survivor's dial attempts appeared and were rejected, twice, +# before it finally got in at 148s. +# +# 148s also lands well past the theoretical lcp-echo-interval(30) x +# failure(3) = 90s budget that 180 was originally sized against, which left only +# 32s of margin. 300 gives roughly 2x the worst observed, on IDLE 2-vCPU sim +# VMs; the VP2440s under kea, BGP and conntrack will be slower, and Vodafone's +# actual policy and timers are unknown. +# +# The cost is real and worth stating: this is also how long a master that is +# alive but genuinely cannot route keeps holding every VIP before yielding -- +# the 2026-09-02 outage shape. That case is mostly covered by bond0.53, which +# satisfies the check within seconds of getting a DHCP lease; GRACE only +# dominates when PPPoE is the only path left. +# +# Do not lower this below the worst measured handover without re-running +# `labsim/labsim-pppoe-ha-test.sh --hard`. Before 2026-09-06 that matrix never +# actually set session-control and reported `deny` at 25s -- a number that did +# not exist. +GRACE=300 # may-dial is a LEASE, not a flag. vrrp-wan-reconcile renews its mtime every # tick; vrrp-wan-guard revokes it once it goes stale. A plain flag survives the