#!/bin/bash # Does the WAN follow VRRP mastership, and does exactly ONE router ever hold the # ISP session? # # The question is not "did a client get internet". A client can be answered by # the wrong path entirely -- for months labsim-vyos's only default route was the # libvirt-NAT scaffold on eth2, so every "the LAN still has internet" verdict was # answered by eth2 rather than by the WAN under test. This script therefore # refuses to run while that is true, and asks its questions of the ROUTERS and # the ACCESS CONCENTRATOR, which cannot be answered by accident. # # The invariant, checked continuously and independently of any individual test: # # the AC never reports two `simdsl` sessions, and no two routers ever have a # pppoe0 interface at the same time # # A run that violates it FAILS regardless of its own verdict, because a single # consumer credential is the whole constraint the design exists to satisfy. # # ./labsim-pppoe-ha-test.sh --list # ./labsim-pppoe-ha-test.sh T3 # ./labsim-pppoe-ha-test.sh --all set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" R1="${R1:-172.31.1.252}"; R2="${R2:-172.31.1.253}" ISP="${ISP:-192.168.122.63}" # the fake access concentrator LANVM="${LANVM:-172.31.10.10}" VIP="${VIP:-172.31.1.1}" PW="${VYOS_PW:-vyos}"; LANPW="${LANPW:-labsim}" EVID="$SCRIPT_DIR/wan-failover-evidence" SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password) r() { timeout 45 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; } isp() { timeout 30 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$ISP" "$@" 2>/dev/null; } # Set the AC's session policy, and PROVE it landed. # # `vbash -c 'source script-template; configure; ...; commit'` does NOT work: the # config session never starts and commit dies with "Invalid command: [commit]", # on stderr, which the isp() helper discards. The whole T4 matrix therefore ran # all three iterations against the accel-ppp DEFAULT while printing # "--- session-control=deny ---" -- it reported coverage it did not have, which # is worse than reporting a failure. Drive it from a real script FILE, then read # the value back and abort the run if it disagrees. isp_session_control() { local mode="$1" printf '#!/bin/vbash\nsource /opt/vyatta/etc/functions/script-template\nconfigure\nset service pppoe-server session-control %s\ncommit\nsave\nexit\n' "$mode" \ | timeout 30 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$ISP" 'cat > /tmp/set-sc.sh && chmod +x /tmp/set-sc.sh && sudo /tmp/set-sc.sh' >/dev/null 2>&1 local got got="$(isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands' \ | sed -n "s/.*session-control '\\(.*\\)'/\\1/p")" if [ "$got" = "$mode" ]; then log " AC session-control=$mode (verified)" return 0 fi fail "could not set AC session-control=$mode (reads '${got:-unset}') -- results would be fiction" return 1 } # The LAN VMs are Alpine and their sshd offers keyboard-interactive, not # `password`. Reusing the routers' option set here made ssh exit 255 BEFORE # running anything, and T5 read that as "the LAN lost the internet" while a # tcpdump on the router showed the pings flowing out pppoe0 and the replies # coming back. An exit code that can mean "the network is broken" or "I could # not log in" is not a connectivity test. LAN_SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=6) lan() { timeout 45 sshpass -p "$LANPW" ssh "${LAN_SSH[@]}" "root@$LANVM" "$@" 2>/dev/null; } # Assert on what the guest actually reported, not on ssh's exit status. lan_online() { [ "$(lan 'ping -c2 -W3 9.9.9.9 >/dev/null 2>&1 && echo ONLINE')" = ONLINE ]; } log() { printf '\033[36m==>\033[0m %s\n' "$*"; } pass() { printf ' \033[32mPASS\033[0m %s\n' "$*"; } fail() { printf ' \033[31mFAIL\033[0m %s\n' "$*"; FAILED=$((FAILED+1)); } FAILED=0 # --- observations ---------------------------------------------------------- ac_sessions() { isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show pppoe-server sessions' \ | grep -c ' simdsl ' || true; } ac_detail() { isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show pppoe-server sessions'; } # A destroyed or unreachable router is emphatically NOT holding pppoe0, but ssh # returns an EMPTY string rather than 0 -- and `[ "" = 0 ]` is false, so a # hard-failover test waited for the dead box to "report" zero and hung until its # timeout, long after the survivor had taken over correctly. Default to 0. ppp_on() { local v; v="$(r "$1" 'ip -4 addr show pppoe0 2>/dev/null | grep -c inet' | tr -d ' \n')" echo "${v:-0}"; } holder() { for h in "$R1" "$R2"; do [ "$(r "$h" "ip -4 -o addr show | grep -c ' ${VIP}/'" | tr -d ' \n')" != 0 ] \ && { echo "$h"; return; }; done; echo none; } status() { r "$1" 'sudo /config/vrrp-wan-reconcile --status'; } # How many routers currently hold a PPPoE interface. The invariant's other half. ppp_holders() { n=0; for h in "$R1" "$R2"; do [ "$(ppp_on "$h")" != 0 ] && n=$((n+1)); done; echo "$n"; } warn() { printf ' \033[33mWARN\033[0m %s\n' "$*"; } check_invariant() { local s p ok=0 s="$(ac_sessions)"; p="$(ppp_holders)" # THE invariant. Two of OUR routers dialled at once is the failure that # matters: one ISP account, and against a real ISP that is how you get # rate-limited or locked out. [ "${p:-0}" -le 1 ] || { fail "INVARIANT: $p routers hold pppoe0"; ok=1; } # The AC's session count is a PROXY for the above, and only a valid one # while the AC enforces single-session. Under session-control=disable it # does not, so a destroyed router's session simply stays in the table and # the count reads 2 while exactly one live router is dialled -- which is AC # bookkeeping, not a double dial. Attribute it rather than failing blind: # only call it a violation when more than one router is ACTUALLY dialled. # # Do not silence it either. An orphaned session still occupies the single # slot at a real ISP, and that is exactly what made session-control=deny # take 148s while the survivor's dial attempts were refused. if [ "${s:-0}" -gt 1 ]; then if [ "${p:-0}" -gt 1 ]; then fail "INVARIANT: AC reports $s simdsl sessions AND $p routers are dialled" ok=1 else warn "AC reports $s simdsl sessions but only ${p:-0} router is dialled -- stale session from the destroyed peer (expected where the AC does not enforce single-session; it is what a hostile AC holds against the survivor)" fi fi return $ok } # --- preconditions --------------------------------------------------------- # The scaffold check is a hard gate, not a warning. A default route via eth2 # means the box can reach the internet without the WAN working at all, and every # connectivity verdict below would be a lie. preflight() { log "preflight" local rc=0 for h in "$R1" "$R2"; do if r "$h" 'ip route show default' | grep -q 'dev eth2'; then fail "$h still routes via eth2 (libvirt-NAT scaffold) -- run sim-net-config.py --drop-scaffold" rc=1 fi if [ "$(r "$h" '[ -f /etc/systemd/system/ppp@pppoe0.service.d/10-vrrp-wan-gate.conf ] && echo y')" != y ]; then fail "$h is missing the ppp gate drop-in -- run migration/vrrp-wan-install" rc=1 fi [ "$(r "$h" 'systemctl is-active vrrp-wan-guard.timer')" = active ] \ || { fail "$h vrrp-wan-guard.timer not active"; rc=1; } # A router with no peers file CANNOT dial, and says so only once in the # journal. Every failover result in the run would then be a false # negative blamed on the ISP. Check both, and check the config that # renders it -- an unsaved commit reverts on reboot and takes pppoe0 # with it, which is how the sim secondary silently stopped dialling. if [ "$(r "$h" '[ -f /etc/ppp/peers/pppoe0 ] && echo y')" != y ]; then fail "$h has no /etc/ppp/peers/pppoe0 -- it cannot dial; re-commit the pppoe subtree" rc=1 fi # The op-mode WRAPPER, not a bare `show`: over non-interactive ssh the # bare form is not on PATH, so this silently matched nothing and failed # both routers that were in fact configured correctly. if ! r "$h" '/opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands' 2>/dev/null | grep -q 'interfaces pppoe pppoe0 source-interface'; then fail "$h has no pppoe0 in config (unsaved commit lost on reboot?)" rc=1 fi done [ "$rc" -eq 0 ] && pass "scaffold dropped, gate present, guard running, both can dial" return $rc } settle() { # wait until exactly one router holds pppoe0, or give up local i for i in $(seq 1 "${1:-24}"); do [ "$(ppp_holders)" = 1 ] && return 0 sleep 5 done return 1 } # Wait until a SPECIFIC router holds pppoe0 and the other does not. # # The obvious `settle` is wrong for a failover: "exactly one holder" is already # true before the handover starts, so it returns instantly and the test reports # that nothing moved while the handover is still in flight. Asking who holds it # is the only useful form of the question. settle_on() { local want="$1" other i other=$([ "$want" = "$R1" ] && echo "$R2" || echo "$R1") for i in $(seq 1 "${2:-30}"); do [ "$(ppp_on "$want")" != 0 ] && [ "$(ppp_on "$other")" = 0 ] && return 0 sleep 5 done return 1 } save_evidence() { local name="$1"; local d="$EVID/$name"; mkdir -p "$d" { echo "=== $(date -Is) ==="; echo "--- AC sessions ---"; ac_detail for h in "$R1" "$R2"; do echo "--- $h ---"; status "$h" r "$h" 'ip -4 -br addr show pppoe0 2>/dev/null; ip route show default; sudo journalctl -t vrrp-wan -n 8 --no-pager' done; } > "$d/state.txt" 2>&1 log "evidence -> wan-failover-evidence/$name/" } # --- tests ----------------------------------------------------------------- T0() { # baseline log "T0 baseline: exactly one session, held by the VIP holder" local h s; h="$(holder)"; s="$(ac_sessions)" [ "$s" = 1 ] && pass "AC reports 1 session" || fail "AC reports $s sessions" [ "$(ppp_on "$h")" != 0 ] && pass "the VIP holder ($h) is the one dialled" \ || fail "VIP holder $h has no pppoe0" local other; other=$([ "$h" = "$R1" ] && echo "$R2" || echo "$R1") [ "$(ppp_on "$other")" = 0 ] && pass "the backup ($other) is not dialled" \ || fail "backup $other also holds pppoe0" save_evidence T0-baseline } T3() { # clean, deliberate failover log "T3 clean failover via force-fault" local from to t0 t1; from="$(holder)" to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1") log " master=$from -> expecting $to" t0=$(date +%s) r "$from" 'sudo mkdir -p /run/vrrp-wan && sudo touch /run/vrrp-wan/force-fault' if settle_on "$to" 30; then t1=$(date +%s) [ "$(ppp_on "$to")" != 0 ] && pass "pppoe0 moved to $to in $((t1-t0))s" \ || fail "pppoe0 did not move to $to" [ "$(ppp_on "$from")" = 0 ] && pass "$from released pppoe0" \ || fail "$from still holds pppoe0" else fail "never settled to exactly one pppoe0 holder" fi check_invariant save_evidence T3-clean-failover r "$from" 'sudo rm -f /run/vrrp-wan/force-fault' settle 30 >/dev/null } T5() { # 10gig down -> PPPoE carries traffic log "T5 10 gig down on the master: traffic must survive on pppoe0" local h; h="$(holder)" r "$h" 'sudo ip link set bond0.53 down' sleep 20 local via; via="$(r "$h" 'ip route show default' | head -1)" if echo "$via" | grep -q pppoe0; then pass "default route moved to pppoe0: $via" else fail "default route did not move to pppoe0: ${via:-}" fi # Poll, do not sample. Judging connectivity on one ping 20s after the link # dropped failed while the path was still reconverging, and reported "the LAN # lost the internet" for a path that came back moments later. A single # negative sample is the least trustworthy verdict this harness can produce. local ok=no i for i in $(seq 1 12); do lan_online && { ok=yes; break; } sleep 5 done [ "$ok" = yes ] && pass "LAN reaches the internet over pppoe0 (after $((i*5))s)" \ || fail "LAN never regained the internet with only pppoe0 up (60s)" save_evidence T5-tengig-down r "$h" 'sudo ip link set bond0.53 up' sleep 20 } T11() { # a blessed box with no peers file must not restart-loop log "T11 missing peers file must not restart-loop" local h; h="$(holder)" # COPY then remove, never move: only a commit touching the pppoe subtree # re-renders this file, so losing it strands the box permanently -- the gate # blocks every dial, systemd says "skipped because of an unmet condition # check" exactly once, and nothing else complains. An earlier `mv` pair did # exactly that to the sim secondary and cost a debugging session. r "$h" 'sudo cp -a /etc/ppp/peers/pppoe0 /run/peers.bak && sudo rm -f /etc/ppp/peers/pppoe0; sudo systemctl restart ppp@pppoe0' sleep 12 local n; n="$(r "$h" 'systemctl show ppp@pppoe0 -p NRestarts --value')" [ "${n:-99}" -le 1 ] && pass "NRestarts=$n (gate refused the start)" \ || fail "NRestarts=$n -- restart loop is back" # Restore on the SAME host we broke, and prove it landed. Do not trust the # copy back: if it silently failed, every later test in the run would be # measuring a router that physically cannot dial. r "$h" 'sudo cp -a /run/peers.bak /etc/ppp/peers/pppoe0' if r "$h" 'test -f /etc/ppp/peers/pppoe0'; then pass "peers file restored on $h" else fail "peers file NOT restored on $h -- that router can no longer dial" fi save_evidence T11-no-peers-file settle 24 >/dev/null } T8() { # lease expiry: the guard must hang up a demoted-but-unreconciled box log "T8 lease expiry revokes the session" local h; h="$(holder)" r "$h" 'sudo systemctl stop vrrp-wan-reconcile.timer' r "$h" 'sudo touch -d "-200 seconds" /run/vrrp-wan/may-dial' sleep 12 [ "$(ppp_on "$h")" = 0 ] && pass "guard hung up on a stale lease" \ || fail "stale lease did not revoke the session" r "$h" 'sudo systemctl start vrrp-wan-reconcile.timer' save_evidence T8-lease-expiry settle 24 >/dev/null } T4() { # hard failover across all three AC session-control policies log "T4 hard failover (destroy the master) x session-control" # Vodafone's policy is unknowable from here, so prove the design survives # every one VyOS can express. `replace` is the accel-ppp default and the # friendly case; `deny` is the hostile one, where the AC refuses the second # session until its own dead-peer timer (lcp-echo-interval 30 x failure 3 = # 90s) frees the first -- which is exactly why GRACE is no longer 90. local mode from to vm t0 t1 for mode in replace deny disable; do log " --- session-control=$mode ---" # Skip the iteration rather than measure the wrong policy. isp_session_control "$mode" || continue sleep 5 from="$(holder)"; to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1") vm=$([ "$from" = "$R1" ] && echo labsim-vyos || echo labsim-vyos2) [ "$from" = none ] && { fail "no master before $mode run"; continue; } log " destroying $vm (master=$from), expecting $to" t0=$(date +%s) sudo virsh destroy "$vm" >/dev/null 2>&1 if settle_on "$to" 48; then t1=$(date +%s) pass "$mode: pppoe0 reached $to in $((t1-t0))s" else fail "$mode: $to never dialled within 240s" fi check_invariant save_evidence "T4-hard-failover-$mode" sudo virsh start "$vm" >/dev/null 2>&1 # Re-bond. A VM restart recreates its taps under NEW names, and the OVS # bond keeps the old ones -- lacp dies, VLAN 1 goes with it, and the box # comes back reachable on some VLANs but not others. ovs_bond_router # detects the stale membership and rebuilds, but nothing runs it # automatically, so a destroy/start test must do it or the survivor # looks like a failover failure. ( source "$SCRIPT_DIR/lib.sh"; source "$SCRIPT_DIR/ovs.sh"; selected_vlans LAG_NAME=$([ "$vm" = labsim-vyos ] && echo lag-vyos || echo lag-vyos2) ovs_bond_router "$vm" ) >/dev/null 2>&1 # Give the returning box time to boot and settle as BACKUP before the # next iteration; it must NOT dial on the way up. sleep 90 [ "$(ppp_on "$from")" = 0 ] && pass "$mode: $from did not dial on reboot" \ || fail "$mode: $from dialled on reboot (gate failed)" done isp_session_control replace >/dev/null log " AC restored to session-control=replace" } T12() { # the flap damper must never tear down an ESTABLISHED session log "T12 flap holdoff must not kill a live session" local h; h="$(holder)" [ "$h" = none ] && { fail "no master to test"; return; } # Forge a holdoff far in the future, as a dial storm would. Before the fix # ppp_dial() returned here BEFORE renewing may-dial, the lease went stale, # and vrrp-wan-guard hung up the master's working WAN ~80s later. r "$h" 'sudo sh -c "echo $(( $(date +%s) + 900 )) > /run/vrrp-wan/holdoff"' # Sleep past LEASE_TTL (75s) so a non-renewed lease would definitely expire. sleep 100 if [ "$(ppp_on "$h")" = 1 ]; then pass "session survived a 900s holdoff (lease still renewed)" else fail "holdoff killed the live session -- damper is tearing down the WAN" fi r "$h" 'sudo rm -f /run/vrrp-wan/holdoff /run/vrrp-wan/dials' save_evidence T12-holdoff-keeps-session settle 24 >/dev/null } case "${1:---all}" in --list) echo "T0 baseline | T3 clean failover | T4 hard failover x policy | T5 10gig-down | T8 lease expiry | T11 no-peers-file | T12 holdoff-keeps-session"; exit 0 ;; --all) preflight || exit 1; T0; T3; T5; T8; T11; T12 ;; --hard) preflight || exit 1; T4 ;; *) preflight || exit 1; "$1" ;; esac echo [ "$FAILED" -eq 0 ] && { echo "ALL PASS"; exit 0; } echo "$FAILED check(s) FAILED"; exit 1