b659e0d47e
Some checks failed
ppp_dial() checked the flap holdoff and returned BEFORE renewing /run/vrrp-wan/may-dial. That lease is what vrrp-wan-guard expires after LEASE_TTL, so tripping the damper stopped the renew and the guard hung up pppoe0 on the MASTER ~80s later. A damper meant to suppress repeated DIALS was tearing down a working WAN instead. Observed in labsim, end to end: DIAL FLAP: >=6 attempts in 600s -- holding off 900s GUARD: lease stale (81s > 75s) -- hanging up pppoe0 An established session now outranks every check below it: ppp_active renews the lease and returns first. Everything after it only decides whether to start a NEW session. Two supporting fixes for how that storm started. The dial attempts were all no-ops because /etc/ppp/peers/pppoe0 was missing, and nothing said so -- systemd logs "skipped because of an unmet condition check" exactly once and the gate looks identical to a healthy backup. ppp_dial() now reports it, and distinguishes "configured but not rendered" (re-commit the subtree) from "no pppoe0 in config at all", which is what a reboot leaves behind when a commit was never saved. That is precisely how the sim secondary lost its WAN. Also `cat | wc -l` rather than `wc -l < file`: redirections are applied left to right, so the missing-file error escapes the 2>/dev/null on every first-ever dial. Harness: T11 copied-then-removed instead of mv, and verifies the restore -- losing that file strands a router permanently, which cost a debugging session. preflight now refuses to run if either router lacks the peers file or the pppoe0 config, since every failover result would otherwise be a false negative blamed on the ISP. New T12 forges a 900s holdoff against a live session and asserts it survives.
330 lines
16 KiB
Bash
Executable File
330 lines
16 KiB
Bash
Executable File
#!/bin/bash
|
|
# Does the WAN follow VRRP mastership, and does exactly ONE router ever hold the
|
|
# ISP session?
|
|
#
|
|
# The question is not "did a client get internet". A client can be answered by
|
|
# the wrong path entirely -- for months labsim-vyos's only default route was the
|
|
# libvirt-NAT scaffold on eth2, so every "the LAN still has internet" verdict was
|
|
# answered by eth2 rather than by the WAN under test. This script therefore
|
|
# refuses to run while that is true, and asks its questions of the ROUTERS and
|
|
# the ACCESS CONCENTRATOR, which cannot be answered by accident.
|
|
#
|
|
# The invariant, checked continuously and independently of any individual test:
|
|
#
|
|
# the AC never reports two `simdsl` sessions, and no two routers ever have a
|
|
# pppoe0 interface at the same time
|
|
#
|
|
# A run that violates it FAILS regardless of its own verdict, because a single
|
|
# consumer credential is the whole constraint the design exists to satisfy.
|
|
#
|
|
# ./labsim-pppoe-ha-test.sh --list
|
|
# ./labsim-pppoe-ha-test.sh T3
|
|
# ./labsim-pppoe-ha-test.sh --all
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
R1="${R1:-172.31.1.252}"; R2="${R2:-172.31.1.253}"
|
|
ISP="${ISP:-192.168.122.63}" # the fake access concentrator
|
|
LANVM="${LANVM:-172.31.10.10}"
|
|
VIP="${VIP:-172.31.1.1}"
|
|
PW="${VYOS_PW:-vyos}"; LANPW="${LANPW:-labsim}"
|
|
EVID="$SCRIPT_DIR/wan-failover-evidence"
|
|
|
|
SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
|
-o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password)
|
|
r() { timeout 45 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; }
|
|
isp() { timeout 30 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$ISP" "$@" 2>/dev/null; }
|
|
# The LAN VMs are Alpine and their sshd offers keyboard-interactive, not
|
|
# `password`. Reusing the routers' option set here made ssh exit 255 BEFORE
|
|
# running anything, and T5 read that as "the LAN lost the internet" while a
|
|
# tcpdump on the router showed the pings flowing out pppoe0 and the replies
|
|
# coming back. An exit code that can mean "the network is broken" or "I could
|
|
# not log in" is not a connectivity test.
|
|
LAN_SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
|
-o LogLevel=ERROR -o ConnectTimeout=6)
|
|
lan() { timeout 45 sshpass -p "$LANPW" ssh "${LAN_SSH[@]}" "root@$LANVM" "$@" 2>/dev/null; }
|
|
|
|
# Assert on what the guest actually reported, not on ssh's exit status.
|
|
lan_online() { [ "$(lan 'ping -c2 -W3 9.9.9.9 >/dev/null 2>&1 && echo ONLINE')" = ONLINE ]; }
|
|
|
|
log() { printf '\033[36m==>\033[0m %s\n' "$*"; }
|
|
pass() { printf ' \033[32mPASS\033[0m %s\n' "$*"; }
|
|
fail() { printf ' \033[31mFAIL\033[0m %s\n' "$*"; FAILED=$((FAILED+1)); }
|
|
FAILED=0
|
|
|
|
# --- observations ----------------------------------------------------------
|
|
ac_sessions() { isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show pppoe-server sessions' \
|
|
| grep -c ' simdsl ' || true; }
|
|
ac_detail() { isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show pppoe-server sessions'; }
|
|
# A destroyed or unreachable router is emphatically NOT holding pppoe0, but ssh
|
|
# returns an EMPTY string rather than 0 -- and `[ "" = 0 ]` is false, so a
|
|
# hard-failover test waited for the dead box to "report" zero and hung until its
|
|
# timeout, long after the survivor had taken over correctly. Default to 0.
|
|
ppp_on() { local v; v="$(r "$1" 'ip -4 addr show pppoe0 2>/dev/null | grep -c inet' | tr -d ' \n')"
|
|
echo "${v:-0}"; }
|
|
holder() { for h in "$R1" "$R2"; do
|
|
[ "$(r "$h" "ip -4 -o addr show | grep -c ' ${VIP}/'" | tr -d ' \n')" != 0 ] \
|
|
&& { echo "$h"; return; }; done; echo none; }
|
|
status() { r "$1" 'sudo /config/vrrp-wan-reconcile --status'; }
|
|
|
|
# How many routers currently hold a PPPoE interface. The invariant's other half.
|
|
ppp_holders() { n=0; for h in "$R1" "$R2"; do
|
|
[ "$(ppp_on "$h")" != 0 ] && n=$((n+1)); done; echo "$n"; }
|
|
|
|
check_invariant() {
|
|
local s p ok=0
|
|
s="$(ac_sessions)"; p="$(ppp_holders)"
|
|
[ "${s:-0}" -le 1 ] || { fail "INVARIANT: AC reports $s simdsl sessions"; ok=1; }
|
|
[ "${p:-0}" -le 1 ] || { fail "INVARIANT: $p routers hold pppoe0"; ok=1; }
|
|
return $ok
|
|
}
|
|
|
|
# --- preconditions ---------------------------------------------------------
|
|
# The scaffold check is a hard gate, not a warning. A default route via eth2
|
|
# means the box can reach the internet without the WAN working at all, and every
|
|
# connectivity verdict below would be a lie.
|
|
preflight() {
|
|
log "preflight"
|
|
local rc=0
|
|
for h in "$R1" "$R2"; do
|
|
if r "$h" 'ip route show default' | grep -q 'dev eth2'; then
|
|
fail "$h still routes via eth2 (libvirt-NAT scaffold) -- run sim-net-config.py --drop-scaffold"
|
|
rc=1
|
|
fi
|
|
if [ "$(r "$h" '[ -f /etc/systemd/system/ppp@pppoe0.service.d/10-vrrp-wan-gate.conf ] && echo y')" != y ]; then
|
|
fail "$h is missing the ppp gate drop-in -- run migration/vrrp-wan-install"
|
|
rc=1
|
|
fi
|
|
[ "$(r "$h" 'systemctl is-active vrrp-wan-guard.timer')" = active ] \
|
|
|| { fail "$h vrrp-wan-guard.timer not active"; rc=1; }
|
|
# A router with no peers file CANNOT dial, and says so only once in the
|
|
# journal. Every failover result in the run would then be a false
|
|
# negative blamed on the ISP. Check both, and check the config that
|
|
# renders it -- an unsaved commit reverts on reboot and takes pppoe0
|
|
# with it, which is how the sim secondary silently stopped dialling.
|
|
if [ "$(r "$h" '[ -f /etc/ppp/peers/pppoe0 ] && echo y')" != y ]; then
|
|
fail "$h has no /etc/ppp/peers/pppoe0 -- it cannot dial; re-commit the pppoe subtree"
|
|
rc=1
|
|
fi
|
|
if ! r "$h" 'show configuration commands' 2>/dev/null | grep -q 'interfaces pppoe pppoe0 source-interface'; then
|
|
fail "$h has no pppoe0 in config (unsaved commit lost on reboot?)"
|
|
rc=1
|
|
fi
|
|
done
|
|
[ "$rc" -eq 0 ] && pass "scaffold dropped, gate present, guard running, both can dial"
|
|
return $rc
|
|
}
|
|
|
|
settle() { # wait until exactly one router holds pppoe0, or give up
|
|
local i
|
|
for i in $(seq 1 "${1:-24}"); do
|
|
[ "$(ppp_holders)" = 1 ] && return 0
|
|
sleep 5
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Wait until a SPECIFIC router holds pppoe0 and the other does not.
|
|
#
|
|
# The obvious `settle` is wrong for a failover: "exactly one holder" is already
|
|
# true before the handover starts, so it returns instantly and the test reports
|
|
# that nothing moved while the handover is still in flight. Asking who holds it
|
|
# is the only useful form of the question.
|
|
settle_on() {
|
|
local want="$1" other i
|
|
other=$([ "$want" = "$R1" ] && echo "$R2" || echo "$R1")
|
|
for i in $(seq 1 "${2:-30}"); do
|
|
[ "$(ppp_on "$want")" != 0 ] && [ "$(ppp_on "$other")" = 0 ] && return 0
|
|
sleep 5
|
|
done
|
|
return 1
|
|
}
|
|
|
|
save_evidence() {
|
|
local name="$1"; local d="$EVID/$name"; mkdir -p "$d"
|
|
{ echo "=== $(date -Is) ==="; echo "--- AC sessions ---"; ac_detail
|
|
for h in "$R1" "$R2"; do echo "--- $h ---"; status "$h"
|
|
r "$h" 'ip -4 -br addr show pppoe0 2>/dev/null; ip route show default; sudo journalctl -t vrrp-wan -n 8 --no-pager'
|
|
done; } > "$d/state.txt" 2>&1
|
|
log "evidence -> wan-failover-evidence/$name/"
|
|
}
|
|
|
|
# --- tests -----------------------------------------------------------------
|
|
T0() { # baseline
|
|
log "T0 baseline: exactly one session, held by the VIP holder"
|
|
local h s; h="$(holder)"; s="$(ac_sessions)"
|
|
[ "$s" = 1 ] && pass "AC reports 1 session" || fail "AC reports $s sessions"
|
|
[ "$(ppp_on "$h")" != 0 ] && pass "the VIP holder ($h) is the one dialled" \
|
|
|| fail "VIP holder $h has no pppoe0"
|
|
local other; other=$([ "$h" = "$R1" ] && echo "$R2" || echo "$R1")
|
|
[ "$(ppp_on "$other")" = 0 ] && pass "the backup ($other) is not dialled" \
|
|
|| fail "backup $other also holds pppoe0"
|
|
save_evidence T0-baseline
|
|
}
|
|
|
|
T3() { # clean, deliberate failover
|
|
log "T3 clean failover via force-fault"
|
|
local from to t0 t1; from="$(holder)"
|
|
to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1")
|
|
log " master=$from -> expecting $to"
|
|
t0=$(date +%s)
|
|
r "$from" 'sudo mkdir -p /run/vrrp-wan && sudo touch /run/vrrp-wan/force-fault'
|
|
if settle_on "$to" 30; then
|
|
t1=$(date +%s)
|
|
[ "$(ppp_on "$to")" != 0 ] && pass "pppoe0 moved to $to in $((t1-t0))s" \
|
|
|| fail "pppoe0 did not move to $to"
|
|
[ "$(ppp_on "$from")" = 0 ] && pass "$from released pppoe0" \
|
|
|| fail "$from still holds pppoe0"
|
|
else
|
|
fail "never settled to exactly one pppoe0 holder"
|
|
fi
|
|
check_invariant
|
|
save_evidence T3-clean-failover
|
|
r "$from" 'sudo rm -f /run/vrrp-wan/force-fault'
|
|
settle 30 >/dev/null
|
|
}
|
|
|
|
T5() { # 10gig down -> PPPoE carries traffic
|
|
log "T5 10 gig down on the master: traffic must survive on pppoe0"
|
|
local h; h="$(holder)"
|
|
r "$h" 'sudo ip link set bond0.53 down'
|
|
sleep 20
|
|
local via; via="$(r "$h" 'ip route show default' | head -1)"
|
|
if echo "$via" | grep -q pppoe0; then
|
|
pass "default route moved to pppoe0: $via"
|
|
else
|
|
fail "default route did not move to pppoe0: ${via:-<none>}"
|
|
fi
|
|
# Poll, do not sample. Judging connectivity on one ping 20s after the link
|
|
# dropped failed while the path was still reconverging, and reported "the LAN
|
|
# lost the internet" for a path that came back moments later. A single
|
|
# negative sample is the least trustworthy verdict this harness can produce.
|
|
local ok=no i
|
|
for i in $(seq 1 12); do
|
|
lan_online && { ok=yes; break; }
|
|
sleep 5
|
|
done
|
|
[ "$ok" = yes ] && pass "LAN reaches the internet over pppoe0 (after $((i*5))s)" \
|
|
|| fail "LAN never regained the internet with only pppoe0 up (60s)"
|
|
save_evidence T5-tengig-down
|
|
r "$h" 'sudo ip link set bond0.53 up'
|
|
sleep 20
|
|
}
|
|
|
|
T11() { # a blessed box with no peers file must not restart-loop
|
|
log "T11 missing peers file must not restart-loop"
|
|
local h; h="$(holder)"
|
|
# COPY then remove, never move: only a commit touching the pppoe subtree
|
|
# re-renders this file, so losing it strands the box permanently -- the gate
|
|
# blocks every dial, systemd says "skipped because of an unmet condition
|
|
# check" exactly once, and nothing else complains. An earlier `mv` pair did
|
|
# exactly that to the sim secondary and cost a debugging session.
|
|
r "$h" 'sudo cp -a /etc/ppp/peers/pppoe0 /run/peers.bak && sudo rm -f /etc/ppp/peers/pppoe0; sudo systemctl restart ppp@pppoe0'
|
|
sleep 12
|
|
local n; n="$(r "$h" 'systemctl show ppp@pppoe0 -p NRestarts --value')"
|
|
[ "${n:-99}" -le 1 ] && pass "NRestarts=$n (gate refused the start)" \
|
|
|| fail "NRestarts=$n -- restart loop is back"
|
|
# Restore on the SAME host we broke, and prove it landed. Do not trust the
|
|
# copy back: if it silently failed, every later test in the run would be
|
|
# measuring a router that physically cannot dial.
|
|
r "$h" 'sudo cp -a /run/peers.bak /etc/ppp/peers/pppoe0'
|
|
if r "$h" 'test -f /etc/ppp/peers/pppoe0'; then
|
|
pass "peers file restored on $h"
|
|
else
|
|
fail "peers file NOT restored on $h -- that router can no longer dial"
|
|
fi
|
|
save_evidence T11-no-peers-file
|
|
settle 24 >/dev/null
|
|
}
|
|
|
|
T8() { # lease expiry: the guard must hang up a demoted-but-unreconciled box
|
|
log "T8 lease expiry revokes the session"
|
|
local h; h="$(holder)"
|
|
r "$h" 'sudo systemctl stop vrrp-wan-reconcile.timer'
|
|
r "$h" 'sudo touch -d "-200 seconds" /run/vrrp-wan/may-dial'
|
|
sleep 12
|
|
[ "$(ppp_on "$h")" = 0 ] && pass "guard hung up on a stale lease" \
|
|
|| fail "stale lease did not revoke the session"
|
|
r "$h" 'sudo systemctl start vrrp-wan-reconcile.timer'
|
|
save_evidence T8-lease-expiry
|
|
settle 24 >/dev/null
|
|
}
|
|
|
|
|
|
T4() { # hard failover across all three AC session-control policies
|
|
log "T4 hard failover (destroy the master) x session-control"
|
|
# Vodafone's policy is unknowable from here, so prove the design survives
|
|
# every one VyOS can express. `replace` is the accel-ppp default and the
|
|
# friendly case; `deny` is the hostile one, where the AC refuses the second
|
|
# session until its own dead-peer timer (lcp-echo-interval 30 x failure 3 =
|
|
# 90s) frees the first -- which is exactly why GRACE is no longer 90.
|
|
local mode from to vm t0 t1
|
|
for mode in replace deny disable; do
|
|
log " --- session-control=$mode ---"
|
|
isp "vbash -c 'source /opt/vyatta/etc/functions/script-template; configure; set service pppoe-server session-control $mode; commit; save; exit'" >/dev/null 2>&1
|
|
sleep 5
|
|
from="$(holder)"; to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1")
|
|
vm=$([ "$from" = "$R1" ] && echo labsim-vyos || echo labsim-vyos2)
|
|
[ "$from" = none ] && { fail "no master before $mode run"; continue; }
|
|
log " destroying $vm (master=$from), expecting $to"
|
|
t0=$(date +%s)
|
|
sudo virsh destroy "$vm" >/dev/null 2>&1
|
|
if settle_on "$to" 48; then
|
|
t1=$(date +%s)
|
|
pass "$mode: pppoe0 reached $to in $((t1-t0))s"
|
|
else
|
|
fail "$mode: $to never dialled within 240s"
|
|
fi
|
|
check_invariant
|
|
save_evidence "T4-hard-failover-$mode"
|
|
sudo virsh start "$vm" >/dev/null 2>&1
|
|
# Re-bond. A VM restart recreates its taps under NEW names, and the OVS
|
|
# bond keeps the old ones -- lacp dies, VLAN 1 goes with it, and the box
|
|
# comes back reachable on some VLANs but not others. ovs_bond_router
|
|
# detects the stale membership and rebuilds, but nothing runs it
|
|
# automatically, so a destroy/start test must do it or the survivor
|
|
# looks like a failover failure.
|
|
( source "$SCRIPT_DIR/lib.sh"; source "$SCRIPT_DIR/ovs.sh"; selected_vlans
|
|
LAG_NAME=$([ "$vm" = labsim-vyos ] && echo lag-vyos || echo lag-vyos2)
|
|
ovs_bond_router "$vm" ) >/dev/null 2>&1
|
|
# Give the returning box time to boot and settle as BACKUP before the
|
|
# next iteration; it must NOT dial on the way up.
|
|
sleep 90
|
|
[ "$(ppp_on "$from")" = 0 ] && pass "$mode: $from did not dial on reboot" \
|
|
|| fail "$mode: $from dialled on reboot (gate failed)"
|
|
done
|
|
isp "vbash -c 'source /opt/vyatta/etc/functions/script-template; configure; set service pppoe-server session-control replace; commit; save; exit'" >/dev/null 2>&1
|
|
log " AC restored to session-control=replace"
|
|
}
|
|
|
|
T12() { # the flap damper must never tear down an ESTABLISHED session
|
|
log "T12 flap holdoff must not kill a live session"
|
|
local h; h="$(holder)"
|
|
[ "$h" = none ] && { fail "no master to test"; return; }
|
|
# Forge a holdoff far in the future, as a dial storm would. Before the fix
|
|
# ppp_dial() returned here BEFORE renewing may-dial, the lease went stale,
|
|
# and vrrp-wan-guard hung up the master's working WAN ~80s later.
|
|
r "$h" 'sudo sh -c "echo $(( $(date +%s) + 900 )) > /run/vrrp-wan/holdoff"'
|
|
# Sleep past LEASE_TTL (75s) so a non-renewed lease would definitely expire.
|
|
sleep 100
|
|
if [ "$(ppp_on "$h")" = 1 ]; then
|
|
pass "session survived a 900s holdoff (lease still renewed)"
|
|
else
|
|
fail "holdoff killed the live session -- damper is tearing down the WAN"
|
|
fi
|
|
r "$h" 'sudo rm -f /run/vrrp-wan/holdoff /run/vrrp-wan/dials'
|
|
save_evidence T12-holdoff-keeps-session
|
|
settle 24 >/dev/null
|
|
}
|
|
|
|
case "${1:---all}" in
|
|
--list) echo "T0 baseline | T3 clean failover | T4 hard failover x policy | T5 10gig-down | T8 lease expiry | T11 no-peers-file | T12 holdoff-keeps-session"; exit 0 ;;
|
|
--all) preflight || exit 1; T0; T3; T5; T8; T11; T12 ;;
|
|
--hard) preflight || exit 1; T4 ;;
|
|
*) preflight || exit 1; "$1" ;;
|
|
esac
|
|
|
|
echo
|
|
[ "$FAILED" -eq 0 ] && { echo "ALL PASS"; exit 0; }
|
|
echo "$FAILED check(s) FAILED"; exit 1
|