9221c71ff0
Some checks failed
T4 prints "--- session-control=deny ---" and then measures whatever policy the AC already had. The setter was isp "vbash -c 'source script-template; configure; set ...; commit; save'" and that form does not start a config session at all -- commit dies with "Invalid command: [commit]" on stderr, which isp() discards. `show configuration commands | grep session-control` on the ISP VM returned nothing after a full matrix run: all three iterations had run against the accel-ppp default. The labels were fiction, and a harness that reports coverage it does not have is worse than one that reports a failure. Driving it from a real script FILE works. isp_session_control() does that, reads the value back, and fails the iteration if it disagrees rather than measuring the wrong policy. `session-control` is a valid node here (checked the template dir on VyOS 2026.08.12-0831-rolling), so this was purely the invocation. Staged the two Pulumi overrides in migration/ rather than adding them to kubernetes-deployment: another agent runs `pulumi up` on that repo, so merging `remove: pppoe0 disable` before the gate exists on vyos002 would let it dial and take the single Vodafone session off the live master. Ordering is written at the top of the file.
358 lines
17 KiB
Bash
Executable File
358 lines
17 KiB
Bash
Executable File
#!/bin/bash
|
|
# Does the WAN follow VRRP mastership, and does exactly ONE router ever hold the
|
|
# ISP session?
|
|
#
|
|
# The question is not "did a client get internet". A client can be answered by
|
|
# the wrong path entirely -- for months labsim-vyos's only default route was the
|
|
# libvirt-NAT scaffold on eth2, so every "the LAN still has internet" verdict was
|
|
# answered by eth2 rather than by the WAN under test. This script therefore
|
|
# refuses to run while that is true, and asks its questions of the ROUTERS and
|
|
# the ACCESS CONCENTRATOR, which cannot be answered by accident.
|
|
#
|
|
# The invariant, checked continuously and independently of any individual test:
|
|
#
|
|
# the AC never reports two `simdsl` sessions, and no two routers ever have a
|
|
# pppoe0 interface at the same time
|
|
#
|
|
# A run that violates it FAILS regardless of its own verdict, because a single
|
|
# consumer credential is the whole constraint the design exists to satisfy.
|
|
#
|
|
# ./labsim-pppoe-ha-test.sh --list
|
|
# ./labsim-pppoe-ha-test.sh T3
|
|
# ./labsim-pppoe-ha-test.sh --all
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
R1="${R1:-172.31.1.252}"; R2="${R2:-172.31.1.253}"
|
|
ISP="${ISP:-192.168.122.63}" # the fake access concentrator
|
|
LANVM="${LANVM:-172.31.10.10}"
|
|
VIP="${VIP:-172.31.1.1}"
|
|
PW="${VYOS_PW:-vyos}"; LANPW="${LANPW:-labsim}"
|
|
EVID="$SCRIPT_DIR/wan-failover-evidence"
|
|
|
|
SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
|
-o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password)
|
|
r() { timeout 45 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; }
|
|
isp() { timeout 30 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$ISP" "$@" 2>/dev/null; }
|
|
|
|
# Set the AC's session policy, and PROVE it landed.
|
|
#
|
|
# `vbash -c 'source script-template; configure; ...; commit'` does NOT work: the
|
|
# config session never starts and commit dies with "Invalid command: [commit]",
|
|
# on stderr, which the isp() helper discards. The whole T4 matrix therefore ran
|
|
# all three iterations against the accel-ppp DEFAULT while printing
|
|
# "--- session-control=deny ---" -- it reported coverage it did not have, which
|
|
# is worse than reporting a failure. Drive it from a real script FILE, then read
|
|
# the value back and abort the run if it disagrees.
|
|
isp_session_control() {
|
|
local mode="$1"
|
|
printf '#!/bin/vbash\nsource /opt/vyatta/etc/functions/script-template\nconfigure\nset service pppoe-server session-control %s\ncommit\nsave\nexit\n' "$mode" \
|
|
| timeout 30 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$ISP" 'cat > /tmp/set-sc.sh && chmod +x /tmp/set-sc.sh && sudo /tmp/set-sc.sh' >/dev/null 2>&1
|
|
local got
|
|
got="$(isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands' \
|
|
| sed -n "s/.*session-control '\\(.*\\)'/\\1/p")"
|
|
if [ "$got" = "$mode" ]; then
|
|
log " AC session-control=$mode (verified)"
|
|
return 0
|
|
fi
|
|
fail "could not set AC session-control=$mode (reads '${got:-unset}') -- results would be fiction"
|
|
return 1
|
|
}
|
|
# The LAN VMs are Alpine and their sshd offers keyboard-interactive, not
|
|
# `password`. Reusing the routers' option set here made ssh exit 255 BEFORE
|
|
# running anything, and T5 read that as "the LAN lost the internet" while a
|
|
# tcpdump on the router showed the pings flowing out pppoe0 and the replies
|
|
# coming back. An exit code that can mean "the network is broken" or "I could
|
|
# not log in" is not a connectivity test.
|
|
LAN_SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
|
-o LogLevel=ERROR -o ConnectTimeout=6)
|
|
lan() { timeout 45 sshpass -p "$LANPW" ssh "${LAN_SSH[@]}" "root@$LANVM" "$@" 2>/dev/null; }
|
|
|
|
# Assert on what the guest actually reported, not on ssh's exit status.
|
|
lan_online() { [ "$(lan 'ping -c2 -W3 9.9.9.9 >/dev/null 2>&1 && echo ONLINE')" = ONLINE ]; }
|
|
|
|
log() { printf '\033[36m==>\033[0m %s\n' "$*"; }
|
|
pass() { printf ' \033[32mPASS\033[0m %s\n' "$*"; }
|
|
fail() { printf ' \033[31mFAIL\033[0m %s\n' "$*"; FAILED=$((FAILED+1)); }
|
|
FAILED=0
|
|
|
|
# --- observations ----------------------------------------------------------
|
|
ac_sessions() { isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show pppoe-server sessions' \
|
|
| grep -c ' simdsl ' || true; }
|
|
ac_detail() { isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show pppoe-server sessions'; }
|
|
# A destroyed or unreachable router is emphatically NOT holding pppoe0, but ssh
|
|
# returns an EMPTY string rather than 0 -- and `[ "" = 0 ]` is false, so a
|
|
# hard-failover test waited for the dead box to "report" zero and hung until its
|
|
# timeout, long after the survivor had taken over correctly. Default to 0.
|
|
ppp_on() { local v; v="$(r "$1" 'ip -4 addr show pppoe0 2>/dev/null | grep -c inet' | tr -d ' \n')"
|
|
echo "${v:-0}"; }
|
|
holder() { for h in "$R1" "$R2"; do
|
|
[ "$(r "$h" "ip -4 -o addr show | grep -c ' ${VIP}/'" | tr -d ' \n')" != 0 ] \
|
|
&& { echo "$h"; return; }; done; echo none; }
|
|
status() { r "$1" 'sudo /config/vrrp-wan-reconcile --status'; }
|
|
|
|
# How many routers currently hold a PPPoE interface. The invariant's other half.
|
|
ppp_holders() { n=0; for h in "$R1" "$R2"; do
|
|
[ "$(ppp_on "$h")" != 0 ] && n=$((n+1)); done; echo "$n"; }
|
|
|
|
check_invariant() {
|
|
local s p ok=0
|
|
s="$(ac_sessions)"; p="$(ppp_holders)"
|
|
[ "${s:-0}" -le 1 ] || { fail "INVARIANT: AC reports $s simdsl sessions"; ok=1; }
|
|
[ "${p:-0}" -le 1 ] || { fail "INVARIANT: $p routers hold pppoe0"; ok=1; }
|
|
return $ok
|
|
}
|
|
|
|
# --- preconditions ---------------------------------------------------------
|
|
# The scaffold check is a hard gate, not a warning. A default route via eth2
|
|
# means the box can reach the internet without the WAN working at all, and every
|
|
# connectivity verdict below would be a lie.
|
|
preflight() {
|
|
log "preflight"
|
|
local rc=0
|
|
for h in "$R1" "$R2"; do
|
|
if r "$h" 'ip route show default' | grep -q 'dev eth2'; then
|
|
fail "$h still routes via eth2 (libvirt-NAT scaffold) -- run sim-net-config.py --drop-scaffold"
|
|
rc=1
|
|
fi
|
|
if [ "$(r "$h" '[ -f /etc/systemd/system/ppp@pppoe0.service.d/10-vrrp-wan-gate.conf ] && echo y')" != y ]; then
|
|
fail "$h is missing the ppp gate drop-in -- run migration/vrrp-wan-install"
|
|
rc=1
|
|
fi
|
|
[ "$(r "$h" 'systemctl is-active vrrp-wan-guard.timer')" = active ] \
|
|
|| { fail "$h vrrp-wan-guard.timer not active"; rc=1; }
|
|
# A router with no peers file CANNOT dial, and says so only once in the
|
|
# journal. Every failover result in the run would then be a false
|
|
# negative blamed on the ISP. Check both, and check the config that
|
|
# renders it -- an unsaved commit reverts on reboot and takes pppoe0
|
|
# with it, which is how the sim secondary silently stopped dialling.
|
|
if [ "$(r "$h" '[ -f /etc/ppp/peers/pppoe0 ] && echo y')" != y ]; then
|
|
fail "$h has no /etc/ppp/peers/pppoe0 -- it cannot dial; re-commit the pppoe subtree"
|
|
rc=1
|
|
fi
|
|
# The op-mode WRAPPER, not a bare `show`: over non-interactive ssh the
|
|
# bare form is not on PATH, so this silently matched nothing and failed
|
|
# both routers that were in fact configured correctly.
|
|
if ! r "$h" '/opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands' 2>/dev/null | grep -q 'interfaces pppoe pppoe0 source-interface'; then
|
|
fail "$h has no pppoe0 in config (unsaved commit lost on reboot?)"
|
|
rc=1
|
|
fi
|
|
done
|
|
[ "$rc" -eq 0 ] && pass "scaffold dropped, gate present, guard running, both can dial"
|
|
return $rc
|
|
}
|
|
|
|
settle() { # wait until exactly one router holds pppoe0, or give up
|
|
local i
|
|
for i in $(seq 1 "${1:-24}"); do
|
|
[ "$(ppp_holders)" = 1 ] && return 0
|
|
sleep 5
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Wait until a SPECIFIC router holds pppoe0 and the other does not.
|
|
#
|
|
# The obvious `settle` is wrong for a failover: "exactly one holder" is already
|
|
# true before the handover starts, so it returns instantly and the test reports
|
|
# that nothing moved while the handover is still in flight. Asking who holds it
|
|
# is the only useful form of the question.
|
|
settle_on() {
|
|
local want="$1" other i
|
|
other=$([ "$want" = "$R1" ] && echo "$R2" || echo "$R1")
|
|
for i in $(seq 1 "${2:-30}"); do
|
|
[ "$(ppp_on "$want")" != 0 ] && [ "$(ppp_on "$other")" = 0 ] && return 0
|
|
sleep 5
|
|
done
|
|
return 1
|
|
}
|
|
|
|
save_evidence() {
|
|
local name="$1"; local d="$EVID/$name"; mkdir -p "$d"
|
|
{ echo "=== $(date -Is) ==="; echo "--- AC sessions ---"; ac_detail
|
|
for h in "$R1" "$R2"; do echo "--- $h ---"; status "$h"
|
|
r "$h" 'ip -4 -br addr show pppoe0 2>/dev/null; ip route show default; sudo journalctl -t vrrp-wan -n 8 --no-pager'
|
|
done; } > "$d/state.txt" 2>&1
|
|
log "evidence -> wan-failover-evidence/$name/"
|
|
}
|
|
|
|
# --- tests -----------------------------------------------------------------
|
|
T0() { # baseline
|
|
log "T0 baseline: exactly one session, held by the VIP holder"
|
|
local h s; h="$(holder)"; s="$(ac_sessions)"
|
|
[ "$s" = 1 ] && pass "AC reports 1 session" || fail "AC reports $s sessions"
|
|
[ "$(ppp_on "$h")" != 0 ] && pass "the VIP holder ($h) is the one dialled" \
|
|
|| fail "VIP holder $h has no pppoe0"
|
|
local other; other=$([ "$h" = "$R1" ] && echo "$R2" || echo "$R1")
|
|
[ "$(ppp_on "$other")" = 0 ] && pass "the backup ($other) is not dialled" \
|
|
|| fail "backup $other also holds pppoe0"
|
|
save_evidence T0-baseline
|
|
}
|
|
|
|
T3() { # clean, deliberate failover
|
|
log "T3 clean failover via force-fault"
|
|
local from to t0 t1; from="$(holder)"
|
|
to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1")
|
|
log " master=$from -> expecting $to"
|
|
t0=$(date +%s)
|
|
r "$from" 'sudo mkdir -p /run/vrrp-wan && sudo touch /run/vrrp-wan/force-fault'
|
|
if settle_on "$to" 30; then
|
|
t1=$(date +%s)
|
|
[ "$(ppp_on "$to")" != 0 ] && pass "pppoe0 moved to $to in $((t1-t0))s" \
|
|
|| fail "pppoe0 did not move to $to"
|
|
[ "$(ppp_on "$from")" = 0 ] && pass "$from released pppoe0" \
|
|
|| fail "$from still holds pppoe0"
|
|
else
|
|
fail "never settled to exactly one pppoe0 holder"
|
|
fi
|
|
check_invariant
|
|
save_evidence T3-clean-failover
|
|
r "$from" 'sudo rm -f /run/vrrp-wan/force-fault'
|
|
settle 30 >/dev/null
|
|
}
|
|
|
|
T5() { # 10gig down -> PPPoE carries traffic
|
|
log "T5 10 gig down on the master: traffic must survive on pppoe0"
|
|
local h; h="$(holder)"
|
|
r "$h" 'sudo ip link set bond0.53 down'
|
|
sleep 20
|
|
local via; via="$(r "$h" 'ip route show default' | head -1)"
|
|
if echo "$via" | grep -q pppoe0; then
|
|
pass "default route moved to pppoe0: $via"
|
|
else
|
|
fail "default route did not move to pppoe0: ${via:-<none>}"
|
|
fi
|
|
# Poll, do not sample. Judging connectivity on one ping 20s after the link
|
|
# dropped failed while the path was still reconverging, and reported "the LAN
|
|
# lost the internet" for a path that came back moments later. A single
|
|
# negative sample is the least trustworthy verdict this harness can produce.
|
|
local ok=no i
|
|
for i in $(seq 1 12); do
|
|
lan_online && { ok=yes; break; }
|
|
sleep 5
|
|
done
|
|
[ "$ok" = yes ] && pass "LAN reaches the internet over pppoe0 (after $((i*5))s)" \
|
|
|| fail "LAN never regained the internet with only pppoe0 up (60s)"
|
|
save_evidence T5-tengig-down
|
|
r "$h" 'sudo ip link set bond0.53 up'
|
|
sleep 20
|
|
}
|
|
|
|
T11() { # a blessed box with no peers file must not restart-loop
|
|
log "T11 missing peers file must not restart-loop"
|
|
local h; h="$(holder)"
|
|
# COPY then remove, never move: only a commit touching the pppoe subtree
|
|
# re-renders this file, so losing it strands the box permanently -- the gate
|
|
# blocks every dial, systemd says "skipped because of an unmet condition
|
|
# check" exactly once, and nothing else complains. An earlier `mv` pair did
|
|
# exactly that to the sim secondary and cost a debugging session.
|
|
r "$h" 'sudo cp -a /etc/ppp/peers/pppoe0 /run/peers.bak && sudo rm -f /etc/ppp/peers/pppoe0; sudo systemctl restart ppp@pppoe0'
|
|
sleep 12
|
|
local n; n="$(r "$h" 'systemctl show ppp@pppoe0 -p NRestarts --value')"
|
|
[ "${n:-99}" -le 1 ] && pass "NRestarts=$n (gate refused the start)" \
|
|
|| fail "NRestarts=$n -- restart loop is back"
|
|
# Restore on the SAME host we broke, and prove it landed. Do not trust the
|
|
# copy back: if it silently failed, every later test in the run would be
|
|
# measuring a router that physically cannot dial.
|
|
r "$h" 'sudo cp -a /run/peers.bak /etc/ppp/peers/pppoe0'
|
|
if r "$h" 'test -f /etc/ppp/peers/pppoe0'; then
|
|
pass "peers file restored on $h"
|
|
else
|
|
fail "peers file NOT restored on $h -- that router can no longer dial"
|
|
fi
|
|
save_evidence T11-no-peers-file
|
|
settle 24 >/dev/null
|
|
}
|
|
|
|
T8() { # lease expiry: the guard must hang up a demoted-but-unreconciled box
|
|
log "T8 lease expiry revokes the session"
|
|
local h; h="$(holder)"
|
|
r "$h" 'sudo systemctl stop vrrp-wan-reconcile.timer'
|
|
r "$h" 'sudo touch -d "-200 seconds" /run/vrrp-wan/may-dial'
|
|
sleep 12
|
|
[ "$(ppp_on "$h")" = 0 ] && pass "guard hung up on a stale lease" \
|
|
|| fail "stale lease did not revoke the session"
|
|
r "$h" 'sudo systemctl start vrrp-wan-reconcile.timer'
|
|
save_evidence T8-lease-expiry
|
|
settle 24 >/dev/null
|
|
}
|
|
|
|
|
|
T4() { # hard failover across all three AC session-control policies
|
|
log "T4 hard failover (destroy the master) x session-control"
|
|
# Vodafone's policy is unknowable from here, so prove the design survives
|
|
# every one VyOS can express. `replace` is the accel-ppp default and the
|
|
# friendly case; `deny` is the hostile one, where the AC refuses the second
|
|
# session until its own dead-peer timer (lcp-echo-interval 30 x failure 3 =
|
|
# 90s) frees the first -- which is exactly why GRACE is no longer 90.
|
|
local mode from to vm t0 t1
|
|
for mode in replace deny disable; do
|
|
log " --- session-control=$mode ---"
|
|
# Skip the iteration rather than measure the wrong policy.
|
|
isp_session_control "$mode" || continue
|
|
sleep 5
|
|
from="$(holder)"; to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1")
|
|
vm=$([ "$from" = "$R1" ] && echo labsim-vyos || echo labsim-vyos2)
|
|
[ "$from" = none ] && { fail "no master before $mode run"; continue; }
|
|
log " destroying $vm (master=$from), expecting $to"
|
|
t0=$(date +%s)
|
|
sudo virsh destroy "$vm" >/dev/null 2>&1
|
|
if settle_on "$to" 48; then
|
|
t1=$(date +%s)
|
|
pass "$mode: pppoe0 reached $to in $((t1-t0))s"
|
|
else
|
|
fail "$mode: $to never dialled within 240s"
|
|
fi
|
|
check_invariant
|
|
save_evidence "T4-hard-failover-$mode"
|
|
sudo virsh start "$vm" >/dev/null 2>&1
|
|
# Re-bond. A VM restart recreates its taps under NEW names, and the OVS
|
|
# bond keeps the old ones -- lacp dies, VLAN 1 goes with it, and the box
|
|
# comes back reachable on some VLANs but not others. ovs_bond_router
|
|
# detects the stale membership and rebuilds, but nothing runs it
|
|
# automatically, so a destroy/start test must do it or the survivor
|
|
# looks like a failover failure.
|
|
( source "$SCRIPT_DIR/lib.sh"; source "$SCRIPT_DIR/ovs.sh"; selected_vlans
|
|
LAG_NAME=$([ "$vm" = labsim-vyos ] && echo lag-vyos || echo lag-vyos2)
|
|
ovs_bond_router "$vm" ) >/dev/null 2>&1
|
|
# Give the returning box time to boot and settle as BACKUP before the
|
|
# next iteration; it must NOT dial on the way up.
|
|
sleep 90
|
|
[ "$(ppp_on "$from")" = 0 ] && pass "$mode: $from did not dial on reboot" \
|
|
|| fail "$mode: $from dialled on reboot (gate failed)"
|
|
done
|
|
isp_session_control replace >/dev/null
|
|
log " AC restored to session-control=replace"
|
|
}
|
|
|
|
T12() { # the flap damper must never tear down an ESTABLISHED session
|
|
log "T12 flap holdoff must not kill a live session"
|
|
local h; h="$(holder)"
|
|
[ "$h" = none ] && { fail "no master to test"; return; }
|
|
# Forge a holdoff far in the future, as a dial storm would. Before the fix
|
|
# ppp_dial() returned here BEFORE renewing may-dial, the lease went stale,
|
|
# and vrrp-wan-guard hung up the master's working WAN ~80s later.
|
|
r "$h" 'sudo sh -c "echo $(( $(date +%s) + 900 )) > /run/vrrp-wan/holdoff"'
|
|
# Sleep past LEASE_TTL (75s) so a non-renewed lease would definitely expire.
|
|
sleep 100
|
|
if [ "$(ppp_on "$h")" = 1 ]; then
|
|
pass "session survived a 900s holdoff (lease still renewed)"
|
|
else
|
|
fail "holdoff killed the live session -- damper is tearing down the WAN"
|
|
fi
|
|
r "$h" 'sudo rm -f /run/vrrp-wan/holdoff /run/vrrp-wan/dials'
|
|
save_evidence T12-holdoff-keeps-session
|
|
settle 24 >/dev/null
|
|
}
|
|
|
|
case "${1:---all}" in
|
|
--list) echo "T0 baseline | T3 clean failover | T4 hard failover x policy | T5 10gig-down | T8 lease expiry | T11 no-peers-file | T12 holdoff-keeps-session"; exit 0 ;;
|
|
--all) preflight || exit 1; T0; T3; T5; T8; T11; T12 ;;
|
|
--hard) preflight || exit 1; T4 ;;
|
|
*) preflight || exit 1; "$1" ;;
|
|
esac
|
|
|
|
echo
|
|
[ "$FAILED" -eq 0 ] && { echo "ALL PASS"; exit 0; }
|
|
echo "$FAILED check(s) FAILED"; exit 1
|