lab/labsim/labsim-pppoe-ha-test.sh

#!/bin/bash
# Does the WAN follow VRRP mastership, and does exactly ONE router ever hold the
# ISP session?
#
# The question is not "did a client get internet". A client can be answered by
# the wrong path entirely -- for months labsim-vyos's only default route was the
# libvirt-NAT scaffold on eth2, so every "the LAN still has internet" verdict was
# answered by eth2 rather than by the WAN under test. This script therefore
# refuses to run while that is true, and asks its questions of the ROUTERS and
# the ACCESS CONCENTRATOR, which cannot be answered by accident.
#
# The invariant, checked continuously and independently of any individual test:
#
#     the AC never reports two `simdsl` sessions, and no two routers ever have a
#     pppoe0 interface at the same time
#
# A run that violates it FAILS regardless of its own verdict, because a single
# consumer credential is the whole constraint the design exists to satisfy.
#
#   ./labsim-pppoe-ha-test.sh --list
#   ./labsim-pppoe-ha-test.sh T3
#   ./labsim-pppoe-ha-test.sh --all
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
R1="${R1:-172.31.1.252}"; R2="${R2:-172.31.1.253}"
ISP="${ISP:-192.168.122.63}"          # the fake access concentrator
LANVM="${LANVM:-172.31.10.10}"
VIP="${VIP:-172.31.1.1}"
PW="${VYOS_PW:-vyos}"; LANPW="${LANPW:-labsim}"
EVID="$SCRIPT_DIR/wan-failover-evidence"

SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
     -o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password)
r()   { timeout 45 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; }
isp() { timeout 30 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$ISP" "$@" 2>/dev/null; }
# The LAN VMs are Alpine and their sshd offers keyboard-interactive, not
# `password`. Reusing the routers' option set here made ssh exit 255 BEFORE
# running anything, and T5 read that as "the LAN lost the internet" while a
# tcpdump on the router showed the pings flowing out pppoe0 and the replies
# coming back. An exit code that can mean "the network is broken" or "I could
# not log in" is not a connectivity test.
LAN_SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
         -o LogLevel=ERROR -o ConnectTimeout=6)
lan() { timeout 45 sshpass -p "$LANPW" ssh "${LAN_SSH[@]}" "root@$LANVM" "$@" 2>/dev/null; }

# Assert on what the guest actually reported, not on ssh's exit status.
lan_online() { [ "$(lan 'ping -c2 -W3 9.9.9.9 >/dev/null 2>&1 && echo ONLINE')" = ONLINE ]; }

log()  { printf '\033[36m==>\033[0m %s\n' "$*"; }
pass() { printf '  \033[32mPASS\033[0m %s\n' "$*"; }
fail() { printf '  \033[31mFAIL\033[0m %s\n' "$*"; FAILED=$((FAILED+1)); }
FAILED=0

# --- observations ----------------------------------------------------------
ac_sessions() { isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show pppoe-server sessions' \
                  | grep -c ' simdsl ' || true; }
ac_detail()   { isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show pppoe-server sessions'; }
# A destroyed or unreachable router is emphatically NOT holding pppoe0, but ssh
# returns an EMPTY string rather than 0 -- and `[ "" = 0 ]` is false, so a
# hard-failover test waited for the dead box to "report" zero and hung until its
# timeout, long after the survivor had taken over correctly. Default to 0.
ppp_on()      { local v; v="$(r "$1" 'ip -4 addr show pppoe0 2>/dev/null | grep -c inet' | tr -d ' \n')"
                echo "${v:-0}"; }
holder()      { for h in "$R1" "$R2"; do
                  [ "$(r "$h" "ip -4 -o addr show | grep -c ' ${VIP}/'" | tr -d ' \n')" != 0 ] \
                    && { echo "$h"; return; }; done; echo none; }
status()      { r "$1" 'sudo /config/vrrp-wan-reconcile --status'; }

# How many routers currently hold a PPPoE interface. The invariant's other half.
ppp_holders() { n=0; for h in "$R1" "$R2"; do
                  [ "$(ppp_on "$h")" != 0 ] && n=$((n+1)); done; echo "$n"; }

check_invariant() {
    local s p ok=0
    s="$(ac_sessions)"; p="$(ppp_holders)"
    [ "${s:-0}" -le 1 ] || { fail "INVARIANT: AC reports $s simdsl sessions"; ok=1; }
    [ "${p:-0}" -le 1 ] || { fail "INVARIANT: $p routers hold pppoe0"; ok=1; }
    return $ok
}

# --- preconditions ---------------------------------------------------------
# The scaffold check is a hard gate, not a warning. A default route via eth2
# means the box can reach the internet without the WAN working at all, and every
# connectivity verdict below would be a lie.
preflight() {
    log "preflight"
    local rc=0
    for h in "$R1" "$R2"; do
        if r "$h" 'ip route show default' | grep -q 'dev eth2'; then
            fail "$h still routes via eth2 (libvirt-NAT scaffold) -- run sim-net-config.py --drop-scaffold"
            rc=1
        fi
        if [ "$(r "$h" '[ -f /etc/systemd/system/ppp@pppoe0.service.d/10-vrrp-wan-gate.conf ] && echo y')" != y ]; then
            fail "$h is missing the ppp gate drop-in -- run migration/vrrp-wan-install"
            rc=1
        fi
        [ "$(r "$h" 'systemctl is-active vrrp-wan-guard.timer')" = active ] \
            || { fail "$h vrrp-wan-guard.timer not active"; rc=1; }
        # A router with no peers file CANNOT dial, and says so only once in the
        # journal. Every failover result in the run would then be a false
        # negative blamed on the ISP. Check both, and check the config that
        # renders it -- an unsaved commit reverts on reboot and takes pppoe0
        # with it, which is how the sim secondary silently stopped dialling.
        if [ "$(r "$h" '[ -f /etc/ppp/peers/pppoe0 ] && echo y')" != y ]; then
            fail "$h has no /etc/ppp/peers/pppoe0 -- it cannot dial; re-commit the pppoe subtree"
            rc=1
        fi
        if ! r "$h" 'show configuration commands' 2>/dev/null | grep -q 'interfaces pppoe pppoe0 source-interface'; then
            fail "$h has no pppoe0 in config (unsaved commit lost on reboot?)"
            rc=1
        fi
    done
    [ "$rc" -eq 0 ] && pass "scaffold dropped, gate present, guard running, both can dial"
    return $rc
}

settle() {  # wait until exactly one router holds pppoe0, or give up
    local i
    for i in $(seq 1 "${1:-24}"); do
        [ "$(ppp_holders)" = 1 ] && return 0
        sleep 5
    done
    return 1
}

# Wait until a SPECIFIC router holds pppoe0 and the other does not.
#
# The obvious `settle` is wrong for a failover: "exactly one holder" is already
# true before the handover starts, so it returns instantly and the test reports
# that nothing moved while the handover is still in flight. Asking who holds it
# is the only useful form of the question.
settle_on() {
    local want="$1" other i
    other=$([ "$want" = "$R1" ] && echo "$R2" || echo "$R1")
    for i in $(seq 1 "${2:-30}"); do
        [ "$(ppp_on "$want")" != 0 ] && [ "$(ppp_on "$other")" = 0 ] && return 0
        sleep 5
    done
    return 1
}

save_evidence() {
    local name="$1"; local d="$EVID/$name"; mkdir -p "$d"
    { echo "=== $(date -Is) ==="; echo "--- AC sessions ---"; ac_detail
      for h in "$R1" "$R2"; do echo "--- $h ---"; status "$h"
        r "$h" 'ip -4 -br addr show pppoe0 2>/dev/null; ip route show default; sudo journalctl -t vrrp-wan -n 8 --no-pager'
      done; } > "$d/state.txt" 2>&1
    log "evidence -> wan-failover-evidence/$name/"
}

# --- tests -----------------------------------------------------------------
T0() {  # baseline
    log "T0  baseline: exactly one session, held by the VIP holder"
    local h s; h="$(holder)"; s="$(ac_sessions)"
    [ "$s" = 1 ] && pass "AC reports 1 session" || fail "AC reports $s sessions"
    [ "$(ppp_on "$h")" != 0 ] && pass "the VIP holder ($h) is the one dialled" \
        || fail "VIP holder $h has no pppoe0"
    local other; other=$([ "$h" = "$R1" ] && echo "$R2" || echo "$R1")
    [ "$(ppp_on "$other")" = 0 ] && pass "the backup ($other) is not dialled" \
        || fail "backup $other also holds pppoe0"
    save_evidence T0-baseline
}

T3() {  # clean, deliberate failover
    log "T3  clean failover via force-fault"
    local from to t0 t1; from="$(holder)"
    to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1")
    log "    master=$from -> expecting $to"
    t0=$(date +%s)
    r "$from" 'sudo mkdir -p /run/vrrp-wan && sudo touch /run/vrrp-wan/force-fault'
    if settle_on "$to" 30; then
        t1=$(date +%s)
        [ "$(ppp_on "$to")" != 0 ] && pass "pppoe0 moved to $to in $((t1-t0))s" \
            || fail "pppoe0 did not move to $to"
        [ "$(ppp_on "$from")" = 0 ] && pass "$from released pppoe0" \
            || fail "$from still holds pppoe0"
    else
        fail "never settled to exactly one pppoe0 holder"
    fi
    check_invariant
    save_evidence T3-clean-failover
    r "$from" 'sudo rm -f /run/vrrp-wan/force-fault'
    settle 30 >/dev/null
}

T5() {  # 10gig down -> PPPoE carries traffic
    log "T5  10 gig down on the master: traffic must survive on pppoe0"
    local h; h="$(holder)"
    r "$h" 'sudo ip link set bond0.53 down'
    sleep 20
    local via; via="$(r "$h" 'ip route show default' | head -1)"
    if echo "$via" | grep -q pppoe0; then
        pass "default route moved to pppoe0: $via"
    else
        fail "default route did not move to pppoe0: ${via:-<none>}"
    fi
    # Poll, do not sample. Judging connectivity on one ping 20s after the link
    # dropped failed while the path was still reconverging, and reported "the LAN
    # lost the internet" for a path that came back moments later. A single
    # negative sample is the least trustworthy verdict this harness can produce.
    local ok=no i
    for i in $(seq 1 12); do
        lan_online && { ok=yes; break; }
        sleep 5
    done
    [ "$ok" = yes ] && pass "LAN reaches the internet over pppoe0 (after $((i*5))s)" \
        || fail "LAN never regained the internet with only pppoe0 up (60s)"
    save_evidence T5-tengig-down
    r "$h" 'sudo ip link set bond0.53 up'
    sleep 20
}

T11() { # a blessed box with no peers file must not restart-loop
    log "T11 missing peers file must not restart-loop"
    local h; h="$(holder)"
    # COPY then remove, never move: only a commit touching the pppoe subtree
    # re-renders this file, so losing it strands the box permanently -- the gate
    # blocks every dial, systemd says "skipped because of an unmet condition
    # check" exactly once, and nothing else complains. An earlier `mv` pair did
    # exactly that to the sim secondary and cost a debugging session.
    r "$h" 'sudo cp -a /etc/ppp/peers/pppoe0 /run/peers.bak && sudo rm -f /etc/ppp/peers/pppoe0; sudo systemctl restart ppp@pppoe0'
    sleep 12
    local n; n="$(r "$h" 'systemctl show ppp@pppoe0 -p NRestarts --value')"
    [ "${n:-99}" -le 1 ] && pass "NRestarts=$n (gate refused the start)" \
        || fail "NRestarts=$n -- restart loop is back"
    # Restore on the SAME host we broke, and prove it landed. Do not trust the
    # copy back: if it silently failed, every later test in the run would be
    # measuring a router that physically cannot dial.
    r "$h" 'sudo cp -a /run/peers.bak /etc/ppp/peers/pppoe0'
    if r "$h" 'test -f /etc/ppp/peers/pppoe0'; then
        pass "peers file restored on $h"
    else
        fail "peers file NOT restored on $h -- that router can no longer dial"
    fi
    save_evidence T11-no-peers-file
    settle 24 >/dev/null
}

T8() {  # lease expiry: the guard must hang up a demoted-but-unreconciled box
    log "T8  lease expiry revokes the session"
    local h; h="$(holder)"
    r "$h" 'sudo systemctl stop vrrp-wan-reconcile.timer'
    r "$h" 'sudo touch -d "-200 seconds" /run/vrrp-wan/may-dial'
    sleep 12
    [ "$(ppp_on "$h")" = 0 ] && pass "guard hung up on a stale lease" \
        || fail "stale lease did not revoke the session"
    r "$h" 'sudo systemctl start vrrp-wan-reconcile.timer'
    save_evidence T8-lease-expiry
    settle 24 >/dev/null
}


T4() {  # hard failover across all three AC session-control policies
    log "T4  hard failover (destroy the master) x session-control"
    # Vodafone's policy is unknowable from here, so prove the design survives
    # every one VyOS can express. `replace` is the accel-ppp default and the
    # friendly case; `deny` is the hostile one, where the AC refuses the second
    # session until its own dead-peer timer (lcp-echo-interval 30 x failure 3 =
    # 90s) frees the first -- which is exactly why GRACE is no longer 90.
    local mode from to vm t0 t1
    for mode in replace deny disable; do
        log "    --- session-control=$mode ---"
        isp "vbash -c 'source /opt/vyatta/etc/functions/script-template; configure; set service pppoe-server session-control $mode; commit; save; exit'" >/dev/null 2>&1
        sleep 5
        from="$(holder)"; to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1")
        vm=$([ "$from" = "$R1" ] && echo labsim-vyos || echo labsim-vyos2)
        [ "$from" = none ] && { fail "no master before $mode run"; continue; }
        log "    destroying $vm (master=$from), expecting $to"
        t0=$(date +%s)
        sudo virsh destroy "$vm" >/dev/null 2>&1
        if settle_on "$to" 48; then
            t1=$(date +%s)
            pass "$mode: pppoe0 reached $to in $((t1-t0))s"
        else
            fail "$mode: $to never dialled within 240s"
        fi
        check_invariant
        save_evidence "T4-hard-failover-$mode"
        sudo virsh start "$vm" >/dev/null 2>&1
        # Re-bond. A VM restart recreates its taps under NEW names, and the OVS
        # bond keeps the old ones -- lacp dies, VLAN 1 goes with it, and the box
        # comes back reachable on some VLANs but not others. ovs_bond_router
        # detects the stale membership and rebuilds, but nothing runs it
        # automatically, so a destroy/start test must do it or the survivor
        # looks like a failover failure.
        ( source "$SCRIPT_DIR/lib.sh"; source "$SCRIPT_DIR/ovs.sh"; selected_vlans
          LAG_NAME=$([ "$vm" = labsim-vyos ] && echo lag-vyos || echo lag-vyos2)
          ovs_bond_router "$vm" ) >/dev/null 2>&1
        # Give the returning box time to boot and settle as BACKUP before the
        # next iteration; it must NOT dial on the way up.
        sleep 90
        [ "$(ppp_on "$from")" = 0 ] && pass "$mode: $from did not dial on reboot" \
            || fail "$mode: $from dialled on reboot (gate failed)"
    done
    isp "vbash -c 'source /opt/vyatta/etc/functions/script-template; configure; set service pppoe-server session-control replace; commit; save; exit'" >/dev/null 2>&1
    log "    AC restored to session-control=replace"
}

T12() { # the flap damper must never tear down an ESTABLISHED session
    log "T12 flap holdoff must not kill a live session"
    local h; h="$(holder)"
    [ "$h" = none ] && { fail "no master to test"; return; }
    # Forge a holdoff far in the future, as a dial storm would. Before the fix
    # ppp_dial() returned here BEFORE renewing may-dial, the lease went stale,
    # and vrrp-wan-guard hung up the master's working WAN ~80s later.
    r "$h" 'sudo sh -c "echo $(( $(date +%s) + 900 )) > /run/vrrp-wan/holdoff"'
    # Sleep past LEASE_TTL (75s) so a non-renewed lease would definitely expire.
    sleep 100
    if [ "$(ppp_on "$h")" = 1 ]; then
        pass "session survived a 900s holdoff (lease still renewed)"
    else
        fail "holdoff killed the live session -- damper is tearing down the WAN"
    fi
    r "$h" 'sudo rm -f /run/vrrp-wan/holdoff /run/vrrp-wan/dials'
    save_evidence T12-holdoff-keeps-session
    settle 24 >/dev/null
}

case "${1:---all}" in
  --list) echo "T0 baseline | T3 clean failover | T4 hard failover x policy | T5 10gig-down | T8 lease expiry | T11 no-peers-file | T12 holdoff-keeps-session"; exit 0 ;;
  --all)  preflight || exit 1; T0; T3; T5; T8; T11; T12 ;;
  --hard) preflight || exit 1; T4 ;;
  *)      preflight || exit 1; "$1" ;;
esac

echo
[ "$FAILED" -eq 0 ] && { echo "ALL PASS"; exit 0; }
echo "$FAILED check(s) FAILED"; exit 1