diff --git a/migration/vyos002-catch.sh b/migration/vyos002-catch.sh new file mode 100755 index 0000000..671eea8 --- /dev/null +++ b/migration/vyos002-catch.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Arm BEFORE powering vyos002 on. Strips the eth2 address the moment the box is +# reachable, then installs the VRRP health-check. +# +# Why this exists: vyos002 boots with `interfaces ethernet eth2 address +# 192.168.8.144/23` still in config.boot -- the same subnet as bond0.2. Linux +# answers ARP for any local address out of any interface on that L2, so eth2 +# answers for addresses that bond0.2 is supposed to route, and traffic lands on +# a port that does not route it. That is the 2026-09-02 cluster outage. +# +# A human "jumping on it fast" loses this race more often than not; the box is +# reachable within a second or two of the interfaces coming up. This polls at +# 1s and commits the moment it gets in. +# +# ./vyos002-catch.sh arm and wait (Ctrl-C to disarm) +# +# Bounded risk while you wait, worth knowing: vyos002 comes up BACKUP (priority +# 100, no-preempt, vyos001 healthy MASTER), so it does NOT hold 192.168.8.1 and +# the GATEWAY cannot be poisoned. The exposure is its own bond0.2 address, which +# is survivable. The unbounded case is it becoming MASTER while eth2 is present +# -- which is exactly what the health-check in step 2 prevents. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PW="${VYOS_PW:-vyos}" +# Management first because it comes up with the box; LoT is the fallback and is +# L2-direct from this workstation (see RECOVERY-CARD-vlan1-move.md). +TARGETS=("192.168.1.253" "10.0.1.253") +SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR -o ConnectTimeout=2 -o PreferredAuthentications=password) + +log() { printf '\033[36m[catch %s]\033[0m %s\n' "$(date +%T)" "$*"; } + +on() { timeout 12 sshpass -p "$PW" ssh "${SSH_OPTS[@]}" "vyos@$1" "$@"; } + +log "armed -- polling ${TARGETS[*]} every 1s. Power on vyos002 now." +HOST="" +while [ -z "$HOST" ]; do + for t in "${TARGETS[@]}"; do + if timeout 4 sshpass -p "$PW" ssh "${SSH_OPTS[@]}" "vyos@$t" true 2>/dev/null; then + HOST="$t"; break + fi + done + [ -z "$HOST" ] && sleep 1 +done +log "CAUGHT on $HOST -- stripping eth2 address" + +# Step 1, on its own commit: get the address off eth2 before anything else. Any +# extra command in this commit is extra seconds of exposure. +timeout 90 sshpass -p "$PW" ssh "${SSH_OPTS[@]}" "vyos@$HOST" 'vbash -s' <<'EOF' 2>&1 | tail -3 +source /opt/vyatta/etc/functions/script-template +delete interfaces ethernet eth2 address +commit +echo "ETH2_RC=$?" +save +exit +EOF +log "eth2 address removed" + +# Step 2: the health-check. Without it this box can hold every floating IP while +# having no WAN -- the outage itself. Copy the script BEFORE referencing it, or +# the commit succeeds and the check silently never passes. +timeout 30 sshpass -p "$PW" scp "${SSH_OPTS[@]}" \ + "$HERE/vrrp-wan-health" "vyos@$HOST:/tmp/vrrp-wan-health" >/dev/null 2>&1 \ + && log "health-check script copied" || log "WARN: scp failed -- step 2 will be skipped" + +timeout 90 sshpass -p "$PW" ssh "${SSH_OPTS[@]}" "vyos@$HOST" 'vbash -s' <<'EOF' 2>&1 | tail -3 +sudo install -o root -g vyattacfg -m 0775 /tmp/vrrp-wan-health /config/vrrp-wan-health +source /opt/vyatta/etc/functions/script-template +set high-availability vrrp sync-group MAIN health-check script '/config/vrrp-wan-health' +set high-availability vrrp sync-group MAIN health-check interval '5' +set high-availability vrrp sync-group MAIN health-check failure-count '3' +commit +echo "HEALTH_RC=$?" +save +exit +EOF +log "health-check installed" + +echo +log "=== state ===" +timeout 30 sshpass -p "$PW" ssh "${SSH_OPTS[@]}" "vyos@$HOST" \ + 'echo "-- eth2 (must show no inet) --"; ip -4 addr show eth2 2>/dev/null | grep inet || echo " none" + echo "-- health-check --"; sudo /config/vrrp-wan-health; echo " exit=$? (non-zero = no WAN = refuses the VIPs, which is CORRECT and safe)" + echo "-- vrrp --"; /opt/vyatta/bin/vyatta-op-cmd-wrapper show vrrp' 2>&1