lab/migration/vyos002-catch.sh

#!/bin/bash
# Arm BEFORE powering vyos002 on. Strips the eth2 address the moment the box is
# reachable, then installs the VRRP health-check.
#
# Why this exists: vyos002 boots with `interfaces ethernet eth2 address
# 192.168.8.144/23` still in config.boot -- the same subnet as bond0.2. Linux
# answers ARP for any local address out of any interface on that L2, so eth2
# answers for addresses that bond0.2 is supposed to route, and traffic lands on
# a port that does not route it. That is the 2026-09-02 cluster outage.
#
# A human "jumping on it fast" loses this race more often than not; the box is
# reachable within a second or two of the interfaces coming up. This polls at
# 1s and commits the moment it gets in.
#
#   ./vyos002-catch.sh            arm and wait (Ctrl-C to disarm)
#
# Bounded risk while you wait, worth knowing: vyos002 comes up BACKUP (priority
# 100, no-preempt, vyos001 healthy MASTER), so it does NOT hold 192.168.8.1 and
# the GATEWAY cannot be poisoned. The exposure is its own bond0.2 address, which
# is survivable. The unbounded case is it becoming MASTER while eth2 is present
# -- which is exactly what the health-check in step 2 prevents.
set -uo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PW="${VYOS_PW:-vyos}"
# Management first because it comes up with the box; LoT is the fallback and is
# L2-direct from this workstation (see RECOVERY-CARD-vlan1-move.md).
TARGETS=("192.168.1.253" "10.0.1.253")
SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
          -o LogLevel=ERROR -o ConnectTimeout=2 -o PreferredAuthentications=password)

log() { printf '\033[36m[catch %s]\033[0m %s\n' "$(date +%T)" "$*"; }

on() { timeout 12 sshpass -p "$PW" ssh "${SSH_OPTS[@]}" "vyos@$1" "$@"; }

log "armed -- polling ${TARGETS[*]} every 1s. Power on vyos002 now."
HOST=""
while [ -z "$HOST" ]; do
  for t in "${TARGETS[@]}"; do
    if timeout 4 sshpass -p "$PW" ssh "${SSH_OPTS[@]}" "vyos@$t" true 2>/dev/null; then
      HOST="$t"; break
    fi
  done
  [ -z "$HOST" ] && sleep 1
done
log "CAUGHT on $HOST -- stripping eth2 address"

# Step 1, on its own commit: get the address off eth2 before anything else. Any
# extra command in this commit is extra seconds of exposure.
timeout 90 sshpass -p "$PW" ssh "${SSH_OPTS[@]}" "vyos@$HOST" 'vbash -s' <<'EOF' 2>&1 | tail -3
source /opt/vyatta/etc/functions/script-template
delete interfaces ethernet eth2 address
commit
echo "ETH2_RC=$?"
save
exit
EOF
log "eth2 address removed"

# Step 2: the health-check. Without it this box can hold every floating IP while
# having no WAN -- the outage itself. Copy the script BEFORE referencing it, or
# the commit succeeds and the check silently never passes.
timeout 30 sshpass -p "$PW" scp "${SSH_OPTS[@]}" \
  "$HERE/vrrp-wan-health" "vyos@$HOST:/tmp/vrrp-wan-health" >/dev/null 2>&1 \
  && log "health-check script copied" || log "WARN: scp failed -- step 2 will be skipped"

timeout 90 sshpass -p "$PW" ssh "${SSH_OPTS[@]}" "vyos@$HOST" 'vbash -s' <<'EOF' 2>&1 | tail -3
sudo install -o root -g vyattacfg -m 0775 /tmp/vrrp-wan-health /config/vrrp-wan-health
source /opt/vyatta/etc/functions/script-template
set high-availability vrrp sync-group MAIN health-check script '/config/vrrp-wan-health'
set high-availability vrrp sync-group MAIN health-check interval '5'
set high-availability vrrp sync-group MAIN health-check failure-count '3'
commit
echo "HEALTH_RC=$?"
save
exit
EOF
log "health-check installed"

echo
log "=== state ==="
timeout 30 sshpass -p "$PW" ssh "${SSH_OPTS[@]}" "vyos@$HOST" \
  'echo "-- eth2 (must show no inet) --"; ip -4 addr show eth2 2>/dev/null | grep inet || echo "   none"
   echo "-- health-check --"; sudo /config/vrrp-wan-health; echo "   exit=$? (non-zero = no WAN = refuses the VIPs, which is CORRECT and safe)"
   echo "-- vrrp --"; /opt/vyatta/bin/vyatta-op-cmd-wrapper show vrrp' 2>&1