lab/labsim/router-up.sh

#!/bin/bash
# Add the VyOS router under test to labsim.
#
# Mirrors the production VP2440 pair: TWO NICs bonded with LACP carrying a
# trunk of every VLAN, then bond0.<vlan> sub-interfaces holding the .1 gateway
# address on each. That is the same config shape the real firewalls run, so a
# rule tested here means something.
#
# NIC model is e1000e, NOT virtio, and that is load-bearing: with virtio the
# guest's bonding driver reports its slaves "MII Status: down" despite
# carrier=1 and never emits a single LACPDU, so the bond sits in
# AD_STATE_DEFAULTED forever. Known issue — see the netdev thread "bonding
# (IEEE 802.3ad) not working with qemu/virtio"; e1000e fixes it with no other
# change. 802.3ad also requires the MII link monitor, which virtio cannot back.
#
#   host                       OVS "switch"                    VyOS VM
#   hostv<vlan> (.2) ──────── ovs-labsim ──── lag-vyos ═════ eth0 + eth1
#                              (tagged)       (LACP, trunk)   └─ bond0.<vlan> = .1
#
# Usage: ./router-up.sh            build + install + configure
#        ./router-up.sh --status   show bond/LACP + interface state
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "$SCRIPT_DIR/lib.sh"
source "$SCRIPT_DIR/ovs.sh"

ROUTER_VM="${ROUTER_VM:-labsim-vyos}"
ROUTER_MEM="${ROUTER_MEM:-2048}"
ROUTER_CPUS="${ROUTER_CPUS:-2}"
ROUTER_DISK_GB="${ROUTER_DISK_GB:-8}"
VYOS_ISO="${VYOS_ISO:-$IMG_DIR/vyos.iso}"
VYOS_CACHE="/var/lib/libvirt/images/lab-pxe-cache"

selected_vlans

if [ "${1:-}" = "--status" ]; then
  ovs_bond_status
  echo "--- vyos gateway addresses (probed from each host leg) ---"
  for entry in "${SELECTED[@]}"; do
    IFS=: read -r vid _n prefix _r <<<"$entry"
    printf '  vlan %-5s %-16s ' "$vid" "${prefix}.1"
    ping -c1 -W2 "${prefix}.1" >/dev/null 2>&1 && echo up || echo down
  done
  exit 0
fi

ovs_require

# --- ISO ------------------------------------------------------------------
if [ ! -f "$VYOS_ISO" ]; then
  # Reuse the bastion's cached nightly if it is already on this box.
  if [ -f "$VYOS_CACHE/vyos.iso" ]; then
    log "reusing cached VyOS ISO"
    sudo cp "$VYOS_CACHE/vyos.iso" "$VYOS_ISO"
  else
    log "resolving latest VyOS nightly ISO..."
    url="$(curl -sSL https://api.github.com/repos/vyos/vyos-nightly-build/releases/latest \
      | python3 -c "import json,sys;print(next(a['browser_download_url'] for a in json.load(sys.stdin)['assets'] if a['name'].endswith('generic-amd64.iso')))")"
    log "downloading $url"
    sudo curl -sSL --max-time 1800 -o "$VYOS_ISO" "$url"
  fi
fi
[ -f "$VYOS_ISO" ] || die "no VyOS ISO at $VYOS_ISO"

# --- VM -------------------------------------------------------------------
if virsh_q dominfo "$ROUTER_VM" >/dev/null 2>&1; then
  log "router VM $ROUTER_VM exists"
  virsh_q start "$ROUTER_VM" >/dev/null 2>&1 || true
else
  log "creating router VM $ROUTER_VM (2 NICs on the trunk, for LACP)"
  sudo qemu-img create -q -f qcow2 "$IMG_DIR/${ROUTER_VM}.qcow2" "${ROUTER_DISK_GB}G" >/dev/null

  # Two trunk NICs — OVS bonds them after boot (libvirt cannot create bonds).
  sudo virt-install \
    --connect "$LIBVIRT_URI" \
    --name "$ROUTER_VM" \
    --memory "$ROUTER_MEM" --vcpus "$ROUTER_CPUS" \
    --disk "path=$IMG_DIR/${ROUTER_VM}.qcow2,format=qcow2,bus=virtio" \
    --disk "path=$VYOS_ISO,device=cdrom,readonly=on" \
    --network "network=$OVS_NET,portgroup=trunk,model=e1000e,trustGuestRxFilters=yes" \
    --network "network=$OVS_NET,portgroup=trunk,model=e1000e,trustGuestRxFilters=yes" \
    --boot cdrom,hd \
    --os-variant debian12 \
    --graphics none --noautoconsole --import >/dev/null
fi

log "waiting for the live system to boot (VyOS live login)..."
python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase live || die "live boot failed"

log "installing VyOS to disk (unattended over the console)..."
python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase install || die "install failed"

# Boot the INSTALLED system from here on. Without this the VM was created with
# --boot cdrom,hd and every restart re-runs the ISO, so the live system comes
# back with no config and every `commit; save` silently evaporates.
log "switching boot to disk and ejecting the install media..."
virsh_q destroy "$ROUTER_VM" >/dev/null 2>&1 || true
sleep 2
sudo virt-xml "$ROUTER_VM" --edit --boot hd >/dev/null
sudo virt-xml "$ROUTER_VM" --remove-device --disk device=cdrom >/dev/null 2>&1 || true
virsh_q start "$ROUTER_VM" >/dev/null
sleep 10

# Bond the taps only now: they are recreated by the restart above, so bonding
# before this would bond stale interfaces.
ovs_bond_router "$ROUTER_VM"

log "applying router config (bond0 LACP + VLAN gateways)..."
python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase configure \
  --vlans "$(printf '%s\n' "${SELECTED[@]}" | tr '\n' ' ')" || die "configure failed"

log "waiting for LACP to negotiate..."
for _ in $(seq 1 30); do
  if sudo ovs-appctl lacp/show "$LAG_NAME" 2>/dev/null | grep -q "current attached"; then
    log "LACP negotiated"; break
  fi
  sleep 5
done

echo
ovs_bond_status
echo
log "router is up. Check reachability with: $SCRIPT_DIR/labsim-matrix.py --watch 2"