lab/labsim/ovs.sh

#!/bin/bash
# Open vSwitch fabric for labsim — the "switch" the whole sim hangs off.
#
# Why OVS and not a Linux bridge: a Linux bridge cannot do LACP at all, and its
# VLAN support is awkward to drive from libvirt. OVS gives real 802.1Q access
# and trunk ports plus real LACP bonds, so a router VM can run the SAME bond0 +
# vif config as the production VP2440s instead of an approximation.
#
# Layout:
#   ovs-labsim            the switch
#     ├─ vm ports         access ports, tag=<vlan>       (micro VM per VLAN)
#     ├─ hostv<vlan>      internal ports, tag=<vlan>     (host leg, for SSH)
#     └─ lag-vyos         LACP bond, trunk of all VLANs  (router under test)
# shellcheck disable=SC2034

OVS_BR="${OVS_BR:-ovs-labsim}"
OVS_NET="${OVS_NET:-labsim-ovs}"          # libvirt network wrapping the bridge
LAG_NAME="${LAG_NAME:-lag-vyos}"

# Native (untagged) VLAN on the trunks to the routers. Empty means NONE: every
# VLAN, Management included, is tagged.
#
# This is not a style choice. A native VLAN is what puts a subnet on the bond
# PARENT (`bond0`) while every other VLAN lives on a sub-interface of it. With
# `dhcp-socket-type: raw`, kea then receives each tagged frame TWICE -- once on
# the sub-interface and once on the parent -- and answers from the parent's pool
# as well, so a client on VLAN 3 is offered a Management address and picks
# whichever reply arrives first (ISC Kea #1117).
#
# Set LABSIM_NATIVE_VLAN=1 to restore the old shape and reproduce the bug:
#   LABSIM_NATIVE_VLAN=1 ./router-up.sh && ./labsim-vlan-leak-test.sh   # FAIL
#                        ./router-up.sh && ./labsim-vlan-leak-test.sh   # PASS
NATIVE_VLAN="${LABSIM_NATIVE_VLAN:-}"

ovs() { sudo ovs-vsctl "$@"; }

ovs_require() {
  command -v ovs-vsctl >/dev/null 2>&1 || die "openvswitch not installed (dnf install openvswitch)"
  systemctl is-active --quiet openvswitch || sudo systemctl start openvswitch \
    || die "could not start openvswitch"
}

# A comma-separated VLAN list, numerically sorted, for comparing two lists that
# came from different places and need not agree on order.
vlan_sorted() { echo "$1" | tr ',' '\n' | grep -v '^$' | sort -n | paste -sd, -; }

# All VLAN ids from the config, comma separated — used for trunk ports.
vlan_id_list() {
  local ids=()
  for entry in "${SELECTED[@]}"; do ids+=("${entry%%:*}"); done
  (IFS=,; echo "${ids[*]}")
}

ovs_up() {
  ovs_require
  ovs --may-exist add-br "$OVS_BR"

  # Host leg per VLAN: an OVS internal port carrying that VLAN's tag, given the
  # .2 address. This is how you SSH to the VMs. It is deliberately NOT their
  # default route (.1 is), so inter-VLAN tests exercise the router, not the
  # host's routing table.
  for entry in "${SELECTED[@]}"; do
    parse_vlan_entry "$entry"
    local port="hostv${V_VID}"
    ovs --may-exist add-port "$OVS_BR" "$port" tag="$V_VID" \
      -- set interface "$port" type=internal
    sudo ip link set "$port" up 2>/dev/null || true
    # Drop any address from a previous mask/octet so a changed vlans.conf does
    # not leave a stale second address on the port.
    sudo ip -4 addr flush dev "$port" 2>/dev/null || true
    # host_octet 0 means "no host leg": the WAN transport VLANs belong to the
    # fake ISPs, and giving the host an address there would misrepresent the
    # segment -- the whole point is that VyOS reaches an ISP, not the host.
    if [ "$V_HOST" != "0" ]; then
      sudo ip addr replace "${V_PREFIX}.${V_HOST}/${V_MASK}" dev "$port"
    fi
  done

  ovs_define_libvirt_net
}

# A libvirt network that hands out OVS ports: one portgroup per VLAN (access)
# plus a trunk portgroup for the router.
ovs_define_libvirt_net() {
  local pg="" ids
  for entry in "${SELECTED[@]}"; do
    IFS=: read -r vid name _p _r <<<"$entry"
    pg+="  <portgroup name='vlan${vid}'>
    <vlan><tag id='${vid}'/></vlan>
  </portgroup>
"
  done

  # Trunk: every VLAN tagged, and by default NO native VLAN (see NATIVE_VLAN at
  # the top of this file for why -- it is the kea #1117 fix, not tidiness).
  # libvirt expresses a native VLAN declaratively via nativeMode='untagged'
  # (see libvirt formatnetwork.html), so it needs no fixing up by hand.
  #
  # The worry that a trunk with no native VLAN has nowhere to put LACPDUs is
  # unfounded, and was tested rather than reasoned about: with vlan_mode=trunk
  # and no tag, `ovs-appctl bond/show` still reports lacp_status: negotiated
  # with both members enabled. LACPDUs are slow-protocol frames handled per
  # member, below the VLAN layer.
  local trunk="  <portgroup name='trunk'>
    <vlan trunk='yes'>
"
  for entry in "${SELECTED[@]}"; do
    IFS=: read -r vid _n _p _r <<<"$entry"
    if [ -n "$NATIVE_VLAN" ] && [ "$vid" = "$NATIVE_VLAN" ]; then
      trunk+="      <tag id='${vid}' nativeMode='untagged'/>
"
    else
      trunk+="      <tag id='${vid}'/>
"
    fi
  done
  trunk+="    </vlan>
  </portgroup>
"

  local xml="<network>
  <name>${OVS_NET}</name>
  <forward mode='bridge'/>
  <bridge name='${OVS_BR}'/>
  <virtualport type='openvswitch'/>
${pg}${trunk}</network>"

  if virsh_q net-info "$OVS_NET" >/dev/null 2>&1; then
    virsh_q net-destroy "$OVS_NET" >/dev/null 2>&1 || true
    virsh_q net-undefine "$OVS_NET" >/dev/null 2>&1 || true
  fi
  echo "$xml" | virsh_q net-define /dev/stdin >/dev/null
  virsh_q net-start "$OVS_NET" >/dev/null
  log "libvirt network $OVS_NET bound to $OVS_BR (access portgroups + trunk)"
}

# Replace the router VM's two individual OVS ports with a single LACP bond.
# libvirt attaches each NIC separately; only ovs-vsctl can bond them, and the
# taps only exist once the VM is running — so this runs post-start.
ovs_bond_router() {
  local vm="$1"
  local taps
  # Two filters, both load-bearing:
  #
  #   $1 ~ /^vnet/ -- domiflist indents its rows, so anchor on the FIELD, not
  #   the line. /^vnet/ silently matches nothing and the bond never gets built.
  #
  #   $3 == OVS_NET -- count only the taps on the sim fabric. The primary also
  #   carries a libvirt-NAT scaffold NIC (see --drop-scaffold in the README), so
  #   an unfiltered count is 3, and this function's "expected 2" guard then
  #   skipped the primary's bond entirely while reporting only a warning.
  taps="$(virsh_q domiflist "$vm" 2>/dev/null \
          | awk -v net="$OVS_NET" '$1 ~ /^vnet/ && $3 == net {print $1}')"
  local count; count="$(echo "$taps" | grep -c .)"
  [ "$count" -eq 2 ] \
    || { warn "router $vm has $count tap(s) on $OVS_NET, expected 2 — skipping bond"; return 1; }

  local t1 t2; t1="$(echo "$taps" | sed -n 1p)"; t2="$(echo "$taps" | sed -n 2p)"
  local want; want="$(vlan_id_list)"

  # Already bonded? Re-runs must still reconcile BOTH the VLAN list and the
  # membership, and each has drawn blood:
  #
  #   VLANs -- adding a VLAN to vlans.conf and finding the bond unchanged is how
  #   a VLAN silently fails to reach a router: interface present, tag missing,
  #   frames dropped by the switch.
  #
  #   MEMBERS -- restarting the VM recreates its taps with NEW names, leaving the
  #   bond holding two interfaces that no longer exist. `list-ports` still shows
  #   the bond, so this early return declared success while the router's real
  #   taps sat in the bridge as two INDEPENDENT ports, each carrying libvirt's
  #   own portgroup VLAN config. That is how labsim-vyos2 ran for weeks with no
  #   LACP at all and a native VLAN nobody had asked for -- and it is invisible
  #   until you change the trunk and only one router follows.
  if ovs list-ports "$OVS_BR" 2>/dev/null | grep -qx "$LAG_NAME"; then
    local members; members="$(ovs-appctl-members)"
    if [ "$members" != "$(printf '%s\n%s' "$t1" "$t2" | sort | paste -sd, -)" ]; then
      warn "bond $LAG_NAME holds stale members [$members], VM has [$t1,$t2] — rebuilding"
      ovs --if-exists del-port "$OVS_BR" "$LAG_NAME"
    else
      # Compare as SETS. vlan_id_list yields config order (1,2,3,9,10,200,51,53)
      # while OVS returns its own sorted order, so a raw string compare reports
      # drift on every run and rewrites a trunk that was already correct.
      local have; have="$(ovs get port "$LAG_NAME" trunks 2>/dev/null | tr -d '[] ')"
      if [ "$(vlan_sorted "$want")" != "$(vlan_sorted "$have")" ]; then
        log "bond $LAG_NAME trunk drift: [$have] -> [$want]; updating"
        ovs set port "$LAG_NAME" trunks="$want"
      else
        log "LACP bond $LAG_NAME already present, trunk correct"
      fi
      ovs_set_native "$LAG_NAME"
      return 0
    fi
  fi

  log "bonding $t1 + $t2 into $LAG_NAME (LACP active, balance-tcp)"
  ovs del-port "$OVS_BR" "$t1" 2>/dev/null || true
  ovs del-port "$OVS_BR" "$t2" 2>/dev/null || true

  # bond_mode=balance-tcp is REQUIRED: OVS defaults a bond to active-backup,
  # which does not speak LACP at all (confirmed on ovs-discuss). It is also the
  # equivalent of VyOS's 802.3ad + layer2+3 hashing.
  #
  # lacp-fallback-ab breaks a genuine deadlock: OVS keeps members disabled
  # until LACP negotiates, while the partner needs carrier before it will send
  # LACPDUs. Falling back to active-backup brings the links up so negotiation
  # can start.
  #
  # The VLAN mode is set inline: libvirt's portgroup config does NOT apply here,
  # because the bond is a port libvirt never created.
  ovs add-bond "$OVS_BR" "$LAG_NAME" "$t1" "$t2" \
      lacp=active bond_mode=balance-tcp trunks="$want" \
    -- set port "$LAG_NAME" other_config:lacp-time=fast \
    -- set port "$LAG_NAME" other_config:lacp-fallback-ab=true
  ovs_set_native "$LAG_NAME"
}

# The bond's current members, sorted and comma-joined, or empty if the bond does
# not resolve at all (which is itself the stale case worth rebuilding for).
ovs-appctl-members() {
  sudo ovs-appctl bond/show "$LAG_NAME" 2>/dev/null \
    | awk '/^member /{gsub(/:/,"",$2); print $2}' | sort | paste -sd, -
}

# Apply NATIVE_VLAN to a trunk port.
#
# `tag` MUST be removed, not merely left alone, when there is no native VLAN.
# Setting vlan_mode=trunk while a stale `tag` remains looks correct in
# `ovs-vsctl list port` -- it prints vlan_mode: trunk right next to tag: 1 --
# but the port keeps egressing that VLAN untagged. Half an hour went into
# "the router is ignoring the trunk change" before the tag was the answer.
ovs_set_native() {
  local port="$1"
  if [ -n "$NATIVE_VLAN" ]; then
    ovs set port "$port" vlan_mode=native-untagged tag="$NATIVE_VLAN"
  else
    ovs set port "$port" vlan_mode=trunk -- clear port "$port" tag
  fi
}

ovs_bond_status() {
  echo "--- ovs bond ---"
  sudo ovs-appctl bond/show "$LAG_NAME" 2>/dev/null | grep -E "bond_mode|lacp_status|^member|may_enable" || echo "(no bond)"
  echo "--- lacp ---"
  sudo ovs-appctl lacp/show "$LAG_NAME" 2>/dev/null | grep -E "status|aggregation key|^member|attached" || true
}

ovs_down() {
  virsh_q net-destroy "$OVS_NET" >/dev/null 2>&1 || true
  virsh_q net-undefine "$OVS_NET" >/dev/null 2>&1 || true
  if command -v ovs-vsctl >/dev/null 2>&1; then
    ovs --if-exists del-br "$OVS_BR" 2>/dev/null || true
  fi
}