#!/bin/bash # Open vSwitch fabric for labsim — the "switch" the whole sim hangs off. # # Why OVS and not a Linux bridge: a Linux bridge cannot do LACP at all, and its # VLAN support is awkward to drive from libvirt. OVS gives real 802.1Q access # and trunk ports plus real LACP bonds, so a router VM can run the SAME bond0 + # vif config as the production VP2440s instead of an approximation. # # Layout: # ovs-labsim the switch # ├─ vm ports access ports, tag= (micro VM per VLAN) # ├─ hostv internal ports, tag= (host leg, for SSH) # └─ lag-vyos LACP bond, trunk of all VLANs (router under test) # shellcheck disable=SC2034 OVS_BR="${OVS_BR:-ovs-labsim}" OVS_NET="${OVS_NET:-labsim-ovs}" # libvirt network wrapping the bridge LAG_NAME="${LAG_NAME:-lag-vyos}" # Native (untagged) VLAN on the trunks to the routers. Empty means NONE: every # VLAN, Management included, is tagged. # # This is not a style choice. A native VLAN is what puts a subnet on the bond # PARENT (`bond0`) while every other VLAN lives on a sub-interface of it. With # `dhcp-socket-type: raw`, kea then receives each tagged frame TWICE -- once on # the sub-interface and once on the parent -- and answers from the parent's pool # as well, so a client on VLAN 3 is offered a Management address and picks # whichever reply arrives first (ISC Kea #1117). # # Set LABSIM_NATIVE_VLAN=1 to restore the old shape and reproduce the bug: # LABSIM_NATIVE_VLAN=1 ./router-up.sh && ./labsim-vlan-leak-test.sh # FAIL # ./router-up.sh && ./labsim-vlan-leak-test.sh # PASS NATIVE_VLAN="${LABSIM_NATIVE_VLAN:-}" ovs() { sudo ovs-vsctl "$@"; } ovs_require() { command -v ovs-vsctl >/dev/null 2>&1 || die "openvswitch not installed (dnf install openvswitch)" systemctl is-active --quiet openvswitch || sudo systemctl start openvswitch \ || die "could not start openvswitch" } # A comma-separated VLAN list, numerically sorted, for comparing two lists that # came from different places and need not agree on order. vlan_sorted() { echo "$1" | tr ',' '\n' | grep -v '^$' | sort -n | paste -sd, -; } # All VLAN ids from the config, comma separated — used for trunk ports. vlan_id_list() { local ids=() for entry in "${SELECTED[@]}"; do ids+=("${entry%%:*}"); done (IFS=,; echo "${ids[*]}") } ovs_up() { ovs_require ovs --may-exist add-br "$OVS_BR" # Host leg per VLAN: an OVS internal port carrying that VLAN's tag, given the # .2 address. This is how you SSH to the VMs. It is deliberately NOT their # default route (.1 is), so inter-VLAN tests exercise the router, not the # host's routing table. for entry in "${SELECTED[@]}"; do parse_vlan_entry "$entry" local port="hostv${V_VID}" ovs --may-exist add-port "$OVS_BR" "$port" tag="$V_VID" \ -- set interface "$port" type=internal sudo ip link set "$port" up 2>/dev/null || true # Drop any address from a previous mask/octet so a changed vlans.conf does # not leave a stale second address on the port. sudo ip -4 addr flush dev "$port" 2>/dev/null || true # host_octet 0 means "no host leg": the WAN transport VLANs belong to the # fake ISPs, and giving the host an address there would misrepresent the # segment -- the whole point is that VyOS reaches an ISP, not the host. if [ "$V_HOST" != "0" ]; then sudo ip addr replace "${V_PREFIX}.${V_HOST}/${V_MASK}" dev "$port" fi done ovs_define_libvirt_net } # A libvirt network that hands out OVS ports: one portgroup per VLAN (access) # plus a trunk portgroup for the router. ovs_define_libvirt_net() { local pg="" ids for entry in "${SELECTED[@]}"; do IFS=: read -r vid name _p _r <<<"$entry" pg+=" " done # Trunk: every VLAN tagged, and by default NO native VLAN (see NATIVE_VLAN at # the top of this file for why -- it is the kea #1117 fix, not tidiness). # libvirt expresses a native VLAN declaratively via nativeMode='untagged' # (see libvirt formatnetwork.html), so it needs no fixing up by hand. # # The worry that a trunk with no native VLAN has nowhere to put LACPDUs is # unfounded, and was tested rather than reasoned about: with vlan_mode=trunk # and no tag, `ovs-appctl bond/show` still reports lacp_status: negotiated # with both members enabled. LACPDUs are slow-protocol frames handled per # member, below the VLAN layer. local trunk=" " for entry in "${SELECTED[@]}"; do IFS=: read -r vid _n _p _r <<<"$entry" if [ -n "$NATIVE_VLAN" ] && [ "$vid" = "$NATIVE_VLAN" ]; then trunk+=" " else trunk+=" " fi done trunk+=" " local xml=" ${OVS_NET} ${pg}${trunk}" if virsh_q net-info "$OVS_NET" >/dev/null 2>&1; then virsh_q net-destroy "$OVS_NET" >/dev/null 2>&1 || true virsh_q net-undefine "$OVS_NET" >/dev/null 2>&1 || true fi echo "$xml" | virsh_q net-define /dev/stdin >/dev/null virsh_q net-start "$OVS_NET" >/dev/null log "libvirt network $OVS_NET bound to $OVS_BR (access portgroups + trunk)" } # Replace the router VM's two individual OVS ports with a single LACP bond. # libvirt attaches each NIC separately; only ovs-vsctl can bond them, and the # taps only exist once the VM is running — so this runs post-start. ovs_bond_router() { local vm="$1" local taps # Two filters, both load-bearing: # # $1 ~ /^vnet/ -- domiflist indents its rows, so anchor on the FIELD, not # the line. /^vnet/ silently matches nothing and the bond never gets built. # # $3 == OVS_NET -- count only the taps on the sim fabric. The primary also # carries a libvirt-NAT scaffold NIC (see --drop-scaffold in the README), so # an unfiltered count is 3, and this function's "expected 2" guard then # skipped the primary's bond entirely while reporting only a warning. taps="$(virsh_q domiflist "$vm" 2>/dev/null \ | awk -v net="$OVS_NET" '$1 ~ /^vnet/ && $3 == net {print $1}')" local count; count="$(echo "$taps" | grep -c .)" [ "$count" -eq 2 ] \ || { warn "router $vm has $count tap(s) on $OVS_NET, expected 2 — skipping bond"; return 1; } local t1 t2; t1="$(echo "$taps" | sed -n 1p)"; t2="$(echo "$taps" | sed -n 2p)" local want; want="$(vlan_id_list)" # Already bonded? Re-runs must still reconcile BOTH the VLAN list and the # membership, and each has drawn blood: # # VLANs -- adding a VLAN to vlans.conf and finding the bond unchanged is how # a VLAN silently fails to reach a router: interface present, tag missing, # frames dropped by the switch. # # MEMBERS -- restarting the VM recreates its taps with NEW names, leaving the # bond holding two interfaces that no longer exist. `list-ports` still shows # the bond, so this early return declared success while the router's real # taps sat in the bridge as two INDEPENDENT ports, each carrying libvirt's # own portgroup VLAN config. That is how labsim-vyos2 ran for weeks with no # LACP at all and a native VLAN nobody had asked for -- and it is invisible # until you change the trunk and only one router follows. if ovs list-ports "$OVS_BR" 2>/dev/null | grep -qx "$LAG_NAME"; then local members; members="$(ovs-appctl-members)" if [ "$members" != "$(printf '%s\n%s' "$t1" "$t2" | sort | paste -sd, -)" ]; then warn "bond $LAG_NAME holds stale members [$members], VM has [$t1,$t2] — rebuilding" ovs --if-exists del-port "$OVS_BR" "$LAG_NAME" else # Compare as SETS. vlan_id_list yields config order (1,2,3,9,10,200,51,53) # while OVS returns its own sorted order, so a raw string compare reports # drift on every run and rewrites a trunk that was already correct. local have; have="$(ovs get port "$LAG_NAME" trunks 2>/dev/null | tr -d '[] ')" if [ "$(vlan_sorted "$want")" != "$(vlan_sorted "$have")" ]; then log "bond $LAG_NAME trunk drift: [$have] -> [$want]; updating" ovs set port "$LAG_NAME" trunks="$want" else log "LACP bond $LAG_NAME already present, trunk correct" fi ovs_set_native "$LAG_NAME" return 0 fi fi log "bonding $t1 + $t2 into $LAG_NAME (LACP active, balance-tcp)" ovs del-port "$OVS_BR" "$t1" 2>/dev/null || true ovs del-port "$OVS_BR" "$t2" 2>/dev/null || true # bond_mode=balance-tcp is REQUIRED: OVS defaults a bond to active-backup, # which does not speak LACP at all (confirmed on ovs-discuss). It is also the # equivalent of VyOS's 802.3ad + layer2+3 hashing. # # lacp-fallback-ab breaks a genuine deadlock: OVS keeps members disabled # until LACP negotiates, while the partner needs carrier before it will send # LACPDUs. Falling back to active-backup brings the links up so negotiation # can start. # # The VLAN mode is set inline: libvirt's portgroup config does NOT apply here, # because the bond is a port libvirt never created. ovs add-bond "$OVS_BR" "$LAG_NAME" "$t1" "$t2" \ lacp=active bond_mode=balance-tcp trunks="$want" \ -- set port "$LAG_NAME" other_config:lacp-time=fast \ -- set port "$LAG_NAME" other_config:lacp-fallback-ab=true ovs_set_native "$LAG_NAME" } # The bond's current members, sorted and comma-joined, or empty if the bond does # not resolve at all (which is itself the stale case worth rebuilding for). ovs-appctl-members() { sudo ovs-appctl bond/show "$LAG_NAME" 2>/dev/null \ | awk '/^member /{gsub(/:/,"",$2); print $2}' | sort | paste -sd, - } # Apply NATIVE_VLAN to a trunk port. # # `tag` MUST be removed, not merely left alone, when there is no native VLAN. # Setting vlan_mode=trunk while a stale `tag` remains looks correct in # `ovs-vsctl list port` -- it prints vlan_mode: trunk right next to tag: 1 -- # but the port keeps egressing that VLAN untagged. Half an hour went into # "the router is ignoring the trunk change" before the tag was the answer. ovs_set_native() { local port="$1" if [ -n "$NATIVE_VLAN" ]; then ovs set port "$port" vlan_mode=native-untagged tag="$NATIVE_VLAN" else ovs set port "$port" vlan_mode=trunk -- clear port "$port" tag fi } ovs_bond_status() { echo "--- ovs bond ---" sudo ovs-appctl bond/show "$LAG_NAME" 2>/dev/null | grep -E "bond_mode|lacp_status|^member|may_enable" || echo "(no bond)" echo "--- lacp ---" sudo ovs-appctl lacp/show "$LAG_NAME" 2>/dev/null | grep -E "status|aggregation key|^member|attached" || true } ovs_down() { virsh_q net-destroy "$OVS_NET" >/dev/null 2>&1 || true virsh_q net-undefine "$OVS_NET" >/dev/null 2>&1 || true if command -v ovs-vsctl >/dev/null 2>&1; then ovs --if-exists del-br "$OVS_BR" 2>/dev/null || true fi }