lab/labsim/labsim-vlan-leak-test.sh

#!/bin/bash
# Does the router offer an address from the WRONG VLAN's pool?
#
# The fault (ISC Kea #1117, "Mix of physical and virtual interfaces (VLAN) does
# not work"): with `dhcp-socket-type: raw`, a frame tagged for a sub-interface is
# ALSO delivered to the PARENT's AF_PACKET socket. Kea then selects a subnet from
# the parent's own address and answers a second time from the wrong pool. Both
# offers race to the client and the CLIENT decides which one wins -- which is why
# the symptom looks device-dependent and unreproducible.
#
# Production and this sim have the identical shape that triggers it: Management
# is the NATIVE/untagged VLAN on `bond0` and therefore has a subnet on the
# parent, while every other VLAN is a `bond0.<vif>` sub-interface of that same
# bond.
#
# Method: make one DHCP client on a TAGGED VLAN send a DISCOVER, and capture
# simultaneously on the parent and on the sub-interface. The verdict is not
# "did the client get the right address" -- the client picking correctly is
# exactly how this hid for weeks. The verdict is how many OFFERs the SERVER
# emitted and which source addresses they carried.
#
#   ./labsim-vlan-leak-test.sh                 test VLAN 3
#   ./labsim-vlan-leak-test.sh --vlan 9        test another VLAN
#   ./labsim-vlan-leak-test.sh --save before   also write the raw captures to
#                                              vlan-leak-evidence/before/
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"

ROUTER_IP="${ROUTER_IP:-172.31.1.1}"
ROUTER_PW="${ROUTER_PW:-vyos}"
CLIENT_PW="${CLIENT_PW:-labsim}"
VLAN=3
CLIENT=""
SAVE=""

while [ $# -gt 0 ]; do
  case "$1" in
    --vlan)   VLAN="$2"; shift 2 ;;
    --client) CLIENT="$2"; shift 2 ;;
    --save)   SAVE="$2"; shift 2 ;;
    *) echo "usage: $0 [--vlan N] [--client IP] [--save LABEL]" >&2; exit 2 ;;
  esac
done
: "${CLIENT:=172.31.${VLAN}.10}"

log() { printf '\033[36m==>\033[0m %s\n' "$*"; }
die() { printf '\033[31merror:\033[0m %s\n' "$*" >&2; exit 1; }

command -v sshpass >/dev/null || die "sshpass required"

# A silent router is the one verdict worth double-checking before reporting.
#
# Kea can be `is-active` and answering nothing -- it reopens sockets on a retry
# loop, and some configurations (`listen-interface`, notably) leave individual
# VLANs dead while the rest work. Both look identical to a one-shot test: "no
# reply at all". Two opposite and equally wrong conclusions about
# `listen-interface` came out of believing a single negative run, in both
# directions, before a retry made the real pattern obvious.
#
# Kea's fallback UDP socket appearing is NOT a readiness signal -- it is bound
# well before the server actually answers. Checked, and it does not work.
RETRIED="${RETRIED:-0}"

router() {
  timeout 40 sshpass -p "$ROUTER_PW" ssh -o StrictHostKeyChecking=no \
    -o ConnectTimeout=8 "vyos@$ROUTER_IP" "$@" 2>/dev/null
}
# VyOS's login shell is vbash, which returns 255 on anything it does not like --
# in particular a backgrounded job. Feeding the script to `bash -s` on stdin
# sidesteps vbash entirely and is the only reliable way to leave a daemon behind.
router_sh() {
  timeout 40 sshpass -p "$ROUTER_PW" ssh -o StrictHostKeyChecking=no \
    -o ConnectTimeout=8 "vyos@$ROUTER_IP" 'bash -s' 2>/dev/null
}
client() {
  timeout 60 sshpass -p "$CLIENT_PW" ssh -o StrictHostKeyChecking=no \
    -o ConnectTimeout=8 "root@$CLIENT" "$@" 2>/dev/null
}

# Which interfaces to watch. The parent is the whole point: after the fix it
# should carry no DHCP traffic of its own at all.
PARENT="bond0"
VIF="bond0.${VLAN}"

log "router $ROUTER_IP -- capturing on $PARENT and $VIF"
# Kill EVERY tcpdump first, not just ones matching this run's pattern, and count
# only afterwards. Counting `pgrep -f 'tcpdump -i bond0'` while a stray tcpdump
# from an earlier session was still running satisfied the >=2 guard with zero of
# THIS run's captures alive -- and a capture that records nothing reports
# "the router sent no reply at all", which reads as a DHCP outage. That sent me
# chasing a fault in the router that was entirely in the test harness.
started="$(router_sh <<EOF
sudo pkill -x tcpdump >/dev/null 2>&1
sleep 1
sudo rm -f /tmp/leak-*.txt
sudo nohup tcpdump -i $PARENT -e -nn -l 'udp port 67 or udp port 68' > /tmp/leak-parent.txt 2>/dev/null &
sudo nohup tcpdump -i $VIF -e -nn -l 'udp port 67 or udp port 68' > /tmp/leak-vif.txt 2>/dev/null &
sleep 3
pgrep -c -x tcpdump
EOF
)"
[ "${started:-0}" -eq 2 ] || die "capture did not start on the router (got ${started:-0}, expected exactly 2)"

# -s /bin/true: ask, observe the answer, apply nothing. The client's existing
# static address is left alone, so this is safe to run against a live sim VM.
log "client $CLIENT -- sending DISCOVER on VLAN $VLAN"
client_out="$(client "udhcpc -n -q -f -i eth0 -s /bin/true -t 3 -T 3 2>&1")"
[ -n "$client_out" ] || die "no response from client $CLIENT"

sleep 2
router "sudo pkill -x tcpdump" >/dev/null
parent="$(router 'sudo cat /tmp/leak-parent.txt')"
vif="$(router 'sudo cat /tmp/leak-vif.txt')"

echo
echo "--- client ---"
echo "$client_out" | sed 's/^/  /'
echo
echo "--- $PARENT (parent) ---"
echo "${parent:-  (nothing)}" | sed 's/^/  /'
echo
echo "--- $VIF (sub-interface) ---"
echo "${vif:-  (nothing)}" | sed 's/^/  /'
echo

if [ -n "$SAVE" ]; then
  d="$SCRIPT_DIR/vlan-leak-evidence/$SAVE"
  mkdir -p "$d"
  printf '%s\n' "$client_out" > "$d/client.txt"
  printf '%s\n' "$parent"     > "$d/capture-parent.txt"
  printf '%s\n' "$vif"        > "$d/capture-vif.txt"
  router '/opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands' \
    | grep -E 'interfaces bonding|vrrp group' > "$d/router-config.txt"
  log "evidence saved to vlan-leak-evidence/$SAVE/"
fi

# --- verdict ---------------------------------------------------------------
# Every BOOTP Reply seen anywhere, reduced to its source address. A reply whose
# source is not this VLAN's router leg is an offer from the wrong subnet.
replies="$(printf '%s\n%s\n' "$parent" "$vif" \
  | grep -o '[0-9.]*\.67 > [0-9.]*\.68' | awk '{print $1}' | sed 's/\.67$//' \
  | sort -u)"
want_prefix="172.31.${VLAN}."

echo "=== verdict ==="
if [ -z "$replies" ]; then
  if [ "$RETRIED" -eq 0 ]; then
    log "no reply -- retrying once in 20s before calling DHCP down"
    sleep 20; RETRIED=1 exec "$0" --vlan "$VLAN" --client "$CLIENT" ${SAVE:+--save "$SAVE"}
  fi
  echo "INCONCLUSIVE: the router sent no reply at all, twice -- DHCP is down on VLAN $VLAN"
  exit 2
fi

bad=0
while read -r src; do
  [ -z "$src" ] && continue
  case "$src" in
    "$want_prefix"*) printf '  ok    offer from %s (this VLAN)\n' "$src" ;;
    *)               printf '  LEAK  offer from %s (WRONG subnet)\n' "$src"; bad=1 ;;
  esac
done <<<"$replies"

# The parent carrying any DHCP of its own is the mechanism, not just a symptom:
# it means the parent still has a subnet kea can match a tagged frame against.
if printf '%s' "$parent" | grep -q 'ethertype IPv4' \
   && printf '%s' "$parent" | grep -v 'vlan ' | grep -q '\.67 > '; then
  echo "  note  $PARENT emitted an UNTAGGED reply -- the parent still serves a subnet"
fi

echo
if [ "$bad" -eq 0 ]; then
  echo "PASS: only this VLAN's pool answered."
  exit 0
fi
echo "FAIL: the router answered from another VLAN's pool (kea #1117)."
exit 1