feat(labsim): a real Kubernetes cluster for rehearsing Cilium <-> VyOS BGP

BGP is about to be added to a network that currently works, where a bad
advertisement blackholes the house. That needs somewhere to fail first.

Three Debian nodes (4 GB / 2 vCPU) on the OVS `vlan2` access ports running k3s
with flannel disabled, so Cilium is the CNI under test. Three rather than two
because ECMP is only meaningfully tested if a node can be drained and more than
one path survives.

VMs rather than k3d: the thing under test is eBGP between Cilium and VyOS across
the switch fabric, nodes peering with the router's bond0.2 leg, directly
connected. k3d would put the nodes on a container bridge -- a different L2 path,
proving something else. (It also wants Docker; this host has podman.) The
existing micro VMs are Alpine with 256 MB and 1 vCPU, which is not close to
enough.

Debian rather than the sim's Alpine base: glibc, a stock kernel that Cilium's
eBPF probes are tested against, and cloud-init that actually applies
network-config -- the Alpine base notably does not.

One trap worth recording. An earlier draft called `selected_vlans "$K8S_VLAN"`
before `ovs_up`, and since `ovs_up` re-defines the libvirt network from
SELECTED, that silently deleted the portgroups for every other VLAN. Running
VMs kept working -- their taps were already attached -- so nothing complained
until the ISP VMs needed vlan51 and vlan53 and could not be attached. It now
selects every VLAN.

The generated kubeconfig is gitignored: it carries cluster-admin credentials and
is one `git add -A` away from being committed. Regenerate with
`k8s-up.sh --kubeconfig`.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DMVzWZgiKW2wquf5z8S1yH

labsim/.gitignore

@@ -2,3 +2,7 @@
 *.log
 labsim_matrix_lib.py
 __pycache__/
+
+# Cluster-admin credentials for the rehearsal cluster, written by
+# `k8s-up.sh --kubeconfig`. Regenerate it rather than commit it.
+*.kubeconfig

labsim/k8s-up.sh

@@ -0,0 +1,252 @@
+#!/bin/bash
+# A real Kubernetes cluster inside labsim, on the OVS fabric, for rehearsing
+# Cilium <-> VyOS BGP before it goes near the production routers.
+#
+# Why VMs and not k3d: the thing under test is eBGP between Cilium and VyOS
+# across the switch fabric — nodes on VLAN 2, peering with the router's bond0.2
+# leg, directly connected. k3d would put the nodes on a container bridge, which
+# is a different L2 path and would prove something else. (It also needs Docker;
+# this host has podman.)
+#
+# Why not the existing micro VMs: they are Alpine with 256 MB and 1 vCPU. k3s
+# plus Cilium needs an order of magnitude more, and a glibc distro with a stock
+# kernel that Cilium's eBPF probes are actually tested against.
+#
+# Three nodes, not two: ECMP is only meaningfully tested if a node can be
+# drained and MORE THAN ONE path survives.
+#
+# Layout (mirrors production's shape, not its addresses):
+#   labsim-k8s1   172.31.2.11   k3s server
+#   labsim-k8s2   172.31.2.12   agent
+#   labsim-k8s3   172.31.2.13   agent
+#   gateway       172.31.2.1    the VRRP VIP of the router pair under test
+#   BGP peers     172.31.2.252 / .253   the routers' real per-box addresses
+#
+# Idempotent: re-running only creates what is missing.
+#
+# Usage:
+#   ./k8s-up.sh              create/start the cluster
+#   ./k8s-up.sh --kubeconfig fetch kubeconfig to ./labsim-k8s.kubeconfig
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+source "$SCRIPT_DIR/lib.sh"
+source "$SCRIPT_DIR/ovs.sh"
+
+# --- knobs ----------------------------------------------------------------
+K8S_VLAN="${K8S_VLAN:-2}"
+K8S_PREFIX="${K8S_PREFIX:-172.31.2}"
+K8S_NODES="${K8S_NODES:-3}"
+K8S_FIRST_OCTET="${K8S_FIRST_OCTET:-11}"
+K8S_MEM="${K8S_MEM:-4096}"          # MB — k3s + cilium + a workload
+K8S_CPUS="${K8S_CPUS:-2}"
+K8S_DISK_GB="${K8S_DISK_GB:-12}"
+K8S_TOKEN="${K8S_TOKEN:-labsim-k3s-token}"
+
+# Debian rather than Alpine: glibc, a stock kernel, and cloud-init that applies
+# network-config properly (the Alpine base in this sim notably does not).
+DEB_URL="${DEB_URL:-https://cloud.debian.org/images/cloud/trixie/latest/debian-13-genericcloud-amd64.qcow2}"
+DEB_BASE="${DEB_BASE:-$IMG_DIR/debian-13-genericcloud-amd64.qcow2}"
+
+# Same version production runs, so CRD shapes and chart flags transfer exactly.
+CILIUM_VERSION="${CILIUM_VERSION:-1.19.1}"
+
+node_name() { echo "labsim-k8s$1"; }
+node_ip()   { echo "${K8S_PREFIX}.$((K8S_FIRST_OCTET + $1 - 1))"; }
+
+# --- base image -----------------------------------------------------------
+ensure_base_image() {
+  if [ -f "$DEB_BASE" ]; then
+    log "base image present: $(basename "$DEB_BASE")"
+    return
+  fi
+  log "fetching Debian cloud image (~330 MB) -> $DEB_BASE"
+  sudo mkdir -p "$IMG_DIR"
+  # .tmp + mv so an interrupted download never leaves a half image that later
+  # runs treat as valid.
+  sudo curl -fsSL --retry 3 -o "${DEB_BASE}.tmp" "$DEB_URL" \
+    || die "could not fetch $DEB_URL"
+  sudo mv "${DEB_BASE}.tmp" "$DEB_BASE"
+  log "base image ready"
+}
+
+# --- cloud-init -----------------------------------------------------------
+# The server node writes the join token; agents wait for the API to answer
+# before joining, because cloud-init ordering across VMs is not guaranteed and
+# a failed join leaves an agent that never retries.
+build_k8s_seed() {
+  local iso="$1" vm="$2" ip="$3" role="$4" server_ip="$5" pubkey="$6"
+  local tmp; tmp="$(mktemp -d)"
+
+  cat > "$tmp/meta-data" <<EOF
+instance-id: $vm
+local-hostname: $vm
+EOF
+
+  cat > "$tmp/network-config" <<EOF
+version: 2
+ethernets:
+  enp1s0:
+    match:
+      name: "en*"
+    addresses: [$ip/24]
+    routes:
+      - to: default
+        via: ${K8S_PREFIX}.1
+    nameservers:
+      addresses: [8.8.8.8, 1.1.1.1]
+EOF
+
+  local k3s_exec
+  if [ "$role" = "server" ]; then
+    # flannel/servicelb/traefik off: Cilium is the CNI under test, and k3s's
+    # own ServiceLB would fight Cilium for LoadBalancer addresses.
+    k3s_exec="server --flannel-backend=none --disable-network-policy --disable=servicelb --disable=traefik --node-ip=$ip --tls-san=$ip --cluster-init"
+  else
+    k3s_exec="agent --server https://${server_ip}:6443 --node-ip=$ip"
+  fi
+
+  cat > "$tmp/user-data" <<EOF
+#cloud-config
+hostname: $vm
+fqdn: $vm
+users:
+  - name: debian
+    groups: [sudo]
+    shell: /bin/bash
+    sudo: ["ALL=(ALL) NOPASSWD:ALL"]
+    lock_passwd: false
+    plain_text_passwd: labsim
+    ssh_authorized_keys:
+      - $pubkey
+ssh_pwauth: true
+disable_root: false
+ssh_authorized_keys:
+  - $pubkey
+
+package_update: true
+packages: [curl, jq, iproute2, tcpdump, bird2]
+
+write_files:
+  # Cilium replaces kube-proxy and needs these; Debian cloud images ship
+  # neither loaded nor persisted.
+  - path: /etc/modules-load.d/cilium.conf
+    content: |
+      br_netfilter
+      overlay
+  - path: /etc/sysctl.d/99-k8s.conf
+    content: |
+      net.ipv4.ip_forward = 1
+      net.bridge.bridge-nf-call-iptables = 1
+
+runcmd:
+  - [ modprobe, br_netfilter ]
+  - [ modprobe, overlay ]
+  - [ sysctl, --system ]
+  - |
+    # Wait for the server's API before an agent tries to join. Without this the
+    # agent fails once and the unit backs off for minutes.
+    if [ "$role" != "server" ]; then
+      for i in \$(seq 1 60); do
+        curl -sk --max-time 3 https://${server_ip}:6443/ping >/dev/null 2>&1 && break
+        sleep 5
+      done
+    fi
+  - |
+    curl -sfL https://get.k3s.io | \
+      INSTALL_K3S_EXEC="$k3s_exec" \
+      K3S_TOKEN="$K8S_TOKEN" \
+      sh -
+EOF
+
+  sudo mkdir -p "$(dirname "$iso")"
+  sudo genisoimage -quiet -output "$iso" -volid cidata -joliet -rock \
+    "$tmp/user-data" "$tmp/meta-data" "$tmp/network-config"
+  rm -rf "$tmp"
+}
+
+# --- VM creation ----------------------------------------------------------
+create_node() {
+  local n="$1" pubkey="$2"
+  local vm; vm="$(node_name "$n")"
+  local ip; ip="$(node_ip "$n")"
+  local role="agent"; [ "$n" -eq 1 ] && role="server"
+  local server_ip; server_ip="$(node_ip 1)"
+
+  if virsh_q dominfo "$vm" >/dev/null 2>&1; then
+    local state; state="$(virsh_q domstate "$vm" 2>/dev/null | head -1 | tr -d '\n')"
+    if [ "$state" = "running" ]; then
+      log "$vm already running ($ip, $role)"
+    else
+      log "$vm exists but is $state — starting"
+      virsh_q start "$vm" >/dev/null
+    fi
+    return
+  fi
+
+  local disk="$IMG_DIR/${vm}.qcow2"
+  local seed="$IMG_DIR/${vm}-seed.iso"
+
+  log "creating $vm ($ip, $role, ${K8S_MEM}MB/${K8S_CPUS}cpu)"
+  sudo qemu-img create -q -f qcow2 -F qcow2 -b "$DEB_BASE" "$disk" "${K8S_DISK_GB}G" >/dev/null
+  build_k8s_seed "$seed" "$vm" "$ip" "$role" "$server_ip" "$pubkey"
+
+  # Access port on the k8s VLAN — same broadcast domain as the routers'
+  # bond0.2 leg, so BGP peering is directly connected exactly as in production.
+  sudo virt-install --connect "$LIBVIRT_URI" --name "$vm" \
+    --memory "$K8S_MEM" --vcpus "$K8S_CPUS" \
+    --disk "path=$disk,format=qcow2,bus=virtio" \
+    --disk "path=$seed,device=cdrom" \
+    --network "network=$OVS_NET,portgroup=vlan${K8S_VLAN},model=virtio" \
+    --os-variant debian12 \
+    --graphics none --noautoconsole --import >/dev/null
+}
+
+fetch_kubeconfig() {
+  local server_ip; server_ip="$(node_ip 1)"
+  local out="$SCRIPT_DIR/labsim-k8s.kubeconfig"
+  log "fetching kubeconfig from $server_ip"
+  ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 \
+      "debian@${server_ip}" "sudo cat /etc/rancher/k3s/k3s.yaml" \
+    | sed "s|127.0.0.1|${server_ip}|" > "$out"
+  chmod 600 "$out"
+  log "wrote $out"
+  log "use: KUBECONFIG=$out kubectl get nodes"
+}
+
+main() {
+  if [ "${1:-}" = "--kubeconfig" ]; then
+    fetch_kubeconfig
+    return
+  fi
+
+  require_tools
+  command -v genisoimage >/dev/null || die "genisoimage missing (dnf install genisoimage)"
+
+  local pubkey; pubkey="$(find_ssh_pubkey)"
+  log "using SSH key: ${pubkey%% *} ...${pubkey##* }"
+
+  ensure_base_image
+
+  # Select EVERY VLAN, not just the k8s one. ovs_up re-defines the libvirt
+  # network from SELECTED, so narrowing it here silently drops the portgroups
+  # for every other VLAN -- running VMs keep working (their taps are already
+  # attached) and nothing complains until the next VM cannot be attached.
+  # Observed: this deleted vlan1/3/9/10/200/51/53 and only surfaced when the
+  # ISP VMs needed vlan51 and vlan53.
+  selected_vlans
+  log "ensuring OVS fabric (all VLANs, so no portgroup is dropped)"
+  ovs_up
+
+  for n in $(seq 1 "$K8S_NODES"); do
+    create_node "$n" "$pubkey"
+  done
+
+  echo
+  log "nodes created. k3s installs on first boot (a few minutes)."
+  log "watch:   ssh debian@$(node_ip 1) 'sudo systemctl status k3s'"
+  log "then:    $0 --kubeconfig"
+  log "then install Cilium $CILIUM_VERSION and the BGP resources (see README)."
+}
+
+main "$@"