diff --git a/labsim/.gitignore b/labsim/.gitignore index 73e9e69..730b6d8 100644 --- a/labsim/.gitignore +++ b/labsim/.gitignore @@ -2,3 +2,7 @@ *.log labsim_matrix_lib.py __pycache__/ + +# Cluster-admin credentials for the rehearsal cluster, written by +# `k8s-up.sh --kubeconfig`. Regenerate it rather than commit it. +*.kubeconfig diff --git a/labsim/k8s-up.sh b/labsim/k8s-up.sh new file mode 100755 index 0000000..ce8cb35 --- /dev/null +++ b/labsim/k8s-up.sh @@ -0,0 +1,252 @@ +#!/bin/bash +# A real Kubernetes cluster inside labsim, on the OVS fabric, for rehearsing +# Cilium <-> VyOS BGP before it goes near the production routers. +# +# Why VMs and not k3d: the thing under test is eBGP between Cilium and VyOS +# across the switch fabric — nodes on VLAN 2, peering with the router's bond0.2 +# leg, directly connected. k3d would put the nodes on a container bridge, which +# is a different L2 path and would prove something else. (It also needs Docker; +# this host has podman.) +# +# Why not the existing micro VMs: they are Alpine with 256 MB and 1 vCPU. k3s +# plus Cilium needs an order of magnitude more, and a glibc distro with a stock +# kernel that Cilium's eBPF probes are actually tested against. +# +# Three nodes, not two: ECMP is only meaningfully tested if a node can be +# drained and MORE THAN ONE path survives. +# +# Layout (mirrors production's shape, not its addresses): +# labsim-k8s1 172.31.2.11 k3s server +# labsim-k8s2 172.31.2.12 agent +# labsim-k8s3 172.31.2.13 agent +# gateway 172.31.2.1 the VRRP VIP of the router pair under test +# BGP peers 172.31.2.252 / .253 the routers' real per-box addresses +# +# Idempotent: re-running only creates what is missing. +# +# Usage: +# ./k8s-up.sh create/start the cluster +# ./k8s-up.sh --kubeconfig fetch kubeconfig to ./labsim-k8s.kubeconfig +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/lib.sh" +source "$SCRIPT_DIR/ovs.sh" + +# --- knobs ---------------------------------------------------------------- +K8S_VLAN="${K8S_VLAN:-2}" +K8S_PREFIX="${K8S_PREFIX:-172.31.2}" +K8S_NODES="${K8S_NODES:-3}" +K8S_FIRST_OCTET="${K8S_FIRST_OCTET:-11}" +K8S_MEM="${K8S_MEM:-4096}" # MB — k3s + cilium + a workload +K8S_CPUS="${K8S_CPUS:-2}" +K8S_DISK_GB="${K8S_DISK_GB:-12}" +K8S_TOKEN="${K8S_TOKEN:-labsim-k3s-token}" + +# Debian rather than Alpine: glibc, a stock kernel, and cloud-init that applies +# network-config properly (the Alpine base in this sim notably does not). +DEB_URL="${DEB_URL:-https://cloud.debian.org/images/cloud/trixie/latest/debian-13-genericcloud-amd64.qcow2}" +DEB_BASE="${DEB_BASE:-$IMG_DIR/debian-13-genericcloud-amd64.qcow2}" + +# Same version production runs, so CRD shapes and chart flags transfer exactly. +CILIUM_VERSION="${CILIUM_VERSION:-1.19.1}" + +node_name() { echo "labsim-k8s$1"; } +node_ip() { echo "${K8S_PREFIX}.$((K8S_FIRST_OCTET + $1 - 1))"; } + +# --- base image ----------------------------------------------------------- +ensure_base_image() { + if [ -f "$DEB_BASE" ]; then + log "base image present: $(basename "$DEB_BASE")" + return + fi + log "fetching Debian cloud image (~330 MB) -> $DEB_BASE" + sudo mkdir -p "$IMG_DIR" + # .tmp + mv so an interrupted download never leaves a half image that later + # runs treat as valid. + sudo curl -fsSL --retry 3 -o "${DEB_BASE}.tmp" "$DEB_URL" \ + || die "could not fetch $DEB_URL" + sudo mv "${DEB_BASE}.tmp" "$DEB_BASE" + log "base image ready" +} + +# --- cloud-init ----------------------------------------------------------- +# The server node writes the join token; agents wait for the API to answer +# before joining, because cloud-init ordering across VMs is not guaranteed and +# a failed join leaves an agent that never retries. +build_k8s_seed() { + local iso="$1" vm="$2" ip="$3" role="$4" server_ip="$5" pubkey="$6" + local tmp; tmp="$(mktemp -d)" + + cat > "$tmp/meta-data" < "$tmp/network-config" < "$tmp/user-data" </dev/null 2>&1 && break + sleep 5 + done + fi + - | + curl -sfL https://get.k3s.io | \ + INSTALL_K3S_EXEC="$k3s_exec" \ + K3S_TOKEN="$K8S_TOKEN" \ + sh - +EOF + + sudo mkdir -p "$(dirname "$iso")" + sudo genisoimage -quiet -output "$iso" -volid cidata -joliet -rock \ + "$tmp/user-data" "$tmp/meta-data" "$tmp/network-config" + rm -rf "$tmp" +} + +# --- VM creation ---------------------------------------------------------- +create_node() { + local n="$1" pubkey="$2" + local vm; vm="$(node_name "$n")" + local ip; ip="$(node_ip "$n")" + local role="agent"; [ "$n" -eq 1 ] && role="server" + local server_ip; server_ip="$(node_ip 1)" + + if virsh_q dominfo "$vm" >/dev/null 2>&1; then + local state; state="$(virsh_q domstate "$vm" 2>/dev/null | head -1 | tr -d '\n')" + if [ "$state" = "running" ]; then + log "$vm already running ($ip, $role)" + else + log "$vm exists but is $state — starting" + virsh_q start "$vm" >/dev/null + fi + return + fi + + local disk="$IMG_DIR/${vm}.qcow2" + local seed="$IMG_DIR/${vm}-seed.iso" + + log "creating $vm ($ip, $role, ${K8S_MEM}MB/${K8S_CPUS}cpu)" + sudo qemu-img create -q -f qcow2 -F qcow2 -b "$DEB_BASE" "$disk" "${K8S_DISK_GB}G" >/dev/null + build_k8s_seed "$seed" "$vm" "$ip" "$role" "$server_ip" "$pubkey" + + # Access port on the k8s VLAN — same broadcast domain as the routers' + # bond0.2 leg, so BGP peering is directly connected exactly as in production. + sudo virt-install --connect "$LIBVIRT_URI" --name "$vm" \ + --memory "$K8S_MEM" --vcpus "$K8S_CPUS" \ + --disk "path=$disk,format=qcow2,bus=virtio" \ + --disk "path=$seed,device=cdrom" \ + --network "network=$OVS_NET,portgroup=vlan${K8S_VLAN},model=virtio" \ + --os-variant debian12 \ + --graphics none --noautoconsole --import >/dev/null +} + +fetch_kubeconfig() { + local server_ip; server_ip="$(node_ip 1)" + local out="$SCRIPT_DIR/labsim-k8s.kubeconfig" + log "fetching kubeconfig from $server_ip" + ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 \ + "debian@${server_ip}" "sudo cat /etc/rancher/k3s/k3s.yaml" \ + | sed "s|127.0.0.1|${server_ip}|" > "$out" + chmod 600 "$out" + log "wrote $out" + log "use: KUBECONFIG=$out kubectl get nodes" +} + +main() { + if [ "${1:-}" = "--kubeconfig" ]; then + fetch_kubeconfig + return + fi + + require_tools + command -v genisoimage >/dev/null || die "genisoimage missing (dnf install genisoimage)" + + local pubkey; pubkey="$(find_ssh_pubkey)" + log "using SSH key: ${pubkey%% *} ...${pubkey##* }" + + ensure_base_image + + # Select EVERY VLAN, not just the k8s one. ovs_up re-defines the libvirt + # network from SELECTED, so narrowing it here silently drops the portgroups + # for every other VLAN -- running VMs keep working (their taps are already + # attached) and nothing complains until the next VM cannot be attached. + # Observed: this deleted vlan1/3/9/10/200/51/53 and only surfaced when the + # ISP VMs needed vlan51 and vlan53. + selected_vlans + log "ensuring OVS fabric (all VLANs, so no portgroup is dropped)" + ovs_up + + for n in $(seq 1 "$K8S_NODES"); do + create_node "$n" "$pubkey" + done + + echo + log "nodes created. k3s installs on first boot (a few minutes)." + log "watch: ssh debian@$(node_ip 1) 'sudo systemctl status k3s'" + log "then: $0 --kubeconfig" + log "then install Cilium $CILIUM_VERSION and the BGP resources (see README)." +} + +main "$@"