lab/labsim/dualstack-lab.sh

#!/usr/bin/env bash
# Differential study: what ACTUALLY differs between a k3s cluster born
# dual-stack and one converted in place?
#
# k3s says dual-stack "cannot be enabled on an existing cluster". The stated
# reason is narrow -- nodes get Pod CIDRs only at join and the Kubernetes IPAM
# controller will not hand out a new IPv6 CIDR later -- and it does not obviously
# apply to a cluster where Cilium owns IPAM. Rather than argue from docs, build
# both shapes and diff them.
#
#   ./dualstack-lab.sh up v4         single-node k3s, IPv4 only     (.21)
#   ./dualstack-lab.sh up dual       single-node k3s, dual-stack    (.22)
#   ./dualstack-lab.sh pristine v4   reflink copy of v4's disk, so the upgrade
#                                    attempt can be rolled back and retried
#   ./dualstack-lab.sh restore v4    put that copy back
#   ./dualstack-lab.sh collect <n>   normalized state dump -> evidence/<n>/
#   ./dualstack-lab.sh compare a b   semantic diff of two collections
#   ./dualstack-lab.sh virtdiff a b  whole-filesystem diff, offline (libguestfs)
#   ./dualstack-lab.sh down [name]
#
# The comparison that matters is `compare dual upgraded`: everything it prints
# is a way the converted cluster failed to reach the shape of a native one.
#
# Single node on purpose. Dual-stack is decided by server flags and CNI config,
# both of which a one-node cluster exercises fully, and it rebuilds in minutes.
# Node-rejoin behaviour needs the 3-node cluster and is a separate question.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "$SCRIPT_DIR/lib.sh"
source "$SCRIPT_DIR/ovs.sh"

K8S_VLAN="${K8S_VLAN:-2}"
DS_PREFIX="${DS_PREFIX:-172.31.2}"
MEM="${MEM:-4096}"; CPUS="${CPUS:-2}"; DISK_GB="${DISK_GB:-12}"
TOKEN="${TOKEN:-labsim-ds-token}"
CILIUM_VERSION="${CILIUM_VERSION:-1.19.1}"   # same as production
DEB_BASE="${DEB_BASE:-$IMG_DIR/debian-13-genericcloud-amd64.qcow2}"
EVIDENCE="$SCRIPT_DIR/dualstack-evidence"

# Pod/Service ranges. IPv4 halves are k3s's own defaults, so the v4-only build is
# a stock cluster and the diff is not polluted by gratuitous differences.
# IPv6 halves are ULA: this cluster never routes off-box, and using the real /48
# here would put lab addresses into a prefix that production also announces.
V4_CLUSTER="10.42.0.0/16"; V4_SERVICE="10.43.0.0/16"
V6_CLUSTER="${V6_CLUSTER:-fd00:42::/56}"
V6_SERVICE="${V6_SERVICE:-fd00:43::/112}"   # /112 -- apiserver caps v6 service ranges
V6_PREFIX="${V6_PREFIX:-fd00:2}"            # node addresses: fd00:2::<octet>

vm_name() { echo "labsim-ds-$1"; }
vm_ip()   { case "$1" in v4) echo "$DS_PREFIX.21";; dual) echo "$DS_PREFIX.22";; *) die "unknown build '$1'";; esac; }
vm_ip6()  { case "$1" in v4) echo "$V6_PREFIX::21";; dual) echo "$V6_PREFIX::22";; *) die "unknown build '$1'";; esac; }
disk_of() { echo "$IMG_DIR/$(vm_name "$1").qcow2"; }

ssh_vm() { local ip="$1"; shift; ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
             -o LogLevel=ERROR -o ConnectTimeout=8 -o BatchMode=yes "debian@$ip" "$@"; }

# --- seed -----------------------------------------------------------------
build_seed() {
  local iso="$1" vm="$2" mode="$3" pubkey="$4"
  local ip ip6 tmp; ip="$(vm_ip "$mode")"; ip6="$(vm_ip6 "$mode")"; tmp="$(mktemp -d)"

  echo "instance-id: $vm" > "$tmp/meta-data"
  # Static v6 on both builds. The v4-only cluster still gets an IPv6 ADDRESS --
  # only its Kubernetes config is v4-only. Otherwise the diff would be dominated
  # by host addressing rather than by what Kubernetes did differently.
  cat > "$tmp/network-config" <<EOF
version: 2
ethernets:
  enp1s0:
    addresses: [${ip}/24, ${ip6}/64]
    routes:
      - to: default
        via: ${DS_PREFIX}.1
    nameservers:
      addresses: [8.8.8.8, 1.1.1.1]
EOF

  local exec_args="server --flannel-backend=none --disable-network-policy --disable=servicelb --disable=traefik --tls-san=$ip --cluster-init"
  if [ "$mode" = dual ]; then
    exec_args="$exec_args --cluster-cidr=${V4_CLUSTER},${V6_CLUSTER} --service-cidr=${V4_SERVICE},${V6_SERVICE} --node-ip=${ip},${ip6}"
  else
    exec_args="$exec_args --node-ip=${ip}"
  fi

  cat > "$tmp/user-data" <<EOF
#cloud-config
hostname: $vm
fqdn: $vm
users:
  - name: debian
    groups: [sudo]
    shell: /bin/bash
    sudo: ["ALL=(ALL) NOPASSWD:ALL"]
    lock_passwd: false
    plain_text_passwd: labsim
    ssh_authorized_keys: [$pubkey]
ssh_pwauth: true
disable_root: false
package_update: true
packages: [curl, jq, iproute2, nftables]
write_files:
  - path: /etc/modules-load.d/cilium.conf
    content: |
      br_netfilter
  - path: /etc/dualstack-lab-mode
    content: |
      $mode
runcmd:
  - modprobe br_netfilter || true
  - |
    curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="$exec_args" K3S_TOKEN="$TOKEN" sh -
  - |
    # Cilium via helm, matching the production version. IPAM stays 'kubernetes'
    # in BOTH builds on purpose: that is what production runs, and it is the
    # mode the k3s objection is actually about. If the converted cluster needs
    # cluster-pool to work, the diff should be what tells us so.
    curl -sfL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash || true
    export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
    helm repo add cilium https://helm.cilium.io >/dev/null 2>&1 || true
    helm repo update >/dev/null 2>&1 || true
    for i in \$(seq 1 60); do kubectl get nodes >/dev/null 2>&1 && break; sleep 5; done
    if [ "$mode" = dual ]; then
      helm install cilium cilium/cilium --version $CILIUM_VERSION -n kube-system \\
        --set kubeProxyReplacement=false --set ipam.mode=kubernetes \\
        --set ipv4.enabled=true --set ipv6.enabled=true \\
        --set k8sServiceHost=$ip --set k8sServicePort=6443 || true
    else
      helm install cilium cilium/cilium --version $CILIUM_VERSION -n kube-system \\
        --set kubeProxyReplacement=false --set ipam.mode=kubernetes \\
        --set ipv4.enabled=true --set ipv6.enabled=false \\
        --set k8sServiceHost=$ip --set k8sServicePort=6443 || true
    fi
    touch /etc/dualstack-lab-ready
EOF
  sudo mkdir -p "$(dirname "$iso")"
  sudo genisoimage -quiet -output "$iso" -volid cidata -joliet -rock \
    "$tmp/user-data" "$tmp/meta-data" "$tmp/network-config"
  rm -rf "$tmp"
}

cmd_up() {
  local mode="${1:?usage: up <v4|dual>}"
  local vm ip disk seed pubkey
  vm="$(vm_name "$mode")"; ip="$(vm_ip "$mode")"; disk="$(disk_of "$mode")"
  seed="$IMG_DIR/${vm}-seed.iso"; pubkey="$(find_ssh_pubkey)"

  [ -f "$DEB_BASE" ] || die "base image missing: $DEB_BASE (run ./k8s-up.sh once)"
  if virsh_q dominfo "$vm" >/dev/null 2>&1; then
    log "$vm exists — starting if stopped"
    [ "$(virsh_q domstate "$vm" | head -1)" = "running" ] || virsh_q start "$vm" >/dev/null
    return
  fi
  selected_vlans; ovs_up
  log "creating $vm ($mode) at $ip / $(vm_ip6 "$mode")"
  sudo qemu-img create -q -f qcow2 -F qcow2 -b "$DEB_BASE" "$disk" "${DISK_GB}G" >/dev/null
  build_seed "$seed" "$vm" "$mode" "$pubkey"
  sudo virt-install --connect "$LIBVIRT_URI" --name "$vm" \
    --memory "$MEM" --vcpus "$CPUS" \
    --disk "path=$disk,format=qcow2,bus=virtio" \
    --disk "path=$seed,device=cdrom" \
    --network "network=$OVS_NET,portgroup=vlan${K8S_VLAN},model=virtio" \
    --os-variant debian12 --graphics none --noautoconsole --import >/dev/null
  log "installing in background; watch: ssh debian@$ip 'ls /etc/dualstack-lab-ready'"
}

# --- pristine copy / restore ---------------------------------------------
# reflink so the copy is instant and independent on btrfs/xfs. A qcow2 backing
# chain would be cheaper still but makes the parent read-only in practice: boot
# the parent again and every child silently corrupts.
cmd_pristine() {
  local mode="${1:?usage: pristine <v4|dual>}" vm disk
  vm="$(vm_name "$mode")"; disk="$(disk_of "$mode")"
  [ "$(virsh_q domstate "$vm" 2>/dev/null | head -1)" = "running" ] && \
    die "$vm is running — shut it down first (virsh shutdown $vm), a copy of a live disk is not consistent"
  sudo cp --reflink=auto "$disk" "${disk}.pristine"
  log "pristine copy: ${disk}.pristine"
}
cmd_restore() {
  local mode="${1:?usage: restore <v4|dual>}" vm disk
  vm="$(vm_name "$mode")"; disk="$(disk_of "$mode")"
  [ -f "${disk}.pristine" ] || die "no pristine copy for $mode"
  [ "$(virsh_q domstate "$vm" 2>/dev/null | head -1)" = "running" ] && \
    die "$vm is running — shut it down first"
  sudo cp --reflink=auto "${disk}.pristine" "$disk"
  log "restored $mode from pristine"
}


# --- the experiment ------------------------------------------------------
# Convert the IPv4-only cluster in place, mirroring the flags the native build
# was BORN with. Each step prints what the cluster did, because the interesting
# output is which step refuses rather than whether the end state is pretty.
cmd_upgrade() {
  local ip; ip="$(vm_ip v4)"; local ip6; ip6="$(vm_ip6 v4)"
  log "step 1/4: add dual CIDRs + dual node-ip to the k3s unit"
  # Done with python on the box, not nested sed: quoting a multi-line systemd
  # continuation through ssh -> sh -> sed produced a literal \\n in the unit, and
  # k3s then saw a dual cluster-cidr with a still-IPv4 service-cidr and refused
  # to start. All three flags go on one line -- systemd does not care, and there
  # is nothing left to escape.
  ssh_vm "$ip" "sudo python3 - <<'PYEOF'
import re
u = '/etc/systemd/system/k3s.service'
s = open(u).read()
old = \"'--node-ip=${ip}'\"
new = \"'--cluster-cidr=${V4_CLUSTER},${V6_CLUSTER}' '--service-cidr=${V4_SERVICE},${V6_SERVICE}' '--node-ip=${ip},${ip6}'\"
assert old in s, 'node-ip flag not found in unit'
open(u,'w').write(s.replace(old, new))
print('    unit rewritten')
PYEOF
sudo systemctl daemon-reload" || die "unit edit failed"
  ssh_vm "$ip" "grep -oE \"'--(cluster|service)-cidr=[^']*'|'--node-ip=[^']*'\" /etc/systemd/system/k3s.service | sed 's/^/    /'"

  log "step 2/4: restart k3s and see whether it accepts the changed ranges"
  ssh_vm "$ip" "sudo systemctl restart k3s" || true
  for i in $(seq 1 40); do
    ssh_vm "$ip" "sudo k3s kubectl get --raw /readyz >/dev/null 2>&1" && break
    sleep 5
  done
  ssh_vm "$ip" "sudo journalctl -u k3s --since '2 min ago' --no-pager 2>/dev/null | grep -iE 'cidr|dual|ipv6|invalid|cannot|fail' | tail -12 | sed 's/^/    /'" || true

  log "step 3/4: what the API says now"
  ssh_vm "$ip" "echo -n '    servicecidr: '; sudo k3s kubectl get servicecidr -o jsonpath='{.items[*].spec.cidrs}'; echo; \
                echo -n '    node podCIDRs: '; sudo k3s kubectl get node -o jsonpath='{.items[0].spec.podCIDRs}'; echo; \
                echo -n '    node addresses: '; sudo k3s kubectl get node -o jsonpath='{.items[0].status.addresses[*].address}'; echo" || true

  log "step 4/4: turn on IPv6 in Cilium"
  ssh_vm "$ip" "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml; sudo -E helm upgrade cilium cilium/cilium --version ${CILIUM_VERSION} -n kube-system --reuse-values --set ipv6.enabled=true >/dev/null 2>&1 && echo '    cilium upgraded' || echo '    cilium upgrade FAILED'" || true
  ssh_vm "$ip" "sudo k3s kubectl -n kube-system rollout restart ds/cilium >/dev/null 2>&1; sleep 20; sudo k3s kubectl -n kube-system get pods -l k8s-app=cilium --no-headers | sed 's/^/    /'" || true
  log "now: ./dualstack-lab.sh collect upgraded ${ip} && ./dualstack-lab.sh compare dual upgraded"
}

# --- evidence collection --------------------------------------------------
# Normalized on purpose. Two independently built clusters differ in certs,
# tokens, UUIDs, timestamps and log lines; left raw, that noise buries the
# handful of differences that actually mean something.
cmd_collect() {
  local name="${1:?usage: collect <name> [ip]}"
  local ip="${2:-}"
  [ -n "$ip" ] || ip="$(vm_ip "$name" 2>/dev/null || true)"
  [ -n "$ip" ] || die "collect: give an ip for a non-standard name"
  local out="$EVIDENCE/$name"; mkdir -p "$out"
  log "collecting from $name ($ip) -> $out"

  ssh_vm "$ip" 'sudo cat /etc/rancher/k3s/config.yaml 2>/dev/null; sudo systemctl cat k3s 2>/dev/null | grep -A30 ExecStart' \
    > "$out/k3s-config.txt" 2>/dev/null || true
  ssh_vm "$ip" 'sudo tr "\0" "\n" < /proc/$(pgrep -f "k3s server" | head -1)/cmdline | grep -v "^$"' \
    > "$out/k3s-cmdline.txt" 2>/dev/null || true
  ssh_vm "$ip" 'ip -o addr show | awk "{print \$2, \$3, \$4}"; echo ---; ip -4 route show; echo ---; ip -6 route show' \
    > "$out/host-net.txt" 2>/dev/null || true
  ssh_vm "$ip" 'sudo sysctl -a 2>/dev/null | grep -E "net\.ipv6\.conf\.(all|default)\.(forwarding|disable_ipv6)|net\.ipv4\.ip_forward"' \
    > "$out/sysctl.txt" 2>/dev/null || true

  local K='sudo k3s kubectl'
  ssh_vm "$ip" "$K get servicecidr -o yaml"       > "$out/servicecidr.yaml" 2>/dev/null || true
  ssh_vm "$ip" "$K get nodes -o yaml"             > "$out/nodes.yaml.raw"   2>/dev/null || true
  ssh_vm "$ip" "$K get ciliumnodes -o yaml"       > "$out/ciliumnodes.yaml.raw" 2>/dev/null || true
  ssh_vm "$ip" "$K -n kube-system get cm cilium-config -o yaml" > "$out/cilium-config.yaml.raw" 2>/dev/null || true
  ssh_vm "$ip" "$K get svc -A -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,FAMILYPOLICY:.spec.ipFamilyPolicy,FAMILIES:.spec.ipFamilies,IPS:.spec.clusterIPs" \
    > "$out/services.txt" 2>/dev/null || true
  ssh_vm "$ip" "$K get pods -A -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,IPS:.status.podIPs" \
    > "$out/podips.txt" 2>/dev/null || true

  # Strip the things that differ every build regardless of configuration.
  for f in "$out"/*.raw; do
    [ -e "$f" ] || continue
    sed -E \
      -e 's/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:]+Z?/<TIME>/g' \
      -e 's/(uid|resourceVersion|creationTimestamp|generation|observedGeneration): .*/\1: <X>/' \
      -e 's/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/<UUID>/g' \
      -e 's/(LS0tLS1|[A-Za-z0-9+\/]{60,}=*)/<B64>/g' \
      "$f" > "${f%.raw}"
    rm -f "$f"
  done
  log "collected $(ls "$out" | wc -l) artefacts"
}

cmd_compare() {
  local a="${1:?usage: compare <a> <b>}" b="${2:?}"
  [ -d "$EVIDENCE/$a" ] && [ -d "$EVIDENCE/$b" ] || die "collect both first"
  echo "### semantic diff: $a (<) vs $b (>)"
  diff -ru "$EVIDENCE/$a" "$EVIDENCE/$b" || true
}

cmd_virtdiff() {
  local a="${1:?usage: virtdiff <a> <b>}" b="${2:?}"
  for m in "$a" "$b"; do
    [ "$(virsh_q domstate "$(vm_name "$m")" 2>/dev/null | head -1)" = "running" ] && \
      die "$(vm_name "$m") is running — virt-diff needs the disks quiescent"
  done
  log "whole-filesystem diff (slow); noise is expected — use it to find what the collector missed"
  sudo virt-diff -a "$(disk_of "$a")" -A "$(disk_of "$b")" \
    | grep -vE '/(var/log|tmp|run|proc|sys)/|\.log$|/var/lib/rancher/k3s/(server/(tls|cred|db)|agent)' || true
}

cmd_down() {
  local only="${1:-}"
  for m in v4 dual; do
    [ -n "$only" ] && [ "$only" != "$m" ] && continue
    local vm; vm="$(vm_name "$m")"
    virsh_q dominfo "$vm" >/dev/null 2>&1 || continue
    log "removing $vm"
    virsh_q destroy "$vm" >/dev/null 2>&1 || true
    virsh_q undefine "$vm" --remove-all-storage >/dev/null 2>&1 || true
  done
}

case "${1:-}" in
  up)       shift; cmd_up "$@" ;;
  upgrade)  shift; cmd_upgrade "$@" ;;
  pristine) shift; cmd_pristine "$@" ;;
  restore)  shift; cmd_restore "$@" ;;
  collect)  shift; cmd_collect "$@" ;;
  compare)  shift; cmd_compare "$@" ;;
  virtdiff) shift; cmd_virtdiff "$@" ;;
  down)     shift; cmd_down "$@" ;;
  *) sed -n '2,30p' "$0"; exit 1 ;;
esac