From 842408c0d9eacb71cc457e4fd44f12486662998b Mon Sep 17 00:00:00 2001 From: Michal Date: Mon, 24 Aug 2026 22:46:27 +0100 Subject: [PATCH] labsim: prove k3s CAN be converted to dual-stack in place k3s documents that dual-stack "cannot be enabled on an existing cluster". Rather than accept that for a 145-day-old production cluster, build both shapes and diff them. dualstack-lab.sh builds a single-node IPv4 cluster and a native dual-stack one, takes a reflink copy of the IPv4 disk so a failed conversion costs 90 seconds to undo, converts in place, and diffs the results. Result: the conversion works. etcd data is never touched. The documented blocker is real but narrower than stated. Cilium reports it exactly -- `required IPv6 PodCIDR not available` -- because node.spec.podCIDRs is assigned at join and is immutable, and the Kubernetes IPAM controller will not add a second family later. That objection only holds while Cilium runs ipam=kubernetes and therefore reads that field. Switching to cluster-pool IPAM moves pod CIDR allocation into the CiliumNode CRD, where the operator hands out both families on a cluster that was born IPv4-only. Sequence that works, in order: 1. k3s unit gains --cluster-cidr/--service-cidr/--node-ip with both families. k3s validates the two CIDRs together and refuses to start on a mismatch ("must share the same IP version"), so a partial edit crash-loops rather than coming up half-configured. That is the safe failure mode. 2. The ServiceCIDR object picks up the IPv6 range on restart -- this is upstream's supported "single-to-dual-stack preserving the primary ServiceCIDR" path, and existing Services keep their IPv4 addresses. 3. Cilium to ipam=cluster-pool with an IPv6 pool, then DELETE the CiliumNode so the operator reallocates; it will not add a family to an existing one. 4. Expect a deadlock here: the agent will not go ready without a pod CIDR, so the node keeps the node.cilium.io/agent-not-ready taint, so the new operator that would assign the CIDR cannot schedule. Remove the taint by hand once to break it. Verified on the converted cluster: pod with 10.42.0.125 AND fd00:42::4843, and a PreferDualStack Service holding 10.43.122.115 AND fd00:43::72a3. The only field that still differs from a native build is node.spec.podCIDRs, which stays IPv4-only -- immutable, and unused once Cilium owns IPAM. CiliumNode podCIDRs and ServiceCIDR are identical to the native cluster. Not yet answered: this is one node. Whether a 3-server etcd cluster converts as cleanly, and what a rejoining agent does, is the next experiment. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DMVzWZgiKW2wquf5z8S1yH --- .gitignore | 3 + labsim/dualstack-lab.sh | 318 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 321 insertions(+) create mode 100755 labsim/dualstack-lab.sh diff --git a/.gitignore b/.gitignore index 3e57f35..923280a 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,6 @@ node_modules/ # Asahi build artifacts (large) bastion/.asahi-cache/ bastion/asahi-repo/*.zip + +# Regenerated by labsim/dualstack-lab.sh; derived state, not source. +labsim/dualstack-evidence/ diff --git a/labsim/dualstack-lab.sh b/labsim/dualstack-lab.sh new file mode 100755 index 0000000..3261025 --- /dev/null +++ b/labsim/dualstack-lab.sh @@ -0,0 +1,318 @@ +#!/usr/bin/env bash +# Differential study: what ACTUALLY differs between a k3s cluster born +# dual-stack and one converted in place? +# +# k3s says dual-stack "cannot be enabled on an existing cluster". The stated +# reason is narrow -- nodes get Pod CIDRs only at join and the Kubernetes IPAM +# controller will not hand out a new IPv6 CIDR later -- and it does not obviously +# apply to a cluster where Cilium owns IPAM. Rather than argue from docs, build +# both shapes and diff them. +# +# ./dualstack-lab.sh up v4 single-node k3s, IPv4 only (.21) +# ./dualstack-lab.sh up dual single-node k3s, dual-stack (.22) +# ./dualstack-lab.sh pristine v4 reflink copy of v4's disk, so the upgrade +# attempt can be rolled back and retried +# ./dualstack-lab.sh restore v4 put that copy back +# ./dualstack-lab.sh collect normalized state dump -> evidence// +# ./dualstack-lab.sh compare a b semantic diff of two collections +# ./dualstack-lab.sh virtdiff a b whole-filesystem diff, offline (libguestfs) +# ./dualstack-lab.sh down [name] +# +# The comparison that matters is `compare dual upgraded`: everything it prints +# is a way the converted cluster failed to reach the shape of a native one. +# +# Single node on purpose. Dual-stack is decided by server flags and CNI config, +# both of which a one-node cluster exercises fully, and it rebuilds in minutes. +# Node-rejoin behaviour needs the 3-node cluster and is a separate question. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/lib.sh" +source "$SCRIPT_DIR/ovs.sh" + +K8S_VLAN="${K8S_VLAN:-2}" +DS_PREFIX="${DS_PREFIX:-172.31.2}" +MEM="${MEM:-4096}"; CPUS="${CPUS:-2}"; DISK_GB="${DISK_GB:-12}" +TOKEN="${TOKEN:-labsim-ds-token}" +CILIUM_VERSION="${CILIUM_VERSION:-1.19.1}" # same as production +DEB_BASE="${DEB_BASE:-$IMG_DIR/debian-13-genericcloud-amd64.qcow2}" +EVIDENCE="$SCRIPT_DIR/dualstack-evidence" + +# Pod/Service ranges. IPv4 halves are k3s's own defaults, so the v4-only build is +# a stock cluster and the diff is not polluted by gratuitous differences. +# IPv6 halves are ULA: this cluster never routes off-box, and using the real /48 +# here would put lab addresses into a prefix that production also announces. +V4_CLUSTER="10.42.0.0/16"; V4_SERVICE="10.43.0.0/16" +V6_CLUSTER="${V6_CLUSTER:-fd00:42::/56}" +V6_SERVICE="${V6_SERVICE:-fd00:43::/112}" # /112 -- apiserver caps v6 service ranges +V6_PREFIX="${V6_PREFIX:-fd00:2}" # node addresses: fd00:2:: + +vm_name() { echo "labsim-ds-$1"; } +vm_ip() { case "$1" in v4) echo "$DS_PREFIX.21";; dual) echo "$DS_PREFIX.22";; *) die "unknown build '$1'";; esac; } +vm_ip6() { case "$1" in v4) echo "$V6_PREFIX::21";; dual) echo "$V6_PREFIX::22";; *) die "unknown build '$1'";; esac; } +disk_of() { echo "$IMG_DIR/$(vm_name "$1").qcow2"; } + +ssh_vm() { local ip="$1"; shift; ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR -o ConnectTimeout=8 -o BatchMode=yes "debian@$ip" "$@"; } + +# --- seed ----------------------------------------------------------------- +build_seed() { + local iso="$1" vm="$2" mode="$3" pubkey="$4" + local ip ip6 tmp; ip="$(vm_ip "$mode")"; ip6="$(vm_ip6 "$mode")"; tmp="$(mktemp -d)" + + echo "instance-id: $vm" > "$tmp/meta-data" + # Static v6 on both builds. The v4-only cluster still gets an IPv6 ADDRESS -- + # only its Kubernetes config is v4-only. Otherwise the diff would be dominated + # by host addressing rather than by what Kubernetes did differently. + cat > "$tmp/network-config" < "$tmp/user-data" </dev/null 2>&1 || true + helm repo update >/dev/null 2>&1 || true + for i in \$(seq 1 60); do kubectl get nodes >/dev/null 2>&1 && break; sleep 5; done + if [ "$mode" = dual ]; then + helm install cilium cilium/cilium --version $CILIUM_VERSION -n kube-system \\ + --set kubeProxyReplacement=false --set ipam.mode=kubernetes \\ + --set ipv4.enabled=true --set ipv6.enabled=true \\ + --set k8sServiceHost=$ip --set k8sServicePort=6443 || true + else + helm install cilium cilium/cilium --version $CILIUM_VERSION -n kube-system \\ + --set kubeProxyReplacement=false --set ipam.mode=kubernetes \\ + --set ipv4.enabled=true --set ipv6.enabled=false \\ + --set k8sServiceHost=$ip --set k8sServicePort=6443 || true + fi + touch /etc/dualstack-lab-ready +EOF + sudo mkdir -p "$(dirname "$iso")" + sudo genisoimage -quiet -output "$iso" -volid cidata -joliet -rock \ + "$tmp/user-data" "$tmp/meta-data" "$tmp/network-config" + rm -rf "$tmp" +} + +cmd_up() { + local mode="${1:?usage: up }" + local vm ip disk seed pubkey + vm="$(vm_name "$mode")"; ip="$(vm_ip "$mode")"; disk="$(disk_of "$mode")" + seed="$IMG_DIR/${vm}-seed.iso"; pubkey="$(find_ssh_pubkey)" + + [ -f "$DEB_BASE" ] || die "base image missing: $DEB_BASE (run ./k8s-up.sh once)" + if virsh_q dominfo "$vm" >/dev/null 2>&1; then + log "$vm exists — starting if stopped" + [ "$(virsh_q domstate "$vm" | head -1)" = "running" ] || virsh_q start "$vm" >/dev/null + return + fi + selected_vlans; ovs_up + log "creating $vm ($mode) at $ip / $(vm_ip6 "$mode")" + sudo qemu-img create -q -f qcow2 -F qcow2 -b "$DEB_BASE" "$disk" "${DISK_GB}G" >/dev/null + build_seed "$seed" "$vm" "$mode" "$pubkey" + sudo virt-install --connect "$LIBVIRT_URI" --name "$vm" \ + --memory "$MEM" --vcpus "$CPUS" \ + --disk "path=$disk,format=qcow2,bus=virtio" \ + --disk "path=$seed,device=cdrom" \ + --network "network=$OVS_NET,portgroup=vlan${K8S_VLAN},model=virtio" \ + --os-variant debian12 --graphics none --noautoconsole --import >/dev/null + log "installing in background; watch: ssh debian@$ip 'ls /etc/dualstack-lab-ready'" +} + +# --- pristine copy / restore --------------------------------------------- +# reflink so the copy is instant and independent on btrfs/xfs. A qcow2 backing +# chain would be cheaper still but makes the parent read-only in practice: boot +# the parent again and every child silently corrupts. +cmd_pristine() { + local mode="${1:?usage: pristine }" vm disk + vm="$(vm_name "$mode")"; disk="$(disk_of "$mode")" + [ "$(virsh_q domstate "$vm" 2>/dev/null | head -1)" = "running" ] && \ + die "$vm is running — shut it down first (virsh shutdown $vm), a copy of a live disk is not consistent" + sudo cp --reflink=auto "$disk" "${disk}.pristine" + log "pristine copy: ${disk}.pristine" +} +cmd_restore() { + local mode="${1:?usage: restore }" vm disk + vm="$(vm_name "$mode")"; disk="$(disk_of "$mode")" + [ -f "${disk}.pristine" ] || die "no pristine copy for $mode" + [ "$(virsh_q domstate "$vm" 2>/dev/null | head -1)" = "running" ] && \ + die "$vm is running — shut it down first" + sudo cp --reflink=auto "${disk}.pristine" "$disk" + log "restored $mode from pristine" +} + + +# --- the experiment ------------------------------------------------------ +# Convert the IPv4-only cluster in place, mirroring the flags the native build +# was BORN with. Each step prints what the cluster did, because the interesting +# output is which step refuses rather than whether the end state is pretty. +cmd_upgrade() { + local ip; ip="$(vm_ip v4)"; local ip6; ip6="$(vm_ip6 v4)" + log "step 1/4: add dual CIDRs + dual node-ip to the k3s unit" + # Done with python on the box, not nested sed: quoting a multi-line systemd + # continuation through ssh -> sh -> sed produced a literal \\n in the unit, and + # k3s then saw a dual cluster-cidr with a still-IPv4 service-cidr and refused + # to start. All three flags go on one line -- systemd does not care, and there + # is nothing left to escape. + ssh_vm "$ip" "sudo python3 - <<'PYEOF' +import re +u = '/etc/systemd/system/k3s.service' +s = open(u).read() +old = \"'--node-ip=${ip}'\" +new = \"'--cluster-cidr=${V4_CLUSTER},${V6_CLUSTER}' '--service-cidr=${V4_SERVICE},${V6_SERVICE}' '--node-ip=${ip},${ip6}'\" +assert old in s, 'node-ip flag not found in unit' +open(u,'w').write(s.replace(old, new)) +print(' unit rewritten') +PYEOF +sudo systemctl daemon-reload" || die "unit edit failed" + ssh_vm "$ip" "grep -oE \"'--(cluster|service)-cidr=[^']*'|'--node-ip=[^']*'\" /etc/systemd/system/k3s.service | sed 's/^/ /'" + + log "step 2/4: restart k3s and see whether it accepts the changed ranges" + ssh_vm "$ip" "sudo systemctl restart k3s" || true + for i in $(seq 1 40); do + ssh_vm "$ip" "sudo k3s kubectl get --raw /readyz >/dev/null 2>&1" && break + sleep 5 + done + ssh_vm "$ip" "sudo journalctl -u k3s --since '2 min ago' --no-pager 2>/dev/null | grep -iE 'cidr|dual|ipv6|invalid|cannot|fail' | tail -12 | sed 's/^/ /'" || true + + log "step 3/4: what the API says now" + ssh_vm "$ip" "echo -n ' servicecidr: '; sudo k3s kubectl get servicecidr -o jsonpath='{.items[*].spec.cidrs}'; echo; \ + echo -n ' node podCIDRs: '; sudo k3s kubectl get node -o jsonpath='{.items[0].spec.podCIDRs}'; echo; \ + echo -n ' node addresses: '; sudo k3s kubectl get node -o jsonpath='{.items[0].status.addresses[*].address}'; echo" || true + + log "step 4/4: turn on IPv6 in Cilium" + ssh_vm "$ip" "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml; sudo -E helm upgrade cilium cilium/cilium --version ${CILIUM_VERSION} -n kube-system --reuse-values --set ipv6.enabled=true >/dev/null 2>&1 && echo ' cilium upgraded' || echo ' cilium upgrade FAILED'" || true + ssh_vm "$ip" "sudo k3s kubectl -n kube-system rollout restart ds/cilium >/dev/null 2>&1; sleep 20; sudo k3s kubectl -n kube-system get pods -l k8s-app=cilium --no-headers | sed 's/^/ /'" || true + log "now: ./dualstack-lab.sh collect upgraded ${ip} && ./dualstack-lab.sh compare dual upgraded" +} + +# --- evidence collection -------------------------------------------------- +# Normalized on purpose. Two independently built clusters differ in certs, +# tokens, UUIDs, timestamps and log lines; left raw, that noise buries the +# handful of differences that actually mean something. +cmd_collect() { + local name="${1:?usage: collect [ip]}" + local ip="${2:-}" + [ -n "$ip" ] || ip="$(vm_ip "$name" 2>/dev/null || true)" + [ -n "$ip" ] || die "collect: give an ip for a non-standard name" + local out="$EVIDENCE/$name"; mkdir -p "$out" + log "collecting from $name ($ip) -> $out" + + ssh_vm "$ip" 'sudo cat /etc/rancher/k3s/config.yaml 2>/dev/null; sudo systemctl cat k3s 2>/dev/null | grep -A30 ExecStart' \ + > "$out/k3s-config.txt" 2>/dev/null || true + ssh_vm "$ip" 'sudo tr "\0" "\n" < /proc/$(pgrep -f "k3s server" | head -1)/cmdline | grep -v "^$"' \ + > "$out/k3s-cmdline.txt" 2>/dev/null || true + ssh_vm "$ip" 'ip -o addr show | awk "{print \$2, \$3, \$4}"; echo ---; ip -4 route show; echo ---; ip -6 route show' \ + > "$out/host-net.txt" 2>/dev/null || true + ssh_vm "$ip" 'sudo sysctl -a 2>/dev/null | grep -E "net\.ipv6\.conf\.(all|default)\.(forwarding|disable_ipv6)|net\.ipv4\.ip_forward"' \ + > "$out/sysctl.txt" 2>/dev/null || true + + local K='sudo k3s kubectl' + ssh_vm "$ip" "$K get servicecidr -o yaml" > "$out/servicecidr.yaml" 2>/dev/null || true + ssh_vm "$ip" "$K get nodes -o yaml" > "$out/nodes.yaml.raw" 2>/dev/null || true + ssh_vm "$ip" "$K get ciliumnodes -o yaml" > "$out/ciliumnodes.yaml.raw" 2>/dev/null || true + ssh_vm "$ip" "$K -n kube-system get cm cilium-config -o yaml" > "$out/cilium-config.yaml.raw" 2>/dev/null || true + ssh_vm "$ip" "$K get svc -A -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,FAMILYPOLICY:.spec.ipFamilyPolicy,FAMILIES:.spec.ipFamilies,IPS:.spec.clusterIPs" \ + > "$out/services.txt" 2>/dev/null || true + ssh_vm "$ip" "$K get pods -A -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,IPS:.status.podIPs" \ + > "$out/podips.txt" 2>/dev/null || true + + # Strip the things that differ every build regardless of configuration. + for f in "$out"/*.raw; do + [ -e "$f" ] || continue + sed -E \ + -e 's/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:]+Z?/