From 527e0798aea370033de927898268d4cdc5275d71 Mon Sep 17 00:00:00 2001 From: Michal Date: Mon, 24 Aug 2026 22:59:55 +0100 Subject: [PATCH] bastion/k3s: bootstrap new clusters with cluster-pool IPAM Every cluster this bastion builds was pinned to ipam=kubernetes, which makes it permanently single-stack: Cilium reads node.spec.podCIDRs, the controller-manager writes that once at node join and never revises it, so adding IPv6 later fails with `required IPv6 PodCIDR not available` and needs a rebuild. cluster-pool puts allocation in the CiliumNode CRD, where the operator can add an address family to a running node. Proven in labsim (842408c). It is also Cilium's own default -- ipam=kubernetes was the deviation. Pool matches the k3s cluster-cidr (10.42.0.0/16, /24 per node), so a node gets the same CIDR shape it would have had. IPv4-only for now: the IPv6 pool wants an apiserver carrying an IPv6 service CIDR, which is a separate change. Mirrors @michal/cilium-values ciliumBaseValues (cilium-values@11ee509), which these paths track but do not import. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DMVzWZgiKW2wquf5z8S1yH --- bastion/src/modules/modules/k3s/src/install.ts | 4 +++- bastion/src/modules/modules/k3s/src/operations/cilium.ts | 4 +++- bastion/tests/integration/k3s-single-node.test.ts | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bastion/src/modules/modules/k3s/src/install.ts b/bastion/src/modules/modules/k3s/src/install.ts index e9202cd..b899668 100644 --- a/bastion/src/modules/modules/k3s/src/install.ts +++ b/bastion/src/modules/modules/k3s/src/install.ts @@ -215,7 +215,9 @@ echo " Using network device: \$DEFAULT_DEV" KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install \\ --set kubeProxyReplacement=true \\ - --set ipam.mode=kubernetes \\ + --set ipam.mode=cluster-pool \\ + --set ipam.operator.clusterPoolIPv4PodCIDRList='{10.42.0.0/16}' \\ + --set ipam.operator.clusterPoolIPv4MaskSize=24 \\ --set devices="\$DEFAULT_DEV" \\ --set nodePort.directRoutingDevice="\$DEFAULT_DEV" diff --git a/bastion/src/modules/modules/k3s/src/operations/cilium.ts b/bastion/src/modules/modules/k3s/src/operations/cilium.ts index 7ac05ff..054f8d2 100644 --- a/bastion/src/modules/modules/k3s/src/operations/cilium.ts +++ b/bastion/src/modules/modules/k3s/src/operations/cilium.ts @@ -47,7 +47,9 @@ export const installCilium: Operation = async (ctx): Promise => const installResult = await ctx.ssh.exec( `KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install \ --set kubeProxyReplacement=true \ - --set ipam.mode=kubernetes \ + --set ipam.mode=cluster-pool \ + --set ipam.operator.clusterPoolIPv4PodCIDRList='{10.42.0.0/16}' \ + --set ipam.operator.clusterPoolIPv4MaskSize=24 \ --set k8sServiceHost=127.0.0.1 \ --set k8sServicePort=6444 \ --set cni.exclusive=false \ diff --git a/bastion/tests/integration/k3s-single-node.test.ts b/bastion/tests/integration/k3s-single-node.test.ts index d52c328..f139965 100644 --- a/bastion/tests/integration/k3s-single-node.test.ts +++ b/bastion/tests/integration/k3s-single-node.test.ts @@ -161,7 +161,7 @@ EOF' CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt) curl -L --fail --silent "https://github.com/cilium/cilium-cli/releases/download/\${CILIUM_CLI_VERSION}/cilium-linux-amd64.tar.gz" | sudo tar xz -C /usr/local/bin DEFAULT_DEV=$(ip -4 route show default | awk '{print $5}' | head -1) - sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install --set kubeProxyReplacement=true --set ipam.mode=kubernetes --set devices=$DEFAULT_DEV --set nodePort.directRoutingDevice=$DEFAULT_DEV + sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install --set kubeProxyReplacement=true --set ipam.mode=cluster-pool --set ipam.operator.clusterPoolIPv4PodCIDRList='{10.42.0.0/16}' --set ipam.operator.clusterPoolIPv4MaskSize=24 --set devices=$DEFAULT_DEV --set nodePort.directRoutingDevice=$DEFAULT_DEV `.trim(), "cilium install", { keyPath: sshKeyPath, timeout: 120_000 }); log("Waiting for Cilium to be ready...");