diff --git a/labsim/labsim-k8s-etcd.sh b/labsim/labsim-k8s-etcd.sh index 839741d..f826599 100755 --- a/labsim/labsim-k8s-etcd.sh +++ b/labsim/labsim-k8s-etcd.sh @@ -146,12 +146,26 @@ $audit_ind content: | br_netfilter overlay - - path: /etc/sysctl.d/99-k8s.conf + # The CIS sysctls the generated config's `protect-kernel-defaults: true` + # REQUIRES -- byte-for-byte from labctl's sysctl.ts (applyCisHardening), plus + # v6 forwarding. Without vm.overcommit_memory=1 / kernel.panic=10 / + # kernel.panic_on_oops=1 the kubelet REFUSES to start ("invalid kernel flag"), + # k3s exits 1 and crash-loops -- which presents downstream as etcd + # re-initialising and the apiserver flapping, i.e. it looks like an etcd/CPU + # problem when it is not. In production these come from install.ks.ts + this + # operation; the sim must set them too. + - path: /etc/sysctl.d/90-k3s-cis.conf content: | - net.ipv4.ip_forward = 1 - net.ipv6.conf.all.forwarding = 1 - net.bridge.bridge-nf-call-iptables = 1 + net.bridge.bridge-nf-call-iptables = 1 net.bridge.bridge-nf-call-ip6tables = 1 + net.ipv4.ip_forward = 1 + net.ipv6.conf.all.forwarding = 1 + vm.panic_on_oom = 0 + vm.overcommit_memory = 1 + kernel.panic = 10 + kernel.panic_on_oops = 1 + fs.inotify.max_user_instances = 524288 + fs.inotify.max_user_watches = 524288 runcmd: - [ modprobe, br_netfilter ] @@ -171,15 +185,13 @@ runcmd: # exactly as production. The token is passed via env for the join; on node 1 # it seeds the cluster token. # - # The etcd-arg tuning is LAB-ONLY and deliberately NOT in the production - # config.yaml/k3s-config.ts: three embedded-etcd servers on a busy labsim - # host starve etcd of CPU, and stock etcd timers (100ms heartbeat / 1s - # election) then trigger a constant leader-re-election storm that never - # settles ("storage is (re)initializing", k3s stuck activating). Relaxing - # the timers lets etcd tolerate the lab's scheduling latency. It changes - # NOTHING the dual-stack conversion exercises (config.yaml CIDR handling, - # ServiceCIDR pickup, quorum across a restart) -- it just keeps etcd alive - # under nested-virt jitter. See dualstack-evidence/etcd-harness-2026-09-07.md. + # The etcd-arg tuning is LAB-ONLY (not in production's config): relaxed + # heartbeat/election timers so etcd tolerates nested-virt scheduling jitter. + # NB: this was NOT what fixed the first run's failure -- that was the missing + # protect-kernel-defaults sysctls above, which crash-looped the kubelet and + # only LOOKED like etcd instability. The tuning is kept as cheap defensive + # insurance for a busy host; it changes nothing the conversion test + # exercises. curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --etcd-arg=heartbeat-interval=500 --etcd-arg=election-timeout=5000" K3S_TOKEN="$TOKEN" sh - EOF