diff --git a/.gitignore b/.gitignore index 923280a..cf26d49 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ bastion/asahi-repo/*.zip # Regenerated by labsim/dualstack-lab.sh; derived state, not source. labsim/dualstack-evidence/ + +# Runtime snapshots from labsim/cilium-ipam-switch.sh +labsim/.ipam-switch-state/ diff --git a/labsim/cilium-ipam-switch.sh b/labsim/cilium-ipam-switch.sh new file mode 100755 index 0000000..e703343 --- /dev/null +++ b/labsim/cilium-ipam-switch.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# Procedure around a Cilium IPAM mode change. Works against any cluster, so the +# rehearsal in labsim and the real thing in production run the SAME steps. +# +# It deliberately does NOT change the mode itself. In labsim that is `helm +# upgrade`; in production Pulumi owns the release and a script racing it would +# just reintroduce drift. What this owns is everything around the apply -- the +# evidence, the deadlock, and the verdict. +# +# ./cilium-ipam-switch.sh preflight record what the cluster looks like now +# ./cilium-ipam-switch.sh unstick break the agent-not-ready taint deadlock +# ./cilium-ipam-switch.sh verify compare against preflight, report renumbering +# +# KUBECONFIG=... ./cilium-ipam-switch.sh preflight +set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +STATE="${STATE:-$SCRIPT_DIR/.ipam-switch-state}" +K="kubectl" + +say() { printf '\033[0;36m[ipam]\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33m[ipam]\033[0m %s\n' "$*" >&2; } + +snapshot() { + echo "## nodes" + $K get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.podCIDRs}{"\n"}{end}' 2>/dev/null + echo "## ciliumnodes" + $K get ciliumnode -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.ipam.podCIDRs}{"\n"}{end}' 2>/dev/null + echo "## pods" + $K get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\t"}{.status.podIP}{"\n"}{end}' 2>/dev/null \ + | grep -vP '\t$' | sort + echo "## ipam" + $K -n kube-system get cm cilium-config -o jsonpath='{.data.ipam}' 2>/dev/null; echo +} + +cmd_preflight() { + mkdir -p "$STATE" + snapshot > "$STATE/before.txt" + say "recorded $(grep -c . "$STATE/before.txt") lines -> $STATE/before.txt" + say "mode now: $(sed -n '/^## ipam/,$p' "$STATE/before.txt" | tail -1)" + # The pod inventory is the rollback reference: if the switch renumbers, this + # is the only record of what an address USED to be. + say "pods on the pod network: $(sed -n '/^## pods/,/^## ipam/p' "$STATE/before.txt" | grep -c '10\.')" +} + +# The deadlock, in one place because it WILL happen and doing it by hand under +# time pressure is how the wrong node gets untainted: +# agent has no pod CIDR -> agent not ready -> node keeps +# node.cilium.io/agent-not-ready:NoSchedule -> the operator that would assign +# the CIDR cannot schedule -> agent still has no pod CIDR. +# Removing the taint is safe: it exists to keep normal workloads off a node +# without working networking, and the operator is precisely the thing that fixes +# that. Kubernetes re-adds it on the next agent restart. +cmd_unstick() { + local stuck=0 + for n in $($K get nodes -o name 2>/dev/null); do + $K get "$n" -o jsonpath='{.spec.taints[*].key}' 2>/dev/null | grep -q 'agent-not-ready' || continue + warn "${n#node/} carries agent-not-ready; removing so the operator can schedule" + $K taint "$n" node.cilium.io/agent-not-ready- >/dev/null 2>&1 && stuck=$((stuck+1)) + done + [ "$stuck" -eq 0 ] && say "no node was stuck" || say "cleared $stuck node(s)" + local pend + pend="$($K -n kube-system get pods -l io.cilium/app=operator --no-headers 2>/dev/null | grep -c Pending)" + [ "${pend:-0}" -gt 0 ] && warn "$pend operator pod(s) still Pending — check tolerations, not just taints" + return 0 +} + +cmd_verify() { + [ -f "$STATE/before.txt" ] || { warn "no preflight snapshot; nothing to compare"; return 1; } + snapshot > "$STATE/after.txt" + echo + say "mode: $(sed -n '/^## ipam/,$p' "$STATE/before.txt" | tail -1) -> $(sed -n '/^## ipam/,$p' "$STATE/after.txt" | tail -1)" + + # The question that decides the size of the maintenance window: did per-node + # CIDRs survive, or was every node renumbered (and every pod with it)? + local moved=0 + while IFS=$'\t' read -r node cidr; do + [ -z "${node:-}" ] && continue + local now; now="$(sed -n '/^## ciliumnodes/,/^## pods/p' "$STATE/after.txt" | awk -F'\t' -v n="$node" '$1==n{print $2}')" + if [ -n "$now" ] && [ "$now" != "$cidr" ]; then + printf ' %-16s %s -> %s\n' "$node" "$cidr" "$now"; moved=$((moved+1)) + fi + done < <(sed -n '/^## ciliumnodes/,/^## pods/p' "$STATE/before.txt" | grep -P '\t') + if [ "$moved" -eq 0 ]; then + say "per-node CIDRs UNCHANGED — the pool adopted the existing allocation" + else + warn "$moved node(s) renumbered — every pod on them must be recycled" + fi + + local before after same + before="$(sed -n '/^## pods/,/^## ipam/p' "$STATE/before.txt" | grep -P '\t10\.' | wc -l)" + after="$(sed -n '/^## pods/,/^## ipam/p' "$STATE/after.txt" | grep -P '\t10\.' | wc -l)" + same="$(comm -12 <(sed -n '/^## pods/,/^## ipam/p' "$STATE/before.txt" | grep -P '\t10\.' | sort) \ + <(sed -n '/^## pods/,/^## ipam/p' "$STATE/after.txt" | grep -P '\t10\.' | sort) | wc -l)" + say "pods: $before before, $after after, $same kept the SAME address" + # Keeping the address is NOT the good outcome. If a node's CIDR moved, its + # existing pods keep IPs that no longer fall inside it, every other node routes + # that prefix to the WRONG node, and those pods go unreachable cross-node while + # looking perfectly healthy. Observed in labsim: two nodes swapped CIDRs and + # cross-node ping to their pods dropped 100%, with every pod still Running. + # This is the check that decides whether a recycle is optional or mandatory. + local stranded=0 + while read -r ns name ip node; do + [ -z "${node:-}" ] && continue + local cidr; cidr="$($K get ciliumnode "$node" -o jsonpath='{.spec.ipam.podCIDRs[0]}' 2>/dev/null)" + [ -z "$cidr" ] && continue + case "$ip" in + "${cidr%.*/*}".*) ;; + *) printf ' STRANDED %-40s %-15s on %s (now %s)\n' "$ns/$name" "$ip" "$node" "$cidr"; stranded=$((stranded+1)) ;; + esac + done < <($K get pods -A -o jsonpath='{range .items[?(@.status.podIP)]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.status.podIP}{" "}{.spec.nodeName}{"\n"}{end}' 2>/dev/null | grep -E ' 10\.') + if [ "$stranded" -gt 0 ]; then + warn "$stranded pod(s) sit OUTSIDE their node CIDR — unreachable cross-node until recycled" + warn "recycle: for ns in $(kubectl get ns -o name | cut -d/ -f2); do kubectl -n $ns rollout restart deploy,ds,sts 2>/dev/null; done" + else + say "every pod is inside its node CIDR — no recycle needed" + fi + say "not-Running pods: $($K get pods -A --no-headers 2>/dev/null | grep -vcE 'Running|Completed')" +} + +case "${1:-}" in + preflight) cmd_preflight ;; + unstick) cmd_unstick ;; + verify) cmd_verify ;; + *) sed -n '2,16p' "$0"; exit 1 ;; +esac