lab/labsim/cilium-ipam-switch.sh

#!/usr/bin/env bash
# Procedure around a Cilium IPAM mode change. Works against any cluster, so the
# rehearsal in labsim and the real thing in production run the SAME steps.
#
# It deliberately does NOT change the mode itself. In labsim that is `helm
# upgrade`; in production Pulumi owns the release and a script racing it would
# just reintroduce drift. What this owns is everything around the apply -- the
# evidence, the deadlock, and the verdict.
#
#   ./cilium-ipam-switch.sh preflight   record what the cluster looks like now
#   ./cilium-ipam-switch.sh unstick     break the agent-not-ready taint deadlock
#   ./cilium-ipam-switch.sh verify      compare against preflight, report renumbering
#
#   KUBECONFIG=... ./cilium-ipam-switch.sh preflight
#
# Whether a recycle is needed is CONDITIONAL, and `verify` is what decides it.
#
# The operator does not preserve which node held which /24 -- it adopts whatever
# CiliumNode.spec.ipam.podCIDRs already says. So:
#
#   * If CiliumNode already agrees with node.spec.podCIDRs on every node -- which
#     is the case for any cluster that has only ever run ipam=kubernetes, because
#     the operator syncs one from the other -- the pool adopts the existing
#     allocation, no node is renumbered, and NO pod recycle is needed. Verified
#     on the 3-node labsim cluster: CIDRs unchanged, nothing stranded, the only
#     blip was the cilium DaemonSet restarting itself.
#
#   * If the two sources DISAGREE, nodes can swap /24s. Their running pods keep
#     addresses that no longer fall inside the node's range, every other node
#     routes that prefix to the wrong node, and those pods go unreachable
#     cross-node while still showing Running. Then a full recycle is mandatory.
#
# Do not skip `verify` on the assumption of the good case. Run it and read it.
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
STATE="${STATE:-$SCRIPT_DIR/.ipam-switch-state}"
K="kubectl"

say()  { printf '\033[0;36m[ipam]\033[0m %s\n' "$*"; }
warn() { printf '\033[1;33m[ipam]\033[0m %s\n' "$*" >&2; }

snapshot() {
  echo "## nodes"
  $K get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.podCIDRs}{"\n"}{end}' 2>/dev/null
  echo "## ciliumnodes"
  $K get ciliumnode -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.ipam.podCIDRs}{"\n"}{end}' 2>/dev/null
  echo "## pods"
  $K get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\t"}{.status.podIP}{"\n"}{end}' 2>/dev/null \
    | grep -vP '\t$' | sort
  echo "## ipam"
  $K -n kube-system get cm cilium-config -o jsonpath='{.data.ipam}' 2>/dev/null; echo
}

cmd_preflight() {
  mkdir -p "$STATE"
  snapshot > "$STATE/before.txt"
  say "recorded $(grep -c . "$STATE/before.txt") lines -> $STATE/before.txt"
  say "mode now: $(sed -n '/^## ipam/,$p' "$STATE/before.txt" | tail -1)"
  # The pod inventory is the rollback reference: if the switch renumbers, this
  # is the only record of what an address USED to be.
  say "pods on the pod network: $(sed -n '/^## pods/,/^## ipam/p' "$STATE/before.txt" | grep -c '10\.')"
}

# The deadlock, in one place because it WILL happen and doing it by hand under
# time pressure is how the wrong node gets untainted:
#   agent has no pod CIDR -> agent not ready -> node keeps
#   node.cilium.io/agent-not-ready:NoSchedule -> the operator that would assign
#   the CIDR cannot schedule -> agent still has no pod CIDR.
# Removing the taint is safe: it exists to keep normal workloads off a node
# without working networking, and the operator is precisely the thing that fixes
# that. Kubernetes re-adds it on the next agent restart.
cmd_unstick() {
  local stuck=0
  for n in $($K get nodes -o name 2>/dev/null); do
    $K get "$n" -o jsonpath='{.spec.taints[*].key}' 2>/dev/null | grep -q 'agent-not-ready' || continue
    warn "${n#node/} carries agent-not-ready; removing so the operator can schedule"
    $K taint "$n" node.cilium.io/agent-not-ready- >/dev/null 2>&1 && stuck=$((stuck+1))
  done
  [ "$stuck" -eq 0 ] && say "no node was stuck" || say "cleared $stuck node(s)"
  local pend
  pend="$($K -n kube-system get pods -l io.cilium/app=operator --no-headers 2>/dev/null | grep -c Pending)"
  [ "${pend:-0}" -gt 0 ] && warn "$pend operator pod(s) still Pending — check tolerations, not just taints"
  return 0
}

cmd_verify() {
  [ -f "$STATE/before.txt" ] || { warn "no preflight snapshot; nothing to compare"; return 1; }
  snapshot > "$STATE/after.txt"
  echo
  say "mode: $(sed -n '/^## ipam/,$p' "$STATE/before.txt" | tail -1) -> $(sed -n '/^## ipam/,$p' "$STATE/after.txt" | tail -1)"

  # The question that decides the size of the maintenance window: did per-node
  # CIDRs survive, or was every node renumbered (and every pod with it)?
  local moved=0
  while IFS=$'\t' read -r node cidr; do
    [ -z "${node:-}" ] && continue
    local now; now="$(sed -n '/^## ciliumnodes/,/^## pods/p' "$STATE/after.txt" | awk -F'\t' -v n="$node" '$1==n{print $2}')"
    if [ -n "$now" ] && [ "$now" != "$cidr" ]; then
      printf '    %-16s %s -> %s\n' "$node" "$cidr" "$now"; moved=$((moved+1))
    fi
  done < <(sed -n '/^## ciliumnodes/,/^## pods/p' "$STATE/before.txt" | grep -P '\t')
  if [ "$moved" -eq 0 ]; then
    say "per-node CIDRs UNCHANGED — the pool adopted the existing allocation"
  else
    warn "$moved node(s) renumbered — every pod on them must be recycled"
  fi

  local before after same
  before="$(sed -n '/^## pods/,/^## ipam/p' "$STATE/before.txt" | grep -P '\t10\.' | wc -l)"
  after="$(sed -n '/^## pods/,/^## ipam/p' "$STATE/after.txt"  | grep -P '\t10\.' | wc -l)"
  same="$(comm -12 <(sed -n '/^## pods/,/^## ipam/p' "$STATE/before.txt" | grep -P '\t10\.' | sort) \
                   <(sed -n '/^## pods/,/^## ipam/p' "$STATE/after.txt"  | grep -P '\t10\.' | sort) | wc -l)"
  say "pods: $before before, $after after, $same kept the SAME address"
  # Keeping the address is NOT the good outcome. If a node's CIDR moved, its
  # existing pods keep IPs that no longer fall inside it, every other node routes
  # that prefix to the WRONG node, and those pods go unreachable cross-node while
  # looking perfectly healthy. Observed in labsim: two nodes swapped CIDRs and
  # cross-node ping to their pods dropped 100%, with every pod still Running.
  # This is the check that decides whether a recycle is optional or mandatory.
  local stranded=0
  while read -r ns name ip node; do
    [ -z "${node:-}" ] && continue
    local cidr; cidr="$($K get ciliumnode "$node" -o jsonpath='{.spec.ipam.podCIDRs[0]}' 2>/dev/null)"
    [ -z "$cidr" ] && continue
    case "$ip" in
      "${cidr%.*/*}".*) ;;
      *) printf '    STRANDED %-40s %-15s on %s (now %s)\n' "$ns/$name" "$ip" "$node" "$cidr"; stranded=$((stranded+1)) ;;
    esac
  done < <($K get pods -A -o jsonpath='{range .items[?(@.status.podIP)]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.status.podIP}{" "}{.spec.nodeName}{"\n"}{end}' 2>/dev/null | grep -E ' 10\.')
  if [ "$stranded" -gt 0 ]; then
    warn "$stranded pod(s) sit OUTSIDE their node CIDR — unreachable cross-node until recycled"
    warn "recycle: for ns in $(kubectl get ns -o name | cut -d/ -f2); do kubectl -n $ns rollout restart deploy,ds,sts 2>/dev/null; done"
  else
    say "every pod is inside its node CIDR — no recycle needed"
  fi
  say "not-Running pods: $($K get pods -A --no-headers 2>/dev/null | grep -vcE 'Running|Completed')"
}

case "${1:-}" in
  preflight) cmd_preflight ;;
  unstick)   cmd_unstick ;;
  verify)    cmd_verify ;;
  *) sed -n '2,16p' "$0"; exit 1 ;;
esac