#!/bin/bash
# A real Kubernetes cluster inside labsim, on the OVS fabric, for rehearsing
# Cilium <-> VyOS BGP before it goes near the production routers.
#
# Why VMs and not k3d: the thing under test is eBGP between Cilium and VyOS
# across the switch fabric — nodes on VLAN 2, peering with the router's bond0.2
# leg, directly connected. k3d would put the nodes on a container bridge, which
# is a different L2 path and would prove something else. (It also needs Docker;
# this host has podman.)
#
# Why not the existing micro VMs: they are Alpine with 256 MB and 1 vCPU. k3s
# plus Cilium needs an order of magnitude more, and a glibc distro with a stock
# kernel that Cilium's eBPF probes are actually tested against.
#
# Three nodes, not two: ECMP is only meaningfully tested if a node can be
# drained and MORE THAN ONE path survives.
#
# Layout (mirrors production's shape, not its addresses):
#   labsim-k8s1   172.31.2.11   k3s server
#   labsim-k8s2   172.31.2.12   agent
#   labsim-k8s3   172.31.2.13   agent
#   gateway       172.31.2.1    the VRRP VIP of the router pair under test
#   BGP peers     172.31.2.252 / .253   the routers' real per-box addresses
#
# Idempotent: re-running only creates what is missing.
#
# Usage:
#   ./k8s-up.sh              create/start the cluster
#   ./k8s-up.sh --kubeconfig fetch kubeconfig to ./labsim-k8s.kubeconfig
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "$SCRIPT_DIR/lib.sh"
source "$SCRIPT_DIR/ovs.sh"

# --- knobs ----------------------------------------------------------------
K8S_VLAN="${K8S_VLAN:-2}"
K8S_PREFIX="${K8S_PREFIX:-172.31.2}"
K8S_NODES="${K8S_NODES:-3}"
K8S_FIRST_OCTET="${K8S_FIRST_OCTET:-11}"
# Dual-stack, mirroring production's shape (not its addresses). VLAN 2 v6 is a
# ULA so nothing here can leak into the real HE /48; node fd00:2::1x matches the
# BGP peers in sim-net-config.py (K8S_NODES_V6). cluster/service v6 are ULAs too;
# the LB pool fd61:1e00::/64 is what Cilium advertises (SERVICE_CIDR_V6 there).
K8S_PREFIX_V6="${K8S_PREFIX_V6:-fd00:2}"        # nodes fd00:2::11/12/13, router ::252/::253
K8S_ROUTER_V6="${K8S_ROUTER_V6:-fd00:2::252}"   # v6 next-hop (primary router bond0.2)
CLUSTER_CIDR_V6="${CLUSTER_CIDR_V6:-fd00:42::/56}"
SERVICE_CIDR_V6="${SERVICE_CIDR_V6:-fd00:43::/112}"
K8S_MEM="${K8S_MEM:-4096}"          # MB — k3s + cilium + a workload
K8S_CPUS="${K8S_CPUS:-2}"
K8S_DISK_GB="${K8S_DISK_GB:-12}"
K8S_TOKEN="${K8S_TOKEN:-labsim-k3s-token}"

# Debian rather than Alpine: glibc, a stock kernel, and cloud-init that applies
# network-config properly (the Alpine base in this sim notably does not).
DEB_URL="${DEB_URL:-https://cloud.debian.org/images/cloud/trixie/latest/debian-13-genericcloud-amd64.qcow2}"
DEB_BASE="${DEB_BASE:-$IMG_DIR/debian-13-genericcloud-amd64.qcow2}"

# Same version production runs, so CRD shapes and chart flags transfer exactly.
CILIUM_VERSION="${CILIUM_VERSION:-1.19.1}"

node_name() { echo "labsim-k8s$1"; }
node_ip()   { echo "${K8S_PREFIX}.$((K8S_FIRST_OCTET + $1 - 1))"; }
node_ip6()  { echo "${K8S_PREFIX_V6}::$((K8S_FIRST_OCTET + $1 - 1))"; }

# --- base image -----------------------------------------------------------
ensure_base_image() {
  if [ -f "$DEB_BASE" ]; then
    log "base image present: $(basename "$DEB_BASE")"
    return
  fi
  log "fetching Debian cloud image (~330 MB) -> $DEB_BASE"
  sudo mkdir -p "$IMG_DIR"
  # .tmp + mv so an interrupted download never leaves a half image that later
  # runs treat as valid.
  sudo curl -fsSL --retry 3 -o "${DEB_BASE}.tmp" "$DEB_URL" \
    || die "could not fetch $DEB_URL"
  sudo mv "${DEB_BASE}.tmp" "$DEB_BASE"
  log "base image ready"
}

# --- cloud-init -----------------------------------------------------------
# The server node writes the join token; agents wait for the API to answer
# before joining, because cloud-init ordering across VMs is not guaranteed and
# a failed join leaves an agent that never retries.
build_k8s_seed() {
  local iso="$1" vm="$2" ip="$3" role="$4" server_ip="$5" pubkey="$6" ip6="$7" server_ip6="$8"
  local tmp; tmp="$(mktemp -d)"

  cat > "$tmp/meta-data" <<EOF
instance-id: $vm
local-hostname: $vm
EOF

  cat > "$tmp/network-config" <<EOF
version: 2
ethernets:
  enp1s0:
    match:
      name: "en*"
    addresses: [$ip/24, $ip6/64]
    routes:
      - to: default
        via: ${K8S_PREFIX}.1
    nameservers:
      addresses: [8.8.8.8, 1.1.1.1]
EOF

  local k3s_exec
  if [ "$role" = "server" ]; then
    # flannel/servicelb/traefik off: Cilium is the CNI under test, and k3s's
    # own ServiceLB would fight Cilium for LoadBalancer addresses. Dual-stack:
    # both families in cluster-cidr/service-cidr and a dual node-ip, mirroring
    # production. Pod IPs come from Cilium cluster-pool (node.spec.podCIDRs is
    # immutable), so cluster-cidr v6 just marks the cluster dual-stack.
    k3s_exec="server --flannel-backend=none --disable-network-policy --disable=servicelb --disable=traefik --node-ip=$ip,$ip6 --tls-san=$ip --cluster-cidr=10.42.0.0/16,${CLUSTER_CIDR_V6} --service-cidr=10.43.0.0/16,${SERVICE_CIDR_V6} --cluster-init"
  else
    k3s_exec="agent --server https://${server_ip}:6443 --node-ip=$ip,$ip6"
  fi

  cat > "$tmp/user-data" <<EOF
#cloud-config
hostname: $vm
fqdn: $vm
users:
  - name: debian
    groups: [sudo]
    shell: /bin/bash
    sudo: ["ALL=(ALL) NOPASSWD:ALL"]
    lock_passwd: false
    plain_text_passwd: labsim
    ssh_authorized_keys:
      - $pubkey
ssh_pwauth: true
disable_root: false
ssh_authorized_keys:
  - $pubkey

package_update: true
packages: [curl, jq, iproute2, tcpdump, bird2]

write_files:
  # Cilium replaces kube-proxy and needs these; Debian cloud images ship
  # neither loaded nor persisted.
  - path: /etc/modules-load.d/cilium.conf
    content: |
      br_netfilter
      overlay
  - path: /etc/sysctl.d/99-k8s.conf
    content: |
      net.ipv4.ip_forward = 1
      net.bridge.bridge-nf-call-iptables = 1

runcmd:
  - [ modprobe, br_netfilter ]
  - [ modprobe, overlay ]
  - [ sysctl, --system ]
  - |
    # Wait for the server's API before an agent tries to join. Without this the
    # agent fails once and the unit backs off for minutes.
    if [ "$role" != "server" ]; then
      for i in \$(seq 1 60); do
        curl -sk --max-time 3 https://${server_ip}:6443/ping >/dev/null 2>&1 && break
        sleep 5
      done
    fi
  - |
    curl -sfL https://get.k3s.io | \
      INSTALL_K3S_EXEC="$k3s_exec" \
      K3S_TOKEN="$K8S_TOKEN" \
      sh -
EOF

  sudo mkdir -p "$(dirname "$iso")"
  sudo genisoimage -quiet -output "$iso" -volid cidata -joliet -rock \
    "$tmp/user-data" "$tmp/meta-data" "$tmp/network-config"
  rm -rf "$tmp"
}

# --- VM creation ----------------------------------------------------------
create_node() {
  local n="$1" pubkey="$2"
  local vm; vm="$(node_name "$n")"
  local ip; ip="$(node_ip "$n")"
  local ip6; ip6="$(node_ip6 "$n")"
  local role="agent"; [ "$n" -eq 1 ] && role="server"
  local server_ip; server_ip="$(node_ip 1)"
  local server_ip6; server_ip6="$(node_ip6 1)"

  if virsh_q dominfo "$vm" >/dev/null 2>&1; then
    local state; state="$(virsh_q domstate "$vm" 2>/dev/null | head -1 | tr -d '\n')"
    if [ "$state" = "running" ]; then
      log "$vm already running ($ip, $role)"
    else
      log "$vm exists but is $state — starting"
      virsh_q start "$vm" >/dev/null
    fi
    return
  fi

  local disk="$IMG_DIR/${vm}.qcow2"
  local seed="$IMG_DIR/${vm}-seed.iso"

  log "creating $vm ($ip, $role, ${K8S_MEM}MB/${K8S_CPUS}cpu)"
  sudo qemu-img create -q -f qcow2 -F qcow2 -b "$DEB_BASE" "$disk" "${K8S_DISK_GB}G" >/dev/null
  build_k8s_seed "$seed" "$vm" "$ip" "$role" "$server_ip" "$pubkey" "$ip6" "$server_ip6"

  # Access port on the k8s VLAN — same broadcast domain as the routers'
  # bond0.2 leg, so BGP peering is directly connected exactly as in production.
  sudo virt-install --connect "$LIBVIRT_URI" --name "$vm" \
    --memory "$K8S_MEM" --vcpus "$K8S_CPUS" \
    --disk "path=$disk,format=qcow2,bus=virtio" \
    --disk "path=$seed,device=cdrom" \
    --network "network=$OVS_NET,portgroup=vlan${K8S_VLAN},model=virtio" \
    --os-variant debian12 \
    --graphics none --noautoconsole --import >/dev/null
}

fetch_kubeconfig() {
  local server_ip; server_ip="$(node_ip 1)"
  local out="$SCRIPT_DIR/labsim-k8s.kubeconfig"
  log "fetching kubeconfig from $server_ip"
  ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 \
      "debian@${server_ip}" "sudo cat /etc/rancher/k3s/k3s.yaml" \
    | sed "s|127.0.0.1|${server_ip}|" > "$out"
  chmod 600 "$out"
  log "wrote $out"
  log "use: KUBECONFIG=$out kubectl get nodes"
}

main() {
  if [ "${1:-}" = "--kubeconfig" ]; then
    fetch_kubeconfig
    return
  fi

  require_tools
  command -v genisoimage >/dev/null || die "genisoimage missing (dnf install genisoimage)"

  local pubkey; pubkey="$(find_ssh_pubkey)"
  log "using SSH key: ${pubkey%% *} ...${pubkey##* }"

  ensure_base_image

  # Select EVERY VLAN, not just the k8s one. ovs_up re-defines the libvirt
  # network from SELECTED, so narrowing it here silently drops the portgroups
  # for every other VLAN -- running VMs keep working (their taps are already
  # attached) and nothing complains until the next VM cannot be attached.
  # Observed: this deleted vlan1/3/9/10/200/51/53 and only surfaced when the
  # ISP VMs needed vlan51 and vlan53.
  selected_vlans
  log "ensuring OVS fabric (all VLANs, so no portgroup is dropped)"
  ovs_up

  for n in $(seq 1 "$K8S_NODES"); do
    create_node "$n" "$pubkey"
  done

  echo
  log "nodes created. k3s installs on first boot (a few minutes)."
  log "watch:   ssh debian@$(node_ip 1) 'sudo systemctl status k3s'"
  log "then:    $0 --kubeconfig"
  log "then install Cilium $CILIUM_VERSION and the BGP resources (see README)."
}

main "$@"