lab/migration/he-tunnel-follow

#!/bin/bash
# Keep the Hurricane Electric 6in4 tunnel pointed at whichever WAN is live.
#
# The tunnel is anchored to a source IPv4. When failover moves the default route
# from the 10 gig to PPPoE, 6in4 packets keep leaving with the old source, HE
# drops them, and IPv6 goes dark while IPv4 keeps working -- a partial outage
# that presents as "some sites are broken", which is far worse to diagnose than
# a clean one.
#
# Changes are made at KERNEL level (`ip tunnel change`), not in VyOS config, on
# purpose:
#   - no commit per WAN flip, so a flapping line cannot churn the config;
#   - no drift against the Pulumi model, so `vyos-verify` stays meaningful;
#   - a reboot restores config.boot, which pins the 10 gig -- the correct
#     default -- so the wrong state cannot survive a restart.
#
#   he-tunnel-follow status     what is live vs what should be (read-only)
#   he-tunnel-follow run        reconcile, updating HE if the source changed
#   he-tunnel-follow run --dry  say what it would do, change nothing
#
# Credentials in /config/he-secrets (0600), NOT in git:
#   HE_USER=<tunnelbroker username>
#   HE_UPDATE_KEY=<from the tunnel's Advanced tab -- replaces the account password>
#   HE_TUNNEL_ID=<numeric tunnel id>
set -uo pipefail

TUNNEL="${TUNNEL:-tun0}"
SECRETS="${SECRETS:-/config/he-secrets}"
STATE="${STATE:-/run/he-tunnel-follow.state}"
# 6in4 costs 20 bytes. The 10 gig path is 1500 -> 1480; PPPoE is 1492 -> 1472.
# Getting this wrong is the classic "IPv6 works until something large" failure.
declare -A WAN_MTU=( ["bond0.53"]=1480 ["pppoe0"]=1472 )
# Require the same answer twice before acting. HE rate-limits updates, and a
# flapping WAN would otherwise hammer the API exactly when it is needed most.
HYSTERESIS="${HYSTERESIS:-2}"

log() { logger -t he-tunnel-follow -- "$*"; printf '  %s\n' "$*"; }
die() { logger -t he-tunnel-follow -p user.err -- "$*"; printf '  ERROR: %s\n' "$*" >&2; exit 1; }

active_wan() { ip -4 route show default 2>/dev/null | awk '/^default/{for(i=1;i<=NF;i++) if($i=="dev") print $(i+1); exit}'; }
addr_of()    { ip -4 -br addr show "$1" 2>/dev/null | awk '{print $3}' | cut -d/ -f1; }
tunnel_src() { ip tunnel show "$TUNNEL" 2>/dev/null | sed -nE 's/.* local ([0-9.]+).*/\1/p'; }
tunnel_mtu() { cat "/sys/class/net/$TUNNEL/mtu" 2>/dev/null; }

# HE's dyndns-style endpoint. `myip` is passed EXPLICITLY rather than letting HE
# infer it from the request source: mid-failover the request itself may egress
# either line, and inferring would happily point the tunnel at the WAN we just
# left.
he_update() {
  local ip="$1"
  [ -r "$SECRETS" ] || die "no $SECRETS -- create it with HE_USER / HE_UPDATE_KEY / HE_TUNNEL_ID (0600)"
  # shellcheck disable=SC1090
  . "$SECRETS"
  [ -n "${HE_USER:-}" ] && [ -n "${HE_UPDATE_KEY:-}" ] && [ -n "${HE_TUNNEL_ID:-}" ] \
    || die "$SECRETS is missing HE_USER, HE_UPDATE_KEY or HE_TUNNEL_ID"

  local out
  out="$(curl -sS --max-time 25 \
      --data-urlencode "username=$HE_USER" \
      --data-urlencode "password=$HE_UPDATE_KEY" \
      --data-urlencode "hostname=$HE_TUNNEL_ID" \
      --data-urlencode "myip=$ip" \
      "https://ipv4.tunnelbroker.net/nic/update" 2>&1)"
  # dyndns protocol: "good <ip>" or "nochg <ip>" are both success.
  case "$out" in
    good*|nochg*) log "HE endpoint set to $ip ($out)"; return 0 ;;
    *)            die "HE update refused: $out" ;;
  esac
}

reconcile() {
  local dry="${1:-}"
  local wan src want_mtu cur_src cur_mtu
  wan="$(active_wan)"; [ -n "$wan" ] || die "no default route; refusing to guess"
  src="$(addr_of "$wan")"; [ -n "$src" ] || die "no IPv4 address on $wan"
  want_mtu="${WAN_MTU[$wan]:-}"
  [ -n "$want_mtu" ] || die "unknown WAN '$wan' -- add it to WAN_MTU rather than guessing an MTU"
  cur_src="$(tunnel_src)"; cur_mtu="$(tunnel_mtu)"

  if [ "$cur_src" = "$src" ] && [ "$cur_mtu" = "$want_mtu" ]; then
    rm -f "$STATE"
    log "in sync: $TUNNEL via $wan src $src mtu $cur_mtu"
    return 0
  fi

  # Hysteresis: count consecutive runs agreeing on the same target.
  local seen=0 last=""
  [ -r "$STATE" ] && { read -r last seen < "$STATE"; }
  if [ "$last" = "$src" ]; then seen=$((seen + 1)); else seen=1; fi
  echo "$src $seen" > "$STATE"
  if [ "$seen" -lt "$HYSTERESIS" ]; then
    log "change seen ($cur_src -> $src) but waiting for stability ($seen/$HYSTERESIS)"
    return 0
  fi

  if [ "$dry" = "--dry" ]; then
    log "DRY RUN: would set HE endpoint to $src, then $TUNNEL local $src mtu $want_mtu"
    return 0
  fi

  # HE first, then local. Either order costs a brief drop, but changing locally
  # first guarantees HE discards our packets for the whole window.
  he_update "$src" || return 1
  sudo ip tunnel change "$TUNNEL" mode sit local "$src" || die "failed to set tunnel local address"
  sudo ip link set "$TUNNEL" mtu "$want_mtu"            || die "failed to set tunnel MTU"
  rm -f "$STATE"
  log "moved $TUNNEL to $wan: src $cur_src -> $src, mtu $cur_mtu -> $want_mtu"
}

case "${1:-status}" in
  status)
    wan="$(active_wan)"
    printf '  active WAN : %s\n' "${wan:-<none>}"
    printf '  wan addr   : %s\n' "$(addr_of "${wan:-lo}")"
    printf '  tunnel src : %s\n' "$(tunnel_src)"
    printf '  tunnel mtu : %s (want %s)\n' "$(tunnel_mtu)" "${WAN_MTU[${wan:-}]:-?}"
    [ -r "$SECRETS" ] && printf '  credentials: present\n' || printf '  credentials: MISSING (%s)\n' "$SECRETS"
    ;;
  run) reconcile "${2:-}" ;;
  *)   die "usage: he-tunnel-follow {status|run [--dry]}" ;;
esac