lab/migration/he-tunnel-follow

#!/bin/bash
# Keep the Hurricane Electric 6in4 tunnel pointed at whichever WAN is live.
#
# The tunnel is anchored to a source IPv4. When failover moves the default route
# from the 10 gig to PPPoE, 6in4 packets keep leaving with the old source, HE
# drops them, and IPv6 goes dark while IPv4 keeps working -- a partial outage
# that presents as "some sites are broken", which is far worse to diagnose than
# a clean one.
#
# Installed on BOTH routers and gated on VRRP mastership: the backup exits
# immediately, and vrrp-wan-reconcile brings tun0 up and calls this script the
# moment it takes the VIP. The HE endpoint itself needs no update when the WAN
# moves between routers -- 87.192.101.48 is the 10 gig lease bound to the cloned
# MAC, so it follows the VIP to the other box unchanged (proven by the 2026-09-06
# drill). HE only has to be told about the WITHIN-box fall back to PPPoE.
#
# Changes are made at KERNEL level (`ip tunnel change`), not in VyOS config, on
# purpose:
#   - no commit per WAN flip, so a flapping line cannot churn the config;
#   - no drift against the Pulumi model, so `vyos-verify` stays meaningful;
#   - a reboot restores config.boot, which pins the 10 gig -- the correct
#     default -- so the wrong state cannot survive a restart.
#
#   he-tunnel-follow status     what is live vs what should be (read-only)
#   he-tunnel-follow run        reconcile, updating HE if the source changed
#   he-tunnel-follow run --dry  say what it would do, change nothing
#
# Credentials in /config/he-secrets (0600), NOT in git:
#   HE_USER=<tunnelbroker username>
#   HE_UPDATE_KEY=<from the tunnel's Advanced tab -- replaces the account password>
#   HE_TUNNEL_ID=<numeric tunnel id>
set -uo pipefail

TUNNEL="${TUNNEL:-tun0}"
SECRETS="${SECRETS:-/config/he-secrets}"
STATE="${STATE:-/run/he-tunnel-follow.state}"
ROLE_STATE="${ROLE_STATE:-/run/he-tunnel-follow.role}"

# HE's update endpoint, as a variable so labsim can point it at a stub. The sim
# has no public IPv4 and no HE account, which is the whole reason the tunnel was
# never rehearsed; with this the sim can exercise the HE-side half too.
HE_UPDATE_URL="${HE_UPDATE_URL:-https://ipv4.tunnelbroker.net/nic/update}"

# This script is installed on BOTH routers -- the same principle as the PPPoE
# gate: configured identically everywhere, gated at runtime. So it must know
# when it is the backup. Left ungated, the backup copy either dies on "no
# default route" every minute, or, far worse, sees its own idle pppoe0 address
# and points the HE endpoint at it. Vodafone hands out a different IPv4 on every
# dial, so that is an IPv6 blackhole plus a wasted write against a rate-limited
# API -- and it would fire on the backup, where nobody is looking.
#
# The VIP comes from the same /config/vrrp-wan.conf the reconciler and the
# health check read, so there is exactly one definition of "master" on the box.
WAN_CONF="${WAN_CONF:-/config/vrrp-wan.conf}"
# shellcheck disable=SC1090
[ -r "$WAN_CONF" ] && . "$WAN_CONF"
VIP="${VRRP_WAN_VIP:-192.168.1.1}"
# 6in4 costs 20 bytes. The 10 gig path is 1500 -> 1480; PPPoE is 1492 -> 1472.
# Getting this wrong is the classic "IPv6 works until something large" failure.
declare -A WAN_MTU=( ["bond0.53"]=1480 ["pppoe0"]=1472 )
# Require the same answer twice before acting. HE rate-limits updates, and a
# flapping WAN would otherwise hammer the API exactly when it is needed most.
HYSTERESIS="${HYSTERESIS:-2}"

log() { logger -t he-tunnel-follow -- "$*"; printf '  %s\n' "$*"; }
die() { logger -t he-tunnel-follow -p user.err -- "$*"; printf '  ERROR: %s\n' "$*" >&2; exit 1; }

holds_vip()  { ip -4 -o addr show 2>/dev/null | grep -q " ${VIP}/"; }
active_wan() { ip -4 route show default 2>/dev/null | awk '/^default/{for(i=1;i<=NF;i++) if($i=="dev") print $(i+1); exit}'; }
addr_of()    { ip -4 -br addr show "$1" 2>/dev/null | awk '{print $3}' | cut -d/ -f1; }
tunnel_src() { ip tunnel show "$TUNNEL" 2>/dev/null | sed -nE 's/.* local ([0-9.]+).*/\1/p'; }
tunnel_mtu() { cat "/sys/class/net/$TUNNEL/mtu" 2>/dev/null; }

# HE's dyndns-style endpoint. `myip` is passed EXPLICITLY rather than letting HE
# infer it from the request source: mid-failover the request itself may egress
# either line, and inferring would happily point the tunnel at the WAN we just
# left.
he_update() {
  local ip="$1"
  [ -r "$SECRETS" ] || die "no $SECRETS -- create it with HE_USER / HE_UPDATE_KEY / HE_TUNNEL_ID (0600)"
  # shellcheck disable=SC1090
  . "$SECRETS"
  [ -n "${HE_USER:-}" ] && [ -n "${HE_UPDATE_KEY:-}" ] && [ -n "${HE_TUNNEL_ID:-}" ] \
    || die "$SECRETS is missing HE_USER, HE_UPDATE_KEY or HE_TUNNEL_ID"

  local out
  out="$(curl -sS --max-time 25 \
      --data-urlencode "username=$HE_USER" \
      --data-urlencode "password=$HE_UPDATE_KEY" \
      --data-urlencode "hostname=$HE_TUNNEL_ID" \
      --data-urlencode "myip=$ip" \
      "$HE_UPDATE_URL" 2>&1)"
  # dyndns protocol: "good <ip>" or "nochg <ip>" are both success.
  case "$out" in
    good*|nochg*) log "HE endpoint set to $ip ($out)"; return 0 ;;
    *)            die "HE update refused: $out" ;;
  esac
}

# Log only when the role CHANGES. On a 1-minute timer an unconditional line
# would be 1440 entries a day on the backup, which is how a real message gets
# lost. The marker lives in /run, so a reboot re-announces the role once.
note_role() {
  local role="$1" last=""
  [ -r "$ROLE_STATE" ] && read -r last < "$ROLE_STATE"
  [ "$last" = "$role" ] && return 0
  echo "$role" > "$ROLE_STATE"
  log "role is now $role"
}

reconcile() {
  local dry="${1:-}"
  local wan src want_mtu cur_src cur_mtu

  # The backup owns nothing here. vrrp-wan-reconcile holds tun0 down on this box
  # and will run this script itself the moment it takes the VIP, so there is
  # nothing to do and nothing to say.
  if ! holds_vip; then
    note_role backup
    rm -f "$STATE"          # start a promoted box with a clean hysteresis count
    return 0
  fi
  note_role master

  wan="$(active_wan)"; [ -n "$wan" ] || die "no default route; refusing to guess"
  src="$(addr_of "$wan")"; [ -n "$src" ] || die "no IPv4 address on $wan"
  want_mtu="${WAN_MTU[$wan]:-}"
  [ -n "$want_mtu" ] || die "unknown WAN '$wan' -- add it to WAN_MTU rather than guessing an MTU"
  cur_src="$(tunnel_src)"; cur_mtu="$(tunnel_mtu)"

  if [ "$cur_src" = "$src" ] && [ "$cur_mtu" = "$want_mtu" ]; then
    rm -f "$STATE"
    log "in sync: $TUNNEL via $wan src $src mtu $cur_mtu"
    return 0
  fi

  # Hysteresis: count consecutive runs agreeing on the same target.
  local seen=0 last=""
  [ -r "$STATE" ] && { read -r last seen < "$STATE"; }
  if [ "$last" = "$src" ]; then seen=$((seen + 1)); else seen=1; fi
  echo "$src $seen" > "$STATE"
  if [ "$seen" -lt "$HYSTERESIS" ]; then
    log "change seen ($cur_src -> $src) but waiting for stability ($seen/$HYSTERESIS)"
    return 0
  fi

  if [ "$dry" = "--dry" ]; then
    log "DRY RUN: would set HE endpoint to $src, then $TUNNEL local $src mtu $want_mtu"
    return 0
  fi

  # HE first, then local. Either order costs a brief drop, but changing locally
  # first guarantees HE discards our packets for the whole window.
  he_update "$src" || return 1
  sudo ip tunnel change "$TUNNEL" mode sit local "$src" || die "failed to set tunnel local address"
  sudo ip link set "$TUNNEL" mtu "$want_mtu"            || die "failed to set tunnel MTU"
  rm -f "$STATE"
  log "moved $TUNNEL to $wan: src $cur_src -> $src, mtu $cur_mtu -> $want_mtu"
}

case "${1:-status}" in
  status)
    wan="$(active_wan)"
    printf '  role       : %s (vip %s)\n' "$(holds_vip && echo master || echo backup)" "$VIP"
    printf '  active WAN : %s\n' "${wan:-<none>}"
    printf '  wan addr   : %s\n' "$(addr_of "${wan:-lo}")"
    printf '  tunnel     : %s\n' "$(ip -br link show "$TUNNEL" 2>/dev/null | awk '{print $2}' || echo '<absent>')"
    printf '  tunnel src : %s\n' "$(tunnel_src)"
    # `${WAN_MTU[$wan]}` with an EMPTY subscript is a hard bash error --
    # "bad array subscript" -- not an empty expansion, and the :- default never
    # gets a chance to apply. A backup router has no default route, so `wan` is
    # empty there and `status` printed an error line on exactly the box whose
    # state you most need to read. Only index the array once there is a key.
    printf '  tunnel mtu : %s (want %s)\n' "$(tunnel_mtu)" \
        "$([ -n "${wan:-}" ] && echo "${WAN_MTU[$wan]:-?}" || echo '- (no WAN; this box is not master)')"
    printf '  he endpoint: %s\n' "$HE_UPDATE_URL"
    [ -r "$SECRETS" ] && printf '  credentials: present\n' || printf '  credentials: MISSING (%s)\n' "$SECRETS"
    ;;
  run) reconcile "${2:-}" ;;
  *)   die "usage: he-tunnel-follow {status|run [--dry]}" ;;
esac