lab/migration/vyos-unifi-switch

#!/bin/bash
# Switch this VyOS box between passive (USG is the gateway) and active
# (VyOS is the gateway). Installed at /config/vyos-unifi-switch, which
# survives image upgrades, so it can be run from a local terminal or the
# JetKVM console with no workstation and no internet.
#
#   vyos-unifi-switch            report the current mode
#   vyos-unifi-switch vyos       take over: VIPs to .1, DHCP, DNS, PPPoE, NAT
#   vyos-unifi-switch unifi      revert; the USG can then be reconnected
#
# READ THIS BEFORE THE CUTOVER
#
# `unifi` is the escape hatch. It runs no health checks, asks no questions and
# has nothing that can refuse. If anything at all looks wrong, run it, then
# plug the USG back in.
#
# `vyos` commits with commit-confirm. If it is not confirmed -- because the
# health checks failed, or because you lost access, or because you walked away
# -- the box returns to the saved configuration on its own. That requires
# `system config-management commit-confirm action reload`; without it VyOS
# REBOOTS instead, which on a gateway is an outage rather than an undo. The
# script refuses to run if that setting is missing.
set -uo pipefail

MODES=/config/modes
UNIFI_BOOT="$MODES/unifi.boot"
DELTA="$MODES/to-vyos.commands"
SECRETS=/config/wan-secrets
MARKER="$MODES/current-mode"
CONFIRM_MINUTES="${CONFIRM_MINUTES:-10}"
OPRUN=/opt/vyatta/bin/vyatta-op-cmd-wrapper

say()  { printf '[switch] %s\n' "$*"; }
warn() { printf '[switch] WARNING: %s\n' "$*" >&2; }
die()  { printf '[switch] ERROR: %s\n' "$*" >&2; exit 1; }

# Run configuration commands in a real config session. Everything the caller
# feeds in runs between `configure` and `exit`.
run_cfg() {
  local script rc
  script="$(mktemp)"
  {
    echo 'source /opt/vyatta/etc/functions/script-template'
    echo 'configure'
    cat
    echo 'exit'
  } > "$script"
  vbash "$script"; rc=$?
  rm -f "$script"
  return $rc
}

# Two traps live in this one function, both of which produced wrong answers
# rather than errors:
#   1. `show configuration commands` quotes values ("action 'reload'"), so the
#      quotes have to go before matching or every value-bearing check fails.
#   2. `... | grep -q` under `set -o pipefail` reports FAILURE even on a match:
#      grep exits at the first hit, the producer takes SIGPIPE, and pipefail
#      surfaces that. Whether it triggers depends on output size, so it fails
#      intermittently. Match against a captured string instead of a pipeline.
cfg_has() {
  local out
  out="$($OPRUN show configuration commands 2>/dev/null | tr -d "'")"
  case "$out" in *"$1"*) return 0 ;; *) return 1 ;; esac
}

# ---------------------------------------------------------------- status ----
current_mode() {
  # The marker records intent; the running config is the truth. Report the
  # config, and complain if the two disagree.
  local live="unknown"
  if cfg_has "set service dhcp-server"; then live="vyos"; else live="unifi"; fi
  echo "$live"
}

show_status() {
  local live marked
  live="$(current_mode)"
  marked="$(cat "$MARKER" 2>/dev/null || echo "never set")"
  echo "host:          $(hostname)"
  echo "mode:          $live   (marker: $marked)"
  [ "$live" != "$marked" ] && [ "$marked" != "never set" ] && \
    warn "marker disagrees with the running config -- trust the config"
  echo "VRRP:"
  $OPRUN show vrrp 2>/dev/null | sed -n '3,$p' | awk '{printf "               %-9s %-12s %s\n", $1, $2, $4}'
  echo "DHCP server:   $(systemctl is-active isc-kea-dhcp4-server 2>/dev/null)"
  echo "DNS forwarder: $(systemctl is-active pdns-recursor 2>/dev/null)"
  echo "WAN (pppoe0):  $(ip -4 -br addr show pppoe0 2>/dev/null | awk '{print $2, $3}' || echo 'not present')"
  echo "default route: $(ip -4 route show default 2>/dev/null | head -1 || echo none)"
  echo "unsaved changes: $(cfg_unsaved)"
}

cfg_unsaved() {
  if /usr/bin/config-mgmt compare >/dev/null 2>&1; then echo "no"; else echo "possibly - check 'compare saved'"; fi
}

# ------------------------------------------------------------- preflight ----
require_files() {
  [ -r "$UNIFI_BOOT" ] || die "missing $UNIFI_BOOT -- capture it while the USG is still the gateway"
  [ -s "$UNIFI_BOOT" ] || die "$UNIFI_BOOT is empty"
}

require_reload_action() {
  cfg_has "set system config-management commit-confirm action reload" && return 0
  die "commit-confirm action is not 'reload'. Without it an unconfirmed switch
       REBOOTS this box instead of reverting. Fix first:
         configure
         set system config-management commit-confirm action reload
         commit; save"
}

# The single worst outcome available is two devices answering on the gateway
# address. It costs one ARP probe to make that impossible.
# Returns 0 (success) when something IS answering on a gateway address we are
# about to claim -- i.e. "yes, still alive, do not proceed".
usg_still_alive() {
  local found=0 ip dev targets
  targets="$(grep -oE "vrrp group [a-z0-9]+ address [0-9.]+" "$DELTA" 2>/dev/null | awk '{print $NF}')"
  if [ -z "$targets" ]; then
    # A delta with no VIPs cannot be probed, which means the single guard
    # against two devices sharing a gateway address is inert. Refuse by
    # default: an unprobeable delta on the real boxes is a broken delta, and
    # "warn and continue" would let the one failure this script exists to
    # prevent through unnoticed. The override is for the lab only.
    if [ "${ALLOW_NO_VIP_DELTA:-0}" = "1" ]; then
      warn "delta claims no VIPs; proceeding because ALLOW_NO_VIP_DELTA=1 (lab only)"
      return 1
    fi
    die "this delta claims no VIPs, so the gateway-address guard cannot run.
       On the real boxes that means a broken delta. If this really is a lab
       run, re-invoke with ALLOW_NO_VIP_DELTA=1."
  fi
  for ip in $targets; do
    dev="$(ip -4 route get "$ip" 2>/dev/null | sed -n 's/.* dev \([^ ]*\).*/\1/p' | head -1)"
    if [ -n "$dev" ] && command -v arping >/dev/null 2>&1; then
      # ARP is the right probe: it answers even when the host filters ICMP.
      if arping -c 2 -w 3 -f -I "$dev" "$ip" >/dev/null 2>&1; then
        warn "something already answers ARP on $ip (via $dev)"; found=1
      fi
    elif ping -c 2 -W 2 "$ip" >/dev/null 2>&1; then
      warn "something already answers ICMP on $ip"; found=1
    fi
  done
  [ "$found" -eq 1 ]
}

# ------------------------------------------------------------- to unifi -----
to_unifi() {
  require_files
  say "reverting to unifi mode (USG is the gateway)"
  run_cfg <<EOF || die "load/commit failed -- the box is unchanged, use the console"
load $UNIFI_BOOT
commit
save
EOF
  echo "unifi" > "$MARKER"
  say "done. The USG can be reconnected."
  say "If it was already connected during this, nothing was disturbed."
}

# -------------------------------------------------------------- to vyos -----
health_checks() {
  local fails=0
  _chk() { # name, command
    if eval "$2" >/dev/null 2>&1; then say "  ok    $1"; else say "  FAIL  $1"; fails=$((fails+1)); fi
  }
  _chk "kea (DHCP) is running"      "systemctl is-active --quiet isc-kea-dhcp4-server"
  _chk "DNS forwarder is running"   "systemctl is-active --quiet pdns-recursor"

  # Only assert on the WAN if this delta actually brings one up. A delta with
  # no PPPoE stanza is a lab/partial delta, and failing it on a missing
  # pppoe0 would make the script untestable anywhere but the live cutover.
  # Announced loudly, because a quietly skipped check is worse than no check.
  # A box whose delta holds its WAN interfaces DOWN is the backup: it has no
  # route out by design, and demanding one reverts a perfectly correct config.
  # This tore down vyos002 on the first successful cutover -- vyos001 went live
  # and its backup was judged unhealthy for lacking the WAN it is deliberately
  # not carrying. Same mistake as requiring the failover line: checks that do
  # not apply to the box being checked.
  if grep -qE "^set interfaces (pppoe pppoe0|bonding bond0 vif 53) disable$" "$DELTA"; then
    say "  note  this box holds its WAN down (backup); skipping WAN checks"
  elif grep -qE "^set interfaces (pppoe|bonding bond0 vif 5)" "$DELTA"; then
    # What matters is that SOME WAN works, not that every WAN works.
    #
    # This reverted a cutover that had genuinely succeeded. The 10 gig line came
    # up on bond0.53 and the cloned MAC was handed the same public address the
    # USG had (87.192.101.48); kea was serving real LAN clients at the same
    # moment. The only failure was pppoe0 -- the Vodafone FAILOVER line -- and
    # requiring it undid a working gateway.
    #
    # Written as [ -n "$(...)" ] rather than `... | grep -q` for the pipefail
    # reason above: a pipeline ending in grep -q cannot be trusted here.
    _chk "a default route exists"   '[ -n "$(ip -4 route show default)" ]'
    _chk "internet reachable"       "ping -c2 -W3 8.8.8.8"
    _chk "DNS resolves through us"  "getent hosts vyos.net"

    # Informational only: report each WAN, fail on neither. A failover line
    # being down is worth seeing, not worth reverting for.
    for _w in pppoe0 bond0.53; do
      if [ -n "$(ip -4 -br addr show "$_w" 2>/dev/null | awk '{print $3}')" ]; then
        say "  ok    WAN $_w has an address (informational)"
      else
        say "  note  WAN $_w has no address (informational, not fatal)"
      fi
    done
  else
    warn "this delta configures no WAN -- skipping all WAN health checks."
    warn "That is expected in the lab and WRONG for the real cutover."
  fi
  return $fails
}

to_vyos() {
  require_files
  require_reload_action
  [ -r "$DELTA" ] || die "missing $DELTA"

  if usg_still_alive; then
    die "refusing: something is still answering on a gateway address.
       Disconnect the USG first. Two devices on the same gateway IP is the
       one failure this script exists to prevent."
  fi

  local pw="" tmp
  if [ -r "$SECRETS" ]; then
    # shellcheck disable=SC1090
    . "$SECRETS"; pw="${WAN_PASSWORD:-}"
  fi
  [ -n "$pw" ] || warn "no WAN_PASSWORD in $SECRETS -- PPPoE will not authenticate"

  tmp="$(mktemp)"; chmod 600 "$tmp"
  sed "s|@@WAN_PASSWORD@@|${pw}|g" "$DELTA" | grep -vE '^\s*(#|$)' > "$tmp"

  say "switching to vyos mode (this box becomes the gateway)"
  say "commit-confirm: ${CONFIRM_MINUTES} min to confirm, else it reverts itself"

  # commit-confirm is TWO steps, and doing only the first commits nothing:
  # `config-mgmt commit_confirm` arms the revert timer, then a normal `commit`
  # applies the candidate config. The interactive prompt lives in the first
  # step, which is why it is invoked directly with -y instead of via the
  # `commit-confirm` alias. IN_COMMIT_CONFIRM is what the real CLI sets, and
  # the commit hooks look at it.
  if ! run_cfg < <(printf 'load %s\n' "$UNIFI_BOOT"; cat "$tmp";
                   printf 'sudo sg vyattacfg "/usr/bin/config-mgmt commit_confirm -y -t=%s"\n' "$CONFIRM_MINUTES";
                   printf 'export IN_COMMIT_CONFIRM=t\ncommit\nunset IN_COMMIT_CONFIRM\n'); then
    rm -f "$tmp"
    die "commit-confirm failed. Nothing was applied; the box is still in its
       previous mode. Run '$0 unifi' if you are unsure."
  fi
  rm -f "$tmp"

  # Poll, do not sample once.
  #
  # A cutover attempt failed here on a fixed 25s wait. That is far too short for
  # a WAN: PPPoE is PADI/PADO/PADR/PADS then LCP, auth and IPCP, routinely 15-30s
  # by itself, and both lines had just been released by the USG seconds earlier.
  # ISPs commonly hold the previous session and MAC binding for minutes before
  # leasing to the "same" CPE again -- which is precisely what a cloned MAC looks
  # like to them. One sample at 25s reported a healthy setup as broken and
  # reverted it.
  #
  # There is still a deadline, because commit-confirm is running: stop well
  # before it so the decision is ours rather than the timer's.
  local budget="${HEALTH_BUDGET:-180}" waited=0 step=15
  say "committed. Polling health for up to ${budget}s (commit-confirm has ${CONFIRM_MINUTES} min)..."
  while :; do
    sleep "$step"; waited=$(( waited + step ))
    if health_checks >/dev/null 2>&1; then
      say "healthy after ${waited}s"
      break
    fi
    if [ "$waited" -ge "$budget" ]; then
      say "still unhealthy after ${waited}s -- final check:"
      break
    fi
    say "  not healthy yet at ${waited}s, still waiting..."
  done

  say "health checks:"
  if health_checks; then
    say "all checks passed -- confirming"
    /usr/bin/config-mgmt confirm >/dev/null 2>&1 || die "confirm failed; it will revert on its own shortly"
    run_cfg <<'EOF' || warn "save failed -- config is live but will not survive a reboot"
save
EOF
    echo "vyos" > "$MARKER"
    say "vyos mode is live and saved."
  else
    warn "health checks FAILED -- reverting now rather than waiting out the timer"
    /usr/bin/config-mgmt revert_soft >/dev/null 2>&1 \
      || warn "revert_soft failed; the commit-confirm timer will still fire within ${CONFIRM_MINUTES} min"
    echo "unifi" > "$MARKER"
    die "reverted to the previous configuration. Reconnect the USG.
       Check: ip addr show pppoe0; journalctl -u pppd; $0 status"
  fi
}

# ----------------------------------------------------------------- main -----
case "${1:-status}" in
  vyos)   to_vyos ;;
  unifi)  to_unifi ;;
  status) show_status ;;
  *)      echo "usage: $(basename "$0") [vyos|unifi|status]" >&2; exit 2 ;;
esac