lab/migration/vrrp-wan-reconcile

#!/bin/sh
# Make the WAN match VRRP mastership. Idempotent; safe to run every 30s and on
# every VRRP transition.
#
# Two WANs, two different control planes, for a reason:
#
#   bond0.53 (10 gig, DHCP)  -- CONFIG plane. Its lease is bound to a cloned MAC
#                               (f0:9f:c2:12:9b:4f, the old USG's), and only VyOS
#                               config can move a MAC. One commit per failover.
#   pppoe0   (Vodafone)      -- SYSTEMD plane. Gated by a drop-in on
#                               ppp@pppoe0; see migration/ppp-vrrp-gate.conf.
#                               No commit, no config lock, no `save`.
#
# PPPoE used to be on the config plane too, via `set interfaces pppoe pppoe0
# disable`. That could not work: `disable` unlinks /etc/ppp/peers/pppoe0, which
# is pppd's options file, so the promotion path deleted the very thing it needed
# and left the unit restart-looping (observed: 47 restarts, zero sessions at the
# access concentrator).
#
# Why a reconciler and not just transition scripts: VyOS delivers
# `transition-script` through keepalived-fifo.py, and on 2026-09-02 that helper
# logged NOTHING for a promotion while Keepalived_vrrp logged all six instances
# entering MASTER. A router held every VIP with no WAN -- the outage, recreated
# by the mechanism meant to prevent it. Scripts give speed; the timer gives
# correctness.
#
#   vrrp-wan-reconcile          reconcile once
#   vrrp-wan-reconcile --status what it thinks, changing nothing

CONF=/config/vrrp-wan.conf
[ -r "$CONF" ] && . "$CONF"
VIP="${VRRP_WAN_VIP:-192.168.1.1}"
WAN_VIF="${WAN_VIF:-53}"
FLAP_MAX="${FLAP_MAX:-6}"
FLAP_WINDOW="${FLAP_WINDOW:-600}"
FLAP_HOLDOFF="${FLAP_HOLDOFF:-900}"

STATE=/run/vrrp-wan
LOCK=/run/vrrp-wan.lock
APPLY=/config/vrrp-wan-apply
DROPIN=/etc/systemd/system/ppp@pppoe0.service.d/10-vrrp-wan-gate.conf

cfg() { /opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands 2>/dev/null; }
holds_vip()   { ip -4 -o addr show 2>/dev/null | grep -q " ${VIP}/"; }
wan_up()      { ip -4 addr show "bond0.${WAN_VIF}" 2>/dev/null | grep -q 'inet '; }
ppp_up()      { ip -4 addr show pppoe0 2>/dev/null | grep -q 'inet '; }
ppp_active()  { systemctl is-active --quiet ppp@pppoe0 2>/dev/null; }
lease_age()   { s=$(stat -c %Y "$STATE/may-dial" 2>/dev/null) || return 1
                echo $(( $(date +%s) - s )); }

# NOTE: there is deliberately no ppp_disabled(). pppoe0 is now ENABLED in config
# on both routers, so such a test would be permanently false and the backup
# early-exit below would never fire -- entering config mode every 30s for ever,
# committing nothing. That exact shape was already live on the sim secondary,
# whose /tmp/vrrp-wan-commit.log read "No configuration changes to commit" while
# the script reported success.
wan_disabled(){ cfg | grep -q "vif ${WAN_VIF} disable"; }

if [ "${1:-}" = "--status" ]; then
    printf 'vip=%s holds_vip=%s wan_disabled=%s wan_up=%s ppp_up=%s ppp_active=%s may_dial=%s lease_age=%s dropin=%s role=%s\n' \
        "$VIP" "$(holds_vip && echo yes || echo no)" \
        "$(wan_disabled && echo yes || echo no)" \
        "$(wan_up && echo yes || echo no)" \
        "$(ppp_up && echo yes || echo no)" \
        "$(ppp_active && echo yes || echo no)" \
        "$([ -f "$STATE/may-dial" ] && echo yes || echo no)" \
        "$(lease_age 2>/dev/null || echo -)" \
        "$([ -f "$DROPIN" ] && echo yes || echo MISSING)" \
        "$(cat "$STATE/role" 2>/dev/null || echo unset)"
    exit 0
fi

# One writer. The lock fd MUST be closed for children (`9>&-` on every call):
# entering VyOS config mode spawns a long-lived unionfs-fuse for the session
# which INHERITS the descriptor and never releases it, so from the first commit
# onward every later run lost the flock and exited 0 having done nothing. That
# is how a demoted router kept the WAN.
exec 9>"$LOCK"
flock -n 9 || exit 0

mkdir -p "$STATE"

# Reap config sessions whose owning process is gone. VyOS leaves a unionfs mount
# per `configure`, one of them holds the commit lock, and after that EVERY commit
# fails -- including the manual one you try in order to fix it.
for d in /opt/vyatta/config/tmp/new_config_*; do
    [ -d "$d" ] || continue
    pid=${d##*_}
    kill -0 "$pid" 2>/dev/null && continue
    umount -l "$d" 2>/dev/null
    rm -rf "$d" 2>/dev/null
done

# `show configuration commands` has been observed returning EMPTY transiently
# under commit-lock contention. Every grep against it then reads false, which on
# the master path looks like "the WAN is disabled" and triggers a pointless
# commit -- one such spurious "releasing" was logged on a box where both WANs
# were already in the right state. A real config is ~500 lines; refuse to act on
# a suspiciously short one.
if [ "$(cfg | wc -l)" -lt 50 ]; then
    logger -t vrrp-wan "config read returned <50 lines; skipping this tick"
    exit 0
fi

# --- PPPoE: the systemd plane ---------------------------------------------
ppp_dial() {
    # Refuse to bless a box whose gate is missing. /etc is per-image, so a VyOS
    # upgrade silently drops the drop-in -- and without it BOTH routers dial on
    # the next commit that touches the pppoe subtree. Failing closed turns a
    # silent loss of protection into "PPPoE never comes up", the safe direction.
    if [ ! -f "$DROPIN" ]; then
        logger -t vrrp-wan "REFUSING to dial: gate drop-in $DROPIN is missing (VyOS upgrade?)"
        return 1
    fi
    now=$(date +%s)
    if [ -f "$STATE/holdoff" ] && [ "$now" -lt "$(cat "$STATE/holdoff" 2>/dev/null || echo 0)" ]; then
        return 1
    fi
    touch "$STATE/may-dial"          # renew the lease every tick
    ppp_active && return 0
    # Trim the dial log to the window, then decide.
    if [ -f "$STATE/dials" ]; then
        awk -v c="$((now - FLAP_WINDOW))" '$1 > c' "$STATE/dials" > "$STATE/dials.new" 2>/dev/null
        mv "$STATE/dials.new" "$STATE/dials" 2>/dev/null
    fi
    if [ "$(wc -l < "$STATE/dials" 2>/dev/null || echo 0)" -ge "$FLAP_MAX" ]; then
        echo $((now + FLAP_HOLDOFF)) > "$STATE/holdoff"
        logger -t vrrp-wan "DIAL FLAP: >=${FLAP_MAX} attempts in ${FLAP_WINDOW}s -- holding off ${FLAP_HOLDOFF}s"
        return 1
    fi
    echo "$now" >> "$STATE/dials"
    logger -t vrrp-wan "MASTER: dialling pppoe0"
    systemctl reset-failed ppp@pppoe0 2>/dev/null
    # `systemctl start` exits 0 even when a Condition blocks the start, so its
    # return code proves nothing. is-active is the only honest answer.
    systemctl start ppp@pppoe0 2>/dev/null
}

ppp_release() {
    # Order matters: revoke the lease FIRST, then stop. The file's absence blocks
    # any NEW start (including one a concurrent VyOS commit would trigger); the
    # stop kills the process that already exists. Stopping first leaves a window
    # in which a commit re-dials a box that is being demoted.
    rm -f "$STATE/may-dial"
    ppp_active || return 0
    logger -t vrrp-wan "not MASTER: hanging up pppoe0"
    systemctl stop ppp@pppoe0 2>/dev/null
}

# --- decide ----------------------------------------------------------------
if holds_vip; then
    echo master > "$STATE/role"
    [ -f "$STATE/since" ] || date +%s > "$STATE/since"
    ppp_dial
    wan_disabled || exit 0
    logger -t vrrp-wan "MASTER with bond0.${WAN_VIF} disabled -> enabling"
    t0=$(date +%s)
    "$APPLY" enable 9>&-
    logger -t vrrp-wan "bond0.${WAN_VIF} enable commit took $(( $(date +%s) - t0 ))s"
else
    echo backup > "$STATE/role"
    rm -f "$STATE/since" "$STATE/holdoff"
    ppp_release
    wan_disabled && exit 0
    logger -t vrrp-wan "not MASTER but bond0.${WAN_VIF} enabled -> releasing"
    t0=$(date +%s)
    "$APPLY" disable 9>&-
    logger -t vrrp-wan "bond0.${WAN_VIF} disable commit took $(( $(date +%s) - t0 ))s"
fi

# No `save`, deliberately. config.boot keeps `vif 53 disable` on BOTH routers, so
# a reboot in any order comes up unable to claim the cloned MAC. PPPoE needs no
# such convention any more: with the gate, config.boot is safe by construction
# and a stray `save` cannot make both boxes dial.