lab/migration/vrrp-wan-reconcile

#!/bin/sh
# Make the WAN match VRRP mastership. Idempotent; safe to run every 30s and on
# every VRRP transition.
#
# Two WANs, two different control planes, for a reason:
#
#   bond0.53 (10 gig, DHCP)  -- CONFIG plane. Its lease is bound to a cloned MAC
#                               (f0:9f:c2:12:9b:4f, the old USG's), and only VyOS
#                               config can move a MAC. One commit per failover.
#   pppoe0   (Vodafone)      -- SYSTEMD plane. Gated by a drop-in on
#                               ppp@pppoe0; see migration/ppp-vrrp-gate.conf.
#                               No commit, no config lock, no `save`.
#
# PPPoE used to be on the config plane too, via `set interfaces pppoe pppoe0
# disable`. That could not work: `disable` unlinks /etc/ppp/peers/pppoe0, which
# is pppd's options file, so the promotion path deleted the very thing it needed
# and left the unit restart-looping (observed: 47 restarts, zero sessions at the
# access concentrator).
#
# Why a reconciler and not just transition scripts: VyOS delivers
# `transition-script` through keepalived-fifo.py, and on 2026-09-02 that helper
# logged NOTHING for a promotion while Keepalived_vrrp logged all six instances
# entering MASTER. A router held every VIP with no WAN -- the outage, recreated
# by the mechanism meant to prevent it. Scripts give speed; the timer gives
# correctness.
#
#   vrrp-wan-reconcile          reconcile once
#   vrrp-wan-reconcile --status what it thinks, changing nothing

CONF=/config/vrrp-wan.conf
[ -r "$CONF" ] && . "$CONF"
VIP="${VRRP_WAN_VIP:-192.168.1.1}"
WAN_VIF="${WAN_VIF:-53}"
FLAP_MAX="${FLAP_MAX:-6}"
FLAP_WINDOW="${FLAP_WINDOW:-600}"
FLAP_HOLDOFF="${FLAP_HOLDOFF:-900}"

STATE=/run/vrrp-wan
LOCK=/run/vrrp-wan.lock
APPLY=/config/vrrp-wan-apply
DROPIN=/etc/systemd/system/ppp@pppoe0.service.d/10-vrrp-wan-gate.conf
V6_TUNNEL="${V6_TUNNEL:-tun0}"
RADVD_CONF="${RADVD_CONF:-/run/radvd/radvd.conf}"

cfg() { /opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands 2>/dev/null; }
holds_vip()   { ip -4 -o addr show 2>/dev/null | grep -q " ${VIP}/"; }
wan_up()      { ip -4 addr show "bond0.${WAN_VIF}" 2>/dev/null | grep -q 'inet '; }
ppp_up()      { ip -4 addr show pppoe0 2>/dev/null | grep -q 'inet '; }
ppp_active()  { systemctl is-active --quiet ppp@pppoe0 2>/dev/null; }
lease_age()   { s=$(stat -c %Y "$STATE/may-dial" 2>/dev/null) || return 1
                echo $(( $(date +%s) - s )); }

# NOTE: there is deliberately no ppp_disabled(). pppoe0 is now ENABLED in config
# on both routers, so such a test would be permanently false and the backup
# early-exit below would never fire -- entering config mode every 30s for ever,
# committing nothing. That exact shape was already live on the sim secondary,
# whose /tmp/vrrp-wan-commit.log read "No configuration changes to commit" while
# the script reported success.
wan_disabled(){ cfg | grep -q "vif ${WAN_VIF} disable"; }

if [ "${1:-}" = "--status" ]; then
    printf 'vip=%s holds_vip=%s wan_disabled=%s wan_up=%s ppp_up=%s ppp_active=%s may_dial=%s lease_age=%s dropin=%s role=%s tun=%s radvd=%s\n' \
        "$VIP" "$(holds_vip && echo yes || echo no)" \
        "$(wan_disabled && echo yes || echo no)" \
        "$(wan_up && echo yes || echo no)" \
        "$(ppp_up && echo yes || echo no)" \
        "$(ppp_active && echo yes || echo no)" \
        "$([ -f "$STATE/may-dial" ] && echo yes || echo no)" \
        "$(lease_age 2>/dev/null || echo -)" \
        "$([ -f "$DROPIN" ] && echo yes || echo MISSING)" \
        "$(cat "$STATE/role" 2>/dev/null || echo unset)" \
        "$(ip -br link show "$V6_TUNNEL" 2>/dev/null | awk '{print $2}' || echo absent)" \
        "$(systemctl is-active radvd 2>/dev/null || echo inactive)"
    exit 0
fi

# One writer. The lock fd MUST be closed for children (`9>&-` on every call):
# entering VyOS config mode spawns a long-lived unionfs-fuse for the session
# which INHERITS the descriptor and never releases it, so from the first commit
# onward every later run lost the flock and exited 0 having done nothing. That
# is how a demoted router kept the WAN.
exec 9>"$LOCK"
flock -n 9 || exit 0

mkdir -p "$STATE"

# Reap config sessions whose owning process is gone. VyOS leaves a unionfs mount
# per `configure`, one of them holds the commit lock, and after that EVERY commit
# fails -- including the manual one you try in order to fix it.
for d in /opt/vyatta/config/tmp/new_config_*; do
    [ -d "$d" ] || continue
    pid=${d##*_}
    kill -0 "$pid" 2>/dev/null && continue
    umount -l "$d" 2>/dev/null
    rm -rf "$d" 2>/dev/null
done

# `show configuration commands` has been observed returning EMPTY transiently
# under commit-lock contention. Every grep against it then reads false, which on
# the master path looks like "the WAN is disabled" and triggers a pointless
# commit -- one such spurious "releasing" was logged on a box where both WANs
# were already in the right state. A real config is ~500 lines; refuse to act on
# a suspiciously short one.
if [ "$(cfg | wc -l)" -lt 50 ]; then
    logger -t vrrp-wan "config read returned <50 lines; skipping this tick"
    exit 0
fi

# --- PPPoE: the systemd plane ---------------------------------------------
ppp_dial() {
    # An ESTABLISHED session outranks every guard below, and this must be the
    # first thing here. `may-dial` is a lease the guard expires after
    # LEASE_TTL, so any early `return 1` before this renew silently hands the
    # guard a live session to kill.
    #
    # Observed in labsim: the flap damper tripped, returned early, the lease
    # went stale at 81s > 75s and the guard hung up pppoe0 ON THE MASTER --
    # a damper meant to suppress repeat DIALS tore down a working WAN instead.
    # Everything below only decides whether to start a NEW session.
    if ppp_active; then
        touch "$STATE/may-dial"
        return 0
    fi
    # Refuse to bless a box whose gate is missing. /etc is per-image, so a VyOS
    # upgrade silently drops the drop-in -- and without it BOTH routers dial on
    # the next commit that touches the pppoe subtree. Failing closed turns a
    # silent loss of protection into "PPPoE never comes up", the safe direction.
    if [ ! -f "$DROPIN" ]; then
        logger -t vrrp-wan "REFUSING to dial: gate drop-in $DROPIN is missing (VyOS upgrade?)"
        return 1
    fi
    # The peers file is pppd's options file AND the gate's second condition, so
    # without it this box silently never dials: systemd logs "skipped because of
    # an unmet condition check" once and nothing else complains. Only a commit
    # that touches the pppoe subtree re-renders it.
    #
    # Seen in labsim: config applied but never `save`d, the router rebooted, and
    # came back with no pppoe0 node at all -- so no peers file, and a master that
    # dialled every tick into silence. Say so loudly rather than looking healthy.
    if [ ! -f /etc/ppp/peers/pppoe0 ]; then
        if cfg | grep -q "interfaces pppoe pppoe0 source-interface"; then
            logger -t vrrp-wan "CANNOT dial: pppoe0 is configured but /etc/ppp/peers/pppoe0 is missing -- re-commit the pppoe subtree to re-render it"
        else
            logger -t vrrp-wan "CANNOT dial: no pppoe0 in config (did a reboot revert an unsaved commit?)"
        fi
        return 1
    fi
    now=$(date +%s)
    if [ -f "$STATE/holdoff" ] && [ "$now" -lt "$(cat "$STATE/holdoff" 2>/dev/null || echo 0)" ]; then
        return 1
    fi
    touch "$STATE/may-dial"          # renew the lease every tick
    # Trim the dial log to the window, then decide.
    if [ -f "$STATE/dials" ]; then
        awk -v c="$((now - FLAP_WINDOW))" '$1 > c' "$STATE/dials" > "$STATE/dials.new" 2>/dev/null
        mv "$STATE/dials.new" "$STATE/dials" 2>/dev/null
    fi
    # `cat | wc`, not `wc -l < file`: the shell applies redirections left to
    # right, so a missing file fails the `<` BEFORE `2>/dev/null` is in effect
    # and dash prints "No such file or directory" on every first-ever dial.
    if [ "$(cat "$STATE/dials" 2>/dev/null | wc -l)" -ge "$FLAP_MAX" ]; then
        echo $((now + FLAP_HOLDOFF)) > "$STATE/holdoff"
        logger -t vrrp-wan "DIAL FLAP: >=${FLAP_MAX} attempts in ${FLAP_WINDOW}s -- holding off ${FLAP_HOLDOFF}s"
        return 1
    fi
    echo "$now" >> "$STATE/dials"
    logger -t vrrp-wan "MASTER: dialling pppoe0"
    systemctl reset-failed ppp@pppoe0 2>/dev/null
    # `systemctl start` exits 0 even when a Condition blocks the start, so its
    # return code proves nothing. is-active is the only honest answer.
    systemctl start ppp@pppoe0 2>/dev/null
}

ppp_release() {
    # Order matters: revoke the lease FIRST, then stop. The file's absence blocks
    # any NEW start (including one a concurrent VyOS commit would trigger); the
    # stop kills the process that already exists. Stopping first leaves a window
    # in which a commit re-dials a box that is being demoted.
    rm -f "$STATE/may-dial"
    ppp_active || return 0
    logger -t vrrp-wan "not MASTER: hanging up pppoe0"
    systemctl stop ppp@pppoe0 2>/dev/null
}

# --- IPv6: the kernel plane -------------------------------------------------
# The HE 6in4 tunnel and the VLAN 9 router advertisements have to follow
# mastership too, or a failover keeps IPv4 and silently drops IPv6 -- the
# partial outage that presents as "some sites are broken".
#
# A THIRD plane, and deliberately not either of the other two. Not config,
# because nothing here needs a commit (unlike the cloned MAC) and a commit per
# transition is the cost the pppoe0 half exists to avoid. Not the systemd gate,
# because there is no equivalent of a peers file to destroy.
#
# What makes this cheap: the tunnel is anchored to 87.192.101.48, the 10 gig
# lease bound to the cloned MAC, so it follows the VIP to the other router
# UNCHANGED. A router-level failover therefore needs no HE API call at all --
# only the link brought up on the box that now owns the address.
#
# Note what is deliberately NOT done here: this does not invoke
# he-tunnel-follow. That script has its own 1-minute task-scheduler cadence and
# a 2-tick hysteresis, and on 2026-09-06 that hysteresis was the only thing that
# stopped a routine `vif53-pin-boot-disable` run from pointing HE at a PPPoE
# address -- by 22 seconds. Calling it from a 30s reconciler as well would halve
# the window it needs. Its job is the WITHIN-box fall back to PPPoE; ours is
# link state.
v6_take() {
    # Absent on a router that has no tunnel in its config -- which is every
    # router until the model change lands. No-op there rather than complain.
    [ -e "/sys/class/net/$V6_TUNNEL" ] || return 0
    # Needs SOME WAN address to source from. Either line will do: if bond0.53 is
    # down but pppoe0 is up, he-tunnel-follow re-points the tunnel on its own
    # schedule, and holding the link down until then would turn a degraded path
    # into no path.
    wan_up || ppp_up || return 0
    ip link show "$V6_TUNNEL" 2>/dev/null | grep -q 'state DOWN' && {
        logger -t vrrp-wan "MASTER: bringing $V6_TUNNEL up"
        ip link set "$V6_TUNNEL" up 2>/dev/null
    }
    # radvd's config is rendered into /run by the VyOS commit, so on a box with
    # no router-advert node there is nothing to start.
    [ -f "$RADVD_CONF" ] || return 0
    systemctl is-active --quiet radvd 2>/dev/null && return 0
    logger -t vrrp-wan "MASTER: starting radvd"
    systemctl start radvd 2>/dev/null
}

v6_release() {
    [ -e "/sys/class/net/$V6_TUNNEL" ] || return 0
    # radvd FIRST, and this ordering is the point: on a graceful stop it emits a
    # final advertisement with router-lifetime 0, which is what tells VLAN 9
    # hosts to stop using this box as their default router. Kill the daemon
    # after tearing things down and they keep a dead gateway until the RA
    # lifetime expires on its own.
    if systemctl is-active --quiet radvd 2>/dev/null; then
        logger -t vrrp-wan "not MASTER: stopping radvd (deprecates the v6 gateway)"
        systemctl stop radvd 2>/dev/null
    fi
    ip link show "$V6_TUNNEL" 2>/dev/null | grep -q 'state DOWN' && return 0
    logger -t vrrp-wan "not MASTER: bringing $V6_TUNNEL down"
    ip link set "$V6_TUNNEL" down 2>/dev/null
}

# --- decide ----------------------------------------------------------------
if holds_vip; then
    echo master > "$STATE/role"
    [ -f "$STATE/since" ] || date +%s > "$STATE/since"
    ppp_dial
    # The WAN block is now an `if` rather than an early exit, so the IPv6 plane
    # below is reached on EVERY tick and not only on the one that enables the
    # WAN. On the promotion tick bond0.53 has no address yet, so v6_take no-ops
    # and the next tick takes it.
    if wan_disabled; then
        logger -t vrrp-wan "MASTER with bond0.${WAN_VIF} disabled -> enabling"
        t0=$(date +%s)
        "$APPLY" enable 9>&-
        logger -t vrrp-wan "bond0.${WAN_VIF} enable commit took $(( $(date +%s) - t0 ))s"
    fi
    v6_take
else
    echo backup > "$STATE/role"
    rm -f "$STATE/since" "$STATE/holdoff"
    ppp_release
    # Before the WAN goes, not after: once bond0.53 is disabled the source
    # address is gone and radvd's farewell advertisement has no path out.
    v6_release
    if ! wan_disabled; then
        logger -t vrrp-wan "not MASTER but bond0.${WAN_VIF} enabled -> releasing"
        t0=$(date +%s)
        "$APPLY" disable 9>&-
        logger -t vrrp-wan "bond0.${WAN_VIF} disable commit took $(( $(date +%s) - t0 ))s"
    fi
fi

# No `save`, deliberately. config.boot keeps `vif 53 disable` on BOTH routers, so
# a reboot in any order comes up unable to claim the cloned MAC. PPPoE needs no
# such convention any more: with the gate, config.boot is safe by construction
# and a stray `save` cannot make both boxes dial.