lab/migration/vif53-pin-boot-disable

#!/bin/sh
# Make config.boot carry `vif 53 disable` while the RUNNING config keeps the
# 10 gig up. Run on the master. Idempotent.
#
# WHY THIS IS AWKWARD
#
# The convention is that BOTH routers' config.boot hold `vif 53 disable`, so a
# reboot in any order comes up unable to claim the cloned MAC
# (f0:9f:c2:12:9b:4f) and vrrp-wan-reconcile then enables it on whichever box
# holds the VIP. The master's RUNNING config must have it enabled -- it is
# carrying the WAN -- so config.boot and the running config must deliberately
# disagree.
#
# VyOS gives no clean way to express that. `save` writes the RUNNING config,
# not the candidate: setting the node, saving, then discarding was tested in
# labsim and config.boot came back without `disable`, the WAN untouched. So the
# only route is to genuinely disable it, save that, and re-enable -- a real,
# brief interruption of the 10 gig.
#
# WHAT THE INTERRUPTION ACTUALLY COSTS
#
# Not the internet, on a healthy pair: pppoe0 is up on the master and the
# failover route falls to it while bond0.53 is down, which is exactly the
# behaviour T5 proves in labsim (LAN back in 5s). Traffic moves to the slower
# line and back. Expect a few seconds, plus however long the ISP takes to
# re-issue the DHCP lease afterwards.
#
# IF THIS SCRIPT DIES HALFWAY it leaves the master with bond0.53 disabled --
# which vrrp-wan-reconcile repairs within 30s ("MASTER with bond0.53 disabled
# -> enabling"). The failure mode is bounded by design, not by luck.
#
#     sudo /config/vif53-pin-boot-disable          do it
#     sudo /config/vif53-pin-boot-disable --check  report only, change nothing

VIF=53
BOOT=/config/config.boot

cfg() { /opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands 2>/dev/null; }
boot_has() { awk "/vif ${VIF} {/,/^        }/" "$BOOT" 2>/dev/null | grep -qc disable 2>/dev/null; }
boot_disable_count() { awk "/vif ${VIF} {/,/^        }/" "$BOOT" 2>/dev/null | grep -c disable; }
run_disable_count() { cfg | grep -c "vif ${VIF} disable"; }

report() {
    printf '  running config disable : %s\n' "$(run_disable_count)"
    printf '  config.boot   disable : %s\n' "$(boot_disable_count)"
    printf '  bond0.%s address      : %s\n' "$VIF" \
        "$(ip -4 addr show "bond0.${VIF}" 2>/dev/null | sed -n 's/.*inet \([0-9.]*\).*/\1/p')"
}

if [ "${1:-}" = "--check" ]; then report; exit 0; fi

if [ "$(boot_disable_count)" -ge 1 ]; then
    echo "  config.boot already pins 'vif ${VIF} disable' -- nothing to do"
    report
    exit 0
fi

# Refuse on a box that is not carrying the WAN: there the running config should
# already have `disable`, and a plain `save` is all that is needed. Doing the
# dance here would be pointless downtime.
if [ "$(run_disable_count)" -ge 1 ]; then
    echo "  this box already has 'vif ${VIF} disable' in the running config;"
    echo "  a plain 'save' is enough and costs nothing. Not touching the WAN."
    exit 0
fi

# Serialise against vrrp-wan-reconcile by taking ITS lock. Without this the two
# commit at the same time and VyOS refuses one of them with "Configuration
# system temporarily locked due to another commit in progress" -- observed in
# labsim, where the `save` landed but the RE-ENABLE did not, leaving the master
# with its 10 gig down. The reconciler uses `flock -n` and simply skips a tick
# it cannot get, so holding this is cheap and safe.
exec 9>/run/vrrp-wan.lock
if ! flock -w 60 9; then
    echo "  could not take /run/vrrp-wan.lock within 60s -- is a commit stuck?"
    echo "  refusing to race vrrp-wan-reconcile for the config lock."
    exit 1
fi

logger -t vif53-pin "pinning 'vif ${VIF} disable' into config.boot -- bond0.${VIF} will bounce"
echo "  bouncing bond0.${VIF} to get 'disable' into config.boot..."
t0=$(date +%s)

# 9>&- so the config session's long-lived unionfs-fuse child does not inherit
# the lock fd and hold it for ever -- the same trap vrrp-wan-reconcile documents.
/bin/vbash 9>&- <<VBASH
source /opt/vyatta/etc/functions/script-template
configure
set interfaces bonding bond0 vif ${VIF} disable
commit
save
delete interfaces bonding bond0 vif ${VIF} disable
commit
exit
VBASH
rc=$?

echo "  window: $(( $(date +%s) - t0 ))s (exit $rc)"
logger -t vif53-pin "done in $(( $(date +%s) - t0 ))s"
report

# Belt and braces: if the re-enable did not take, say so loudly. The reconciler
# will fix it within 30s on a master, but silence here would look like success.
if [ "$(run_disable_count)" -ge 1 ]; then
    echo "  *** WARNING: bond0.${VIF} is STILL disabled in the running config."
    echo "  *** vrrp-wan-reconcile should re-enable it within 30s on the master."
    echo "  *** Force it now with: sudo /config/vrrp-wan-reconcile"
    exit 1
fi
[ "$(boot_disable_count)" -ge 1 ] || { echo "  *** config.boot still not pinned"; exit 1; }
echo "  OK: config.boot pinned, running config still has the WAN"