lab/migration/vlan2-v6-watchdog

#!/bin/bash
# Auto-revert the VLAN 2 IPv6 change if IPv4 stops working.
#
# The change it guards is additive and IPv6-only, so in theory it cannot affect
# IPv4 at all. This exists because "in theory" is exactly what has been wrong
# repeatedly, and because it is applied in an unattended window: the person who
# would notice is away, and a router that has lost IPv4 takes the house and the
# cluster with it.
#
# Modelled on wan-drill-watchdog: arm before the risky thing, disarm after, and
# in between let the ROUTER decide for itself rather than depending on anything
# off-box. A watchdog that needs the network it is protecting is not a watchdog.
#
#   vlan2-v6-watchdog arm [seconds]   start watching (default 2400 = 40 min)
#   vlan2-v6-watchdog disarm          stop
#   vlan2-v6-watchdog status          is it armed, and what has it seen
#
# Runs ON the router, out of /config, detached via setsid so it outlives the ssh
# session that started it.
set -uo pipefail

STATE=/run/vlan2-v6-watchdog
PIDF=$STATE/pid
LOGF=$STATE/log
V6_PREFIX="${V6_PREFIX:-2001:470:187e:2}"

# How long IPv4 must be continuously bad before reverting. Long enough that a
# commit's own brief disruption, or one lost probe, does not trigger it; short
# enough to matter. The reconciler ticks at 30s, so 90s is three chances.
GRACE="${GRACE:-90}"
PROBE="${PROBE:-9.9.9.9}"

log() { mkdir -p "$STATE"; printf '%s %s\n' "$(date -Is)" "$*" >> "$LOGF"; }

# IPv4 health, asked of the box itself -- and it MUST be role-aware.
#
# The first version required a default route plus internet on both routers, and
# would have reverted on vyos002 within 90s of being armed. That box is the
# gated BACKUP: by design it holds no VIP, has bond0.53 disabled and pppoe0
# down, so it has no default route and no internet, and that is the correct
# resting state rather than a fault. Caught at 20s of the 90s grace.
#
# So: the master must be able to reach the internet. The backup only has to
# still be on the network and able to see its peer -- which is what would
# actually be at risk if a VLAN 2 change went wrong.
VIP="${VIP:-192.168.1.1}"
holds_vip() { ip -4 -o addr show 2>/dev/null | grep -q " ${VIP}/"; }

v4_ok() {
    if holds_vip; then
        ip -4 route show default 2>/dev/null | grep -q . || return 1
        ping -c1 -W2 "$PROBE" >/dev/null 2>&1
    else
        # Backup: management address present and the VIP answers. If the VIP has
        # gone too then the pair has a bigger problem than this change, and
        # reverting an IPv6 addition would not help -- so this deliberately does
        # not fire on peer loss alone.
        ip -4 -o addr show bond0.1 2>/dev/null | grep -q 'inet ' || return 1
        ping -c1 -W2 "$VIP" >/dev/null 2>&1
    fi
}

revert() {
    log "REVERTING: IPv4 has been down for ${GRACE}s"
    # A script file, not vbash -c: the latter never starts a config session and
    # the commit fails to stderr where nobody sees it.
    cat > /tmp/vlan2-v6-revert.sh <<'REOF'
#!/bin/vbash
source /opt/vyatta/etc/functions/script-template
configure
delete service dhcpv6-server
delete service router-advert interface bond0.2
commit
save
exit
REOF
    chmod +x /tmp/vlan2-v6-revert.sh
    /tmp/vlan2-v6-revert.sh >> "$LOGF" 2>&1
    # The interface address is deleted separately: it is the one node whose
    # removal could plausibly disturb something else, so it goes last and its
    # failure does not block the rest.
    for n in 1 2; do
        ip -6 addr del "${V6_PREFIX}::${n}/64" dev bond0.2 2>/dev/null
    done
    log "revert done; IPv4 now $(v4_ok && echo OK || echo STILL BAD)"
}

watch_loop() {
    local deadline=$(( $(date +%s) + $1 )) bad=0
    log "armed for $1s (grace ${GRACE}s, probe ${PROBE})"
    while [ "$(date +%s)" -lt "$deadline" ]; do
        if v4_ok; then
            [ "$bad" -ne 0 ] && log "IPv4 recovered after ${bad}s"
            bad=0
        else
            bad=$((bad + 10))
            log "IPv4 bad (${bad}s/${GRACE}s)"
            if [ "$bad" -ge "$GRACE" ]; then
                revert
                log "disarming after revert"
                rm -f "$PIDF"
                return 0
            fi
        fi
        sleep 10
    done
    log "expired without incident"
    rm -f "$PIDF"
}

case "${1:-status}" in
    arm)
        mkdir -p "$STATE"
        [ -f "$PIDF" ] && kill -0 "$(cat "$PIDF")" 2>/dev/null && { echo "already armed"; exit 0; }
        setsid "$0" _run "${2:-2400}" >/dev/null 2>&1 < /dev/null &
        sleep 1
        [ -f "$PIDF" ] && echo "  armed (pid $(cat "$PIDF"), ${2:-2400}s)" || echo "  FAILED to arm"
        ;;
    _run)
        echo $$ > "$PIDF"
        watch_loop "${2:-2400}"
        ;;
    disarm)
        if [ -f "$PIDF" ]; then kill "$(cat "$PIDF")" 2>/dev/null; rm -f "$PIDF"; echo "  disarmed"
        else echo "  not armed"; fi
        ;;
    status)
        if [ -f "$PIDF" ] && kill -0 "$(cat "$PIDF")" 2>/dev/null; then echo "  armed (pid $(cat "$PIDF"))"
        else echo "  not armed"; fi
        echo "  --- log ---"; tail -12 "$LOGF" 2>/dev/null | sed 's/^/  /' || echo "  (none)"
        ;;
    *) echo "usage: $0 {arm [seconds]|disarm|status}" >&2; exit 2 ;;
esac