lab/labsim/labsim-dualstack-net.sh

#!/bin/bash
# VLAN 2 gets IPv6 in labsim: addresses, router advertisements and a DHCPv6
# server with per-MAC reservations.
#
# This is the rehearsal of the production change (dual-stack plan, phase 2b) and
# runs the SAME VyOS config, against the sim router pair, so the production
# apply is a repeat rather than a first attempt.
#
# WHY DHCPv6 AND NOT SLAAC. The cluster needs each node's IPv6 to be knowable in
# advance and stable: k3s resolves node-ip once at start-up, and a node's
# identity cannot be allowed to change under it. The estate already answers that
# question for IPv4 with kea reservations keyed on MAC, so IPv6 answers it the
# same way and stays one source of truth.
#
# DHCPv6 normally keys on DUID, not MAC -- a DUID is generated by the client and
# is not derivable from its MAC, which would have meant a second, client-owned
# source of truth. VyOS's static-mapping accepts `mac` as well as `duid`
# (verified on the sim: the node.tag directory offers duid, mac, ipv6-address,
# ipv6-prefix), so the reservation can key on the same MAC the v4 one does.
#
# The RA carries managed-flag WITH no-autonomous-flag. That combination is what
# makes the node's address unambiguous: managed sends it to DHCPv6, and
# non-autonomous stops it also forming a SLAAC address from the same prefix.
# Leave autonomous on and every node has two global addresses, only one of which
# anybody reserved -- and whichever labctl happens to find is the one that ends
# up in node-ip.
#
# Both are VALUELESS nodes on this VyOS: `managed-flag` not `managed-flag true`,
# and `prefix <p> no-autonomous-flag` not `autonomous-flag false`. The value
# forms are rejected with "is not valid".
#
# A FAILED COMMIT DOES NOT MEAN NOTHING CHANGED. VyOS commits node groups
# independently, so an earlier run of this script left the bond0.2 address and
# most of the router-advert block applied while `[[service dhcpv6-server]]
# failed` -- the interface and RA groups had already succeeded. Re-read the
# config after any failure rather than assuming a clean rollback; that matters
# more in production than here.
#
# TWO THINGS THIS REHEARSAL FOUND, both of which would have bitten production:
#
# 1. managed-flag is NECESSARY BUT NOT SUFFICIENT. It only tells the host to use
#    DHCPv6; the kernel's accept_ra implements SLAAC and nothing else, so a node
#    with no DHCPv6 *client* running takes no lease at all. The sim's Debian
#    nodes have neither NetworkManager nor a networkd .network file managing the
#    interface, so they ended up with a kernel SLAAC address
#    (2001:db8:187e:2:5054:ff:fe53:731 -- EUI-64 from the MAC) and never asked
#    for the reserved ::11. Production's Fedora nodes use NetworkManager, which
#    does run a DHCPv6 client on the managed flag, and the DGX Sparks run
#    NetworkManager too -- but that must be VERIFIED per node, not assumed. The
#    k3s module's preflight catches the consequence; the fix is node-side.
#
# 2. TURNING AUTONOMOUS OFF DOES NOT RETRACT ADDRESSES ALREADY FORMED. An
#    earlier partial apply advertised the prefix while autonomous was still on;
#    the nodes autoconfigured, and adding no-autonomous-flag afterwards left
#    those addresses in place with a 30-day valid lifetime. So in production,
#    where VLAN 2 has no IPv6 at all yet, no-autonomous-flag MUST be in the same
#    commit that first advertises the prefix. Advertise first and tighten later
#    and every node carries an unreserved EUI-64 address that labctl might pick
#    up as node-ip.
#
#   ./labsim-dualstack-net.sh up       apply to both sim routers
#   ./labsim-dualstack-net.sh down     remove it again
#   ./labsim-dualstack-net.sh status   what the routers and nodes think
#   ./labsim-dualstack-net.sh leases   who has taken an address
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
R1="${R1:-172.31.1.252}"   # sim vyos001 -> ::1
R2="${R2:-172.31.1.253}"   # sim vyos002 -> ::2
PW="${VYOS_PW:-vyos}"

# Mirrors production's scheme (2001:470:187e:<vlan>::/64) using the
# documentation prefix, so the shape is rehearsed without putting sim addresses
# inside a range production announces.
V6_PREFIX="${V6_PREFIX:-2001:db8:187e:2}"
LINK_MTU="${LINK_MTU:-1472}"
# VyOS requires a unique subnet-id per DHCPv6 subnet ("Unique subnet ID not
# specified for subnet"). Using the VLAN id keeps it self-documenting and
# collision-free across VLANs.
VLAN_ID="${VLAN_ID:-2}"

# node -> last hextet. Mirrors the v4 host part (172.31.2.11 -> ::11) so a
# reservation is readable next to its IPv4 twin.
NODES=(labsim-k8s1:11 labsim-k8s2:12 labsim-k8s3:13)

SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
     -o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password)

log()  { printf '\033[0;36m[ds-net]\033[0m %s\n' "$*"; }
warn() { printf '\033[1;33m[ds-net]\033[0m %s\n' "$*" >&2; }
die()  { printf '\033[0;31m[ds-net]\033[0m %s\n' "$*" >&2; exit 1; }

# Drive VyOS from a script FILE, never `vbash -c`. The latter never starts a
# config session: commit fails to stderr, a helper discards it, and the run
# reports success having changed nothing. labsim-pppoe-ha-test.sh lost a whole
# policy matrix to exactly that.
vyos_apply() {  # host, set-lines on stdin
    local h="$1" out
    out="$({ printf '#!/bin/vbash\nsource /opt/vyatta/etc/functions/script-template\nconfigure\n'
             cat
             printf 'commit\nsave\nexit\n'
           } | timeout 120 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$h" \
                 'cat > /tmp/ds-net.sh && chmod +x /tmp/ds-net.sh && sudo /tmp/ds-net.sh' 2>&1)"
    # Read the outcome instead of assuming it. The first version of this script
    # printed "up" after BOTH routers had failed to commit -- the same shape of
    # lie this repo has been bitten by before (a matrix reporting coverage it did
    # not have). vbash exits 0 even when the commit fails, so the text is the
    # only honest signal.
    printf '%s\n' "$out" | grep -vE '^\s*$' | sed 's/^/      /' | tail -6
    if printf '%s' "$out" | grep -qiE 'Commit failed|\[\[.*\]\] failed|Set failed'; then
        return 1
    fi
    return 0
}

r() { timeout 40 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; }

mac_of() { sudo virsh domiflist "$1" 2>/dev/null | awk '/52:54/{print $5; exit}'; }

up() {
    log "collecting node MACs (the reservation key, same as IPv4)"
    local mappings="" name hextet mac
    for entry in "${NODES[@]}"; do
        name="${entry%%:*}"; hextet="${entry##*:}"
        mac="$(mac_of "$name")"
        [ -n "$mac" ] || die "no MAC for $name -- is the sim cluster up? (./k8s-up.sh)"
        log "    $name $mac -> ${V6_PREFIX}::${hextet}"
        # No trailing newline: `${mappings}` sits on its own line in the heredoc
        # below and supplies its own. A heredoc terminator is matched LITERALLY
        # in the source before any expansion, so `${mappings}EOF` is not the
        # delimiter -- the heredoc ran on, swallowed the rest of this function
        # and part of down(), and the apply failed with "Invalid command: [EOF]".
        [ -n "$mappings" ] && mappings+=$'\n'
        mappings+="set service dhcpv6-server shared-network-name K8S subnet ${V6_PREFIX}::/64 static-mapping ${name} mac '${mac}'
set service dhcpv6-server shared-network-name K8S subnet ${V6_PREFIX}::/64 static-mapping ${name} ipv6-address '${V6_PREFIX}::${hextet}'"
    done

    local host pref hextet_self
    for host in "$R1" "$R2"; do
        if [ "$host" = "$R1" ]; then pref=high; hextet_self=1; else pref=low; hextet_self=2; fi
        log "applying to $host (router ${V6_PREFIX}::${hextet_self}, RA preference $pref)"
        vyos_apply "$host" <<EOF || die "commit failed on $host -- nothing applied there"
set interfaces bonding bond0 vif 2 address '${V6_PREFIX}::${hextet_self}/64'
set service router-advert interface bond0.2 prefix ${V6_PREFIX}::/64 valid-lifetime '2592000'
set service router-advert interface bond0.2 prefix ${V6_PREFIX}::/64 preferred-lifetime '604800'
set service router-advert interface bond0.2 prefix ${V6_PREFIX}::/64 no-autonomous-flag
set service router-advert interface bond0.2 managed-flag
set service router-advert interface bond0.2 link-mtu '${LINK_MTU}'
set service router-advert interface bond0.2 default-preference '${pref}'
set service dhcpv6-server shared-network-name K8S subnet ${V6_PREFIX}::/64 subnet-id '${VLAN_ID}'
${mappings}
EOF
    done
    log "up. Nodes need a DHCPv6 client on their VLAN 2 interface -- see 'status'."
}

down() {
    local host
    for host in "$R1" "$R2"; do
        log "removing from $host"
        vyos_apply "$host" <<EOF
delete service dhcpv6-server
delete service router-advert interface bond0.2
delete interfaces bonding bond0 vif 2 address '${V6_PREFIX}::$([ "$host" = "$R1" ] && echo 1 || echo 2)/64'
EOF
    done
    log "down"
}

status() {
    local host
    for host in "$R1" "$R2"; do
        printf '  --- %s ---\n' "$host"
        printf '    bond0.2 v6 : %s\n' "$(r "$host" 'ip -6 -br addr show bond0.2 | tr -s " "' || echo '<unreachable>')"
        printf '    radvd      : %s\n' "$(r "$host" 'systemctl is-active radvd')"
        # ps|grep, not pgrep: the bracket idiom that stops a self-match gets
        # mangled through ssh -> vbash quoting and reported "NOT running" for a
        # daemon that was plainly up.
        printf '    kea-dhcp6  : %s\n' "$(r "$host" 'c=$(ps -ef | grep -c "[k]ea-dhcp6"); [ "$c" -gt 0 ] && echo running || echo "NOT running"')"
        # radvd is expected INACTIVE on the backup: vrrp-wan-reconcile's IPv6
        # plane stops it there so only the master advertises on any VLAN. Not a
        # fault -- see the step-0 IPv6-follows-master work.
        printf '    role       : %s\n' "$(r "$host" 'sudo /config/vrrp-wan-reconcile --status 2>/dev/null | grep -o "role=[a-z]*"')"
    done
    printf '  --- nodes ---\n'
    local entry name
    for entry in "${NODES[@]}"; do
        name="${entry%%:*}"
        printf '    %-14s %s\n' "$name" "$(node_v6 "$name")"
    done
}

# What global IPv6 does the node actually hold? This is the question labctl asks
# before it will write node-ip, so ask it the same way.
node_v6() {
    # Ask the node, not the hypervisor: virsh domifaddr --source agent needs
    # qemu-guest-agent, which these images do not carry, and returned nothing
    # while the nodes plainly had addresses.
    local name="$1" v4 out
    case "$name" in *k8s1) v4=172.31.2.11 ;; *k8s2) v4=172.31.2.12 ;; *k8s3) v4=172.31.2.13 ;; *) echo "<unknown>"; return ;; esac
    out="$(timeout 20 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
             -o LogLevel=ERROR -o ConnectTimeout=6 -o BatchMode=yes "debian@$v4" \
             'ip -6 -br addr show scope global 2>/dev/null | tr -s " "' 2>/dev/null)"
    echo "${out:-<unreachable>}"
}

leases() {
    local host
    for host in "$R1" "$R2"; do
        printf '  --- %s ---\n' "$host"
        r "$host" '/opt/vyatta/bin/vyatta-op-cmd-wrapper show dhcpv6-server leases 2>/dev/null || echo "    (no leases command / no leases)"'
    done
}

case "${1:-status}" in
    up)     up ;;
    down)   down ;;
    status) status ;;
    leases) leases ;;
    *) die "usage: $0 {up|down|status|leases}" ;;
esac