diff --git a/labsim/labsim-dualstack-net.sh b/labsim/labsim-dualstack-net.sh new file mode 100755 index 0000000..1f5714c --- /dev/null +++ b/labsim/labsim-dualstack-net.sh @@ -0,0 +1,221 @@ +#!/bin/bash +# VLAN 2 gets IPv6 in labsim: addresses, router advertisements and a DHCPv6 +# server with per-MAC reservations. +# +# This is the rehearsal of the production change (dual-stack plan, phase 2b) and +# runs the SAME VyOS config, against the sim router pair, so the production +# apply is a repeat rather than a first attempt. +# +# WHY DHCPv6 AND NOT SLAAC. The cluster needs each node's IPv6 to be knowable in +# advance and stable: k3s resolves node-ip once at start-up, and a node's +# identity cannot be allowed to change under it. The estate already answers that +# question for IPv4 with kea reservations keyed on MAC, so IPv6 answers it the +# same way and stays one source of truth. +# +# DHCPv6 normally keys on DUID, not MAC -- a DUID is generated by the client and +# is not derivable from its MAC, which would have meant a second, client-owned +# source of truth. VyOS's static-mapping accepts `mac` as well as `duid` +# (verified on the sim: the node.tag directory offers duid, mac, ipv6-address, +# ipv6-prefix), so the reservation can key on the same MAC the v4 one does. +# +# The RA carries managed-flag WITH no-autonomous-flag. That combination is what +# makes the node's address unambiguous: managed sends it to DHCPv6, and +# non-autonomous stops it also forming a SLAAC address from the same prefix. +# Leave autonomous on and every node has two global addresses, only one of which +# anybody reserved -- and whichever labctl happens to find is the one that ends +# up in node-ip. +# +# Both are VALUELESS nodes on this VyOS: `managed-flag` not `managed-flag true`, +# and `prefix

no-autonomous-flag` not `autonomous-flag false`. The value +# forms are rejected with "is not valid". +# +# A FAILED COMMIT DOES NOT MEAN NOTHING CHANGED. VyOS commits node groups +# independently, so an earlier run of this script left the bond0.2 address and +# most of the router-advert block applied while `[[service dhcpv6-server]] +# failed` -- the interface and RA groups had already succeeded. Re-read the +# config after any failure rather than assuming a clean rollback; that matters +# more in production than here. +# +# TWO THINGS THIS REHEARSAL FOUND, both of which would have bitten production: +# +# 1. managed-flag is NECESSARY BUT NOT SUFFICIENT. It only tells the host to use +# DHCPv6; the kernel's accept_ra implements SLAAC and nothing else, so a node +# with no DHCPv6 *client* running takes no lease at all. The sim's Debian +# nodes have neither NetworkManager nor a networkd .network file managing the +# interface, so they ended up with a kernel SLAAC address +# (2001:db8:187e:2:5054:ff:fe53:731 -- EUI-64 from the MAC) and never asked +# for the reserved ::11. Production's Fedora nodes use NetworkManager, which +# does run a DHCPv6 client on the managed flag, and the DGX Sparks run +# NetworkManager too -- but that must be VERIFIED per node, not assumed. The +# k3s module's preflight catches the consequence; the fix is node-side. +# +# 2. TURNING AUTONOMOUS OFF DOES NOT RETRACT ADDRESSES ALREADY FORMED. An +# earlier partial apply advertised the prefix while autonomous was still on; +# the nodes autoconfigured, and adding no-autonomous-flag afterwards left +# those addresses in place with a 30-day valid lifetime. So in production, +# where VLAN 2 has no IPv6 at all yet, no-autonomous-flag MUST be in the same +# commit that first advertises the prefix. Advertise first and tighten later +# and every node carries an unreserved EUI-64 address that labctl might pick +# up as node-ip. +# +# ./labsim-dualstack-net.sh up apply to both sim routers +# ./labsim-dualstack-net.sh down remove it again +# ./labsim-dualstack-net.sh status what the routers and nodes think +# ./labsim-dualstack-net.sh leases who has taken an address +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +R1="${R1:-172.31.1.252}" # sim vyos001 -> ::1 +R2="${R2:-172.31.1.253}" # sim vyos002 -> ::2 +PW="${VYOS_PW:-vyos}" + +# Mirrors production's scheme (2001:470:187e:::/64) using the +# documentation prefix, so the shape is rehearsed without putting sim addresses +# inside a range production announces. +V6_PREFIX="${V6_PREFIX:-2001:db8:187e:2}" +LINK_MTU="${LINK_MTU:-1472}" +# VyOS requires a unique subnet-id per DHCPv6 subnet ("Unique subnet ID not +# specified for subnet"). Using the VLAN id keeps it self-documenting and +# collision-free across VLANs. +VLAN_ID="${VLAN_ID:-2}" + +# node -> last hextet. Mirrors the v4 host part (172.31.2.11 -> ::11) so a +# reservation is readable next to its IPv4 twin. +NODES=(labsim-k8s1:11 labsim-k8s2:12 labsim-k8s3:13) + +SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password) + +log() { printf '\033[0;36m[ds-net]\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33m[ds-net]\033[0m %s\n' "$*" >&2; } +die() { printf '\033[0;31m[ds-net]\033[0m %s\n' "$*" >&2; exit 1; } + +# Drive VyOS from a script FILE, never `vbash -c`. The latter never starts a +# config session: commit fails to stderr, a helper discards it, and the run +# reports success having changed nothing. labsim-pppoe-ha-test.sh lost a whole +# policy matrix to exactly that. +vyos_apply() { # host, set-lines on stdin + local h="$1" out + out="$({ printf '#!/bin/vbash\nsource /opt/vyatta/etc/functions/script-template\nconfigure\n' + cat + printf 'commit\nsave\nexit\n' + } | timeout 120 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$h" \ + 'cat > /tmp/ds-net.sh && chmod +x /tmp/ds-net.sh && sudo /tmp/ds-net.sh' 2>&1)" + # Read the outcome instead of assuming it. The first version of this script + # printed "up" after BOTH routers had failed to commit -- the same shape of + # lie this repo has been bitten by before (a matrix reporting coverage it did + # not have). vbash exits 0 even when the commit fails, so the text is the + # only honest signal. + printf '%s\n' "$out" | grep -vE '^\s*$' | sed 's/^/ /' | tail -6 + if printf '%s' "$out" | grep -qiE 'Commit failed|\[\[.*\]\] failed|Set failed'; then + return 1 + fi + return 0 +} + +r() { timeout 40 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; } + +mac_of() { sudo virsh domiflist "$1" 2>/dev/null | awk '/52:54/{print $5; exit}'; } + +up() { + log "collecting node MACs (the reservation key, same as IPv4)" + local mappings="" name hextet mac + for entry in "${NODES[@]}"; do + name="${entry%%:*}"; hextet="${entry##*:}" + mac="$(mac_of "$name")" + [ -n "$mac" ] || die "no MAC for $name -- is the sim cluster up? (./k8s-up.sh)" + log " $name $mac -> ${V6_PREFIX}::${hextet}" + # No trailing newline: `${mappings}` sits on its own line in the heredoc + # below and supplies its own. A heredoc terminator is matched LITERALLY + # in the source before any expansion, so `${mappings}EOF` is not the + # delimiter -- the heredoc ran on, swallowed the rest of this function + # and part of down(), and the apply failed with "Invalid command: [EOF]". + [ -n "$mappings" ] && mappings+=$'\n' + mappings+="set service dhcpv6-server shared-network-name K8S subnet ${V6_PREFIX}::/64 static-mapping ${name} mac '${mac}' +set service dhcpv6-server shared-network-name K8S subnet ${V6_PREFIX}::/64 static-mapping ${name} ipv6-address '${V6_PREFIX}::${hextet}'" + done + + local host pref hextet_self + for host in "$R1" "$R2"; do + if [ "$host" = "$R1" ]; then pref=high; hextet_self=1; else pref=low; hextet_self=2; fi + log "applying to $host (router ${V6_PREFIX}::${hextet_self}, RA preference $pref)" + vyos_apply "$host" <')" + printf ' radvd : %s\n' "$(r "$host" 'systemctl is-active radvd')" + # ps|grep, not pgrep: the bracket idiom that stops a self-match gets + # mangled through ssh -> vbash quoting and reported "NOT running" for a + # daemon that was plainly up. + printf ' kea-dhcp6 : %s\n' "$(r "$host" 'c=$(ps -ef | grep -c "[k]ea-dhcp6"); [ "$c" -gt 0 ] && echo running || echo "NOT running"')" + # radvd is expected INACTIVE on the backup: vrrp-wan-reconcile's IPv6 + # plane stops it there so only the master advertises on any VLAN. Not a + # fault -- see the step-0 IPv6-follows-master work. + printf ' role : %s\n' "$(r "$host" 'sudo /config/vrrp-wan-reconcile --status 2>/dev/null | grep -o "role=[a-z]*"')" + done + printf ' --- nodes ---\n' + local entry name + for entry in "${NODES[@]}"; do + name="${entry%%:*}" + printf ' %-14s %s\n' "$name" "$(node_v6 "$name")" + done +} + +# What global IPv6 does the node actually hold? This is the question labctl asks +# before it will write node-ip, so ask it the same way. +node_v6() { + # Ask the node, not the hypervisor: virsh domifaddr --source agent needs + # qemu-guest-agent, which these images do not carry, and returned nothing + # while the nodes plainly had addresses. + local name="$1" v4 out + case "$name" in *k8s1) v4=172.31.2.11 ;; *k8s2) v4=172.31.2.12 ;; *k8s3) v4=172.31.2.13 ;; *) echo ""; return ;; esac + out="$(timeout 20 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR -o ConnectTimeout=6 -o BatchMode=yes "debian@$v4" \ + 'ip -6 -br addr show scope global 2>/dev/null | tr -s " "' 2>/dev/null)" + echo "${out:-}" +} + +leases() { + local host + for host in "$R1" "$R2"; do + printf ' --- %s ---\n' "$host" + r "$host" '/opt/vyatta/bin/vyatta-op-cmd-wrapper show dhcpv6-server leases 2>/dev/null || echo " (no leases command / no leases)"' + done +} + +case "${1:-status}" in + up) up ;; + down) down ;; + status) status ;; + leases) leases ;; + *) die "usage: $0 {up|down|status|leases}" ;; +esac