diff --git a/migration/vlan2-v6-apply b/migration/vlan2-v6-apply new file mode 100755 index 0000000..3f21446 --- /dev/null +++ b/migration/vlan2-v6-apply @@ -0,0 +1,189 @@ +#!/bin/bash +# Give VLAN 2 (the k8s VLAN) IPv6: addresses, router advertisements and DHCPv6 +# reservations. Production half of dual-stack phase 2b. +# +# Rehearsed first as labsim/labsim-dualstack-net.sh, which is where the four +# VyOS facts below were paid for rather than guessed. +# +# ADDRESSING ONLY -- NOT EGRESS. The prefix is advertised with +# `default-lifetime 0`, so nodes get their reserved addresses but neither router +# becomes an IPv6 default router. Turning on real v6 egress moves cluster image +# pulls onto the HE tunnel (1480, or 1472 on PPPoE) whose throughput has never +# been measured, and that is not a thing to switch on unattended. Flipping it is +# one line: `set service router-advert interface bond0.2 default-lifetime '1800'` +# plus a default-preference, once somebody is watching. +# +# LISTEN-INTERFACE IS NOT OPTIONAL. Without it kea6 renders +# `interfaces: [ "*" ]` and serves DHCPv6 on EVERY VLAN, not just this one. +# Observed in production the moment this was first applied: kea started +# answering SOLICIT/REQUEST from an unrelated device on bond0.10 (LoT). That is +# a DHCPv6 server switched on estate-wide as a side effect of configuring one +# VLAN -- the same family of mistake as the kea IPv4 cross-VLAN bug (ISC #1117) +# this estate already fought. Pin the interface. +# +# WHY DHCPv6 RATHER THAN SLAAC: k3s resolves node-ip once at start-up, so a +# node's address must be knowable in advance and stable. The estate already +# answers that for IPv4 with kea reservations keyed on MAC; IPv6 answers it the +# same way, from the same MACs, so there is one source of truth. VyOS's +# static-mapping accepts `mac` as well as `duid`, which is what makes that +# possible -- DHCPv6 normally keys on a client-generated DUID. +# +# vlan2-v6-apply plan print what would be applied, change nothing +# vlan2-v6-apply apply apply to both routers +# vlan2-v6-apply verify what the routers and nodes now hold +# vlan2-v6-apply revert remove it again +set -uo pipefail + +R1="${R1:-10.0.1.252}" # vyos001 -> ::1 +R2="${R2:-10.0.1.253}" # vyos002 -> ::2 +PW="${VYOS_PW:-vyos}" +V6_PREFIX="${V6_PREFIX:-2001:470:187e:2}" +LINK_MTU="${LINK_MTU:-1472}" +SUBNET_ID="${SUBNET_ID:-2}" # VyOS requires a unique id per DHCPv6 subnet +SHARED_NET="${SHARED_NET:-TheLab-k8s}" + +# name:v4-host-octet -- the IPv6 host part mirrors the IPv4 one so a reservation +# is readable next to its twin. MACs are read from the LIVE IPv4 reservations at +# run time, never duplicated here: one source of truth, and a node that is +# re-homed cannot end up with a stale v6 mapping. +NODES=(worker0-k8s0:23 worker1-k8s0:13 worker2-k8s0:25 spark-2935:12 aitopatom-3a1c:27) + +SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR -o ConnectTimeout=8) + +# stderr, NOT stdout. build_mappings() is captured with $(...) and log() lines +# went straight into the config stream, where VyOS rejected each one as +# "Invalid command: [[0" -- ANSI escapes and all. The valid sets still applied, +# so the routers ended up correct but NOT identical: different lines were lost +# on each. A progress message is not data; keep it off the data channel. +log() { printf '\033[0;36m[vlan2-v6]\033[0m %s\n' "$*" >&2; } +die() { printf '\033[0;31m[vlan2-v6]\033[0m %s\n' "$*" >&2; exit 1; } + +r() { timeout 45 ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; } + +# Drive VyOS from a script FILE with plain commit + save. +# - `vbash -c` never starts a config session; the commit fails to stderr and a +# helper discards it, so the run reports success having changed nothing. +# - `commit-confirm` hangs non-interactively and strands an orphaned +# config-mgmt commit_confirm holding the config lock. +# - vbash exits 0 even when the commit fails, so the OUTPUT is the only honest +# signal. Read it. +vyos_apply() { + local h="$1" out + out="$({ printf '#!/bin/vbash\nsource /opt/vyatta/etc/functions/script-template\nconfigure\n' + cat + printf 'commit\nsave\nexit\n' + } | timeout 150 ssh "${SSH[@]}" "vyos@$h" \ + 'cat > /tmp/vlan2-v6.sh && chmod +x /tmp/vlan2-v6.sh && sudo /tmp/vlan2-v6.sh' 2>&1)" + printf '%s\n' "$out" | grep -vE '^\s*$' | sed 's/^/ /' | tail -8 + # "Invalid command" was NOT in this list the first time, so a run that fed + # rubbish to VyOS reported success. vbash exits 0 regardless, so every + # rejection shape has to be named explicitly. + printf '%s' "$out" | grep -qiE 'Commit failed|\[\[.*\]\] failed|Set failed|Invalid command' && return 1 + return 0 +} + +# Read each node's MAC out of the live IPv4 reservation. +# +# Scoped to the IPv4 subnet on purpose. Once this script has run once, the node +# has TWO `static-mapping mac` lines -- the v4 one and the v6 one it just +# created -- and an unscoped match returned both concatenated +# ("9c:76:0e:49:e9:179c:76:0e:49:e9:17"), which VyOS then rejected as an invalid +# value. Self-inflicted on the second run: the lookup has to name the subnet it +# means, or the script poisons its own input as soon as it succeeds. +V4_SUBNET="${V4_SUBNET:-192.168.8.0/23}" +mac_of() { + r "$R1" "/opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands 2>/dev/null \ + | grep -F 'subnet $V4_SUBNET' \ + | sed -n \"s/.*static-mapping $1 mac '\\(.*\\)'/\\1/p\"" | tr -d ' \n' +} + +build_mappings() { + local out="" name hextet mac + for entry in "${NODES[@]}"; do + name="${entry%%:*}"; hextet="${entry##*:}" + mac="$(mac_of "$name")" + [ -n "$mac" ] || die "no IPv4 reservation found for $name -- refusing to invent a MAC" + log " $name $mac -> ${V6_PREFIX}::${hextet}" + [ -n "$out" ] && out+=$'\n' + out+="set service dhcpv6-server shared-network-name ${SHARED_NET} subnet ${V6_PREFIX}::/64 static-mapping ${name} mac '${mac}' +set service dhcpv6-server shared-network-name ${SHARED_NET} subnet ${V6_PREFIX}::/64 static-mapping ${name} ipv6-address '${V6_PREFIX}::${hextet}'" + done + printf '%s' "$out" +} + +# One commit per router, and `no-autonomous-flag` is in it. That is not a +# style choice: turning autonomous off LATER does not retract addresses already +# formed, so a prefix advertised even briefly without it leaves every node +# holding an unreserved EUI-64 address with a 30-day lifetime. Observed in the +# sim. VLAN 2 has no IPv6 today, so this is the one chance to get it right. +config_for() { + local self="$1" mappings="$2" + cat </dev/null | grep -o "role=[a-z]*"')" + printf ' IPv4 sane : %s\n' "$(r "$h" 'ip -4 route show default | head -1')" + done +} + +cmd_revert() { + local h self + for h in "$R1" "$R2"; do + [ "$h" = "$R1" ] && self=1 || self=2 + log "reverting $h" + vyos_apply "$h" <> "$LOGF"; } + +# IPv4 health, asked of the box itself -- and it MUST be role-aware. +# +# The first version required a default route plus internet on both routers, and +# would have reverted on vyos002 within 90s of being armed. That box is the +# gated BACKUP: by design it holds no VIP, has bond0.53 disabled and pppoe0 +# down, so it has no default route and no internet, and that is the correct +# resting state rather than a fault. Caught at 20s of the 90s grace. +# +# So: the master must be able to reach the internet. The backup only has to +# still be on the network and able to see its peer -- which is what would +# actually be at risk if a VLAN 2 change went wrong. +VIP="${VIP:-192.168.1.1}" +holds_vip() { ip -4 -o addr show 2>/dev/null | grep -q " ${VIP}/"; } + +v4_ok() { + if holds_vip; then + ip -4 route show default 2>/dev/null | grep -q . || return 1 + ping -c1 -W2 "$PROBE" >/dev/null 2>&1 + else + # Backup: management address present and the VIP answers. If the VIP has + # gone too then the pair has a bigger problem than this change, and + # reverting an IPv6 addition would not help -- so this deliberately does + # not fire on peer loss alone. + ip -4 -o addr show bond0.1 2>/dev/null | grep -q 'inet ' || return 1 + ping -c1 -W2 "$VIP" >/dev/null 2>&1 + fi +} + +revert() { + log "REVERTING: IPv4 has been down for ${GRACE}s" + # A script file, not vbash -c: the latter never starts a config session and + # the commit fails to stderr where nobody sees it. + cat > /tmp/vlan2-v6-revert.sh <<'REOF' +#!/bin/vbash +source /opt/vyatta/etc/functions/script-template +configure +delete service dhcpv6-server +delete service router-advert interface bond0.2 +commit +save +exit +REOF + chmod +x /tmp/vlan2-v6-revert.sh + /tmp/vlan2-v6-revert.sh >> "$LOGF" 2>&1 + # The interface address is deleted separately: it is the one node whose + # removal could plausibly disturb something else, so it goes last and its + # failure does not block the rest. + for n in 1 2; do + ip -6 addr del "${V6_PREFIX}::${n}/64" dev bond0.2 2>/dev/null + done + log "revert done; IPv4 now $(v4_ok && echo OK || echo STILL BAD)" +} + +watch_loop() { + local deadline=$(( $(date +%s) + $1 )) bad=0 + log "armed for $1s (grace ${GRACE}s, probe ${PROBE})" + while [ "$(date +%s)" -lt "$deadline" ]; do + if v4_ok; then + [ "$bad" -ne 0 ] && log "IPv4 recovered after ${bad}s" + bad=0 + else + bad=$((bad + 10)) + log "IPv4 bad (${bad}s/${GRACE}s)" + if [ "$bad" -ge "$GRACE" ]; then + revert + log "disarming after revert" + rm -f "$PIDF" + return 0 + fi + fi + sleep 10 + done + log "expired without incident" + rm -f "$PIDF" +} + +case "${1:-status}" in + arm) + mkdir -p "$STATE" + [ -f "$PIDF" ] && kill -0 "$(cat "$PIDF")" 2>/dev/null && { echo "already armed"; exit 0; } + setsid "$0" _run "${2:-2400}" >/dev/null 2>&1 < /dev/null & + sleep 1 + [ -f "$PIDF" ] && echo " armed (pid $(cat "$PIDF"), ${2:-2400}s)" || echo " FAILED to arm" + ;; + _run) + echo $$ > "$PIDF" + watch_loop "${2:-2400}" + ;; + disarm) + if [ -f "$PIDF" ]; then kill "$(cat "$PIDF")" 2>/dev/null; rm -f "$PIDF"; echo " disarmed" + else echo " not armed"; fi + ;; + status) + if [ -f "$PIDF" ] && kill -0 "$(cat "$PIDF")" 2>/dev/null; then echo " armed (pid $(cat "$PIDF"))" + else echo " not armed"; fi + echo " --- log ---"; tail -12 "$LOGF" 2>/dev/null | sed 's/^/ /' || echo " (none)" + ;; + *) echo "usage: $0 {arm [seconds]|disarm|status}" >&2; exit 2 ;; +esac diff --git a/migration/window-evidence/2026-09-06-baseline.txt b/migration/window-evidence/2026-09-06-baseline.txt new file mode 100644 index 0000000..b115288 --- /dev/null +++ b/migration/window-evidence/2026-09-06-baseline.txt @@ -0,0 +1,35 @@ +=== W0 baseline 2026-09-06T22:25:20+01:00 === +--- 10.0.1.252 --- +vyos001 + holds VIP : YES + WAN : bond0.53=87.192.101.48 pppoe0=90.251.152.236 + route : default via 87.192.96.1 dev bond0.53 proto failover metric 1 + internet: UP + ipv6 : UP + vrrp : 6 MASTER + vlan2 v6 present already? : 0 +--- 10.0.1.253 --- +vyos002 + holds VIP : no + WAN : + route : + internet: DOWN + ipv6 : DOWN + vrrp : 6 BACKUP + vlan2 v6 present already? : 0 +--- cluster --- + aitopatom-3a1c Ready + spark-2935 Ready + worker0-k8s0.ad.itaz.eu Ready + worker1-k8s0.ad.itaz.eu Ready + worker2-k8s0.ad.itaz.eu Ready +--- known-good save --- +[known-good] pinned 1118 lines as known-good on vyos001 +[known-good] restore with: /config/vyos-known-good restore +[known-good] pinned 1119 lines as known-good on vyos002 +[known-good] restore with: /config/vyos-known-good restore +--- vrrp-wan-install --check --- + 10.0.1.252: vrrp-wan in sync + 10.0.1.253: vrrp-wan in sync +vyos001: in sync (533 nodes) +vyos002: in sync (534 nodes) diff --git a/migration/window-evidence/2026-09-06-dhcpv6.txt b/migration/window-evidence/2026-09-06-dhcpv6.txt new file mode 100644 index 0000000..7c16d2d --- /dev/null +++ b/migration/window-evidence/2026-09-06-dhcpv6.txt @@ -0,0 +1,56 @@ +=== W2: does a production node take a DHCPv6 reservation? === +Window of 2026-09-06, operator offline. + +SHORT ANSWER: the RA half works; the RESERVATION half does not, and the reason +is structural rather than a misconfiguration. + +WHAT WORKS + - RA is emitted on bond0.2 with AdvManagedFlag on, AdvAutonomous off, + AdvLinkMTU 1472, AdvDefaultLifetime 0 (deliberately not a default router). + - NetworkManager on worker0 SAW it and acted: + dhcp6 (eno1): activation: beginning transaction (timeout in 45 seconds) + So "ipv6.method=auto follows the managed flag" is now EVIDENCE, not + inference. That was the open question this window set out to close. + - kea-dhcp6 runs on both routers and receives SOLICITs on bond0.2. + +WHAT DOES NOT WORK, AND WHY + Every DHCPv6 packet kea logs is annotated [no hwaddr info]: + + duid=[00:04:69:d9:30:64:c3:4e:...] [no hwaddr info] <- DUID-UUID (type 4) + duid=[00:04:25:11:e0:0a:7b:c4:...] [no hwaddr info] <- DUID-UUID + duid=[00:01:00:01:31:78:0a:ae:...] [no hwaddr info] <- DUID-LLT (type 1) + + DHCPv6 identifies a client by DUID, not by MAC. kea can sometimes DERIVE a MAC + (its mac-sources machinery), but here it derives nothing -- including from a + DUID-LLT, which does embed one. So `hw-address` reservations cannot match, and + a node that solicits gets no reserved address. + + VyOS accepting `static-mapping mac` is therefore necessary but NOT + sufficient: it renders valid kea config ("hw-address": "78:55:36:08:28:fb", + confirmed in /run/kea/kea-dhcp6.conf) that simply never matches these clients. + The plan's assumption -- "reservations keyed on MAC, one source of truth with + IPv4" -- does not survive contact with DHCPv6. + +OPTIONS, none free, for an attended session: + a) Key reservations on DUID (VyOS static-mapping supports `duid`). Works, but a + DUID is client-generated: it is a SECOND source of truth, it must be + harvested per node, and it changes if a node is rebuilt -- which is exactly + the "out of the box for new nodes" property we were trying to preserve. + b) Set kea `mac-sources` explicitly and see whether any method recovers a MAC + from these clients. Cheap to try; may simply not work for DUID-UUID. + c) Drop reservations: hand out a dynamic range and have labctl DISCOVER the + node's address rather than predict it. The k3s preflight (914135c) already + refuses to write a node-ip the node does not hold, so discovery is safe -- + but the address is then not knowable before the node boots. + d) SLAAC with no-autonomous cleared, and accept EUI-64 addresses. Stable per + NIC and needs no server state at all, but the address is not ours to choose. + +A SECOND FINDING, unrelated and worse if unnoticed + `service dhcpv6-server` with no listen-interface renders kea6 with + interfaces: [ "*" ] -- a DHCPv6 server on EVERY VLAN. Observed live: kea began + answering SOLICIT/REQUEST from an unrelated device on bond0.10 (LoT) within + seconds of the first apply. Same family as the kea IPv4 cross-VLAN bug (ISC + #1117) this estate already fought. Fixed by pinning + `set service dhcpv6-server listen-interface bond0.2` plus a subnet-level + `interface bond0.2`; both routers now render interfaces: ["bond0.2"]. +