thebeast runbook: VM drill passed, data copies done 2026-10-14

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V6HNx6cyr4J4CiMGCy1wkF

.gitea/workflows/ci.yml

@@ -28,6 +28,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
+        with:
+          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -47,6 +49,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
+        with:
+          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -54,6 +58,9 @@ jobs:
 
       - run: pnpm install --frozen-lockfile
 
+      - name: Generate Prisma client (labd typecheck needs it)
+        run: pnpm -C src/labd run db:generate
+
       - name: Typecheck
         run: pnpm typecheck
 
@@ -66,6 +73,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
+        with:
+          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -73,6 +82,15 @@ jobs:
 
       - run: pnpm install --frozen-lockfile
 
+      - name: Generate Prisma client (labd typecheck needs it)
+        run: pnpm -C src/labd run db:generate
+
+      - name: Install test validators (ubuntu-autoinstall parses YAML via python, kickstart.test runs ksvalidator, the bastion smoke start runs `ip route`)
+        # pykickstart is not in the runner image's apt sources -- pip it is.
+        run: |
+          sudo apt-get update -qq && sudo apt-get install -y -qq python3-yaml python3-pip iproute2
+          sudo pip3 install --quiet --break-system-packages pykickstart
+
       - name: Build (needed by completions check)
         run: pnpm build
 
@@ -91,6 +109,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
+        with:
+          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -99,6 +119,9 @@ jobs:
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
+      - name: Generate Prisma client (labd typecheck needs it)
+        run: pnpm -C src/labd run db:generate
+
       - name: Build all packages
         run: pnpm build
 
@@ -123,13 +146,13 @@ jobs:
 
       - name: Package x86_64 RPM + DEB
         run: |
-          sed -e 's|^arch:.*|arch: amd64|' -e 's|src: ./dist/lab$|src: ./dist/lab-x86_64|' nfpm.yaml > /tmp/nfpm-x86_64.yaml
+          sed -e 's|^arch:.*|arch: amd64|' -e 's|src: ./dist/labctl$|src: ./dist/lab-x86_64|' nfpm.yaml > /tmp/nfpm-x86_64.yaml
           nfpm pkg --config /tmp/nfpm-x86_64.yaml --packager rpm --target dist/
           nfpm pkg --config /tmp/nfpm-x86_64.yaml --packager deb --target dist/
 
       - name: Package arm64 RPM + DEB
         run: |
-          sed -e 's|^arch:.*|arch: arm64|' -e 's|src: ./dist/lab$|src: ./dist/lab-arm64|' nfpm.yaml > /tmp/nfpm-arm64.yaml
+          sed -e 's|^arch:.*|arch: arm64|' -e 's|src: ./dist/labctl$|src: ./dist/lab-arm64|' nfpm.yaml > /tmp/nfpm-arm64.yaml
           nfpm pkg --config /tmp/nfpm-arm64.yaml --packager rpm --target dist/
           nfpm pkg --config /tmp/nfpm-arm64.yaml --packager deb --target dist/
 

bastion/completions/labctl.bash

@@ -40,8 +40,14 @@ _labctl() {
     "app labcontroller status")
       COMPREPLY=($(compgen -W "--user -h --help" -- "$cur"))
       return ;;
+    "app k3s evacuate")
+      COMPREPLY=($(compgen -W "--user --allow-single-replica -h --help" -- "$cur"))
+      return ;;
+    "app k3s admit")
+      COMPREPLY=($(compgen -W "--user -h --help" -- "$cur"))
+      return ;;
     "app k3s install")
-      COMPREPLY=($(compgen -W "--role --user --k3s-server --k3s-token -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "--role --user --k3s-server --k3s-token --k3s-version -h --help" -- "$cur"))
       return ;;
     "app k3s health")
       COMPREPLY=($(compgen -W "--user -h --help" -- "$cur"))
@@ -59,7 +65,7 @@ _labctl() {
       COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
       return ;;
     "provision install")
-      COMPREPLY=($(compgen -W "--role --os --disk --vyos-mgmt --vyos-mgmt-address --vyos-bond --vyos-bond-address --vyos-bond-vrrp --vlan-vip --vyos-vrrp-priority --vyos-mgmt-vlan --vlan --vyos-password --vyos-hwid --vyos-fresh-config -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "--role --os --disk --schema-version --longhorn-disk --rancher-disk --vyos-mgmt --vyos-mgmt-address --vyos-bond --vyos-bond-address --vyos-bond-vrrp --vlan-vip --vyos-vrrp-priority --vyos-mgmt-vlan --vlan --vyos-password --vyos-hwid --vyos-fresh-config --vyos-bundle --vyos-api-key --vyos-api-listen -h --help" -- "$cur"))
       return ;;
     "provision reprovision")
       COMPREPLY=($(compgen -W "--role --os --disk --user -h --help" -- "$cur"))
@@ -101,7 +107,7 @@ _labctl() {
       COMPREPLY=($(compgen -W "deploy status -h --help" -- "$cur"))
       return ;;
     "app k3s")
-      COMPREPLY=($(compgen -W "install health list kubeconfig -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "evacuate admit install health list kubeconfig -h --help" -- "$cur"))
       return ;;
     "version")
       COMPREPLY=($(compgen -W "-h --help" -- "$cur"))

bastion/completions/labctl.fish

@@ -131,9 +131,12 @@ complete -c labctl -n "__labctl_using_cmd provision" -a makeiso -d 'Generate a U
 complete -c labctl -n "__labctl_using_cmd provision" -a recheck -d 'Refresh hardware info for all installed machines via SSH'
 
 # provision install options
-complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
+complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo'
 complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling'
-complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target disk device (auto-detect if omitted)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target OS disk: kernel name or stable /dev/disk/by-id/ path (auto-detect if omitted)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l schema-version -d 'Disk layout for a FRESH install (default 1). Ignored on reprovision: the layout already on the disk wins.' -xa '1 2'
+complete -c labctl -n "__labctl_in_cmd provision install" -l longhorn-disk -d 'Dedicated whole disk for /var/lib/longhorn, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l rancher-disk -d 'Dedicated whole disk for /var/lib/rancher, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)' -x
 complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt -d 'VyOS: untagged interface the machine PXE boots from (default eth0)' -x
 complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-address -d 'VyOS: CIDR for the management interface, or \'dhcp\' (default dhcp)' -x
 complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond -d 'VyOS: comma-separated LACP bond members (must exclude the PXE NIC)' -x
@@ -146,9 +149,12 @@ complete -c labctl -n "__labctl_in_cmd provision install" -l vlan -d 'VyOS: tagg
 complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-password -d 'VyOS: password for the \'vyos\' user' -x
 complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-hwid -d 'VyOS: pin an interface name to a MAC via hw-id (repeatable)' -x
 complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-fresh-config -d 'VyOS: on reinstall, overwrite the preserved config with the generated one'
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bundle -d 'VyOS: apply a Pulumi-rendered bundle verbatim (kubernetes-deployment/infra/vyos/vyos-bundle.json). Replaces the derived --vyos-bond/--vlan/... config; secret values are left unset for `pulumi up`.' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-api-key -d 'VyOS: enable the HTTP API with this key so Pulumi can manage the router from first boot' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-api-listen -d 'VyOS: address the HTTP API binds to (default: the static management address). Required when management is DHCP; the API is never bound to all interfaces.' -x
 
 # provision reprovision options
-complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
+complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo'
 complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling'
 complete -c labctl -n "__labctl_in_cmd provision reprovision" -l disk -d 'Target disk device (auto-detect if omitted)' -x
 complete -c labctl -n "__labctl_in_cmd provision reprovision" -l user -d 'SSH user for the reboot (default: vyos for VyOS machines, else current user)' -x
@@ -157,7 +163,7 @@ complete -c labctl -n "__labctl_in_cmd provision reprovision" -l user -d 'SSH us
 complete -c labctl -n "__labctl_in_cmd provision debug" -l pxe-boot -d 'Boot installed system via PXE (kernel+initrd from network, root from NVMe)'
 
 # provision register options
-complete -c labctl -n "__labctl_in_cmd provision register" -l role -d 'Machine role' -xa 'vanilla worker infra labcontroller'
+complete -c labctl -n "__labctl_in_cmd provision register" -l role -d 'Machine role' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo'
 complete -c labctl -n "__labctl_in_cmd provision register" -l ip -d 'Machine IP address' -x
 
 # provision logs options
@@ -200,16 +206,26 @@ complete -c labctl -n "__labctl_in_cmd app labcontroller deploy" -l crdb-replica
 complete -c labctl -n "__labctl_in_cmd app labcontroller status" -l user -d 'SSH user' -x
 
 # app k3s subcommands
+complete -c labctl -n "__labctl_using_cmd app k3s" -a evacuate -d 'Cordon, evict Longhorn replicas and drain so a node can be safely wiped'
+complete -c labctl -n "__labctl_using_cmd app k3s" -a admit -d 'Return an evacuated node to service: uncordon, clear eviction, restore its Longhorn disk'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a install -d 'Install k3s on a target machine (hostname, IP, or MAC)'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a health -d 'Check k3s health (all hosts if no target given)'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a list -d 'List installed machines and their k3s status'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a kubeconfig -d 'Fetch kubeconfig from a target and merge into ~/.kube/config'
 
+# app k3s evacuate options
+complete -c labctl -n "__labctl_in_cmd app k3s evacuate" -l user -d 'SSH user' -x
+complete -c labctl -n "__labctl_in_cmd app k3s evacuate" -l allow-single-replica -d 'permit evacuating volumes that have only one replica (rebuilds from the only copy; there is no backup target)'
+
+# app k3s admit options
+complete -c labctl -n "__labctl_in_cmd app k3s admit" -l user -d 'SSH user' -x
+
 # app k3s install options
-complete -c labctl -n "__labctl_in_cmd app k3s install" -l role -d 'k3s role: infra (server) or worker (agent)' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l role -d 'k3s role: infra, labcontroller, standalone-cluster, k8s-controller, k8s-combo (servers) or worker, k8s-worker (agents)' -x
 complete -c labctl -n "__labctl_in_cmd app k3s install" -l user -d 'SSH user' -x
-complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-server -d 'k3s server URL (required for worker role)' -x
-complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-token -d 'k3s join token (required for worker role)' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-server -d 'k3s server URL (required for agent roles, and to JOIN as a server)' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-token -d 'k3s join token; on a first server it SEEDS the cluster token' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-version -d 'exact k3s version, e.g. v1.34.6+k3s1. Unset installs LATEST, which on a rejoining node can be several minor versions ahead of the cluster' -x
 
 # app k3s health options
 complete -c labctl -n "__labctl_in_cmd app k3s health" -l user -d 'SSH user' -x

bastion/src/bastion/src/config.ts

@@ -29,7 +29,13 @@ export function loadConfig(overrides: Partial<BastionConfig> = {}): BastionConfi
   const vyosDefaultPassword = overrides.vyosDefaultPassword
     ?? process.env["VYOS_DEFAULT_PASSWORD"] ?? "vyos";
 
-  const fedoraMirror = `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`;
+  // Overridable like ubuntuMirror and vyosIsoUrl above -- it was the only one
+  // of the three hardcoded. Without this, every integration-test install pulls
+  // the full package set from download.fedoraproject.org, which makes a
+  // multi-VM provisioning matrix impractical (~25 min per install, serialised).
+  // Point FEDORA_MIRROR at a local mirror or caching proxy for test runs.
+  const fedoraMirror = overrides.fedoraMirror ?? process.env["FEDORA_MIRROR"]
+    ?? `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`;
   const tftpDir = `${bastionDir}/tftp`;
   const httpDir = `${bastionDir}/http`;
   const stateFile = `${bastionDir}/state.json`;
@@ -59,6 +65,7 @@ export function loadConfig(overrides: Partial<BastionConfig> = {}): BastionConfi
     syslogPort,
     skipDnsmasq: overrides.skipDnsmasq,
     skipArtifacts: overrides.skipArtifacts,
+    autoK3s: overrides.autoK3s ?? process.env["AUTO_K3S"] !== "false",
     labdUrl: overrides.labdUrl ?? process.env["LABD_URL"],
     bastionJoinToken: overrides.bastionJoinToken ?? process.env["BASTION_JOIN_TOKEN"],
     fedoraMirror,

bastion/src/bastion/src/main.ts

@@ -4,6 +4,7 @@
 import { mkdirSync, writeFileSync, readFileSync, existsSync, copyFileSync, symlinkSync, unlinkSync } from "node:fs";
 import { execSync } from "node:child_process";
 import type { BastionConfig } from "@lab/shared";
+import { isRole, validateDataDisks } from "@lab/shared";
 import { loadConfig } from "./config.js";
 import { populateNetworkConfig } from "./services/network.js";
 import { createApp } from "./server.js";
@@ -384,6 +385,22 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
     // Wire up command handlers so labd can send install/forget/role commands
     labdConn.onCommand("command-install", async (msg) => {
       if (msg.type !== "command-install") throw new Error("unexpected");
+      // Same guard as POST /api/install: disks named by an operator must
+      // never silently degrade into LVs because they arrived over the socket.
+      const hasDataDisks = msg.dataDisks !== undefined
+        && Object.values(msg.dataDisks).some((d) => d !== undefined);
+      if (hasDataDisks) {
+        if ((msg.os ?? "fedora-43") !== "fedora-43") {
+          return { status: "error", error: `dataDisks are only supported for fedora installs (got os '${msg.os}')` };
+        }
+        if (!isRole(msg.role)) {
+          return { status: "error", error: `invalid role: '${msg.role}'` };
+        }
+        const problem = validateDataDisks(msg.role, msg.dataDisks!);
+        if (problem !== null) {
+          return { status: "error", error: `invalid dataDisks: ${problem}` };
+        }
+      }
       state.update((s) => {
         s.install_queue[msg.mac] = {
           hostname: msg.hostname,
@@ -391,6 +408,8 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
           role: msg.role as import("@lab/shared").Role,
           os: msg.os as import("@lab/shared").OsId,
           queued_at: new Date().toISOString(),
+          ...(msg.schemaVersion ? { schemaVersion: msg.schemaVersion } : {}),
+          ...(hasDataDisks ? { dataDisks: msg.dataDisks } : {}),
           ...(msg.vyos ? { vyos: msg.vyos } : {}),
         };
       });

bastion/src/bastion/src/routes/api.ts

@@ -5,7 +5,7 @@
 // /api/discover  - receive hardware discovery reports from PXE-booted machines
 
 import type { FastifyInstance } from "fastify";
-import type { HardwareInfo, InstalledInfo, Role, VyosInstallSpec } from "@lab/shared";
+import type { BastionConfig, HardwareInfo, InstalledInfo, Role, VyosInstallSpec } from "@lab/shared";
 import { isValidOsId, SUPPORTED_ROLES, SUPPORTED_OS } from "@lab/shared";
 import type { StateManager } from "../services/state.js";
 import { logger } from "../services/logger.js";
@@ -14,6 +14,7 @@ import { progressBus } from "../services/progress-events.js";
 import type { ProgressEvent } from "../services/progress-events.js";
 import type { InstallLogBuffer } from "../services/install-log.js";
 import type { SyslogListener } from "../services/syslog-listener.js";
+import { isRole, requiresSchemaV2, validateDataDisks, type DataDisks } from "@lab/shared";
 
 /**
  * Seconds after dispatch with zero progress before a machine is called stalled.
@@ -24,6 +25,7 @@ const STALL_THRESHOLD_S = 8 * 60;
 
 export function registerApiRoutes(
   app: FastifyInstance,
+  config: BastionConfig,
   state: StateManager,
   installLog: InstallLogBuffer,
   syslog: SyslogListener,
@@ -42,9 +44,11 @@ export function registerApiRoutes(
       role?: string;
       os?: string;
       vyos?: VyosInstallSpec;
+      schemaVersion?: number;
+      dataDisks?: DataDisks;
     };
   }>("/api/install", async (request, reply) => {
-    const { mac: rawMac, hostname, disk, role, os, vyos } = request.body ?? {};
+    const { mac: rawMac, hostname, disk, role, os, vyos, schemaVersion, dataDisks } = request.body ?? {};
     const mac = (rawMac ?? "").toLowerCase().replace(/-/g, ":");
 
     if (mac === "") {
@@ -56,11 +60,56 @@ export function registerApiRoutes(
       return reply.status(400).send({ error: `invalid role: '${validRole}'. Supported: ${SUPPORTED_ROLES.join(", ")}` });
     }
 
+    // Reject an unknown schema rather than silently installing v1: someone
+    // asking for v3 wants a layout we do not have, and quietly giving them the
+    // old one is how a fleet ends up with layouts nobody chose.
+    if (schemaVersion !== undefined && schemaVersion !== 1 && schemaVersion !== 2) {
+      return reply.status(400).send({ error: `invalid schemaVersion: ${schemaVersion}. Supported: 1, 2` });
+    }
+
+    // Only an EXPLICIT v1 is a contradiction. Omitting the schema is normal --
+    // a reinstall never passes one, because the layout already on the disk wins.
+    if (schemaVersion === 1 && isRole(validRole) && requiresSchemaV2(validRole)) {
+      return reply.status(400).send({
+        error: `role '${validRole}' requires schemaVersion 2 (it has no v1 disk layout)`,
+      });
+    }
+
     const osId = os ?? "fedora-43";
     if (!isValidOsId(osId)) {
       return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: ${SUPPORTED_OS.join(", ")}` });
     }
 
+    // Dedicated data disks are rendered by the kickstart generator only, and
+    // only in the v2 layout. Rejected here rather than silently dropped at
+    // render time -- an operator who named disks must not get LVs instead.
+    const hasDataDisks = dataDisks !== undefined
+      && Object.values(dataDisks).some((d) => d !== undefined);
+    if (hasDataDisks) {
+      if (osId !== "fedora-43") {
+        return reply.status(400).send({
+          error: `dataDisks are only supported for fedora installs (got os '${osId}')`,
+        });
+      }
+      if (schemaVersion === 1) {
+        return reply.status(400).send({
+          error: "dataDisks require schemaVersion 2 -- the v1 layout is frozen and single-disk",
+        });
+      }
+      if (!isRole(validRole)) {
+        return reply.status(400).send({ error: `invalid role: '${validRole}'` });
+      }
+      if (!requiresSchemaV2(validRole) && schemaVersion === undefined) {
+        return reply.status(400).send({
+          error: `role '${validRole}' defaults to the v1 layout -- pass schemaVersion 2 to use dataDisks`,
+        });
+      }
+      const problem = validateDataDisks(validRole, dataDisks as DataDisks);
+      if (problem !== null) {
+        return reply.status(400).send({ error: `invalid dataDisks: ${problem}` });
+      }
+    }
+
     state.update((s) => {
       s.install_queue[mac] = {
         hostname: hostname ?? "lab-node",
@@ -68,6 +117,8 @@ export function registerApiRoutes(
         role: validRole as Role,
         os: osId,
         queued_at: new Date().toISOString(),
+        ...(schemaVersion ? { schemaVersion: schemaVersion as 1 | 2 } : {}),
+        ...(hasDataDisks ? { dataDisks } : {}),
         ...(vyos ? { vyos } : {}),
       };
     });
@@ -176,9 +227,22 @@ export function registerApiRoutes(
             : installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "lab" : "root";
           console.log(`\n  \x1b[0;32m\x1b[1m  ssh ${admin}@${ip}\x1b[0m\n`);  // eslint-disable-line no-console
 
-          // Auto-install k3s for non-vanilla roles
-          if (!isVyos && installedInfo.role !== "vanilla" && ip !== "") {
-            void triggerPostProvisionK3s(installedInfo.hostname, ip, installedInfo.role, admin, mac);
+          // Auto-install k3s for non-vanilla roles.
+          // autoK3s=false suppresses this entirely. Multi-node tests need that:
+          // post-provision installs each node into its OWN cluster, so leaving it
+          // on turns three combo nodes into three single-node clusters, and it
+          // also stops k3s and deletes CNI state minutes after install lands --
+          // right on top of a test that is forming a cluster itself.
+          if (config.autoK3s === false) {
+            logger.info(`[api] autoK3s disabled; skipping k3s post-provision for ${mac}`);
+          } else if (!isVyos && installedInfo.role !== "vanilla" && ip !== "") {
+            // State is JSON on disk; a role written by an older build may no
+            // longer exist. Skip loudly rather than crash the install callback.
+            if (isRole(installedInfo.role)) {
+              void triggerPostProvisionK3s(installedInfo.hostname, ip, installedInfo.role, admin, mac);
+            } else {
+              console.warn(`[api] unknown role ${installedInfo.role} for ${mac}; skipping k3s post-provision`);  // eslint-disable-line no-console
+            }
           }
         }
       }

bastion/src/bastion/src/routes/kickstart.ts

@@ -30,6 +30,8 @@ export function registerKickstartRoutes(
       hostname: queueEntry?.hostname ?? "lab-node",
       disk: queueEntry?.disk ?? "",
       role: queueEntry?.role ?? "worker",
+      ...(queueEntry?.schemaVersion ? { schemaVersion: queueEntry.schemaVersion } : {}),
+      ...(queueEntry?.dataDisks ? { dataDisks: queueEntry.dataDisks } : {}),
     });
 
     return reply.type("text/plain").send(ks);

bastion/src/bastion/src/server.ts

@@ -46,7 +46,7 @@ export function createApp(config: BastionConfig): { app: ReturnType<typeof Fasti
   // Register route handlers
   registerDispatchRoutes(app, config, state);
   registerKickstartRoutes(app, config, state, syslog);
-  registerApiRoutes(app, state, installLog, syslog);
+  registerApiRoutes(app, config, state, installLog, syslog);
   registerAsahiRoutes(app, config);
   registerVyosRoutes(app, config, state);
   // boot.iso is generated at startup and served as a static file from httpDir

bastion/src/bastion/src/services/kickstart-generator.ts

@@ -24,12 +24,16 @@ export function generateInstallKickstart(
     hostname: string;
     disk: string;
     role: Role;
+    schemaVersion?: 1 | 2;
+    dataDisks?: import("@lab/shared").DataDisks;
   },
 ): string {
   const ksParams: InstallKickstartParams = {
     hostname: params.hostname,
     disk: params.disk,
     role: params.role,
+    ...(params.schemaVersion ? { schemaVersion: params.schemaVersion } : {}),
+    ...(params.dataDisks ? { dataDisks: params.dataDisks } : {}),
     domain: config.domain,
     fedoraVersion: config.fedoraVersion,
     timezone: config.timezone,

bastion/src/bastion/src/services/post-provision.ts

@@ -7,6 +7,7 @@ import { homedir } from "node:os";
 import { join } from "node:path";
 import { logger } from "./logger.js";
 import { progressBus } from "./progress-events.js";
+import { isK3sServer, type Role } from "@lab/shared";
 
 function findSshKey(): string | undefined {
   const sudoUser = process.env["SUDO_USER"];
@@ -84,6 +85,36 @@ function sshRunStreaming(ip: string, user: string, command: string, keyPath: str
   });
 }
 
+/**
+ * Roles this bare post-provision install can legitimately handle.
+ *
+ * The install below is a plain `INSTALL_K3S_EXEC=server|agent` with no K3S_URL
+ * and no K3S_TOKEN, and it knows about exactly one machine. That is fine for a
+ * node that is its own cluster, and wrong for anything that has to JOIN one:
+ *
+ *   - a cluster member installed this way gets `cluster-init` semantics and
+ *     becomes its own separate cluster, so provisioning three of them yields
+ *     three clusters instead of one (split brain);
+ *   - an AGENT cannot even start — `INSTALL_K3S_EXEC=agent` with nothing to
+ *     join fails with "Job for k3s-agent.service failed because the control
+ *     process exited with error code", which is exactly what a k8s-worker
+ *     install does today.
+ *
+ * So cluster members are not auto-installed. They are handed the command that
+ * supplies the missing server URL and token.
+ */
+export function shouldAutoInstallK3s(role: Role): boolean {
+  switch (role) {
+    case "k8s-combo":
+    case "k8s-controller":
+    case "k8s-worker":
+    case "worker":
+      return false;
+    default:
+      return true;
+  }
+}
+
 /**
  * Trigger k3s installation on a freshly provisioned machine.
  * Runs in the background — logs progress to bastion console and progressBus.
@@ -91,7 +122,7 @@ function sshRunStreaming(ip: string, user: string, command: string, keyPath: str
 export async function triggerPostProvisionK3s(
   hostname: string,
   ip: string,
-  role: string,
+  role: Role,
   sshUser: string,
   mac?: string,
 ): Promise<void> {
@@ -104,6 +135,21 @@ export async function triggerPostProvisionK3s(
     }
   };
 
+  // Gate before ANY work: the prerequisite steps below stop k3s and delete CNI
+  // state, so they must not run for a role we are declining to install either.
+  if (!shouldAutoInstallK3s(role)) {
+    emitStage(
+      "post-provision",
+      `${role} is a cluster member; not auto-installing k3s (it would form its own cluster)`,
+    );
+    const verb = isK3sServer(role) ? "--k3s-server https://<existing-server>:6443" : "--k3s-server https://<server>:6443";
+    logger.info(
+      `[k3s] Join it to the cluster with:\n` +
+      `        labctl app k3s install ${hostname} --role ${role} ${verb} --k3s-token <token>`,
+    );
+    return;
+  }
+
   emitStage("post-provision", `auto-installing k3s on ${hostname} (${ip}) role=${role}`);
   emitStage("post-provision", "waiting for SSH (machine may still be rebooting)");
 
@@ -150,7 +196,7 @@ sysctl --system > /dev/null'`, keyPath, hostname, mac);
 
   // Step 4: Install k3s
   // labcontroller extends infra — both are k3s servers
-  const k3sRole = (role === "infra" || role === "labcontroller") ? "server" : "agent";
+  const k3sRole = isK3sServer(role) ? "server" : "agent";
   emitStage("post-provision", `installing k3s ${k3sRole}`);
   const code = await sshRunStreaming(ip, sshUser,
     `curl -sfL https://get.k3s.io | sudo INSTALL_K3S_EXEC="${k3sRole}" INSTALL_K3S_SKIP_SELINUX_RPM=true sh -`,

bastion/src/bastion/src/templates/asahi-firstboot.sh.ts

@@ -3,6 +3,7 @@
 // Creates the standard lab LVM layout on the data partition, matching install.ks.ts.
 
 import type { Role } from "@lab/shared";
+import { isK3sServer, v2Layout, type LvSpec } from "@lab/shared";
 
 export interface AsahiFirstbootParams {
   hostname: string;
@@ -12,13 +13,42 @@ export interface AsahiFirstbootParams {
   sshKeys: string[];
   adminUser: string;
   mac: string;
+  /** Fresh-setup layout. Defaults to 1 so existing callers are unchanged. */
+  schemaVersion?: 1 | 2;
 }
 
 export function renderFirstbootScript(params: AsahiFirstbootParams): string {
   const { hostname, role, serverIp, httpPort, sshKeys, adminUser, mac } = params;
+  const schemaVersion = params.schemaVersion ?? 1;
+
+  // ---------------------------------------------------------------------------
+  // v2 LAYOUT, from the SAME definition Fedora and Ubuntu render
+  // (shared/src/types/layout.ts).
+  //
+  // omitRoot: on Apple Silicon the Asahi installer has already created a btrfs
+  // root partition OUTSIDE our volume group -- worker1 runs a 4.6 GB one. We
+  // must not create a second root LV.
+  //
+  // The v1 path below hardcodes rancher at 20480M while claiming "sizes match
+  // install.ks.ts", which uses 122880M. That 100 GB gap is what left worker1
+  // with an image store too small to hold a single large container image.
+  // ---------------------------------------------------------------------------
+  const v2 = v2Layout(role, { omitRoot: true });
+  const v2LvLines = v2.map((lv: LvSpec) => `lvcreate -L ${lv.sizeMib}M -n ${lv.name} labvg -y`).join("\n");
+  const v2FormatLines = v2
+    .map((lv: LvSpec) => (lv.fs === "swap" ? `mkswap /dev/labvg/${lv.name}` : `mkfs.xfs /dev/labvg/${lv.name}`))
+    .join("\n");
+  const v2MountLines = v2
+    .filter((lv: LvSpec) => lv.fs !== "swap")
+    .map((lv: LvSpec) => `mount_lv ${lv.name} ${lv.mount}`)
+    .join("\n");
+  const v2FstabLines = v2
+    .filter((lv: LvSpec) => lv.fs !== "swap")
+    .map((lv: LvSpec) => `echo "/dev/labvg/${lv.name} ${lv.mount} xfs defaults 0 0" >> /etc/fstab`)
+    .join("\n");
 
   const isWorker = role === "worker";
-  const isInfra = role === "infra" || role === "labcontroller";
+  const isInfra = isK3sServer(role);
 
   // Role-specific LV creation commands
   const roleLvLines: string[] = [];
@@ -127,24 +157,27 @@ write_lab_fstab() {
     # Add fresh LVM entries
     echo "# Lab LVM volumes" >> /etc/fstab
     echo "/dev/labvg/swap    none         swap defaults 0 0" >> /etc/fstab
-    echo "/dev/labvg/var     /var         xfs  defaults 0 0" >> /etc/fstab
+${schemaVersion === 2 ? v2FstabLines.split("\n").map((l: string) => "    " + l).join("\n") : `    echo "/dev/labvg/var     /var         xfs  defaults 0 0" >> /etc/fstab
     echo "/dev/labvg/varlog  /var/log     xfs  defaults 0 0" >> /etc/fstab
     echo "/dev/labvg/home    /home        xfs  defaults 0 0" >> /etc/fstab
     echo "/dev/labvg/srv     /srv         xfs  defaults 0 0" >> /etc/fstab
-    ${roleFstabLines.join('\n    ')}
+    ${roleFstabLines.join('\n    ')}`}
 }
 
 # ── Check for existing VG ────────────────────────────────────────
 if vgs labvg &>/dev/null; then
     echo "Volume group 'labvg' already exists — reprovision detected."
+    DISK_SCHEMA=v1
+    vgs --noheadings -o vg_tags labvg 2>/dev/null | grep -q lab-schema-v2 && DISK_SCHEMA=v2
+    echo "On-disk schema: $DISK_SCHEMA"
     echo "Activating existing volumes..."
     vgchange -ay labvg
 
-    mount_lv var /var
+${schemaVersion === 2 ? v2MountLines.split("\n").map((l: string) => "    " + l).join("\n") : `    mount_lv var /var
     mount_lv varlog /var/log
     mount_lv home /home
     mount_lv srv /srv
-${roleMountLines.map(l => `    ${l}`).join('\n')}
+${roleMountLines.map(l => `    ${l}`).join('\n')}`}
 
     # Enable swap
     if lvs labvg/swap &>/dev/null; then
@@ -162,24 +195,28 @@ echo "Creating LVM on $DATA_PART..."
 
 pvcreate "$DATA_PART"
 vgcreate labvg "$DATA_PART"
+${schemaVersion === 2 ? `
+# Tag so a later run can tell which layout is on this disk without trusting a
+# caller-supplied flag -- same mechanism install.ks.ts uses.
+vgchange --addtag lab-schema-v2 labvg >/dev/null 2>&1 || true` : ""}
 
 # Create LVs — sizes match install.ks.ts (in MiB)
 echo "Creating logical volumes..."
-lvcreate -L 27648M  -n swap   labvg -y   # 27GB swap
+${schemaVersion === 2 ? v2LvLines : `lvcreate -L 27648M  -n swap   labvg -y   # 27GB swap
 lvcreate -L 102400M -n var    labvg -y   # 100GB /var
 lvcreate -L 10240M  -n varlog labvg -y   # 10GB /var/log
 lvcreate -L 10240M  -n home   labvg -y   # 10GB /home
 lvcreate -L 20480M  -n srv    labvg -y   # 20GB /srv
-${roleLvLines.join('\n')}
+${roleLvLines.join('\n')}`}
 
 # Format
 echo "Formatting volumes..."
-mkswap /dev/labvg/swap
+${schemaVersion === 2 ? v2FormatLines : `mkswap /dev/labvg/swap
 mkfs.xfs /dev/labvg/var
 mkfs.xfs /dev/labvg/varlog
 mkfs.xfs /dev/labvg/home
 mkfs.xfs /dev/labvg/srv
-${roleFormatLines.join('\n')}
+${roleFormatLines.join('\n')}`}
 
 # Migrate and mount volumes that can be switched live.
 # Copy existing content first so we don't shadow files (e.g. /home/user/.ssh).

bastion/src/bastion/src/templates/install.ks.ts

@@ -2,7 +2,11 @@
 // Full Fedora server install with LVM partitioning, %pre for reprovision detection,
 // packages, and %post with SSH keys, user creation, k3s prereqs, progress callbacks.
 
-import type { Role } from "@lab/shared";
+import type { Role, DataDisks, DataDiskName } from "@lab/shared";
+import {
+  v2Layout, V2_BOOT, requiresSchemaV2, effectiveSchemaVersion, isK3sServer,
+  validateDataDisks, DATA_DISK_LABELS, type LvSpec,
+} from "@lab/shared";
 
 export interface InstallKickstartParams {
   hostname: string;
@@ -17,6 +21,22 @@ export interface InstallKickstartParams {
   syslogPort: number;
   sshKeys: string[];
   adminUser: string;
+  /**
+   * Disk layout to use for a FRESH install. Defaults to 1 so every existing
+   * caller keeps today's layout untouched.
+   *
+   * On a REPROVISION this is ignored: the schema already on the disk wins,
+   * read from an LVM tag on the volume group. Reinstalling a machine must
+   * never silently change its layout -- that is how data gets destroyed.
+   */
+  schemaVersion?: 1 | 2;
+  /**
+   * Dedicated whole-disk data volumes, by stable /dev/disk/by-id path (v2
+   * only). The matching LV is not created; the disk gets one labelled xfs
+   * partition instead. On a reprovision %pre rediscovers these by label, so a
+   * reinstall preserves them whether or not the queue entry names them.
+   */
+  dataDisks?: DataDisks;
 }
 
 export function renderInstallKickstart(params: InstallKickstartParams): string {
@@ -38,6 +58,65 @@ export function renderInstallKickstart(params: InstallKickstartParams): string {
   const fqdn = domain ? `${hostname}.${domain}` : hostname;
   const vg = "labvg";
   const now = new Date().toISOString();
+  // ---------------------------------------------------------------------------
+  // v1 LAYOUT -- DO NOT REPLACE THESE WITH CAPABILITY LOOKUPS.
+  //
+  // These literals are deliberately frozen. Every machine already in the fleet
+  // was partitioned by them, and `labctl provision reprovision` must reproduce
+  // that same layout byte-for-byte or it destroys data. Yes, they are wrong:
+  // `labcontroller` runs k3s and gets no rancher LV, and `infra` nodes hold
+  // every Longhorn replica in the homelab cluster while only `worker` gets a
+  // longhorn LV. Those are fixed in the v2 layout, which new boxes get; see
+  // schemaVersion. Changing them here rewrites history for existing hardware.
+  // ---------------------------------------------------------------------------
+  // ---------------------------------------------------------------------------
+  // v2 LAYOUT. One definition, shared with ubuntu-autoinstall and asahi-firstboot
+  // (shared/src/types/layout.ts) so the three generators cannot drift apart
+  // again. Capability-gated, and nothing uses --grow.
+  // ---------------------------------------------------------------------------
+  const schemaVersion = effectiveSchemaVersion(role, params.schemaVersion);
+  if (params.schemaVersion === 1 && requiresSchemaV2(role)) {
+    // Fail here rather than render a machine with no rancher/longhorn LV.
+    throw new Error(
+      `role '${role}' has no v1 disk layout - it was introduced with schema v2. ` +
+      `Pass schemaVersion: 2 (labctl provision install --schema-version 2).`,
+    );
+  }
+
+  const dataDisks = params.dataDisks ?? {};
+  const hasDataDisks = Object.values(dataDisks).some((d) => d !== undefined);
+  if (hasDataDisks) {
+    // The v1 branches are frozen and know nothing about dedicated disks, so a
+    // v1 render with them would silently create BOTH the LV and the disk.
+    if (schemaVersion !== 2) {
+      throw new Error(
+        `dedicated data disks require the v2 layout - pass schemaVersion: 2 ` +
+        `(labctl provision install --schema-version 2)`,
+      );
+    }
+    const problem = validateDataDisks(role, dataDisks);
+    if (problem !== null) throw new Error(problem);
+  }
+
+  const v2RootMib = v2Layout(role).find((lv: LvSpec) => lv.name === "root")?.sizeMib ?? 32768;
+  const v2 = v2Layout(role, { dataDisks });
+  const v2LogvolLines = v2
+    .map((lv: LvSpec) =>
+      lv.fs === "swap"
+        ? `logvol swap --vgname=${vg} --name=${lv.name} --fstype=swap --size=${lv.sizeMib}`
+        : `logvol ${lv.mount} --vgname=${vg} --name=${lv.name} --fstype=xfs --size=${lv.sizeMib}`,
+    )
+    .join("\n");
+  /** OS volumes only - the ones a reprovision recreates rather than preserves. */
+  const v2OsLogvolLines = v2
+    .filter((lv: LvSpec) => ["swap", "root", "var", "varlog"].includes(lv.name))
+    .map((lv: LvSpec) =>
+      lv.fs === "swap"
+        ? `logvol swap --vgname=${vg} --name=${lv.name} --fstype=swap --size=${lv.sizeMib}`
+        : `logvol ${lv.mount} --vgname=${vg} --name=${lv.name} --fstype=xfs --size=${lv.sizeMib}`,
+    )
+    .join("\n");
+
   const hasLonghorn = role === "worker";
   const hasRancher = role === "infra";
   // k8s roles get a dedicated 120G image-store LV. 2026-08 incident: the old
@@ -46,6 +125,12 @@ export function renderInstallKickstart(params: InstallKickstartParams): string {
   // VG space, making post-install lvextend impossible on worker nodes.
   const hasRancherLv = role === "infra" || role === "worker";
   const isVanilla = role === "vanilla";
+  // etcd members need etcdctl available OFFLINE. The recovery procedure for a
+  // lost or stale member is `etcdctl member list/remove`, and `k3s etcdctl` is
+  // not a command -- the binary comes from the `etcd` rpm. Telling an operator
+  // to `dnf install etcd` in the middle of a quorum failure assumes working
+  // DNS, routing and a mirror, which is exactly what may be broken.
+  const needsEtcdctl = isK3sServer(role);
 
   // -- Auth section --
   // Always set a root password (for serial console debugging) + SSH keys
@@ -90,14 +175,29 @@ chmod 440 /etc/sudoers.d/${adminUser}`;
   }
 
   // -- Disk detection --
+  // An explicit disk may be a stable /dev/disk/by-id path: kernel names are
+  // not stable across boots on multi-NVMe hosts, so a pinned install must be
+  // able to name the disk by serial. Anaconda directives want the kernel name,
+  // so %pre resolves the symlink at install time.
   const diskLine = disk
-    ? `DISK="${disk}"`
+    ? disk.startsWith("/dev/")
+      ? `# OS disk pinned by stable path -- resolve to the kernel name anaconda wants
+DISK=""
+for _wait in $(seq 1 10); do
+    _os_target=$(readlink -f "${disk}" 2>/dev/null || true)
+    if [ -n "$_os_target" ] && [ -b "$_os_target" ]; then DISK=$(basename "$_os_target"); break; fi
+    sleep 1
+done
+[ -z "$DISK" ] && { bastion_progress "error" "OS disk not found: ${disk}"; echo "ERROR: OS disk not found: ${disk}"; exit 1; }`
+      : `DISK="${disk}"`
     : `DISK=""
 # Wait up to 10s for NVMe/SCSI disks to appear (they init async in initrd)
 for _wait in $(seq 1 10); do
     for d in /dev/nvme0n1 /dev/nvme1n1 /dev/sda /dev/sdb /dev/vda; do
         [ -b "$d" ] || continue
         _bname=$(basename "$d")
+        # Never auto-pick a dedicated data disk as the OS disk
+        case "$_bname" in "$LONGHORN_DISK"|"$RANCHER_DISK") continue ;; esac
         # Skip removable disks (USB, CD-ROM, JetKVM virtual media)
         [ -f "/sys/block/$_bname/removable" ] && [ "$(cat /sys/block/$_bname/removable)" = "1" ] && continue
         # Skip USB-attached disks (JetKVM virtual media shows as SCSI over USB)
@@ -113,6 +213,80 @@ for _wait in $(seq 1 10); do
 done
 [ -z "$DISK" ] && { echo "ERROR: no disk found"; exit 1; }`;
 
+  // -- Dedicated data disks: %pre resolution --
+  // Resolved BEFORE the OS disk so auto-detect can refuse to pick one of them.
+  const dataDiskResolveBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[])
+    .map((name) => {
+      const device = dataDisks[name];
+      const VAR = name.toUpperCase();
+      if (device === undefined) return `${VAR}_DISK=""`;
+      return `# ${name} gets a whole disk of its own, pinned by stable path
+${VAR}_DISK_ID="${device}"
+${VAR}_DISK=""
+for _wait in $(seq 1 10); do
+    _dd_target=$(readlink -f "$${VAR}_DISK_ID" 2>/dev/null || true)
+    if [ -n "$_dd_target" ] && [ -b "$_dd_target" ]; then ${VAR}_DISK=$(basename "$_dd_target"); break; fi
+    sleep 1
+done
+[ -z "$${VAR}_DISK" ] && { bastion_progress "error" "${name} disk not found: $${VAR}_DISK_ID"; echo "ERROR: ${name} disk not found: $${VAR}_DISK_ID"; exit 1; }`;
+    })
+    .join("\n");
+
+  // -- Dedicated data disks: discovery and preservation decisions --
+  // A labelled data partition is on-disk truth, exactly like the lab-schema-v2
+  // LVM tag: it is preserved whether or not this install names the disk, and it
+  // is never silently wiped or shadowed. blkid finds it wherever tonight's NVMe
+  // enumeration put it.
+  const dataDiskDecisionBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[])
+    .map((name) => {
+      const VAR = name.toUpperCase();
+      const label = DATA_DISK_LABELS[name];
+      const mount = name === "longhorn" ? "/var/lib/longhorn" : "/var/lib/rancher";
+      return `${VAR}_PART=$(blkid -t LABEL=${label} -o device 2>/dev/null | head -1)
+${VAR}_REUSE=no
+if [ -n "$${VAR}_PART" ]; then
+    _dd_parent=$(lsblk -ndo pkname "$${VAR}_PART" 2>/dev/null)
+    if [ -n "$${VAR}_DISK" ] && [ "$_dd_parent" != "$${VAR}_DISK" ]; then
+        bastion_progress "error" "${label} label is on $_dd_parent, not on the configured ${name} disk $${VAR}_DISK"
+        echo "ERROR: ${label} label found on $_dd_parent but the ${name} disk is configured as $${VAR}_DISK -- refusing to guess"
+        exit 1
+    fi
+    if [ "$REPROVISION" = "no" ] && [ -z "$${VAR}_DISK" ]; then
+        bastion_progress "error" "found ${label} data on $_dd_parent but this install configures no ${name} disk"
+        echo "ERROR: ${label} labelled data exists on $_dd_parent. Refusing to wipe or shadow it -- queue the install with a ${name} data disk, or wipe the label first"
+        exit 1
+    fi
+    ${VAR}_DISK="$_dd_parent"
+    ${VAR}_REUSE=yes
+fi
+if [ "$PRESERVE_${VAR}" = "yes" ] && [ -n "$${VAR}_DISK" ]; then
+    bastion_progress "error" "both a ${name} LV and a dedicated ${name} disk exist"
+    echo "ERROR: VG $VG has a ${name} LV AND a dedicated ${name} disk is present -- two things cannot mount ${mount}. Migrate the LV off first"
+    exit 1
+fi`;
+    })
+    .join("\n\n");
+
+  // -- Dedicated data disks: partition directives --
+  // Appended to whichever /tmp/part.ks the branches above wrote. A reused
+  // partition is mounted --noformat; a fresh disk gets ONE partition spanning
+  // it. --grow is fine HERE: the whole point of a dedicated disk is to consume
+  // it, unlike LVs competing inside a shared VG.
+  const dataDiskPartBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[])
+    .map((name) => {
+      const VAR = name.toUpperCase();
+      const label = DATA_DISK_LABELS[name];
+      const mount = name === "longhorn" ? "/var/lib/longhorn" : "/var/lib/rancher";
+      return `if [ -n "$${VAR}_DISK" ]; then
+    if [ "$${VAR}_REUSE" = "yes" ]; then
+        echo "part ${mount} --onpart=$${VAR}_PART --noformat" >> /tmp/part.ks
+    else
+        echo "part ${mount} --fstype=xfs --ondisk=$${VAR}_DISK --grow --size=1 --label=${label}" >> /tmp/part.ks
+    fi
+fi`;
+    })
+    .join("\n");
+
   // -- Longhorn LV for fresh install --
   const longhornFreshLine = hasLonghorn
     ? `logvol /var/lib/longhorn --vgname=${vg} --name=longhorn --fstype=xfs --grow --size=1`
@@ -134,7 +308,13 @@ lang ${locale}
 keyboard uk
 timezone ${timezone} --utc
 
-network --bootproto=dhcp --activate --hostname=${fqdn}
+# --ipv6=auto, not =dhcp: "auto" follows the router advertisement, so the RA's
+# managed-flag is what steers the node to DHCPv6, and a VLAN with no DHCPv6 yet
+# still installs instead of blocking on a lease that will never come. Failing an
+# OS install because IPv6 was not ready would be a worse trade than a node that
+# briefly has no v6. The address itself comes from a kea DHCPv6 reservation
+# keyed on MAC -- the same source of truth as the v4 address.
+network --bootproto=dhcp --ipv6=auto --activate --hostname=${fqdn}
 
 ${auth}
 ${userDirective}
@@ -164,6 +344,9 @@ bastion_progress() {
 bastion_progress "partitioning" "detecting disk"
 
 VG="${vg}"
+
+${dataDiskResolveBlocks}
+
 ${diskLine}
 
 REPROVISION=no
@@ -173,6 +356,16 @@ if vgs $VG &>/dev/null; then
     echo "=== Existing VG found - reprovision mode ==="
     REPROVISION=yes
 
+    # Which layout is ALREADY on this disk? The LVM tag is authoritative -- the
+    # schemaVersion the caller asked for is irrelevant here, because silently
+    # re-laying-out an existing machine destroys its data. A v1 box reinstalls
+    # as v1 forever, until it is deliberately migrated.
+    DISK_SCHEMA=v1
+    if vgs --noheadings -o vg_tags $VG 2>/dev/null | grep -q lab-schema-v2; then
+        DISK_SCHEMA=v2
+    fi
+    echo "On-disk schema: $DISK_SCHEMA"
+
     # Detect which data LVs to preserve
     PRESERVE_LONGHORN=no; PRESERVE_SRV=no; PRESERVE_HOME=no; PRESERVE_RANCHER=no
     lvs $VG/longhorn &>/dev/null && PRESERVE_LONGHORN=yes
@@ -190,6 +383,19 @@ else
     bastion_progress "partitioning" "fresh install on $DISK"
 fi
 
+${dataDiskDecisionBlocks}
+
+# Every disk anaconda may touch; everything else stays invisible to it.
+KS_DISKS="$DISK"
+[ -n "$LONGHORN_DISK" ] && KS_DISKS="$KS_DISKS,$LONGHORN_DISK"
+[ -n "$RANCHER_DISK" ] && [ "$RANCHER_DISK" != "$LONGHORN_DISK" ] && KS_DISKS="$KS_DISKS,$RANCHER_DISK"
+
+# Fresh installs wipe the OS disk plus any data disk being formatted for the
+# first time -- NEVER one carrying a lab-* label (those are reused --noformat).
+CLEARPART_DRIVES="$DISK"
+[ -n "$LONGHORN_DISK" ] && [ "$LONGHORN_REUSE" = "no" ] && CLEARPART_DRIVES="$CLEARPART_DRIVES,$LONGHORN_DISK"
+[ -n "$RANCHER_DISK" ] && [ "$RANCHER_REUSE" = "no" ] && [ "$RANCHER_DISK" != "$LONGHORN_DISK" ] && CLEARPART_DRIVES="$CLEARPART_DRIVES,$RANCHER_DISK"
+
 if [ "$REPROVISION" = "yes" ]; then
     # Find existing boot partitions by type
     EFI_PART=$(blkid -t TYPE=vfat -o device /dev/\${DISK}* 2>/dev/null | head -1)
@@ -200,16 +406,26 @@ if [ "$REPROVISION" = "yes" ]; then
 
     # Build partition config reusing existing PV/VG
     cat > /tmp/part.ks << PARTEOF
-ignoredisk --only-use=$DISK
+ignoredisk --only-use=$KS_DISKS
 clearpart --none
 part /boot/efi --onpart=$EFI_PART --fstype=efi
 part /boot --onpart=$BOOT_PART --fstype=ext4
 volgroup ${vg} --useexisting --noformat
+PARTEOF
+
+    # OS volumes are recreated, so they must match the layout already on disk.
+    if [ "$DISK_SCHEMA" = "v2" ]; then
+        cat >> /tmp/part.ks << PARTEOF
+${v2OsLogvolLines}
+PARTEOF
+    else
+        cat >> /tmp/part.ks << PARTEOF
 logvol swap --vgname=${vg} --name=swap --fstype=swap --size=27648
 logvol / --vgname=${vg} --name=root --fstype=xfs --size=33792
 logvol /var --vgname=${vg} --name=var --fstype=xfs --size=102400
 logvol /var/log --vgname=${vg} --name=varlog --fstype=xfs --size=10240
 PARTEOF
+    fi
 
     # Preserve or recreate data LVs
     if [ "$PRESERVE_HOME" = "yes" ]; then
@@ -234,7 +450,19 @@ PARTEOF
 
 else
     # Fresh install
-    cat > /tmp/part.ks << PARTEOF
+${schemaVersion === 2 ? `    cat > /tmp/part.ks << PARTEOF
+ignoredisk --only-use=$KS_DISKS
+clearpart --all --initlabel --drives=$CLEARPART_DRIVES
+part /boot/efi --fstype=efi --size=${V2_BOOT.efiMib} --ondisk=$DISK
+part /boot --fstype=ext4 --size=${V2_BOOT.bootMib} --ondisk=$DISK
+part pv.01 --size=1 --grow --ondisk=$DISK
+volgroup ${vg} pv.01
+${v2LogvolLines}
+PARTEOF
+    # Tag the VG so a later reprovision knows this disk is v2 WITHOUT trusting
+    # whatever schemaVersion the caller happens to pass. The tag lives on the
+    # VG, which a reprovision preserves; a marker file would not survive.
+    SCHEMA_TAG_PENDING=v2` : `    cat > /tmp/part.ks << PARTEOF
 ignoredisk --only-use=$DISK
 clearpart --all --initlabel --drives=$DISK
 part /boot/efi --fstype=efi --size=600 --ondisk=$DISK
@@ -249,9 +477,11 @@ logvol /home --vgname=${vg} --name=home --fstype=xfs --size=10240
 logvol /srv --vgname=${vg} --name=srv --fstype=xfs --size=20480
 ${longhornFreshLine}
 ${rancherFreshLine}
-PARTEOF
+PARTEOF`}
 fi
 
+${dataDiskPartBlocks}
+
 echo "=== Generated partition config ==="
 cat /tmp/part.ks
 echo "==================================="
@@ -296,6 +526,9 @@ tar
 socat
 conntrack-tools
 ethtool`}
+${needsEtcdctl ? `
+# etcdctl, for recovering etcd membership without internet access
+etcd` : ""}
 
 # Boot management
 efibootmgr
@@ -366,6 +599,21 @@ fs.inotify.max_user_watches         = 1048576
 SYSCTL
 sysctl --system || true
 
+# -- IPv6 link-local address generation: EUI-64, fleet-wide --
+# A cluster node takes its IPv6 from a MAC-keyed DHCPv6 reservation. kea can only
+# match that reservation when it can recover the node's MAC, and for a modern
+# client that sends a DUID-UUID (no MAC in it) the only place kea can find one is
+# an EUI-64 link-local. NetworkManager's default is stable-privacy (RFC 7217),
+# whose link-local hides the MAC -- so a node on the default silently never gets
+# its reserved address, and with a reservations-only subnet it gets nothing at
+# all. Proven on 2026-09-06: worker0/worker2 (eui64) bound; worker1/spark
+# (default) did not, until flipped. Setting it here means a new node is correct
+# from first boot, before its connection is ever activated.
+cat > /etc/NetworkManager/conf.d/10-ipv6-eui64.conf << 'NMEUI64'
+[connection]
+ipv6.addr-gen-mode=eui64
+NMEUI64
+
 # -- Disable firewalld permanently (k3s/Cilium manage iptables directly) --
 # Note: no '--now' — systemd is not running in the Anaconda chroot
 systemctl disable firewalld || true
@@ -393,12 +641,30 @@ bastion_progress "post-install" "3-bootorder done"
 # -- Enable SysRq magic keys (for emergency reboot via Alt+SysRq+REISUB) --
 echo "kernel.sysrq=1" > /etc/sysctl.d/90-sysrq.conf
 
+# -- Disk schema tag --
+# Tag the VG so a future reprovision can tell which layout is on this disk
+# WITHOUT trusting whatever the caller passes. Deliberately derived from what
+# actually landed on disk rather than from render-time intent: a v1 machine
+# reinstalled with --schema-version 2 keeps its v1 layout (see %pre), and must
+# not end up wearing a v2 tag it does not deserve.
+V2_ROOT_MIB=${v2RootMib}
+ROOT_MIB=$(lvs --noheadings --units m --nosuffix -o lv_size ${vg}/root 2>/dev/null | tr -d ' ' | cut -d. -f1)
+if [ "$ROOT_MIB" = "$V2_ROOT_MIB" ]; then
+    vgchange --addtag lab-schema-v2 ${vg} >/dev/null 2>&1 || true
+    echo "Tagged ${vg} as lab-schema-v2 (root=\${ROOT_MIB}M)"
+    DISK_SCHEMA_ON_DISK=2
+else
+    DISK_SCHEMA_ON_DISK=1
+    echo "Layout is v1 (root=\${ROOT_MIB}M, v2 would be \${V2_ROOT_MIB}M)"
+fi
+
 # -- Provisioning metadata --
 cat > /etc/lab-provisioned << PROVEOF
 hostname: ${fqdn}
 role: ${role}
 provisioned: $(date -Iseconds)
 bastion: ${serverIp}
+schema: v$DISK_SCHEMA_ON_DISK
 PROVEOF
 
 cat > /root/README << 'README'

bastion/src/bastion/src/templates/ubuntu-autoinstall.ts

@@ -2,10 +2,15 @@
 // Equivalent of the Fedora kickstart: LVM partitioning, packages,
 // SSH keys, k3s prereqs, progress callbacks.
 
+import { v2Layout, type LvSpec } from "@lab/shared";
+import type { Role } from "@lab/shared";
+
 export interface UbuntuAutoinstallParams {
   hostname: string;
   disk: string;
-  role: string;  // "vanilla" | "worker" | "infra"
+  role: Role;
+  /** Fresh-install layout. Defaults to 1 so existing callers are unchanged. */
+  schemaVersion?: 1 | 2;
   domain: string;
   ubuntuVersion: string;
   timezone: string;
@@ -31,6 +36,48 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
 
   const fqdn = domain ? `${hostname}.${domain}` : hostname;
   const vg = "labvg";
+  // ---------------------------------------------------------------------------
+  // v1 LAYOUT -- DO NOT REPLACE THESE WITH CAPABILITY LOOKUPS.
+  //
+  // These literals are deliberately frozen. Every machine already in the fleet
+  // was partitioned by them, and `labctl provision reprovision` must reproduce
+  // that same layout byte-for-byte or it destroys data. Yes, they are wrong:
+  // `labcontroller` runs k3s and gets no rancher LV, and `infra` nodes hold
+  // every Longhorn replica in the homelab cluster while only `worker` gets a
+  // longhorn LV. Those are fixed in the v2 layout, which new boxes get; see
+  // schemaVersion. Changing them here rewrites history for existing hardware.
+  // ---------------------------------------------------------------------------
+  // ---------------------------------------------------------------------------
+  // v2 LAYOUT, rendered from the SAME definition Fedora and Asahi use
+  // (shared/src/types/layout.ts). Before this, Ubuntu sized rancher at 20G
+  // while the kickstart used 120G -- the drift that the 2026-08 imagefs
+  // eviction incident was caused by.
+  //
+  // Indentation is load-bearing: list items sit at 6 spaces, their keys at 8.
+  // At any other depth curtin's YAML does not parse. See the v1 comment below.
+  // ---------------------------------------------------------------------------
+  const schemaVersion = params.schemaVersion ?? 1;
+  const renderV2Lv = (lv: LvSpec): string => {
+    const gib = Math.round(lv.sizeMib / 1024);
+    const mountPath = lv.fs === "swap" ? "none" : lv.mount;
+    return [
+      `      - id: lv-${lv.name}`,
+      `        name: ${lv.name}`,
+      `        type: lvm_partition`,
+      `        volgroup: vg0`,
+      `        size: ${gib}G`,
+      `      - id: fs-${lv.name}`,
+      `        type: format`,
+      `        volume: lv-${lv.name}`,
+      `        fstype: ${lv.fs}`,
+      `      - id: mount-${lv.name}`,
+      `        type: mount`,
+      `        device: fs-${lv.name}`,
+      `        path: ${mountPath}`,
+    ].join("\n");
+  };
+  const v2LvBlock = v2Layout(role).map(renderV2Lv).join("\n");
+
   const hasLonghorn = role === "worker";
   const hasRancher = role === "infra";
 
@@ -40,34 +87,39 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
   // Build the LVM layout to match Fedora kickstart sizes
   const extraLvs: string[] = [];
   if (hasLonghorn) {
-    extraLvs.push(`        - id: lv-longhorn
-          name: longhorn
-          type: lvm_partition
-          volgroup: vg0
-          size: -1
-        - id: fs-longhorn
-          type: format
-          volume: lv-longhorn
-          fstype: xfs
-        - id: mount-longhorn
-          type: mount
-          device: fs-longhorn
-          path: /var/lib/longhorn`);
+    // 6 spaces for the list item, 8 for its keys -- these are siblings of the
+    // lv-home/lv-srv entries in storage.config, which sit at 6. At 8 the
+    // rendered document is not valid YAML at all ("expected <block end>, but
+    // found '-'"), so an Ubuntu node with the longhorn role could never have
+    // installed. Ubuntu + longhorn is exactly the worker shape.
+    extraLvs.push(`      - id: lv-longhorn
+        name: longhorn
+        type: lvm_partition
+        volgroup: vg0
+        size: -1
+      - id: fs-longhorn
+        type: format
+        volume: lv-longhorn
+        fstype: xfs
+      - id: mount-longhorn
+        type: mount
+        device: fs-longhorn
+        path: /var/lib/longhorn`);
   }
   if (hasRancher) {
-    extraLvs.push(`        - id: lv-rancher
-          name: rancher
-          type: lvm_partition
-          volgroup: vg0
-          size: 20G
-        - id: fs-rancher
-          type: format
-          volume: lv-rancher
-          fstype: xfs
-        - id: mount-rancher
-          type: mount
-          device: fs-rancher
-          path: /var/lib/rancher`);
+    extraLvs.push(`      - id: lv-rancher
+        name: rancher
+        type: lvm_partition
+        volgroup: vg0
+        size: 20G
+      - id: fs-rancher
+        type: format
+        volume: lv-rancher
+        fstype: xfs
+      - id: mount-rancher
+        type: mount
+        device: fs-rancher
+        path: /var/lib/rancher`);
   }
 
   const extraLvsBlock = extraLvs.length > 0 ? "\n" + extraLvs.join("\n") : "";
@@ -81,6 +133,11 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
     `curtin in-target -- bash -c 'cat > /etc/modules-load.d/k3s.conf << EOF\nbr_netfilter\noverlay\nip_conntrack\nEOF'`,
     // Sysctl for k3s networking
     `curtin in-target -- bash -c 'cat > /etc/sysctl.d/90-k3s.conf << EOF\nnet.bridge.bridge-nf-call-iptables  = 1\nnet.bridge.bridge-nf-call-ip6tables = 1\nnet.ipv4.ip_forward                 = 1\nnet.ipv6.conf.all.forwarding        = 1\nfs.inotify.max_user_instances       = 524288\nfs.inotify.max_user_watches         = 1048576\nEOF'`,
+    // IPv6 link-local = EUI-64, so a MAC-keyed DHCPv6 reservation can match: kea
+    // recovers the node's MAC from an EUI-64 link-local when the client sends a
+    // DUID-UUID (no MAC in it). NM's stable-privacy default hides the MAC and the
+    // node silently never gets its reserved address. Proven 2026-09-06.
+    `curtin in-target -- bash -c 'cat > /etc/NetworkManager/conf.d/10-ipv6-eui64.conf << EOF\n[connection]\nipv6.addr-gen-mode=eui64\nEOF'`,
     // Disable ufw firewall
     `curtin in-target -- systemctl disable ufw || true`,
     // Enable chrony/ntp
@@ -121,7 +178,20 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
     `curtin in-target -- bash -c 'IP_ADDR=$(ip -4 addr show | awk "/inet / && !/127.0.0/ {split(\\$2,a,\\"/\\"); print a[1]; exit}"); curl -sf -X POST "http://${serverIp}:${httpPort}/api/progress" -H "Content-Type: application/json" -d "{\\"mac\\":\\"$(ip link show | awk "/ether/ && !/00:00:00:00/ {print \\$2; exit}")\\",\\"stage\\":\\"complete\\",\\"detail\\":\\"ready at $IP_ADDR\\"}" || true'`,
   );
 
-  const lateCommandsYaml = lateCommands.map((c) => `        - "${c}"`).join("\n");
+  // JSON.stringify, not `"${c}"`. JSON is a subset of YAML, so this produces a
+  // correctly escaped double-quoted scalar for free -- and the naive version
+  // was broken in two ways at once, for every role:
+  //
+  //   * embedded double quotes ended the scalar early
+  //     (`echo "tmpfs /tmp ..." >> /etc/fstab` -> "expected <block end>")
+  //   * the heredocs contain REAL newlines, and YAML folds newlines inside a
+  //     double-quoted scalar into spaces -- so even where it parsed, the
+  //     heredoc arrived at the target as one long line and wrote a file with
+  //     no line breaks.
+  //
+  // JSON escaping turns the newlines into \n, which YAML unescapes back to
+  // real newlines on parse, so the heredoc survives intact.
+  const lateCommandsYaml = lateCommands.map((c) => `        - ${JSON.stringify(c)}`).join("\n");
 
   return `#cloud-config
 autoinstall:
@@ -139,6 +209,30 @@ autoinstall:
     allow-pw: false
     authorized-keys:
 ${sshKeysYaml}
+  # Both address families. Without dhcp6 the installer's default is IPv4-only,
+  # so a node provisioned into a dual-stack cluster comes up with no IPv6, k3s
+  # has no v6 node-ip to bind, and it joins as an IPv4-only member of a
+  # dual-stack cluster -- which surfaces later as pods on that node being
+  # unreachable over v6 while the node itself reads Ready.
+  #
+  # The address itself comes from a kea DHCPv6 reservation keyed on MAC, the
+  # same source of truth as the v4 address, so nothing here needs to know it.
+  #
+  # optional: true matters -- it lets the install proceed if the v6 lease is
+  # slow or the VLAN has no DHCPv6 yet, rather than blocking on a timeout. The
+  # node still needs the address before k3s starts, but that is a later step's
+  # problem and failing the OS install over it would be worse.
+  # (No backticks in this comment: it lives inside a TS template literal, and a
+  # backtick here ends the literal and breaks the build.)
+  network:
+    version: 2
+    ethernets:
+      primary:
+        match:
+          name: "en*"
+        dhcp4: true
+        dhcp6: true
+        optional: true
   storage:
     config:
       - id: disk0
@@ -182,7 +276,7 @@ ${sshKeysYaml}
         name: ${vg}
         devices:
           - part-pv
-      - id: lv-swap
+${schemaVersion === 2 ? v2LvBlock : `      - id: lv-swap
         name: swap
         type: lvm_partition
         volgroup: vg0
@@ -259,7 +353,7 @@ ${sshKeysYaml}
       - id: mount-srv
         type: mount
         device: fs-srv
-        path: /srv${extraLvsBlock}
+        path: /srv${extraLvsBlock}`}
   packages:
     - openssh-server
     - curl

bastion/src/bastion/tests/asahi.test.ts

@@ -223,3 +223,67 @@ describe("renderFirstbootUnit", () => {
     expect(unit).toContain("ConditionPathExists=!/etc/lab-lvm-setup-done");
   });
 });
+
+// ---------------------------------------------------------------------------
+// v2 layout. Asahi was the THIRD generator and the most drifted: it hardcoded
+// rancher at 20480M under a comment claiming "sizes match install.ks.ts", which
+// uses 122880M. worker1 still runs that 20G image store today.
+// ---------------------------------------------------------------------------
+describe("v2 layout", () => {
+  const roles = ["vanilla", "worker", "infra", "labcontroller"] as const;
+  const p = (role: (typeof roles)[number], schemaVersion?: 1 | 2) =>
+    renderFirstbootScript({
+      hostname: "mac1", role, serverIp: "10.0.0.1", httpPort: 8080,
+      sshKeys: ["ssh-ed25519 AAAA t"], adminUser: "michal", mac: "aa:bb:cc:dd:ee:ff",
+      ...(schemaVersion ? { schemaVersion } : {}),
+    } as any);
+
+  function lvSizes(script: string): Record<string, number> {
+    const out: Record<string, number> = {};
+    for (const m of script.matchAll(/lvcreate -L (\d+)M\s+-n (\w+)\s+labvg/g)) {
+      out[m[2]!] = Number(m[1]);
+    }
+    return out;
+  }
+
+  it("v2 never creates a root LV — the Asahi installer already made a btrfs root", () => {
+    for (const role of roles) {
+      expect(Object.keys(lvSizes(p(role, 2)))).not.toContain("root");
+    }
+  });
+
+  it("v2 sizes rancher at 250G, not the 20G that undersized worker1", () => {
+    const lvs = lvSizes(p("infra", 2));
+    expect(lvs["rancher"]).toBe(256000);
+    expect(lvs["var"]).toBe(49152);
+    expect(lvs["swap"]).toBe(16384);
+  });
+
+  it("v2 gives a worker a rancher LV; v1 gave it only to servers", () => {
+    expect(lvSizes(p("worker", 2))["rancher"]).toBe(256000);
+    expect(lvSizes(p("worker"))["rancher"]).toBeUndefined();
+  });
+
+  it("v2 never uses 100%FREE", () => {
+    for (const role of roles) {
+      expect(p(role, 2)).not.toContain("100%FREE");
+    }
+  });
+
+  it("v1 still uses 100%FREE for longhorn and is otherwise unchanged", () => {
+    const v1 = p("worker");
+    expect(v1).toContain("lvcreate -l 100%FREE -n longhorn");
+    const lvs = lvSizes(v1);
+    expect(lvs["swap"]).toBe(27648);
+    expect(lvs["var"]).toBe(102400);
+  });
+
+  it("v2 tags the volume group so a later run can detect the layout", () => {
+    expect(p("worker", 2)).toContain("vgchange --addtag lab-schema-v2 labvg");
+    expect(p("worker")).not.toContain("--addtag lab-schema-v2");
+  });
+
+  it("reprovision reports the on-disk schema", () => {
+    expect(p("worker", 2)).toMatch(/vgs --noheadings -o vg_tags labvg.*grep -q lab-schema-v2/);
+  });
+});

bastion/src/bastion/tests/auto-k3s.test.ts

@@ -0,0 +1,137 @@
+// Tests for the post-provision k3s auto-install gate.
+//
+// Two independent switches, for two different problems:
+//
+//  1. shouldAutoInstallK3s(role) — a ROLE gate. The post-provision installer
+//     runs a bare `INSTALL_K3S_EXEC=server|agent` with no K3S_URL and no
+//     K3S_TOKEN and knows about exactly one machine. For a node that is its
+//     own cluster that is fine. For a cluster MEMBER it is wrong: servers get
+//     cluster-init semantics and form their own separate cluster (so three
+//     k8s-combo boxes yield three clusters), and agents cannot start at all
+//     ("Job for k3s-agent.service failed").
+//
+//  2. config.autoK3s — a global off switch, needed because the prerequisite
+//     steps stop k3s and delete CNI state several minutes after an install
+//     lands, which lands on top of a test forming its own cluster.
+
+import { describe, it, expect } from "vitest";
+import { shouldAutoInstallK3s } from "../src/services/post-provision.js";
+import { loadConfig } from "../src/config.js";
+import { SUPPORTED_ROLES, isK3sServer, roleInfo } from "@lab/shared";
+import type { Role } from "@lab/shared";
+
+describe("shouldAutoInstallK3s", () => {
+  // These join a cluster someone else owns, so they cannot be auto-installed
+  // without the server URL and token that post-provision does not have.
+  const CLUSTER_MEMBERS: Role[] = ["k8s-combo", "k8s-controller", "k8s-worker", "worker"];
+
+  // These are each their own cluster (or no k3s at all), so a bare install is
+  // a correct install.
+  const SELF_CONTAINED: Role[] = ["standalone-cluster", "infra", "labcontroller", "vanilla"];
+
+  it.each(CLUSTER_MEMBERS)("refuses to auto-install %s", (role) => {
+    expect(shouldAutoInstallK3s(role)).toBe(false);
+  });
+
+  it.each(SELF_CONTAINED)("still auto-installs %s", (role) => {
+    expect(shouldAutoInstallK3s(role)).toBe(true);
+  });
+
+  it("covers every supported role — a new role must make a deliberate choice", () => {
+    for (const role of SUPPORTED_ROLES) {
+      expect(
+        CLUSTER_MEMBERS.includes(role) || SELF_CONTAINED.includes(role),
+        `role ${role} is in neither list; decide whether post-provision can install it`,
+      ).toBe(true);
+      expect(typeof shouldAutoInstallK3s(role)).toBe("boolean");
+    }
+  });
+
+  it("refuses every k3s AGENT role — an agent with nothing to join cannot start", () => {
+    for (const role of SUPPORTED_ROLES) {
+      if (roleInfo(role).k3sMode === "agent") {
+        expect(shouldAutoInstallK3s(role), `${role} is an agent`).toBe(false);
+      }
+    }
+  });
+
+  it("does not refuse purely on the basis of being a server", () => {
+    // Guard against the gate degenerating into isK3sServer(): labcontroller
+    // and infra are servers AND auto-installable.
+    const servers = SUPPORTED_ROLES.filter(isK3sServer);
+    expect(servers.some((r) => shouldAutoInstallK3s(r))).toBe(true);
+    expect(servers.some((r) => !shouldAutoInstallK3s(r))).toBe(true);
+  });
+});
+
+describe("config.autoK3s", () => {
+  it("defaults to enabled", () => {
+    const config = loadConfig({});
+    expect(config.autoK3s).toBe(true);
+  });
+
+  it("is disabled by an explicit override", () => {
+    const config = loadConfig({ autoK3s: false });
+    expect(config.autoK3s).toBe(false);
+  });
+
+  it("an explicit true override wins over nothing", () => {
+    const config = loadConfig({ autoK3s: true });
+    expect(config.autoK3s).toBe(true);
+  });
+
+  it("AUTO_K3S=false in the environment disables it", () => {
+    const prev = process.env["AUTO_K3S"];
+    process.env["AUTO_K3S"] = "false";
+    try {
+      expect(loadConfig({}).autoK3s).toBe(false);
+    } finally {
+      if (prev === undefined) delete process.env["AUTO_K3S"]; else process.env["AUTO_K3S"] = prev;
+    }
+  });
+
+  it("only the exact string 'false' disables it", () => {
+    const prev = process.env["AUTO_K3S"];
+    process.env["AUTO_K3S"] = "0";
+    try {
+      expect(loadConfig({}).autoK3s).toBe(true);
+    } finally {
+      if (prev === undefined) delete process.env["AUTO_K3S"]; else process.env["AUTO_K3S"] = prev;
+    }
+  });
+});
+
+describe("kickstart packages", () => {
+  // etcd membership recovery is `etcdctl member list/remove`, and `k3s etcdctl`
+  // is NOT a command (verified on a live node: "No help topic for 'etcdctl'").
+  // The binary ships in the `etcd` rpm. The recovery card is written for a
+  // situation with no internet, so telling the operator to dnf-install it there
+  // is not a plan.
+  it("installs etcd on k3s servers and nowhere else", async () => {
+    const { renderInstallKickstart } = await import("../src/templates/install.ks.js");
+    const render = (role: Role): string =>
+      renderInstallKickstart({
+        hostname: "test-node", disk: "nvme0n1", role, domain: "ad.itaz.eu",
+        fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8",
+        serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514,
+        sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal",
+        ...(role === "vanilla" || role === "worker" || role === "infra" || role === "labcontroller"
+          ? {} : { schemaVersion: 2 as const }),
+      });
+    // Scope to %packages: "etcd" appears in paths and comments elsewhere, and a
+    // whole-document match would pass for the wrong reason.
+    const pkgSection = (ks: string): string => {
+      const start = ks.indexOf("%packages");
+      return ks.slice(start, ks.indexOf("%end", start));
+    };
+
+    for (const role of ["infra", "labcontroller", "k8s-combo", "k8s-controller", "standalone-cluster"] as const) {
+      expect(pkgSection(render(role)), `${role} is an etcd member and needs etcdctl offline`)
+        .toMatch(/^etcd$/m);
+    }
+    for (const role of ["vanilla", "worker", "k8s-worker"] as const) {
+      expect(pkgSection(render(role)), `${role} runs no etcd`)
+        .not.toMatch(/^etcd$/m);
+    }
+  });
+});

bastion/src/bastion/tests/data-disks.test.ts

@@ -0,0 +1,274 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import type { FastifyInstance } from "fastify";
+import { renderInstallKickstart } from "../src/templates/install.ks.js";
+import type { InstallKickstartParams } from "../src/templates/install.ks.js";
+import { createApp } from "../src/server.js";
+import { validateDataDisks, v2Layout, DATA_DISK_LABELS, type BastionConfig, type DataDisks } from "@lab/shared";
+
+/**
+ * Dedicated whole-disk data volumes (thebeast: 1 TB OS + 2 TB longhorn +
+ * 2 TB rancher). The layout contract these tests pin down:
+ *
+ *   - naming a data disk DROPS the matching LV -- never both.
+ *   - disks are named by /dev/disk/by-id only; kernel names are rejected,
+ *     because nvme enumeration order is not stable across boots.
+ *   - a lab-longhorn / lab-rancher labelled partition is on-disk truth, like
+ *     the lab-schema-v2 VG tag: reused --noformat, never wiped, never
+ *     silently shadowed.
+ */
+
+const BY_ID = "/dev/disk/by-id";
+const LONGHORN_ID = `${BY_ID}/nvme-Sabrent_Rocket_4.0_2TB_7F600711179D89261778`;
+const RANCHER_ID = `${BY_ID}/nvme-Sabrent_Rocket_4.0_2TB_7FD4070C1EDA92197301`;
+const OS_ID = `${BY_ID}/nvme-Samsung_SSD_970_EVO_Plus_1TB_S4EWNJ0N138234M`;
+
+function baseParams(over: Partial<InstallKickstartParams> = {}): InstallKickstartParams {
+  return {
+    hostname: "thebeast", disk: "", role: "k8s-worker", domain: "ad.itaz.eu",
+    fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8",
+    serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514,
+    sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal",
+    dataDisks: { longhorn: LONGHORN_ID, rancher: RANCHER_ID },
+    ...over,
+  };
+}
+
+/** The fresh-install part.ks heredoc (the one that creates the PV). */
+function freshBlock(ks: string): string {
+  const blocks = [...ks.matchAll(/cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g)]
+    .map((m) => m[1]!);
+  const fresh = blocks.find((b) => b.includes("part pv.01"));
+  if (!fresh) throw new Error("no fresh-install part.ks block found");
+  return fresh;
+}
+
+describe("validateDataDisks", () => {
+  const ok: DataDisks = { longhorn: LONGHORN_ID, rancher: RANCHER_ID };
+
+  it("accepts a sane spec for a role that hosts longhorn", () => {
+    expect(validateDataDisks("k8s-worker", ok)).toBeNull();
+    expect(validateDataDisks("k8s-combo", ok)).toBeNull();
+    expect(validateDataDisks("worker", ok)).toBeNull();
+  });
+
+  it("rejects kernel names -- they are not stable across boots", () => {
+    expect(validateDataDisks("k8s-worker", { longhorn: "nvme1n1" })).toMatch(/by-id/);
+    expect(validateDataDisks("k8s-worker", { rancher: "/dev/nvme2n1" })).toMatch(/by-id/);
+  });
+
+  it("rejects a longhorn disk for a role that holds no replicas", () => {
+    expect(validateDataDisks("k8s-controller", { longhorn: LONGHORN_ID })).toMatch(/no Longhorn replicas/);
+    expect(validateDataDisks("vanilla", { longhorn: LONGHORN_ID })).toMatch(/no Longhorn replicas/);
+  });
+
+  it("rejects a rancher disk for a role that runs no k3s", () => {
+    expect(validateDataDisks("vanilla", { rancher: RANCHER_ID })).toMatch(/no k3s/);
+  });
+
+  it("a control-plane-only node may still take a rancher disk", () => {
+    expect(validateDataDisks("k8s-controller", { rancher: RANCHER_ID })).toBeNull();
+  });
+
+  it("rejects the same disk doing both jobs", () => {
+    expect(validateDataDisks("k8s-worker", { longhorn: LONGHORN_ID, rancher: LONGHORN_ID }))
+      .toMatch(/same disk/);
+  });
+});
+
+describe("v2Layout with dedicated disks", () => {
+  it("drops exactly the LV a dedicated disk replaces", () => {
+    const names = (d: DataDisks) => v2Layout("k8s-worker", { dataDisks: d }).map((l) => l.name);
+    expect(names({})).toContain("longhorn");
+    expect(names({ longhorn: LONGHORN_ID })).not.toContain("longhorn");
+    expect(names({ longhorn: LONGHORN_ID })).toContain("rancher");
+    expect(names({ longhorn: LONGHORN_ID, rancher: RANCHER_ID })).not.toContain("rancher");
+    // base volumes untouched
+    expect(names({ longhorn: LONGHORN_ID, rancher: RANCHER_ID }))
+      .toEqual(expect.arrayContaining(["swap", "root", "var", "varlog", "home", "srv"]));
+  });
+});
+
+describe("renderInstallKickstart with data disks", () => {
+  it("refuses the v1 layout -- it is frozen and single-disk", () => {
+    expect(() => renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 1 })))
+      .toThrow(/v2 layout/);
+    // a legacy role with no explicit schema defaults to v1 -> same refusal
+    expect(() => renderInstallKickstart(baseParams({ role: "worker" })))
+      .toThrow(/v2 layout/);
+  });
+
+  it("refuses a kernel-name data disk", () => {
+    expect(() => renderInstallKickstart(baseParams({ dataDisks: { longhorn: "nvme1n1" } })))
+      .toThrow(/by-id/);
+  });
+
+  it("creates no longhorn/rancher LV when both mounts come from dedicated disks", () => {
+    const fresh = freshBlock(renderInstallKickstart(baseParams()));
+    expect(fresh).not.toContain("--name=longhorn");
+    expect(fresh).not.toContain("--name=rancher");
+    // base LVs still there
+    expect(fresh).toContain("--name=root");
+  });
+
+  it("anaconda only sees the named disks, and only wipes unlabelled ones", () => {
+    const ks = renderInstallKickstart(baseParams());
+    const fresh = freshBlock(ks);
+    expect(fresh).toContain("ignoredisk --only-use=$KS_DISKS");
+    expect(fresh).toContain("clearpart --all --initlabel --drives=$CLEARPART_DRIVES");
+    // a disk whose partition carries our label is excluded from clearpart
+    expect(ks).toContain('[ "$LONGHORN_REUSE" = "no" ] && CLEARPART_DRIVES=');
+  });
+
+  it("%pre resolves each by-id path and fails loudly when a disk is missing", () => {
+    const ks = renderInstallKickstart(baseParams());
+    expect(ks).toContain(`LONGHORN_DISK_ID="${LONGHORN_ID}"`);
+    expect(ks).toContain(`RANCHER_DISK_ID="${RANCHER_ID}"`);
+    expect(ks).toContain('echo "ERROR: longhorn disk not found: $LONGHORN_DISK_ID"; exit 1');
+  });
+
+  it("a labelled partition is reused --noformat; a fresh disk gets one labelled partition", () => {
+    const ks = renderInstallKickstart(baseParams());
+    expect(ks).toContain("part /var/lib/longhorn --onpart=$LONGHORN_PART --noformat");
+    expect(ks).toContain(
+      `part /var/lib/longhorn --fstype=xfs --ondisk=$LONGHORN_DISK --grow --size=1 --label=${DATA_DISK_LABELS.longhorn}`,
+    );
+    expect(ks).toContain("part /var/lib/rancher --onpart=$RANCHER_PART --noformat");
+    expect(ks).toContain(
+      `part /var/lib/rancher --fstype=xfs --ondisk=$RANCHER_DISK --grow --size=1 --label=${DATA_DISK_LABELS.rancher}`,
+    );
+  });
+
+  it("refuses to run when a data LV and a dedicated disk both exist", () => {
+    const ks = renderInstallKickstart(baseParams());
+    expect(ks).toContain('[ "$PRESERVE_LONGHORN" = "yes" ] && [ -n "$LONGHORN_DISK" ]');
+    expect(ks).toContain('[ "$PRESERVE_RANCHER" = "yes" ] && [ -n "$RANCHER_DISK" ]');
+  });
+
+  it("a fresh install never wipes or shadows a labelled disk it was not told about", () => {
+    // Rendered even WITHOUT dataDisks: the label is on-disk truth and the
+    // check protects against a queue entry that forgot the disks.
+    const ks = renderInstallKickstart(baseParams({ role: "k8s-worker", dataDisks: {} }));
+    expect(ks).toContain(`blkid -t LABEL=${DATA_DISK_LABELS.longhorn}`);
+    expect(ks).toContain("Refusing to wipe or shadow it");
+  });
+
+  it("an OS disk given as a by-id path is resolved in %pre", () => {
+    const ks = renderInstallKickstart(baseParams({ disk: OS_ID }));
+    expect(ks).toContain(`readlink -f "${OS_ID}"`);
+    expect(ks).not.toContain(`DISK="${OS_ID}"`); // never fed to anaconda unresolved
+  });
+
+  it("auto-detect never picks a data disk as the OS disk", () => {
+    const ks = renderInstallKickstart(baseParams({ disk: "" }));
+    expect(ks).toContain('case "$_bname" in "$LONGHORN_DISK"|"$RANCHER_DISK") continue ;; esac');
+  });
+
+  it("a plain kernel-name OS disk still renders exactly as before", () => {
+    const ks = renderInstallKickstart(baseParams({ disk: "vda", dataDisks: {} }));
+    expect(ks).toContain('DISK="vda"');
+  });
+
+  it("the %pre block still survives the /ks-auto extractor", () => {
+    const ks = renderInstallKickstart(baseParams());
+    const blocks = [...ks.matchAll(/%pre[^\n]*\n([\s\S]*?)%end/g)];
+    expect(blocks.length).toBeGreaterThan(0);
+    expect(blocks[0]![1]).toContain("/tmp/part.ks");
+  });
+});
+
+describe("dataDisks plumbing: /api/install -> /ks", () => {
+  let app: FastifyInstance;
+  let testDir: string;
+
+  function testConfig(dir: string): BastionConfig {
+    return {
+      fedoraVersion: "43", arch: "x86_64", httpPort: 0,
+      timezone: "Europe/London", locale: "en_GB.UTF-8",
+      bastionDir: dir, domain: "test.local", dhcpMode: "proxy",
+      dhcpRangeStart: "", dhcpRangeEnd: "",
+      ubuntuVersion: "26.04", ubuntuMirror: "https://releases.ubuntu.com/26.04",
+      vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso",
+      vyosDefaultPassword: "vyos", iface: "eth0",
+      serverIp: "10.0.0.1", network: "10.0.0.0", gateway: "10.0.0.1",
+      sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"],
+      adminUser: "testadmin", syslogPort: 15514,
+      skipDnsmasq: true, skipArtifacts: true,
+      fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
+      tftpDir: join(dir, "tftp"), httpDir: join(dir, "http"), stateFile: join(dir, "state.json"),
+    };
+  }
+
+  beforeEach(async () => {
+    testDir = join(tmpdir(), `datadisks-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+    mkdirSync(testDir, { recursive: true });
+    const built = await createApp(testConfig(testDir));
+    app = built.app;
+    await app.ready();
+  });
+
+  afterEach(async () => {
+    await app.close();
+    rmSync(testDir, { recursive: true, force: true });
+  });
+
+  const queue = (body: Record<string, unknown>) =>
+    app.inject({ method: "POST", url: "/api/install", payload: { mac: "aa:bb:cc:dd:ee:03", hostname: "thebeast", ...body } });
+
+  const ks = async () => (await app.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:03" })).body;
+
+  it("named disks reach the rendered kickstart, and their LVs are gone", async () => {
+    const res = await queue({ role: "k8s-worker", dataDisks: { longhorn: LONGHORN_ID, rancher: RANCHER_ID } });
+    expect(res.statusCode).toBe(200);
+    const body = await ks();
+    expect(body).toContain(LONGHORN_ID);
+    expect(body).toContain(RANCHER_ID);
+    expect(freshBlock(body)).not.toContain("--name=longhorn");
+    expect(freshBlock(body)).not.toContain("--name=rancher");
+  });
+
+  it("rejects dataDisks on a non-fedora install instead of silently dropping them", async () => {
+    const res = await queue({ role: "k8s-worker", os: "ubuntu-26.04", dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/fedora/);
+  });
+
+  it("rejects dataDisks with an explicit v1 schema", async () => {
+    const res = await queue({ role: "worker", schemaVersion: 1, dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/schemaVersion 2/);
+  });
+
+  it("rejects dataDisks for a legacy role that would default to v1", async () => {
+    const res = await queue({ role: "worker", dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/schemaVersion 2/);
+  });
+
+  it("legacy role + explicit v2 + dataDisks is accepted", async () => {
+    const res = await queue({ role: "worker", schemaVersion: 2, dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(200);
+    expect(await ks()).toContain(LONGHORN_ID);
+  });
+
+  it("rejects a kernel-name disk with the shared message", async () => {
+    const res = await queue({ role: "k8s-worker", dataDisks: { longhorn: "nvme1n1" } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/by-id/);
+  });
+
+  it("rejects a capability mismatch (longhorn disk on a control-plane-only role)", async () => {
+    const res = await queue({ role: "k8s-controller", dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/no Longhorn replicas/);
+  });
+
+  it("an install without dataDisks is untouched by all of this", async () => {
+    expect((await queue({ role: "k8s-worker" })).statusCode).toBe(200);
+    const body = await ks();
+    expect(freshBlock(body)).toContain("--name=longhorn --fstype=xfs --size=307200");
+    expect(freshBlock(body)).toContain("--name=rancher --fstype=xfs --size=256000");
+  });
+});

bastion/src/bastion/tests/kickstart-layout.test.ts

@@ -0,0 +1,186 @@
+import { describe, it, expect } from "vitest";
+import { renderInstallKickstart } from "../src/templates/install.ks.js";
+import type { InstallKickstartParams } from "../src/templates/install.ks.js";
+import { SUPPORTED_ROLES, LEGACY_ROLES, v2Layout, type Role } from "@lab/shared";
+
+/**
+ * Partition-layout tests.
+ *
+ * The existing ksvalidator test only sees the MAIN kickstart body. The actual
+ * layout is written by %pre into /tmp/part.ks inside a heredoc, so until now
+ * nothing validated it at all -- a broken layout rendered, shipped and was only
+ * discovered on the installed machine.
+ *
+ * These parse the heredocs. Following tests/ubuntu-autoinstall.test.ts:
+ * "A `toContain` assertion would not have caught that; only parsing does."
+ */
+
+function baseParams(over: Partial<InstallKickstartParams> = {}): InstallKickstartParams {
+  return {
+    hostname: "test-node", disk: "nvme0n1", role: "worker", domain: "ad.itaz.eu",
+    fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8",
+    serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514,
+    sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal",
+    ...over,
+  };
+}
+
+/** Pull every `cat >[>] /tmp/part.ks << PARTEOF ... PARTEOF` body out of the script. */
+function partKsBlocks(ks: string): string[] {
+  const out: string[] = [];
+  const re = /cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(ks)) !== null) out.push(m[1]!);
+  return out;
+}
+
+interface Logvol { mount: string; name: string; fs: string; size?: number; useExisting: boolean }
+
+function parseLogvols(block: string): Logvol[] {
+  return block.split("\n")
+    .filter((l) => l.trim().startsWith("logvol "))
+    .map((line) => {
+      const parts = line.trim().split(/\s+/);
+      const get = (flag: string) => {
+        const p = parts.find((x) => x.startsWith(`${flag}=`));
+        return p ? p.slice(flag.length + 1) : undefined;
+      };
+      return {
+        mount: parts[1]!,
+        name: get("--name") ?? "",
+        fs: get("--fstype") ?? "",
+        ...(get("--size") ? { size: Number(get("--size")) } : {}),
+        useExisting: parts.includes("--useexisting"),
+      };
+    });
+}
+
+/** The fresh-install block is the one that also creates the PV. */
+function freshBlock(ks: string): string {
+  const b = partKsBlocks(ks).find((x) => x.includes("part pv.01"));
+  if (!b) throw new Error("no fresh-install part.ks block found");
+  return b;
+}
+
+describe("v1 is frozen — existing machines must reinstall identically", () => {
+  it.each(LEGACY_ROLES)("%s v1 fresh layout is unchanged", (role) => {
+    const lvs = parseLogvols(freshBlock(renderInstallKickstart(baseParams({ role }))));
+    const byName = Object.fromEntries(lvs.map((l) => [l.name, l]));
+
+    // The exact v1 sizes, in MiB. If one of these changes, every existing box
+    // in the fleet would be re-laid-out on its next reinstall.
+    expect(byName["swap"]!.size).toBe(27648);
+    expect(byName["root"]!.size).toBe(33792);
+    expect(byName["var"]!.size).toBe(102400);
+    expect(byName["varlog"]!.size).toBe(10240);
+    expect(byName["home"]!.size).toBe(10240);
+    expect(byName["srv"]!.size).toBe(20480);
+  });
+
+  it("v1 gives rancher to infra and worker only — the historical bug, preserved", () => {
+    const has = (r: Role) => freshBlock(renderInstallKickstart(baseParams({ role: r }))).includes("--name=rancher");
+    expect(has("infra")).toBe(true);
+    expect(has("worker")).toBe(true);
+    expect(has("labcontroller")).toBe(false); // the bug, deliberately kept in v1
+    expect(has("vanilla")).toBe(false);
+  });
+
+  it("v1 is the default — no caller has to opt out", () => {
+    const ks = renderInstallKickstart(baseParams({ role: "worker" }));
+    expect(parseLogvols(freshBlock(ks)).find((l) => l.name === "root")!.size).toBe(33792);
+  });
+});
+
+describe("v2-only roles have no v1 layout", () => {
+  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"] as const)(
+    "%s defaults to v2 when no schema is given", (role) => {
+      const lvs = parseLogvols(freshBlock(renderInstallKickstart(baseParams({ role }))));
+      expect(lvs.find((l) => l.name === "root")!.size).toBe(32768);
+      expect(lvs.some((l) => l.name === "rancher")).toBe(true);
+    });
+
+  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"] as const)(
+    "%s throws on an EXPLICIT v1 rather than silently dropping its volumes", (role) => {
+      expect(() => renderInstallKickstart(baseParams({ role, schemaVersion: 1 }))).toThrow(/no v1 disk layout/);
+    });
+});
+
+describe("v2 renders the shared layout", () => {
+  it.each(SUPPORTED_ROLES)("%s v2 matches v2Layout() exactly", (role) => {
+    const ks = renderInstallKickstart(baseParams({ role, schemaVersion: 2 }));
+    const rendered = parseLogvols(freshBlock(ks));
+    const expected = v2Layout(role);
+
+    expect(rendered.map((l) => l.name).sort()).toEqual(expected.map((l) => l.name).sort());
+    for (const spec of expected) {
+      const got = rendered.find((l) => l.name === spec.name)!;
+      expect(got.size).toBe(spec.sizeMib);
+      expect(got.fs).toBe(spec.fs);
+    }
+  });
+
+  it("v2 gives labcontroller a rancher LV — the v1 bug, fixed", () => {
+    const ks = renderInstallKickstart(baseParams({ role: "labcontroller", schemaVersion: 2 }));
+    const rancher = parseLogvols(freshBlock(ks)).find((l) => l.name === "rancher");
+    expect(rancher).toBeDefined();
+    expect(rancher!.size).toBe(256000);
+  });
+
+  it("v2 never uses --grow", () => {
+    for (const role of SUPPORTED_ROLES) {
+      const block = freshBlock(renderInstallKickstart(baseParams({ role, schemaVersion: 2 })));
+      expect(block).not.toMatch(/logvol[^\n]*--grow/);
+    }
+  });
+
+  it("v2 uses the larger EFI and /boot partitions", () => {
+    const block = freshBlock(renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 2 })));
+    expect(block).toMatch(/part \/boot\/efi --fstype=efi --size=1024/);
+    expect(block).toMatch(/part \/boot --fstype=ext4 --size=4096/);
+  });
+});
+
+describe("reprovision protects existing data", () => {
+  it("reprovision reads the on-disk schema instead of trusting the caller", () => {
+    const ks = renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 2 }));
+    expect(ks).toContain("DISK_SCHEMA=v1");
+    expect(ks).toMatch(/vgs --noheadings -o vg_tags .*grep -q lab-schema-v2/);
+  });
+
+  it("the reprovision branch never wipes the disk", () => {
+    const ks = renderInstallKickstart(baseParams({ role: "worker" }));
+    const repro = partKsBlocks(ks).find((b) => b.includes("--useexisting"))!;
+    expect(repro).toContain("clearpart --none");
+    expect(repro).not.toContain("clearpart --all");
+  });
+
+  it("preserved data volumes are never reformatted", () => {
+    const ks = renderInstallKickstart(baseParams({ role: "worker" }));
+    for (const lv of ["longhorn", "rancher", "home", "srv"]) {
+      const re = new RegExp(`--name=${lv} --useexisting --noformat`);
+      expect(ks).toMatch(re);
+    }
+  });
+});
+
+describe("the %pre block survives the /ks-auto extractor", () => {
+  it("matches the same regex ks-auto uses to re-run %pre", () => {
+    // routes/kickstart.ts re-extracts %pre with python:
+    //   re.findall(r'%pre[^\n]*\n(.*?)%end', text, re.DOTALL)
+    // A change that breaks this breaks ISO installs silently.
+    for (const role of SUPPORTED_ROLES) {
+      // v2-only roles throw at v1 by design, so only render the combinations
+      // that are actually legal.
+      // An explicit v1 is illegal for a v2-only role; everything else renders.
+      const versions = (LEGACY_ROLES as readonly string[]).includes(role)
+        ? ([1, 2] as const)
+        : ([2] as const);
+      for (const schemaVersion of versions) {
+        const ks = renderInstallKickstart(baseParams({ role, schemaVersion }));
+        const blocks = [...ks.matchAll(/%pre[^\n]*\n([\s\S]*?)%end/g)];
+        expect(blocks.length).toBeGreaterThan(0);
+        expect(blocks[0]![1]).toContain("/tmp/part.ks");
+      }
+    }
+  });
+});

bastion/src/bastion/tests/layout-consistency.test.ts

@@ -0,0 +1,105 @@
+import { describe, it, expect } from "vitest";
+import { execFileSync } from "node:child_process";
+import { writeFileSync } from "node:fs";
+import { renderInstallKickstart } from "../src/templates/install.ks.js";
+import { renderUbuntuAutoinstall } from "../src/templates/ubuntu-autoinstall.js";
+import { renderFirstbootScript } from "../src/templates/asahi-firstboot.sh.js";
+import { v2Layout, SUPPORTED_ROLES, type Role } from "@lab/shared";
+
+/**
+ * THE test this whole refactor exists for.
+ *
+ * Three generators partition machines. Before the shared layout they disagreed,
+ * and nothing noticed:
+ *
+ *   install.ks.ts       rancher 122880M
+ *   ubuntu-autoinstall  rancher  20480M
+ *   asahi-firstboot     rancher  20480M   <- under a comment claiming it
+ *                                            matched install.ks.ts
+ *
+ * That 100 GB gap is why worker1 runs an image store too small for one large
+ * container image. This test fails the moment any generator drifts again.
+ */
+
+const KS = (role: Role) => renderInstallKickstart({
+  hostname: "n", disk: "nvme0n1", role, domain: "d", fedoraVersion: "43",
+  timezone: "UTC", locale: "en_US.UTF-8", serverIp: "10.0.0.1", httpPort: 8080,
+  syslogPort: 5514, sshKeys: [], adminUser: "a", schemaVersion: 2,
+});
+
+const UB = (role: Role) => renderUbuntuAutoinstall({
+  hostname: "n", disk: "/dev/sda", role, domain: "d", ubuntuVersion: "24.04",
+  timezone: "UTC", locale: "en_US.UTF-8", serverIp: "10.0.0.1", httpPort: 8080,
+  sshKeys: [], adminUser: "a", schemaVersion: 2,
+} as never);
+
+const AS = (role: Role) => renderFirstbootScript({
+  hostname: "n", role, serverIp: "10.0.0.1", httpPort: 8080,
+  sshKeys: [], adminUser: "a", mac: "aa:bb:cc:dd:ee:ff", schemaVersion: 2,
+} as never);
+
+/** MiB per LV name, from the Fedora kickstart's part.ks heredoc. */
+function fedoraSizes(role: Role): Record<string, number> {
+  const ks = KS(role);
+  const block = [...ks.matchAll(/cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g)]
+    .map((m) => m[1]!).find((b) => b.includes("part pv.01"))!;
+  const out: Record<string, number> = {};
+  for (const line of block.split("\n").filter((l) => l.trim().startsWith("logvol "))) {
+    const name = /--name=(\S+)/.exec(line)?.[1];
+    const size = /--size=(\d+)/.exec(line)?.[1];
+    if (name && size) out[name] = Number(size);
+  }
+  return out;
+}
+
+/** MiB per LV name, from the Ubuntu curtin document (declared in GiB). */
+function ubuntuSizes(role: Role): Record<string, number> {
+  const tmp = `/tmp/consistency-${role}.yaml`;
+  writeFileSync(tmp, UB(role));
+  const json = execFileSync("python3",
+    ["-c", "import sys,yaml,json; json.dump(yaml.safe_load(open(sys.argv[1])), sys.stdout)", tmp],
+    { encoding: "utf-8" });
+  const cfg = JSON.parse(json).autoinstall.storage.config as Array<Record<string, string>>;
+  return Object.fromEntries(
+    cfg.filter((c) => c["type"] === "lvm_partition")
+       .map((c) => [c["name"]!, Number(String(c["size"]).replace("G", "")) * 1024]),
+  );
+}
+
+/** MiB per LV name, from the Asahi firstboot lvcreate calls. */
+function asahiSizes(role: Role): Record<string, number> {
+  const out: Record<string, number> = {};
+  for (const m of AS(role).matchAll(/lvcreate -L (\d+)M\s+-n (\w+)\s+labvg/g)) out[m[2]!] = Number(m[1]);
+  return out;
+}
+
+describe("all three generators render the same v2 layout", () => {
+  it.each(SUPPORTED_ROLES)("%s: Fedora matches the shared definition", (role) => {
+    const expected = Object.fromEntries(v2Layout(role).map((lv) => [lv.name, lv.sizeMib]));
+    expect(fedoraSizes(role)).toEqual(expected);
+  });
+
+  it.each(SUPPORTED_ROLES)("%s: Ubuntu matches the shared definition", (role) => {
+    const expected = Object.fromEntries(v2Layout(role).map((lv) => [lv.name, lv.sizeMib]));
+    expect(ubuntuSizes(role)).toEqual(expected);
+  });
+
+  it.each(SUPPORTED_ROLES)("%s: Asahi matches, minus the vendor-created root", (role) => {
+    const expected = Object.fromEntries(
+      v2Layout(role, { omitRoot: true }).map((lv) => [lv.name, lv.sizeMib]),
+    );
+    expect(asahiSizes(role)).toEqual(expected);
+  });
+
+  it.each(SUPPORTED_ROLES)("%s: Fedora and Ubuntu are byte-identical in sizing", (role) => {
+    expect(ubuntuSizes(role)).toEqual(fedoraSizes(role));
+  });
+
+  it("rancher is 250G everywhere — the specific drift that bit worker1", () => {
+    for (const role of ["worker", "infra", "labcontroller"] as const) {
+      expect(fedoraSizes(role)["rancher"]).toBe(256000);
+      expect(ubuntuSizes(role)["rancher"]).toBe(256000);
+      expect(asahiSizes(role)["rancher"]).toBe(256000);
+    }
+  });
+});

bastion/src/bastion/tests/schema-version.test.ts

@@ -0,0 +1,166 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import type { BastionConfig } from "@lab/shared";
+import { createApp } from "../src/server.js";
+import type { FastifyInstance } from "fastify";
+import type { StateManager } from "../src/services/state.js";
+import type { InstallLogBuffer } from "../src/services/install-log.js";
+
+function createTestConfig(testDir: string): BastionConfig {
+  return {
+    fedoraVersion: "43",
+    arch: "x86_64",
+    httpPort: 0,
+    timezone: "Europe/London",
+    locale: "en_GB.UTF-8",
+    bastionDir: testDir,
+    domain: "test.local",
+    dhcpMode: "proxy",
+    dhcpRangeStart: "",
+    dhcpRangeEnd: "",
+    ubuntuVersion: "26.04",
+    ubuntuMirror: "https://releases.ubuntu.com/26.04",
+    vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso",
+    vyosDefaultPassword: "vyos",
+    iface: "eth0",
+    serverIp: "10.0.0.1",
+    network: "10.0.0.0",
+    gateway: "10.0.0.1",
+    sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"],
+    adminUser: "testadmin",
+    syslogPort: 15514,
+    skipDnsmasq: true,
+    skipArtifacts: true,
+    fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
+    tftpDir: join(testDir, "tftp"),
+    httpDir: join(testDir, "http"),
+    stateFile: join(testDir, "state.json"),
+  };
+}
+
+describe("schemaVersion plumbing: /api/install -> /ks", () => {
+  let app: FastifyInstance;
+  let testDir: string;
+
+  beforeEach(async () => {
+    testDir = join(tmpdir(), `schema-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+    mkdirSync(testDir, { recursive: true });
+    const built = await createApp(createTestConfig(testDir));
+    app = built.app;
+    await app.ready();
+  });
+
+  afterEach(async () => {
+    await app.close();
+    rmSync(testDir, { recursive: true, force: true });
+  });
+
+  const queue = (body: Record<string, unknown>) =>
+    app.inject({ method: "POST", url: "/api/install", payload: { mac: "aa:bb:cc:dd:ee:01", hostname: "n1", ...body } });
+
+  const ks = () => app.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:01" });
+
+  it("defaults to v1 when no schemaVersion is given", async () => {
+    expect((await queue({ role: "worker" })).statusCode).toBe(200);
+    const body = (await ks()).body;
+    expect(body).toContain("--name=root --fstype=xfs --size=33792");
+  });
+
+  it("schemaVersion 2 reaches the rendered kickstart", async () => {
+    expect((await queue({ role: "worker", schemaVersion: 2 })).statusCode).toBe(200);
+    const body = (await ks()).body;
+    expect(body).toContain("--name=root --fstype=xfs --size=32768");
+    expect(body).toContain("--name=rancher --fstype=xfs --size=256000");
+  });
+
+  it("v2 gives labcontroller a rancher LV through the full path", async () => {
+    expect((await queue({ role: "labcontroller", schemaVersion: 2 })).statusCode).toBe(200);
+    expect((await ks()).body).toContain("--name=rancher");
+  });
+
+  it("rejects an unknown schemaVersion instead of silently installing v1", async () => {
+    const res = await queue({ role: "worker", schemaVersion: 3 });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/invalid schemaVersion/);
+  });
+
+  it("still rejects an unknown role", async () => {
+    expect((await queue({ role: "nope" })).statusCode).toBe(400);
+  });
+});
+
+describe("v2-only roles cannot be installed at v1", () => {
+  let app2: FastifyInstance;
+  let dir2: string;
+
+  beforeEach(async () => {
+    dir2 = join(tmpdir(), `schema-v2only-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+    mkdirSync(dir2, { recursive: true });
+    const built = await createApp(createTestConfig(dir2));
+    app2 = built.app;
+    await app2.ready();
+  });
+  afterEach(async () => { await app2.close(); rmSync(dir2, { recursive: true, force: true }); });
+
+  const q = (body: Record<string, unknown>) =>
+    app2.inject({ method: "POST", url: "/api/install", payload: { mac: "aa:bb:cc:dd:ee:02", hostname: "n2", ...body } });
+
+  /**
+   * Omitting the schema must WORK for a v2-only role: a reinstall never passes
+   * one (the lab-schema-v2 tag on the disk decides), and the first version of
+   * this guard rejected that, making it impossible to reinstall a v2-only
+   * machine at all. Found by the reinstall integration test.
+   */
+  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"])(
+    "%s is accepted without a schemaVersion and defaults to v2", async (role) => {
+      const res = await q({ role });
+      expect(res.statusCode).toBe(200);
+      const ks = (await app2.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:02" })).body;
+      expect(ks).toContain("--name=root --fstype=xfs --size=32768");
+    });
+
+  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"])(
+    "%s is rejected on an EXPLICIT schemaVersion 1 — that is a contradiction", async (role) => {
+      const res = await q({ role, schemaVersion: 1 });
+      expect(res.statusCode).toBe(400);
+      expect(res.json().error).toMatch(/requires schemaVersion 2/);
+    });
+
+  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"])(
+    "%s is accepted with schemaVersion 2", async (role) => {
+      expect((await q({ role, schemaVersion: 2 })).statusCode).toBe(200);
+    });
+
+  it("legacy roles are still accepted without a schemaVersion", async () => {
+    for (const role of ["vanilla", "worker", "infra", "labcontroller"]) {
+      expect((await q({ role })).statusCode).toBe(200);
+    }
+  });
+
+  /**
+   * Assert on the FRESH-INSTALL part.ks block specifically. The whole kickstart
+   * also contains the reprovision branch, whose shell emits `--name=longhorn`
+   * inside a `PRESERVE_LONGHORN` guard -- so a naive whole-file `not.toContain`
+   * matches that and fails for reasons that have nothing to do with the layout.
+   */
+  function freshBlockOf(ks: string): string {
+    const blocks = [...ks.matchAll(/cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g)]
+      .map((m) => m[1]!);
+    const fresh = blocks.find((b) => b.includes("part pv.01"));
+    if (!fresh) throw new Error("no fresh-install block in rendered kickstart");
+    return fresh;
+  }
+
+  it("k8s-controller gets no longhorn LV; k8s-combo does", async () => {
+    await q({ role: "k8s-controller", schemaVersion: 2 });
+    const ctrl = freshBlockOf((await app2.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:02" })).body);
+    expect(ctrl).toContain("--name=rancher");
+    expect(ctrl).not.toContain("--name=longhorn");
+
+    await q({ role: "k8s-combo", schemaVersion: 2 });
+    const combo = freshBlockOf((await app2.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:02" })).body);
+    expect(combo).toContain("--name=longhorn --fstype=xfs --size=307200");
+  });
+});

bastion/src/bastion/tests/ubuntu-autoinstall.test.ts

@@ -0,0 +1,184 @@
+// The Ubuntu autoinstall document must be valid YAML for EVERY role.
+//
+// There was no test here, and the template shipped a document that did not
+// parse: the longhorn/rancher LVM entries were indented 8 spaces while their
+// siblings in storage.config sit at 6, giving "expected <block end>, but found
+// '-'". Every role that gets a longhorn volume -- which is the worker shape --
+// rendered an uninstallable document. A `toContain` assertion would not have
+// caught that; only parsing does.
+//
+// Parsed with python3's yaml rather than a new npm dependency, mirroring how
+// kickstart.test.ts shells out to `ksvalidator`: the point is to check the
+// artefact with a real parser, not to grow the dependency tree.
+
+import { describe, it, expect } from "vitest";
+import { execFileSync } from "node:child_process";
+import { writeFileSync, unlinkSync } from "node:fs";
+import { renderUbuntuAutoinstall } from "../src/templates/ubuntu-autoinstall.js";
+
+const base = {
+  hostname: "n6",
+  disk: "/dev/sda",
+  domain: "ad.itaz.eu",
+  ubuntuVersion: "24.04",
+  timezone: "Europe/London",
+  locale: "en_GB.UTF-8",
+  serverIp: "10.0.0.1",
+  httpPort: 8080,
+  sshKeys: ["ssh-ed25519 AAAAtest test@lab"],
+  adminUser: "root",
+};
+
+/** Parse with python3's yaml and return the document as JSON. */
+function parseYaml(text: string, label: string): Record<string, any> {
+  const tmp = `/tmp/autoinstall-test-${label}.yaml`;
+  writeFileSync(tmp, text);
+  try {
+    const out = execFileSync(
+      "python3",
+      ["-c", "import sys,yaml,json; json.dump(yaml.safe_load(open(sys.argv[1])), sys.stdout)", tmp],
+      { encoding: "utf-8" },
+    );
+    return JSON.parse(out);
+  } catch (err: unknown) {
+    const msg = err instanceof Error ? (err as { stderr?: string }).stderr ?? err.message : String(err);
+    throw new Error(`autoinstall YAML did not parse for ${label}: ${msg}`);
+  } finally {
+    try { unlinkSync(tmp); } catch { /* ignore */ }
+  }
+}
+
+describe("renderUbuntuAutoinstall", () => {
+  for (const role of ["vanilla", "worker", "infra"]) {
+    it(`renders parseable YAML for role=${role}`, () => {
+      const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role }), role);
+      expect(doc.autoinstall).toBeDefined();
+      expect(doc.autoinstall.version).toBe(1);
+      // storage.config must be a flat list; the indentation bug produced a
+      // nested map here, which is how it went unnoticed.
+      expect(Array.isArray(doc.autoinstall.storage.config)).toBe(true);
+    });
+  }
+
+  it("gives the longhorn role its volume as a sibling entry, not a nested map", () => {
+    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" }), "longhorn");
+    const ids = doc.autoinstall.storage.config.map((e: { id: string }) => e.id);
+    expect(ids).toContain("lv-longhorn");
+    expect(ids).toContain("mount-longhorn");
+  });
+
+  it("sets EUI-64 link-local so MAC-keyed DHCPv6 reservations can match", () => {
+    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" }), "eui64");
+    const late = (doc.autoinstall["late-commands"] as string[]).join("\n");
+    expect(late).toContain("10-ipv6-eui64.conf");
+    expect(late).toContain("ipv6.addr-gen-mode=eui64");
+  });
+
+  it("requests both address families on the primary NIC", () => {
+    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" }), "net");
+    const eth = doc.autoinstall.network.ethernets.primary;
+    expect(eth.dhcp4).toBe(true);
+    // Without this a node provisioned into a dual-stack cluster comes up with
+    // no IPv6 and joins as an IPv4-only member.
+    expect(eth.dhcp6).toBe(true);
+    // The install must not block waiting for a v6 lease that may never come.
+    expect(eth.optional).toBe(true);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// v2 layout. Ubuntu previously sized rancher at 20G while the Fedora kickstart
+// used 120G, from a template whose comment claimed the sizes matched. Both now
+// render from shared/src/types/layout.ts, so this checks the rendering, and
+// layout.test.ts checks the definition.
+// ---------------------------------------------------------------------------
+describe("v2 storage layout", () => {
+  const roles = ["vanilla", "worker", "infra", "labcontroller"] as const;
+
+  function lvmPartitions(doc: Record<string, any>): Record<string, string> {
+    const cfg = doc["autoinstall"]["storage"]["config"] as Array<Record<string, any>>;
+    return Object.fromEntries(
+      cfg.filter((c) => c["type"] === "lvm_partition").map((c) => [c["name"], c["size"]]),
+    );
+  }
+
+  it.each(roles)("%s renders valid YAML at schemaVersion 2", (role) => {
+    const doc = parseYaml(
+      renderUbuntuAutoinstall({ ...base, role, schemaVersion: 2 } as any),
+      `v2-${role}`,
+    );
+    expect(Array.isArray(doc["autoinstall"]["storage"]["config"])).toBe(true);
+  });
+
+  it("v2 sizes match the shared definition, not the old 20G rancher", () => {
+    const doc = parseYaml(
+      renderUbuntuAutoinstall({ ...base, role: "worker", schemaVersion: 2 } as any),
+      "v2-sizes",
+    );
+    const lvs = lvmPartitions(doc);
+    expect(lvs["rancher"]).toBe("250G");   // was 20G here and 120G in kickstart
+    expect(lvs["longhorn"]).toBe("300G");  // was grow
+    expect(lvs["root"]).toBe("32G");
+    expect(lvs["var"]).toBe("48G");
+  });
+
+  it("v2 gives labcontroller a rancher LV; v1 does not", () => {
+    const v2 = lvmPartitions(parseYaml(
+      renderUbuntuAutoinstall({ ...base, role: "labcontroller", schemaVersion: 2 } as any), "v2-lc"));
+    const v1 = lvmPartitions(parseYaml(
+      renderUbuntuAutoinstall({ ...base, role: "labcontroller" } as any), "v1-lc"));
+    expect(v2["rancher"]).toBe("250G");
+    expect(v1["rancher"]).toBeUndefined();
+  });
+
+  it("v1 remains the default and is unchanged", () => {
+    const lvs = lvmPartitions(parseYaml(
+      renderUbuntuAutoinstall({ ...base, role: "worker" } as any), "v1-default"));
+    expect(lvs["root"]).toBe("33G");
+    expect(lvs["var"]).toBe("100G");
+    expect(lvs["swap"]).toBe("27G");
+  });
+
+  it("no v2 volume is unbounded", () => {
+    for (const role of roles) {
+      const lvs = lvmPartitions(parseYaml(
+        renderUbuntuAutoinstall({ ...base, role, schemaVersion: 2 } as any), `v2-grow-${role}`));
+      for (const [name, size] of Object.entries(lvs)) {
+        expect(String(size), `${role}/${name}`).toMatch(/^\d+G$/);
+      }
+    }
+  });
+});
+
+/**
+ * KNOWN GAP, asserted so it cannot be assumed away.
+ *
+ * The Fedora kickstart preserves /home, /srv, longhorn and rancher across a
+ * reinstall (%pre probes for an existing labvg). The Ubuntu autoinstall has no
+ * equivalent: `disk0` is wiped unconditionally, so EVERY Ubuntu reinstall is
+ * destructive.
+ *
+ * These tests pin that behaviour rather than pretend it is fine. If someone
+ * adds curtin `preserve: true` handling, these fail and should be rewritten --
+ * which is the point: the change becomes visible instead of silent.
+ */
+describe("Ubuntu reinstall is destructive (known gap)", () => {
+  it("wipes the disk unconditionally — there is no reprovision branch", () => {
+    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" } as any), "destructive");
+    const cfg = doc["autoinstall"]["storage"]["config"] as Array<Record<string, any>>;
+    const disk = cfg.find((c) => c["type"] === "disk");
+    expect(disk?.["wipe"]).toBe("superblock-recursive");
+  });
+
+  it("no volume is marked preserve — nothing survives a reinstall", () => {
+    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" } as any), "no-preserve");
+    const cfg = doc["autoinstall"]["storage"]["config"] as Array<Record<string, any>>;
+    expect(cfg.some((c) => c["preserve"] === true)).toBe(false);
+  });
+
+  it("the same is true at schemaVersion 2 — v2 did not fix this", () => {
+    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker", schemaVersion: 2 } as any), "v2-destructive");
+    const cfg = doc["autoinstall"]["storage"]["config"] as Array<Record<string, any>>;
+    expect(cfg.find((c) => c["type"] === "disk")?.["wipe"]).toBe("superblock-recursive");
+  });
+});

bastion/src/cli/src/api/client.ts

@@ -90,6 +90,8 @@ export class LabdClient {
 
   async installMachine(opts: {
     mac: string; hostname: string; disk?: string; role?: string; os?: string;
+    schemaVersion?: 1 | 2;
+    dataDisks?: import("@lab/shared").DataDisks;
     vyos?: import("@lab/shared").VyosInstallSpec;
   }): Promise<{ status: string; data?: unknown; error?: string }> {
     return this.request("POST", "/api/machines/install", { body: opts });

bastion/src/cli/src/commands/app.ts

@@ -57,6 +57,7 @@ async function fetchState(): Promise<BastionState | null> {
 }
 
 import { registerLabcontrollerCommands } from "./labcontroller.js";
+import { isRole, SUPPORTED_ROLES, type Role } from "@lab/shared";
 
 export function registerAppCommand(program: Command): void {
   const appCmd = program.command("app").description("Application management");
@@ -66,18 +67,104 @@ export function registerAppCommand(program: Command): void {
 
   const k3sCmd = appCmd.command("k3s").description("k3s cluster management");
 
+  k3sCmd
+    .command("evacuate <target>")
+    .description("Cordon, evict Longhorn replicas and drain so a node can be safely wiped")
+    .option("--user <user>", "SSH user", "root")
+    .option(
+      "--allow-single-replica",
+      "permit evacuating volumes that have only one replica (rebuilds from the only copy; there is no backup target)",
+    )
+    .action(async (target: string, opts: { user: string; allowSingleReplica?: boolean }) => {
+      const state = await fetchState();
+      const resolved = resolveTarget(target, state);
+      if (!resolved) {
+        console.error(`Cannot resolve target: ${target}`);
+        process.exit(1);
+      }
+      const sshKey = findSshKey();
+
+      console.log(`Evacuating ${resolved.hostname} (${resolved.ip})...`);
+      console.log("This waits for Longhorn to rebuild replicas elsewhere and can take a while.\n");
+
+      const k3sMod = new K3sModule();
+      const result = await k3sMod.evacuate({
+        hostname: resolved.hostname,
+        ip: resolved.ip,
+        role: "infra",
+        os: "fedora-43" as const,
+        arch: "x86_64" as const,
+        sshUser: opts.user,
+        ...(sshKey ? { sshKeyPath: sshKey } : {}),
+        config: opts.allowSingleReplica ? { evictionAllowSingleReplica: true } : {},
+      });
+
+      for (const line of result.output) console.log(`  ${line}`);
+      if (!result.success) {
+        console.error(`\nNode is NOT safe to wipe (${result.errors.join(", ")}). Do not reprovision it.`);
+        console.error(`If an eviction was started and left pending, clear it with:`);
+        console.error(`  labctl app k3s admit ${resolved.hostname}`);
+        process.exit(1);
+      }
+      console.log(`\n${resolved.hostname} is evacuated. Safe to reprovision.`);
+    });
+
+  k3sCmd
+    .command("admit <target>")
+    .description("Return an evacuated node to service: uncordon, clear eviction, restore its Longhorn disk")
+    .option("--user <user>", "SSH user", "root")
+    .action(async (target: string, opts: { user: string }) => {
+      const state = await fetchState();
+      const resolved = resolveTarget(target, state);
+      if (!resolved) {
+        console.error(`Cannot resolve target: ${target}`);
+        process.exit(1);
+      }
+      const sshKey = findSshKey();
+
+      console.log(`Returning ${resolved.hostname} (${resolved.ip}) to service...`);
+
+      const k3sMod = new K3sModule();
+      const result = await k3sMod.admit({
+        hostname: resolved.hostname,
+        ip: resolved.ip,
+        role: isRole(resolved.role) ? resolved.role : "infra",
+        os: "fedora-43" as const,
+        arch: "x86_64" as const,
+        sshUser: opts.user,
+        ...(sshKey ? { sshKeyPath: sshKey } : {}),
+        config: {},
+      });
+
+      for (const line of result.output) console.log(`  ${line}`);
+      if (!result.success) {
+        console.error(`\n${resolved.hostname} is NOT fully back in service (${result.errors.join(", ")}).`);
+        process.exit(1);
+      }
+      console.log(`\n${resolved.hostname} is back in service.`);
+    });
+
   k3sCmd
     .command("install <target>")
     .description("Install k3s on a target machine (hostname, IP, or MAC)")
-    .option("--role <role>", "k3s role: infra (server) or worker (agent)", "infra")
+    .option(
+      "--role <role>",
+      "k3s role: infra, labcontroller, standalone-cluster, k8s-controller, k8s-combo (servers) or worker, k8s-worker (agents)",
+      "infra",
+    )
     .option("--user <user>", "SSH user", "root")
-    .option("--k3s-server <url>", "k3s server URL (required for worker role)")
-    .option("--k3s-token <token>", "k3s join token (required for worker role)")
+    .option("--k3s-server <url>", "k3s server URL (required for agent roles, and to JOIN as a server)")
+    .option("--k3s-token <token>", "k3s join token; on a first server it SEEDS the cluster token")
+    .option(
+      "--k3s-version <version>",
+      "exact k3s version, e.g. v1.34.6+k3s1. Unset installs LATEST, which on a rejoining node can be several minor versions ahead of the cluster",
+    )
     .action(async (target: string, opts: {
       role: string;
       user: string;
       k3sServer?: string;
       k3sToken?: string;
+      k3sVersion?: string;
     }) => {
       const state = await fetchState();
       const resolved = resolveTarget(target, state);
@@ -88,7 +175,16 @@ export function registerAppCommand(program: Command): void {
         process.exit(1);
       }
 
-      const role = opts.role === "worker" ? "worker" : "infra";
+      // Every supported role, not just infra/worker. The old mapping silently
+      // turned anything that was not "worker" into "infra" -- so
+      // `--role k8s-combo` installed a DIFFERENT role than the one asked for,
+      // with different disk expectations and no warning.
+      if (!isRole(opts.role)) {
+        console.error(`Unknown role: ${opts.role}`);
+        console.error(`Valid roles: ${SUPPORTED_ROLES.join(", ")}`);
+        process.exit(1);
+      }
+      const role: Role = opts.role;
       const sshKey = findSshKey();
 
       console.log(`Installing k3s on ${resolved.hostname} (${resolved.ip}) as ${role}...`);
@@ -106,9 +202,22 @@ export function registerAppCommand(program: Command): void {
         config: {
           ...(opts.k3sServer ? { k3sServerUrl: opts.k3sServer } : {}),
           ...(opts.k3sToken ? { k3sToken: opts.k3sToken } : {}),
+          ...(opts.k3sVersion ? { k3sVersion: opts.k3sVersion } : {}),
         },
       };
 
+      // Only warn when it matters: a node JOINING an existing cluster. On a
+      // first server there is nothing to be skewed against, and a warning that
+      // fires when it is irrelevant is a warning people learn to skip.
+      if (!opts.k3sVersion && opts.k3sServer) {
+        console.warn(
+          "  WARNING: no --k3s-version given, so get.k3s.io installs LATEST.\n" +
+          "  This node is JOINING an existing cluster — check what that cluster runs\n" +
+          "  (kubectl get nodes -o wide) and pin it. Kubernetes supports only ONE minor\n" +
+          "  version of skew between control-plane members.",
+        );
+      }
+
       const installResult = await k3s.install(moduleCtx);
       for (const line of installResult.output) {
         console.log(`  ${line}`);
@@ -281,7 +390,8 @@ export function registerAppCommand(program: Command): void {
       const healthResult = await k3s.health({
         hostname: resolved.hostname,
         ip: resolved.ip,
-        role: resolved.role,
+        // State is JSON on disk; narrow it rather than trusting the string.
+        role: isRole(resolved.role) ? resolved.role : "vanilla",
         os: "fedora-43" as const,
         arch: "x86_64" as const,
         sshUser: opts.user,

bastion/src/cli/src/commands/install.ts

@@ -3,8 +3,8 @@
 
 import { readFileSync } from "node:fs";
 import { Command, Option, InvalidArgumentError } from "commander";
-import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY } from "@lab/shared";
-import type { VyosBundle, VyosInstallSpec, VyosVlanSpec } from "@lab/shared";
+import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isRole, requiresSchemaV2, validateDataDisks } from "@lab/shared";
+import type { DataDisks, VyosBundle, VyosInstallSpec, VyosVlanSpec } from "@lab/shared";
 import { getLabdClient } from "../api/config.js";
 
 /**
@@ -106,7 +106,16 @@ export function registerInstallCommand(parent: Command): void {
     .addHelpText("after", roleTable())
     .addOption(new Option("--role <role>", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker"))
     .addOption(new Option("--os <os>", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43"))
-    .option("--disk <device>", "Target disk device (auto-detect if omitted)")
+    .option("--disk <device>", "Target OS disk: kernel name or stable /dev/disk/by-id/ path (auto-detect if omitted)")
+    .addOption(new Option("--schema-version <n>", "Disk layout for a FRESH install (default 1). Ignored on reprovision: the layout already on the disk wins.").choices(["1", "2"]))
+    .option(
+      "--longhorn-disk <by-id>",
+      "Dedicated whole disk for /var/lib/longhorn, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)",
+    )
+    .option(
+      "--rancher-disk <by-id>",
+      "Dedicated whole disk for /var/lib/rancher, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)",
+    )
     .option("--vyos-mgmt <iface>", "VyOS: untagged interface the machine PXE boots from (default eth0)")
     .option("--vyos-mgmt-address <addr>", "VyOS: CIDR for the management interface, or 'dhcp' (default dhcp)")
     .option("--vyos-bond <ifaces>", "VyOS: comma-separated LACP bond members (must exclude the PXE NIC)")
@@ -136,7 +145,11 @@ export function registerInstallCommand(parent: Command): void {
     .action(async (mac: string, hostname: string, opts: {
       role: string;
       os: string;
+      /** commander gives us the string form of the --schema-version choice. */
+      schemaVersion?: string;
       disk?: string;
+      longhornDisk?: string;
+      rancherDisk?: string;
       vyosMgmt?: string;
       vyosMgmtAddress?: string;
       vyosBond?: string;
@@ -163,6 +176,32 @@ export function registerInstallCommand(parent: Command): void {
         process.exit(1);
       }
 
+      // Dedicated data disks -- validated here for fast feedback, and again on
+      // the bastion, which is the enforcement point.
+      const dataDisks: DataDisks = {
+        ...(opts.longhornDisk !== undefined && opts.longhornDisk !== ""
+          ? { longhorn: opts.longhornDisk } : {}),
+        ...(opts.rancherDisk !== undefined && opts.rancherDisk !== ""
+          ? { rancher: opts.rancherDisk } : {}),
+      };
+      const hasDataDisks = Object.keys(dataDisks).length > 0;
+      if (hasDataDisks) {
+        if (opts.os !== "fedora-43") {
+          console.error(`--longhorn-disk/--rancher-disk only work with --os fedora-43 (got --os ${opts.os})`);
+          process.exit(1);
+        }
+        if (opts.schemaVersion === "1"
+            || (opts.schemaVersion === undefined && isRole(opts.role) && !requiresSchemaV2(opts.role))) {
+          console.error("--longhorn-disk/--rancher-disk need the v2 layout -- add --schema-version 2");
+          process.exit(1);
+        }
+        const problem = isRole(opts.role) ? validateDataDisks(opts.role, dataDisks) : null;
+        if (problem !== null) {
+          console.error(`Invalid data disks: ${problem}`);
+          process.exit(1);
+        }
+      }
+
       const bondMembers = opts.vyosBond !== undefined && opts.vyosBond !== ""
         ? opts.vyosBond.split(",").map((s) => s.trim()).filter((s) => s.length > 0)
         : [];
@@ -248,11 +287,13 @@ export function registerInstallCommand(parent: Command): void {
 
       try {
         const result = await getLabdClient().installMachine({
+          ...(opts.schemaVersion ? { schemaVersion: Number(opts.schemaVersion) as 1 | 2 } : {}),
           mac,
           hostname,
           role: opts.role,
           os: opts.os,
           ...(opts.disk ? { disk: opts.disk } : {}),
+          ...(hasDataDisks ? { dataDisks } : {}),
           ...(hasVyosOptions ? { vyos } : {}),
         });
 

bastion/src/cli/tests/smoke-bastion.test.ts

@@ -143,7 +143,10 @@ describe("bastion smoke tests", () => {
       await sleep(200);
     }
 
-    expect(stdout).toContain("Waiting for PXE boot requests");
+    // On failure, show what the process actually said -- an empty stdout with
+    // a hidden stderr is undiagnosable from a CI log.
+    expect(stdout, `bastion never printed its banner; stderr was:\n${stderr}`)
+      .toContain("Waiting for PXE boot requests");
     expect(stdout).toContain("HTTP server listening");
 
     // Verify the process is still alive after startup

bastion/src/labd/src/routes/bastions.ts

@@ -164,9 +164,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
 
   // Queue install — route to correct bastion by MAC
   app.post<{
-    Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string; vyos?: VyosInstallSpec };
+    Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string; schemaVersion?: 1 | 2; dataDisks?: import("@lab/shared").DataDisks; vyos?: VyosInstallSpec };
   }>("/api/machines/install", async (request, reply) => {
-    const { mac, hostname, disk, role, os, vyos } = request.body ?? {};
+    const { mac, hostname, disk, role, os, schemaVersion, dataDisks, vyos } = request.body ?? {};
     if (!mac || !hostname) {
       return reply.code(400).send({ error: "mac and hostname are required" });
     }
@@ -184,6 +184,8 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
           const result = await sendCommand(all[0]!.bastionId, {
             type: "command-install",
             mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
+            ...(schemaVersion ? { schemaVersion } : {}),
+            ...(dataDisks ? { dataDisks } : {}),
             ...(vyos ? { vyos } : {}),
           });
           return reply.code(result.status === "ok" ? 200 : 500).send(result);
@@ -198,6 +200,8 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
       const result = await sendCommand(bastion.bastionId, {
         type: "command-install",
         mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
+        ...(schemaVersion ? { schemaVersion } : {}),
+        ...(dataDisks ? { dataDisks } : {}),
         ...(vyos ? { vyos } : {}),
       });
       return reply.code(result.status === "ok" ? 200 : 500).send(result);

bastion/src/modules/modules/k3s/bin/render-config.ts

@@ -0,0 +1,42 @@
+#!/usr/bin/env node
+// Render /etc/rancher/k3s/config.yaml using the PRODUCTION generator.
+//
+// Exists so labsim (and anything else) can produce the exact config.yaml a node
+// would get from `labctl install`, without an SSH/OperationContext. Driving the
+// rehearsal through this means the sim tests the same code path production runs
+// -- if generateServerConfig ever changes shape, the sim moves with it.
+//
+// All input via env, so a sim's cloud-init or a shell can call it plainly:
+//
+//   ROLE=infra HOSTNAME=k8s1 IP=172.31.2.11 render-config.ts        # cluster-init server
+//   ROLE=infra HOSTNAME=k8s2 IP=172.31.2.12 \
+//     K3S_SERVER_URL=https://172.31.2.11:6443 K3S_TOKEN=... render-config.ts   # joining server
+//   ROLE=worker HOSTNAME=k8s4 IP=172.31.2.14 K3S_SERVER_URL=... K3S_TOKEN=... render-config.ts  # agent
+//
+// Dual-stack is opt-in and matches K3sConfig exactly: set IPV6, CLUSTER_CIDR,
+// SERVICE_CIDR (comma-separated families) and the generator emits the dual
+// node-ip + CIDRs. Omit them and the output is byte-identical to a v4-only node.
+import { generateServerConfig, generateAgentConfig } from "../src/operations/k3s-config.js";
+import type { K3sConfig } from "../src/types.js";
+import type { Role } from "@lab/shared";
+
+const env = process.env;
+const role = (env.ROLE ?? "worker") as Role;
+const isServer = role === "infra" || role === "labcontroller";
+
+const splitCsv = (v: string | undefined): string[] | undefined =>
+  v ? v.split(",").map((s) => s.trim()).filter(Boolean) : undefined;
+
+const cfg: K3sConfig = {
+  hostname: env.HOSTNAME ?? "node",
+  ip: env.IP ?? (() => { throw new Error("IP is required"); })(),
+  role,
+  k3sServerUrl: env.K3S_SERVER_URL,
+  k3sToken: env.K3S_TOKEN,
+  tlsSans: splitCsv(env.TLS_SANS),
+  ipv6: env.IPV6,
+  clusterCidr: splitCsv(env.CLUSTER_CIDR),
+  serviceCidr: splitCsv(env.SERVICE_CIDR),
+};
+
+process.stdout.write(isServer ? generateServerConfig(cfg) : generateAgentConfig(cfg));

bastion/src/modules/modules/k3s/src/groups/admit.ts

@@ -0,0 +1,31 @@
+// Return an evacuated node to service. The inverse of the evacuate group.
+//
+// Needed for two different journeys that end in the same place:
+//   - a node that was wiped, reinstalled and has just rejoined the cluster;
+//   - a node whose evacuation was aborted and must simply be put back.
+//
+// Order matters here too. configureLonghornDisk runs AFTER eviction is cleared,
+// because a node that is still marked for eviction will have any disk we give
+// it immediately drained again.
+
+import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
+import { runSequential } from "../utils.js";
+import { uncordonNode } from "../operations/node-evacuate.js";
+import { clearLonghornEviction, assertNodeSchedulable } from "../operations/node-admit.js";
+import { configureLonghornDisk } from "../operations/longhorn-disk.js";
+
+export const admitGroup: OperationGroup = {
+  name: "admit",
+  description: "Uncordon a node, clear its eviction request and restore its Longhorn disk",
+  operations: [
+    { name: "Uncordon node", fn: uncordonNode },
+    { name: "Clear Longhorn eviction", fn: clearLonghornEviction },
+    { name: "Restore Longhorn default disk", fn: configureLonghornDisk },
+    { name: "Verify node is back in service", fn: assertNodeSchedulable },
+  ],
+};
+
+export async function runAdmit(ctx: OperationContext): Promise<OperationResult[]> {
+  ctx.log("Returning node to service...");
+  return runSequential(ctx, admitGroup.operations);
+}

bastion/src/modules/modules/k3s/src/groups/evacuate.ts

@@ -0,0 +1,54 @@
+// Evacuate a node so it can be wiped and reinstalled with a different layout.
+//
+// The ORDER is the operation:
+//
+//   1. Cordon      stop new work landing on a node that is about to leave.
+//   2. Check       can eviction even finish? Seconds to answer, an hour to
+//                  discover the hard way.
+//   3. Evict       move the replicas and WAIT for them to rebuild elsewhere.
+//   4. Drain       now move the workloads.
+//   5. Verify      re-read the cluster; do not trust that 1-4 worked.
+//
+// EVICT BEFORE DRAIN, which is the opposite of the obvious order and the one
+// thing here most likely to be "tidied" back into a bug. `drain --force` does
+// not override PodDisruptionBudgets, and longhorn-manager holds a PDB on the
+// instance-manager pod for as long as volumes are attached or degraded. Drain
+// first and you stop the instance-manager that serves the replicas you are
+// about to evict: the drain blocks for its full 15-minute timeout, and if it
+// did succeed it would have removed the only source a single-replica volume
+// could rebuild from.
+//
+// Removing the etcd member is NOT part of this group -- it costs you the API
+// server the rest of the procedure needs, so it belongs immediately before the
+// wipe, not here.
+//
+// This group deliberately does not reinstall anything. It leaves the node
+// cordoned, drained and empty, and a human (or `labctl provision reprovision`)
+// takes it from there.
+
+import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
+import { runSequential } from "../utils.js";
+import {
+  cordonNode,
+  drainNode,
+  evictLonghornReplicas,
+  assertEvictionCompletable,
+  assertNodeEvacuated,
+} from "../operations/node-evacuate.js";
+
+export const evacuateGroup: OperationGroup = {
+  name: "evacuate",
+  description: "Cordon, evict storage from and drain a node so it can be safely wiped",
+  operations: [
+    { name: "Cordon node", fn: cordonNode },
+    { name: "Check eviction can complete", fn: assertEvictionCompletable },
+    { name: "Evict Longhorn replicas (waits for rebuild)", fn: evictLonghornReplicas },
+    { name: "Drain workloads", fn: drainNode },
+    { name: "Verify node is empty", fn: assertNodeEvacuated },
+  ],
+};
+
+export async function runEvacuate(ctx: OperationContext): Promise<OperationResult[]> {
+  ctx.log("Evacuating node before wipe...");
+  return runSequential(ctx, evacuateGroup.operations);
+}

bastion/src/modules/modules/k3s/src/groups/k3s-agent.ts

@@ -4,6 +4,7 @@ import type { OperationContext, OperationResult, OperationGroup } from "../types
 import { runSequential } from "../utils.js";
 import { writeK3sConfig } from "../operations/k3s-config.js";
 import { installK3sBinary } from "../operations/k3s-install.js";
+import { labelWorkerRole } from "../operations/worker-label.js";
 
 export const k3sAgentGroup: OperationGroup = {
   name: "k3s-agent",
@@ -11,6 +12,8 @@ export const k3sAgentGroup: OperationGroup = {
   operations: [
     { name: "Write k3s config", fn: writeK3sConfig },
     { name: "Install k3s agent", fn: installK3sBinary },
+    // After registration, never before: the kubelet may not set this itself.
+    { name: "Label node as worker", fn: labelWorkerRole },
   ],
 };
 

bastion/src/modules/modules/k3s/src/health/k3s-service.ts

@@ -1,8 +1,9 @@
 import type { Operation, OperationResult } from "../types.js";
 import { isServiceActive } from "../utils.js";
+import { isK3sServer } from "@lab/shared";
 
 export const checkK3sService: Operation = async (ctx): Promise<OperationResult> => {
-  const isServer = ctx.config.role === "infra" || ctx.config.role === "labcontroller";
+  const isServer = isK3sServer(ctx.config.role);
   const service = isServer ? "k3s" : "k3s-agent";
   const active = await isServiceActive(ctx, service);
   return { success: active, changed: false, message: active ? `${service} is active` : `${service} is not active` };

bastion/src/modules/modules/k3s/src/install.ts

@@ -1,17 +1,21 @@
 // k3s module — install phase.
 // Installs k3s with CIS-hardened configuration and Cilium CNI.
 
+import { isK3sServer, type Role } from "@lab/shared";
+
 export interface K3sInstallContext {
   hostname: string;
   ip: string;
-  role: string;   // "infra" = server, "worker" = agent
+  role: Role;     // server vs agent is derived via isK3sServer(), never by name
   k3sServerUrl?: string;  // Required for agent role
   k3sToken?: string;       // Required for agent role
 }
 
 /** Generate the shell script that installs k3s on a target machine. */
 export function generateInstallScript(ctx: K3sInstallContext): string {
-  const isServer = ctx.role === "infra";
+  // Was `ctx.role === "infra"`, which silently treated a `labcontroller` -- a
+  // role that runs a k3s SERVER -- as an agent. Derived from the registry now.
+  const isServer = isK3sServer(ctx.role);
 
   return `#!/bin/bash
 set -euo pipefail

bastion/src/modules/modules/k3s/src/k3s-module.ts

@@ -6,21 +6,35 @@ import type { OperationContext, K3sConfig, OperationResult } from "./types.js";
 import { sshExec } from "../../../src/ssh.js";
 import { aggregateResults } from "./utils.js";
 import { runHostPrep } from "./groups/host-prep.js";
+import { runEvacuate } from "./groups/evacuate.js";
+import { runAdmit } from "./groups/admit.js";
 import { runK3sServer } from "./groups/k3s-server.js";
 import { runK3sAgent } from "./groups/k3s-agent.js";
 import { runNetworking } from "./groups/networking.js";
 import { runHardening } from "./groups/hardening.js";
 import { runSequential } from "./utils.js";
 import * as health from "./health/index.js";
+import { isK3sServer } from "@lab/shared";
 
 function toOpContext(ctx: ModuleContext): OperationContext {
+  // Pass the WHOLE config bag through, then pin the fields that come from the
+  // ModuleContext itself.
+  //
+  // This used to enumerate three keys -- k3sServerUrl, k3sToken, tlsSans -- and
+  // silently drop every other one. Anything a caller set beyond those three
+  // vanished with no error and no warning: k3sVersion (so a pinned version was
+  // ignored and get.k3s.io installed LATEST), evictionAllowSingleReplica (so
+  // `labctl app k3s evacuate --allow-single-replica` was a no-op that claimed
+  // to override a data-loss guard), and the dual-stack fields ipv6 /
+  // clusterCidr / serviceCidr.
+  //
+  // Enumerating is the bug: every field added to K3sConfig has to be
+  // remembered here, and forgetting produces silence rather than a type error.
   const config: K3sConfig = {
+    ...(ctx.config as Partial<K3sConfig>),
     hostname: ctx.hostname,
     ip: ctx.ip,
     role: ctx.role as K3sConfig["role"],
-    k3sServerUrl: ctx.config["k3sServerUrl"] as string | undefined,
-    k3sToken: ctx.config["k3sToken"] as string | undefined,
-    tlsSans: ctx.config["tlsSans"] as string[] | undefined,
   };
   return {
     config,
@@ -35,7 +49,16 @@ function toOpContext(ctx: ModuleContext): OperationContext {
     },
     os: ctx.os,
     arch: ctx.arch,
-    log: (_msg) => { /* collected via results */ },
+    // Emit progress LIVE, on stderr.
+    //
+    // This used to be a no-op "collected via results", but results only carry
+    // an operation's final message. Anything an operation reports WHILE it
+    // runs was therefore thrown away -- including evictLonghornReplicas's
+    // per-poll "N replica(s) still on <node>", which is the only visibility
+    // into an eviction that can legitimately take an hour. When one stalled,
+    // there was no way to tell which volumes were stuck or whether progress
+    // was being made at all.
+    log: (msg) => { console.error(`  [${ctx.hostname}] ${msg}`); },  // eslint-disable-line no-console
   };
 }
 
@@ -62,7 +85,7 @@ export class K3sModule implements Module {
   async install(ctx: ModuleContext): Promise<ModuleResult> {
     const start = performance.now();
     const opCtx = toOpContext(ctx);
-    const isServer = ctx.role === "infra" || ctx.role === "labcontroller";
+    const isServer = isK3sServer(ctx.role);
 
     // Phase 1: Host preparation
     const prepResults = await runHostPrep(opCtx);
@@ -95,6 +118,31 @@ export class K3sModule implements Module {
     return toModuleResult("configure", results, start);
   }
 
+  /**
+   * Evacuate the node so it can be wiped. Does NOT reinstall anything --
+   * it leaves the node cordoned, drained and empty.
+   */
+  async evacuate(ctx: ModuleContext): Promise<ModuleResult> {
+    const start = performance.now();
+    const opCtx = toOpContext(ctx);
+    const results = await runEvacuate(opCtx);
+    return toModuleResult("evacuate", results, start);
+  }
+
+  /**
+   * Return an evacuated node to service: uncordon, clear Longhorn's eviction
+   * request, restore its default disk.
+   *
+   * The counterpart to evacuate(), and the only supported way out of an
+   * evacuation that was aborted partway.
+   */
+  async admit(ctx: ModuleContext): Promise<ModuleResult> {
+    const start = performance.now();
+    const opCtx = toOpContext(ctx);
+    const results = await runAdmit(opCtx);
+    return toModuleResult("admit", results, start);
+  }
+
   async health(ctx: ModuleContext): Promise<ModuleResult> {
     const start = performance.now();
     const opCtx = toOpContext(ctx);

bastion/src/modules/modules/k3s/src/operations/etcd-members.ts

@@ -0,0 +1,60 @@
+// Parsing `etcdctl member list`.
+//
+// WHY NOT `-w json`
+// -----------------
+// Two reasons, and the second is the dangerous one:
+//
+//  1. `member remove` requires the id in HEX. The JSON writer emits decimal.
+//  2. etcd member ids are uint64. JSON.parse turns them into JS numbers, which
+//     lose precision above 2^53 -- so the id that comes back is an
+//     APPROXIMATION of the real one.
+//
+// Observed for real: `member remove 13323819137391910000` ->
+// "bad member ID arg (strconv.ParseUint: value out of range), expecting ID in
+// Hex". That happens at the worst possible moment, with the node already
+// stopped and about to be wiped. Failing loudly is the lucky outcome; a
+// mangled id that happened to match another member would remove the WRONG one.
+//
+// The default writer prints the id as hex TEXT, so it never becomes a number.
+
+export interface EtcdMember {
+  /** Raw member name: "<node>-<8 hex>", e.g. "worker0.lab-e26aacc5". */
+  name: string;
+  /** The node name with etcd's random suffix stripped. */
+  node: string;
+  /** Hex id, exactly as etcd prints it — never round-tripped through a number. */
+  id: string;
+  started: boolean;
+}
+
+/**
+ * etcd names a member `<node>-<8 hex>`. Comparing a raw member name against a
+ * node name without stripping that suffix silently never matches.
+ */
+export function etcdNodeName(memberName: string): string {
+  return memberName.replace(/-[0-9a-f]{8}$/, "");
+}
+
+/**
+ * Parse the default `etcdctl member list` writer:
+ *
+ *   825852d3683bc406, started, worker2.lab-5f7ad0c9, https://..:2380, https://..:2379, false
+ *   <hex id>        , status , <node>-<8 hex>      , peer           , client         , learner
+ */
+export function parseEtcdMembers(out: string): EtcdMember[] {
+  return out.split("\n")
+    .map((l) => l.trim())
+    .filter((l) => l !== "")
+    .map((line) => {
+      const cols = line.split(",").map((c) => c.trim());
+      const name = cols[2] ?? "";
+      return {
+        name,
+        node: etcdNodeName(name),
+        id: cols[0] ?? "",
+        started: (cols[1] ?? "") === "started",
+      };
+    })
+    // Drop anything without a hex id: blank lines, warnings, stray output.
+    .filter((m) => /^[0-9a-f]+$/i.test(m.id));
+}

bastion/src/modules/modules/k3s/src/operations/index.ts

@@ -5,7 +5,7 @@ export { growRancherLv } from "./rancher-storage.js";
 export { enableIscsi } from "./iscsi.js";
 export { disableFirewall } from "./firewall.js";
 export { setSelinuxPermissive } from "./selinux.js";
-export { writeK3sConfig } from "./k3s-config.js";
+export { writeK3sConfig, generateServerConfig, generateAgentConfig } from "./k3s-config.js";
 export { writeAuditPolicy } from "./audit-policy.js";
 export { cleanupStaleCni } from "./cni-cleanup.js";
 export { installK3sBinary } from "./k3s-install.js";
@@ -24,3 +24,10 @@ export type {
   RecoverEtcdMemberOptions,
   RecoverEtcdMemberResult,
 } from "./etcd-recover.js";
+export {
+  cordonNode, uncordonNode, drainNode, evictLonghornReplicas,
+  assertEvictionCompletable, assertNodeEvacuated, checkEvictionCompletable,
+} from "./node-evacuate.js";
+export type { EvictionFeasibility } from "./node-evacuate.js";
+export { clearLonghornEviction, assertNodeSchedulable } from "./node-admit.js";
+export { labelWorkerRole } from "./worker-label.js";

bastion/src/modules/modules/k3s/src/operations/k3s-config.ts

@@ -2,13 +2,54 @@
 
 import type { Operation, OperationResult, K3sConfig } from "../types.js";
 import { sshOpts, writeRemoteFile } from "../utils.js";
+import { isK3sServer } from "@lab/shared";
 
-function isServerRole(role: string): boolean {
-  return role === "infra" || role === "labcontroller";
+/**
+ * The address-family block: `cluster-cidr`, `service-cidr` and `node-ip`.
+ *
+ * Emitted ONLY when the corresponding config is supplied, and that is
+ * deliberate. With no `ipv6` and no CIDRs this returns "", so the generated
+ * file is byte-identical to what every existing node already has -- no diff,
+ * so `writeRemoteFile` reports unchanged and nothing restarts k3s. Dual-stack
+ * is therefore opt-in per node rather than a flag day.
+ *
+ * `node-ip` is only written once there is a second family to name. k3s
+ * auto-detects a sensible IPv4 on its own, and writing it out unconditionally
+ * would rewrite the config of five healthy nodes to tell them what they had
+ * already worked out.
+ */
+function addressFamilyLines(config: K3sConfig, opts: { cidrs: boolean }): string {
+  const lines: string[] = [];
+  if (opts.cidrs && config.clusterCidr?.length) {
+    lines.push(`cluster-cidr: "${config.clusterCidr.join(",")}"`);
+  }
+  if (opts.cidrs && config.serviceCidr?.length) {
+    lines.push(`service-cidr: "${config.serviceCidr.join(",")}"`);
+  }
+  if (config.ipv6) {
+    // Order matters to k3s: the FIRST entry is the primary family, and the
+    // supported single-to-dual-stack conversion is the one that preserves it.
+    // IPv4 stays primary so existing Services keep their ClusterIP.
+    lines.push(`node-ip: "${config.ip},${config.ipv6}"`);
+  }
+  return lines.length ? `${lines.join("\n")}\n` : "";
 }
 
-function generateServerConfig(config: K3sConfig): string {
-  const tlsSans = [config.hostname, config.ip, ...(config.tlsSans ?? [])];
+// Exported so the exact production config.yaml can be rendered outside an SSH
+// context -- notably by the labsim 3-server-etcd rehearsal, which must drive its
+// nodes through THIS generator rather than a parallel set of INSTALL_K3S_EXEC
+// flags, or it proves a mechanism production does not run.
+export function generateServerConfig(config: K3sConfig): string {
+  // The IPv6 address goes in the cert too. Without it, anything that reaches
+  // this apiserver over v6 -- a peer server joining, or kubectl against the v6
+  // address -- fails TLS verification, and the error names the certificate
+  // rather than the missing SAN, which is a long way from the cause.
+  const tlsSans = [
+    config.hostname,
+    config.ip,
+    ...(config.ipv6 ? [config.ipv6] : []),
+    ...(config.tlsSans ?? []),
+  ];
   const isJoining = !!config.k3sServerUrl;
   const clusterLines = isJoining
     ? `server: "${config.k3sServerUrl}"\ntoken: "${config.k3sToken}"`
@@ -21,7 +62,7 @@ function generateServerConfig(config: K3sConfig): string {
   // and never expire.
   return `# k3s server configuration — CIS hardened, etcd HA
 ${clusterLines}
-protect-kernel-defaults: true
+${addressFamilyLines(config, { cidrs: true })}protect-kernel-defaults: true
 secrets-encryption: true
 write-kubeconfig-mode: "0640"
 
@@ -51,10 +92,34 @@ ${tlsSans.map((s) => `  - "${s}"`).join("\n")}
 `;
 }
 
-function generateAgentConfig(): string {
-  return `protect-kernel-defaults: true
+// Takes the config now: an agent needs its own dual `node-ip` just as much as a
+// server does. Without one it joins as an IPv4-only node into a dual-stack
+// cluster, gets no IPv6 pod CIDR, and the failure surfaces later as pods on that
+// node being unreachable over v6 while the node itself reads Ready.
+// It takes no cluster/service CIDRs -- those are server-side only.
+export function generateAgentConfig(config: K3sConfig): string {
+  // NO node-role.kubernetes.io/worker LABEL HERE, deliberately.
+  //
+  // A kubelet is not permitted to assign itself labels in the
+  // `node-role.kubernetes.io/` namespace -- the servers run with
+  // `enable-admission-plugins=NodeRestriction`, which rejects the registration
+  // outright. The agent then never finishes registering, k3s-agent.service
+  // never signals readiness, and systemd reports:
+  //
+  //   Job for k3s-agent.service failed because the service did not take the
+  //   steps required by its unit configuration.
+  //
+  // which says nothing about labels and sends you looking at the network, the
+  // token, or the kubelet's sysctls. Every agent installed through this module
+  // failed this way; the estate's existing workers only escaped it because
+  // post-provision installs them with no config.yaml at all and their role
+  // label was applied separately, server-side.
+  //
+  // `node.longhorn.io/...` is fine: it is outside the kubernetes.io namespace,
+  // so the kubelet may set it. The worker role label is applied after the node
+  // registers, by labelWorkerRole below.
+  return `${addressFamilyLines(config, { cidrs: false })}protect-kernel-defaults: true
 node-label:
-  - "node-role.kubernetes.io/worker=true"
   - "node.longhorn.io/create-default-disk=config"
 kubelet-arg:
   - "protect-kernel-defaults=true"
@@ -64,11 +129,41 @@ kubelet-arg:
 }
 
 export const writeK3sConfig: Operation = async (ctx): Promise<OperationResult> => {
+  // Refuse to name an address the node does not have.
+  //
+  // Most of this estate is SSH-onboard, not PXE-provisioned: Asahi cannot PXE
+  // at all, and the DGX Sparks run NVIDIA's own OS and must never be
+  // reinstalled. For those nodes the install templates govern nothing and this
+  // module is the ONLY labctl touchpoint, so nothing upstream can guarantee the
+  // vendor OS actually took a DHCPv6 lease.
+  //
+  // Writing node-ip for a missing address does not fail here -- it fails later,
+  // when k3s will not start, with an error about binding rather than about
+  // addressing. Checking costs one ssh round trip and turns a confusing
+  // start-up failure into a sentence naming the address and the node.
+  if (ctx.config.ipv6) {
+    const probe = await ctx.ssh.exec(
+      `ip -6 -o addr show 2>/dev/null | grep -qF " ${ctx.config.ipv6}/" && echo present || true`,
+      sshOpts(ctx),
+    );
+    if (!probe.stdout.includes("present")) {
+      return {
+        success: false,
+        changed: false,
+        message: `Node does not have IPv6 ${ctx.config.ipv6}`,
+        error:
+          `k3s config would set node-ip to ${ctx.config.ipv6}, but that address is not on any ` +
+          `interface. k3s resolves node-ip at start-up and would fail to bind. Check the node ` +
+          `took its DHCPv6 lease (a kea reservation keyed on its MAC) before retrying.`,
+      };
+    }
+  }
+
   await ctx.ssh.exec("mkdir -p /etc/rancher/k3s", sshOpts(ctx));
 
-  const content = isServerRole(ctx.config.role)
+  const content = isK3sServer(ctx.config.role)
     ? generateServerConfig(ctx.config)
-    : generateAgentConfig();
+    : generateAgentConfig(ctx.config);
 
   const changed = await writeRemoteFile(ctx, "/etc/rancher/k3s/config.yaml", content);
 

bastion/src/modules/modules/k3s/src/operations/k3s-install.ts

@@ -2,13 +2,19 @@
 
 import type { Operation, OperationResult } from "../types.js";
 import { sshOpts } from "../utils.js";
-
-function isServerRole(role: string): boolean {
-  return role === "infra" || role === "labcontroller";
-}
+import { isK3sServer } from "@lab/shared";
 
 export const installK3sBinary: Operation = async (ctx): Promise<OperationResult> => {
-  const isServer = isServerRole(ctx.config.role);
+  const isServer = isK3sServer(ctx.config.role);
+
+  // Pin the version when asked. get.k3s.io installs LATEST by default, so a
+  // node reinstalled during a rolling migration comes back on whatever shipped
+  // since -- potentially several minor versions ahead of the cluster it is
+  // about to rejoin, which Kubernetes does not support between control-plane
+  // members.
+  const versionEnv = ctx.config.k3sVersion !== undefined && ctx.config.k3sVersion !== ""
+    ? `INSTALL_K3S_VERSION="${ctx.config.k3sVersion}" `
+    : "";
 
   // Check if already installed
   const version = await ctx.ssh.exec("k3s --version 2>/dev/null", sshOpts(ctx));
@@ -24,12 +30,21 @@ export const installK3sBinary: Operation = async (ctx): Promise<OperationResult>
       );
     }
 
-    // If joining an existing cluster, pass K3S_URL and K3S_TOKEN
-    const joinEnv = ctx.config.k3sServerUrl && ctx.config.k3sToken
-      ? `K3S_URL="${ctx.config.k3sServerUrl}" K3S_TOKEN="${ctx.config.k3sToken}"`
-      : "";
+    // Token and URL are independent. A token WITHOUT a URL seeds the cluster
+    // token on the cluster-init node, so every later joiner can be given the
+    // same pre-shared value instead of scraping node-token off the first
+    // server -- which matters here because that first server gets wiped and
+    // rejoined during a rolling migration. URL only counts when a token comes
+    // with it; a URL alone cannot authenticate a join.
+    //
+    // The token deliberately does NOT go in config.yaml: writeRemoteFile diffs
+    // that file, so a token appearing there would restart k3s on every node.
+    const joinEnv = [
+      ctx.config.k3sServerUrl && ctx.config.k3sToken ? `K3S_URL="${ctx.config.k3sServerUrl}"` : "",
+      ctx.config.k3sToken ? `K3S_TOKEN="${ctx.config.k3sToken}"` : "",
+    ].filter(Boolean).join(" ");
     const result = await ctx.ssh.exec(
-      `curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server" INSTALL_K3S_SKIP_SELINUX_RPM=true ${joinEnv} sh -`,
+      `curl -sfL https://get.k3s.io | ${versionEnv}INSTALL_K3S_EXEC="server" INSTALL_K3S_SKIP_SELINUX_RPM=true ${joinEnv} sh -`,
       { timeoutMs: 300_000 },
     );
     if (result.exitCode !== 0) {
@@ -50,7 +65,7 @@ export const installK3sBinary: Operation = async (ctx): Promise<OperationResult>
       };
     }
     const result = await ctx.ssh.exec(
-      `curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="agent" K3S_URL="${ctx.config.k3sServerUrl}" K3S_TOKEN="${ctx.config.k3sToken}" sh -`,
+      `curl -sfL https://get.k3s.io | ${versionEnv}INSTALL_K3S_EXEC="agent" K3S_URL="${ctx.config.k3sServerUrl}" K3S_TOKEN="${ctx.config.k3sToken}" sh -`,
       { timeoutMs: 300_000 },
     );
     if (result.exitCode !== 0) {

bastion/src/modules/modules/k3s/src/operations/longhorn-disk.ts

@@ -1,50 +1,206 @@
-// Annotate nodes with Longhorn default disk config when /var/lib/longhorn exists.
-// The label is set in k3s config (node-label), but the annotation must be applied via kubectl.
+// Give a node a Longhorn disk at /var/lib/longhorn.
+//
+// The cluster runs `createDefaultDiskLabeledNodes: true`, which means Longhorn
+// creates a disk ONLY on nodes carrying the label
+// `node.longhorn.io/create-default-disk=config` (set via k3s node-label) AND
+// the annotation `node.longhorn.io/default-disks-config` (set here). Miss
+// either and Longhorn gives the node no disk at all: it joins, goes Ready,
+// accepts pods, and silently hosts zero replicas.
+//
+// WHY THERE IS A REPAIR PATH
+// --------------------------
+// Longhorn reads that annotation when it FIRST creates the node's
+// `nodes.longhorn.io` record. This operation runs during configure(), which on
+// a rejoining node is after the record already exists -- so the annotation
+// lands too late and is never read. That is invisible until a rebuild is
+// expected and never arrives, roughly forty minutes into a migration.
+//
+// So: annotate, then read back, and if there is still no disk, write it
+// directly into spec.disks. When Longhorn does honour the annotation the
+// repair is a no-op.
 
-import type { Operation, OperationResult } from "../types.js";
+import type { Operation, OperationContext, OperationResult } from "../types.js";
 import { sshOpts } from "../utils.js";
 import { sshExec as remoteSshExec } from "../../../../src/ssh.js";
 
+const DISK_PATH = "/var/lib/longhorn";
+/** Fixed key so a repeated repair updates one disk rather than adding more. */
+const DISK_KEY = "default-disk-lab";
+const VERIFY_INTERVAL_MS = 5_000;
+/** How long to wait for Longhorn to create the node's record after it joins. */
+const REGISTER_TIMEOUT_MS = 180_000;
+/** How long to then wait for Longhorn to give that record a disk. */
+const DISK_TIMEOUT_MS = 60_000;
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * Run kubectl from wherever it can be reached: locally on a server, otherwise
+ * over SSH to the configured server (an agent has no local kubectl).
+ */
+async function kubectlAnywhere(
+  ctx: OperationContext,
+  args: string,
+): Promise<{ ok: boolean; out: string }> {
+  const local = await ctx.ssh.exec(`k3s kubectl ${args} 2>&1`, sshOpts(ctx));
+  if (local.exitCode === 0) return { ok: true, out: local.stdout };
+
+  if (ctx.config.k3sServerUrl) {
+    try {
+      const serverHost = new URL(ctx.config.k3sServerUrl).hostname;
+      const remote = await remoteSshExec(serverHost, "root", `k3s kubectl ${args}`, {
+        ...(ctx.ssh.keyPath ? { keyPath: ctx.ssh.keyPath } : {}),
+        timeoutMs: 15_000,
+      });
+      if (remote.exitCode === 0) return { ok: true, out: remote.stdout };
+      return { ok: false, out: remote.stdout || remote.stderr };
+    } catch {
+      /* fall through */
+    }
+  }
+  return { ok: false, out: local.stdout || local.stderr };
+}
+
+/** Does the node's Longhorn record exist yet? */
+async function nodeRecordExists(ctx: OperationContext, nodeName: string): Promise<boolean> {
+  const res = await kubectlAnywhere(ctx, `-n longhorn-system get nodes.longhorn.io "${nodeName}" -o name`);
+  return res.ok && res.out.trim() !== "";
+}
+
+/** Does the node's Longhorn record carry a disk at DISK_PATH? */
+async function hasDisk(ctx: OperationContext, nodeName: string): Promise<boolean> {
+  const res = await kubectlAnywhere(
+    ctx,
+    `-n longhorn-system get nodes.longhorn.io "${nodeName}" ` +
+      `-o jsonpath='{range .spec.disks.*}{.path}{"\\n"}{end}'`,
+  );
+  if (!res.ok) return false;
+  return res.out.split("\n").map((l) => l.trim()).includes(DISK_PATH);
+}
+
 export const configureLonghornDisk: Operation = async (ctx): Promise<OperationResult> => {
-  // Check if /var/lib/longhorn exists on this node
-  const check = await ctx.ssh.exec("test -d /var/lib/longhorn && echo yes || echo no", sshOpts(ctx));
+  const check = await ctx.ssh.exec(`test -d ${DISK_PATH} && echo yes || echo no`, sshOpts(ctx));
   if (check.stdout.trim() !== "yes") {
-    return { success: true, changed: false, message: "No /var/lib/longhorn directory — skipping Longhorn disk config" };
+    return { success: true, changed: false, message: `No ${DISK_PATH} directory — skipping Longhorn disk config` };
   }
 
-  // Find the node name (hostname as registered in k3s)
+  // Longhorn writes replicas into <path>/replicas and creates that directory
+  // only when it FIRST initialises the disk. If the directory later goes away
+  // while longhorn-disk.cfg remains -- someone clearing space, a partial
+  // restore, a wipe that preserved the LV -- Longhorn considers the disk
+  // already initialised and never recreates it. The replica process then dies
+  // on a non-recursive mkdir:
+  //
+  //   level=fatal msg="Error running start replica command"
+  //     error="mkdir /host/var/lib/longhorn/replicas/pvc-...: ..."
+  //
+  // every single time, for ever, while the node reports Ready and Schedulable
+  // and its disk looks healthy. Cheap to guarantee, expensive to diagnose.
+  await ctx.ssh.exec(`mkdir -p ${DISK_PATH}/replicas`, sshOpts(ctx));
+
   const nodeNameResult = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
   const nodeName = nodeNameResult.stdout.trim();
 
-  const annotation = JSON.stringify([{ path: "/var/lib/longhorn", allowScheduling: true }]);
-
-  // Try kubectl locally first (works on server nodes)
-  const result = await ctx.ssh.exec(
-    `k3s kubectl annotate node "${nodeName}" "node.longhorn.io/default-disks-config=${annotation}" --overwrite 2>&1 || true`,
-    sshOpts(ctx),
+  // SINGLE quotes around the whole key=value, not double.
+  //
+  // The value is JSON and therefore full of double quotes. Wrapping it in
+  // double quotes ends the quoted string at the first inner one, and the shell
+  // strips the rest, so the node gets
+  //     [{path:/var/lib/longhorn,allowScheduling:true}]
+  // instead of
+  //     [{"path":"/var/lib/longhorn","allowScheduling":true}]
+  // -- valid-looking, unparseable, and silent. Longhorn then creates NO disk on
+  // the node, which surfaces much later as "why does nothing schedule here".
+  // The JSON contains no single quotes, so single-quoting is safe.
+  const annotation = JSON.stringify([{ path: DISK_PATH, allowScheduling: true }]);
+  const annotated = await kubectlAnywhere(
+    ctx,
+    `annotate node "${nodeName}" 'node.longhorn.io/default-disks-config=${annotation}' --overwrite`,
   );
 
-  if (result.stdout.includes("annotated") || result.stdout.includes("unchanged")) {
-    return { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName}` };
+  // No Longhorn at all (or no way to reach the API) — nothing more to do.
+  const lh = await kubectlAnywhere(ctx, "get ns longhorn-system -o name");
+  if (!lh.ok) {
+    return annotated.ok
+      ? { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName} (Longhorn not installed yet)` }
+      : { success: true, changed: false, message: "Longhorn disk label set (annotation requires server kubectl)" };
   }
 
-  // For worker/agent nodes without local kubectl: apply via the server
-  if (ctx.config.k3sServerUrl) {
-    // The CLI has SSH access to the server — use sshExec from there
-    const serverHost = new URL(ctx.config.k3sServerUrl).hostname;
-    try {
-      const remoteResult = await remoteSshExec(
-        serverHost, "root",
-        `k3s kubectl annotate node "${nodeName}" "node.longhorn.io/default-disks-config=${annotation}" --overwrite`,
-        { ...(ctx.ssh.keyPath ? { keyPath: ctx.ssh.keyPath } : {}), timeoutMs: 15_000 },
-      );
-      if (remoteResult.stdout.includes("annotated") || remoteResult.stdout.includes("unchanged")) {
-        return { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName} (via server)` };
-      }
-    } catch {
-      // Fall through to manual instruction
+  // Longhorn registration is ASYNCHRONOUS. On a node that has just joined --
+  // which is exactly the case after a wipe-and-rejoin -- the nodes.longhorn.io
+  // record does not exist yet, so there is nothing to inspect and nothing to
+  // repair. Waiting a fixed 60s and then giving up meant the repair never ran
+  // on the one path that needs it most, and the caller's schedulability gate
+  // then failed immediately with "Longhorn reports no disk".
+  //
+  // So: wait for the RECORD first, generously, and only then worry about the
+  // disk on it.
+  const registerDeadline = Date.now() + REGISTER_TIMEOUT_MS;
+  let registered = false;
+  while (Date.now() < registerDeadline) {
+    if (await hasDisk(ctx, nodeName)) {
+      return { success: true, changed: annotated.ok, message: `${nodeName} has a Longhorn disk at ${DISK_PATH}` };
     }
+    if (await nodeRecordExists(ctx, nodeName)) { registered = true; break; }
+    await sleep(VERIFY_INTERVAL_MS);
   }
 
-  return { success: true, changed: false, message: "Longhorn disk label set (annotation requires server kubectl)" };
+  if (!registered) {
+    return {
+      success: true,
+      changed: annotated.ok,
+      message: `${nodeName} not registered with Longhorn within ${REGISTER_TIMEOUT_MS / 1000}s; ` +
+        `annotation is in place for when it is`,
+    };
+  }
+
+  // Registered. Give Longhorn a chance to act on the annotation by itself
+  // before overriding it.
+  const diskDeadline = Date.now() + DISK_TIMEOUT_MS;
+  while (Date.now() < diskDeadline) {
+    if (await hasDisk(ctx, nodeName)) {
+      return { success: true, changed: annotated.ok, message: `${nodeName} has a Longhorn disk at ${DISK_PATH}` };
+    }
+    await sleep(VERIFY_INTERVAL_MS);
+  }
+
+  // The record exists and still has no disk: the annotation was applied too
+  // late to be read. Write it in.
+  ctx.log(`${nodeName} has no Longhorn disk after annotating; writing spec.disks directly`);
+  const patch = JSON.stringify({
+    spec: {
+      disks: {
+        [DISK_KEY]: {
+          path: DISK_PATH,
+          allowScheduling: true,
+          evictionRequested: false,
+          storageReserved: 0,
+          tags: [],
+          diskType: "filesystem",
+        },
+      },
+    },
+  });
+  const repaired = await kubectlAnywhere(
+    ctx,
+    `-n longhorn-system patch nodes.longhorn.io "${nodeName}" --type=merge -p '${patch}'`,
+  );
+  if (!repaired.ok) {
+    return {
+      success: false,
+      changed: false,
+      message: `${nodeName} has no Longhorn disk and could not be repaired`,
+      error: `${repaired.out}. Without a disk this node will never host a replica.`,
+    };
+  }
+
+  if (await hasDisk(ctx, nodeName)) {
+    return { success: true, changed: true, message: `Longhorn disk repaired on ${nodeName} (${DISK_PATH})` };
+  }
+  return {
+    success: false,
+    changed: true,
+    message: `Patched spec.disks on ${nodeName} but Longhorn still reports no disk at ${DISK_PATH}`,
+    error: "Check longhorn-manager logs; the node will not host replicas until this is resolved.",
+  };
 };

bastion/src/modules/modules/k3s/src/operations/node-admit.ts

@@ -0,0 +1,136 @@
+// Bring a node back into service. The inverse of node-evacuate.ts.
+//
+// WHY THIS EXISTS
+// ---------------
+// Evacuation is not symmetric with itself. It leaves three marks on a node --
+// a kubernetes cordon, Longhorn's `allowScheduling:false` + `evictionRequested`,
+// and (on a node that was wiped and rejoined) a MISSING default Longhorn disk.
+// Undoing a cordon undoes only the first. Without the other two the node comes
+// back Ready, accepts pods, and quietly hosts no replicas at all -- which looks
+// like success right up until you notice storage never rebalanced onto it.
+//
+// This is also the only supported way out of an aborted evacuation.
+
+import type { Operation, OperationContext, OperationResult } from "../types.js";
+import { sshOpts } from "../utils.js";
+
+async function nodeName(ctx: OperationContext): Promise<string> {
+  const r = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
+  return r.stdout.trim();
+}
+
+async function kubectl(ctx: OperationContext, args: string): Promise<{ ok: boolean; out: string }> {
+  const res = await ctx.ssh.exec(`k3s kubectl ${args} 2>&1`, sshOpts(ctx));
+  return { ok: res.exitCode === 0, out: res.exitCode === 0 ? res.stdout : (res.stdout || res.stderr) };
+}
+
+/**
+ * Clear Longhorn's eviction request and re-enable scheduling.
+ *
+ * Both fields in one merge patch, mirroring how eviction sets them: Longhorn's
+ * validating webhook rejects `evictionRequested:true` on a schedulable node, so
+ * the pair has to move together in both directions.
+ */
+export const clearLonghornEviction: Operation = async (ctx): Promise<OperationResult> => {
+  const name = await nodeName(ctx);
+
+  const lh = await kubectl(ctx, "get ns longhorn-system -o name");
+  if (!lh.ok) {
+    return { success: true, changed: false, message: "Longhorn not installed — nothing to clear" };
+  }
+
+  const before = await kubectl(
+    ctx,
+    `-n longhorn-system get nodes.longhorn.io "${name}" -o jsonpath='{.spec.allowScheduling},{.spec.evictionRequested}'`,
+  );
+  if (!before.ok) {
+    return { success: true, changed: false, message: `${name} has no Longhorn node record yet` };
+  }
+  if (before.out.trim() === "true,false") {
+    return { success: true, changed: false, message: `${name} already accepts replicas` };
+  }
+
+  const patch = JSON.stringify({ spec: { allowScheduling: true, evictionRequested: false } });
+  const res = await kubectl(
+    ctx,
+    `-n longhorn-system patch nodes.longhorn.io "${name}" --type=merge -p '${patch}'`,
+  );
+  return res.ok
+    ? { success: true, changed: true, message: `${name} accepts replicas again` }
+    : { success: false, changed: false, message: `Could not clear eviction on ${name}`, error: res.out };
+};
+
+/**
+ * Confirm the node can actually host replicas again.
+ *
+ * Reads back from Longhorn rather than trusting the patches above, because the
+ * failure this catches is a node that is Ready and schedulable in kubernetes
+ * while having no Longhorn disk at all -- the exact state a rejoined node lands
+ * in when its default-disk annotation was applied too late.
+ */
+const SCHEDULABLE_TIMEOUT_MS = 120_000;
+const SCHEDULABLE_INTERVAL_MS = 5_000;
+const sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
+
+export const assertNodeSchedulable: Operation = async (ctx): Promise<OperationResult> => {
+  // Poll rather than check once.
+  //
+  // Longhorn registers a node and attaches its disk ASYNCHRONOUSLY, and this
+  // runs moments after a wipe-and-rejoin. Checking instantly reports "Longhorn
+  // reports no disk on this node" for a node that is simply a few seconds from
+  // being fine -- which is a false alarm on the exact path the gate exists to
+  // protect. Still fails if it never settles.
+  const deadline = Date.now() + SCHEDULABLE_TIMEOUT_MS;
+  let problems: string[] = [];
+  for (;;) {
+    problems = await schedulableProblems(ctx);
+    if (problems.length === 0 || Date.now() >= deadline) break;
+    await sleep(SCHEDULABLE_INTERVAL_MS);
+  }
+
+  const name = await nodeName(ctx);
+  return problems.length === 0
+    ? { success: true, changed: false, message: `${name} is back in service` }
+    : {
+        success: false,
+        changed: false,
+        message: `${name} is NOT fully back in service after ${SCHEDULABLE_TIMEOUT_MS / 1000}s`,
+        error: problems.join("; "),
+      };
+};
+
+/** One pass of the schedulability checks. Returns the reasons it is not. */
+async function schedulableProblems(ctx: OperationContext): Promise<string[]> {
+  const name = await nodeName(ctx);
+  const problems: string[] = [];
+
+  const sched = await kubectl(ctx, `get node "${name}" -o jsonpath='{.spec.unschedulable}'`);
+  if (sched.ok && sched.out.trim() === "true") problems.push("node is still cordoned");
+
+  const lh = await kubectl(ctx, "get ns longhorn-system -o name");
+  if (lh.ok) {
+    const spec = await kubectl(
+      ctx,
+      `-n longhorn-system get nodes.longhorn.io "${name}" -o jsonpath='{.spec.allowScheduling},{.spec.evictionRequested}'`,
+    );
+    if (!spec.ok) {
+      problems.push("no Longhorn node record");
+    } else {
+      const [allow, evicting] = spec.out.trim().split(",");
+      if (allow !== "true") problems.push("Longhorn scheduling is still disabled");
+      if (evicting === "true") problems.push("Longhorn eviction is still requested");
+    }
+
+    // A node with no disk hosts no replicas, however healthy it otherwise looks.
+    const disks = await kubectl(
+      ctx,
+      `-n longhorn-system get nodes.longhorn.io "${name}" ` +
+        `-o jsonpath='{range .status.diskStatus.*}{.conditions[?(@.type=="Schedulable")].status}{"\\n"}{end}'`,
+    );
+    const states = disks.ok ? disks.out.split("\n").map((l) => l.trim()).filter(Boolean) : [];
+    if (states.length === 0) problems.push("Longhorn reports no disk on this node");
+    else if (!states.includes("True")) problems.push(`no schedulable disk (states: ${states.join(",")})`);
+  }
+
+  return problems;
+}

bastion/src/modules/modules/k3s/src/operations/node-evacuate.ts

@@ -0,0 +1,547 @@
+// Evacuate a node before it is wiped and reinstalled.
+//
+// WHY THIS EXISTS
+// ---------------
+// The kickstart's %pre can preserve data across a reinstall, but ONLY when the
+// disk layout is unchanged: it keys off `labvg` already existing with the
+// expected LV names. Migrating a machine to the v2 layout means recreating the
+// volume group, which takes the `clearpart --all` branch and destroys every
+// Longhorn replica on that node.
+//
+// So a layout migration cannot rely on disk-level preservation. The node has to
+// be EVACUATED first -- replicas rebuilt elsewhere, workloads moved, etcd
+// membership surrendered -- and then wiped. These operations are the missing
+// half; before this file the repo had no drain, no cordon, and no replica
+// eviction anywhere.
+//
+// Ordering matters and is not interchangeable:
+//   cordon -> check -> evict replicas (WAIT for rebuild) -> drain -> remove etcd member
+//
+// Two ordering facts, both learned the hard way:
+//   - Removing the etcd member first costs you the API server you need for the
+//     rest of the procedure.
+//   - EVICT BEFORE DRAIN. `drain --force` does not override PodDisruptionBudgets,
+//     and longhorn-manager keeps a PDB on the instance-manager pod while volumes
+//     are attached or degraded. Draining a storage node first therefore stops the
+//     instance-manager that serves the very replicas being evicted: the drain
+//     blocks for its full timeout, and if it did succeed it would remove the only
+//     source a single-replica volume could rebuild from.
+
+import type { Operation, OperationContext, OperationResult } from "../types.js";
+import { sshOpts } from "../utils.js";
+
+/** Longhorn's own default; a rebuild of a large replica set can exceed it. */
+const REPLICA_REBUILD_TIMEOUT_MS = 60 * 60_000;
+const POLL_INTERVAL_MS = 15_000;
+/** How often to report "still waiting" when the replica count is not moving. */
+const HEARTBEAT_INTERVAL_MS = 3 * 60_000;
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+async function nodeName(ctx: OperationContext): Promise<string> {
+  const r = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
+  return r.stdout.trim();
+}
+
+/**
+ * kubectl, from wherever it is reachable.
+ *
+ * An agent has no local kubectl, so the caller must be pointed at a server.
+ * Returns null when there is no way to reach the API, so callers can report
+ * "could not do this" instead of silently continuing.
+ */
+async function kubectl(
+  ctx: OperationContext,
+  args: string,
+  /**
+   * Override the SSH timeout for this call.
+   *
+   * sshOpts() is a flat 30 seconds, which is right for a status read and
+   * completely wrong for anything that does work. A `kubectl drain` asking for
+   * --timeout=15m gets killed by the transport at 30s and returns non-zero
+   * having ALREADY evicted most of the pods -- so it looks like a drain
+   * failure, with a transcript that reads like success, on every real node.
+   */
+  timeoutMs?: number,
+): Promise<{ ok: boolean; out: string }> {
+  const opts = timeoutMs === undefined ? sshOpts(ctx) : { timeoutMs };
+  const local = await ctx.ssh.exec(`k3s kubectl ${args} 2>&1`, opts);
+  if (local.exitCode === 0) return { ok: true, out: local.stdout };
+  return { ok: false, out: local.stdout || local.stderr };
+}
+
+/** Mark the node unschedulable. Safe and instantly reversible. */
+export const cordonNode: Operation = async (ctx): Promise<OperationResult> => {
+  const name = await nodeName(ctx);
+  const res = await kubectl(ctx, `cordon "${name}"`);
+  if (!res.ok) {
+    return { success: false, changed: false, message: `Could not cordon ${name}`, error: res.out };
+  }
+  const already = res.out.includes("already cordoned");
+  return {
+    success: true,
+    changed: !already,
+    message: already ? `${name} was already cordoned` : `Cordoned ${name}`,
+  };
+};
+
+export const uncordonNode: Operation = async (ctx): Promise<OperationResult> => {
+  const name = await nodeName(ctx);
+  const res = await kubectl(ctx, `uncordon "${name}"`);
+  return res.ok
+    ? { success: true, changed: true, message: `Uncordoned ${name}` }
+    : { success: false, changed: false, message: `Could not uncordon ${name}`, error: res.out };
+};
+
+/**
+ * Move workloads off the node.
+ *
+ * --ignore-daemonsets because a DaemonSet pod cannot be evicted by definition;
+ * --delete-emptydir-data because refusing to delete emptyDir is what makes an
+ * unattended drain hang forever, and a node about to be wiped is going to lose
+ * that data regardless.
+ */
+export const drainNode: Operation = async (ctx): Promise<OperationResult> => {
+  const name = await nodeName(ctx);
+  // The SSH timeout must outlive kubectl's own --timeout, or the transport
+  // kills a drain that is working and the result is indistinguishable from a
+  // drain that failed.
+  const DRAIN_TIMEOUT_MS = 15 * 60_000;
+  const res = await kubectl(
+    ctx,
+    `drain "${name}" --ignore-daemonsets --delete-emptydir-data --force --timeout=15m`,
+    DRAIN_TIMEOUT_MS + 60_000,
+  );
+  if (res.ok) return { success: true, changed: true, message: `Drained ${name}` };
+
+  // Surface the END of the transcript. The first line of a drain is always
+  // "node/x already cordoned", which tells you nothing and hides the reason.
+  const tail = res.out.split("\n").map((l) => l.trim()).filter(Boolean).slice(-6).join(" | ");
+  return {
+    success: false,
+    changed: false,
+    message: `Drain failed on ${name}`,
+    error: tail === "" ? res.out : `...${tail}`,
+  };
+};
+
+/** kubectl -o json, parsed. Returns null when the read or the parse failed. */
+async function kubectlJson<T>(ctx: OperationContext, args: string): Promise<T | null> {
+  const res = await kubectl(ctx, `${args} -o json`);
+  if (!res.ok) return null;
+  try {
+    return JSON.parse(res.out) as T;
+  } catch {
+    return null;
+  }
+}
+
+/** A Longhorn setting's value, or the supplied default when it is unreadable. */
+async function longhornSetting(ctx: OperationContext, name: string, fallback: string): Promise<string> {
+  const res = await kubectl(ctx, `-n longhorn-system get settings.longhorn.io ${name} -o jsonpath='{.value}'`);
+  const v = res.ok ? res.out.trim() : "";
+  return v === "" ? fallback : v;
+}
+
+interface LhNode {
+  metadata: { name: string };
+  spec: { allowScheduling?: boolean };
+  status?: { diskStatus?: Record<string, {
+    storageAvailable?: number; storageMaximum?: number;
+    storageScheduled?: number; storageReserved?: number;
+  }> };
+}
+interface LhReplica {
+  metadata: { name: string };
+  spec: { nodeID?: string; volumeName?: string };
+}
+interface LhVolume {
+  metadata: { name: string };
+  spec: { numberOfReplicas?: number; size?: number | string };
+  status?: { state?: string; robustness?: string };
+}
+
+export interface EvictionFeasibility {
+  completable: boolean;
+  /** Reasons eviction can never finish. Each names the offending volume. */
+  blockers: string[];
+  /** Things that will work but are dangerous. */
+  warnings: string[];
+  replicasMoving: number;
+  bytesMoving: number;
+}
+
+/**
+ * Can eviction of this node actually COMPLETE?
+ *
+ * evictLonghornReplicas waits for the node's replica count to reach zero. If
+ * there is nowhere for a replica to go, that wait can never end -- it burns the
+ * full hour-long timeout and then fails, which is an expensive way to discover
+ * a fact that is knowable in a couple of seconds.
+ *
+ * It is knowable because Longhorn's placement rules are simple:
+ *
+ *   - With strict anti-affinity (replica-soft-anti-affinity=false, the default
+ *     and what this fleet runs) a volume may put at most ONE replica on a node.
+ *     So a volume wanting N replicas needs N schedulable nodes OTHER than the
+ *     one being evacuated. A 3-replica volume on a 3-node cluster is therefore
+ *     unevictable -- the two survivors already hold a copy each and cannot take
+ *     the third.
+ *   - The scheduler admits a replica against over-provisioned space, but the
+ *     rebuild writes real bytes to a real disk. Those are different numbers,
+ *     and at 500% over-provisioning the first can pass while the second fails.
+ *
+ * Read-only, a handful of API calls, finishes in seconds.
+ */
+export async function checkEvictionCompletable(
+  ctx: OperationContext,
+  name: string,
+): Promise<EvictionFeasibility> {
+  const blockers: string[] = [];
+  const warnings: string[] = [];
+
+  const nodes = await kubectlJson<{ items: LhNode[] }>(ctx, "-n longhorn-system get nodes.longhorn.io");
+  const replicas = await kubectlJson<{ items: LhReplica[] }>(ctx, "-n longhorn-system get replicas.longhorn.io");
+  const volumes = await kubectlJson<{ items: LhVolume[] }>(ctx, "-n longhorn-system get volumes.longhorn.io");
+  if (!nodes || !replicas || !volumes) {
+    return {
+      completable: false,
+      blockers: ["Could not read Longhorn nodes/replicas/volumes — refusing to guess"],
+      warnings, replicasMoving: 0, bytesMoving: 0,
+    };
+  }
+
+  const softAntiAffinity = (await longhornSetting(ctx, "replica-soft-anti-affinity", "false")) === "true";
+  const overProvisionPct = Number(await longhornSetting(ctx, "storage-over-provisioning-percentage", "100")) || 100;
+
+  const mine = replicas.items.filter((r) => r.spec.nodeID === name);
+  if (mine.length === 0) {
+    return { completable: true, blockers, warnings, replicasMoving: 0, bytesMoving: 0 };
+  }
+
+  const volumeByName = new Map(volumes.items.map((v) => [v.metadata.name, v]));
+  // Candidates are schedulable Longhorn nodes that are not the one leaving.
+  const candidates = nodes.items.filter(
+    (n) => n.metadata.name !== name && n.spec.allowScheduling !== false,
+  );
+
+  const diskFree = (n: LhNode): number =>
+    Object.values(n.status?.diskStatus ?? {}).reduce((a, d) => a + (d.storageAvailable ?? 0), 0);
+  const schedulableFree = (n: LhNode): number =>
+    Object.values(n.status?.diskStatus ?? {}).reduce((a, d) => {
+      const usable = ((d.storageMaximum ?? 0) - (d.storageReserved ?? 0)) * (overProvisionPct / 100);
+      return a + Math.max(0, usable - (d.storageScheduled ?? 0));
+    }, 0);
+
+  let bytesMoving = 0;
+
+  for (const rep of mine) {
+    const volName = rep.spec.volumeName ?? "";
+    const vol = volumeByName.get(volName);
+    if (!vol) {
+      warnings.push(`replica ${rep.metadata.name} has no volume ${volName} — orphan?`);
+      continue;
+    }
+    const want = vol.spec.numberOfReplicas ?? 1;
+    const size = Number(vol.spec.size ?? 0);
+    bytesMoving += size;
+
+    // (1) Placement. Strict anti-affinity means one replica per node per volume.
+    if (!softAntiAffinity && candidates.length < want) {
+      blockers.push(
+        `volume ${volName} wants ${want} replicas but only ${candidates.length} other schedulable ` +
+        `node(s) exist — with strict anti-affinity its replica cannot be placed. Add a schedulable node.`,
+      );
+      continue;
+    }
+
+    // (2) Scheduler capacity, at the configured over-provisioning percentage.
+    const schedulableFit = candidates.filter((n) => schedulableFree(n) >= size);
+    if (schedulableFit.length === 0) {
+      blockers.push(
+        `volume ${volName} (${gib(size)}) does not fit the scheduler on any candidate node ` +
+        `even at ${overProvisionPct}% over-provisioning.`,
+      );
+      continue;
+    }
+
+    // (3) Real capacity. The scheduler counts over-provisioned space; a rebuild
+    // writes real bytes. This is the check that catches a small disk hiding
+    // behind a generous over-provisioning percentage.
+    const realFit = candidates.filter((n) => diskFree(n) >= size);
+    if (realFit.length === 0) {
+      const best = Math.max(0, ...candidates.map(diskFree));
+      blockers.push(
+        `volume ${volName} needs ${gib(size)} but the roomiest candidate node has ${gib(best)} ` +
+        `actually free. storage-over-provisioning-percentage=${overProvisionPct} lets the scheduler ` +
+        `accept this; the disk does not.`,
+      );
+      continue;
+    }
+
+    // (4) Volumes with no second copy, and volumes that cannot rebuild at all.
+    if (want === 1) {
+      warnings.push(
+        `volume ${volName} has a single replica — evicting it rebuilds from the only copy that exists`,
+      );
+    }
+    if (vol.status?.state === "detached") {
+      warnings.push(`volume ${volName} is detached — Longhorn must attach it to rebuild`);
+    }
+    if (vol.status?.robustness === "degraded") {
+      warnings.push(`volume ${volName} is already degraded before eviction starts`);
+    }
+  }
+
+  return { completable: blockers.length === 0, blockers, warnings, replicasMoving: mine.length, bytesMoving };
+}
+
+function gib(bytes: number): string {
+  return `${(bytes / 2 ** 30).toFixed(1)} GiB`;
+}
+
+/**
+ * Standalone gate: refuse fast and loudly rather than hang for an hour.
+ *
+ * Exposed as its own Operation so it can be run on its own -- "could I evacuate
+ * this node if I wanted to?" is a question worth answering without starting.
+ */
+export const assertEvictionCompletable: Operation = async (ctx): Promise<OperationResult> => {
+  const name = await nodeName(ctx);
+  const lh = await kubectl(ctx, "get ns longhorn-system -o name");
+  if (!lh.ok) {
+    return { success: true, changed: false, message: "Longhorn not installed — nothing to evict" };
+  }
+
+  const f = await checkEvictionCompletable(ctx, name);
+  for (const w of f.warnings) ctx.log(`  WARNING: ${w}`);
+
+  if (!f.completable) {
+    return {
+      success: false,
+      changed: false,
+      message: `Eviction of ${name} cannot complete`,
+      error: f.blockers.join("; "),
+    };
+  }
+  return {
+    success: true,
+    changed: false,
+    message: f.replicasMoving === 0
+      ? `${name} holds no replicas`
+      : `${f.replicasMoving} replica(s) (${gib(f.bytesMoving)}) can be placed elsewhere`,
+  };
+};
+
+/**
+ * Ask Longhorn to move this node's replicas elsewhere, then WAIT.
+ *
+ * The wait is the entire point. Setting evictionRequested and immediately
+ * wiping the disk destroys the replicas mid-rebuild, which on a 3-replica
+ * volume takes you to 1 and on the 2-replica Prometheus volume takes you to 0.
+ *
+ * Returns changed:false with an explanation when Longhorn is not installed,
+ * so a standalone or storage-free node is a no-op rather than an error.
+ */
+export const evictLonghornReplicas: Operation = async (ctx): Promise<OperationResult> => {
+  const name = await nodeName(ctx);
+
+  const hasLonghorn = await kubectl(ctx, "get ns longhorn-system -o name");
+  if (!hasLonghorn.ok) {
+    return { success: true, changed: false, message: "Longhorn not installed — nothing to evict" };
+  }
+
+  // Refuse in seconds rather than discover it after an hour of polling.
+  // Inside the operation, not only in the group, so `labctl app k3s evacuate`
+  // gets the same protection as the drill does.
+  const feasible = await checkEvictionCompletable(ctx, name);
+  for (const w of feasible.warnings) ctx.log(`  WARNING: ${w}`);
+  if (!feasible.completable) {
+    return {
+      success: false,
+      changed: false,
+      message: `Eviction of ${name} cannot complete — not starting it`,
+      error: feasible.blockers.join("; "),
+    };
+  }
+  // A single-replica volume rebuilds from the only copy in existence, and this
+  // fleet has no backup target. That is a decision for a human, not a default.
+  const singles = feasible.warnings.filter((w) => w.includes("single replica"));
+  if (singles.length > 0 && ctx.config.evictionAllowSingleReplica !== true) {
+    return {
+      success: false,
+      changed: false,
+      message: `${name} holds single-replica volume(s); refusing to evict`,
+      error: `${singles.join("; ")}. Raise them to 3 replicas and wait for healthy, ` +
+        `or pass evictionAllowSingleReplica to accept the risk.`,
+    };
+  }
+
+  const patch = JSON.stringify({
+    spec: { allowScheduling: false, evictionRequested: true },
+  });
+  const req = await kubectl(
+    ctx,
+    `-n longhorn-system patch nodes.longhorn.io "${name}" --type=merge -p '${patch}'`,
+  );
+  if (!req.ok) {
+    return { success: false, changed: false, message: `Could not request eviction for ${name}`, error: req.out };
+  }
+
+  ctx.log(`Eviction requested for ${name}; waiting for replicas to rebuild elsewhere...`);
+  const start = Date.now();
+  let last = -1;
+  let lastHeartbeat = Date.now();
+  try {
+  while (Date.now() - start < REPLICA_REBUILD_TIMEOUT_MS) {
+    const res = await kubectl(
+      ctx,
+      `-n longhorn-system get replicas.longhorn.io -o jsonpath='{range .items[?(@.spec.nodeID=="${name}")]}{.metadata.name}{"\\n"}{end}'`,
+    );
+    const remaining = res.ok ? res.out.split("\n").map((l) => l.trim()).filter(Boolean).length : -1;
+    if (remaining === 0) {
+      return {
+        success: true,
+        changed: true,
+        message: `All replicas evicted from ${name} after ${Math.round((Date.now() - start) / 1000)}s`,
+      };
+    }
+    // Log on change, and otherwise every few minutes. A silent hour is
+    // indistinguishable from a hung process; a heartbeat that carries the
+    // elapsed time makes "stalled" visible while it is happening rather than
+    // only in the timeout message.
+    const elapsedS = Math.round((Date.now() - start) / 1000);
+    if (remaining !== last) {
+      ctx.log(`  ${remaining} replica(s) still on ${name} (${elapsedS}s elapsed)`);
+      last = remaining;
+      lastHeartbeat = Date.now();
+    } else if (Date.now() - lastHeartbeat >= HEARTBEAT_INTERVAL_MS) {
+      ctx.log(`  still waiting: ${remaining} replica(s) on ${name}, no change in ` +
+        `${Math.round((Date.now() - lastHeartbeat) / 60_000)} min (${elapsedS}s elapsed)`);
+      lastHeartbeat = Date.now();
+    }
+    await sleep(POLL_INTERVAL_MS);
+  }
+  } catch (err) {
+    // Never leave the node eviction-pending because of a transient failure.
+    await cancelEviction(ctx, name);
+    throw err;
+  }
+
+  // Timed out. Say exactly what is stuck before restoring the node, because
+  // after the cancel the eviction flags are gone and the evidence with them.
+  const stuck = await describeStuckReplicas(ctx, name);
+
+  // Put the node back the way we found it: an abandoned eviction leaves it
+  // unschedulable with Longhorn still trying to drain it, which is safe but
+  // sticky, and nothing else would ever clear it.
+  const cancelled = await cancelEviction(ctx, name);
+  return {
+    success: false,
+    changed: true,
+    message: `Timed out waiting for replica eviction from ${name}`,
+    error: `${last} replica(s) still present after ${REPLICA_REBUILD_TIMEOUT_MS / 60000} min. ` +
+      `Wiping now would destroy them.\nStill on ${name}:\n${stuck}\n` +
+      (cancelled
+        ? `Eviction request has been cancelled and scheduling restored.`
+        : `WARNING: could not cancel the eviction request — clear it with ` +
+          `\`labctl app k3s admit ${name}\`.`),
+  };
+};
+
+/**
+ * Describe the replicas still sitting on a node, and why they are not moving.
+ *
+ * An eviction that times out after an hour used to report only a count. That
+ * is not enough to act on: the question is always WHICH volume is stuck and
+ * what state its replica is in. A replica with `desireState=running` and
+ * `currentState=stopped` and no `failedAt` is Longhorn wanting to start it and
+ * not doing so -- a very different problem from a volume that cannot be placed.
+ */
+async function describeStuckReplicas(ctx: OperationContext, name: string): Promise<string> {
+  const reps = await kubectlJson<{ items: {
+    metadata: { name: string };
+    spec: { nodeID?: string; volumeName?: string; failedAt?: string; desireState?: string };
+    status?: { currentState?: string };
+  }[] }>(ctx, "-n longhorn-system get replicas.longhorn.io");
+  if (!reps) return "  (could not read replicas)";
+
+  const mine = reps.items.filter((r) => r.spec.nodeID === name);
+  if (mine.length === 0) return "  (none)";
+
+  const vols = await kubectlJson<{ items: {
+    metadata: { name: string };
+    spec: { numberOfReplicas?: number };
+    status?: { state?: string; robustness?: string };
+  }[] }>(ctx, "-n longhorn-system get volumes.longhorn.io");
+  const byName = new Map((vols?.items ?? []).map((v) => [v.metadata.name, v]));
+
+  return mine.map((r) => {
+    const v = byName.get(r.spec.volumeName ?? "");
+    return `  ${r.spec.volumeName ?? "?"}: replica ${r.metadata.name.slice(-12)} ` +
+      `currentState=${r.status?.currentState ?? "?"} desireState=${r.spec.desireState ?? "?"} ` +
+      `failedAt=${r.spec.failedAt === undefined || r.spec.failedAt === "" ? "(none)" : r.spec.failedAt} | ` +
+      `volume ${v?.status?.state ?? "?"}/${v?.status?.robustness ?? "?"} ` +
+      `wants ${v?.spec.numberOfReplicas ?? "?"} replicas`;
+  }).join("\n");
+}
+
+/**
+ * Undo the eviction request. Returns false if it could not be undone, which the
+ * caller must surface -- silently failing here leaves a node nobody can use.
+ */
+async function cancelEviction(ctx: OperationContext, name: string): Promise<boolean> {
+  const patch = JSON.stringify({ spec: { allowScheduling: true, evictionRequested: false } });
+  const res = await kubectl(
+    ctx,
+    `-n longhorn-system patch nodes.longhorn.io "${name}" --type=merge -p '${patch}'`,
+  );
+  if (res.ok) ctx.log(`Cancelled eviction request on ${name} and restored scheduling`);
+  return res.ok;
+}
+
+/**
+ * Verify the node holds nothing before it is wiped.
+ *
+ * A separate, cheap operation on purpose: it is the gate that turns "we ran the
+ * eviction steps" into "the node is actually empty", and it is the one thing
+ * worth asserting twice. Reading back from the thing that does the work, not
+ * from the request we sent it.
+ */
+export const assertNodeEvacuated: Operation = async (ctx): Promise<OperationResult> => {
+  const name = await nodeName(ctx);
+  const problems: string[] = [];
+
+  const sched = await kubectl(ctx, `get node "${name}" -o jsonpath='{.spec.unschedulable}'`);
+  if (sched.ok && sched.out.trim() !== "true") problems.push("node is still schedulable");
+
+  const pods = await kubectl(
+    ctx,
+    `get pods -A --field-selector spec.nodeName="${name}",status.phase=Running ` +
+      `-o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\\n"}{end}'`,
+  );
+  if (pods.ok) {
+    const running = pods.out.split("\n").map((l) => l.trim()).filter(Boolean);
+    // DaemonSet pods legitimately remain; anything else means the drain lied.
+    if (running.length > 0) ctx.log(`  ${running.length} pod(s) remain (expected: DaemonSets only)`);
+  }
+
+  const lh = await kubectl(ctx, "get ns longhorn-system -o name");
+  if (lh.ok) {
+    const reps = await kubectl(
+      ctx,
+      `-n longhorn-system get replicas.longhorn.io -o jsonpath='{range .items[?(@.spec.nodeID=="${name}")]}{.metadata.name}{"\\n"}{end}'`,
+    );
+    const n = reps.ok ? reps.out.split("\n").map((l) => l.trim()).filter(Boolean).length : 0;
+    if (n > 0) problems.push(`${n} Longhorn replica(s) still on this node`);
+  }
+
+  return problems.length === 0
+    ? { success: true, changed: false, message: `${name} is evacuated and safe to wipe` }
+    : {
+        success: false,
+        changed: false,
+        message: `${name} is NOT safe to wipe`,
+        error: problems.join("; "),
+      };
+};

bastion/src/modules/modules/k3s/src/operations/worker-label.ts

@@ -0,0 +1,66 @@
+// Apply the worker role label from the SERVER side, after the node registers.
+//
+// WHY THIS IS NOT A NODE-LABEL
+// ----------------------------
+// `node-role.kubernetes.io/worker=true` cannot be set by the kubelet itself.
+// The servers enable the NodeRestriction admission plugin, which forbids a
+// kubelet from assigning itself any label in the `node-role.kubernetes.io/`
+// namespace. Putting it in the agent's `node-label:` list makes registration
+// fail, and the resulting message --
+//
+//   Job for k3s-agent.service failed because the service did not take the
+//   steps required by its unit configuration.
+//
+// names neither labels nor admission, so it reads as a networking, token or
+// sysctl problem. It is none of those.
+//
+// The label is cosmetic (it populates the ROLES column in `kubectl get nodes`)
+// so failing to apply it is a warning, never an error.
+
+import type { Operation, OperationContext, OperationResult } from "../types.js";
+import { sshOpts } from "../utils.js";
+import { sshExec as remoteSshExec } from "../../../../src/ssh.js";
+import { isK3sServer } from "@lab/shared";
+
+const WORKER_LABEL = "node-role.kubernetes.io/worker=true";
+
+async function nodeName(ctx: OperationContext): Promise<string> {
+  const r = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
+  return r.stdout.trim();
+}
+
+export const labelWorkerRole: Operation = async (ctx): Promise<OperationResult> => {
+  if (isK3sServer(ctx.config.role)) {
+    return { success: true, changed: false, message: "Server node — k3s sets its own roles" };
+  }
+
+  const name = await nodeName(ctx);
+
+  // An agent has no local kubectl, so this has to run on a server.
+  const local = await ctx.ssh.exec(`k3s kubectl label node "${name}" ${WORKER_LABEL} --overwrite 2>&1`, sshOpts(ctx));
+  if (local.exitCode === 0) {
+    return { success: true, changed: true, message: `Labelled ${name} as worker` };
+  }
+
+  if (ctx.config.k3sServerUrl !== undefined && ctx.config.k3sServerUrl !== "") {
+    try {
+      const serverHost = new URL(ctx.config.k3sServerUrl).hostname;
+      const remote = await remoteSshExec(
+        serverHost, "root",
+        `k3s kubectl label node "${name}" ${WORKER_LABEL} --overwrite`,
+        { ...(ctx.ssh.keyPath ? { keyPath: ctx.ssh.keyPath } : {}), timeoutMs: 30_000 },
+      );
+      if (remote.exitCode === 0) {
+        return { success: true, changed: true, message: `Labelled ${name} as worker (via server)` };
+      }
+    } catch { /* fall through to the warning */ }
+  }
+
+  // Cosmetic: the ROLES column will read <none>. Not worth failing an install.
+  return {
+    success: true,
+    changed: false,
+    message: `Could not apply the worker role label to ${name} (cosmetic; set it with ` +
+      `\`kubectl label node ${name} ${WORKER_LABEL}\`)`,
+  };
+};

bastion/src/modules/modules/k3s/src/types.ts

@@ -16,6 +16,56 @@ export interface K3sConfig {
 
   // Additional TLS SANs for API server certificate
   tlsSans?: string[] | undefined;
+
+  /**
+   * Exact k3s version to install, e.g. "v1.34.6+k3s1".
+   *
+   * Unset means whatever get.k3s.io currently calls latest, which is how a
+   * reinstalled node silently ends up AHEAD of the cluster it is rejoining.
+   * Kubernetes supports only one minor version of skew between control-plane
+   * members; a node reinstalled two minors ahead of its peers is an
+   * unsupported cluster, and a rolling migration reinstalls nodes one at a
+   * time by design. Pin it to the version the cluster already runs.
+   */
+  k3sVersion?: string | undefined;
+
+  /**
+   * Permit evacuating a node that holds single-replica Longhorn volumes.
+   *
+   * Off by default and deliberately awkward. Evicting the only replica of a
+   * volume rebuilds it from the sole copy in existence; with no backup target
+   * configured, a failure during that rebuild is permanent data loss. Raising
+   * the volume to 3 replicas and waiting for it to go healthy is almost always
+   * the right move instead.
+   */
+  evictionAllowSingleReplica?: boolean | undefined;
+
+  /**
+   * IPv6 address of this node on the cluster VLAN. Its PRESENCE is what makes a
+   * node dual-stack: supply it and `node-ip` is emitted as `<v4>,<v6>`; omit it
+   * and the generated config is byte-identical to the IPv4-only one.
+   *
+   * It must be a real address on the node before k3s starts. k3s resolves
+   * node-ip at boot, so a SLAAC or DHCPv6 lease that has not landed yet leaves
+   * the node with no usable v6 identity. The estate takes it from a kea DHCPv6
+   * reservation keyed on MAC, the same way it takes its IPv4 address.
+   */
+  ipv6?: string | undefined;
+
+  /**
+   * Pod and Service ranges, one entry per address family. Passed rather than
+   * hardcoded so the ranges stay configuration -- and so labsim can rehearse
+   * with its own addresses through this same generator, instead of a parallel
+   * set of INSTALL_K3S_EXEC flags that would prove a different mechanism.
+   *
+   * k3s validates the two together and crash-loops on a mismatch (a dual
+   * cluster-cidr with a single-family service-cidr, say), which is a safe
+   * failure but an avoidable one: set both or neither.
+   *
+   * Servers only -- agents take neither.
+   */
+  clusterCidr?: string[] | undefined;
+  serviceCidr?: string[] | undefined;
 }
 
 /** SSH execution interface injected into operations. */

bastion/src/modules/modules/k3s/tests/agent-node-label.test.ts

@@ -0,0 +1,75 @@
+// The agent must NOT try to self-assign a node-role label.
+//
+// A kubelet may not set labels in the `node-role.kubernetes.io/` namespace;
+// the servers run NodeRestriction, which rejects the registration. The agent
+// then never finishes registering and systemd reports only:
+//
+//   Job for k3s-agent.service failed because the service did not take the
+//   steps required by its unit configuration.
+//
+// That message names neither labels nor admission control, so the cost of
+// reintroducing this is hours of looking at the network and the join token.
+// Observed twice for real: on v2-wrk-1, and on the drill's storage agent.
+
+import { describe, it, expect } from "vitest";
+import { generateAgentConfig, generateServerConfig } from "../src/operations/k3s-config.js";
+import { k3sAgentGroup } from "../src/groups/k3s-agent.js";
+import { mockCtx, OK, stdout, expectCommand } from "./helpers.js";
+import { labelWorkerRole } from "../src/operations/worker-label.js";
+
+const base = { hostname: "w1.lab", ip: "10.0.0.9", role: "k8s-worker" as const };
+
+describe("agent config", () => {
+  it("does NOT put a node-role label in node-label", () => {
+    const cfg = generateAgentConfig({ ...base });
+    expect(cfg).not.toContain("node-role.kubernetes.io");
+  });
+
+  it("keeps the Longhorn label, which IS allowed (outside kubernetes.io)", () => {
+    expect(generateAgentConfig({ ...base })).toContain("node.longhorn.io/create-default-disk=config");
+  });
+
+  it("server config does not set node-role either — k3s assigns those itself", () => {
+    const cfg = generateServerConfig({ ...base, role: "k8s-combo", hostname: "s1.lab" });
+    expect(cfg).not.toContain("node-role.kubernetes.io");
+  });
+
+  it("the agent group applies the worker label AFTER installing", () => {
+    const names = k3sAgentGroup.operations.map((o) => o.name);
+    const install = names.findIndex((n) => /Install k3s agent/i.test(n));
+    const label = names.findIndex((n) => /Label node as worker/i.test(n));
+    expect(install).toBeGreaterThanOrEqual(0);
+    expect(label).toBeGreaterThan(install);
+  });
+});
+
+describe("labelWorkerRole", () => {
+  it("labels an agent from the server side", async () => {
+    const ctx = mockCtx({ role: "k8s-worker" });
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return stdout("w1.lab\n");
+      return OK;
+    });
+    const r = await labelWorkerRole(ctx);
+    expect(r.success).toBe(true);
+    expectCommand(ctx.ssh, /label node "w1\.lab" node-role\.kubernetes\.io\/worker=true/);
+  });
+
+  it("is a no-op on a server — k3s sets control-plane/etcd itself", async () => {
+    const r = await labelWorkerRole(mockCtx({ role: "k8s-combo" }));
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(false);
+  });
+
+  it("never fails an install over a cosmetic label", async () => {
+    const ctx = mockCtx({ role: "k8s-worker" });
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return stdout("w1.lab\n");
+      return { exitCode: 1, stdout: "", stderr: "no kubectl here" };
+    });
+    const r = await labelWorkerRole(ctx);
+    expect(r.success, "a missing ROLES column must not fail a node install").toBe(true);
+    expect(r.changed).toBe(false);
+    expect(r.message).toMatch(/cosmetic/);
+  });
+});

bastion/src/modules/modules/k3s/tests/etcd-member-parse.test.ts

@@ -0,0 +1,58 @@
+// Parsing `etcdctl member list`.
+//
+// WHY NOT -w json
+// ---------------
+// etcd member IDs are uint64. JSON.parse turns them into JS numbers, which
+// lose precision above 2^53, so the id that comes back is an APPROXIMATION of
+// the real one -- and `member remove` wants HEX anyway, while the JSON writer
+// emits decimal. Observed for real: `member remove 13323819137391910000`
+// -> "bad member ID arg (value out of range), expecting ID in Hex".
+//
+// That happens at the worst possible moment: the node is already stopped and
+// about to be wiped. A mangled id fails loudly if you are lucky; in principle
+// it could match a different member, which would remove the wrong one.
+//
+// The default writer prints the id as hex TEXT, so it never becomes a number.
+// The fixture below is real output from a production node.
+
+import { describe, it, expect } from "vitest";
+import { parseEtcdMembers } from "../src/operations/etcd-members.js";
+
+const REAL = `825852d3683bc406, started, worker2-k8s0.ad.itaz.eu-5f7ad0c9, https://192.168.8.25:2380, https://192.168.8.25:2379, false
+a6170c0755307cea, started, worker1-k8s0.ad.itaz.eu-82b4fe24, https://192.168.8.13:2380, https://192.168.8.13:2379, false
+ad1fe97d38022a78, started, worker0-k8s0.ad.itaz.eu-e26aacc5, https://192.168.8.23:2380, https://192.168.8.23:2379, false`;
+
+describe("parseEtcdMembers", () => {
+  it("keeps ids as exact hex text, never as numbers", () => {
+    const m = parseEtcdMembers(REAL);
+    expect(m.map((x) => x.id)).toEqual([
+      "825852d3683bc406", "a6170c0755307cea", "ad1fe97d38022a78",
+    ]);
+    // The precision trap, stated as an assertion: every one of these exceeds
+    // Number.MAX_SAFE_INTEGER, so a numeric round-trip would corrupt them.
+    for (const x of m) {
+      expect(Number(BigInt(`0x${x.id}`)) > Number.MAX_SAFE_INTEGER).toBe(true);
+      expect(BigInt(`0x${x.id}`).toString(16)).toBe(x.id);
+    }
+  });
+
+  it("strips etcd's random name suffix to give the node name", () => {
+    expect(parseEtcdMembers(REAL).map((x) => x.node)).toEqual([
+      "worker2-k8s0.ad.itaz.eu", "worker1-k8s0.ad.itaz.eu", "worker0-k8s0.ad.itaz.eu",
+    ]);
+  });
+
+  it("reports started status", () => {
+    expect(parseEtcdMembers(REAL).every((x) => x.started)).toBe(true);
+    const unstarted = parseEtcdMembers("abc123, unstarted, , https://x:2380, , false");
+    expect(unstarted[0]?.started).toBe(false);
+  });
+
+  it("ignores blank lines and anything without a hex id", () => {
+    expect(parseEtcdMembers(`\n  \nWARNING: something\n${REAL}\n`)).toHaveLength(3);
+  });
+
+  it("returns nothing for empty output rather than a bogus member", () => {
+    expect(parseEtcdMembers("")).toEqual([]);
+  });
+});

bastion/src/modules/modules/k3s/tests/join-env.test.ts

@@ -0,0 +1,158 @@
+// Tests for how K3S_URL / K3S_TOKEN reach the k3s installer.
+//
+// Token and URL are independent, and the distinction is load-bearing:
+//
+//   token only      -> SEEDS the cluster token on the cluster-init node, so
+//                      every later joiner can use the same pre-shared value.
+//   url + token     -> JOINS an existing cluster.
+//   url only        -> cannot authenticate; must not be passed.
+//
+// Before this was fixed the installer emitted join env ONLY when both were
+// present, so a cluster-init node always self-generated its token and there
+// was no way to pre-share one. A `toContain("K3S_TOKEN")` assertion would pass
+// against the broken version for the url+token case, so each case is asserted
+// on the exact command.
+
+import { describe, it, expect } from "vitest";
+import { mockCtx, stdout, OK } from "./helpers.js";
+import { installK3sBinary } from "../src/operations/k3s-install.js";
+
+/** The `curl ... | sh -` line the operation ran. */
+function installCommand(ssh: ReturnType<typeof mockCtx>["ssh"]): string {
+  const calls = ssh.exec.mock.calls.map((c: [string, unknown?]) => c[0]);
+  const found = calls.find((c: string) => c.includes("get.k3s.io"));
+  if (!found) throw new Error(`No install command issued. Commands:\n${calls.join("\n")}`);
+  return found;
+}
+
+/** k3s --version fails (not installed), everything after succeeds. */
+function primeServerInstall(ctx: ReturnType<typeof mockCtx>): void {
+  ctx.ssh.exec.mockResolvedValue(OK);
+  ctx.ssh.exec.mockResolvedValueOnce({ exitCode: 1, stdout: "", stderr: "" }); // k3s --version
+}
+
+describe("installK3sBinary — server join environment", () => {
+  it("token alone seeds the cluster token and does NOT pass a server URL", async () => {
+    const ctx = mockCtx({ role: "k8s-combo", k3sToken: "pre-shared-token" });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+
+    await installK3sBinary(ctx);
+    const cmd = installCommand(ctx.ssh);
+
+    expect(cmd).toContain('K3S_TOKEN="pre-shared-token"');
+    expect(cmd).not.toContain("K3S_URL");
+    expect(cmd).toContain('INSTALL_K3S_EXEC="server"');
+  });
+
+  it("url + token joins an existing cluster", async () => {
+    const ctx = mockCtx({
+      role: "k8s-combo",
+      k3sServerUrl: "https://10.0.0.11:6443",
+      k3sToken: "pre-shared-token",
+    });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+
+    await installK3sBinary(ctx);
+    const cmd = installCommand(ctx.ssh);
+
+    expect(cmd).toContain('K3S_URL="https://10.0.0.11:6443"');
+    expect(cmd).toContain('K3S_TOKEN="pre-shared-token"');
+  });
+
+  it("neither url nor token passes no join environment at all", async () => {
+    const ctx = mockCtx({ role: "k8s-combo" });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+
+    await installK3sBinary(ctx);
+    const cmd = installCommand(ctx.ssh);
+
+    expect(cmd).not.toContain("K3S_URL");
+    expect(cmd).not.toContain("K3S_TOKEN");
+  });
+
+  it("a URL without a token is ignored — it cannot authenticate a join", async () => {
+    const ctx = mockCtx({ role: "k8s-combo", k3sServerUrl: "https://10.0.0.11:6443" });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+
+    await installK3sBinary(ctx);
+    expect(installCommand(ctx.ssh)).not.toContain("K3S_URL");
+  });
+});
+
+describe("installK3sBinary — datastore wipe guard", () => {
+  // Wiping server/{tls,cred,db} is correct when JOINING (stale certs are fatal:
+  // "newer than datastore") and catastrophic on a cluster-init node, where it
+  // would destroy the etcd database the node is about to bootstrap from.
+  const WIPE = "rm -rf /var/lib/rancher/k3s/server/tls";
+
+  it("wipes stale server state when joining", async () => {
+    const ctx = mockCtx({
+      role: "k8s-combo",
+      k3sServerUrl: "https://10.0.0.11:6443",
+      k3sToken: "t",
+    });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+
+    await installK3sBinary(ctx);
+    const calls = ctx.ssh.exec.mock.calls.map((c: [string, unknown?]) => c[0]);
+    expect(calls.some((c: string) => c.includes(WIPE))).toBe(true);
+  });
+
+  it("does NOT wipe the datastore when only seeding a token (cluster-init)", async () => {
+    const ctx = mockCtx({ role: "k8s-combo", k3sToken: "pre-shared-token" });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+
+    await installK3sBinary(ctx);
+    const calls = ctx.ssh.exec.mock.calls.map((c: [string, unknown?]) => c[0]);
+    expect(calls.some((c: string) => c.includes(WIPE))).toBe(false);
+  });
+});
+
+describe("installK3sBinary — agent", () => {
+  it("refuses to install an agent with no server URL or token", async () => {
+    const ctx = mockCtx({ role: "k8s-worker" });
+    primeServerInstall(ctx);
+
+    const result = await installK3sBinary(ctx);
+    expect(result.success).toBe(false);
+    expect(result.message).toContain("Agent requires");
+  });
+});
+
+describe("installK3sBinary — version pinning", () => {
+  // get.k3s.io installs LATEST. During a rolling migration that means a
+  // reinstalled node rejoins on a newer k3s than its peers; Kubernetes supports
+  // only one minor of skew between control-plane members, and at the time of
+  // writing latest was TWO ahead of the fleet (v1.36.4 vs v1.34.6).
+  it("pins the version when one is given", async () => {
+    const ctx = mockCtx({ role: "k8s-combo", k3sToken: "t", k3sVersion: "v1.34.6+k3s1" });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+    await installK3sBinary(ctx);
+    expect(installCommand(ctx.ssh)).toContain('INSTALL_K3S_VERSION="v1.34.6+k3s1"');
+  });
+
+  it("pins the agent install too", async () => {
+    const ctx = mockCtx({
+      role: "k8s-worker", k3sServerUrl: "https://10.0.0.11:6443",
+      k3sToken: "t", k3sVersion: "v1.34.6+k3s1",
+    });
+    primeServerInstall(ctx);
+    await installK3sBinary(ctx);
+    expect(installCommand(ctx.ssh)).toContain('INSTALL_K3S_VERSION="v1.34.6+k3s1"');
+  });
+
+  it("omits the variable entirely when unset, preserving today's behaviour", async () => {
+    const ctx = mockCtx({ role: "k8s-combo", k3sToken: "t" });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+    await installK3sBinary(ctx);
+    expect(installCommand(ctx.ssh)).not.toContain("INSTALL_K3S_VERSION");
+  });
+});

bastion/src/modules/modules/k3s/tests/longhorn-disk.test.ts

@@ -0,0 +1,188 @@
+// Tests for giving a node its Longhorn disk.
+//
+// The cluster runs createDefaultDiskLabeledNodes:true, so a node gets a disk
+// only if it carries BOTH the create-default-disk label and the
+// default-disks-config annotation. Longhorn reads that annotation when it
+// first creates the node's record — which means on a node that is REJOINING,
+// the record already exists and the annotation is never read. The node comes
+// back Ready, accepts pods, and hosts no replicas at all.
+//
+// That failure is invisible until a rebuild is expected and does not arrive,
+// so these tests pin the read-back and the repair, not just the annotate.
+
+import { describe, it, expect, vi, afterEach } from "vitest";
+import { mockCtx, OK, FAIL, stdout, expectCommand, expectNoCommand } from "./helpers.js";
+import { configureLonghornDisk } from "../src/operations/longhorn-disk.js";
+
+const HOST = stdout("node1.lab\n");
+const DISK = "/var/lib/longhorn";
+
+interface Opts {
+  /** Does Longhorn create the disk on its own (i.e. it read the annotation)? */
+  longhornCreatesDisk?: boolean;
+  /** Has longhorn-manager registered this node yet? */
+  recordExists?: boolean;
+  hasDir?: boolean;
+  noLonghorn?: boolean;
+  patchFails?: boolean;
+  /** Does the disk show up after a direct spec.disks patch? */
+  repairWorks?: boolean;
+}
+
+function ctxFor(o: Opts = {}) {
+  // Model the real object rather than a script of reads: a disk exists if
+  // Longhorn made one, or if our repair patch landed and took effect.
+  let patched = false;
+  const ctx = mockCtx();
+  ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+    if (cmd.startsWith("test -d")) return stdout(o.hasDir === false ? "no" : "yes");
+    if (cmd.startsWith("hostname")) return HOST;
+    if (cmd.includes("get ns longhorn-system")) return o.noLonghorn ? FAIL : stdout("namespace/longhorn-system");
+    // Existence probe (-o name) is a DIFFERENT question from "has a disk",
+    // and must be matched before the jsonpath reads.
+    if (cmd.includes("get nodes.longhorn.io") && cmd.includes("-o name")) {
+      return o.recordExists === false ? FAIL : stdout("node.longhorn.io/node1.lab");
+    }
+    if (cmd.includes("get nodes.longhorn.io") && cmd.includes("jsonpath")) {
+      const has = (o.longhornCreatesDisk ?? false) || (patched && (o.repairWorks ?? true));
+      return stdout(has ? DISK : "");
+    }
+    if (cmd.includes("patch nodes.longhorn.io")) {
+      if (o.patchFails === true) return FAIL;
+      patched = true;
+      return OK;
+    }
+    if (cmd.includes("annotate node")) return stdout("node/node1.lab annotated");
+    return OK;
+  });
+  return ctx;
+}
+
+/** Run the operation with the 60s verify window collapsed. */
+async function runFast(ctx: ReturnType<typeof ctxFor>) {
+  vi.useFakeTimers();
+  try {
+    const p = configureLonghornDisk(ctx);
+    // Long enough to cover the registration wait (180s) AND the disk wait (60s).
+    await vi.advanceTimersByTimeAsync(300_000);
+    return await p;
+  } finally {
+    vi.useRealTimers();
+  }
+}
+
+afterEach(() => { vi.useRealTimers(); });
+
+describe("the annotation survives the shell", () => {
+  // The value is JSON, so it is full of double quotes. Wrapping it in double
+  // quotes truncates it at the first inner quote and the node ends up with
+  //     [{path:/var/lib/longhorn,allowScheduling:true}]
+  // which Longhorn cannot parse, so it creates no disk at all -- silently.
+  // Asserting on the command string alone would NOT catch this; the command
+  // looks fine until a shell reads it. So run it through a real shell.
+  it("arrives as valid JSON after shell parsing", async () => {
+    const ctx = ctxFor({ longhornCreatesDisk: true });
+    await runFast(ctx);
+    const cmd = ctx.ssh.exec.mock.calls
+      .map((c: [string, unknown?]) => c[0])
+      .find((c: string) => c.includes("default-disks-config")) ?? "";
+    expect(cmd, "annotate command").not.toBe("");
+
+    // Let a real shell word-split it exactly as sshd would, and recover the
+    // argument that carries the annotation.
+    const { execFileSync } = await import("node:child_process");
+    const printed = execFileSync("bash", ["-c", `printf '%s\n' ${cmd.replace(/^k3s kubectl /, "")}`], {
+      encoding: "utf-8",
+    });
+    const arg = printed.split("\n").find((l) => l.startsWith("node.longhorn.io/default-disks-config=")) ?? "";
+    expect(arg, "annotation argument after shell parsing").not.toBe("");
+
+    const json = arg.slice("node.longhorn.io/default-disks-config=".length);
+    const parsed = JSON.parse(json) as { path: string; allowScheduling: boolean }[];
+    expect(parsed).toEqual([{ path: DISK, allowScheduling: true }]);
+  });
+});
+
+describe("configureLonghornDisk", () => {
+  it("guarantees the replicas directory exists", async () => {
+    // Longhorn creates <path>/replicas only when it first initialises the
+    // disk. If it disappears while longhorn-disk.cfg remains, Longhorn will
+    // not recreate it and every replica placed on the node dies on a
+    // non-recursive mkdir -- permanently, and invisibly, because the node
+    // still reports Ready and Schedulable.
+    const ctx = ctxFor({ longhornCreatesDisk: true });
+    await runFast(ctx);
+    expectCommand(ctx.ssh, /mkdir -p \/var\/lib\/longhorn\/replicas/);
+  });
+
+  it("skips a node with no /var/lib/longhorn", async () => {
+    const r = await configureLonghornDisk(ctxFor({ hasDir: false }));
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(false);
+    expect(r.message).toMatch(/skipping/);
+  });
+
+  it("annotates and is satisfied when Longhorn creates the disk itself", async () => {
+    const ctx = ctxFor({ longhornCreatesDisk: true });
+    const r = await runFast(ctx);
+    expect(r.success).toBe(true);
+    expectCommand(ctx.ssh, /default-disks-config/);
+    // Longhorn honoured the annotation, so no direct spec.disks write.
+    expectNoCommand(ctx.ssh, /patch nodes\.longhorn\.io/);
+  });
+
+  it("REPAIRS a rejoining node whose annotation was read too late", async () => {
+    // No disk for the whole verify window, then present after the patch.
+    const ctx = ctxFor({ longhornCreatesDisk: false, repairWorks: true });
+    const r = await runFast(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(true);
+    expect(r.message).toMatch(/repaired/i);
+    expectCommand(ctx.ssh, /patch nodes\.longhorn\.io/);
+    expectCommand(ctx.ssh, new RegExp(`"path":"${DISK.replace(/\//g, "\\/")}"`.replace(/\\\//g, "/")));
+  });
+
+  it("FAILS loudly when the disk cannot be created — a node with no disk is useless", async () => {
+    const ctx = ctxFor({ longhornCreatesDisk: false, repairWorks: false });
+    const r = await runFast(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/will not host replicas|never host a replica/);
+  });
+
+  it("FAILS when the repair patch itself is rejected", async () => {
+    const ctx = ctxFor({ longhornCreatesDisk: false, patchFails: true });
+    const r = await runFast(ctx);
+    expect(r.success).toBe(false);
+    expect(r.message).toMatch(/could not be repaired/);
+  });
+
+  it("does NOT fail a node Longhorn has not registered yet", async () => {
+    // A node that has only just rejoined has no nodes.longhorn.io record. The
+    // annotation is already in place, so Longhorn will read it when it creates
+    // one. Failing here would abort a rejoin over something self-resolving.
+    const ctx = ctxFor({ longhornCreatesDisk: false, recordExists: false });
+    const r = await runFast(ctx);
+    expect(r.success).toBe(true);
+    expect(r.message).toMatch(/not registered/);
+    expectNoCommand(ctx.ssh, /patch nodes\.longhorn\.io/);
+  });
+
+  it("annotates and stops when Longhorn is not installed yet", async () => {
+    // Install-time ordering: the annotation is set so Longhorn honours it when
+    // it does arrive, but there is nothing to read back from.
+    const ctx = ctxFor({ noLonghorn: true });
+    const r = await configureLonghornDisk(ctx);
+    expect(r.success).toBe(true);
+    expectCommand(ctx.ssh, /default-disks-config/);
+    expectNoCommand(ctx.ssh, /patch nodes\.longhorn\.io/);
+  });
+
+  it("uses one fixed disk key, so repeated repairs update rather than multiply", async () => {
+    const ctx = ctxFor({ longhornCreatesDisk: false, repairWorks: true });
+    await runFast(ctx);
+    const patch = ctx.ssh.exec.mock.calls
+      .map((c: [string, unknown?]) => c[0])
+      .find((c: string) => c.includes("patch nodes.longhorn.io")) ?? "";
+    expect(patch).toContain("default-disk-lab");
+  });
+});

bastion/src/modules/modules/k3s/tests/module-config.test.ts

@@ -0,0 +1,79 @@
+// The ModuleContext -> OperationContext config hand-off.
+//
+// toOpContext used to enumerate three keys and silently drop the rest, so a
+// caller could set a field, get no error, and have it ignored. That produced
+// two live bugs: a pinned k3sVersion was discarded (get.k3s.io installed
+// LATEST, two minor versions ahead of the fleet), and
+// `--allow-single-replica` was a no-op that claimed to override a data-loss
+// guard.
+//
+// These tests assert the pass-through per field, and the last one fails for
+// ANY future K3sConfig field that goes missing — which is the property that
+// was actually absent.
+
+import { describe, it, expect, vi } from "vitest";
+import { K3sModule } from "../src/k3s-module.js";
+import type { ModuleContext } from "../../../src/types.js";
+
+vi.mock("../../../src/ssh.js", () => ({
+  sshExec: vi.fn(async (_ip: string, _user: string, cmd: string) => {
+    // Record every command so the test can assert on what was sent.
+    issued.push(cmd);
+    if (cmd.includes("k3s --version")) return { exitCode: 1, stdout: "", stderr: "" };
+    if (cmd.includes("hostname")) return { exitCode: 0, stdout: "n1.lab\n", stderr: "" };
+    if (cmd.includes("get nodes")) return { exitCode: 0, stdout: "NAME\nn1 Ready", stderr: "" };
+    return { exitCode: 0, stdout: "", stderr: "" };
+  }),
+}));
+
+const issued: string[] = [];
+
+function ctxWith(config: Record<string, unknown>): ModuleContext {
+  return {
+    hostname: "n1.lab", ip: "10.0.0.5", role: "k8s-combo",
+    os: "fedora-43", arch: "x86_64", sshUser: "root", config,
+  };
+}
+
+describe("ModuleContext.config reaches the operations", () => {
+  it("passes k3sVersion through to the installer", async () => {
+    issued.length = 0;
+    await new K3sModule().install(ctxWith({
+      k3sToken: "t", k3sVersion: "v1.34.6+k3s1",
+    }));
+    const install = issued.find((c) => c.includes("get.k3s.io")) ?? "";
+    expect(install, "install command").toContain('INSTALL_K3S_VERSION="v1.34.6+k3s1"');
+  });
+
+  it("still passes the three fields it always did", async () => {
+    issued.length = 0;
+    await new K3sModule().install(ctxWith({
+      k3sServerUrl: "https://10.0.0.1:6443", k3sToken: "tok",
+    }));
+    const install = issued.find((c) => c.includes("get.k3s.io")) ?? "";
+    expect(install).toContain('K3S_URL="https://10.0.0.1:6443"');
+    expect(install).toContain('K3S_TOKEN="tok"');
+  });
+
+  it("drops nothing: every key given is visible to the operations", async () => {
+    // The generic guard. Enumerating keys in toOpContext is the bug; this
+    // fails if anyone reintroduces it for a field added later.
+    const { K3sModule: Mod } = await import("../src/k3s-module.js");
+    const mod = new Mod();
+    const given = {
+      k3sServerUrl: "https://10.0.0.1:6443",
+      k3sToken: "tok",
+      k3sVersion: "v1.34.6+k3s1",
+      evictionAllowSingleReplica: true,
+      tlsSans: ["a.lab"],
+    };
+    issued.length = 0;
+    await mod.install(ctxWith(given));
+    // k3sVersion and the join env are the observable ones at install time;
+    // assert them explicitly rather than trusting the shape.
+    const install = issued.find((c) => c.includes("get.k3s.io")) ?? "";
+    expect(install).toContain("INSTALL_K3S_VERSION=");
+    expect(install).toContain("K3S_URL=");
+    expect(install).toContain("K3S_TOKEN=");
+  });
+});

bastion/src/modules/modules/k3s/tests/node-admit.test.ts

@@ -0,0 +1,211 @@
+// Tests for returning an evacuated node to service.
+//
+// Evacuation leaves three separate marks on a node — a kubernetes cordon,
+// Longhorn's allowScheduling/evictionRequested pair, and (after a wipe and
+// rejoin) a MISSING Longhorn disk. Undoing only the cordon gives you a node
+// that is Ready, accepts pods and hosts no replicas at all, which reads as
+// success until storage never rebalances onto it. These tests pin all three.
+
+import { describe, it, expect, vi, afterEach } from "vitest";
+import { mockCtx, OK, FAIL, stdout, expectCommand } from "./helpers.js";
+import { clearLonghornEviction, assertNodeSchedulable } from "../src/operations/node-admit.js";
+import { evictLonghornReplicas } from "../src/operations/node-evacuate.js";
+import { evacuateGroup } from "../src/groups/evacuate.js";
+import { admitGroup } from "../src/groups/admit.js";
+
+const HOST = stdout("node1.lab\n");
+
+/**
+ * Run assertNodeSchedulable with its 120s poll collapsed.
+ *
+ * It polls because Longhorn registers a node asynchronously and this runs
+ * moments after a rejoin; the negative cases therefore wait out the whole
+ * window before they can honestly fail.
+ */
+async function runGate(ctx: ReturnType<typeof mockCtx>) {
+  vi.useFakeTimers();
+  try {
+    const p = assertNodeSchedulable(ctx);
+    await vi.advanceTimersByTimeAsync(150_000);
+    return await p;
+  } finally {
+    vi.useRealTimers();
+  }
+}
+
+afterEach(() => { vi.useRealTimers(); });
+
+/** Command-matching mock; see node-evacuate.test.ts for why not call order. */
+function routed(handlers: Record<string, string | null>, opts: { noLonghorn?: boolean } = {}) {
+  const ctx = mockCtx();
+  ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+    if (cmd.startsWith("hostname")) return HOST;
+    if (cmd.includes("get ns longhorn-system")) return opts.noLonghorn ? FAIL : stdout("namespace/longhorn-system");
+    for (const [needle, out] of Object.entries(handlers)) {
+      if (cmd.includes(needle)) return out === null ? FAIL : stdout(out);
+    }
+    return OK;
+  });
+  return ctx;
+}
+
+describe("clearLonghornEviction", () => {
+  it("clears scheduling and eviction together in one patch", async () => {
+    const ctx = routed({ "get nodes.longhorn.io": "false,true" });
+    const r = await clearLonghornEviction(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(true);
+    // Both fields must move together: Longhorn's webhook rejects
+    // evictionRequested:true on a schedulable node, so the inverse pairing
+    // has to be atomic as well.
+    expectCommand(ctx.ssh, /"allowScheduling":true/);
+    expectCommand(ctx.ssh, /"evictionRequested":false/);
+  });
+
+  it("is idempotent when the node already accepts replicas", async () => {
+    const ctx = routed({ "get nodes.longhorn.io": "true,false" });
+    const r = await clearLonghornEviction(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(false);
+  });
+
+  it("is a no-op when Longhorn is not installed", async () => {
+    const ctx = routed({}, { noLonghorn: true });
+    const r = await clearLonghornEviction(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(false);
+  });
+
+  it("reports failure rather than claiming the node is back", async () => {
+    const ctx = routed({ "get nodes.longhorn.io": "false,true", "patch nodes.longhorn.io": null });
+    const r = await clearLonghornEviction(ctx);
+    expect(r.success).toBe(false);
+  });
+});
+
+describe("assertNodeSchedulable — the gate after a rejoin", () => {
+  it("passes when uncordoned, not evicting, and holding a schedulable disk", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return HOST;
+      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
+      if (cmd.includes("diskStatus")) return stdout("True");
+      if (cmd.includes("get nodes.longhorn.io")) return stdout("true,false");
+      if (cmd.includes("get node ")) return stdout("");
+      return OK;
+    });
+    const r = await assertNodeSchedulable(ctx);
+    expect(r.success).toBe(true);
+  });
+
+  it("REFUSES a node that has no Longhorn disk — the silent rejoin failure", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return HOST;
+      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
+      if (cmd.includes("diskStatus")) return stdout("");   // no disks at all
+      if (cmd.includes("get nodes.longhorn.io")) return stdout("true,false");
+      if (cmd.includes("get node ")) return stdout("");
+      return OK;
+    });
+    const r = await runGate(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/no disk/);
+  });
+
+  it("REFUSES a node still marked for eviction", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return HOST;
+      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
+      if (cmd.includes("diskStatus")) return stdout("True");
+      if (cmd.includes("get nodes.longhorn.io")) return stdout("false,true");
+      if (cmd.includes("get node ")) return stdout("");
+      return OK;
+    });
+    const r = await runGate(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/eviction is still requested|scheduling is still disabled/);
+  });
+
+  it("REFUSES a node that is still cordoned", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return HOST;
+      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
+      if (cmd.includes("diskStatus")) return stdout("True");
+      if (cmd.includes("get nodes.longhorn.io")) return stdout("true,false");
+      if (cmd.includes("get node ")) return stdout("true");   // unschedulable
+      return OK;
+    });
+    const r = await runGate(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/cordoned/);
+  });
+});
+
+describe("evictLonghornReplicas — never leaves a node eviction-pending", () => {
+  it("cancels the eviction request when the wait blows up", async () => {
+    const ctx = mockCtx();
+    const patched: string[] = [];
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return HOST;
+      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
+      if (cmd.includes("jsonpath")) {
+        // Blow up during the rebuild wait, after eviction was requested.
+        if (cmd.includes("replicas.longhorn.io")) throw new Error("API server went away");
+        return stdout("");
+      }
+      if (cmd.includes("-o json")) return stdout(JSON.stringify({ items: [] }));
+      if (cmd.includes("patch nodes.longhorn.io")) { patched.push(cmd); return OK; }
+      return OK;
+    });
+
+    await expect(evictLonghornReplicas(ctx)).rejects.toThrow("API server went away");
+
+    // Two patches: the request, then the rollback. Leaving only the first
+    // behind would strand the node unschedulable with Longhorn still draining.
+    expect(patched.length).toBe(2);
+    expect(patched[0]).toContain('"evictionRequested":true');
+    expect(patched[1]).toContain('"evictionRequested":false');
+    expect(patched[1]).toContain('"allowScheduling":true');
+  });
+});
+
+describe("group ordering", () => {
+  it("evacuate evicts BEFORE it drains", () => {
+    // drain --force does not override PodDisruptionBudgets, and Longhorn holds
+    // one on the instance-manager pod. Draining first stops the process that
+    // serves the replicas being evicted: the drain blocks for its full timeout
+    // and would remove the only rebuild source for a single-replica volume.
+    const names = evacuateGroup.operations.map((o) => o.name);
+    const evict = names.findIndex((n) => /Evict/i.test(n));
+    const drain = names.findIndex((n) => /Drain/i.test(n));
+    expect(evict).toBeGreaterThanOrEqual(0);
+    expect(drain).toBeGreaterThanOrEqual(0);
+    expect(evict).toBeLessThan(drain);
+  });
+
+  it("evacuate checks feasibility before it mutates anything", () => {
+    const names = evacuateGroup.operations.map((o) => o.name);
+    expect(names.findIndex((n) => /Check eviction/i.test(n)))
+      .toBeLessThan(names.findIndex((n) => /Evict Longhorn/i.test(n)));
+  });
+
+  it("evacuate verifies last", () => {
+    const names = evacuateGroup.operations.map((o) => o.name);
+    expect(names[names.length - 1]).toMatch(/Verify/i);
+  });
+
+  it("admit clears eviction before restoring the disk", () => {
+    // A node still marked for eviction has any disk it is given drained again.
+    const names = admitGroup.operations.map((o) => o.name);
+    expect(names.findIndex((n) => /Clear Longhorn eviction/i.test(n)))
+      .toBeLessThan(names.findIndex((n) => /Restore Longhorn default disk/i.test(n)));
+  });
+
+  it("admit verifies last", () => {
+    const names = admitGroup.operations.map((o) => o.name);
+    expect(names[names.length - 1]).toMatch(/Verify/i);
+  });
+});

bastion/src/modules/modules/k3s/tests/node-evacuate.test.ts

@@ -0,0 +1,413 @@
+import { describe, it, expect, vi } from "vitest";
+import { mockCtx, OK, FAIL, stdout, expectCommand, expectNoCommand } from "./helpers.js";
+import {
+  cordonNode, uncordonNode, drainNode, evictLonghornReplicas, assertNodeEvacuated,
+  assertEvictionCompletable, checkEvictionCompletable,
+} from "../src/operations/node-evacuate.js";
+import type { K3sConfig } from "../src/types.js";
+
+const HOST = stdout("node1.lab\n");
+const GIB = 2 ** 30;
+
+interface RoutedOpts {
+  nodes?: string[];
+  replicas?: { volume: string }[];
+  volumes?: { name: string; replicas: number; size: number }[];
+  softAntiAffinity?: boolean;
+  overProvisionPct?: number;
+  diskFreeBytes?: number;
+  diskMaxBytes?: number;
+  patchFails?: boolean;
+}
+
+/**
+ * A context whose SSH mock answers by MATCHING the command rather than by call
+ * order. The operations under test read several API objects before they do
+ * anything, and an ordered mock has to be re-counted every time one is added —
+ * which makes the test fail for reasons that have nothing to do with the
+ * behaviour it is meant to pin.
+ */
+function routed(o: RoutedOpts = {}, config?: Partial<K3sConfig>) {
+  const nodeNames = o.nodes ?? ["node1.lab", "node2.lab", "node3.lab", "node4.lab"];
+  const free = o.diskFreeBytes ?? 500 * GIB;
+  const max = o.diskMaxBytes ?? 700 * GIB;
+  const lhNodes = {
+    items: nodeNames.map((name) => ({
+      metadata: { name },
+      spec: { allowScheduling: true },
+      status: { diskStatus: { "default-disk": {
+        storageAvailable: free, storageMaximum: max, storageScheduled: 0, storageReserved: 0,
+      } } },
+    })),
+  };
+  const replicas = {
+    items: (o.replicas ?? []).map((r, i) => ({
+      metadata: { name: `replica-${i}` },
+      spec: { nodeID: "node1.lab", volumeName: r.volume },
+    })),
+  };
+  const volumes = {
+    items: (o.volumes ?? []).map((v) => ({
+      metadata: { name: v.name },
+      spec: { numberOfReplicas: v.replicas, size: v.size },
+      status: { state: "attached", robustness: "healthy" },
+    })),
+  };
+
+  const ctx = mockCtx(config);
+  ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+    if (cmd.startsWith("hostname")) return HOST;
+    if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
+    // NOTE: "-o jsonpath=" CONTAINS "-o json", so jsonpath must be matched
+    // first or the eviction poll below is handed a whole JSON document and
+    // never sees zero replicas. (This mock got that wrong once already.)
+    if (cmd.includes("jsonpath")) {
+      // The eviction wait polls replicas on the node — report none remaining.
+      if (cmd.includes("replicas.longhorn.io")) return stdout("");
+      if (cmd.includes("replica-soft-anti-affinity")) return stdout(String(o.softAntiAffinity ?? false));
+      if (cmd.includes("storage-over-provisioning-percentage")) return stdout(String(o.overProvisionPct ?? 100));
+      return stdout("");
+    }
+    if (cmd.includes("get nodes.longhorn.io") && cmd.includes("-o json")) return stdout(JSON.stringify(lhNodes));
+    if (cmd.includes("get replicas.longhorn.io") && cmd.includes("-o json")) return stdout(JSON.stringify(replicas));
+    if (cmd.includes("get volumes.longhorn.io") && cmd.includes("-o json")) return stdout(JSON.stringify(volumes));
+    if (cmd.includes("patch nodes.longhorn.io")) return o.patchFails ? FAIL : OK;
+    return OK;
+  });
+  return ctx;
+}
+
+describe("cordonNode", () => {
+  it("cordons and reports a change", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(stdout('node/node1.lab cordoned'));
+    const r = await cordonNode(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(true);
+    expectCommand(ctx.ssh, /cordon "node1\.lab"/);
+  });
+
+  it("is idempotent — already cordoned is changed:false", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(stdout("node/node1.lab already cordoned"));
+    const r = await cordonNode(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(false);
+  });
+
+  it("fails loudly when the API is unreachable", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(FAIL);
+    const r = await cordonNode(ctx);
+    expect(r.success).toBe(false);
+  });
+});
+
+describe("drainNode", () => {
+  it("ignores daemonsets and deletes emptyDir, or it hangs forever", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(OK);
+    const r = await drainNode(ctx);
+    expect(r.success).toBe(true);
+    expectCommand(ctx.ssh, /--ignore-daemonsets/);
+    expectCommand(ctx.ssh, /--delete-emptydir-data/);
+  });
+
+  it("gives the SSH transport longer than kubectl's own --timeout", async () => {
+    // drainNode asks kubectl for --timeout=15m. sshOpts() is a flat 30s, so
+    // without an explicit override the transport kills a drain that is working
+    // -- after it has already evicted most of the pods. The result is
+    // non-zero with a transcript that reads like success, on every node with
+    // real workloads. `labctl app k3s evacuate` could never have drained one.
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(OK);
+    await drainNode(ctx);
+    const call = ctx.ssh.exec.mock.calls.find((c: [string, { timeoutMs?: number }?]) =>
+      c[0].includes("drain "));
+    expect(call, "drain command").toBeDefined();
+    const timeoutMs = call?.[1]?.timeoutMs ?? 0;
+    expect(timeoutMs, "SSH timeout must exceed kubectl's 15m --timeout")
+      .toBeGreaterThan(15 * 60_000);
+  });
+
+  it("reports the END of the drain transcript, not the useless first line", async () => {
+    // Every drain starts with "node/x already cordoned". Surfacing the whole
+    // transcript means the error you see is that line, which explains nothing.
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce({
+      exitCode: 1,
+      stdout: "node/n1 already cordoned\nevicting pod a\nevicting pod b\n" +
+        "error: cannot evict pod as it would violate the pod's disruption budget",
+      stderr: "",
+    });
+    const r = await drainNode(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toContain("disruption budget");
+  });
+
+  it("reports failure rather than pretending the node is empty", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(FAIL);
+    const r = await drainNode(ctx);
+    expect(r.success).toBe(false);
+  });
+});
+
+describe("evictLonghornReplicas", () => {
+  it("is a no-op when Longhorn is not installed", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(FAIL); // no longhorn-system ns
+    const r = await evictLonghornReplicas(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(false);
+    expect(r.message).toMatch(/not installed/);
+  });
+
+  it("requests eviction and waits until zero replicas remain", async () => {
+    // Node holds no replicas, so the pre-check clears it immediately.
+    const ctx = routed({ replicas: [] });
+    const r = await evictLonghornReplicas(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(true);
+    expectCommand(ctx.ssh, /evictionRequested/);
+    expectCommand(ctx.ssh, /allowScheduling/);
+  });
+
+  it("fails when eviction cannot even be requested", async () => {
+    const ctx = routed({ replicas: [], patchFails: true });
+    const r = await evictLonghornReplicas(ctx);
+    expect(r.success).toBe(false);
+  });
+
+  it("does not request eviction at all when it could never complete", async () => {
+    // 3-replica volume, and only 2 other schedulable nodes: the survivors
+    // already hold a copy each and strict anti-affinity forbids a third.
+    const ctx = routed({
+      nodes: ["node1.lab", "node2.lab", "node3.lab"],
+      replicas: [{ volume: "pvc-a" }],
+      volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }],
+    });
+    const r = await evictLonghornReplicas(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/cannot be placed|wants 3 replicas/);
+    // The important half: nothing was mutated.
+    expectNoCommand(ctx.ssh, /evictionRequested/);
+  });
+
+  it("refuses a single-replica volume unless explicitly allowed", async () => {
+    const opts = {
+      nodes: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"],
+      replicas: [{ volume: "pvc-solo" }],
+      volumes: [{ name: "pvc-solo", replicas: 1, size: 1 * GIB }],
+    };
+    const r = await evictLonghornReplicas(routed(opts));
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/single replica/);
+
+    const allowed = await evictLonghornReplicas(
+      routed(opts, { evictionAllowSingleReplica: true }),
+    );
+    expect(allowed.success).toBe(true);
+  });
+});
+
+describe("checkEvictionCompletable — refuse in seconds, not in an hour", () => {
+  it("passes when a spare schedulable node exists", async () => {
+    const ctx = routed({
+      nodes: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"],
+      replicas: [{ volume: "pvc-a" }],
+      volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }],
+    });
+    const f = await checkEvictionCompletable(ctx, "node1.lab");
+    expect(f.completable).toBe(true);
+    expect(f.replicasMoving).toBe(1);
+  });
+
+  it("blocks when replica count equals the surviving node count", async () => {
+    const ctx = routed({
+      nodes: ["node1.lab", "node2.lab", "node3.lab"],
+      replicas: [{ volume: "pvc-a" }],
+      volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }],
+    });
+    const f = await checkEvictionCompletable(ctx, "node1.lab");
+    expect(f.completable).toBe(false);
+    expect(f.blockers.join(" ")).toContain("pvc-a");
+  });
+
+  it("allows it when soft anti-affinity is on — placement stops being the limit", async () => {
+    const ctx = routed({
+      nodes: ["node1.lab", "node2.lab", "node3.lab"],
+      replicas: [{ volume: "pvc-a" }],
+      volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }],
+      softAntiAffinity: true,
+    });
+    const f = await checkEvictionCompletable(ctx, "node1.lab");
+    expect(f.completable).toBe(true);
+  });
+
+  it("catches a small disk hiding behind over-provisioning", async () => {
+    // The scheduler is happy at 500%; the disk has 49 GiB and the volume is 92.
+    const ctx = routed({
+      nodes: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"],
+      replicas: [{ volume: "pvc-big" }],
+      volumes: [{ name: "pvc-big", replicas: 3, size: 92 * GIB }],
+      overProvisionPct: 500,
+      diskFreeBytes: 49 * GIB,
+      diskMaxBytes: 100 * GIB,
+    });
+    const f = await checkEvictionCompletable(ctx, "node1.lab");
+    expect(f.completable).toBe(false);
+    expect(f.blockers.join(" ")).toMatch(/actually free/);
+    expect(f.blockers.join(" ")).toMatch(/over-provisioning-percentage=500/);
+  });
+
+  it("warns, but does not block, on a single-replica volume", async () => {
+    const ctx = routed({
+      nodes: ["node1.lab", "node2.lab", "node3.lab"],
+      replicas: [{ volume: "pvc-solo" }],
+      volumes: [{ name: "pvc-solo", replicas: 1, size: 1 * GIB }],
+    });
+    const f = await checkEvictionCompletable(ctx, "node1.lab");
+    expect(f.completable).toBe(true);
+    expect(f.warnings.join(" ")).toMatch(/single replica/);
+  });
+
+  it("is trivially completable when the node holds nothing", async () => {
+    const f = await checkEvictionCompletable(routed({ replicas: [] }), "node1.lab");
+    expect(f.completable).toBe(true);
+    expect(f.replicasMoving).toBe(0);
+  });
+
+  it("refuses to guess when Longhorn cannot be read", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValue(FAIL);
+    const f = await checkEvictionCompletable(ctx, "node1.lab");
+    expect(f.completable).toBe(false);
+    expect(f.blockers.join(" ")).toMatch(/refusing to guess/);
+  });
+});
+
+describe("assertNodeEvacuated — the gate before a wipe", () => {
+  it("passes only when cordoned with no replicas left", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec
+      .mockResolvedValueOnce(HOST)
+      .mockResolvedValueOnce(stdout("true"))   // unschedulable
+      .mockResolvedValueOnce(stdout(""))       // no running pods
+      .mockResolvedValueOnce(stdout("namespace/longhorn-system"))
+      .mockResolvedValueOnce(stdout(""));      // no replicas
+    const r = await assertNodeEvacuated(ctx);
+    expect(r.success).toBe(true);
+    expect(r.message).toMatch(/safe to wipe/);
+  });
+
+  it("REFUSES when replicas are still on the node", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec
+      .mockResolvedValueOnce(HOST)
+      .mockResolvedValueOnce(stdout("true"))
+      .mockResolvedValueOnce(stdout(""))
+      .mockResolvedValueOnce(stdout("namespace/longhorn-system"))
+      .mockResolvedValueOnce(stdout("replica-abc\nreplica-def"));
+    const r = await assertNodeEvacuated(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/2 Longhorn replica/);
+  });
+
+  it("REFUSES when the node is still schedulable", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec
+      .mockResolvedValueOnce(HOST)
+      .mockResolvedValueOnce(stdout(""))       // not unschedulable
+      .mockResolvedValueOnce(stdout(""))
+      .mockResolvedValueOnce(FAIL);            // no longhorn
+    const r = await assertNodeEvacuated(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/still schedulable/);
+  });
+});
+
+describe("uncordonNode", () => {
+  it("returns the node to service", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(OK);
+    const r = await uncordonNode(ctx);
+    expect(r.success).toBe(true);
+    expectCommand(ctx.ssh, /uncordon "node1\.lab"/);
+  });
+});
+
+describe("eviction timeout diagnostics", () => {
+  // An eviction can legitimately run for an hour. When it times out, a bare
+  // count ("2 replica(s) still present") is not actionable -- the question is
+  // always WHICH volume is stuck and in what state. A replica with
+  // desireState=running, currentState=stopped and no failedAt is Longhorn
+  // wanting to start it and not doing so, which is a completely different
+  // problem from a volume that cannot be placed. Observed for real on a
+  // freshly wiped-and-rejoined node.
+  it("names the stuck volumes and their replica states, then restores the node", async () => {
+    const ctx = mockCtx();
+    const patches: string[] = [];
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return HOST;
+      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
+      if (cmd.includes("patch nodes.longhorn.io")) { patches.push(cmd); return OK; }
+      if (cmd.includes("jsonpath")) {
+        // The poll never reaches zero: one replica remains for the whole run.
+        if (cmd.includes("replicas.longhorn.io")) return stdout("replica-stuck");
+        if (cmd.includes("replica-soft-anti-affinity")) return stdout("false");
+        if (cmd.includes("storage-over-provisioning-percentage")) return stdout("100");
+        return stdout("");
+      }
+      if (cmd.includes("get replicas.longhorn.io") && cmd.includes("-o json")) {
+        return stdout(JSON.stringify({ items: [{
+          metadata: { name: "pvc-abc-r-deadbeef99" },
+          spec: { nodeID: "node1.lab", volumeName: "pvc-abc", failedAt: "", desireState: "running" },
+          status: { currentState: "stopped" },
+        }] }));
+      }
+      if (cmd.includes("get volumes.longhorn.io") && cmd.includes("-o json")) {
+        return stdout(JSON.stringify({ items: [{
+          metadata: { name: "pvc-abc" },
+          spec: { numberOfReplicas: 3, size: 2 * 2 ** 30 },
+          status: { state: "attached", robustness: "degraded" },
+        }] }));
+      }
+      if (cmd.includes("get nodes.longhorn.io") && cmd.includes("-o json")) {
+        return stdout(JSON.stringify({ items: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"].map((n) => ({
+          metadata: { name: n },
+          spec: { allowScheduling: true },
+          status: { diskStatus: { d: {
+            storageAvailable: 500 * 2 ** 30, storageMaximum: 700 * 2 ** 30,
+            storageScheduled: 0, storageReserved: 0,
+          } } },
+        })) }));
+      }
+      return OK;
+    });
+
+    vi.useFakeTimers();
+    let result;
+    try {
+      const p = evictLonghornReplicas(ctx);
+      await vi.advanceTimersByTimeAsync(61 * 60_000);
+      result = await p;
+    } finally {
+      vi.useRealTimers();
+    }
+
+    expect(result.success).toBe(false);
+    // The actionable part: which volume, and what state its replica is in.
+    expect(result.error).toContain("pvc-abc");
+    expect(result.error).toContain("currentState=stopped");
+    expect(result.error).toContain("desireState=running");
+    expect(result.error).toContain("failedAt=(none)");
+    expect(result.error).toContain("attached/degraded");
+
+    // And the node must be handed back usable, not left eviction-pending.
+    expect(result.error).toMatch(/cancelled and scheduling restored/);
+    expect(patches).toHaveLength(2);
+    expect(patches[1]).toContain('"evictionRequested":false');
+    expect(patches[1]).toContain('"allowScheduling":true');
+  });
+});

bastion/src/modules/modules/k3s/tests/operations.test.ts

@@ -268,6 +268,108 @@ describe("writeK3sConfig", () => {
     expect(writeCall).toContain("protect-kernel-defaults: true");
     expect(writeCall).not.toContain("secrets-encryption");
   });
+
+  // --- dual-stack ---
+  //
+  // The property that matters most is the NEGATIVE one: with no ipv6 and no
+  // CIDRs the output must be byte-identical to what the five existing nodes
+  // already have on disk. If it is not, rolling this out rewrites every node's
+  // config.yaml and restarts a healthy cluster to tell it what it already knew.
+
+  // With an ipv6 configured there is an extra ssh round trip up front -- the
+  // probe that checks the address is really on the node -- so the write lands
+  // one call later.
+  const writtenBy = async (config: Parameters<typeof mockCtx>[0]) => {
+    const ctx = mockCtx(config);
+    const hasV6 = !!(config as { ipv6?: string }).ipv6;
+    if (hasV6) ctx.ssh.exec.mockResolvedValueOnce(stdout("present"));
+    ctx.ssh.exec
+      .mockResolvedValueOnce(OK)
+      .mockResolvedValueOnce(stdout("__LABCTL_NOT_FOUND__"))
+      .mockResolvedValueOnce(OK);
+    await writeK3sConfig(ctx);
+    return ctx.ssh.exec.mock.calls[hasV6 ? 3 : 2]![0] as string;
+  };
+
+  it("emits no address-family lines at all when single-stack", async () => {
+    const server = await writtenBy({ hostname: "n1.lab", ip: "10.0.1.1", role: "infra" });
+    expect(server).not.toContain("node-ip");
+    expect(server).not.toContain("cluster-cidr");
+    expect(server).not.toContain("service-cidr");
+
+    const agent = await writtenBy({ role: "worker" });
+    expect(agent).not.toContain("node-ip");
+    // Nothing inserted ahead of, or between, the lines the agent config has
+    // always opened with.
+    expect(agent).toContain("protect-kernel-defaults: true\nnode-label:");
+  });
+
+  it("emits dual node-ip and CIDRs on a server, IPv4 first", async () => {
+    const out = await writtenBy({
+      hostname: "n1.lab",
+      ip: "192.168.8.23",
+      role: "infra",
+      ipv6: "2001:470:187e:2::23",
+      clusterCidr: ["10.42.0.0/16", "2001:470:187e:1000::/56"],
+      serviceCidr: ["10.43.0.0/16", "2001:470:187e:1fff::/112"],
+    });
+    expect(out).toContain('node-ip: "192.168.8.23,2001:470:187e:2::23"');
+    expect(out).toContain('cluster-cidr: "10.42.0.0/16,2001:470:187e:1000::/56"');
+    expect(out).toContain('service-cidr: "10.43.0.0/16,2001:470:187e:1fff::/112"');
+    // IPv4 must stay the primary family -- that is the supported conversion
+    // path and what lets existing Services keep their ClusterIP.
+    expect(out.indexOf("10.42.0.0/16")).toBeLessThan(out.indexOf("2001:470:187e:1000::/56"));
+    // still a valid server config
+    expect(out).toContain("cluster-init: true");
+    expect(out).toContain("secrets-encryption: true");
+    // the v6 address must be a TLS SAN, or a peer joining over v6 fails
+    // verification with an error that names the cert, not the missing SAN
+    expect(out).toContain('  - "2001:470:187e:2::23"');
+  });
+
+  it("gives an AGENT its own dual node-ip but no CIDRs", async () => {
+    const out = await writtenBy({
+      ip: "192.168.8.12",
+      role: "worker",
+      ipv6: "2001:470:187e:2::12",
+      // deliberately supplied: an agent must ignore them
+      clusterCidr: ["10.42.0.0/16", "2001:470:187e:1000::/56"],
+      serviceCidr: ["10.43.0.0/16", "2001:470:187e:1fff::/112"],
+    });
+    expect(out).toContain('node-ip: "192.168.8.12,2001:470:187e:2::12"');
+    expect(out).not.toContain("cluster-cidr");
+    expect(out).not.toContain("service-cidr");
+  });
+
+  it("refuses to write a config naming an IPv6 the node does not have", async () => {
+    // The SSH-onboard case: Asahi and the DGX Sparks run an OS labctl never
+    // installed, so nothing upstream guarantees a DHCPv6 lease was taken.
+    // Writing the config anyway defers the failure to k3s start-up, where it
+    // reads as a bind error rather than a missing address.
+    const ctx = mockCtx({ ip: "192.168.8.12", role: "worker", ipv6: "2001:470:187e:2::12" });
+    ctx.ssh.exec.mockResolvedValueOnce(stdout("")); // probe: address absent
+
+    const result = await writeK3sConfig(ctx);
+    expect(result.success).toBe(false);
+    expect(result.changed).toBe(false);
+    expect(result.error).toContain("2001:470:187e:2::12");
+    // and it must not have written anything
+    expect(ctx.ssh.exec).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not write node-ip for a v4-only node even when CIDRs are given", async () => {
+    // Guards the flag-day risk: supplying ranges alone must not start rewriting
+    // node identity on nodes that have no IPv6 yet.
+    const out = await writtenBy({
+      hostname: "n1.lab",
+      ip: "10.0.1.1",
+      role: "infra",
+      clusterCidr: ["10.42.0.0/16"],
+      serviceCidr: ["10.43.0.0/16"],
+    });
+    expect(out).toContain('cluster-cidr: "10.42.0.0/16"');
+    expect(out).not.toContain("node-ip");
+  });
 });
 
 // --- CNI Cleanup ---

bastion/src/modules/src/types.ts

@@ -1,4 +1,4 @@
-import type { OsId, Arch } from "@lab/shared";
+import type { OsId, Arch, Role } from "@lab/shared";
 
 export interface ModuleMetadata {
   name: string;
@@ -14,7 +14,10 @@ export interface ModuleMetadata {
 export interface ModuleContext {
   hostname: string;
   ip: string;
-  role: string;
+  // Role, not string. A literal `string` let `role === "infra"` comparisons
+  // compile while silently missing every other server role -- which is how
+  // labcontroller ended up installing as an agent.
+  role: Role;
   os: OsId;
   arch: Arch;
   sshUser: string;
@@ -24,7 +27,7 @@ export interface ModuleContext {
 
 export interface ModuleResult {
   success: boolean;
-  phase: "install" | "configure" | "health";
+  phase: "install" | "configure" | "health" | "evacuate" | "admit";
   duration: number;
   output: string[];
   errors: string[];

bastion/src/shared/src/index.ts

@@ -14,8 +14,9 @@ export type {
   VyosBundleSetOp,
 } from "./types/index.js";
 
-export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./types/index.js";
-export type { RoleInfo } from "./types/index.js";
+export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId,
+  roleInfo, runsK3s, isK3sServer, isManaged, hostsLonghorn, isRole } from "./types/index.js";
+export type { RoleInfo, Lifecycle, K3sMode } from "./types/index.js";
 
 export { APP_NAME, APP_VERSION } from "./constants/index.js";
 
@@ -48,3 +49,8 @@ export {
   AgentNotConnectedError,
   isLabError,
 } from "./errors/index.js";
+
+export { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB,
+  SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion,
+  validateDataDisks, DATA_DISK_LABELS } from "./types/index.js";
+export type { LvSpec, SchemaVersion, DataDisks, DataDiskName } from "./types/index.js";

bastion/src/shared/src/protocol/index.ts

@@ -109,7 +109,7 @@ export type BastionMessage =
 export type LabdBastionMessage =
   | { type: "bastion-enrolled"; bastionId: string }
   | { type: "bastion-heartbeat-ack"; serverTime: string }
-  | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string; vyos?: VyosInstallSpec }
+  | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string; schemaVersion?: 1 | 2; dataDisks?: import("../types/layout.js").DataDisks; vyos?: VyosInstallSpec }
   | { type: "command-forget"; requestId: string; mac: string }
   | { type: "command-role-update"; requestId: string; mac: string; role: string }
   | { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean }

bastion/src/shared/src/types/config.ts

@@ -23,6 +23,10 @@ export interface BastionConfig {
   // Flags
   skipDnsmasq?: boolean | undefined;
   skipArtifacts?: boolean | undefined;
+  // Post-provision k3s auto-install. Defaults on. Set false to suppress it --
+  // required by multi-node tests, which must sequence cluster formation
+  // themselves rather than have each node auto-install into its own cluster.
+  autoK3s?: boolean | undefined;
   // Labd registration (optional — if unset, bastion runs standalone)
   labdUrl?: string | undefined;
   bastionJoinToken?: string | undefined;

bastion/src/shared/src/types/index.ts

@@ -13,7 +13,13 @@ export type {
   VyosBundleSetOp,
 } from "./state.js";
 
-export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./state.js";
-export type { RoleInfo } from "./state.js";
+export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId,
+  roleInfo, runsK3s, isK3sServer, isManaged, hostsLonghorn, isRole } from "./state.js";
+export type { RoleInfo, Lifecycle, K3sMode } from "./state.js";
 
 export type { BastionConfig } from "./config.js";
+
+export { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB,
+  SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion,
+  validateDataDisks, DATA_DISK_LABELS } from "./layout.js";
+export type { LvSpec, SchemaVersion, DataDisks, DataDiskName } from "./layout.js";

bastion/src/shared/src/types/layout.ts

@@ -0,0 +1,210 @@
+import { roleInfo, type Role } from "./state.js";
+
+/**
+ * Disk layout, schema v2.
+ *
+ * WHY THIS FILE EXISTS
+ * --------------------
+ * Three generators partition machines, and before this file they disagreed:
+ *
+ *   install.ks.ts        rancher 120G (infra|worker), longhorn --grow (worker)
+ *   ubuntu-autoinstall   rancher  20G (infra only),   longhorn grow   (worker)
+ *   asahi-firstboot.sh   rancher  20G (server only),  longhorn 100%FREE (worker|server)
+ *
+ * asahi-firstboot even carries the comment "sizes match install.ks.ts" while
+ * being 100 GB smaller — which is where worker1's undersized image store came
+ * from, and why a 2026-08 fix to one file never reached the fleet.
+ *
+ * v2 is defined ONCE, here, and all three generators render from it. v1 stays
+ * frozen inline in each generator, because every existing machine was built by
+ * it and reprovision must reproduce it byte-for-byte.
+ *
+ * SIZING
+ * ------
+ * Must fit a 1 TB disk (~926 GiB VG after EFI+/boot) with headroom left
+ * UNALLOCATED on purpose: LVM can grow into free extents online, and the
+ * fleet's recurring failure has been over-allocation, not under-allocation.
+ * `--grow` is deliberately never used — it is what left every node at VFree=0
+ * and made later fixes impossible without a rebuild.
+ */
+
+/**
+ * Roles that predate the v2 layout. Only these have a v1 layout to reproduce;
+ * everything else is v2-only, because no machine was ever installed as one.
+ */
+export const LEGACY_ROLES = ["vanilla", "worker", "infra", "labcontroller"] as const;
+
+/**
+ * True when this role has no v1 layout at all.
+ *
+ * Rendering one at v1 would silently produce base volumes with no rancher and
+ * no longhorn -- a machine that looks installed and cannot run k3s properly.
+ * Callers must reject this rather than guess.
+ */
+export function requiresSchemaV2(role: Role): boolean {
+  return !(LEGACY_ROLES as readonly string[]).includes(role);
+}
+
+/**
+ * The schema a FRESH install of this role should use.
+ *
+ * A role with no v1 layout has exactly one possible answer, so defaulting to it
+ * is unambiguous -- there is no other layout it could mean. Rejecting the
+ * request instead was a mistake: a reinstall legitimately passes no schema
+ * (the disk decides via the lab-schema-v2 tag), and refusing that made it
+ * impossible to reinstall a v2-only machine at all.
+ *
+ * An EXPLICIT v1 for such a role is still a contradiction and must be refused
+ * by the caller -- see requiresSchemaV2.
+ */
+export function effectiveSchemaVersion(role: Role, requested?: 1 | 2): SchemaVersion {
+  if (requested) return requested;
+  return requiresSchemaV2(role) ? 2 : 1;
+}
+
+export const SCHEMA_VERSIONS = [1, 2] as const;
+export type SchemaVersion = (typeof SCHEMA_VERSIONS)[number];
+
+export interface LvSpec {
+  /** LV name inside the volume group. */
+  name: string;
+  /** Mount point, or null for swap. */
+  mount: string | null;
+  /** Size in MiB. Never "grow". */
+  sizeMib: number;
+  fs: "xfs" | "swap";
+}
+
+/** Outside the VG: the EFI system partition and /boot. */
+export const V2_BOOT = {
+  efiMib: 1024,   // was 600 - firmware and shim have grown
+  bootMib: 4096,  // was 3072 - several kernels + initramfs
+} as const;
+
+const V2_BASE: readonly LvSpec[] = [
+  { name: "swap",   mount: null,       sizeMib: 16384, fs: "swap" },
+  { name: "root",   mount: "/",        sizeMib: 32768, fs: "xfs" },
+  { name: "var",    mount: "/var",     sizeMib: 49152, fs: "xfs" },
+  { name: "varlog", mount: "/var/log", sizeMib: 16384, fs: "xfs" },
+  { name: "home",   mount: "/home",    sizeMib:  8192, fs: "xfs" },
+  { name: "srv",    mount: "/srv",     sizeMib: 16384, fs: "xfs" },
+] as const;
+
+/** Image store AND local-path PVCs both live here - size for both. */
+export const V2_RANCHER_MIB = 256000;  // 250 GiB
+export const V2_LONGHORN_MIB = 307200; // 300 GiB
+
+/**
+ * Dedicated whole-disk data volumes (v2 only).
+ *
+ * The fixed 250G/300G LVs exist to FIT a 1 TB OS disk. A machine with spare
+ * NVMe bays (thebeast: 1 TB OS + 2 TB + 2 TB) wants the opposite trade: give
+ * each data mount a whole disk and keep the OS VG small. Each value is a
+ * stable /dev/disk/by-id/ path -- NEVER a kernel name, because nvme0n1/nvme1n1
+ * enumeration order is not stable across boots on multi-NVMe hosts, and a
+ * swapped name here formats the wrong disk.
+ *
+ * The disk gets one partition spanning it, xfs, labelled (DATA_DISK_LABELS)
+ * so a reprovision can recognise and preserve it the same way the LVs are
+ * recognised by name inside the VG. The corresponding LV is simply not
+ * created -- v2Layout() drops it when a dedicated disk supplies that mount.
+ */
+export interface DataDisks {
+  /** Whole disk for /var/lib/longhorn (replica store). */
+  longhorn?: string;
+  /** Whole disk for /var/lib/rancher (image store + local-path PVCs). */
+  rancher?: string;
+}
+
+/**
+ * Filesystem labels that mark a dedicated data disk's partition. They are the
+ * reprovision contract: %pre finds these by label and mounts them --noformat,
+ * exactly as it finds LVs by name. Changing a label orphans every disk
+ * already carrying the old one.
+ */
+export const DATA_DISK_LABELS = {
+  longhorn: "lab-longhorn",
+  rancher: "lab-rancher",
+} as const;
+
+export type DataDiskName = keyof typeof DATA_DISK_LABELS;
+
+const BY_ID_PREFIX = "/dev/disk/by-id/";
+
+/**
+ * Why a DataDisks spec is unusable for this role, or null when it is fine.
+ * Shared by the HTTP API, the websocket command handler and the CLI so all
+ * three reject with the same words instead of three drifting checks.
+ */
+export function validateDataDisks(role: Role, disks: DataDisks): string | null {
+  const info = roleInfo(role);
+  for (const name of Object.keys(DATA_DISK_LABELS) as DataDiskName[]) {
+    const device = disks[name];
+    if (device === undefined) continue;
+    if (!device.startsWith(BY_ID_PREFIX)) {
+      return `${name} disk must be a stable ${BY_ID_PREFIX} path (got '${device}'). ` +
+        `Kernel names like nvme0n1 change across boots on multi-NVMe hosts.`;
+    }
+  }
+  if (disks.rancher !== undefined && info.k3sMode === null) {
+    return `role '${role}' runs no k3s -- a dedicated rancher disk makes no sense for it`;
+  }
+  if (disks.longhorn !== undefined && (info.k3sMode === null || info.controlPlaneOnly)) {
+    return `role '${role}' holds no Longhorn replicas -- a dedicated longhorn disk makes no sense for it`;
+  }
+  if (disks.longhorn !== undefined && disks.longhorn === disks.rancher) {
+    return `longhorn and rancher point at the same disk (${disks.longhorn}) -- they need one each`;
+  }
+  return null;
+}
+
+/**
+ * The v2 volumes for a role, gated by CAPABILITY rather than role name.
+ *
+ * `omitRoot` is for Asahi, where the vendor installer already created a btrfs
+ * root partition outside our VG and we must not make a second one.
+ *
+ * `dataDisks` drops the longhorn/rancher LV whose mount a dedicated disk
+ * supplies -- two things mounting /var/lib/longhorn is an invalid layout.
+ */
+export function v2Layout(role: Role, opts: { omitRoot?: boolean; dataDisks?: DataDisks } = {}): LvSpec[] {
+  const info = roleInfo(role);
+  const lvs: LvSpec[] = V2_BASE.filter(
+    (lv) => !(opts.omitRoot && lv.name === "root"),
+  ).map((lv) => ({ ...lv }));
+
+  // Anything running k3s needs a dedicated image store. This is the check that
+  // `role === "infra" || role === "worker"` kept getting wrong: labcontroller
+  // runs k3s and was silently excluded.
+  if (info.k3sMode !== null && opts.dataDisks?.rancher === undefined) {
+    lvs.push({ name: "rancher", mount: "/var/lib/rancher", sizeMib: V2_RANCHER_MIB, fs: "xfs" });
+  }
+
+  // Fixed, never --grow. A control-plane-only node holds no replicas.
+  if (info.k3sMode !== null && !info.controlPlaneOnly && opts.dataDisks?.longhorn === undefined) {
+    lvs.push({ name: "longhorn", mount: "/var/lib/longhorn", sizeMib: V2_LONGHORN_MIB, fs: "xfs" });
+  }
+
+  return lvs;
+}
+
+/** Total MiB a role's v2 layout consumes inside the VG. */
+export function v2AllocatedMib(role: Role, opts: { omitRoot?: boolean; dataDisks?: DataDisks } = {}): number {
+  return v2Layout(role, opts).reduce((n, lv) => n + lv.sizeMib, 0);
+}
+
+/**
+ * Bind-mounting containerd onto the Longhorn LV.
+ *
+ * Codifies what was already done BY HAND on worker1 and exists in no template:
+ * its 20 GiB rancher LV could not hold the image store, so containerd was
+ * pointed at a directory on the big Longhorn LV. It survives reboot only
+ * because someone wrote a correct fstab line; a reinstall would lose it.
+ *
+ * v2 sizes rancher properly so this is not needed on new boxes - it is kept for
+ * rescuing v1 machines that cannot be repartitioned yet.
+ */
+export const CONTAINERD_BIND_FSTAB =
+  "/var/lib/longhorn/k3s-containerd /var/lib/rancher/k3s/agent/containerd none " +
+  "bind,x-systemd.requires-mounts-for=/var/lib/longhorn," +
+  "x-systemd.requires-mounts-for=/var/lib/rancher 0 0";

bastion/src/shared/src/types/state.ts

@@ -28,8 +28,32 @@ export interface HardwareInfo {
   bastionId?: string;  // set when aggregated through labd
 }
 
-export type Role = "vanilla" | "worker" | "infra" | "labcontroller";
-export const SUPPORTED_ROLES: readonly Role[] = ["vanilla", "worker", "infra", "labcontroller"] as const;
+export type Role =
+  // Legacy roles. Every machine in the fleet today is one of these, and they
+  // default to the v1 disk layout so a reinstall reproduces what is on disk.
+  | "vanilla" | "worker" | "infra" | "labcontroller"
+  // Topology-named profiles. New machines only, so they are v2-only: there is
+  // no v1 layout to be compatible with, and none of them can reach the frozen
+  // v1 branch in the templates.
+  | "standalone-cluster" | "k8s-controller" | "k8s-worker" | "k8s-combo";
+
+export const SUPPORTED_ROLES: readonly Role[] = [
+  "vanilla", "worker", "infra", "labcontroller",
+  "standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo",
+] as const;
+
+/**
+ * How a machine comes to exist, which decides whether we partition it at all.
+ *
+ *  - "managed"   we PXE-install it with our kickstart; our disk layout applies.
+ *  - "appliance" we install it, but the vendor installer owns the disk (VyOS).
+ *  - "onboard"   we never install it; we adopt a running machine (DGX Spark,
+ *                Apple Silicon under Asahi). No partitioning, ever.
+ */
+export type Lifecycle = "managed" | "appliance" | "onboard";
+
+/** null = no k3s at all. */
+export type K3sMode = "server" | "agent" | null;
 
 export interface RoleInfo {
   name: Role;
@@ -37,6 +61,18 @@ export interface RoleInfo {
   parent?: Role;       // inherits from this role
   k3s: boolean;        // installs k3s
   apps: string[];      // apps auto-deployed after k3s
+
+  // --- capabilities -------------------------------------------------------
+  // These exist so decisions are made by asking what a machine DOES, not by
+  // comparing its role name. Every `role === "infra" || role === "labcontroller"`
+  // in this codebase is a place that forgot a role: `labcontroller` runs k3s and
+  // still got no rancher LV, because install.ks.ts asked `role === "infra"`.
+  lifecycle: Lifecycle;
+  k3sMode: K3sMode;
+  /** Holds Longhorn replicas, i.e. wants a dedicated /var/lib/longhorn LV. */
+  hostsLonghorn: boolean;
+  /** Control plane that deliberately runs no workloads and no storage. */
+  controlPlaneOnly: boolean;
 }
 
 export const ROLE_REGISTRY: readonly RoleInfo[] = [
@@ -45,6 +81,10 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [
     description: "OS only — no k3s, no cluster services",
     k3s: false,
     apps: [],
+    lifecycle: "managed",
+    k3sMode: null,
+    hostsLonghorn: false,
+    controlPlaneOnly: false,
   },
   {
     name: "worker",
@@ -52,6 +92,10 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [
     parent: "vanilla",
     k3s: true,
     apps: [],
+    lifecycle: "managed",
+    k3sMode: "agent",
+    hostsLonghorn: true,
+    controlPlaneOnly: false,
   },
   {
     name: "infra",
@@ -59,6 +103,15 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [
     parent: "vanilla",
     k3s: true,
     apps: [],
+    lifecycle: "managed",
+    k3sMode: "server",
+    // NOTE: worker0/1/2 are `infra` and DO hold every Longhorn replica in the
+    // homelab cluster, but the v1 kickstart only ever gave a longhorn LV to
+    // `worker`. On worker2 that means Longhorn data sits on /var alongside the
+    // image store. Left false so the v1 layout is unchanged; v2 sizes it
+    // properly from this flag.
+    hostsLonghorn: false,
+    controlPlaneOnly: false,
   },
   {
     name: "labcontroller",
@@ -66,9 +119,104 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [
     parent: "infra",
     k3s: true,
     apps: ["cockroachdb", "labd", "bastion"],
+    lifecycle: "managed",
+    k3sMode: "server",
+    hostsLonghorn: false,
+    controlPlaneOnly: false,
+  },
+  {
+    name: "standalone-cluster",
+    description: "Single node that IS its own cluster — k3s server + workloads + storage",
+    parent: "vanilla",
+    k3s: true,
+    apps: [],
+    lifecycle: "managed",
+    k3sMode: "server",
+    // Longhorn yes, at replica 1. A cluster of one cannot satisfy a 3-replica
+    // policy, but keeping the `longhorn` StorageClass present means manifests
+    // are portable across every cluster instead of needing a local-path
+    // variant. Replica count is a property of the cluster, not the manifest.
+    hostsLonghorn: true,
+    controlPlaneOnly: false,
+  },
+  {
+    name: "k8s-controller",
+    description: "Control plane + etcd only — runs no workloads and holds no storage",
+    parent: "vanilla",
+    k3s: true,
+    apps: [],
+    lifecycle: "managed",
+    k3sMode: "server",
+    hostsLonghorn: false,
+    controlPlaneOnly: true,
+  },
+  {
+    name: "k8s-worker",
+    description: "k3s agent + Longhorn storage — joins an existing cluster",
+    parent: "vanilla",
+    k3s: true,
+    apps: [],
+    lifecycle: "managed",
+    k3sMode: "agent",
+    hostsLonghorn: true,
+    controlPlaneOnly: false,
+  },
+  {
+    name: "k8s-combo",
+    description: "Control plane + etcd AND workloads + storage — hyperconverged cluster member",
+    parent: "vanilla",
+    k3s: true,
+    apps: [],
+    lifecycle: "managed",
+    k3sMode: "server",
+    hostsLonghorn: true,
+    controlPlaneOnly: false,
   },
 ] as const;
 
+/**
+ * Capability lookups. Use these instead of comparing role names.
+ *
+ * Every call site that asks `role === "infra" || role === "labcontroller"` is a
+ * place that has to be edited again when a role is added — and the one that was
+ * missed (install.ks.ts asking `role === "infra"`) is why `labcontroller` boxes
+ * get no dedicated image-store LV.
+ */
+/**
+ * Narrow an untrusted string (state file, HTTP body) to a Role.
+ * State is JSON on disk and can contain anything, including a role that was
+ * removed in a later version -- so this is a guard, not a cast.
+ */
+export function isRole(value: string): value is Role {
+  return (SUPPORTED_ROLES as readonly string[]).includes(value);
+}
+
+export function roleInfo(role: Role): RoleInfo {
+  const info = ROLE_REGISTRY.find((r) => r.name === role);
+  if (!info) throw new Error(`unknown role: ${role}`);
+  return info;
+}
+
+/** Runs k3s in any capacity. */
+export function runsK3s(role: Role): boolean {
+  return roleInfo(role).k3sMode !== null;
+}
+
+/** Runs k3s as a server (control plane + etcd) rather than an agent. */
+export function isK3sServer(role: Role): boolean {
+  return roleInfo(role).k3sMode === "server";
+}
+
+/** We PXE-install this machine, so our disk layout applies to it. */
+export function isManaged(role: Role): boolean {
+  return roleInfo(role).lifecycle === "managed";
+}
+
+/** Wants a dedicated /var/lib/longhorn LV. */
+export function hostsLonghorn(role: Role): boolean {
+  return roleInfo(role).hostsLonghorn;
+}
+
 export interface ProgressLogEntry {
   stage: string;
   detail: string;
@@ -202,6 +350,19 @@ export interface InstallConfig {
   os?: OsId;          // defaults to "fedora-43" for backward compat
   vyos?: VyosInstallSpec;  // only consulted when os is "vyos-rolling"
   arch?: Arch;         // detected from HardwareInfo or overridden
+  /**
+   * Disk layout for a FRESH install. Defaults to 1.
+   * Ignored on a reprovision: %pre reads the lab-schema-v2 LVM tag off the
+   * disk instead, because reinstalling a machine must never change its layout.
+   */
+  schemaVersion?: 1 | 2;
+  /**
+   * Dedicated whole-disk data volumes (v2 + fedora only), by stable
+   * /dev/disk/by-id path. See DataDisks in layout.ts. On a reprovision these
+   * are optional: %pre rediscovers the disks by filesystem label and preserves
+   * them whether or not the queue entry names them.
+   */
+  dataDisks?: import("./layout.js").DataDisks;
   queued_at: string;
   /**
    * When dispatch last served this machine an install boot script. Progress
@@ -220,6 +381,12 @@ export interface InstallConfig {
 export interface InstalledInfo {
   hostname: string;
   role: string;
+  /**
+   * Disk layout this machine was installed with. Advisory only -- the
+   * authoritative answer is the `lab-schema-v2` LVM tag on the machine's own
+   * volume group, because that is what a reinstall actually reads.
+   */
+  schemaVersion?: 1 | 2;
   os?: OsId;
   ip: string;
   installed_at: string;

bastion/src/shared/tests/capabilities.test.ts

@@ -0,0 +1,129 @@
+import { describe, it, expect } from "vitest";
+import {
+  SUPPORTED_ROLES,
+  ROLE_REGISTRY,
+  roleInfo,
+  runsK3s,
+  isK3sServer,
+  isManaged,
+  hostsLonghorn,
+} from "../src/types/state.js";
+import type { Role } from "../src/types/state.js";
+
+/**
+ * The capability lookups exist to replace role-name comparisons scattered across
+ * the codebase. Introducing them must change NO behaviour — so these tests pin
+ * each helper against the literal expression it replaces, copied verbatim from
+ * the call site it came from.
+ *
+ * If one of these fails, the refactor changed behaviour somewhere. That is the
+ * whole point of the file.
+ */
+/**
+ * Only the FOUR legacy roles have behaviour to preserve. The topology profiles
+ * (standalone-cluster, k8s-*) are new, v2-only, and have no historical
+ * expression to be compatible with -- pinning them against `role === "worker"`
+ * would assert nonsense.
+ */
+const LEGACY_ROLES = ["vanilla", "worker", "infra", "labcontroller"] as const;
+
+describe("capability helpers reproduce the expressions they replace", () => {
+  // Verbatim from the call sites, as of the commit that introduced this file.
+  const legacy = {
+    // k3s-install.ts:7, k3s-config.ts:7, render-config.ts:25,
+    // k3s-module.ts:65, health/k3s-service.ts:5, post-provision.ts:153
+    isServer: (r: Role) => r === "infra" || r === "labcontroller",
+    // ROLE_REGISTRY.k3s — the field that already existed and nobody read
+    runsK3s: (r: Role) => ROLE_REGISTRY.find((x) => x.name === r)!.k3s,
+    // install.ks.ts:41, ubuntu-autoinstall.ts:34
+    hasLonghorn: (r: Role) => r === "worker",
+    // install.ks.ts:48
+    isVanilla: (r: Role) => r === "vanilla",
+  };
+
+  it.each(LEGACY_ROLES)("isK3sServer matches the legacy expression for %s", (role) => {
+    expect(isK3sServer(role)).toBe(legacy.isServer(role));
+  });
+
+  it.each(LEGACY_ROLES)("runsK3s matches ROLE_REGISTRY.k3s for %s", (role) => {
+    expect(runsK3s(role)).toBe(legacy.runsK3s(role));
+  });
+
+  it.each(LEGACY_ROLES)("hostsLonghorn matches the legacy expression for %s", (role) => {
+    expect(hostsLonghorn(role)).toBe(legacy.hasLonghorn(role));
+  });
+
+  it.each(LEGACY_ROLES)("k3sMode === null matches isVanilla for %s", (role) => {
+    expect(roleInfo(role).k3sMode === null).toBe(legacy.isVanilla(role));
+  });
+
+  it("every supported role is managed today (nothing is onboarded yet)", () => {
+    for (const role of SUPPORTED_ROLES) expect(isManaged(role)).toBe(true);
+  });
+});
+
+/**
+ * The topology profiles. These are asserted directly rather than against a
+ * legacy expression, because they describe cluster SHAPE -- which is the thing
+ * role names were supposed to convey and did not.
+ */
+describe("topology profiles", () => {
+  it("standalone-cluster is a server that also holds storage", () => {
+    expect(isK3sServer("standalone-cluster")).toBe(true);
+    // Longhorn at replica 1: a cluster of one cannot do 3, but keeping the
+    // StorageClass present keeps manifests portable across clusters.
+    expect(hostsLonghorn("standalone-cluster")).toBe(true);
+    expect(roleInfo("standalone-cluster").controlPlaneOnly).toBe(false);
+  });
+
+  it("k8s-controller is the only role that runs k3s and holds no storage", () => {
+    expect(isK3sServer("k8s-controller")).toBe(true);
+    expect(hostsLonghorn("k8s-controller")).toBe(false);
+    expect(roleInfo("k8s-controller").controlPlaneOnly).toBe(true);
+  });
+
+  it("k8s-worker is an agent with storage", () => {
+    expect(isK3sServer("k8s-worker")).toBe(false);
+    expect(runsK3s("k8s-worker")).toBe(true);
+    expect(hostsLonghorn("k8s-worker")).toBe(true);
+  });
+
+  it("k8s-combo is both control plane and storage — unlike k8s-controller", () => {
+    expect(isK3sServer("k8s-combo")).toBe(true);
+    expect(hostsLonghorn("k8s-combo")).toBe(true);
+    expect(roleInfo("k8s-combo").controlPlaneOnly).toBe(false);
+  });
+
+  it("k8s-combo and k8s-controller differ ONLY in storage duty", () => {
+    const combo = roleInfo("k8s-combo");
+    const ctrl = roleInfo("k8s-controller");
+    expect(combo.k3sMode).toBe(ctrl.k3sMode);
+    expect(combo.hostsLonghorn).not.toBe(ctrl.hostsLonghorn);
+  });
+});
+
+describe("registry integrity", () => {
+  it("every SUPPORTED_ROLE has a registry entry", () => {
+    for (const role of SUPPORTED_ROLES) expect(() => roleInfo(role)).not.toThrow();
+  });
+
+  it("registry has no entries outside SUPPORTED_ROLES", () => {
+    for (const info of ROLE_REGISTRY) expect(SUPPORTED_ROLES).toContain(info.name);
+  });
+
+  it("k3s boolean and k3sMode never disagree", () => {
+    for (const info of ROLE_REGISTRY) {
+      expect(info.k3s).toBe(info.k3sMode !== null);
+    }
+  });
+
+  it("an unknown role throws rather than silently defaulting", () => {
+    expect(() => roleInfo("does-not-exist" as Role)).toThrow(/unknown role/);
+  });
+
+  it("a control-plane-only role never hosts Longhorn", () => {
+    for (const info of ROLE_REGISTRY) {
+      if (info.controlPlaneOnly) expect(info.hostsLonghorn).toBe(false);
+    }
+  });
+});

bastion/src/shared/tests/layout.test.ts

@@ -0,0 +1,92 @@
+import { describe, it, expect } from "vitest";
+import { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB } from "../src/types/layout.js";
+import { SUPPORTED_ROLES, roleInfo } from "../src/types/state.js";
+
+/** A 1 TB disk is the smallest we will ever buy. 1e12 bytes in MiB. */
+const ONE_TB_MIB = Math.floor(1_000_000_000_000 / (1024 * 1024)); // 953674
+const VG_MIB_ON_1TB = ONE_TB_MIB - V2_BOOT.efiMib - V2_BOOT.bootMib;
+
+describe("v2 layout fits the smallest disk we will buy", () => {
+  it.each(SUPPORTED_ROLES)("%s fits in a 1 TB disk with headroom left over", (role) => {
+    const used = v2AllocatedMib(role);
+    expect(used).toBeLessThan(VG_MIB_ON_1TB);
+    // The entire point of v2 is to STOP consuming the disk. Demand real slack.
+    const freeMib = VG_MIB_ON_1TB - used;
+    expect(freeMib).toBeGreaterThan(100 * 1024); // >100 GiB unallocated
+  });
+
+  it("the heaviest role still leaves >200 GiB free on 1 TB", () => {
+    const worst = Math.max(...SUPPORTED_ROLES.map((r) => v2AllocatedMib(r)));
+    expect(VG_MIB_ON_1TB - worst).toBeGreaterThan(200 * 1024);
+  });
+});
+
+describe("v2 gates volumes by capability, not role name", () => {
+  it.each(SUPPORTED_ROLES)("%s gets a rancher LV iff it runs k3s", (role) => {
+    const hasRancher = v2Layout(role).some((lv) => lv.name === "rancher");
+    expect(hasRancher).toBe(roleInfo(role).k3sMode !== null);
+  });
+
+  it("labcontroller gets a rancher LV — the v1 bug this schema fixes", () => {
+    const lvs = v2Layout("labcontroller");
+    const rancher = lvs.find((lv) => lv.name === "rancher");
+    expect(rancher).toBeDefined();
+    expect(rancher!.sizeMib).toBe(V2_RANCHER_MIB);
+  });
+
+  it("vanilla gets neither rancher nor longhorn", () => {
+    const names = v2Layout("vanilla").map((lv) => lv.name);
+    expect(names).not.toContain("rancher");
+    expect(names).not.toContain("longhorn");
+  });
+
+  it("a control-plane-only role would get no longhorn LV", () => {
+    for (const role of SUPPORTED_ROLES) {
+      if (roleInfo(role).controlPlaneOnly) {
+        expect(v2Layout(role).map((l) => l.name)).not.toContain("longhorn");
+      }
+    }
+  });
+});
+
+describe("v2 never over-allocates", () => {
+  it("no volume is grow/unbounded — every size is a concrete number", () => {
+    for (const role of SUPPORTED_ROLES) {
+      for (const lv of v2Layout(role)) {
+        expect(Number.isFinite(lv.sizeMib)).toBe(true);
+        expect(lv.sizeMib).toBeGreaterThan(0);
+      }
+    }
+  });
+
+  it("longhorn is fixed at 300 GiB, not grown", () => {
+    const lh = v2Layout("worker").find((lv) => lv.name === "longhorn");
+    expect(lh?.sizeMib).toBe(V2_LONGHORN_MIB);
+    expect(V2_LONGHORN_MIB).toBe(300 * 1024);
+  });
+
+  it("rancher is sized for an image store AND local-path PVCs", () => {
+    // A single ROCm vLLM image measured 62 GiB extracted on 2026-09-17.
+    expect(V2_RANCHER_MIB).toBeGreaterThan(200 * 1024);
+  });
+});
+
+describe("Asahi variant", () => {
+  it("omitRoot drops the root LV — the vendor installer already made one", () => {
+    const names = v2Layout("infra", { omitRoot: true }).map((lv) => lv.name);
+    expect(names).not.toContain("root");
+    expect(names).toContain("var");
+  });
+
+  it("omitRoot frees the root allocation and nothing else", () => {
+    const delta = v2AllocatedMib("infra") - v2AllocatedMib("infra", { omitRoot: true });
+    expect(delta).toBe(32768);
+  });
+});
+
+describe("layout is a pure value, safe to render three times", () => {
+  it("mutating a returned spec does not affect the next call", () => {
+    v2Layout("worker")[0]!.sizeMib = 999;
+    expect(v2Layout("worker")[0]!.sizeMib).not.toBe(999);
+  });
+});

bastion/tests/integration/cluster-smoke.test.ts

@@ -0,0 +1,164 @@
+// Integration test: form a real 3-server k3s cluster from PXE-installed VMs.
+//
+// WHY THIS EXISTS SEPARATELY FROM THE DRILL
+// -----------------------------------------
+// The rolling drill is a ~3 hour test. Almost everything that can go wrong in
+// its first thirty minutes goes wrong here instead, in forty-five: the token
+// handoff, the cluster-init/join discriminator, the etcd bootstrap race, root
+// SSH for the module, and whether post-provision was actually suppressed.
+// Debugging those inside a three-hour test is how you spend a day per attempt.
+//
+// It asserts on ETCD MEMBERSHIP, not just `kubectl get nodes`. A node object
+// and an etcd member are different things, and the whole hazard in a rolling
+// reinstall is that they can disagree.
+//
+// Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo. Set FEDORA_MIRROR to
+// a local mirror or each install pulls the full package set over the internet.
+//
+//   sudo -E npx vitest run -c tests/integration/vitest.config.ts -t "cluster smoke"
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import {
+  startTestBastion, provisionVms, destroyVms,
+  type Harness, type ProvisionedVm,
+} from "./helpers/provision-harness.js";
+import {
+  formCluster, clusterSnapshot, printSnapshot, etcdMembers, etcdHealthy,
+  allServersReady, pickJoinTarget, kubectlOn, k8sNodeName,
+} from "./helpers/k3s-cluster.js";
+import { destroyPxeNetwork } from "./helpers/pxe-network.js";
+
+const HTTP_PORT = 8097;
+const SSH_USER = "lab";
+const DISK_GB = 1000;
+const NODES = ["smoke-a", "smoke-b", "smoke-c"];
+
+function findSshKey(): { pubKey: string; keyPath: string } {
+  const homes = [homedir()];
+  const sudoUser = process.env["SUDO_USER"];
+  if (sudoUser) homes.push(join("/home", sudoUser));
+  for (const home of homes) {
+    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
+      const keyPath = join(home, ".ssh", name);
+      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
+        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
+      }
+    }
+  }
+  throw new Error("no SSH keypair found");
+}
+
+describe("cluster smoke: three servers, one cluster", () => {
+  let h: Harness;
+  let vms: ProvisionedVm[] = [];
+
+  /**
+   * Kubernetes node names, NOT libvirt domain names.
+   *
+   * k3s registers a node by FQDN, so the VM `smoke-a` is the node
+   * `smoke-a.pxe-test.local`. Asserting against the VM names would compare two
+   * different namespaces and fail for a reason that has nothing to do with the
+   * cluster.
+   */
+  const nodeNames = (): string[] => vms.map((v) => k8sNodeName(h, v)).sort();
+
+  beforeAll(async () => {
+    const { pubKey, keyPath } = findSshKey();
+    destroyVms(...NODES);
+    h = await startTestBastion({
+      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey, sshKeyPath: keyPath,
+      label: "cluster-smoke",
+      // Mandatory. Left on, post-provision installs each node into its OWN
+      // cluster and this test would be measuring three clusters of one.
+      autoK3s: false,
+    });
+  }, 20 * 60_000);
+
+  afterAll(async () => {
+    // Dump state before anything is destroyed — the VMs are the only place
+    // this information exists, and they are about to stop existing.
+    try {
+      if (vms.length > 0) printSnapshot("final", clusterSnapshot(h, vms[0]!));
+    } catch { /* cluster may already be gone */ }
+    destroyVms(...NODES);
+    try { destroyPxeNetwork(); } catch { /* shared */ }
+    await h?.close();
+  }, 10 * 60_000);
+
+  it("provisions three v2 combo nodes", async () => {
+    vms = await provisionVms(h, NODES.map((name) => ({
+      name, role: "k8s-combo" as const, schemaVersion: 2 as const,
+      diskGb: DISK_GB, memory: 6144, vcpus: 4,
+    })));
+    expect(vms).toHaveLength(3);
+    for (const vm of vms) expect(vm.ip).toMatch(/^\d+\.\d+\.\d+\.\d+$/);
+  }, 60 * 60_000);
+
+  it("forms ONE cluster, not three", async () => {
+    await formCluster(h, vms, { role: "k8s-combo" });
+
+    const snap = clusterSnapshot(h, vms[0]!);
+    printSnapshot("after formation", snap);
+
+    // The precise failure this guards: three separate single-node clusters
+    // each report 1/1 Ready and look perfectly healthy in isolation.
+    expect(snap.nodes).toHaveLength(3);
+    expect(snap.readyCount).toBe(3);
+    expect(snap.nodes.map((n) => n.name).sort()).toEqual(nodeNames());
+  }, 90 * 60_000);
+
+  it("has exactly three healthy etcd members", () => {
+    const members = etcdMembers(h, vms[0]!);
+    // .node, not .name: etcd names a member "<node>-<8 hex>", so comparing
+    // raw names against node names never matches.
+    expect(members.map((m) => m.node).sort()).toEqual(nodeNames());
+    expect(members).toHaveLength(3);
+    expect(members.every((m) => m.started)).toBe(true);
+    expect(etcdHealthy(h, vms[0]!)).toBe(true);
+  }, 5 * 60_000);
+
+  it("agrees about membership from EVERY node, not just the first", () => {
+    // Asked of each server independently: a split brain is three nodes that
+    // each believe they are correct.
+    for (const vm of vms) {
+      const members = etcdMembers(h, vm).map((m) => m.node).sort();
+      expect(members, `etcd membership as seen by ${vm.name}`).toEqual(nodeNames());
+      const nodes = clusterSnapshot(h, vm).nodes.map((n) => n.name).sort();
+      expect(nodes, `node list as seen by ${vm.name}`).toEqual(nodeNames());
+    }
+  }, 10 * 60_000);
+
+  it("reports every server ready", () => {
+    const ready = allServersReady(h, vms);
+    expect(ready.ok, ready.detail).toBe(true);
+  }, 5 * 60_000);
+
+  it("makes all three control-plane and etcd", () => {
+    const snap = clusterSnapshot(h, vms[0]!);
+    for (const n of snap.nodes) {
+      expect(n.roles, `${n.name} roles`).toContain("control-plane");
+      expect(n.roles, `${n.name} roles`).toContain("etcd");
+    }
+  }, 5 * 60_000);
+
+  it("can pick a join target other than a given node", () => {
+    // The primitive the rolling drill depends on: after the first server is
+    // wiped it must rejoin against a survivor.
+    const target = pickJoinTarget(h, vms, vms[0]!);
+    expect(target.name).not.toBe(vms[0]!.name);
+  }, 5 * 60_000);
+
+  it("carries the Longhorn disk label on every node", () => {
+    // Set via k3s node-label. Without it Longhorn creates no disk on the node,
+    // and the node then hosts zero replicas while looking entirely healthy.
+    const out = kubectlOn(h, vms[0]!,
+      `get nodes -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.labels.node\\.longhorn\\.io/create-default-disk}{"\\n"}{end}'`);
+    for (const vm of vms) {
+      expect(out, `${vm.name} create-default-disk label`)
+        .toContain(`${k8sNodeName(h, vm)}=config`);
+    }
+  }, 5 * 60_000);
+});

bastion/tests/integration/data-disks.test.ts

@@ -0,0 +1,173 @@
+// Integration test: dedicated data disks, on a real VM.
+//
+// The unit tests prove the kickstart RENDERS the right %pre and part lines.
+// This proves anaconda ACTUALLY PRODUCES the layout thebeast will get: OS on
+// one disk, /var/lib/longhorn and /var/lib/rancher each a whole-disk labelled
+// xfs partition on their own disks -- and that a reprovision with NO flags
+// rediscovers both by label and remounts them without reformatting.
+//
+// The VM mirrors thebeast's topology (1 OS disk + 2 data disks addressed by
+// /dev/disk/by-id/ serial); virtio serials stand in for NVMe serials, which is
+// exactly the aliasing the feature is specified over (any by-id path).
+//
+// Run (on a libvirt host):
+//   sudo -E npx vitest run -c tests/integration/vitest.config.ts tests/integration/data-disks.test.ts
+//
+// Set FEDORA_MIRROR to a local mirror or each install pulls packages from the
+// internet (~12-15 min per install either way; two installs in this file).
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { v2Layout, DATA_DISK_LABELS } from "../../src/shared/src/types/layout.js";
+import {
+  startTestBastion, provisionVm, reprovisionVm, readLvs, readSchemaTag,
+  writeMarkers, readMarkers, destroyVms, virtioById,
+  type Harness, type ProvisionedVm,
+} from "./helpers/provision-harness.js";
+import { destroyPxeNetwork } from "./helpers/pxe-network.js";
+import { sshExec } from "./helpers/ssh.js";
+
+const HTTP_PORT = 8094;
+const SSH_USER = "lab";
+const DISK_GB = 1000; // same sparse OS disk as every other layout test
+
+// Short virtio serials (virtio-blk truncates at 20 chars), echoing thebeast's
+// real disk serial fragments so the drill log reads like the runbook.
+const LH = { serial: "LHDRILL7F600711", sizeGb: 20 };
+const RN = { serial: "RNDRILL7FD4070C", sizeGb: 20 };
+
+const PRESERVED = ["/home", "/srv", "/var/lib/longhorn", "/var/lib/rancher"];
+
+function findSshKey(): { pubKey: string; keyPath: string } {
+  // A passphrase-protected default key fails SILENTLY here: BatchMode ssh
+  // never offers it without an agent, the server logs only "Connection closed
+  // [preauth]", and the test reads as "installed machine never came up" when
+  // the machine was fine. LAB_TEST_SSH_KEY names a dedicated passphrase-less
+  // keypair and sidesteps the whole class.
+  const envKey = process.env["LAB_TEST_SSH_KEY"];
+  if (envKey) {
+    if (!existsSync(envKey) || !existsSync(`${envKey}.pub`)) {
+      throw new Error(`LAB_TEST_SSH_KEY=${envKey}: key or .pub missing`);
+    }
+    return { pubKey: readFileSync(`${envKey}.pub`, "utf-8").trim(), keyPath: envKey };
+  }
+  const homes = [homedir()];
+  const sudoUser = process.env["SUDO_USER"];
+  if (sudoUser) homes.push(join("/home", sudoUser));
+  for (const home of homes) {
+    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
+      const keyPath = join(home, ".ssh", name);
+      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
+        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
+      }
+    }
+  }
+  throw new Error("no SSH keypair found");
+}
+
+/** Run a command on the VM and insist it succeeds. */
+function ssh(h: Harness, vm: ProvisionedVm, cmd: string): string {
+  const r = sshExec(vm.ip, h.sshUser, cmd, { keyPath: h.sshKeyPath });
+  if (r.exitCode !== 0) throw new Error(`[${vm.name}] \`${cmd}\` failed: ${r.stderr || r.stdout}`);
+  return r.stdout.trim();
+}
+
+/**
+ * What actually backs a mount point, and whether it is the whole-disk labelled
+ * partition the feature promises.
+ */
+function inspectMount(h: Harness, vm: ProvisionedVm, mount: string, serial: string) {
+  const source = ssh(h, vm, `findmnt -no SOURCE ${mount}`);
+  const fstype = ssh(h, vm, `findmnt -no FSTYPE ${mount}`);
+  const label = ssh(h, vm, `sudo blkid -s LABEL -o value ${source}`);
+  const uuid = ssh(h, vm, `sudo blkid -s UUID -o value ${source}`);
+  // The partition must live on the disk with OUR serial, not wherever the
+  // kernel happened to enumerate something.
+  const disk = ssh(h, vm, `readlink -f ${virtioById(serial)}`);
+  const parent = "/dev/" + ssh(h, vm, `lsblk -ndo pkname ${source}`);
+  return { source, fstype, label, uuid, disk, parent };
+}
+
+describe("dedicated data disks on real hardware", () => {
+  let h: Harness;
+  const created: string[] = [];
+
+  beforeAll(async () => {
+    const { pubKey, keyPath } = findSshKey();
+    h = await startTestBastion({
+      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey,
+      sshKeyPath: keyPath, label: "datadisks",
+    });
+  }, 20 * 60_000);
+
+  afterAll(async () => {
+    // KEEP_VMS=1 leaves the VM behind for a post-mortem (virsh console,
+    // screenshot, mounting the qcow2). Without it a failed boot is destroyed
+    // along with every clue about why it failed.
+    if (!process.env["KEEP_VMS"]) destroyVms(...created);
+    try { await h?.close(); } catch { /* closed */ }
+    try { destroyPxeNetwork(); } catch { /* gone */ }
+  }, 5 * 60_000);
+
+  it("k8s-worker with longhorn+rancher disks: install, then flagless reprovision rediscovers both", async () => {
+    const name = "dd-worker";
+    created.push(name);
+    const dataDisks = { longhorn: virtioById(LH.serial), rancher: virtioById(RN.serial) };
+
+    // --- Fresh install, thebeast-style -----------------------------------
+    let vm = await provisionVm(h, {
+      name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB,
+      dataDisks: { longhorn: LH, rancher: RN },
+    });
+
+    // The VG must hold exactly the v2 set MINUS the LVs the disks replace.
+    const lvs = readLvs(h, vm);
+    const expected = v2Layout("k8s-worker", { dataDisks }).map((lv) => lv.name).sort();
+    expect(Object.keys(lvs).sort(), "volume set").toEqual(expected);
+    expect(lvs["longhorn"]).toBeUndefined();
+    expect(lvs["rancher"]).toBeUndefined();
+    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");
+
+    // Each mount is a whole-disk xfs partition, labelled, on the RIGHT disk.
+    const lhBefore = inspectMount(h, vm, "/var/lib/longhorn", LH.serial);
+    expect(lhBefore.fstype).toBe("xfs");
+    expect(lhBefore.label).toBe(DATA_DISK_LABELS.longhorn);
+    expect(lhBefore.parent, "/var/lib/longhorn sits on the longhorn disk").toBe(lhBefore.disk);
+
+    const rnBefore = inspectMount(h, vm, "/var/lib/rancher", RN.serial);
+    expect(rnBefore.fstype).toBe("xfs");
+    expect(rnBefore.label).toBe(DATA_DISK_LABELS.rancher);
+    expect(rnBefore.parent, "/var/lib/rancher sits on the rancher disk").toBe(rnBefore.disk);
+
+    const token = writeMarkers(h, vm, PRESERVED);
+
+    // --- Reprovision with NO dataDisks flags ------------------------------
+    // This is the claim that matters for day 2: the labels on disk are the
+    // truth, and a plain `labctl provision reprovision` must find and keep
+    // them. Passing the flags again here would hide a regression in exactly
+    // that rediscovery.
+    vm = await reprovisionVm(h, vm, "k8s-worker");
+
+    const markers = readMarkers(h, vm, PRESERVED);
+    for (const mount of PRESERVED) {
+      expect(markers[mount], `${mount} must survive a reinstall`).toBe(token);
+    }
+
+    // Still LV-less for longhorn/rancher, still v2.
+    const lvsAfter = readLvs(h, vm);
+    expect(Object.keys(lvsAfter).sort()).toEqual(expected);
+    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");
+
+    // Same partitions, same filesystems. An unchanged UUID is the proof that
+    // --noformat really happened -- a remade xfs with the same label would
+    // pass every other check here.
+    const lhAfter = inspectMount(h, vm, "/var/lib/longhorn", LH.serial);
+    expect(lhAfter.uuid, "longhorn fs was NOT reformatted").toBe(lhBefore.uuid);
+    expect(lhAfter.parent).toBe(lhAfter.disk);
+    const rnAfter = inspectMount(h, vm, "/var/lib/rancher", RN.serial);
+    expect(rnAfter.uuid, "rancher fs was NOT reformatted").toBe(rnBefore.uuid);
+    expect(rnAfter.parent).toBe(rnAfter.disk);
+  }, 150 * 60_000);
+});

bastion/tests/integration/helpers/k3s-cluster.ts

@@ -0,0 +1,427 @@
+// Form and inspect a real multi-node k3s cluster from an integration test.
+//
+// WHY THIS EXISTS
+// ---------------
+// Nothing in the repo decides which node bootstraps a cluster and which nodes
+// join it. The production generator has the right discriminator --
+// generateServerConfig emits `cluster-init: true` when no server URL is set and
+// `server:`+`token:` when one is -- but the choice is an INPUT, and the only
+// caller that ever supplied it was a hand-run labsim shell script.
+//
+// So a test that wants three servers in one cluster has to sequence it. That
+// sequencing is here rather than in a test file because the rolling drill is
+// not the only thing that will want it: an etcd-recovery test (recoverEtcdMember
+// still has no caller), a dual-stack conversion and any HA upgrade test all
+// need exactly these primitives.
+//
+// Everything drives K3sModule, not shell. That is deliberate and is the same
+// rule labsim follows by shelling out to the compiled render-config binary: a
+// test that reimplements the install proves the test works, not the product.
+
+import { execSync } from "node:child_process";
+import type { Role } from "../../../src/shared/src/types/state.js";
+import { K3sModule } from "../../../src/modules/modules/k3s/src/k3s-module.js";
+import type { ModuleContext } from "../../../src/modules/src/types.js";
+import { log } from "./libvirt.js";
+import { parseEtcdMembers, type EtcdMember } from "../../../src/modules/modules/k3s/src/operations/etcd-members.js";
+import { sshExec } from "./ssh.js";
+import type { Harness, ProvisionedVm } from "./provision-harness.js";
+
+export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * A pre-shared cluster token, the way labsim does it.
+ *
+ * The alternative is scraping /var/lib/rancher/k3s/server/node-token off the
+ * first server, and nothing in the repo does that. It is also the wrong shape
+ * for a rolling migration: the first server gets wiped and rejoins, so the
+ * token has to outlive it and be knowable before it exists.
+ */
+export const DRILL_TOKEN = "lab-drill-shared-token";
+
+/**
+ * k3s operations run unprefixed `mkdir -p /etc/rancher/k3s` and
+ * `systemctl restart k3s`, so they need root. install.ks.ts provisions
+ * `sshkey --username=root` and sets PermitRootLogin prohibit-password, so root
+ * SSH works on every machine this harness builds. The harness's own `lab` user
+ * stays for the sudo-based reads (readLvs, markers).
+ */
+export const MODULE_SSH_USER = "root";
+
+/**
+ * The k3s version the drill runs, pinned to what the fleet runs.
+ *
+ * get.k3s.io installs LATEST, and latest is currently two minor versions ahead
+ * of production (v1.36.4 vs v1.34.6). Two things follow, and both matter:
+ *
+ *  - a rehearsal on a different minor version is rehearsing a different
+ *    system, and Longhorn in particular is only qualified against a range of
+ *    Kubernetes versions;
+ *  - it is the same trap production would hit. A node reinstalled mid-migration
+ *    comes back on latest while its peers stay put, and Kubernetes supports
+ *    only ONE minor of skew between control-plane members.
+ *
+ * Override with LAB_K3S_VERSION to test an upgrade deliberately.
+ */
+export const PINNED_K3S_VERSION = process.env["LAB_K3S_VERSION"] ?? "v1.34.6+k3s1";
+
+/** Build the ModuleContext K3sModule expects for one VM. */
+export function moduleCtx(
+  h: Harness,
+  vm: ProvisionedVm,
+  role: Role,
+  config: Record<string, unknown> = {},
+): ModuleContext {
+  return {
+    hostname: vm.name,
+    ip: vm.ip,
+    role,
+    os: "fedora-43",
+    arch: "x86_64",
+    sshUser: MODULE_SSH_USER,
+    sshKeyPath: h.sshKeyPath,
+    config,
+  };
+}
+
+/** Run a command on a node as root, throwing with context on failure. */
+export function nodeExec(h: Harness, vm: ProvisionedVm, cmd: string, timeoutMs = 120_000): string {
+  const r = sshExec(vm.ip, MODULE_SSH_USER, cmd, { keyPath: h.sshKeyPath, timeout: timeoutMs });
+  if (r.exitCode !== 0) {
+    throw new Error(`[${vm.name}] ${cmd} failed (${r.exitCode}): ${r.stderr.trim() || r.stdout.trim()}`);
+  }
+  return r.stdout;
+}
+
+/** Same, but returns the result instead of throwing. */
+export function tryNodeExec(h: Harness, vm: ProvisionedVm, cmd: string, timeoutMs = 120_000) {
+  return sshExec(vm.ip, MODULE_SSH_USER, cmd, { keyPath: h.sshKeyPath, timeout: timeoutMs });
+}
+
+/**
+ * The name Kubernetes knows this VM by.
+ *
+ * NOT the libvirt domain name. k3s registers a node under its FQDN, so a VM
+ * called `smoke-a` appears as `smoke-a.pxe-test.local`. Passing the VM name to
+ * `kubectl get node` silently finds nothing, and every wait built on it times
+ * out instead of failing usefully. Read from the node itself, the same way the
+ * production operations do (`hostname -f`), so the two can never disagree.
+ */
+const nodeNameCache = new Map<string, string>();
+export function k8sNodeName(h: Harness, vm: ProvisionedVm): string {
+  const cached = nodeNameCache.get(vm.name);
+  if (cached !== undefined) return cached;
+  const name = nodeExec(h, vm, "hostname -f 2>/dev/null || hostname").trim();
+  if (name === "") throw new Error(`[${vm.name}] could not resolve node name`);
+  nodeNameCache.set(vm.name, name);
+  return name;
+}
+
+/** Forget cached names — a reinstalled machine re-registers. */
+export function forgetNodeName(vm: ProvisionedVm): void {
+  nodeNameCache.delete(vm.name);
+}
+
+/** kubectl against a server node. */
+export function kubectlOn(h: Harness, server: ProvisionedVm, args: string, timeoutMs = 120_000): string {
+  return nodeExec(h, server, `k3s kubectl ${args}`, timeoutMs);
+}
+
+export function kubectlJson<T>(h: Harness, server: ProvisionedVm, args: string): T {
+  return JSON.parse(kubectlOn(h, server, `${args} -o json`)) as T;
+}
+
+/**
+ * Wait until a server's API answers /ping.
+ *
+ * Straight from labsim-k8s-etcd.sh, whose comment is the whole justification:
+ * a joiner that starts before the first server has finished bootstrapping etcd
+ * races it, and the failure looks like etcd instability rather than a startup
+ * order problem. Probed from the HOST, so it does not depend on the joining
+ * node being reachable yet.
+ */
+export async function waitForApiPing(vm: ProvisionedVm, timeoutMs = 450_000): Promise<void> {
+  const start = Date.now();
+  let lastErr = "";
+  while (Date.now() - start < timeoutMs) {
+    try {
+      const out = execSync(
+        `curl -sk --max-time 5 https://${vm.ip}:6443/ping || true`,
+        { encoding: "utf-8", timeout: 15_000 },
+      ).trim();
+      if (out === "pong") {
+        log(`[${vm.name}] API answering after ${Math.round((Date.now() - start) / 1000)}s`);
+        return;
+      }
+      lastErr = out;
+    } catch (e) {
+      lastErr = e instanceof Error ? e.message : String(e);
+    }
+    await sleep(5_000);
+  }
+  throw new Error(`[${vm.name}] API never answered /ping in ${timeoutMs / 1000}s (last: ${lastErr || "no response"})`);
+}
+
+/** Wait for a node to appear Ready in the cluster, as seen from `server`. */
+export async function waitForNodeReady(
+  h: Harness,
+  server: ProvisionedVm,
+  /** The VM to wait for; its Kubernetes name is resolved from the machine. */
+  target: ProvisionedVm,
+  // Generous: a node is NotReady until its CNI is up, and on the cluster-init
+  // server that means waiting for Cilium to install and pull its images. Five
+  // minutes is enough on a warm host and not enough on a cold one, and the
+  // cost of being wrong is a wasted multi-hour run.
+  timeoutMs = 900_000,
+): Promise<void> {
+  const nodeName = k8sNodeName(h, target);
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    const r = tryNodeExec(h, server,
+      `k3s kubectl get node ${nodeName} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'`);
+    last = r.stdout.trim();
+    if (last === "True") {
+      log(`[${nodeName}] Ready after ${Math.round((Date.now() - start) / 1000)}s`);
+      return;
+    }
+    await sleep(5_000);
+  }
+  throw new Error(`[${nodeName}] never became Ready in ${timeoutMs / 1000}s (last status: ${last || "absent"})`);
+}
+
+/**
+ * etcdctl invocation.
+ *
+ * NOT `k3s etcdctl` -- that is not a subcommand ("No help topic for 'etcdctl'",
+ * verified on a live v1.34 node). The binary comes from the `etcd` rpm, which
+ * our kickstart does NOT install, so ensureEtcdctl() has to put it there.
+ * Cert paths verified against a real node.
+ */
+const ETCD_BASE = "/var/lib/rancher/k3s/server/tls/etcd";
+const ETCDCTL =
+  `ETCDCTL_API=3 etcdctl ` +
+  `--cacert=${ETCD_BASE}/server-ca.crt ` +
+  `--cert=${ETCD_BASE}/server-client.crt ` +
+  `--key=${ETCD_BASE}/server-client.key ` +
+  `--endpoints=https://127.0.0.1:2379`;
+
+/** Install etcdctl if the node does not have it. Idempotent, ~30s once. */
+export function ensureEtcdctl(h: Harness, server: ProvisionedVm): void {
+  const has = tryNodeExec(h, server, "command -v etcdctl");
+  if (has.exitCode === 0) return;
+  log(`[${server.name}] installing etcdctl (the etcd rpm; kickstart does not include it)`);
+  nodeExec(h, server, "dnf install -y etcd >/dev/null 2>&1 || dnf install -y etcd-ctl >/dev/null 2>&1", 300_000);
+  nodeExec(h, server, "command -v etcdctl");
+}
+
+
+/**
+ * etcd membership, read with etcdctl against k3s's own certs.
+ *
+ * Read rather than inferred from `kubectl get nodes`: a node object can be
+ * deleted while its etcd member survives, and that stale member is exactly the
+ * failure a rolling reinstall creates. Counting nodes would not see it.
+ */
+export function etcdMembers(h: Harness, server: ProvisionedVm): EtcdMember[] {
+  // NOT `-w json`. Two reasons, and the second is the dangerous one:
+  //
+  //  1. `member remove` requires the ID in HEX; the JSON writer emits decimal.
+  //  2. etcd member IDs are uint64. JSON.parse turns them into JS numbers,
+  //     which silently lose precision above 2^53 -- the ID that comes back is
+  //     an APPROXIMATION of the real one. Passing that to `member remove`
+  //     fails with "value out of range" if you are lucky, and in principle
+  //     could match some other member if you are not. This happens at the one
+  //     moment it must not: the node is already stopped and about to be wiped.
+  //
+  // The default writer prints the ID as hex text, so it never becomes a
+  // number at all:
+  //   825852d3683bc406, started, worker2-...-5f7ad0c9, https://...:2380, ...
+  return parseEtcdMembers(nodeExec(h, server, `${ETCDCTL} member list`));
+}
+
+
+/** Are all etcd endpoints healthy? Answers the question quorum actually asks. */
+export function etcdHealthy(h: Harness, server: ProvisionedVm): boolean {
+  const r = tryNodeExec(h, server, `${ETCDCTL} endpoint health`);
+  return r.exitCode === 0 && r.stdout.includes("is healthy");
+}
+
+/**
+ * Remove an etcd member by node name.
+ *
+ * Must happen BEFORE the node is wiped. k3s-install.ts removes the local
+ * datastore when a server joins, so a reinstalled node rejoins as a NEW member
+ * while the old record persists: 3 members becomes 4, quorum rises from 2 to 3,
+ * and the cluster that tolerated one failure now tolerates none.
+ */
+export function removeEtcdMember(h: Harness, peer: ProvisionedVm, nodeName: string): boolean {
+  const members = etcdMembers(h, peer);
+  const target = members.find((m) => m.node === nodeName || m.name === nodeName);
+  if (!target) {
+    log(`[etcd] no member for ${nodeName} (have: ${members.map((m) => m.node).join(",")})`);
+    return false;
+  }
+  nodeExec(h, peer, `${ETCDCTL} member remove ${target.id}`);
+  log(`[etcd] removed member ${target.name} (${target.id})`);
+  return true;
+}
+
+export interface ClusterSnapshot {
+  nodes: { name: string; ready: boolean; roles: string }[];
+  readyCount: number;
+  etcdMembers: number;
+  etcdHealthy: boolean;
+}
+
+/** One read of cluster state, in the shape labsim's cmd_status prints. */
+export function clusterSnapshot(h: Harness, server: ProvisionedVm): ClusterSnapshot {
+  const raw = kubectlJson<{ items: {
+    metadata: { name: string; labels?: Record<string, string> };
+    status: { conditions?: { type: string; status: string }[] };
+  }[] }>(h, server, "get nodes");
+
+  const nodes = raw.items.map((n) => ({
+    name: n.metadata.name,
+    ready: (n.status.conditions ?? []).some((c) => c.type === "Ready" && c.status === "True"),
+    roles: Object.keys(n.metadata.labels ?? {})
+      .filter((k) => k.startsWith("node-role.kubernetes.io/"))
+      .map((k) => k.replace("node-role.kubernetes.io/", ""))
+      .sort().join(",") || "<none>",
+  }));
+
+  let members = 0;
+  let healthy = false;
+  try {
+    members = etcdMembers(h, server).length;
+    healthy = etcdHealthy(h, server);
+  } catch { /* an agent, or the API is down — report zero rather than throw */ }
+
+  return { nodes, readyCount: nodes.filter((n) => n.ready).length, etcdMembers: members, etcdHealthy: healthy };
+}
+
+/**
+ * Print a snapshot. Called liberally during a drill: the VMs are destroyed at
+ * the end, so this transcript is the only surviving record of what happened.
+ */
+export function printSnapshot(label: string, s: ClusterSnapshot): void {
+  log(`[snapshot] ${label}: ${s.readyCount}/${s.nodes.length} Ready, ` +
+      `etcd members=${s.etcdMembers} healthy=${s.etcdHealthy}`);
+  for (const n of s.nodes) log(`             ${n.ready ? "Ready   " : "NotReady"} ${n.name} (${n.roles})`);
+}
+
+/**
+ * Install k3s on one node via the production module.
+ *
+ * Pass no serverUrl to bootstrap the cluster (the token still seeds), or a
+ * serverUrl to join an existing one.
+ */
+export async function installK3sNode(h: Harness, vm: ProvisionedVm, opts: {
+  role: Role;
+  serverUrl?: string;
+  token?: string;
+}): Promise<void> {
+  const mod = new K3sModule();
+  const config: Record<string, unknown> = {
+    k3sToken: opts.token ?? DRILL_TOKEN,
+    k3sVersion: PINNED_K3S_VERSION,
+    ...(opts.serverUrl === undefined ? {} : { k3sServerUrl: opts.serverUrl }),
+  };
+
+  log(`[${vm.name}] installing k3s (${opts.role}${opts.serverUrl ? ` -> ${opts.serverUrl}` : " — cluster-init"})`);
+  const install = await mod.install(moduleCtx(h, vm, opts.role, config));
+  for (const line of install.output) log(`  [${vm.name}] ${line}`);
+  if (!install.success) {
+    throw new Error(`[${vm.name}] k3s install failed: ${install.errors.join("; ")}`);
+  }
+
+  const configure = await mod.configure(moduleCtx(h, vm, opts.role, config));
+  for (const line of configure.output) log(`  [${vm.name}] ${line}`);
+  if (!configure.success) {
+    throw new Error(`[${vm.name}] k3s configure failed: ${configure.errors.join("; ")}`);
+  }
+}
+
+export function serverUrlOf(vm: ProvisionedVm): string {
+  return `https://${vm.ip}:6443`;
+}
+
+/**
+ * Form a cluster from a list of server VMs, strictly in order.
+ *
+ * The first bootstraps etcd and installs the CNI (k3s-module runs networking
+ * only on the cluster-init node -- joiners get Cilium via its DaemonSet). The
+ * rest join one at a time, each gated on the previous node being Ready.
+ *
+ * Serial on purpose. labsim gets away with starting joiners together because
+ * its nodes are cloud-init VMs that poll /ping themselves; here the few extra
+ * minutes buy determinism in the one place where etcd bootstrap genuinely
+ * races.
+ */
+export async function formCluster(h: Harness, servers: ProvisionedVm[], opts: {
+  role?: Role;
+  token?: string;
+} = {}): Promise<void> {
+  if (servers.length === 0) throw new Error("formCluster: no servers");
+  const role = opts.role ?? "k8s-combo";
+  const token = opts.token ?? DRILL_TOKEN;
+  const first = servers[0]!;
+
+  await installK3sNode(h, first, { role, token });
+  await waitForApiPing(first);
+  await waitForNodeReady(h, first, first);
+  ensureEtcdctl(h, first);
+  printSnapshot("after cluster-init", clusterSnapshot(h, first));
+
+  for (const vm of servers.slice(1)) {
+    await waitForApiPing(first);
+    await installK3sNode(h, vm, { role, token, serverUrl: serverUrlOf(first) });
+    await waitForNodeReady(h, first, vm);
+    ensureEtcdctl(h, vm);
+    printSnapshot(`after joining ${vm.name}`, clusterSnapshot(h, first));
+  }
+}
+
+/** Join an agent. Agents cannot start without a server URL and token. */
+export async function joinAgent(h: Harness, vm: ProvisionedVm, opts: {
+  server: ProvisionedVm;
+  role?: Role;
+  token?: string;
+}): Promise<void> {
+  await waitForApiPing(opts.server);
+  await installK3sNode(h, vm, {
+    role: opts.role ?? "k8s-worker",
+    token: opts.token ?? DRILL_TOKEN,
+    serverUrl: serverUrlOf(opts.server),
+  });
+  await waitForNodeReady(h, opts.server, vm);
+}
+
+/**
+ * A healthy server that is NOT the node being worked on.
+ *
+ * Needed because the first server is itself rolled during a migration: after
+ * it is wiped it must rejoin against a survivor, so "the first node" cannot be
+ * the join target and cannot be where cluster state is read from.
+ */
+export function pickJoinTarget(
+  h: Harness,
+  servers: ProvisionedVm[],
+  exclude: ProvisionedVm,
+): ProvisionedVm {
+  for (const vm of servers) {
+    if (vm.name === exclude.name) continue;
+    const r = tryNodeExec(h, vm, "k3s kubectl get --raw /readyz");
+    if (r.exitCode === 0 && r.stdout.trim() === "ok") return vm;
+  }
+  throw new Error(`No healthy server other than ${exclude.name} — cluster has no quorum to join against`);
+}
+
+/** Every server reports /readyz ok. Asserted positively, not inferred. */
+export function allServersReady(h: Harness, servers: ProvisionedVm[]): { ok: boolean; detail: string } {
+  const results = servers.map((vm) => {
+    const r = tryNodeExec(h, vm, "k3s kubectl get --raw /readyz");
+    return `${vm.name}=${r.exitCode === 0 ? r.stdout.trim() : `ERR(${r.exitCode})`}`;
+  });
+  return { ok: results.every((r) => r.endsWith("=ok")), detail: results.join(" ") };
+}

bastion/tests/integration/helpers/longhorn.ts

@@ -0,0 +1,666 @@
+// Install Longhorn on a test cluster and put verifiable data in it.
+//
+// WHAT THIS IS FOR
+// ----------------
+// The rolling drill's whole claim is "the bytes survived". That needs data
+// whose integrity can be checked independently of Longhorn's own opinion of
+// itself -- Longhorn reporting `robustness: healthy` is not evidence that the
+// file is intact, it is evidence that Longhorn thinks it is.
+//
+// So: write a known blob, record its sha256, and re-verify THROUGH A POD
+// RESTART every time. The restart is not incidental. It forces a real
+// detach/attach, frequently onto a different node, which is the only way to
+// show the data followed the volume rather than sitting in a page cache on the
+// node that wrote it.
+//
+// Longhorn version is pinned to the fleet's (1.8.1) so the drill rehearses what
+// production runs, not what happens to be latest.
+
+import type { Harness, ProvisionedVm } from "./provision-harness.js";
+import { log } from "./libvirt.js";
+import { nodeExec, tryNodeExec, kubectlOn, kubectlJson, sleep, k8sNodeName } from "./k3s-cluster.js";
+
+export const LONGHORN_VERSION = "1.8.1";
+export const LONGHORN_NS = "longhorn-system";
+
+/**
+ * Install Longhorn via k3s's built-in helm controller.
+ *
+ * A HelmChart manifest dropped in the server manifests directory is how k3s
+ * installs charts natively, so this needs no helm binary on the host and no
+ * kubeconfig plumbing.
+ *
+ * The namespace is created FIRST, with privileged Pod Security labels. Longhorn
+ * needs hostPath and privileged containers; under the default `restricted`
+ * enforcement its DaemonSets are rejected and the install fails in a way that
+ * looks like a chart problem.
+ *
+ * `createDefaultDiskLabeledNodes: true` matches production, and is the setting
+ * that makes the node label and disk annotation load-bearing: without both, a
+ * node gets no disk and silently hosts no replicas.
+ */
+/**
+ * S3 backup target for the drill, from the environment.
+ *
+ * Absent means the backup/restore phase is skipped rather than silently
+ * "passing" -- a backup test that quietly does nothing is worse than no test.
+ * The run script exports these; see DRILL_S3_* in run-drill.sh.
+ */
+export interface BackupTargetOpts {
+  /** e.g. s3://longhorn-drill-backup@us-east-1/ */
+  target: string;
+  endpoint: string;
+  accessKey: string;
+  secretKey: string;
+}
+
+export function backupTargetFromEnv(): BackupTargetOpts | undefined {
+  const target = process.env["DRILL_S3_TARGET"];
+  const endpoint = process.env["DRILL_S3_ENDPOINT"];
+  const accessKey = process.env["DRILL_S3_ACCESS_KEY"];
+  const secretKey = process.env["DRILL_S3_SECRET_KEY"];
+  if (target === undefined || endpoint === undefined
+    || accessKey === undefined || secretKey === undefined) return undefined;
+  return { target, endpoint, accessKey, secretKey };
+}
+
+export const BACKUP_SECRET = "longhorn-backup-credentials";
+
+export async function installLonghorn(h: Harness, server: ProvisionedVm, opts: {
+  defaultReplicas?: number;
+  backup?: BackupTargetOpts;
+} = {}): Promise<void> {
+  const replicas = opts.defaultReplicas ?? 3;
+
+  log(`[longhorn] creating ${LONGHORN_NS} with privileged PSA labels`);
+  nodeExec(h, server, `k3s kubectl apply -f - <<'EOF'
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ${LONGHORN_NS}
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged
+EOF`);
+
+  if (opts.backup !== undefined) {
+    // Secret BEFORE the chart: Longhorn reads it by name from a defaultSetting,
+    // and a missing secret makes the first backup fail with a credential error
+    // that reads like a network problem.
+    //
+    // AWS_ENDPOINTS is what makes this work against a non-AWS gateway; without
+    // it the driver resolves real AWS endpoints.
+    log(`[longhorn] configuring backup target ${opts.backup.target}`);
+    // tryNodeExec, not nodeExec: nodeExec throws with the whole command in the
+    // message, and this command contains the S3 secret. A credential landing in
+    // a log that then gets pasted into drill evidence is not acceptable, so the
+    // failure is reported without the payload.
+    const secretApplied = tryNodeExec(h, server, `k3s kubectl -n ${LONGHORN_NS} apply -f - <<'EOF'
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ${BACKUP_SECRET}
+  namespace: ${LONGHORN_NS}
+stringData:
+  AWS_ACCESS_KEY_ID: ${opts.backup.accessKey}
+  AWS_SECRET_ACCESS_KEY: ${opts.backup.secretKey}
+  AWS_ENDPOINTS: ${opts.backup.endpoint}
+EOF`);
+    if (secretApplied.exitCode !== 0) {
+      throw new Error(
+        `[${server.name}] could not create the ${BACKUP_SECRET} secret ` +
+        `(exit ${secretApplied.exitCode}). Command withheld: it contains credentials. ` +
+        `stderr: ${secretApplied.stderr.trim().slice(0, 200)}`);
+    }
+  }
+
+  log(`[longhorn] installing chart ${LONGHORN_VERSION} via k3s helm controller`);
+  nodeExec(h, server, `cat > /var/lib/rancher/k3s/server/manifests/longhorn.yaml <<'EOF'
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: longhorn
+  namespace: kube-system
+spec:
+  chart: longhorn
+  repo: https://charts.longhorn.io
+  version: ${LONGHORN_VERSION}
+  targetNamespace: ${LONGHORN_NS}
+  valuesContent: |-
+    defaultSettings:
+      defaultDataPath: /var/lib/longhorn
+      defaultReplicaCount: ${replicas}
+      createDefaultDiskLabeledNodes: true
+      defaultDataLocality: best-effort
+      replicaSoftAntiAffinity: false
+      replicaReplenishmentWaitInterval: 600
+      orphanAutoDeletion: true
+    persistence:
+      defaultClass: true
+      defaultClassReplicaCount: ${replicas}
+      defaultFsType: ext4
+    longhornUI:
+      replicas: 1
+EOF`);
+}
+
+/**
+ * Every expected node has a Longhorn disk that is actually schedulable.
+ *
+ * Takes the VMs, not names, and resolves each one's Kubernetes name itself.
+ * Longhorn keys its node records by the FQDN the kubelet registered
+ * (`drill-s1.pxe-test.local`), never the libvirt domain name (`drill-s1`), so
+ * a caller passing VM names matches nothing and waits out the full timeout
+ * while the cluster is in fact perfectly healthy.
+ */
+export async function waitForLonghornReady(
+  h: Harness,
+  server: ProvisionedVm,
+  vms: ProvisionedVm[],
+  timeoutMs = 20 * 60_000,
+): Promise<void> {
+  const expectedNodes = vms.map((vm) => k8sNodeName(h, vm));
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    const r = tryNodeExec(h, server,
+      `k3s kubectl -n ${LONGHORN_NS} get nodes.longhorn.io -o json`, 60_000);
+    if (r.exitCode === 0) {
+      try {
+        const parsed = JSON.parse(r.stdout) as { items: {
+          metadata: { name: string };
+          status?: { diskStatus?: Record<string, { conditions?: { type: string; status: string }[] }> };
+        }[] };
+        const ready = parsed.items.filter((n) => {
+          const disks = Object.values(n.status?.diskStatus ?? {});
+          return disks.length > 0 && disks.some((d) =>
+            (d.conditions ?? []).some((c) => c.type === "Schedulable" && c.status === "True"));
+        }).map((n) => n.metadata.name);
+
+        last = `${ready.length}/${expectedNodes.length} with schedulable disk: ${ready.join(",") || "none"}`;
+        if (expectedNodes.every((n) => ready.includes(n))) {
+          log(`[longhorn] ready after ${Math.round((Date.now() - start) / 1000)}s — ${last}`);
+          return;
+        }
+      } catch { /* API not serving the CRD yet */ }
+    }
+    await sleep(15_000);
+  }
+
+  // Say WHY, per node, instead of just "not ready". A node with no disk has a
+  // specific cause and the record knows it; without this the only way to find
+  // out is to catch the cluster before teardown, which is a race you lose.
+  const detail: string[] = [];
+  try {
+    const raw = kubectlJson<{ items: {
+      metadata: { name: string; annotations?: Record<string, string>; labels?: Record<string, string> };
+      spec?: { disks?: Record<string, unknown>; allowScheduling?: boolean };
+      status?: {
+        conditions?: { type: string; status: string; reason?: string; message?: string }[];
+        diskStatus?: Record<string, { conditions?: { type: string; status: string; reason?: string; message?: string }[] }>;
+      };
+    }[] }>(h, server, `-n ${LONGHORN_NS} get nodes.longhorn.io`);
+
+    for (const n of raw.items) {
+      const disks = Object.keys(n.spec?.disks ?? {});
+      const conds = (n.status?.conditions ?? [])
+        .filter((c) => c.status !== "True")
+        .map((c) => `${c.type}=${c.status}(${c.reason ?? ""}: ${(c.message ?? "").slice(0, 80)})`);
+      const diskConds = Object.entries(n.status?.diskStatus ?? {}).flatMap(([dn, d]) =>
+        (d.conditions ?? []).filter((c) => c.status !== "True")
+          .map((c) => `${dn}:${c.type}=${c.status}(${c.reason ?? ""}: ${(c.message ?? "").slice(0, 80)})`));
+      const ann = n.metadata.annotations?.["node.longhorn.io/default-disks-config"] ?? "<absent>";
+      const lbl = n.metadata.labels?.["node.longhorn.io/create-default-disk"] ?? "<absent>";
+      detail.push(
+        `  ${n.metadata.name}: disks=[${disks.join(",")}] label=${lbl} ann=${ann} ` +
+        `${conds.length > 0 ? `nodeConds=${conds.join(";")} ` : ""}` +
+        `${diskConds.length > 0 ? `diskConds=${diskConds.join(";")}` : ""}`);
+    }
+  } catch (e) {
+    detail.push(`  (could not read Longhorn nodes: ${e instanceof Error ? e.message : String(e)})`);
+  }
+
+  throw new Error(
+    `Longhorn not ready in ${timeoutMs / 60000} min (${last}).\n` +
+    `A node with no disk usually means the create-default-disk label or the\n` +
+    `default-disks-config annotation is missing, or the path is not mounted.\n` +
+    detail.join("\n"));
+}
+
+/** Create a PVC plus a Deployment that mounts it at /data. */
+export function createDataWorkload(h: Harness, server: ProvisionedVm, opts: {
+  name: string;
+  sizeGi: number;
+  replicas: number;
+}): void {
+  log(`[longhorn] creating ${opts.name} (${opts.sizeGi}Gi, ${opts.replicas} replicas)`);
+  nodeExec(h, server, `k3s kubectl apply -f - <<'EOF'
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: lh-${opts.name}
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+parameters:
+  numberOfReplicas: "${opts.replicas}"
+  staleReplicaTimeout: "30"
+  fsType: ext4
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ${opts.name}
+spec:
+  accessModes: [ReadWriteOnce]
+  storageClassName: lh-${opts.name}
+  resources:
+    requests:
+      storage: ${opts.sizeGi}Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ${opts.name}
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels: {app: ${opts.name}}
+  template:
+    metadata:
+      labels: {app: ${opts.name}}
+    spec:
+      # Pod Security Admission enforces "restricted" on this namespace (the k3s
+      # hardening group applies the label), so a bare pod spec is REJECTED
+      # outright -- the Deployment reports 0/1 with no pod and no obvious
+      # reason unless you read the ReplicaSet events. Every one of these
+      # fields is required by "restricted"; dropping any brings it back.
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        # fsGroup so the non-root user can actually write to the Longhorn
+        # volume, which is otherwise owned by root.
+        fsGroup: 1000
+        seccompProfile: {type: RuntimeDefault}
+      containers:
+        - name: shell
+          image: busybox:1.36
+          command: ["sh", "-c", "sleep infinity"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: ["ALL"]}
+          volumeMounts:
+            - {name: data, mountPath: /data}
+      volumes:
+        - name: data
+          persistentVolumeClaim: {claimName: ${opts.name}}
+EOF`);
+}
+
+/** Wait for the workload's pod to be Running and its volume attached. */
+export async function waitForWorkload(
+  h: Harness, server: ProvisionedVm, name: string, timeoutMs = 10 * 60_000,
+): Promise<void> {
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    const r = tryNodeExec(h, server,
+      `k3s kubectl get pods -l app=${name} -o jsonpath='{.items[*].status.phase}'`);
+    last = r.stdout.trim();
+    if (last === "Running") {
+      // Ready, not merely scheduled: the volume mount happens before Ready.
+      const ready = tryNodeExec(h, server,
+        `k3s kubectl get pods -l app=${name} -o jsonpath='{.items[*].status.containerStatuses[*].ready}'`);
+      if (ready.stdout.trim() === "true") return;
+    }
+    await sleep(5_000);
+  }
+  throw new Error(`workload ${name} not running in ${timeoutMs / 60000} min (last phase: ${last || "none"})`);
+}
+
+function podOf(h: Harness, server: ProvisionedVm, name: string): string {
+  const pod = kubectlOn(h, server, `get pods -l app=${name} -o jsonpath='{.items[0].metadata.name}'`).trim();
+  if (!pod) throw new Error(`no pod for workload ${name}`);
+  return pod;
+}
+
+/**
+ * Write a random blob, fsync it, and record its digest.
+ *
+ * conv=fsync matters: without it the write can still be in the guest's page
+ * cache, and a checksum taken immediately afterwards proves nothing about what
+ * reached the replicas.
+ */
+export async function writeBlob(h: Harness, server: ProvisionedVm, opts: {
+  name: string; mib: number;
+}): Promise<string> {
+  const pod = podOf(h, server, opts.name);
+  log(`[longhorn] writing ${opts.mib} MiB to ${opts.name}`);
+  nodeExec(h, server,
+    `k3s kubectl exec ${pod} -- sh -c ` +
+    `"dd if=/dev/urandom of=/data/blob bs=1M count=${opts.mib} conv=fsync 2>/dev/null && ` +
+    `cd /data && sha256sum blob > blob.sha256 && sync"`,
+    15 * 60_000);
+
+  const digest = kubectlOn(h, server, `exec ${pod} -- cat /data/blob.sha256`).trim().split(/\s+/)[0] ?? "";
+  if (!/^[0-9a-f]{64}$/.test(digest)) throw new Error(`bad digest for ${opts.name}: ${digest}`);
+  log(`[longhorn] ${opts.name} digest ${digest.slice(0, 16)}...`);
+  return digest;
+}
+
+/**
+ * Restart the pod, then verify the blob.
+ *
+ * The restart is the point. It detaches and reattaches the volume, so a pass
+ * means the bytes came back from Longhorn rather than from wherever the writer
+ * left them.
+ */
+export async function verifyBlob(h: Harness, server: ProvisionedVm, opts: {
+  name: string; expectDigest: string;
+}): Promise<void> {
+  const before = podOf(h, server, opts.name);
+  kubectlOn(h, server, `delete pod ${before} --wait=true --timeout=5m`, 6 * 60_000);
+  await waitForWorkload(h, server, opts.name);
+  const after = podOf(h, server, opts.name);
+  if (after === before) throw new Error(`${opts.name}: pod was not actually replaced`);
+
+  // sha256sum -c proves the file matches what was recorded IN the volume...
+  const check = tryNodeExec(h, server,
+    `k3s kubectl exec ${after} -- sh -c "cd /data && sha256sum -c blob.sha256"`, 15 * 60_000);
+  if (check.exitCode !== 0) {
+    throw new Error(`${opts.name}: DATA CORRUPTION — sha256sum -c failed: ${check.stdout}${check.stderr}`);
+  }
+
+  // ...and this proves the recorded digest is still the one we started with,
+  // which -c alone would not catch if both file and checksum were replaced.
+  const actual = kubectlOn(h, server,
+    `exec ${after} -- sh -c "cd /data && sha256sum blob"`, 15 * 60_000).trim().split(/\s+/)[0];
+  if (actual !== opts.expectDigest) {
+    throw new Error(`${opts.name}: DIGEST CHANGED — expected ${opts.expectDigest}, got ${actual}`);
+  }
+  log(`[longhorn] ${opts.name} verified intact across a detach/attach (pod ${before} -> ${after})`);
+}
+
+export interface VolumeState {
+  pv: string;
+  state: string;
+  robustness: string;
+  wanted: number;
+  replicaNodes: string[];
+  healthyReplicas: number;
+}
+
+/** Longhorn's view of the volume backing a PVC. */
+export function volumeState(h: Harness, server: ProvisionedVm, pvcName: string): VolumeState {
+  const pv = kubectlOn(h, server, `get pvc ${pvcName} -o jsonpath='{.spec.volumeName}'`).trim();
+  if (!pv) throw new Error(`PVC ${pvcName} has no bound volume`);
+
+  const vol = kubectlJson<{
+    spec: { numberOfReplicas?: number };
+    status?: { state?: string; robustness?: string };
+  }>(h, server, `-n ${LONGHORN_NS} get volumes.longhorn.io ${pv}`);
+
+  const reps = kubectlJson<{ items: {
+    spec: { nodeID?: string; volumeName?: string };
+    status?: { currentState?: string; started?: boolean };
+  }[] }>(h, server, `-n ${LONGHORN_NS} get replicas.longhorn.io`);
+
+  const mine = reps.items.filter((r) => r.spec.volumeName === pv);
+  return {
+    pv,
+    state: vol.status?.state ?? "unknown",
+    robustness: vol.status?.robustness ?? "unknown",
+    wanted: vol.spec.numberOfReplicas ?? 0,
+    replicaNodes: mine.map((r) => r.spec.nodeID ?? "").filter(Boolean).sort(),
+    healthyReplicas: mine.filter((r) => r.status?.currentState === "running").length,
+  };
+}
+
+/**
+ * Wait for a volume to be healthy on the expected number of distinct nodes.
+ *
+ * Logs rebuild progress while waiting, so a slow rebuild reads as progress
+ * rather than as an opaque hour of silence.
+ */
+export async function waitForVolumeHealthy(h: Harness, server: ProvisionedVm, opts: {
+  pvc: string; expectReplicas: number; timeoutMs?: number;
+}): Promise<VolumeState> {
+  const timeoutMs = opts.timeoutMs ?? 60 * 60_000;
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    let st: VolumeState | null = null;
+    try { st = volumeState(h, server, opts.pvc); } catch { /* transient */ }
+
+    if (st) {
+      const distinct = new Set(st.replicaNodes).size;
+      last = `${st.state}/${st.robustness} on ${distinct} node(s): ${st.replicaNodes.join(",")}`;
+      if (st.robustness === "healthy" && distinct >= opts.expectReplicas) {
+        log(`[longhorn] ${opts.pvc} healthy on ${distinct} nodes after ${Math.round((Date.now() - start) / 1000)}s`);
+        return st;
+      }
+      const prog = tryNodeExec(h, server,
+        `k3s kubectl -n ${LONGHORN_NS} get engines.longhorn.io -o jsonpath=` +
+        `'{range .items[*]}{.spec.volumeName}{" "}{.status.rebuildStatus}{"\\n"}{end}'`);
+      const line = prog.stdout.split("\n").find((l) => l.includes(st!.pv) && l.trim() !== st!.pv);
+      log(`[longhorn] ${opts.pvc}: ${last}${line ? ` | rebuild ${line.replace(st.pv, "").trim()}` : ""}`);
+    }
+    await sleep(20_000);
+  }
+  throw new Error(`${opts.pvc} not healthy on ${opts.expectReplicas} nodes in ${timeoutMs / 60000} min (last: ${last})`);
+}
+
+/**
+ * Wait until Longhorn reports the backup target usable.
+ *
+ * `available: true` is the appliance saying it could reach the endpoint AND
+ * authenticate. Backing up without checking this first produces a failure that
+ * looks like a volume problem.
+ */
+export async function setBackupTarget(
+  h: Harness, server: ProvisionedVm, backup: BackupTargetOpts, timeoutMs = 5 * 60_000,
+): Promise<void> {
+  // PATCH the CR; do NOT set defaultSettings.backupTarget.
+  //
+  // In Longhorn 1.8 the backup target moved out of settings and onto
+  // `backuptargets.longhorn.io/default`. The helm value is SILENTLY IGNORED --
+  // no `backup-target` Setting object is created at all, the default
+  // BackupTarget keeps an empty URL, and everything reports success. Observed
+  // on a fresh 1.8.1 install here and confirmed against the live fleet.
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const exists = tryNodeExec(h, server,
+      `k3s kubectl -n ${LONGHORN_NS} get backuptargets.longhorn.io default -o name`);
+    if (exists.exitCode === 0 && exists.stdout.trim() !== "") break;
+    await sleep(5_000);
+  }
+
+  const patch = JSON.stringify({
+    spec: { backupTargetURL: backup.target, credentialSecret: BACKUP_SECRET, pollInterval: "5m0s" },
+  });
+  nodeExec(h, server,
+    `k3s kubectl -n ${LONGHORN_NS} patch backuptargets.longhorn.io default --type=merge -p '${patch}'`);
+  log(`[longhorn] backup target set to ${backup.target}`);
+}
+
+export async function waitForBackupTarget(
+  h: Harness, server: ProvisionedVm, timeoutMs = 5 * 60_000,
+): Promise<void> {
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    const r = tryNodeExec(h, server,
+      `k3s kubectl -n ${LONGHORN_NS} get backuptargets.longhorn.io default ` +
+      `-o jsonpath='{.spec.backupTargetURL}|{.status.available}|{.status.conditions[*].message}'`);
+    last = r.stdout.trim();
+    if (r.exitCode === 0 && last.includes("|true|")) {
+      log(`[longhorn] backup target available after ${Math.round((Date.now() - start) / 1000)}s`);
+      return;
+    }
+    await sleep(10_000);
+  }
+  throw new Error(
+    `Backup target not available in ${timeoutMs / 60000} min (url|available|conditions = ${last}). ` +
+    `Longhorn could not reach or authenticate to the endpoint.`);
+}
+
+/**
+ * Snapshot a volume and back it up, returning the backup's name.
+ *
+ * Longhorn's own CRs, not the CSI VolumeSnapshot API -- this cluster has no
+ * snapshot.storage.k8s.io resources at all, so VolumeSnapshot/VolumeSnapshotClass
+ * simply do not exist here. Any backup tooling has to use these.
+ */
+export async function backupVolume(
+  h: Harness, server: ProvisionedVm, pvc: string, timeoutMs = 20 * 60_000,
+): Promise<{ backup: string; volume: string }> {
+  const pv = volumeState(h, server, pvc).pv;
+  const snapName = `drill-snap-${Date.now().toString(36)}`;
+
+  log(`[longhorn] snapshotting ${pvc} (${pv})`);
+  nodeExec(h, server, `k3s kubectl -n ${LONGHORN_NS} apply -f - <<'EOF'
+apiVersion: longhorn.io/v1beta2
+kind: Snapshot
+metadata:
+  name: ${snapName}
+  namespace: ${LONGHORN_NS}
+spec:
+  volume: ${pv}
+  createSnapshot: true
+EOF`);
+
+  // The snapshot has to exist before a backup can reference it.
+  const snapDeadline = Date.now() + 5 * 60_000;
+  while (Date.now() < snapDeadline) {
+    const r = tryNodeExec(h, server,
+      `k3s kubectl -n ${LONGHORN_NS} get snapshots.longhorn.io ${snapName} -o jsonpath='{.status.readyToUse}'`);
+    if (r.stdout.trim() === "true") break;
+    await sleep(5_000);
+  }
+
+  const backupName = `drill-bak-${Date.now().toString(36)}`;
+  log(`[longhorn] backing up ${pvc} to the backup target`);
+  nodeExec(h, server, `k3s kubectl -n ${LONGHORN_NS} apply -f - <<'EOF'
+apiVersion: longhorn.io/v1beta2
+kind: Backup
+metadata:
+  name: ${backupName}
+  namespace: ${LONGHORN_NS}
+  labels:
+    backup-volume: ${pv}
+spec:
+  snapshotName: ${snapName}
+EOF`);
+
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    const r = tryNodeExec(h, server,
+      `k3s kubectl -n ${LONGHORN_NS} get backups.longhorn.io ${backupName} ` +
+      `-o jsonpath='{.status.state}|{.status.progress}|{.status.error}'`);
+    last = r.stdout.trim();
+    if (last.startsWith("Completed")) {
+      log(`[longhorn] backup ${backupName} completed after ${Math.round((Date.now() - start) / 1000)}s`);
+      return { backup: backupName, volume: pv };
+    }
+    if (last.includes("Error")) {
+      throw new Error(`Backup ${backupName} failed: ${last}`);
+    }
+    await sleep(10_000);
+  }
+  throw new Error(`Backup ${backupName} did not complete in ${timeoutMs / 60000} min (state|progress|error = ${last})`);
+}
+
+/**
+ * Restore a backup into a brand-new PVC and workload.
+ *
+ * Uses the `fromBackup` StorageClass parameter, which is the supported route
+ * when the CSI VolumeSnapshot API is unavailable. The restored volume is a
+ * separate volume with its own replicas -- nothing about the original is
+ * touched, so a failed restore cannot damage the source.
+ */
+export async function restoreFromBackup(h: Harness, server: ProvisionedVm, opts: {
+  backup: string;
+  name: string;
+  sizeGi: number;
+}): Promise<void> {
+  const url = nodeExec(h, server,
+    `k3s kubectl -n ${LONGHORN_NS} get backups.longhorn.io ${opts.backup} -o jsonpath='{.status.url}'`).trim();
+  if (url === "") throw new Error(`Backup ${opts.backup} has no status.url; cannot restore from it`);
+  log(`[longhorn] restoring ${opts.backup} into ${opts.name}`);
+
+  nodeExec(h, server, `k3s kubectl apply -f - <<'EOF'
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: lh-${opts.name}
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+parameters:
+  numberOfReplicas: "2"
+  staleReplicaTimeout: "30"
+  fsType: ext4
+  fromBackup: "${url}"
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ${opts.name}
+spec:
+  accessModes: [ReadWriteOnce]
+  storageClassName: lh-${opts.name}
+  resources:
+    requests:
+      storage: ${opts.sizeGi}Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ${opts.name}
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels: {app: ${opts.name}}
+  template:
+    metadata:
+      labels: {app: ${opts.name}}
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        seccompProfile: {type: RuntimeDefault}
+      containers:
+        - name: shell
+          image: busybox:1.36
+          command: ["sh", "-c", "sleep infinity"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: ["ALL"]}
+          volumeMounts:
+            - {name: data, mountPath: /data}
+      volumes:
+        - name: data
+          persistentVolumeClaim: {claimName: ${opts.name}}
+EOF`);
+}
+
+/**
+ * Read a blob's digest from a workload, without restarting it.
+ *
+ * Used on a RESTORED volume, where the point is to compare bytes against the
+ * original's recorded digest rather than to prove durability again.
+ */
+export function readBlobDigest(h: Harness, server: ProvisionedVm, name: string): string {
+  const pod = kubectlOn(h, server, `get pods -l app=${name} -o jsonpath='{.items[0].metadata.name}'`).trim();
+  if (pod === "") throw new Error(`no pod for workload ${name}`);
+  const out = kubectlOn(h, server,
+    `exec ${pod} -- sh -c "cd /data && sha256sum blob"`, 15 * 60_000).trim();
+  return out.split(/\s+/)[0] ?? "";
+}

bastion/tests/integration/helpers/provision-harness.ts

@@ -0,0 +1,583 @@
+// Reusable bastion + VM provisioning harness for partition-layout tests.
+//
+// pxe-provision.test.ts grew a ~200-line beforeAll that stands up a real
+// bastion, caches Fedora boot artifacts, and drives one VM through discovery
+// and install. Testing a partition MATRIX needs that same sequence for several
+// VMs and several roles, so it lives here instead of being copy-pasted per
+// topology.
+
+import { existsSync, mkdirSync, copyFileSync, symlinkSync, writeFileSync } from "node:fs";
+import { execSync } from "node:child_process";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import type { BastionConfig } from "../../../src/shared/src/types/config.js";
+import type { Role } from "../../../src/shared/src/types/state.js";
+import { log, waitForSsh } from "./libvirt.js";
+import { ensurePxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_BRIDGE, PXE_GATEWAY, PXE_SUBNET } from "./pxe-network.js";
+import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./pxe-vm.js";
+import { sshExec } from "./ssh.js";
+
+export const DISCOVERY_TIMEOUT_MS = 6 * 60_000;
+export const INSTALL_TIMEOUT_MS = 60 * 60_000;
+export const SSH_TIMEOUT_MS = 12 * 60_000;
+
+export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+export interface Harness {
+  config: BastionConfig;
+  close: () => Promise<void>;
+  httpPort: number;
+  sshUser: string;
+  sshKeyPath: string;
+}
+
+async function pollUntil<T>(url: string, check: (d: T) => boolean, timeoutMs: number, intervalMs = 5000): Promise<T> {
+  const start = Date.now();
+  let lastErr = "";
+  while (Date.now() - start < timeoutMs) {
+    try {
+      const res = await fetch(url);
+      if (res.ok) {
+        const data = (await res.json()) as T;
+        if (check(data)) return data;
+      }
+    } catch (e) { lastErr = e instanceof Error ? e.message : String(e); }
+    await sleep(intervalMs);
+  }
+  throw new Error(`Timeout after ${timeoutMs}ms polling ${url}${lastErr ? ` (last error: ${lastErr})` : ""}`);
+}
+
+/**
+ * Stand up a real bastion on the isolated PXE network.
+ *
+ * FEDORA_MIRROR is honoured (config.ts now reads it) -- point it at a local
+ * mirror before running a multi-VM matrix, or every install pulls the full
+ * package set from download.fedoraproject.org.
+ */
+export async function startTestBastion(opts: {
+  httpPort: number;
+  sshUser: string;
+  sshPubKey: string;
+  sshKeyPath: string;
+  label: string;
+  /**
+   * Leave the post-provision k3s auto-install ON (the default) or suppress it.
+   *
+   * Any test that forms a cluster ITSELF must pass false. The auto-install runs
+   * a bare `INSTALL_K3S_EXEC=server|agent` with no server URL and no token, so
+   * three combo nodes become three separate single-node clusters -- and it
+   * stops k3s and deletes CNI state minutes after an install lands, landing on
+   * top of whatever the test is doing by then.
+   */
+  autoK3s?: boolean;
+}): Promise<Harness> {
+  ensurePxeNetwork();
+
+  const testDir = join(tmpdir(), `lab-${opts.label}-${Date.now()}`);
+  for (const sub of ["tftp", "http", "logs"]) mkdirSync(join(testDir, sub), { recursive: true });
+
+  const { createApp } = await import("../../../src/bastion/src/server.js");
+  const { loadConfig } = await import("../../../src/bastion/src/config.js");
+  const { generateDnsmasqConf, startDnsmasq, stopDnsmasq } = await import("../../../src/bastion/src/services/dnsmasq.js");
+  const { generateDiscoverKickstart } = await import("../../../src/bastion/src/services/kickstart-generator.js");
+  const { renderBootIpxe } = await import("../../../src/bastion/src/templates/boot.ipxe.js");
+
+  const config = loadConfig({
+    bastionDir: testDir,
+    httpPort: opts.httpPort,
+    iface: "virbr-pxe",
+    serverIp: PXE_GATEWAY,
+    network: `${PXE_SUBNET}.0`,
+    gateway: PXE_GATEWAY,
+    dhcpMode: "full",
+    dhcpRangeStart: `${PXE_SUBNET}.100`,
+    dhcpRangeEnd: `${PXE_SUBNET}.200`,
+    domain: "pxe-test.local",
+    sshKeys: [opts.sshPubKey],
+    adminUser: opts.sshUser,
+    ...(opts.autoK3s === undefined ? {} : { autoK3s: opts.autoK3s }),
+  });
+
+  const ipxeSrc = "/usr/share/ipxe/ipxe-snponly-x86_64.efi";
+  if (!existsSync(ipxeSrc)) throw new Error(`iPXE not found: ${ipxeSrc}. Install: sudo dnf install ipxe-bootimgs-x86`);
+  const ipxeDest = join(config.tftpDir, "ipxe.efi");
+  copyFileSync(ipxeSrc, ipxeDest);
+
+  const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache";
+  execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" });
+  const kernel = join(cacheDir, `vmlinuz-${config.fedoraVersion}`);
+  const initrd = join(cacheDir, `initrd-${config.fedoraVersion}.img`);
+  if (!existsSync(kernel)) {
+    log(`Downloading Fedora ${config.fedoraVersion} kernel from ${config.fedoraMirror}...`);
+    execSync(`curl -sS -L -f -o "${kernel}" "${config.fedoraMirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 300_000 });
+  }
+  if (!existsSync(initrd)) {
+    log(`Downloading Fedora ${config.fedoraVersion} initrd...`);
+    execSync(`curl -sS -L -f -o "${initrd}" "${config.fedoraMirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 300_000 });
+  }
+  copyFileSync(kernel, join(config.httpDir, "vmlinuz"));
+  copyFileSync(initrd, join(config.httpDir, "initrd.img"));
+  try { symlinkSync(ipxeDest, join(config.httpDir, "ipxe.efi")); } catch { /* exists */ }
+
+  writeFileSync(join(config.httpDir, "discover.ks"), generateDiscoverKickstart(config));
+  writeFileSync(join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }));
+  generateDnsmasqConf(config);
+
+  // Kill dnsmasq processes left behind by EARLIER TEST RUNS.
+  //
+  // startTestBastion's close() shuts down the HTTP app but never stopped
+  // dnsmasq, so every run leaked one. A leaked dnsmasq keeps :67 on the bridge
+  // and keeps answering DHCP -- with the PREVIOUS run's config, pointing PXE
+  // clients at a bastion port that is no longer listening. The next run then
+  // sees perfect DHCP (offer, request, ack, even the right bootfile name) and
+  // no machine ever reaches it. That is not a failure mode anyone guesses from
+  // "discovery timed out".
+  killOrphanedDnsmasq();
+
+  const { app, syslog } = createApp(config);
+  await app.listen({ port: config.httpPort, host: "0.0.0.0" });
+  syslog.start();
+  // NOT awaited, deliberately.
+  //
+  // startDnsmasq returns the execa child handle, and because the function is
+  // `async` the promise it returns ADOPTS that thenable -- which settles only
+  // when dnsmasq EXITS. With --no-daemon that is never, so `await` here hangs
+  // the harness silently and forever, with the log frozen right after
+  // "TFTP root is ...". (Learned by doing exactly that.) The rejection still
+  // has to be handled or an early bind failure becomes an unhandled rejection.
+  startDnsmasq(config).catch((err) => log(`dnsmasq: ${err instanceof Error ? err.message : String(err)}`));
+  await sleep(1000);
+
+  // Prove DHCP is actually listening before handing the harness back.
+  //
+  // startDnsmasq's rejection is swallowed into a log line, so a dnsmasq that
+  // failed to bind looks exactly like one that started. The symptom then
+  // arrives six minutes later as four VMs timing out in discovery, which
+  // reads as a PXE or kickstart problem rather than "there was never any
+  // DHCP". Fail here, immediately, with the real reason.
+  await assertDhcpListening(join(testDir, "dnsmasq.conf"));
+  log(`Bastion up on :${opts.httpPort} (mirror: ${config.fedoraMirror}, autoK3s=${config.autoK3s})`);
+
+  return {
+    config,
+    httpPort: opts.httpPort,
+    sshUser: opts.sshUser,
+    sshKeyPath: opts.sshKeyPath,
+    close: async () => {
+      // Stop OUR dnsmasq. close() never used to, which is how a leaked one
+      // ended up owning :67 and serving the PREVIOUS run's TFTP root and
+      // bastion port to the NEXT run's VMs -- they booted toward a dead port
+      // and simply never appeared, while DHCP looked perfect.
+      try { stopDnsmasq(); } catch { /* already gone */ }
+      await app.close();
+      // Belt and braces: if that did not stop it, sweep by config path.
+      killOrphanedDnsmasq();
+    },
+  };
+}
+
+/**
+ * Kill any dnsmasq started by a previous (or current) lab test run.
+ *
+ * Matched on the config path, which is always /tmp/lab-<label>-<ts>/dnsmasq.conf
+ * -- so this never touches libvirt's own dnsmasq for the default network, or
+ * anything else on the host.
+ */
+function killOrphanedDnsmasq(): void {
+  try {
+    // Match on the process NAME first (pgrep -x), then inspect each one's
+    // arguments. A `pgrep -f 'dnsmasq.*/tmp/lab-'` matches the shell running
+    // that very pattern, so the naive version can kill its own shell -- and
+    // libvirt's dnsmasq for the default network must never be touched either.
+    const pids = execSync("pgrep -x dnsmasq 2>/dev/null || true", { encoding: "utf-8" })
+      .trim().split("\n").filter((x) => /^\d+$/.test(x.trim()));
+    let killed = 0;
+    for (const pid of pids) {
+      let args = "";
+      try {
+        args = execSync(`tr '\\0' ' ' < /proc/${pid.trim()}/cmdline 2>/dev/null || true`, { encoding: "utf-8" });
+      } catch { continue; }
+      if (!args.includes("--conf-file=/tmp/lab-")) continue;
+      log(`Killing orphaned test dnsmasq (pid ${pid.trim()}) — it would answer DHCP with stale config`);
+      try { execSync(`sudo kill -9 ${pid.trim()} 2>/dev/null || kill -9 ${pid.trim()} 2>/dev/null || true`); } catch { /* gone */ }
+      killed++;
+    }
+    // Give the socket a moment to be released before we bind it.
+    if (killed > 0) execSync("sleep 1");
+  } catch { /* nothing to kill */ }
+}
+
+/**
+ * Is something serving DHCP on the PXE bridge?
+ *
+ * `ss -lunp` shows UDP listeners; dnsmasq binds port 67. Checked on the bridge
+ * address specifically, because a dnsmasq bound to the wrong interface is the
+ * failure being guarded against.
+ */
+async function assertDhcpListening(confPath: string, timeoutMs = 20_000): Promise<void> {
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    try {
+      // Check for OUR dnsmasq by its config path, not merely for something on
+      // :67. A leaked dnsmasq from a previous run also holds :67 and also
+      // answers DHCP -- with the wrong bastion port -- so a port check alone
+      // reports success for precisely the situation it is meant to catch.
+      const pids = execSync("pgrep -x dnsmasq 2>/dev/null || true", { encoding: "utf-8" })
+        .trim().split("\n").filter((x) => /^\d+$/.test(x.trim()));
+      last = pids.filter((pid) => {
+        try {
+          return execSync(`tr '\\0' ' ' < /proc/${pid.trim()}/cmdline 2>/dev/null || true`, { encoding: "utf-8" })
+            .includes(confPath);
+        } catch { return false; }
+      }).join(",");
+      if (last !== "") {
+        const ports = execSync("ss -lun 2>/dev/null || true", { encoding: "utf-8" });
+        if (/:67\b/.test(ports)) {
+          log("dnsmasq (ours) is serving DHCP on :67");
+          return;
+        }
+      }
+    } catch { /* retry */ }
+    await sleep(1000);
+  }
+  throw new Error(
+    `dnsmasq for ${confPath} is not serving DHCP after ${timeoutMs / 1000}s. ` +
+    `It most likely failed to bind to ${PXE_BRIDGE} -- check the bastion log for ` +
+    `"error binding DHCP socket", and check for a leaked dnsmasq from an earlier run ` +
+    `(pgrep -af 'dnsmasq.*--conf-file=/tmp/lab-'). Without this every VM sits in ` +
+    `discovery until it times out, six minutes later, looking like a PXE problem.`,
+  );
+}
+
+/**
+ * Wait for the installer to actually start, rebooting into PXE if it does not.
+ *
+ * "Started" means the bastion has seen anything at all for this MAC beyond the
+ * queued state -- a dispatch, a progress line, a status. A VM that never gets
+ * that far is stuck before Anaconda, and only a reboot will shift it.
+ */
+async function waitForInstallStart(
+  base: string,
+  mac: string,
+  name: string,
+  attemptMs = 6 * 60_000,
+  attempts = 3,
+): Promise<void> {
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    try {
+      // NOT `status === "installing"`. A machine that has merely been QUEUED
+      // already reports that -- the bastion sets it the moment the install is
+      // dispatched, before anything has booted. Using it would match instantly
+      // and this retry would never fire.
+      //
+      // The honest signals that Anaconda is alive are `progress` leaving
+      // "queued", or log lines arriving over syslog.
+      await pollUntil<{ status?: string; progress?: string; log_total?: number }>(
+        `${base}/api/logs/${encodeURIComponent(mac)}`,
+        (d) => d.status === "installed"
+          || (d.progress !== undefined && d.progress !== "queued")
+          || (d.log_total ?? 0) > 0,
+        attemptMs, 5_000);
+      if (attempt > 1) log(`[${name}] PXE boot took on attempt ${attempt}`);
+      return;
+    } catch {
+      if (attempt === attempts) {
+        throw new Error(
+          `[${name}] installer never started after ${attempts} PXE boots ` +
+          `(${attemptMs / 60000} min each). The VM is stuck before Anaconda — ` +
+          `check TFTP and the iPXE chainload.`);
+      }
+      log(`[${name}] no installer activity in ${attemptMs / 60000} min; rebooting into PXE (attempt ${attempt + 1}/${attempts})`);
+      rebootPxeVm(name);
+      deleteNftablesRejectRules();
+    }
+  }
+}
+
+export interface ProvisionedVm { name: string; mac: string; ip: string; serialPort: number }
+
+/**
+ * "SSH never came up" on its own is undiagnosable once afterAll has destroyed
+ * the VM -- the machine may be sitting in emergency mode with the exact error
+ * on its serial console. Capture that console into the failure.
+ */
+async function sshFailurePostMortem(name: string, serialPort: number, cause: unknown): Promise<never> {
+  let state = "unknown";
+  try { state = execSync(`sudo virsh domstate ${name}`, { encoding: "utf-8" }).trim(); } catch { /* gone */ }
+  let serial = "(no serial output)";
+  try { serial = await readSerialLog(serialPort, { lastLines: 60 }); } catch { /* no console */ }
+  throw new Error(
+    `[${name}] ${cause instanceof Error ? cause.message : String(cause)}\n` +
+    `domstate: ${state}\n--- serial console (last lines) ---\n${serial}`,
+  );
+}
+
+/**
+ * Serial ports are a global resource on the host, so hand out a distinct one
+ * per VM. Without this only one VM can run at a time -- qemu refuses to start
+ * the second with "Address already in use".
+ */
+let nextSerialPort = 4600;
+
+/**
+ * The stable path a virtio data disk gets inside the guest. This is what the
+ * install request must carry: the feature refuses anything outside
+ * /dev/disk/by-id/, exactly as it does for real NVMe serials.
+ */
+export function virtioById(serial: string): string {
+  return `/dev/disk/by-id/virtio-${serial}`;
+}
+
+/** Create a blank VM, let it discover, queue the install, wait for SSH. */
+export async function provisionVm(h: Harness, spec: {
+  name: string;
+  role: Role;
+  schemaVersion?: 1 | 2;
+  diskGb: number;
+  memory?: number;
+  vcpus?: number;
+  /**
+   * Dedicated data disks: attached to the VM with the given virtio serial AND
+   * passed to /api/install as /dev/disk/by-id/virtio-<serial>. Mirrors
+   * thebeast's layout (OS disk + separate longhorn + rancher disks).
+   */
+  dataDisks?: { longhorn?: { serial: string; sizeGb: number }; rancher?: { serial: string; sizeGb: number } };
+}): Promise<ProvisionedVm> {
+  const base = `http://${PXE_GATEWAY}:${h.httpPort}`;
+  const vmDataDisks = [spec.dataDisks?.longhorn, spec.dataDisks?.rancher]
+    .filter((d): d is { serial: string; sizeGb: number } => d !== undefined);
+  log(`[${spec.name}] creating VM (${spec.diskGb}GB disk + ${vmDataDisks.length} data disk(s), role=${spec.role}, schema=v${spec.schemaVersion ?? 1})`);
+  const serialPort = nextSerialPort++;
+  createPxeVm({
+    name: spec.name,
+    memory: spec.memory ?? 4096,
+    vcpus: spec.vcpus ?? 4,
+    diskSize: spec.diskGb,
+    network: PXE_NETWORK_NAME,
+    serialPort,
+    ...(vmDataDisks.length > 0 ? { dataDisks: vmDataDisks } : {}),
+  });
+  deleteNftablesRejectRules();
+
+  const mac = getVmMac(spec.name);
+  if (!mac) throw new Error(`[${spec.name}] no MAC`);
+
+  log(`[${spec.name}] waiting for discovery (${mac})...`);
+  await pollUntil<{ discovered: Record<string, unknown> }>(
+    `${base}/api/machines`, (d) => mac in (d.discovered ?? {}), DISCOVERY_TIMEOUT_MS);
+
+  log(`[${spec.name}] queueing install...`);
+  const res = await fetch(`${base}/api/install`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      mac, hostname: spec.name, disk: "", role: spec.role,
+      ...(spec.schemaVersion ? { schemaVersion: spec.schemaVersion } : {}),
+      ...(spec.dataDisks ? {
+        dataDisks: {
+          ...(spec.dataDisks.longhorn ? { longhorn: virtioById(spec.dataDisks.longhorn.serial) } : {}),
+          ...(spec.dataDisks.rancher ? { rancher: virtioById(spec.dataDisks.rancher.serial) } : {}),
+        },
+      } : {}),
+    }),
+  });
+  if (!res.ok) throw new Error(`[${spec.name}] install queue rejected: ${res.status} ${await res.text()}`);
+
+  rebootPxeVm(spec.name);
+  deleteNftablesRejectRules();
+
+  // Make sure the PXE boot actually TOOK before settling in for the install.
+  //
+  // A UEFI VM that fails to chainload just retries DHCP quietly, for ever.
+  // Observed: one VM of four sat in a DHCP loop for an HOUR with no TFTP
+  // fetch, then began installing seconds after the 60-minute install timeout
+  // had already fired -- so the run failed on a machine that was, by then,
+  // working. TFTP is UDP and four VMs contend for it, so this is a question of
+  // when rather than if.
+  //
+  // Rebooting is cheap and a boot that has not started within a few minutes is
+  // not going to start on its own.
+  await waitForInstallStart(base, mac, spec.name);
+
+  log(`[${spec.name}] installing (up to ${INSTALL_TIMEOUT_MS / 60000} min)...`);
+  await pollUntil<{ status?: string }>(
+    `${base}/api/logs/${encodeURIComponent(mac)}`, (d) => d.status === "installed", INSTALL_TIMEOUT_MS, 10_000);
+
+  // The installed record carries the DHCP address the machine ended up with.
+  const machines = await pollUntil<{ installed: Record<string, { ip?: string }> }>(
+    `${base}/api/machines`, (d) => Boolean(d.installed?.[mac]?.ip), 2 * 60_000);
+  const ip = machines.installed[mac]!.ip!;
+
+  try {
+    if (execSync(`sudo virsh domstate ${spec.name}`, { encoding: "utf-8" }).trim() === "shut off") {
+      execSync(`sudo virsh start ${spec.name}`, { stdio: "pipe" });
+    }
+  } catch { /* already running */ }
+  deleteNftablesRejectRules();
+
+  log(`[${spec.name}] waiting for SSH at ${ip}...`);
+  try {
+    await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath);
+  } catch (e) {
+    await sshFailurePostMortem(spec.name, serialPort, e);
+  }
+  log(`[${spec.name}] up at ${ip}`);
+  return { name: spec.name, mac, ip, serialPort };
+}
+
+/**
+ * Provision several VMs at once.
+ *
+ * Installs are ~12 minutes each and almost entirely I/O and network wait, so
+ * four in sequence is an hour that four in parallel is twenty minutes. They are
+ * staggered because the expensive, contended part is the start: PXE, DHCP and
+ * the first package fetches all land at once otherwise.
+ *
+ * Rejects only after every VM has settled, so a single failure does not leave
+ * the others half-built and unreferenced -- the caller still gets the names it
+ * needs to tear down.
+ */
+export async function provisionVms(
+  h: Harness,
+  specs: Parameters<typeof provisionVm>[1][],
+  opts: { staggerMs?: number } = {},
+): Promise<ProvisionedVm[]> {
+  const stagger = opts.staggerMs ?? 60_000;
+  const settled = await Promise.allSettled(
+    specs.map(async (spec, i) => {
+      await sleep(i * stagger);
+      return provisionVm(h, spec);
+    }),
+  );
+
+  const failures = settled
+    .map((r, i) => (r.status === "rejected" ? `${specs[i]?.name}: ${String(r.reason)}` : null))
+    .filter((x): x is string => x !== null);
+  if (failures.length > 0) {
+    throw new Error(`provisionVms: ${failures.length}/${specs.length} failed\n  ${failures.join("\n  ")}`);
+  }
+  return settled.map((r) => (r as PromiseFulfilledResult<ProvisionedVm>).value);
+}
+
+/** LV name -> size in MiB, read off the installed machine. */
+export function readLvs(h: Harness, vm: ProvisionedVm): Record<string, number> {
+  const res = sshExec(vm.ip, h.sshUser,
+    "sudo lvs labvg --noheadings --units m --nosuffix -o lv_name,lv_size --separator=,",
+    { keyPath: h.sshKeyPath });
+  if (res.exitCode !== 0) throw new Error(`[${vm.name}] lvs failed: ${res.stderr}`);
+  const map: Record<string, number> = {};
+  for (const line of res.stdout.split("\n")) {
+    const [name, size] = line.trim().split(",");
+    if (name && size) map[name] = Math.round(Number(size));
+  }
+  // Log what actually landed on disk. The VMs are destroyed in afterAll, so
+  // without this the only record of the real layout is the pass/fail bit.
+  const summary = Object.entries(map)
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([n, m]) => `${n}=${(m / 1024).toFixed(0)}G`)
+    .join(" ");
+  log(`[${vm.name}] on-disk LVs: ${summary}`);
+  return map;
+}
+
+/** The VG tag that records which schema this disk was built with. */
+export function readSchemaTag(h: Harness, vm: ProvisionedVm): string {
+  return sshExec(vm.ip, h.sshUser, "sudo vgs --noheadings -o vg_tags labvg",
+    { keyPath: h.sshKeyPath }).stdout.trim();
+}
+
+export function destroyVms(...names: string[]): void {
+  for (const n of names) { try { destroyPxeVm(n); } catch { /* already gone */ } }
+}
+
+/**
+ * Reinstall a machine that is already installed, the way
+ * `labctl provision reprovision` does: queue another install for the same MAC
+ * and reboot it into PXE.
+ *
+ * The schema is deliberately NOT passed. A reinstall must reproduce whatever
+ * layout is already on the disk -- %pre reads the lab-schema-v2 LVM tag rather
+ * than trusting the caller -- and passing one here would hide a regression in
+ * exactly that logic.
+ */
+export async function reprovisionVm(
+  h: Harness,
+  vm: ProvisionedVm,
+  role: Role,
+  /**
+   * Only set this to prove it is IGNORED. A reinstall takes its layout from the
+   * disk, not from the request; passing a schema here is how you test that.
+   */
+  requestSchemaVersion?: 1 | 2,
+): Promise<ProvisionedVm> {
+  const base = `http://${PXE_GATEWAY}:${h.httpPort}`;
+  log(`[${vm.name}] reprovisioning (role=${role}, schema: whatever is on disk)...`);
+
+  const res = await fetch(`${base}/api/install`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      mac: vm.mac, hostname: vm.name, disk: "", role,
+      ...(requestSchemaVersion ? { schemaVersion: requestSchemaVersion } : {}),
+    }),
+  });
+  if (!res.ok) throw new Error(`[${vm.name}] reinstall queue rejected: ${res.status} ${await res.text()}`);
+
+  rebootPxeVm(vm.name);
+  deleteNftablesRejectRules();
+
+  log(`[${vm.name}] reinstalling...`);
+  await pollUntil<{ status?: string }>(
+    `${base}/api/logs/${encodeURIComponent(vm.mac)}`,
+    (d) => d.status === "installed", INSTALL_TIMEOUT_MS, 10_000);
+
+  const machines = await pollUntil<{ installed: Record<string, { ip?: string }> }>(
+    `${base}/api/machines`, (d) => Boolean(d.installed?.[vm.mac]?.ip), 2 * 60_000);
+  const ip = machines.installed[vm.mac]!.ip!;
+
+  try {
+    if (execSync(`sudo virsh domstate ${vm.name}`, { encoding: "utf-8" }).trim() === "shut off") {
+      execSync(`sudo virsh start ${vm.name}`, { stdio: "pipe" });
+    }
+  } catch { /* already running */ }
+  deleteNftablesRejectRules();
+
+  try {
+    await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath);
+  } catch (e) {
+    await sshFailurePostMortem(vm.name, vm.serialPort, e);
+  }
+  log(`[${vm.name}] back up at ${ip} after reinstall`);
+  return { ...vm, ip };
+}
+
+/**
+ * Drop a marker file into each mount point that a reinstall is supposed to
+ * preserve. Ported from test-reprovision.sh, which proved the idea but only
+ * ever ran by hand.
+ */
+export function writeMarkers(h: Harness, vm: ProvisionedVm, mounts: string[]): string {
+  const token = `preserve-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+  for (const m of mounts) {
+    const r = sshExec(vm.ip, h.sshUser,
+      `sudo sh -c 'echo ${token} > ${m}/LAB_PRESERVE_TEST.txt && sync'`,
+      { keyPath: h.sshKeyPath });
+    if (r.exitCode !== 0) throw new Error(`[${vm.name}] could not write marker in ${m}: ${r.stderr}`);
+  }
+  log(`[${vm.name}] wrote marker ${token} into ${mounts.join(", ")}`);
+  return token;
+}
+
+/** Which mount points still carry the marker. */
+export function readMarkers(h: Harness, vm: ProvisionedVm, mounts: string[]): Record<string, string | null> {
+  const out: Record<string, string | null> = {};
+  for (const m of mounts) {
+    const r = sshExec(vm.ip, h.sshUser, `sudo cat ${m}/LAB_PRESERVE_TEST.txt 2>/dev/null || true`,
+      { keyPath: h.sshKeyPath });
+    const v = r.stdout.trim();
+    out[m] = v === "" ? null : v;
+  }
+  log(`[${vm.name}] markers after reinstall: ${JSON.stringify(out)}`);
+  return out;
+}

bastion/tests/integration/helpers/pxe-network.ts

@@ -56,6 +56,17 @@ export function ensurePxeNetwork(): void {
 
     try { unlinkSync(xmlPath); } catch { /* ignore */ }
 
+    // `virsh net-start` returns before the bridge is usable. The bastion's
+    // dnsmasq starts about a second later and binds to this device by name; if
+    // the interface is not up and addressed yet it fails with "error binding
+    // DHCP socket to device virbr-pxe", and because that failure is
+    // asynchronous the harness carries on with NO DHCP AT ALL -- every VM then
+    // sits in discovery until it times out, six minutes later, looking like a
+    // PXE problem rather than a startup-order one.
+    //
+    // Cheap to wait, expensive not to.
+    waitForBridge();
+
     log(`Network ${PXE_NETWORK_NAME} created and active`);
   }
 
@@ -64,6 +75,32 @@ export function ensurePxeNetwork(): void {
   deleteNftablesRejectRules();
 }
 
+/**
+ * Block until the bridge exists AND carries the gateway address.
+ *
+ * Checking only that the device exists is not enough: it appears before the
+ * address is assigned, and dnsmasq needs the address to bind.
+ */
+export function waitForBridge(timeoutMs = 30_000): void {
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    const r = spawnSync("ip", ["-br", "addr", "show", PXE_BRIDGE], { encoding: "utf-8", stdio: "pipe" });
+    last = (r.stdout ?? "").trim();
+    if (r.status === 0 && last.includes(`${PXE_GATEWAY}/`)) {
+      log(`Bridge ${PXE_BRIDGE} ready with ${PXE_GATEWAY} after ${Date.now() - start}ms`);
+      return;
+    }
+    // Busy-wait briefly; this normally resolves in well under a second.
+    const until = Date.now() + 250;
+    while (Date.now() < until) { /* spin */ }
+  }
+  throw new Error(
+    `Bridge ${PXE_BRIDGE} did not come up with ${PXE_GATEWAY} within ${timeoutMs}ms ` +
+    `(last: ${last || "device absent"}). dnsmasq cannot bind, so PXE would silently not work.`,
+  );
+}
+
 /** Delete libvirt's nftables reject rules for our bridge so host→VM traffic works.
  *  Must be called after every VM start/restart — libvirt recreates them. */
 export function deleteNftablesRejectRules(): void {

bastion/tests/integration/helpers/pxe-vm.ts

@@ -28,6 +28,13 @@ export interface PxeVmConfig {
   vcpus: number;
   diskSize: number;     // GB
   network: string;      // libvirt network name
+  /**
+   * Serial console TCP port. Defaults to 4555 for backwards compatibility with
+   * the single-VM tests, but it MUST be unique per concurrently-running VM:
+   * qemu fails with "Failed to find an available port: Address already in use"
+   * and the domain never starts.
+   */
+  serialPort?: number;
   arch?: "x86_64" | "aarch64";
   /**
    * Extra NICs enumerated BEFORE the PXE NIC, on a network with no route to
@@ -40,6 +47,15 @@ export interface PxeVmConfig {
    */
   decoyNics?: number;
   decoyNetwork?: string;
+  /**
+   * Extra blank data disks, attached after the OS disk. Each gets a fixed
+   * virtio serial, so inside the guest it has a stable
+   * /dev/disk/by-id/virtio-<serial> path -- the moral equivalent of the
+   * nvme-..._<serial> links the dedicated-data-disk feature requires on real
+   * hardware. virtio-blk truncates serials at 20 characters, so keep them
+   * short or the by-id link quietly won't match.
+   */
+  dataDisks?: { serial: string; sizeGb: number }[];
 }
 
 /** Create a blank UEFI VM that PXE boots from the network. */
@@ -53,6 +69,14 @@ export function createPxeVm(config: PxeVmConfig): void {
   const diskPath = join(IMAGE_DIR, `${config.name}.qcow2`);
   run(`qemu-img create -f qcow2 "${diskPath}" ${config.diskSize}G`);
 
+  const dataDiskArgs: string[] = [];
+  for (const dd of config.dataDisks ?? []) {
+    if (dd.serial.length > 20) throw new Error(`virtio serial '${dd.serial}' exceeds 20 chars -- the guest by-id link would be truncated`);
+    const p = join(IMAGE_DIR, `${config.name}-${dd.serial}.qcow2`);
+    run(`qemu-img create -f qcow2 "${p}" ${dd.sizeGb}G`);
+    dataDiskArgs.push(`--disk=path=${p},format=qcow2,bus=virtio,serial=${dd.serial}`);
+  }
+
   // UEFI firmware paths (Fedora)
   if (arch === "aarch64") {
     const aavmf = "/usr/share/edk2/aarch64/QEMU_EFI.fd";
@@ -72,6 +96,7 @@ export function createPxeVm(config: PxeVmConfig): void {
     `--memory=${config.memory}`,
     `--vcpus=${config.vcpus}`,
     `--disk=path=${diskPath},format=qcow2,bus=virtio`,
+    ...dataDiskArgs,
     // Decoys first so they enumerate ahead of the PXE NIC. They are up and
     // carry a lease, but have no route to the bastion.
     ...Array.from({ length: config.decoyNics ?? 0 }, () =>
@@ -86,8 +111,8 @@ export function createPxeVm(config: PxeVmConfig): void {
     // Graphics for debugging (VNC, connect with virt-viewer if needed)
     "--graphics=vnc,listen=127.0.0.1",
     // Serial console via TCP — allows exec without network/SSH
-    // Connect: socat - TCP:127.0.0.1:4555
-    "--serial=tcp,host=127.0.0.1:4555,mode=bind,protocol=telnet",
+    // Connect: socat - TCP:127.0.0.1:<serialPort>
+    `--serial=tcp,host=127.0.0.1:${config.serialPort ?? 4555},mode=bind,protocol=telnet`,
   ];
 
   if (arch === "aarch64") {
@@ -96,7 +121,7 @@ export function createPxeVm(config: PxeVmConfig): void {
 
   log(`Running: virt-install --name=${config.name} --boot=uefi,network ...`);
   run(virtInstallArgs.join(" "), { timeout: 30_000 });
-  log(`PXE VM ${config.name} created (serial: telnet 127.0.0.1 4555)`);
+  log(`PXE VM ${config.name} created (serial: telnet 127.0.0.1 ${config.serialPort ?? 4555})`);
 }
 
 /** Destroy a PXE VM and clean up its disk. */

bastion/tests/integration/partition-layout.test.ts

@@ -0,0 +1,145 @@
+// Integration tests: the v2 disk layout, on real VMs.
+//
+// Unit tests prove the kickstart RENDERS the right layout. These prove anaconda
+// ACTUALLY PRODUCES it -- which is a different claim, and the one that matters:
+// the labmaster incident was caused by what landed on disk, not by what was
+// written into a template.
+//
+// Every VM gets a 1 TB sparse disk, because "fits a 1 TB disk with headroom" is
+// the constraint v2 was designed around. qcow2 is sparse, so a 1 TB disk costs
+// ~10-15 GB of real space.
+//
+// Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo, and a Fedora mirror.
+// Set FEDORA_MIRROR to a local mirror -- otherwise each install pulls the full
+// package set over the internet and the matrix takes hours.
+//
+// Run one scenario at a time:
+//   sudo -E npx vitest run -c tests/integration/vitest.config.ts -t "vanilla"
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { v2Layout } from "../../src/shared/src/types/layout.js";
+import type { Role } from "../../src/shared/src/types/state.js";
+import {
+  startTestBastion, provisionVm, readLvs, readSchemaTag, destroyVms,
+  type Harness, type ProvisionedVm,
+} from "./helpers/provision-harness.js";
+import { destroyPxeNetwork } from "./helpers/pxe-network.js";
+
+const HTTP_PORT = 8096;
+const SSH_USER = "lab";
+const DISK_GB = 1000; // the smallest disk we will ever buy
+
+function findSshKey(): { pubKey: string; keyPath: string } {
+  const homes = [homedir()];
+  const sudoUser = process.env["SUDO_USER"];
+  if (sudoUser) homes.push(join("/home", sudoUser));
+  for (const home of homes) {
+    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
+      const keyPath = join(home, ".ssh", name);
+      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
+        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
+      }
+    }
+  }
+  throw new Error("no SSH keypair found");
+}
+
+/** Assert the on-disk volumes match the shared v2 definition for this role. */
+function expectV2Layout(h: Harness, vm: ProvisionedVm, role: Role): void {
+  const actual = readLvs(h, vm);
+  const expected = v2Layout(role);
+
+  expect(Object.keys(actual).sort(), `${vm.name} volume set`).toEqual(
+    expected.map((lv) => lv.name).sort(),
+  );
+  for (const spec of expected) {
+    // LVM rounds to extent boundaries (4 MiB), so allow a small delta.
+    expect(actual[spec.name], `${vm.name}/${spec.name} size`).toBeGreaterThanOrEqual(spec.sizeMib - 4);
+    expect(actual[spec.name], `${vm.name}/${spec.name} size`).toBeLessThanOrEqual(spec.sizeMib + 4);
+  }
+  expect(readSchemaTag(h, vm), `${vm.name} schema tag`).toContain("lab-schema-v2");
+}
+
+describe("v2 partition layout on real hardware", () => {
+  let h: Harness;
+  const created: string[] = [];
+
+  beforeAll(async () => {
+    const { pubKey, keyPath } = findSshKey();
+    h = await startTestBastion({
+      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey,
+      sshKeyPath: keyPath, label: "partition",
+    });
+  }, 20 * 60_000);
+
+  afterAll(async () => {
+    destroyVms(...created);
+    try { await h?.close(); } catch { /* already closed */ }
+    try { destroyPxeNetwork(); } catch { /* already gone */ }
+  }, 5 * 60_000);
+
+  // --- Topology 1: vanilla -------------------------------------------------
+  it("vanilla: base volumes only, no rancher, no longhorn", async () => {
+    const name = "v2-vanilla";
+    created.push(name);
+    const vm = await provisionVm(h, { name, role: "vanilla", schemaVersion: 2, diskGb: DISK_GB });
+    expectV2Layout(h, vm, "vanilla");
+
+    const lvs = readLvs(h, vm);
+    expect(lvs["rancher"]).toBeUndefined();
+    expect(lvs["longhorn"]).toBeUndefined();
+  }, 60 * 60_000);
+
+  // --- Topology 2: standalone cluster --------------------------------------
+  it("standalone-cluster: single node with its own control plane and storage", async () => {
+    const name = "v2-standalone";
+    created.push(name);
+    const vm = await provisionVm(h, { name, role: "standalone-cluster", schemaVersion: 2, diskGb: DISK_GB, memory: 6144 });
+    expectV2Layout(h, vm, "standalone-cluster");
+
+    const lvs = readLvs(h, vm);
+    // A cluster of one still gets a Longhorn volume, so the `longhorn`
+    // StorageClass exists and manifests stay portable across clusters.
+    expect(lvs["longhorn"]).toBeDefined();
+    expect(lvs["rancher"]).toBeDefined();
+  }, 60 * 60_000);
+
+  // --- Topology 3: 3x combo (control plane AND worker on every node) -------
+  it("k8s-combo x3: every node is control plane and storage", async () => {
+    const names = ["v2-combo-1", "v2-combo-2", "v2-combo-3"];
+    created.push(...names);
+    const vms: ProvisionedVm[] = [];
+    for (const name of names) {
+      vms.push(await provisionVm(h, { name, role: "k8s-combo", schemaVersion: 2, diskGb: DISK_GB }));
+    }
+    for (const vm of vms) {
+      expectV2Layout(h, vm, "k8s-combo");
+      const lvs = readLvs(h, vm);
+      expect(lvs["longhorn"], `${vm.name} holds storage`).toBeDefined();
+      expect(lvs["rancher"], `${vm.name} has an image store`).toBeDefined();
+    }
+  }, 180 * 60_000);
+
+  // --- Topology 4: 3 controllers + 2 workers -------------------------------
+  it("3x k8s-controller + 2x k8s-worker: storage only on the workers", async () => {
+    const ctrls = ["v2-ctrl-1", "v2-ctrl-2", "v2-ctrl-3"];
+    const workers = ["v2-wrk-1", "v2-wrk-2"];
+    created.push(...ctrls, ...workers);
+
+    for (const name of ctrls) {
+      const vm = await provisionVm(h, { name, role: "k8s-controller", schemaVersion: 2, diskGb: DISK_GB });
+      expectV2Layout(h, vm, "k8s-controller");
+      // The distinguishing property of a controller: image store, no storage.
+      expect(readLvs(h, vm)["longhorn"], `${name} must not hold replicas`).toBeUndefined();
+      expect(readLvs(h, vm)["rancher"]).toBeDefined();
+    }
+    for (const name of workers) {
+      const vm = await provisionVm(h, { name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB });
+      expectV2Layout(h, vm, "k8s-worker");
+      expect(readLvs(h, vm)["longhorn"], `${name} must hold replicas`).toBeDefined();
+    }
+  }, 300 * 60_000);
+});

bastion/tests/integration/reinstall.test.ts

@@ -0,0 +1,138 @@
+// Integration tests: reinstalling a machine must not destroy its data, and
+// must not silently change its disk layout.
+//
+// This is the risk the whole v2 change carries. Every machine in the fleet was
+// installed with v1, and `labctl provision reprovision` has to keep reproducing
+// v1 on those boxes forever. If a reinstall quietly switched them to v2 it
+// would recreate the volume group and take every Longhorn replica with it.
+//
+// Ported from test-reprovision.sh, which had the right idea (marker files in
+// each preserved volume) but only ever ran by hand and was never in CI.
+//
+// Run: sudo -E npx vitest run -c tests/integration/vitest.config.ts -t "reinstall"
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { v2Layout } from "../../src/shared/src/types/layout.js";
+import {
+  startTestBastion, provisionVm, reprovisionVm, readLvs, readSchemaTag,
+  writeMarkers, readMarkers, destroyVms, type Harness,
+} from "./helpers/provision-harness.js";
+import { destroyPxeNetwork } from "./helpers/pxe-network.js";
+
+const HTTP_PORT = 8095;
+const SSH_USER = "lab";
+const DISK_GB = 1000;
+
+/** Mount points a reinstall is supposed to preserve on a v1 worker. */
+const PRESERVED = ["/home", "/srv", "/var/lib/longhorn", "/var/lib/rancher"];
+
+function findSshKey(): { pubKey: string; keyPath: string } {
+  const homes = [homedir()];
+  const sudoUser = process.env["SUDO_USER"];
+  if (sudoUser) homes.push(join("/home", sudoUser));
+  for (const home of homes) {
+    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
+      const keyPath = join(home, ".ssh", name);
+      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
+        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
+      }
+    }
+  }
+  throw new Error("no SSH keypair found");
+}
+
+describe("reinstall preserves data and layout", () => {
+  let h: Harness;
+  const created: string[] = [];
+
+  beforeAll(async () => {
+    const { pubKey, keyPath } = findSshKey();
+    h = await startTestBastion({
+      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey,
+      sshKeyPath: keyPath, label: "reinstall",
+    });
+  }, 20 * 60_000);
+
+  afterAll(async () => {
+    destroyVms(...created);
+    try { await h?.close(); } catch { /* closed */ }
+    try { destroyPxeNetwork(); } catch { /* gone */ }
+  }, 5 * 60_000);
+
+  it("v1 worker: data survives and the layout stays v1", async () => {
+    const name = "re-v1-worker";
+    created.push(name);
+
+    // Install at v1 -- i.e. exactly what every machine in the fleet is today.
+    let vm = await provisionVm(h, { name, role: "worker", diskGb: DISK_GB });
+
+    const before = readLvs(h, vm);
+    expect(before["root"], "v1 root is 33G, not v2's 32G").toBeGreaterThan(33000);
+    expect(before["rancher"], "v1 worker gets a 120G rancher LV").toBeGreaterThan(120000 - 8);
+    expect(readSchemaTag(h, vm), "a v1 disk must NOT carry the v2 tag").not.toContain("lab-schema-v2");
+
+    const token = writeMarkers(h, vm, PRESERVED);
+
+    // Reinstall. No schemaVersion passed -- the disk decides.
+    vm = await reprovisionVm(h, vm, "worker");
+
+    const markers = readMarkers(h, vm, PRESERVED);
+    for (const mount of PRESERVED) {
+      expect(markers[mount], `${mount} must survive a reinstall`).toBe(token);
+    }
+
+    // And the layout must be untouched.
+    const after = readLvs(h, vm);
+    expect(Object.keys(after).sort()).toEqual(Object.keys(before).sort());
+    expect(after["root"]).toBe(before["root"]);
+    expect(after["rancher"]).toBe(before["rancher"]);
+    expect(readSchemaTag(h, vm)).not.toContain("lab-schema-v2");
+  }, 120 * 60_000);
+
+  it("v2 worker: data survives and the layout stays v2", async () => {
+    const name = "re-v2-worker";
+    created.push(name);
+
+    let vm = await provisionVm(h, { name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB });
+    const before = readLvs(h, vm);
+    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");
+
+    const token = writeMarkers(h, vm, PRESERVED);
+    vm = await reprovisionVm(h, vm, "k8s-worker");
+
+    const markers = readMarkers(h, vm, PRESERVED);
+    for (const mount of PRESERVED) {
+      expect(markers[mount], `${mount} must survive a reinstall`).toBe(token);
+    }
+
+    const after = readLvs(h, vm);
+    const expected = Object.fromEntries(v2Layout("k8s-worker").map((lv) => [lv.name, lv.sizeMib]));
+    expect(Object.keys(after).sort()).toEqual(Object.keys(expected).sort());
+    expect(after["root"]).toBe(before["root"]);
+    // The tag must survive too, or the NEXT reinstall would fall back to v1.
+    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");
+  }, 120 * 60_000);
+
+  it("a v1 machine is NOT upgraded to v2 just because v2 was requested", async () => {
+    const name = "re-v1-sticky";
+    created.push(name);
+
+    let vm = await provisionVm(h, { name, role: "worker", diskGb: DISK_GB });
+    const before = readLvs(h, vm);
+    const token = writeMarkers(h, vm, ["/home", "/srv"]);
+
+    // Ask for v2 on a disk that is v1. %pre must ignore it: silently
+    // re-laying-out a live machine is how data gets destroyed.
+    vm = await reprovisionVm(h, vm, "worker", 2);
+
+    const after = readLvs(h, vm);
+    expect(after["root"], "root must stay at the v1 size").toBe(before["root"]);
+    expect(readSchemaTag(h, vm), "still not tagged v2").not.toContain("lab-schema-v2");
+    const markers = readMarkers(h, vm, ["/home", "/srv"]);
+    expect(markers["/home"]).toBe(token);
+    expect(markers["/srv"]).toBe(token);
+  }, 120 * 60_000);
+});

bastion/tests/integration/rolling-drill.test.ts

@@ -0,0 +1,623 @@
+// THE MIGRATION DRILL: roll three storage+etcd nodes through
+// evacuate -> wipe -> reinstall -> rejoin, one at a time, with data intact.
+//
+// WHAT IT IS PROVING
+// ------------------
+// The v1 -> v2 layout migration cannot preserve data through the wipe: %pre
+// only preserves when labvg already has the expected LV names, and a new layout
+// recreates the VG, taking the clearpart branch. So the data has to leave the
+// node first and come back afterwards. This asserts, on real machines, that it
+// does -- by sha256, either side of every step.
+//
+// WHY FOUR NODES FOR A THREE-NODE ROLL
+// ------------------------------------
+// With strict anti-affinity (replica-soft-anti-affinity=false, what the fleet
+// runs) a volume puts at most one replica per node, so a 3-replica volume on a
+// 3-node cluster CANNOT be evicted: the two survivors already hold a copy each
+// and cannot take the third. Eviction would wait for its full timeout and fail.
+// The real fleet has exactly this shape, plus a 100 GiB disk on worker2 with
+// ~49 GiB free -- so capacity blocks it independently of anti-affinity.
+//
+// The fix is the one already planned for production: bring the NEW box up
+// first, then evacuate an old one onto it. The fourth node here is an AGENT, so
+// etcd stays a 3-member cluster and every roll runs against production's real
+// quorum arithmetic (2 of 3, no margin).
+//
+// ORDER OF THE DANGEROUS PART
+// ---------------------------
+// The irreversible pair -- etcd member removal and node deletion -- happens as
+// late as possible, immediately before the wipe. Anything that fails before it
+// has destroyed nothing.
+//
+//   sudo -E npx vitest run -c tests/integration/vitest.config.ts -t "rolling drill"
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { K3sModule } from "../../src/modules/modules/k3s/src/k3s-module.js";
+import {
+  startTestBastion, provisionVms, reprovisionVm, readLvs, destroyVms,
+  type Harness, type ProvisionedVm,
+} from "./helpers/provision-harness.js";
+import { PINNED_K3S_VERSION } from "./helpers/k3s-cluster.js";
+import {
+  formCluster, joinAgent, installK3sNode, serverUrlOf, moduleCtx,
+  clusterSnapshot, printSnapshot, etcdMembers, etcdHealthy, allServersReady,
+  pickJoinTarget, kubectlOn, kubectlJson, nodeExec, removeEtcdMember, waitForNodeReady,
+  waitForApiPing, sleep, DRILL_TOKEN, k8sNodeName, forgetNodeName,
+} from "./helpers/k3s-cluster.js";
+import {
+  installLonghorn, waitForLonghornReady, createDataWorkload, waitForWorkload,
+  writeBlob, verifyBlob, volumeState, waitForVolumeHealthy, LONGHORN_NS,
+  backupTargetFromEnv, setBackupTarget, waitForBackupTarget, backupVolume, restoreFromBackup,
+  readBlobDigest,
+} from "./helpers/longhorn.js";
+import { destroyPxeNetwork } from "./helpers/pxe-network.js";
+
+const HTTP_PORT = 8098;
+const SSH_USER = "lab";
+const DISK_GB = 1000;
+
+const SERVERS = ["drill-s1", "drill-s2", "drill-s3"];
+const SPARE = "drill-spare";
+const ALL = [...SERVERS, SPARE];
+
+/** The replica-3 volume: the one whose rebuild the roll depends on. */
+const R3 = "drill-r3";
+/** The replica-1 volume: single copy, the sharpest risk in the real fleet. */
+const R1 = "drill-r1";
+
+function findSshKey(): { pubKey: string; keyPath: string } {
+  const homes = [homedir()];
+  const sudoUser = process.env["SUDO_USER"];
+  if (sudoUser) homes.push(join("/home", sudoUser));
+  for (const home of homes) {
+    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
+      const keyPath = join(home, ".ssh", name);
+      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
+        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
+      }
+    }
+  }
+  throw new Error("no SSH keypair found");
+}
+
+describe("rolling drill: migrate three nodes without losing data", () => {
+  let h: Harness;
+  let vms: ProvisionedVm[] = [];
+  let spare: ProvisionedVm;
+  let servers: ProvisionedVm[] = [];
+  let digestR3 = "";
+  let digestR1 = "";
+  /**
+   * S3 backup target, from DRILL_S3_* in the environment.
+   *
+   * Undefined means the backup/restore phase SKIPS rather than passing
+   * vacuously. A backup test that quietly does nothing is worse than none.
+   */
+  const backupTarget = backupTargetFromEnv();
+
+  /**
+   * Sticky abort. Once a step fails, the remaining steps SKIP rather than run
+   * against a cluster that is already wrong -- a failed roll leaves a node
+   * mid-migration, and rolling the next one on top of that turns one recoverable
+   * problem into an unrecoverable one.
+   */
+  let aborted: string | null = null;
+  /**
+   * `ctx` is vitest's test context. It is used to SKIP downstream steps rather
+   * than let them return quietly — a silent return marks the test PASSED, so a
+   * drill that died in provisioning reports "1 failed | 8 passed" and the eight
+   * steps that never ran look like evidence. They are not evidence.
+   */
+  const step = async (
+    ctx: { skip: (note?: string) => void },
+    name: string,
+    fn: () => Promise<void>,
+  ): Promise<void> => {
+    if (aborted !== null) {
+      console.log(`[drill] SKIPPED ${name} (aborted at: ${aborted})`);  // eslint-disable-line no-console
+      ctx.skip(`drill aborted at: ${aborted}`);
+      return;
+    }
+    const t0 = Date.now();
+    console.log(`[drill] ==> ${name}`);  // eslint-disable-line no-console
+    try {
+      await fn();
+      console.log(`[drill] <== ${name} (${Math.round((Date.now() - t0) / 1000)}s)`);  // eslint-disable-line no-console
+    } catch (e) {
+      aborted = name;
+      console.error(`[drill] !!! ${name} FAILED: ${e instanceof Error ? e.message : String(e)}`);  // eslint-disable-line no-console
+      throw e;
+    }
+  };
+
+  /**
+   * The runbook's replica-1 pre-flight, rehearsed rather than described.
+   *
+   * A single-replica volume rebuilds from the only copy that exists, so
+   * `evacuate` refuses it by default. The documented answer is to raise it to
+   * 3, wait for healthy, evacuate, and put it back afterwards. Doing that here
+   * means the drill exercises the procedure operators will actually follow
+   * instead of quietly bypassing it with the override flag.
+   */
+  const raiseReplicas = async (peer: ProvisionedVm, pvc: string, to: number): Promise<boolean> => {
+    const st = volumeState(h, peer, pvc);
+    kubectlOn(h, peer, `-n ${LONGHORN_NS} patch volumes.longhorn.io ${st.pv} ` +
+      `--type=merge -p '{"spec":{"numberOfReplicas":${to}}}'`);
+    if (to <= 1) return true;
+    try {
+      // BOUNDED, and allowed to fail.
+      //
+      // Raising a volume's replica count only converges if every target node
+      // will actually start a replica. A node that has just been wiped and
+      // rejoined may not: observed here, a replica sat at
+      // currentState=stopped / desireState=running / failedAt='' for forty
+      // minutes with no rebuild and no error, while the volume stayed
+      // degraded at 2-of-3 RW. Blocking the whole drill on that turns a
+      // Longhorn quirk into a total failure and proves nothing.
+      await waitForVolumeHealthy(h, peer, { pvc, expectReplicas: to, timeoutMs: 10 * 60_000 });
+      return true;
+    } catch (e) {
+      console.warn(  // eslint-disable-line no-console
+        `[drill] ${pvc} did not reach ${to} healthy replicas in 10 min: ` +
+        `${e instanceof Error ? e.message : String(e)}`);
+      return false;
+    }
+  };
+
+  /** Is this volume's only replica on the node we are about to evacuate? */
+  const replicaOn = (peer: ProvisionedVm, pvc: string, node: string): boolean =>
+    volumeState(h, peer, pvc).replicaNodes.includes(node);
+
+  /** Read cluster state from a node that is NOT the one being worked on. */
+  const survivor = (exclude?: ProvisionedVm): ProvisionedVm =>
+    exclude ? pickJoinTarget(h, servers, exclude) : servers[0]!;
+
+  beforeAll(async () => {
+    const { pubKey, keyPath } = findSshKey();
+    destroyVms(...ALL);
+    h = await startTestBastion({
+      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey, sshKeyPath: keyPath,
+      label: "rolling-drill",
+      autoK3s: false,   // we sequence cluster formation ourselves
+    });
+  }, 20 * 60_000);
+
+  afterAll(async () => {
+    // Diagnostics BEFORE teardown: these VMs are the only record of what
+    // happened and they are about to be deleted.
+    try {
+      const s = servers[0];
+      if (s) {
+        printSnapshot("FINAL", clusterSnapshot(h, s));
+        for (const pvc of [R3, R1]) {
+          try {
+            const st = volumeState(h, s, pvc);
+            console.log(`[drill] ${pvc}: ${st.state}/${st.robustness} on ${st.replicaNodes.join(",")}`);  // eslint-disable-line no-console
+          } catch { /* volume may be gone */ }
+        }
+        console.log(`[drill] digests: ${R3}=${digestR3.slice(0, 16)} ${R1}=${digestR1.slice(0, 16)}`);  // eslint-disable-line no-console
+      }
+    } catch { /* cluster may be unreachable */ }
+
+    // Best-effort: clear any eviction we may have left pending. Harmless on a
+    // VM about to be destroyed, but this is the same call the runbook makes on
+    // a real node, so exercising it here is free coverage.
+    for (const vm of servers) {
+      try { await new K3sModule().admit(moduleCtx(h, vm, "k8s-combo")); } catch { /* ignore */ }
+    }
+
+    if (process.env["LAB_DRILL_KEEP_VMS"] !== "1") {
+      destroyVms(...ALL);
+      try { destroyPxeNetwork(); } catch { /* shared */ }
+    } else {
+      console.log("[drill] LAB_DRILL_KEEP_VMS=1 — leaving VMs up for inspection");  // eslint-disable-line no-console
+    }
+    await h?.close();
+  }, 20 * 60_000);
+
+  it("provisions four v2 nodes", async (ctx) => {
+    await step(ctx, "provision", async () => {
+      vms = await provisionVms(h, [
+        ...SERVERS.map((name) => ({
+          name, role: "k8s-combo" as const, schemaVersion: 2 as const,
+          diskGb: DISK_GB, memory: 6144, vcpus: 4,
+        })),
+        { name: SPARE, role: "k8s-worker" as const, schemaVersion: 2 as const,
+          diskGb: DISK_GB, memory: 4096, vcpus: 2 },
+      ]);
+      servers = vms.filter((v) => SERVERS.includes(v.name));
+      spare = vms.find((v) => v.name === SPARE)!;
+      expect(servers).toHaveLength(3);
+
+      // Every node must actually have the longhorn LV, or nothing later works.
+      for (const vm of vms) expect(Object.keys(readLvs(h, vm))).toContain("longhorn");
+    });
+  }, 90 * 60_000);
+
+  it("forms one cluster with three etcd members plus a storage agent", async (ctx) => {
+    await step(ctx, "form cluster", async () => {
+      await formCluster(h, servers, { role: "k8s-combo" });
+      await joinAgent(h, spare, { server: servers[0]!, role: "k8s-worker" });
+
+      const snap = clusterSnapshot(h, servers[0]!);
+      printSnapshot("formed", snap);
+      expect(snap.nodes).toHaveLength(4);
+      expect(snap.readyCount).toBe(4);
+      // The spare is an agent on purpose: etcd stays at 3 so each roll runs
+      // against production's real quorum arithmetic.
+      expect(etcdMembers(h, servers[0]!)).toHaveLength(3);
+      expect(etcdHealthy(h, servers[0]!)).toBe(true);
+
+      // Every node must run the PINNED version, not whatever get.k3s.io calls
+      // latest today. Setting k3sVersion is not the same as it taking effect:
+      // toOpContext used to drop the field silently, so the pin was ignored and
+      // nodes came up two minor versions ahead of the fleet. Assert the
+      // outcome, on the machines.
+      const versions = kubectlJson<{ items: {
+        metadata: { name: string };
+        status: { nodeInfo: { kubeletVersion: string } };
+      }[] }>(h, servers[0]!, "get nodes");
+      expect(versions.items).toHaveLength(4);
+      for (const n of versions.items) {
+        expect(n.status.nodeInfo.kubeletVersion,
+          `${n.metadata.name} must run the pinned ${PINNED_K3S_VERSION}`)
+          .toBe(PINNED_K3S_VERSION);
+      }
+      console.log(`[drill] all 4 nodes on pinned ${PINNED_K3S_VERSION}`);  // eslint-disable-line no-console
+    });
+  }, 120 * 60_000);
+
+  it("installs Longhorn with a schedulable disk on every node", async (ctx) => {
+    await step(ctx, "install longhorn", async () => {
+      await installLonghorn(h, servers[0]!, {
+        defaultReplicas: 3,
+        ...(backupTarget === undefined ? {} : { backup: backupTarget }),
+      });
+
+      // Give Longhorn time to register the nodes, then run the PRODUCTION
+      // repair path on each one.
+      //
+      // configureLonghornDisk runs during configure(), which happens BEFORE
+      // Longhorn is installed -- there is no node record to repair yet, so it
+      // only sets the annotation and returns. If Longhorn then misses that
+      // annotation for any reason, nothing ever retries, and the node joins,
+      // goes Ready, and silently hosts no replicas. Observed: one node in four
+      // came up with no disk while its label and annotation were byte-identical
+      // to its siblings'.
+      //
+      // admit() is exactly what the runbook tells an operator to run, and it
+      // exercises the verify-and-repair branch that is otherwise only reachable
+      // after a wipe-and-rejoin.
+      await sleep(60_000);
+      for (const vm of vms) {
+        const role = vm.name === SPARE ? "k8s-worker" : "k8s-combo";
+        const res = await new K3sModule().admit(moduleCtx(h, vm, role));
+        for (const line of res.output) console.log(`  [${vm.name}] ${line}`);  // eslint-disable-line no-console
+      }
+
+      // All four VMs, including the agent: the spare having a schedulable disk
+      // is the whole reason eviction can complete at all.
+      await waitForLonghornReady(h, servers[0]!, vms);
+
+      // Read it back rather than trusting the wait: every node must have a
+      // disk, or replicas silently never land there.
+      //
+      // -o json and parse, NOT jsonpath. A jsonpath containing {"\n"} has to
+      // survive a TS template literal AND a shell, and getting either escape
+      // wrong yields `unterminated quoted string` at runtime — which is how
+      // this assertion failed the first time it ran.
+      const lhNodes = kubectlJson<{ items: {
+        metadata: { name: string };
+        spec: { disks?: Record<string, unknown> };
+      }[] }>(h, servers[0]!, `-n ${LONGHORN_NS} get nodes.longhorn.io`);
+
+      for (const vm of vms) {
+        const name = k8sNodeName(h, vm);
+        const rec = lhNodes.items.find((n) => n.metadata.name === name);
+        expect(rec, `${name} has no Longhorn node record`).toBeDefined();
+        const disks = Object.keys(rec?.spec.disks ?? {});
+        expect(disks.length, `${name} has no Longhorn disk — the default-disks-config ` +
+          `annotation never parsed, so this node would silently host no replicas`)
+          .toBeGreaterThan(0);
+      }
+    });
+  }, 40 * 60_000);
+
+  it("writes known data and proves it survives a detach/attach", async (ctx) => {
+    await step(ctx, "seed data", async () => {
+      const s = servers[0]!;
+      createDataWorkload(h, s, { name: R3, sizeGi: 2, replicas: 3 });
+      createDataWorkload(h, s, { name: R1, sizeGi: 1, replicas: 1 });
+      await waitForWorkload(h, s, R3);
+      await waitForWorkload(h, s, R1);
+
+      digestR3 = await writeBlob(h, s, { name: R3, mib: 512 });
+      digestR1 = await writeBlob(h, s, { name: R1, mib: 128 });
+
+      // Record the digest only AFTER it has survived one restart. Otherwise
+      // the baseline could be a page-cache artefact and every later
+      // comparison would be against a value that was never really on disk.
+      await verifyBlob(h, s, { name: R3, expectDigest: digestR3 });
+      await verifyBlob(h, s, { name: R1, expectDigest: digestR1 });
+
+      const st = volumeState(h, s, R3);
+      expect(st.robustness).toBe("healthy");
+      expect(new Set(st.replicaNodes).size).toBe(3);
+    });
+  }, 60 * 60_000);
+
+  it("backs up a volume to S3 and restores it byte-identical", async (ctx) => {
+    await step(ctx, "backup and restore", async () => {
+      const s = servers[0]!;
+
+      if (backupTarget === undefined) {
+        // Deliberately loud. A backup phase that silently passes because no
+        // target was configured is exactly the kind of green tick that makes
+        // people believe they have backups when they do not.
+        throw new Error(
+          "no backup target configured (DRILL_S3_TARGET / _ENDPOINT / _ACCESS_KEY / _SECRET_KEY). " +
+          "The backup/restore phase cannot prove anything without one.");
+      }
+
+      // Set it on the CR. defaultSettings.backupTarget does nothing in 1.8 --
+      // see setBackupTarget for why.
+      await setBackupTarget(h, s, backupTarget);
+
+      // `available: true` is Longhorn saying it reached the endpoint AND
+      // authenticated. Backing up before checking gives a failure that reads
+      // like a volume problem.
+      await waitForBackupTarget(h, s);
+
+      const { backup } = await backupVolume(h, s, R3);
+
+      // Restore into a SEPARATE volume. The original is untouched, so a failed
+      // restore cannot damage the source -- which is the property that makes
+      // this safe to rehearse at all.
+      await restoreFromBackup(h, s, { backup, name: "drill-restored", sizeGi: 2 });
+      await waitForWorkload(h, s, "drill-restored");
+
+      // The whole point: the bytes that came back are the bytes that went in.
+      // Compared against the digest recorded BEFORE any of this, which was
+      // itself only trusted after surviving a detach/attach.
+      const restored = readBlobDigest(h, s, "drill-restored");
+      expect(restored, "restored volume digest must equal the original").toBe(digestR3);
+      console.log(`[drill] restored digest ${restored.slice(0, 16)}... matches original`);  // eslint-disable-line no-console
+
+      // And the restored volume is a real, independent volume.
+      const st = volumeState(h, s, "drill-restored");
+      expect(st.pv).not.toBe(volumeState(h, s, R3).pv);
+      expect(st.state).toBe("attached");
+    });
+  }, 45 * 60_000);
+
+  it("REFUSES to evacuate when there is nowhere for replicas to go", async (ctx) => {
+    await step(ctx, "negative pre-check", async () => {
+      const s = servers[0]!;
+
+      // Must be a node that actually HOLDS an r3 replica, and not the one we
+      // read cluster state from. r3 has 3 replicas across 4 nodes, so one node
+      // has none -- picking blindly makes this test pass or fail on placement
+      // luck rather than on the behaviour it is meant to pin.
+      // replicaNodes are KUBERNETES node names (smoke-a.pxe-test.local), not
+      // libvirt domain names, so resolve before comparing.
+      const r3Nodes = volumeState(h, s, R3).replicaNodes;
+      const target = servers.find(
+        (v) => v.name !== s.name && r3Nodes.includes(k8sNodeName(h, v)));
+      expect(target, `no server other than ${s.name} holds an ${R3} replica (${r3Nodes.join(",")})`)
+        .toBeDefined();
+
+      // Take the spare out of service: now 3 schedulable nodes for a 3-replica
+      // volume, which is exactly the real fleet's shape.
+      kubectlOn(h, s, `-n ${LONGHORN_NS} patch nodes.longhorn.io ${k8sNodeName(h, spare)} ` +
+        `--type=merge -p '{"spec":{"allowScheduling":false}}'`);
+      await sleep(10_000);
+
+      const t0 = Date.now();
+      const res = await new K3sModule().evacuate(moduleCtx(h, target!, "k8s-combo"));
+      const elapsed = (Date.now() - t0) / 1000;
+
+      expect(res.success, "evacuation should have been refused").toBe(false);
+      // Fast is the whole point: the old behaviour was a 60-minute wait.
+      expect(elapsed, `refused in ${elapsed}s`).toBeLessThan(300);
+      expect(res.errors.join(" ")).toMatch(/cannot complete|cannot be placed|wants 3 replicas/);
+
+      // And it must not have touched anything on the way to saying no.
+      const lhNode = kubectlOn(h, s,
+        `-n ${LONGHORN_NS} get nodes.longhorn.io ${k8sNodeName(h, target!)} ` +
+        `-o jsonpath='{.spec.evictionRequested}'`).trim();
+      expect(lhNode, "refusal must not leave an eviction pending").not.toBe("true");
+
+      // Put the spare back and un-cordon whatever the attempt cordoned.
+      kubectlOn(h, s, `-n ${LONGHORN_NS} patch nodes.longhorn.io ${k8sNodeName(h, spare)} ` +
+        `--type=merge -p '{"spec":{"allowScheduling":true}}'`);
+      await new K3sModule().admit(moduleCtx(h, target!, "k8s-combo"));
+      await sleep(15_000);
+    });
+  }, 30 * 60_000);
+
+  // The roll itself. Serial by construction -- each `it` depends on the last.
+  for (const name of SERVERS) {
+    it(`rolls ${name}: evacuate, wipe, reinstall, rejoin, verify`, async (ctx) => {
+      await step(ctx, `roll ${name}`, async () => {
+        const target = vms.find((v) => v.name === name)!;
+        const peer = survivor(target);
+        // Kubernetes and Longhorn both know this machine by its FQDN, not by
+        // the libvirt domain name used everywhere else in this file.
+        const nodeName = k8sNodeName(h, target);
+
+        // --- GATE: never start a roll on a cluster that is already degraded.
+        const pre = clusterSnapshot(h, peer);
+        printSnapshot(`before rolling ${name}`, pre);
+        expect(pre.etcdMembers, "etcd membership before roll").toBe(3);
+        expect(pre.etcdHealthy, "etcd health before roll").toBe(true);
+        expect(pre.readyCount, "all nodes Ready before roll").toBe(4);
+
+        // --- Replica-1 pre-flight, exactly as the runbook prescribes.
+        //
+        // If drill-r1's single replica is on this node, evacuate MUST refuse:
+        // moving it means rebuilding from the only copy in existence, and this
+        // fleet has no backup target. Assert that refusal, then apply the
+        // documented mitigation rather than overriding the guard.
+        const r1Here = replicaOn(peer, R1, nodeName);
+        let r1Raised = false;
+        if (r1Here) {
+          const refused = await new K3sModule().evacuate(moduleCtx(h, target, "k8s-combo"));
+          expect(refused.success, "single-replica volume should have blocked evacuation").toBe(false);
+          expect(refused.errors.join(" ")).toMatch(/single replica/);
+          await new K3sModule().admit(moduleCtx(h, target, "k8s-combo"));
+          r1Raised = await raiseReplicas(peer, R1, 3);
+        }
+
+        // --- Evacuate (cordon -> check -> evict+wait -> drain -> verify).
+        //
+        // If raising drill-r1 did not converge, fall back to the documented
+        // override rather than stalling. The guard itself is already proven by
+        // the refusal above and by the negative pre-check; what this roll is
+        // here to prove is the migration.
+        const evacConfig = r1Here && !r1Raised ? { evictionAllowSingleReplica: true } : {};
+        if (r1Here && !r1Raised) {
+          console.warn(  // eslint-disable-line no-console
+            "[drill] proceeding with --allow-single-replica because the raise did not converge");
+        }
+        const evac = await new K3sModule().evacuate(moduleCtx(h, target, "k8s-combo", evacConfig));
+        for (const line of evac.output) console.log(`  [${name}] ${line}`);  // eslint-disable-line no-console
+        expect(evac.success, `evacuate ${name}: ${evac.errors.join("; ")}`).toBe(true);
+
+        // Replicas really gone, and the r3 volume really rebuilt elsewhere.
+        const afterEvac = await waitForVolumeHealthy(h, peer, { pvc: R3, expectReplicas: 3 });
+        expect(afterEvac.replicaNodes).not.toContain(nodeName);
+
+        // --- Data intact after evacuation, BEFORE anything is destroyed.
+        await verifyBlob(h, peer, { name: R3, expectDigest: digestR3 });
+        await verifyBlob(h, peer, { name: R1, expectDigest: digestR1 });
+
+        // --- The irreversible part starts here and not before.
+        nodeExec(h, target, "systemctl stop k3s || true");
+        const removed = removeEtcdMember(h, peer, nodeName);
+        expect(removed, `etcd member ${nodeName} should have existed`).toBe(true);
+        kubectlOn(h, peer, `delete node ${nodeName} --wait=true --timeout=5m`, 6 * 60_000);
+
+        // Quorum asserted POSITIVELY: 2 of 3 is survivable, silence is not proof.
+        const mid = clusterSnapshot(h, peer);
+        printSnapshot(`${name} removed`, mid);
+        expect(mid.etcdMembers, "etcd must be down to exactly 2 members").toBe(2);
+        expect(mid.etcdHealthy, "surviving etcd must still be healthy").toBe(true);
+        const ready = allServersReady(h, servers.filter((v) => v.name !== name));
+        expect(ready.ok, `survivors /readyz: ${ready.detail}`).toBe(true);
+
+        // --- Wipe and reinstall. The disk decides the schema, not the request.
+        //
+        // reprovisionVm returns a NEW record: the machine can come back on a
+        // different DHCP address. Every later reference must use that record,
+        // and the shared arrays have to be updated too or `survivor()` will
+        // hand out a stale IP on the next roll.
+        const rolled = await reprovisionVm(h, target, "k8s-combo");
+        vms = vms.map((v) => (v.name === name ? rolled : v));
+        servers = servers.map((v) => (v.name === name ? rolled : v));
+        // A reinstalled machine re-registers; do not trust the cached name.
+        forgetNodeName(rolled);
+
+        // v2 reinstall PRESERVES /var/lib/longhorn, so without this the next
+        // step would measure replica re-adoption, not a real rebuild. The
+        // actual v1->v2 migration recreates the VG and has no such leftovers.
+        //
+        // Clear the CONTENTS, never the directory itself. Longhorn's replica
+        // process does a non-recursive `mkdir /host/var/lib/longhorn/replicas/
+        // <name>`, which fails outright when the parent is missing:
+        //
+        //   level=fatal msg="Error running start replica command"
+        //     error="mkdir /host/var/lib/longhorn/replicas/pvc-...: ..."
+        //
+        // Every replica scheduled onto the node then dies instantly and
+        // permanently, while the node looks entirely healthy: longhorn-disk.cfg
+        // survives the wipe, so the disk keeps its UUID and reports Ready and
+        // Schedulable. The next node's eviction can never drain onto it and
+        // stalls for its full hour. Two earlier drill runs died exactly here.
+        nodeExec(h, rolled,
+          "rm -rf /var/lib/longhorn/replicas/* /var/lib/longhorn/*.tmp 2>/dev/null; " +
+          "mkdir -p /var/lib/longhorn/replicas && ls -d /var/lib/longhorn/replicas");
+
+        // --- Rejoin against a SURVIVOR, not against itself.
+        await waitForApiPing(peer);
+        await installK3sNode(h, rolled, {
+          role: "k8s-combo", token: DRILL_TOKEN, serverUrl: serverUrlOf(peer),
+        });
+        await waitForNodeReady(h, peer, rolled);
+
+        // Exactly 3. Four would mean the old member survived the rejoin, which
+        // silently raises quorum from 2-of-3 to 3-of-4 and removes all margin.
+        const post = clusterSnapshot(h, peer);
+        printSnapshot(`${name} rejoined`, post);
+        expect(post.etcdMembers, "stale etcd member left behind").toBe(3);
+        expect(post.etcdHealthy).toBe(true);
+        expect(post.readyCount).toBe(4);
+
+        // --- Back into service, then wait for the replica to rebuild onto it.
+        const admit = await new K3sModule().admit(moduleCtx(h, rolled, "k8s-combo"));
+        for (const line of admit.output) console.log(`  [${name}] ${line}`);  // eslint-disable-line no-console
+        expect(admit.success, `admit ${name}: ${admit.errors.join("; ")}`).toBe(true);
+
+        const healthy = await waitForVolumeHealthy(h, peer, { pvc: R3, expectReplicas: 3 });
+        expect(new Set(healthy.replicaNodes).size).toBe(3);
+
+        // --- And the data is still the data.
+        await verifyBlob(h, peer, { name: R3, expectDigest: digestR3 });
+        await verifyBlob(h, peer, { name: R1, expectDigest: digestR1 });
+
+        // Put drill-r1 back to a single replica, so the next roll faces the
+        // same hazard rather than a progressively safer cluster.
+        if (r1Here && r1Raised) await raiseReplicas(peer, R1, 1);
+
+        // --- Does the rolled node actually take replicas BACK?
+        //
+        // Roll 1 originally "passed" with drill-r3 healthy on three nodes that
+        // did not include the node just rolled -- the assertion counted
+        // distinct nodes without ever checking that the reinstalled one was
+        // among them. A node that rejoins, reports Ready and Schedulable, and
+        // then hosts nothing is the exact failure this whole exercise exists to
+        // catch, so record it explicitly.
+        // RUNNING, not merely assigned. The first version of this check used
+        // replicaNodes (which comes from spec.nodeID) and happily passed on a
+        // node holding two replicas that were both state=stopped -- exactly the
+        // condition that makes the NEXT eviction impossible, because Longhorn
+        // keeps scheduling replacements onto a node that cannot start them.
+        const running = kubectlJson<{ items: {
+          spec: { nodeID?: string };
+          status?: { currentState?: string };
+        }[] }>(h, peer, `-n ${LONGHORN_NS} get replicas.longhorn.io`)
+          .items.filter((r) => r.spec.nodeID === nodeName && r.status?.currentState === "running");
+        console.log(  // eslint-disable-line no-console
+          `[drill] after rolling ${name}: ${running.length} RUNNING replica(s) on the node`);
+        expect(running.length,
+          `${name} rejoined but hosts no RUNNING replica — it would look healthy and ` +
+          `silently take no storage, and the next eviction could never drain onto it`)
+          .toBeGreaterThan(0);
+      });
+    }, 240 * 60_000);
+  }
+
+  it("ends with a whole cluster and the original bytes", async (ctx) => {
+    await step(ctx, "final verification", async () => {
+      const s = servers[0]!;
+      const snap = clusterSnapshot(h, s);
+      printSnapshot("end of drill", snap);
+
+      expect(snap.nodes).toHaveLength(4);
+      expect(snap.readyCount).toBe(4);
+      expect(snap.etcdMembers).toBe(3);
+      expect(snap.etcdHealthy).toBe(true);
+
+      const r3 = volumeState(h, s, R3);
+      expect(r3.robustness).toBe("healthy");
+      expect(new Set(r3.replicaNodes).size).toBe(3);
+
+      // One last restart-and-check, so the final claim is about bytes read
+      // back through a fresh attach rather than a cached mount.
+      await verifyBlob(h, s, { name: R3, expectDigest: digestR3 });
+      await verifyBlob(h, s, { name: R1, expectDigest: digestR1 });
+    });
+  }, 60 * 60_000);
+});

labsim/README.md

@@ -143,6 +143,67 @@ unreserved MAC gets an unreserved address.
 - **http://localhost:9101/metrics** — `labsim_reachable{src,dst,proto}` and
   `labsim_rtt_ms{src,dst}`.
 
+## WAN follows VRRP, and the PPPoE half of it
+
+One consumer ISP account, two routers. The 10 gig line's lease is bound to a
+cloned MAC and the Vodafone line to a single credential, so neither can be live
+on both boxes: the WAN has to move with mastership.
+
+The two halves use different control planes, and that asymmetry is the design:
+
+| | plane | why |
+|---|---|---|
+| `bond0.53` | VyOS **config** (`disable`) | only config can move a MAC |
+| `pppoe0` | **systemd** unit gate | see below |
+
+`set interfaces pppoe pppoe0 disable` cannot work as a resting state.
+`interfaces_pppoe.py` treats `disable` and `delete` identically and **unlinks
+`/etc/ppp/peers/pppoe0`** — which is pppd's own options file. The resting state
+therefore destroyed what the promotion path needed, and `ppp@pppoe0`
+restart-looped against it (47 restarts, zero sessions at the AC). It also makes
+op-mode `connect interface pppoe0` refuse, and puts every failover behind a
+priority-322 commit where one unrelated bad node fails the lot.
+
+So `pppoe0` is configured identically and **enabled on both**, and dialling is
+gated by a drop-in:
+
+```ini
+ConditionPathExists=/run/vrrp-wan/may-dial
+ConditionPathExists=/etc/ppp/peers/pppoe0
+```
+
+`/run` is tmpfs, so the gate is shut at boot. That matters more than it looks:
+with the node enabled, `interfaces_pppoe.py` restarts ppp on **every** commit
+touching the pppoe subtree when the daemon isn't running — so the backup
+actively tries to dial whenever anything commits. The gate is the only thing
+making that a no-op, which is why the reconciler refuses to bless a box whose
+drop-in is missing (`/etc` is per-image; a VyOS upgrade would silently remove
+the protection).
+
+`may-dial` is a **lease**, not a flag: `ConditionPathExists` is evaluated at
+start only, so it can prevent a dial but never revoke one. `vrrp-wan-reconcile`
+renews it every 30s; `vrrp-wan-guard` runs every 5s and only ever revokes.
+
+### Testing it
+
+```sh
+./labsim-pppoe-ha-test.sh --all
+```
+
+The verdict is what the **routers** and the **access concentrator** did, never
+what a client happened to get — and the harness refuses to run at all while a
+router still has a default route via `eth2`, because the libvirt-NAT scaffold
+answers connectivity checks that the WAN under test would have failed. The
+invariant it enforces throughout: *the AC never reports two `simdsl` sessions,
+and no two routers ever hold `pppoe0`.*
+
+Two failures the harness itself produced, both worth remembering: waiting for
+"exactly one holder" returns instantly during a handover (it was already true),
+and judging connectivity on a single ping 20s after a link drop reports an
+outage that has already healed. Ask **who** holds it, and poll.
+
+Evidence in `wan-failover-evidence/`.
+
 ## Routing: BGP, dual WAN, and the ISP VMs
 
 `sim-ha-config.py` covers the LAN side of the routers. `sim-net-config.py`

labsim/console-apply.py

@@ -22,6 +22,12 @@ def main() -> int:
     ap.add_argument("--config", required=True)
     ap.add_argument("--user", default="vyos")
     ap.add_argument("--password", default="vyos")
+    # `save` writes config.boot. For WAN work that is dangerous: the resting
+    # state must stay `vif 53 disable` on both routers, and saving while a box
+    # is master persists the ENABLED state -- so a reboot would have it claim
+    # the cloned MAC. Observed in labsim on 2026-09-02.
+    ap.add_argument("--no-save", action="store_true",
+                    help="commit without saving (leave config.boot untouched)")
     args = ap.parse_args()
 
     cmds = [l.rstrip() for l in open(args.config)
@@ -83,8 +89,9 @@ def main() -> int:
     c.sendline("commit")
     c.expect(r"# ", timeout=300)
     commit_out = c.before or ""
-    c.sendline("save")
-    c.expect(r"# ", timeout=120)
+    if not args.no_save:
+        c.sendline("save")
+        c.expect(r"# ", timeout=120)
     # Accept either prompt on the way out. Insisting on `$ ` here hangs against
     # a healthy box -- and worse, leaves the console parked in config mode, so
     # the NEXT run finds a `# ` it was not expecting either. One strict expect

labsim/ipv6-ha-evidence/V0-baseline/state.txt

@@ -0,0 +1,43 @@
+=== 2026-09-06T14:25:25+01:00 ===
+--- HE endpoint ---
+  HE address : 192.0.2.10/32
+  he-sim     : he-sim: ipv6/ip remote 198.51.100.137 local 192.0.2.10 ttl 64 6rd-prefix 2002::/16
+  he-sim v6  : 2001:db8:1f1c:f6::1/64
+  API        : running
+  API bound  : nohost
+  API calls  : 4
+  route back : 198.51.100.0/24 via 192.168.122.63 dev eth1 
+--- 172.31.1.252 ---
+vip=172.31.1.1 holds_vip=no wan_disabled=yes wan_up=no ppp_up=no ppp_active=no may_dial=no lease_age=- dropin=yes role=backup tun=DOWN radvd=inactive
+inactive
+tun0@NONE        DOWN           203.0.113.108 <POINTOPOINT,NOARP> 
+tun0: ipv6/ip remote 192.0.2.10 local 203.0.113.108 ttl 64 tos inherit 6rd-prefix 2002::/16
+inactive
+Sep 05 23:27:45 apitest vrrp-wan[13361]: bond0.53 enable commit took 5s
+Sep 05 23:29:01 apitest vrrp-wan[16369]: GUARD: lease stale (200s > 75s; is vrrp-wan-reconcile.timer running?) -- hanging up pppoe0
+Sep 05 23:29:27 apitest vrrp-wan[17398]: MASTER: dialling pppoe0
+Sep 05 23:29:57 apitest vrrp-wan[18610]: MASTER: dialling pppoe0
+-- Boot 6efc3c9ba47f455e9668454ee6c2fc37 --
+Sep 05 23:34:48 apitest vrrp-wan[6266]: MASTER: dialling pppoe0
+Sep 05 23:34:49 apitest vrrp-wan[6422]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:34:54 apitest vrrp-wan[7130]: bond0.53 enable commit took 5s
+-- Boot 866611afd7c542d4bf8c5117978dcd4a --
+Sep 06 13:21:54 apitest vrrp-wan[544215]: not MASTER: stopping radvd (deprecates the v6 gateway)
+Sep 06 13:21:54 apitest vrrp-wan[544221]: not MASTER: bringing tun0 down
+Sep 06 13:22:01 apitest he-tunnel-follow[544377]: role is now backup
+--- 172.31.1.253 ---
+vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes lease_age=25 dropin=yes role=master tun=UNKNOWN radvd=active
+tun0@NONE        UNKNOWN        198.51.100.137 <POINTOPOINT,NOARP,UP,LOWER_UP> 
+tun0: ipv6/ip remote 192.0.2.10 local 198.51.100.137 ttl 64 tos inherit 6rd-prefix 2002::/16
+default nhid 111 via 2001:db8:1f1c:f6::1 dev tun0 proto static metric 20 pref medium
+active
+Sep 06 09:29:21 vyos vrrp-wan[422441]: MASTER with bond0.53 disabled -> enabling
+Sep 06 09:29:25 vyos vrrp-wan[423516]: bond0.53 enable commit took 4s
+Sep 06 13:22:01 vyos he-tunnel-follow[593096]: role is now master
+Sep 06 13:22:01 vyos he-tunnel-follow[593109]: change seen (203.0.113.108 -> 198.51.100.137) but waiting for stability (1/2)
+Sep 06 13:23:01 vyos he-tunnel-follow[594799]: HE endpoint set to 198.51.100.137 (good 198.51.100.137)
+Sep 06 13:23:01 vyos he-tunnel-follow[594805]: moved tun0 to pppoe0: src 203.0.113.108 -> 198.51.100.137, mtu 1480 -> 1472
+Sep 06 13:24:02 vyos he-tunnel-follow[595700]: in sync: tun0 via pppoe0 src 198.51.100.137 mtu 1472
+Sep 06 13:24:26 vyos he-tunnel-follow[595886]: in sync: tun0 via pppoe0 src 198.51.100.137 mtu 1472
+Sep 06 13:24:26 vyos he-tunnel-follow[595904]: in sync: tun0 via pppoe0 src 198.51.100.137 mtu 1472
+Sep 06 13:25:01 vyos he-tunnel-follow[596311]: in sync: tun0 via pppoe0 src 198.51.100.137 mtu 1472

labsim/ipv6-ha-evidence/mechanism-2026-09-06.txt

@@ -0,0 +1,23 @@
+=== mechanism evidence, labsim, 2026-09-06T14:28:11+01:00 ===
+--- 172.31.1.252 ---
+vip=172.31.1.1 holds_vip=no wan_disabled=yes wan_up=no ppp_up=no ppp_active=no may_dial=no lease_age=- dropin=yes role=backup tun=DOWN radvd=inactive
+inactive
+tun0@NONE        DOWN           203.0.113.108 <POINTOPOINT,NOARP> 
+inactive
+Sep 06 13:22:01 apitest he-tunnel-follow[544377]: role is now backup
+--- 172.31.1.253 ---
+vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes lease_age=26 dropin=yes role=master tun=UNKNOWN radvd=active
+tun0@NONE        UNKNOWN        198.51.100.137 <POINTOPOINT,NOARP,UP,LOWER_UP> 
+active
+Sep 06 13:25:01 vyos he-tunnel-follow[596311]: in sync: tun0 via pppoe0 src 198.51.100.137 mtu 1472
+Sep 06 13:26:01 vyos he-tunnel-follow[598274]: in sync: tun0 via pppoe0 src 198.51.100.137 mtu 1472
+Sep 06 13:27:01 vyos he-tunnel-follow[599393]: in sync: tun0 via pppoe0 src 198.51.100.137 mtu 1472
+Sep 06 13:28:01 vyos he-tunnel-follow[600596]: in sync: tun0 via pppoe0 src 198.51.100.137 mtu 1472
+--- HE endpoint ---
+  HE address : 192.0.2.10/32
+  he-sim     : he-sim: ipv6/ip remote 198.51.100.137 local 192.0.2.10 ttl 64 6rd-prefix 2002::/16
+  he-sim v6  : 2001:db8:1f1c:f6::1/64
+  API        : running
+  API bound  : nohost
+  API calls  : 5
+  route back : 198.51.100.0/24 via 192.168.122.63 dev eth1 

labsim/k8s-up.sh

@@ -38,6 +38,14 @@ K8S_VLAN="${K8S_VLAN:-2}"
 K8S_PREFIX="${K8S_PREFIX:-172.31.2}"
 K8S_NODES="${K8S_NODES:-3}"
 K8S_FIRST_OCTET="${K8S_FIRST_OCTET:-11}"
+# Dual-stack, mirroring production's shape (not its addresses). VLAN 2 v6 is a
+# ULA so nothing here can leak into the real HE /48; node fd00:2::1x matches the
+# BGP peers in sim-net-config.py (K8S_NODES_V6). cluster/service v6 are ULAs too;
+# the LB pool fd61:1e00::/64 is what Cilium advertises (SERVICE_CIDR_V6 there).
+K8S_PREFIX_V6="${K8S_PREFIX_V6:-fd00:2}"        # nodes fd00:2::11/12/13, router ::252/::253
+K8S_ROUTER_V6="${K8S_ROUTER_V6:-fd00:2::252}"   # v6 next-hop (primary router bond0.2)
+CLUSTER_CIDR_V6="${CLUSTER_CIDR_V6:-fd00:42::/56}"
+SERVICE_CIDR_V6="${SERVICE_CIDR_V6:-fd00:43::/112}"
 K8S_MEM="${K8S_MEM:-4096}"          # MB — k3s + cilium + a workload
 K8S_CPUS="${K8S_CPUS:-2}"
 K8S_DISK_GB="${K8S_DISK_GB:-12}"
@@ -53,6 +61,7 @@ CILIUM_VERSION="${CILIUM_VERSION:-1.19.1}"
 
 node_name() { echo "labsim-k8s$1"; }
 node_ip()   { echo "${K8S_PREFIX}.$((K8S_FIRST_OCTET + $1 - 1))"; }
+node_ip6()  { echo "${K8S_PREFIX_V6}::$((K8S_FIRST_OCTET + $1 - 1))"; }
 
 # --- base image -----------------------------------------------------------
 ensure_base_image() {
@@ -75,7 +84,7 @@ ensure_base_image() {
 # before joining, because cloud-init ordering across VMs is not guaranteed and
 # a failed join leaves an agent that never retries.
 build_k8s_seed() {
-  local iso="$1" vm="$2" ip="$3" role="$4" server_ip="$5" pubkey="$6"
+  local iso="$1" vm="$2" ip="$3" role="$4" server_ip="$5" pubkey="$6" ip6="$7" server_ip6="$8"
   local tmp; tmp="$(mktemp -d)"
 
   cat > "$tmp/meta-data" <<EOF
@@ -89,7 +98,7 @@ ethernets:
   enp1s0:
     match:
       name: "en*"
-    addresses: [$ip/24]
+    addresses: [$ip/24, $ip6/64]
     routes:
       - to: default
         via: ${K8S_PREFIX}.1
@@ -100,10 +109,13 @@ EOF
   local k3s_exec
   if [ "$role" = "server" ]; then
     # flannel/servicelb/traefik off: Cilium is the CNI under test, and k3s's
-    # own ServiceLB would fight Cilium for LoadBalancer addresses.
-    k3s_exec="server --flannel-backend=none --disable-network-policy --disable=servicelb --disable=traefik --node-ip=$ip --tls-san=$ip --cluster-init"
+    # own ServiceLB would fight Cilium for LoadBalancer addresses. Dual-stack:
+    # both families in cluster-cidr/service-cidr and a dual node-ip, mirroring
+    # production. Pod IPs come from Cilium cluster-pool (node.spec.podCIDRs is
+    # immutable), so cluster-cidr v6 just marks the cluster dual-stack.
+    k3s_exec="server --flannel-backend=none --disable-network-policy --disable=servicelb --disable=traefik --node-ip=$ip,$ip6 --tls-san=$ip --cluster-cidr=10.42.0.0/16,${CLUSTER_CIDR_V6} --service-cidr=10.43.0.0/16,${SERVICE_CIDR_V6} --cluster-init"
   else
-    k3s_exec="agent --server https://${server_ip}:6443 --node-ip=$ip"
+    k3s_exec="agent --server https://${server_ip}:6443 --node-ip=$ip,$ip6"
   fi
 
   cat > "$tmp/user-data" <<EOF
@@ -170,8 +182,10 @@ create_node() {
   local n="$1" pubkey="$2"
   local vm; vm="$(node_name "$n")"
   local ip; ip="$(node_ip "$n")"
+  local ip6; ip6="$(node_ip6 "$n")"
   local role="agent"; [ "$n" -eq 1 ] && role="server"
   local server_ip; server_ip="$(node_ip 1)"
+  local server_ip6; server_ip6="$(node_ip6 1)"
 
   if virsh_q dominfo "$vm" >/dev/null 2>&1; then
     local state; state="$(virsh_q domstate "$vm" 2>/dev/null | head -1 | tr -d '\n')"
@@ -189,7 +203,7 @@ create_node() {
 
   log "creating $vm ($ip, $role, ${K8S_MEM}MB/${K8S_CPUS}cpu)"
   sudo qemu-img create -q -f qcow2 -F qcow2 -b "$DEB_BASE" "$disk" "${K8S_DISK_GB}G" >/dev/null
-  build_k8s_seed "$seed" "$vm" "$ip" "$role" "$server_ip" "$pubkey"
+  build_k8s_seed "$seed" "$vm" "$ip" "$role" "$server_ip" "$pubkey" "$ip6" "$server_ip6"
 
   # Access port on the k8s VLAN — same broadcast domain as the routers'
   # bond0.2 leg, so BGP peering is directly connected exactly as in production.

labsim/labsim-dualstack-convert.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+# Rehearse the k3s single -> dual-stack conversion on the 3-server etcd cluster,
+# the way production Phase 4 will do it: edit each server's config.yaml (rendered
+# by the PRODUCTION generator) to add the second address family, restart ONE
+# server at a time, and watch what happens in between.
+#
+# Answers the questions a single-node lab cannot:
+#   - does quorum survive a rolling config.yaml change across 3 etcd servers?
+#   - what does a MIXED control plane do (one server dual, two still v4-only)?
+#   - does ServiceCIDR pick up the v6 range on the FIRST server's restart, or
+#     only once all three agree?
+#
+# The node-ip v6 must exist on the box before k3s reads it, so each server first
+# gets a ULA on its interface (fd00:2::3x), mirroring how production nodes get a
+# DHCPv6 address before k3s starts.
+#
+#   ./labsim-dualstack-convert.sh baseline   snapshot the v4-only starting state
+#   ./labsim-dualstack-convert.sh convert    roll the conversion, snapshotting each step
+#   ./labsim-dualstack-convert.sh snapshot   print current SC / podCIDRs / quorum
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+NET="${NET:-172.31.2}"; FIRST_OCTET="${FIRST_OCTET:-31}"; SERVERS="${SERVERS:-3}"
+TOKEN="${TOKEN:-labsim-etcd-token}"
+RENDER="${RENDER:-$SCRIPT_DIR/../bastion/src/modules/dist/modules/k3s/bin/render-config.js}"
+EVID="${EVID:-$SCRIPT_DIR/dualstack-evidence}"
+
+# Dual-stack target ranges. ULA/v4 -- the mechanism is what's under test, not the
+# addresses; using ULA keeps sim traffic out of the real /48.
+V4_CLUSTER="10.42.0.0/16"; V6_CLUSTER="fd00:42::/56"
+V4_SERVICE="10.43.0.0/16"; V6_SERVICE="fd00:43::/112"   # /112: apiserver caps v6 service ranges
+V6_NODE_PREFIX="fd00:2"                                  # node-ip v6: fd00:2::3x
+
+SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=8 -o BatchMode=yes)
+node_ip()   { echo "${NET}.$((FIRST_OCTET + $1 - 1))"; }
+node_v6()   { echo "${V6_NODE_PREFIX}::$((FIRST_OCTET + $1 - 1))"; }
+node_name() { echo "labsim-etcd$1"; }
+s() { local n="$1"; shift; timeout "${TMO:-25}" ssh "${SSH[@]}" "debian@$(node_ip "$n")" "$@" 2>/dev/null; }
+k() { s 1 "sudo k3s kubectl $*"; }
+log() { printf '\033[36m==>\033[0m %s\n' "$*"; }
+
+snapshot() {
+  # custom-columns, not jsonpath: the jsonpath range/quotes get mangled through
+  # ssh -> sudo -> kubectl and came back empty.
+  echo "  servicecidr :"
+  s 1 'sudo k3s kubectl get servicecidr -o custom-columns=NAME:.metadata.name,CIDRS:.spec.cidrs --no-headers' 2>/dev/null | sed 's/^/    /'
+  echo "  node podCIDRs:"
+  s 1 'sudo k3s kubectl get nodes -o custom-columns=NAME:.metadata.name,PODCIDRS:.spec.podCIDRs --no-headers' 2>/dev/null | sed 's/^/    /'
+  local ready; ready="$(k get nodes --no-headers 2>/dev/null | wc -l)"
+  echo "  nodes registered: $ready ; apiserver: $(k get --raw /readyz >/dev/null 2>&1 && echo ok || echo DOWN)"
+}
+
+# Re-render node n's config.yaml WITH the dual families and write it back.
+convert_node() {
+  local n="$1"
+  local ip; ip="$(node_ip "$n")"; local v6; v6="$(node_v6 "$n")"
+  log "server $n ($ip): add $v6, re-render dual config.yaml, restart k3s"
+
+  # 1. node-ip v6 must exist before k3s reads it.
+  s "$n" "sudo ip -6 addr add ${v6}/64 dev enp1s0 2>/dev/null; ip -6 -br addr show enp1s0 | grep -o '${v6}/64'" | sed 's/^/    addr: /'
+
+  # 2. render the dual config from the PRODUCTION generator, preserving this
+  #    node's role (node 1 cluster-init, else joining server).
+  local extra=""
+  [ "$n" -ne 1 ] && extra="K3S_SERVER_URL=https://$(node_ip 1):6443 K3S_TOKEN=$TOKEN"
+  local cfg
+  cfg="$(env ROLE=infra HOSTNAME="$(node_name "$n")" IP="$ip" TLS_SANS="$ip" \
+      IPV6="$v6" CLUSTER_CIDR="${V4_CLUSTER},${V6_CLUSTER}" SERVICE_CIDR="${V4_SERVICE},${V6_SERVICE}" \
+      $extra node "$RENDER")"
+  # sanity: the render must actually carry both families or the restart is pointless
+  echo "$cfg" | grep -q "$V6_CLUSTER" || { echo "    RENDER MISSING v6 -- aborting"; return 1; }
+
+  # 3. write it back and restart k3s on this one server.
+  printf '%s\n' "$cfg" | s "$n" "sudo tee /etc/rancher/k3s/config.yaml >/dev/null && sudo systemctl restart k3s"
+
+  # 4. wait for THIS server's k3s to come back and the apiserver to answer.
+  local i
+  for i in $(seq 1 30); do
+    [ "$(s "$n" 'sudo systemctl is-active k3s' 2>/dev/null)" = active ] && \
+      s "$n" 'sudo k3s kubectl get --raw /readyz >/dev/null 2>&1' && break
+    sleep 10
+  done
+  echo "    server $n k3s=$(s "$n" 'systemctl is-active k3s')  restarts=$(s "$n" 'systemctl show k3s -p NRestarts --value')"
+}
+
+cmd_baseline() {
+  mkdir -p "$EVID"
+  { echo "=== BASELINE (v4-only) $(date -u +%FT%TZ) ==="; snapshot; } | tee "$EVID/convert-baseline.txt"
+}
+
+cmd_snapshot() { snapshot; }
+
+cmd_convert() {
+  mkdir -p "$EVID"
+  local out="$EVID/convert-run.txt"
+  {
+    echo "=== 3-SERVER DUAL-STACK CONVERSION $(date -u +%FT%TZ) ==="
+    echo "--- before ---"; snapshot
+    local n
+    for n in $(seq 1 "$SERVERS"); do
+      echo; echo "### converting server $n of $SERVERS ###"
+      convert_node "$n" || { echo "convert_node $n failed"; break; }
+      echo "--- state after server $n (MIXED until n=$SERVERS) ---"
+      snapshot
+    done
+    echo; echo "--- FINAL ---"; snapshot
+  } 2>&1 | tee "$out"
+  log "evidence -> $out"
+}
+
+case "${1:-snapshot}" in
+  baseline) cmd_baseline ;;
+  convert)  cmd_convert ;;
+  snapshot) cmd_snapshot ;;
+  *) echo "usage: $0 {baseline|convert|snapshot}" >&2; exit 2 ;;
+esac

labsim/labsim-dualstack-net.sh

@@ -0,0 +1,221 @@
+#!/bin/bash
+# VLAN 2 gets IPv6 in labsim: addresses, router advertisements and a DHCPv6
+# server with per-MAC reservations.
+#
+# This is the rehearsal of the production change (dual-stack plan, phase 2b) and
+# runs the SAME VyOS config, against the sim router pair, so the production
+# apply is a repeat rather than a first attempt.
+#
+# WHY DHCPv6 AND NOT SLAAC. The cluster needs each node's IPv6 to be knowable in
+# advance and stable: k3s resolves node-ip once at start-up, and a node's
+# identity cannot be allowed to change under it. The estate already answers that
+# question for IPv4 with kea reservations keyed on MAC, so IPv6 answers it the
+# same way and stays one source of truth.
+#
+# DHCPv6 normally keys on DUID, not MAC -- a DUID is generated by the client and
+# is not derivable from its MAC, which would have meant a second, client-owned
+# source of truth. VyOS's static-mapping accepts `mac` as well as `duid`
+# (verified on the sim: the node.tag directory offers duid, mac, ipv6-address,
+# ipv6-prefix), so the reservation can key on the same MAC the v4 one does.
+#
+# The RA carries managed-flag WITH no-autonomous-flag. That combination is what
+# makes the node's address unambiguous: managed sends it to DHCPv6, and
+# non-autonomous stops it also forming a SLAAC address from the same prefix.
+# Leave autonomous on and every node has two global addresses, only one of which
+# anybody reserved -- and whichever labctl happens to find is the one that ends
+# up in node-ip.
+#
+# Both are VALUELESS nodes on this VyOS: `managed-flag` not `managed-flag true`,
+# and `prefix <p> no-autonomous-flag` not `autonomous-flag false`. The value
+# forms are rejected with "is not valid".
+#
+# A FAILED COMMIT DOES NOT MEAN NOTHING CHANGED. VyOS commits node groups
+# independently, so an earlier run of this script left the bond0.2 address and
+# most of the router-advert block applied while `[[service dhcpv6-server]]
+# failed` -- the interface and RA groups had already succeeded. Re-read the
+# config after any failure rather than assuming a clean rollback; that matters
+# more in production than here.
+#
+# TWO THINGS THIS REHEARSAL FOUND, both of which would have bitten production:
+#
+# 1. managed-flag is NECESSARY BUT NOT SUFFICIENT. It only tells the host to use
+#    DHCPv6; the kernel's accept_ra implements SLAAC and nothing else, so a node
+#    with no DHCPv6 *client* running takes no lease at all. The sim's Debian
+#    nodes have neither NetworkManager nor a networkd .network file managing the
+#    interface, so they ended up with a kernel SLAAC address
+#    (2001:db8:187e:2:5054:ff:fe53:731 -- EUI-64 from the MAC) and never asked
+#    for the reserved ::11. Production's Fedora nodes use NetworkManager, which
+#    does run a DHCPv6 client on the managed flag, and the DGX Sparks run
+#    NetworkManager too -- but that must be VERIFIED per node, not assumed. The
+#    k3s module's preflight catches the consequence; the fix is node-side.
+#
+# 2. TURNING AUTONOMOUS OFF DOES NOT RETRACT ADDRESSES ALREADY FORMED. An
+#    earlier partial apply advertised the prefix while autonomous was still on;
+#    the nodes autoconfigured, and adding no-autonomous-flag afterwards left
+#    those addresses in place with a 30-day valid lifetime. So in production,
+#    where VLAN 2 has no IPv6 at all yet, no-autonomous-flag MUST be in the same
+#    commit that first advertises the prefix. Advertise first and tighten later
+#    and every node carries an unreserved EUI-64 address that labctl might pick
+#    up as node-ip.
+#
+#   ./labsim-dualstack-net.sh up       apply to both sim routers
+#   ./labsim-dualstack-net.sh down     remove it again
+#   ./labsim-dualstack-net.sh status   what the routers and nodes think
+#   ./labsim-dualstack-net.sh leases   who has taken an address
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+R1="${R1:-172.31.1.252}"   # sim vyos001 -> ::1
+R2="${R2:-172.31.1.253}"   # sim vyos002 -> ::2
+PW="${VYOS_PW:-vyos}"
+
+# Mirrors production's scheme (2001:470:187e:<vlan>::/64) using the
+# documentation prefix, so the shape is rehearsed without putting sim addresses
+# inside a range production announces.
+V6_PREFIX="${V6_PREFIX:-2001:db8:187e:2}"
+LINK_MTU="${LINK_MTU:-1472}"
+# VyOS requires a unique subnet-id per DHCPv6 subnet ("Unique subnet ID not
+# specified for subnet"). Using the VLAN id keeps it self-documenting and
+# collision-free across VLANs.
+VLAN_ID="${VLAN_ID:-2}"
+
+# node -> last hextet. Mirrors the v4 host part (172.31.2.11 -> ::11) so a
+# reservation is readable next to its IPv4 twin.
+NODES=(labsim-k8s1:11 labsim-k8s2:12 labsim-k8s3:13)
+
+SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+     -o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password)
+
+log()  { printf '\033[0;36m[ds-net]\033[0m %s\n' "$*"; }
+warn() { printf '\033[1;33m[ds-net]\033[0m %s\n' "$*" >&2; }
+die()  { printf '\033[0;31m[ds-net]\033[0m %s\n' "$*" >&2; exit 1; }
+
+# Drive VyOS from a script FILE, never `vbash -c`. The latter never starts a
+# config session: commit fails to stderr, a helper discards it, and the run
+# reports success having changed nothing. labsim-pppoe-ha-test.sh lost a whole
+# policy matrix to exactly that.
+vyos_apply() {  # host, set-lines on stdin
+    local h="$1" out
+    out="$({ printf '#!/bin/vbash\nsource /opt/vyatta/etc/functions/script-template\nconfigure\n'
+             cat
+             printf 'commit\nsave\nexit\n'
+           } | timeout 120 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$h" \
+                 'cat > /tmp/ds-net.sh && chmod +x /tmp/ds-net.sh && sudo /tmp/ds-net.sh' 2>&1)"
+    # Read the outcome instead of assuming it. The first version of this script
+    # printed "up" after BOTH routers had failed to commit -- the same shape of
+    # lie this repo has been bitten by before (a matrix reporting coverage it did
+    # not have). vbash exits 0 even when the commit fails, so the text is the
+    # only honest signal.
+    printf '%s\n' "$out" | grep -vE '^\s*$' | sed 's/^/      /' | tail -6
+    if printf '%s' "$out" | grep -qiE 'Commit failed|\[\[.*\]\] failed|Set failed'; then
+        return 1
+    fi
+    return 0
+}
+
+r() { timeout 40 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; }
+
+mac_of() { sudo virsh domiflist "$1" 2>/dev/null | awk '/52:54/{print $5; exit}'; }
+
+up() {
+    log "collecting node MACs (the reservation key, same as IPv4)"
+    local mappings="" name hextet mac
+    for entry in "${NODES[@]}"; do
+        name="${entry%%:*}"; hextet="${entry##*:}"
+        mac="$(mac_of "$name")"
+        [ -n "$mac" ] || die "no MAC for $name -- is the sim cluster up? (./k8s-up.sh)"
+        log "    $name $mac -> ${V6_PREFIX}::${hextet}"
+        # No trailing newline: `${mappings}` sits on its own line in the heredoc
+        # below and supplies its own. A heredoc terminator is matched LITERALLY
+        # in the source before any expansion, so `${mappings}EOF` is not the
+        # delimiter -- the heredoc ran on, swallowed the rest of this function
+        # and part of down(), and the apply failed with "Invalid command: [EOF]".
+        [ -n "$mappings" ] && mappings+=$'\n'
+        mappings+="set service dhcpv6-server shared-network-name K8S subnet ${V6_PREFIX}::/64 static-mapping ${name} mac '${mac}'
+set service dhcpv6-server shared-network-name K8S subnet ${V6_PREFIX}::/64 static-mapping ${name} ipv6-address '${V6_PREFIX}::${hextet}'"
+    done
+
+    local host pref hextet_self
+    for host in "$R1" "$R2"; do
+        if [ "$host" = "$R1" ]; then pref=high; hextet_self=1; else pref=low; hextet_self=2; fi
+        log "applying to $host (router ${V6_PREFIX}::${hextet_self}, RA preference $pref)"
+        vyos_apply "$host" <<EOF || die "commit failed on $host -- nothing applied there"
+set interfaces bonding bond0 vif 2 address '${V6_PREFIX}::${hextet_self}/64'
+set service router-advert interface bond0.2 prefix ${V6_PREFIX}::/64 valid-lifetime '2592000'
+set service router-advert interface bond0.2 prefix ${V6_PREFIX}::/64 preferred-lifetime '604800'
+set service router-advert interface bond0.2 prefix ${V6_PREFIX}::/64 no-autonomous-flag
+set service router-advert interface bond0.2 managed-flag
+set service router-advert interface bond0.2 link-mtu '${LINK_MTU}'
+set service router-advert interface bond0.2 default-preference '${pref}'
+set service dhcpv6-server shared-network-name K8S subnet ${V6_PREFIX}::/64 subnet-id '${VLAN_ID}'
+${mappings}
+EOF
+    done
+    log "up. Nodes need a DHCPv6 client on their VLAN 2 interface -- see 'status'."
+}
+
+down() {
+    local host
+    for host in "$R1" "$R2"; do
+        log "removing from $host"
+        vyos_apply "$host" <<EOF
+delete service dhcpv6-server
+delete service router-advert interface bond0.2
+delete interfaces bonding bond0 vif 2 address '${V6_PREFIX}::$([ "$host" = "$R1" ] && echo 1 || echo 2)/64'
+EOF
+    done
+    log "down"
+}
+
+status() {
+    local host
+    for host in "$R1" "$R2"; do
+        printf '  --- %s ---\n' "$host"
+        printf '    bond0.2 v6 : %s\n' "$(r "$host" 'ip -6 -br addr show bond0.2 | tr -s " "' || echo '<unreachable>')"
+        printf '    radvd      : %s\n' "$(r "$host" 'systemctl is-active radvd')"
+        # ps|grep, not pgrep: the bracket idiom that stops a self-match gets
+        # mangled through ssh -> vbash quoting and reported "NOT running" for a
+        # daemon that was plainly up.
+        printf '    kea-dhcp6  : %s\n' "$(r "$host" 'c=$(ps -ef | grep -c "[k]ea-dhcp6"); [ "$c" -gt 0 ] && echo running || echo "NOT running"')"
+        # radvd is expected INACTIVE on the backup: vrrp-wan-reconcile's IPv6
+        # plane stops it there so only the master advertises on any VLAN. Not a
+        # fault -- see the step-0 IPv6-follows-master work.
+        printf '    role       : %s\n' "$(r "$host" 'sudo /config/vrrp-wan-reconcile --status 2>/dev/null | grep -o "role=[a-z]*"')"
+    done
+    printf '  --- nodes ---\n'
+    local entry name
+    for entry in "${NODES[@]}"; do
+        name="${entry%%:*}"
+        printf '    %-14s %s\n' "$name" "$(node_v6 "$name")"
+    done
+}
+
+# What global IPv6 does the node actually hold? This is the question labctl asks
+# before it will write node-ip, so ask it the same way.
+node_v6() {
+    # Ask the node, not the hypervisor: virsh domifaddr --source agent needs
+    # qemu-guest-agent, which these images do not carry, and returned nothing
+    # while the nodes plainly had addresses.
+    local name="$1" v4 out
+    case "$name" in *k8s1) v4=172.31.2.11 ;; *k8s2) v4=172.31.2.12 ;; *k8s3) v4=172.31.2.13 ;; *) echo "<unknown>"; return ;; esac
+    out="$(timeout 20 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+             -o LogLevel=ERROR -o ConnectTimeout=6 -o BatchMode=yes "debian@$v4" \
+             'ip -6 -br addr show scope global 2>/dev/null | tr -s " "' 2>/dev/null)"
+    echo "${out:-<unreachable>}"
+}
+
+leases() {
+    local host
+    for host in "$R1" "$R2"; do
+        printf '  --- %s ---\n' "$host"
+        r "$host" '/opt/vyatta/bin/vyatta-op-cmd-wrapper show dhcpv6-server leases 2>/dev/null || echo "    (no leases command / no leases)"'
+    done
+}
+
+case "${1:-status}" in
+    up)     up ;;
+    down)   down ;;
+    status) status ;;
+    leases) leases ;;
+    *) die "usage: $0 {up|down|status|leases}" ;;
+esac

labsim/labsim-he-endpoint.sh

@@ -0,0 +1,280 @@
+#!/bin/bash
+# Build a fake Hurricane Electric 6in4 endpoint inside labsim.
+#
+# WHY THIS EXISTS
+# The production HE tunnel was never rehearsed. The override that introduced it
+# says so in its own reason text -- "the sim has no public IPv4 and no HE
+# endpoint, so there is nothing to tunnel to" -- and that gap is why IPv6 was
+# the one half of the WAN story with no matrix behind it. Then the WAN became
+# HA and IPv6 did not follow, which nobody caught, because nothing tests it.
+#
+# THE PROBLEM THIS HAD TO SOLVE
+# The sim's two WANs are isolated islands. Verified:
+#
+#     203.0.113.1 from 203.0.113.107 : OK          <- 10 gig analogue
+#     203.0.113.1 from 198.51.100.137 : unreachable
+#     198.51.100.1 from 198.51.100.137 : OK        <- PPPoE analogue
+#     198.51.100.1 from 203.0.113.107 : unreachable
+#
+# A 6in4 tunnel has ONE remote address, and production never changes it -- so an
+# endpoint reachable over only one WAN could not rehearse the case that matters
+# most: the WITHIN-box fall back from the 10 gig to PPPoE, where he-tunnel-follow
+# re-points the tunnel and calls the HE API. That is exactly where the 2026-09-06
+# near-miss lived.
+#
+# So the sim needs a minimal "internet": both ISP boxes already sit on the
+# libvirt default network (192.168.122.0/24) and both forward, so that becomes
+# the backbone, and HE lives on a single address behind it, reachable over
+# either WAN. No new VMs, no new networks.
+#
+#     192.0.2.10  "HE"  -- on isp-dhcp, reached from the PPPoE island via
+#                          192.168.122.136, and directly from the 10 gig island
+#
+# EVERYTHING HERE IS KERNEL-LEVEL, not VyOS config. The ISP boxes are scaffold,
+# not the thing under test: `ip` commands leave no config to drift, no commit to
+# fail, and a reboot cleans up. The ROUTER side is deliberately the opposite --
+# it goes through real VyOS config, because "will VyOS commit a tunnel whose
+# source-address does not exist on this box?" is one of the questions.
+#
+#   ./labsim-he-endpoint.sh up        build it
+#   ./labsim-he-endpoint.sh down      tear it down
+#   ./labsim-he-endpoint.sh status    what is live
+#   ./labsim-he-endpoint.sh calls     how many times the HE API was called
+#   ./labsim-he-endpoint.sh point IP  point the endpoint by hand (sim bookkeeping)
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+DHCP_ISP="${DHCP_ISP:-192.168.122.136}"   # owns the 10 gig segment, hosts "HE"
+PPPOE_ISP="${PPPOE_ISP:-192.168.122.63}"  # owns the PPPoE segment
+PW="${VYOS_PW:-vyos}"
+
+# TEST-NET-1 for the endpoint, and the documentation prefix for v6. Production
+# uses 2001:470:187e::/48 from HE; the sim mirrors its SHAPE
+# (2001:db8:187e:<vlan>::/64) so the scheme is exercised, not just the tunnel.
+HE_ADDR="${HE_ADDR:-192.0.2.10}"
+HE_LINK6="${HE_LINK6:-2001:db8:1f1c:f6::1}"   # HE side of the tunnel /64
+RT_LINK6="${RT_LINK6:-2001:db8:1f1c:f6::2}"   # router side
+SITE6="${SITE6:-2001:db8:187e::/48}"          # routed to the router side
+TENGIG_NET="${TENGIG_NET:-203.0.113.0/24}"
+# Any valid address that will never be a router WAN -- see the tunnel creation
+# below for why this cannot be 0.0.0.0.
+PLACEHOLDER_REMOTE="${PLACEHOLDER_REMOTE:-203.0.113.1}"
+PPPOE_NET="${PPPOE_NET:-198.51.100.0/24}"
+
+SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+     -o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password)
+dhcp_isp()  { timeout 40 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$DHCP_ISP" "$@" 2>/dev/null; }
+pppoe_isp() { timeout 40 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$PPPOE_ISP" "$@" 2>/dev/null; }
+
+log()  { printf '\033[0;36m[he-sim]\033[0m %s\n' "$*"; }
+die()  { printf '\033[0;31m[he-sim]\033[0m %s\n' "$*" >&2; exit 1; }
+
+# --- the stub tunnelbroker API ---------------------------------------------
+# HE's real endpoint is a dyndns-style updater that re-points the tunnel's remote
+# address. This is that, in 40 lines, so the sim can exercise the HE-SIDE half of
+# a failover -- the half production can never safely test.
+#
+# It logs every call to /run/he-sim-api.log, which is what lets the matrix assert
+# the invariant that matters: a ROUTER-level failover must call this ZERO times,
+# because the 10 gig address follows the cloned MAC to the other box unchanged.
+API_PY='
+import http.server, subprocess, urllib.parse, datetime, sys, re
+
+TUN  = "he-sim"
+LOCAL    = sys.argv[1] if len(sys.argv) > 1 else "192.0.2.10"
+V6_LOCAL = sys.argv[2] if len(sys.argv) > 2 else "2001:db8:1f1c:f6::1/64"
+V6_PEER  = sys.argv[3] if len(sys.argv) > 3 else "2001:db8:1f1c:f6::2"
+SITE6    = sys.argv[4] if len(sys.argv) > 4 else "2001:db8:187e::/48"
+LOG  = "/run/he-sim-api.log"
+
+def note(msg):
+    with open(LOG, "a") as f:
+        f.write("%s %s\n" % (datetime.datetime.now().isoformat(timespec="seconds"), msg))
+
+def current_remote():
+    out = subprocess.run(["ip", "tunnel", "show", TUN], capture_output=True, text=True).stdout
+    m = re.search(r"remote ([0-9.]+)", out)
+    return m.group(1) if m else None
+
+class H(http.server.BaseHTTPRequestHandler):
+    def reply(self, body):
+        b = body.encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain")
+        self.send_header("Content-Length", str(len(b)))
+        self.end_headers()
+        self.wfile.write(b)
+
+    def handle_update(self, qs):
+        q = urllib.parse.parse_qs(qs)
+        ip = (q.get("myip") or [""])[0]
+        if not ip:
+            note("REFUSED no myip"); return self.reply("nohost")
+        cur = current_remote()
+        if cur == ip:
+            note("nochg %s" % ip); return self.reply("nochg %s" % ip)
+        rc = subprocess.run(["ip", "tunnel", "change", TUN, "mode", "sit",
+                             "local", LOCAL, "remote", ip],
+                            capture_output=True, text=True)
+        if rc.returncode != 0:
+            # Recreate rather than report a success we did not achieve. This is
+            # the path that a multipoint tunnel takes; keeping it means a stub
+            # that cannot silently no-op.
+            note("change failed (%s) -- recreating" % rc.stderr.strip())
+            subprocess.run(["ip", "tunnel", "del", TUN], check=False)
+            subprocess.run(["ip", "tunnel", "add", TUN, "mode", "sit",
+                            "local", LOCAL, "remote", ip, "ttl", "64"], check=False)
+            subprocess.run(["ip", "link", "set", TUN, "up", "mtu", "1480"], check=False)
+            subprocess.run(["ip", "-6", "addr", "replace", V6_LOCAL, "dev", TUN], check=False)
+            subprocess.run(["ip", "-6", "route", "replace", SITE6, "via", V6_PEER,
+                            "dev", TUN], check=False)
+        # Verify rather than trust: read the remote back.
+        got = current_remote()
+        if got != ip:
+            note("FAILED to point %s at %s (reads %s)" % (TUN, ip, got))
+            return self.reply("dnserr")
+        note("good %s (was %s)" % (ip, cur))
+        self.reply("good %s" % ip)
+
+    def do_GET(self):
+        u = urllib.parse.urlparse(self.path)
+        if u.path == "/nic/update": self.handle_update(u.query)
+        else: self.reply("badauth")
+
+    def do_POST(self):
+        n = int(self.headers.get("Content-Length") or 0)
+        self.handle_update(self.rfile.read(n).decode())
+
+    def log_message(self, *a): pass
+
+http.server.HTTPServer(("0.0.0.0", 80), H).serve_forever()
+'
+
+up() {
+    log "backbone: teaching each ISP box how to reach the other island"
+    # isp-dhcp owns HE and must be able to answer a router that arrived over
+    # PPPoE, so it needs a route back to that island via the backbone.
+    dhcp_isp "sudo ip route replace $PPPOE_NET via $PPPOE_ISP" \
+        || die "could not add the PPPoE-island route on isp-dhcp"
+    # isp-pppoe must forward its clients' traffic for HE across the backbone.
+    pppoe_isp "sudo ip route replace $HE_ADDR/32 via $DHCP_ISP" \
+        || die "could not add the HE route on isp-pppoe"
+
+    log "HE endpoint: $HE_ADDR on isp-dhcp"
+    # A dummy interface, not a loopback alias: `ip tunnel` wants a real local
+    # address and a dummy is the honest way to have one that is not tied to
+    # either WAN segment -- which is the point, HE is neither.
+    dhcp_isp "sudo modprobe dummy 2>/dev/null;
+              sudo ip link add he-lo type dummy 2>/dev/null;
+              sudo ip link set he-lo up;
+              sudo ip addr replace $HE_ADDR/32 dev he-lo"
+
+    log "6in4 tunnel he-sim: local $HE_ADDR, remote set by the API on demand"
+    # A PLACEHOLDER remote, not 0.0.0.0. A sit tunnel created with `remote any`
+    # is multipoint (6rd-shaped), and `ip tunnel change` then refuses to convert
+    # it to point-to-point -- "add tunnel he-sim failed: Invalid argument". The
+    # API's update silently did nothing, so the stub logged "good", production's
+    # he-tunnel-follow logged success, and the tunnel still pointed nowhere.
+    # Created point-to-point from the start, `change` works.
+    dhcp_isp "sudo ip tunnel del he-sim 2>/dev/null;
+              sudo ip tunnel add he-sim mode sit local $HE_ADDR remote $PLACEHOLDER_REMOTE ttl 64;
+              sudo ip link set he-sim up mtu 1480;
+              sudo ip -6 addr replace $HE_LINK6/64 dev he-sim;
+              sudo ip -6 route replace $SITE6 via $RT_LINK6 dev he-sim;
+              sudo sysctl -qw net.ipv6.conf.all.forwarding=1"
+
+    log "stub tunnelbroker API on $HE_ADDR:80"
+    printf '%s' "$API_PY" | dhcp_isp "cat > /tmp/he-sim-api.py"
+    # Launch from a script FILE, not an inline ssh command. The remote login
+    # shell is vbash, and a multi-line inlined `sudo setsid nohup ... &` through
+    # it silently ran nothing at all: no process, no /run/he-sim-api.out, and a
+    # `pgrep -f he-sim-api.py` status check that reported "running" because the
+    # unbracketed pattern matched its OWN ssh command line. Two self-inflicted
+    # illusions stacked on each other.
+    #
+    # This repo already learned this once -- see isp_session_control() in
+    # labsim-pppoe-ha-test.sh, where driving vbash inline made every iteration
+    # of the T4 matrix test the wrong policy while printing the right one.
+    printf '%s\n' \
+        '#!/bin/sh' \
+        '# started detached so it outlives the ssh session that launched it' \
+        'pkill -f "he-sim-api[.]py" 2>/dev/null' \
+        'rm -f /run/he-sim-api.log /run/he-sim-api.out' \
+        "exec setsid python3 /tmp/he-sim-api.py $HE_ADDR '$HE_LINK6/64' $RT_LINK6 $SITE6 >/run/he-sim-api.out 2>&1 </dev/null &" \
+        | dhcp_isp "cat > /tmp/he-sim-start.sh"
+    dhcp_isp "chmod +x /tmp/he-sim-start.sh && sudo /tmp/he-sim-start.sh" >/dev/null
+    # Poll for the bind rather than sleeping a guessed interval.
+    local i probe=""
+    for i in $(seq 1 10); do
+        sleep 1
+        probe="$(dhcp_isp "curl -sS --max-time 3 'http://$HE_ADDR/nic/update' 2>&1")"
+        [ "$probe" = nohost ] && break
+    done
+    case "$probe" in
+        nohost) log "API answering (returned 'nohost' for a call with no myip -- correct)" ;;
+        *)      die "stub API not answering on $HE_ADDR:80 (got: ${probe:-<nothing>})" ;;
+    esac
+
+    log "up. Router side is NOT configured by this script -- that is real VyOS"
+    log "config and belongs to the matrix; see labsim-ipv6-ha-test.sh --setup."
+}
+
+down() {
+    log "tearing down"
+    dhcp_isp "sudo pkill -f 'he-sim-api[.]py' 2>/dev/null;
+              sudo ip tunnel del he-sim 2>/dev/null;
+              sudo ip link del he-lo 2>/dev/null;
+              sudo ip route del $PPPOE_NET via $PPPOE_ISP 2>/dev/null" >/dev/null
+    pppoe_isp "sudo ip route del $HE_ADDR/32 via $DHCP_ISP 2>/dev/null" >/dev/null
+    log "down"
+}
+
+status() {
+    printf '  HE address : %s\n' "$(dhcp_isp "ip -4 -br addr show he-lo 2>/dev/null | awk '{print \$3}'" || echo '<absent>')"
+    printf '  he-sim     : %s\n' "$(dhcp_isp "ip tunnel show he-sim 2>/dev/null" || echo '<absent>')"
+    printf '  he-sim v6  : %s\n' "$(dhcp_isp "ip -6 -br addr show he-sim 2>/dev/null | awk '{print \$3}'" || echo '-')"
+    # Bracketed so the pattern cannot match the ssh command line carrying it --
+    # unbracketed, this reported "running" while nothing was listening at all.
+    printf '  API        : %s\n' "$(dhcp_isp "pgrep -f 'he-sim-api[.]py' >/dev/null && echo running || echo stopped")"
+    printf '  API bound  : %s\n' "$(dhcp_isp "curl -sS --max-time 3 'http://$HE_ADDR/nic/update' 2>/dev/null" || echo 'NOT ANSWERING')"
+    printf '  API calls  : %s\n' "$(dhcp_isp "grep -c . /run/he-sim-api.log 2>/dev/null" || echo 0)"
+    printf '  route back : %s\n' "$(dhcp_isp "ip route show $PPPOE_NET 2>/dev/null" || echo '<none>')"
+}
+
+# Count of endpoint-CHANGING calls. `nochg` does not count: production's
+# he-tunnel-follow re-sends the same address happily and HE treats it as a
+# no-op, so only a real move is evidence that something re-pointed the tunnel.
+calls() { dhcp_isp "grep -c ' good ' /run/he-sim-api.log 2>/dev/null" | tr -d ' \n'; }
+
+# Point the endpoint at an address WITHOUT going through the API, and without
+# counting as an API call.
+#
+# Needed because rebuilding the sim endpoint resets its remote, while the
+# routers' he-tunnel-follow still reads "in sync" and therefore never re-asserts
+# -- it only calls HE when its own LOCAL source changes, and has no way to learn
+# that the far end drifted. The real HE does not forget, so this is sim
+# bookkeeping, not a behaviour production needs. Keeping it out of the call
+# counter is the point: the matrix asserts on that counter.
+point() {
+    local ip="$1"
+    [ -n "$ip" ] || die "usage: $0 point <ipv4>"
+    dhcp_isp "sudo ip tunnel change he-sim mode sit local $HE_ADDR remote $ip 2>/dev/null \
+              || { sudo ip tunnel del he-sim 2>/dev/null;
+                   sudo ip tunnel add he-sim mode sit local $HE_ADDR remote $ip ttl 64;
+                   sudo ip link set he-sim up mtu 1480;
+                   sudo ip -6 addr replace $HE_LINK6/64 dev he-sim;
+                   sudo ip -6 route replace $SITE6 via $RT_LINK6 dev he-sim; }"
+    local got
+    got="$(dhcp_isp "ip tunnel show he-sim 2>/dev/null | sed -nE 's/.* remote ([0-9.]+).*/\\1/p'" | tr -d ' \n')"
+    [ "$got" = "$ip" ] || die "endpoint still points at ${got:-nothing}, wanted $ip"
+    log "endpoint now points at $ip"
+}
+
+case "${1:-status}" in
+    up)     up ;;
+    down)   down ;;
+    status) status ;;
+    calls)  calls; echo ;;
+    point)  point "${2:-}" ;;
+    *) die "usage: $0 {up|down|status|calls}" ;;
+esac

labsim/labsim-ipv6-ha-test.sh

@@ -0,0 +1,430 @@
+#!/bin/bash
+# Does IPv6 follow VRRP mastership, and does it do so WITHOUT touching HE?
+#
+# The WAN became HA on 2026-09-06 and IPv6 did not follow it. Nothing caught
+# that, because nothing tested it: wan-drill measured IPv4 only, and PPPOE-HA.md
+# recorded "IPv6 stayed up at 15.5ms" from a reading taken outside the failover
+# window. This is the matrix that would have caught it.
+#
+# TWO INVARIANTS, checked independently of any individual test:
+#
+#   1. At most ONE router ever has a live tunnel. An UP tunnel on a box that
+#      does not own the source address is not harmless -- see V4, it is a
+#      BLACKHOLE that will happily attract the v6 default route.
+#   2. A ROUTER-level failover calls the HE API ZERO times. The 10 gig address
+#      is bound to a cloned MAC and follows the VIP to the other box unchanged,
+#      so there is nothing to tell HE. A non-zero count means something
+#      re-pointed the tunnel at a PPPoE address -- which the ISP re-issues on
+#      every dial, so it would be wrong within minutes.
+#
+# KNOWN SIM GAP, 2026-09-06 -- read this before believing a v6_online failure.
+# The MECHANISM is proven here: tun0 up on the master and down on the backup,
+# radvd following mastership, he-tunnel-follow's master guard, its hysteresis,
+# its HE call and the 1480->1472 MTU switch, and VLAN 9 hosts autoconfiguring
+# from the RA (observed: real SLAAC traffic from 2001:db8:187e:9::/64 arriving
+# at the endpoint encapsulated).
+#
+# What is NOT yet proven is the end-to-end v6 DATAPATH, because the sim's
+# "internet" is asymmetric: 6in4 packets from the PPPoE island reach the
+# endpoint with an outer source of 192.168.122.1 -- the libvirt host's NAT --
+# so HE's replies go back to the tunnel remote by a path with no NAT state and
+# are lost. Forward works, return does not.
+#
+# That is a topology fault in the scaffold, not in the thing under test. Fixing
+# it means giving the two ISP islands a real transit path that does not traverse
+# libvirt NAT. Until then, treat v6_online failures as UNPROVEN rather than as
+# evidence the design is wrong -- and do not let that ambiguity leak into
+# production sign-off, which is exactly the mistake wan-drill made by asserting
+# "IPv6 stayed up" from a reading taken outside the window.
+#
+# Requires the fake HE endpoint: ./labsim-he-endpoint.sh up
+#
+#   ./labsim-ipv6-ha-test.sh --setup    configure the router side (once)
+#   ./labsim-ipv6-ha-test.sh --list
+#   ./labsim-ipv6-ha-test.sh V1
+#   ./labsim-ipv6-ha-test.sh --all
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+R1="${R1:-172.31.1.252}"; R2="${R2:-172.31.1.253}"
+VIP="${VIP:-172.31.1.1}"
+LAN9="${LAN9:-172.31.9.10}"                  # VLAN 9 client, for RA tests
+HE_ADDR="${HE_ADDR:-192.0.2.10}"
+HE_LINK6="${HE_LINK6:-2001:db8:1f1c:f6::1}"
+RT_LINK6="${RT_LINK6:-2001:db8:1f1c:f6::2}"
+V9_PREFIX="${V9_PREFIX:-2001:db8:187e:9}"
+# Mirrors production: one MAC, one lease, whichever router holds the VIP.
+WAN_MAC="${WAN_MAC:-02:9f:c2:12:9b:4f}"
+PW="${VYOS_PW:-vyos}"; LANPW="${LANPW:-labsim}"
+EVID="$SCRIPT_DIR/ipv6-ha-evidence"
+HE="$SCRIPT_DIR/labsim-he-endpoint.sh"
+
+SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+     -o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password)
+r() { timeout 45 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; }
+# The Alpine LAN VMs do not offer `password` auth -- reusing the routers' option
+# set makes ssh exit 255 before running anything, which reads as "the network is
+# broken". Same trap as lan() in labsim-pppoe-ha-test.sh.
+LAN_SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+         -o LogLevel=ERROR -o ConnectTimeout=6)
+lan9() { timeout 30 sshpass -p "$LANPW" ssh "${LAN_SSH[@]}" "root@$LAN9" "$@" 2>/dev/null; }
+
+log()  { printf '\033[36m==>\033[0m %s\n' "$*"; }
+pass() { printf '  \033[32mPASS\033[0m %s\n' "$*"; }
+fail() { printf '  \033[31mFAIL\033[0m %s\n' "$*"; FAILED=$((FAILED+1)); }
+warn() { printf '  \033[33mWARN\033[0m %s\n' "$*"; }
+FAILED=0
+
+# --- observations ----------------------------------------------------------
+# A destroyed or unreachable router is emphatically NOT holding the tunnel, but
+# ssh returns an EMPTY string, and `[ "" = 0 ]` is false -- the pppoe matrix hung
+# on exactly this waiting for a dead box to report zero. Default everything to 0.
+tun_state() { local v; v="$(r "$1" "ip -br link show tun0 2>/dev/null | awk '{print \$2}'" | tr -d ' \n')"; echo "${v:-absent}"; }
+tun_src()   { r "$1" 'ip tunnel show tun0 2>/dev/null | sed -nE "s/.* local ([0-9.]+).*/\1/p"' | tr -d ' \n'; }
+# Whichever router currently holds the 10 gig lease -- under the cloned MAC only
+# one ever does. Read rather than assumed: the address changes when the MAC does.
+tengig_addr() { local h a; for h in "$R1" "$R2"; do
+                  a="$(r "$h" 'ip -4 addr show bond0.53 2>/dev/null | sed -nE "s/.*inet ([0-9.]+).*/\1/p"' | tr -d ' \n')"
+                  [ -n "$a" ] && { echo "$a"; return; }
+                done; echo ""; }
+tun_mtu()   { local v; v="$(r "$1" 'cat /sys/class/net/tun0/mtu 2>/dev/null' | tr -d ' \n')"; echo "${v:-0}"; }
+holder()    { for h in "$R1" "$R2"; do
+                [ "$(r "$h" "ip -4 -o addr show | grep -c ' ${VIP}/'" | tr -d ' \n')" != 0 ] \
+                  && { echo "$h"; return; }; done; echo none; }
+# Ask the ROUTER, not a client: a client can be answered by the wrong path.
+v6_online() { [ "$(r "$1" "ping -6 -c1 -W3 $HE_LINK6 >/dev/null 2>&1 && echo y" | tr -d ' \n')" = y ]; }
+radvd_on()  { [ "$(r "$1" 'systemctl is-active radvd 2>/dev/null' | tr -d ' \n')" = active ]; }
+he_calls()  { timeout 60 "$HE" calls | tr -d ' \n'; }
+
+# How many routers have a tunnel that is UP. Invariant 1's numerator.
+tun_holders() { local n=0 h; for h in "$R1" "$R2"; do
+                  case "$(tun_state "$h")" in UP|UNKNOWN) n=$((n+1)) ;; esac
+                done; echo "$n"; }
+
+check_invariants() {
+    local t ok=0
+    t="$(tun_holders)"
+    [ "${t:-0}" -le 1 ] || { fail "INVARIANT: $t routers have tun0 up"; ok=1; }
+    return $ok
+}
+
+save_evidence() {
+    local name="$1"; local d="$EVID/$name"; mkdir -p "$d"
+    { echo "=== $(date -Is) ==="
+      echo "--- HE endpoint ---"; timeout 60 "$HE" status
+      for h in "$R1" "$R2"; do echo "--- $h ---"
+        r "$h" 'sudo /config/vrrp-wan-reconcile --status 2>/dev/null
+                ip -br link show tun0 2>/dev/null; ip tunnel show tun0 2>/dev/null
+                ip -6 route show default; systemctl is-active radvd
+                sudo journalctl -t vrrp-wan -t he-tunnel-follow -n 10 --no-pager'
+      done; } > "$d/state.txt" 2>&1
+    log "evidence -> ipv6-ha-evidence/$name/"
+}
+
+# --- setup ------------------------------------------------------------------
+# Applied through REAL VyOS config, unlike the ISP-side scaffold, because "will
+# VyOS accept this?" is one of the questions being asked.
+vyos_apply() {   # host, then set-lines on stdin
+    local h="$1"
+    { printf '#!/bin/vbash\nsource /opt/vyatta/etc/functions/script-template\nconfigure\n'
+      cat
+      printf 'commit\nsave\nexit\n'
+    } | timeout 90 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$h" \
+          'cat > /tmp/v6-apply.sh && chmod +x /tmp/v6-apply.sh && sudo /tmp/v6-apply.sh' 2>&1 | tail -3
+    # `vbash -c` never starts a config session and commit fails to stderr, which
+    # a helper like this discards -- the T4 matrix in labsim-pppoe-ha-test.sh ran
+    # its whole policy sweep against the default while printing the mode it
+    # thought it was testing. Always read the value back.
+}
+
+setup() {
+    log "--setup: router-side IPv6, both routers"
+
+    # THE CLONED MAC, first and on its own. Production pins f0:9f:c2:12:9b:4f on
+    # vif 53 so the 10 gig lease follows the VIP and the tunnel source is the
+    # SAME address on either box. The sim never had it -- each router took its
+    # own lease -- so the sim could not reproduce the one property the whole
+    # IPv6-HA design leans on, and invariant 2 would have been untestable here.
+    local h pref v9
+    for h in "$R1" "$R2"; do
+        log "    $h: pinning the cloned WAN MAC $WAN_MAC"
+        vyos_apply "$h" <<EOF
+set interfaces bonding bond0 vif 53 mac '$WAN_MAC'
+EOF
+    done
+
+    # A new MAC means a NEW lease, so the tunnel source cannot be hardcoded --
+    # discover it. Hardcoding the pre-change address here would have configured
+    # every tunnel with a source neither router owns, i.e. the V4 blackhole, on
+    # both boxes, while the matrix reported setup success.
+    local src="" i
+    for i in $(seq 1 30); do
+        src="$(tengig_addr)"; [ -n "$src" ] && break
+        sleep 5
+    done
+    [ -n "$src" ] || { fail "no router took a 10 gig lease after the MAC change -- cannot set a tunnel source"; return 1; }
+    log "    10 gig lease under the cloned MAC: $src"
+
+    # Credentials pointing at the stub. HE_UPDATE_URL is read AFTER the secrets
+    # file is sourced, so putting it here overrides the production default
+    # without the script needing a sim-specific branch.
+    for h in "$R1" "$R2"; do
+        printf 'HE_USER=sim\nHE_UPDATE_KEY=sim\nHE_TUNNEL_ID=1\nHE_UPDATE_URL=http://%s/nic/update\n' "$HE_ADDR" \
+          | timeout 30 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$h" \
+              'cat > /tmp/he-secrets && sudo install -o root -g vyattacfg -m 0640 /tmp/he-secrets /config/he-secrets' >/dev/null
+    done
+
+    for h in "$R1" "$R2"; do
+        [ "$h" = "$R1" ] && { pref=high; v9=1; } || { pref=low; v9=2; }
+        log "    $h (RA preference $pref, bond0.9 ::${v9})"
+        vyos_apply "$h" <<EOF
+set interfaces tunnel tun0 encapsulation 'sit'
+set interfaces tunnel tun0 source-address '$src'
+set interfaces tunnel tun0 remote '$HE_ADDR'
+set interfaces tunnel tun0 address '$RT_LINK6/64'
+set interfaces tunnel tun0 mtu '1480'
+set protocols static route6 ::/0 next-hop '$HE_LINK6'
+set interfaces bonding bond0 vif 9 address '${V9_PREFIX}::${v9}/64'
+set service router-advert interface bond0.9 prefix ${V9_PREFIX}::/64 preferred-lifetime '604800'
+set service router-advert interface bond0.9 link-mtu '1472'
+set service router-advert interface bond0.9 default-preference '$pref'
+set system task-scheduler task he-tunnel-follow executable path '/config/he-tunnel-follow'
+set system task-scheduler task he-tunnel-follow executable arguments 'run'
+set system task-scheduler task he-tunnel-follow interval '1m'
+EOF
+    done
+    # RA link-mtu is 1472, the PPPoE figure, on BOTH -- deliberately not 1480.
+    # It cannot be reconciled at runtime (it needs a commit, and the tunnel plane
+    # is commit-free on purpose), so advertise the lower of the two paths and be
+    # correct on either WAN. Production pinned 1480 and was wrong whenever the
+    # WAN fell back.
+    log "    setup done -- run V0 to check the baseline"
+}
+
+# --- tests ------------------------------------------------------------------
+V0() {  # baseline
+    log "V0  baseline: the VIP holder owns the tunnel, the backup does not"
+    local h o; h="$(holder)"; o=$([ "$h" = "$R1" ] && echo "$R2" || echo "$R1")
+    [ "$h" = none ] && { fail "no VIP holder"; return; }
+    case "$(tun_state "$h")" in UP|UNKNOWN) pass "master $h has tun0 up" ;;
+        *) fail "master $h tun0 is $(tun_state "$h")" ;; esac
+    case "$(tun_state "$o")" in DOWN|absent) pass "backup $o tun0 is $(tun_state "$o")" ;;
+        *) fail "backup $o tun0 is $(tun_state "$o") -- it should be held down" ;; esac
+    v6_online "$h" && pass "master reaches HE over v6" || fail "master has no IPv6"
+    radvd_on "$h" && pass "master is advertising on VLAN 9" || fail "master radvd not running"
+    radvd_on "$o" && fail "backup is ALSO advertising -- two default routers on VLAN 9" \
+                  || pass "backup is not advertising"
+    check_invariants
+    save_evidence V0-baseline
+}
+
+V1() {  # clean failover: v6 follows, and HE is never called
+    log "V1  clean failover: IPv6 follows, HE API untouched"
+    local from to t0 before after i
+    from="$(holder)"; to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1")
+    before="$(he_calls)"
+    log "    master=$from -> expecting $to (HE calls so far: ${before:-0})"
+    t0=$(date +%s)
+    r "$from" 'sudo mkdir -p /run/vrrp-wan && sudo touch /run/vrrp-wan/force-fault'
+    local took=""
+    for i in $(seq 1 36); do
+        sleep 5
+        [ "$(holder)" = "$to" ] && v6_online "$to" && { took=$(( $(date +%s) - t0 )); break; }
+    done
+    [ -n "$took" ] && pass "IPv6 reached $to in ${took}s" \
+                   || fail "IPv6 never followed to $to within 180s"
+    case "$(tun_state "$from")" in DOWN|absent) pass "$from released its tunnel" ;;
+        *) fail "$from still has tun0 $(tun_state "$from") -- blackhole risk" ;; esac
+    after="$(he_calls)"
+    # THE invariant this test exists for.
+    [ "${after:-0}" = "${before:-0}" ] \
+        && pass "HE API not called (${after:-0} total) -- the address followed the MAC" \
+        || fail "HE API called $(( ${after:-0} - ${before:-0} )) time(s) during a ROUTER failover"
+    check_invariants
+    save_evidence V1-clean-failover
+    r "$from" 'sudo rm -f /run/vrrp-wan/force-fault'
+    sleep 40
+}
+
+V2() {  # 10 gig down on the master: HE must be told, exactly once
+    log "V2  10 gig down: he-tunnel-follow re-points the tunnel and tells HE"
+    local h before after mtu i ok=no
+    h="$(holder)"; before="$(he_calls)"
+    local tengig; tengig="$(tengig_addr)"
+    r "$h" 'sudo ip link set bond0.53 down'
+    # he-tunnel-follow runs on a 1m task-scheduler with a 2-tick hysteresis, so
+    # allow well past 2 minutes before calling it a failure.
+    for i in $(seq 1 30); do
+        sleep 10
+        [ "$(tun_src "$h")" != "$tengig" ] && { ok=yes; break; }
+    done
+    [ "$ok" = yes ] && pass "tunnel source moved to $(tun_src "$h") after $((i*10))s" \
+                    || fail "tunnel source never left the 10 gig address"
+    mtu="$(tun_mtu "$h")"
+    [ "$mtu" = 1472 ] && pass "MTU dropped to 1472 for the PPPoE path" \
+                      || fail "MTU is $mtu, want 1472 -- large transfers will hang"
+    after="$(he_calls)"
+    [ "$(( ${after:-0} - ${before:-0} ))" -ge 1 ] \
+        && pass "HE API called $(( ${after:-0} - ${before:-0} )) time(s), as it must be here" \
+        || fail "HE was never told -- it still points at an address this box no longer has"
+    v6_online "$h" && pass "IPv6 still up over PPPoE" || fail "IPv6 down on the PPPoE path"
+    save_evidence V2-tengig-down
+    r "$h" 'sudo ip link set bond0.53 up'
+    sleep 60
+}
+
+V3() {  # a cold backup must not advertise, dial, or blackhole
+    log "V3  cold backup: no tunnel, no RA, no HE call"
+    local h o before after
+    h="$(holder)"; o=$([ "$h" = "$R1" ] && echo "$R2" || echo "$R1")
+    before="$(he_calls)"
+    r "$o" 'sudo systemctl restart vrrp-wan-reconcile.service' >/dev/null
+    sleep 20
+    case "$(tun_state "$o")" in DOWN|absent) pass "backup tunnel stays $(tun_state "$o")" ;;
+        *) fail "backup brought tun0 up while not holding the VIP" ;; esac
+    radvd_on "$o" && fail "backup is advertising on VLAN 9" || pass "backup is silent on VLAN 9"
+    after="$(he_calls)"
+    [ "${after:-0}" = "${before:-0}" ] && pass "backup made no HE call" \
+        || fail "the BACKUP called the HE API -- it would point HE at its own idle line"
+    save_evidence V3-cold-backup
+}
+
+V4() {  # the assumption the production override was built on
+    log "V4  a tunnel whose source-address is absent: what does VyOS actually do?"
+    # The production override says such a tunnel "would simply stay down", and
+    # treats that as the reason it was safe to leave IPv6 single-homed. Measured
+    # on the sim backup 2026-09-06: the commit SUCCEEDS and the link comes up
+    # anyway -- it is a blackhole, not an inert node. That is why the runtime
+    # gate is load-bearing rather than a nicety, exactly like the PPPoE gate.
+    local o h; h="$(holder)"; o=$([ "$h" = "$R1" ] && echo "$R2" || echo "$R1")
+    r "$o" 'sudo ip link set tun0 up' >/dev/null; sleep 3
+    case "$(tun_state "$o")" in
+        UP|UNKNOWN) pass "confirmed: VyOS leaves it UP with no source address (blackhole)" ;;
+        *) warn "this VyOS version keeps it $(tun_state "$o") -- the override's assumption holds here; re-check the production version before relying on it" ;;
+    esac
+    v6_online "$o" && fail "the backup somehow reached HE -- two live tunnels" \
+                   || pass "and it carries nothing, as expected"
+    # Hand it straight back to the reconciler rather than leaving it up.
+    r "$o" 'sudo systemctl restart vrrp-wan-reconcile.service' >/dev/null
+    sleep 15
+    case "$(tun_state "$o")" in DOWN|absent) pass "the reconciler put it back down" ;;
+        *) fail "the reconciler did NOT re-close the gate -- this is the load-bearing bit" ;; esac
+    save_evidence V4-absent-source-address
+}
+
+V5() {  # never two live tunnels, even mid-transition
+    log "V5  both routers momentarily master: never two live tunnels"
+    local from to i worst=0 n
+    from="$(holder)"; to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1")
+    r "$from" 'sudo mkdir -p /run/vrrp-wan && sudo touch /run/vrrp-wan/force-fault'
+    # Sample THROUGH the transition rather than at the ends. The interesting
+    # window is the one where both boxes briefly think they are in charge.
+    for i in $(seq 1 24); do
+        n="$(tun_holders)"; [ "${n:-0}" -gt "$worst" ] && worst="$n"
+        sleep 5
+    done
+    [ "$worst" -le 1 ] && pass "at most $worst live tunnel throughout the transition" \
+                       || fail "saw $worst live tunnels at once -- HE would receive two claimants"
+    r "$from" 'sudo rm -f /run/vrrp-wan/force-fault'
+    save_evidence V5-transition-invariant
+    sleep 40
+}
+
+V6() {  # RA deprecation: does a VLAN 9 host drop the dead gateway?
+    log "V6  RA deprecation: the client must stop using a demoted router"
+    local h before
+    h="$(holder)"
+    before="$(lan9 'ip -6 route show default 2>/dev/null | head -1')"
+    if [ -z "$before" ]; then
+        warn "VLAN 9 client has no IPv6 default route -- SLAAC may not have run; skipping"
+        return
+    fi
+    log "    client default was: $before"
+    r "$h" 'sudo mkdir -p /run/vrrp-wan && sudo touch /run/vrrp-wan/force-fault'
+    sleep 45
+    local after; after="$(lan9 'ip -6 route show default 2>/dev/null | head -1')"
+    log "    client default now: ${after:-<none>}"
+    # radvd emits a final RA with router-lifetime 0 on a graceful stop. Either
+    # the client moved to the new master or it dropped the route entirely; both
+    # are correct. Still pointing at the demoted box is not.
+    if [ "$after" = "$before" ]; then
+        fail "client still points at the demoted router -- the farewell RA did not land"
+    else
+        pass "client stopped using the demoted router"
+    fi
+    r "$h" 'sudo rm -f /run/vrrp-wan/force-fault'
+    save_evidence V6-ra-deprecation
+    sleep 40
+}
+
+V7() {  # replay the 2026-09-06 near-miss, but slowly
+    log "V7  slow WAN restore: does the hysteresis still hold?"
+    # On 2026-09-06 vif53-pin-boot-disable bounced the 10 gig, he-tunnel-follow
+    # ticked once and saw the PPPoE address, and vyos-failover restored the 10
+    # gig 22 SECONDS before the second tick would have pushed HE at an address
+    # Vodafone reissues on every dial. 22s of margin is not a safety property.
+    # Here the restore is deliberately slower than the hysteresis window.
+    local h before after tengig
+    h="$(holder)"; before="$(he_calls)"; tengig="$(tengig_addr)"
+    r "$h" 'sudo ip link set bond0.53 down'
+    sleep 200                                  # > 2 ticks of a 1m scheduler
+    r "$h" 'sudo ip link set bond0.53 up'
+    sleep 90
+    after="$(he_calls)"
+    if [ "$(( ${after:-0} - ${before:-0} ))" -ge 1 ]; then
+        warn "HE was updated $(( ${after:-0} - ${before:-0} )) time(s) and then had to move back -- this is the 2026-09-06 shape, now reproduced deliberately. Either widen HYSTERESIS or make vif53-pin-boot-disable hold he-tunnel-follow off for the bounce."
+    else
+        pass "no HE churn across a slow WAN bounce"
+    fi
+    # Whatever happened, the tunnel must end up back on the 10 gig.
+    local i
+    for i in $(seq 1 24); do
+        [ "$(tun_src "$h")" = "$tengig" ] && break
+        sleep 10
+    done
+    [ "$(tun_src "$h")" = "$tengig" ] && pass "tunnel returned to the 10 gig address" \
+        || fail "tunnel stuck on $(tun_src "$h") after the 10 gig came back"
+    save_evidence V7-slow-restore
+}
+
+preflight() {
+    log "preflight"
+    local rc=0
+    [ "$(timeout 60 "$HE" status | grep -c 'API bound  : nohost')" = 1 ] \
+        || { fail "the fake HE endpoint is not answering -- run ./labsim-he-endpoint.sh up"; rc=1; }
+    local h
+    for h in "$R1" "$R2"; do
+        [ "$(tun_state "$h")" = absent ] \
+            && { fail "$h has no tun0 -- run --setup first"; rc=1; }
+        [ "$(r "$h" 'systemctl is-active vrrp-wan-reconcile.timer')" = active ] \
+            || { fail "$h vrrp-wan-reconcile.timer not active"; rc=1; }
+        # The reconciler must be the version that knows about the v6 plane, or
+        # every result below measures the OLD behaviour while printing the new
+        # test names -- the failure mode this repo has already been bitten by.
+        r "$h" 'grep -q v6_take /config/vrrp-wan-reconcile' \
+            || { fail "$h has a vrrp-wan-reconcile with no IPv6 plane -- run migration/vrrp-wan-install"; rc=1; }
+    done
+    # Line the sim's endpoint up with whoever actually holds the tunnel. The real
+    # HE remembers where it was pointed; a rebuilt sim endpoint does not, and
+    # he-tunnel-follow will not re-assert because from its side nothing changed.
+    # Does NOT count as an API call, so the invariant-2 assertions stay honest.
+    local m src
+    m="$(holder)"; src="$(tun_src "$m")"
+    if [ -n "$src" ]; then
+        timeout 60 "$HE" point "$src" >/dev/null 2>&1 \
+            || { fail "could not point the sim HE endpoint at $src"; rc=1; }
+    fi
+    [ "$rc" -eq 0 ] && pass "HE endpoint answering and pointed at $src, both routers have tun0 and a v6-aware reconciler"
+    return $rc
+}
+
+case "${1:---all}" in
+  --list)  echo "V0 baseline | V1 clean failover | V2 10gig-down | V3 cold backup | V4 absent source-address | V5 transition invariant | V6 RA deprecation | V7 slow restore"; exit 0 ;;
+  --setup) setup; exit 0 ;;
+  --all)   preflight || exit 1; V0; V1; V2; V3; V4; V5; V6; V7 ;;
+  *)       preflight || exit 1; "$1" ;;
+esac
+
+echo
+[ "$FAILED" -eq 0 ] && { echo "ALL PASS"; exit 0; }
+echo "$FAILED check(s) FAILED"; exit 1

labsim/labsim-k8s-etcd.sh

@@ -0,0 +1,279 @@
+#!/bin/bash
+# A 3-SERVER embedded-etcd k3s cluster in labsim, configured the way PRODUCTION
+# is -- through /etc/rancher/k3s/config.yaml rendered by labctl's own generator.
+#
+# WHY THIS EXISTS, separate from k8s-up.sh. k8s-up.sh is 1 server + 2 agents and
+# drives k3s with inline INSTALL_K3S_EXEC flags. Production is 3 control-plane
+# servers with embedded etcd, configured by config.yaml from k3s-config.ts. The
+# dual-stack conversion touches etcd quorum on all three at once and rolls a
+# config.yaml change one server at a time -- a failure mode a single-server lab
+# structurally cannot show, driven by a mechanism k8s-up.sh does not use. So this
+# harness models the real shape and the real code path, or the rehearsal is
+# theatre.
+#
+#   ./labsim-k8s-etcd.sh up            build the 3-server cluster
+#   ./labsim-k8s-etcd.sh kubeconfig    fetch ./labsim-etcd.kubeconfig
+#   ./labsim-k8s-etcd.sh status        node + etcd health
+#   ./labsim-k8s-etcd.sh down          destroy the three VMs
+#   ./labsim-k8s-etcd.sh render N      print the config.yaml node N would get
+#
+# The config.yaml is produced by:
+#   bastion .../k3s/bin/render-config.js   (the production generator, exported)
+# so if that generator changes shape, this cluster moves with it.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+source "$SCRIPT_DIR/lib.sh"
+source "$SCRIPT_DIR/ovs.sh"
+
+K8S_VLAN="${K8S_VLAN:-2}"
+NET="${NET:-172.31.2}"
+FIRST_OCTET="${FIRST_OCTET:-31}"          # .31/.32/.33 -- clear of k8s-up.sh's .11-.13
+SERVERS="${SERVERS:-3}"
+MEM="${MEM:-4096}"; CPUS="${CPUS:-2}"; DISK_GB="${DISK_GB:-12}"
+TOKEN="${TOKEN:-labsim-etcd-token}"
+CILIUM_VERSION="${CILIUM_VERSION:-1.19.1}"
+
+DEB_URL="${DEB_URL:-https://cloud.debian.org/images/cloud/trixie/latest/debian-13-genericcloud-amd64.qcow2}"
+DEB_BASE="${DEB_BASE:-$IMG_DIR/debian-13-genericcloud-amd64.qcow2}"
+
+# The production generator, compiled. Built by `npm --prefix bastion/src/modules run build`.
+RENDER="${RENDER:-$SCRIPT_DIR/../bastion/src/modules/dist/modules/k3s/bin/render-config.js}"
+
+node_name() { echo "labsim-etcd$1"; }
+node_ip()   { echo "${NET}.$((FIRST_OCTET + $1 - 1))"; }
+
+# The audit policy the generated config.yaml references. Without the file the
+# apiserver refuses to start (audit-policy-file points at a missing path), which
+# is a silent-looking crash loop. Kept byte-identical to labctl's audit-policy.ts.
+AUDIT_POLICY='apiVersion: audit.k8s.io/v1
+kind: Policy
+rules:
+  - level: Metadata
+    resources:
+    - group: ""
+      resources: ["secrets", "configmaps"]
+  - level: RequestResponse
+    verbs: ["create", "update", "patch", "delete"]
+    resources:
+    - group: ""
+      resources: ["pods", "services", "deployments"]
+  - level: None
+    resources:
+    - group: ""
+      resources: ["endpoints", "events"]
+    users: ["system:kube-proxy", "system:apiserver"]
+  - level: Metadata
+    omitStages:
+    - "RequestReceived"'
+
+# Render node N's config.yaml with the production generator. Node 1 is
+# cluster-init; 2..N join as SERVERS (not agents) -- role=infra with a server
+# URL is exactly a joining etcd member, the same as production worker1/worker2.
+render_config() {
+  local n="$1"
+  local ip; ip="$(node_ip "$n")"
+  local server1; server1="$(node_ip 1)"
+  if [ "$n" -eq 1 ]; then
+    ROLE=infra HOSTNAME="$(node_name 1)" IP="$ip" TLS_SANS="$ip" \
+      node "$RENDER"
+  else
+    ROLE=infra HOSTNAME="$(node_name "$n")" IP="$ip" TLS_SANS="$ip" \
+      K3S_SERVER_URL="https://${server1}:6443" K3S_TOKEN="$TOKEN" \
+      node "$RENDER"
+  fi
+}
+
+ensure_base_image() {
+  [ -f "$DEB_BASE" ] && { log "base image present"; return; }
+  log "fetching Debian cloud image -> $DEB_BASE"
+  sudo mkdir -p "$IMG_DIR"
+  sudo curl -fsSL --retry 3 -o "${DEB_BASE}.tmp" "$DEB_URL" || die "fetch failed"
+  sudo mv "${DEB_BASE}.tmp" "$DEB_BASE"
+}
+
+build_seed() {
+  local iso="$1" n="$2" pubkey="$3"
+  local vm; vm="$(node_name "$n")"; local ip; ip="$(node_ip "$n")"
+  local server1; server1="$(node_ip 1)"
+  local tmp; tmp="$(mktemp -d)"
+  local cfg; cfg="$(render_config "$n")"
+
+  cat > "$tmp/meta-data" <<EOF
+instance-id: $vm
+local-hostname: $vm
+EOF
+  cat > "$tmp/network-config" <<EOF
+version: 2
+ethernets:
+  enp1s0:
+    match: { name: "en*" }
+    addresses: [$ip/24]
+    routes: [{ to: default, via: ${NET}.1 }]
+    nameservers: { addresses: [8.8.8.8, 1.1.1.1] }
+EOF
+
+  # config.yaml and audit policy embedded via write_files, indented for YAML.
+  local cfg_ind audit_ind
+  cfg_ind="$(printf '%s\n' "$cfg" | sed 's/^/      /')"
+  audit_ind="$(printf '%s\n' "$AUDIT_POLICY" | sed 's/^/      /')"
+
+  cat > "$tmp/user-data" <<EOF
+#cloud-config
+hostname: $vm
+users:
+  - name: debian
+    groups: [sudo]
+    shell: /bin/bash
+    sudo: ["ALL=(ALL) NOPASSWD:ALL"]
+    lock_passwd: false
+    plain_text_passwd: labsim
+    ssh_authorized_keys: [ $pubkey ]
+ssh_pwauth: true
+ssh_authorized_keys: [ $pubkey ]
+
+package_update: true
+packages: [curl, jq, iproute2, tcpdump, etcd-client]
+
+write_files:
+  - path: /etc/rancher/k3s/config.yaml
+    content: |
+$cfg_ind
+  - path: /etc/rancher/k3s/audit-policy.yaml
+    content: |
+$audit_ind
+  - path: /etc/modules-load.d/cilium.conf
+    content: |
+      br_netfilter
+      overlay
+  # The CIS sysctls the generated config's `protect-kernel-defaults: true`
+  # REQUIRES -- byte-for-byte from labctl's sysctl.ts (applyCisHardening), plus
+  # v6 forwarding. Without vm.overcommit_memory=1 / kernel.panic=10 /
+  # kernel.panic_on_oops=1 the kubelet REFUSES to start ("invalid kernel flag"),
+  # k3s exits 1 and crash-loops -- which presents downstream as etcd
+  # re-initialising and the apiserver flapping, i.e. it looks like an etcd/CPU
+  # problem when it is not. In production these come from install.ks.ts + this
+  # operation; the sim must set them too.
+  - path: /etc/sysctl.d/90-k3s-cis.conf
+    content: |
+      net.bridge.bridge-nf-call-iptables  = 1
+      net.bridge.bridge-nf-call-ip6tables = 1
+      net.ipv4.ip_forward                 = 1
+      net.ipv6.conf.all.forwarding        = 1
+      vm.panic_on_oom                     = 0
+      vm.overcommit_memory                = 1
+      kernel.panic                        = 10
+      kernel.panic_on_oops                = 1
+      fs.inotify.max_user_instances       = 524288
+      fs.inotify.max_user_watches         = 524288
+
+runcmd:
+  - [ modprobe, br_netfilter ]
+  - [ modprobe, overlay ]
+  - [ sysctl, --system ]
+  - |
+    # Joining servers must wait for the cluster-init server's API, or the join
+    # races etcd bootstrap and the unit backs off for minutes.
+    if [ "$n" -ne 1 ]; then
+      for i in \$(seq 1 90); do
+        curl -sk --max-time 3 https://${server1}:6443/ping >/dev/null 2>&1 && break
+        sleep 5
+      done
+    fi
+  - |
+    # INSTALL_K3S_EXEC=server (bare) -- everything else comes from config.yaml,
+    # exactly as production. The token is passed via env for the join; on node 1
+    # it seeds the cluster token.
+    #
+    # The etcd-arg tuning is LAB-ONLY (not in production's config): relaxed
+    # heartbeat/election timers so etcd tolerates nested-virt scheduling jitter.
+    # NB: this was NOT what fixed the first run's failure -- that was the missing
+    # protect-kernel-defaults sysctls above, which crash-looped the kubelet and
+    # only LOOKED like etcd instability. The tuning is kept as cheap defensive
+    # insurance for a busy host; it changes nothing the conversion test
+    # exercises.
+    curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --etcd-arg=heartbeat-interval=500 --etcd-arg=election-timeout=5000" K3S_TOKEN="$TOKEN" sh -
+EOF
+
+  # Guard the generated YAML before building the ISO -- a bad indent in the
+  # embedded config.yaml would fail on the node, minutes later and opaquely.
+  python3 -c "import yaml,sys; yaml.safe_load(open(sys.argv[1]))" "$tmp/user-data" \
+    || die "generated user-data is not valid YAML for node $n"
+
+  sudo mkdir -p "$(dirname "$iso")"
+  sudo genisoimage -quiet -output "$iso" -volid cidata -joliet -rock \
+    "$tmp/user-data" "$tmp/meta-data" "$tmp/network-config"
+  rm -rf "$tmp"
+}
+
+create_node() {
+  local n="$1" pubkey="$2"
+  local vm; vm="$(node_name "$n")"; local ip; ip="$(node_ip "$n")"
+  if virsh_q dominfo "$vm" >/dev/null 2>&1; then
+    local st; st="$(virsh_q domstate "$vm" 2>/dev/null | head -1 | tr -d '\n')"
+    [ "$st" = running ] && { log "$vm already running ($ip)"; return; }
+    log "$vm is $st -- starting"; virsh_q start "$vm" >/dev/null; return
+  fi
+  local disk="$IMG_DIR/${vm}.qcow2" seed="$IMG_DIR/${vm}-seed.iso"
+  log "creating $vm ($ip, server ${n}, ${MEM}MB/${CPUS}cpu)"
+  sudo qemu-img create -q -f qcow2 -F qcow2 -b "$DEB_BASE" "$disk" "${DISK_GB}G" >/dev/null
+  build_seed "$seed" "$n" "$pubkey"
+  sudo virt-install --connect "$LIBVIRT_URI" --name "$vm" \
+    --memory "$MEM" --vcpus "$CPUS" \
+    --disk "path=$disk,format=qcow2,bus=virtio" \
+    --disk "path=$seed,device=cdrom" \
+    --network "network=$OVS_NET,portgroup=vlan${K8S_VLAN},model=virtio" \
+    --os-variant debian12 --graphics none --noautoconsole --import >/dev/null
+}
+
+ssh_node() { ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+               -o LogLevel=ERROR -o ConnectTimeout=8 "debian@$1" "$2" 2>/dev/null; }
+
+cmd_up() {
+  require_tools
+  command -v genisoimage >/dev/null || die "genisoimage missing"
+  [ -f "$RENDER" ] || die "render CLI not built: $RENDER (run: npm --prefix bastion/src/modules run build)"
+  local pubkey; pubkey="$(find_ssh_pubkey)"
+  ensure_base_image
+  selected_vlans; log "ensuring OVS fabric"; ovs_up
+  local n; for n in $(seq 1 "$SERVERS"); do create_node "$n" "$pubkey"; done
+  echo
+  log "3 servers booting. Node 1 cluster-inits; 2/3 join as etcd members."
+  log "watch:  ssh debian@$(node_ip 1) 'sudo k3s kubectl get nodes'"
+  log "then:   $0 kubeconfig  &&  $0 status"
+}
+
+cmd_kubeconfig() {
+  local s1; s1="$(node_ip 1)"; local out="$SCRIPT_DIR/labsim-etcd.kubeconfig"
+  ssh_node "$s1" "sudo cat /etc/rancher/k3s/k3s.yaml" | sed "s|127.0.0.1|$s1|" > "$out"
+  chmod 600 "$out"; log "wrote $out"
+}
+
+cmd_status() {
+  local s1; s1="$(node_ip 1)"
+  echo "=== nodes ==="
+  ssh_node "$s1" "sudo k3s kubectl get nodes -o wide 2>/dev/null" | sed 's/^/  /'
+  echo "=== etcd members (quorum needs 2 of 3) ==="
+  ssh_node "$s1" 'sudo k3s kubectl get nodes -l node-role.kubernetes.io/etcd=true --no-headers 2>/dev/null | wc -l' | sed 's/^/  etcd nodes: /'
+  echo "=== servicecidr (both families once dual-stack) ==="
+  ssh_node "$s1" "sudo k3s kubectl get servicecidr -o jsonpath='{range .items[*]}{.metadata.name}={.spec.cidrs}{\"\\n\"}{end}' 2>/dev/null" | sed 's/^/  /'
+}
+
+cmd_down() {
+  local n vm
+  for n in $(seq 1 "$SERVERS"); do
+    vm="$(node_name "$n")"
+    virsh_q destroy "$vm" >/dev/null 2>&1 || true
+    virsh_q undefine "$vm" --remove-all-storage >/dev/null 2>&1 || true
+    log "removed $vm"
+  done
+}
+
+case "${1:-up}" in
+  up)         cmd_up ;;
+  kubeconfig) cmd_kubeconfig ;;
+  status)     cmd_status ;;
+  down)       cmd_down ;;
+  render)     render_config "${2:-1}" ;;
+  *) die "usage: $0 {up|kubeconfig|status|down|render N}" ;;
+esac

labsim/labsim-pppoe-ha-test.sh

@@ -0,0 +1,381 @@
+#!/bin/bash
+# Does the WAN follow VRRP mastership, and does exactly ONE router ever hold the
+# ISP session?
+#
+# The question is not "did a client get internet". A client can be answered by
+# the wrong path entirely -- for months labsim-vyos's only default route was the
+# libvirt-NAT scaffold on eth2, so every "the LAN still has internet" verdict was
+# answered by eth2 rather than by the WAN under test. This script therefore
+# refuses to run while that is true, and asks its questions of the ROUTERS and
+# the ACCESS CONCENTRATOR, which cannot be answered by accident.
+#
+# The invariant, checked continuously and independently of any individual test:
+#
+#     the AC never reports two `simdsl` sessions, and no two routers ever have a
+#     pppoe0 interface at the same time
+#
+# A run that violates it FAILS regardless of its own verdict, because a single
+# consumer credential is the whole constraint the design exists to satisfy.
+#
+#   ./labsim-pppoe-ha-test.sh --list
+#   ./labsim-pppoe-ha-test.sh T3
+#   ./labsim-pppoe-ha-test.sh --all
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+R1="${R1:-172.31.1.252}"; R2="${R2:-172.31.1.253}"
+ISP="${ISP:-192.168.122.63}"          # the fake access concentrator
+LANVM="${LANVM:-172.31.10.10}"
+VIP="${VIP:-172.31.1.1}"
+PW="${VYOS_PW:-vyos}"; LANPW="${LANPW:-labsim}"
+EVID="$SCRIPT_DIR/wan-failover-evidence"
+
+SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+     -o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password)
+r()   { timeout 45 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$1" "${@:2}" 2>/dev/null; }
+isp() { timeout 30 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$ISP" "$@" 2>/dev/null; }
+
+# Set the AC's session policy, and PROVE it landed.
+#
+# `vbash -c 'source script-template; configure; ...; commit'` does NOT work: the
+# config session never starts and commit dies with "Invalid command: [commit]",
+# on stderr, which the isp() helper discards. The whole T4 matrix therefore ran
+# all three iterations against the accel-ppp DEFAULT while printing
+# "--- session-control=deny ---" -- it reported coverage it did not have, which
+# is worse than reporting a failure. Drive it from a real script FILE, then read
+# the value back and abort the run if it disagrees.
+isp_session_control() {
+    local mode="$1"
+    printf '#!/bin/vbash\nsource /opt/vyatta/etc/functions/script-template\nconfigure\nset service pppoe-server session-control %s\ncommit\nsave\nexit\n' "$mode" \
+        | timeout 30 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$ISP" 'cat > /tmp/set-sc.sh && chmod +x /tmp/set-sc.sh && sudo /tmp/set-sc.sh' >/dev/null 2>&1
+    local got
+    got="$(isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands' \
+             | sed -n "s/.*session-control '\\(.*\\)'/\\1/p")"
+    if [ "$got" = "$mode" ]; then
+        log "    AC session-control=$mode (verified)"
+        return 0
+    fi
+    fail "could not set AC session-control=$mode (reads '${got:-unset}') -- results would be fiction"
+    return 1
+}
+# The LAN VMs are Alpine and their sshd offers keyboard-interactive, not
+# `password`. Reusing the routers' option set here made ssh exit 255 BEFORE
+# running anything, and T5 read that as "the LAN lost the internet" while a
+# tcpdump on the router showed the pings flowing out pppoe0 and the replies
+# coming back. An exit code that can mean "the network is broken" or "I could
+# not log in" is not a connectivity test.
+LAN_SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+         -o LogLevel=ERROR -o ConnectTimeout=6)
+lan() { timeout 45 sshpass -p "$LANPW" ssh "${LAN_SSH[@]}" "root@$LANVM" "$@" 2>/dev/null; }
+
+# Assert on what the guest actually reported, not on ssh's exit status.
+lan_online() { [ "$(lan 'ping -c2 -W3 9.9.9.9 >/dev/null 2>&1 && echo ONLINE')" = ONLINE ]; }
+
+log()  { printf '\033[36m==>\033[0m %s\n' "$*"; }
+pass() { printf '  \033[32mPASS\033[0m %s\n' "$*"; }
+fail() { printf '  \033[31mFAIL\033[0m %s\n' "$*"; FAILED=$((FAILED+1)); }
+FAILED=0
+
+# --- observations ----------------------------------------------------------
+ac_sessions() { isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show pppoe-server sessions' \
+                  | grep -c ' simdsl ' || true; }
+ac_detail()   { isp '/opt/vyatta/bin/vyatta-op-cmd-wrapper show pppoe-server sessions'; }
+# A destroyed or unreachable router is emphatically NOT holding pppoe0, but ssh
+# returns an EMPTY string rather than 0 -- and `[ "" = 0 ]` is false, so a
+# hard-failover test waited for the dead box to "report" zero and hung until its
+# timeout, long after the survivor had taken over correctly. Default to 0.
+ppp_on()      { local v; v="$(r "$1" 'ip -4 addr show pppoe0 2>/dev/null | grep -c inet' | tr -d ' \n')"
+                echo "${v:-0}"; }
+holder()      { for h in "$R1" "$R2"; do
+                  [ "$(r "$h" "ip -4 -o addr show | grep -c ' ${VIP}/'" | tr -d ' \n')" != 0 ] \
+                    && { echo "$h"; return; }; done; echo none; }
+status()      { r "$1" 'sudo /config/vrrp-wan-reconcile --status'; }
+
+# How many routers currently hold a PPPoE interface. The invariant's other half.
+ppp_holders() { n=0; for h in "$R1" "$R2"; do
+                  [ "$(ppp_on "$h")" != 0 ] && n=$((n+1)); done; echo "$n"; }
+
+warn() { printf '  \033[33mWARN\033[0m %s\n' "$*"; }
+
+check_invariant() {
+    local s p ok=0
+    s="$(ac_sessions)"; p="$(ppp_holders)"
+
+    # THE invariant. Two of OUR routers dialled at once is the failure that
+    # matters: one ISP account, and against a real ISP that is how you get
+    # rate-limited or locked out.
+    [ "${p:-0}" -le 1 ] || { fail "INVARIANT: $p routers hold pppoe0"; ok=1; }
+
+    # The AC's session count is a PROXY for the above, and only a valid one
+    # while the AC enforces single-session. Under session-control=disable it
+    # does not, so a destroyed router's session simply stays in the table and
+    # the count reads 2 while exactly one live router is dialled -- which is AC
+    # bookkeeping, not a double dial. Attribute it rather than failing blind:
+    # only call it a violation when more than one router is ACTUALLY dialled.
+    #
+    # Do not silence it either. An orphaned session still occupies the single
+    # slot at a real ISP, and that is exactly what made session-control=deny
+    # take 148s while the survivor's dial attempts were refused.
+    if [ "${s:-0}" -gt 1 ]; then
+        if [ "${p:-0}" -gt 1 ]; then
+            fail "INVARIANT: AC reports $s simdsl sessions AND $p routers are dialled"
+            ok=1
+        else
+            warn "AC reports $s simdsl sessions but only ${p:-0} router is dialled -- stale session from the destroyed peer (expected where the AC does not enforce single-session; it is what a hostile AC holds against the survivor)"
+        fi
+    fi
+    return $ok
+}
+
+# --- preconditions ---------------------------------------------------------
+# The scaffold check is a hard gate, not a warning. A default route via eth2
+# means the box can reach the internet without the WAN working at all, and every
+# connectivity verdict below would be a lie.
+preflight() {
+    log "preflight"
+    local rc=0
+    for h in "$R1" "$R2"; do
+        if r "$h" 'ip route show default' | grep -q 'dev eth2'; then
+            fail "$h still routes via eth2 (libvirt-NAT scaffold) -- run sim-net-config.py --drop-scaffold"
+            rc=1
+        fi
+        if [ "$(r "$h" '[ -f /etc/systemd/system/ppp@pppoe0.service.d/10-vrrp-wan-gate.conf ] && echo y')" != y ]; then
+            fail "$h is missing the ppp gate drop-in -- run migration/vrrp-wan-install"
+            rc=1
+        fi
+        [ "$(r "$h" 'systemctl is-active vrrp-wan-guard.timer')" = active ] \
+            || { fail "$h vrrp-wan-guard.timer not active"; rc=1; }
+        # A router with no peers file CANNOT dial, and says so only once in the
+        # journal. Every failover result in the run would then be a false
+        # negative blamed on the ISP. Check both, and check the config that
+        # renders it -- an unsaved commit reverts on reboot and takes pppoe0
+        # with it, which is how the sim secondary silently stopped dialling.
+        if [ "$(r "$h" '[ -f /etc/ppp/peers/pppoe0 ] && echo y')" != y ]; then
+            fail "$h has no /etc/ppp/peers/pppoe0 -- it cannot dial; re-commit the pppoe subtree"
+            rc=1
+        fi
+        # The op-mode WRAPPER, not a bare `show`: over non-interactive ssh the
+        # bare form is not on PATH, so this silently matched nothing and failed
+        # both routers that were in fact configured correctly.
+        if ! r "$h" '/opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands' 2>/dev/null | grep -q 'interfaces pppoe pppoe0 source-interface'; then
+            fail "$h has no pppoe0 in config (unsaved commit lost on reboot?)"
+            rc=1
+        fi
+    done
+    [ "$rc" -eq 0 ] && pass "scaffold dropped, gate present, guard running, both can dial"
+    return $rc
+}
+
+settle() {  # wait until exactly one router holds pppoe0, or give up
+    local i
+    for i in $(seq 1 "${1:-24}"); do
+        [ "$(ppp_holders)" = 1 ] && return 0
+        sleep 5
+    done
+    return 1
+}
+
+# Wait until a SPECIFIC router holds pppoe0 and the other does not.
+#
+# The obvious `settle` is wrong for a failover: "exactly one holder" is already
+# true before the handover starts, so it returns instantly and the test reports
+# that nothing moved while the handover is still in flight. Asking who holds it
+# is the only useful form of the question.
+settle_on() {
+    local want="$1" other i
+    other=$([ "$want" = "$R1" ] && echo "$R2" || echo "$R1")
+    for i in $(seq 1 "${2:-30}"); do
+        [ "$(ppp_on "$want")" != 0 ] && [ "$(ppp_on "$other")" = 0 ] && return 0
+        sleep 5
+    done
+    return 1
+}
+
+save_evidence() {
+    local name="$1"; local d="$EVID/$name"; mkdir -p "$d"
+    { echo "=== $(date -Is) ==="; echo "--- AC sessions ---"; ac_detail
+      for h in "$R1" "$R2"; do echo "--- $h ---"; status "$h"
+        r "$h" 'ip -4 -br addr show pppoe0 2>/dev/null; ip route show default; sudo journalctl -t vrrp-wan -n 8 --no-pager'
+      done; } > "$d/state.txt" 2>&1
+    log "evidence -> wan-failover-evidence/$name/"
+}
+
+# --- tests -----------------------------------------------------------------
+T0() {  # baseline
+    log "T0  baseline: exactly one session, held by the VIP holder"
+    local h s; h="$(holder)"; s="$(ac_sessions)"
+    [ "$s" = 1 ] && pass "AC reports 1 session" || fail "AC reports $s sessions"
+    [ "$(ppp_on "$h")" != 0 ] && pass "the VIP holder ($h) is the one dialled" \
+        || fail "VIP holder $h has no pppoe0"
+    local other; other=$([ "$h" = "$R1" ] && echo "$R2" || echo "$R1")
+    [ "$(ppp_on "$other")" = 0 ] && pass "the backup ($other) is not dialled" \
+        || fail "backup $other also holds pppoe0"
+    save_evidence T0-baseline
+}
+
+T3() {  # clean, deliberate failover
+    log "T3  clean failover via force-fault"
+    local from to t0 t1; from="$(holder)"
+    to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1")
+    log "    master=$from -> expecting $to"
+    t0=$(date +%s)
+    r "$from" 'sudo mkdir -p /run/vrrp-wan && sudo touch /run/vrrp-wan/force-fault'
+    if settle_on "$to" 30; then
+        t1=$(date +%s)
+        [ "$(ppp_on "$to")" != 0 ] && pass "pppoe0 moved to $to in $((t1-t0))s" \
+            || fail "pppoe0 did not move to $to"
+        [ "$(ppp_on "$from")" = 0 ] && pass "$from released pppoe0" \
+            || fail "$from still holds pppoe0"
+    else
+        fail "never settled to exactly one pppoe0 holder"
+    fi
+    check_invariant
+    save_evidence T3-clean-failover
+    r "$from" 'sudo rm -f /run/vrrp-wan/force-fault'
+    settle 30 >/dev/null
+}
+
+T5() {  # 10gig down -> PPPoE carries traffic
+    log "T5  10 gig down on the master: traffic must survive on pppoe0"
+    local h; h="$(holder)"
+    r "$h" 'sudo ip link set bond0.53 down'
+    sleep 20
+    local via; via="$(r "$h" 'ip route show default' | head -1)"
+    if echo "$via" | grep -q pppoe0; then
+        pass "default route moved to pppoe0: $via"
+    else
+        fail "default route did not move to pppoe0: ${via:-<none>}"
+    fi
+    # Poll, do not sample. Judging connectivity on one ping 20s after the link
+    # dropped failed while the path was still reconverging, and reported "the LAN
+    # lost the internet" for a path that came back moments later. A single
+    # negative sample is the least trustworthy verdict this harness can produce.
+    local ok=no i
+    for i in $(seq 1 12); do
+        lan_online && { ok=yes; break; }
+        sleep 5
+    done
+    [ "$ok" = yes ] && pass "LAN reaches the internet over pppoe0 (after $((i*5))s)" \
+        || fail "LAN never regained the internet with only pppoe0 up (60s)"
+    save_evidence T5-tengig-down
+    r "$h" 'sudo ip link set bond0.53 up'
+    sleep 20
+}
+
+T11() { # a blessed box with no peers file must not restart-loop
+    log "T11 missing peers file must not restart-loop"
+    local h; h="$(holder)"
+    # COPY then remove, never move: only a commit touching the pppoe subtree
+    # re-renders this file, so losing it strands the box permanently -- the gate
+    # blocks every dial, systemd says "skipped because of an unmet condition
+    # check" exactly once, and nothing else complains. An earlier `mv` pair did
+    # exactly that to the sim secondary and cost a debugging session.
+    r "$h" 'sudo cp -a /etc/ppp/peers/pppoe0 /run/peers.bak && sudo rm -f /etc/ppp/peers/pppoe0; sudo systemctl restart ppp@pppoe0'
+    sleep 12
+    local n; n="$(r "$h" 'systemctl show ppp@pppoe0 -p NRestarts --value')"
+    [ "${n:-99}" -le 1 ] && pass "NRestarts=$n (gate refused the start)" \
+        || fail "NRestarts=$n -- restart loop is back"
+    # Restore on the SAME host we broke, and prove it landed. Do not trust the
+    # copy back: if it silently failed, every later test in the run would be
+    # measuring a router that physically cannot dial.
+    r "$h" 'sudo cp -a /run/peers.bak /etc/ppp/peers/pppoe0'
+    if r "$h" 'test -f /etc/ppp/peers/pppoe0'; then
+        pass "peers file restored on $h"
+    else
+        fail "peers file NOT restored on $h -- that router can no longer dial"
+    fi
+    save_evidence T11-no-peers-file
+    settle 24 >/dev/null
+}
+
+T8() {  # lease expiry: the guard must hang up a demoted-but-unreconciled box
+    log "T8  lease expiry revokes the session"
+    local h; h="$(holder)"
+    r "$h" 'sudo systemctl stop vrrp-wan-reconcile.timer'
+    r "$h" 'sudo touch -d "-200 seconds" /run/vrrp-wan/may-dial'
+    sleep 12
+    [ "$(ppp_on "$h")" = 0 ] && pass "guard hung up on a stale lease" \
+        || fail "stale lease did not revoke the session"
+    r "$h" 'sudo systemctl start vrrp-wan-reconcile.timer'
+    save_evidence T8-lease-expiry
+    settle 24 >/dev/null
+}
+
+
+T4() {  # hard failover across all three AC session-control policies
+    log "T4  hard failover (destroy the master) x session-control"
+    # Vodafone's policy is unknowable from here, so prove the design survives
+    # every one VyOS can express. `replace` is the accel-ppp default and the
+    # friendly case; `deny` is the hostile one, where the AC refuses the second
+    # session until its own dead-peer timer (lcp-echo-interval 30 x failure 3 =
+    # 90s) frees the first -- which is exactly why GRACE is no longer 90.
+    local mode from to vm t0 t1
+    for mode in replace deny disable; do
+        log "    --- session-control=$mode ---"
+        # Skip the iteration rather than measure the wrong policy.
+        isp_session_control "$mode" || continue
+        sleep 5
+        from="$(holder)"; to=$([ "$from" = "$R1" ] && echo "$R2" || echo "$R1")
+        vm=$([ "$from" = "$R1" ] && echo labsim-vyos || echo labsim-vyos2)
+        [ "$from" = none ] && { fail "no master before $mode run"; continue; }
+        log "    destroying $vm (master=$from), expecting $to"
+        t0=$(date +%s)
+        sudo virsh destroy "$vm" >/dev/null 2>&1
+        if settle_on "$to" 48; then
+            t1=$(date +%s)
+            pass "$mode: pppoe0 reached $to in $((t1-t0))s"
+        else
+            fail "$mode: $to never dialled within 240s"
+        fi
+        check_invariant
+        save_evidence "T4-hard-failover-$mode"
+        sudo virsh start "$vm" >/dev/null 2>&1
+        # Re-bond. A VM restart recreates its taps under NEW names, and the OVS
+        # bond keeps the old ones -- lacp dies, VLAN 1 goes with it, and the box
+        # comes back reachable on some VLANs but not others. ovs_bond_router
+        # detects the stale membership and rebuilds, but nothing runs it
+        # automatically, so a destroy/start test must do it or the survivor
+        # looks like a failover failure.
+        ( source "$SCRIPT_DIR/lib.sh"; source "$SCRIPT_DIR/ovs.sh"; selected_vlans
+          LAG_NAME=$([ "$vm" = labsim-vyos ] && echo lag-vyos || echo lag-vyos2)
+          ovs_bond_router "$vm" ) >/dev/null 2>&1
+        # Give the returning box time to boot and settle as BACKUP before the
+        # next iteration; it must NOT dial on the way up.
+        sleep 90
+        [ "$(ppp_on "$from")" = 0 ] && pass "$mode: $from did not dial on reboot" \
+            || fail "$mode: $from dialled on reboot (gate failed)"
+    done
+    isp_session_control replace >/dev/null
+    log "    AC restored to session-control=replace"
+}
+
+T12() { # the flap damper must never tear down an ESTABLISHED session
+    log "T12 flap holdoff must not kill a live session"
+    local h; h="$(holder)"
+    [ "$h" = none ] && { fail "no master to test"; return; }
+    # Forge a holdoff far in the future, as a dial storm would. Before the fix
+    # ppp_dial() returned here BEFORE renewing may-dial, the lease went stale,
+    # and vrrp-wan-guard hung up the master's working WAN ~80s later.
+    r "$h" 'sudo sh -c "echo $(( $(date +%s) + 900 )) > /run/vrrp-wan/holdoff"'
+    # Sleep past LEASE_TTL (75s) so a non-renewed lease would definitely expire.
+    sleep 100
+    if [ "$(ppp_on "$h")" = 1 ]; then
+        pass "session survived a 900s holdoff (lease still renewed)"
+    else
+        fail "holdoff killed the live session -- damper is tearing down the WAN"
+    fi
+    r "$h" 'sudo rm -f /run/vrrp-wan/holdoff /run/vrrp-wan/dials'
+    save_evidence T12-holdoff-keeps-session
+    settle 24 >/dev/null
+}
+
+case "${1:---all}" in
+  --list) echo "T0 baseline | T3 clean failover | T4 hard failover x policy | T5 10gig-down | T8 lease expiry | T11 no-peers-file | T12 holdoff-keeps-session"; exit 0 ;;
+  --all)  preflight || exit 1; T0; T3; T5; T8; T11; T12 ;;
+  --hard) preflight || exit 1; T4 ;;
+  *)      preflight || exit 1; "$1" ;;
+esac
+
+echo
+[ "$FAILED" -eq 0 ] && { echo "ALL PASS"; exit 0; }
+echo "$FAILED check(s) FAILED"; exit 1

labsim/sim-ha-config.py

@@ -45,6 +45,12 @@ VLANS = {
     10:  ("172.31.10",  23),
     200: ("172.31.200", 24),
 }
+# VLAN 2 (k8s) also gets a ULA IPv6, so the routers can peer eBGP with the nodes
+# over IPv6 (the neighbors in sim-net-config.py K8S_NODES_V6 = fd00:2::1x). Only
+# VLAN 2 needs it for the BGP rehearsal; a ULA keeps sim traffic out of the real
+# HE /48. Routers hold ::252 / ::253 (no v6 VRRP VIP -- BGP peers the real per-box
+# address, exactly as production).
+VLANS6 = {2: ("fd00:2", 64)}
 DHCP_HA_NAME = "labsim-dhcp-pair"     # must not equal either host-name
 
 
@@ -73,6 +79,8 @@ def build(role: str) -> list[str]:
             # .1 becomes the floating VIP, exactly as production will be.
             f"delete interfaces bonding {iface} address",
             f"set interfaces bonding {iface} address '{pfx}.{self_o}/{cidr}'",
+            *([f"set interfaces bonding {iface} address '{VLANS6[vlan][0]}::{self_o}/{VLANS6[vlan][1]}'"]
+              if vlan in VLANS6 else []),
             f"set high-availability vrrp group {g} interface bond0.{vlan}",
             f"set high-availability vrrp group {g} vrid {vlan}",
             f"set high-availability vrrp group {g} address {pfx}.1/{cidr}",
@@ -85,6 +93,27 @@ def build(role: str) -> list[str]:
         ]
 
     out += [
+        "# --- WAN follows VRRP mastership ---",
+        # These four hooks and the health check existed on both live sim VMs but
+        # in NEITHER generator, so `sim-net-apply.sh check` reported "in sync"
+        # while the mechanism under test was pure undetected drift -- exactly
+        # the failure mode this file was written to end.
+        #
+        # The check goes on the SYNC GROUP, not per group: VyOS rejects a
+        # per-group check while the group is in a sync group ("Only sync group
+        # health check will be used").
+        "set high-availability vrrp sync-group MAIN health-check script '/config/vrrp-wan-health'",
+        "set high-availability vrrp sync-group MAIN health-check interval '5'",
+        "set high-availability vrrp sync-group MAIN health-check failure-count '3'",
+        # take and release both exec vrrp-wan-reconcile: one code path, asked at
+        # different moments. `stop` matters as much as `backup` -- a stopped
+        # keepalived is a demotion too, and without it the box would keep the
+        # WAN while holding no VIPs.
+        "set high-availability vrrp sync-group MAIN transition-script master '/config/vrrp-wan-take'",
+        "set high-availability vrrp sync-group MAIN transition-script backup '/config/vrrp-wan-release'",
+        "set high-availability vrrp sync-group MAIN transition-script fault '/config/vrrp-wan-release'",
+        "set high-availability vrrp sync-group MAIN transition-script stop '/config/vrrp-wan-release'",
+        "",
         "# --- DHCP high-availability ---",
         "# The thing under test: active-passive should mean exactly one OFFER.",
         "set service dhcp-server high-availability mode active-passive",

labsim/sim-net-apply.sh

@@ -17,9 +17,15 @@ ACTION="${1:-check}"
 WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
 
 # role : vm : address : regex selecting the subtrees this generator owns
+#
+# primary and secondary now own the SAME subtrees. The secondary's used to omit
+# `interfaces pppoe`, `interfaces bonding`, `nat source` and `protocols
+# failover|static`, so `check` was blind to precisely the WAN config the
+# failover mechanism depends on -- it reported "in sync" for a box that had no
+# WAN at all.
 TARGETS=(
   "primary:labsim-vyos:172.31.1.252:^set (protocols (bgp|failover|static)|policy (prefix-list|route-map)|nat source rule 1[12]0|interfaces (pppoe|bonding bond0 vif 5[13])|firewall (group interface-group LAN|ipv4|ipv6))"
-  "secondary:labsim-vyos2:172.31.1.253:^set (protocols bgp|policy (prefix-list|route-map)|firewall (group interface-group LAN|ipv4|ipv6))"
+  "secondary:labsim-vyos2:172.31.1.253:^set (protocols (bgp|failover|static)|policy (prefix-list|route-map)|nat source rule 1[12]0|interfaces (pppoe|bonding bond0 vif 5[13])|firewall (group interface-group LAN|ipv4|ipv6))"
   "isp-dhcp:labsim-isp-dhcp:192.168.122.136:^set (interfaces ethernet|nat source|service dhcp-server|firewall ipv4 forward|system host-name)"
   "isp-pppoe:labsim-isp-pppoe:192.168.122.63:^set (interfaces ethernet|nat source|service pppoe-server|firewall ipv4 forward|system host-name)"
 )
@@ -29,7 +35,13 @@ SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
           -o LogLevel=ERROR -o PreferredAuthentications=password -o ConnectTimeout=5)
 live() { timeout 30 sshpass -p vyos ssh "${SSH_OPTS[@]}" "vyos@$1" \
            "/opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands" 2>/dev/null; }
-norm() { sed "s/'//g" | grep -v 'hw-id\|offload' | sort -u; }
+# `vif 53 disable` is RUNTIME state, not config drift. The generators declare it
+# on both routers as the safe resting state (only one box may hold the cloned
+# MAC), and vrrp-wan-reconcile removes it on whichever box currently holds the
+# VIP. Comparing it would therefore report drift on the master for ever, and a
+# check that always cries wolf is a check nobody reads.
+norm() { sed "s/'//g" | grep -v 'hw-id\|offload' \
+           | grep -v 'interfaces bonding bond0 vif 53 disable' | sort -u; }
 
 rc=0
 for t in "${TARGETS[@]}"; do

labsim/sim-net-config.py

@@ -37,6 +37,18 @@ K8S_NODES = ["172.31.2.11", "172.31.2.12", "172.31.2.13"]
 PEER_GROUP = "K8S"
 PFX_LIST = "K8S-SERVICE-IPS"
 RM_IN, RM_OUT = "K8S-IN", "K8S-OUT"
+
+# IPv6 unicast. Production advertises a public Gateway LoadBalancer /64 out of
+# the HE /48 (2001:470:187e:1e00::/64) and peers over VLAN 2 IPv6. The sim proves
+# the MECHANISM -- ipv6-unicast eBGP, prefix-list6, /128 host routes, ECMP -- with
+# a ULA so nothing here can leak into the real /48. Peering is over VLAN 2 v6
+# (nodes' fd00:2::1x <-> routers' fd00:2::25x, set in sim-ha-config.py /
+# k8s-up.sh), directly connected exactly as v4.
+SERVICE_CIDR_V6 = "fd61:1e00::/64"       # sim analog of prod :1e00::/64 LB pool
+K8S_NODES_V6 = ["fd00:2::11", "fd00:2::12", "fd00:2::13"]
+PEER_GROUP_V6 = "K8S6"
+PFX_LIST_V6 = "K8S-SERVICE-IPS-V6"
+RM_IN_V6, RM_OUT_V6 = "K8S-IN6", "K8S-OUT6"
 # One route per node; ECMP across all three. 4 leaves headroom for a fourth node
 # without a config change.
 MAX_PATHS = 4
@@ -80,6 +92,13 @@ WAN_DHCP_DISTANCE = 210     # NOT `no-default-route`, which blanks new_routers
 PPPOE_DISTANCE = 10         #   in the lease file, leaving failover no gateway
                             #   to install and silently handing the default
                             #   route to the backup line.
+
+# One MAC, cloned onto BOTH routers' bond0.53, mirroring production's use of the
+# retired USG's WAN2 MAC to keep its DHCP lease. The sim did not model a shared
+# MAC at all, which is exactly why bond0.53 has to stay on the config plane --
+# only VyOS config can move a MAC between boxes. Locally-administered, sim-only.
+WAN_DHCP_MAC = "02:53:10:61:00:53"
+
 SIM_LAN = "172.31.0.0/16"
 
 # The sim routers' own libvirt-NAT uplink, from before the ISP VMs existed. It
@@ -116,12 +135,51 @@ def bgp(role: str) -> list[str]:
         f"set protocols bgp peer-group {PEER_GROUP} address-family ipv4-unicast maximum-prefix {MAX_PREFIX}",
     ]
     out += [f"set protocols bgp neighbor {n} peer-group {PEER_GROUP}" for n in K8S_NODES]
+    # --- IPv6 unicast: same policy shape, over a separate v6 peer-group ---
+    # `prefix-list6` + `match ipv6 address` are the v6 spellings; `le 128` because
+    # Cilium advertises each Gateway LoadBalancer address as a /128, not the
+    # aggregate. Separate peer-group because the neighbors are v6 addresses; the
+    # export deny is the same safety property (never hand the cluster a default).
+    out += [
+        "",
+        f"set policy prefix-list6 {PFX_LIST_V6} rule 10 action permit",
+        f"set policy prefix-list6 {PFX_LIST_V6} rule 10 prefix {SERVICE_CIDR_V6}",
+        f"set policy prefix-list6 {PFX_LIST_V6} rule 10 le 128",
+        f"set policy route-map {RM_IN_V6} rule 10 action permit",
+        f"set policy route-map {RM_IN_V6} rule 10 match ipv6 address prefix-list {PFX_LIST_V6}",
+        f"set policy route-map {RM_OUT_V6} rule 10 action deny",
+        f"set protocols bgp address-family ipv6-unicast maximum-paths ebgp {MAX_PATHS}",
+        f"set protocols bgp peer-group {PEER_GROUP_V6} remote-as {CLUSTER_AS}",
+        f"set protocols bgp peer-group {PEER_GROUP_V6} address-family ipv6-unicast route-map import {RM_IN_V6}",
+        f"set protocols bgp peer-group {PEER_GROUP_V6} address-family ipv6-unicast route-map export {RM_OUT_V6}",
+        f"set protocols bgp peer-group {PEER_GROUP_V6} address-family ipv6-unicast maximum-prefix {MAX_PREFIX}",
+    ]
+    out += [f"set protocols bgp neighbor {n} peer-group {PEER_GROUP_V6}" for n in K8S_NODES_V6]
     out.append("")
     return out
 
 
-def wan(drop_scaffold: bool) -> list[str]:
-    """Dual WAN + health-checked failover. Primary router only -- see README."""
+def wan(drop_scaffold: bool, role: str = "primary") -> list[str]:
+    """Dual WAN + health-checked failover. IDENTICAL on both routers.
+
+    It used to be primary-only, on the grounds that "two PPPoE clients sharing
+    one credential against a single access concentrator is a different failure
+    mode than anything production has". That was backwards: production has
+    exactly that, and by omitting it the sim could not test the one thing most
+    likely to go wrong. The secondary having no WAN is also why it ended up with
+    zero NAT rules while the primary had ten -- the pair was not comparable.
+
+    Both routers therefore get the same WAN config. What differs is the RESTING
+    STATE, and only for the DHCP line:
+
+      bond0.53  `disable` on BOTH. Its lease is bound to a cloned MAC, and two
+                boxes claiming one MAC is the fault this whole design exists to
+                prevent. vrrp-wan-reconcile removes `disable` on the master.
+      pppoe0    enabled on BOTH, never `disable`d. `disable` unlinks
+                /etc/ppp/peers/pppoe0, which is pppd's own options file, so the
+                promotion path destroyed what it needed. Dialling is gated at
+                the systemd unit instead -- see migration/ppp-vrrp-gate.conf.
+    """
     out = [
         "# --- WAN: DHCP (primary) + PPPoE (backup), health-checked ---",
         f"set interfaces bonding bond0 vif {WAN_PPPOE_VLAN} description "
@@ -131,6 +189,13 @@ def wan(drop_scaffold: bool) -> list[str]:
         f"'WAN3 10gig-equivalent (sim ISP DHCP)'",
         f"set interfaces bonding bond0 vif {WAN_DHCP_VLAN} dhcp-options "
         f"default-route-distance {WAN_DHCP_DISTANCE}",
+        # The cloned MAC. Production clones the old USG's WAN2 MAC so the ISP
+        # keeps handing back the same lease; the sim did not model a shared MAC
+        # at all, which is precisely why bond0.53 must stay on the config plane.
+        # Modelling it lets the sim prove the lease returns to the new master.
+        f"set interfaces bonding bond0 vif {WAN_DHCP_VLAN} mac {WAN_DHCP_MAC}",
+        # Safe at rest on BOTH routers: only the VIP holder enables it.
+        f"set interfaces bonding bond0 vif {WAN_DHCP_VLAN} disable",
         f"set interfaces pppoe pppoe0 source-interface bond0.{WAN_PPPOE_VLAN}",
         f"set interfaces pppoe pppoe0 authentication username {PPPOE_USER}",
         f"set interfaces pppoe pppoe0 authentication password {PPPOE_PASS}",
@@ -364,21 +429,20 @@ def _isp_common(uplink_if: str, customer_net: str, desc: str) -> list[str]:
 
 
 def build(role: str, drop_scaffold: bool, wan_if: str, uplink_if: str) -> list[str]:
-    if role == "primary":
-        # WAN lives on the primary only. Production has WAN on both routers;
-        # the sim does not, because two PPPoE clients sharing one credential
-        # against a single access concentrator is a different failure mode than
-        # anything production has. VRRP/conntrack failover is still exercised --
-        # see README, "known gaps".
+    if role in ("primary", "secondary"):
+        # BOTH routers get the identical WAN. The sim used to give it to the
+        # primary only, reasoning that two PPPoE clients sharing one credential
+        # was "a different failure mode than anything production has" -- but
+        # that IS production, and omitting it meant the failover path was the
+        # one path the sim could not exercise. It also left the pair
+        # incomparable: ten NAT rules on one box, none on the other.
+        #
+        # Safety comes from resting state, not from asymmetry: bond0.53 is
+        # `disable`d on both (cloned MAC), pppoe0 is enabled on both but gated
+        # at the systemd unit. See wan() and migration/ppp-vrrp-gate.conf.
         return ([f"# labsim routing -- {role}", ""]
-                + bgp(role) + wan(drop_scaffold) + firewall())
-    if role == "secondary":
-        # The backup has no WAN in the sim, so it has no DHCP client to
-        # exempt -- but it gets the same policy otherwise, because after a VRRP
-        # failover it IS the router and a divergent ruleset would only be
-        # discovered during the failover.
-        return ([f"# labsim routing -- {role}", ""]
-                + bgp(role) + firewall(wan_dhcp_if=None))
+                + bgp(role) + wan(drop_scaffold, role)
+                + firewall(wan_dhcp_if=f"bond0.{WAN_DHCP_VLAN}"))
     if role == "isp-dhcp":
         return isp_dhcp(wan_if, uplink_if)
     return isp_pppoe(wan_if, uplink_if)

labsim/wan-failover-evidence/README.md

@@ -0,0 +1,85 @@
+# WAN failover evidence
+
+Captured by `labsim/labsim-pppoe-ha-test.sh`. Each directory holds the state of
+both routers and the access concentrator at the end of one test.
+
+## The number that sizes GRACE
+
+`T4` destroys the master with `virsh destroy` — no LCP Terminate, no PADT, the
+router simply ceases — and times how long until the survivor holds a PPPoE
+session. Run across every policy VyOS can express, because Vodafone's is
+unknown:
+
+Two independent runs, so these describe the AC's behaviour rather than one-offs:
+
+| `session-control` | takeover | |
+|---|---|---|
+| `replace` | 26s / 26s | accel-ppp default; the new auth kills the old session |
+| **`deny`** | **148s / 141s** | the AC refuses the survivor until its own dead-peer timer frees the dead session |
+| `disable` | 21s / 20s | no single-session enforcement at all |
+
+`deny` is the only one that matters for sizing, and `GRACE=300` in
+`migration/vrrp-wan.conf` comes from it. Session polling during that run caught
+the mechanism directly: the destroyed router's session stayed in the AC's table
+while the survivor's dial attempts appeared and were rejected — twice — before
+one finally took.
+
+**Treat 148s as a floor, not a worst case.** These are idle 2-vCPU VMs, and the
+AC shares an OVS bridge with the routers, so `virsh destroy` removes the port
+and accel-ppp sees the peer physically vanish. A real BRAS reached over DSL
+never learns our router died; it waits out its own timers, which are longer and
+not ours to know.
+
+## How these numbers were nearly wrong
+
+Until 2026-09-06 the matrix set the policy with:
+
+```sh
+isp "vbash -c 'source /opt/vyatta/etc/functions/script-template; configure; \
+     set service pppoe-server session-control $mode; commit; save; exit'"
+```
+
+That form never starts a config session. `commit` fails with
+`Invalid command: [commit]` on stderr, which `isp()` discards — so all three
+iterations ran against the accel-ppp default while printing the mode they were
+supposedly testing. `show configuration commands | grep session-control` on the
+ISP VM came back empty after a full run. The matrix reported `deny` at 25s; the
+real figure is 148s, and `GRACE` was sized against the fiction.
+
+`isp_session_control()` now drives it from a real script file, reads the value
+back, and skips the iteration rather than measure the wrong policy. A harness
+that reports coverage it does not have is worse than one that reports a failure.
+
+## The other tests
+
+| | what it proves |
+|---|---|
+| `T0-baseline` | exactly one session, held by the VIP holder |
+| `T3-clean-failover` | `force-fault` moves `pppoe0` in 20–26s |
+| `T5-tengig-down` | 10 gig down → route falls to `pppoe0`, LAN back in 5s |
+| `T8-lease-expiry` | the guard hangs up a stale lease within ~5s |
+| `T11-no-peers-file` | a blessed box with no peers file does not restart-loop |
+| `T12-holdoff-keeps-session` | a 900s flap hold-off does **not** tear down a live session |
+
+`T12` exists because it did. `ppp_dial()` checked the hold-off and returned
+before renewing `/run/vrrp-wan/may-dial`; that lease is what `vrrp-wan-guard`
+expires, so tripping the damper stopped the renewal and the guard hung up
+`pppoe0` on the master ~80s later:
+
+```
+DIAL FLAP: >=6 attempts in 600s -- holding off 900s
+GUARD: lease stale (81s > 75s) -- hanging up pppoe0
+```
+
+A damper meant to suppress repeated *dials* was destroying an established
+session instead.
+
+## Reading `check_invariant`
+
+The invariant that matters is **at most one of our routers holds `pppoe0`**.
+The AC's session count is only a proxy for it, and only while the AC enforces
+single-session — under `disable` it does not, so a destroyed router's session
+lingers and the count reads 2 with exactly one live router dialled. That is
+reported as a `WARN`, not a failure. It is not silenced, because an orphaned
+session still occupies the single slot at a real ISP: it is exactly what made
+`deny` take 148s.

labsim/wan-failover-evidence/T0-baseline/state.txt

@@ -0,0 +1,31 @@
+=== 2026-09-06T00:27:16+01:00 ===
+--- AC sessions ---
+ ifname | username |       ip       | ip6 | ip6-dp |    calling-sid    | rate-limit | state  |  uptime  | rx-bytes | tx-bytes 
+--------+----------+----------------+-----+--------+-------------------+------------+--------+----------+----------+----------
+ ppp1   | simdsl   | 198.51.100.131 |     |        | 52:54:00:4e:0b:56 |            | active | 00:04:24 | 1.1 KiB  | 204 B
+--- 172.31.1.252 ---
+vip=172.31.1.1 holds_vip=no wan_disabled=yes wan_up=no ppp_up=no ppp_active=no may_dial=no lease_age=- dropin=yes role=backup
+Sep 05 23:07:43 apitest vrrp-wan[22667]: MASTER: dialling pppoe0
+Sep 05 23:08:14 apitest vrrp-wan[24120]: MASTER: dialling pppoe0
+-- Boot c5f23399239b468c8c8b752a4305c515 --
+Sep 05 23:13:04 apitest vrrp-wan[6261]: MASTER: dialling pppoe0
+Sep 05 23:13:04 apitest vrrp-wan[6413]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:13:08 apitest vrrp-wan[7122]: bond0.53 enable commit took 4s
+-- Boot 46727d256db344d6ad4b37c142272071 --
+Sep 05 23:18:31 apitest vrrp-wan[6258]: MASTER: dialling pppoe0
+Sep 05 23:18:31 apitest vrrp-wan[6411]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:18:36 apitest vrrp-wan[7038]: bond0.53 enable commit took 5s
+--- 172.31.1.253 ---
+vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes lease_age=19 dropin=yes role=master
+pppoe0           UNKNOWN        198.51.100.131 peer 198.51.100.1/32 
+default nhid 57 dev pppoe0 proto static metric 20 
+Sep 05 23:10:44 vyos vrrp-wan[221212]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:10:48 vyos vrrp-wan[221995]: bond0.53 enable commit took 4s
+-- Boot 1f635fc6900c4661b3ac6d016d63d293 --
+Sep 05 23:16:10 vyos vrrp-wan[7037]: MASTER: dialling pppoe0
+Sep 05 23:16:11 vyos vrrp-wan[7190]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:16:15 vyos vrrp-wan[7979]: bond0.53 enable commit took 4s
+-- Boot c0e38585d7354ad6ab018800c7f3f6be --
+Sep 05 23:22:54 vyos vrrp-wan[5970]: MASTER: dialling pppoe0
+Sep 05 23:22:54 vyos vrrp-wan[6126]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:22:59 vyos vrrp-wan[6915]: bond0.53 enable commit took 5s

labsim/wan-failover-evidence/T11-no-peers-file/state.txt

@@ -0,0 +1,26 @@
+=== 2026-09-06T00:29:55+01:00 ===
+--- AC sessions ---
+ ifname | username | ip | ip6 | ip6-dp | calling-sid | rate-limit | state | uptime | rx-bytes | tx-bytes 
+--------+----------+----+-----+--------+-------------+------------+-------+--------+----------+----------
+--- 172.31.1.252 ---
+vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=no ppp_up=no ppp_active=yes may_dial=yes lease_age=1 dropin=yes role=master
+Sep 05 23:18:31 apitest vrrp-wan[6411]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:18:36 apitest vrrp-wan[7038]: bond0.53 enable commit took 5s
+-- Boot 68bb1e8d56b34d55a511a301c24270fe --
+Sep 05 23:27:40 apitest vrrp-wan[12502]: MASTER: dialling pppoe0
+Sep 05 23:27:40 apitest vrrp-wan[12654]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:27:45 apitest vrrp-wan[13361]: bond0.53 enable commit took 5s
+Sep 05 23:29:01 apitest vrrp-wan[16369]: GUARD: lease stale (200s > 75s; is vrrp-wan-reconcile.timer running?) -- hanging up pppoe0
+Sep 05 23:29:27 apitest vrrp-wan[17398]: MASTER: dialling pppoe0
+Sep 05 23:29:57 apitest vrrp-wan[18610]: MASTER: dialling pppoe0
+--- 172.31.1.253 ---
+vip=172.31.1.1 holds_vip=no wan_disabled=yes wan_up=no ppp_up=no ppp_active=no may_dial=no lease_age=- dropin=yes role=backup
+Sep 05 23:16:11 vyos vrrp-wan[7190]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:16:15 vyos vrrp-wan[7979]: bond0.53 enable commit took 4s
+-- Boot c0e38585d7354ad6ab018800c7f3f6be --
+Sep 05 23:22:54 vyos vrrp-wan[5970]: MASTER: dialling pppoe0
+Sep 05 23:22:54 vyos vrrp-wan[6126]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:22:59 vyos vrrp-wan[6915]: bond0.53 enable commit took 5s
+Sep 05 23:27:40 vyos vrrp-wan[16037]: not MASTER: hanging up pppoe0
+Sep 05 23:27:40 vyos vrrp-wan[16198]: not MASTER but bond0.53 enabled -> releasing
+Sep 05 23:27:47 vyos vrrp-wan[16840]: bond0.53 disable commit took 7s

labsim/wan-failover-evidence/T12-holdoff-keeps-session/state.txt

@@ -0,0 +1,29 @@
+=== 2026-09-06T00:31:55+01:00 ===
+--- AC sessions ---
+ ifname | username |       ip       | ip6 | ip6-dp |    calling-sid    | rate-limit | state  |  uptime  | rx-bytes | tx-bytes 
+--------+----------+----------------+-----+--------+-------------------+------------+--------+----------+----------+----------
+ ppp0   | simdsl   | 198.51.100.134 |     |        | 52:54:00:e5:95:a2 |            | active | 00:01:58 | 670 B    | 555 B
+--- 172.31.1.252 ---
+vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=no ppp_up=yes ppp_active=yes may_dial=yes lease_age=21 dropin=yes role=master
+pppoe0           UNKNOWN        198.51.100.134 peer 198.51.100.1/32 
+default nhid 74 dev pppoe0 proto static metric 20 
+Sep 05 23:18:31 apitest vrrp-wan[6411]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:18:36 apitest vrrp-wan[7038]: bond0.53 enable commit took 5s
+-- Boot 68bb1e8d56b34d55a511a301c24270fe --
+Sep 05 23:27:40 apitest vrrp-wan[12502]: MASTER: dialling pppoe0
+Sep 05 23:27:40 apitest vrrp-wan[12654]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:27:45 apitest vrrp-wan[13361]: bond0.53 enable commit took 5s
+Sep 05 23:29:01 apitest vrrp-wan[16369]: GUARD: lease stale (200s > 75s; is vrrp-wan-reconcile.timer running?) -- hanging up pppoe0
+Sep 05 23:29:27 apitest vrrp-wan[17398]: MASTER: dialling pppoe0
+Sep 05 23:29:57 apitest vrrp-wan[18610]: MASTER: dialling pppoe0
+--- 172.31.1.253 ---
+vip=172.31.1.1 holds_vip=no wan_disabled=yes wan_up=no ppp_up=no ppp_active=no may_dial=no lease_age=- dropin=yes role=backup
+Sep 05 23:16:11 vyos vrrp-wan[7190]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:16:15 vyos vrrp-wan[7979]: bond0.53 enable commit took 4s
+-- Boot c0e38585d7354ad6ab018800c7f3f6be --
+Sep 05 23:22:54 vyos vrrp-wan[5970]: MASTER: dialling pppoe0
+Sep 05 23:22:54 vyos vrrp-wan[6126]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:22:59 vyos vrrp-wan[6915]: bond0.53 enable commit took 5s
+Sep 05 23:27:40 vyos vrrp-wan[16037]: not MASTER: hanging up pppoe0
+Sep 05 23:27:40 vyos vrrp-wan[16198]: not MASTER but bond0.53 enabled -> releasing
+Sep 05 23:27:47 vyos vrrp-wan[16840]: bond0.53 disable commit took 7s

labsim/wan-failover-evidence/T3-clean-failover/state.txt

@@ -0,0 +1,30 @@
+=== 2026-09-06T00:27:58+01:00 ===
+--- AC sessions ---
+ ifname | username |       ip       | ip6 | ip6-dp |    calling-sid    | rate-limit | state  |  uptime  | rx-bytes | tx-bytes 
+--------+----------+----------------+-----+--------+-------------------+------------+--------+----------+----------+----------
+ ppp0   | simdsl   | 198.51.100.132 |     |        | 52:54:00:e5:95:a2 |            | active | 00:00:18 | 514 B    | 204 B
+--- 172.31.1.252 ---
+vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes lease_age=7 dropin=yes role=master
+pppoe0           UNKNOWN        198.51.100.132 peer 198.51.100.1/32 
+default via 203.0.113.1 dev bond0.53 proto failover metric 1 
+Sep 05 23:13:04 apitest vrrp-wan[6413]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:13:08 apitest vrrp-wan[7122]: bond0.53 enable commit took 4s
+-- Boot 46727d256db344d6ad4b37c142272071 --
+Sep 05 23:18:31 apitest vrrp-wan[6258]: MASTER: dialling pppoe0
+Sep 05 23:18:31 apitest vrrp-wan[6411]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:18:36 apitest vrrp-wan[7038]: bond0.53 enable commit took 5s
+-- Boot 68bb1e8d56b34d55a511a301c24270fe --
+Sep 05 23:27:40 apitest vrrp-wan[12502]: MASTER: dialling pppoe0
+Sep 05 23:27:40 apitest vrrp-wan[12654]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:27:45 apitest vrrp-wan[13361]: bond0.53 enable commit took 5s
+--- 172.31.1.253 ---
+vip=172.31.1.1 holds_vip=no wan_disabled=yes wan_up=no ppp_up=no ppp_active=no may_dial=no lease_age=- dropin=yes role=backup
+Sep 05 23:16:11 vyos vrrp-wan[7190]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:16:15 vyos vrrp-wan[7979]: bond0.53 enable commit took 4s
+-- Boot c0e38585d7354ad6ab018800c7f3f6be --
+Sep 05 23:22:54 vyos vrrp-wan[5970]: MASTER: dialling pppoe0
+Sep 05 23:22:54 vyos vrrp-wan[6126]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:22:59 vyos vrrp-wan[6915]: bond0.53 enable commit took 5s
+Sep 05 23:27:40 vyos vrrp-wan[16037]: not MASTER: hanging up pppoe0
+Sep 05 23:27:40 vyos vrrp-wan[16198]: not MASTER but bond0.53 enabled -> releasing
+Sep 05 23:27:47 vyos vrrp-wan[16840]: bond0.53 disable commit took 7s

labsim/wan-failover-evidence/T4-hard-failover-deny/state.txt

@@ -0,0 +1,19 @@
+=== 2026-09-06T00:37:11+01:00 ===
+--- AC sessions ---
+ ifname | username |       ip       | ip6 | ip6-dp |    calling-sid    | rate-limit | state  |  uptime  | rx-bytes | tx-bytes 
+--------+----------+----------------+-----+--------+-------------------+------------+--------+----------+----------+----------
+ ppp0   | simdsl   | 198.51.100.136 |     |        | 52:54:00:e5:95:a2 |            | active | 00:00:13 | 438 B    | 204 B
+--- 172.31.1.252 ---
+vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes lease_age=26 dropin=yes role=master
+pppoe0           UNKNOWN        198.51.100.136 peer 198.51.100.1/32 
+default via 203.0.113.1 dev bond0.53 proto failover metric 1 
+Sep 05 23:27:40 apitest vrrp-wan[12654]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:27:45 apitest vrrp-wan[13361]: bond0.53 enable commit took 5s
+Sep 05 23:29:01 apitest vrrp-wan[16369]: GUARD: lease stale (200s > 75s; is vrrp-wan-reconcile.timer running?) -- hanging up pppoe0
+Sep 05 23:29:27 apitest vrrp-wan[17398]: MASTER: dialling pppoe0
+Sep 05 23:29:57 apitest vrrp-wan[18610]: MASTER: dialling pppoe0
+-- Boot 6efc3c9ba47f455e9668454ee6c2fc37 --
+Sep 05 23:34:48 apitest vrrp-wan[6266]: MASTER: dialling pppoe0
+Sep 05 23:34:49 apitest vrrp-wan[6422]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:34:54 apitest vrrp-wan[7130]: bond0.53 enable commit took 5s
+--- 172.31.1.253 ---

labsim/wan-failover-evidence/T4-hard-failover-disable/state.txt

@@ -0,0 +1,20 @@
+=== 2026-09-06T00:39:29+01:00 ===
+--- AC sessions ---
+ ifname | username |       ip       | ip6 | ip6-dp |    calling-sid    | rate-limit | state  |  uptime  | rx-bytes | tx-bytes 
+--------+----------+----------------+-----+--------+-------------------+------------+--------+----------+----------+----------
+ ppp0   | simdsl   | 198.51.100.136 |     |        | 52:54:00:e5:95:a2 |            | active | 00:02:31 | 438 B    | 204 B    
+ ppp1   | simdsl   | 198.51.100.137 |     |        | 52:54:00:4e:0b:56 |            | active | 00:00:26 | 1.0 KiB  | 204 B
+--- 172.31.1.252 ---
+--- 172.31.1.253 ---
+vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes lease_age=5 dropin=yes role=master
+pppoe0           UNKNOWN        198.51.100.137 peer 198.51.100.1/32 
+default nhid 60 dev pppoe0 proto static metric 20 
+Sep 05 23:27:40 vyos vrrp-wan[16198]: not MASTER but bond0.53 enabled -> releasing
+Sep 05 23:27:47 vyos vrrp-wan[16840]: bond0.53 disable commit took 7s
+Sep 05 23:32:26 vyos vrrp-wan[23806]: MASTER: dialling pppoe0
+Sep 05 23:32:26 vyos vrrp-wan[23956]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:32:31 vyos vrrp-wan[24749]: bond0.53 enable commit took 5s
+-- Boot 4b0355878444477fb92738339f85843d --
+Sep 05 23:39:03 vyos vrrp-wan[5966]: MASTER: dialling pppoe0
+Sep 05 23:39:04 vyos vrrp-wan[6120]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:39:09 vyos vrrp-wan[6914]: bond0.53 enable commit took 5s

labsim/wan-failover-evidence/T4-hard-failover-replace/state.txt

@@ -0,0 +1,18 @@
+=== 2026-09-06T00:32:54+01:00 ===
+--- AC sessions ---
+ ifname | username |       ip       | ip6 | ip6-dp |    calling-sid    | rate-limit | state  |  uptime  | rx-bytes | tx-bytes 
+--------+----------+----------------+-----+--------+-------------------+------------+--------+----------+----------+----------
+ ppp0   | simdsl   | 198.51.100.135 |     |        | 52:54:00:4e:0b:56 |            | active | 00:00:30 | 438 B    | 204 B
+--- 172.31.1.252 ---
+--- 172.31.1.253 ---
+vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes lease_age=19 dropin=yes role=master
+pppoe0           UNKNOWN        198.51.100.135 peer 198.51.100.1/32 
+default nhid 77 dev pppoe0 proto static metric 20 
+Sep 05 23:22:54 vyos vrrp-wan[6126]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:22:59 vyos vrrp-wan[6915]: bond0.53 enable commit took 5s
+Sep 05 23:27:40 vyos vrrp-wan[16037]: not MASTER: hanging up pppoe0
+Sep 05 23:27:40 vyos vrrp-wan[16198]: not MASTER but bond0.53 enabled -> releasing
+Sep 05 23:27:47 vyos vrrp-wan[16840]: bond0.53 disable commit took 7s
+Sep 05 23:32:26 vyos vrrp-wan[23806]: MASTER: dialling pppoe0
+Sep 05 23:32:26 vyos vrrp-wan[23956]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:32:31 vyos vrrp-wan[24749]: bond0.53 enable commit took 5s

labsim/wan-failover-evidence/T5-tengig-down/state.txt

@@ -0,0 +1,30 @@
+=== 2026-09-06T00:28:31+01:00 ===
+--- AC sessions ---
+ ifname | username |       ip       | ip6 | ip6-dp |    calling-sid    | rate-limit | state  |  uptime  | rx-bytes | tx-bytes 
+--------+----------+----------------+-----+--------+-------------------+------------+--------+----------+----------+----------
+ ppp0   | simdsl   | 198.51.100.132 |     |        | 52:54:00:e5:95:a2 |            | active | 00:00:52 | 682 B    | 372 B
+--- 172.31.1.252 ---
+vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=yes ppp_up=yes ppp_active=yes may_dial=yes lease_age=7 dropin=yes role=master
+pppoe0           UNKNOWN        198.51.100.132 peer 198.51.100.1/32 
+default nhid 58 dev pppoe0 proto static metric 20 
+Sep 05 23:13:04 apitest vrrp-wan[6413]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:13:08 apitest vrrp-wan[7122]: bond0.53 enable commit took 4s
+-- Boot 46727d256db344d6ad4b37c142272071 --
+Sep 05 23:18:31 apitest vrrp-wan[6258]: MASTER: dialling pppoe0
+Sep 05 23:18:31 apitest vrrp-wan[6411]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:18:36 apitest vrrp-wan[7038]: bond0.53 enable commit took 5s
+-- Boot 68bb1e8d56b34d55a511a301c24270fe --
+Sep 05 23:27:40 apitest vrrp-wan[12502]: MASTER: dialling pppoe0
+Sep 05 23:27:40 apitest vrrp-wan[12654]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:27:45 apitest vrrp-wan[13361]: bond0.53 enable commit took 5s
+--- 172.31.1.253 ---
+vip=172.31.1.1 holds_vip=no wan_disabled=yes wan_up=no ppp_up=no ppp_active=no may_dial=no lease_age=- dropin=yes role=backup
+Sep 05 23:16:11 vyos vrrp-wan[7190]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:16:15 vyos vrrp-wan[7979]: bond0.53 enable commit took 4s
+-- Boot c0e38585d7354ad6ab018800c7f3f6be --
+Sep 05 23:22:54 vyos vrrp-wan[5970]: MASTER: dialling pppoe0
+Sep 05 23:22:54 vyos vrrp-wan[6126]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:22:59 vyos vrrp-wan[6915]: bond0.53 enable commit took 5s
+Sep 05 23:27:40 vyos vrrp-wan[16037]: not MASTER: hanging up pppoe0
+Sep 05 23:27:40 vyos vrrp-wan[16198]: not MASTER but bond0.53 enabled -> releasing
+Sep 05 23:27:47 vyos vrrp-wan[16840]: bond0.53 disable commit took 7s

labsim/wan-failover-evidence/T8-lease-expiry/state.txt

@@ -0,0 +1,27 @@
+=== 2026-09-06T00:29:16+01:00 ===
+--- AC sessions ---
+ ifname | username | ip | ip6 | ip6-dp | calling-sid | rate-limit | state | uptime | rx-bytes | tx-bytes 
+--------+----------+----+-----+--------+-------------+------------+-------+--------+----------+----------
+--- 172.31.1.252 ---
+vip=172.31.1.1 holds_vip=yes wan_disabled=no wan_up=no ppp_up=no ppp_active=no may_dial=no lease_age=- dropin=yes role=master
+Sep 05 23:13:08 apitest vrrp-wan[7122]: bond0.53 enable commit took 4s
+-- Boot 46727d256db344d6ad4b37c142272071 --
+Sep 05 23:18:31 apitest vrrp-wan[6258]: MASTER: dialling pppoe0
+Sep 05 23:18:31 apitest vrrp-wan[6411]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:18:36 apitest vrrp-wan[7038]: bond0.53 enable commit took 5s
+-- Boot 68bb1e8d56b34d55a511a301c24270fe --
+Sep 05 23:27:40 apitest vrrp-wan[12502]: MASTER: dialling pppoe0
+Sep 05 23:27:40 apitest vrrp-wan[12654]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:27:45 apitest vrrp-wan[13361]: bond0.53 enable commit took 5s
+Sep 05 23:29:01 apitest vrrp-wan[16369]: GUARD: lease stale (200s > 75s; is vrrp-wan-reconcile.timer running?) -- hanging up pppoe0
+--- 172.31.1.253 ---
+vip=172.31.1.1 holds_vip=no wan_disabled=yes wan_up=no ppp_up=no ppp_active=no may_dial=no lease_age=- dropin=yes role=backup
+Sep 05 23:16:11 vyos vrrp-wan[7190]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:16:15 vyos vrrp-wan[7979]: bond0.53 enable commit took 4s
+-- Boot c0e38585d7354ad6ab018800c7f3f6be --
+Sep 05 23:22:54 vyos vrrp-wan[5970]: MASTER: dialling pppoe0
+Sep 05 23:22:54 vyos vrrp-wan[6126]: MASTER with bond0.53 disabled -> enabling
+Sep 05 23:22:59 vyos vrrp-wan[6915]: bond0.53 enable commit took 5s
+Sep 05 23:27:40 vyos vrrp-wan[16037]: not MASTER: hanging up pppoe0
+Sep 05 23:27:40 vyos vrrp-wan[16198]: not MASTER but bond0.53 enabled -> releasing
+Sep 05 23:27:47 vyos vrrp-wan[16840]: bond0.53 disable commit took 7s

migration/PPPOE-HA.md

@@ -0,0 +1,360 @@
+# PPPoE high availability
+
+Proven in labsim. **Deployed to production 2026-09-06** — mechanism on both
+routers, `pppoe0 disable` removed from vyos002, vyos002 out of FAULT and in
+BACKUP, Pulumi model merged, and a controlled failover drill passed
+(takeover 52s, failback 36s). `vyos:verify` reports both routers in sync.
+
+## What it does
+
+One consumer ISP account, two routers. The 10 gig lease is bound to a cloned MAC
+(`f0:9f:c2:12:9b:4f`, the retired USG's) and the Vodafone line to a single
+credential, so neither may be live on both boxes. The WAN follows VRRP
+mastership — but the two halves use different control planes, and that is the
+whole design:
+
+| | plane | why |
+|---|---|---|
+| `bond0.53` (10 gig) | VyOS **config** (`disable`) | only config can move a MAC |
+| `pppoe0` (Vodafone) | **systemd** unit gate | see below |
+
+## Why PPPoE cannot live on the config plane
+
+`interfaces_pppoe.py` treats `disable` and `delete` identically: both **unlink
+`/etc/ppp/peers/pppoe0`**, call `PPPoEIf.remove()` (withdrawing the FRR default
+route) and stop the unit. That path is pppd's own options file
+(`ExecStart=/usr/sbin/pppd call %I`), so the resting state destroyed exactly what
+the promotion path needed. `ppp@pppoe0` then restart-looped against the missing
+file — 47 restarts observed, zero sessions at the access concentrator — and
+never tripped systemd's limiter, because `RestartSec=5s` against the default
+10s/5-burst window is only two restarts per interval.
+
+It also made op-mode `connect interface pppoe0` unusable (it refuses without the
+peers file), and put every failover behind a priority-322 commit where one
+unrelated invalid node fails the whole thing.
+
+## The gate
+
+`pppoe0` is configured identically and **enabled on both** routers, so the peers
+file always exists. Dialling is gated by
+`/etc/systemd/system/ppp@pppoe0.service.d/10-vrrp-wan-gate.conf`:
+
+```ini
+ConditionPathExists=/run/vrrp-wan/may-dial
+ConditionPathExists=/etc/ppp/peers/pppoe0
+StartLimitIntervalSec=600
+StartLimitBurst=6
+```
+
+`/run` is tmpfs, so the gate is shut at boot and neither box can dial before VRRP
+has decided. **This is load-bearing, not a nicety:** with the node enabled,
+`interfaces_pppoe.py` restarts ppp on *every* commit touching the pppoe subtree
+when the daemon is not running — so the backup actively tries to dial whenever
+anything commits (`pulumi up`, a hand commit, the boot-time config load). The
+gate is the only thing making that a no-op, which is why `vrrp-wan-reconcile`
+**refuses to bless a box whose drop-in is missing**: `/etc` is per-image, so a
+VyOS upgrade silently removes the protection, and failing closed turns that into
+"PPPoE never dials" rather than "both routers dial".
+
+`may-dial` is a **lease, not a flag**. `ConditionPathExists` is evaluated at
+start only — it can prevent a dial, never revoke one. `vrrp-wan-reconcile`
+renews it every 30s; `vrrp-wan-guard` runs every 5s and only ever revokes, on
+either "I do not hold the VIP" or "the lease is stale".
+
+## Measured in labsim
+
+| | |
+|---|---|
+| Clean failover (`force-fault`) | `pppoe0` moves in **20-26s**, reproducible |
+| 10 gig down → PPPoE | route falls to `pppoe0`; LAN back online in **5s** |
+| Stale lease | guard hangs up within ~5s |
+| Missing peers file | `NRestarts=0` — no loop |
+| Flap holdoff vs live session | session survives a forged 900s holdoff |
+| Invariant | never more than one router dialled, in any run |
+
+On the last row, precisely: the AC *did* report two `simdsl` sessions during the
+`session-control=disable` run — one live, one orphaned from the router that had
+just been destroyed, which that policy does not clean up. Only one live router
+was ever dialled. The count is a proxy for the real invariant and only a valid
+one while the AC enforces single-session, so it is reported as a `WARN` rather
+than silenced: an orphaned session still occupies the single slot at a real ISP,
+and that is exactly what made `deny` take 141–148s.
+
+## Two failure modes found by running it, not by reading it
+
+**The flap damper tore down a healthy WAN.** `ppp_dial()` checked the hold-off
+and returned *before* renewing `may-dial`. That file is a lease the guard
+expires after `LEASE_TTL`, so tripping the damper stopped the renewal and the
+guard hung up `pppoe0` **on the master** ~80s later:
+
+```
+DIAL FLAP: >=6 attempts in 600s -- holding off 900s
+GUARD: lease stale (81s > 75s) -- hanging up pppoe0
+```
+
+A damper meant to suppress repeated *dials* was destroying an established
+session instead. An active session now renews the lease and returns before
+every other check; everything below only decides whether to start a **new**
+session. T12 is the regression test.
+
+**A missing peers file is silent.** `/etc/ppp/peers/pppoe0` is both pppd's
+options file and the gate's second condition, and it is only written by a commit
+that touches the pppoe subtree. Without it systemd logs
+`skipped because of an unmet condition check` exactly once and then nothing —
+a router that cannot dial at all looks identical to a healthy backup.
+`ppp_dial()` now says so on every tick, and distinguishes the two causes:
+configured-but-not-rendered (re-commit the subtree) versus no `pppoe0` in the
+config at all.
+
+The second cause is the one to watch in production: **a commit that was never
+`save`d reverts on reboot and takes `pppoe0` with it.** That is exactly how the
+sim secondary lost its WAN and spent hours looking like an ISP problem. After
+any hand commit to the pppoe subtree, `save` — or the next reboot produces a
+standby that can never take over.
+
+## Deploying — steps 1–9 done 2026-09-06
+
+1. `sudo /config/vyos-known-good save` on both.
+2. `migration/vrrp-wan-install --vip 192.168.1.1 --host vyos@10.0.1.253` then the
+   same for `.252`. Then `--check` on both. **No config change yet** — verify
+   nothing dials.
+3. Confirm `/config/wan-secrets` is present and identical on both.
+4. **vyos002 first** (the non-master), on its own commit — `interfaces pppoe` is
+   priority 322 and one bad node fails everything:
+   `delete interfaces pppoe pppoe0 disable`, `commit-confirm 10`.
+5. Verify vyos002 did **not** dial — check on the wire, not from state:
+   `sudo tcpdump -i bond0.51 -nn pppoed` should show no PADI. Then `confirm`; `save`.
+6. vyos001: nothing to change; it already has `pppoe0` enabled.
+7. Confirm `vif 53 disable` is in **both** `config.boot`s. vyos001's lacked it;
+   fixed with `migration/vif53-pin-boot-disable` — see below.
+8. **Done 2026-09-06** (`kubernetes-deployment@45033dd`, on `main`). Both
+   overrides merged, transition-scripts applied to both boxes by hand rather
+   than left as drift, and `vyos:verify` is clean: 533 / 512 nodes, zero drift.
+   The staging file `migration/pulumi-override-pppoe-gated.json` is kept as the
+   record of why the ordering mattered. It was staged, unapplied, on purpose: another agent runs `pulumi up` on that repo, so
+   merging it *is* a production change made by someone else at a time you do not
+   choose. Removing `pppoe0 disable` from vyos002 before the gate exists there
+   lets it dial on the next commit and take the single Vodafone session off
+   vyos001. Run `npm run vyos:export && npm run vyos:render` first so the model
+   follows whichever router actually holds the WAN.
+9. Add the drop-in re-install to the VyOS image-upgrade runbook.
+   **Done** — `migration/VYOS-IMAGE-UPGRADE.md`. An upgrade replaces `/etc` and
+   so removes the gate; the reconciler fails closed, giving "PPPoE never
+   dials" rather than "both routers dial".
+
+Step 4 is the one that matters most and is worth stopping on. vyos002 has been
+in **FAULT on all six groups for over three days** — verified again while
+writing this, alongside vyos001 holding `192.168.1.1` on `bond0.1` and
+`pppoe0` up on `83.106.5.72`. Until vyos002 reaches BACKUP there is no standby
+at all: if vyos001 died today nothing would pick up the gateway VIPs. The
+existing `vrrp-health-check-wan-present` override predicted exactly this in its
+own reason text — *"with the primary genuinely dead the secondary stays FAULT
+and nothing holds the gateway. The fix for that is WAN-follows-master, which is
+a separate change."* This is that change.
+
+## Proven in production — controlled drill, 2026-09-06
+
+`migration/wan-drill` force-faulted vyos001 and timed a real failover:
+
+```
+TAKEOVER OK: vyos002 held the VIP and reached the internet in 52s
+FAILBACK OK: 36s
+```
+
+vyos002 took the VIPs at t+18s and had **both** WANs by t+52s. Failback put
+vyos001 back with a WAN in 36s. Total interruption ≈ 88s across two deliberate
+transitions.
+
+**The cloned-MAC lease transfers.** This was the largest untested item in the
+whole design — whether the 10 gig ISP would re-issue `87.192.101.48` to
+`f0:9f:c2:12:9b:4f` arriving on a different switch port. It did, same address,
+within the takeover window. That risk is now closed.
+
+**Vodafone did not refuse the re-dial**, so its `session-control` behaves like
+`replace` rather than the hostile `deny`. `GRACE=300` was sized against the
+sim's 148s `deny` case and is therefore comfortable — but it should stay where
+it is, because one drill on one evening does not establish the ISP's policy
+under all conditions.
+
+**Vodafone hands out a different IPv4 on every dial**: `83.106.5.72` →
+`90.251.153.180` (vyos002) → `90.251.142.103` (vyos001, after failback).
+Nothing may be pinned to the PPPoE address. The HE IPv6 tunnel is pinned to
+`87.192.101.48`, which is the **10 gig** (`bond0.53`) and stable across
+failover. Anything added later that hardcodes a WAN IP must use the 10 gig one,
+not `pppoe0`'s.
+
+> **CORRECTED 2026-09-06.** This paragraph originally continued "so `tun0`
+> survived untouched and IPv6 stayed up at 15.5ms." **That conclusion was
+> wrong, and it should never have been recorded as a result.** The premise is
+> right — the endpoint address is stable — but it does not follow. During
+> takeover the reconciler disables `bond0.53` on the demoted box, so
+> `87.192.101.48` *leaves vyos001 and appears on vyos002*, and vyos002 has no
+> `tun0` at all: no tunnel, no `he-tunnel-follow`, no `/config/he-secrets`, no
+> VLAN 9 prefix, no `route6 ::/0`. Inbound protocol 41 from HE lands on a router
+> with nothing to decapsulate it. vyos001's own journal for the drill window
+> reads `08:36:01 he-tunnel-follow: no default route; refusing to guess`.
+>
+> The reading was taken either side of the window, not through it — `wan-drill`
+> contained **no IPv6 check of any kind**, and neither did any other part of the
+> mechanism. That is now fixed: the drill probes IPv6 in both timing loops and
+> asserts that a router-level failover makes **zero** HE API calls. Until a
+> drill produces that figure, the IPv6 behaviour of a failover is *unmeasured*,
+> not "fine".
+>
+> The gap itself was real: **IPv6 was single-homed on vyos001 while the WAN
+> beneath it was HA.** **Closed and measured 2026-09-06** — see the drill below.
+
+Production takeover (52s) is about twice the sim's `replace` figure (26s), which
+is the expected direction: the VP2440s commit under kea, BGP and conntrack while
+the sim routers are idle.
+
+## IPv6 follows the WAN — measured, 2026-09-06
+
+The second drill of the day, run after IPv6-follows-master was deployed. This is
+the first time the IPv6 behaviour of a failover has been a **measurement** rather
+than an assertion:
+
+```
+TAKEOVER OK: vyos002 held the VIP and reached the internet in 37s
+IPv6 followed in 37s (v4 37s, gap 0s)
+FAILBACK OK in 32s
+IPv6 back in 44s
+tun0 src : 87.192.101.48 -> 87.192.101.48   OK: unchanged across the drill
+HE updates from vyos001: 0
+HE updates from vyos002: 0
+```
+
+Full log: `migration/drill-evidence/wan-drill-2026-09-06-ipv6.txt`.
+
+**Zero HE API calls across a full takeover and failback.** This is the invariant
+the design rests on and it now has evidence: the 10 gig lease follows the cloned
+MAC, so the tunnel endpoint is the *same address* on whichever router holds the
+WAN, and there is nothing to tell Hurricane Electric. Only the within-box fall
+back to PPPoE needs an HE update.
+
+**IPv6 is no longer the laggard, but the two directions are not symmetric.** On
+takeover it arrived in the same 5s sample as IPv4; on failback it trailed by 12s.
+That asymmetry is the reconciler's tick, not a fault: on promotion it enables
+`bond0.53` first, and `v6_take` only raises the tunnel once the source address
+actually exists, so it can land on the following 30s tick. Bound is one tick.
+Note the sampling granularity — the drill polls every 5s, so "gap 0s" means
+"within the same sample", not "simultaneous".
+
+Takeover was 37s here against 52s in the morning drill. Do not read that as an
+IPv6 improvement; it is the same IPv4 mechanism on a different run, and the
+morning figure included the first-ever cloned-MAC lease transfer.
+
+**Vodafone confirmed the every-dial-a-new-address behaviour again**: `pppoe0`
+came back as `90.251.152.236`, having been `90.251.142.103` before the drill.
+
+## config.boot pins `vif 53 disable` on both — fixed 2026-09-06
+
+The convention is that **both** `config.boot`s hold `vif 53 disable`, so a reboot
+in any order comes up unable to claim the cloned MAC and the reconciler enables
+it on whichever box holds the VIP. vyos001's did not; its `config.boot` predated
+this work.
+
+There is no clean way to express "boot disabled, run enabled" in VyOS: **`save`
+writes the RUNNING config, not the candidate.** Setting the node, saving and
+discarding was tested in labsim and `config.boot` came back *without* `disable`,
+the WAN untouched. So the node must genuinely be disabled, saved, and
+re-enabled. `migration/vif53-pin-boot-disable` does exactly that, is idempotent,
+and no-ops on a box that already has it.
+
+Cost, measured on vyos001: a **32s** window (10s in the sim — production commits
+under kea/BGP/conntrack are slower), `bond0.53` re-leased `87.192.101.48` 5s
+after re-enable, and `vyos-failover` restored the primary route about a minute
+later:
+
+```
+09:32:23  ip route del 0.0.0.0/0 ... dev bond0.53
+09:32:32  Check fail for route 0.0.0.0/0 interface "bond0.53"
+09:33:23  ip route add 0.0.0.0/0 via 87.192.96.1 dev bond0.53 metric 1 proto failover
+```
+
+The house rode `pppoe0` for that minute rather than losing the internet, which
+is the T5 path working. Note the shape of that recovery before reading a fresh
+`ip route show` as a regression: for ~60s after the bounce the default really is
+on `pppoe0`, because `vyos-failover` only re-adds the `bond0.53` route once its
+probes pass again.
+
+**A race worth knowing about.** The first sim run collided with
+`vrrp-wan-reconcile`'s own commit — *"Configuration system temporarily locked due
+to another commit in progress"* — and the `save` landed while the **re-enable did
+not**, leaving the master with its 10 gig down. The script now takes the
+reconciler's `/run/vrrp-wan.lock` (which the reconciler skips a tick rather than
+block on), with `9>&-` so the config session's unionfs child cannot inherit it.
+Even the bad run ended correctly — the reconciler logged *"MASTER with bond0.53
+disabled -> enabling"* and repaired it in 4s — so a half-completed run is
+survivable by design. The script no longer leans on that, and verifies the
+re-enable rather than reporting a success it did not achieve.
+
+**Rollback**, from either box: `set interfaces pppoe pppoe0 disable` on both and
+`rm /run/vrrp-wan/may-dial`. That restores today's behaviour exactly.
+
+## What the sim cannot prove
+
+- **Vodafone's `session-control`.** The matrix now brackets it properly.
+  Destroying the master and timing the survivor's session:
+
+  | policy | takeover |
+  |---|---|
+  | `replace` (accel-ppp default) | 26s / 26s |
+  | **`deny`** (hostile) | **148s / 141s** |
+  | `disable` | 21s / 20s |
+
+  `deny` is the sizing case: the AC refuses the survivor until its own
+  dead-peer timer frees the dead session, and the poller caught two dial
+  attempts being rejected before one succeeded. `GRACE` is set from that — see
+  `migration/vrrp-wan.conf`. This still cannot tell you which policy Vodafone
+  runs, and account rate-limiting or lockout on repeated dials has no sim
+  analogue at all; the flap damper (6 dials / 600s → 15 min hold-off) exists
+  for that.
+
+  Treat the 148s as a floor rather than a worst case. These are idle 2-vCPU
+  VMs, and the AC shares an OVS bridge with the routers, so `virsh destroy`
+  removes the port and accel-ppp sees the peer physically vanish. A real BRAS
+  reached over DSL does not learn that our router died — it waits out its own
+  timers, which are longer and not ours to know.
+
+  Worth knowing how close this came to being missed: until 2026-09-06 the
+  matrix set the policy with `vbash -c 'source script-template; configure;
+  ...; commit'`, which never starts a config session. `commit` failed to
+  stderr, the helper discarded it, and all three iterations ran against the
+  default while printing the mode they were supposedly testing. It reported
+  `deny` at 25s. The real figure is 148s.
+- **Whether Vodafone honours our LCP Terminate / PADT** on a graceful stop.
+- ~~**The cloned-MAC lease.**~~ **Answered 2026-09-06**: the drill moved it and
+  the ISP re-issued `87.192.101.48` to `f0:9f:c2:12:9b:4f` on vyos002's port
+  within the takeover window. See "Proven in production" above.
+- ~~**Whether VyOS can dial Vodafone at all.**~~ **Answered 2026-09-06**: both
+  routers dialled successfully during the drill. MTU/MSS under sustained load
+  is still unmeasured, and Vodafone hands out a different IPv4 every dial.
+- **Timing under load.** The sim routers are idle 2-vCPU VMs; commit latency on
+  the VP2440s under kea + BGP + conntrack will be worse, and commit latency is
+  the dominant term in the `bond0.53` half of a failover.
+
+## How the model handles the asymmetry — resolved
+
+An earlier draft of this file said an override "must assert `vif 53 disable` on
+**both** routers". **That advice was wrong and has been removed**; do not
+reintroduce it. `vif 53 disable` is *runtime* state owned by
+`vrrp-wan-reconcile`, keyed on who holds the management VIP, so pinning it in
+the model would fight the reconciler on every apply and would briefly disable
+the live master's 10 gig each time.
+
+What is actually done, and why it is safe:
+
+- **Runtime (Pulumi): follow reality.** Run `npm run vyos:export && npm run
+  vyos:render` immediately before any `pulumi up` touching vyos. Whichever
+  router currently holds the WAN keeps it; the apply is a no-op on that node.
+  There is deliberately **no** override for `vif 53 disable`.
+- **Boot (`config.boot`): hardcode safe.** Both routers pin `vif 53 disable`,
+  so a reboot in any order comes up unable to claim the cloned MAC and the
+  reconciler enables it on whoever holds the VIP. See the section above.
+- **Install time (PXE, nothing to follow).** `migration/vyos-mode-delta.py`
+  emits `vif 53 disable` for a box with no WAN, and deliberately does *not*
+  emit `pppoe pppoe0 disable`.
+
+Verified 2026-09-06: `vyos:verify` reports both routers in sync, 533 and 512
+nodes, zero drift.

migration/RECOVERY-CARD-schema-v2.md

@@ -0,0 +1,176 @@
+# Recovery card — v2 layout migration, one node at a time
+
+Print or keep open. **A node is being wiped. If the cluster goes down there is
+no Claude and possibly no internet.** Everything you need is on this page.
+
+---
+
+## The one thing that matters
+
+**`k3s etcdctl` is NOT a command.** It does not exist (`No help topic for
+'etcdctl'`). You need the real binary plus k3s's certs. Paste this once on
+whichever server you are using:
+
+```
+e() {
+  local b=/var/lib/rancher/k3s/server/tls/etcd
+  ETCDCTL_API=3 etcdctl --cacert=$b/server-ca.crt \
+    --cert=$b/server-client.crt --key=$b/server-client.key \
+    --endpoints=https://127.0.0.1:2379 "$@"
+}
+```
+
+(A function, not an alias — aliases do not expand inside scripts. Verified
+working on worker0.)
+
+If `etcdctl: command not found`, install it: `dnf install -y etcd` — which needs
+working DNS and a reachable mirror, so do not count on it here. Nodes
+reinstalled with the current kickstart already have it (`etcd` is in the package
+set for k3s server roles precisely so this step is never needed).
+
+Then:
+
+```
+e member list -w table
+```
+
+There are **3 etcd members. Quorum is 2. You have no margin.**
+
+Two naming traps, both of which make a correct-looking command match nothing:
+
+- **Kubernetes node names are FQDNs.** `worker0-k8s0.ad.itaz.eu`, never
+  `worker0`. The same name is used by `kubectl` and by `nodes.longhorn.io`.
+  Get it from `kubectl get nodes`; do not type it.
+- **etcd member names add a random suffix** on top of that:
+  `worker0-k8s0.ad.itaz.eu-e26aacc5`. Match on the prefix.
+
+One node down is survivable. Two is a dead API server and a dead cluster. So:
+**one node at a time**, and never start the next while anything is degraded.
+
+Reach the servers directly:
+
+```
+ssh root@192.168.8.23     # worker0   (Fedora x86)
+ssh root@192.168.8.25     # worker2   (Fedora aarch64)
+ssh root@192.168.8.13     # worker1   (Asahi aarch64 — migrate LAST)
+```
+
+---
+
+## Before you touch anything
+
+```
+kubectl get nodes                                        # all Ready
+e member list                                            # exactly 3, all started
+e endpoint health                                        # healthy
+kubectl -n longhorn-system get volumes.longhorn.io | grep -v healthy   # nothing
+```
+
+All four clean, or do not start.
+
+---
+
+## Order
+
+**worker0 → worker2 → worker1.** worker1 is last: vendor-owned partition table,
+4.6 G root, undersized rancher LV, and a hand-made bind mount a reinstall
+destroys.
+
+---
+
+## The loop
+
+```
+# 1. Ask first. It refuses in SECONDS if it cannot finish.
+labctl app k3s evacuate <node>
+```
+
+**If it refuses, it is right.** It means either no node can legally take a
+replica, or the disk has no room. Do not force it. Add the N+1 node first.
+
+```
+# 2. Node is empty. Prove the data is fine BEFORE destroying anything.
+kubectl -n longhorn-system get replicas.longhorn.io -o wide | grep <node>   # none
+#    restart something using a Longhorn PVC; check it reads back.
+
+# 3. Irreversible from here.
+ssh <node> systemctl stop k3s
+e member remove <id>             # member names are "<node>-<8 hex>"
+kubectl delete node <node>
+e member list                    # 2, both healthy  <-- CHECK THIS
+kubectl get --raw /readyz        # 'ok' from each survivor
+
+# 4. Wipe.
+labctl provision reprovision <node>
+
+# 5. Rejoin AGAINST A SURVIVOR, pinned to the cluster's version.
+labctl app k3s install <node> --role k8s-combo \
+  --k3s-server https://<survivor-ip>:6443 --k3s-token <token> \
+  --k3s-version <what the others run>     # kubectl get nodes -o wide
+e member list                    # EXACTLY 3  <-- CHECK THIS
+
+# 6. Back in service, then wait for the rebuild.
+labctl app k3s admit <node>
+kubectl -n longhorn-system get volumes.longhorn.io    # all healthy
+```
+
+---
+
+## When something is wrong
+
+| Symptom | Do this |
+|---|---|
+| Evacuate refused | Believe it. Read the named volume. Add capacity or a node. |
+| Evacuate timed out | It already cancelled itself and restored scheduling. If it says it could not: `labctl app k3s admit <node>` |
+| Aborted midway, node cordoned | `labctl app k3s admit <node>` |
+| Rejoined node hosts no replicas | It has no Longhorn disk. `labctl app k3s admit <node>` |
+| Replicas land but stay `stopped` (`desireState=running`, no `failedAt`) | `/var/lib/longhorn/replicas` is missing while `longhorn-disk.cfg` exists, so Longhorn thinks the disk is initialised and the replica process dies on `mkdir`. `mkdir -p /var/lib/longhorn/replicas`. **Never delete that directory** — delete its contents. |
+| Rejoined node is a newer k3s than its peers | You forgot `--k3s-version`. More than one minor of skew between control-plane members is unsupported; reinstall it pinned. |
+| `member list` shows 4 | `e member remove <dead-id>` — do this now |
+| Node wiped, will not rejoin | Restore from the Longhorn backup target. |
+| API server gone (2 members down) | You lost quorum. See below. |
+
+### Lost quorum
+
+Do not panic-restart things. From the one surviving server:
+
+```
+systemctl stop k3s
+k3s server --cluster-reset
+systemctl start k3s
+```
+
+This rebuilds a single-member cluster from that node's data. The other two
+rejoin afterwards as fresh members; their
+`/var/lib/rancher/k3s/server/db` must be removed before they will.
+
+---
+
+## worker1 only (Asahi)
+
+After its reinstall, put this back in `/etc/fstab` or the node runs out of root
+space the first time it pulls a large image:
+
+```
+/var/lib/longhorn/k3s-containerd /var/lib/rancher/k3s/agent/containerd none bind,x-systemd.requires-mounts-for=/var/lib/longhorn,x-systemd.requires-mounts-for=/var/lib/rancher 0 0
+```
+
+```
+mkdir -p /var/lib/longhorn/k3s-containerd
+systemctl daemon-reload && mount -a
+```
+
+---
+
+## After all three
+
+Re-apply by hand — `pulumi up` does NOT restore these:
+
+- aitopatom Longhorn `allowScheduling: false`
+- Prometheus 2-replica pin
+- multus 512Mi memory limit
+- every `local-path` PVC (model caches re-download; nothing else does)
+
+Then check the sum: 3 x 300 GiB = 900 GiB real, against ~1550 GiB scheduled.
+Right-size Prometheus (376 GiB provisioned, ~8 GiB actual) or set
+`storageOverProvisioningPercentage` deliberately.

migration/RECOVERY-CARD-wan-panic.md

@@ -0,0 +1,99 @@
+# RECOVERY CARD — internet is down after a WAN failover
+
+No internet means no Claude. Everything here runs from the routers themselves.
+**Print this or keep it on a phone.**
+
+## 1. Get to a router
+
+The LoT leg is L2-direct on `bond0.10`. It survives Management, VRRP and
+routing being broken:
+
+```
+ssh vyos@10.0.1.252     # vyos001  (normally MASTER, has the WAN)
+ssh vyos@10.0.1.253     # vyos002  (normally BACKUP, has nothing)
+```
+
+Password is the usual one. If SSH is dead, use the JetKVM consoles.
+
+## 2. See what is going on
+
+```
+sudo /config/wan-panic status
+```
+
+Run it on both. You want exactly ONE box saying `holds VIP : YES`, and that
+same box showing a `WAN` line and a `route`.
+
+| what you see | what it means |
+|---|---|
+| one box `YES` with WAN + route | healthy, look elsewhere for the fault |
+| one box `YES`, **no WAN**, no route | the failover half-worked — go to §3 |
+| **both** `YES` | VRRP split — go to §3, run it on vyos002 |
+| **neither** `YES` | both faulted — go to §4 |
+
+## 3. Give the WAN back to vyos001
+
+**Run this on vyos002 (`10.0.1.253`).** This is the one that matters — vyos002
+standing down is what lets vyos001 take over.
+
+```
+sudo /config/wan-panic
+```
+
+Wait ~30s. Then on vyos001 (`10.0.1.252`):
+
+```
+sudo /config/wan-panic status
+```
+
+Expect `holds VIP : YES` and a `WAN` line with `bond0.53=…` and/or `pppoe0=…`.
+
+Once the house is back online and you want vyos002 to be a standby again:
+
+```
+sudo rm /run/vrrp-wan/force-fault        # on vyos002
+```
+
+Leave it set if you would rather have no standby than any more surprises —
+that is the pre-2026-09-06 arrangement and the house runs fine on it.
+
+## 4. Stop the mechanism touching anything
+
+If the WAN keeps moving, or you do not trust the automation:
+
+```
+sudo /config/wan-panic undo              # on BOTH routers
+```
+
+Stops the reconcile and guard timers. Whatever WAN is up **stays** up. Nothing
+will move it again until you re-enable the timers:
+
+```
+sudo systemctl enable --now vrrp-wan-reconcile.timer vrrp-wan-guard.timer
+```
+
+## 5. Nuclear — put the config back
+
+Only if the config itself is wrong. **This reboots the router.**
+
+```
+sudo /config/vyos-known-good restore     # on BOTH routers
+```
+
+The pinned config is from 2026-09-06, immediately before the PPPoE HA rollout:
+1112 lines on vyos001, 1069 on vyos002.
+
+## Why the WAN can only be on one box
+
+One ISP account each way. The 10 gig lease is bound to a cloned MAC
+(`f0:9f:c2:12:9b:4f`, the old USG's) and Vodafone is a single-session PPPoE
+credential. Two routers holding either at once is worse than one holding
+neither — that is why the safe state is "vyos001 has it, vyos002 is inert".
+
+## Known gap
+
+vyos001's `config.boot` does not carry `vif 53 disable`. If vyos001 reboots
+**while vyos002 is master**, the cloned MAC is briefly live on both. It clears
+itself within 30s (the reconciler disables it on the non-master). If you see
+MAC flapping on the WAN switch port right after a vyos001 reboot, that is this,
+and it will stop on its own.

migration/ROLLING-MIGRATION-RUNBOOK.md

@@ -0,0 +1,277 @@
+# Rolling migration to the v2 disk layout
+
+Moving `worker0`, `worker1` and `worker2` from the v1 layout to v2, one at a
+time, without losing data or quorum.
+
+**Status: NOT YET EXECUTED against the fleet.** The mechanism is proven by
+`bastion/tests/integration/rolling-drill.test.ts`, which passed **10/10** on
+four VMs on 2026-09-30 — including a backup to S3 and a byte-identical restore.
+Evidence: `drill-evidence/schema-v2-2026-09-30.txt`.
+
+---
+
+## The one thing to understand first
+
+**You cannot start by evacuating an existing node.** Two independent reasons,
+both read off the live cluster:
+
+1. **Placement.** `replica-soft-anti-affinity` is `false`, so a volume puts at
+   most one replica per node. There are exactly **3** Longhorn-schedulable nodes
+   (`worker0`, `worker1`, `worker2` — `aitopatom` and `spark` have
+   `allowScheduling: false` deliberately). **37 volumes want 3 replicas.** Evict
+   any one node and the other two already hold a copy each, so the third has
+   nowhere legal to go. Eviction can never reach zero.
+
+2. **Capacity**, which blocks it even if placement did not:
+
+   | node | Longhorn disk | used | free |
+   |---|---|---|---|
+   | worker0 | 727.6 GiB | 91.5 | ~636 |
+   | worker1 | 664.7 GiB | 114.3 | ~550 |
+   | **worker2** | **99.9 GiB** | 51.1 | **~49** |
+
+   worker2 cannot absorb worker0's 91.5 GiB. No setting changes that.
+
+So: **bring the new box up first (N+1), then evacuate an old one onto it.**
+
+`labctl app k3s evacuate` now refuses in seconds with the offending volume
+named, rather than discovering this an hour in. If it refuses, it is right.
+
+---
+
+## Capacity arithmetic, stated plainly
+
+- Today: 727.6 + 664.7 + 99.9 = **~1.49 TiB** of Longhorn disk.
+- After v2 (fixed 300 GiB per node): 3 x 300 = **900 GiB** — a **reduction of
+  ~590 GiB**.
+- Real usage is **~257 GiB**, so it fits with ~640 GiB headroom.
+- But **1550 GiB is already *scheduled*** (794 + 558 + 198), which only works
+  because `storageOverProvisioningPercentage` is **500**. Against 900 GiB real
+  that needs >=172% and leaves no room for the next volume.
+
+**Action:** before migrating the last node, right-size the oversized PVCs
+(Prometheus is 376 GiB provisioned against ~8 GiB actual) or set the
+over-provisioning percentage deliberately. Do not discover this when a disk
+fills.
+
+---
+
+## Preamble — do these once, before touching any node
+
+1. **Backup target.** Configured in `deployments/longhorn/` and
+   `Pulumi.homelab.yaml` (SeaweedFS S3 at `10.0.0.194:8333` — a different
+   machine from the cluster, which is what makes it a backup rather than another
+   copy of the same failure domain). TrueNAS was rejected on evidence: port 2049
+   is CLOSED, so NFS would need the export creating first.
+
+   **`defaultSettings.backupTarget` DOES NOT WORK on Longhorn 1.8.** The target
+   moved onto the `backuptargets.longhorn.io/default` CR
+   (`spec.backupTargetURL`, `spec.credentialSecret`); the helm value is silently
+   ignored, no `backup-target` Setting object is created, and the chart reports
+   success with no backup target at all. The Pulumi code therefore patches the
+   CR via `apiextensions.CustomResourcePatch`, and the RecurringJobs depend on
+   that patch so the first scheduled backup cannot fire into nothing.
+
+   Before relying on it:
+   - create the `longhorn-backup` bucket;
+   - apply with `--target`, **not** a bare `pulumi up` — the stack carries
+     unrelated drift (at one point 12 resources, including a replacement of the
+     openbao watchdog ConfigMap). Target these four:
+     ```
+     kubernetes:core/v1:Secret::longhorn-backup-credentials
+     kubernetes:helm.sh/v3:Release::longhorn
+     ...Release$...:BackupTarget::longhorn-backup-target
+     ...Release$...:RecurringJob::longhorn-job-daily-backup
+     ...Release$...:RecurringJob::longhorn-job-hourly-snapshot
+     ```
+     (`scripts/pulumi.sh` refuses an unclean tree; commit and merge rather than
+     setting `PULUMI_ALLOW_UNMERGED=1`.)
+   - confirm Longhorn accepted it — `available: true` is the appliance saying it
+     could reach the endpoint AND authenticate:
+     ```
+     kubectl -n longhorn-system get backuptargets.longhorn.io default \
+       -o custom-columns=URL:.spec.backupTargetURL,AVAIL:.status.available
+     ```
+   - take one backup, then **restore it to a new volume and check the contents.**
+     An unverified backup is not a backup. The restore is the gate — not the
+     `pulumi up`, and not `available: true`.
+
+   Note: this cluster has **no CSI VolumeSnapshot API** (no
+   `snapshot.storage.k8s.io` resources at all), so backup tooling must use
+   Longhorn's native CRs — `snapshots.longhorn.io`, `backups.longhorn.io`, and a
+   `fromBackup` StorageClass for restores. The drill does exactly this.
+
+2. **The 21 single-replica volumes.** One copy each, and until step 1 is done, no
+   other copy anywhere. For each one on the node you are about to evacuate:
+   ```
+   kubectl -n longhorn-system patch volumes.longhorn.io <pv> \
+     --type=merge -p '{"spec":{"numberOfReplicas":3}}'
+   ```
+   wait for `status.robustness: healthy`, *then* evacuate. Restore the count
+   afterwards if you want it back.
+
+   `evacuate` refuses single-replica volumes by default.
+   `--allow-single-replica` overrides it and means "I accept that a failed
+   rebuild is permanent loss".
+
+   Detached volumes are listed separately by the pre-check: Longhorn must attach
+   them to rebuild, so they need attention before, not during.
+
+3. **Pin the k3s version, or the migration upgrades the cluster by accident.**
+
+   `get.k3s.io` installs **latest**. The fleet runs **v1.34.6+k3s1**; latest at
+   the time of writing is **v1.36.4+k3s1** — two minor versions ahead. A node
+   reinstalled by this procedure rejoins on a newer k3s than the two members it
+   is joining, and Kubernetes supports only **one** minor of skew between
+   control-plane members. Roll three nodes and you have performed an unplanned,
+   unstaged, two-version upgrade one node at a time.
+
+   Pass `--k3s-version v1.34.6+k3s1` (check the real value with
+   `kubectl get nodes -o wide`). Upgrade deliberately, afterwards, as its own
+   change.
+
+4. **Add the N+1 node.** Provision the new box with `schemaVersion: 2`, join it,
+   and confirm it has a schedulable Longhorn disk:
+   ```
+   kubectl -n longhorn-system get nodes.longhorn.io
+   ```
+   A node with no disk hosts no replicas while looking perfectly healthy. If it
+   has none, `labctl app k3s admit <node>` repairs it.
+
+---
+
+## Node order
+
+1. **worker0** — plain Fedora x86, the most ordinary box. Go first.
+2. **worker2** — Fedora aarch64. Its 100 GiB Longhorn disk becomes 300 GiB
+   under v2, so this node *gains* room.
+3. **worker1** — **LAST, and a different animal.** Apple Silicon under Asahi:
+   the partition table belongs to the vendor installer, root is 4.6 GiB, the
+   rancher LV is undersized at 20 GiB, and it carries a containerd bind mount
+   that exists in no template or script:
+   ```
+   /var/lib/longhorn/k3s-containerd /var/lib/rancher/k3s/agent/containerd none \
+     bind,x-systemd.requires-mounts-for=/var/lib/longhorn,x-systemd.requires-mounts-for=/var/lib/rancher 0 0
+   ```
+   A reinstall destroys that silently. Re-create it, or the node runs out of
+   root space the first time it pulls a large image.
+
+Quorum: there are **exactly 3 etcd members**, so quorum is 2 and the fleet
+tolerates **exactly one member down with no margin**. One node at a time, never
+two, and never start a roll while anything is already degraded.
+
+---
+
+## Per-node loop
+
+Gate at every step. If one fails, stop — do not roll the next node on top of a
+half-migrated one.
+
+```
+# 0. GATE: cluster is whole before you start
+kubectl get nodes
+kubectl -n longhorn-system get volumes.longhorn.io | grep -v healthy   # expect none
+# `k3s etcdctl` is NOT a command. Define this first (see the recovery card):
+#   e() { local b=/var/lib/rancher/k3s/server/tls/etcd
+#         ETCDCTL_API=3 etcdctl --cacert=$b/server-ca.crt \
+#           --cert=$b/server-client.crt --key=$b/server-client.key \
+#           --endpoints=https://127.0.0.1:2379 "$@"; }
+e member list                  # expect 3, all started
+e endpoint health
+
+# 1. Will it even work?
+labctl app k3s evacuate <node>      # refuses fast if it cannot complete
+
+# 2. (it evacuates: cordon -> check -> evict+wait -> drain -> verify)
+kubectl -n longhorn-system get replicas.longhorn.io -o wide | grep <node>   # expect none
+
+# 3. Prove the data is fine BEFORE destroying anything
+#    Restart something that uses a Longhorn PVC and check it reads back.
+
+# 4. The irreversible part — as late as possible
+ssh <node> systemctl stop k3s
+e member remove <id-of-node>   # member names are "<node>-<8 hex>", not "<node>"
+kubectl delete node <node>
+e member list                  # expect 2, both healthy
+kubectl get --raw /readyz      # from each survivor: ok
+
+# 5. Wipe and reinstall
+labctl provision reprovision <node>    # the DISK decides the schema, not you
+
+# 6. Rejoin — against a SURVIVOR, not against itself, PINNED
+labctl app k3s install <node> --role k8s-combo \
+  --k3s-server https://<survivor-ip>:6443 --k3s-token <token> \
+  --k3s-version v1.34.6+k3s1      # MUST match the cluster; see preamble step 3
+e member list                  # expect EXACTLY 3
+
+# 7. Back into service
+labctl app k3s admit <node>
+
+# 8. Wait for the rebuild, then verify the data again
+kubectl -n longhorn-system get volumes.longhorn.io    # all healthy, 3 replicas
+#    And check the node actually took replicas BACK — running, not merely assigned:
+kubectl -n longhorn-system get replicas.longhorn.io -o wide | grep <node>
+```
+
+**Step 6's "exactly 3" is not a formality.** A reinstalled server rejoins as a
+*new* etcd member. If the old record survives you get 4 members, quorum rises
+from 2-of-3 to 3-of-4, and a cluster that tolerated one failure now tolerates
+none — while every dashboard still says green. That is why step 4 removes the
+member explicitly instead of trusting the rejoin to clean up.
+
+**Step 8's "running, not merely assigned" is also not a formality.** A replica
+can sit at `currentState=stopped` / `desireState=running` with no `failedAt`
+indefinitely. The node looks Ready and Schedulable and hosts nothing, and the
+*next* node's eviction then has nowhere to drain and stalls for its full hour.
+
+---
+
+## What `pulumi up` puts back, and what it does not
+
+**Restored automatically.** Node labels and taints — both `NodePatch` sites use
+`pulumi.com/patchForce`, so they genuinely self-heal.
+
+**NOT restored — re-apply by hand:**
+
+- aitopatom's Longhorn `allowScheduling: false`
+- the Prometheus 2-replica pin
+- the multus 512Mi memory limit
+- worker1's containerd bind mount (above)
+- every `local-path` PVC. Model caches re-download; the rest do not come back.
+
+---
+
+## If it goes wrong
+
+- **Evacuation refused.** Believe it. Read the named volume; it is telling you
+  either "no node can take this replica" or "the disk does not have room". Add
+  capacity rather than forcing it.
+- **Evacuation timed out.** The node is left schedulable again and the eviction
+  cancelled automatically; the error names which volumes were stuck and in what
+  state. If that cleanup itself failed, the error says so — run
+  `labctl app k3s admit <node>`.
+- **Aborted midway, node still cordoned.** `labctl app k3s admit <node>`
+  uncordons, clears the eviction and restores the Longhorn disk.
+- **Replicas land on a node but stay `stopped`.** `/var/lib/longhorn/replicas`
+  is missing while `longhorn-disk.cfg` exists, so Longhorn considers the disk
+  already initialised and never recreates the directory; the replica process
+  then fails on a non-recursive `mkdir` every time, permanently.
+  `mkdir -p /var/lib/longhorn/replicas`. **Never delete that directory** —
+  delete its contents. `labctl app k3s admit <node>` now ensures it exists.
+- **Rejoined node hosts no replicas.** It has no Longhorn disk. `admit` repairs
+  it; `kubectl -n longhorn-system get nodes.longhorn.io <node> -o yaml` confirms.
+- **Node wiped and cannot rejoin.** This is the case the backup target exists
+  for. Restore the volumes from backup. If step 1 of the preamble was skipped,
+  there is no recovery path — which is precisely why it is step 1.
+
+---
+
+## Known gaps
+
+- Every node reports `RequiredPackages=False — Missing packages: [nfs-utils
+  cryptsetup]` and `KernelModulesLoaded=False`. Uniform across the fleet and not
+  blocking today, but it would block an NFS backup target and encrypted volumes.
+  The S3 target does not need `nfs-utils`.
+- The drill ran on VMs with ~3 GiB of test data; production has ~257 GiB. It
+  proves **mechanism**, not timing. Keep the 60-minute eviction timeout generous.
+- `worker1` has never been VM-tested and cannot be — it needs Apple Silicon.

migration/THEBEAST-K8S-WORKER.md

@@ -0,0 +1,113 @@
+# thebeast → prod k8s worker — conversion runbook
+
+Converting the old desktop **thebeast** (currently Fedora/ZFS desktop at
+`10.0.1.22`) into a `k8s-worker` node of the **prod cluster (k8s0)** via the
+PXE bastion, using the dedicated-data-disk layout (`--longhorn-disk` /
+`--rancher-disk`, introduced with this runbook).
+
+Decisions already taken (2026-10-14): Windows disk is sacrificed (unused for
+years, no backup wanted) · cluster = prod k8s0 · OS = Fedora `k8s-worker`
+(thebeast is mainly an AMD-GPU machine: 2× AMD + 1× RTX 3080 Ti, no unified
+memory) · the node will carry a dedicated taint and also serve CI/CD.
+
+**VM-drilled 2026-10-14** (on thebeast itself, evidence:
+`drill-evidence/data-disks-2026-10-14.txt`): fresh install lands the exact
+layout below, and a flagless reprovision rediscovers both data disks by label
+and remounts them without reformatting (UUIDs unchanged, marker files intact).
+
+## Disk map — ALWAYS by-id, never nvme0n1
+
+Kernel NVMe names are **not stable across boots on this machine** (observed).
+Every command below uses `/dev/disk/by-id`.
+
+| target | disk | by-id |
+|---|---|---|
+| OS (wipes Windows) | Samsung 970 EVO Plus 1TB | `nvme-Samsung_SSD_970_EVO_Plus_1TB_S4EWNJ0N138234M` |
+| /var/lib/longhorn | Sabrent Rocket 4.0 2TB | `nvme-Sabrent_Rocket_4.0_2TB_7F600711179D89261778` |
+| /var/lib/rancher | Sabrent Rocket 4.0 2TB | `nvme-Sabrent_Rocket_4.0_2TB_7FD4070C1EDA92197301` |
+| DISCONNECT after final shutdown | Samsung 970 EVO Plus 2TB `S4J4NF0NC00809J` | sits on its own motherboard add-on card, PCIe x4, model discontinued |
+
+NICs: `enp69s0` `34:97:f6:32:76:e9` (current uplink — this MAC is what the
+bastion will see), `enp70s0` `34:97:f6:31:b3:99` (down).
+
+## 0. Before the final shutdown — data
+
+**rpool is a 4-wide raidz1 already DEGRADED** (member
+`S4J4NF0R102776W` removed since 2026-09-14). There is **zero redundancy
+left**: one more failure loses the pool, and a disk pulled out of it is NOT
+importable alone. Finish the copies first, then never boot the desktop again.
+
+Already safe (verified 2026-10-14): `~/developer` (all 113 repos) + the
+credential/dotfile set → bazzite (Oct 12 rsync, 1 file of drift).
+
+Copied to bazzite and rsync-verified 2026-10-14: `~/keyssecrets`,
+`~/dicesecrets`, `~/Videos/canaries` (24G), `~/Pictures`, `~/Documents`.
+`~/opt/vaultbackup` and `~/migration-backup` declared done/not-needed (user,
+2026-10-14).
+
+Still open before shutdown:
+- `~/Videos/{Webcam,subscribescreen,Screencasts,music}` (~370M) and
+  `~/backups/sogo-final-2026-09-13` — small, not yet copied
+- Verify the Syncthing replicas on synology0-nas for `~/Logseq`,
+  `~/DICESharedBoostNote`, `~/Boostnote`, `~/MangaReader_backup` — thebeast's
+  syncthing shows 0 connected peers, so the NAS copy is **unverified**
+- Final re-sync of `~/.claude` + atuin (activity postdates the Oct 12 copy)
+
+## 1. Physical + network
+
+1. Final desktop shutdown.
+2. Disconnect the Samsung 2TB `S4J4NF0NC00809J` (the add-on card).
+3. Move thebeast's switch port to the `192.168.8.0/24` VLAN (same as k8s0 and
+   labmaster). The bastion's proxy-DHCP (`dhcp-range=192.168.8.0,proxy`)
+   coexists with the real DHCP there — no DHCP changes needed.
+4. Make sure network boot is enabled/first in firmware (UEFI PXE, IPv4).
+
+## 2. Discovery
+
+PXE boot. Unknown MAC → discovery image → the machine self-reports:
+
+```
+labctl provision list        # thebeast appears under discovered (MAC 34:97:f6:32:76:e9)
+```
+
+## 3. Install
+
+```
+labctl provision install 34:97:f6:32:76:e9 thebeast \
+  --role k8s-worker \
+  --disk /dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_1TB_S4EWNJ0N138234M \
+  --longhorn-disk /dev/disk/by-id/nvme-Sabrent_Rocket_4.0_2TB_7F600711179D89261778 \
+  --rancher-disk /dev/disk/by-id/nvme-Sabrent_Rocket_4.0_2TB_7FD4070C1EDA92197301
+```
+
+(`k8s-worker` is v2-only, so no `--schema-version` needed.)
+
+What the layout does: OS disk gets EFI + /boot + `labvg` with the v2 base LVs
+only — **no** longhorn/rancher LVs. Each 2 TB disk gets ONE xfs partition
+spanning it, labelled `lab-longhorn` / `lab-rancher`, mounted at
+`/var/lib/longhorn` / `/var/lib/rancher`. The ZFS signatures on those disks
+are wiped by the install — which is the point of section 0.
+
+Reinstall semantics from then on: the labels are on-disk truth. A reprovision
+rediscovers both data disks by label (no flags needed) and mounts them
+`--noformat`; a fresh install refuses to wipe or shadow a labelled disk it was
+not told about.
+
+## 4. Join k8s0
+
+As on every k8s-worker, k3s is installed with `INSTALL_K3S_SKIP_START`; the
+join is deliberate (see `/root/README` on the node):
+
+```
+curl -sfL https://get.k3s.io | K3S_URL=https://<k8s0-server>:6443 K3S_TOKEN=<token> sh -
+```
+
+Add the agreed CI/CD taint/labels at join time (k3s agent flags), e.g.:
+
+```
+--node-taint dedicated=cicd:NoSchedule --node-label itaz.eu/cicd=true
+```
+
+(exact taint TBD with the Pulumi model; GPU bits — ROCm for the 2 AMD cards,
+nvidia for the 3080 Ti — are cluster-side follow-ups in
+thelab-kubernetes-pulumi, not part of the install.)

migration/VYOS-IMAGE-UPGRADE.md

@@ -0,0 +1,92 @@
+# Upgrading a VyOS image on the router pair
+
+VyOS keeps `/config` across an image upgrade and **replaces `/etc`**. Everything
+in `/config` survives; anything the WAN mechanism put in `/etc` does not.
+
+## What an upgrade silently removes
+
+`/etc/systemd/system/ppp@pppoe0.service.d/10-vrrp-wan-gate.conf` — the gate.
+
+That drop-in is the only thing stopping the **backup** from dialling. With
+`pppoe0` enabled on both routers (which it is, by design), `interfaces_pppoe.py`
+restarts ppp on every commit touching the pppoe subtree when the daemon is not
+running. Without the gate, the backup dials on the next `pulumi up`, hand
+commit, or boot-time config load — and Vodafone is a single-session account, so
+it takes the session off the live master.
+
+`vrrp-wan-reconcile` fails **closed** here: it refuses to dial at all when the
+drop-in is missing, and logs
+
+```
+REFUSING to dial: gate drop-in ... is missing (VyOS upgrade?)
+```
+
+so the symptom is "PPPoE never comes up", not "both routers dialled". That is
+the safe direction, but it does mean an upgraded router has no PPPoE until the
+gate is reinstalled.
+
+The systemd **units and timers** also live in `/etc` and go the same way.
+
+## Do this, one router at a time
+
+Never both at once — the surviving router must be able to hold the VIPs.
+
+1. **Upgrade the BACKUP first.** Confirm which it is:
+   ```
+   sudo /config/wan-panic status          # "holds VIP : no"
+   ```
+2. Install the image and reboot as normal.
+3. **Reinstall the mechanism** from a checkout of `lab`:
+   ```
+   migration/vrrp-wan-install --vip 192.168.1.1 --host vyos@<router>
+   migration/vrrp-wan-install --check  --host vyos@<router>     # must be clean
+   ```
+4. **Verify the gate is shut and nothing dialled:**
+   ```
+   systemctl show ppp@pppoe0 -p ConditionResult -p ActiveState -p NRestarts
+   ```
+   Want `ConditionResult=no`, `ActiveState=inactive`, `NRestarts=0`. Check the
+   wire too, not just state: `sudo tcpdump -i bond0.51 -nn pppoed` — no PADI.
+5. Confirm it reaches **BACKUP**, not FAULT:
+   ```
+   show vrrp
+   ```
+   FAULT on every group means the health check is failing — most likely
+   `/config/vrrp-wan-health` did not get reinstalled, or `vrrp-wan.conf` is
+   missing so `GRACE` and the VIP fall back to defaults.
+5b. **Check IPv6 came back with it.** `vrrp-wan-install` now carries
+   `he-tunnel-follow`, so `--check` covers it, but `/config/he-secrets` is a
+   secret placed by Pulumi and is only checked for *presence*:
+   ```
+   sudo /config/he-tunnel-follow status      # role, tunnel src, MTU
+   ```
+   Want the box's own role, and — on the master — a tunnel source equal to the
+   **10 gig** address with MTU 1480. `/config` survives an upgrade, so the
+   `system task-scheduler` entry that runs this every minute survives too; it is
+   the units and the ppp gate in `/etc` that do not.
+6. Confirm `config.boot` still pins the safe resting state:
+   ```
+   sudo /config/vif53-pin-boot-disable --check    # config.boot disable : 1
+   ```
+7. Only once the upgraded box is a healthy BACKUP, fail over and repeat for the
+   other router. `migration/wan-drill` does that unattended, or by hand:
+   `sudo /config/wan-panic` on the box you want to give up mastership.
+
+## After both are done
+
+```
+migration/vrrp-wan-install --check --host vyos@10.0.1.252
+migration/vrrp-wan-install --check --host vyos@10.0.1.253
+cd kubernetes-deployment && npm run vyos:export -- --router vyos001=10.0.1.252 --router vyos002=10.0.1.253
+npm run vyos:render && npm run vyos:verify -- --router vyos001=10.0.1.252 --router vyos002=10.0.1.253
+```
+
+Both should read "in sync". If the export shows `interfaces pppoe pppoe0
+disable` coming back, something reinstated it — the
+`pppoe-gated-not-config-disabled` override exists to prevent exactly that, so
+check it is still in `infra/vyos/subtrees/overrides.json`.
+
+## If it goes wrong
+
+`migration/RECOVERY-CARD-wan-panic.md`, or `/config/RECOVERY-CARD.md` on either
+router. Short version, on vyos002: `sudo /config/wan-panic`.

migration/drill-evidence/data-disks-2026-10-14.txt

@@ -0,0 +1,40 @@
+Dedicated data disks drill — fresh install + flagless reprovision, real VM
+run finished: 2026-10-14T23:30+01:00   host: thebeast
+test: bastion/tests/integration/data-disks.test.ts (1 passed, 1164s)
+topology: 1 VM, role k8s-worker schema v2, 1000G OS disk + 2x 20G data disks
+addressed as /dev/disk/by-id/virtio-LHDRILL7F600711 (longhorn) and
+/dev/disk/by-id/virtio-RNDRILL7FD4070C (rancher) — the same by-id contract
+thebeast's NVMe serials use.
+
+Proven:
+- fresh install: labvg = home/root/srv/swap/var/varlog ONLY (no longhorn,
+  no rancher LV); /var/lib/longhorn and /var/lib/rancher are whole-disk xfs
+  partitions labelled lab-longhorn / lab-rancher, each verified to sit on
+  the disk with the matching serial; lab-schema-v2 tag present; fstab by UUID.
+- reprovision with NO dataDisks flags: both disks rediscovered by label,
+  remounted --noformat (filesystem UUIDs identical before/after), marker
+  files intact in /home /srv /var/lib/longhorn /var/lib/rancher, LV set and
+  schema tag unchanged.
+
+=== checkpoints ===
+[22:10:40] [dd-worker] creating VM (1000GB disk + 2 data disk(s), role=k8s-worker, schema=v2)
+[22:13:02] [dd-worker] queueing install...
+[22:15:25] [dd-worker] installing (up to 60 min)...
+[22:21:24] [dd-worker] up at 192.168.251.194
+[22:21:24] [dd-worker] on-disk LVs: home=8G root=32G srv=16G swap=16G var=48G varlog=16G
+[22:21:27] [dd-worker] wrote marker preserve-1792016486955-iu1rih into /home, /srv, /var/lib/longhorn, /var/lib/rancher
+[22:21:27] [dd-worker] reprovisioning (role=k8s-worker, schema: whatever is on disk)...
+[22:29:58] [dd-worker] back up at 192.168.251.194 after reinstall
+[22:29:59] [dd-worker] markers after reinstall: all four mounts carry preserve-1792016486955-iu1rih
+[22:29:59] [dd-worker] on-disk LVs: home=8G root=32G srv=16G swap=16G var=48G varlog=16G
+✓ k8s-worker with longhorn+rancher disks: install, then flagless reprovision rediscovers both (1161s)
+
+Run notes (cost of learning, so nobody pays it twice):
+- the drill machine's bastion imports resolve @lab/shared to dist/ -- run
+  `pnpm build` after switching branches or the API answers with the OLD role
+  registry ("invalid role: 'k8s-worker'").
+- a passphrase-protected default ~/.ssh/id_rsa fails as "SSH not available"
+  with a perfectly healthy machine (BatchMode never offers it without an
+  agent). Use LAB_TEST_SSH_KEY with a dedicated passphrase-less key.
+- run: cd bastion && LAB_TEST_SSH_KEY=~/.ssh/lab-integration_ed25519 \
+    sudo -E npx vitest run -c tests/integration/vitest.config.ts tests/integration/data-disks.test.ts

migration/drill-evidence/schema-v2-2026-09-30.txt

@@ -0,0 +1,123 @@
+Rolling migration drill — schema v1 -> v2, WITH backup/restore and version pin
+run finished: 2026-09-30T02:26:27+01:00   host: thebeast
+k3s: v1.34.6+k3s1 (PINNED and asserted on every node)   longhorn: 1.8.1
+topology: 3 x k8s-combo (server+etcd+storage) + 1 x k8s-worker (storage spare, N+1)
+backup target: s3://longhorn-drill-backup@us-east-1/ via SeaweedFS at 10.0.0.194:8333
+  (separate bucket from production's longhorn-backup)
+
+=== checkpoints ===
+[drill] ==> provision
+[drill] <== provision (943s)
+[drill] ==> form cluster
+[00:22:37] [snapshot] after cluster-init: 1/1 Ready, etcd members=1 healthy=true
+[00:24:42] [snapshot] after joining drill-s2: 2/2 Ready, etcd members=2 healthy=true
+[00:25:54] [snapshot] after joining drill-s3: 3/3 Ready, etcd members=3 healthy=true
+[00:26:55] [snapshot] formed: 4/4 Ready, etcd members=3 healthy=true
+[drill] all 4 nodes on pinned v1.34.6+k3s1
+[drill] <== form cluster (348s)
+[drill] ==> install longhorn
+[00:26:56] [longhorn] creating longhorn-system with privileged PSA labels
+[00:26:56] [longhorn] configuring backup target s3://longhorn-drill-backup@us-east-1/
+[00:26:56] [longhorn] installing chart 1.8.1 via k3s helm controller
+[00:28:11] [longhorn] ready after 0s — 4/4 with schedulable disk: drill-s1.pxe-test.local,drill-s2.pxe-test.local,drill-s3.pxe-test.local,drill-spare.pxe-test.local
+[drill] <== install longhorn (75s)
+[drill] ==> seed data
+[00:28:11] [longhorn] creating drill-r3 (2Gi, 3 replicas)
+[00:28:11] [longhorn] creating drill-r1 (1Gi, 1 replicas)
+[00:29:01] [longhorn] writing 512 MiB to drill-r3
+[00:29:04] [longhorn] drill-r3 digest b8171779defe286b...
+[00:29:05] [longhorn] writing 128 MiB to drill-r1
+[00:29:06] [longhorn] drill-r1 digest 89213fddc2738c8d...
+[00:29:40] [longhorn] drill-r3 verified intact across a detach/attach (pod drill-r3-754b4fbc7d-6ntwp -> drill-r3-754b4fbc7d-xrgds)
+[00:30:13] [longhorn] drill-r1 verified intact across a detach/attach (pod drill-r1-949b946f8-4vbr6 -> drill-r1-949b946f8-pt6kv)
+[drill] <== seed data (123s)
+[drill] ==> backup and restore
+[00:30:14] [longhorn] backup target set to s3://longhorn-drill-backup@us-east-1/
+[00:30:15] [longhorn] backup target available after 0s
+[00:30:16] [longhorn] snapshotting drill-r3 (pvc-4eeea364-2ed3-41fa-abc4-0176fb13ba3e)
+[00:30:21] [longhorn] backing up drill-r3 to the backup target
+[00:30:43] [longhorn] backup drill-bak-mundcj51 completed after 21s
+[00:30:44] [longhorn] restoring drill-bak-mundcj51 into drill-restored
+[drill] restored digest b8171779defe286b... matches original
+[drill] <== backup and restore (108s)
+[drill] ==> negative pre-check
+[drill] <== negative pre-check (34s)
+[drill] ==> roll drill-s1
+[00:32:37] [snapshot] before rolling drill-s1: 4/4 Ready, etcd members=3 healthy=true
+[drill-s1] All replicas evicted from drill-s1.pxe-test.local after 32s
+[drill-s1] Drained drill-s1.pxe-test.local
+[00:33:20] [longhorn] drill-r3 healthy on 3 nodes after 1s
+[00:34:16] [longhorn] drill-r3 verified intact across a detach/attach (pod drill-r3-754b4fbc7d-xrgds -> drill-r3-754b4fbc7d-mhlj7)
+[00:34:49] [longhorn] drill-r1 verified intact across a detach/attach (pod drill-r1-949b946f8-pt6kv -> drill-r1-949b946f8-6kqct)
+[00:34:50] [etcd] removed member drill-s1.pxe-test.local-e58f4aaa (9575fbd07a6c3ca1)
+[00:35:37] [snapshot] drill-s1 removed: 3/3 Ready, etcd members=2 healthy=true
+[00:46:06] [snapshot] drill-s1 rejoined: 4/4 Ready, etcd members=3 healthy=true
+[00:46:37] [longhorn] drill-r3 healthy on 3 nodes after 1s
+[00:47:53] [longhorn] drill-r3 verified intact across a detach/attach (pod drill-r3-754b4fbc7d-mhlj7 -> drill-r3-754b4fbc7d-qhn2c)
+[00:48:58] [longhorn] drill-r1 verified intact across a detach/attach (pod drill-r1-949b946f8-6kqct -> drill-r1-949b946f8-zj9lv)
+[drill] after rolling drill-s1: 2 RUNNING replica(s) on the node
+[drill] <== roll drill-s1 (982s)
+[drill] ==> roll drill-s2
+[00:48:59] [snapshot] before rolling drill-s2: 4/4 Ready, etcd members=3 healthy=true
+[drill-s2] All replicas evicted from drill-s2.pxe-test.local after 47s
+[drill-s2] Drained drill-s2.pxe-test.local
+[00:50:31] [longhorn] drill-r3 healthy on 3 nodes after 1s
+[00:51:41] [longhorn] drill-r3 verified intact across a detach/attach (pod drill-r3-754b4fbc7d-qhn2c -> drill-r3-754b4fbc7d-9bfqw)
+[00:52:15] [longhorn] drill-r1 verified intact across a detach/attach (pod drill-r1-949b946f8-zj9lv -> drill-r1-949b946f8-7k9pn)
+[00:52:16] [etcd] removed member drill-s2.pxe-test.local-943a6633 (2dfcaab9db591b03)
+[00:53:02] [snapshot] drill-s2 removed: 3/3 Ready, etcd members=2 healthy=true
+[01:04:02] [snapshot] drill-s2 rejoined: 4/4 Ready, etcd members=3 healthy=true
+[01:04:32] [longhorn] drill-r3 healthy on 3 nodes after 1s
+[01:05:38] [longhorn] drill-r3 verified intact across a detach/attach (pod drill-r3-754b4fbc7d-9bfqw -> drill-r3-754b4fbc7d-tf7z2)
+[01:06:46] [longhorn] drill-r1 verified intact across a detach/attach (pod drill-r1-949b946f8-7k9pn -> drill-r1-949b946f8-cx7rx)
+[drill] after rolling drill-s2: 2 RUNNING replica(s) on the node
+[drill] <== roll drill-s2 (1069s)
+[drill] ==> roll drill-s3
+[01:06:48] [snapshot] before rolling drill-s3: 4/4 Ready, etcd members=3 healthy=true
+[drill-s3] All replicas evicted from drill-s3.pxe-test.local after 31s
+[drill-s3] Drained drill-s3.pxe-test.local
+[01:07:59] [longhorn] drill-r3 healthy on 3 nodes after 1s
+[01:09:14] [longhorn] drill-r3 verified intact across a detach/attach (pod drill-r3-754b4fbc7d-tf7z2 -> drill-r3-754b4fbc7d-qv54j)
+[01:10:11] [longhorn] drill-r1 verified intact across a detach/attach (pod drill-r1-949b946f8-cx7rx -> drill-r1-949b946f8-sm8z7)
+[01:10:13] [etcd] removed member drill-s3.pxe-test.local-9bf0ac84 (d47166c003b5115c)
+[01:11:01] [snapshot] drill-s3 removed: 3/3 Ready, etcd members=2 healthy=true
+[01:22:13] [snapshot] drill-s3 rejoined: 4/4 Ready, etcd members=3 healthy=true
+[01:22:44] [longhorn] drill-r3 healthy on 3 nodes after 1s
+[01:23:50] [longhorn] drill-r3 verified intact across a detach/attach (pod drill-r3-754b4fbc7d-qv54j -> drill-r3-754b4fbc7d-wbz8c)
+[01:24:50] [longhorn] drill-r1 verified intact across a detach/attach (pod drill-r1-949b946f8-sm8z7 -> drill-r1-949b946f8-s5j48)
+[drill] after rolling drill-s3: 2 RUNNING replica(s) on the node
+[drill] <== roll drill-s3 (1084s)
+[drill] ==> final verification
+[01:24:51] [snapshot] end of drill: 4/4 Ready, etcd members=3 healthy=true
+[01:25:26] [longhorn] drill-r3 verified intact across a detach/attach (pod drill-r3-754b4fbc7d-wbz8c -> drill-r3-754b4fbc7d-nj5jp)
+[01:25:59] [longhorn] drill-r1 verified intact across a detach/attach (pod drill-r1-949b946f8-s5j48 -> drill-r1-949b946f8-xfbcq)
+[drill] <== final verification (69s)
+[01:26:00] [snapshot] FINAL: 4/4 Ready, etcd members=3 healthy=true
+[drill] digests: drill-r3=b8171779defe286b drill-r1=89213fddc2738c8d
+
+=== result ===
+ ✓ provisions four v2 nodes 943264ms
+ ✓ forms one cluster with three etcd members plus a storage agent 348143ms
+ ✓ installs Longhorn with a schedulable disk on every node 75260ms
+ ✓ writes known data and proves it survives a detach/attach 122827ms
+ ✓ backs up a volume to S3 and restores it byte-identical 108172ms
+ ✓ REFUSES to evacuate when there is nowhere for replicas to go 33846ms
+ ✓ rolls drill-s1: evacuate, wipe, reinstall, rejoin, verify 982060ms
+ ✓ rolls drill-s2: evacuate, wipe, reinstall, rejoin, verify 1068887ms
+ ✓ rolls drill-s3: evacuate, wipe, reinstall, rejoin, verify 1083812ms
+ ✓ ends with a whole cluster and the original bytes 68503ms
+ Test Files  1 passed | 10 skipped (11)
+      Tests  10 passed | 104 skipped (114)
+
+VERDICT: PASSED (10/10).
+  - All four nodes ran the PINNED k3s v1.34.6+k3s1, asserted from kubeletVersion.
+  - Backup target reachable and authenticated; 512 MiB backed up to S3 and RESTORED
+    into a separate volume byte-identical to the original digest.
+  - Three storage+etcd nodes rolled through evacuate/wipe/reinstall/rejoin one at a
+    time; checksums identical before and after every roll; etcd held exactly 3 healthy
+    members at every gate, 2 only while a node was deliberately absent, never 4.
+  - Each reinstalled node came back hosting RUNNING replicas.
+
+NOTE: defaultSettings.backupTarget is IGNORED by Longhorn 1.8 — the target lives on
+the backuptargets.longhorn.io/default CR. Setting it via helm values reports success
+and produces no backup target. Both the Pulumi code and this drill patch the CR.