thebeast runbook: VM drill passed, data copies done 2026-10-14

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V6HNx6cyr4J4CiMGCy1wkF

.gitea/workflows/ci.yml

@@ -28,6 +28,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
+        with:
+          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -47,6 +49,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
+        with:
+          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -54,6 +58,9 @@ jobs:
 
       - run: pnpm install --frozen-lockfile
 
+      - name: Generate Prisma client (labd typecheck needs it)
+        run: pnpm -C src/labd run db:generate
+
       - name: Typecheck
         run: pnpm typecheck
 
@@ -66,6 +73,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
+        with:
+          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -73,6 +82,15 @@ jobs:
 
       - run: pnpm install --frozen-lockfile
 
+      - name: Generate Prisma client (labd typecheck needs it)
+        run: pnpm -C src/labd run db:generate
+
+      - name: Install test validators (ubuntu-autoinstall parses YAML via python, kickstart.test runs ksvalidator, the bastion smoke start runs `ip route`)
+        # pykickstart is not in the runner image's apt sources -- pip it is.
+        run: |
+          sudo apt-get update -qq && sudo apt-get install -y -qq python3-yaml python3-pip iproute2
+          sudo pip3 install --quiet --break-system-packages pykickstart
+
       - name: Build (needed by completions check)
         run: pnpm build
 
@@ -91,6 +109,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
+        with:
+          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -99,6 +119,9 @@ jobs:
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
+      - name: Generate Prisma client (labd typecheck needs it)
+        run: pnpm -C src/labd run db:generate
+
       - name: Build all packages
         run: pnpm build
 
@@ -123,13 +146,13 @@ jobs:
 
       - name: Package x86_64 RPM + DEB
         run: |
-          sed -e 's|^arch:.*|arch: amd64|' -e 's|src: ./dist/lab$|src: ./dist/lab-x86_64|' nfpm.yaml > /tmp/nfpm-x86_64.yaml
+          sed -e 's|^arch:.*|arch: amd64|' -e 's|src: ./dist/labctl$|src: ./dist/lab-x86_64|' nfpm.yaml > /tmp/nfpm-x86_64.yaml
           nfpm pkg --config /tmp/nfpm-x86_64.yaml --packager rpm --target dist/
           nfpm pkg --config /tmp/nfpm-x86_64.yaml --packager deb --target dist/
 
       - name: Package arm64 RPM + DEB
         run: |
-          sed -e 's|^arch:.*|arch: arm64|' -e 's|src: ./dist/lab$|src: ./dist/lab-arm64|' nfpm.yaml > /tmp/nfpm-arm64.yaml
+          sed -e 's|^arch:.*|arch: arm64|' -e 's|src: ./dist/labctl$|src: ./dist/lab-arm64|' nfpm.yaml > /tmp/nfpm-arm64.yaml
           nfpm pkg --config /tmp/nfpm-arm64.yaml --packager rpm --target dist/
           nfpm pkg --config /tmp/nfpm-arm64.yaml --packager deb --target dist/
 

.gitignore

@@ -31,3 +31,9 @@ node_modules/
 # Asahi build artifacts (large)
 bastion/.asahi-cache/
 bastion/asahi-repo/*.zip
+
+# Regenerated by labsim/dualstack-lab.sh; derived state, not source.
+labsim/dualstack-evidence/
+
+# Runtime snapshots from labsim/cilium-ipam-switch.sh
+labsim/.ipam-switch-state/

bastion/completions/labctl.bash

@@ -40,8 +40,14 @@ _labctl() {
     "app labcontroller status")
       COMPREPLY=($(compgen -W "--user -h --help" -- "$cur"))
       return ;;
+    "app k3s evacuate")
+      COMPREPLY=($(compgen -W "--user --allow-single-replica -h --help" -- "$cur"))
+      return ;;
+    "app k3s admit")
+      COMPREPLY=($(compgen -W "--user -h --help" -- "$cur"))
+      return ;;
     "app k3s install")
-      COMPREPLY=($(compgen -W "--role --user --k3s-server --k3s-token -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "--role --user --k3s-server --k3s-token --k3s-version -h --help" -- "$cur"))
       return ;;
     "app k3s health")
       COMPREPLY=($(compgen -W "--user -h --help" -- "$cur"))
@@ -59,10 +65,10 @@ _labctl() {
       COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
       return ;;
     "provision install")
-      COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "--role --os --disk --schema-version --longhorn-disk --rancher-disk --vyos-mgmt --vyos-mgmt-address --vyos-bond --vyos-bond-address --vyos-bond-vrrp --vlan-vip --vyos-vrrp-priority --vyos-mgmt-vlan --vlan --vyos-password --vyos-hwid --vyos-fresh-config --vyos-bundle --vyos-api-key --vyos-api-listen -h --help" -- "$cur"))
       return ;;
     "provision reprovision")
-      COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "--role --os --disk --user -h --help" -- "$cur"))
       return ;;
     "provision debug")
       COMPREPLY=($(compgen -W "--pxe-boot -h --help" -- "$cur"))
@@ -101,7 +107,7 @@ _labctl() {
       COMPREPLY=($(compgen -W "deploy status -h --help" -- "$cur"))
       return ;;
     "app k3s")
-      COMPREPLY=($(compgen -W "install health list kubeconfig -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "evacuate admit install health list kubeconfig -h --help" -- "$cur"))
       return ;;
     "version")
       COMPREPLY=($(compgen -W "-h --help" -- "$cur"))

bastion/completions/labctl.fish

@@ -131,20 +131,39 @@ complete -c labctl -n "__labctl_using_cmd provision" -a makeiso -d 'Generate a U
 complete -c labctl -n "__labctl_using_cmd provision" -a recheck -d 'Refresh hardware info for all installed machines via SSH'
 
 # provision install options
-complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
-complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04'
-complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target disk device (auto-detect if omitted)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo'
+complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling'
+complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target OS disk: kernel name or stable /dev/disk/by-id/ path (auto-detect if omitted)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l schema-version -d 'Disk layout for a FRESH install (default 1). Ignored on reprovision: the layout already on the disk wins.' -xa '1 2'
+complete -c labctl -n "__labctl_in_cmd provision install" -l longhorn-disk -d 'Dedicated whole disk for /var/lib/longhorn, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l rancher-disk -d 'Dedicated whole disk for /var/lib/rancher, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt -d 'VyOS: untagged interface the machine PXE boots from (default eth0)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-address -d 'VyOS: CIDR for the management interface, or \'dhcp\' (default dhcp)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond -d 'VyOS: comma-separated LACP bond members (must exclude the PXE NIC)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond-address -d 'VyOS: address on the untagged bond (trunk native VLAN)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond-vrrp -d 'VyOS: VRRP VIP floated on the untagged bond' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vlan-vip -d 'VyOS: VRRP VIP for a --vlan entry (repeatable)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-vrrp-priority -d 'VyOS: VRRP priority for all groups on this box (higher = master)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-vlan -d 'VyOS: tagged management VLAN on the PXE port' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vlan -d 'VyOS: tagged VLAN sub-interface on the bond (repeatable)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-password -d 'VyOS: password for the \'vyos\' user' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-hwid -d 'VyOS: pin an interface name to a MAC via hw-id (repeatable)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-fresh-config -d 'VyOS: on reinstall, overwrite the preserved config with the generated one'
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bundle -d 'VyOS: apply a Pulumi-rendered bundle verbatim (kubernetes-deployment/infra/vyos/vyos-bundle.json). Replaces the derived --vyos-bond/--vlan/... config; secret values are left unset for `pulumi up`.' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-api-key -d 'VyOS: enable the HTTP API with this key so Pulumi can manage the router from first boot' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-api-listen -d 'VyOS: address the HTTP API binds to (default: the static management address). Required when management is DHCP; the API is never bound to all interfaces.' -x
 
 # provision reprovision options
-complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
-complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04'
+complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo'
+complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling'
 complete -c labctl -n "__labctl_in_cmd provision reprovision" -l disk -d 'Target disk device (auto-detect if omitted)' -x
+complete -c labctl -n "__labctl_in_cmd provision reprovision" -l user -d 'SSH user for the reboot (default: vyos for VyOS machines, else current user)' -x
 
 # provision debug options
 complete -c labctl -n "__labctl_in_cmd provision debug" -l pxe-boot -d 'Boot installed system via PXE (kernel+initrd from network, root from NVMe)'
 
 # provision register options
-complete -c labctl -n "__labctl_in_cmd provision register" -l role -d 'Machine role' -xa 'vanilla worker infra labcontroller'
+complete -c labctl -n "__labctl_in_cmd provision register" -l role -d 'Machine role' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo'
 complete -c labctl -n "__labctl_in_cmd provision register" -l ip -d 'Machine IP address' -x
 
 # provision logs options
@@ -187,16 +206,26 @@ complete -c labctl -n "__labctl_in_cmd app labcontroller deploy" -l crdb-replica
 complete -c labctl -n "__labctl_in_cmd app labcontroller status" -l user -d 'SSH user' -x
 
 # app k3s subcommands
+complete -c labctl -n "__labctl_using_cmd app k3s" -a evacuate -d 'Cordon, evict Longhorn replicas and drain so a node can be safely wiped'
+complete -c labctl -n "__labctl_using_cmd app k3s" -a admit -d 'Return an evacuated node to service: uncordon, clear eviction, restore its Longhorn disk'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a install -d 'Install k3s on a target machine (hostname, IP, or MAC)'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a health -d 'Check k3s health (all hosts if no target given)'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a list -d 'List installed machines and their k3s status'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a kubeconfig -d 'Fetch kubeconfig from a target and merge into ~/.kube/config'
 
+# app k3s evacuate options
+complete -c labctl -n "__labctl_in_cmd app k3s evacuate" -l user -d 'SSH user' -x
+complete -c labctl -n "__labctl_in_cmd app k3s evacuate" -l allow-single-replica -d 'permit evacuating volumes that have only one replica (rebuilds from the only copy; there is no backup target)'
+
+# app k3s admit options
+complete -c labctl -n "__labctl_in_cmd app k3s admit" -l user -d 'SSH user' -x
+
 # app k3s install options
-complete -c labctl -n "__labctl_in_cmd app k3s install" -l role -d 'k3s role: infra (server) or worker (agent)' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l role -d 'k3s role: infra, labcontroller, standalone-cluster, k8s-controller, k8s-combo (servers) or worker, k8s-worker (agents)' -x
 complete -c labctl -n "__labctl_in_cmd app k3s install" -l user -d 'SSH user' -x
-complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-server -d 'k3s server URL (required for worker role)' -x
-complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-token -d 'k3s join token (required for worker role)' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-server -d 'k3s server URL (required for agent roles, and to JOIN as a server)' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-token -d 'k3s join token; on a first server it SEEDS the cluster token' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-version -d 'exact k3s version, e.g. v1.34.6+k3s1. Unset installs LATEST, which on a rejoining node can be several minor versions ahead of the cluster' -x
 
 # app k3s health options
 complete -c labctl -n "__labctl_in_cmd app k3s health" -l user -d 'SSH user' -x

bastion/docs/ARCHITECTURE.md

@@ -89,83 +89,6 @@ Side paths:
 
 ---
 
-## Multi-architecture PXE
-
-The bastion serves both `x86_64` and `aarch64` over the network. Nothing about this is
-operator-configured -- there is no `--arch` flag, by design.
-
-### How a client's architecture is decided
-
-1. **DHCP option 93** (Client System Architecture) picks the *bootloader*. dnsmasq matches
-   it and hands out a matching iPXE binary:
-
-   | Option 93 | Client | Served |
-   |---|---|---|
-   | `0` | x86 BIOS | `undionly.kpxe` (TFTP) |
-   | `7`, `9` | x64 UEFI | `ipxe.efi` (TFTP) |
-   | `11` | **ARM64 UEFI** | `ipxe-arm64.efi` (TFTP) |
-   | `16` | x64 UEFI HTTP Boot | `http://…/ipxe.efi` |
-   | `19` | **ARM64 UEFI HTTP Boot** | `http://…/ipxe-arm64.efi` |
-
-   Values come from the IANA Processor Architecture Types registry. Note `19`, not `20` --
-   `20` is *pc/at bios boot from http*. EDK2/AAVMF prefers HTTP Boot over TFTP PXE, so the
-   iPXE binaries are staged in **both** `tftpDir` and `httpDir` (symlinked by `main.ts`).
-
-2. **`/dispatch` picks the kernel.** Option 93 never reaches the HTTP endpoint, so
-   `boot.ipxe` passes iPXE's own `${buildarch}` as `?arch=`. `resolveArch()` prefers, in
-   order: the tracked machine record → the reported `?arch=` → the configured default.
-   The record wins because it is what we observed on the machine itself.
-
-### Artifact naming
-
-`x86_64` keeps the original unsuffixed paths so its rendered iPXE scripts are unchanged;
-everything else is suffixed. `kernelPath()` / `initrdPath()` in `templates/boot.ipxe.ts`
-are the single source of truth, used by both the templates and `main.ts` staging.
-
-| arch | kernel | initrd |
-|---|---|---|
-| `x86_64` | `/vmlinuz` | `/initrd.img` |
-| `aarch64` | `/vmlinuz-aarch64` | `/initrd-aarch64.img` |
-
-`tests/ipxe-x86-regression.test.ts` pins the x86_64 output against a golden fixture.
-
-### arm64 gotchas
-
-- **LoadFile2 is mandatory.** arm64 has no `HdrS` boot protocol; the kernel's EFI stub
-  fetches the initrd over the UEFI `EFI_LOAD_FILE2_PROTOCOL`. An iPXE build without it
-  accepts the `initrd` line, silently drops it, and the kernel panics with
-  `VFS: Unable to mount root fs on unknown-block(0,0)`. Fedora's
-  `ipxe-bootimgs-aarch64` implements it; the integration test asserts this up front so
-  the failure names itself instead of looking like a disk problem.
-- **`nomodeset` is x86-only.** On arm64 there is no VGA path to fall back to. aarch64 gets
-  `console=tty0 console=ttyAMA0,115200` instead — the last `console=` wins for
-  `/dev/console`, so serial is the interactive one.
-- **Ubuntu is x86_64-only.** `releases.ubuntu.com` publishes no arm64 netboot artifacts.
-  `osSupportsArch()` encodes this, and both the install guard and `/dispatch` refuse the
-  combination rather than serving an x86 kernel to an ARM machine.
-
----
-
-## Onboarding classification (vendor OS)
-
-Machines carry an `onboard` field: `"pxe"` (default) or `"ssh"`, plus `vendor_os` naming
-what they run. `classifyOnboard()` in `@lab/shared` sets it from DMI identity, with known
-hardware also matched by MAC — a machine can sit in state for a long time with no DMI, and
-a DMI-only rule would fail open exactly where it matters.
-
-`onboard: "ssh"` means *we cannot rebuild this machine's OS*. Installs are refused at both
-entry points (`/api/install` and the labd `command-install` handler) with an error naming
-the machine and pointing at `provision debug`. **Rescue is never guarded** — being unable
-to reinstall a machine is precisely when a rescue shell is needed.
-
-This is a fact about the machine, not a blocklist. The refusal follows from "no image in
-our pipeline restores `vendor_os`", so adding a DGX OS image to the pipeline is what
-unblocks the DGX Sparks — no entry needs deleting.
-
-Current classifications: NVIDIA DGX Spark (`spark-2935`, `spark-3a1c`) → `dgx-os`.
-
----
-
 ## Packages
 
 ### Monorepo Structure
@@ -481,36 +404,6 @@ Hardcoded `/dev/sda` default broke NVMe-only machines. Fix: default to empty str
 ### Anaconda Rescue Mode Limitations
 `%pre` and `%post` sections do not execute in `inst.rescue` mode. SSH in rescue mode is provided by Anaconda's `inst.sshd` kernel parameter + `sshpw` kickstart directive. Manual setup via `curl bastion:8080/debug-setup.sh | bash` for nc listener.
 
-**Unresolved (2026-08-11): rescue SSH has never been observed working.** Adding the first
-integration coverage for `provision debug` (`tests/integration/pxe-rescue.test.ts`) showed the
-rescue environment coming up correctly — the bastion serves the kernel and initrd, Anaconda
-boots, fetches `debug.ks`, and reaches its installer environment — but **nothing ever listens on
-port 22**.
-
-Strength of the evidence, stated precisely because it decides where to look next:
-- **aarch64 — direct.** Port 22 probed every 20s for 30 minutes while the Anaconda installer
-  environment was demonstrably running (NetworkManager, polkitd, rsyslog on the console). Never
-  opened.
-- **x86_64 — corroborating, not conclusive.** One clean KVM run (943s) where SSH never became
-  available inside a 15-minute budget. That VM's progress into the rescue environment was *not*
-  observed — vitest's final reporter discards the streamed log — so it is consistent with the
-  aarch64 result but does not independently prove it. Re-run with `KEEP_VM=1` and probe port 22
-  directly to settle it.
-
-If the x86_64 result holds up, this is orthogonal to the multi-architecture work, since x86_64 is
-untouched by it. Leads worth checking, in order:
-- Does `inst.sshd` actually start `sshd` in `inst.rescue` mode, or only in install mode? The
-  port never opens, so this is the prime suspect — an auth problem would still show an open port.
-- `sshkey` may apply only to the *installed* system, leaving the installer environment
-  password-only via `sshpw`. That would matter once sshd does listen: the test authenticates
-  key-only (`BatchMode=yes`).
-- The `%anaconda`-context directives in `debug.ks` may be skipped entirely when a kickstart is
-  supplied alongside `inst.rescue`.
-
-Until this is resolved, `provision debug` gets you a booted rescue environment on the console
-(including on arm64), but not an SSH shell. The `debug-setup.sh` nc-listener path is the
-documented workaround and is unaffected.
-
 ---
 
 ## Planned Work (Taskmaster)

bastion/package.json

@@ -21,14 +21,10 @@
     "test:integration:pxe:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'PXE boot'",
     "test:integration:iso": "vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'",
     "test:integration:iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'",
+    "test:integration:vyos": "vitest run -c tests/integration/vitest.config.ts -t 'VyOS provisioning'",
+    "test:integration:vyos:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'VyOS provisioning'",
     "test:integration:arm-iso": "vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'",
     "test:integration:arm-iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'",
-    "test:integration:rescue": "vitest run -c tests/integration/vitest.config.ts -t 'x86 rescue boot'",
-    "test:integration:rescue:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'x86 rescue boot'",
-    "test:integration:arm-pxe": "vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE rescue'",
-    "test:integration:arm-pxe:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE rescue'",
-    "test:integration:arm-pxe-full": "ARM_PXE_FULL=1 vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE'",
-    "test:integration:arm-pxe-full:host": "sudo -E ARM_PXE_FULL=1 $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE'",
     "test:integration:asahi": "vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'",
     "test:integration:asahi:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'",
     "test:integration:asahi-validate": "vitest run -c tests/integration/vitest.config.ts -t 'asahi.*validation'",

bastion/scripts/test-provision.sh

@@ -2,19 +2,16 @@
 # Run PXE and/or ISO boot integration tests.
 #
 # Usage:
-#   sudo ./scripts/test-provision.sh              # run PXE + ISO (x86_64)
-#   sudo ./scripts/test-provision.sh pxe          # PXE only
-#   sudo ./scripts/test-provision.sh iso          # ISO only (x86_64)
-#   sudo ./scripts/test-provision.sh rescue       # x86_64 Anaconda rescue boot + SSH (~15min)
-#   sudo ./scripts/test-provision.sh arm          # ARM ISO boot (emulated, SLOW ~60min)
-#   sudo ./scripts/test-provision.sh arm-pxe      # ARM network PXE rescue: NBP + rescue over SSH (~25-30min)
-#   sudo ./scripts/test-provision.sh arm-pxe-full # ARM network PXE incl. discover + full install (~75-95min)
-#   sudo ./scripts/test-provision.sh all          # all tests including ARM
+#   sudo ./scripts/test-provision.sh          # run PXE + ISO (x86_64)
+#   sudo ./scripts/test-provision.sh pxe      # PXE only
+#   sudo ./scripts/test-provision.sh iso      # ISO only (x86_64)
+#   sudo ./scripts/test-provision.sh arm      # ARM ISO boot (emulated, SLOW ~60min)
+#   sudo ./scripts/test-provision.sh all      # all tests including ARM
 #
 # Prerequisites:
 #   libvirtd, OVMF (edk2-ovmf), iPXE (ipxe-bootimgs-x86),
 #   dnsmasq, xorriso, mtools, virt-install, qemu-img
-#   ARM: qemu-system-aarch64, edk2-aarch64, ipxe-bootimgs-aarch64
+#   ARM: qemu-system-aarch64, edk2-aarch64
 set -e
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
@@ -61,10 +58,6 @@ if [ ! -f /usr/share/edk2/ovmf/OVMF_CODE.fd ]; then
     exit 1
 fi
 
-MODE="${1:-both}"
-
-# iPXE binaries are per-architecture. x86_64 is always required (the dnsmasq config
-# references it); arm64 only for the ARM network-PXE modes.
 IPXE_EFI=""
 for f in /usr/share/ipxe/ipxe-snponly-x86_64.efi /usr/share/ipxe/ipxe-snp-x86_64.efi /usr/share/ipxe/ipxe-x86_64.efi; do
     [ -f "$f" ] && IPXE_EFI="$f" && break
@@ -74,20 +67,6 @@ if [ -z "$IPXE_EFI" ]; then
     exit 1
 fi
 
-IPXE_EFI_ARM64=""
-for f in /usr/share/ipxe/arm64-efi/snponly.efi /usr/share/ipxe/arm64-efi/ipxe.efi; do
-    [ -f "$f" ] && IPXE_EFI_ARM64="$f" && break
-done
-
-case "$MODE" in
-    arm-pxe|arm-pxe-full|all)
-        if [ -z "$IPXE_EFI_ARM64" ] && [ "$MODE" != "all" ]; then
-            echo -e "${RED}arm64 iPXE binary not found.${RESET} Install: sudo dnf install ipxe-bootimgs-aarch64"
-            exit 1
-        fi
-        ;;
-esac
-
 # Find SSH key
 SSH_KEY=""
 for name in id_ed25519 id_ecdsa id_rsa; do
@@ -104,19 +83,10 @@ fi
 echo -e "  User:    ${BOLD}$REAL_USER${RESET}"
 echo -e "  SSH key: ${BOLD}$SSH_KEY${RESET}"
 echo -e "  iPXE:    ${BOLD}$IPXE_EFI${RESET}"
-echo -e "  iPXE a64:${BOLD} ${IPXE_EFI_ARM64:-not installed}${RESET}"
 echo ""
 
-require_arm_emulation() {
-    if ! command -v qemu-system-aarch64 &>/dev/null; then
-        echo -e "${RED}qemu-system-aarch64 not found.${RESET} Install: sudo dnf install qemu-system-aarch64 edk2-aarch64"
-        exit 1
-    fi
-    if [ ! -f /usr/share/edk2/aarch64/QEMU_EFI.fd ]; then
-        echo -e "${RED}AAVMF firmware not found.${RESET} Install: sudo dnf install edk2-aarch64"
-        exit 1
-    fi
-}
+# --- Determine which tests to run ---
+MODE="${1:-both}"
 
 run_test() {
     local name="$1" pattern="$2"
@@ -146,26 +116,13 @@ case "$MODE" in
         run_test "ISO boot" "ISO boot" || FAILED=1
         ;;
     arm|arm-iso)
-        require_arm_emulation
+        if ! command -v qemu-system-aarch64 &>/dev/null; then
+            echo -e "${RED}qemu-system-aarch64 not found.${RESET} Install: sudo dnf install qemu-system-aarch64 edk2-aarch64"
+            exit 1
+        fi
         echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 30-60 minutes.${RESET}"
         run_test "ARM ISO boot" "ARM ISO" || FAILED=1
         ;;
-    rescue)
-        echo -e "${YELLOW}x86_64 rescue boot (KVM). Expect ~15 minutes.${RESET}"
-        run_test "x86 rescue boot" "x86 rescue boot" || FAILED=1
-        ;;
-    arm-pxe)
-        require_arm_emulation
-        echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 25-30 minutes.${RESET}"
-        echo -e "${YELLOW}Covers option 93 -> arm64 NBP, arch resolution, and rescue over SSH.${RESET}"
-        echo -e "${YELLOW}For the full install too, use: $0 arm-pxe-full${RESET}"
-        run_test "ARM PXE rescue" "ARM PXE rescue" || FAILED=1
-        ;;
-    arm-pxe-full)
-        require_arm_emulation
-        echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 75-95 minutes.${RESET}"
-        ARM_PXE_FULL=1 run_test "ARM PXE (rescue + install)" "ARM PXE" || FAILED=1
-        ;;
     both)
         run_test "PXE boot" "PXE boot" || FAILED=1
         run_test "ISO boot" "ISO boot" || FAILED=1
@@ -176,17 +133,12 @@ case "$MODE" in
         if command -v qemu-system-aarch64 &>/dev/null; then
             echo -e "${YELLOW}ARM emulation is ~10x slower than native.${RESET}"
             run_test "ARM ISO boot" "ARM ISO" || FAILED=1
-            if [ -n "$IPXE_EFI_ARM64" ]; then
-                run_test "ARM PXE rescue" "ARM PXE rescue" || FAILED=1
-            else
-                echo -e "${YELLOW}Skipping ARM PXE test (ipxe-bootimgs-aarch64 not installed)${RESET}"
-            fi
         else
-            echo -e "${YELLOW}Skipping ARM tests (qemu-system-aarch64 not installed)${RESET}"
+            echo -e "${YELLOW}Skipping ARM test (qemu-system-aarch64 not installed)${RESET}"
         fi
         ;;
     *)
-        echo "Usage: $0 [pxe|iso|rescue|arm|arm-pxe|arm-pxe-full|both|all]"
+        echo "Usage: $0 [pxe|iso|arm|both|all]"
         exit 1
         ;;
 esac

bastion/src/bastion/src/config.ts

@@ -20,7 +20,22 @@ export function loadConfig(overrides: Partial<BastionConfig> = {}): BastionConfi
   const ubuntuMirror = overrides.ubuntuMirror ?? process.env["UBUNTU_MIRROR"]
     ?? `https://releases.ubuntu.com/${ubuntuVersion}`;
 
-  const fedoraMirror = `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`;
+  // "latest" resolves the newest nightly ISO from the vyos-nightly-build GitHub
+  // releases at startup. downloads.vyos.io no longer serves direct rolling ISOs
+  // (it returns the vyos.io site, and nightly builds sit behind a signup form);
+  // GitHub releases are the remaining free, unauthenticated direct source.
+  // LTS ISOs are subscription-only. Set VYOS_ISO_URL to pin a specific build.
+  const vyosIsoUrl = overrides.vyosIsoUrl ?? process.env["VYOS_ISO_URL"] ?? "latest";
+  const vyosDefaultPassword = overrides.vyosDefaultPassword
+    ?? process.env["VYOS_DEFAULT_PASSWORD"] ?? "vyos";
+
+  // Overridable like ubuntuMirror and vyosIsoUrl above -- it was the only one
+  // of the three hardcoded. Without this, every integration-test install pulls
+  // the full package set from download.fedoraproject.org, which makes a
+  // multi-VM provisioning matrix impractical (~25 min per install, serialised).
+  // Point FEDORA_MIRROR at a local mirror or caching proxy for test runs.
+  const fedoraMirror = overrides.fedoraMirror ?? process.env["FEDORA_MIRROR"]
+    ?? `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`;
   const tftpDir = `${bastionDir}/tftp`;
   const httpDir = `${bastionDir}/http`;
   const stateFile = `${bastionDir}/state.json`;
@@ -38,6 +53,8 @@ export function loadConfig(overrides: Partial<BastionConfig> = {}): BastionConfi
     dhcpRangeEnd,
     ubuntuVersion,
     ubuntuMirror,
+    vyosIsoUrl,
+    vyosDefaultPassword,
     // These are populated at runtime by the network service
     iface: overrides.iface ?? "",
     serverIp: overrides.serverIp ?? "",
@@ -48,6 +65,7 @@ export function loadConfig(overrides: Partial<BastionConfig> = {}): BastionConfi
     syslogPort,
     skipDnsmasq: overrides.skipDnsmasq,
     skipArtifacts: overrides.skipArtifacts,
+    autoK3s: overrides.autoK3s ?? process.env["AUTO_K3S"] !== "false",
     labdUrl: overrides.labdUrl ?? process.env["LABD_URL"],
     bastionJoinToken: overrides.bastionJoinToken ?? process.env["BASTION_JOIN_TOKEN"],
     fedoraMirror,

bastion/src/bastion/src/main.ts

@@ -3,9 +3,8 @@
 
 import { mkdirSync, writeFileSync, readFileSync, existsSync, copyFileSync, symlinkSync, unlinkSync } from "node:fs";
 import { execSync } from "node:child_process";
-import type { Arch, BastionConfig } from "@lab/shared";
-import { SUPPORTED_ARCHES, fedoraMirrorFor, classifyOnboard } from "@lab/shared";
-import { kernelPath, initrdPath } from "./templates/boot.ipxe.js";
+import type { BastionConfig } from "@lab/shared";
+import { isRole, validateDataDisks } from "@lab/shared";
 import { loadConfig } from "./config.js";
 import { populateNetworkConfig } from "./services/network.js";
 import { createApp } from "./server.js";
@@ -15,7 +14,6 @@ import { renderBootIpxe } from "./templates/boot.ipxe.js";
 import { logger } from "./services/logger.js";
 import { BastionConnection } from "./services/labd-connection.js";
 import { progressBus } from "./services/progress-events.js";
-import { checkInstallAllowed } from "./services/install-guard.js";
 import { ensureBootIso } from "./routes/boot-iso.js";
 
 function copyIfMissing(src: string, dest: string, label: string): void {
@@ -43,6 +41,125 @@ function download(url: string, dest: string, label: string): void {
   }
 }
 
+/**
+ * Pick the largest regular-file initrd from an `xorriso -lsl` listing.
+ *
+ * /live carries decoys: a 0-byte initrd.img placeholder on some images, or an
+ * initrd.img SYMLINK to the real version-suffixed file on others. Parsing is
+ * field-based (ls -l layout: perms links uid gid size month day time 'name')
+ * and considers only lines whose mode string marks a regular file — symlinks
+ * report their link size, not the target's, and must not win.
+ */
+export function pickLargestInitrd(
+  listing: string,
+): { name: string; size: number } | undefined {
+  let best: { name: string; size: number } | undefined;
+  for (const line of listing.split("\n")) {
+    if (!line.startsWith("-")) continue;          // regular files only
+    const quoted = /'([^']+)'/.exec(line);
+    const fields = line.trim().split(/\s+/);
+    const size = parseInt(fields[4] ?? "", 10);
+    const name = quoted?.[1] ?? "";
+    if (!name.startsWith("initrd")) continue;
+    if (!Number.isFinite(size) || size <= 0) continue;
+    if (best === undefined || size > best.size) {
+      best = { name, size };
+    }
+  }
+  return best;
+}
+
+const VYOS_NIGHTLY_RELEASES =
+  "https://api.github.com/repos/vyos/vyos-nightly-build/releases/latest";
+
+/**
+ * Resolve the configured VyOS ISO URL, expanding the "latest" sentinel.
+ *
+ * The nightly asset filename embeds a build date, so there is no stable
+ * "latest.iso" path to hardcode — the newest release has to be looked up.
+ * Any other value is used verbatim, which is how VYOS_ISO_URL pins a build
+ * or points at a locally mirrored copy.
+ */
+function resolveVyosIsoUrl(configured: string): string {
+  if (configured !== "latest") return configured;
+
+  const body = execSync(`curl -sSfL "${VYOS_NIGHTLY_RELEASES}"`, {
+    encoding: "utf-8",
+    stdio: ["pipe", "pipe", "pipe"],
+  });
+  const release = JSON.parse(body) as {
+    tag_name?: string;
+    assets?: Array<{ name: string; browser_download_url: string }>;
+  };
+
+  const asset = (release.assets ?? []).find((a) =>
+    /generic-amd64\.iso$/.test(a.name),
+  );
+  if (!asset) {
+    throw new Error(
+      `No generic-amd64 ISO asset in VyOS nightly release ${release.tag_name ?? "?"}`,
+    );
+  }
+
+  logger.info(`  VyOS ISO resolved to ${asset.name} (${release.tag_name ?? "?"})`);
+  return asset.browser_download_url;
+}
+
+/**
+ * Extract VyOS netboot artifacts from the release ISO.
+ *
+ * VyOS publishes no netboot bundle, so kernel/initrd/squashfs have to come out
+ * of the ISO. xorriso is already in the bastion image (used for boot.iso) and
+ * extracts without root or a loop mount.
+ *
+ * The initrd needs care: /live contains an empty initrd.img placeholder
+ * alongside the real one, which carries a version-suffixed name. Booting the
+ * 0-byte file fails with no useful diagnostic, so pick the largest initrd*.
+ */
+export function prepareVyosArtifacts(config: BastionConfig): void {
+  const kernel = `${config.httpDir}/vyos-vmlinuz`;
+  const initrd = `${config.httpDir}/vyos-initrd`;
+  const squashfs = `${config.httpDir}/vyos-filesystem.squashfs`;
+
+  if (existsSync(kernel) && existsSync(initrd) && existsSync(squashfs)) {
+    logger.info("  VyOS netboot artifacts -- cached");
+    return;
+  }
+
+  const iso = `${config.bastionDir}/vyos.iso`;
+  download(resolveVyosIsoUrl(config.vyosIsoUrl), iso, "VyOS ISO");
+
+  const extract = (isoPath: string, dest: string, label: string): void => {
+    execSync(
+      `xorriso -osirrox on -indev "${iso}" -extract "${isoPath}" "${dest}"`,
+      { stdio: "pipe" },
+    );
+    logger.info(`  ${label} -- extracted from ${isoPath}`);
+  };
+
+  extract("/live/vmlinuz", kernel, "VyOS kernel");
+  extract("/live/filesystem.squashfs", squashfs, "VyOS squashfs");
+
+  // Pick the real initrd by size from the ISO's own directory listing.
+  const listing = execSync(`xorriso -indev "${iso}" -lsl /live/ --`, {
+    encoding: "utf-8",
+    stdio: ["pipe", "pipe", "pipe"],
+  });
+
+  const best = pickLargestInitrd(listing);
+  if (best === undefined) {
+    throw new Error("No non-empty initrd found in /live on the VyOS ISO");
+  }
+  extract(`/live/${best.name}`, initrd, `VyOS initrd (${best.name}, ${best.size} bytes)`);
+
+  // The ISO is only needed to produce the three artifacts above.
+  try {
+    unlinkSync(iso);
+  } catch {
+    // Non-fatal: leaving it costs disk but nothing else.
+  }
+}
+
 function symlinkSafe(target: string, linkPath: string): void {
   try {
     symlinkSync(target, linkPath);
@@ -133,14 +250,9 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
   mkdirSync(config.tftpDir, { recursive: true });
   mkdirSync(config.httpDir, { recursive: true });
 
-  // Architectures we can actually network boot, reported in the banner so a missing
-  // arm64 payload is visible at startup instead of at 2am when a rescue is needed.
-  const bootArches: Arch[] = [];
-  let ipxeArm64Ready = false;
-
   // Prepare boot artifacts
   if (config.skipArtifacts !== true) {
-    logger.info(`Preparing boot artifacts (Fedora ${config.fedoraVersion}, ${SUPPORTED_ARCHES.join(" + ")})...`);
+    logger.info(`Preparing boot artifacts (Fedora ${config.fedoraVersion} ${config.arch})...`);
 
     copyIfMissing(
       "/usr/share/ipxe/undionly.kpxe",
@@ -158,41 +270,20 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
         `${config.tftpDir}/ipxe-arm64.efi`,
         "iPXE UEFI arm64",
       );
-      ipxeArm64Ready = true;
     } catch {
-      logger.warn("arm64 iPXE not available -- arm64 machines cannot network boot.");
-      logger.warn("  Install with: sudo dnf install ipxe-bootimgs-aarch64");
+      logger.warn("arm64 iPXE not available -- skipping");
     }
 
-    // Fedora pxeboot kernel + initrd per architecture. x86_64 keeps the unsuffixed
-    // names it has always used; other architectures are suffixed. The iPXE templates
-    // resolve the same paths via kernelPath()/initrdPath().
-    for (const arch of SUPPORTED_ARCHES) {
-      const mirror = fedoraMirrorFor(config.fedoraVersion, arch);
-      try {
-        download(
-          `${mirror}/images/pxeboot/vmlinuz`,
-          `${config.httpDir}${kernelPath(arch)}`,
-          `Fedora ${arch} kernel`,
-        );
-        download(
-          `${mirror}/images/pxeboot/initrd.img`,
-          `${config.httpDir}${initrdPath(arch)}`,
-          `Fedora ${arch} initrd`,
-        );
-        bootArches.push(arch);
-      } catch (err) {
-        // Non-fatal: a bastion with no arm64 artifacts still serves x86_64 fine.
-        // Failing startup over an unreachable mirror for an architecture that may not
-        // even be present on this network would be worse.
-        logger.warn(`Fedora ${arch} kernel/initrd unavailable -- ${arch} PXE disabled`);
-        logger.warn(`  ${err instanceof Error ? err.message : String(err)}`);
-      }
-    }
-
-    if (!bootArches.includes("x86_64")) {
-      throw new Error("Fedora x86_64 kernel/initrd could not be staged -- cannot serve PXE");
-    }
+    download(
+      `${config.fedoraMirror}/images/pxeboot/vmlinuz`,
+      `${config.httpDir}/vmlinuz`,
+      "Fedora kernel",
+    );
+    download(
+      `${config.fedoraMirror}/images/pxeboot/initrd.img`,
+      `${config.httpDir}/initrd.img`,
+      "Fedora initrd",
+    );
 
     // Ubuntu netboot artifacts (non-fatal — Ubuntu version may not be released yet)
     try {
@@ -211,6 +302,17 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
       logger.warn(`Ubuntu ${config.ubuntuVersion} artifacts not available -- Ubuntu provisioning disabled`);
     }
 
+    // VyOS netboot artifacts (non-fatal — same policy as Ubuntu)
+    try {
+      logger.info("Preparing VyOS netboot artifacts...");
+      prepareVyosArtifacts(config);
+    } catch (err) {
+      logger.warn(
+        `VyOS artifacts not available -- VyOS provisioning disabled ` +
+        `(${err instanceof Error ? err.message : String(err)})`,
+      );
+    }
+
     // Symlink iPXE binaries into HTTP dir for UEFI HTTP Boot
     for (const name of ["ipxe.efi", "ipxe-arm64.efi"]) {
       const src = `${config.tftpDir}/${name}`;
@@ -283,12 +385,21 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
     // Wire up command handlers so labd can send install/forget/role commands
     labdConn.onCommand("command-install", async (msg) => {
       if (msg.type !== "command-install") throw new Error("unexpected");
-      const installMac = msg.mac.toLowerCase().replace(/-/g, ":");
-      const osId = (msg.os as import("@lab/shared").OsId | undefined) ?? "fedora-43";
-      const check = checkInstallAllowed(state.load(), installMac, osId);
-      if (check.allowed === false) {
-        logger.warn(`INSTALL REFUSED: ${installMac} -- ${check.error}`);
-        return { status: "error", error: check.error };
+      // Same guard as POST /api/install: disks named by an operator must
+      // never silently degrade into LVs because they arrived over the socket.
+      const hasDataDisks = msg.dataDisks !== undefined
+        && Object.values(msg.dataDisks).some((d) => d !== undefined);
+      if (hasDataDisks) {
+        if ((msg.os ?? "fedora-43") !== "fedora-43") {
+          return { status: "error", error: `dataDisks are only supported for fedora installs (got os '${msg.os}')` };
+        }
+        if (!isRole(msg.role)) {
+          return { status: "error", error: `invalid role: '${msg.role}'` };
+        }
+        const problem = validateDataDisks(msg.role, msg.dataDisks!);
+        if (problem !== null) {
+          return { status: "error", error: `invalid dataDisks: ${problem}` };
+        }
       }
       state.update((s) => {
         s.install_queue[msg.mac] = {
@@ -297,6 +408,9 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
           role: msg.role as import("@lab/shared").Role,
           os: msg.os as import("@lab/shared").OsId,
           queued_at: new Date().toISOString(),
+          ...(msg.schemaVersion ? { schemaVersion: msg.schemaVersion } : {}),
+          ...(hasDataDisks ? { dataDisks: msg.dataDisks } : {}),
+          ...(msg.vyos ? { vyos: msg.vyos } : {}),
         };
       });
       return { status: "ok", data: { mac: msg.mac, hostname: msg.hostname } };
@@ -350,24 +464,13 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
       const mac = (msg.mac as string).toLowerCase();
       const now = new Date().toISOString();
       const existing = state.load().discovered[mac];
-      const identity = {
-        mac,
-        manufacturer: (msg.manufacturer as string) ?? "unknown",
-        product: (msg.product as string) ?? "unknown",
-        board: (msg.board as string) ?? "unknown",
-        ...(existing?.onboard !== undefined ? { onboard: existing.onboard } : {}),
-        ...(existing?.vendor_os !== undefined ? { vendor_os: existing.vendor_os } : {}),
-      };
-      const onboarding = classifyOnboard(identity);
-      const rootDevice = msg.root_device ?? existing?.root_device;
-      const rootArgs = msg.root_args ?? existing?.root_args;
       state.update((s) => {
         s.discovered[mac] = {
           mac,
-          product: identity.product,
-          board: identity.board,
+          product: (msg.product as string) ?? "unknown",
+          board: (msg.board as string) ?? "unknown",
           serial: (msg.serial as string) ?? "unknown",
-          manufacturer: identity.manufacturer,
+          manufacturer: (msg.manufacturer as string) ?? "unknown",
           cpu_model: (msg.cpu_model as string) ?? "unknown",
           cpu_cores: (msg.cpu_cores as number) ?? 0,
           memory_gb: (msg.memory_gb as number) ?? 0,
@@ -376,20 +479,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
           nics: (msg.nics as Array<{ name: string; mac: string; state: string }>) ?? [],
           first_seen: existing?.first_seen ?? now,
           last_seen: now,
-          onboard: onboarding.onboard,
-          ...(onboarding.vendor_os !== undefined ? { vendor_os: onboarding.vendor_os } : {}),
-          ...(rootDevice !== undefined ? { root_device: rootDevice } : {}),
-          ...(rootArgs !== undefined ? { root_args: rootArgs } : {}),
         };
-        // Keep the installed record in step -- the guard and --pxe-boot both read it.
-        const inst = s.installed[mac];
-        if (inst) {
-          inst.arch = (msg.arch as string) ?? inst.arch;
-          inst.onboard = onboarding.onboard;
-          if (onboarding.vendor_os !== undefined) inst.vendor_os = onboarding.vendor_os;
-          if (rootDevice !== undefined) inst.root_device = rootDevice;
-          if (rootArgs !== undefined) inst.root_args = rootArgs;
-        }
       });
       logger.info(`HARDWARE UPDATED: ${mac} -- ${msg.manufacturer ?? "?"} ${msg.product ?? "?"} (${msg.cpu_model ?? "?"}, ${msg.cpu_cores ?? "?"} cores, ${msg.memory_gb ?? "?"}GB RAM)`);
       return { status: "ok", data: { mac } };
@@ -424,7 +514,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
   }
 
   // Print banner
-  printBanner(config, bootArches, ipxeArm64Ready);
+  printBanner(config);
 
   // Graceful shutdown
   const shutdown = async (): Promise<void> => {
@@ -446,22 +536,11 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
   await new Promise(() => {});
 }
 
-function printBanner(config: BastionConfig, bootArches: Arch[], ipxeArm64Ready: boolean): void {
+function printBanner(config: BastionConfig): void {
   const dhcpInfo = config.dhcpMode === "full"
     ? `full (${config.dhcpRangeStart}-${config.dhcpRangeEnd})`
     : "proxy (alongside existing DHCP)";
 
-  // arm64 needs both an iPXE binary (DHCP hands it out on option 93 = 0x0b) and a
-  // kernel/initrd pair. Report the combination, since either missing breaks it.
-  const archInfo = config.skipArtifacts === true
-    ? "(artifacts skipped)"
-    : SUPPORTED_ARCHES
-        .map((a) => {
-          const ready = bootArches.includes(a) && (a !== "aarch64" || ipxeArm64Ready);
-          return ready ? a : `${a} (unavailable)`;
-        })
-        .join(", ");
-
   console.log("");
   console.log("\x1b[36m\x1b[1m" + "=".repeat(60) + "\x1b[0m");
   console.log("\x1b[36m\x1b[1m  Lab PXE Bastion -- Discovery Mode\x1b[0m");
@@ -470,8 +549,7 @@ function printBanner(config: BastionConfig, bootArches: Arch[], ipxeArm64Ready:
   console.log(`  Network:   \x1b[1m${config.network}/24\x1b[0m via \x1b[1m${config.iface}\x1b[0m`);
   console.log(`  DHCP:      \x1b[1m${dhcpInfo}\x1b[0m`);
   console.log(`  HTTP:      \x1b[1mhttp://${config.serverIp}:${config.httpPort}/\x1b[0m`);
-  console.log(`  OS:        \x1b[1mFedora ${config.fedoraVersion}\x1b[0m`);
-  console.log(`  Net boot:  \x1b[1m${archInfo}\x1b[0m`);
+  console.log(`  OS:        \x1b[1mFedora ${config.fedoraVersion} (${config.arch})\x1b[0m`);
   console.log(`  Domain:    \x1b[1m${config.domain}\x1b[0m`);
   console.log(`  State:     \x1b[1m${config.stateFile}\x1b[0m`);
   console.log("");

bastion/src/bastion/src/routes/api.ts

@@ -5,19 +5,27 @@
 // /api/discover  - receive hardware discovery reports from PXE-booted machines
 
 import type { FastifyInstance } from "fastify";
-import type { HardwareInfo, InstalledInfo, Role } from "@lab/shared";
-import { isValidOsId, SUPPORTED_ROLES, classifyOnboard } from "@lab/shared";
+import type { BastionConfig, HardwareInfo, InstalledInfo, Role, VyosInstallSpec } from "@lab/shared";
+import { isValidOsId, SUPPORTED_ROLES, SUPPORTED_OS } from "@lab/shared";
 import type { StateManager } from "../services/state.js";
 import { logger } from "../services/logger.js";
 import { triggerPostProvisionK3s } from "../services/post-provision.js";
-import { checkInstallAllowed } from "../services/install-guard.js";
 import { progressBus } from "../services/progress-events.js";
 import type { ProgressEvent } from "../services/progress-events.js";
 import type { InstallLogBuffer } from "../services/install-log.js";
 import type { SyslogListener } from "../services/syslog-listener.js";
+import { isRole, requiresSchemaV2, validateDataDisks, type DataDisks } from "@lab/shared";
+
+/**
+ * Seconds after dispatch with zero progress before a machine is called stalled.
+ * Generous: the slowest legitimate gap is fetching a ~600MB VyOS squashfs over
+ * HTTP before the hook can report anything.
+ */
+const STALL_THRESHOLD_S = 8 * 60;
 
 export function registerApiRoutes(
   app: FastifyInstance,
+  config: BastionConfig,
   state: StateManager,
   installLog: InstallLogBuffer,
   syslog: SyslogListener,
@@ -35,9 +43,12 @@ export function registerApiRoutes(
       disk?: string;
       role?: string;
       os?: string;
+      vyos?: VyosInstallSpec;
+      schemaVersion?: number;
+      dataDisks?: DataDisks;
     };
   }>("/api/install", async (request, reply) => {
-    const { mac: rawMac, hostname, disk, role, os } = request.body ?? {};
+    const { mac: rawMac, hostname, disk, role, os, vyos, schemaVersion, dataDisks } = request.body ?? {};
     const mac = (rawMac ?? "").toLowerCase().replace(/-/g, ":");
 
     if (mac === "") {
@@ -49,15 +60,54 @@ export function registerApiRoutes(
       return reply.status(400).send({ error: `invalid role: '${validRole}'. Supported: ${SUPPORTED_ROLES.join(", ")}` });
     }
 
-    const osId = os ?? "fedora-43";
-    if (!isValidOsId(osId)) {
-      return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: fedora-43, ubuntu-26.04` });
+    // Reject an unknown schema rather than silently installing v1: someone
+    // asking for v3 wants a layout we do not have, and quietly giving them the
+    // old one is how a fleet ends up with layouts nobody chose.
+    if (schemaVersion !== undefined && schemaVersion !== 1 && schemaVersion !== 2) {
+      return reply.status(400).send({ error: `invalid schemaVersion: ${schemaVersion}. Supported: 1, 2` });
     }
 
-    const check = checkInstallAllowed(state.load(), mac, osId);
-    if (check.allowed === false) {
-      logger.warn(`INSTALL REFUSED: ${mac} -- ${check.error}`);
-      return reply.status(409).send({ error: check.error });
+    // Only an EXPLICIT v1 is a contradiction. Omitting the schema is normal --
+    // a reinstall never passes one, because the layout already on the disk wins.
+    if (schemaVersion === 1 && isRole(validRole) && requiresSchemaV2(validRole)) {
+      return reply.status(400).send({
+        error: `role '${validRole}' requires schemaVersion 2 (it has no v1 disk layout)`,
+      });
+    }
+
+    const osId = os ?? "fedora-43";
+    if (!isValidOsId(osId)) {
+      return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: ${SUPPORTED_OS.join(", ")}` });
+    }
+
+    // Dedicated data disks are rendered by the kickstart generator only, and
+    // only in the v2 layout. Rejected here rather than silently dropped at
+    // render time -- an operator who named disks must not get LVs instead.
+    const hasDataDisks = dataDisks !== undefined
+      && Object.values(dataDisks).some((d) => d !== undefined);
+    if (hasDataDisks) {
+      if (osId !== "fedora-43") {
+        return reply.status(400).send({
+          error: `dataDisks are only supported for fedora installs (got os '${osId}')`,
+        });
+      }
+      if (schemaVersion === 1) {
+        return reply.status(400).send({
+          error: "dataDisks require schemaVersion 2 -- the v1 layout is frozen and single-disk",
+        });
+      }
+      if (!isRole(validRole)) {
+        return reply.status(400).send({ error: `invalid role: '${validRole}'` });
+      }
+      if (!requiresSchemaV2(validRole) && schemaVersion === undefined) {
+        return reply.status(400).send({
+          error: `role '${validRole}' defaults to the v1 layout -- pass schemaVersion 2 to use dataDisks`,
+        });
+      }
+      const problem = validateDataDisks(validRole, dataDisks as DataDisks);
+      if (problem !== null) {
+        return reply.status(400).send({ error: `invalid dataDisks: ${problem}` });
+      }
     }
 
     state.update((s) => {
@@ -67,6 +117,9 @@ export function registerApiRoutes(
         role: validRole as Role,
         os: osId,
         queued_at: new Date().toISOString(),
+        ...(schemaVersion ? { schemaVersion: schemaVersion as 1 | 2 } : {}),
+        ...(hasDataDisks ? { dataDisks } : {}),
+        ...(vyos ? { vyos } : {}),
       };
     });
 
@@ -165,12 +218,31 @@ export function registerApiRoutes(
           };
           s.installed[mac] = installedInfo;
 
-          const admin = installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "lab" : "root";
+          // VyOS: the only login user is "vyos", and a router never runs k3s —
+          // without this guard a non-vanilla role + recorded IP would trigger
+          // the k3s post-provision against a VyOS box.
+          const isVyos = (installedInfo.os ?? "").startsWith("vyos");
+          const admin = isVyos
+            ? "vyos"
+            : installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "lab" : "root";
           console.log(`\n  \x1b[0;32m\x1b[1m  ssh ${admin}@${ip}\x1b[0m\n`);  // eslint-disable-line no-console
 
-          // Auto-install k3s for non-vanilla roles
-          if (installedInfo.role !== "vanilla" && ip !== "") {
-            void triggerPostProvisionK3s(installedInfo.hostname, ip, installedInfo.role, admin, mac);
+          // Auto-install k3s for non-vanilla roles.
+          // autoK3s=false suppresses this entirely. Multi-node tests need that:
+          // post-provision installs each node into its OWN cluster, so leaving it
+          // on turns three combo nodes into three single-node clusters, and it
+          // also stops k3s and deletes CNI state minutes after install lands --
+          // right on top of a test that is forming a cluster itself.
+          if (config.autoK3s === false) {
+            logger.info(`[api] autoK3s disabled; skipping k3s post-provision for ${mac}`);
+          } else if (!isVyos && installedInfo.role !== "vanilla" && ip !== "") {
+            // State is JSON on disk; a role written by an older build may no
+            // longer exist. Skip loudly rather than crash the install callback.
+            if (isRole(installedInfo.role)) {
+              void triggerPostProvisionK3s(installedInfo.hostname, ip, installedInfo.role, admin, mac);
+            } else {
+              console.warn(`[api] unknown role ${installedInfo.role} for ${mac}; skipping k3s post-provision`);  // eslint-disable-line no-console
+            }
           }
         }
       }
@@ -291,10 +363,6 @@ export function registerApiRoutes(
       arch?: string;
       disks?: Array<{ name: string; size_gb: number; model: string }>;
       nics?: Array<{ name: string; mac: string; state: string }>;
-      // Root filesystem, when the reporter could observe it (recheck over SSH, or the
-      // probe script run from a rescue shell). Used by --pxe-boot.
-      root_device?: string;
-      root_args?: string;
     };
   }>("/api/discover", async (request, reply) => {
     const data = request.body;
@@ -309,53 +377,22 @@ export function registerApiRoutes(
 
     state.update((s) => {
       const existing = s.discovered[mac];
-      // Classify onboarding from the DMI identity we just received. An explicit
-      // classification already on the record wins (see classifyOnboard).
-      const onboarding = classifyOnboard({
-        mac,
-        manufacturer: data.manufacturer ?? existing?.manufacturer ?? "unknown",
-        product: data.product ?? existing?.product ?? "unknown",
-        board: data.board ?? existing?.board ?? "unknown",
-        ...(existing?.onboard !== undefined ? { onboard: existing.onboard } : {}),
-        ...(existing?.vendor_os !== undefined ? { vendor_os: existing.vendor_os } : {}),
-      });
-      const rootDevice = data.root_device ?? existing?.root_device;
-      const rootArgs = data.root_args ?? existing?.root_args;
-
-      // Absent fields keep whatever we already knew. Reporters are not all the full
-      // discovery kickstart: the rescue-shell probe posts only a root device, and
-      // blanking a machine's hardware inventory as a side effect of that would be
-      // silent data loss.
       const hwInfo: HardwareInfo = {
         mac,
-        product: data.product ?? existing?.product ?? "unknown",
-        board: data.board ?? existing?.board ?? "unknown",
-        serial: data.serial ?? existing?.serial ?? "unknown",
-        manufacturer: data.manufacturer ?? existing?.manufacturer ?? "unknown",
-        cpu_model: data.cpu_model ?? existing?.cpu_model ?? "unknown",
-        cpu_cores: data.cpu_cores ?? existing?.cpu_cores ?? 0,
-        memory_gb: data.memory_gb ?? existing?.memory_gb ?? 0,
-        arch: data.arch ?? existing?.arch ?? "unknown",
-        disks: data.disks ?? existing?.disks ?? [],
-        nics: data.nics ?? existing?.nics ?? [],
+        product: data.product ?? "unknown",
+        board: data.board ?? "unknown",
+        serial: data.serial ?? "unknown",
+        manufacturer: data.manufacturer ?? "unknown",
+        cpu_model: data.cpu_model ?? "unknown",
+        cpu_cores: data.cpu_cores ?? 0,
+        memory_gb: data.memory_gb ?? 0,
+        arch: data.arch ?? "unknown",
+        disks: data.disks ?? [],
+        nics: data.nics ?? [],
         first_seen: existing?.first_seen ?? now,
         last_seen: now,
-        onboard: onboarding.onboard,
-        ...(onboarding.vendor_os !== undefined ? { vendor_os: onboarding.vendor_os } : {}),
-        ...(rootDevice !== undefined ? { root_device: rootDevice } : {}),
-        ...(rootArgs !== undefined ? { root_args: rootArgs } : {}),
       };
       s.discovered[mac] = hwInfo;
-
-      // Keep the installed record in step -- the install guard and --pxe-boot read it.
-      const inst = s.installed[mac];
-      if (inst) {
-        if (data.arch !== undefined) inst.arch = data.arch;
-        inst.onboard = onboarding.onboard;
-        if (onboarding.vendor_os !== undefined) inst.vendor_os = onboarding.vendor_os;
-        if (rootDevice !== undefined) inst.root_device = rootDevice;
-        if (rootArgs !== undefined) inst.root_args = rootArgs;
-      }
     });
 
     const label = isNew ? "NEW MACHINE DISCOVERED" : "MACHINE RE-DISCOVERED";
@@ -476,6 +513,15 @@ export function registerApiRoutes(
     const installedEntry = currentState.installed[mac];
 
     if (queueEntry) {
+      // A machine that was handed an install script but has reported nothing
+      // since is wedged BEFORE the installer environment came up — a bad
+      // kernel/initrd, no network in the initramfs, or the wrong NIC picked.
+      // Surfacing it here is what makes that diagnosable without a console.
+      const since = queueEntry.progress_at ?? queueEntry.dispatched_at;
+      const stalledForS = since !== undefined && queueEntry.progress === undefined
+        ? Math.floor((Date.now() - new Date(since).getTime()) / 1000)
+        : 0;
+
       return reply.send({
         mac,
         hostname: queueEntry.hostname,
@@ -483,6 +529,9 @@ export function registerApiRoutes(
         progress: queueEntry.progress ?? "queued",
         progress_detail: queueEntry.progress_detail ?? "",
         progress_at: queueEntry.progress_at ?? queueEntry.queued_at,
+        dispatched_at: queueEntry.dispatched_at,
+        stalled_for_s: stalledForS,
+        stalled: stalledForS > STALL_THRESHOLD_S,
         role: queueEntry.role,
         os: queueEntry.os,
         stages: queueEntry.log ?? [],

bastion/src/bastion/src/routes/dispatch.ts

@@ -5,8 +5,7 @@
 //   - unknown       -> discovery mode (collect hardware, POST to bastion)
 
 import type { FastifyInstance } from "fastify";
-import type { Arch, BastionConfig, BastionState, OsId } from "@lab/shared";
-import { normalizeArch, fedoraMirrorFor, osSupportsArch } from "@lab/shared";
+import type { BastionConfig } from "@lab/shared";
 import type { StateManager } from "../services/state.js";
 import {
   renderDiscoverIpxe,
@@ -14,51 +13,12 @@ import {
   renderDebugIpxe,
   renderPxeBootDebugIpxe,
   renderLocalBootIpxe,
-  renderUnsupportedIpxe,
 } from "../templates/boot.ipxe.js";
 import { renderUbuntuInstallIpxe } from "../templates/ubuntu-boot.ipxe.js";
+import { renderVyosInstallIpxe } from "../templates/vyos-boot.ipxe.js";
 import { renderDebugKickstart } from "../templates/debug.ks.js";
 import { logger } from "../services/logger.js";
 
-/**
- * Resolve a booting machine's architecture.
- *
- * Order matters. The tracked record is what we actually observed on the machine, so it
- * wins. `reported` is iPXE's ${buildarch}, which is only as good as the binary DHCP
- * handed the client -- correct in practice, but a misconfigured option 93 mapping would
- * make it lie. The configured default is the last resort.
- *
- * There is deliberately no operator-supplied architecture anywhere in this path.
- */
-export function resolveArch(
-  state: BastionState,
-  mac: string,
-  reported: string | undefined,
-  config: BastionConfig,
-): Arch {
-  return normalizeArch(state.installed[mac]?.arch)
-    ?? normalizeArch(state.install_queue[mac]?.arch)
-    ?? normalizeArch(state.discovered[mac]?.arch)
-    ?? normalizeArch(reported)
-    ?? normalizeArch(config.arch)
-    ?? "x86_64";
-}
-
-/** The root filesystem to boot for --pxe-boot, if the machine's record carries one. */
-function resolveRoot(
-  state: BastionState,
-  mac: string,
-): { rootDevice: string; rootArgs?: string } | null {
-  const installed = state.installed[mac];
-  const discovered = state.discovered[mac];
-  const rootDevice = installed?.root_device ?? discovered?.root_device;
-  if (rootDevice === undefined || rootDevice === "") return null;
-  const rootArgs = installed?.root_args ?? discovered?.root_args;
-  return rootArgs !== undefined && rootArgs !== ""
-    ? { rootDevice, rootArgs }
-    : { rootDevice };
-}
-
 export function registerDispatchRoutes(
   app: FastifyInstance,
   config: BastionConfig,
@@ -93,68 +53,18 @@ curl -sf -X POST "http://${config.serverIp}:${config.httpPort}/api/progress" \\
     -H "Content-Type: application/json" \\
     -d "{\\"mac\\":\\"$MAC_ADDR\\",\\"stage\\":\\"debug-ready\\",\\"detail\\":\\"nc $IP_ADDR 2323\\"}" 2>/dev/null || true
 
-# --- Find the installed root filesystem and report it ---
-# This is what 'labctl provision debug --pxe-boot' needs. The rescue image cannot
-# report it by itself: %pre/%post do not run in rescue mode, so it happens here.
-vgchange -ay >/dev/null 2>&1 || true
-
-ROOT_DEVICE=""
-ROOT_ARGS=""
-PROBE_MNT=/tmp/lab-rootprobe
-mkdir -p "$PROBE_MNT"
-
-# Candidates: every LVM logical volume plus every non-LVM partition with a filesystem.
-for CAND in $(lvs --noheadings -o lv_path 2>/dev/null) \\
-            $(blkid -o device 2>/dev/null | grep -v '^/dev/mapper/'); do
-    [ -b "$CAND" ] || continue
-    mount -o ro "$CAND" "$PROBE_MNT" >/dev/null 2>&1 || continue
-    # A root filesystem has both of these; /boot and /home do not.
-    if [ -f "$PROBE_MNT/etc/fstab" ] && [ -d "$PROBE_MNT/usr" ]; then
-        ROOT_DEVICE="$CAND"
-        PRETTY=$(. "$PROBE_MNT/etc/os-release" 2>/dev/null && echo "$PRETTY_NAME")
-        echo "  found root: $CAND \${PRETTY:+($PRETTY)}"
-        if [ "$(lsblk -no TYPE "$CAND" 2>/dev/null | head -1)" = "lvm" ]; then
-            VGLV=$(lvs --noheadings -o vg_name,lv_name "$CAND" 2>/dev/null | awk '{print $1"/"$2}')
-            [ -n "$VGLV" ] && ROOT_ARGS="rd.lvm.lv=$VGLV"
-            # Swap comes from fstab here — /proc/swaps is the rescue image's, not the host's.
-            SWLV=$(awk '$3=="swap" && $1 ~ /^\\/dev\\// {print $1; exit}' "$PROBE_MNT/etc/fstab" 2>/dev/null)
-            if [ -n "$SWLV" ]; then
-                SWVGLV=$(lvs --noheadings -o vg_name,lv_name "$SWLV" 2>/dev/null | awk '{print $1"/"$2}')
-                [ -n "$SWVGLV" ] && [ "$SWVGLV" != "$VGLV" ] && ROOT_ARGS="$ROOT_ARGS rd.lvm.lv=$SWVGLV"
-            fi
-        fi
-        umount "$PROBE_MNT" >/dev/null 2>&1 || true
-        break
-    fi
-    umount "$PROBE_MNT" >/dev/null 2>&1 || true
-done
-
-if [ -n "$ROOT_DEVICE" ]; then
-    curl -sf -X POST "http://${config.serverIp}:${config.httpPort}/api/discover" \\
-        -H "Content-Type: application/json" \\
-        -d "{\\"mac\\":\\"$MAC_ADDR\\",\\"root_device\\":\\"$ROOT_DEVICE\\",\\"root_args\\":\\"$ROOT_ARGS\\"}" 2>/dev/null \\
-        && echo "  reported to bastion — 'labctl provision debug --pxe-boot' will work now"
-else
-    echo "  no root filesystem found — --pxe-boot cannot be used on this machine"
-fi
-
 echo ""
 echo "=== Debug environment ready ==="
 echo "  nc $IP_ADDR 2323    (remote shell)"
 echo "  ssh root@$IP_ADDR   (password: debug)"
-if [ -n "$ROOT_DEVICE" ]; then
-    echo "  root: $ROOT_DEVICE $ROOT_ARGS"
-fi
 echo "==============================="
 `;
     return reply.type("text/plain").send(script);
   });
 
-  app.get<{ Querystring: { mac?: string; arch?: string } }>("/dispatch", async (request, reply) => {
+  app.get<{ Querystring: { mac?: string } }>("/dispatch", async (request, reply) => {
     const mac = (request.query.mac ?? "").toLowerCase().replace(/-/g, ":");
     const currentState = state.load();
-    const arch = resolveArch(currentState, mac, request.query.arch, config);
-    const fedoraMirror = fedoraMirrorFor(config.fedoraVersion, arch);
 
     // Debug mode takes highest priority — auto-clear after serving once
     const debugEntry = currentState.debug[mac];
@@ -163,48 +73,22 @@ echo "==============================="
       state.update((s) => { delete s.debug[mac]; });
 
       let script: string;
-      const wantsPxeBoot = debugEntry.pxeBoot === true;
-      const root = wantsPxeBoot ? resolveRoot(currentState, mac) : null;
-
-      if (root !== null) {
-        logger.info(`PXE BOOT DEBUG: ${mac} -> ${hostname} (${arch}, root=${root.rootDevice})`);
+      if (debugEntry.pxeBoot) {
+        logger.info(`PXE BOOT DEBUG: ${mac} -> ${hostname} (kernel+initrd from PXE, root from NVMe)`);
         script = renderPxeBootDebugIpxe({
           mac,
           hostname,
           serverIp: config.serverIp,
           httpPort: config.httpPort,
-          arch,
-          ...root,
         });
       } else {
-        // --pxe-boot without a known root device falls back to rescue rather than
-        // guessing. A wrong root= leaves the machine unbootable, and rescue is where
-        // the operator can find the real one (curl /debug-setup.sh reports it back).
-        const notice = wantsPxeBoot
-          ? [
-              "",
-              "NOTE: --pxe-boot requested, but no root device is recorded",
-              "      for this machine. Booting rescue instead.",
-              "      From the rescue shell, run:",
-              // No pipe or && here: iPXE treats || and && as command separators, so keep
-              // the printed command free of anything its parser might claim.
-              `        curl -s http://${config.serverIp}:${config.httpPort}/debug-setup.sh -o /tmp/s.sh ; sh /tmp/s.sh`,
-              "      then retry --pxe-boot.",
-            ]
-          : undefined;
-        if (wantsPxeBoot) {
-          logger.warn(`PXE BOOT DEBUG: ${mac} -> ${hostname} has no recorded root device -- serving rescue instead`);
-        } else {
-          logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (${arch}, rescue mode)`);
-        }
+        logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (rescue mode)`);
         script = renderDebugIpxe({
           mac,
           hostname,
           serverIp: config.serverIp,
           httpPort: config.httpPort,
-          fedoraMirror,
-          arch,
-          ...(notice ? { notice } : {}),
+          fedoraMirror: config.fedoraMirror,
         });
       }
       return reply.type("text/plain").send(script);
@@ -214,24 +98,24 @@ echo "==============================="
     if (queueEntry) {
       const hostname = queueEntry.hostname ?? "lab-node";
       const os = queueEntry.os ?? "fedora-43";
-      logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os}, ${arch})`);
+      logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os})`);
+
+      // Stamp the handoff so a machine that boots the installer but never
+      // reports can be spotted without a console.
+      state.update((s) => {
+        const entry = s.install_queue[mac];
+        if (entry) entry.dispatched_at = new Date().toISOString();
+      });
 
       let script: string;
-      if (os.startsWith("ubuntu")) {
-        // Last line of defence. The install guard refuses this combination when the
-        // machine's architecture is already known, but a machine queued before it was
-        // discovered can reach here. Serving the x86-only Ubuntu kernel to an arm64
-        // client is precisely the bug this work exists to fix, so stop instead.
-        if (!osSupportsArch(os as OsId, arch)) {
-          logger.error(`INSTALL BLOCKED: ${mac} -> ${hostname} -- ${os} has no ${arch} artifacts`);
-          script = renderUnsupportedIpxe({
-            hostname,
-            mac,
-            reason: `${os} publishes no ${arch} netboot artifacts`,
-            action: `labctl provision install ${mac} ${hostname} --os fedora-43`,
-          });
-          return reply.type("text/plain").send(script);
-        }
+      if (os.startsWith("vyos")) {
+        script = renderVyosInstallIpxe({
+          mac,
+          hostname,
+          serverIp: config.serverIp,
+          httpPort: config.httpPort,
+        });
+      } else if (os.startsWith("ubuntu")) {
         script = renderUbuntuInstallIpxe({
           mac,
           hostname,
@@ -246,8 +130,7 @@ echo "==============================="
           serverIp: config.serverIp,
           httpPort: config.httpPort,
           fedoraVersion: config.fedoraVersion,
-          fedoraMirror,
-          arch,
+          fedoraMirror: config.fedoraMirror,
         });
       }
 
@@ -264,14 +147,13 @@ echo "==============================="
     }
 
     // Unknown MAC -> discovery mode
-    logger.info(`PXE request from ${mac} (${arch}) -> discovery mode`);
+    logger.info(`PXE request from ${mac} -> discovery mode`);
 
     const script = renderDiscoverIpxe({
       mac,
       serverIp: config.serverIp,
       httpPort: config.httpPort,
-      fedoraMirror,
-      arch,
+      fedoraMirror: config.fedoraMirror,
     });
 
     return reply.type("text/plain").send(script);

bastion/src/bastion/src/routes/kickstart.ts

@@ -30,6 +30,8 @@ export function registerKickstartRoutes(
       hostname: queueEntry?.hostname ?? "lab-node",
       disk: queueEntry?.disk ?? "",
       role: queueEntry?.role ?? "worker",
+      ...(queueEntry?.schemaVersion ? { schemaVersion: queueEntry.schemaVersion } : {}),
+      ...(queueEntry?.dataDisks ? { dataDisks: queueEntry.dataDisks } : {}),
     });
 
     return reply.type("text/plain").send(ks);

bastion/src/bastion/src/routes/vyos.ts

@@ -0,0 +1,71 @@
+// VyOS network install routes.
+//
+// VyOS has no unattended installer, so the automation is injected via
+// live-config's `hooks` component: the iPXE script passes
+// live-config.hooks=<.../vyos/autoinstall.sh>, live-config wgets it and runs it
+// as root, and that script fetches and executes the generated install driver.
+
+import type { FastifyInstance } from "fastify";
+import type { BastionConfig } from "@lab/shared";
+import type { StateManager } from "../services/state.js";
+import { buildVyosConfigSpec } from "../templates/vyos-config-spec.js";
+import { renderVyosInstallPy } from "../templates/vyos-install.py.js";
+import { logger } from "../services/logger.js";
+
+function normalizeMac(value: string | undefined): string {
+  return (value ?? "").toLowerCase().replace(/-/g, ":");
+}
+
+export function registerVyosRoutes(
+  app: FastifyInstance,
+  config: BastionConfig,
+  state: StateManager,
+): void {
+  // live-config hook. Kept minimal: everything version-specific lives in the
+  // generated Python. wget is guaranteed present -- live-config used it to
+  // fetch this very script.
+  app.get<{ Querystring: { mac?: string } }>("/vyos/autoinstall.sh", async (request, reply) => {
+    const mac = normalizeMac(request.query.mac);
+    const base = `http://${config.serverIp}:${config.httpPort}`;
+
+    logger.info(`VYOS AUTOINSTALL HOOK served to ${mac || "unknown MAC"}`);
+
+    const script = `#!/bin/sh
+# Lab PXE Bastion -- VyOS unattended install hook (run by live-config as root)
+set -eu
+
+wget -q "${base}/vyos/install.py?mac=${mac}" -O /tmp/vyos-install.py
+exec python3 /tmp/vyos-install.py
+`;
+    return reply.type("text/plain").send(script);
+  });
+
+  // Per-MAC install driver, with the machine's config spec baked in.
+  app.get<{ Querystring: { mac?: string } }>("/vyos/install.py", async (request, reply) => {
+    const mac = normalizeMac(request.query.mac);
+    const queueEntry = state.load().install_queue[mac];
+
+    const spec = buildVyosConfigSpec({
+      hostname: queueEntry?.hostname ?? "vyos",
+      spec: queueEntry?.vyos,
+      defaultPassword: config.vyosDefaultPassword,
+      sshKeys: config.sshKeys,
+      disk: queueEntry?.disk,
+    });
+
+    logger.info(
+      `VYOS INSTALL DRIVER served to ${mac} (${spec.hostname}, ` +
+      `${spec.sets.length} config ops, disk="${spec.disk || "auto"}")`,
+    );
+
+    const script = renderVyosInstallPy({
+      spec,
+      mac,
+      serverIp: config.serverIp,
+      httpPort: config.httpPort,
+      role: queueEntry?.role ?? "vanilla",
+    });
+
+    return reply.type("text/plain").send(script);
+  });
+}

bastion/src/bastion/src/server.ts

@@ -12,6 +12,7 @@ import { registerDispatchRoutes } from "./routes/dispatch.js";
 import { registerKickstartRoutes } from "./routes/kickstart.js";
 import { registerApiRoutes } from "./routes/api.js";
 import { registerAsahiRoutes } from "./routes/asahi.js";
+import { registerVyosRoutes } from "./routes/vyos.js";
 
 
 export function createApp(config: BastionConfig): { app: ReturnType<typeof Fastify>; state: StateManager; installLog: InstallLogBuffer; syslog: SyslogListener } {
@@ -45,8 +46,9 @@ export function createApp(config: BastionConfig): { app: ReturnType<typeof Fasti
   // Register route handlers
   registerDispatchRoutes(app, config, state);
   registerKickstartRoutes(app, config, state, syslog);
-  registerApiRoutes(app, state, installLog, syslog);
+  registerApiRoutes(app, config, state, installLog, syslog);
   registerAsahiRoutes(app, config);
+  registerVyosRoutes(app, config, state);
   // boot.iso is generated at startup and served as a static file from httpDir
   // (static serving supports HTTP Range requests, required by JetKVM streaming)
 

bastion/src/bastion/src/services/install-guard.ts

@@ -1,94 +0,0 @@
-// Pre-flight checks for queuing an OS install.
-//
-// Both entry points -- the HTTP /api/install route and the labd command-install handler
-// -- run this, so `labctl provision install` and `provision reprovision` are covered
-// whichever way the request arrives.
-//
-// Rescue/debug is deliberately NOT guarded. Being unable to reinstall a machine is
-// exactly when you most need to boot it into a rescue shell.
-
-import type { Arch, BastionState, OsId } from "@lab/shared";
-import { classifyOnboard, normalizeArch, osSupportsArch, vendorOsDescription, archesForOs } from "@lab/shared";
-
-export type InstallCheck =
-  | { allowed: true }
-  | { allowed: false; error: string };
-
-interface MachineIdentity {
-  hostname: string;
-  arch: Arch | undefined;
-  identity: Parameters<typeof classifyOnboard>[0];
-}
-
-/** Best-known identity for a MAC, merged across the three state maps. */
-function identify(state: BastionState, mac: string): MachineIdentity {
-  const discovered = state.discovered[mac];
-  const installed = state.installed[mac];
-  const queued = state.install_queue[mac];
-
-  const manufacturer = discovered?.manufacturer ?? installed?.manufacturer;
-  const product = discovered?.product ?? installed?.product;
-  const board = discovered?.board;
-  const onboard = installed?.onboard ?? discovered?.onboard;
-  const vendorOs = installed?.vendor_os ?? discovered?.vendor_os;
-
-  return {
-    hostname: installed?.hostname ?? queued?.hostname ?? discovered?.product ?? mac,
-    arch: normalizeArch(installed?.arch ?? queued?.arch ?? discovered?.arch),
-    identity: {
-      mac,
-      ...(manufacturer !== undefined ? { manufacturer } : {}),
-      ...(product !== undefined ? { product } : {}),
-      ...(board !== undefined ? { board } : {}),
-      ...(onboard !== undefined ? { onboard } : {}),
-      ...(vendorOs !== undefined ? { vendor_os: vendorOs } : {}),
-    },
-  };
-}
-
-/**
- * Decide whether `mac` may be queued for an install of `os`.
- *
- * Refusals name the machine and the reason, and point at the action that is available
- * instead. An operator hitting this at 2am should not have to read the source to work
- * out what happened.
- */
-export function checkInstallAllowed(
-  state: BastionState,
-  mac: string,
-  os: OsId,
-): InstallCheck {
-  const machine = identify(state, mac);
-  const { onboard, vendor_os } = classifyOnboard(machine.identity);
-
-  // 1. Machines running a vendor OS we cannot rebuild.
-  if (onboard === "ssh") {
-    const what = vendorOsDescription(vendor_os);
-    return {
-      allowed: false,
-      error:
-        `Refusing to install ${machine.hostname} (${mac}): it runs ${what}. ` +
-        `No image in our pipeline can restore it, so installing ${os} would destroy that ` +
-        `driver and firmware stack permanently. This machine is SSH-onboard: we manage its ` +
-        `userspace, not its OS. ` +
-        `To boot it into a rescue shell instead, run: labctl provision debug ${machine.hostname}`,
-      // TODO: when a DGX OS / SparkOS image joins the pipeline, an install targeting a
-      // machine whose vendor_os matches that image should be allowed through here.
-    };
-  }
-
-  // 2. Architecture the OS has no netboot artifacts for.
-  if (machine.arch !== undefined && !osSupportsArch(os, machine.arch)) {
-    const supported = archesForOs(os);
-    return {
-      allowed: false,
-      error:
-        `Refusing to install ${os} on ${machine.hostname} (${mac}): ` +
-        `${os} has no ${machine.arch} netboot artifacts` +
-        (supported.length > 0 ? ` (only ${supported.join(", ")})` : "") +
-        `. Use an OS that supports ${machine.arch}.`,
-    };
-  }
-
-  return { allowed: true };
-}

bastion/src/bastion/src/services/kickstart-generator.ts

@@ -24,12 +24,16 @@ export function generateInstallKickstart(
     hostname: string;
     disk: string;
     role: Role;
+    schemaVersion?: 1 | 2;
+    dataDisks?: import("@lab/shared").DataDisks;
   },
 ): string {
   const ksParams: InstallKickstartParams = {
     hostname: params.hostname,
     disk: params.disk,
     role: params.role,
+    ...(params.schemaVersion ? { schemaVersion: params.schemaVersion } : {}),
+    ...(params.dataDisks ? { dataDisks: params.dataDisks } : {}),
     domain: config.domain,
     fedoraVersion: config.fedoraVersion,
     timezone: config.timezone,

bastion/src/bastion/src/services/post-provision.ts

@@ -7,6 +7,7 @@ import { homedir } from "node:os";
 import { join } from "node:path";
 import { logger } from "./logger.js";
 import { progressBus } from "./progress-events.js";
+import { isK3sServer, type Role } from "@lab/shared";
 
 function findSshKey(): string | undefined {
   const sudoUser = process.env["SUDO_USER"];
@@ -84,6 +85,36 @@ function sshRunStreaming(ip: string, user: string, command: string, keyPath: str
   });
 }
 
+/**
+ * Roles this bare post-provision install can legitimately handle.
+ *
+ * The install below is a plain `INSTALL_K3S_EXEC=server|agent` with no K3S_URL
+ * and no K3S_TOKEN, and it knows about exactly one machine. That is fine for a
+ * node that is its own cluster, and wrong for anything that has to JOIN one:
+ *
+ *   - a cluster member installed this way gets `cluster-init` semantics and
+ *     becomes its own separate cluster, so provisioning three of them yields
+ *     three clusters instead of one (split brain);
+ *   - an AGENT cannot even start — `INSTALL_K3S_EXEC=agent` with nothing to
+ *     join fails with "Job for k3s-agent.service failed because the control
+ *     process exited with error code", which is exactly what a k8s-worker
+ *     install does today.
+ *
+ * So cluster members are not auto-installed. They are handed the command that
+ * supplies the missing server URL and token.
+ */
+export function shouldAutoInstallK3s(role: Role): boolean {
+  switch (role) {
+    case "k8s-combo":
+    case "k8s-controller":
+    case "k8s-worker":
+    case "worker":
+      return false;
+    default:
+      return true;
+  }
+}
+
 /**
  * Trigger k3s installation on a freshly provisioned machine.
  * Runs in the background — logs progress to bastion console and progressBus.
@@ -91,7 +122,7 @@ function sshRunStreaming(ip: string, user: string, command: string, keyPath: str
 export async function triggerPostProvisionK3s(
   hostname: string,
   ip: string,
-  role: string,
+  role: Role,
   sshUser: string,
   mac?: string,
 ): Promise<void> {
@@ -104,6 +135,21 @@ export async function triggerPostProvisionK3s(
     }
   };
 
+  // Gate before ANY work: the prerequisite steps below stop k3s and delete CNI
+  // state, so they must not run for a role we are declining to install either.
+  if (!shouldAutoInstallK3s(role)) {
+    emitStage(
+      "post-provision",
+      `${role} is a cluster member; not auto-installing k3s (it would form its own cluster)`,
+    );
+    const verb = isK3sServer(role) ? "--k3s-server https://<existing-server>:6443" : "--k3s-server https://<server>:6443";
+    logger.info(
+      `[k3s] Join it to the cluster with:\n` +
+      `        labctl app k3s install ${hostname} --role ${role} ${verb} --k3s-token <token>`,
+    );
+    return;
+  }
+
   emitStage("post-provision", `auto-installing k3s on ${hostname} (${ip}) role=${role}`);
   emitStage("post-provision", "waiting for SSH (machine may still be rebooting)");
 
@@ -150,7 +196,7 @@ sysctl --system > /dev/null'`, keyPath, hostname, mac);
 
   // Step 4: Install k3s
   // labcontroller extends infra — both are k3s servers
-  const k3sRole = (role === "infra" || role === "labcontroller") ? "server" : "agent";
+  const k3sRole = isK3sServer(role) ? "server" : "agent";
   emitStage("post-provision", `installing k3s ${k3sRole}`);
   const code = await sshRunStreaming(ip, sshUser,
     `curl -sfL https://get.k3s.io | sudo INSTALL_K3S_EXEC="${k3sRole}" INSTALL_K3S_SKIP_SELINUX_RPM=true sh -`,

bastion/src/bastion/src/templates/asahi-firstboot.sh.ts

@@ -3,6 +3,7 @@
 // Creates the standard lab LVM layout on the data partition, matching install.ks.ts.
 
 import type { Role } from "@lab/shared";
+import { isK3sServer, v2Layout, type LvSpec } from "@lab/shared";
 
 export interface AsahiFirstbootParams {
   hostname: string;
@@ -12,13 +13,42 @@ export interface AsahiFirstbootParams {
   sshKeys: string[];
   adminUser: string;
   mac: string;
+  /** Fresh-setup layout. Defaults to 1 so existing callers are unchanged. */
+  schemaVersion?: 1 | 2;
 }
 
 export function renderFirstbootScript(params: AsahiFirstbootParams): string {
   const { hostname, role, serverIp, httpPort, sshKeys, adminUser, mac } = params;
+  const schemaVersion = params.schemaVersion ?? 1;
+
+  // ---------------------------------------------------------------------------
+  // v2 LAYOUT, from the SAME definition Fedora and Ubuntu render
+  // (shared/src/types/layout.ts).
+  //
+  // omitRoot: on Apple Silicon the Asahi installer has already created a btrfs
+  // root partition OUTSIDE our volume group -- worker1 runs a 4.6 GB one. We
+  // must not create a second root LV.
+  //
+  // The v1 path below hardcodes rancher at 20480M while claiming "sizes match
+  // install.ks.ts", which uses 122880M. That 100 GB gap is what left worker1
+  // with an image store too small to hold a single large container image.
+  // ---------------------------------------------------------------------------
+  const v2 = v2Layout(role, { omitRoot: true });
+  const v2LvLines = v2.map((lv: LvSpec) => `lvcreate -L ${lv.sizeMib}M -n ${lv.name} labvg -y`).join("\n");
+  const v2FormatLines = v2
+    .map((lv: LvSpec) => (lv.fs === "swap" ? `mkswap /dev/labvg/${lv.name}` : `mkfs.xfs /dev/labvg/${lv.name}`))
+    .join("\n");
+  const v2MountLines = v2
+    .filter((lv: LvSpec) => lv.fs !== "swap")
+    .map((lv: LvSpec) => `mount_lv ${lv.name} ${lv.mount}`)
+    .join("\n");
+  const v2FstabLines = v2
+    .filter((lv: LvSpec) => lv.fs !== "swap")
+    .map((lv: LvSpec) => `echo "/dev/labvg/${lv.name} ${lv.mount} xfs defaults 0 0" >> /etc/fstab`)
+    .join("\n");
 
   const isWorker = role === "worker";
-  const isInfra = role === "infra" || role === "labcontroller";
+  const isInfra = isK3sServer(role);
 
   // Role-specific LV creation commands
   const roleLvLines: string[] = [];
@@ -127,24 +157,27 @@ write_lab_fstab() {
     # Add fresh LVM entries
     echo "# Lab LVM volumes" >> /etc/fstab
     echo "/dev/labvg/swap    none         swap defaults 0 0" >> /etc/fstab
-    echo "/dev/labvg/var     /var         xfs  defaults 0 0" >> /etc/fstab
+${schemaVersion === 2 ? v2FstabLines.split("\n").map((l: string) => "    " + l).join("\n") : `    echo "/dev/labvg/var     /var         xfs  defaults 0 0" >> /etc/fstab
     echo "/dev/labvg/varlog  /var/log     xfs  defaults 0 0" >> /etc/fstab
     echo "/dev/labvg/home    /home        xfs  defaults 0 0" >> /etc/fstab
     echo "/dev/labvg/srv     /srv         xfs  defaults 0 0" >> /etc/fstab
-    ${roleFstabLines.join('\n    ')}
+    ${roleFstabLines.join('\n    ')}`}
 }
 
 # ── Check for existing VG ────────────────────────────────────────
 if vgs labvg &>/dev/null; then
     echo "Volume group 'labvg' already exists — reprovision detected."
+    DISK_SCHEMA=v1
+    vgs --noheadings -o vg_tags labvg 2>/dev/null | grep -q lab-schema-v2 && DISK_SCHEMA=v2
+    echo "On-disk schema: $DISK_SCHEMA"
     echo "Activating existing volumes..."
     vgchange -ay labvg
 
-    mount_lv var /var
+${schemaVersion === 2 ? v2MountLines.split("\n").map((l: string) => "    " + l).join("\n") : `    mount_lv var /var
     mount_lv varlog /var/log
     mount_lv home /home
     mount_lv srv /srv
-${roleMountLines.map(l => `    ${l}`).join('\n')}
+${roleMountLines.map(l => `    ${l}`).join('\n')}`}
 
     # Enable swap
     if lvs labvg/swap &>/dev/null; then
@@ -162,24 +195,28 @@ echo "Creating LVM on $DATA_PART..."
 
 pvcreate "$DATA_PART"
 vgcreate labvg "$DATA_PART"
+${schemaVersion === 2 ? `
+# Tag so a later run can tell which layout is on this disk without trusting a
+# caller-supplied flag -- same mechanism install.ks.ts uses.
+vgchange --addtag lab-schema-v2 labvg >/dev/null 2>&1 || true` : ""}
 
 # Create LVs — sizes match install.ks.ts (in MiB)
 echo "Creating logical volumes..."
-lvcreate -L 27648M  -n swap   labvg -y   # 27GB swap
+${schemaVersion === 2 ? v2LvLines : `lvcreate -L 27648M  -n swap   labvg -y   # 27GB swap
 lvcreate -L 102400M -n var    labvg -y   # 100GB /var
 lvcreate -L 10240M  -n varlog labvg -y   # 10GB /var/log
 lvcreate -L 10240M  -n home   labvg -y   # 10GB /home
 lvcreate -L 20480M  -n srv    labvg -y   # 20GB /srv
-${roleLvLines.join('\n')}
+${roleLvLines.join('\n')}`}
 
 # Format
 echo "Formatting volumes..."
-mkswap /dev/labvg/swap
+${schemaVersion === 2 ? v2FormatLines : `mkswap /dev/labvg/swap
 mkfs.xfs /dev/labvg/var
 mkfs.xfs /dev/labvg/varlog
 mkfs.xfs /dev/labvg/home
 mkfs.xfs /dev/labvg/srv
-${roleFormatLines.join('\n')}
+${roleFormatLines.join('\n')}`}
 
 # Migrate and mount volumes that can be switched live.
 # Copy existing content first so we don't shadow files (e.g. /home/user/.ssh).

bastion/src/bastion/src/templates/boot.ipxe.ts

@@ -1,55 +1,4 @@
 // iPXE boot script templates for dispatch routing.
-//
-// Architecture handling: the bastion serves one kernel/initrd pair per architecture.
-// x86_64 keeps the original unsuffixed paths so its output is unchanged; every other
-// architecture gets an arch-suffixed pair. See stageBootArtifacts() in main.ts for the
-// matching staging side, and boot-iso.ts for the same scheme on the ISO path.
-
-import type { Arch } from "@lab/shared";
-
-/** Kernel/initrd URL paths, keyed by architecture. */
-export function kernelPath(arch: Arch): string {
-  return arch === "x86_64" ? "/vmlinuz" : `/vmlinuz-${arch}`;
-}
-
-export function initrdPath(arch: Arch): string {
-  return arch === "x86_64" ? "/initrd.img" : `/initrd-${arch}.img`;
-}
-
-/**
- * Console arguments per architecture.
- *
- * arm64 has no VGA text console: a headless machine only talks over the SoC UART, so
- * ttyAMA0 must be listed as well. The last console= wins for /dev/console, so serial
- * is the interactive one while tty0 still receives boot output on machines with a
- * display attached.
- */
-const CONSOLE_ARGS: Record<Arch, string> = {
-  x86_64: "console=tty0",
-  aarch64: "console=tty0 console=ttyAMA0,115200",
-};
-
-/**
- * Anaconda arguments for the graphical-suppression / console setup.
- *
- * `nomodeset` disables kernel mode setting, which on x86 forces the generic VGA path
- * and makes flaky GPU drivers survive the installer. On arm64 it does not mean the
- * same thing -- there is no VGA fallback to drop back to, and it can leave the machine
- * with no usable console at all -- so arm64 gets explicit console arguments instead.
- */
-function installerArgs(arch: Arch): string {
-  return arch === "x86_64" ? "inst.text nomodeset" : `inst.text ${CONSOLE_ARGS[arch]}`;
-}
-
-/** Extra console arguments appended to templates that don't already set them. */
-function extraConsoleArgs(arch: Arch): string {
-  return arch === "x86_64" ? "" : ` ${CONSOLE_ARGS[arch]}`;
-}
-
-/** Join kernel arguments, dropping empties so callers can pass optional groups. */
-function joinArgs(...parts: Array<string | undefined>): string {
-  return parts.filter((p) => p !== undefined && p !== "").join(" ");
-}
 
 export interface BootIpxeParams {
   serverIp: string;
@@ -59,11 +8,6 @@ export interface BootIpxeParams {
 /**
  * Initial iPXE boot script that chains to the dispatch endpoint.
  * This is what dnsmasq serves to iPXE clients via HTTP.
- *
- * `${buildarch}` is iPXE's own build architecture ("x86_64" or "arm64"), which is the
- * one architecture signal available on every path -- network PXE, UEFI HTTP boot and
- * the boot ISO alike. DHCP option 93 only reaches dnsmasq, never this HTTP endpoint.
- * dispatch prefers the tracked machine record and falls back to this.
  */
 export function renderBootIpxe(params: BootIpxeParams): string {
   return `#!ipxe
@@ -75,7 +19,7 @@ echo   Contacting server for instructions...
 echo ============================================
 echo
 
-chain http://${params.serverIp}:${params.httpPort}/dispatch?mac=\${net0/mac}&arch=\${buildarch}
+chain http://${params.serverIp}:${params.httpPort}/dispatch?mac=\${net0/mac}
 `;
 }
 
@@ -87,9 +31,7 @@ export function renderDiscoverIpxe(params: {
   serverIp: string;
   httpPort: number;
   fedoraMirror: string;
-  arch: Arch;
 }): string {
-  const base = `http://${params.serverIp}:${params.httpPort}`;
   return `#!ipxe
 
 echo
@@ -100,8 +42,8 @@ echo   Collecting hardware info...
 echo =============================================
 echo
 
-kernel ${base}${kernelPath(params.arch)} inst.ks=${base}/discover.ks inst.stage2=${params.fedoraMirror} ${installerArgs(params.arch)}
-initrd ${base}${initrdPath(params.arch)}
+kernel http://${params.serverIp}:${params.httpPort}/vmlinuz inst.ks=http://${params.serverIp}:${params.httpPort}/discover.ks inst.stage2=${params.fedoraMirror} inst.text nomodeset
+initrd http://${params.serverIp}:${params.httpPort}/initrd.img
 boot
 `;
 }
@@ -116,9 +58,7 @@ export function renderInstallIpxe(params: {
   httpPort: number;
   fedoraVersion: string;
   fedoraMirror: string;
-  arch: Arch;
 }): string {
-  const base = `http://${params.serverIp}:${params.httpPort}`;
   return `#!ipxe
 
 echo
@@ -129,8 +69,8 @@ echo   MAC:    ${params.mac}
 echo =============================================
 echo
 
-kernel ${base}${kernelPath(params.arch)} inst.ks=${base}/ks?mac=${params.mac} inst.repo=${params.fedoraMirror} ${installerArgs(params.arch)}
-initrd ${base}${initrdPath(params.arch)}
+kernel http://${params.serverIp}:${params.httpPort}/vmlinuz inst.ks=http://${params.serverIp}:${params.httpPort}/ks?mac=${params.mac} inst.repo=${params.fedoraMirror} inst.text nomodeset
+initrd http://${params.serverIp}:${params.httpPort}/initrd.img
 boot
 `;
 }
@@ -138,9 +78,6 @@ boot
 /**
  * iPXE script for debug/rescue mode -- boots Fedora installer in rescue mode.
  * Provides a shell with LVM tools, network, and SSH for inspecting installed systems.
- *
- * `notice` is shown before the boot line. dispatch uses it to explain why a requested
- * --pxe-boot fell back to rescue.
  */
 export function renderDebugIpxe(params: {
   mac: string;
@@ -148,11 +85,7 @@ export function renderDebugIpxe(params: {
   serverIp: string;
   httpPort: number;
   fedoraMirror: string;
-  arch: Arch;
-  notice?: string[];
 }): string {
-  const base = `http://${params.serverIp}:${params.httpPort}`;
-  const notice = (params.notice ?? []).map((line) => `echo   ${line}\n`).join("");
   return `#!ipxe
 
 echo
@@ -160,11 +93,11 @@ echo =============================================
 echo   Lab PXE Bastion - DEBUG/RESCUE MODE
 echo   Target: ${params.hostname}
 echo   MAC:    ${params.mac}
-${notice}echo =============================================
+echo =============================================
 echo
 
-kernel ${base}${kernelPath(params.arch)} inst.rescue inst.text inst.sshd inst.ks=${base}/debug.ks?mac=${params.mac} inst.stage2=${params.fedoraMirror}${extraConsoleArgs(params.arch)}
-initrd ${base}${initrdPath(params.arch)}
+kernel http://${params.serverIp}:${params.httpPort}/vmlinuz inst.rescue inst.text inst.sshd inst.ks=http://${params.serverIp}:${params.httpPort}/debug.ks?mac=${params.mac} inst.stage2=${params.fedoraMirror}
+initrd http://${params.serverIp}:${params.httpPort}/initrd.img
 boot
 `;
 }
@@ -173,28 +106,13 @@ boot
  * iPXE script for PXE-boot debug mode -- boots the installed system's root
  * filesystem using the bastion's PXE kernel+initrd instead of local GRUB.
  * Workaround for UEFI firmware bugs that make local disk boot slow.
- *
- * rootDevice/rootArgs come from the machine's record -- they are not assumed. Our
- * Fedora installs use an LVM layout, but nothing guarantees any given machine does,
- * and a wrong root= here means an unbootable machine. dispatch refuses to render this
- * script without them.
  */
 export function renderPxeBootDebugIpxe(params: {
   mac: string;
   hostname: string;
   serverIp: string;
   httpPort: number;
-  arch: Arch;
-  rootDevice: string;
-  rootArgs?: string;
 }): string {
-  const base = `http://${params.serverIp}:${params.httpPort}`;
-  const cmdline = joinArgs(
-    `root=${params.rootDevice}`,
-    "ro",
-    params.rootArgs,
-    CONSOLE_ARGS[params.arch],
-  );
   return `#!ipxe
 
 echo
@@ -206,40 +124,12 @@ echo   Kernel+initrd from PXE, root from NVMe
 echo =============================================
 echo
 
-kernel ${base}${kernelPath(params.arch)} ${cmdline}
-initrd ${base}${initrdPath(params.arch)}
+kernel http://${params.serverIp}:${params.httpPort}/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0
+initrd http://${params.serverIp}:${params.httpPort}/initrd.img
 boot
 `;
 }
 
-/**
- * iPXE script for a request we refuse to serve.
- *
- * Better a machine that stops with a legible reason on its console than one handed a
- * kernel it cannot execute, which fails much later and much less clearly.
- */
-export function renderUnsupportedIpxe(params: {
-  mac: string;
-  hostname: string;
-  reason: string;
-  action?: string;
-}): string {
-  return `#!ipxe
-
-echo
-echo =============================================
-echo   Lab PXE Bastion - CANNOT BOOT THIS MACHINE
-echo   Target: ${params.hostname}
-echo   MAC:    ${params.mac}
-echo
-echo   ${params.reason}
-${params.action !== undefined ? `echo\necho   Try: ${params.action}\n` : ""}echo =============================================
-echo
-sleep 10
-exit 1
-`;
-}
-
 /**
  * iPXE script for already-installed machines -- exits to boot from local disk.
  */

bastion/src/bastion/src/templates/dnsmasq.conf.ts

@@ -48,20 +48,15 @@ enable-tftp
 tftp-root=${tftpDir}
 tftp-no-blocksize
 
-# Detect client architecture -- PXE (TFTP) clients.
-# Values are DHCP option 93 (Client System Architecture), IANA "Processor Architecture
-# Types". Getting these wrong means the machine is handed a bootloader its firmware
-# cannot execute, and it loops or hangs with no console output.
+# Detect client architecture -- PXE (TFTP) clients
 dhcp-match=set:bios,option:client-arch,0
 dhcp-match=set:efi-x86_64,option:client-arch,7
 dhcp-match=set:efi-x86_64,option:client-arch,9
 dhcp-match=set:efi-arm64,option:client-arch,11
 
-# Detect client architecture -- UEFI HTTP Boot clients (no TFTP size limit).
-# 16 = x64 uefi boot from http, 19 = arm uefi 64 boot from http.
-# (20 is pc/at bios boot from http -- not arm64.)
+# Detect client architecture -- UEFI HTTP Boot clients (no TFTP size limit)
 dhcp-match=set:httpboot-x86_64,option:client-arch,16
-dhcp-match=set:httpboot-arm64,option:client-arch,19
+dhcp-match=set:httpboot-arm64,option:client-arch,20
 
 # Detect iPXE clients (already chainloaded)
 dhcp-userclass=set:ipxe,iPXE

bastion/src/bastion/src/templates/install.ks.ts

@@ -2,7 +2,11 @@
 // Full Fedora server install with LVM partitioning, %pre for reprovision detection,
 // packages, and %post with SSH keys, user creation, k3s prereqs, progress callbacks.
 
-import type { Role } from "@lab/shared";
+import type { Role, DataDisks, DataDiskName } from "@lab/shared";
+import {
+  v2Layout, V2_BOOT, requiresSchemaV2, effectiveSchemaVersion, isK3sServer,
+  validateDataDisks, DATA_DISK_LABELS, type LvSpec,
+} from "@lab/shared";
 
 export interface InstallKickstartParams {
   hostname: string;
@@ -17,6 +21,22 @@ export interface InstallKickstartParams {
   syslogPort: number;
   sshKeys: string[];
   adminUser: string;
+  /**
+   * Disk layout to use for a FRESH install. Defaults to 1 so every existing
+   * caller keeps today's layout untouched.
+   *
+   * On a REPROVISION this is ignored: the schema already on the disk wins,
+   * read from an LVM tag on the volume group. Reinstalling a machine must
+   * never silently change its layout -- that is how data gets destroyed.
+   */
+  schemaVersion?: 1 | 2;
+  /**
+   * Dedicated whole-disk data volumes, by stable /dev/disk/by-id path (v2
+   * only). The matching LV is not created; the disk gets one labelled xfs
+   * partition instead. On a reprovision %pre rediscovers these by label, so a
+   * reinstall preserves them whether or not the queue entry names them.
+   */
+  dataDisks?: DataDisks;
 }
 
 export function renderInstallKickstart(params: InstallKickstartParams): string {
@@ -38,9 +58,79 @@ export function renderInstallKickstart(params: InstallKickstartParams): string {
   const fqdn = domain ? `${hostname}.${domain}` : hostname;
   const vg = "labvg";
   const now = new Date().toISOString();
+  // ---------------------------------------------------------------------------
+  // v1 LAYOUT -- DO NOT REPLACE THESE WITH CAPABILITY LOOKUPS.
+  //
+  // These literals are deliberately frozen. Every machine already in the fleet
+  // was partitioned by them, and `labctl provision reprovision` must reproduce
+  // that same layout byte-for-byte or it destroys data. Yes, they are wrong:
+  // `labcontroller` runs k3s and gets no rancher LV, and `infra` nodes hold
+  // every Longhorn replica in the homelab cluster while only `worker` gets a
+  // longhorn LV. Those are fixed in the v2 layout, which new boxes get; see
+  // schemaVersion. Changing them here rewrites history for existing hardware.
+  // ---------------------------------------------------------------------------
+  // ---------------------------------------------------------------------------
+  // v2 LAYOUT. One definition, shared with ubuntu-autoinstall and asahi-firstboot
+  // (shared/src/types/layout.ts) so the three generators cannot drift apart
+  // again. Capability-gated, and nothing uses --grow.
+  // ---------------------------------------------------------------------------
+  const schemaVersion = effectiveSchemaVersion(role, params.schemaVersion);
+  if (params.schemaVersion === 1 && requiresSchemaV2(role)) {
+    // Fail here rather than render a machine with no rancher/longhorn LV.
+    throw new Error(
+      `role '${role}' has no v1 disk layout - it was introduced with schema v2. ` +
+      `Pass schemaVersion: 2 (labctl provision install --schema-version 2).`,
+    );
+  }
+
+  const dataDisks = params.dataDisks ?? {};
+  const hasDataDisks = Object.values(dataDisks).some((d) => d !== undefined);
+  if (hasDataDisks) {
+    // The v1 branches are frozen and know nothing about dedicated disks, so a
+    // v1 render with them would silently create BOTH the LV and the disk.
+    if (schemaVersion !== 2) {
+      throw new Error(
+        `dedicated data disks require the v2 layout - pass schemaVersion: 2 ` +
+        `(labctl provision install --schema-version 2)`,
+      );
+    }
+    const problem = validateDataDisks(role, dataDisks);
+    if (problem !== null) throw new Error(problem);
+  }
+
+  const v2RootMib = v2Layout(role).find((lv: LvSpec) => lv.name === "root")?.sizeMib ?? 32768;
+  const v2 = v2Layout(role, { dataDisks });
+  const v2LogvolLines = v2
+    .map((lv: LvSpec) =>
+      lv.fs === "swap"
+        ? `logvol swap --vgname=${vg} --name=${lv.name} --fstype=swap --size=${lv.sizeMib}`
+        : `logvol ${lv.mount} --vgname=${vg} --name=${lv.name} --fstype=xfs --size=${lv.sizeMib}`,
+    )
+    .join("\n");
+  /** OS volumes only - the ones a reprovision recreates rather than preserves. */
+  const v2OsLogvolLines = v2
+    .filter((lv: LvSpec) => ["swap", "root", "var", "varlog"].includes(lv.name))
+    .map((lv: LvSpec) =>
+      lv.fs === "swap"
+        ? `logvol swap --vgname=${vg} --name=${lv.name} --fstype=swap --size=${lv.sizeMib}`
+        : `logvol ${lv.mount} --vgname=${vg} --name=${lv.name} --fstype=xfs --size=${lv.sizeMib}`,
+    )
+    .join("\n");
+
   const hasLonghorn = role === "worker";
   const hasRancher = role === "infra";
+  // k8s roles get a dedicated 120G image-store LV. 2026-08 incident: the old
+  // 20G LV idled at 85% used, so a single ~5G image pull tripped imagefs
+  // eviction. Must be sized here — longhorn's --grow consumes all remaining
+  // VG space, making post-install lvextend impossible on worker nodes.
+  const hasRancherLv = role === "infra" || role === "worker";
   const isVanilla = role === "vanilla";
+  // etcd members need etcdctl available OFFLINE. The recovery procedure for a
+  // lost or stale member is `etcdctl member list/remove`, and `k3s etcdctl` is
+  // not a command -- the binary comes from the `etcd` rpm. Telling an operator
+  // to `dnf install etcd` in the middle of a quorum failure assumes working
+  // DNS, routing and a mirror, which is exactly what may be broken.
+  const needsEtcdctl = isK3sServer(role);
 
   // -- Auth section --
   // Always set a root password (for serial console debugging) + SSH keys
@@ -85,14 +175,29 @@ chmod 440 /etc/sudoers.d/${adminUser}`;
   }
 
   // -- Disk detection --
+  // An explicit disk may be a stable /dev/disk/by-id path: kernel names are
+  // not stable across boots on multi-NVMe hosts, so a pinned install must be
+  // able to name the disk by serial. Anaconda directives want the kernel name,
+  // so %pre resolves the symlink at install time.
   const diskLine = disk
-    ? `DISK="${disk}"`
+    ? disk.startsWith("/dev/")
+      ? `# OS disk pinned by stable path -- resolve to the kernel name anaconda wants
+DISK=""
+for _wait in $(seq 1 10); do
+    _os_target=$(readlink -f "${disk}" 2>/dev/null || true)
+    if [ -n "$_os_target" ] && [ -b "$_os_target" ]; then DISK=$(basename "$_os_target"); break; fi
+    sleep 1
+done
+[ -z "$DISK" ] && { bastion_progress "error" "OS disk not found: ${disk}"; echo "ERROR: OS disk not found: ${disk}"; exit 1; }`
+      : `DISK="${disk}"`
     : `DISK=""
 # Wait up to 10s for NVMe/SCSI disks to appear (they init async in initrd)
 for _wait in $(seq 1 10); do
     for d in /dev/nvme0n1 /dev/nvme1n1 /dev/sda /dev/sdb /dev/vda; do
         [ -b "$d" ] || continue
         _bname=$(basename "$d")
+        # Never auto-pick a dedicated data disk as the OS disk
+        case "$_bname" in "$LONGHORN_DISK"|"$RANCHER_DISK") continue ;; esac
         # Skip removable disks (USB, CD-ROM, JetKVM virtual media)
         [ -f "/sys/block/$_bname/removable" ] && [ "$(cat /sys/block/$_bname/removable)" = "1" ] && continue
         # Skip USB-attached disks (JetKVM virtual media shows as SCSI over USB)
@@ -108,14 +213,88 @@ for _wait in $(seq 1 10); do
 done
 [ -z "$DISK" ] && { echo "ERROR: no disk found"; exit 1; }`;
 
+  // -- Dedicated data disks: %pre resolution --
+  // Resolved BEFORE the OS disk so auto-detect can refuse to pick one of them.
+  const dataDiskResolveBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[])
+    .map((name) => {
+      const device = dataDisks[name];
+      const VAR = name.toUpperCase();
+      if (device === undefined) return `${VAR}_DISK=""`;
+      return `# ${name} gets a whole disk of its own, pinned by stable path
+${VAR}_DISK_ID="${device}"
+${VAR}_DISK=""
+for _wait in $(seq 1 10); do
+    _dd_target=$(readlink -f "$${VAR}_DISK_ID" 2>/dev/null || true)
+    if [ -n "$_dd_target" ] && [ -b "$_dd_target" ]; then ${VAR}_DISK=$(basename "$_dd_target"); break; fi
+    sleep 1
+done
+[ -z "$${VAR}_DISK" ] && { bastion_progress "error" "${name} disk not found: $${VAR}_DISK_ID"; echo "ERROR: ${name} disk not found: $${VAR}_DISK_ID"; exit 1; }`;
+    })
+    .join("\n");
+
+  // -- Dedicated data disks: discovery and preservation decisions --
+  // A labelled data partition is on-disk truth, exactly like the lab-schema-v2
+  // LVM tag: it is preserved whether or not this install names the disk, and it
+  // is never silently wiped or shadowed. blkid finds it wherever tonight's NVMe
+  // enumeration put it.
+  const dataDiskDecisionBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[])
+    .map((name) => {
+      const VAR = name.toUpperCase();
+      const label = DATA_DISK_LABELS[name];
+      const mount = name === "longhorn" ? "/var/lib/longhorn" : "/var/lib/rancher";
+      return `${VAR}_PART=$(blkid -t LABEL=${label} -o device 2>/dev/null | head -1)
+${VAR}_REUSE=no
+if [ -n "$${VAR}_PART" ]; then
+    _dd_parent=$(lsblk -ndo pkname "$${VAR}_PART" 2>/dev/null)
+    if [ -n "$${VAR}_DISK" ] && [ "$_dd_parent" != "$${VAR}_DISK" ]; then
+        bastion_progress "error" "${label} label is on $_dd_parent, not on the configured ${name} disk $${VAR}_DISK"
+        echo "ERROR: ${label} label found on $_dd_parent but the ${name} disk is configured as $${VAR}_DISK -- refusing to guess"
+        exit 1
+    fi
+    if [ "$REPROVISION" = "no" ] && [ -z "$${VAR}_DISK" ]; then
+        bastion_progress "error" "found ${label} data on $_dd_parent but this install configures no ${name} disk"
+        echo "ERROR: ${label} labelled data exists on $_dd_parent. Refusing to wipe or shadow it -- queue the install with a ${name} data disk, or wipe the label first"
+        exit 1
+    fi
+    ${VAR}_DISK="$_dd_parent"
+    ${VAR}_REUSE=yes
+fi
+if [ "$PRESERVE_${VAR}" = "yes" ] && [ -n "$${VAR}_DISK" ]; then
+    bastion_progress "error" "both a ${name} LV and a dedicated ${name} disk exist"
+    echo "ERROR: VG $VG has a ${name} LV AND a dedicated ${name} disk is present -- two things cannot mount ${mount}. Migrate the LV off first"
+    exit 1
+fi`;
+    })
+    .join("\n\n");
+
+  // -- Dedicated data disks: partition directives --
+  // Appended to whichever /tmp/part.ks the branches above wrote. A reused
+  // partition is mounted --noformat; a fresh disk gets ONE partition spanning
+  // it. --grow is fine HERE: the whole point of a dedicated disk is to consume
+  // it, unlike LVs competing inside a shared VG.
+  const dataDiskPartBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[])
+    .map((name) => {
+      const VAR = name.toUpperCase();
+      const label = DATA_DISK_LABELS[name];
+      const mount = name === "longhorn" ? "/var/lib/longhorn" : "/var/lib/rancher";
+      return `if [ -n "$${VAR}_DISK" ]; then
+    if [ "$${VAR}_REUSE" = "yes" ]; then
+        echo "part ${mount} --onpart=$${VAR}_PART --noformat" >> /tmp/part.ks
+    else
+        echo "part ${mount} --fstype=xfs --ondisk=$${VAR}_DISK --grow --size=1 --label=${label}" >> /tmp/part.ks
+    fi
+fi`;
+    })
+    .join("\n");
+
   // -- Longhorn LV for fresh install --
   const longhornFreshLine = hasLonghorn
     ? `logvol /var/lib/longhorn --vgname=${vg} --name=longhorn --fstype=xfs --grow --size=1`
     : "";
 
-  // -- Rancher LV for fresh install (infra role) --
-  const rancherFreshLine = hasRancher
-    ? `logvol /var/lib/rancher --vgname=${vg} --name=rancher --fstype=xfs --size=20480`
+  // -- Rancher LV for fresh install (k8s roles: worker + infra) --
+  const rancherFreshLine = hasRancherLv
+    ? `logvol /var/lib/rancher --vgname=${vg} --name=rancher --fstype=xfs --size=122880`
     : "";
 
   return `# Lab Bastion -- Fedora ${fedoraVersion} server install
@@ -129,7 +308,13 @@ lang ${locale}
 keyboard uk
 timezone ${timezone} --utc
 
-network --bootproto=dhcp --activate --hostname=${fqdn}
+# --ipv6=auto, not =dhcp: "auto" follows the router advertisement, so the RA's
+# managed-flag is what steers the node to DHCPv6, and a VLAN with no DHCPv6 yet
+# still installs instead of blocking on a lease that will never come. Failing an
+# OS install because IPv6 was not ready would be a worse trade than a node that
+# briefly has no v6. The address itself comes from a kea DHCPv6 reservation
+# keyed on MAC -- the same source of truth as the v4 address.
+network --bootproto=dhcp --ipv6=auto --activate --hostname=${fqdn}
 
 ${auth}
 ${userDirective}
@@ -159,6 +344,9 @@ bastion_progress() {
 bastion_progress "partitioning" "detecting disk"
 
 VG="${vg}"
+
+${dataDiskResolveBlocks}
+
 ${diskLine}
 
 REPROVISION=no
@@ -168,6 +356,16 @@ if vgs $VG &>/dev/null; then
     echo "=== Existing VG found - reprovision mode ==="
     REPROVISION=yes
 
+    # Which layout is ALREADY on this disk? The LVM tag is authoritative -- the
+    # schemaVersion the caller asked for is irrelevant here, because silently
+    # re-laying-out an existing machine destroys its data. A v1 box reinstalls
+    # as v1 forever, until it is deliberately migrated.
+    DISK_SCHEMA=v1
+    if vgs --noheadings -o vg_tags $VG 2>/dev/null | grep -q lab-schema-v2; then
+        DISK_SCHEMA=v2
+    fi
+    echo "On-disk schema: $DISK_SCHEMA"
+
     # Detect which data LVs to preserve
     PRESERVE_LONGHORN=no; PRESERVE_SRV=no; PRESERVE_HOME=no; PRESERVE_RANCHER=no
     lvs $VG/longhorn &>/dev/null && PRESERVE_LONGHORN=yes
@@ -185,6 +383,19 @@ else
     bastion_progress "partitioning" "fresh install on $DISK"
 fi
 
+${dataDiskDecisionBlocks}
+
+# Every disk anaconda may touch; everything else stays invisible to it.
+KS_DISKS="$DISK"
+[ -n "$LONGHORN_DISK" ] && KS_DISKS="$KS_DISKS,$LONGHORN_DISK"
+[ -n "$RANCHER_DISK" ] && [ "$RANCHER_DISK" != "$LONGHORN_DISK" ] && KS_DISKS="$KS_DISKS,$RANCHER_DISK"
+
+# Fresh installs wipe the OS disk plus any data disk being formatted for the
+# first time -- NEVER one carrying a lab-* label (those are reused --noformat).
+CLEARPART_DRIVES="$DISK"
+[ -n "$LONGHORN_DISK" ] && [ "$LONGHORN_REUSE" = "no" ] && CLEARPART_DRIVES="$CLEARPART_DRIVES,$LONGHORN_DISK"
+[ -n "$RANCHER_DISK" ] && [ "$RANCHER_REUSE" = "no" ] && [ "$RANCHER_DISK" != "$LONGHORN_DISK" ] && CLEARPART_DRIVES="$CLEARPART_DRIVES,$RANCHER_DISK"
+
 if [ "$REPROVISION" = "yes" ]; then
     # Find existing boot partitions by type
     EFI_PART=$(blkid -t TYPE=vfat -o device /dev/\${DISK}* 2>/dev/null | head -1)
@@ -195,16 +406,26 @@ if [ "$REPROVISION" = "yes" ]; then
 
     # Build partition config reusing existing PV/VG
     cat > /tmp/part.ks << PARTEOF
-ignoredisk --only-use=$DISK
+ignoredisk --only-use=$KS_DISKS
 clearpart --none
 part /boot/efi --onpart=$EFI_PART --fstype=efi
 part /boot --onpart=$BOOT_PART --fstype=ext4
 volgroup ${vg} --useexisting --noformat
+PARTEOF
+
+    # OS volumes are recreated, so they must match the layout already on disk.
+    if [ "$DISK_SCHEMA" = "v2" ]; then
+        cat >> /tmp/part.ks << PARTEOF
+${v2OsLogvolLines}
+PARTEOF
+    else
+        cat >> /tmp/part.ks << PARTEOF
 logvol swap --vgname=${vg} --name=swap --fstype=swap --size=27648
 logvol / --vgname=${vg} --name=root --fstype=xfs --size=33792
 logvol /var --vgname=${vg} --name=var --fstype=xfs --size=102400
 logvol /var/log --vgname=${vg} --name=varlog --fstype=xfs --size=10240
 PARTEOF
+    fi
 
     # Preserve or recreate data LVs
     if [ "$PRESERVE_HOME" = "yes" ]; then
@@ -229,7 +450,19 @@ PARTEOF
 
 else
     # Fresh install
-    cat > /tmp/part.ks << PARTEOF
+${schemaVersion === 2 ? `    cat > /tmp/part.ks << PARTEOF
+ignoredisk --only-use=$KS_DISKS
+clearpart --all --initlabel --drives=$CLEARPART_DRIVES
+part /boot/efi --fstype=efi --size=${V2_BOOT.efiMib} --ondisk=$DISK
+part /boot --fstype=ext4 --size=${V2_BOOT.bootMib} --ondisk=$DISK
+part pv.01 --size=1 --grow --ondisk=$DISK
+volgroup ${vg} pv.01
+${v2LogvolLines}
+PARTEOF
+    # Tag the VG so a later reprovision knows this disk is v2 WITHOUT trusting
+    # whatever schemaVersion the caller happens to pass. The tag lives on the
+    # VG, which a reprovision preserves; a marker file would not survive.
+    SCHEMA_TAG_PENDING=v2` : `    cat > /tmp/part.ks << PARTEOF
 ignoredisk --only-use=$DISK
 clearpart --all --initlabel --drives=$DISK
 part /boot/efi --fstype=efi --size=600 --ondisk=$DISK
@@ -244,9 +477,11 @@ logvol /home --vgname=${vg} --name=home --fstype=xfs --size=10240
 logvol /srv --vgname=${vg} --name=srv --fstype=xfs --size=20480
 ${longhornFreshLine}
 ${rancherFreshLine}
-PARTEOF
+PARTEOF`}
 fi
 
+${dataDiskPartBlocks}
+
 echo "=== Generated partition config ==="
 cat /tmp/part.ks
 echo "==================================="
@@ -291,6 +526,9 @@ tar
 socat
 conntrack-tools
 ethtool`}
+${needsEtcdctl ? `
+# etcdctl, for recovering etcd membership without internet access
+etcd` : ""}
 
 # Boot management
 efibootmgr
@@ -361,6 +599,21 @@ fs.inotify.max_user_watches         = 1048576
 SYSCTL
 sysctl --system || true
 
+# -- IPv6 link-local address generation: EUI-64, fleet-wide --
+# A cluster node takes its IPv6 from a MAC-keyed DHCPv6 reservation. kea can only
+# match that reservation when it can recover the node's MAC, and for a modern
+# client that sends a DUID-UUID (no MAC in it) the only place kea can find one is
+# an EUI-64 link-local. NetworkManager's default is stable-privacy (RFC 7217),
+# whose link-local hides the MAC -- so a node on the default silently never gets
+# its reserved address, and with a reservations-only subnet it gets nothing at
+# all. Proven on 2026-09-06: worker0/worker2 (eui64) bound; worker1/spark
+# (default) did not, until flipped. Setting it here means a new node is correct
+# from first boot, before its connection is ever activated.
+cat > /etc/NetworkManager/conf.d/10-ipv6-eui64.conf << 'NMEUI64'
+[connection]
+ipv6.addr-gen-mode=eui64
+NMEUI64
+
 # -- Disable firewalld permanently (k3s/Cilium manage iptables directly) --
 # Note: no '--now' — systemd is not running in the Anaconda chroot
 systemctl disable firewalld || true
@@ -388,12 +641,30 @@ bastion_progress "post-install" "3-bootorder done"
 # -- Enable SysRq magic keys (for emergency reboot via Alt+SysRq+REISUB) --
 echo "kernel.sysrq=1" > /etc/sysctl.d/90-sysrq.conf
 
+# -- Disk schema tag --
+# Tag the VG so a future reprovision can tell which layout is on this disk
+# WITHOUT trusting whatever the caller passes. Deliberately derived from what
+# actually landed on disk rather than from render-time intent: a v1 machine
+# reinstalled with --schema-version 2 keeps its v1 layout (see %pre), and must
+# not end up wearing a v2 tag it does not deserve.
+V2_ROOT_MIB=${v2RootMib}
+ROOT_MIB=$(lvs --noheadings --units m --nosuffix -o lv_size ${vg}/root 2>/dev/null | tr -d ' ' | cut -d. -f1)
+if [ "$ROOT_MIB" = "$V2_ROOT_MIB" ]; then
+    vgchange --addtag lab-schema-v2 ${vg} >/dev/null 2>&1 || true
+    echo "Tagged ${vg} as lab-schema-v2 (root=\${ROOT_MIB}M)"
+    DISK_SCHEMA_ON_DISK=2
+else
+    DISK_SCHEMA_ON_DISK=1
+    echo "Layout is v1 (root=\${ROOT_MIB}M, v2 would be \${V2_ROOT_MIB}M)"
+fi
+
 # -- Provisioning metadata --
 cat > /etc/lab-provisioned << PROVEOF
 hostname: ${fqdn}
 role: ${role}
 provisioned: $(date -Iseconds)
 bastion: ${serverIp}
+schema: v$DISK_SCHEMA_ON_DISK
 PROVEOF
 
 cat > /root/README << 'README'

bastion/src/bastion/src/templates/ubuntu-autoinstall.ts

@@ -2,10 +2,15 @@
 // Equivalent of the Fedora kickstart: LVM partitioning, packages,
 // SSH keys, k3s prereqs, progress callbacks.
 
+import { v2Layout, type LvSpec } from "@lab/shared";
+import type { Role } from "@lab/shared";
+
 export interface UbuntuAutoinstallParams {
   hostname: string;
   disk: string;
-  role: string;  // "vanilla" | "worker" | "infra"
+  role: Role;
+  /** Fresh-install layout. Defaults to 1 so existing callers are unchanged. */
+  schemaVersion?: 1 | 2;
   domain: string;
   ubuntuVersion: string;
   timezone: string;
@@ -31,6 +36,48 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
 
   const fqdn = domain ? `${hostname}.${domain}` : hostname;
   const vg = "labvg";
+  // ---------------------------------------------------------------------------
+  // v1 LAYOUT -- DO NOT REPLACE THESE WITH CAPABILITY LOOKUPS.
+  //
+  // These literals are deliberately frozen. Every machine already in the fleet
+  // was partitioned by them, and `labctl provision reprovision` must reproduce
+  // that same layout byte-for-byte or it destroys data. Yes, they are wrong:
+  // `labcontroller` runs k3s and gets no rancher LV, and `infra` nodes hold
+  // every Longhorn replica in the homelab cluster while only `worker` gets a
+  // longhorn LV. Those are fixed in the v2 layout, which new boxes get; see
+  // schemaVersion. Changing them here rewrites history for existing hardware.
+  // ---------------------------------------------------------------------------
+  // ---------------------------------------------------------------------------
+  // v2 LAYOUT, rendered from the SAME definition Fedora and Asahi use
+  // (shared/src/types/layout.ts). Before this, Ubuntu sized rancher at 20G
+  // while the kickstart used 120G -- the drift that the 2026-08 imagefs
+  // eviction incident was caused by.
+  //
+  // Indentation is load-bearing: list items sit at 6 spaces, their keys at 8.
+  // At any other depth curtin's YAML does not parse. See the v1 comment below.
+  // ---------------------------------------------------------------------------
+  const schemaVersion = params.schemaVersion ?? 1;
+  const renderV2Lv = (lv: LvSpec): string => {
+    const gib = Math.round(lv.sizeMib / 1024);
+    const mountPath = lv.fs === "swap" ? "none" : lv.mount;
+    return [
+      `      - id: lv-${lv.name}`,
+      `        name: ${lv.name}`,
+      `        type: lvm_partition`,
+      `        volgroup: vg0`,
+      `        size: ${gib}G`,
+      `      - id: fs-${lv.name}`,
+      `        type: format`,
+      `        volume: lv-${lv.name}`,
+      `        fstype: ${lv.fs}`,
+      `      - id: mount-${lv.name}`,
+      `        type: mount`,
+      `        device: fs-${lv.name}`,
+      `        path: ${mountPath}`,
+    ].join("\n");
+  };
+  const v2LvBlock = v2Layout(role).map(renderV2Lv).join("\n");
+
   const hasLonghorn = role === "worker";
   const hasRancher = role === "infra";
 
@@ -40,34 +87,39 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
   // Build the LVM layout to match Fedora kickstart sizes
   const extraLvs: string[] = [];
   if (hasLonghorn) {
-    extraLvs.push(`        - id: lv-longhorn
-          name: longhorn
-          type: lvm_partition
-          volgroup: vg0
-          size: -1
-        - id: fs-longhorn
-          type: format
-          volume: lv-longhorn
-          fstype: xfs
-        - id: mount-longhorn
-          type: mount
-          device: fs-longhorn
-          path: /var/lib/longhorn`);
+    // 6 spaces for the list item, 8 for its keys -- these are siblings of the
+    // lv-home/lv-srv entries in storage.config, which sit at 6. At 8 the
+    // rendered document is not valid YAML at all ("expected <block end>, but
+    // found '-'"), so an Ubuntu node with the longhorn role could never have
+    // installed. Ubuntu + longhorn is exactly the worker shape.
+    extraLvs.push(`      - id: lv-longhorn
+        name: longhorn
+        type: lvm_partition
+        volgroup: vg0
+        size: -1
+      - id: fs-longhorn
+        type: format
+        volume: lv-longhorn
+        fstype: xfs
+      - id: mount-longhorn
+        type: mount
+        device: fs-longhorn
+        path: /var/lib/longhorn`);
   }
   if (hasRancher) {
-    extraLvs.push(`        - id: lv-rancher
-          name: rancher
-          type: lvm_partition
-          volgroup: vg0
-          size: 20G
-        - id: fs-rancher
-          type: format
-          volume: lv-rancher
-          fstype: xfs
-        - id: mount-rancher
-          type: mount
-          device: fs-rancher
-          path: /var/lib/rancher`);
+    extraLvs.push(`      - id: lv-rancher
+        name: rancher
+        type: lvm_partition
+        volgroup: vg0
+        size: 20G
+      - id: fs-rancher
+        type: format
+        volume: lv-rancher
+        fstype: xfs
+      - id: mount-rancher
+        type: mount
+        device: fs-rancher
+        path: /var/lib/rancher`);
   }
 
   const extraLvsBlock = extraLvs.length > 0 ? "\n" + extraLvs.join("\n") : "";
@@ -81,6 +133,11 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
     `curtin in-target -- bash -c 'cat > /etc/modules-load.d/k3s.conf << EOF\nbr_netfilter\noverlay\nip_conntrack\nEOF'`,
     // Sysctl for k3s networking
     `curtin in-target -- bash -c 'cat > /etc/sysctl.d/90-k3s.conf << EOF\nnet.bridge.bridge-nf-call-iptables  = 1\nnet.bridge.bridge-nf-call-ip6tables = 1\nnet.ipv4.ip_forward                 = 1\nnet.ipv6.conf.all.forwarding        = 1\nfs.inotify.max_user_instances       = 524288\nfs.inotify.max_user_watches         = 1048576\nEOF'`,
+    // IPv6 link-local = EUI-64, so a MAC-keyed DHCPv6 reservation can match: kea
+    // recovers the node's MAC from an EUI-64 link-local when the client sends a
+    // DUID-UUID (no MAC in it). NM's stable-privacy default hides the MAC and the
+    // node silently never gets its reserved address. Proven 2026-09-06.
+    `curtin in-target -- bash -c 'cat > /etc/NetworkManager/conf.d/10-ipv6-eui64.conf << EOF\n[connection]\nipv6.addr-gen-mode=eui64\nEOF'`,
     // Disable ufw firewall
     `curtin in-target -- systemctl disable ufw || true`,
     // Enable chrony/ntp
@@ -121,7 +178,20 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
     `curtin in-target -- bash -c 'IP_ADDR=$(ip -4 addr show | awk "/inet / && !/127.0.0/ {split(\\$2,a,\\"/\\"); print a[1]; exit}"); curl -sf -X POST "http://${serverIp}:${httpPort}/api/progress" -H "Content-Type: application/json" -d "{\\"mac\\":\\"$(ip link show | awk "/ether/ && !/00:00:00:00/ {print \\$2; exit}")\\",\\"stage\\":\\"complete\\",\\"detail\\":\\"ready at $IP_ADDR\\"}" || true'`,
   );
 
-  const lateCommandsYaml = lateCommands.map((c) => `        - "${c}"`).join("\n");
+  // JSON.stringify, not `"${c}"`. JSON is a subset of YAML, so this produces a
+  // correctly escaped double-quoted scalar for free -- and the naive version
+  // was broken in two ways at once, for every role:
+  //
+  //   * embedded double quotes ended the scalar early
+  //     (`echo "tmpfs /tmp ..." >> /etc/fstab` -> "expected <block end>")
+  //   * the heredocs contain REAL newlines, and YAML folds newlines inside a
+  //     double-quoted scalar into spaces -- so even where it parsed, the
+  //     heredoc arrived at the target as one long line and wrote a file with
+  //     no line breaks.
+  //
+  // JSON escaping turns the newlines into \n, which YAML unescapes back to
+  // real newlines on parse, so the heredoc survives intact.
+  const lateCommandsYaml = lateCommands.map((c) => `        - ${JSON.stringify(c)}`).join("\n");
 
   return `#cloud-config
 autoinstall:
@@ -139,6 +209,30 @@ autoinstall:
     allow-pw: false
     authorized-keys:
 ${sshKeysYaml}
+  # Both address families. Without dhcp6 the installer's default is IPv4-only,
+  # so a node provisioned into a dual-stack cluster comes up with no IPv6, k3s
+  # has no v6 node-ip to bind, and it joins as an IPv4-only member of a
+  # dual-stack cluster -- which surfaces later as pods on that node being
+  # unreachable over v6 while the node itself reads Ready.
+  #
+  # The address itself comes from a kea DHCPv6 reservation keyed on MAC, the
+  # same source of truth as the v4 address, so nothing here needs to know it.
+  #
+  # optional: true matters -- it lets the install proceed if the v6 lease is
+  # slow or the VLAN has no DHCPv6 yet, rather than blocking on a timeout. The
+  # node still needs the address before k3s starts, but that is a later step's
+  # problem and failing the OS install over it would be worse.
+  # (No backticks in this comment: it lives inside a TS template literal, and a
+  # backtick here ends the literal and breaks the build.)
+  network:
+    version: 2
+    ethernets:
+      primary:
+        match:
+          name: "en*"
+        dhcp4: true
+        dhcp6: true
+        optional: true
   storage:
     config:
       - id: disk0
@@ -182,7 +276,7 @@ ${sshKeysYaml}
         name: ${vg}
         devices:
           - part-pv
-      - id: lv-swap
+${schemaVersion === 2 ? v2LvBlock : `      - id: lv-swap
         name: swap
         type: lvm_partition
         volgroup: vg0
@@ -259,7 +353,7 @@ ${sshKeysYaml}
       - id: mount-srv
         type: mount
         device: fs-srv
-        path: /srv${extraLvsBlock}
+        path: /srv${extraLvsBlock}`}
   packages:
     - openssh-server
     - curl

bastion/src/bastion/src/templates/vyos-boot.ipxe.ts

@@ -0,0 +1,53 @@
+// iPXE boot script template for VyOS network install.
+//
+// VyOS ships no unattended installer: `install image` is unconditionally
+// interactive (image_installer.py's install action takes no arguments, and
+// --no-prompt is wired only to `add`). So PXE boots the *live* system and the
+// automation is injected through live-config's `hooks` component, which fetches
+// a script over HTTP and runs it as root late in live boot.
+//
+// Unlike the Fedora/Ubuntu paths this boots a live image rather than an
+// installer, so there is no kickstart/autoinstall equivalent — see
+// routes/vyos.ts for the hook that actually drives the install.
+
+export function renderVyosInstallIpxe(params: {
+  mac: string;
+  hostname: string;
+  serverIp: string;
+  httpPort: number;
+}): string {
+  const base = `http://${params.serverIp}:${params.httpPort}`;
+
+  // Pin the boot NIC by MAC. live-boot otherwise scans for the first
+  // *connected* interface, and on a multi-NIC box that race is lost by
+  // whichever port negotiates slowest: on the Protectli VP2440 the SFP+
+  // pair links first, so live-boot picked the fiber ports (which have no
+  // DHCP), burned 15s per port, and gave up with "Unable to find a live
+  // file system on the network" -- while the copper port that actually PXE
+  // booted came up at 4.6s and was never tried.
+  //
+  // live-boot's Device_from_bootif() strips the "01-" and matches the MAC
+  // against /sys/class/net/*. params.mac is the dispatch key, i.e. exactly
+  // the NIC that PXE booted -- more reliable than iPXE's ${net0} on a box
+  // where the booting NIC may not be net0.
+  const bootif = `01-${params.mac.toLowerCase().replace(/:/g, "-")}`;
+
+  // Deliberately NOT passing `nonetworking` (present in VyOS's own PXE docs):
+  // live-config's hook component needs networking up to fetch the hook over
+  // HTTP. Also no `console=ttyS0` — on hardware without a physical UART that
+  // costs 30s at every systemd boot phase.
+  return `#!ipxe
+
+echo
+echo =============================================
+echo   Lab PXE Bastion - INSTALLING VyOS
+echo   Target: ${params.hostname}
+echo   MAC:    ${params.mac}
+echo =============================================
+echo
+
+kernel ${base}/vyos-vmlinuz boot=live nopersistence noautologin BOOTIF=${bootif} fetch=${base}/vyos-filesystem.squashfs live-config.hooks=${base}/vyos/autoinstall.sh?mac=${params.mac}
+initrd ${base}/vyos-initrd
+boot
+`;
+}

bastion/src/bastion/src/templates/vyos-config-spec.ts

@@ -0,0 +1,352 @@
+// Builds the VyOS configuration spec applied by the autoinstall hook.
+//
+// We deliberately do NOT emit a config.boot file as text. A config.boot carries a
+// `vyos-config-version` trailer; without a trailer matching the running image,
+// VyOS runs its migration scripts from version 0 on first boot. Instead the hook
+// loads the image's own /opt/vyatta/etc/config.boot.default through vyos.configtree
+// and applies these set operations on top, so syntax and version trailer always
+// match the exact image being installed.
+
+import type { VyosInstallSpec } from "@lab/shared";
+
+export interface VyosSetOp {
+  path: string[];
+  value?: string;
+  /** false appends to a multi-value node (e.g. bond members) instead of replacing. */
+  replace?: boolean;
+}
+
+export interface VyosConfigSpec {
+  hostname: string;
+  /** "" means accept the installer default (the running image's version string). */
+  imageName: string;
+  password: string;
+  console: "K" | "S";
+  /** Target disk name (e.g. "nvme0n1"); "" accepts the installer's first-disk default. */
+  disk: string;
+  /**
+   * IP the driver should report in the "complete" callback ("ready at <ip>" —
+   * the exact format routes/api.ts parses installed.ip from). The mgmt
+   * address when static; "" means detect the live DHCP address at runtime.
+   */
+  reportAddress: string;
+  /** Whether to accept RAID-1 when the installer finds more than one disk. */
+  raid: boolean;
+  /** Overwrite the installed config.boot with the generated one on reinstall. */
+  freshConfig: boolean;
+  sets: VyosSetOp[];
+  /** Paths that are VyOS tag nodes — must be marked as such in the ConfigTree. */
+  tags: string[][];
+}
+
+/**
+ * Normalise a target disk to the form the installer expects.
+ *
+ * find_disks() enumerates via `lsblk -Jbp` (-p = full paths), so its valid
+ * responses are "/dev/mmcblk0"-style. A bare "mmcblk0" is rejected by
+ * ask_input()'s valid_responses check and re-prompts forever.
+ */
+function normalizeDiskPath(value: string | undefined): string {
+  const raw = (value ?? "").trim();
+  if (raw === "") return "";
+  return raw.startsWith("/dev/") ? raw : `/dev/${raw}`;
+}
+
+/** Sentinel marking a value that lives in Pulumi config, not in the bundle. */
+const SECRET_PREFIX = "@secret:";
+
+/**
+ * Enable the VyOS HTTP API so the router is manageable the moment it boots.
+ *
+ * This belongs at install time rather than in the Pulumi model: the model is
+ * applied THROUGH this API, so a router that lacks it cannot be brought under
+ * management without a hand-run change on a live firewall. It is also why the
+ * model excludes `service https` outright -- a provider able to rewrite its own
+ * transport can lock itself out permanently.
+ *
+ * `listen-address` is always set. Leaving it unbound would expose a
+ * config-write endpoint on every segment the router touches, the WAN included.
+ */
+function apiSets(apiKey: string, listenAddress: string): VyosSetOp[] {
+  const sets: VyosSetOp[] = [
+    { path: ["service", "https", "api", "keys", "id", "pulumi", "key"], value: apiKey },
+    { path: ["service", "https", "api", "rest"] },
+  ];
+  if (listenAddress !== "") {
+    sets.push({ path: ["service", "https", "listen-address"], value: listenAddress });
+  }
+  return sets;
+}
+
+/** Tag nodes introduced by the API config, needed by the installer's ConfigTree. */
+const API_TAGS: string[][] = [["service", "https", "api", "keys", "id"]];
+
+/**
+ * The address to bind the API to: an explicit choice, else the management
+ * address with its prefix length stripped. Under DHCP there is no address to
+ * bind at build time, so the caller must pass one or the listener stays unbound
+ * and the API is not enabled at all.
+ */
+function apiListenAddress(spec: VyosInstallSpec, mgmtAddress: string): string {
+  if (spec.apiListenAddress !== undefined && spec.apiListenAddress !== "") {
+    return spec.apiListenAddress;
+  }
+  return mgmtAddress.includes("/") ? (mgmtAddress.split("/")[0] ?? "") : "";
+}
+
+/**
+ * Use a Pulumi-rendered bundle as the router's config verbatim.
+ *
+ * Secret-valued nodes are dropped rather than installed with their sentinel
+ * text: writing `@secret:pppoePassword` into config.boot would look configured
+ * while being wrong, which is worse than being absent. The router comes up
+ * without those values and the first `pulumi up` fills them in.
+ *
+ * `system host-name` is forced to the hostname the install was asked for. The
+ * bundle carries the name of whichever router it was exported from, and
+ * installing vyos001's hostname onto vyos002 would collide on the network.
+ */
+function buildFromBundle(
+  params: { hostname: string; defaultPassword: string; disk?: string | undefined },
+  spec: VyosInstallSpec,
+  bundle: NonNullable<VyosInstallSpec["bundle"]>,
+  mgmtAddress: string,
+): VyosConfigSpec {
+  const sets: VyosSetOp[] = [];
+  const dropped: string[] = [];
+  for (const op of bundle.sets) {
+    if (op.value !== undefined && op.value.startsWith(SECRET_PREFIX)) {
+      dropped.push(op.path.join(" "));
+      continue;
+    }
+    if (op.path.length === 2 && op.path[0] === "system" && op.path[1] === "host-name") {
+      continue;
+    }
+    sets.push({
+      path: op.path,
+      ...(op.value === undefined ? {} : { value: op.value }),
+      ...(op.replace === undefined ? {} : { replace: op.replace }),
+    });
+  }
+  sets.unshift({ path: ["system", "host-name"], value: params.hostname });
+
+  if (dropped.length > 0) {
+    console.warn(
+      `vyos ${params.hostname}: ${dropped.length} secret-valued node(s) left unset by the ` +
+        `bundle; run \`pulumi up\` to supply them: ${dropped.join(", ")}`,
+    );
+  }
+
+  const tags = [...bundle.tags];
+  const api = enableApi(spec, params.hostname, mgmtAddress);
+  if (api.length > 0) {
+    sets.push(...api);
+    tags.push(...API_TAGS);
+  }
+
+  return {
+    hostname: params.hostname,
+    imageName: "",
+    password: spec.password ?? params.defaultPassword,
+    console: "K",
+    disk: normalizeDiskPath(params.disk),
+    reportAddress: mgmtAddress.includes("/") ? (mgmtAddress.split("/")[0] ?? "") : "",
+    raid: false,
+    freshConfig: spec.freshConfig ?? false,
+    sets,
+    tags,
+  };
+}
+
+/**
+ * The API config for this install, or nothing when it cannot be enabled safely.
+ *
+ * Refusing to enable it unbound is deliberate. Under DHCP there is no address
+ * known at build time, and the alternative -- binding to every interface --
+ * would publish a config-write endpoint on the WAN. Better to leave the router
+ * SSH-only and say so than to open it everywhere.
+ */
+function enableApi(spec: VyosInstallSpec, hostname: string, mgmtAddress: string): VyosSetOp[] {
+  if (spec.apiKey === undefined || spec.apiKey === "") return [];
+  const listen = apiListenAddress(spec, mgmtAddress);
+  if (listen === "") {
+    console.warn(
+      `vyos ${hostname}: --vyos-api-key given but no address to bind to ` +
+        `(management is "${mgmtAddress}"). Pass --vyos-api-listen <addr>; the HTTP API ` +
+        `has NOT been enabled, so Pulumi cannot manage this router yet.`,
+    );
+    return [];
+  }
+  return apiSets(spec.apiKey, listen);
+}
+
+export function buildVyosConfigSpec(params: {
+  hostname: string;
+  spec?: VyosInstallSpec | undefined;
+  defaultPassword: string;
+  sshKeys?: string[] | undefined;
+  disk?: string | undefined;
+}): VyosConfigSpec {
+  const spec = params.spec ?? {};
+  const mgmt = spec.mgmtInterface ?? "eth0";
+  const mgmtAddress = spec.mgmtAddress ?? "dhcp";
+
+  // A rendered bundle replaces the derived config entirely. Deriving a second
+  // opinion alongside it is the drift the bundle exists to prevent: Pulumi and
+  // labctl would each believe they knew the router's config, and the box would
+  // end up with whichever ran last.
+  if (spec.bundle !== undefined) {
+    return buildFromBundle(params, spec, spec.bundle, mgmtAddress);
+  }
+  const bondMembers = spec.bondMembers ?? [];
+  const vlans = spec.vlans ?? [];
+
+  const sets: VyosSetOp[] = [];
+  const tags: string[][] = [
+    ["interfaces", "ethernet"],
+    ["system", "login", "user"],
+  ];
+
+  const hwIds = spec.hwIds ?? {};
+  const pinHwId = (iface: string): void => {
+    const mac = hwIds[iface];
+    if (mac !== undefined && mac !== "") {
+      sets.push({ path: ["interfaces", "ethernet", iface, "hw-id"], value: mac });
+    }
+  };
+
+  sets.push({ path: ["system", "host-name"], value: params.hostname });
+
+  // Management interface — the NIC that PXE booted, left untagged and unbonded.
+  sets.push({ path: ["interfaces", "ethernet", mgmt, "address"], value: mgmtAddress });
+  pinHwId(mgmt);
+
+  // Tagged management VLAN on the PXE port. Emitted regardless of bonding, so
+  // the box stays reachable on the management VLAN while still booting untagged
+  // on whichever VLAN the bastion's proxy DHCP serves.
+  const mgmtVlan = spec.mgmtVlan;
+  if (mgmtVlan !== undefined) {
+    tags.push(["interfaces", "ethernet", mgmt, "vif"]);
+    const vif = ["interfaces", "ethernet", mgmt, "vif", String(mgmtVlan.id)];
+    sets.push({ path: [...vif, "address"], value: mgmtVlan.address });
+    if (mgmtVlan.description !== undefined && mgmtVlan.description !== "") {
+      sets.push({ path: [...vif, "description"], value: mgmtVlan.description });
+    }
+  }
+
+  // LACP bond. Members must exclude the PXE NIC; firmware PXE cannot run over LACP.
+  const bonded = bondMembers.length > 0;
+  if (bonded) {
+    tags.push(["interfaces", "bonding"]);
+    sets.push({ path: ["interfaces", "bonding", "bond0", "mode"], value: "802.3ad" });
+    sets.push({ path: ["interfaces", "bonding", "bond0", "hash-policy"], value: "layer2+3" });
+    for (const member of bondMembers) {
+      sets.push({
+        path: ["interfaces", "bonding", "bond0", "member", "interface"],
+        value: member,
+        replace: false,
+      });
+      pinHwId(member);
+    }
+    // Address on the trunk's native/untagged VLAN.
+    if (spec.bondAddress !== undefined && spec.bondAddress !== "") {
+      sets.push({ path: ["interfaces", "bonding", "bond0", "address"], value: spec.bondAddress });
+    }
+  }
+
+  // VRRP groups accumulate here; emitted (plus a sync group) after the VLANs.
+  // interface accepts dotted vifs (constraint regex `[0-9]+(.\d+)?`), address
+  // is a tag node (the VIP is the tag value itself), vrid range is 1-255.
+  const vrrpGroups: Array<{ name: string; iface: string; vrid: number; vip: string }> = [];
+  if (bonded && spec.bondVrrp !== undefined && spec.bondVrrp !== "") {
+    // vrid 1 for the untagged group: the native VLAN is never a vif, so this
+    // cannot collide with a vlan-id-derived vrid.
+    vrrpGroups.push({ name: "native", iface: "bond0", vrid: 1, vip: spec.bondVrrp });
+  }
+
+  // Tagged VLAN sub-interfaces hang off the bond when there is one, else off mgmt.
+  const parent = bonded
+    ? ["interfaces", "bonding", "bond0"]
+    : ["interfaces", "ethernet", mgmt];
+  if (vlans.length > 0) {
+    tags.push([...parent, "vif"]);
+    const parentName = bonded ? "bond0" : mgmt;
+    for (const vlan of vlans) {
+      const vif = [...parent, "vif", String(vlan.id)];
+      sets.push({ path: [...vif, "address"], value: vlan.address });
+      if (vlan.description !== undefined && vlan.description !== "") {
+        sets.push({ path: [...vif, "description"], value: vlan.description });
+      }
+      if (vlan.vrrp !== undefined && vlan.vrrp !== "") {
+        vrrpGroups.push({
+          name: `vlan${vlan.id}`,
+          iface: `${parentName}.${vlan.id}`,
+          vrid: vlan.id,
+          vip: vlan.vrrp,
+        });
+      }
+    }
+  }
+
+  // Emit VRRP groups plus one sync group so all VLANs fail over together —
+  // without it a single-link event could split mastership across the pair.
+  if (vrrpGroups.length > 0) {
+    tags.push(["high-availability", "vrrp", "group"]);
+    tags.push(["high-availability", "vrrp", "sync-group"]);
+    const priority = String(spec.vrrpPriority ?? 100);
+    for (const g of vrrpGroups) {
+      const base = ["high-availability", "vrrp", "group", g.name];
+      sets.push({ path: [...base, "interface"], value: g.iface });
+      sets.push({ path: [...base, "vrid"], value: String(g.vrid) });
+      sets.push({ path: [...base, "priority"], value: priority });
+      // address is a tag node: the VIP is the path's final segment, no value.
+      sets.push({ path: [...base, "address", g.vip] });
+      tags.push([...base, "address"]);
+      sets.push({
+        path: ["high-availability", "vrrp", "sync-group", "MAIN", "member"],
+        value: g.name,
+        replace: false,
+      });
+    }
+  }
+
+  sets.push({ path: ["service", "ssh", "port"], value: "22" });
+
+  const sshKeys = params.sshKeys ?? [];
+  if (sshKeys.length > 0) {
+    tags.push(["system", "login", "user", "vyos", "authentication", "public-keys"]);
+    sshKeys.forEach((entry, index) => {
+      const parts = entry.trim().split(/\s+/);
+      const type = parts[0] ?? "";
+      const key = parts[1] ?? "";
+      if (!type.startsWith("ssh-") && !type.startsWith("ecdsa-")) return;
+      if (!key) return;
+      const name = parts[2] ?? `lab-key-${index}`;
+      const base = ["system", "login", "user", "vyos", "authentication", "public-keys", name];
+      sets.push({ path: [...base, "type"], value: type });
+      sets.push({ path: [...base, "key"], value: key });
+    });
+  }
+
+  // Enabled here too, not just for bundle installs: every VyOS this bastion
+  // provisions should be manageable from first boot.
+  const api = enableApi(spec, params.hostname, mgmtAddress);
+  if (api.length > 0) {
+    sets.push(...api);
+    tags.push(...API_TAGS);
+  }
+
+  return {
+    hostname: params.hostname,
+    imageName: "",
+    password: spec.password ?? params.defaultPassword,
+    console: "K",
+    disk: normalizeDiskPath(params.disk),
+    // Static mgmt address wins; under DHCP the driver detects the live IP.
+    reportAddress: mgmtAddress.includes("/") ? (mgmtAddress.split("/")[0] ?? "") : "",
+    raid: false,
+    freshConfig: spec.freshConfig ?? false,
+    sets,
+    tags,
+  };
+}

bastion/src/bastion/src/templates/vyos-install.py.ts

@@ -0,0 +1,514 @@
+// Renders the Python program that performs the unattended VyOS install.
+//
+// It runs as root inside the live system, fetched and executed by live-config's
+// `hooks` component (see vyos-boot.ipxe.ts). It does three things:
+//   1. builds config.boot from the image's own default via vyos.configtree
+//   2. drives the interactive `install image` through a pty
+//   3. reports progress back to the bastion, then reboots
+//
+// A pty is used rather than piping stdin because the installer reads the
+// password through getpass(), which opens /dev/tty directly and would ignore a
+// pipe. Prompts are matched by text rather than replayed positionally: the
+// installer skips the boot-config question when it finds a previous
+// installation, so a fixed answer sequence desyncs on reinstall.
+
+import type { VyosConfigSpec } from "./vyos-config-spec.js";
+
+export function renderVyosInstallPy(params: {
+  spec: VyosConfigSpec;
+  mac: string;
+  serverIp: string;
+  httpPort: number;
+  role: string;
+}): string {
+  // Base64 so arbitrary values (passwords, descriptions, SSH keys) can never
+  // terminate the Python string literal that carries them.
+  const specB64 = Buffer.from(JSON.stringify(params.spec), "utf-8").toString("base64");
+
+  return `#!/usr/bin/env python3
+"""Unattended VyOS install driver -- generated by the lab PXE bastion."""
+
+import base64
+import json
+import os
+import pty
+import re
+import select
+import subprocess
+import sys
+import time
+import urllib.request
+
+SPEC = json.loads(base64.b64decode("${specB64}").decode("utf-8"))
+BASTION = "http://${params.serverIp}:${params.httpPort}"
+MAC = "${params.mac}"
+ROLE = ${JSON.stringify(params.role ?? "vanilla")}
+
+INSTALLER = "/usr/libexec/vyos/op_mode/image_installer.py"
+CONFIG_DIR = "/opt/vyatta/etc/config"
+# The installer copies the rootfs from the boot MEDIUM path -- which only a
+# CD/USB boot provides. With fetch= (HTTP netboot) nothing is mounted there
+# (verified in VM: Errno 2), so the squashfs must be linked or re-fetched into
+# place before 'install image' runs.
+ROOTFS_EXPECTED = "/usr/lib/live/mount/medium/live/filesystem.squashfs"
+SQUASHFS_URL = "http://${params.serverIp}:${params.httpPort}/vyos-filesystem.squashfs"
+# The live-config hook runs BEFORE vyos-router creates the /opt/vyatta compat
+# path, so the squashfs's own location must be tried too (verified in VM: only
+# /usr/share/vyos/config.boot.default exists at hook time).
+DEFAULT_CONFIG_CANDIDATES = [
+    "/opt/vyatta/etc/config.boot.default",
+    "/usr/share/vyos/config.boot.default",
+]
+STALL_TIMEOUT = 900  # seconds without installer output before giving up
+
+
+def detect_ip():
+    """Best-effort local IP as seen on the route toward the bastion.
+
+    Matches Fedora's semantics (IP captured during install): under DHCP the
+    installed system will renew on the same NIC/subnet the live env used.
+    """
+    import socket
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        s.connect(("${params.serverIp}", ${params.httpPort}))
+        ip = s.getsockname()[0]
+        s.close()
+        return ip
+    except Exception:
+        return ""
+
+
+class LogStreamer:
+    """Stream install output to the bastion's /api/log so 'labctl provision
+    logs -f' works live for VyOS, like Anaconda's syslog does for Fedora.
+
+    Strictly best-effort: a failed POST drops the batch and must never stall
+    the pty read loop or fail the install.
+    """
+
+    ANSI = re.compile(rb"\\x1b\\[[0-9;?]*[a-zA-Z]|\\x1b[=>]|\\r")
+
+    def __init__(self):
+        self.partial = b""
+        self.pending = []
+        self.last_flush = time.time()
+
+    def feed(self, chunk):
+        """Raw pty bytes: split into lines, strip ANSI noise, queue."""
+        self.partial += chunk
+        while b"\\n" in self.partial:
+            raw, self.partial = self.partial.split(b"\\n", 1)
+            text = self.ANSI.sub(b"", raw).decode("utf-8", "replace").rstrip()
+            if text:
+                self.pending.append(text)
+        self.maybe_flush()
+
+    def line(self, text):
+        """A driver-originated message (already a clean string)."""
+        self.pending.append(text)
+        self.maybe_flush()
+
+    def maybe_flush(self):
+        if len(self.pending) >= 20 or (self.pending and time.time() - self.last_flush >= 2):
+            self.flush()
+
+    def flush(self):
+        if not self.pending:
+            return
+        batch, self.pending = self.pending[:200], self.pending[200:]
+        self.last_flush = time.time()
+        try:
+            body = json.dumps({"mac": MAC, "lines": batch}).encode()
+            req = urllib.request.Request(
+                BASTION + "/api/log",
+                data=body,
+                headers={"Content-Type": "application/json"},
+            )
+            urllib.request.urlopen(req, timeout=5).read()
+        except Exception:
+            pass
+
+
+STREAM = LogStreamer()
+
+
+def say(msg):
+    """Print locally and stream to the bastion log buffer."""
+    print(msg)
+    STREAM.line(str(msg))
+
+
+def report(stage, detail=""):
+    """Best-effort progress callback; never fatal."""
+    STREAM.flush()
+    try:
+        body = json.dumps({"mac": MAC, "stage": stage, "detail": detail}).encode()
+        req = urllib.request.Request(
+            BASTION + "/api/progress",
+            data=body,
+            headers={"Content-Type": "application/json"},
+        )
+        urllib.request.urlopen(req, timeout=5).read()
+    except Exception:
+        pass
+
+
+def build_config():
+    """Apply our set operations onto the image's own default config.
+
+    Using config.boot.default as the base keeps the vyos-config-version trailer
+    consistent with the running image, so first boot does not run migrations.
+    """
+    from vyos.configtree import ConfigTree
+
+    default_config = next(
+        (p for p in DEFAULT_CONFIG_CANDIDATES if os.path.exists(p)), None)
+    if default_config is None:
+        raise FileNotFoundError(
+            "no config.boot.default found (tried %s)" % ", ".join(DEFAULT_CONFIG_CANDIDATES))
+    say("base config: %s" % default_config)
+
+    with open(default_config) as handle:
+        config = ConfigTree(handle.read())
+
+    for op in SPEC["sets"]:
+        replace = op.get("replace", True)
+        if "value" in op and op["value"] is not None:
+            config.set(op["path"], value=op["value"], replace=replace)
+        else:
+            config.set(op["path"])
+
+    # Tag nodes must be marked after the nodes exist, as the installer itself does.
+    for tag in SPEC["tags"]:
+        try:
+            config.set_tag(tag)
+        except Exception as err:
+            say("warning: set_tag %s failed: %s" % (tag, err))
+
+    os.makedirs(CONFIG_DIR, exist_ok=True)
+    target = os.path.join(CONFIG_DIR, "config.boot")
+
+    # Re-attach the vyos-config-version footer: ConfigTree.to_string() emits
+    # only the config body, and a config without the footer is treated as
+    # ancient -- the boot migrator then runs every migration over it and (as
+    # observed in the VM test) crashes in system/31-to-32. Building the footer
+    # from the running system pins it to the exact image being installed.
+    body = config.to_string()
+    try:
+        from vyos.component_version import version_info_from_system
+        info = version_info_from_system()
+        info.update_config_body(body)
+        info.write(target)
+        say("wrote %s (footer: %s)" % (target, info.release))
+    except Exception as err:
+        say("warning: version footer failed (%s); writing bare config" % err)
+        with open(target, "w") as handle:
+            handle.write(body)
+    return target
+
+
+def find_live_squashfs():
+    """Locate the squashfs live-boot fetched, without walking into the mounted
+    rootfs or overlay (each would mean traversing the entire OS tree)."""
+    explicit = [
+        "/run/live/medium/live/filesystem.squashfs",
+        "/lib/live/mount/medium/live/filesystem.squashfs",
+    ]
+    for path in explicit:
+        if os.path.isfile(path) and os.path.getsize(path) > 0:
+            return path
+    for root in ("/run/live", "/lib/live/mount", "/usr/lib/live/mount"):
+        for dirpath, dirs, files in os.walk(root):
+            depth = dirpath.count(os.sep) - root.count(os.sep)
+            dirs[:] = [d for d in dirs
+                       if d not in ("rootfs", "overlay")
+                       and not d.endswith(".squashfs")
+                       and depth < 3]
+            if "filesystem.squashfs" in files:
+                path = os.path.join(dirpath, "filesystem.squashfs")
+                if os.path.isfile(path) and os.path.getsize(path) > 0:
+                    return path
+    return None
+
+
+def ensure_rootfs():
+    """Make FILE_ROOTFS_SRC exist so the installer can copy the system image."""
+    if os.path.isfile(ROOTFS_EXPECTED) and os.path.getsize(ROOTFS_EXPECTED) > 0:
+        return
+    src = find_live_squashfs()
+    if src is None:
+        say("squashfs not in live mounts; re-fetching %s" % SQUASHFS_URL)
+        src = "/tmp/filesystem.squashfs"
+        urllib.request.urlretrieve(SQUASHFS_URL, src)
+    os.makedirs(os.path.dirname(ROOTFS_EXPECTED), exist_ok=True)
+    if os.path.lexists(ROOTFS_EXPECTED):
+        os.remove(ROOTFS_EXPECTED)
+    os.symlink(src, ROOTFS_EXPECTED)
+    say("rootfs source: %s -> %s" % (ROOTFS_EXPECTED, src))
+
+
+def build_rules():
+    """Prompt -> response table for the interactive installer."""
+    password = SPEC["password"].encode() + b"\\n"
+    image_name = SPEC["imageName"].encode() + b"\\n"
+    disk = SPEC["disk"].encode() + b"\\n"
+    console = SPEC["console"].encode() + b"\\n"
+    raid = (b"yes\\n" if SPEC["raid"] else b"no\\n")
+
+    return [
+        (re.compile(rb"Would you like to continue\\?"), b"yes\\n"),
+        (re.compile(rb"What would you like to name this image\\?"), image_name),
+        (re.compile(rb"Please confirm password for the .vyos. user:"), password),
+        (re.compile(rb"Please enter a password for the .vyos. user:"), password),
+        (re.compile(rb"What console should be used by default"), console),
+        # Three RAID variants: "configure RAID-1 mirroring?", "...on them?",
+        # and "choose two disks for RAID-1 mirroring?" -- all default to YES,
+        # so a missed one both hangs the install and risks an unwanted mirror.
+        (re.compile(rb"Would you like to [^?]*RAID-1 mirroring"), raid),
+        (re.compile(rb"Installation will delete all data on (?:the drive|both drives)\\. Continue\\?"), b"yes\\n"),
+        (re.compile(rb"Which one should be used for installation\\?"), disk),
+        (re.compile(rb"Would you like to use all the free space on the drive\\?"), b"yes\\n"),
+        (re.compile(rb"Which file would you like as boot config\\?"), b"1\\n"),
+        # Reinstall path only (search_previous_installation): carrying the old
+        # /config and SSH host keys forward is VyOS's "reinstall without losing
+        # data". Always yes -- freshConfig replaces config.boot afterwards, so
+        # answering no here would also discard non-config data under /config.
+        (re.compile(rb"Would you like to copy data to the new image\\?"), b"yes\\n"),
+        (re.compile(rb"Would you like to copy the encrypted config to the new image\\?"), b"yes\\n"),
+        # More than one previous image found -- take the first offered.
+        (re.compile(rb"From which image would you like to save config information\\?"), b"1\\n"),
+        (re.compile(rb"From which image would you like to copy the encrypted config\\?"), b"1\\n"),
+    ]
+
+
+def run_installer():
+    """Drive image_installer.py over a pty, answering prompts as they appear."""
+    rules = build_rules()
+    master, slave = pty.openpty()
+
+    proc = subprocess.Popen(
+        [INSTALLER, "--action", "install"],
+        stdin=slave,
+        stdout=slave,
+        stderr=slave,
+        close_fds=True,
+        preexec_fn=os.setsid,
+    )
+    os.close(slave)
+
+    buf = b""
+    transcript = b""   # rolling tail of everything the installer printed
+    last_output = time.time()
+
+    while True:
+        ready, _, _ = select.select([master], [], [], 1.0)
+
+        if ready:
+            try:
+                chunk = os.read(master, 4096)
+            except OSError:
+                break
+            if not chunk:
+                break
+
+            sys.stdout.buffer.write(chunk)
+            sys.stdout.buffer.flush()
+            buf += chunk
+            transcript = (transcript + chunk)[-8000:]
+            STREAM.feed(chunk)
+            last_output = time.time()
+
+            # Answer every prompt currently in the buffer, earliest first, so
+            # ordering is preserved even when the installer skips questions --
+            # and so a single chunk carrying two prompts gets both answers.
+            while True:
+                best = None
+                for pattern, response in rules:
+                    found = pattern.search(buf)
+                    if found and (best is None or found.start() < best[0].start()):
+                        best = (found, response)
+                if best is None:
+                    break
+                found, response = best
+                os.write(master, response)
+                transcript = (transcript + b"\\n>>> answered: " + response)[-8000:]
+                STREAM.line(">>> answered: " + response.decode("utf-8", "replace").strip())
+                buf = buf[found.end():]
+
+            # Bound memory if the installer emits a lot without prompting.
+            if len(buf) > 65536:
+                buf = buf[-8192:]
+
+        elif proc.poll() is not None:
+            break
+
+        STREAM.maybe_flush()
+
+        if time.time() - last_output > STALL_TIMEOUT:
+            proc.kill()
+            raise SystemExit("installer produced no output for %ds" % STALL_TIMEOUT)
+
+    os.close(master)
+    return proc.wait(), transcript.decode("utf-8", "replace")
+
+
+def ensure_network_boot_first():
+    """Keep network boot first so the bastion intercepts every reboot.
+
+    Port of the Fedora kickstart's %post efibootmgr step (install.ks.ts) --
+    what makes reprovision-by-reboot work. Best-effort: skipped on BIOS boots
+    or when efibootmgr is absent. Runs from the live env after the installer;
+    efibootmgr edits NVRAM, not the disk, so installer cleanup is irrelevant.
+    """
+    import shutil
+    if not os.path.isdir("/sys/firmware/efi") or shutil.which("efibootmgr") is None:
+        say("boot order: skipped (BIOS boot or efibootmgr missing)")
+        return
+    try:
+        out = subprocess.run(["efibootmgr"], capture_output=True, text=True, timeout=30).stdout
+        order = []
+        network_entry = None
+        for line in out.splitlines():
+            m = re.match(r"^BootOrder:\\s*(.*)$", line)
+            if m:
+                order = [x.strip() for x in m.group(1).split(",") if x.strip()]
+                continue
+            m = re.match(r"^Boot([0-9A-Fa-f]{4})\\*?\\s+(.*)$", line)
+            if m and network_entry is None:
+                if re.search(r"network|pxe|ipv4|ipv6|http", m.group(2), re.IGNORECASE):
+                    network_entry = m.group(1).upper()
+        if network_entry is None or not order:
+            say("boot order: no network boot entry found; leaving as is")
+            return
+        new_order = [network_entry] + [x for x in order if x.upper() != network_entry]
+        if [x.upper() for x in order] == [x.upper() for x in new_order]:
+            say("boot order: network entry Boot%s already first" % network_entry)
+            return
+        subprocess.run(["efibootmgr", "-o", ",".join(new_order)],
+                       capture_output=True, timeout=30)
+        say("boot order: moved network entry Boot%s first" % network_entry)
+    except Exception as err:
+        say("warning: boot order adjustment failed: %s" % err)
+
+
+def with_target_mounted(fn):
+    """Mount the installed root partition, call fn(rw_dir), always unmount.
+
+    The installer has unmounted and cleaned the target by the time this runs,
+    so the block device is free. The partition holding boot/<image>/rw is the
+    VyOS root; the glob also yields the installed image's rw dir directly.
+    """
+    import glob
+    disk = SPEC["disk"]
+    if not disk:
+        # No pinned disk (installer picked the default) -- enumerate all disks.
+        candidates = ["/dev/" + b for b in os.listdir("/sys/block")
+                      if not b.startswith(("loop", "ram", "zram", "sr"))]
+    else:
+        candidates = [disk]
+
+    mnt = "/mnt/lab-target"
+    os.makedirs(mnt, exist_ok=True)
+    for dev in candidates:
+        name = os.path.basename(dev)
+        parts = sorted(p for p in os.listdir("/sys/block/%s" % name)
+                       if p.startswith(name)) if os.path.isdir("/sys/block/%s" % name) else []
+        for part in parts:
+            pdev = "/dev/" + part
+            if subprocess.run(["mount", pdev, mnt], capture_output=True).returncode != 0:
+                continue
+            try:
+                rw_dirs = glob.glob(os.path.join(mnt, "boot", "*", "rw"))
+                if rw_dirs:
+                    fn(rw_dirs[0])
+                    return True
+            finally:
+                subprocess.run(["umount", mnt], capture_output=True)
+    return False
+
+
+def post_install_target_steps():
+    """Metadata + optional fresh-config overwrite inside the installed image."""
+    def apply(rw_dir):
+        config_dir = os.path.join(rw_dir, "opt/vyatta/etc/config")
+        os.makedirs(config_dir, exist_ok=True)
+
+        # /config/lab-provisioned -- survives VyOS image upgrades. Mirrors the
+        # Fedora kickstart's /etc/lab-provisioned.
+        try:
+            with open(os.path.join(config_dir, "lab-provisioned"), "w") as handle:
+                handle.write("hostname=%s\\n" % SPEC["hostname"])
+                handle.write("role=%s\\n" % ROLE)
+                handle.write("provisioned=%s\\n" % time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
+                handle.write("bastion=%s\\n" % BASTION)
+            say("wrote /config/lab-provisioned")
+        except Exception as err:
+            say("warning: lab-provisioned metadata failed: %s" % err)
+
+        # freshConfig: make the bastion-generated config win over the previous
+        # installation's carried-forward config. Explicit intent -- failure is
+        # fatal (raised out of with_target_mounted).
+        if SPEC.get("freshConfig"):
+            import shutil
+            shutil.copyfile(os.path.join(CONFIG_DIR, "config.boot"),
+                            os.path.join(config_dir, "config.boot"))
+            say("freshConfig: replaced installed config.boot with generated config")
+
+    mounted = with_target_mounted(apply)
+    if not mounted:
+        if SPEC.get("freshConfig"):
+            raise RuntimeError("freshConfig requested but installed root partition not found")
+        say("warning: installed root partition not found; skipping metadata")
+
+
+def main():
+    report("vyos-install", "building config.boot")
+    try:
+        build_config()
+    except Exception as err:
+        report("error", "config generation failed: %s" % err)
+        raise
+
+    report("vyos-install", "staging rootfs for installer")
+    try:
+        ensure_rootfs()
+    except Exception as err:
+        report("error", "rootfs staging failed: %s" % err)
+        raise
+
+    report("vyos-install", "running install image")
+    code, transcript = run_installer()
+
+    if code != 0:
+        # Surface the installer's last words in bastion progress -- the console
+        # they were printed on is usually invisible during unattended installs.
+        report("error", "install image exited %d | tail: %s" % (code, transcript[-4000:]))
+        raise SystemExit(code)
+
+    report("post-install", "boot order + metadata")
+    ensure_network_boot_first()
+    try:
+        post_install_target_steps()
+    except Exception as err:
+        report("error", "post-install target steps failed: %s" % err)
+        raise
+
+    # "complete" is the stage the bastion uses to move a machine out of the
+    # install queue into installed state, and "ready at <ip>" is the exact
+    # detail format it parses installed.ip from -- see routes/api.ts.
+    ip = SPEC.get("reportAddress") or detect_ip()
+    report("complete", "ready at %s" % ip if ip else "VyOS installed, rebooting")
+    os.system("sync")
+    # --force: this driver is a child of live-config.service, whose start job is
+    # still running -- a normal reboot deadlocks waiting for it (verified in VM:
+    # shutdown blocked >1min on "start job is running for live-config"). The
+    # installer has already unmounted and cleaned the target, so an immediate
+    # reboot is safe.
+    os.system("systemctl reboot --force")
+
+
+if __name__ == "__main__":
+    main()
+`;
+}

bastion/src/bastion/tests/arch-dispatch.test.ts

@@ -1,291 +0,0 @@
-// aarch64 support in the PXE dispatch path.
-//
-// The x86_64 side is pinned separately by ipxe-x86-regression.test.ts.
-
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { mkdirSync, rmSync } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import type { BastionConfig, BastionState, HardwareInfo } from "@lab/shared";
-import { createApp } from "../src/server.js";
-import { resolveArch } from "../src/routes/dispatch.js";
-import { renderDnsmasqConf } from "../src/templates/dnsmasq.conf.js";
-import type { FastifyInstance } from "fastify";
-import type { StateManager } from "../src/services/state.js";
-
-function createTestConfig(testDir: string): BastionConfig {
-  return {
-    fedoraVersion: "43",
-    arch: "x86_64",
-    httpPort: 0,
-    timezone: "Europe/London",
-    locale: "en_GB.UTF-8",
-    bastionDir: testDir,
-    domain: "test.local",
-    dhcpMode: "proxy",
-    dhcpRangeStart: "",
-    dhcpRangeEnd: "",
-    ubuntuVersion: "26.04",
-    ubuntuMirror: "https://releases.ubuntu.com/26.04",
-    iface: "eth0",
-    serverIp: "10.0.0.1",
-    network: "10.0.0.0",
-    gateway: "10.0.0.1",
-    sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"],
-    adminUser: "testadmin",
-    syslogPort: 15514,
-    skipDnsmasq: true,
-    skipArtifacts: true,
-    fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
-    tftpDir: join(testDir, "tftp"),
-    httpDir: join(testDir, "http"),
-    stateFile: join(testDir, "state.json"),
-  };
-}
-
-function hardware(mac: string, over: Partial<HardwareInfo> = {}): HardwareInfo {
-  return {
-    mac,
-    product: "TestBox",
-    board: "TestBoard",
-    serial: "SN123",
-    manufacturer: "TestCorp",
-    cpu_model: "Test CPU",
-    cpu_cores: 4,
-    memory_gb: 16,
-    arch: "x86_64",
-    disks: [],
-    nics: [],
-    first_seen: new Date().toISOString(),
-    last_seen: new Date().toISOString(),
-    ...over,
-  };
-}
-
-const emptyState = (): BastionState => ({
-  discovered: {}, install_queue: {}, installed: {}, debug: {},
-});
-
-describe("architecture resolution", () => {
-  const config = createTestConfig("/tmp/unused");
-  const mac = "aa:bb:cc:dd:ee:ff";
-
-  it("prefers the tracked record over what the client reports", () => {
-    const state = emptyState();
-    state.discovered[mac] = hardware(mac, { arch: "aarch64" });
-    // Client claims x86_64; the machine record says otherwise and wins.
-    expect(resolveArch(state, mac, "x86_64", config)).toBe("aarch64");
-  });
-
-  it("falls back to the architecture reported at boot", () => {
-    expect(resolveArch(emptyState(), mac, "arm64", config)).toBe("aarch64");
-  });
-
-  it("normalises iPXE's arm64 spelling to aarch64", () => {
-    expect(resolveArch(emptyState(), mac, "arm64", config)).toBe("aarch64");
-    expect(resolveArch(emptyState(), mac, "x86_64", config)).toBe("x86_64");
-  });
-
-  it("falls back to the configured default for unknown architectures", () => {
-    expect(resolveArch(emptyState(), mac, "riscv64", config)).toBe("x86_64");
-    expect(resolveArch(emptyState(), mac, undefined, config)).toBe("x86_64");
-  });
-
-  it("reads arch from the installed record for already-provisioned machines", () => {
-    const state = emptyState();
-    state.installed[mac] = {
-      hostname: "spark", role: "worker", ip: "10.0.0.5",
-      installed_at: new Date().toISOString(), arch: "aarch64",
-    };
-    expect(resolveArch(state, mac, undefined, config)).toBe("aarch64");
-  });
-});
-
-describe("aarch64 dispatch", () => {
-  let testDir: string;
-  let app: FastifyInstance;
-  let state: StateManager;
-  const mac = "aa:bb:cc:dd:ee:ff";
-
-  beforeEach(() => {
-    testDir = join(tmpdir(), `bastion-arch-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
-    mkdirSync(join(testDir, "http"), { recursive: true });
-    mkdirSync(join(testDir, "tftp"), { recursive: true });
-    const result = createApp(createTestConfig(testDir));
-    app = result.app;
-    state = result.state;
-  });
-
-  afterEach(async () => {
-    await app.close();
-    rmSync(testDir, { recursive: true, force: true });
-  });
-
-  it("serves the aarch64 kernel and initrd to an arm64 client", async () => {
-    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}&arch=arm64` });
-    expect(res.statusCode).toBe(200);
-    expect(res.body).toContain("/vmlinuz-aarch64");
-    expect(res.body).toContain("/initrd-aarch64.img");
-    expect(res.body).not.toContain("/vmlinuz ");
-  });
-
-  it("points an arm64 client at the aarch64 Fedora mirror", async () => {
-    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}&arch=arm64` });
-    expect(res.body).toContain("Everything/aarch64/os");
-    expect(res.body).not.toContain("Everything/x86_64/os");
-  });
-
-  it("uses serial console arguments and not nomodeset on arm64", async () => {
-    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}&arch=arm64` });
-    expect(res.body).toContain("console=ttyAMA0,115200");
-    expect(res.body).not.toContain("nomodeset");
-  });
-
-  it("refuses to serve the x86-only Ubuntu kernel to an arm64 client", async () => {
-    // A machine queued for Ubuntu before it was discovered as aarch64 reaches dispatch
-    // with no guard having run. Serving it /ubuntu-vmlinuz is the original bug.
-    state.update((s) => {
-      s.install_queue[mac] = {
-        hostname: "arm-node", disk: "", role: "worker",
-        os: "ubuntu-26.04", queued_at: new Date().toISOString(),
-      };
-    });
-
-    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}&arch=arm64` });
-    expect(res.statusCode).toBe(200);
-    expect(res.body).toContain("CANNOT BOOT THIS MACHINE");
-    expect(res.body).toContain("no aarch64 netboot artifacts");
-    expect(res.body).not.toContain("ubuntu-vmlinuz");
-  });
-
-  it("still serves Ubuntu to an x86_64 client", async () => {
-    state.update((s) => {
-      s.install_queue[mac] = {
-        hostname: "x86-node", disk: "", role: "worker",
-        os: "ubuntu-26.04", queued_at: new Date().toISOString(),
-      };
-    });
-
-    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}&arch=x86_64` });
-    expect(res.body).toContain("ubuntu-vmlinuz");
-    expect(res.body).not.toContain("CANNOT BOOT");
-  });
-
-  it("serves a rescue kernel for the recorded architecture, not the requester's", async () => {
-    // The Spark case: machine known to be aarch64, queued for rescue.
-    state.update((s) => {
-      s.discovered[mac] = hardware(mac, { arch: "aarch64" });
-      s.debug[mac] = { hostname: "spark-2935", queued_at: new Date().toISOString() };
-    });
-
-    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}` });
-    expect(res.body).toContain("DEBUG/RESCUE MODE");
-    expect(res.body).toContain("/vmlinuz-aarch64");
-    expect(res.body).toContain("inst.rescue");
-    expect(res.body).toContain("inst.sshd");
-  });
-});
-
-describe("--pxe-boot root device", () => {
-  let testDir: string;
-  let app: FastifyInstance;
-  let state: StateManager;
-  const mac = "aa:bb:cc:dd:ee:ff";
-
-  beforeEach(() => {
-    testDir = join(tmpdir(), `bastion-root-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
-    mkdirSync(join(testDir, "http"), { recursive: true });
-    mkdirSync(join(testDir, "tftp"), { recursive: true });
-    const result = createApp(createTestConfig(testDir));
-    app = result.app;
-    state = result.state;
-  });
-
-  afterEach(async () => {
-    await app.close();
-    rmSync(testDir, { recursive: true, force: true });
-  });
-
-  it("uses the root device recorded on the machine", async () => {
-    state.update((s) => {
-      s.installed[mac] = {
-        hostname: "worker-1", role: "worker", ip: "10.0.0.50",
-        installed_at: new Date().toISOString(),
-        root_device: "/dev/mapper/otherVG-root",
-        root_args: "rd.lvm.lv=otherVG/root",
-      };
-      s.debug[mac] = { hostname: "worker-1", queued_at: new Date().toISOString(), pxeBoot: true };
-    });
-
-    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}` });
-    expect(res.body).toContain("PXE BOOT (debug)");
-    expect(res.body).toContain("root=/dev/mapper/otherVG-root");
-    expect(res.body).toContain("rd.lvm.lv=otherVG/root");
-    // The old hardcoded layout must not leak back in.
-    expect(res.body).not.toContain("labvg");
-  });
-
-  it("falls back to rescue rather than guessing when no root device is known", async () => {
-    state.update((s) => {
-      s.installed[mac] = {
-        hostname: "spark-2935", role: "worker", ip: "192.168.8.12",
-        installed_at: new Date().toISOString(), arch: "aarch64",
-      };
-      s.debug[mac] = { hostname: "spark-2935", queued_at: new Date().toISOString(), pxeBoot: true };
-    });
-
-    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}` });
-    expect(res.body).toContain("DEBUG/RESCUE MODE");
-    expect(res.body).toContain("no root device is recorded");
-    expect(res.body).toContain("debug-setup.sh");
-    expect(res.body).not.toContain("root=");
-    // And it is still the right architecture.
-    expect(res.body).toContain("/vmlinuz-aarch64");
-  });
-
-  it("records a root device reported from a rescue shell without erasing hardware info", async () => {
-    state.update((s) => {
-      s.discovered[mac] = hardware(mac, { product: "DGX Spark", manufacturer: "NVIDIA", arch: "aarch64" });
-    });
-
-    const res = await app.inject({
-      method: "POST",
-      url: "/api/discover",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ mac, root_device: "/dev/nvme0n1p2" }),
-    });
-    expect(res.statusCode).toBe(200);
-
-    const hw = state.load().discovered[mac];
-    expect(hw?.root_device).toBe("/dev/nvme0n1p2");
-    // The partial report must not blank what we already knew.
-    expect(hw?.product).toBe("DGX Spark");
-    expect(hw?.cpu_cores).toBe(4);
-    expect(hw?.arch).toBe("aarch64");
-  });
-});
-
-describe("dnsmasq architecture detection", () => {
-  const conf = renderDnsmasqConf(createTestConfig("/tmp/unused"));
-
-  it("maps DHCP option 93 values to per-architecture bootloaders", () => {
-    // 11 = ARM 64-bit UEFI
-    expect(conf).toContain("dhcp-match=set:efi-arm64,option:client-arch,11");
-    expect(conf).toContain("dhcp-boot=tag:efi-arm64,tag:!ipxe,ipxe-arm64.efi");
-    // 7 / 9 = x64 UEFI, 0 = x86 BIOS
-    expect(conf).toContain("dhcp-match=set:efi-x86_64,option:client-arch,7");
-    expect(conf).toContain("dhcp-match=set:efi-x86_64,option:client-arch,9");
-    expect(conf).toContain("dhcp-match=set:bios,option:client-arch,0");
-  });
-
-  it("matches arm64 UEFI HTTP boot on 19, not 20", () => {
-    // IANA: 19 = arm uefi 64 boot from http, 20 = pc/at bios boot from http.
-    expect(conf).toContain("dhcp-match=set:httpboot-arm64,option:client-arch,19");
-    expect(conf).not.toContain("dhcp-match=set:httpboot-arm64,option:client-arch,20");
-    expect(conf).toContain("dhcp-match=set:httpboot-x86_64,option:client-arch,16");
-  });
-
-  it("offers an arm64 PXE service directive in proxy mode", () => {
-    expect(conf).toContain('pxe-service=tag:!ipxe,ARM64_EFI,"PXE Boot",ipxe-arm64.efi');
-  });
-});

bastion/src/bastion/tests/asahi.test.ts

@@ -223,3 +223,67 @@ describe("renderFirstbootUnit", () => {
     expect(unit).toContain("ConditionPathExists=!/etc/lab-lvm-setup-done");
   });
 });
+
+// ---------------------------------------------------------------------------
+// v2 layout. Asahi was the THIRD generator and the most drifted: it hardcoded
+// rancher at 20480M under a comment claiming "sizes match install.ks.ts", which
+// uses 122880M. worker1 still runs that 20G image store today.
+// ---------------------------------------------------------------------------
+describe("v2 layout", () => {
+  const roles = ["vanilla", "worker", "infra", "labcontroller"] as const;
+  const p = (role: (typeof roles)[number], schemaVersion?: 1 | 2) =>
+    renderFirstbootScript({
+      hostname: "mac1", role, serverIp: "10.0.0.1", httpPort: 8080,
+      sshKeys: ["ssh-ed25519 AAAA t"], adminUser: "michal", mac: "aa:bb:cc:dd:ee:ff",
+      ...(schemaVersion ? { schemaVersion } : {}),
+    } as any);
+
+  function lvSizes(script: string): Record<string, number> {
+    const out: Record<string, number> = {};
+    for (const m of script.matchAll(/lvcreate -L (\d+)M\s+-n (\w+)\s+labvg/g)) {
+      out[m[2]!] = Number(m[1]);
+    }
+    return out;
+  }
+
+  it("v2 never creates a root LV — the Asahi installer already made a btrfs root", () => {
+    for (const role of roles) {
+      expect(Object.keys(lvSizes(p(role, 2)))).not.toContain("root");
+    }
+  });
+
+  it("v2 sizes rancher at 250G, not the 20G that undersized worker1", () => {
+    const lvs = lvSizes(p("infra", 2));
+    expect(lvs["rancher"]).toBe(256000);
+    expect(lvs["var"]).toBe(49152);
+    expect(lvs["swap"]).toBe(16384);
+  });
+
+  it("v2 gives a worker a rancher LV; v1 gave it only to servers", () => {
+    expect(lvSizes(p("worker", 2))["rancher"]).toBe(256000);
+    expect(lvSizes(p("worker"))["rancher"]).toBeUndefined();
+  });
+
+  it("v2 never uses 100%FREE", () => {
+    for (const role of roles) {
+      expect(p(role, 2)).not.toContain("100%FREE");
+    }
+  });
+
+  it("v1 still uses 100%FREE for longhorn and is otherwise unchanged", () => {
+    const v1 = p("worker");
+    expect(v1).toContain("lvcreate -l 100%FREE -n longhorn");
+    const lvs = lvSizes(v1);
+    expect(lvs["swap"]).toBe(27648);
+    expect(lvs["var"]).toBe(102400);
+  });
+
+  it("v2 tags the volume group so a later run can detect the layout", () => {
+    expect(p("worker", 2)).toContain("vgchange --addtag lab-schema-v2 labvg");
+    expect(p("worker")).not.toContain("--addtag lab-schema-v2");
+  });
+
+  it("reprovision reports the on-disk schema", () => {
+    expect(p("worker", 2)).toMatch(/vgs --noheadings -o vg_tags labvg.*grep -q lab-schema-v2/);
+  });
+});

bastion/src/bastion/tests/auto-k3s.test.ts

@@ -0,0 +1,137 @@
+// Tests for the post-provision k3s auto-install gate.
+//
+// Two independent switches, for two different problems:
+//
+//  1. shouldAutoInstallK3s(role) — a ROLE gate. The post-provision installer
+//     runs a bare `INSTALL_K3S_EXEC=server|agent` with no K3S_URL and no
+//     K3S_TOKEN and knows about exactly one machine. For a node that is its
+//     own cluster that is fine. For a cluster MEMBER it is wrong: servers get
+//     cluster-init semantics and form their own separate cluster (so three
+//     k8s-combo boxes yield three clusters), and agents cannot start at all
+//     ("Job for k3s-agent.service failed").
+//
+//  2. config.autoK3s — a global off switch, needed because the prerequisite
+//     steps stop k3s and delete CNI state several minutes after an install
+//     lands, which lands on top of a test forming its own cluster.
+
+import { describe, it, expect } from "vitest";
+import { shouldAutoInstallK3s } from "../src/services/post-provision.js";
+import { loadConfig } from "../src/config.js";
+import { SUPPORTED_ROLES, isK3sServer, roleInfo } from "@lab/shared";
+import type { Role } from "@lab/shared";
+
+describe("shouldAutoInstallK3s", () => {
+  // These join a cluster someone else owns, so they cannot be auto-installed
+  // without the server URL and token that post-provision does not have.
+  const CLUSTER_MEMBERS: Role[] = ["k8s-combo", "k8s-controller", "k8s-worker", "worker"];
+
+  // These are each their own cluster (or no k3s at all), so a bare install is
+  // a correct install.
+  const SELF_CONTAINED: Role[] = ["standalone-cluster", "infra", "labcontroller", "vanilla"];
+
+  it.each(CLUSTER_MEMBERS)("refuses to auto-install %s", (role) => {
+    expect(shouldAutoInstallK3s(role)).toBe(false);
+  });
+
+  it.each(SELF_CONTAINED)("still auto-installs %s", (role) => {
+    expect(shouldAutoInstallK3s(role)).toBe(true);
+  });
+
+  it("covers every supported role — a new role must make a deliberate choice", () => {
+    for (const role of SUPPORTED_ROLES) {
+      expect(
+        CLUSTER_MEMBERS.includes(role) || SELF_CONTAINED.includes(role),
+        `role ${role} is in neither list; decide whether post-provision can install it`,
+      ).toBe(true);
+      expect(typeof shouldAutoInstallK3s(role)).toBe("boolean");
+    }
+  });
+
+  it("refuses every k3s AGENT role — an agent with nothing to join cannot start", () => {
+    for (const role of SUPPORTED_ROLES) {
+      if (roleInfo(role).k3sMode === "agent") {
+        expect(shouldAutoInstallK3s(role), `${role} is an agent`).toBe(false);
+      }
+    }
+  });
+
+  it("does not refuse purely on the basis of being a server", () => {
+    // Guard against the gate degenerating into isK3sServer(): labcontroller
+    // and infra are servers AND auto-installable.
+    const servers = SUPPORTED_ROLES.filter(isK3sServer);
+    expect(servers.some((r) => shouldAutoInstallK3s(r))).toBe(true);
+    expect(servers.some((r) => !shouldAutoInstallK3s(r))).toBe(true);
+  });
+});
+
+describe("config.autoK3s", () => {
+  it("defaults to enabled", () => {
+    const config = loadConfig({});
+    expect(config.autoK3s).toBe(true);
+  });
+
+  it("is disabled by an explicit override", () => {
+    const config = loadConfig({ autoK3s: false });
+    expect(config.autoK3s).toBe(false);
+  });
+
+  it("an explicit true override wins over nothing", () => {
+    const config = loadConfig({ autoK3s: true });
+    expect(config.autoK3s).toBe(true);
+  });
+
+  it("AUTO_K3S=false in the environment disables it", () => {
+    const prev = process.env["AUTO_K3S"];
+    process.env["AUTO_K3S"] = "false";
+    try {
+      expect(loadConfig({}).autoK3s).toBe(false);
+    } finally {
+      if (prev === undefined) delete process.env["AUTO_K3S"]; else process.env["AUTO_K3S"] = prev;
+    }
+  });
+
+  it("only the exact string 'false' disables it", () => {
+    const prev = process.env["AUTO_K3S"];
+    process.env["AUTO_K3S"] = "0";
+    try {
+      expect(loadConfig({}).autoK3s).toBe(true);
+    } finally {
+      if (prev === undefined) delete process.env["AUTO_K3S"]; else process.env["AUTO_K3S"] = prev;
+    }
+  });
+});
+
+describe("kickstart packages", () => {
+  // etcd membership recovery is `etcdctl member list/remove`, and `k3s etcdctl`
+  // is NOT a command (verified on a live node: "No help topic for 'etcdctl'").
+  // The binary ships in the `etcd` rpm. The recovery card is written for a
+  // situation with no internet, so telling the operator to dnf-install it there
+  // is not a plan.
+  it("installs etcd on k3s servers and nowhere else", async () => {
+    const { renderInstallKickstart } = await import("../src/templates/install.ks.js");
+    const render = (role: Role): string =>
+      renderInstallKickstart({
+        hostname: "test-node", disk: "nvme0n1", role, domain: "ad.itaz.eu",
+        fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8",
+        serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514,
+        sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal",
+        ...(role === "vanilla" || role === "worker" || role === "infra" || role === "labcontroller"
+          ? {} : { schemaVersion: 2 as const }),
+      });
+    // Scope to %packages: "etcd" appears in paths and comments elsewhere, and a
+    // whole-document match would pass for the wrong reason.
+    const pkgSection = (ks: string): string => {
+      const start = ks.indexOf("%packages");
+      return ks.slice(start, ks.indexOf("%end", start));
+    };
+
+    for (const role of ["infra", "labcontroller", "k8s-combo", "k8s-controller", "standalone-cluster"] as const) {
+      expect(pkgSection(render(role)), `${role} is an etcd member and needs etcdctl offline`)
+        .toMatch(/^etcd$/m);
+    }
+    for (const role of ["vanilla", "worker", "k8s-worker"] as const) {
+      expect(pkgSection(render(role)), `${role} runs no etcd`)
+        .not.toMatch(/^etcd$/m);
+    }
+  });
+});

bastion/src/bastion/tests/data-disks.test.ts

@@ -0,0 +1,274 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import type { FastifyInstance } from "fastify";
+import { renderInstallKickstart } from "../src/templates/install.ks.js";
+import type { InstallKickstartParams } from "../src/templates/install.ks.js";
+import { createApp } from "../src/server.js";
+import { validateDataDisks, v2Layout, DATA_DISK_LABELS, type BastionConfig, type DataDisks } from "@lab/shared";
+
+/**
+ * Dedicated whole-disk data volumes (thebeast: 1 TB OS + 2 TB longhorn +
+ * 2 TB rancher). The layout contract these tests pin down:
+ *
+ *   - naming a data disk DROPS the matching LV -- never both.
+ *   - disks are named by /dev/disk/by-id only; kernel names are rejected,
+ *     because nvme enumeration order is not stable across boots.
+ *   - a lab-longhorn / lab-rancher labelled partition is on-disk truth, like
+ *     the lab-schema-v2 VG tag: reused --noformat, never wiped, never
+ *     silently shadowed.
+ */
+
+const BY_ID = "/dev/disk/by-id";
+const LONGHORN_ID = `${BY_ID}/nvme-Sabrent_Rocket_4.0_2TB_7F600711179D89261778`;
+const RANCHER_ID = `${BY_ID}/nvme-Sabrent_Rocket_4.0_2TB_7FD4070C1EDA92197301`;
+const OS_ID = `${BY_ID}/nvme-Samsung_SSD_970_EVO_Plus_1TB_S4EWNJ0N138234M`;
+
+function baseParams(over: Partial<InstallKickstartParams> = {}): InstallKickstartParams {
+  return {
+    hostname: "thebeast", disk: "", role: "k8s-worker", domain: "ad.itaz.eu",
+    fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8",
+    serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514,
+    sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal",
+    dataDisks: { longhorn: LONGHORN_ID, rancher: RANCHER_ID },
+    ...over,
+  };
+}
+
+/** The fresh-install part.ks heredoc (the one that creates the PV). */
+function freshBlock(ks: string): string {
+  const blocks = [...ks.matchAll(/cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g)]
+    .map((m) => m[1]!);
+  const fresh = blocks.find((b) => b.includes("part pv.01"));
+  if (!fresh) throw new Error("no fresh-install part.ks block found");
+  return fresh;
+}
+
+describe("validateDataDisks", () => {
+  const ok: DataDisks = { longhorn: LONGHORN_ID, rancher: RANCHER_ID };
+
+  it("accepts a sane spec for a role that hosts longhorn", () => {
+    expect(validateDataDisks("k8s-worker", ok)).toBeNull();
+    expect(validateDataDisks("k8s-combo", ok)).toBeNull();
+    expect(validateDataDisks("worker", ok)).toBeNull();
+  });
+
+  it("rejects kernel names -- they are not stable across boots", () => {
+    expect(validateDataDisks("k8s-worker", { longhorn: "nvme1n1" })).toMatch(/by-id/);
+    expect(validateDataDisks("k8s-worker", { rancher: "/dev/nvme2n1" })).toMatch(/by-id/);
+  });
+
+  it("rejects a longhorn disk for a role that holds no replicas", () => {
+    expect(validateDataDisks("k8s-controller", { longhorn: LONGHORN_ID })).toMatch(/no Longhorn replicas/);
+    expect(validateDataDisks("vanilla", { longhorn: LONGHORN_ID })).toMatch(/no Longhorn replicas/);
+  });
+
+  it("rejects a rancher disk for a role that runs no k3s", () => {
+    expect(validateDataDisks("vanilla", { rancher: RANCHER_ID })).toMatch(/no k3s/);
+  });
+
+  it("a control-plane-only node may still take a rancher disk", () => {
+    expect(validateDataDisks("k8s-controller", { rancher: RANCHER_ID })).toBeNull();
+  });
+
+  it("rejects the same disk doing both jobs", () => {
+    expect(validateDataDisks("k8s-worker", { longhorn: LONGHORN_ID, rancher: LONGHORN_ID }))
+      .toMatch(/same disk/);
+  });
+});
+
+describe("v2Layout with dedicated disks", () => {
+  it("drops exactly the LV a dedicated disk replaces", () => {
+    const names = (d: DataDisks) => v2Layout("k8s-worker", { dataDisks: d }).map((l) => l.name);
+    expect(names({})).toContain("longhorn");
+    expect(names({ longhorn: LONGHORN_ID })).not.toContain("longhorn");
+    expect(names({ longhorn: LONGHORN_ID })).toContain("rancher");
+    expect(names({ longhorn: LONGHORN_ID, rancher: RANCHER_ID })).not.toContain("rancher");
+    // base volumes untouched
+    expect(names({ longhorn: LONGHORN_ID, rancher: RANCHER_ID }))
+      .toEqual(expect.arrayContaining(["swap", "root", "var", "varlog", "home", "srv"]));
+  });
+});
+
+describe("renderInstallKickstart with data disks", () => {
+  it("refuses the v1 layout -- it is frozen and single-disk", () => {
+    expect(() => renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 1 })))
+      .toThrow(/v2 layout/);
+    // a legacy role with no explicit schema defaults to v1 -> same refusal
+    expect(() => renderInstallKickstart(baseParams({ role: "worker" })))
+      .toThrow(/v2 layout/);
+  });
+
+  it("refuses a kernel-name data disk", () => {
+    expect(() => renderInstallKickstart(baseParams({ dataDisks: { longhorn: "nvme1n1" } })))
+      .toThrow(/by-id/);
+  });
+
+  it("creates no longhorn/rancher LV when both mounts come from dedicated disks", () => {
+    const fresh = freshBlock(renderInstallKickstart(baseParams()));
+    expect(fresh).not.toContain("--name=longhorn");
+    expect(fresh).not.toContain("--name=rancher");
+    // base LVs still there
+    expect(fresh).toContain("--name=root");
+  });
+
+  it("anaconda only sees the named disks, and only wipes unlabelled ones", () => {
+    const ks = renderInstallKickstart(baseParams());
+    const fresh = freshBlock(ks);
+    expect(fresh).toContain("ignoredisk --only-use=$KS_DISKS");
+    expect(fresh).toContain("clearpart --all --initlabel --drives=$CLEARPART_DRIVES");
+    // a disk whose partition carries our label is excluded from clearpart
+    expect(ks).toContain('[ "$LONGHORN_REUSE" = "no" ] && CLEARPART_DRIVES=');
+  });
+
+  it("%pre resolves each by-id path and fails loudly when a disk is missing", () => {
+    const ks = renderInstallKickstart(baseParams());
+    expect(ks).toContain(`LONGHORN_DISK_ID="${LONGHORN_ID}"`);
+    expect(ks).toContain(`RANCHER_DISK_ID="${RANCHER_ID}"`);
+    expect(ks).toContain('echo "ERROR: longhorn disk not found: $LONGHORN_DISK_ID"; exit 1');
+  });
+
+  it("a labelled partition is reused --noformat; a fresh disk gets one labelled partition", () => {
+    const ks = renderInstallKickstart(baseParams());
+    expect(ks).toContain("part /var/lib/longhorn --onpart=$LONGHORN_PART --noformat");
+    expect(ks).toContain(
+      `part /var/lib/longhorn --fstype=xfs --ondisk=$LONGHORN_DISK --grow --size=1 --label=${DATA_DISK_LABELS.longhorn}`,
+    );
+    expect(ks).toContain("part /var/lib/rancher --onpart=$RANCHER_PART --noformat");
+    expect(ks).toContain(
+      `part /var/lib/rancher --fstype=xfs --ondisk=$RANCHER_DISK --grow --size=1 --label=${DATA_DISK_LABELS.rancher}`,
+    );
+  });
+
+  it("refuses to run when a data LV and a dedicated disk both exist", () => {
+    const ks = renderInstallKickstart(baseParams());
+    expect(ks).toContain('[ "$PRESERVE_LONGHORN" = "yes" ] && [ -n "$LONGHORN_DISK" ]');
+    expect(ks).toContain('[ "$PRESERVE_RANCHER" = "yes" ] && [ -n "$RANCHER_DISK" ]');
+  });
+
+  it("a fresh install never wipes or shadows a labelled disk it was not told about", () => {
+    // Rendered even WITHOUT dataDisks: the label is on-disk truth and the
+    // check protects against a queue entry that forgot the disks.
+    const ks = renderInstallKickstart(baseParams({ role: "k8s-worker", dataDisks: {} }));
+    expect(ks).toContain(`blkid -t LABEL=${DATA_DISK_LABELS.longhorn}`);
+    expect(ks).toContain("Refusing to wipe or shadow it");
+  });
+
+  it("an OS disk given as a by-id path is resolved in %pre", () => {
+    const ks = renderInstallKickstart(baseParams({ disk: OS_ID }));
+    expect(ks).toContain(`readlink -f "${OS_ID}"`);
+    expect(ks).not.toContain(`DISK="${OS_ID}"`); // never fed to anaconda unresolved
+  });
+
+  it("auto-detect never picks a data disk as the OS disk", () => {
+    const ks = renderInstallKickstart(baseParams({ disk: "" }));
+    expect(ks).toContain('case "$_bname" in "$LONGHORN_DISK"|"$RANCHER_DISK") continue ;; esac');
+  });
+
+  it("a plain kernel-name OS disk still renders exactly as before", () => {
+    const ks = renderInstallKickstart(baseParams({ disk: "vda", dataDisks: {} }));
+    expect(ks).toContain('DISK="vda"');
+  });
+
+  it("the %pre block still survives the /ks-auto extractor", () => {
+    const ks = renderInstallKickstart(baseParams());
+    const blocks = [...ks.matchAll(/%pre[^\n]*\n([\s\S]*?)%end/g)];
+    expect(blocks.length).toBeGreaterThan(0);
+    expect(blocks[0]![1]).toContain("/tmp/part.ks");
+  });
+});
+
+describe("dataDisks plumbing: /api/install -> /ks", () => {
+  let app: FastifyInstance;
+  let testDir: string;
+
+  function testConfig(dir: string): BastionConfig {
+    return {
+      fedoraVersion: "43", arch: "x86_64", httpPort: 0,
+      timezone: "Europe/London", locale: "en_GB.UTF-8",
+      bastionDir: dir, domain: "test.local", dhcpMode: "proxy",
+      dhcpRangeStart: "", dhcpRangeEnd: "",
+      ubuntuVersion: "26.04", ubuntuMirror: "https://releases.ubuntu.com/26.04",
+      vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso",
+      vyosDefaultPassword: "vyos", iface: "eth0",
+      serverIp: "10.0.0.1", network: "10.0.0.0", gateway: "10.0.0.1",
+      sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"],
+      adminUser: "testadmin", syslogPort: 15514,
+      skipDnsmasq: true, skipArtifacts: true,
+      fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
+      tftpDir: join(dir, "tftp"), httpDir: join(dir, "http"), stateFile: join(dir, "state.json"),
+    };
+  }
+
+  beforeEach(async () => {
+    testDir = join(tmpdir(), `datadisks-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+    mkdirSync(testDir, { recursive: true });
+    const built = await createApp(testConfig(testDir));
+    app = built.app;
+    await app.ready();
+  });
+
+  afterEach(async () => {
+    await app.close();
+    rmSync(testDir, { recursive: true, force: true });
+  });
+
+  const queue = (body: Record<string, unknown>) =>
+    app.inject({ method: "POST", url: "/api/install", payload: { mac: "aa:bb:cc:dd:ee:03", hostname: "thebeast", ...body } });
+
+  const ks = async () => (await app.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:03" })).body;
+
+  it("named disks reach the rendered kickstart, and their LVs are gone", async () => {
+    const res = await queue({ role: "k8s-worker", dataDisks: { longhorn: LONGHORN_ID, rancher: RANCHER_ID } });
+    expect(res.statusCode).toBe(200);
+    const body = await ks();
+    expect(body).toContain(LONGHORN_ID);
+    expect(body).toContain(RANCHER_ID);
+    expect(freshBlock(body)).not.toContain("--name=longhorn");
+    expect(freshBlock(body)).not.toContain("--name=rancher");
+  });
+
+  it("rejects dataDisks on a non-fedora install instead of silently dropping them", async () => {
+    const res = await queue({ role: "k8s-worker", os: "ubuntu-26.04", dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/fedora/);
+  });
+
+  it("rejects dataDisks with an explicit v1 schema", async () => {
+    const res = await queue({ role: "worker", schemaVersion: 1, dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/schemaVersion 2/);
+  });
+
+  it("rejects dataDisks for a legacy role that would default to v1", async () => {
+    const res = await queue({ role: "worker", dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/schemaVersion 2/);
+  });
+
+  it("legacy role + explicit v2 + dataDisks is accepted", async () => {
+    const res = await queue({ role: "worker", schemaVersion: 2, dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(200);
+    expect(await ks()).toContain(LONGHORN_ID);
+  });
+
+  it("rejects a kernel-name disk with the shared message", async () => {
+    const res = await queue({ role: "k8s-worker", dataDisks: { longhorn: "nvme1n1" } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/by-id/);
+  });
+
+  it("rejects a capability mismatch (longhorn disk on a control-plane-only role)", async () => {
+    const res = await queue({ role: "k8s-controller", dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/no Longhorn replicas/);
+  });
+
+  it("an install without dataDisks is untouched by all of this", async () => {
+    expect((await queue({ role: "k8s-worker" })).statusCode).toBe(200);
+    const body = await ks();
+    expect(freshBlock(body)).toContain("--name=longhorn --fstype=xfs --size=307200");
+    expect(freshBlock(body)).toContain("--name=rancher --fstype=xfs --size=256000");
+  });
+});

bastion/src/bastion/tests/dispatch.test.ts

@@ -22,6 +22,8 @@ function createTestConfig(testDir: string): BastionConfig {
     dhcpRangeEnd: "",
     ubuntuVersion: "26.04",
     ubuntuMirror: "https://releases.ubuntu.com/26.04",
+    vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso",
+    vyosDefaultPassword: "vyos",
     iface: "eth0",
     serverIp: "10.0.0.1",
     network: "10.0.0.0",

bastion/src/bastion/tests/fixtures/ipxe-x86_64-golden.json

@@ -1,8 +0,0 @@
-{
-  "boot": "#!ipxe\n\necho\necho ============================================\necho   Lab PXE Bastion\necho   Contacting server for instructions...\necho ============================================\necho\n\nchain http://10.0.0.1:8080/dispatch?mac=${net0/mac}\n",
-  "discover": "#!ipxe\n\necho\necho =============================================\necho   Lab PXE Bastion - DISCOVERY MODE\necho   MAC: aa:bb:cc:dd:ee:ff\necho   Collecting hardware info...\necho =============================================\necho\n\nkernel http://10.0.0.1:8080/vmlinuz inst.ks=http://10.0.0.1:8080/discover.ks inst.stage2=https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os inst.text nomodeset\ninitrd http://10.0.0.1:8080/initrd.img\nboot\n",
-  "install": "#!ipxe\n\necho\necho =============================================\necho   Lab PXE Bastion - INSTALLING Fedora 43\necho   Target: worker-1\necho   MAC:    aa:bb:cc:dd:ee:ff\necho =============================================\necho\n\nkernel http://10.0.0.1:8080/vmlinuz inst.ks=http://10.0.0.1:8080/ks?mac=aa:bb:cc:dd:ee:ff inst.repo=https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os inst.text nomodeset\ninitrd http://10.0.0.1:8080/initrd.img\nboot\n",
-  "debug": "#!ipxe\n\necho\necho =============================================\necho   Lab PXE Bastion - DEBUG/RESCUE MODE\necho   Target: worker-1\necho   MAC:    aa:bb:cc:dd:ee:ff\necho =============================================\necho\n\nkernel http://10.0.0.1:8080/vmlinuz inst.rescue inst.text inst.sshd inst.ks=http://10.0.0.1:8080/debug.ks?mac=aa:bb:cc:dd:ee:ff inst.stage2=https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os\ninitrd http://10.0.0.1:8080/initrd.img\nboot\n",
-  "pxeBoot": "#!ipxe\n\necho\necho =============================================\necho   Lab PXE Bastion - PXE BOOT (debug)\necho   Target: worker-1\necho   MAC:    aa:bb:cc:dd:ee:ff\necho   Kernel+initrd from PXE, root from NVMe\necho =============================================\necho\n\nkernel http://10.0.0.1:8080/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0\ninitrd http://10.0.0.1:8080/initrd.img\nboot\n",
-  "localBoot": "#!ipxe\n\necho\necho =============================================\necho   Lab PXE Bastion - worker-1\necho   Already installed, booting from local disk\necho =============================================\necho\nsleep 3\nexit 1\n"
-}

bastion/src/bastion/tests/install-guard.test.ts

@@ -1,194 +0,0 @@
-// Installs must never reach a machine running a vendor OS we cannot restore.
-//
-// This is the guardrail that stops someone reinstalling a DGX Spark at 2am. Rescue is
-// deliberately still allowed for the same machines -- that is the whole point.
-
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { mkdirSync, rmSync } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import type { BastionConfig, BastionState, HardwareInfo } from "@lab/shared";
-import { classifyOnboard } from "@lab/shared";
-import { createApp } from "../src/server.js";
-import { checkInstallAllowed } from "../src/services/install-guard.js";
-import type { FastifyInstance } from "fastify";
-import type { StateManager } from "../src/services/state.js";
-
-// The real machines this exists to protect.
-const SPARK_2935 = "4c:bb:47:7f:29:35";
-const SPARK_3A1C = "48:21:0b:96:3a:1c";
-const ORDINARY = "aa:bb:cc:dd:ee:ff";
-
-function createTestConfig(testDir: string): BastionConfig {
-  return {
-    fedoraVersion: "43", arch: "x86_64", httpPort: 0,
-    timezone: "Europe/London", locale: "en_GB.UTF-8", bastionDir: testDir,
-    domain: "test.local", dhcpMode: "proxy", dhcpRangeStart: "", dhcpRangeEnd: "",
-    ubuntuVersion: "26.04", ubuntuMirror: "https://releases.ubuntu.com/26.04",
-    iface: "eth0", serverIp: "10.0.0.1", network: "10.0.0.0", gateway: "10.0.0.1",
-    sshKeys: [], adminUser: "testadmin", syslogPort: 15514,
-    skipDnsmasq: true, skipArtifacts: true,
-    fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
-    tftpDir: join(testDir, "tftp"), httpDir: join(testDir, "http"),
-    stateFile: join(testDir, "state.json"),
-  };
-}
-
-function hardware(mac: string, over: Partial<HardwareInfo> = {}): HardwareInfo {
-  return {
-    mac, product: "TestBox", board: "TestBoard", serial: "SN1",
-    manufacturer: "TestCorp", cpu_model: "Test CPU", cpu_cores: 4, memory_gb: 16,
-    arch: "x86_64", disks: [], nics: [],
-    first_seen: new Date().toISOString(), last_seen: new Date().toISOString(),
-    ...over,
-  };
-}
-
-const emptyState = (): BastionState => ({
-  discovered: {}, install_queue: {}, installed: {}, debug: {},
-});
-
-describe("classifyOnboard", () => {
-  it("recognises a DGX Spark from its DMI identity", () => {
-    expect(classifyOnboard({
-      mac: ORDINARY, manufacturer: "NVIDIA", product: "NVIDIA DGX Spark", board: "GB10",
-    })).toEqual({ onboard: "ssh", vendor_os: "dgx-os" });
-  });
-
-  it("recognises the known Sparks even with no DMI recorded", () => {
-    // Neither Spark has hardware info in bastion state today. A DMI-only rule would
-    // fail open on exactly the machines this protects.
-    expect(classifyOnboard({ mac: SPARK_2935 }).onboard).toBe("ssh");
-    expect(classifyOnboard({ mac: SPARK_3A1C }).onboard).toBe("ssh");
-  });
-
-  it("treats ordinary hardware as PXE-installable", () => {
-    expect(classifyOnboard({
-      mac: ORDINARY, manufacturer: "Beelink", product: "SER9", board: "SER9",
-    })).toEqual({ onboard: "pxe" });
-  });
-
-  it("does not override an explicit classification already on the record", () => {
-    expect(classifyOnboard({
-      mac: SPARK_2935, onboard: "pxe",
-    })).toEqual({ onboard: "pxe" });
-  });
-});
-
-describe("checkInstallAllowed", () => {
-  it("refuses a DGX Spark and explains why", () => {
-    const state = emptyState();
-    state.installed[SPARK_2935] = {
-      hostname: "spark-2935", role: "worker", ip: "192.168.8.12",
-      installed_at: new Date().toISOString(), arch: "aarch64",
-    };
-
-    const result = checkInstallAllowed(state, SPARK_2935, "fedora-43");
-    expect(result.allowed).toBe(false);
-    if (result.allowed === false) {
-      expect(result.error).toContain("spark-2935");
-      expect(result.error).toContain("DGX OS");
-      expect(result.error).toContain("provision debug");
-    }
-  });
-
-  it("refuses a Spark that is only known by MAC", () => {
-    expect(checkInstallAllowed(emptyState(), SPARK_3A1C, "fedora-43").allowed).toBe(false);
-  });
-
-  it("allows an ordinary discovered machine", () => {
-    const state = emptyState();
-    state.discovered[ORDINARY] = hardware(ORDINARY);
-    expect(checkInstallAllowed(state, ORDINARY, "fedora-43").allowed).toBe(true);
-  });
-
-  it("allows Fedora on aarch64", () => {
-    const state = emptyState();
-    state.discovered[ORDINARY] = hardware(ORDINARY, { arch: "aarch64" });
-    expect(checkInstallAllowed(state, ORDINARY, "fedora-43").allowed).toBe(true);
-  });
-
-  it("refuses Ubuntu on aarch64 -- no netboot artifacts are published", () => {
-    const state = emptyState();
-    state.discovered[ORDINARY] = hardware(ORDINARY, { arch: "aarch64" });
-    const result = checkInstallAllowed(state, ORDINARY, "ubuntu-26.04");
-    expect(result.allowed).toBe(false);
-    if (result.allowed === false) {
-      expect(result.error).toContain("aarch64");
-    }
-  });
-
-  it("allows Ubuntu on x86_64", () => {
-    const state = emptyState();
-    state.discovered[ORDINARY] = hardware(ORDINARY, { arch: "x86_64" });
-    expect(checkInstallAllowed(state, ORDINARY, "ubuntu-26.04").allowed).toBe(true);
-  });
-});
-
-describe("install route enforces the guard", () => {
-  let testDir: string;
-  let app: FastifyInstance;
-  let state: StateManager;
-
-  beforeEach(() => {
-    testDir = join(tmpdir(), `bastion-guard-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
-    mkdirSync(join(testDir, "http"), { recursive: true });
-    mkdirSync(join(testDir, "tftp"), { recursive: true });
-    const result = createApp(createTestConfig(testDir));
-    app = result.app;
-    state = result.state;
-  });
-
-  afterEach(async () => {
-    await app.close();
-    rmSync(testDir, { recursive: true, force: true });
-  });
-
-  it("rejects POST /api/install for a Spark and queues nothing", async () => {
-    const res = await app.inject({
-      method: "POST",
-      url: "/api/install",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ mac: SPARK_2935, hostname: "spark-2935", role: "worker" }),
-    });
-
-    expect(res.statusCode).toBe(409);
-    expect(JSON.parse(res.body).error).toContain("Refusing to install");
-    expect(state.load().install_queue[SPARK_2935]).toBeUndefined();
-  });
-
-  it("still serves rescue to a Spark -- debug is never guarded", async () => {
-    state.update((s) => {
-      s.installed[SPARK_2935] = {
-        hostname: "spark-2935", role: "worker", ip: "192.168.8.12",
-        installed_at: new Date().toISOString(), arch: "aarch64",
-      };
-      s.debug[SPARK_2935] = { hostname: "spark-2935", queued_at: new Date().toISOString() };
-    });
-
-    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${SPARK_2935}` });
-    expect(res.statusCode).toBe(200);
-    expect(res.body).toContain("DEBUG/RESCUE MODE");
-    expect(res.body).toContain("/vmlinuz-aarch64");
-  });
-
-  it("a Spark that PXE boots unqueued gets discovery, never an install", async () => {
-    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${SPARK_2935}&arch=arm64` });
-    expect(res.body).toContain("DISCOVERY MODE");
-    expect(res.body).not.toContain("INSTALLING");
-  });
-
-  it("still accepts an ordinary machine", async () => {
-    state.update((s) => { s.discovered[ORDINARY] = hardware(ORDINARY); });
-
-    const res = await app.inject({
-      method: "POST",
-      url: "/api/install",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ mac: ORDINARY, hostname: "worker-1", role: "worker" }),
-    });
-
-    expect(res.statusCode).toBe(200);
-    expect(state.load().install_queue[ORDINARY]).toBeDefined();
-  });
-});

bastion/src/bastion/tests/ipxe-x86-regression.test.ts

@@ -1,89 +0,0 @@
-// x86_64 iPXE output regression gate.
-//
-// The aarch64 PXE work must not change what an x86_64 machine is served. The golden
-// fixture was dumped from the templates as they stood before that work started, so
-// any diff here is a regression, not an improvement.
-//
-// The one deliberate exception is renderBootIpxe: its chain URL gained
-// `&arch=${buildarch}` so the dispatch endpoint can observe the client's
-// architecture at boot time. That single change is asserted explicitly below
-// rather than being allowed to slip through the byte-for-byte comparison.
-
-import { describe, it, expect } from "vitest";
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
-import { fileURLToPath } from "node:url";
-import { dirname } from "node:path";
-import {
-  renderBootIpxe,
-  renderDiscoverIpxe,
-  renderInstallIpxe,
-  renderDebugIpxe,
-  renderPxeBootDebugIpxe,
-  renderLocalBootIpxe,
-} from "../src/templates/boot.ipxe.js";
-
-const here = dirname(fileURLToPath(import.meta.url));
-const golden = JSON.parse(
-  readFileSync(join(here, "fixtures", "ipxe-x86_64-golden.json"), "utf-8"),
-) as Record<string, string>;
-
-// Exactly the parameters used to dump the fixture.
-const serverIp = "10.0.0.1";
-const httpPort = 8080;
-const mac = "aa:bb:cc:dd:ee:ff";
-const hostname = "worker-1";
-const fedoraVersion = "43";
-const fedoraMirror =
-  "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os";
-
-// The x86_64 LVM layout the fixture was captured with. Before this work the values
-// were hardcoded in the template; they are now supplied by the caller from machine
-// state, so the fixture pins the rendering, not the defaults.
-const x86Root = {
-  rootDevice: "/dev/mapper/labvg-root",
-  rootArgs: "rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap",
-};
-
-describe("x86_64 iPXE output is unchanged", () => {
-  it("discover script is byte-identical", () => {
-    const rendered = renderDiscoverIpxe({
-      mac, serverIp, httpPort, fedoraMirror, arch: "x86_64",
-    });
-    expect(rendered).toBe(golden["discover"]);
-  });
-
-  it("install script is byte-identical", () => {
-    const rendered = renderInstallIpxe({
-      mac, hostname, serverIp, httpPort, fedoraVersion, fedoraMirror, arch: "x86_64",
-    });
-    expect(rendered).toBe(golden["install"]);
-  });
-
-  it("debug/rescue script is byte-identical", () => {
-    const rendered = renderDebugIpxe({
-      mac, hostname, serverIp, httpPort, fedoraMirror, arch: "x86_64",
-    });
-    expect(rendered).toBe(golden["debug"]);
-  });
-
-  it("--pxe-boot script is byte-identical when state carries the Fedora LVM layout", () => {
-    const rendered = renderPxeBootDebugIpxe({
-      mac, hostname, serverIp, httpPort, arch: "x86_64", ...x86Root,
-    });
-    expect(rendered).toBe(golden["pxeBoot"]);
-  });
-
-  it("local boot script is byte-identical", () => {
-    expect(renderLocalBootIpxe(hostname)).toBe(golden["localBoot"]);
-  });
-
-  it("boot.ipxe differs only by the &arch= chain parameter", () => {
-    const rendered = renderBootIpxe({ serverIp, httpPort });
-    // The sole intended difference.
-    expect(rendered).toBe(golden["boot"].replace(
-      "/dispatch?mac=${net0/mac}",
-      "/dispatch?mac=${net0/mac}&arch=${buildarch}",
-    ));
-  });
-});

bastion/src/bastion/tests/kickstart-layout.test.ts

@@ -0,0 +1,186 @@
+import { describe, it, expect } from "vitest";
+import { renderInstallKickstart } from "../src/templates/install.ks.js";
+import type { InstallKickstartParams } from "../src/templates/install.ks.js";
+import { SUPPORTED_ROLES, LEGACY_ROLES, v2Layout, type Role } from "@lab/shared";
+
+/**
+ * Partition-layout tests.
+ *
+ * The existing ksvalidator test only sees the MAIN kickstart body. The actual
+ * layout is written by %pre into /tmp/part.ks inside a heredoc, so until now
+ * nothing validated it at all -- a broken layout rendered, shipped and was only
+ * discovered on the installed machine.
+ *
+ * These parse the heredocs. Following tests/ubuntu-autoinstall.test.ts:
+ * "A `toContain` assertion would not have caught that; only parsing does."
+ */
+
+function baseParams(over: Partial<InstallKickstartParams> = {}): InstallKickstartParams {
+  return {
+    hostname: "test-node", disk: "nvme0n1", role: "worker", domain: "ad.itaz.eu",
+    fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8",
+    serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514,
+    sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal",
+    ...over,
+  };
+}
+
+/** Pull every `cat >[>] /tmp/part.ks << PARTEOF ... PARTEOF` body out of the script. */
+function partKsBlocks(ks: string): string[] {
+  const out: string[] = [];
+  const re = /cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(ks)) !== null) out.push(m[1]!);
+  return out;
+}
+
+interface Logvol { mount: string; name: string; fs: string; size?: number; useExisting: boolean }
+
+function parseLogvols(block: string): Logvol[] {
+  return block.split("\n")
+    .filter((l) => l.trim().startsWith("logvol "))
+    .map((line) => {
+      const parts = line.trim().split(/\s+/);
+      const get = (flag: string) => {
+        const p = parts.find((x) => x.startsWith(`${flag}=`));
+        return p ? p.slice(flag.length + 1) : undefined;
+      };
+      return {
+        mount: parts[1]!,
+        name: get("--name") ?? "",
+        fs: get("--fstype") ?? "",
+        ...(get("--size") ? { size: Number(get("--size")) } : {}),
+        useExisting: parts.includes("--useexisting"),
+      };
+    });
+}
+
+/** The fresh-install block is the one that also creates the PV. */
+function freshBlock(ks: string): string {
+  const b = partKsBlocks(ks).find((x) => x.includes("part pv.01"));
+  if (!b) throw new Error("no fresh-install part.ks block found");
+  return b;
+}
+
+describe("v1 is frozen — existing machines must reinstall identically", () => {
+  it.each(LEGACY_ROLES)("%s v1 fresh layout is unchanged", (role) => {
+    const lvs = parseLogvols(freshBlock(renderInstallKickstart(baseParams({ role }))));
+    const byName = Object.fromEntries(lvs.map((l) => [l.name, l]));
+
+    // The exact v1 sizes, in MiB. If one of these changes, every existing box
+    // in the fleet would be re-laid-out on its next reinstall.
+    expect(byName["swap"]!.size).toBe(27648);
+    expect(byName["root"]!.size).toBe(33792);
+    expect(byName["var"]!.size).toBe(102400);
+    expect(byName["varlog"]!.size).toBe(10240);
+    expect(byName["home"]!.size).toBe(10240);
+    expect(byName["srv"]!.size).toBe(20480);
+  });
+
+  it("v1 gives rancher to infra and worker only — the historical bug, preserved", () => {
+    const has = (r: Role) => freshBlock(renderInstallKickstart(baseParams({ role: r }))).includes("--name=rancher");
+    expect(has("infra")).toBe(true);
+    expect(has("worker")).toBe(true);
+    expect(has("labcontroller")).toBe(false); // the bug, deliberately kept in v1
+    expect(has("vanilla")).toBe(false);
+  });
+
+  it("v1 is the default — no caller has to opt out", () => {
+    const ks = renderInstallKickstart(baseParams({ role: "worker" }));
+    expect(parseLogvols(freshBlock(ks)).find((l) => l.name === "root")!.size).toBe(33792);
+  });
+});
+
+describe("v2-only roles have no v1 layout", () => {
+  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"] as const)(
+    "%s defaults to v2 when no schema is given", (role) => {
+      const lvs = parseLogvols(freshBlock(renderInstallKickstart(baseParams({ role }))));
+      expect(lvs.find((l) => l.name === "root")!.size).toBe(32768);
+      expect(lvs.some((l) => l.name === "rancher")).toBe(true);
+    });
+
+  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"] as const)(
+    "%s throws on an EXPLICIT v1 rather than silently dropping its volumes", (role) => {
+      expect(() => renderInstallKickstart(baseParams({ role, schemaVersion: 1 }))).toThrow(/no v1 disk layout/);
+    });
+});
+
+describe("v2 renders the shared layout", () => {
+  it.each(SUPPORTED_ROLES)("%s v2 matches v2Layout() exactly", (role) => {
+    const ks = renderInstallKickstart(baseParams({ role, schemaVersion: 2 }));
+    const rendered = parseLogvols(freshBlock(ks));
+    const expected = v2Layout(role);
+
+    expect(rendered.map((l) => l.name).sort()).toEqual(expected.map((l) => l.name).sort());
+    for (const spec of expected) {
+      const got = rendered.find((l) => l.name === spec.name)!;
+      expect(got.size).toBe(spec.sizeMib);
+      expect(got.fs).toBe(spec.fs);
+    }
+  });
+
+  it("v2 gives labcontroller a rancher LV — the v1 bug, fixed", () => {
+    const ks = renderInstallKickstart(baseParams({ role: "labcontroller", schemaVersion: 2 }));
+    const rancher = parseLogvols(freshBlock(ks)).find((l) => l.name === "rancher");
+    expect(rancher).toBeDefined();
+    expect(rancher!.size).toBe(256000);
+  });
+
+  it("v2 never uses --grow", () => {
+    for (const role of SUPPORTED_ROLES) {
+      const block = freshBlock(renderInstallKickstart(baseParams({ role, schemaVersion: 2 })));
+      expect(block).not.toMatch(/logvol[^\n]*--grow/);
+    }
+  });
+
+  it("v2 uses the larger EFI and /boot partitions", () => {
+    const block = freshBlock(renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 2 })));
+    expect(block).toMatch(/part \/boot\/efi --fstype=efi --size=1024/);
+    expect(block).toMatch(/part \/boot --fstype=ext4 --size=4096/);
+  });
+});
+
+describe("reprovision protects existing data", () => {
+  it("reprovision reads the on-disk schema instead of trusting the caller", () => {
+    const ks = renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 2 }));
+    expect(ks).toContain("DISK_SCHEMA=v1");
+    expect(ks).toMatch(/vgs --noheadings -o vg_tags .*grep -q lab-schema-v2/);
+  });
+
+  it("the reprovision branch never wipes the disk", () => {
+    const ks = renderInstallKickstart(baseParams({ role: "worker" }));
+    const repro = partKsBlocks(ks).find((b) => b.includes("--useexisting"))!;
+    expect(repro).toContain("clearpart --none");
+    expect(repro).not.toContain("clearpart --all");
+  });
+
+  it("preserved data volumes are never reformatted", () => {
+    const ks = renderInstallKickstart(baseParams({ role: "worker" }));
+    for (const lv of ["longhorn", "rancher", "home", "srv"]) {
+      const re = new RegExp(`--name=${lv} --useexisting --noformat`);
+      expect(ks).toMatch(re);
+    }
+  });
+});
+
+describe("the %pre block survives the /ks-auto extractor", () => {
+  it("matches the same regex ks-auto uses to re-run %pre", () => {
+    // routes/kickstart.ts re-extracts %pre with python:
+    //   re.findall(r'%pre[^\n]*\n(.*?)%end', text, re.DOTALL)
+    // A change that breaks this breaks ISO installs silently.
+    for (const role of SUPPORTED_ROLES) {
+      // v2-only roles throw at v1 by design, so only render the combinations
+      // that are actually legal.
+      // An explicit v1 is illegal for a v2-only role; everything else renders.
+      const versions = (LEGACY_ROLES as readonly string[]).includes(role)
+        ? ([1, 2] as const)
+        : ([2] as const);
+      for (const schemaVersion of versions) {
+        const ks = renderInstallKickstart(baseParams({ role, schemaVersion }));
+        const blocks = [...ks.matchAll(/%pre[^\n]*\n([\s\S]*?)%end/g)];
+        expect(blocks.length).toBeGreaterThan(0);
+        expect(blocks[0]![1]).toContain("/tmp/part.ks");
+      }
+    }
+  });
+});

bastion/src/bastion/tests/kickstart.test.ts

@@ -96,9 +96,9 @@ describe("renderInstallKickstart", () => {
     expect(ks).toContain("/api/progress");
   });
 
-  it("infra role has /var/lib/rancher partition", () => {
+  it("infra role has 120G /var/lib/rancher partition", () => {
     const ks = renderInstallKickstart(baseParams({ role: "infra" }));
-    expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=20480");
+    expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=122880");
   });
 
   it("infra role has k3s install", () => {
@@ -106,10 +106,14 @@ describe("renderInstallKickstart", () => {
     expect(ks).toContain("curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true sh -");
   });
 
-  it("worker role does NOT have /var/lib/rancher partition in fresh install", () => {
+  it("worker role has 120G /var/lib/rancher partition (imageFs must be sized before longhorn --grow)", () => {
     const ks = renderInstallKickstart(baseParams({ role: "worker" }));
-    // Worker should not have the fresh-install rancher partition line
-    expect(ks).not.toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=20480");
+    expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=122880");
+  });
+
+  it("vanilla role does NOT have /var/lib/rancher partition in fresh install", () => {
+    const ks = renderInstallKickstart(baseParams({ role: "vanilla" }));
+    expect(ks).not.toContain("--name=rancher --fstype=xfs");
   });
 
   it("worker role does NOT have k3s install", () => {

bastion/src/bastion/tests/layout-consistency.test.ts

@@ -0,0 +1,105 @@
+import { describe, it, expect } from "vitest";
+import { execFileSync } from "node:child_process";
+import { writeFileSync } from "node:fs";
+import { renderInstallKickstart } from "../src/templates/install.ks.js";
+import { renderUbuntuAutoinstall } from "../src/templates/ubuntu-autoinstall.js";
+import { renderFirstbootScript } from "../src/templates/asahi-firstboot.sh.js";
+import { v2Layout, SUPPORTED_ROLES, type Role } from "@lab/shared";
+
+/**
+ * THE test this whole refactor exists for.
+ *
+ * Three generators partition machines. Before the shared layout they disagreed,
+ * and nothing noticed:
+ *
+ *   install.ks.ts       rancher 122880M
+ *   ubuntu-autoinstall  rancher  20480M
+ *   asahi-firstboot     rancher  20480M   <- under a comment claiming it
+ *                                            matched install.ks.ts
+ *
+ * That 100 GB gap is why worker1 runs an image store too small for one large
+ * container image. This test fails the moment any generator drifts again.
+ */
+
+const KS = (role: Role) => renderInstallKickstart({
+  hostname: "n", disk: "nvme0n1", role, domain: "d", fedoraVersion: "43",
+  timezone: "UTC", locale: "en_US.UTF-8", serverIp: "10.0.0.1", httpPort: 8080,
+  syslogPort: 5514, sshKeys: [], adminUser: "a", schemaVersion: 2,
+});
+
+const UB = (role: Role) => renderUbuntuAutoinstall({
+  hostname: "n", disk: "/dev/sda", role, domain: "d", ubuntuVersion: "24.04",
+  timezone: "UTC", locale: "en_US.UTF-8", serverIp: "10.0.0.1", httpPort: 8080,
+  sshKeys: [], adminUser: "a", schemaVersion: 2,
+} as never);
+
+const AS = (role: Role) => renderFirstbootScript({
+  hostname: "n", role, serverIp: "10.0.0.1", httpPort: 8080,
+  sshKeys: [], adminUser: "a", mac: "aa:bb:cc:dd:ee:ff", schemaVersion: 2,
+} as never);
+
+/** MiB per LV name, from the Fedora kickstart's part.ks heredoc. */
+function fedoraSizes(role: Role): Record<string, number> {
+  const ks = KS(role);
+  const block = [...ks.matchAll(/cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g)]
+    .map((m) => m[1]!).find((b) => b.includes("part pv.01"))!;
+  const out: Record<string, number> = {};
+  for (const line of block.split("\n").filter((l) => l.trim().startsWith("logvol "))) {
+    const name = /--name=(\S+)/.exec(line)?.[1];
+    const size = /--size=(\d+)/.exec(line)?.[1];
+    if (name && size) out[name] = Number(size);
+  }
+  return out;
+}
+
+/** MiB per LV name, from the Ubuntu curtin document (declared in GiB). */
+function ubuntuSizes(role: Role): Record<string, number> {
+  const tmp = `/tmp/consistency-${role}.yaml`;
+  writeFileSync(tmp, UB(role));
+  const json = execFileSync("python3",
+    ["-c", "import sys,yaml,json; json.dump(yaml.safe_load(open(sys.argv[1])), sys.stdout)", tmp],
+    { encoding: "utf-8" });
+  const cfg = JSON.parse(json).autoinstall.storage.config as Array<Record<string, string>>;
+  return Object.fromEntries(
+    cfg.filter((c) => c["type"] === "lvm_partition")
+       .map((c) => [c["name"]!, Number(String(c["size"]).replace("G", "")) * 1024]),
+  );
+}
+
+/** MiB per LV name, from the Asahi firstboot lvcreate calls. */
+function asahiSizes(role: Role): Record<string, number> {
+  const out: Record<string, number> = {};
+  for (const m of AS(role).matchAll(/lvcreate -L (\d+)M\s+-n (\w+)\s+labvg/g)) out[m[2]!] = Number(m[1]);
+  return out;
+}
+
+describe("all three generators render the same v2 layout", () => {
+  it.each(SUPPORTED_ROLES)("%s: Fedora matches the shared definition", (role) => {
+    const expected = Object.fromEntries(v2Layout(role).map((lv) => [lv.name, lv.sizeMib]));
+    expect(fedoraSizes(role)).toEqual(expected);
+  });
+
+  it.each(SUPPORTED_ROLES)("%s: Ubuntu matches the shared definition", (role) => {
+    const expected = Object.fromEntries(v2Layout(role).map((lv) => [lv.name, lv.sizeMib]));
+    expect(ubuntuSizes(role)).toEqual(expected);
+  });
+
+  it.each(SUPPORTED_ROLES)("%s: Asahi matches, minus the vendor-created root", (role) => {
+    const expected = Object.fromEntries(
+      v2Layout(role, { omitRoot: true }).map((lv) => [lv.name, lv.sizeMib]),
+    );
+    expect(asahiSizes(role)).toEqual(expected);
+  });
+
+  it.each(SUPPORTED_ROLES)("%s: Fedora and Ubuntu are byte-identical in sizing", (role) => {
+    expect(ubuntuSizes(role)).toEqual(fedoraSizes(role));
+  });
+
+  it("rancher is 250G everywhere — the specific drift that bit worker1", () => {
+    for (const role of ["worker", "infra", "labcontroller"] as const) {
+      expect(fedoraSizes(role)["rancher"]).toBe(256000);
+      expect(ubuntuSizes(role)["rancher"]).toBe(256000);
+      expect(asahiSizes(role)["rancher"]).toBe(256000);
+    }
+  });
+});

bastion/src/bastion/tests/schema-version.test.ts

@@ -0,0 +1,166 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import type { BastionConfig } from "@lab/shared";
+import { createApp } from "../src/server.js";
+import type { FastifyInstance } from "fastify";
+import type { StateManager } from "../src/services/state.js";
+import type { InstallLogBuffer } from "../src/services/install-log.js";
+
+function createTestConfig(testDir: string): BastionConfig {
+  return {
+    fedoraVersion: "43",
+    arch: "x86_64",
+    httpPort: 0,
+    timezone: "Europe/London",
+    locale: "en_GB.UTF-8",
+    bastionDir: testDir,
+    domain: "test.local",
+    dhcpMode: "proxy",
+    dhcpRangeStart: "",
+    dhcpRangeEnd: "",
+    ubuntuVersion: "26.04",
+    ubuntuMirror: "https://releases.ubuntu.com/26.04",
+    vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso",
+    vyosDefaultPassword: "vyos",
+    iface: "eth0",
+    serverIp: "10.0.0.1",
+    network: "10.0.0.0",
+    gateway: "10.0.0.1",
+    sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"],
+    adminUser: "testadmin",
+    syslogPort: 15514,
+    skipDnsmasq: true,
+    skipArtifacts: true,
+    fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
+    tftpDir: join(testDir, "tftp"),
+    httpDir: join(testDir, "http"),
+    stateFile: join(testDir, "state.json"),
+  };
+}
+
+describe("schemaVersion plumbing: /api/install -> /ks", () => {
+  let app: FastifyInstance;
+  let testDir: string;
+
+  beforeEach(async () => {
+    testDir = join(tmpdir(), `schema-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+    mkdirSync(testDir, { recursive: true });
+    const built = await createApp(createTestConfig(testDir));
+    app = built.app;
+    await app.ready();
+  });
+
+  afterEach(async () => {
+    await app.close();
+    rmSync(testDir, { recursive: true, force: true });
+  });
+
+  const queue = (body: Record<string, unknown>) =>
+    app.inject({ method: "POST", url: "/api/install", payload: { mac: "aa:bb:cc:dd:ee:01", hostname: "n1", ...body } });
+
+  const ks = () => app.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:01" });
+
+  it("defaults to v1 when no schemaVersion is given", async () => {
+    expect((await queue({ role: "worker" })).statusCode).toBe(200);
+    const body = (await ks()).body;
+    expect(body).toContain("--name=root --fstype=xfs --size=33792");
+  });
+
+  it("schemaVersion 2 reaches the rendered kickstart", async () => {
+    expect((await queue({ role: "worker", schemaVersion: 2 })).statusCode).toBe(200);
+    const body = (await ks()).body;
+    expect(body).toContain("--name=root --fstype=xfs --size=32768");
+    expect(body).toContain("--name=rancher --fstype=xfs --size=256000");
+  });
+
+  it("v2 gives labcontroller a rancher LV through the full path", async () => {
+    expect((await queue({ role: "labcontroller", schemaVersion: 2 })).statusCode).toBe(200);
+    expect((await ks()).body).toContain("--name=rancher");
+  });
+
+  it("rejects an unknown schemaVersion instead of silently installing v1", async () => {
+    const res = await queue({ role: "worker", schemaVersion: 3 });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/invalid schemaVersion/);
+  });
+
+  it("still rejects an unknown role", async () => {
+    expect((await queue({ role: "nope" })).statusCode).toBe(400);
+  });
+});
+
+describe("v2-only roles cannot be installed at v1", () => {
+  let app2: FastifyInstance;
+  let dir2: string;
+
+  beforeEach(async () => {
+    dir2 = join(tmpdir(), `schema-v2only-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+    mkdirSync(dir2, { recursive: true });
+    const built = await createApp(createTestConfig(dir2));
+    app2 = built.app;
+    await app2.ready();
+  });
+  afterEach(async () => { await app2.close(); rmSync(dir2, { recursive: true, force: true }); });
+
+  const q = (body: Record<string, unknown>) =>
+    app2.inject({ method: "POST", url: "/api/install", payload: { mac: "aa:bb:cc:dd:ee:02", hostname: "n2", ...body } });
+
+  /**
+   * Omitting the schema must WORK for a v2-only role: a reinstall never passes
+   * one (the lab-schema-v2 tag on the disk decides), and the first version of
+   * this guard rejected that, making it impossible to reinstall a v2-only
+   * machine at all. Found by the reinstall integration test.
+   */
+  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"])(
+    "%s is accepted without a schemaVersion and defaults to v2", async (role) => {
+      const res = await q({ role });
+      expect(res.statusCode).toBe(200);
+      const ks = (await app2.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:02" })).body;
+      expect(ks).toContain("--name=root --fstype=xfs --size=32768");
+    });
+
+  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"])(
+    "%s is rejected on an EXPLICIT schemaVersion 1 — that is a contradiction", async (role) => {
+      const res = await q({ role, schemaVersion: 1 });
+      expect(res.statusCode).toBe(400);
+      expect(res.json().error).toMatch(/requires schemaVersion 2/);
+    });
+
+  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"])(
+    "%s is accepted with schemaVersion 2", async (role) => {
+      expect((await q({ role, schemaVersion: 2 })).statusCode).toBe(200);
+    });
+
+  it("legacy roles are still accepted without a schemaVersion", async () => {
+    for (const role of ["vanilla", "worker", "infra", "labcontroller"]) {
+      expect((await q({ role })).statusCode).toBe(200);
+    }
+  });
+
+  /**
+   * Assert on the FRESH-INSTALL part.ks block specifically. The whole kickstart
+   * also contains the reprovision branch, whose shell emits `--name=longhorn`
+   * inside a `PRESERVE_LONGHORN` guard -- so a naive whole-file `not.toContain`
+   * matches that and fails for reasons that have nothing to do with the layout.
+   */
+  function freshBlockOf(ks: string): string {
+    const blocks = [...ks.matchAll(/cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g)]
+      .map((m) => m[1]!);
+    const fresh = blocks.find((b) => b.includes("part pv.01"));
+    if (!fresh) throw new Error("no fresh-install block in rendered kickstart");
+    return fresh;
+  }
+
+  it("k8s-controller gets no longhorn LV; k8s-combo does", async () => {
+    await q({ role: "k8s-controller", schemaVersion: 2 });
+    const ctrl = freshBlockOf((await app2.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:02" })).body);
+    expect(ctrl).toContain("--name=rancher");
+    expect(ctrl).not.toContain("--name=longhorn");
+
+    await q({ role: "k8s-combo", schemaVersion: 2 });
+    const combo = freshBlockOf((await app2.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:02" })).body);
+    expect(combo).toContain("--name=longhorn --fstype=xfs --size=307200");
+  });
+});

bastion/src/bastion/tests/ubuntu-autoinstall.test.ts

@@ -0,0 +1,184 @@
+// The Ubuntu autoinstall document must be valid YAML for EVERY role.
+//
+// There was no test here, and the template shipped a document that did not
+// parse: the longhorn/rancher LVM entries were indented 8 spaces while their
+// siblings in storage.config sit at 6, giving "expected <block end>, but found
+// '-'". Every role that gets a longhorn volume -- which is the worker shape --
+// rendered an uninstallable document. A `toContain` assertion would not have
+// caught that; only parsing does.
+//
+// Parsed with python3's yaml rather than a new npm dependency, mirroring how
+// kickstart.test.ts shells out to `ksvalidator`: the point is to check the
+// artefact with a real parser, not to grow the dependency tree.
+
+import { describe, it, expect } from "vitest";
+import { execFileSync } from "node:child_process";
+import { writeFileSync, unlinkSync } from "node:fs";
+import { renderUbuntuAutoinstall } from "../src/templates/ubuntu-autoinstall.js";
+
+const base = {
+  hostname: "n6",
+  disk: "/dev/sda",
+  domain: "ad.itaz.eu",
+  ubuntuVersion: "24.04",
+  timezone: "Europe/London",
+  locale: "en_GB.UTF-8",
+  serverIp: "10.0.0.1",
+  httpPort: 8080,
+  sshKeys: ["ssh-ed25519 AAAAtest test@lab"],
+  adminUser: "root",
+};
+
+/** Parse with python3's yaml and return the document as JSON. */
+function parseYaml(text: string, label: string): Record<string, any> {
+  const tmp = `/tmp/autoinstall-test-${label}.yaml`;
+  writeFileSync(tmp, text);
+  try {
+    const out = execFileSync(
+      "python3",
+      ["-c", "import sys,yaml,json; json.dump(yaml.safe_load(open(sys.argv[1])), sys.stdout)", tmp],
+      { encoding: "utf-8" },
+    );
+    return JSON.parse(out);
+  } catch (err: unknown) {
+    const msg = err instanceof Error ? (err as { stderr?: string }).stderr ?? err.message : String(err);
+    throw new Error(`autoinstall YAML did not parse for ${label}: ${msg}`);
+  } finally {
+    try { unlinkSync(tmp); } catch { /* ignore */ }
+  }
+}
+
+describe("renderUbuntuAutoinstall", () => {
+  for (const role of ["vanilla", "worker", "infra"]) {
+    it(`renders parseable YAML for role=${role}`, () => {
+      const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role }), role);
+      expect(doc.autoinstall).toBeDefined();
+      expect(doc.autoinstall.version).toBe(1);
+      // storage.config must be a flat list; the indentation bug produced a
+      // nested map here, which is how it went unnoticed.
+      expect(Array.isArray(doc.autoinstall.storage.config)).toBe(true);
+    });
+  }
+
+  it("gives the longhorn role its volume as a sibling entry, not a nested map", () => {
+    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" }), "longhorn");
+    const ids = doc.autoinstall.storage.config.map((e: { id: string }) => e.id);
+    expect(ids).toContain("lv-longhorn");
+    expect(ids).toContain("mount-longhorn");
+  });
+
+  it("sets EUI-64 link-local so MAC-keyed DHCPv6 reservations can match", () => {
+    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" }), "eui64");
+    const late = (doc.autoinstall["late-commands"] as string[]).join("\n");
+    expect(late).toContain("10-ipv6-eui64.conf");
+    expect(late).toContain("ipv6.addr-gen-mode=eui64");
+  });
+
+  it("requests both address families on the primary NIC", () => {
+    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" }), "net");
+    const eth = doc.autoinstall.network.ethernets.primary;
+    expect(eth.dhcp4).toBe(true);
+    // Without this a node provisioned into a dual-stack cluster comes up with
+    // no IPv6 and joins as an IPv4-only member.
+    expect(eth.dhcp6).toBe(true);
+    // The install must not block waiting for a v6 lease that may never come.
+    expect(eth.optional).toBe(true);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// v2 layout. Ubuntu previously sized rancher at 20G while the Fedora kickstart
+// used 120G, from a template whose comment claimed the sizes matched. Both now
+// render from shared/src/types/layout.ts, so this checks the rendering, and
+// layout.test.ts checks the definition.
+// ---------------------------------------------------------------------------
+describe("v2 storage layout", () => {
+  const roles = ["vanilla", "worker", "infra", "labcontroller"] as const;
+
+  function lvmPartitions(doc: Record<string, any>): Record<string, string> {
+    const cfg = doc["autoinstall"]["storage"]["config"] as Array<Record<string, any>>;
+    return Object.fromEntries(
+      cfg.filter((c) => c["type"] === "lvm_partition").map((c) => [c["name"], c["size"]]),
+    );
+  }
+
+  it.each(roles)("%s renders valid YAML at schemaVersion 2", (role) => {
+    const doc = parseYaml(
+      renderUbuntuAutoinstall({ ...base, role, schemaVersion: 2 } as any),
+      `v2-${role}`,
+    );
+    expect(Array.isArray(doc["autoinstall"]["storage"]["config"])).toBe(true);
+  });
+
+  it("v2 sizes match the shared definition, not the old 20G rancher", () => {
+    const doc = parseYaml(
+      renderUbuntuAutoinstall({ ...base, role: "worker", schemaVersion: 2 } as any),
+      "v2-sizes",
+    );
+    const lvs = lvmPartitions(doc);
+    expect(lvs["rancher"]).toBe("250G");   // was 20G here and 120G in kickstart
+    expect(lvs["longhorn"]).toBe("300G");  // was grow
+    expect(lvs["root"]).toBe("32G");
+    expect(lvs["var"]).toBe("48G");
+  });
+
+  it("v2 gives labcontroller a rancher LV; v1 does not", () => {
+    const v2 = lvmPartitions(parseYaml(
+      renderUbuntuAutoinstall({ ...base, role: "labcontroller", schemaVersion: 2 } as any), "v2-lc"));
+    const v1 = lvmPartitions(parseYaml(
+      renderUbuntuAutoinstall({ ...base, role: "labcontroller" } as any), "v1-lc"));
+    expect(v2["rancher"]).toBe("250G");
+    expect(v1["rancher"]).toBeUndefined();
+  });
+
+  it("v1 remains the default and is unchanged", () => {
+    const lvs = lvmPartitions(parseYaml(
+      renderUbuntuAutoinstall({ ...base, role: "worker" } as any), "v1-default"));
+    expect(lvs["root"]).toBe("33G");
+    expect(lvs["var"]).toBe("100G");
+    expect(lvs["swap"]).toBe("27G");
+  });
+
+  it("no v2 volume is unbounded", () => {
+    for (const role of roles) {
+      const lvs = lvmPartitions(parseYaml(
+        renderUbuntuAutoinstall({ ...base, role, schemaVersion: 2 } as any), `v2-grow-${role}`));
+      for (const [name, size] of Object.entries(lvs)) {
+        expect(String(size), `${role}/${name}`).toMatch(/^\d+G$/);
+      }
+    }
+  });
+});
+
+/**
+ * KNOWN GAP, asserted so it cannot be assumed away.
+ *
+ * The Fedora kickstart preserves /home, /srv, longhorn and rancher across a
+ * reinstall (%pre probes for an existing labvg). The Ubuntu autoinstall has no
+ * equivalent: `disk0` is wiped unconditionally, so EVERY Ubuntu reinstall is
+ * destructive.
+ *
+ * These tests pin that behaviour rather than pretend it is fine. If someone
+ * adds curtin `preserve: true` handling, these fail and should be rewritten --
+ * which is the point: the change becomes visible instead of silent.
+ */
+describe("Ubuntu reinstall is destructive (known gap)", () => {
+  it("wipes the disk unconditionally — there is no reprovision branch", () => {
+    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" } as any), "destructive");
+    const cfg = doc["autoinstall"]["storage"]["config"] as Array<Record<string, any>>;
+    const disk = cfg.find((c) => c["type"] === "disk");
+    expect(disk?.["wipe"]).toBe("superblock-recursive");
+  });
+
+  it("no volume is marked preserve — nothing survives a reinstall", () => {
+    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" } as any), "no-preserve");
+    const cfg = doc["autoinstall"]["storage"]["config"] as Array<Record<string, any>>;
+    expect(cfg.some((c) => c["preserve"] === true)).toBe(false);
+  });
+
+  it("the same is true at schemaVersion 2 — v2 did not fix this", () => {
+    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker", schemaVersion: 2 } as any), "v2-destructive");
+    const cfg = doc["autoinstall"]["storage"]["config"] as Array<Record<string, any>>;
+    expect(cfg.find((c) => c["type"] === "disk")?.["wipe"]).toBe("superblock-recursive");
+  });
+});

bastion/src/bastion/tests/vyos-bundle.test.ts

@@ -0,0 +1,155 @@
+import { describe, it, expect, vi } from "vitest";
+import type { VyosBundle } from "@lab/shared";
+import { buildVyosConfigSpec } from "../src/templates/vyos-config-spec.js";
+
+/**
+ * A bundle is what makes "one config, two apply paths" true rather than
+ * aspirational: `pulumi up` POSTs the subtree model to a running router, labctl
+ * writes the same model into config.boot during a PXE install. These tests pin
+ * the properties that keep the two honest.
+ */
+const bundle: VyosBundle = {
+  sets: [
+    { path: ["system", "host-name"], value: "vyos001" },
+    { path: ["interfaces", "bonding", "bond0", "address"], value: "192.168.1.252/24" },
+    { path: ["interfaces", "bonding", "bond0", "member", "interface"], value: "eth1", replace: false },
+    { path: ["interfaces", "bonding", "bond0", "vif", "53", "disable"] },
+    { path: ["interfaces", "pppoe", "pppoe0", "authentication", "password"], value: "@secret:pppoePassword" },
+    { path: ["interfaces", "pppoe", "pppoe0", "mtu"], value: "1492" },
+  ],
+  tags: [["interfaces", "bonding", "bond0"], ["interfaces", "ethernet"]],
+};
+
+const build = (hostname: string, extra: Record<string, unknown> = {}) =>
+  buildVyosConfigSpec({
+    hostname,
+    spec: { bundle, ...extra },
+    defaultPassword: "changeme",
+  });
+
+describe("vyos config spec from a Pulumi bundle", () => {
+  it("applies non-secret nodes verbatim, preserving valuelessness and replace:false", () => {
+    const spec = build("vyos001");
+
+    expect(spec.sets).toContainEqual({
+      path: ["interfaces", "bonding", "bond0", "address"],
+      value: "192.168.1.252/24",
+    });
+    // A multi-value node must keep replace:false or the second bond member
+    // overwrites the first.
+    expect(spec.sets).toContainEqual({
+      path: ["interfaces", "bonding", "bond0", "member", "interface"],
+      value: "eth1",
+      replace: false,
+    });
+    // A valueless node must not acquire a value on the way through.
+    expect(spec.sets).toContainEqual({
+      path: ["interfaces", "bonding", "bond0", "vif", "53", "disable"],
+    });
+    expect(spec.tags).toEqual(bundle.tags);
+  });
+
+  it("drops secret-valued nodes instead of installing the sentinel text", () => {
+    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+    const spec = build("vyos001");
+
+    const values = spec.sets.map((s) => s.value ?? "");
+    expect(values.some((v) => v.startsWith("@secret:"))).toBe(false);
+    expect(spec.sets.some((s) => s.path.includes("authentication"))).toBe(false);
+    // Silently dropping the WAN credential would leave someone debugging a dead
+    // PPPoE link, so it has to be said out loud.
+    expect(warn).toHaveBeenCalledWith(expect.stringContaining("pulumi up"));
+    warn.mockRestore();
+  });
+
+  it("forces the hostname the install was asked for, not the bundle's", () => {
+    // The bundle is exported from one router and reused for its peer; taking the
+    // hostname from it would put two vyos001s on the network.
+    const spec = build("vyos002");
+    const hostnames = spec.sets.filter(
+      (s) => s.path.length === 2 && s.path[0] === "system" && s.path[1] === "host-name",
+    );
+    expect(hostnames).toEqual([{ path: ["system", "host-name"], value: "vyos002" }]);
+  });
+
+  it("still honours installer inputs, which are not router config", () => {
+    const spec = buildVyosConfigSpec({
+      hostname: "vyos001",
+      spec: { bundle, password: "s3cret", freshConfig: true },
+      defaultPassword: "changeme",
+      disk: "nvme0n1",
+    });
+    expect(spec.password).toBe("s3cret");
+    expect(spec.freshConfig).toBe(true);
+    expect(spec.disk).toBe("/dev/nvme0n1");
+  });
+
+  it("enables the HTTP API at install so Pulumi can manage the router from first boot", () => {
+    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+    const spec = buildVyosConfigSpec({
+      hostname: "vyos001",
+      spec: { bundle, apiKey: "k3y", apiListenAddress: "10.0.1.252" },
+      defaultPassword: "changeme",
+    });
+
+    expect(spec.sets).toContainEqual({
+      path: ["service", "https", "api", "keys", "id", "pulumi", "key"],
+      value: "k3y",
+    });
+    expect(spec.sets).toContainEqual({ path: ["service", "https", "api", "rest"] });
+    expect(spec.sets).toContainEqual({
+      path: ["service", "https", "listen-address"],
+      value: "10.0.1.252",
+    });
+    // The key id is a tag node; without this the installer's ConfigTree rejects it.
+    expect(spec.tags).toContainEqual(["service", "https", "api", "keys", "id"]);
+    warn.mockRestore();
+  });
+
+  it("binds the API to the static management address when none is given", () => {
+    const spec = buildVyosConfigSpec({
+      hostname: "vyos001",
+      spec: { apiKey: "k3y", mgmtAddress: "192.168.1.252/24" },
+      defaultPassword: "changeme",
+    });
+    expect(spec.sets).toContainEqual({
+      path: ["service", "https", "listen-address"],
+      value: "192.168.1.252",
+    });
+  });
+
+  it("refuses to enable the API unbound rather than exposing it on the WAN", () => {
+    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+    // Management is DHCP, so there is no address to bind at build time. Binding
+    // to everything would put a config-write endpoint on the WAN.
+    const spec = buildVyosConfigSpec({
+      hostname: "vyos001",
+      spec: { apiKey: "k3y", mgmtAddress: "dhcp" },
+      defaultPassword: "changeme",
+    });
+    expect(spec.sets.some((s) => s.path[0] === "service" && s.path[1] === "https")).toBe(false);
+    expect(warn).toHaveBeenCalledWith(expect.stringContaining("has NOT been enabled"));
+    warn.mockRestore();
+  });
+
+  it("does not enable the API when no key is supplied", () => {
+    const spec = buildVyosConfigSpec({
+      hostname: "vyos001",
+      spec: { mgmtAddress: "192.168.1.252/24" },
+      defaultPassword: "changeme",
+    });
+    expect(spec.sets.some((s) => s.path[0] === "service" && s.path[1] === "https")).toBe(false);
+  });
+
+  it("ignores the derived path entirely when a bundle is present", () => {
+    // Belt and braces: even if topology flags reach this far (the CLI rejects
+    // them), the bundle must win rather than merge.
+    const spec = buildVyosConfigSpec({
+      hostname: "vyos001",
+      spec: { bundle, bondMembers: ["eth2", "eth3"], vlans: [{ id: 99, address: "10.9.9.1/24" }] },
+      defaultPassword: "changeme",
+    });
+    expect(spec.sets.some((s) => s.path.includes("99"))).toBe(false);
+    expect(spec.sets.filter((s) => s.value === "eth2" || s.value === "eth3")).toEqual([]);
+  });
+});

bastion/src/bastion/tests/vyos.test.ts

@@ -0,0 +1,548 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import type { BastionConfig } from "@lab/shared";
+import type { FastifyInstance } from "fastify";
+import { createApp } from "../src/server.js";
+import type { StateManager } from "../src/services/state.js";
+import { buildVyosConfigSpec } from "../src/templates/vyos-config-spec.js";
+import { renderVyosInstallPy } from "../src/templates/vyos-install.py.js";
+
+function createTestConfig(testDir: string): BastionConfig {
+  return {
+    fedoraVersion: "43",
+    arch: "x86_64",
+    httpPort: 0,
+    timezone: "Europe/London",
+    locale: "en_GB.UTF-8",
+    bastionDir: testDir,
+    domain: "test.local",
+    dhcpMode: "proxy",
+    dhcpRangeStart: "",
+    dhcpRangeEnd: "",
+    ubuntuVersion: "26.04",
+    ubuntuMirror: "https://releases.ubuntu.com/26.04",
+    vyosIsoUrl: "https://example.invalid/vyos.iso",
+    vyosDefaultPassword: "test-pw",
+    iface: "eth0",
+    serverIp: "10.0.0.1",
+    network: "10.0.0.0",
+    gateway: "10.0.0.1",
+    sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST lab@test"],
+    adminUser: "testadmin",
+    syslogPort: 15515,
+    skipDnsmasq: true,
+    skipArtifacts: true,
+    fedoraMirror: "https://example.invalid/fedora",
+    tftpDir: join(testDir, "tftp"),
+    httpDir: join(testDir, "http"),
+    stateFile: join(testDir, "state.json"),
+  };
+}
+
+/** Pull the base64 spec back out of the generated Python driver. */
+function decodeSpecFrom(python: string): Record<string, unknown> {
+  const match = /base64\.b64decode\("([^"]+)"\)/.exec(python);
+  if (!match?.[1]) throw new Error("no base64 spec found in generated driver");
+  return JSON.parse(Buffer.from(match[1], "base64").toString("utf-8"));
+}
+
+describe("vyos config spec", () => {
+  it("puts VLANs on the bond when members are given", () => {
+    const spec = buildVyosConfigSpec({
+      hostname: "fw1",
+      defaultPassword: "pw",
+      spec: {
+        mgmtInterface: "eth0",
+        mgmtAddress: "10.0.8.2/24",
+        bondMembers: ["eth2", "eth3"],
+        vlans: [{ id: 10, address: "10.0.10.1/24", description: "k8s" }],
+      },
+    });
+
+    const paths = spec.sets.map((s) => s.path.join(" "));
+    expect(paths).toContain("interfaces bonding bond0 mode");
+    expect(paths).toContain("interfaces bonding bond0 vif 10 address");
+    // VLANs must hang off the bond, not the management NIC.
+    expect(paths).not.toContain("interfaces ethernet eth0 vif 10 address");
+
+    // Bond members are a multi-value node — appending, not replacing, is what
+    // keeps the second member from overwriting the first.
+    const members = spec.sets.filter(
+      (s) => s.path.join(" ") === "interfaces bonding bond0 member interface",
+    );
+    expect(members.map((m) => m.value)).toEqual(["eth2", "eth3"]);
+    expect(members.every((m) => m.replace === false)).toBe(true);
+  });
+
+  it("falls back to VLANs on the management NIC when unbonded", () => {
+    const spec = buildVyosConfigSpec({
+      hostname: "fw2",
+      defaultPassword: "pw",
+      spec: { mgmtInterface: "eth1", vlans: [{ id: 20, address: "10.0.20.1/24" }] },
+    });
+
+    const paths = spec.sets.map((s) => s.path.join(" "));
+    expect(paths).toContain("interfaces ethernet eth1 vif 20 address");
+  });
+
+  it("normalises the target disk to a full /dev path", () => {
+    // find_disks() enumerates with `lsblk -Jbp`, so valid responses are full
+    // paths; a bare name fails valid_responses and re-prompts forever.
+    expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw", disk: "/dev/mmcblk0" }).disk)
+      .toBe("/dev/mmcblk0");
+    expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw", disk: "mmcblk0" }).disk)
+      .toBe("/dev/mmcblk0");
+    expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw" }).disk).toBe("");
+  });
+
+  it("defaults to dhcp on eth0 and never opts into RAID", () => {
+    const spec = buildVyosConfigSpec({ hostname: "fw4", defaultPassword: "pw" });
+    const address = spec.sets.find(
+      (s) => s.path.join(" ") === "interfaces ethernet eth0 address",
+    );
+    expect(address?.value).toBe("dhcp");
+    // The installer's RAID prompt defaults to yes; a second disk must not
+    // silently produce a mirror.
+    expect(spec.raid).toBe(false);
+  });
+});
+
+describe("vyos routes", () => {
+  let testDir: string;
+  let app: FastifyInstance;
+  let state: StateManager;
+  const mac = "aa:bb:cc:11:22:33";
+
+  beforeEach(() => {
+    testDir = join(tmpdir(), `bastion-vyos-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+    mkdirSync(join(testDir, "http"), { recursive: true });
+    mkdirSync(join(testDir, "tftp"), { recursive: true });
+
+    const result = createApp(createTestConfig(testDir));
+    app = result.app;
+    state = result.state;
+  });
+
+  afterEach(async () => {
+    await app.close();
+    rmSync(testDir, { recursive: true, force: true });
+  });
+
+  it("dispatches a queued vyos machine to the live-boot script", async () => {
+    state.update((s) => {
+      s.install_queue[mac] = {
+        hostname: "fw1",
+        disk: "/dev/nvme0n1",
+        role: "worker",
+        os: "vyos-rolling",
+        queued_at: new Date().toISOString(),
+      };
+    });
+
+    const response = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}` });
+    expect(response.statusCode).toBe(200);
+
+    expect(response.body).toContain("/vyos-vmlinuz");
+    expect(response.body).toContain("fetch=http://10.0.0.1:0/vyos-filesystem.squashfs");
+    expect(response.body).toContain(`live-config.hooks=http://10.0.0.1:0/vyos/autoinstall.sh?mac=${mac}`);
+
+    // `nonetworking` appears in VyOS's own PXE docs but breaks the hook fetch,
+    // and console=ttyS0 costs 30s per systemd phase on boards with no UART.
+    expect(response.body).not.toContain("nonetworking");
+    expect(response.body).not.toContain("console=ttyS0");
+  });
+
+  it("serves a hook that fetches and executes the install driver", async () => {
+    const response = await app.inject({ method: "GET", url: `/vyos/autoinstall.sh?mac=${mac}` });
+    expect(response.statusCode).toBe(200);
+    expect(response.body).toContain(`/vyos/install.py?mac=${mac}`);
+    expect(response.body).toContain("python3 /tmp/vyos-install.py");
+  });
+
+  it("bakes the machine's config into the generated install driver", async () => {
+    state.update((s) => {
+      s.install_queue[mac] = {
+        hostname: "fw1",
+        disk: "/dev/nvme0n1",
+        role: "worker",
+        os: "vyos-rolling",
+        queued_at: new Date().toISOString(),
+        vyos: {
+          mgmtInterface: "eth0",
+          mgmtAddress: "10.0.8.2/24",
+          bondMembers: ["eth2", "eth3"],
+          vlans: [{ id: 10, address: "10.0.10.1/24" }],
+          password: "s3cret",
+        },
+      };
+    });
+
+    const response = await app.inject({ method: "GET", url: `/vyos/install.py?mac=${mac}` });
+    expect(response.statusCode).toBe(200);
+
+    // Builds config from the image's own default so the vyos-config-version
+    // trailer matches and first boot skips migrations.
+    expect(response.body).toContain("/opt/vyatta/etc/config.boot.default");
+    expect(response.body).toContain("/usr/libexec/vyos/op_mode/image_installer.py");
+    // "complete" is what moves the machine out of the install queue.
+    expect(response.body).toContain('report("complete"');
+
+    const spec = decodeSpecFrom(response.body);
+    expect(spec["hostname"]).toBe("fw1");
+    expect(spec["password"]).toBe("s3cret");
+    expect(spec["disk"]).toBe("/dev/nvme0n1");
+
+    const paths = (spec["sets"] as Array<{ path: string[] }>).map((s) => s.path.join(" "));
+    expect(paths).toContain("interfaces bonding bond0 vif 10 address");
+    expect(paths).toContain("system host-name");
+  });
+
+  it("falls back to the bastion default password when none is set", async () => {
+    state.update((s) => {
+      s.install_queue[mac] = {
+        hostname: "fw9",
+        disk: "",
+        role: "worker",
+        os: "vyos-rolling",
+        queued_at: new Date().toISOString(),
+      };
+    });
+
+    const response = await app.inject({ method: "GET", url: `/vyos/install.py?mac=${mac}` });
+    const spec = decodeSpecFrom(response.body);
+    expect(spec["password"]).toBe("test-pw");
+    // Empty disk means "accept the installer's first-disk default".
+    expect(spec["disk"]).toBe("");
+  });
+});
+
+describe("vyos hw-id pinning", () => {
+  it("emits hw-id for the mgmt interface and each bond member", () => {
+    // Discovery sees enp2s0/enp1s0f0np0 under Fedora, but VyOS enumerates its
+    // own eth<N>. Pinning by MAC is what makes the mapping deterministic.
+    const spec = buildVyosConfigSpec({
+      hostname: "fw1",
+      defaultPassword: "pw",
+      spec: {
+        mgmtInterface: "eth2",
+        bondMembers: ["eth0", "eth1"],
+        hwIds: {
+          eth2: "64:62:66:25:96:47",
+          eth0: "64:62:66:25:96:45",
+          eth1: "64:62:66:25:96:46",
+        },
+      },
+    });
+
+    const hw = spec.sets.filter((s) => s.path[s.path.length - 1] === "hw-id");
+    expect(hw.map((s) => [s.path[2], s.value])).toEqual([
+      ["eth2", "64:62:66:25:96:47"],
+      ["eth0", "64:62:66:25:96:45"],
+      ["eth1", "64:62:66:25:96:46"],
+    ]);
+  });
+
+  it("omits hw-id entirely when no mapping is given", () => {
+    const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" });
+    expect(spec.sets.some((s) => s.path.includes("hw-id"))).toBe(false);
+  });
+});
+
+describe("vyos management VLAN", () => {
+  it("puts the mgmt VLAN on the PXE port while the bond carries routed VLANs", () => {
+    // Trunked PXE port: boots untagged on the VLAN the bastion serves, stays
+    // reachable on the tagged management VLAN.
+    const spec = buildVyosConfigSpec({
+      hostname: "vyos001",
+      defaultPassword: "pw",
+      spec: {
+        mgmtInterface: "eth2",
+        mgmtAddress: "dhcp",
+        mgmtVlan: { id: 3, address: "192.168.3.4/24", description: "kvm" },
+        bondMembers: ["eth0", "eth1"],
+        vlans: [{ id: 2, address: "192.168.8.2/23" }],
+      },
+    });
+
+    const paths = spec.sets.map((s) => s.path.join(" "));
+    expect(paths).toContain("interfaces ethernet eth2 vif 3 address");
+    expect(paths).toContain("interfaces bonding bond0 vif 2 address");
+    // The mgmt VLAN must not land on the bond.
+    expect(paths).not.toContain("interfaces bonding bond0 vif 3 address");
+    expect(spec.tags.map((t) => t.join(" "))).toContain("interfaces ethernet eth2 vif");
+  });
+});
+
+describe("vyos VRRP HA", () => {
+  const haSpec = {
+    mgmtInterface: "eth2",
+    mgmtAddress: "dhcp",
+    bondMembers: ["eth0", "eth1"],
+    bondAddress: "192.168.1.252/24",
+    bondVrrp: "192.168.1.254/24",
+    vrrpPriority: 200,
+    vlans: [
+      { id: 3, address: "192.168.3.4/24", vrrp: "192.168.3.254/24" },
+      { id: 200, address: "192.168.2.252/24" },   // no VIP on this one
+    ],
+  };
+
+  it("emits a vrrp group per VIP with vrid = VLAN id and dotted vif interface", () => {
+    const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw", spec: haSpec });
+    const paths = spec.sets.map((s) => `${s.path.join(" ")}${s.value !== undefined ? "=" + s.value : ""}`);
+
+    expect(paths).toContain("interfaces bonding bond0 address=192.168.1.252/24");
+    // untagged bond group: vrid 1, interface bond0 itself
+    expect(paths).toContain("high-availability vrrp group native interface=bond0");
+    expect(paths).toContain("high-availability vrrp group native vrid=1");
+    // address is a tag node -- VIP is the final path segment, no value
+    expect(paths).toContain("high-availability vrrp group native address 192.168.1.254/24");
+    // VLAN group: vrid = VLAN id, dotted vif
+    expect(paths).toContain("high-availability vrrp group vlan3 interface=bond0.3");
+    expect(paths).toContain("high-availability vrrp group vlan3 vrid=3");
+    expect(paths).toContain("high-availability vrrp group vlan3 address 192.168.3.254/24");
+    // VLAN without a VIP gets no group
+    expect(paths.some((p) => p.includes("group vlan200"))).toBe(false);
+  });
+
+  it("applies the box-wide priority and one sync group over all groups", () => {
+    const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw", spec: haSpec });
+    const prio = spec.sets.filter((s) => s.path[s.path.length - 1] === "priority"
+      && s.path[0] === "high-availability");
+    expect(prio).toHaveLength(2);
+    expect(prio.every((s) => s.value === "200")).toBe(true);
+
+    // sync group binds the pair: all groups fail over together
+    const members = spec.sets.filter(
+      (s) => s.path.join(" ") === "high-availability vrrp sync-group MAIN member",
+    );
+    expect(members.map((m) => m.value)).toEqual(["native", "vlan3"]);
+    expect(members.every((m) => m.replace === false)).toBe(true);
+  });
+
+  it("emits no high-availability nodes when no VIPs are given", () => {
+    const spec = buildVyosConfigSpec({
+      hostname: "fw1",
+      defaultPassword: "pw",
+      spec: { bondMembers: ["eth0", "eth1"], vlans: [{ id: 3, address: "192.168.3.4/24" }] },
+    });
+    expect(spec.sets.some((s) => s.path[0] === "high-availability")).toBe(false);
+  });
+});
+
+describe("pickLargestInitrd", async () => {
+  const { pickLargestInitrd } = await import("../src/main.js");
+
+  // Verbatim from `xorriso -lsl /live/` on vyos-2026.08.05-0033-rolling.
+  const realListing = `total 8
+-r--r--r--    1 0        0           22255 Aug  5 01:33 'filesystem.packages'
+-r--r--r--    1 0        0               6 Aug  5 01:33 'filesystem.packages-remove'
+-r--r--r--    1 0        0        541192192 Aug  5 01:33 'filesystem.squashfs'
+-r--r--r--    1 0        0        50352547 Aug  5 01:33 'initrd.img'
+-r--r--r--    1 0        0        50352547 Aug  5 01:33 'initrd.img-6.18.41-vyos'
+-r--r--r--    1 0        0              20 Aug  5 01:33 'packages.txt'
+-r--r--r--    1 0        0         9135104 Aug  2 19:54 'vmlinuz'
+-r--r--r--    1 0        0         9135104 Aug  2 19:54 'vmlinuz-6.18.41-vyos'
+`;
+
+  it("picks a full-size initrd from a real nightly listing", () => {
+    expect(pickLargestInitrd(realListing)).toEqual({ name: "initrd.img", size: 50352547 });
+  });
+
+  it("ignores 0-byte decoys and symlinks (which report link size, not target size)", () => {
+    const listing = `total 8
+-r--r--r--    1 0        0               0 Aug  5 01:33 'initrd.img'
+lrwxrwxrwx    1 0        0              24 Aug  5 01:33 'initrd.img-link' -> 'initrd.img-6.18.41-vyos'
+-r--r--r--    1 0        0        50352547 Aug  5 01:33 'initrd.img-6.18.41-vyos'
+`;
+    expect(pickLargestInitrd(listing)).toEqual({ name: "initrd.img-6.18.41-vyos", size: 50352547 });
+  });
+
+  it("returns undefined when only decoys exist", () => {
+    expect(pickLargestInitrd("-r--r--r-- 1 0 0 0 Aug 5 01:33 'initrd.img'\n")).toBeUndefined();
+  });
+});
+
+describe("vyos fedora-parity features", () => {
+  it("computes reportAddress from a static mgmt address, empty for dhcp", () => {
+    const staticSpec = buildVyosConfigSpec({
+      hostname: "fw1", defaultPassword: "pw",
+      spec: { mgmtAddress: "192.168.8.2/23" },
+    });
+    expect(staticSpec.reportAddress).toBe("192.168.8.2");
+
+    const dhcpSpec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" });
+    expect(dhcpSpec.reportAddress).toBe("");
+  });
+
+  it("defaults freshConfig off (reinstall preserves the on-disk config)", () => {
+    expect(buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" }).freshConfig).toBe(false);
+    expect(buildVyosConfigSpec({
+      hostname: "fw1", defaultPassword: "pw", spec: { freshConfig: true },
+    }).freshConfig).toBe(true);
+  });
+
+  it("driver streams logs to /api/log and reports 'ready at' on completion", async () => {
+    const testDir = join(tmpdir(), `bastion-vyos-parity-${Date.now()}`);
+    mkdirSync(join(testDir, "http"), { recursive: true });
+    mkdirSync(join(testDir, "tftp"), { recursive: true });
+    const { app: parityApp, state: parityState } = createApp(createTestConfig(testDir));
+    try {
+      parityState.update((s) => {
+        s.install_queue["aa:bb:cc:44:55:66"] = {
+          hostname: "fw9", disk: "/dev/vda", role: "vanilla",
+          os: "vyos-rolling", queued_at: new Date().toISOString(),
+        };
+      });
+      const response = await parityApp.inject({
+        method: "GET", url: "/vyos/install.py?mac=aa:bb:cc:44:55:66",
+      });
+      expect(response.body).toContain("/api/log");
+      expect(response.body).toContain('"lines": batch');
+      expect(response.body).toContain('report("complete", "ready at %s"');
+      expect(response.body).toContain("ensure_network_boot_first");
+      expect(response.body).toContain("lab-provisioned");
+      expect(response.body).toContain('ROLE = "vanilla"');
+    } finally {
+      await parityApp.close();
+      rmSync(testDir, { recursive: true, force: true });
+    }
+  });
+
+  it("complete with 'ready at' records installed.ip for a vyos machine", async () => {
+    const testDir = join(tmpdir(), `bastion-vyos-complete-${Date.now()}`);
+    mkdirSync(join(testDir, "http"), { recursive: true });
+    mkdirSync(join(testDir, "tftp"), { recursive: true });
+    const { app: cApp, state: cState } = createApp(createTestConfig(testDir));
+    try {
+      const mac2 = "aa:bb:cc:77:88:99";
+      cState.update((s) => {
+        s.install_queue[mac2] = {
+          hostname: "fw1", disk: "/dev/vda", role: "vanilla",
+          os: "vyos-rolling", queued_at: new Date().toISOString(),
+        };
+      });
+      const response = await cApp.inject({
+        method: "POST", url: "/api/progress",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ mac: mac2, stage: "complete", detail: "ready at 192.168.8.2" }),
+      });
+      expect(response.statusCode).toBe(200);
+      const installed = cState.load().installed[mac2];
+      expect(installed?.ip).toBe("192.168.8.2");
+      expect(installed?.os).toBe("vyos-rolling");
+    } finally {
+      await cApp.close();
+      rmSync(testDir, { recursive: true, force: true });
+    }
+  });
+});
+
+describe("vyos installer prompt coverage", () => {
+  // Every interactive prompt image_installer.py can emit, copied verbatim from
+  // the MSG_* constants (including the reinstall-only search_previous_installation
+  // ones). An unanswered prompt does not fail loudly -- the installer simply
+  // blocks on stdin until the driver's stall timeout, which is how the reinstall
+  // path silently hung for 15 minutes in the VM test.
+  const PROMPTS: Record<string, string> = {
+    continue: "Would you like to continue? [y/N] ",
+    imageName: "What would you like to name this image? (Default: 1.5-rolling) ",
+    password: 'Please enter a password for the "vyos" user: ',
+    passwordConfirm: 'Please confirm password for the "vyos" user: ',
+    console: "What console should be used by default? (K: KVM, S: Serial)? (Default: K) ",
+    raidConfigure: "Would you like to configure RAID-1 mirroring? [Y/n] ",
+    raidFoundDisks: "Would you like to configure RAID-1 mirroring on them? [Y/n] ",
+    raidChooseDisks: "Would you like to choose two disks for RAID-1 mirroring? [Y/n] ",
+    diskSelect: "Which one should be used for installation? (Default: /dev/vda) ",
+    diskConfirm: "Installation will delete all data on the drive. Continue? [y/N] ",
+    raidConfirm: "Installation will delete all data on both drives. Continue? [y/N] ",
+    rootSizeAll: "Would you like to use all the free space on the drive? [Y/n] ",
+    bootConfig: "Which file would you like as boot config? ",
+    copyData: "Would you like to copy data to the new image? [Y/n] ",
+    chooseCopyData: "From which image would you like to save config information? ",
+    copyEncData: "Would you like to copy the encrypted config to the new image? [Y/n] ",
+    chooseCopyEncData: "From which image would you like to copy the encrypted config? ",
+  };
+
+  it("answers every installer prompt exactly once", () => {
+    const { execFileSync } = require("node:child_process") as typeof import("node:child_process");
+    const { writeFileSync, unlinkSync, mkdtempSync } = require("node:fs") as typeof import("node:fs");
+
+    // Skip cleanly where python3 is unavailable (same spirit as the
+    // ksvalidator-backed kickstart test).
+    try {
+      execFileSync("python3", ["--version"], { stdio: "pipe" });
+    } catch {
+      return;
+    }
+
+    const spec = buildVyosConfigSpec({
+      hostname: "fw1", defaultPassword: "pw", disk: "/dev/vda",
+    });
+    const driver = renderVyosInstallPy({
+      spec, mac: "aa:bb:cc:11:22:33", serverIp: "10.0.0.1", httpPort: 8080, role: "vanilla",
+    });
+
+    const dir = mkdtempSync(join(tmpdir(), "vyos-rules-"));
+    const driverPath = join(dir, "driver.py");
+    const checkPath = join(dir, "check.py");
+    writeFileSync(driverPath, driver);
+    writeFileSync(checkPath, `
+import importlib.util, json, sys
+spec = importlib.util.spec_from_file_location("drv", ${JSON.stringify(driverPath)})
+drv = importlib.util.module_from_spec(spec); spec.loader.exec_module(drv)
+rules = drv.build_rules()
+prompts = json.loads(sys.argv[1])
+out = {}
+for label, text in prompts.items():
+    out[label] = len([r for p, r in rules if p.search(text.encode())])
+print(json.dumps(out))
+`);
+
+    try {
+      const stdout = execFileSync("python3", [checkPath, JSON.stringify(PROMPTS)], {
+        encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"],
+      });
+      const counts = JSON.parse(stdout) as Record<string, number>;
+      const unanswered = Object.entries(counts).filter(([, n]) => n !== 1);
+      expect(unanswered).toEqual([]);
+    } finally {
+      try { unlinkSync(driverPath); unlinkSync(checkPath); } catch { /* best effort */ }
+      try { rmSync(dir, { recursive: true, force: true }); } catch { /* best effort */ }
+    }
+  });
+});
+
+describe("vyos boot NIC pinning", () => {
+  it("pins the boot interface by MAC via BOOTIF", async () => {
+    // Without this, live-boot picks the first *connected* NIC. On the VP2440
+    // the SFP+ pair links before the copper PXE port, so live-boot tried the
+    // fiber ports (no DHCP), timed out 15s each, and failed with "Unable to
+    // find a live file system on the network".
+    const testDir = join(tmpdir(), `bastion-vyos-bootif-${Date.now()}`);
+    mkdirSync(join(testDir, "http"), { recursive: true });
+    mkdirSync(join(testDir, "tftp"), { recursive: true });
+    const { app: a, state: st } = createApp(createTestConfig(testDir));
+    try {
+      const m = "64:62:66:25:96:47";
+      st.update((s) => {
+        s.install_queue[m] = {
+          hostname: "vyos001", disk: "/dev/mmcblk0", role: "vanilla",
+          os: "vyos-rolling", queued_at: new Date().toISOString(),
+        };
+      });
+      const res = await a.inject({ method: "GET", url: `/dispatch?mac=${m}` });
+      // live-boot's Device_from_bootif() expects 01-<mac with dashes>
+      expect(res.body).toContain("BOOTIF=01-64-62-66-25-96-47");
+      // and it must be on the kernel line, before fetch= is attempted
+      const kernelLine = res.body.split("\n").find((l) => l.startsWith("kernel "));
+      expect(kernelLine).toContain("BOOTIF=01-64-62-66-25-96-47");
+      expect(kernelLine).toContain("fetch=");
+    } finally {
+      await a.close();
+      rmSync(testDir, { recursive: true, force: true });
+    }
+  });
+});

bastion/src/cli/src/api/client.ts

@@ -90,6 +90,9 @@ export class LabdClient {
 
   async installMachine(opts: {
     mac: string; hostname: string; disk?: string; role?: string; os?: string;
+    schemaVersion?: 1 | 2;
+    dataDisks?: import("@lab/shared").DataDisks;
+    vyos?: import("@lab/shared").VyosInstallSpec;
   }): Promise<{ status: string; data?: unknown; error?: string }> {
     return this.request("POST", "/api/machines/install", { body: opts });
   }
@@ -110,7 +113,6 @@ export class LabdClient {
     memory_gb?: number; arch?: string;
     disks?: Array<{ name: string; size_gb: number; model: string }>;
     nics?: Array<{ name: string; mac: string; state: string }>;
-    root_device?: string; root_args?: string;
   }): Promise<{ status: string; error?: string }> {
     return this.request("POST", "/api/machines/discover", { body: data });
   }

bastion/src/cli/src/commands/app.ts

@@ -57,6 +57,7 @@ async function fetchState(): Promise<BastionState | null> {
 }
 
 import { registerLabcontrollerCommands } from "./labcontroller.js";
+import { isRole, SUPPORTED_ROLES, type Role } from "@lab/shared";
 
 export function registerAppCommand(program: Command): void {
   const appCmd = program.command("app").description("Application management");
@@ -66,18 +67,104 @@ export function registerAppCommand(program: Command): void {
 
   const k3sCmd = appCmd.command("k3s").description("k3s cluster management");
 
+  k3sCmd
+    .command("evacuate <target>")
+    .description("Cordon, evict Longhorn replicas and drain so a node can be safely wiped")
+    .option("--user <user>", "SSH user", "root")
+    .option(
+      "--allow-single-replica",
+      "permit evacuating volumes that have only one replica (rebuilds from the only copy; there is no backup target)",
+    )
+    .action(async (target: string, opts: { user: string; allowSingleReplica?: boolean }) => {
+      const state = await fetchState();
+      const resolved = resolveTarget(target, state);
+      if (!resolved) {
+        console.error(`Cannot resolve target: ${target}`);
+        process.exit(1);
+      }
+      const sshKey = findSshKey();
+
+      console.log(`Evacuating ${resolved.hostname} (${resolved.ip})...`);
+      console.log("This waits for Longhorn to rebuild replicas elsewhere and can take a while.\n");
+
+      const k3sMod = new K3sModule();
+      const result = await k3sMod.evacuate({
+        hostname: resolved.hostname,
+        ip: resolved.ip,
+        role: "infra",
+        os: "fedora-43" as const,
+        arch: "x86_64" as const,
+        sshUser: opts.user,
+        ...(sshKey ? { sshKeyPath: sshKey } : {}),
+        config: opts.allowSingleReplica ? { evictionAllowSingleReplica: true } : {},
+      });
+
+      for (const line of result.output) console.log(`  ${line}`);
+      if (!result.success) {
+        console.error(`\nNode is NOT safe to wipe (${result.errors.join(", ")}). Do not reprovision it.`);
+        console.error(`If an eviction was started and left pending, clear it with:`);
+        console.error(`  labctl app k3s admit ${resolved.hostname}`);
+        process.exit(1);
+      }
+      console.log(`\n${resolved.hostname} is evacuated. Safe to reprovision.`);
+    });
+
+  k3sCmd
+    .command("admit <target>")
+    .description("Return an evacuated node to service: uncordon, clear eviction, restore its Longhorn disk")
+    .option("--user <user>", "SSH user", "root")
+    .action(async (target: string, opts: { user: string }) => {
+      const state = await fetchState();
+      const resolved = resolveTarget(target, state);
+      if (!resolved) {
+        console.error(`Cannot resolve target: ${target}`);
+        process.exit(1);
+      }
+      const sshKey = findSshKey();
+
+      console.log(`Returning ${resolved.hostname} (${resolved.ip}) to service...`);
+
+      const k3sMod = new K3sModule();
+      const result = await k3sMod.admit({
+        hostname: resolved.hostname,
+        ip: resolved.ip,
+        role: isRole(resolved.role) ? resolved.role : "infra",
+        os: "fedora-43" as const,
+        arch: "x86_64" as const,
+        sshUser: opts.user,
+        ...(sshKey ? { sshKeyPath: sshKey } : {}),
+        config: {},
+      });
+
+      for (const line of result.output) console.log(`  ${line}`);
+      if (!result.success) {
+        console.error(`\n${resolved.hostname} is NOT fully back in service (${result.errors.join(", ")}).`);
+        process.exit(1);
+      }
+      console.log(`\n${resolved.hostname} is back in service.`);
+    });
+
   k3sCmd
     .command("install <target>")
     .description("Install k3s on a target machine (hostname, IP, or MAC)")
-    .option("--role <role>", "k3s role: infra (server) or worker (agent)", "infra")
+    .option(
+      "--role <role>",
+      "k3s role: infra, labcontroller, standalone-cluster, k8s-controller, k8s-combo (servers) or worker, k8s-worker (agents)",
+      "infra",
+    )
     .option("--user <user>", "SSH user", "root")
-    .option("--k3s-server <url>", "k3s server URL (required for worker role)")
-    .option("--k3s-token <token>", "k3s join token (required for worker role)")
+    .option("--k3s-server <url>", "k3s server URL (required for agent roles, and to JOIN as a server)")
+    .option("--k3s-token <token>", "k3s join token; on a first server it SEEDS the cluster token")
+    .option(
+      "--k3s-version <version>",
+      "exact k3s version, e.g. v1.34.6+k3s1. Unset installs LATEST, which on a rejoining node can be several minor versions ahead of the cluster",
+    )
     .action(async (target: string, opts: {
       role: string;
       user: string;
       k3sServer?: string;
       k3sToken?: string;
+      k3sVersion?: string;
     }) => {
       const state = await fetchState();
       const resolved = resolveTarget(target, state);
@@ -88,7 +175,16 @@ export function registerAppCommand(program: Command): void {
         process.exit(1);
       }
 
-      const role = opts.role === "worker" ? "worker" : "infra";
+      // Every supported role, not just infra/worker. The old mapping silently
+      // turned anything that was not "worker" into "infra" -- so
+      // `--role k8s-combo` installed a DIFFERENT role than the one asked for,
+      // with different disk expectations and no warning.
+      if (!isRole(opts.role)) {
+        console.error(`Unknown role: ${opts.role}`);
+        console.error(`Valid roles: ${SUPPORTED_ROLES.join(", ")}`);
+        process.exit(1);
+      }
+      const role: Role = opts.role;
       const sshKey = findSshKey();
 
       console.log(`Installing k3s on ${resolved.hostname} (${resolved.ip}) as ${role}...`);
@@ -106,9 +202,22 @@ export function registerAppCommand(program: Command): void {
         config: {
           ...(opts.k3sServer ? { k3sServerUrl: opts.k3sServer } : {}),
           ...(opts.k3sToken ? { k3sToken: opts.k3sToken } : {}),
+          ...(opts.k3sVersion ? { k3sVersion: opts.k3sVersion } : {}),
         },
       };
 
+      // Only warn when it matters: a node JOINING an existing cluster. On a
+      // first server there is nothing to be skewed against, and a warning that
+      // fires when it is irrelevant is a warning people learn to skip.
+      if (!opts.k3sVersion && opts.k3sServer) {
+        console.warn(
+          "  WARNING: no --k3s-version given, so get.k3s.io installs LATEST.\n" +
+          "  This node is JOINING an existing cluster — check what that cluster runs\n" +
+          "  (kubectl get nodes -o wide) and pin it. Kubernetes supports only ONE minor\n" +
+          "  version of skew between control-plane members.",
+        );
+      }
+
       const installResult = await k3s.install(moduleCtx);
       for (const line of installResult.output) {
         console.log(`  ${line}`);
@@ -281,7 +390,8 @@ export function registerAppCommand(program: Command): void {
       const healthResult = await k3s.health({
         hostname: resolved.hostname,
         ip: resolved.ip,
-        role: resolved.role,
+        // State is JSON on disk; narrow it rather than trusting the string.
+        role: isRole(resolved.role) ? resolved.role : "vanilla",
         os: "fedora-43" as const,
         arch: "x86_64" as const,
         sshUser: opts.user,

bastion/src/cli/src/commands/debug.ts

@@ -8,7 +8,6 @@ import { join } from "node:path";
 import { Command } from "commander";
 import type { BastionState } from "@lab/shared";
 import { getLabdClient } from "../api/config.js";
-import { ROOT_DEVICE_PROBE, parseRootProbe } from "../utils/hardware-probe.js";
 
 /** Resolve a target (hostname, MAC, or IP) to {mac, hostname, ip} from state. */
 function resolveTarget(
@@ -45,54 +44,6 @@ function resolveTarget(
   return null;
 }
 
-/** The local admin account to SSH as (root is not usable — it has no key here). */
-function sshUser(): string {
-  const adminUser = process.env["SUDO_USER"] ?? process.env["USER"] ?? "";
-  return adminUser === "root" ? "" : adminUser;
-}
-
-/** Common ssh arguments, ending with user@host. Null when there is no usable user. */
-function sshBaseArgs(ip: string): string[] | null {
-  const user = sshUser();
-  if (user === "") return null;
-
-  const sudoUser = process.env["SUDO_USER"];
-  const realHome = sudoUser !== undefined ? join("/home", sudoUser) : homedir();
-  const sshKey = ["id_ed25519", "id_rsa", "id_ecdsa"]
-    .map((name) => join(realHome, ".ssh", name))
-    .find((k) => existsSync(k));
-
-  return [
-    "-o", "StrictHostKeyChecking=no",
-    "-o", "UserKnownHostsFile=/dev/null",
-    "-o", "ConnectTimeout=10",
-    ...(sshKey !== undefined ? ["-i", sshKey] : []),
-    `${user}@${ip}`,
-  ];
-}
-
-/**
- * Run a shell script on the target as root and return its stdout, or null.
- *
- * The script goes over stdin rather than the command line so it can contain quotes
- * without a second round of shell escaping. `sudo -n` fails fast instead of hanging on
- * a password prompt that would then eat the script.
- */
-function sshCapture(ip: string, script: string): string | null {
-  const base = sshBaseArgs(ip);
-  if (base === null) return null;
-  try {
-    return execFileSync("ssh", [...base, "sudo", "-n", "sh", "-s"], {
-      input: script,
-      encoding: "utf-8",
-      stdio: ["pipe", "pipe", "pipe"],
-      timeout: 30_000,
-    });
-  } catch {
-    return null;
-  }
-}
-
 export function registerDebugCommand(parent: Command): void {
   parent
     .command("debug <target>")
@@ -120,31 +71,6 @@ export function registerDebugCommand(parent: Command): void {
       }
 
       const { mac, hostname, ip } = resolved;
-
-      // --pxe-boot needs a root= for the installed system. If the machine is still
-      // reachable, observe it now rather than assuming a disk layout: a wrong root=
-      // leaves the machine unbootable. If it isn't reachable, dispatch falls back to
-      // rescue and the operator reports the real one from there.
-      if (opts.pxeBoot === true && ip !== "") {
-        const known = state.installed[mac]?.root_device ?? state.discovered[mac]?.root_device;
-        if (known === undefined || known === "") {
-          console.log(`No root device recorded for ${hostname}. Probing over SSH...`);
-          const probe = sshCapture(ip, ROOT_DEVICE_PROBE);
-          const root = probe === null ? {} : parseRootProbe(probe);
-          if (root.root_device !== undefined) {
-            console.log(`  root=${root.root_device}${root.root_args !== undefined ? ` ${root.root_args}` : ""}`);
-            try {
-              await client.discoverMachine({ mac, ...root });
-            } catch (err) {
-              console.error(`  Could not record it: ${err instanceof Error ? err.message : String(err)}`);
-            }
-          } else {
-            console.log("  Probe failed. Booting rescue instead; report the root device with:");
-            console.log("    curl http://<bastion>:8080/debug-setup.sh | bash");
-          }
-        }
-      }
-
       console.log(`Queuing debug mode for ${hostname} (${mac})...`);
 
       try {
@@ -160,15 +86,32 @@ export function registerDebugCommand(parent: Command): void {
 
       // Try SSH reboot into PXE
       if (ip !== "") {
-        const base = sshBaseArgs(ip);
-        if (base !== null) {
-          console.log(`\nAttempting SSH reboot into PXE (${sshUser()}@${ip})...`);
+        const adminUser = process.env["SUDO_USER"] ?? process.env["USER"] ?? "";
+        const effectiveUser = adminUser === "root" ? "" : adminUser;
+
+        if (effectiveUser !== "") {
+          console.log(`\nAttempting SSH reboot into PXE (${effectiveUser}@${ip})...`);
+
+          const sudoUser = process.env["SUDO_USER"];
+          const realHome = sudoUser !== undefined ? join("/home", sudoUser) : homedir();
+          const keyPaths = [
+            join(realHome, ".ssh", "id_ed25519"),
+            join(realHome, ".ssh", "id_rsa"),
+            join(realHome, ".ssh", "id_ecdsa"),
+          ];
+          const sshKey = keyPaths.find(k => existsSync(k));
+
+          const sshArgs = [
+            "-o", "StrictHostKeyChecking=no",
+            "-o", "UserKnownHostsFile=/dev/null",
+            "-o", "ConnectTimeout=10",
+            ...(sshKey !== undefined ? ["-i", sshKey] : []),
+            `${effectiveUser}@${ip}`,
+            'PXE_ENTRY=$(sudo efibootmgr | grep -iE "pxe|network|ipv4" | head -1 | grep -oP "Boot\\K[0-9A-F]+"); if [ -n "$PXE_ENTRY" ]; then sudo efibootmgr --bootnext "$PXE_ENTRY" && echo "PXE set as next boot" && sudo reboot; else echo "No PXE boot entry found, rebooting anyway..." && sudo reboot; fi',
+          ];
 
           try {
-            execFileSync("ssh", [
-              ...base,
-              'PXE_ENTRY=$(sudo efibootmgr | grep -iE "pxe|network|ipv4" | head -1 | grep -oP "Boot\\K[0-9A-F]+"); if [ -n "$PXE_ENTRY" ]; then sudo efibootmgr --bootnext "$PXE_ENTRY" && echo "PXE set as next boot" && sudo reboot; else echo "No PXE boot entry found, rebooting anyway..." && sudo reboot; fi',
-            ], { stdio: "inherit" });
+            execFileSync("ssh", sshArgs, { stdio: "inherit" });
           } catch {
             // SSH connection closing during reboot is expected
           }

bastion/src/cli/src/commands/install.ts

@@ -1,10 +1,61 @@
 // CLI command: provision install
 // Queue a discovered machine for OS installation via labd.
 
-import { Command, Option } from "commander";
-import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY } from "@lab/shared";
+import { readFileSync } from "node:fs";
+import { Command, Option, InvalidArgumentError } from "commander";
+import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isRole, requiresSchemaV2, validateDataDisks } from "@lab/shared";
+import type { DataDisks, VyosBundle, VyosInstallSpec, VyosVlanSpec } from "@lab/shared";
 import { getLabdClient } from "../api/config.js";
 
+/**
+ * Load one router's config out of a Pulumi-rendered bundle.
+ *
+ * The bundle is produced by `kubernetes-deployment` (npm run vyos:bundle) and
+ * holds every router it manages, keyed by name. Selecting by hostname here is
+ * what keeps bring-up and `pulumi up` describing the same box: labctl replays
+ * the declared config rather than deriving its own.
+ */
+export function loadVyosBundle(path: string, hostname: string): VyosBundle {
+  let parsed: { version?: number; routers?: Record<string, VyosBundle> };
+  try {
+    parsed = JSON.parse(readFileSync(path, "utf8"));
+  } catch (e) {
+    throw new InvalidArgumentError(`Cannot read VyOS bundle ${path}: ${(e as Error).message}`);
+  }
+  if (parsed.version !== 1) {
+    throw new InvalidArgumentError(
+      `VyOS bundle ${path} has version ${parsed.version ?? "<none>"}; this labctl understands 1`,
+    );
+  }
+  const router = parsed.routers?.[hostname];
+  if (router === undefined) {
+    const known = Object.keys(parsed.routers ?? {}).join(", ") || "<none>";
+    throw new InvalidArgumentError(
+      `VyOS bundle ${path} has no entry for "${hostname}" (has: ${known})`,
+    );
+  }
+  return router;
+}
+
+/** Parse a repeated --vlan flag: "<id>:<cidr>[:<description>]". */
+export function parseVlan(value: string, previous: VyosVlanSpec[] = []): VyosVlanSpec[] {
+  const parts = value.split(":");
+  const id = Number(parts[0]);
+  const address = parts[1] ?? "";
+  // InvalidArgumentError makes commander print a clean message instead of
+  // dumping a stack trace at the operator.
+  if (!Number.isInteger(id) || id < 1 || id > 4094) {
+    throw new InvalidArgumentError(`Invalid VLAN id in "${value}" (expected 1-4094)`);
+  }
+  if (!address.includes("/")) {
+    throw new InvalidArgumentError(
+      `Invalid VLAN address in "${value}" (expected CIDR, e.g. 10.0.10.1/24)`,
+    );
+  }
+  const description = parts.slice(2).join(":");
+  return [...previous, { id, address, ...(description ? { description } : {}) }];
+}
+
 function roleTable(): string {
   const lines: string[] = ["", "Available roles:"];
   for (const r of ROLE_REGISTRY) {
@@ -15,6 +66,38 @@ function roleTable(): string {
   return lines.join("\n");
 }
 
+/** Parse a repeated --vlan-vip flag: "<id>:<cidr>" — VRRP VIP for a --vlan entry. */
+export function parseVlanVip(
+  value: string,
+  previous: Record<number, string> = {},
+): Record<number, string> {
+  const index = value.indexOf(":");
+  const id = Number(index === -1 ? Number.NaN : value.slice(0, index));
+  const cidr = index === -1 ? "" : value.slice(index + 1).trim();
+  if (!Number.isInteger(id) || id < 1 || id > 4094 || !cidr.includes("/")) {
+    throw new InvalidArgumentError(
+      `Invalid VLAN VIP "${value}" (expected <id>:<cidr>, e.g. 3:192.168.3.254/24)`,
+    );
+  }
+  return { ...previous, [id]: cidr };
+}
+
+/** Parse a repeated --vyos-hwid flag: "<iface>=<mac>". */
+export function parseHwId(
+  value: string,
+  previous: Record<string, string> = {},
+): Record<string, string> {
+  const index = value.indexOf("=");
+  const iface = index === -1 ? "" : value.slice(0, index).trim();
+  const mac = index === -1 ? "" : value.slice(index + 1).trim().toLowerCase();
+  if (iface === "" || !/^([0-9a-f]{2}:){5}[0-9a-f]{2}$/.test(mac)) {
+    throw new InvalidArgumentError(
+      `Invalid hw-id "${value}" (expected <iface>=<mac>, e.g. eth2=64:62:66:25:96:47)`,
+    );
+  }
+  return { ...previous, [iface]: mac };
+}
+
 export function registerInstallCommand(parent: Command): void {
   parent
     .command("install <mac> <hostname>")
@@ -23,11 +106,65 @@ export function registerInstallCommand(parent: Command): void {
     .addHelpText("after", roleTable())
     .addOption(new Option("--role <role>", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker"))
     .addOption(new Option("--os <os>", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43"))
-    .option("--disk <device>", "Target disk device (auto-detect if omitted)")
+    .option("--disk <device>", "Target OS disk: kernel name or stable /dev/disk/by-id/ path (auto-detect if omitted)")
+    .addOption(new Option("--schema-version <n>", "Disk layout for a FRESH install (default 1). Ignored on reprovision: the layout already on the disk wins.").choices(["1", "2"]))
+    .option(
+      "--longhorn-disk <by-id>",
+      "Dedicated whole disk for /var/lib/longhorn, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)",
+    )
+    .option(
+      "--rancher-disk <by-id>",
+      "Dedicated whole disk for /var/lib/rancher, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)",
+    )
+    .option("--vyos-mgmt <iface>", "VyOS: untagged interface the machine PXE boots from (default eth0)")
+    .option("--vyos-mgmt-address <addr>", "VyOS: CIDR for the management interface, or 'dhcp' (default dhcp)")
+    .option("--vyos-bond <ifaces>", "VyOS: comma-separated LACP bond members (must exclude the PXE NIC)")
+    .option("--vyos-bond-address <cidr>", "VyOS: address on the untagged bond (trunk native VLAN)")
+    .option("--vyos-bond-vrrp <cidr>", "VyOS: VRRP VIP floated on the untagged bond")
+    .option("--vlan-vip <id:cidr>", "VyOS: VRRP VIP for a --vlan entry (repeatable)", parseVlanVip)
+    .option("--vyos-vrrp-priority <n>", "VyOS: VRRP priority for all groups on this box (higher = master)")
+    .option("--vyos-mgmt-vlan <id:cidr[:desc]>", "VyOS: tagged management VLAN on the PXE port")
+    .option("--vlan <id:cidr[:desc]>", "VyOS: tagged VLAN sub-interface on the bond (repeatable)", parseVlan)
+    .option("--vyos-password <password>", "VyOS: password for the 'vyos' user")
+    .option("--vyos-hwid <iface=mac>", "VyOS: pin an interface name to a MAC via hw-id (repeatable)", parseHwId)
+    .option("--vyos-fresh-config", "VyOS: on reinstall, overwrite the preserved config with the generated one")
+    .option(
+      "--vyos-bundle <path>",
+      "VyOS: apply a Pulumi-rendered bundle verbatim (kubernetes-deployment/infra/vyos/vyos-bundle.json). " +
+        "Replaces the derived --vyos-bond/--vlan/... config; secret values are left unset for `pulumi up`.",
+    )
+    .option(
+      "--vyos-api-key <key>",
+      "VyOS: enable the HTTP API with this key so Pulumi can manage the router from first boot",
+    )
+    .option(
+      "--vyos-api-listen <addr>",
+      "VyOS: address the HTTP API binds to (default: the static management address). " +
+        "Required when management is DHCP; the API is never bound to all interfaces.",
+    )
     .action(async (mac: string, hostname: string, opts: {
       role: string;
       os: string;
+      /** commander gives us the string form of the --schema-version choice. */
+      schemaVersion?: string;
       disk?: string;
+      longhornDisk?: string;
+      rancherDisk?: string;
+      vyosMgmt?: string;
+      vyosMgmtAddress?: string;
+      vyosBond?: string;
+      vyosBondAddress?: string;
+      vyosBondVrrp?: string;
+      vlan?: VyosVlanSpec[];
+      vlanVip?: Record<number, string>;
+      vyosVrrpPriority?: string;
+      vyosMgmtVlan?: string;
+      vyosPassword?: string;
+      vyosHwid?: Record<string, string>;
+      vyosFreshConfig?: boolean;
+      vyosBundle?: string;
+      vyosApiKey?: string;
+      vyosApiListen?: string;
     }) => {
       if (!isValidOsId(opts.os)) {
         console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`);
@@ -39,18 +176,132 @@ export function registerInstallCommand(parent: Command): void {
         process.exit(1);
       }
 
+      // Dedicated data disks -- validated here for fast feedback, and again on
+      // the bastion, which is the enforcement point.
+      const dataDisks: DataDisks = {
+        ...(opts.longhornDisk !== undefined && opts.longhornDisk !== ""
+          ? { longhorn: opts.longhornDisk } : {}),
+        ...(opts.rancherDisk !== undefined && opts.rancherDisk !== ""
+          ? { rancher: opts.rancherDisk } : {}),
+      };
+      const hasDataDisks = Object.keys(dataDisks).length > 0;
+      if (hasDataDisks) {
+        if (opts.os !== "fedora-43") {
+          console.error(`--longhorn-disk/--rancher-disk only work with --os fedora-43 (got --os ${opts.os})`);
+          process.exit(1);
+        }
+        if (opts.schemaVersion === "1"
+            || (opts.schemaVersion === undefined && isRole(opts.role) && !requiresSchemaV2(opts.role))) {
+          console.error("--longhorn-disk/--rancher-disk need the v2 layout -- add --schema-version 2");
+          process.exit(1);
+        }
+        const problem = isRole(opts.role) ? validateDataDisks(opts.role, dataDisks) : null;
+        if (problem !== null) {
+          console.error(`Invalid data disks: ${problem}`);
+          process.exit(1);
+        }
+      }
+
+      const bondMembers = opts.vyosBond !== undefined && opts.vyosBond !== ""
+        ? opts.vyosBond.split(",").map((s) => s.trim()).filter((s) => s.length > 0)
+        : [];
+
+      // Attach --vlan-vip entries to their --vlan definitions. A VIP for a VLAN
+      // that was never defined is a typo that would otherwise vanish silently.
+      const vips = opts.vlanVip ?? {};
+      const vlans = (opts.vlan ?? []).map((v) =>
+        vips[v.id] !== undefined ? { ...v, vrrp: vips[v.id] as string } : v,
+      );
+      for (const id of Object.keys(vips)) {
+        if (!vlans.some((v) => String(v.id) === id)) {
+          console.error(`--vlan-vip ${id}:... has no matching --vlan ${id}:... entry`);
+          process.exit(1);
+        }
+      }
+
+      const vrrpPriority = opts.vyosVrrpPriority !== undefined && opts.vyosVrrpPriority !== ""
+        ? Number(opts.vyosVrrpPriority)
+        : undefined;
+      if (vrrpPriority !== undefined
+          && (!Number.isInteger(vrrpPriority) || vrrpPriority < 1 || vrrpPriority > 255)) {
+        console.error(`--vyos-vrrp-priority must be an integer 1-255 (got ${opts.vyosVrrpPriority})`);
+        process.exit(1);
+      }
+      const vyos: VyosInstallSpec = {
+        ...(opts.vyosMgmt !== undefined && opts.vyosMgmt !== ""
+          ? { mgmtInterface: opts.vyosMgmt } : {}),
+        ...(opts.vyosMgmtAddress !== undefined && opts.vyosMgmtAddress !== ""
+          ? { mgmtAddress: opts.vyosMgmtAddress } : {}),
+        ...(bondMembers.length > 0 ? { bondMembers } : {}),
+        ...(opts.vyosBondAddress !== undefined && opts.vyosBondAddress !== ""
+          ? { bondAddress: opts.vyosBondAddress } : {}),
+        ...(opts.vyosBondVrrp !== undefined && opts.vyosBondVrrp !== ""
+          ? { bondVrrp: opts.vyosBondVrrp } : {}),
+        ...(vrrpPriority !== undefined ? { vrrpPriority } : {}),
+        ...(vlans.length > 0 ? { vlans } : {}),
+        ...(opts.vyosPassword !== undefined && opts.vyosPassword !== ""
+          ? { password: opts.vyosPassword } : {}),
+        ...(opts.vyosHwid !== undefined && Object.keys(opts.vyosHwid).length > 0
+          ? { hwIds: opts.vyosHwid } : {}),
+        ...(opts.vyosMgmtVlan !== undefined && opts.vyosMgmtVlan !== ""
+          ? { mgmtVlan: parseVlan(opts.vyosMgmtVlan)[0] as VyosVlanSpec } : {}),
+        ...(opts.vyosFreshConfig === true ? { freshConfig: true } : {}),
+        ...(opts.vyosBundle !== undefined && opts.vyosBundle !== ""
+          ? { bundle: loadVyosBundle(opts.vyosBundle, hostname) } : {}),
+        ...(opts.vyosApiKey !== undefined && opts.vyosApiKey !== ""
+          ? { apiKey: opts.vyosApiKey } : {}),
+        ...(opts.vyosApiListen !== undefined && opts.vyosApiListen !== ""
+          ? { apiListenAddress: opts.vyosApiListen } : {}),
+      };
+      const hasVyosOptions = Object.keys(vyos).length > 0;
+
+      // A bundle already describes the whole router. Accepting derived topology
+      // flags alongside it would silently discard them (the bundle wins in
+      // buildVyosConfigSpec), so say so rather than appear to honour both.
+      if (vyos.bundle !== undefined) {
+        const derived = ["mgmtInterface", "mgmtAddress", "bondMembers", "bondAddress",
+                         "bondVrrp", "vrrpPriority", "vlans", "mgmtVlan"] as const;
+        const conflicting = derived.filter((k) => vyos[k] !== undefined);
+        if (conflicting.length > 0) {
+          console.error(
+            `--vyos-bundle describes the whole router; these would be ignored: ${conflicting.join(", ")}`,
+          );
+          console.error("Remove them, or change the bundle in kubernetes-deployment and re-render.");
+          process.exit(1);
+        }
+      }
+
+      if (hasVyosOptions && !opts.os.startsWith("vyos")) {
+        console.error(`VyOS options require --os vyos-rolling (got --os ${opts.os})`);
+        process.exit(1);
+      }
+
+      // Firmware PXE cannot run over LACP, so the NIC that boots the installer
+      // must stay out of the bond — otherwise the next reinstall has no path in.
+      const mgmt = vyos.mgmtInterface ?? "eth0";
+      if (bondMembers.includes(mgmt)) {
+        console.error(`--vyos-bond must not include the PXE/management interface "${mgmt}"`);
+        console.error("PXE cannot boot over an LACP bond; keep that NIC unbonded.");
+        process.exit(1);
+      }
+
       try {
         const result = await getLabdClient().installMachine({
+          ...(opts.schemaVersion ? { schemaVersion: Number(opts.schemaVersion) as 1 | 2 } : {}),
           mac,
           hostname,
           role: opts.role,
           os: opts.os,
           ...(opts.disk ? { disk: opts.disk } : {}),
+          ...(hasDataDisks ? { dataDisks } : {}),
+          ...(hasVyosOptions ? { vyos } : {}),
         });
 
         console.log(JSON.stringify(result, null, 2));
         console.log("");
-        const osLabel = opts.os.startsWith("ubuntu") ? "Ubuntu" : "Fedora";
+        const osLabel = opts.os.startsWith("ubuntu")
+          ? "Ubuntu"
+          : opts.os.startsWith("vyos") ? "VyOS" : "Fedora";
         console.log(`Power on the machine to start ${osLabel} installation.`);
 
         const roleInfo = ROLE_REGISTRY.find(r => r.name === opts.role);

bastion/src/cli/src/commands/recheck.ts

@@ -4,7 +4,6 @@
 import type { Command } from "commander";
 import { sshExec } from "@lab/modules";
 import { getLabdClient } from "../api/config.js";
-import { ROOT_DEVICE_PROBE } from "../utils/hardware-probe.js";
 
 const BOLD = "\x1b[1m";
 const GREEN = "\x1b[0;32m";
@@ -25,9 +24,7 @@ const HW_COLLECT_SCRIPT = [
   'N=$(grep -c "^processor" /proc/cpuinfo 2>/dev/null || echo 0)',
   'R=$(awk "/MemTotal/ {printf \\"%d\\", \\$2/1024/1024}" /proc/meminfo 2>/dev/null || echo 0)',
   'A=$(uname -m)',
-  // Root filesystem, so --pxe-boot has a root= to use instead of assuming our layout.
-  ROOT_DEVICE_PROBE,
-  'printf \'{"product":"%s","board":"%s","serial":"%s","manufacturer":"%s","cpu_model":"%s","cpu_cores":%s,"memory_gb":%s,"arch":"%s","root_device":"%s","root_args":"%s"}\\n\' "$P" "$B" "$S" "$M" "$C" "$N" "$R" "$A" "$RD" "$RA"',
+  'printf \'{"product":"%s","board":"%s","serial":"%s","manufacturer":"%s","cpu_model":"%s","cpu_cores":%s,"memory_gb":%s,"arch":"%s"}\\n\' "$P" "$B" "$S" "$M" "$C" "$N" "$R" "$A"',
 ].join("; ");
 
 export function registerRecheckCommand(parent: Command): void {
@@ -47,11 +44,14 @@ export function registerRecheckCommand(parent: Command): void {
       }
 
       // Build list of machines to check
-      const targets: Array<{ mac: string; hostname: string; ip: string }> = [];
+      const targets: Array<{ mac: string; hostname: string; ip: string; sshUser: string }> = [];
+      const userIsDefault = opts.user === "root";
       for (const [mac, info] of Object.entries(state.installed)) {
         if (!info.ip) continue;
         if (opts.target && info.hostname !== opts.target && mac !== opts.target) continue;
-        targets.push({ mac, hostname: info.hostname, ip: info.ip });
+        // VyOS boxes only have the "vyos" login; honor an explicit --user.
+        const sshUser = userIsDefault && (info.os ?? "").startsWith("vyos") ? "vyos" : opts.user;
+        targets.push({ mac, hostname: info.hostname, ip: info.ip, sshUser });
       }
 
       if (targets.length === 0) {
@@ -64,12 +64,12 @@ export function registerRecheckCommand(parent: Command): void {
       let updated = 0;
       let failed = 0;
 
-      for (const { mac, hostname, ip } of targets) {
+      for (const { mac, hostname, ip, sshUser } of targets) {
         process.stdout.write(`  ${hostname.padEnd(24)} ${DIM}(${ip})${RESET}  `);
 
         try {
           const t0 = Date.now();
-          const result = await sshExec(ip, opts.user, HW_COLLECT_SCRIPT, SSH_OPTS);
+          const result = await sshExec(ip, sshUser, HW_COLLECT_SCRIPT, SSH_OPTS);
           const elapsed = Date.now() - t0;
           if (result.exitCode !== 0) {
             console.log(`${RED}SSH failed (exit ${result.exitCode}, ${elapsed}ms)${RESET}`);
@@ -84,10 +84,7 @@ export function registerRecheckCommand(parent: Command): void {
           const cpu = hwData.cpu_model || "?";
           const cores = hwData.cpu_cores || "?";
           const mem = hwData.memory_gb || "?";
-          const root = typeof hwData.root_device === "string" && hwData.root_device !== ""
-            ? `, root=${hwData.root_device}`
-            : "";
-          console.log(`${GREEN}OK${RESET} ${DIM}${cpu}, ${cores} cores, ${mem}GB${root}${RESET}`);
+          console.log(`${GREEN}OK${RESET} ${DIM}${cpu}, ${cores} cores, ${mem}GB${RESET}`);
           updated++;
         } catch (err) {
           console.log(`${RED}FAIL${RESET} ${DIM}${err instanceof Error ? err.message : String(err)}${RESET}`);

bastion/src/cli/src/commands/reprovision.ts

@@ -24,12 +24,12 @@ function roleTable(): string {
 function resolveTarget(
   target: string,
   state: BastionState,
-): { mac: string; hostname: string; ip: string } | null {
+): { mac: string; hostname: string; ip: string; os?: string } | null {
   const normalized = target.toLowerCase().replace(/-/g, ":");
 
   if (state.installed[normalized]) {
     const info = state.installed[normalized];
-    return { mac: normalized, hostname: info.hostname, ip: info.ip };
+    return { mac: normalized, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
   }
 
   if (state.discovered[normalized]) {
@@ -38,13 +38,13 @@ function resolveTarget(
 
   for (const [mac, info] of Object.entries(state.installed)) {
     if (info.hostname === target || info.hostname.startsWith(target + ".")) {
-      return { mac, hostname: info.hostname, ip: info.ip };
+      return { mac, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
     }
   }
 
   for (const [mac, info] of Object.entries(state.installed)) {
     if (info.ip === target) {
-      return { mac, hostname: info.hostname, ip: info.ip };
+      return { mac, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
     }
   }
 
@@ -60,10 +60,12 @@ export function registerReprovisionCommand(parent: Command): void {
     .addOption(new Option("--role <role>", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker"))
     .addOption(new Option("--os <os>", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43"))
     .option("--disk <device>", "Target disk device (auto-detect if omitted)")
+    .option("--user <user>", "SSH user for the reboot (default: vyos for VyOS machines, else current user)")
     .action(async (target: string, hostnameOverride: string | undefined, opts: {
       role: string;
       os: string;
       disk?: string;
+      user?: string;
     }) => {
       if (!isValidOsId(opts.os)) {
         console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`);
@@ -123,7 +125,11 @@ export function registerReprovisionCommand(parent: Command): void {
         return;
       }
 
-      const adminUser = process.env["SUDO_USER"] ?? process.env["USER"] ?? "";
+      // SSH user: explicit flag > the machine's current OS (VyOS boxes only
+      // have the "vyos" login) > the invoking user.
+      const currentOsIsVyos = (resolved.os ?? "").startsWith("vyos");
+      const adminUser = opts.user
+        ?? (currentOsIsVyos ? "vyos" : (process.env["SUDO_USER"] ?? process.env["USER"] ?? ""));
       const effectiveUser = adminUser === "root" ? "" : adminUser;
 
       if (effectiveUser === "") {

bastion/src/cli/src/utils/hardware-probe.ts

@@ -1,59 +0,0 @@
-// Shell snippets for observing a machine's hardware over SSH.
-//
-// Pure shell + awk, no Python: these run on whatever the target happens to be,
-// including a minimal rescue environment.
-
-/**
- * Report the root filesystem and any dracut arguments needed to assemble it.
- *
- * Emits two lines:
- *   ROOT_DEVICE=<device>
- *   ROOT_ARGS=<args>
- *
- * Used by `--pxe-boot`, which boots the installed system with a kernel and initrd from
- * the network. Getting root= wrong there leaves the machine unbootable, so this observes
- * the machine rather than assuming our Fedora LVM layout.
- *
- * Device form is chosen for stability across reboots: LVM logical volumes keep their
- * /dev/mapper path, anything else is reported by UUID, which survives device renumbering.
- */
-export const ROOT_DEVICE_PROBE = [
-  'RD=$(findmnt -no SOURCE / 2>/dev/null | head -1)',
-  'RA=""',
-  'RT=$(lsblk -no TYPE "$RD" 2>/dev/null | head -1)',
-  'if [ "$RT" = "lvm" ]; then',
-  '  VGLV=$(lvs --noheadings -o vg_name,lv_name "$RD" 2>/dev/null | awk \'{print $1"/"$2}\')',
-  '  [ -n "$VGLV" ] && RA="rd.lvm.lv=$VGLV"',
-  // Swap must be assembled too or resume= stalls the boot waiting for it.
-  '  SW=$(awk \'NR>1 {print $1; exit}\' /proc/swaps 2>/dev/null)',
-  '  if [ -n "$SW" ] && [ "$(lsblk -no TYPE "$SW" 2>/dev/null | head -1)" = "lvm" ]; then',
-  '    SWVGLV=$(lvs --noheadings -o vg_name,lv_name "$SW" 2>/dev/null | awk \'{print $1"/"$2}\')',
-  '    [ -n "$SWVGLV" ] && [ "$SWVGLV" != "$VGLV" ] && RA="$RA rd.lvm.lv=$SWVGLV"',
-  '  fi',
-  'elif [ -n "$RD" ]; then',
-  '  U=$(findmnt -no UUID / 2>/dev/null | head -1)',
-  '  [ -n "$U" ] && RD="UUID=$U"',
-  'fi',
-  'printf \'ROOT_DEVICE=%s\\nROOT_ARGS=%s\\n\' "$RD" "$RA"',
-].join("; ");
-
-export interface RootInfo {
-  root_device?: string;
-  root_args?: string;
-}
-
-/** Parse the ROOT_DEVICE/ROOT_ARGS lines emitted by ROOT_DEVICE_PROBE. */
-export function parseRootProbe(stdout: string): RootInfo {
-  const out: RootInfo = {};
-  for (const line of stdout.split("\n")) {
-    const trimmed = line.trim();
-    if (trimmed.startsWith("ROOT_DEVICE=")) {
-      const v = trimmed.slice("ROOT_DEVICE=".length).trim();
-      if (v !== "") out.root_device = v;
-    } else if (trimmed.startsWith("ROOT_ARGS=")) {
-      const v = trimmed.slice("ROOT_ARGS=".length).trim();
-      if (v !== "") out.root_args = v;
-    }
-  }
-  return out;
-}

bastion/src/cli/tests/install-vyos.test.ts

@@ -0,0 +1,35 @@
+// Tests for VyOS install option parsing.
+
+import { describe, it, expect } from "vitest";
+import { parseVlan } from "../src/commands/install.js";
+
+describe("parseVlan", () => {
+  it("parses id and CIDR", () => {
+    expect(parseVlan("10:10.0.10.1/24")).toEqual([{ id: 10, address: "10.0.10.1/24" }]);
+  });
+
+  it("accumulates across repeated flags", () => {
+    const first = parseVlan("10:10.0.10.1/24");
+    const both = parseVlan("20:10.0.20.1/24", first);
+    expect(both).toHaveLength(2);
+    expect(both[1]).toEqual({ id: 20, address: "10.0.20.1/24" });
+  });
+
+  it("keeps a description, including one containing colons", () => {
+    expect(parseVlan("30:10.0.30.1/24:mgmt:secondary")).toEqual([
+      { id: 30, address: "10.0.30.1/24", description: "mgmt:secondary" },
+    ]);
+  });
+
+  it("rejects an address that is not CIDR", () => {
+    // A bare address would produce a VyOS config that fails to commit on first
+    // boot, long after the operator has stopped watching.
+    expect(() => parseVlan("10:10.0.10.1")).toThrow(/CIDR/);
+  });
+
+  it("rejects out-of-range and non-numeric VLAN ids", () => {
+    expect(() => parseVlan("0:10.0.10.1/24")).toThrow(/1-4094/);
+    expect(() => parseVlan("4095:10.0.10.1/24")).toThrow(/1-4094/);
+    expect(() => parseVlan("abc:10.0.10.1/24")).toThrow(/1-4094/);
+  });
+});

bastion/src/cli/tests/smoke-bastion.test.ts

@@ -143,7 +143,10 @@ describe("bastion smoke tests", () => {
       await sleep(200);
     }
 
-    expect(stdout).toContain("Waiting for PXE boot requests");
+    // On failure, show what the process actually said -- an empty stdout with
+    // a hidden stderr is undiagnosable from a CI log.
+    expect(stdout, `bastion never printed its banner; stderr was:\n${stderr}`)
+      .toContain("Waiting for PXE boot requests");
     expect(stdout).toContain("HTTP server listening");
 
     // Verify the process is still alive after startup

bastion/src/labd/src/routes/bastions.ts

@@ -10,6 +10,7 @@ import type { FastifyInstance } from "fastify";
 import type { DbClient } from "../server.js";
 import { bastionRegistry } from "../services/bastion-registry.js";
 import { generateRequestId } from "@lab/shared";
+import type { VyosInstallSpec } from "@lab/shared";
 
 const COMMAND_TIMEOUT_MS = 15_000;
 
@@ -163,9 +164,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
 
   // Queue install — route to correct bastion by MAC
   app.post<{
-    Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string };
+    Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string; schemaVersion?: 1 | 2; dataDisks?: import("@lab/shared").DataDisks; vyos?: VyosInstallSpec };
   }>("/api/machines/install", async (request, reply) => {
-    const { mac, hostname, disk, role, os } = request.body ?? {};
+    const { mac, hostname, disk, role, os, schemaVersion, dataDisks, vyos } = request.body ?? {};
     if (!mac || !hostname) {
       return reply.code(400).send({ error: "mac and hostname are required" });
     }
@@ -183,6 +184,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
           const result = await sendCommand(all[0]!.bastionId, {
             type: "command-install",
             mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
+            ...(schemaVersion ? { schemaVersion } : {}),
+            ...(dataDisks ? { dataDisks } : {}),
+            ...(vyos ? { vyos } : {}),
           });
           return reply.code(result.status === "ok" ? 200 : 500).send(result);
         } catch (err) {
@@ -196,6 +200,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
       const result = await sendCommand(bastion.bastionId, {
         type: "command-install",
         mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
+        ...(schemaVersion ? { schemaVersion } : {}),
+        ...(dataDisks ? { dataDisks } : {}),
+        ...(vyos ? { vyos } : {}),
       });
       return reply.code(result.status === "ok" ? 200 : 500).send(result);
     } catch (err) {
@@ -299,7 +306,6 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
       memory_gb?: number; arch?: string;
       disks?: Array<{ name: string; size_gb: number; model: string }>;
       nics?: Array<{ name: string; mac: string; state: string }>;
-      root_device?: string; root_args?: string;
     };
   }>("/api/machines/discover", async (request, reply) => {
     const data = request.body ?? {};

bastion/src/modules/modules/k3s/bin/render-config.ts

@@ -0,0 +1,42 @@
+#!/usr/bin/env node
+// Render /etc/rancher/k3s/config.yaml using the PRODUCTION generator.
+//
+// Exists so labsim (and anything else) can produce the exact config.yaml a node
+// would get from `labctl install`, without an SSH/OperationContext. Driving the
+// rehearsal through this means the sim tests the same code path production runs
+// -- if generateServerConfig ever changes shape, the sim moves with it.
+//
+// All input via env, so a sim's cloud-init or a shell can call it plainly:
+//
+//   ROLE=infra HOSTNAME=k8s1 IP=172.31.2.11 render-config.ts        # cluster-init server
+//   ROLE=infra HOSTNAME=k8s2 IP=172.31.2.12 \
+//     K3S_SERVER_URL=https://172.31.2.11:6443 K3S_TOKEN=... render-config.ts   # joining server
+//   ROLE=worker HOSTNAME=k8s4 IP=172.31.2.14 K3S_SERVER_URL=... K3S_TOKEN=... render-config.ts  # agent
+//
+// Dual-stack is opt-in and matches K3sConfig exactly: set IPV6, CLUSTER_CIDR,
+// SERVICE_CIDR (comma-separated families) and the generator emits the dual
+// node-ip + CIDRs. Omit them and the output is byte-identical to a v4-only node.
+import { generateServerConfig, generateAgentConfig } from "../src/operations/k3s-config.js";
+import type { K3sConfig } from "../src/types.js";
+import type { Role } from "@lab/shared";
+
+const env = process.env;
+const role = (env.ROLE ?? "worker") as Role;
+const isServer = role === "infra" || role === "labcontroller";
+
+const splitCsv = (v: string | undefined): string[] | undefined =>
+  v ? v.split(",").map((s) => s.trim()).filter(Boolean) : undefined;
+
+const cfg: K3sConfig = {
+  hostname: env.HOSTNAME ?? "node",
+  ip: env.IP ?? (() => { throw new Error("IP is required"); })(),
+  role,
+  k3sServerUrl: env.K3S_SERVER_URL,
+  k3sToken: env.K3S_TOKEN,
+  tlsSans: splitCsv(env.TLS_SANS),
+  ipv6: env.IPV6,
+  clusterCidr: splitCsv(env.CLUSTER_CIDR),
+  serviceCidr: splitCsv(env.SERVICE_CIDR),
+};
+
+process.stdout.write(isServer ? generateServerConfig(cfg) : generateAgentConfig(cfg));

bastion/src/modules/modules/k3s/src/groups/admit.ts

@@ -0,0 +1,31 @@
+// Return an evacuated node to service. The inverse of the evacuate group.
+//
+// Needed for two different journeys that end in the same place:
+//   - a node that was wiped, reinstalled and has just rejoined the cluster;
+//   - a node whose evacuation was aborted and must simply be put back.
+//
+// Order matters here too. configureLonghornDisk runs AFTER eviction is cleared,
+// because a node that is still marked for eviction will have any disk we give
+// it immediately drained again.
+
+import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
+import { runSequential } from "../utils.js";
+import { uncordonNode } from "../operations/node-evacuate.js";
+import { clearLonghornEviction, assertNodeSchedulable } from "../operations/node-admit.js";
+import { configureLonghornDisk } from "../operations/longhorn-disk.js";
+
+export const admitGroup: OperationGroup = {
+  name: "admit",
+  description: "Uncordon a node, clear its eviction request and restore its Longhorn disk",
+  operations: [
+    { name: "Uncordon node", fn: uncordonNode },
+    { name: "Clear Longhorn eviction", fn: clearLonghornEviction },
+    { name: "Restore Longhorn default disk", fn: configureLonghornDisk },
+    { name: "Verify node is back in service", fn: assertNodeSchedulable },
+  ],
+};
+
+export async function runAdmit(ctx: OperationContext): Promise<OperationResult[]> {
+  ctx.log("Returning node to service...");
+  return runSequential(ctx, admitGroup.operations);
+}

bastion/src/modules/modules/k3s/src/groups/evacuate.ts

@@ -0,0 +1,54 @@
+// Evacuate a node so it can be wiped and reinstalled with a different layout.
+//
+// The ORDER is the operation:
+//
+//   1. Cordon      stop new work landing on a node that is about to leave.
+//   2. Check       can eviction even finish? Seconds to answer, an hour to
+//                  discover the hard way.
+//   3. Evict       move the replicas and WAIT for them to rebuild elsewhere.
+//   4. Drain       now move the workloads.
+//   5. Verify      re-read the cluster; do not trust that 1-4 worked.
+//
+// EVICT BEFORE DRAIN, which is the opposite of the obvious order and the one
+// thing here most likely to be "tidied" back into a bug. `drain --force` does
+// not override PodDisruptionBudgets, and longhorn-manager holds a PDB on the
+// instance-manager pod for as long as volumes are attached or degraded. Drain
+// first and you stop the instance-manager that serves the replicas you are
+// about to evict: the drain blocks for its full 15-minute timeout, and if it
+// did succeed it would have removed the only source a single-replica volume
+// could rebuild from.
+//
+// Removing the etcd member is NOT part of this group -- it costs you the API
+// server the rest of the procedure needs, so it belongs immediately before the
+// wipe, not here.
+//
+// This group deliberately does not reinstall anything. It leaves the node
+// cordoned, drained and empty, and a human (or `labctl provision reprovision`)
+// takes it from there.
+
+import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
+import { runSequential } from "../utils.js";
+import {
+  cordonNode,
+  drainNode,
+  evictLonghornReplicas,
+  assertEvictionCompletable,
+  assertNodeEvacuated,
+} from "../operations/node-evacuate.js";
+
+export const evacuateGroup: OperationGroup = {
+  name: "evacuate",
+  description: "Cordon, evict storage from and drain a node so it can be safely wiped",
+  operations: [
+    { name: "Cordon node", fn: cordonNode },
+    { name: "Check eviction can complete", fn: assertEvictionCompletable },
+    { name: "Evict Longhorn replicas (waits for rebuild)", fn: evictLonghornReplicas },
+    { name: "Drain workloads", fn: drainNode },
+    { name: "Verify node is empty", fn: assertNodeEvacuated },
+  ],
+};
+
+export async function runEvacuate(ctx: OperationContext): Promise<OperationResult[]> {
+  ctx.log("Evacuating node before wipe...");
+  return runSequential(ctx, evacuateGroup.operations);
+}

bastion/src/modules/modules/k3s/src/groups/host-prep.ts

@@ -1,21 +1,23 @@
-// Host preparation: kernel modules, sysctl, swap, firewall, SELinux.
+// Host preparation: kernel modules, sysctl, swap, storage, firewall, SELinux.
 
 import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
 import { runSequential } from "../utils.js";
 import { loadKernelModules } from "../operations/kernel-modules.js";
 import { applyCisHardening } from "../operations/sysctl.js";
-import { disableSwap } from "../operations/swap.js";
+import { enableSwap } from "../operations/swap.js";
+import { growRancherLv } from "../operations/rancher-storage.js";
 import { disableFirewall } from "../operations/firewall.js";
 import { setSelinuxPermissive } from "../operations/selinux.js";
 import { enableIscsi } from "../operations/iscsi.js";
 
 export const hostPrepGroup: OperationGroup = {
   name: "host-prep",
-  description: "Prepare host for k3s: kernel modules, sysctl, swap, firewall, SELinux, iSCSI",
+  description: "Prepare host for k3s: kernel modules, sysctl, swap, imageFs sizing, firewall, SELinux, iSCSI",
   operations: [
     { name: "Load kernel modules", fn: loadKernelModules },
     { name: "Apply CIS sysctl", fn: applyCisHardening },
-    { name: "Disable swap", fn: disableSwap },
+    { name: "Enable swap", fn: enableSwap },
+    { name: "Grow rancher LV", fn: growRancherLv },
     { name: "Disable firewall", fn: disableFirewall },
     { name: "Set SELinux permissive", fn: setSelinuxPermissive },
     { name: "Enable iSCSI", fn: enableIscsi },

bastion/src/modules/modules/k3s/src/groups/k3s-agent.ts

@@ -4,6 +4,7 @@ import type { OperationContext, OperationResult, OperationGroup } from "../types
 import { runSequential } from "../utils.js";
 import { writeK3sConfig } from "../operations/k3s-config.js";
 import { installK3sBinary } from "../operations/k3s-install.js";
+import { labelWorkerRole } from "../operations/worker-label.js";
 
 export const k3sAgentGroup: OperationGroup = {
   name: "k3s-agent",
@@ -11,6 +12,8 @@ export const k3sAgentGroup: OperationGroup = {
   operations: [
     { name: "Write k3s config", fn: writeK3sConfig },
     { name: "Install k3s agent", fn: installK3sBinary },
+    // After registration, never before: the kubelet may not set this itself.
+    { name: "Label node as worker", fn: labelWorkerRole },
   ],
 };
 

bastion/src/modules/modules/k3s/src/health/k3s-service.ts

@@ -1,8 +1,9 @@
 import type { Operation, OperationResult } from "../types.js";
 import { isServiceActive } from "../utils.js";
+import { isK3sServer } from "@lab/shared";
 
 export const checkK3sService: Operation = async (ctx): Promise<OperationResult> => {
-  const isServer = ctx.config.role === "infra" || ctx.config.role === "labcontroller";
+  const isServer = isK3sServer(ctx.config.role);
   const service = isServer ? "k3s" : "k3s-agent";
   const active = await isServiceActive(ctx, service);
   return { success: active, changed: false, message: active ? `${service} is active` : `${service} is not active` };

bastion/src/modules/modules/k3s/src/install.ts

@@ -1,17 +1,21 @@
 // k3s module — install phase.
 // Installs k3s with CIS-hardened configuration and Cilium CNI.
 
+import { isK3sServer, type Role } from "@lab/shared";
+
 export interface K3sInstallContext {
   hostname: string;
   ip: string;
-  role: string;   // "infra" = server, "worker" = agent
+  role: Role;     // server vs agent is derived via isK3sServer(), never by name
   k3sServerUrl?: string;  // Required for agent role
   k3sToken?: string;       // Required for agent role
 }
 
 /** Generate the shell script that installs k3s on a target machine. */
 export function generateInstallScript(ctx: K3sInstallContext): string {
-  const isServer = ctx.role === "infra";
+  // Was `ctx.role === "infra"`, which silently treated a `labcontroller` -- a
+  // role that runs a k3s SERVER -- as an agent. Derived from the registry now.
+  const isServer = isK3sServer(ctx.role);
 
   return `#!/bin/bash
 set -euo pipefail
@@ -215,7 +219,9 @@ echo "  Using network device: \$DEFAULT_DEV"
 
 KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install \\
   --set kubeProxyReplacement=true \\
-  --set ipam.mode=kubernetes \\
+  --set ipam.mode=cluster-pool \\
+  --set ipam.operator.clusterPoolIPv4PodCIDRList='{10.42.0.0/16}' \\
+  --set ipam.operator.clusterPoolIPv4MaskSize=24 \\
   --set devices="\$DEFAULT_DEV" \\
   --set nodePort.directRoutingDevice="\$DEFAULT_DEV"
 

bastion/src/modules/modules/k3s/src/k3s-module.ts

@@ -6,21 +6,35 @@ import type { OperationContext, K3sConfig, OperationResult } from "./types.js";
 import { sshExec } from "../../../src/ssh.js";
 import { aggregateResults } from "./utils.js";
 import { runHostPrep } from "./groups/host-prep.js";
+import { runEvacuate } from "./groups/evacuate.js";
+import { runAdmit } from "./groups/admit.js";
 import { runK3sServer } from "./groups/k3s-server.js";
 import { runK3sAgent } from "./groups/k3s-agent.js";
 import { runNetworking } from "./groups/networking.js";
 import { runHardening } from "./groups/hardening.js";
 import { runSequential } from "./utils.js";
 import * as health from "./health/index.js";
+import { isK3sServer } from "@lab/shared";
 
 function toOpContext(ctx: ModuleContext): OperationContext {
+  // Pass the WHOLE config bag through, then pin the fields that come from the
+  // ModuleContext itself.
+  //
+  // This used to enumerate three keys -- k3sServerUrl, k3sToken, tlsSans -- and
+  // silently drop every other one. Anything a caller set beyond those three
+  // vanished with no error and no warning: k3sVersion (so a pinned version was
+  // ignored and get.k3s.io installed LATEST), evictionAllowSingleReplica (so
+  // `labctl app k3s evacuate --allow-single-replica` was a no-op that claimed
+  // to override a data-loss guard), and the dual-stack fields ipv6 /
+  // clusterCidr / serviceCidr.
+  //
+  // Enumerating is the bug: every field added to K3sConfig has to be
+  // remembered here, and forgetting produces silence rather than a type error.
   const config: K3sConfig = {
+    ...(ctx.config as Partial<K3sConfig>),
     hostname: ctx.hostname,
     ip: ctx.ip,
     role: ctx.role as K3sConfig["role"],
-    k3sServerUrl: ctx.config["k3sServerUrl"] as string | undefined,
-    k3sToken: ctx.config["k3sToken"] as string | undefined,
-    tlsSans: ctx.config["tlsSans"] as string[] | undefined,
   };
   return {
     config,
@@ -35,7 +49,16 @@ function toOpContext(ctx: ModuleContext): OperationContext {
     },
     os: ctx.os,
     arch: ctx.arch,
-    log: (_msg) => { /* collected via results */ },
+    // Emit progress LIVE, on stderr.
+    //
+    // This used to be a no-op "collected via results", but results only carry
+    // an operation's final message. Anything an operation reports WHILE it
+    // runs was therefore thrown away -- including evictLonghornReplicas's
+    // per-poll "N replica(s) still on <node>", which is the only visibility
+    // into an eviction that can legitimately take an hour. When one stalled,
+    // there was no way to tell which volumes were stuck or whether progress
+    // was being made at all.
+    log: (msg) => { console.error(`  [${ctx.hostname}] ${msg}`); },  // eslint-disable-line no-console
   };
 }
 
@@ -62,7 +85,7 @@ export class K3sModule implements Module {
   async install(ctx: ModuleContext): Promise<ModuleResult> {
     const start = performance.now();
     const opCtx = toOpContext(ctx);
-    const isServer = ctx.role === "infra" || ctx.role === "labcontroller";
+    const isServer = isK3sServer(ctx.role);
 
     // Phase 1: Host preparation
     const prepResults = await runHostPrep(opCtx);
@@ -95,6 +118,31 @@ export class K3sModule implements Module {
     return toModuleResult("configure", results, start);
   }
 
+  /**
+   * Evacuate the node so it can be wiped. Does NOT reinstall anything --
+   * it leaves the node cordoned, drained and empty.
+   */
+  async evacuate(ctx: ModuleContext): Promise<ModuleResult> {
+    const start = performance.now();
+    const opCtx = toOpContext(ctx);
+    const results = await runEvacuate(opCtx);
+    return toModuleResult("evacuate", results, start);
+  }
+
+  /**
+   * Return an evacuated node to service: uncordon, clear Longhorn's eviction
+   * request, restore its default disk.
+   *
+   * The counterpart to evacuate(), and the only supported way out of an
+   * evacuation that was aborted partway.
+   */
+  async admit(ctx: ModuleContext): Promise<ModuleResult> {
+    const start = performance.now();
+    const opCtx = toOpContext(ctx);
+    const results = await runAdmit(opCtx);
+    return toModuleResult("admit", results, start);
+  }
+
   async health(ctx: ModuleContext): Promise<ModuleResult> {
     const start = performance.now();
     const opCtx = toOpContext(ctx);

bastion/src/modules/modules/k3s/src/operations/cilium.ts

@@ -47,7 +47,9 @@ export const installCilium: Operation = async (ctx): Promise<OperationResult> =>
   const installResult = await ctx.ssh.exec(
     `KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install \
       --set kubeProxyReplacement=true \
-      --set ipam.mode=kubernetes \
+      --set ipam.mode=cluster-pool \
+      --set ipam.operator.clusterPoolIPv4PodCIDRList='{10.42.0.0/16}' \
+      --set ipam.operator.clusterPoolIPv4MaskSize=24 \
       --set k8sServiceHost=127.0.0.1 \
       --set k8sServicePort=6444 \
       --set cni.exclusive=false \

bastion/src/modules/modules/k3s/src/operations/etcd-members.ts

@@ -0,0 +1,60 @@
+// Parsing `etcdctl member list`.
+//
+// WHY NOT `-w json`
+// -----------------
+// Two reasons, and the second is the dangerous one:
+//
+//  1. `member remove` requires the id in HEX. The JSON writer emits decimal.
+//  2. etcd member ids are uint64. JSON.parse turns them into JS numbers, which
+//     lose precision above 2^53 -- so the id that comes back is an
+//     APPROXIMATION of the real one.
+//
+// Observed for real: `member remove 13323819137391910000` ->
+// "bad member ID arg (strconv.ParseUint: value out of range), expecting ID in
+// Hex". That happens at the worst possible moment, with the node already
+// stopped and about to be wiped. Failing loudly is the lucky outcome; a
+// mangled id that happened to match another member would remove the WRONG one.
+//
+// The default writer prints the id as hex TEXT, so it never becomes a number.
+
+export interface EtcdMember {
+  /** Raw member name: "<node>-<8 hex>", e.g. "worker0.lab-e26aacc5". */
+  name: string;
+  /** The node name with etcd's random suffix stripped. */
+  node: string;
+  /** Hex id, exactly as etcd prints it — never round-tripped through a number. */
+  id: string;
+  started: boolean;
+}
+
+/**
+ * etcd names a member `<node>-<8 hex>`. Comparing a raw member name against a
+ * node name without stripping that suffix silently never matches.
+ */
+export function etcdNodeName(memberName: string): string {
+  return memberName.replace(/-[0-9a-f]{8}$/, "");
+}
+
+/**
+ * Parse the default `etcdctl member list` writer:
+ *
+ *   825852d3683bc406, started, worker2.lab-5f7ad0c9, https://..:2380, https://..:2379, false
+ *   <hex id>        , status , <node>-<8 hex>      , peer           , client         , learner
+ */
+export function parseEtcdMembers(out: string): EtcdMember[] {
+  return out.split("\n")
+    .map((l) => l.trim())
+    .filter((l) => l !== "")
+    .map((line) => {
+      const cols = line.split(",").map((c) => c.trim());
+      const name = cols[2] ?? "";
+      return {
+        name,
+        node: etcdNodeName(name),
+        id: cols[0] ?? "",
+        started: (cols[1] ?? "") === "started",
+      };
+    })
+    // Drop anything without a hex id: blank lines, warnings, stray output.
+    .filter((m) => /^[0-9a-f]+$/i.test(m.id));
+}

bastion/src/modules/modules/k3s/src/operations/index.ts

@@ -1,10 +1,11 @@
 export { loadKernelModules } from "./kernel-modules.js";
 export { applyCisHardening } from "./sysctl.js";
-export { disableSwap } from "./swap.js";
+export { enableSwap } from "./swap.js";
+export { growRancherLv } from "./rancher-storage.js";
 export { enableIscsi } from "./iscsi.js";
 export { disableFirewall } from "./firewall.js";
 export { setSelinuxPermissive } from "./selinux.js";
-export { writeK3sConfig } from "./k3s-config.js";
+export { writeK3sConfig, generateServerConfig, generateAgentConfig } from "./k3s-config.js";
 export { writeAuditPolicy } from "./audit-policy.js";
 export { cleanupStaleCni } from "./cni-cleanup.js";
 export { installK3sBinary } from "./k3s-install.js";
@@ -23,3 +24,10 @@ export type {
   RecoverEtcdMemberOptions,
   RecoverEtcdMemberResult,
 } from "./etcd-recover.js";
+export {
+  cordonNode, uncordonNode, drainNode, evictLonghornReplicas,
+  assertEvictionCompletable, assertNodeEvacuated, checkEvictionCompletable,
+} from "./node-evacuate.js";
+export type { EvictionFeasibility } from "./node-evacuate.js";
+export { clearLonghornEviction, assertNodeSchedulable } from "./node-admit.js";
+export { labelWorkerRole } from "./worker-label.js";

bastion/src/modules/modules/k3s/src/operations/k3s-config.ts

@@ -2,13 +2,54 @@
 
 import type { Operation, OperationResult, K3sConfig } from "../types.js";
 import { sshOpts, writeRemoteFile } from "../utils.js";
+import { isK3sServer } from "@lab/shared";
 
-function isServerRole(role: string): boolean {
-  return role === "infra" || role === "labcontroller";
+/**
+ * The address-family block: `cluster-cidr`, `service-cidr` and `node-ip`.
+ *
+ * Emitted ONLY when the corresponding config is supplied, and that is
+ * deliberate. With no `ipv6` and no CIDRs this returns "", so the generated
+ * file is byte-identical to what every existing node already has -- no diff,
+ * so `writeRemoteFile` reports unchanged and nothing restarts k3s. Dual-stack
+ * is therefore opt-in per node rather than a flag day.
+ *
+ * `node-ip` is only written once there is a second family to name. k3s
+ * auto-detects a sensible IPv4 on its own, and writing it out unconditionally
+ * would rewrite the config of five healthy nodes to tell them what they had
+ * already worked out.
+ */
+function addressFamilyLines(config: K3sConfig, opts: { cidrs: boolean }): string {
+  const lines: string[] = [];
+  if (opts.cidrs && config.clusterCidr?.length) {
+    lines.push(`cluster-cidr: "${config.clusterCidr.join(",")}"`);
+  }
+  if (opts.cidrs && config.serviceCidr?.length) {
+    lines.push(`service-cidr: "${config.serviceCidr.join(",")}"`);
+  }
+  if (config.ipv6) {
+    // Order matters to k3s: the FIRST entry is the primary family, and the
+    // supported single-to-dual-stack conversion is the one that preserves it.
+    // IPv4 stays primary so existing Services keep their ClusterIP.
+    lines.push(`node-ip: "${config.ip},${config.ipv6}"`);
+  }
+  return lines.length ? `${lines.join("\n")}\n` : "";
 }
 
-function generateServerConfig(config: K3sConfig): string {
-  const tlsSans = [config.hostname, config.ip, ...(config.tlsSans ?? [])];
+// Exported so the exact production config.yaml can be rendered outside an SSH
+// context -- notably by the labsim 3-server-etcd rehearsal, which must drive its
+// nodes through THIS generator rather than a parallel set of INSTALL_K3S_EXEC
+// flags, or it proves a mechanism production does not run.
+export function generateServerConfig(config: K3sConfig): string {
+  // The IPv6 address goes in the cert too. Without it, anything that reaches
+  // this apiserver over v6 -- a peer server joining, or kubectl against the v6
+  // address -- fails TLS verification, and the error names the certificate
+  // rather than the missing SAN, which is a long way from the cause.
+  const tlsSans = [
+    config.hostname,
+    config.ip,
+    ...(config.ipv6 ? [config.ipv6] : []),
+    ...(config.tlsSans ?? []),
+  ];
   const isJoining = !!config.k3sServerUrl;
   const clusterLines = isJoining
     ? `server: "${config.k3sServerUrl}"\ntoken: "${config.k3sToken}"`
@@ -21,7 +62,7 @@ function generateServerConfig(config: K3sConfig): string {
   // and never expire.
   return `# k3s server configuration — CIS hardened, etcd HA
 ${clusterLines}
-protect-kernel-defaults: true
+${addressFamilyLines(config, { cidrs: true })}protect-kernel-defaults: true
 secrets-encryption: true
 write-kubeconfig-mode: "0640"
 
@@ -51,10 +92,34 @@ ${tlsSans.map((s) => `  - "${s}"`).join("\n")}
 `;
 }
 
-function generateAgentConfig(): string {
-  return `protect-kernel-defaults: true
+// Takes the config now: an agent needs its own dual `node-ip` just as much as a
+// server does. Without one it joins as an IPv4-only node into a dual-stack
+// cluster, gets no IPv6 pod CIDR, and the failure surfaces later as pods on that
+// node being unreachable over v6 while the node itself reads Ready.
+// It takes no cluster/service CIDRs -- those are server-side only.
+export function generateAgentConfig(config: K3sConfig): string {
+  // NO node-role.kubernetes.io/worker LABEL HERE, deliberately.
+  //
+  // A kubelet is not permitted to assign itself labels in the
+  // `node-role.kubernetes.io/` namespace -- the servers run with
+  // `enable-admission-plugins=NodeRestriction`, which rejects the registration
+  // outright. The agent then never finishes registering, k3s-agent.service
+  // never signals readiness, and systemd reports:
+  //
+  //   Job for k3s-agent.service failed because the service did not take the
+  //   steps required by its unit configuration.
+  //
+  // which says nothing about labels and sends you looking at the network, the
+  // token, or the kubelet's sysctls. Every agent installed through this module
+  // failed this way; the estate's existing workers only escaped it because
+  // post-provision installs them with no config.yaml at all and their role
+  // label was applied separately, server-side.
+  //
+  // `node.longhorn.io/...` is fine: it is outside the kubernetes.io namespace,
+  // so the kubelet may set it. The worker role label is applied after the node
+  // registers, by labelWorkerRole below.
+  return `${addressFamilyLines(config, { cidrs: false })}protect-kernel-defaults: true
 node-label:
-  - "node-role.kubernetes.io/worker=true"
   - "node.longhorn.io/create-default-disk=config"
 kubelet-arg:
   - "protect-kernel-defaults=true"
@@ -64,11 +129,41 @@ kubelet-arg:
 }
 
 export const writeK3sConfig: Operation = async (ctx): Promise<OperationResult> => {
+  // Refuse to name an address the node does not have.
+  //
+  // Most of this estate is SSH-onboard, not PXE-provisioned: Asahi cannot PXE
+  // at all, and the DGX Sparks run NVIDIA's own OS and must never be
+  // reinstalled. For those nodes the install templates govern nothing and this
+  // module is the ONLY labctl touchpoint, so nothing upstream can guarantee the
+  // vendor OS actually took a DHCPv6 lease.
+  //
+  // Writing node-ip for a missing address does not fail here -- it fails later,
+  // when k3s will not start, with an error about binding rather than about
+  // addressing. Checking costs one ssh round trip and turns a confusing
+  // start-up failure into a sentence naming the address and the node.
+  if (ctx.config.ipv6) {
+    const probe = await ctx.ssh.exec(
+      `ip -6 -o addr show 2>/dev/null | grep -qF " ${ctx.config.ipv6}/" && echo present || true`,
+      sshOpts(ctx),
+    );
+    if (!probe.stdout.includes("present")) {
+      return {
+        success: false,
+        changed: false,
+        message: `Node does not have IPv6 ${ctx.config.ipv6}`,
+        error:
+          `k3s config would set node-ip to ${ctx.config.ipv6}, but that address is not on any ` +
+          `interface. k3s resolves node-ip at start-up and would fail to bind. Check the node ` +
+          `took its DHCPv6 lease (a kea reservation keyed on its MAC) before retrying.`,
+      };
+    }
+  }
+
   await ctx.ssh.exec("mkdir -p /etc/rancher/k3s", sshOpts(ctx));
 
-  const content = isServerRole(ctx.config.role)
+  const content = isK3sServer(ctx.config.role)
     ? generateServerConfig(ctx.config)
-    : generateAgentConfig();
+    : generateAgentConfig(ctx.config);
 
   const changed = await writeRemoteFile(ctx, "/etc/rancher/k3s/config.yaml", content);
 

bastion/src/modules/modules/k3s/src/operations/k3s-install.ts

@@ -2,13 +2,19 @@
 
 import type { Operation, OperationResult } from "../types.js";
 import { sshOpts } from "../utils.js";
-
-function isServerRole(role: string): boolean {
-  return role === "infra" || role === "labcontroller";
-}
+import { isK3sServer } from "@lab/shared";
 
 export const installK3sBinary: Operation = async (ctx): Promise<OperationResult> => {
-  const isServer = isServerRole(ctx.config.role);
+  const isServer = isK3sServer(ctx.config.role);
+
+  // Pin the version when asked. get.k3s.io installs LATEST by default, so a
+  // node reinstalled during a rolling migration comes back on whatever shipped
+  // since -- potentially several minor versions ahead of the cluster it is
+  // about to rejoin, which Kubernetes does not support between control-plane
+  // members.
+  const versionEnv = ctx.config.k3sVersion !== undefined && ctx.config.k3sVersion !== ""
+    ? `INSTALL_K3S_VERSION="${ctx.config.k3sVersion}" `
+    : "";
 
   // Check if already installed
   const version = await ctx.ssh.exec("k3s --version 2>/dev/null", sshOpts(ctx));
@@ -24,12 +30,21 @@ export const installK3sBinary: Operation = async (ctx): Promise<OperationResult>
       );
     }
 
-    // If joining an existing cluster, pass K3S_URL and K3S_TOKEN
-    const joinEnv = ctx.config.k3sServerUrl && ctx.config.k3sToken
-      ? `K3S_URL="${ctx.config.k3sServerUrl}" K3S_TOKEN="${ctx.config.k3sToken}"`
-      : "";
+    // Token and URL are independent. A token WITHOUT a URL seeds the cluster
+    // token on the cluster-init node, so every later joiner can be given the
+    // same pre-shared value instead of scraping node-token off the first
+    // server -- which matters here because that first server gets wiped and
+    // rejoined during a rolling migration. URL only counts when a token comes
+    // with it; a URL alone cannot authenticate a join.
+    //
+    // The token deliberately does NOT go in config.yaml: writeRemoteFile diffs
+    // that file, so a token appearing there would restart k3s on every node.
+    const joinEnv = [
+      ctx.config.k3sServerUrl && ctx.config.k3sToken ? `K3S_URL="${ctx.config.k3sServerUrl}"` : "",
+      ctx.config.k3sToken ? `K3S_TOKEN="${ctx.config.k3sToken}"` : "",
+    ].filter(Boolean).join(" ");
     const result = await ctx.ssh.exec(
-      `curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server" INSTALL_K3S_SKIP_SELINUX_RPM=true ${joinEnv} sh -`,
+      `curl -sfL https://get.k3s.io | ${versionEnv}INSTALL_K3S_EXEC="server" INSTALL_K3S_SKIP_SELINUX_RPM=true ${joinEnv} sh -`,
       { timeoutMs: 300_000 },
     );
     if (result.exitCode !== 0) {
@@ -50,7 +65,7 @@ export const installK3sBinary: Operation = async (ctx): Promise<OperationResult>
       };
     }
     const result = await ctx.ssh.exec(
-      `curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="agent" K3S_URL="${ctx.config.k3sServerUrl}" K3S_TOKEN="${ctx.config.k3sToken}" sh -`,
+      `curl -sfL https://get.k3s.io | ${versionEnv}INSTALL_K3S_EXEC="agent" K3S_URL="${ctx.config.k3sServerUrl}" K3S_TOKEN="${ctx.config.k3sToken}" sh -`,
       { timeoutMs: 300_000 },
     );
     if (result.exitCode !== 0) {

bastion/src/modules/modules/k3s/src/operations/longhorn-disk.ts

@@ -1,50 +1,206 @@
-// Annotate nodes with Longhorn default disk config when /var/lib/longhorn exists.
-// The label is set in k3s config (node-label), but the annotation must be applied via kubectl.
+// Give a node a Longhorn disk at /var/lib/longhorn.
+//
+// The cluster runs `createDefaultDiskLabeledNodes: true`, which means Longhorn
+// creates a disk ONLY on nodes carrying the label
+// `node.longhorn.io/create-default-disk=config` (set via k3s node-label) AND
+// the annotation `node.longhorn.io/default-disks-config` (set here). Miss
+// either and Longhorn gives the node no disk at all: it joins, goes Ready,
+// accepts pods, and silently hosts zero replicas.
+//
+// WHY THERE IS A REPAIR PATH
+// --------------------------
+// Longhorn reads that annotation when it FIRST creates the node's
+// `nodes.longhorn.io` record. This operation runs during configure(), which on
+// a rejoining node is after the record already exists -- so the annotation
+// lands too late and is never read. That is invisible until a rebuild is
+// expected and never arrives, roughly forty minutes into a migration.
+//
+// So: annotate, then read back, and if there is still no disk, write it
+// directly into spec.disks. When Longhorn does honour the annotation the
+// repair is a no-op.
 
-import type { Operation, OperationResult } from "../types.js";
+import type { Operation, OperationContext, OperationResult } from "../types.js";
 import { sshOpts } from "../utils.js";
 import { sshExec as remoteSshExec } from "../../../../src/ssh.js";
 
+const DISK_PATH = "/var/lib/longhorn";
+/** Fixed key so a repeated repair updates one disk rather than adding more. */
+const DISK_KEY = "default-disk-lab";
+const VERIFY_INTERVAL_MS = 5_000;
+/** How long to wait for Longhorn to create the node's record after it joins. */
+const REGISTER_TIMEOUT_MS = 180_000;
+/** How long to then wait for Longhorn to give that record a disk. */
+const DISK_TIMEOUT_MS = 60_000;
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * Run kubectl from wherever it can be reached: locally on a server, otherwise
+ * over SSH to the configured server (an agent has no local kubectl).
+ */
+async function kubectlAnywhere(
+  ctx: OperationContext,
+  args: string,
+): Promise<{ ok: boolean; out: string }> {
+  const local = await ctx.ssh.exec(`k3s kubectl ${args} 2>&1`, sshOpts(ctx));
+  if (local.exitCode === 0) return { ok: true, out: local.stdout };
+
+  if (ctx.config.k3sServerUrl) {
+    try {
+      const serverHost = new URL(ctx.config.k3sServerUrl).hostname;
+      const remote = await remoteSshExec(serverHost, "root", `k3s kubectl ${args}`, {
+        ...(ctx.ssh.keyPath ? { keyPath: ctx.ssh.keyPath } : {}),
+        timeoutMs: 15_000,
+      });
+      if (remote.exitCode === 0) return { ok: true, out: remote.stdout };
+      return { ok: false, out: remote.stdout || remote.stderr };
+    } catch {
+      /* fall through */
+    }
+  }
+  return { ok: false, out: local.stdout || local.stderr };
+}
+
+/** Does the node's Longhorn record exist yet? */
+async function nodeRecordExists(ctx: OperationContext, nodeName: string): Promise<boolean> {
+  const res = await kubectlAnywhere(ctx, `-n longhorn-system get nodes.longhorn.io "${nodeName}" -o name`);
+  return res.ok && res.out.trim() !== "";
+}
+
+/** Does the node's Longhorn record carry a disk at DISK_PATH? */
+async function hasDisk(ctx: OperationContext, nodeName: string): Promise<boolean> {
+  const res = await kubectlAnywhere(
+    ctx,
+    `-n longhorn-system get nodes.longhorn.io "${nodeName}" ` +
+      `-o jsonpath='{range .spec.disks.*}{.path}{"\\n"}{end}'`,
+  );
+  if (!res.ok) return false;
+  return res.out.split("\n").map((l) => l.trim()).includes(DISK_PATH);
+}
+
 export const configureLonghornDisk: Operation = async (ctx): Promise<OperationResult> => {
-  // Check if /var/lib/longhorn exists on this node
-  const check = await ctx.ssh.exec("test -d /var/lib/longhorn && echo yes || echo no", sshOpts(ctx));
+  const check = await ctx.ssh.exec(`test -d ${DISK_PATH} && echo yes || echo no`, sshOpts(ctx));
   if (check.stdout.trim() !== "yes") {
-    return { success: true, changed: false, message: "No /var/lib/longhorn directory — skipping Longhorn disk config" };
+    return { success: true, changed: false, message: `No ${DISK_PATH} directory — skipping Longhorn disk config` };
   }
 
-  // Find the node name (hostname as registered in k3s)
+  // Longhorn writes replicas into <path>/replicas and creates that directory
+  // only when it FIRST initialises the disk. If the directory later goes away
+  // while longhorn-disk.cfg remains -- someone clearing space, a partial
+  // restore, a wipe that preserved the LV -- Longhorn considers the disk
+  // already initialised and never recreates it. The replica process then dies
+  // on a non-recursive mkdir:
+  //
+  //   level=fatal msg="Error running start replica command"
+  //     error="mkdir /host/var/lib/longhorn/replicas/pvc-...: ..."
+  //
+  // every single time, for ever, while the node reports Ready and Schedulable
+  // and its disk looks healthy. Cheap to guarantee, expensive to diagnose.
+  await ctx.ssh.exec(`mkdir -p ${DISK_PATH}/replicas`, sshOpts(ctx));
+
   const nodeNameResult = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
   const nodeName = nodeNameResult.stdout.trim();
 
-  const annotation = JSON.stringify([{ path: "/var/lib/longhorn", allowScheduling: true }]);
-
-  // Try kubectl locally first (works on server nodes)
-  const result = await ctx.ssh.exec(
-    `k3s kubectl annotate node "${nodeName}" "node.longhorn.io/default-disks-config=${annotation}" --overwrite 2>&1 || true`,
-    sshOpts(ctx),
+  // SINGLE quotes around the whole key=value, not double.
+  //
+  // The value is JSON and therefore full of double quotes. Wrapping it in
+  // double quotes ends the quoted string at the first inner one, and the shell
+  // strips the rest, so the node gets
+  //     [{path:/var/lib/longhorn,allowScheduling:true}]
+  // instead of
+  //     [{"path":"/var/lib/longhorn","allowScheduling":true}]
+  // -- valid-looking, unparseable, and silent. Longhorn then creates NO disk on
+  // the node, which surfaces much later as "why does nothing schedule here".
+  // The JSON contains no single quotes, so single-quoting is safe.
+  const annotation = JSON.stringify([{ path: DISK_PATH, allowScheduling: true }]);
+  const annotated = await kubectlAnywhere(
+    ctx,
+    `annotate node "${nodeName}" 'node.longhorn.io/default-disks-config=${annotation}' --overwrite`,
   );
 
-  if (result.stdout.includes("annotated") || result.stdout.includes("unchanged")) {
-    return { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName}` };
+  // No Longhorn at all (or no way to reach the API) — nothing more to do.
+  const lh = await kubectlAnywhere(ctx, "get ns longhorn-system -o name");
+  if (!lh.ok) {
+    return annotated.ok
+      ? { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName} (Longhorn not installed yet)` }
+      : { success: true, changed: false, message: "Longhorn disk label set (annotation requires server kubectl)" };
   }
 
-  // For worker/agent nodes without local kubectl: apply via the server
-  if (ctx.config.k3sServerUrl) {
-    // The CLI has SSH access to the server — use sshExec from there
-    const serverHost = new URL(ctx.config.k3sServerUrl).hostname;
-    try {
-      const remoteResult = await remoteSshExec(
-        serverHost, "root",
-        `k3s kubectl annotate node "${nodeName}" "node.longhorn.io/default-disks-config=${annotation}" --overwrite`,
-        { ...(ctx.ssh.keyPath ? { keyPath: ctx.ssh.keyPath } : {}), timeoutMs: 15_000 },
-      );
-      if (remoteResult.stdout.includes("annotated") || remoteResult.stdout.includes("unchanged")) {
-        return { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName} (via server)` };
-      }
-    } catch {
-      // Fall through to manual instruction
+  // Longhorn registration is ASYNCHRONOUS. On a node that has just joined --
+  // which is exactly the case after a wipe-and-rejoin -- the nodes.longhorn.io
+  // record does not exist yet, so there is nothing to inspect and nothing to
+  // repair. Waiting a fixed 60s and then giving up meant the repair never ran
+  // on the one path that needs it most, and the caller's schedulability gate
+  // then failed immediately with "Longhorn reports no disk".
+  //
+  // So: wait for the RECORD first, generously, and only then worry about the
+  // disk on it.
+  const registerDeadline = Date.now() + REGISTER_TIMEOUT_MS;
+  let registered = false;
+  while (Date.now() < registerDeadline) {
+    if (await hasDisk(ctx, nodeName)) {
+      return { success: true, changed: annotated.ok, message: `${nodeName} has a Longhorn disk at ${DISK_PATH}` };
     }
+    if (await nodeRecordExists(ctx, nodeName)) { registered = true; break; }
+    await sleep(VERIFY_INTERVAL_MS);
   }
 
-  return { success: true, changed: false, message: "Longhorn disk label set (annotation requires server kubectl)" };
+  if (!registered) {
+    return {
+      success: true,
+      changed: annotated.ok,
+      message: `${nodeName} not registered with Longhorn within ${REGISTER_TIMEOUT_MS / 1000}s; ` +
+        `annotation is in place for when it is`,
+    };
+  }
+
+  // Registered. Give Longhorn a chance to act on the annotation by itself
+  // before overriding it.
+  const diskDeadline = Date.now() + DISK_TIMEOUT_MS;
+  while (Date.now() < diskDeadline) {
+    if (await hasDisk(ctx, nodeName)) {
+      return { success: true, changed: annotated.ok, message: `${nodeName} has a Longhorn disk at ${DISK_PATH}` };
+    }
+    await sleep(VERIFY_INTERVAL_MS);
+  }
+
+  // The record exists and still has no disk: the annotation was applied too
+  // late to be read. Write it in.
+  ctx.log(`${nodeName} has no Longhorn disk after annotating; writing spec.disks directly`);
+  const patch = JSON.stringify({
+    spec: {
+      disks: {
+        [DISK_KEY]: {
+          path: DISK_PATH,
+          allowScheduling: true,
+          evictionRequested: false,
+          storageReserved: 0,
+          tags: [],
+          diskType: "filesystem",
+        },
+      },
+    },
+  });
+  const repaired = await kubectlAnywhere(
+    ctx,
+    `-n longhorn-system patch nodes.longhorn.io "${nodeName}" --type=merge -p '${patch}'`,
+  );
+  if (!repaired.ok) {
+    return {
+      success: false,
+      changed: false,
+      message: `${nodeName} has no Longhorn disk and could not be repaired`,
+      error: `${repaired.out}. Without a disk this node will never host a replica.`,
+    };
+  }
+
+  if (await hasDisk(ctx, nodeName)) {
+    return { success: true, changed: true, message: `Longhorn disk repaired on ${nodeName} (${DISK_PATH})` };
+  }
+  return {
+    success: false,
+    changed: true,
+    message: `Patched spec.disks on ${nodeName} but Longhorn still reports no disk at ${DISK_PATH}`,
+    error: "Check longhorn-manager logs; the node will not host replicas until this is resolved.",
+  };
 };

bastion/src/modules/modules/k3s/src/operations/node-admit.ts

@@ -0,0 +1,136 @@
+// Bring a node back into service. The inverse of node-evacuate.ts.
+//
+// WHY THIS EXISTS
+// ---------------
+// Evacuation is not symmetric with itself. It leaves three marks on a node --
+// a kubernetes cordon, Longhorn's `allowScheduling:false` + `evictionRequested`,
+// and (on a node that was wiped and rejoined) a MISSING default Longhorn disk.
+// Undoing a cordon undoes only the first. Without the other two the node comes
+// back Ready, accepts pods, and quietly hosts no replicas at all -- which looks
+// like success right up until you notice storage never rebalanced onto it.
+//
+// This is also the only supported way out of an aborted evacuation.
+
+import type { Operation, OperationContext, OperationResult } from "../types.js";
+import { sshOpts } from "../utils.js";
+
+async function nodeName(ctx: OperationContext): Promise<string> {
+  const r = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
+  return r.stdout.trim();
+}
+
+async function kubectl(ctx: OperationContext, args: string): Promise<{ ok: boolean; out: string }> {
+  const res = await ctx.ssh.exec(`k3s kubectl ${args} 2>&1`, sshOpts(ctx));
+  return { ok: res.exitCode === 0, out: res.exitCode === 0 ? res.stdout : (res.stdout || res.stderr) };
+}
+
+/**
+ * Clear Longhorn's eviction request and re-enable scheduling.
+ *
+ * Both fields in one merge patch, mirroring how eviction sets them: Longhorn's
+ * validating webhook rejects `evictionRequested:true` on a schedulable node, so
+ * the pair has to move together in both directions.
+ */
+export const clearLonghornEviction: Operation = async (ctx): Promise<OperationResult> => {
+  const name = await nodeName(ctx);
+
+  const lh = await kubectl(ctx, "get ns longhorn-system -o name");
+  if (!lh.ok) {
+    return { success: true, changed: false, message: "Longhorn not installed — nothing to clear" };
+  }
+
+  const before = await kubectl(
+    ctx,
+    `-n longhorn-system get nodes.longhorn.io "${name}" -o jsonpath='{.spec.allowScheduling},{.spec.evictionRequested}'`,
+  );
+  if (!before.ok) {
+    return { success: true, changed: false, message: `${name} has no Longhorn node record yet` };
+  }
+  if (before.out.trim() === "true,false") {
+    return { success: true, changed: false, message: `${name} already accepts replicas` };
+  }
+
+  const patch = JSON.stringify({ spec: { allowScheduling: true, evictionRequested: false } });
+  const res = await kubectl(
+    ctx,
+    `-n longhorn-system patch nodes.longhorn.io "${name}" --type=merge -p '${patch}'`,
+  );
+  return res.ok
+    ? { success: true, changed: true, message: `${name} accepts replicas again` }
+    : { success: false, changed: false, message: `Could not clear eviction on ${name}`, error: res.out };
+};
+
+/**
+ * Confirm the node can actually host replicas again.
+ *
+ * Reads back from Longhorn rather than trusting the patches above, because the
+ * failure this catches is a node that is Ready and schedulable in kubernetes
+ * while having no Longhorn disk at all -- the exact state a rejoined node lands
+ * in when its default-disk annotation was applied too late.
+ */
+const SCHEDULABLE_TIMEOUT_MS = 120_000;
+const SCHEDULABLE_INTERVAL_MS = 5_000;
+const sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
+
+export const assertNodeSchedulable: Operation = async (ctx): Promise<OperationResult> => {
+  // Poll rather than check once.
+  //
+  // Longhorn registers a node and attaches its disk ASYNCHRONOUSLY, and this
+  // runs moments after a wipe-and-rejoin. Checking instantly reports "Longhorn
+  // reports no disk on this node" for a node that is simply a few seconds from
+  // being fine -- which is a false alarm on the exact path the gate exists to
+  // protect. Still fails if it never settles.
+  const deadline = Date.now() + SCHEDULABLE_TIMEOUT_MS;
+  let problems: string[] = [];
+  for (;;) {
+    problems = await schedulableProblems(ctx);
+    if (problems.length === 0 || Date.now() >= deadline) break;
+    await sleep(SCHEDULABLE_INTERVAL_MS);
+  }
+
+  const name = await nodeName(ctx);
+  return problems.length === 0
+    ? { success: true, changed: false, message: `${name} is back in service` }
+    : {
+        success: false,
+        changed: false,
+        message: `${name} is NOT fully back in service after ${SCHEDULABLE_TIMEOUT_MS / 1000}s`,
+        error: problems.join("; "),
+      };
+};
+
+/** One pass of the schedulability checks. Returns the reasons it is not. */
+async function schedulableProblems(ctx: OperationContext): Promise<string[]> {
+  const name = await nodeName(ctx);
+  const problems: string[] = [];
+
+  const sched = await kubectl(ctx, `get node "${name}" -o jsonpath='{.spec.unschedulable}'`);
+  if (sched.ok && sched.out.trim() === "true") problems.push("node is still cordoned");
+
+  const lh = await kubectl(ctx, "get ns longhorn-system -o name");
+  if (lh.ok) {
+    const spec = await kubectl(
+      ctx,
+      `-n longhorn-system get nodes.longhorn.io "${name}" -o jsonpath='{.spec.allowScheduling},{.spec.evictionRequested}'`,
+    );
+    if (!spec.ok) {
+      problems.push("no Longhorn node record");
+    } else {
+      const [allow, evicting] = spec.out.trim().split(",");
+      if (allow !== "true") problems.push("Longhorn scheduling is still disabled");
+      if (evicting === "true") problems.push("Longhorn eviction is still requested");
+    }
+
+    // A node with no disk hosts no replicas, however healthy it otherwise looks.
+    const disks = await kubectl(
+      ctx,
+      `-n longhorn-system get nodes.longhorn.io "${name}" ` +
+        `-o jsonpath='{range .status.diskStatus.*}{.conditions[?(@.type=="Schedulable")].status}{"\\n"}{end}'`,
+    );
+    const states = disks.ok ? disks.out.split("\n").map((l) => l.trim()).filter(Boolean) : [];
+    if (states.length === 0) problems.push("Longhorn reports no disk on this node");
+    else if (!states.includes("True")) problems.push(`no schedulable disk (states: ${states.join(",")})`);
+  }
+
+  return problems;
+}

bastion/src/modules/modules/k3s/src/operations/node-evacuate.ts

@@ -0,0 +1,547 @@
+// Evacuate a node before it is wiped and reinstalled.
+//
+// WHY THIS EXISTS
+// ---------------
+// The kickstart's %pre can preserve data across a reinstall, but ONLY when the
+// disk layout is unchanged: it keys off `labvg` already existing with the
+// expected LV names. Migrating a machine to the v2 layout means recreating the
+// volume group, which takes the `clearpart --all` branch and destroys every
+// Longhorn replica on that node.
+//
+// So a layout migration cannot rely on disk-level preservation. The node has to
+// be EVACUATED first -- replicas rebuilt elsewhere, workloads moved, etcd
+// membership surrendered -- and then wiped. These operations are the missing
+// half; before this file the repo had no drain, no cordon, and no replica
+// eviction anywhere.
+//
+// Ordering matters and is not interchangeable:
+//   cordon -> check -> evict replicas (WAIT for rebuild) -> drain -> remove etcd member
+//
+// Two ordering facts, both learned the hard way:
+//   - Removing the etcd member first costs you the API server you need for the
+//     rest of the procedure.
+//   - EVICT BEFORE DRAIN. `drain --force` does not override PodDisruptionBudgets,
+//     and longhorn-manager keeps a PDB on the instance-manager pod while volumes
+//     are attached or degraded. Draining a storage node first therefore stops the
+//     instance-manager that serves the very replicas being evicted: the drain
+//     blocks for its full timeout, and if it did succeed it would remove the only
+//     source a single-replica volume could rebuild from.
+
+import type { Operation, OperationContext, OperationResult } from "../types.js";
+import { sshOpts } from "../utils.js";
+
+/** Longhorn's own default; a rebuild of a large replica set can exceed it. */
+const REPLICA_REBUILD_TIMEOUT_MS = 60 * 60_000;
+const POLL_INTERVAL_MS = 15_000;
+/** How often to report "still waiting" when the replica count is not moving. */
+const HEARTBEAT_INTERVAL_MS = 3 * 60_000;
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+async function nodeName(ctx: OperationContext): Promise<string> {
+  const r = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
+  return r.stdout.trim();
+}
+
+/**
+ * kubectl, from wherever it is reachable.
+ *
+ * An agent has no local kubectl, so the caller must be pointed at a server.
+ * Returns null when there is no way to reach the API, so callers can report
+ * "could not do this" instead of silently continuing.
+ */
+async function kubectl(
+  ctx: OperationContext,
+  args: string,
+  /**
+   * Override the SSH timeout for this call.
+   *
+   * sshOpts() is a flat 30 seconds, which is right for a status read and
+   * completely wrong for anything that does work. A `kubectl drain` asking for
+   * --timeout=15m gets killed by the transport at 30s and returns non-zero
+   * having ALREADY evicted most of the pods -- so it looks like a drain
+   * failure, with a transcript that reads like success, on every real node.
+   */
+  timeoutMs?: number,
+): Promise<{ ok: boolean; out: string }> {
+  const opts = timeoutMs === undefined ? sshOpts(ctx) : { timeoutMs };
+  const local = await ctx.ssh.exec(`k3s kubectl ${args} 2>&1`, opts);
+  if (local.exitCode === 0) return { ok: true, out: local.stdout };
+  return { ok: false, out: local.stdout || local.stderr };
+}
+
+/** Mark the node unschedulable. Safe and instantly reversible. */
+export const cordonNode: Operation = async (ctx): Promise<OperationResult> => {
+  const name = await nodeName(ctx);
+  const res = await kubectl(ctx, `cordon "${name}"`);
+  if (!res.ok) {
+    return { success: false, changed: false, message: `Could not cordon ${name}`, error: res.out };
+  }
+  const already = res.out.includes("already cordoned");
+  return {
+    success: true,
+    changed: !already,
+    message: already ? `${name} was already cordoned` : `Cordoned ${name}`,
+  };
+};
+
+export const uncordonNode: Operation = async (ctx): Promise<OperationResult> => {
+  const name = await nodeName(ctx);
+  const res = await kubectl(ctx, `uncordon "${name}"`);
+  return res.ok
+    ? { success: true, changed: true, message: `Uncordoned ${name}` }
+    : { success: false, changed: false, message: `Could not uncordon ${name}`, error: res.out };
+};
+
+/**
+ * Move workloads off the node.
+ *
+ * --ignore-daemonsets because a DaemonSet pod cannot be evicted by definition;
+ * --delete-emptydir-data because refusing to delete emptyDir is what makes an
+ * unattended drain hang forever, and a node about to be wiped is going to lose
+ * that data regardless.
+ */
+export const drainNode: Operation = async (ctx): Promise<OperationResult> => {
+  const name = await nodeName(ctx);
+  // The SSH timeout must outlive kubectl's own --timeout, or the transport
+  // kills a drain that is working and the result is indistinguishable from a
+  // drain that failed.
+  const DRAIN_TIMEOUT_MS = 15 * 60_000;
+  const res = await kubectl(
+    ctx,
+    `drain "${name}" --ignore-daemonsets --delete-emptydir-data --force --timeout=15m`,
+    DRAIN_TIMEOUT_MS + 60_000,
+  );
+  if (res.ok) return { success: true, changed: true, message: `Drained ${name}` };
+
+  // Surface the END of the transcript. The first line of a drain is always
+  // "node/x already cordoned", which tells you nothing and hides the reason.
+  const tail = res.out.split("\n").map((l) => l.trim()).filter(Boolean).slice(-6).join(" | ");
+  return {
+    success: false,
+    changed: false,
+    message: `Drain failed on ${name}`,
+    error: tail === "" ? res.out : `...${tail}`,
+  };
+};
+
+/** kubectl -o json, parsed. Returns null when the read or the parse failed. */
+async function kubectlJson<T>(ctx: OperationContext, args: string): Promise<T | null> {
+  const res = await kubectl(ctx, `${args} -o json`);
+  if (!res.ok) return null;
+  try {
+    return JSON.parse(res.out) as T;
+  } catch {
+    return null;
+  }
+}
+
+/** A Longhorn setting's value, or the supplied default when it is unreadable. */
+async function longhornSetting(ctx: OperationContext, name: string, fallback: string): Promise<string> {
+  const res = await kubectl(ctx, `-n longhorn-system get settings.longhorn.io ${name} -o jsonpath='{.value}'`);
+  const v = res.ok ? res.out.trim() : "";
+  return v === "" ? fallback : v;
+}
+
+interface LhNode {
+  metadata: { name: string };
+  spec: { allowScheduling?: boolean };
+  status?: { diskStatus?: Record<string, {
+    storageAvailable?: number; storageMaximum?: number;
+    storageScheduled?: number; storageReserved?: number;
+  }> };
+}
+interface LhReplica {
+  metadata: { name: string };
+  spec: { nodeID?: string; volumeName?: string };
+}
+interface LhVolume {
+  metadata: { name: string };
+  spec: { numberOfReplicas?: number; size?: number | string };
+  status?: { state?: string; robustness?: string };
+}
+
+export interface EvictionFeasibility {
+  completable: boolean;
+  /** Reasons eviction can never finish. Each names the offending volume. */
+  blockers: string[];
+  /** Things that will work but are dangerous. */
+  warnings: string[];
+  replicasMoving: number;
+  bytesMoving: number;
+}
+
+/**
+ * Can eviction of this node actually COMPLETE?
+ *
+ * evictLonghornReplicas waits for the node's replica count to reach zero. If
+ * there is nowhere for a replica to go, that wait can never end -- it burns the
+ * full hour-long timeout and then fails, which is an expensive way to discover
+ * a fact that is knowable in a couple of seconds.
+ *
+ * It is knowable because Longhorn's placement rules are simple:
+ *
+ *   - With strict anti-affinity (replica-soft-anti-affinity=false, the default
+ *     and what this fleet runs) a volume may put at most ONE replica on a node.
+ *     So a volume wanting N replicas needs N schedulable nodes OTHER than the
+ *     one being evacuated. A 3-replica volume on a 3-node cluster is therefore
+ *     unevictable -- the two survivors already hold a copy each and cannot take
+ *     the third.
+ *   - The scheduler admits a replica against over-provisioned space, but the
+ *     rebuild writes real bytes to a real disk. Those are different numbers,
+ *     and at 500% over-provisioning the first can pass while the second fails.
+ *
+ * Read-only, a handful of API calls, finishes in seconds.
+ */
+export async function checkEvictionCompletable(
+  ctx: OperationContext,
+  name: string,
+): Promise<EvictionFeasibility> {
+  const blockers: string[] = [];
+  const warnings: string[] = [];
+
+  const nodes = await kubectlJson<{ items: LhNode[] }>(ctx, "-n longhorn-system get nodes.longhorn.io");
+  const replicas = await kubectlJson<{ items: LhReplica[] }>(ctx, "-n longhorn-system get replicas.longhorn.io");
+  const volumes = await kubectlJson<{ items: LhVolume[] }>(ctx, "-n longhorn-system get volumes.longhorn.io");
+  if (!nodes || !replicas || !volumes) {
+    return {
+      completable: false,
+      blockers: ["Could not read Longhorn nodes/replicas/volumes — refusing to guess"],
+      warnings, replicasMoving: 0, bytesMoving: 0,
+    };
+  }
+
+  const softAntiAffinity = (await longhornSetting(ctx, "replica-soft-anti-affinity", "false")) === "true";
+  const overProvisionPct = Number(await longhornSetting(ctx, "storage-over-provisioning-percentage", "100")) || 100;
+
+  const mine = replicas.items.filter((r) => r.spec.nodeID === name);
+  if (mine.length === 0) {
+    return { completable: true, blockers, warnings, replicasMoving: 0, bytesMoving: 0 };
+  }
+
+  const volumeByName = new Map(volumes.items.map((v) => [v.metadata.name, v]));
+  // Candidates are schedulable Longhorn nodes that are not the one leaving.
+  const candidates = nodes.items.filter(
+    (n) => n.metadata.name !== name && n.spec.allowScheduling !== false,
+  );
+
+  const diskFree = (n: LhNode): number =>
+    Object.values(n.status?.diskStatus ?? {}).reduce((a, d) => a + (d.storageAvailable ?? 0), 0);
+  const schedulableFree = (n: LhNode): number =>
+    Object.values(n.status?.diskStatus ?? {}).reduce((a, d) => {
+      const usable = ((d.storageMaximum ?? 0) - (d.storageReserved ?? 0)) * (overProvisionPct / 100);
+      return a + Math.max(0, usable - (d.storageScheduled ?? 0));
+    }, 0);
+
+  let bytesMoving = 0;
+
+  for (const rep of mine) {
+    const volName = rep.spec.volumeName ?? "";
+    const vol = volumeByName.get(volName);
+    if (!vol) {
+      warnings.push(`replica ${rep.metadata.name} has no volume ${volName} — orphan?`);
+      continue;
+    }
+    const want = vol.spec.numberOfReplicas ?? 1;
+    const size = Number(vol.spec.size ?? 0);
+    bytesMoving += size;
+
+    // (1) Placement. Strict anti-affinity means one replica per node per volume.
+    if (!softAntiAffinity && candidates.length < want) {
+      blockers.push(
+        `volume ${volName} wants ${want} replicas but only ${candidates.length} other schedulable ` +
+        `node(s) exist — with strict anti-affinity its replica cannot be placed. Add a schedulable node.`,
+      );
+      continue;
+    }
+
+    // (2) Scheduler capacity, at the configured over-provisioning percentage.
+    const schedulableFit = candidates.filter((n) => schedulableFree(n) >= size);
+    if (schedulableFit.length === 0) {
+      blockers.push(
+        `volume ${volName} (${gib(size)}) does not fit the scheduler on any candidate node ` +
+        `even at ${overProvisionPct}% over-provisioning.`,
+      );
+      continue;
+    }
+
+    // (3) Real capacity. The scheduler counts over-provisioned space; a rebuild
+    // writes real bytes. This is the check that catches a small disk hiding
+    // behind a generous over-provisioning percentage.
+    const realFit = candidates.filter((n) => diskFree(n) >= size);
+    if (realFit.length === 0) {
+      const best = Math.max(0, ...candidates.map(diskFree));
+      blockers.push(
+        `volume ${volName} needs ${gib(size)} but the roomiest candidate node has ${gib(best)} ` +
+        `actually free. storage-over-provisioning-percentage=${overProvisionPct} lets the scheduler ` +
+        `accept this; the disk does not.`,
+      );
+      continue;
+    }
+
+    // (4) Volumes with no second copy, and volumes that cannot rebuild at all.
+    if (want === 1) {
+      warnings.push(
+        `volume ${volName} has a single replica — evicting it rebuilds from the only copy that exists`,
+      );
+    }
+    if (vol.status?.state === "detached") {
+      warnings.push(`volume ${volName} is detached — Longhorn must attach it to rebuild`);
+    }
+    if (vol.status?.robustness === "degraded") {
+      warnings.push(`volume ${volName} is already degraded before eviction starts`);
+    }
+  }
+
+  return { completable: blockers.length === 0, blockers, warnings, replicasMoving: mine.length, bytesMoving };
+}
+
+function gib(bytes: number): string {
+  return `${(bytes / 2 ** 30).toFixed(1)} GiB`;
+}
+
+/**
+ * Standalone gate: refuse fast and loudly rather than hang for an hour.
+ *
+ * Exposed as its own Operation so it can be run on its own -- "could I evacuate
+ * this node if I wanted to?" is a question worth answering without starting.
+ */
+export const assertEvictionCompletable: Operation = async (ctx): Promise<OperationResult> => {
+  const name = await nodeName(ctx);
+  const lh = await kubectl(ctx, "get ns longhorn-system -o name");
+  if (!lh.ok) {
+    return { success: true, changed: false, message: "Longhorn not installed — nothing to evict" };
+  }
+
+  const f = await checkEvictionCompletable(ctx, name);
+  for (const w of f.warnings) ctx.log(`  WARNING: ${w}`);
+
+  if (!f.completable) {
+    return {
+      success: false,
+      changed: false,
+      message: `Eviction of ${name} cannot complete`,
+      error: f.blockers.join("; "),
+    };
+  }
+  return {
+    success: true,
+    changed: false,
+    message: f.replicasMoving === 0
+      ? `${name} holds no replicas`
+      : `${f.replicasMoving} replica(s) (${gib(f.bytesMoving)}) can be placed elsewhere`,
+  };
+};
+
+/**
+ * Ask Longhorn to move this node's replicas elsewhere, then WAIT.
+ *
+ * The wait is the entire point. Setting evictionRequested and immediately
+ * wiping the disk destroys the replicas mid-rebuild, which on a 3-replica
+ * volume takes you to 1 and on the 2-replica Prometheus volume takes you to 0.
+ *
+ * Returns changed:false with an explanation when Longhorn is not installed,
+ * so a standalone or storage-free node is a no-op rather than an error.
+ */
+export const evictLonghornReplicas: Operation = async (ctx): Promise<OperationResult> => {
+  const name = await nodeName(ctx);
+
+  const hasLonghorn = await kubectl(ctx, "get ns longhorn-system -o name");
+  if (!hasLonghorn.ok) {
+    return { success: true, changed: false, message: "Longhorn not installed — nothing to evict" };
+  }
+
+  // Refuse in seconds rather than discover it after an hour of polling.
+  // Inside the operation, not only in the group, so `labctl app k3s evacuate`
+  // gets the same protection as the drill does.
+  const feasible = await checkEvictionCompletable(ctx, name);
+  for (const w of feasible.warnings) ctx.log(`  WARNING: ${w}`);
+  if (!feasible.completable) {
+    return {
+      success: false,
+      changed: false,
+      message: `Eviction of ${name} cannot complete — not starting it`,
+      error: feasible.blockers.join("; "),
+    };
+  }
+  // A single-replica volume rebuilds from the only copy in existence, and this
+  // fleet has no backup target. That is a decision for a human, not a default.
+  const singles = feasible.warnings.filter((w) => w.includes("single replica"));
+  if (singles.length > 0 && ctx.config.evictionAllowSingleReplica !== true) {
+    return {
+      success: false,
+      changed: false,
+      message: `${name} holds single-replica volume(s); refusing to evict`,
+      error: `${singles.join("; ")}. Raise them to 3 replicas and wait for healthy, ` +
+        `or pass evictionAllowSingleReplica to accept the risk.`,
+    };
+  }
+
+  const patch = JSON.stringify({
+    spec: { allowScheduling: false, evictionRequested: true },
+  });
+  const req = await kubectl(
+    ctx,
+    `-n longhorn-system patch nodes.longhorn.io "${name}" --type=merge -p '${patch}'`,
+  );
+  if (!req.ok) {
+    return { success: false, changed: false, message: `Could not request eviction for ${name}`, error: req.out };
+  }
+
+  ctx.log(`Eviction requested for ${name}; waiting for replicas to rebuild elsewhere...`);
+  const start = Date.now();
+  let last = -1;
+  let lastHeartbeat = Date.now();
+  try {
+  while (Date.now() - start < REPLICA_REBUILD_TIMEOUT_MS) {
+    const res = await kubectl(
+      ctx,
+      `-n longhorn-system get replicas.longhorn.io -o jsonpath='{range .items[?(@.spec.nodeID=="${name}")]}{.metadata.name}{"\\n"}{end}'`,
+    );
+    const remaining = res.ok ? res.out.split("\n").map((l) => l.trim()).filter(Boolean).length : -1;
+    if (remaining === 0) {
+      return {
+        success: true,
+        changed: true,
+        message: `All replicas evicted from ${name} after ${Math.round((Date.now() - start) / 1000)}s`,
+      };
+    }
+    // Log on change, and otherwise every few minutes. A silent hour is
+    // indistinguishable from a hung process; a heartbeat that carries the
+    // elapsed time makes "stalled" visible while it is happening rather than
+    // only in the timeout message.
+    const elapsedS = Math.round((Date.now() - start) / 1000);
+    if (remaining !== last) {
+      ctx.log(`  ${remaining} replica(s) still on ${name} (${elapsedS}s elapsed)`);
+      last = remaining;
+      lastHeartbeat = Date.now();
+    } else if (Date.now() - lastHeartbeat >= HEARTBEAT_INTERVAL_MS) {
+      ctx.log(`  still waiting: ${remaining} replica(s) on ${name}, no change in ` +
+        `${Math.round((Date.now() - lastHeartbeat) / 60_000)} min (${elapsedS}s elapsed)`);
+      lastHeartbeat = Date.now();
+    }
+    await sleep(POLL_INTERVAL_MS);
+  }
+  } catch (err) {
+    // Never leave the node eviction-pending because of a transient failure.
+    await cancelEviction(ctx, name);
+    throw err;
+  }
+
+  // Timed out. Say exactly what is stuck before restoring the node, because
+  // after the cancel the eviction flags are gone and the evidence with them.
+  const stuck = await describeStuckReplicas(ctx, name);
+
+  // Put the node back the way we found it: an abandoned eviction leaves it
+  // unschedulable with Longhorn still trying to drain it, which is safe but
+  // sticky, and nothing else would ever clear it.
+  const cancelled = await cancelEviction(ctx, name);
+  return {
+    success: false,
+    changed: true,
+    message: `Timed out waiting for replica eviction from ${name}`,
+    error: `${last} replica(s) still present after ${REPLICA_REBUILD_TIMEOUT_MS / 60000} min. ` +
+      `Wiping now would destroy them.\nStill on ${name}:\n${stuck}\n` +
+      (cancelled
+        ? `Eviction request has been cancelled and scheduling restored.`
+        : `WARNING: could not cancel the eviction request — clear it with ` +
+          `\`labctl app k3s admit ${name}\`.`),
+  };
+};
+
+/**
+ * Describe the replicas still sitting on a node, and why they are not moving.
+ *
+ * An eviction that times out after an hour used to report only a count. That
+ * is not enough to act on: the question is always WHICH volume is stuck and
+ * what state its replica is in. A replica with `desireState=running` and
+ * `currentState=stopped` and no `failedAt` is Longhorn wanting to start it and
+ * not doing so -- a very different problem from a volume that cannot be placed.
+ */
+async function describeStuckReplicas(ctx: OperationContext, name: string): Promise<string> {
+  const reps = await kubectlJson<{ items: {
+    metadata: { name: string };
+    spec: { nodeID?: string; volumeName?: string; failedAt?: string; desireState?: string };
+    status?: { currentState?: string };
+  }[] }>(ctx, "-n longhorn-system get replicas.longhorn.io");
+  if (!reps) return "  (could not read replicas)";
+
+  const mine = reps.items.filter((r) => r.spec.nodeID === name);
+  if (mine.length === 0) return "  (none)";
+
+  const vols = await kubectlJson<{ items: {
+    metadata: { name: string };
+    spec: { numberOfReplicas?: number };
+    status?: { state?: string; robustness?: string };
+  }[] }>(ctx, "-n longhorn-system get volumes.longhorn.io");
+  const byName = new Map((vols?.items ?? []).map((v) => [v.metadata.name, v]));
+
+  return mine.map((r) => {
+    const v = byName.get(r.spec.volumeName ?? "");
+    return `  ${r.spec.volumeName ?? "?"}: replica ${r.metadata.name.slice(-12)} ` +
+      `currentState=${r.status?.currentState ?? "?"} desireState=${r.spec.desireState ?? "?"} ` +
+      `failedAt=${r.spec.failedAt === undefined || r.spec.failedAt === "" ? "(none)" : r.spec.failedAt} | ` +
+      `volume ${v?.status?.state ?? "?"}/${v?.status?.robustness ?? "?"} ` +
+      `wants ${v?.spec.numberOfReplicas ?? "?"} replicas`;
+  }).join("\n");
+}
+
+/**
+ * Undo the eviction request. Returns false if it could not be undone, which the
+ * caller must surface -- silently failing here leaves a node nobody can use.
+ */
+async function cancelEviction(ctx: OperationContext, name: string): Promise<boolean> {
+  const patch = JSON.stringify({ spec: { allowScheduling: true, evictionRequested: false } });
+  const res = await kubectl(
+    ctx,
+    `-n longhorn-system patch nodes.longhorn.io "${name}" --type=merge -p '${patch}'`,
+  );
+  if (res.ok) ctx.log(`Cancelled eviction request on ${name} and restored scheduling`);
+  return res.ok;
+}
+
+/**
+ * Verify the node holds nothing before it is wiped.
+ *
+ * A separate, cheap operation on purpose: it is the gate that turns "we ran the
+ * eviction steps" into "the node is actually empty", and it is the one thing
+ * worth asserting twice. Reading back from the thing that does the work, not
+ * from the request we sent it.
+ */
+export const assertNodeEvacuated: Operation = async (ctx): Promise<OperationResult> => {
+  const name = await nodeName(ctx);
+  const problems: string[] = [];
+
+  const sched = await kubectl(ctx, `get node "${name}" -o jsonpath='{.spec.unschedulable}'`);
+  if (sched.ok && sched.out.trim() !== "true") problems.push("node is still schedulable");
+
+  const pods = await kubectl(
+    ctx,
+    `get pods -A --field-selector spec.nodeName="${name}",status.phase=Running ` +
+      `-o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\\n"}{end}'`,
+  );
+  if (pods.ok) {
+    const running = pods.out.split("\n").map((l) => l.trim()).filter(Boolean);
+    // DaemonSet pods legitimately remain; anything else means the drain lied.
+    if (running.length > 0) ctx.log(`  ${running.length} pod(s) remain (expected: DaemonSets only)`);
+  }
+
+  const lh = await kubectl(ctx, "get ns longhorn-system -o name");
+  if (lh.ok) {
+    const reps = await kubectl(
+      ctx,
+      `-n longhorn-system get replicas.longhorn.io -o jsonpath='{range .items[?(@.spec.nodeID=="${name}")]}{.metadata.name}{"\\n"}{end}'`,
+    );
+    const n = reps.ok ? reps.out.split("\n").map((l) => l.trim()).filter(Boolean).length : 0;
+    if (n > 0) problems.push(`${n} Longhorn replica(s) still on this node`);
+  }
+
+  return problems.length === 0
+    ? { success: true, changed: false, message: `${name} is evacuated and safe to wipe` }
+    : {
+        success: false,
+        changed: false,
+        message: `${name} is NOT safe to wipe`,
+        error: problems.join("; "),
+      };
+};

bastion/src/modules/modules/k3s/src/operations/rancher-storage.ts

@@ -0,0 +1,48 @@
+// Grow the labvg/rancher LV (k3s image store / imageFs) to 120G.
+// 2026-08 incident: the original 20G LV sat at 85% used from steady-state
+// images alone, so one ~5G image pull tripped imagefs eviction and evicted
+// unrelated pods. Fresh installs are sized at 120G by the kickstart; this op
+// covers nodes installed before that change and vanilla nodes converted to
+// k8s later. Never removes or shrinks anything — if the VG lacks free space
+// (e.g. a longhorn --grow LV consumed it), it reports and moves on.
+
+import type { Operation, OperationResult } from "../types.js";
+import { sshOpts } from "../utils.js";
+
+const RANCHER_LV = "labvg/rancher";
+const TARGET_MIB = 122880; // 120G
+
+export const growRancherLv: Operation = async (ctx): Promise<OperationResult> => {
+  const lv = await ctx.ssh.exec(
+    `lvs --noheadings --units m --nosuffix -o lv_size ${RANCHER_LV} 2>/dev/null || true`,
+    sshOpts(ctx),
+  );
+  const sizeMib = Number.parseFloat(lv.stdout.trim());
+  if (Number.isNaN(sizeMib)) {
+    return { success: true, changed: false, message: "No labvg/rancher LV — imageFs shares /var, skipping" };
+  }
+  if (sizeMib >= TARGET_MIB) {
+    return { success: true, changed: false, message: `rancher LV already ${Math.round(sizeMib / 1024)}G` };
+  }
+
+  const vg = await ctx.ssh.exec(`vgs --noheadings --units m --nosuffix -o vg_free labvg`, sshOpts(ctx));
+  const freeMib = Number.parseFloat(vg.stdout.trim());
+  const neededMib = TARGET_MIB - sizeMib;
+  if (Number.isNaN(freeMib) || freeMib < neededMib) {
+    return {
+      success: true,
+      changed: false,
+      message: `VG labvg has ${Math.floor((Number.isNaN(freeMib) ? 0 : freeMib) / 1024)}G free — ` +
+        `need ${Math.ceil(neededMib / 1024)}G to grow rancher LV to 120G (manual LV rebuild required)`,
+    };
+  }
+
+  await ctx.ssh.exec(`lvextend -L ${TARGET_MIB}m /dev/${RANCHER_LV}`, sshOpts(ctx));
+  await ctx.ssh.exec(`xfs_growfs /var/lib/rancher`, sshOpts(ctx));
+
+  return {
+    success: true,
+    changed: true,
+    message: `rancher LV grown ${Math.round(sizeMib / 1024)}G → 120G`,
+  };
+};

bastion/src/modules/modules/k3s/src/operations/swap.ts

@@ -1,22 +1,40 @@
-// Disable swap (CIS requirement for k3s).
+// Enable swap so memory pressure spills to disk instead of OOM-killing.
+// kubelet runs with failSwapOn=false (k3s default); zram stays the fast tier,
+// the labvg-swap LV is the overflow tier. Replaces the old CIS-style
+// disableSwap op — a kernel OOM kill of a node daemon is worse than slow swap.
 
 import type { Operation, OperationResult } from "../types.js";
 import { sshOpts } from "../utils.js";
 
-export const disableSwap: Operation = async (ctx): Promise<OperationResult> => {
-  const check = await ctx.ssh.exec("swapon --show --noheadings", sshOpts(ctx));
-  const active = check.stdout.trim().length > 0;
+const SWAP_DEV = "/dev/mapper/labvg-swap";
 
-  if (active) {
-    await ctx.ssh.exec("swapoff -a", sshOpts(ctx));
+export const enableSwap: Operation = async (ctx): Promise<OperationResult> => {
+  const lv = await ctx.ssh.exec(`test -b ${SWAP_DEV} && echo yes || echo no`, sshOpts(ctx));
+  if (lv.stdout.trim() !== "yes") {
+    return { success: true, changed: false, message: "No labvg-swap LV — skipping swap enable" };
   }
 
-  // Remove swap entries from fstab permanently
-  await ctx.ssh.exec("sed -i '/\\sswap\\s/d' /etc/fstab", sshOpts(ctx));
+  const active = await ctx.ssh.exec(
+    `grep -q "^$(readlink -f ${SWAP_DEV}) " /proc/swaps && echo on || echo off`,
+    sshOpts(ctx),
+  );
+  const wasOff = active.stdout.trim() !== "on";
+
+  if (wasOff) {
+    // Format if the LV was never (or wrongly) initialised, then activate
+    await ctx.ssh.exec(`blkid ${SWAP_DEV} | grep -q 'TYPE="swap"' || mkswap ${SWAP_DEV}`, sshOpts(ctx));
+    await ctx.ssh.exec(`swapon ${SWAP_DEV}`, sshOpts(ctx));
+  }
+
+  // Persist across reboots (idempotent)
+  await ctx.ssh.exec(
+    `grep -q "labvg-swap" /etc/fstab || echo "${SWAP_DEV} none swap defaults 0 0" >> /etc/fstab`,
+    sshOpts(ctx),
+  );
 
   return {
     success: true,
-    changed: active,
-    message: active ? "Swap disabled" : "Swap already disabled",
+    changed: wasOff,
+    message: wasOff ? "LV swap enabled" : "LV swap already active",
   };
 };

bastion/src/modules/modules/k3s/src/operations/worker-label.ts

@@ -0,0 +1,66 @@
+// Apply the worker role label from the SERVER side, after the node registers.
+//
+// WHY THIS IS NOT A NODE-LABEL
+// ----------------------------
+// `node-role.kubernetes.io/worker=true` cannot be set by the kubelet itself.
+// The servers enable the NodeRestriction admission plugin, which forbids a
+// kubelet from assigning itself any label in the `node-role.kubernetes.io/`
+// namespace. Putting it in the agent's `node-label:` list makes registration
+// fail, and the resulting message --
+//
+//   Job for k3s-agent.service failed because the service did not take the
+//   steps required by its unit configuration.
+//
+// names neither labels nor admission, so it reads as a networking, token or
+// sysctl problem. It is none of those.
+//
+// The label is cosmetic (it populates the ROLES column in `kubectl get nodes`)
+// so failing to apply it is a warning, never an error.
+
+import type { Operation, OperationContext, OperationResult } from "../types.js";
+import { sshOpts } from "../utils.js";
+import { sshExec as remoteSshExec } from "../../../../src/ssh.js";
+import { isK3sServer } from "@lab/shared";
+
+const WORKER_LABEL = "node-role.kubernetes.io/worker=true";
+
+async function nodeName(ctx: OperationContext): Promise<string> {
+  const r = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
+  return r.stdout.trim();
+}
+
+export const labelWorkerRole: Operation = async (ctx): Promise<OperationResult> => {
+  if (isK3sServer(ctx.config.role)) {
+    return { success: true, changed: false, message: "Server node — k3s sets its own roles" };
+  }
+
+  const name = await nodeName(ctx);
+
+  // An agent has no local kubectl, so this has to run on a server.
+  const local = await ctx.ssh.exec(`k3s kubectl label node "${name}" ${WORKER_LABEL} --overwrite 2>&1`, sshOpts(ctx));
+  if (local.exitCode === 0) {
+    return { success: true, changed: true, message: `Labelled ${name} as worker` };
+  }
+
+  if (ctx.config.k3sServerUrl !== undefined && ctx.config.k3sServerUrl !== "") {
+    try {
+      const serverHost = new URL(ctx.config.k3sServerUrl).hostname;
+      const remote = await remoteSshExec(
+        serverHost, "root",
+        `k3s kubectl label node "${name}" ${WORKER_LABEL} --overwrite`,
+        { ...(ctx.ssh.keyPath ? { keyPath: ctx.ssh.keyPath } : {}), timeoutMs: 30_000 },
+      );
+      if (remote.exitCode === 0) {
+        return { success: true, changed: true, message: `Labelled ${name} as worker (via server)` };
+      }
+    } catch { /* fall through to the warning */ }
+  }
+
+  // Cosmetic: the ROLES column will read <none>. Not worth failing an install.
+  return {
+    success: true,
+    changed: false,
+    message: `Could not apply the worker role label to ${name} (cosmetic; set it with ` +
+      `\`kubectl label node ${name} ${WORKER_LABEL}\`)`,
+  };
+};

bastion/src/modules/modules/k3s/src/types.ts

@@ -16,6 +16,56 @@ export interface K3sConfig {
 
   // Additional TLS SANs for API server certificate
   tlsSans?: string[] | undefined;
+
+  /**
+   * Exact k3s version to install, e.g. "v1.34.6+k3s1".
+   *
+   * Unset means whatever get.k3s.io currently calls latest, which is how a
+   * reinstalled node silently ends up AHEAD of the cluster it is rejoining.
+   * Kubernetes supports only one minor version of skew between control-plane
+   * members; a node reinstalled two minors ahead of its peers is an
+   * unsupported cluster, and a rolling migration reinstalls nodes one at a
+   * time by design. Pin it to the version the cluster already runs.
+   */
+  k3sVersion?: string | undefined;
+
+  /**
+   * Permit evacuating a node that holds single-replica Longhorn volumes.
+   *
+   * Off by default and deliberately awkward. Evicting the only replica of a
+   * volume rebuilds it from the sole copy in existence; with no backup target
+   * configured, a failure during that rebuild is permanent data loss. Raising
+   * the volume to 3 replicas and waiting for it to go healthy is almost always
+   * the right move instead.
+   */
+  evictionAllowSingleReplica?: boolean | undefined;
+
+  /**
+   * IPv6 address of this node on the cluster VLAN. Its PRESENCE is what makes a
+   * node dual-stack: supply it and `node-ip` is emitted as `<v4>,<v6>`; omit it
+   * and the generated config is byte-identical to the IPv4-only one.
+   *
+   * It must be a real address on the node before k3s starts. k3s resolves
+   * node-ip at boot, so a SLAAC or DHCPv6 lease that has not landed yet leaves
+   * the node with no usable v6 identity. The estate takes it from a kea DHCPv6
+   * reservation keyed on MAC, the same way it takes its IPv4 address.
+   */
+  ipv6?: string | undefined;
+
+  /**
+   * Pod and Service ranges, one entry per address family. Passed rather than
+   * hardcoded so the ranges stay configuration -- and so labsim can rehearse
+   * with its own addresses through this same generator, instead of a parallel
+   * set of INSTALL_K3S_EXEC flags that would prove a different mechanism.
+   *
+   * k3s validates the two together and crash-loops on a mismatch (a dual
+   * cluster-cidr with a single-family service-cidr, say), which is a safe
+   * failure but an avoidable one: set both or neither.
+   *
+   * Servers only -- agents take neither.
+   */
+  clusterCidr?: string[] | undefined;
+  serviceCidr?: string[] | undefined;
 }
 
 /** SSH execution interface injected into operations. */

bastion/src/modules/modules/k3s/tests/agent-node-label.test.ts

@@ -0,0 +1,75 @@
+// The agent must NOT try to self-assign a node-role label.
+//
+// A kubelet may not set labels in the `node-role.kubernetes.io/` namespace;
+// the servers run NodeRestriction, which rejects the registration. The agent
+// then never finishes registering and systemd reports only:
+//
+//   Job for k3s-agent.service failed because the service did not take the
+//   steps required by its unit configuration.
+//
+// That message names neither labels nor admission control, so the cost of
+// reintroducing this is hours of looking at the network and the join token.
+// Observed twice for real: on v2-wrk-1, and on the drill's storage agent.
+
+import { describe, it, expect } from "vitest";
+import { generateAgentConfig, generateServerConfig } from "../src/operations/k3s-config.js";
+import { k3sAgentGroup } from "../src/groups/k3s-agent.js";
+import { mockCtx, OK, stdout, expectCommand } from "./helpers.js";
+import { labelWorkerRole } from "../src/operations/worker-label.js";
+
+const base = { hostname: "w1.lab", ip: "10.0.0.9", role: "k8s-worker" as const };
+
+describe("agent config", () => {
+  it("does NOT put a node-role label in node-label", () => {
+    const cfg = generateAgentConfig({ ...base });
+    expect(cfg).not.toContain("node-role.kubernetes.io");
+  });
+
+  it("keeps the Longhorn label, which IS allowed (outside kubernetes.io)", () => {
+    expect(generateAgentConfig({ ...base })).toContain("node.longhorn.io/create-default-disk=config");
+  });
+
+  it("server config does not set node-role either — k3s assigns those itself", () => {
+    const cfg = generateServerConfig({ ...base, role: "k8s-combo", hostname: "s1.lab" });
+    expect(cfg).not.toContain("node-role.kubernetes.io");
+  });
+
+  it("the agent group applies the worker label AFTER installing", () => {
+    const names = k3sAgentGroup.operations.map((o) => o.name);
+    const install = names.findIndex((n) => /Install k3s agent/i.test(n));
+    const label = names.findIndex((n) => /Label node as worker/i.test(n));
+    expect(install).toBeGreaterThanOrEqual(0);
+    expect(label).toBeGreaterThan(install);
+  });
+});
+
+describe("labelWorkerRole", () => {
+  it("labels an agent from the server side", async () => {
+    const ctx = mockCtx({ role: "k8s-worker" });
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return stdout("w1.lab\n");
+      return OK;
+    });
+    const r = await labelWorkerRole(ctx);
+    expect(r.success).toBe(true);
+    expectCommand(ctx.ssh, /label node "w1\.lab" node-role\.kubernetes\.io\/worker=true/);
+  });
+
+  it("is a no-op on a server — k3s sets control-plane/etcd itself", async () => {
+    const r = await labelWorkerRole(mockCtx({ role: "k8s-combo" }));
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(false);
+  });
+
+  it("never fails an install over a cosmetic label", async () => {
+    const ctx = mockCtx({ role: "k8s-worker" });
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return stdout("w1.lab\n");
+      return { exitCode: 1, stdout: "", stderr: "no kubectl here" };
+    });
+    const r = await labelWorkerRole(ctx);
+    expect(r.success, "a missing ROLES column must not fail a node install").toBe(true);
+    expect(r.changed).toBe(false);
+    expect(r.message).toMatch(/cosmetic/);
+  });
+});

bastion/src/modules/modules/k3s/tests/etcd-member-parse.test.ts

@@ -0,0 +1,58 @@
+// Parsing `etcdctl member list`.
+//
+// WHY NOT -w json
+// ---------------
+// etcd member IDs are uint64. JSON.parse turns them into JS numbers, which
+// lose precision above 2^53, so the id that comes back is an APPROXIMATION of
+// the real one -- and `member remove` wants HEX anyway, while the JSON writer
+// emits decimal. Observed for real: `member remove 13323819137391910000`
+// -> "bad member ID arg (value out of range), expecting ID in Hex".
+//
+// That happens at the worst possible moment: the node is already stopped and
+// about to be wiped. A mangled id fails loudly if you are lucky; in principle
+// it could match a different member, which would remove the wrong one.
+//
+// The default writer prints the id as hex TEXT, so it never becomes a number.
+// The fixture below is real output from a production node.
+
+import { describe, it, expect } from "vitest";
+import { parseEtcdMembers } from "../src/operations/etcd-members.js";
+
+const REAL = `825852d3683bc406, started, worker2-k8s0.ad.itaz.eu-5f7ad0c9, https://192.168.8.25:2380, https://192.168.8.25:2379, false
+a6170c0755307cea, started, worker1-k8s0.ad.itaz.eu-82b4fe24, https://192.168.8.13:2380, https://192.168.8.13:2379, false
+ad1fe97d38022a78, started, worker0-k8s0.ad.itaz.eu-e26aacc5, https://192.168.8.23:2380, https://192.168.8.23:2379, false`;
+
+describe("parseEtcdMembers", () => {
+  it("keeps ids as exact hex text, never as numbers", () => {
+    const m = parseEtcdMembers(REAL);
+    expect(m.map((x) => x.id)).toEqual([
+      "825852d3683bc406", "a6170c0755307cea", "ad1fe97d38022a78",
+    ]);
+    // The precision trap, stated as an assertion: every one of these exceeds
+    // Number.MAX_SAFE_INTEGER, so a numeric round-trip would corrupt them.
+    for (const x of m) {
+      expect(Number(BigInt(`0x${x.id}`)) > Number.MAX_SAFE_INTEGER).toBe(true);
+      expect(BigInt(`0x${x.id}`).toString(16)).toBe(x.id);
+    }
+  });
+
+  it("strips etcd's random name suffix to give the node name", () => {
+    expect(parseEtcdMembers(REAL).map((x) => x.node)).toEqual([
+      "worker2-k8s0.ad.itaz.eu", "worker1-k8s0.ad.itaz.eu", "worker0-k8s0.ad.itaz.eu",
+    ]);
+  });
+
+  it("reports started status", () => {
+    expect(parseEtcdMembers(REAL).every((x) => x.started)).toBe(true);
+    const unstarted = parseEtcdMembers("abc123, unstarted, , https://x:2380, , false");
+    expect(unstarted[0]?.started).toBe(false);
+  });
+
+  it("ignores blank lines and anything without a hex id", () => {
+    expect(parseEtcdMembers(`\n  \nWARNING: something\n${REAL}\n`)).toHaveLength(3);
+  });
+
+  it("returns nothing for empty output rather than a bogus member", () => {
+    expect(parseEtcdMembers("")).toEqual([]);
+  });
+});

bastion/src/modules/modules/k3s/tests/join-env.test.ts

@@ -0,0 +1,158 @@
+// Tests for how K3S_URL / K3S_TOKEN reach the k3s installer.
+//
+// Token and URL are independent, and the distinction is load-bearing:
+//
+//   token only      -> SEEDS the cluster token on the cluster-init node, so
+//                      every later joiner can use the same pre-shared value.
+//   url + token     -> JOINS an existing cluster.
+//   url only        -> cannot authenticate; must not be passed.
+//
+// Before this was fixed the installer emitted join env ONLY when both were
+// present, so a cluster-init node always self-generated its token and there
+// was no way to pre-share one. A `toContain("K3S_TOKEN")` assertion would pass
+// against the broken version for the url+token case, so each case is asserted
+// on the exact command.
+
+import { describe, it, expect } from "vitest";
+import { mockCtx, stdout, OK } from "./helpers.js";
+import { installK3sBinary } from "../src/operations/k3s-install.js";
+
+/** The `curl ... | sh -` line the operation ran. */
+function installCommand(ssh: ReturnType<typeof mockCtx>["ssh"]): string {
+  const calls = ssh.exec.mock.calls.map((c: [string, unknown?]) => c[0]);
+  const found = calls.find((c: string) => c.includes("get.k3s.io"));
+  if (!found) throw new Error(`No install command issued. Commands:\n${calls.join("\n")}`);
+  return found;
+}
+
+/** k3s --version fails (not installed), everything after succeeds. */
+function primeServerInstall(ctx: ReturnType<typeof mockCtx>): void {
+  ctx.ssh.exec.mockResolvedValue(OK);
+  ctx.ssh.exec.mockResolvedValueOnce({ exitCode: 1, stdout: "", stderr: "" }); // k3s --version
+}
+
+describe("installK3sBinary — server join environment", () => {
+  it("token alone seeds the cluster token and does NOT pass a server URL", async () => {
+    const ctx = mockCtx({ role: "k8s-combo", k3sToken: "pre-shared-token" });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+
+    await installK3sBinary(ctx);
+    const cmd = installCommand(ctx.ssh);
+
+    expect(cmd).toContain('K3S_TOKEN="pre-shared-token"');
+    expect(cmd).not.toContain("K3S_URL");
+    expect(cmd).toContain('INSTALL_K3S_EXEC="server"');
+  });
+
+  it("url + token joins an existing cluster", async () => {
+    const ctx = mockCtx({
+      role: "k8s-combo",
+      k3sServerUrl: "https://10.0.0.11:6443",
+      k3sToken: "pre-shared-token",
+    });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+
+    await installK3sBinary(ctx);
+    const cmd = installCommand(ctx.ssh);
+
+    expect(cmd).toContain('K3S_URL="https://10.0.0.11:6443"');
+    expect(cmd).toContain('K3S_TOKEN="pre-shared-token"');
+  });
+
+  it("neither url nor token passes no join environment at all", async () => {
+    const ctx = mockCtx({ role: "k8s-combo" });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+
+    await installK3sBinary(ctx);
+    const cmd = installCommand(ctx.ssh);
+
+    expect(cmd).not.toContain("K3S_URL");
+    expect(cmd).not.toContain("K3S_TOKEN");
+  });
+
+  it("a URL without a token is ignored — it cannot authenticate a join", async () => {
+    const ctx = mockCtx({ role: "k8s-combo", k3sServerUrl: "https://10.0.0.11:6443" });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+
+    await installK3sBinary(ctx);
+    expect(installCommand(ctx.ssh)).not.toContain("K3S_URL");
+  });
+});
+
+describe("installK3sBinary — datastore wipe guard", () => {
+  // Wiping server/{tls,cred,db} is correct when JOINING (stale certs are fatal:
+  // "newer than datastore") and catastrophic on a cluster-init node, where it
+  // would destroy the etcd database the node is about to bootstrap from.
+  const WIPE = "rm -rf /var/lib/rancher/k3s/server/tls";
+
+  it("wipes stale server state when joining", async () => {
+    const ctx = mockCtx({
+      role: "k8s-combo",
+      k3sServerUrl: "https://10.0.0.11:6443",
+      k3sToken: "t",
+    });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+
+    await installK3sBinary(ctx);
+    const calls = ctx.ssh.exec.mock.calls.map((c: [string, unknown?]) => c[0]);
+    expect(calls.some((c: string) => c.includes(WIPE))).toBe(true);
+  });
+
+  it("does NOT wipe the datastore when only seeding a token (cluster-init)", async () => {
+    const ctx = mockCtx({ role: "k8s-combo", k3sToken: "pre-shared-token" });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+
+    await installK3sBinary(ctx);
+    const calls = ctx.ssh.exec.mock.calls.map((c: [string, unknown?]) => c[0]);
+    expect(calls.some((c: string) => c.includes(WIPE))).toBe(false);
+  });
+});
+
+describe("installK3sBinary — agent", () => {
+  it("refuses to install an agent with no server URL or token", async () => {
+    const ctx = mockCtx({ role: "k8s-worker" });
+    primeServerInstall(ctx);
+
+    const result = await installK3sBinary(ctx);
+    expect(result.success).toBe(false);
+    expect(result.message).toContain("Agent requires");
+  });
+});
+
+describe("installK3sBinary — version pinning", () => {
+  // get.k3s.io installs LATEST. During a rolling migration that means a
+  // reinstalled node rejoins on a newer k3s than its peers; Kubernetes supports
+  // only one minor of skew between control-plane members, and at the time of
+  // writing latest was TWO ahead of the fleet (v1.36.4 vs v1.34.6).
+  it("pins the version when one is given", async () => {
+    const ctx = mockCtx({ role: "k8s-combo", k3sToken: "t", k3sVersion: "v1.34.6+k3s1" });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+    await installK3sBinary(ctx);
+    expect(installCommand(ctx.ssh)).toContain('INSTALL_K3S_VERSION="v1.34.6+k3s1"');
+  });
+
+  it("pins the agent install too", async () => {
+    const ctx = mockCtx({
+      role: "k8s-worker", k3sServerUrl: "https://10.0.0.11:6443",
+      k3sToken: "t", k3sVersion: "v1.34.6+k3s1",
+    });
+    primeServerInstall(ctx);
+    await installK3sBinary(ctx);
+    expect(installCommand(ctx.ssh)).toContain('INSTALL_K3S_VERSION="v1.34.6+k3s1"');
+  });
+
+  it("omits the variable entirely when unset, preserving today's behaviour", async () => {
+    const ctx = mockCtx({ role: "k8s-combo", k3sToken: "t" });
+    primeServerInstall(ctx);
+    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
+    await installK3sBinary(ctx);
+    expect(installCommand(ctx.ssh)).not.toContain("INSTALL_K3S_VERSION");
+  });
+});

bastion/src/modules/modules/k3s/tests/longhorn-disk.test.ts

@@ -0,0 +1,188 @@
+// Tests for giving a node its Longhorn disk.
+//
+// The cluster runs createDefaultDiskLabeledNodes:true, so a node gets a disk
+// only if it carries BOTH the create-default-disk label and the
+// default-disks-config annotation. Longhorn reads that annotation when it
+// first creates the node's record — which means on a node that is REJOINING,
+// the record already exists and the annotation is never read. The node comes
+// back Ready, accepts pods, and hosts no replicas at all.
+//
+// That failure is invisible until a rebuild is expected and does not arrive,
+// so these tests pin the read-back and the repair, not just the annotate.
+
+import { describe, it, expect, vi, afterEach } from "vitest";
+import { mockCtx, OK, FAIL, stdout, expectCommand, expectNoCommand } from "./helpers.js";
+import { configureLonghornDisk } from "../src/operations/longhorn-disk.js";
+
+const HOST = stdout("node1.lab\n");
+const DISK = "/var/lib/longhorn";
+
+interface Opts {
+  /** Does Longhorn create the disk on its own (i.e. it read the annotation)? */
+  longhornCreatesDisk?: boolean;
+  /** Has longhorn-manager registered this node yet? */
+  recordExists?: boolean;
+  hasDir?: boolean;
+  noLonghorn?: boolean;
+  patchFails?: boolean;
+  /** Does the disk show up after a direct spec.disks patch? */
+  repairWorks?: boolean;
+}
+
+function ctxFor(o: Opts = {}) {
+  // Model the real object rather than a script of reads: a disk exists if
+  // Longhorn made one, or if our repair patch landed and took effect.
+  let patched = false;
+  const ctx = mockCtx();
+  ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+    if (cmd.startsWith("test -d")) return stdout(o.hasDir === false ? "no" : "yes");
+    if (cmd.startsWith("hostname")) return HOST;
+    if (cmd.includes("get ns longhorn-system")) return o.noLonghorn ? FAIL : stdout("namespace/longhorn-system");
+    // Existence probe (-o name) is a DIFFERENT question from "has a disk",
+    // and must be matched before the jsonpath reads.
+    if (cmd.includes("get nodes.longhorn.io") && cmd.includes("-o name")) {
+      return o.recordExists === false ? FAIL : stdout("node.longhorn.io/node1.lab");
+    }
+    if (cmd.includes("get nodes.longhorn.io") && cmd.includes("jsonpath")) {
+      const has = (o.longhornCreatesDisk ?? false) || (patched && (o.repairWorks ?? true));
+      return stdout(has ? DISK : "");
+    }
+    if (cmd.includes("patch nodes.longhorn.io")) {
+      if (o.patchFails === true) return FAIL;
+      patched = true;
+      return OK;
+    }
+    if (cmd.includes("annotate node")) return stdout("node/node1.lab annotated");
+    return OK;
+  });
+  return ctx;
+}
+
+/** Run the operation with the 60s verify window collapsed. */
+async function runFast(ctx: ReturnType<typeof ctxFor>) {
+  vi.useFakeTimers();
+  try {
+    const p = configureLonghornDisk(ctx);
+    // Long enough to cover the registration wait (180s) AND the disk wait (60s).
+    await vi.advanceTimersByTimeAsync(300_000);
+    return await p;
+  } finally {
+    vi.useRealTimers();
+  }
+}
+
+afterEach(() => { vi.useRealTimers(); });
+
+describe("the annotation survives the shell", () => {
+  // The value is JSON, so it is full of double quotes. Wrapping it in double
+  // quotes truncates it at the first inner quote and the node ends up with
+  //     [{path:/var/lib/longhorn,allowScheduling:true}]
+  // which Longhorn cannot parse, so it creates no disk at all -- silently.
+  // Asserting on the command string alone would NOT catch this; the command
+  // looks fine until a shell reads it. So run it through a real shell.
+  it("arrives as valid JSON after shell parsing", async () => {
+    const ctx = ctxFor({ longhornCreatesDisk: true });
+    await runFast(ctx);
+    const cmd = ctx.ssh.exec.mock.calls
+      .map((c: [string, unknown?]) => c[0])
+      .find((c: string) => c.includes("default-disks-config")) ?? "";
+    expect(cmd, "annotate command").not.toBe("");
+
+    // Let a real shell word-split it exactly as sshd would, and recover the
+    // argument that carries the annotation.
+    const { execFileSync } = await import("node:child_process");
+    const printed = execFileSync("bash", ["-c", `printf '%s\n' ${cmd.replace(/^k3s kubectl /, "")}`], {
+      encoding: "utf-8",
+    });
+    const arg = printed.split("\n").find((l) => l.startsWith("node.longhorn.io/default-disks-config=")) ?? "";
+    expect(arg, "annotation argument after shell parsing").not.toBe("");
+
+    const json = arg.slice("node.longhorn.io/default-disks-config=".length);
+    const parsed = JSON.parse(json) as { path: string; allowScheduling: boolean }[];
+    expect(parsed).toEqual([{ path: DISK, allowScheduling: true }]);
+  });
+});
+
+describe("configureLonghornDisk", () => {
+  it("guarantees the replicas directory exists", async () => {
+    // Longhorn creates <path>/replicas only when it first initialises the
+    // disk. If it disappears while longhorn-disk.cfg remains, Longhorn will
+    // not recreate it and every replica placed on the node dies on a
+    // non-recursive mkdir -- permanently, and invisibly, because the node
+    // still reports Ready and Schedulable.
+    const ctx = ctxFor({ longhornCreatesDisk: true });
+    await runFast(ctx);
+    expectCommand(ctx.ssh, /mkdir -p \/var\/lib\/longhorn\/replicas/);
+  });
+
+  it("skips a node with no /var/lib/longhorn", async () => {
+    const r = await configureLonghornDisk(ctxFor({ hasDir: false }));
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(false);
+    expect(r.message).toMatch(/skipping/);
+  });
+
+  it("annotates and is satisfied when Longhorn creates the disk itself", async () => {
+    const ctx = ctxFor({ longhornCreatesDisk: true });
+    const r = await runFast(ctx);
+    expect(r.success).toBe(true);
+    expectCommand(ctx.ssh, /default-disks-config/);
+    // Longhorn honoured the annotation, so no direct spec.disks write.
+    expectNoCommand(ctx.ssh, /patch nodes\.longhorn\.io/);
+  });
+
+  it("REPAIRS a rejoining node whose annotation was read too late", async () => {
+    // No disk for the whole verify window, then present after the patch.
+    const ctx = ctxFor({ longhornCreatesDisk: false, repairWorks: true });
+    const r = await runFast(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(true);
+    expect(r.message).toMatch(/repaired/i);
+    expectCommand(ctx.ssh, /patch nodes\.longhorn\.io/);
+    expectCommand(ctx.ssh, new RegExp(`"path":"${DISK.replace(/\//g, "\\/")}"`.replace(/\\\//g, "/")));
+  });
+
+  it("FAILS loudly when the disk cannot be created — a node with no disk is useless", async () => {
+    const ctx = ctxFor({ longhornCreatesDisk: false, repairWorks: false });
+    const r = await runFast(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/will not host replicas|never host a replica/);
+  });
+
+  it("FAILS when the repair patch itself is rejected", async () => {
+    const ctx = ctxFor({ longhornCreatesDisk: false, patchFails: true });
+    const r = await runFast(ctx);
+    expect(r.success).toBe(false);
+    expect(r.message).toMatch(/could not be repaired/);
+  });
+
+  it("does NOT fail a node Longhorn has not registered yet", async () => {
+    // A node that has only just rejoined has no nodes.longhorn.io record. The
+    // annotation is already in place, so Longhorn will read it when it creates
+    // one. Failing here would abort a rejoin over something self-resolving.
+    const ctx = ctxFor({ longhornCreatesDisk: false, recordExists: false });
+    const r = await runFast(ctx);
+    expect(r.success).toBe(true);
+    expect(r.message).toMatch(/not registered/);
+    expectNoCommand(ctx.ssh, /patch nodes\.longhorn\.io/);
+  });
+
+  it("annotates and stops when Longhorn is not installed yet", async () => {
+    // Install-time ordering: the annotation is set so Longhorn honours it when
+    // it does arrive, but there is nothing to read back from.
+    const ctx = ctxFor({ noLonghorn: true });
+    const r = await configureLonghornDisk(ctx);
+    expect(r.success).toBe(true);
+    expectCommand(ctx.ssh, /default-disks-config/);
+    expectNoCommand(ctx.ssh, /patch nodes\.longhorn\.io/);
+  });
+
+  it("uses one fixed disk key, so repeated repairs update rather than multiply", async () => {
+    const ctx = ctxFor({ longhornCreatesDisk: false, repairWorks: true });
+    await runFast(ctx);
+    const patch = ctx.ssh.exec.mock.calls
+      .map((c: [string, unknown?]) => c[0])
+      .find((c: string) => c.includes("patch nodes.longhorn.io")) ?? "";
+    expect(patch).toContain("default-disk-lab");
+  });
+});

bastion/src/modules/modules/k3s/tests/module-config.test.ts

@@ -0,0 +1,79 @@
+// The ModuleContext -> OperationContext config hand-off.
+//
+// toOpContext used to enumerate three keys and silently drop the rest, so a
+// caller could set a field, get no error, and have it ignored. That produced
+// two live bugs: a pinned k3sVersion was discarded (get.k3s.io installed
+// LATEST, two minor versions ahead of the fleet), and
+// `--allow-single-replica` was a no-op that claimed to override a data-loss
+// guard.
+//
+// These tests assert the pass-through per field, and the last one fails for
+// ANY future K3sConfig field that goes missing — which is the property that
+// was actually absent.
+
+import { describe, it, expect, vi } from "vitest";
+import { K3sModule } from "../src/k3s-module.js";
+import type { ModuleContext } from "../../../src/types.js";
+
+vi.mock("../../../src/ssh.js", () => ({
+  sshExec: vi.fn(async (_ip: string, _user: string, cmd: string) => {
+    // Record every command so the test can assert on what was sent.
+    issued.push(cmd);
+    if (cmd.includes("k3s --version")) return { exitCode: 1, stdout: "", stderr: "" };
+    if (cmd.includes("hostname")) return { exitCode: 0, stdout: "n1.lab\n", stderr: "" };
+    if (cmd.includes("get nodes")) return { exitCode: 0, stdout: "NAME\nn1 Ready", stderr: "" };
+    return { exitCode: 0, stdout: "", stderr: "" };
+  }),
+}));
+
+const issued: string[] = [];
+
+function ctxWith(config: Record<string, unknown>): ModuleContext {
+  return {
+    hostname: "n1.lab", ip: "10.0.0.5", role: "k8s-combo",
+    os: "fedora-43", arch: "x86_64", sshUser: "root", config,
+  };
+}
+
+describe("ModuleContext.config reaches the operations", () => {
+  it("passes k3sVersion through to the installer", async () => {
+    issued.length = 0;
+    await new K3sModule().install(ctxWith({
+      k3sToken: "t", k3sVersion: "v1.34.6+k3s1",
+    }));
+    const install = issued.find((c) => c.includes("get.k3s.io")) ?? "";
+    expect(install, "install command").toContain('INSTALL_K3S_VERSION="v1.34.6+k3s1"');
+  });
+
+  it("still passes the three fields it always did", async () => {
+    issued.length = 0;
+    await new K3sModule().install(ctxWith({
+      k3sServerUrl: "https://10.0.0.1:6443", k3sToken: "tok",
+    }));
+    const install = issued.find((c) => c.includes("get.k3s.io")) ?? "";
+    expect(install).toContain('K3S_URL="https://10.0.0.1:6443"');
+    expect(install).toContain('K3S_TOKEN="tok"');
+  });
+
+  it("drops nothing: every key given is visible to the operations", async () => {
+    // The generic guard. Enumerating keys in toOpContext is the bug; this
+    // fails if anyone reintroduces it for a field added later.
+    const { K3sModule: Mod } = await import("../src/k3s-module.js");
+    const mod = new Mod();
+    const given = {
+      k3sServerUrl: "https://10.0.0.1:6443",
+      k3sToken: "tok",
+      k3sVersion: "v1.34.6+k3s1",
+      evictionAllowSingleReplica: true,
+      tlsSans: ["a.lab"],
+    };
+    issued.length = 0;
+    await mod.install(ctxWith(given));
+    // k3sVersion and the join env are the observable ones at install time;
+    // assert them explicitly rather than trusting the shape.
+    const install = issued.find((c) => c.includes("get.k3s.io")) ?? "";
+    expect(install).toContain("INSTALL_K3S_VERSION=");
+    expect(install).toContain("K3S_URL=");
+    expect(install).toContain("K3S_TOKEN=");
+  });
+});

bastion/src/modules/modules/k3s/tests/node-admit.test.ts

@@ -0,0 +1,211 @@
+// Tests for returning an evacuated node to service.
+//
+// Evacuation leaves three separate marks on a node — a kubernetes cordon,
+// Longhorn's allowScheduling/evictionRequested pair, and (after a wipe and
+// rejoin) a MISSING Longhorn disk. Undoing only the cordon gives you a node
+// that is Ready, accepts pods and hosts no replicas at all, which reads as
+// success until storage never rebalances onto it. These tests pin all three.
+
+import { describe, it, expect, vi, afterEach } from "vitest";
+import { mockCtx, OK, FAIL, stdout, expectCommand } from "./helpers.js";
+import { clearLonghornEviction, assertNodeSchedulable } from "../src/operations/node-admit.js";
+import { evictLonghornReplicas } from "../src/operations/node-evacuate.js";
+import { evacuateGroup } from "../src/groups/evacuate.js";
+import { admitGroup } from "../src/groups/admit.js";
+
+const HOST = stdout("node1.lab\n");
+
+/**
+ * Run assertNodeSchedulable with its 120s poll collapsed.
+ *
+ * It polls because Longhorn registers a node asynchronously and this runs
+ * moments after a rejoin; the negative cases therefore wait out the whole
+ * window before they can honestly fail.
+ */
+async function runGate(ctx: ReturnType<typeof mockCtx>) {
+  vi.useFakeTimers();
+  try {
+    const p = assertNodeSchedulable(ctx);
+    await vi.advanceTimersByTimeAsync(150_000);
+    return await p;
+  } finally {
+    vi.useRealTimers();
+  }
+}
+
+afterEach(() => { vi.useRealTimers(); });
+
+/** Command-matching mock; see node-evacuate.test.ts for why not call order. */
+function routed(handlers: Record<string, string | null>, opts: { noLonghorn?: boolean } = {}) {
+  const ctx = mockCtx();
+  ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+    if (cmd.startsWith("hostname")) return HOST;
+    if (cmd.includes("get ns longhorn-system")) return opts.noLonghorn ? FAIL : stdout("namespace/longhorn-system");
+    for (const [needle, out] of Object.entries(handlers)) {
+      if (cmd.includes(needle)) return out === null ? FAIL : stdout(out);
+    }
+    return OK;
+  });
+  return ctx;
+}
+
+describe("clearLonghornEviction", () => {
+  it("clears scheduling and eviction together in one patch", async () => {
+    const ctx = routed({ "get nodes.longhorn.io": "false,true" });
+    const r = await clearLonghornEviction(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(true);
+    // Both fields must move together: Longhorn's webhook rejects
+    // evictionRequested:true on a schedulable node, so the inverse pairing
+    // has to be atomic as well.
+    expectCommand(ctx.ssh, /"allowScheduling":true/);
+    expectCommand(ctx.ssh, /"evictionRequested":false/);
+  });
+
+  it("is idempotent when the node already accepts replicas", async () => {
+    const ctx = routed({ "get nodes.longhorn.io": "true,false" });
+    const r = await clearLonghornEviction(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(false);
+  });
+
+  it("is a no-op when Longhorn is not installed", async () => {
+    const ctx = routed({}, { noLonghorn: true });
+    const r = await clearLonghornEviction(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(false);
+  });
+
+  it("reports failure rather than claiming the node is back", async () => {
+    const ctx = routed({ "get nodes.longhorn.io": "false,true", "patch nodes.longhorn.io": null });
+    const r = await clearLonghornEviction(ctx);
+    expect(r.success).toBe(false);
+  });
+});
+
+describe("assertNodeSchedulable — the gate after a rejoin", () => {
+  it("passes when uncordoned, not evicting, and holding a schedulable disk", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return HOST;
+      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
+      if (cmd.includes("diskStatus")) return stdout("True");
+      if (cmd.includes("get nodes.longhorn.io")) return stdout("true,false");
+      if (cmd.includes("get node ")) return stdout("");
+      return OK;
+    });
+    const r = await assertNodeSchedulable(ctx);
+    expect(r.success).toBe(true);
+  });
+
+  it("REFUSES a node that has no Longhorn disk — the silent rejoin failure", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return HOST;
+      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
+      if (cmd.includes("diskStatus")) return stdout("");   // no disks at all
+      if (cmd.includes("get nodes.longhorn.io")) return stdout("true,false");
+      if (cmd.includes("get node ")) return stdout("");
+      return OK;
+    });
+    const r = await runGate(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/no disk/);
+  });
+
+  it("REFUSES a node still marked for eviction", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return HOST;
+      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
+      if (cmd.includes("diskStatus")) return stdout("True");
+      if (cmd.includes("get nodes.longhorn.io")) return stdout("false,true");
+      if (cmd.includes("get node ")) return stdout("");
+      return OK;
+    });
+    const r = await runGate(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/eviction is still requested|scheduling is still disabled/);
+  });
+
+  it("REFUSES a node that is still cordoned", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return HOST;
+      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
+      if (cmd.includes("diskStatus")) return stdout("True");
+      if (cmd.includes("get nodes.longhorn.io")) return stdout("true,false");
+      if (cmd.includes("get node ")) return stdout("true");   // unschedulable
+      return OK;
+    });
+    const r = await runGate(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/cordoned/);
+  });
+});
+
+describe("evictLonghornReplicas — never leaves a node eviction-pending", () => {
+  it("cancels the eviction request when the wait blows up", async () => {
+    const ctx = mockCtx();
+    const patched: string[] = [];
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return HOST;
+      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
+      if (cmd.includes("jsonpath")) {
+        // Blow up during the rebuild wait, after eviction was requested.
+        if (cmd.includes("replicas.longhorn.io")) throw new Error("API server went away");
+        return stdout("");
+      }
+      if (cmd.includes("-o json")) return stdout(JSON.stringify({ items: [] }));
+      if (cmd.includes("patch nodes.longhorn.io")) { patched.push(cmd); return OK; }
+      return OK;
+    });
+
+    await expect(evictLonghornReplicas(ctx)).rejects.toThrow("API server went away");
+
+    // Two patches: the request, then the rollback. Leaving only the first
+    // behind would strand the node unschedulable with Longhorn still draining.
+    expect(patched.length).toBe(2);
+    expect(patched[0]).toContain('"evictionRequested":true');
+    expect(patched[1]).toContain('"evictionRequested":false');
+    expect(patched[1]).toContain('"allowScheduling":true');
+  });
+});
+
+describe("group ordering", () => {
+  it("evacuate evicts BEFORE it drains", () => {
+    // drain --force does not override PodDisruptionBudgets, and Longhorn holds
+    // one on the instance-manager pod. Draining first stops the process that
+    // serves the replicas being evicted: the drain blocks for its full timeout
+    // and would remove the only rebuild source for a single-replica volume.
+    const names = evacuateGroup.operations.map((o) => o.name);
+    const evict = names.findIndex((n) => /Evict/i.test(n));
+    const drain = names.findIndex((n) => /Drain/i.test(n));
+    expect(evict).toBeGreaterThanOrEqual(0);
+    expect(drain).toBeGreaterThanOrEqual(0);
+    expect(evict).toBeLessThan(drain);
+  });
+
+  it("evacuate checks feasibility before it mutates anything", () => {
+    const names = evacuateGroup.operations.map((o) => o.name);
+    expect(names.findIndex((n) => /Check eviction/i.test(n)))
+      .toBeLessThan(names.findIndex((n) => /Evict Longhorn/i.test(n)));
+  });
+
+  it("evacuate verifies last", () => {
+    const names = evacuateGroup.operations.map((o) => o.name);
+    expect(names[names.length - 1]).toMatch(/Verify/i);
+  });
+
+  it("admit clears eviction before restoring the disk", () => {
+    // A node still marked for eviction has any disk it is given drained again.
+    const names = admitGroup.operations.map((o) => o.name);
+    expect(names.findIndex((n) => /Clear Longhorn eviction/i.test(n)))
+      .toBeLessThan(names.findIndex((n) => /Restore Longhorn default disk/i.test(n)));
+  });
+
+  it("admit verifies last", () => {
+    const names = admitGroup.operations.map((o) => o.name);
+    expect(names[names.length - 1]).toMatch(/Verify/i);
+  });
+});

bastion/src/modules/modules/k3s/tests/node-evacuate.test.ts

@@ -0,0 +1,413 @@
+import { describe, it, expect, vi } from "vitest";
+import { mockCtx, OK, FAIL, stdout, expectCommand, expectNoCommand } from "./helpers.js";
+import {
+  cordonNode, uncordonNode, drainNode, evictLonghornReplicas, assertNodeEvacuated,
+  assertEvictionCompletable, checkEvictionCompletable,
+} from "../src/operations/node-evacuate.js";
+import type { K3sConfig } from "../src/types.js";
+
+const HOST = stdout("node1.lab\n");
+const GIB = 2 ** 30;
+
+interface RoutedOpts {
+  nodes?: string[];
+  replicas?: { volume: string }[];
+  volumes?: { name: string; replicas: number; size: number }[];
+  softAntiAffinity?: boolean;
+  overProvisionPct?: number;
+  diskFreeBytes?: number;
+  diskMaxBytes?: number;
+  patchFails?: boolean;
+}
+
+/**
+ * A context whose SSH mock answers by MATCHING the command rather than by call
+ * order. The operations under test read several API objects before they do
+ * anything, and an ordered mock has to be re-counted every time one is added —
+ * which makes the test fail for reasons that have nothing to do with the
+ * behaviour it is meant to pin.
+ */
+function routed(o: RoutedOpts = {}, config?: Partial<K3sConfig>) {
+  const nodeNames = o.nodes ?? ["node1.lab", "node2.lab", "node3.lab", "node4.lab"];
+  const free = o.diskFreeBytes ?? 500 * GIB;
+  const max = o.diskMaxBytes ?? 700 * GIB;
+  const lhNodes = {
+    items: nodeNames.map((name) => ({
+      metadata: { name },
+      spec: { allowScheduling: true },
+      status: { diskStatus: { "default-disk": {
+        storageAvailable: free, storageMaximum: max, storageScheduled: 0, storageReserved: 0,
+      } } },
+    })),
+  };
+  const replicas = {
+    items: (o.replicas ?? []).map((r, i) => ({
+      metadata: { name: `replica-${i}` },
+      spec: { nodeID: "node1.lab", volumeName: r.volume },
+    })),
+  };
+  const volumes = {
+    items: (o.volumes ?? []).map((v) => ({
+      metadata: { name: v.name },
+      spec: { numberOfReplicas: v.replicas, size: v.size },
+      status: { state: "attached", robustness: "healthy" },
+    })),
+  };
+
+  const ctx = mockCtx(config);
+  ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+    if (cmd.startsWith("hostname")) return HOST;
+    if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
+    // NOTE: "-o jsonpath=" CONTAINS "-o json", so jsonpath must be matched
+    // first or the eviction poll below is handed a whole JSON document and
+    // never sees zero replicas. (This mock got that wrong once already.)
+    if (cmd.includes("jsonpath")) {
+      // The eviction wait polls replicas on the node — report none remaining.
+      if (cmd.includes("replicas.longhorn.io")) return stdout("");
+      if (cmd.includes("replica-soft-anti-affinity")) return stdout(String(o.softAntiAffinity ?? false));
+      if (cmd.includes("storage-over-provisioning-percentage")) return stdout(String(o.overProvisionPct ?? 100));
+      return stdout("");
+    }
+    if (cmd.includes("get nodes.longhorn.io") && cmd.includes("-o json")) return stdout(JSON.stringify(lhNodes));
+    if (cmd.includes("get replicas.longhorn.io") && cmd.includes("-o json")) return stdout(JSON.stringify(replicas));
+    if (cmd.includes("get volumes.longhorn.io") && cmd.includes("-o json")) return stdout(JSON.stringify(volumes));
+    if (cmd.includes("patch nodes.longhorn.io")) return o.patchFails ? FAIL : OK;
+    return OK;
+  });
+  return ctx;
+}
+
+describe("cordonNode", () => {
+  it("cordons and reports a change", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(stdout('node/node1.lab cordoned'));
+    const r = await cordonNode(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(true);
+    expectCommand(ctx.ssh, /cordon "node1\.lab"/);
+  });
+
+  it("is idempotent — already cordoned is changed:false", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(stdout("node/node1.lab already cordoned"));
+    const r = await cordonNode(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(false);
+  });
+
+  it("fails loudly when the API is unreachable", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(FAIL);
+    const r = await cordonNode(ctx);
+    expect(r.success).toBe(false);
+  });
+});
+
+describe("drainNode", () => {
+  it("ignores daemonsets and deletes emptyDir, or it hangs forever", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(OK);
+    const r = await drainNode(ctx);
+    expect(r.success).toBe(true);
+    expectCommand(ctx.ssh, /--ignore-daemonsets/);
+    expectCommand(ctx.ssh, /--delete-emptydir-data/);
+  });
+
+  it("gives the SSH transport longer than kubectl's own --timeout", async () => {
+    // drainNode asks kubectl for --timeout=15m. sshOpts() is a flat 30s, so
+    // without an explicit override the transport kills a drain that is working
+    // -- after it has already evicted most of the pods. The result is
+    // non-zero with a transcript that reads like success, on every node with
+    // real workloads. `labctl app k3s evacuate` could never have drained one.
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(OK);
+    await drainNode(ctx);
+    const call = ctx.ssh.exec.mock.calls.find((c: [string, { timeoutMs?: number }?]) =>
+      c[0].includes("drain "));
+    expect(call, "drain command").toBeDefined();
+    const timeoutMs = call?.[1]?.timeoutMs ?? 0;
+    expect(timeoutMs, "SSH timeout must exceed kubectl's 15m --timeout")
+      .toBeGreaterThan(15 * 60_000);
+  });
+
+  it("reports the END of the drain transcript, not the useless first line", async () => {
+    // Every drain starts with "node/x already cordoned". Surfacing the whole
+    // transcript means the error you see is that line, which explains nothing.
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce({
+      exitCode: 1,
+      stdout: "node/n1 already cordoned\nevicting pod a\nevicting pod b\n" +
+        "error: cannot evict pod as it would violate the pod's disruption budget",
+      stderr: "",
+    });
+    const r = await drainNode(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toContain("disruption budget");
+  });
+
+  it("reports failure rather than pretending the node is empty", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(FAIL);
+    const r = await drainNode(ctx);
+    expect(r.success).toBe(false);
+  });
+});
+
+describe("evictLonghornReplicas", () => {
+  it("is a no-op when Longhorn is not installed", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(FAIL); // no longhorn-system ns
+    const r = await evictLonghornReplicas(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(false);
+    expect(r.message).toMatch(/not installed/);
+  });
+
+  it("requests eviction and waits until zero replicas remain", async () => {
+    // Node holds no replicas, so the pre-check clears it immediately.
+    const ctx = routed({ replicas: [] });
+    const r = await evictLonghornReplicas(ctx);
+    expect(r.success).toBe(true);
+    expect(r.changed).toBe(true);
+    expectCommand(ctx.ssh, /evictionRequested/);
+    expectCommand(ctx.ssh, /allowScheduling/);
+  });
+
+  it("fails when eviction cannot even be requested", async () => {
+    const ctx = routed({ replicas: [], patchFails: true });
+    const r = await evictLonghornReplicas(ctx);
+    expect(r.success).toBe(false);
+  });
+
+  it("does not request eviction at all when it could never complete", async () => {
+    // 3-replica volume, and only 2 other schedulable nodes: the survivors
+    // already hold a copy each and strict anti-affinity forbids a third.
+    const ctx = routed({
+      nodes: ["node1.lab", "node2.lab", "node3.lab"],
+      replicas: [{ volume: "pvc-a" }],
+      volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }],
+    });
+    const r = await evictLonghornReplicas(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/cannot be placed|wants 3 replicas/);
+    // The important half: nothing was mutated.
+    expectNoCommand(ctx.ssh, /evictionRequested/);
+  });
+
+  it("refuses a single-replica volume unless explicitly allowed", async () => {
+    const opts = {
+      nodes: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"],
+      replicas: [{ volume: "pvc-solo" }],
+      volumes: [{ name: "pvc-solo", replicas: 1, size: 1 * GIB }],
+    };
+    const r = await evictLonghornReplicas(routed(opts));
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/single replica/);
+
+    const allowed = await evictLonghornReplicas(
+      routed(opts, { evictionAllowSingleReplica: true }),
+    );
+    expect(allowed.success).toBe(true);
+  });
+});
+
+describe("checkEvictionCompletable — refuse in seconds, not in an hour", () => {
+  it("passes when a spare schedulable node exists", async () => {
+    const ctx = routed({
+      nodes: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"],
+      replicas: [{ volume: "pvc-a" }],
+      volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }],
+    });
+    const f = await checkEvictionCompletable(ctx, "node1.lab");
+    expect(f.completable).toBe(true);
+    expect(f.replicasMoving).toBe(1);
+  });
+
+  it("blocks when replica count equals the surviving node count", async () => {
+    const ctx = routed({
+      nodes: ["node1.lab", "node2.lab", "node3.lab"],
+      replicas: [{ volume: "pvc-a" }],
+      volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }],
+    });
+    const f = await checkEvictionCompletable(ctx, "node1.lab");
+    expect(f.completable).toBe(false);
+    expect(f.blockers.join(" ")).toContain("pvc-a");
+  });
+
+  it("allows it when soft anti-affinity is on — placement stops being the limit", async () => {
+    const ctx = routed({
+      nodes: ["node1.lab", "node2.lab", "node3.lab"],
+      replicas: [{ volume: "pvc-a" }],
+      volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }],
+      softAntiAffinity: true,
+    });
+    const f = await checkEvictionCompletable(ctx, "node1.lab");
+    expect(f.completable).toBe(true);
+  });
+
+  it("catches a small disk hiding behind over-provisioning", async () => {
+    // The scheduler is happy at 500%; the disk has 49 GiB and the volume is 92.
+    const ctx = routed({
+      nodes: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"],
+      replicas: [{ volume: "pvc-big" }],
+      volumes: [{ name: "pvc-big", replicas: 3, size: 92 * GIB }],
+      overProvisionPct: 500,
+      diskFreeBytes: 49 * GIB,
+      diskMaxBytes: 100 * GIB,
+    });
+    const f = await checkEvictionCompletable(ctx, "node1.lab");
+    expect(f.completable).toBe(false);
+    expect(f.blockers.join(" ")).toMatch(/actually free/);
+    expect(f.blockers.join(" ")).toMatch(/over-provisioning-percentage=500/);
+  });
+
+  it("warns, but does not block, on a single-replica volume", async () => {
+    const ctx = routed({
+      nodes: ["node1.lab", "node2.lab", "node3.lab"],
+      replicas: [{ volume: "pvc-solo" }],
+      volumes: [{ name: "pvc-solo", replicas: 1, size: 1 * GIB }],
+    });
+    const f = await checkEvictionCompletable(ctx, "node1.lab");
+    expect(f.completable).toBe(true);
+    expect(f.warnings.join(" ")).toMatch(/single replica/);
+  });
+
+  it("is trivially completable when the node holds nothing", async () => {
+    const f = await checkEvictionCompletable(routed({ replicas: [] }), "node1.lab");
+    expect(f.completable).toBe(true);
+    expect(f.replicasMoving).toBe(0);
+  });
+
+  it("refuses to guess when Longhorn cannot be read", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValue(FAIL);
+    const f = await checkEvictionCompletable(ctx, "node1.lab");
+    expect(f.completable).toBe(false);
+    expect(f.blockers.join(" ")).toMatch(/refusing to guess/);
+  });
+});
+
+describe("assertNodeEvacuated — the gate before a wipe", () => {
+  it("passes only when cordoned with no replicas left", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec
+      .mockResolvedValueOnce(HOST)
+      .mockResolvedValueOnce(stdout("true"))   // unschedulable
+      .mockResolvedValueOnce(stdout(""))       // no running pods
+      .mockResolvedValueOnce(stdout("namespace/longhorn-system"))
+      .mockResolvedValueOnce(stdout(""));      // no replicas
+    const r = await assertNodeEvacuated(ctx);
+    expect(r.success).toBe(true);
+    expect(r.message).toMatch(/safe to wipe/);
+  });
+
+  it("REFUSES when replicas are still on the node", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec
+      .mockResolvedValueOnce(HOST)
+      .mockResolvedValueOnce(stdout("true"))
+      .mockResolvedValueOnce(stdout(""))
+      .mockResolvedValueOnce(stdout("namespace/longhorn-system"))
+      .mockResolvedValueOnce(stdout("replica-abc\nreplica-def"));
+    const r = await assertNodeEvacuated(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/2 Longhorn replica/);
+  });
+
+  it("REFUSES when the node is still schedulable", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec
+      .mockResolvedValueOnce(HOST)
+      .mockResolvedValueOnce(stdout(""))       // not unschedulable
+      .mockResolvedValueOnce(stdout(""))
+      .mockResolvedValueOnce(FAIL);            // no longhorn
+    const r = await assertNodeEvacuated(ctx);
+    expect(r.success).toBe(false);
+    expect(r.error).toMatch(/still schedulable/);
+  });
+});
+
+describe("uncordonNode", () => {
+  it("returns the node to service", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(OK);
+    const r = await uncordonNode(ctx);
+    expect(r.success).toBe(true);
+    expectCommand(ctx.ssh, /uncordon "node1\.lab"/);
+  });
+});
+
+describe("eviction timeout diagnostics", () => {
+  // An eviction can legitimately run for an hour. When it times out, a bare
+  // count ("2 replica(s) still present") is not actionable -- the question is
+  // always WHICH volume is stuck and in what state. A replica with
+  // desireState=running, currentState=stopped and no failedAt is Longhorn
+  // wanting to start it and not doing so, which is a completely different
+  // problem from a volume that cannot be placed. Observed for real on a
+  // freshly wiped-and-rejoined node.
+  it("names the stuck volumes and their replica states, then restores the node", async () => {
+    const ctx = mockCtx();
+    const patches: string[] = [];
+    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
+      if (cmd.startsWith("hostname")) return HOST;
+      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
+      if (cmd.includes("patch nodes.longhorn.io")) { patches.push(cmd); return OK; }
+      if (cmd.includes("jsonpath")) {
+        // The poll never reaches zero: one replica remains for the whole run.
+        if (cmd.includes("replicas.longhorn.io")) return stdout("replica-stuck");
+        if (cmd.includes("replica-soft-anti-affinity")) return stdout("false");
+        if (cmd.includes("storage-over-provisioning-percentage")) return stdout("100");
+        return stdout("");
+      }
+      if (cmd.includes("get replicas.longhorn.io") && cmd.includes("-o json")) {
+        return stdout(JSON.stringify({ items: [{
+          metadata: { name: "pvc-abc-r-deadbeef99" },
+          spec: { nodeID: "node1.lab", volumeName: "pvc-abc", failedAt: "", desireState: "running" },
+          status: { currentState: "stopped" },
+        }] }));
+      }
+      if (cmd.includes("get volumes.longhorn.io") && cmd.includes("-o json")) {
+        return stdout(JSON.stringify({ items: [{
+          metadata: { name: "pvc-abc" },
+          spec: { numberOfReplicas: 3, size: 2 * 2 ** 30 },
+          status: { state: "attached", robustness: "degraded" },
+        }] }));
+      }
+      if (cmd.includes("get nodes.longhorn.io") && cmd.includes("-o json")) {
+        return stdout(JSON.stringify({ items: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"].map((n) => ({
+          metadata: { name: n },
+          spec: { allowScheduling: true },
+          status: { diskStatus: { d: {
+            storageAvailable: 500 * 2 ** 30, storageMaximum: 700 * 2 ** 30,
+            storageScheduled: 0, storageReserved: 0,
+          } } },
+        })) }));
+      }
+      return OK;
+    });
+
+    vi.useFakeTimers();
+    let result;
+    try {
+      const p = evictLonghornReplicas(ctx);
+      await vi.advanceTimersByTimeAsync(61 * 60_000);
+      result = await p;
+    } finally {
+      vi.useRealTimers();
+    }
+
+    expect(result.success).toBe(false);
+    // The actionable part: which volume, and what state its replica is in.
+    expect(result.error).toContain("pvc-abc");
+    expect(result.error).toContain("currentState=stopped");
+    expect(result.error).toContain("desireState=running");
+    expect(result.error).toContain("failedAt=(none)");
+    expect(result.error).toContain("attached/degraded");
+
+    // And the node must be handed back usable, not left eviction-pending.
+    expect(result.error).toMatch(/cancelled and scheduling restored/);
+    expect(patches).toHaveLength(2);
+    expect(patches[1]).toContain('"evictionRequested":false');
+    expect(patches[1]).toContain('"allowScheduling":true');
+  });
+});

bastion/src/modules/modules/k3s/tests/operations.test.ts

@@ -72,31 +72,97 @@ describe("applyCisHardening", () => {
 
 // --- Swap ---
 
-import { disableSwap } from "../src/operations/swap.js";
+import { enableSwap } from "../src/operations/swap.js";
 
-describe("disableSwap", () => {
-  it("disables active swap", async () => {
+describe("enableSwap", () => {
+  it("activates LV swap when present but off", async () => {
     const ctx = mockCtx();
     ctx.ssh.exec
-      .mockResolvedValueOnce(stdout("/dev/sda2 partition 2G"))  // swap active
-      .mockResolvedValueOnce(OK)   // swapoff
-      .mockResolvedValueOnce(OK);  // sed fstab
+      .mockResolvedValueOnce(stdout("yes"))  // LV exists
+      .mockResolvedValueOnce(stdout("off"))  // not in /proc/swaps
+      .mockResolvedValueOnce(OK)             // blkid || mkswap
+      .mockResolvedValueOnce(OK)             // swapon
+      .mockResolvedValueOnce(OK);            // fstab entry
 
-    const result = await disableSwap(ctx);
+    const result = await enableSwap(ctx);
     expect(result.success).toBe(true);
     expect(result.changed).toBe(true);
-    expectCommand(ctx.ssh, "swapoff -a");
+    expectCommand(ctx.ssh, "swapon /dev/mapper/labvg-swap");
   });
 
-  it("is idempotent when swap already off", async () => {
+  it("is idempotent when LV swap already active", async () => {
     const ctx = mockCtx();
     ctx.ssh.exec
-      .mockResolvedValueOnce(stdout(""))  // no swap
-      .mockResolvedValueOnce(OK);          // sed fstab (always runs)
+      .mockResolvedValueOnce(stdout("yes"))  // LV exists
+      .mockResolvedValueOnce(stdout("on"))   // already in /proc/swaps
+      .mockResolvedValueOnce(OK);            // fstab entry (always ensured)
 
-    const result = await disableSwap(ctx);
+    const result = await enableSwap(ctx);
     expect(result.changed).toBe(false);
-    expectNoCommand(ctx.ssh, "swapoff");
+    expectNoCommand(ctx.ssh, "swapon /dev/mapper/labvg-swap");
+  });
+
+  it("skips when no labvg-swap LV exists", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(stdout("no")); // LV missing
+
+    const result = await enableSwap(ctx);
+    expect(result.success).toBe(true);
+    expect(result.changed).toBe(false);
+    expectNoCommand(ctx.ssh, "swapon");
+  });
+});
+
+// --- Rancher LV (imageFs sizing) ---
+
+import { growRancherLv } from "../src/operations/rancher-storage.js";
+
+describe("growRancherLv", () => {
+  it("grows a 20G LV to 120G when the VG has space", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec
+      .mockResolvedValueOnce(stdout("  20480.00"))   // lv_size
+      .mockResolvedValueOnce(stdout("  747807.00"))  // vg_free
+      .mockResolvedValueOnce(OK)                     // lvextend
+      .mockResolvedValueOnce(OK);                    // xfs_growfs
+
+    const result = await growRancherLv(ctx);
+    expect(result.success).toBe(true);
+    expect(result.changed).toBe(true);
+    expectCommand(ctx.ssh, "lvextend -L 122880m /dev/labvg/rancher");
+    expectCommand(ctx.ssh, "xfs_growfs /var/lib/rancher");
+  });
+
+  it("is idempotent when the LV is already 120G", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(stdout("  122880.00")); // lv_size
+
+    const result = await growRancherLv(ctx);
+    expect(result.changed).toBe(false);
+    expectNoCommand(ctx.ssh, "lvextend");
+  });
+
+  it("reports without failing when the VG has no free space", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec
+      .mockResolvedValueOnce(stdout("  20480.00"))  // lv_size
+      .mockResolvedValueOnce(stdout("  0.00"));     // vg_free
+
+    const result = await growRancherLv(ctx);
+    expect(result.success).toBe(true);
+    expect(result.changed).toBe(false);
+    expect(result.message).toContain("free");
+    expectNoCommand(ctx.ssh, "lvextend");
+  });
+
+  it("skips when there is no rancher LV", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(stdout("")); // lvs empty
+
+    const result = await growRancherLv(ctx);
+    expect(result.success).toBe(true);
+    expect(result.changed).toBe(false);
+    expectNoCommand(ctx.ssh, "lvextend");
   });
 });
 
@@ -202,6 +268,108 @@ describe("writeK3sConfig", () => {
     expect(writeCall).toContain("protect-kernel-defaults: true");
     expect(writeCall).not.toContain("secrets-encryption");
   });
+
+  // --- dual-stack ---
+  //
+  // The property that matters most is the NEGATIVE one: with no ipv6 and no
+  // CIDRs the output must be byte-identical to what the five existing nodes
+  // already have on disk. If it is not, rolling this out rewrites every node's
+  // config.yaml and restarts a healthy cluster to tell it what it already knew.
+
+  // With an ipv6 configured there is an extra ssh round trip up front -- the
+  // probe that checks the address is really on the node -- so the write lands
+  // one call later.
+  const writtenBy = async (config: Parameters<typeof mockCtx>[0]) => {
+    const ctx = mockCtx(config);
+    const hasV6 = !!(config as { ipv6?: string }).ipv6;
+    if (hasV6) ctx.ssh.exec.mockResolvedValueOnce(stdout("present"));
+    ctx.ssh.exec
+      .mockResolvedValueOnce(OK)
+      .mockResolvedValueOnce(stdout("__LABCTL_NOT_FOUND__"))
+      .mockResolvedValueOnce(OK);
+    await writeK3sConfig(ctx);
+    return ctx.ssh.exec.mock.calls[hasV6 ? 3 : 2]![0] as string;
+  };
+
+  it("emits no address-family lines at all when single-stack", async () => {
+    const server = await writtenBy({ hostname: "n1.lab", ip: "10.0.1.1", role: "infra" });
+    expect(server).not.toContain("node-ip");
+    expect(server).not.toContain("cluster-cidr");
+    expect(server).not.toContain("service-cidr");
+
+    const agent = await writtenBy({ role: "worker" });
+    expect(agent).not.toContain("node-ip");
+    // Nothing inserted ahead of, or between, the lines the agent config has
+    // always opened with.
+    expect(agent).toContain("protect-kernel-defaults: true\nnode-label:");
+  });
+
+  it("emits dual node-ip and CIDRs on a server, IPv4 first", async () => {
+    const out = await writtenBy({
+      hostname: "n1.lab",
+      ip: "192.168.8.23",
+      role: "infra",
+      ipv6: "2001:470:187e:2::23",
+      clusterCidr: ["10.42.0.0/16", "2001:470:187e:1000::/56"],
+      serviceCidr: ["10.43.0.0/16", "2001:470:187e:1fff::/112"],
+    });
+    expect(out).toContain('node-ip: "192.168.8.23,2001:470:187e:2::23"');
+    expect(out).toContain('cluster-cidr: "10.42.0.0/16,2001:470:187e:1000::/56"');
+    expect(out).toContain('service-cidr: "10.43.0.0/16,2001:470:187e:1fff::/112"');
+    // IPv4 must stay the primary family -- that is the supported conversion
+    // path and what lets existing Services keep their ClusterIP.
+    expect(out.indexOf("10.42.0.0/16")).toBeLessThan(out.indexOf("2001:470:187e:1000::/56"));
+    // still a valid server config
+    expect(out).toContain("cluster-init: true");
+    expect(out).toContain("secrets-encryption: true");
+    // the v6 address must be a TLS SAN, or a peer joining over v6 fails
+    // verification with an error that names the cert, not the missing SAN
+    expect(out).toContain('  - "2001:470:187e:2::23"');
+  });
+
+  it("gives an AGENT its own dual node-ip but no CIDRs", async () => {
+    const out = await writtenBy({
+      ip: "192.168.8.12",
+      role: "worker",
+      ipv6: "2001:470:187e:2::12",
+      // deliberately supplied: an agent must ignore them
+      clusterCidr: ["10.42.0.0/16", "2001:470:187e:1000::/56"],
+      serviceCidr: ["10.43.0.0/16", "2001:470:187e:1fff::/112"],
+    });
+    expect(out).toContain('node-ip: "192.168.8.12,2001:470:187e:2::12"');
+    expect(out).not.toContain("cluster-cidr");
+    expect(out).not.toContain("service-cidr");
+  });
+
+  it("refuses to write a config naming an IPv6 the node does not have", async () => {
+    // The SSH-onboard case: Asahi and the DGX Sparks run an OS labctl never
+    // installed, so nothing upstream guarantees a DHCPv6 lease was taken.
+    // Writing the config anyway defers the failure to k3s start-up, where it
+    // reads as a bind error rather than a missing address.
+    const ctx = mockCtx({ ip: "192.168.8.12", role: "worker", ipv6: "2001:470:187e:2::12" });
+    ctx.ssh.exec.mockResolvedValueOnce(stdout("")); // probe: address absent
+
+    const result = await writeK3sConfig(ctx);
+    expect(result.success).toBe(false);
+    expect(result.changed).toBe(false);
+    expect(result.error).toContain("2001:470:187e:2::12");
+    // and it must not have written anything
+    expect(ctx.ssh.exec).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not write node-ip for a v4-only node even when CIDRs are given", async () => {
+    // Guards the flag-day risk: supplying ranges alone must not start rewriting
+    // node identity on nodes that have no IPv6 yet.
+    const out = await writtenBy({
+      hostname: "n1.lab",
+      ip: "10.0.1.1",
+      role: "infra",
+      clusterCidr: ["10.42.0.0/16"],
+      serviceCidr: ["10.43.0.0/16"],
+    });
+    expect(out).toContain('cluster-cidr: "10.42.0.0/16"');
+    expect(out).not.toContain("node-ip");
+  });
 });
 
 // --- CNI Cleanup ---

bastion/src/modules/modules/k3s/tests/smoke.test.ts

@@ -16,7 +16,7 @@ describe("smoke: full server install pipeline", () => {
     const pipeline: NamedOperation[] = [
       { name: "Kernel modules", fn: ops.loadKernelModules },
       { name: "Sysctl hardening", fn: ops.applyCisHardening },
-      { name: "Disable swap", fn: ops.disableSwap },
+      { name: "Enable swap", fn: ops.enableSwap },
       { name: "Disable firewall", fn: ops.disableFirewall },
       { name: "SELinux permissive", fn: ops.setSelinuxPermissive },
       { name: "Write k3s config", fn: ops.writeK3sConfig },
@@ -73,7 +73,7 @@ describe("smoke: pipeline stops on failure", () => {
     };
 
     const results = await runSequential(ctx, [
-      { name: "OK op", fn: ops.disableSwap },
+      { name: "OK op", fn: ops.enableSwap },
       { name: "Failing op", fn: failingOp },
       { name: "Never called", fn: neverCalled },
     ]);
@@ -98,11 +98,12 @@ describe("smoke: agent install rejects missing config", () => {
 });
 
 describe("smoke: all operations are exported", () => {
-  it("exports all 15 operations", () => {
+  it("exports all 16 operations", () => {
     const exported = [
       ops.loadKernelModules,
       ops.applyCisHardening,
-      ops.disableSwap,
+      ops.enableSwap,
+      ops.growRancherLv,
       ops.disableFirewall,
       ops.setSelinuxPermissive,
       ops.writeK3sConfig,
@@ -117,7 +118,7 @@ describe("smoke: all operations are exported", () => {
       ops.checkCertExpiry,
     ];
 
-    expect(exported).toHaveLength(15);
+    expect(exported).toHaveLength(16);
     for (const op of exported) {
       expect(typeof op).toBe("function");
     }

bastion/src/modules/src/types.ts

@@ -1,4 +1,4 @@
-import type { OsId, Arch } from "@lab/shared";
+import type { OsId, Arch, Role } from "@lab/shared";
 
 export interface ModuleMetadata {
   name: string;
@@ -14,7 +14,10 @@ export interface ModuleMetadata {
 export interface ModuleContext {
   hostname: string;
   ip: string;
-  role: string;
+  // Role, not string. A literal `string` let `role === "infra"` comparisons
+  // compile while silently missing every other server role -- which is how
+  // labcontroller ended up installing as an agent.
+  role: Role;
   os: OsId;
   arch: Arch;
   sshUser: string;
@@ -24,7 +27,7 @@ export interface ModuleContext {
 
 export interface ModuleResult {
   success: boolean;
-  phase: "install" | "configure" | "health";
+  phase: "install" | "configure" | "health" | "evacuate" | "admit";
   duration: number;
   output: string[];
   errors: string[];

bastion/src/shared/src/hardware/index.ts

@@ -1,154 +0,0 @@
-// Architecture normalisation and machine classification.
-//
-// Both are derived from what the system already observes about a machine -- never from
-// an operator-supplied flag.
-
-import type { Arch, HardwareInfo, OnboardMethod, OsId } from "../types/index.js";
-
-export const SUPPORTED_ARCHES: readonly Arch[] = ["x86_64", "aarch64"] as const;
-
-/**
- * Normalise an architecture string to one we serve boot artifacts for.
- *
- * Sources and their spellings:
- *   uname -m         -> "x86_64" / "aarch64"
- *   iPXE ${buildarch}-> "x86_64" / "arm64"
- *   dpkg/Debian      -> "amd64"  / "arm64"
- *
- * Returns undefined for anything we don't serve, so callers fall back rather than
- * inventing a kernel path that would 404.
- */
-export function normalizeArch(value: string | undefined | null): Arch | undefined {
-  switch ((value ?? "").trim().toLowerCase()) {
-    case "x86_64":
-    case "x86-64":
-    case "amd64":
-      return "x86_64";
-    case "aarch64":
-    case "arm64":
-      return "aarch64";
-    default:
-      return undefined;
-  }
-}
-
-/** Fedora pxeboot artifact base URL for an architecture. */
-export function fedoraMirrorFor(fedoraVersion: string, arch: Arch): string {
-  return `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`;
-}
-
-/**
- * Which architectures each OS in the pipeline can actually be installed on.
- *
- * Fedora publishes pxeboot vmlinuz/initrd for both. Ubuntu does not: as of 26.04,
- * releases.ubuntu.com publishes amd64 artifacts only, so there is nothing to netboot an
- * arm64 machine with. Claiming support would fail at download time with a 404 instead
- * of a useful message.
- */
-const OS_ARCH_SUPPORT: Record<OsId, readonly Arch[]> = {
-  "fedora-43": ["x86_64", "aarch64"],
-  "ubuntu-26.04": ["x86_64"],
-};
-
-export function osSupportsArch(os: OsId, arch: Arch): boolean {
-  return (OS_ARCH_SUPPORT[os] ?? []).includes(arch);
-}
-
-export function archesForOs(os: OsId): readonly Arch[] {
-  return OS_ARCH_SUPPORT[os] ?? [];
-}
-
-/**
- * Machines that run a vendor OS we have no image for.
- *
- * These are SSH-onboard: we manage userspace, but reinstalling destroys a driver and
- * firmware stack our pipeline cannot rebuild. Matched on DMI identity, which is what
- * discovery and `provision recheck` both collect.
- *
- * This is deliberately a property of the machine ("it runs DGX OS"), not a blocklist
- * ("never install this MAC"). When a DGX OS image joins the pipeline, teaching the
- * installer about vendor_os "dgx-os" is what unblocks these machines -- no entry here
- * needs deleting.
- */
-interface VendorOsRule {
-  vendorOs: string;
-  description: string;
-  matches: (hw: DmiIdentity) => boolean;
-}
-
-interface DmiIdentity {
-  manufacturer: string;
-  product: string;
-  board: string;
-}
-
-const VENDOR_OS_RULES: readonly VendorOsRule[] = [
-  {
-    vendorOs: "dgx-os",
-    description: "NVIDIA DGX OS (proprietary driver + firmware stack, no image in our pipeline)",
-    matches: ({ manufacturer, product, board }) =>
-      (manufacturer.includes("nvidia") || product.includes("nvidia")) &&
-      (product.includes("dgx") || product.includes("spark") ||
-       board.includes("gb10") || product.includes("gb10")),
-  },
-];
-
-/**
- * Machines known to run a vendor OS, by MAC.
- *
- * The DMI rules above only fire once discovery or `provision recheck` has populated a
- * hardware record. Machines onboarded over SSH may sit in state for a long time with no
- * DMI at all -- which is exactly the state both DGX Sparks are in today -- so a
- * DMI-only classifier would fail open on the machines this guard exists to protect.
- *
- * This is a statement of fact about known hardware ("this box runs DGX OS"), not an
- * install policy. Whether that means "refuse" is decided by whether the pipeline has an
- * image for that vendor OS.
- */
-const KNOWN_VENDOR_OS_MACS: Record<string, string> = {
-  "4c:bb:47:7f:29:35": "dgx-os",  // spark-2935
-  "48:21:0b:96:3a:1c": "dgx-os",  // spark-3a1c
-};
-
-/**
- * Classify how a machine should be onboarded, from its hardware record.
- *
- * An explicit `onboard` already on the record wins: it may have been set by an operator
- * or by a rule that has since changed, and silently overriding it would be worse than
- * leaving it.
- */
-export function classifyOnboard(
-  hw: Partial<Pick<HardwareInfo, "mac" | "manufacturer" | "product" | "board">>
-    & { onboard?: OnboardMethod; vendor_os?: string },
-): { onboard: OnboardMethod; vendor_os?: string } {
-  if (hw.onboard !== undefined) {
-    return hw.vendor_os !== undefined
-      ? { onboard: hw.onboard, vendor_os: hw.vendor_os }
-      : { onboard: hw.onboard };
-  }
-
-  const knownVendorOs = KNOWN_VENDOR_OS_MACS[(hw.mac ?? "").toLowerCase().replace(/-/g, ":")];
-  if (knownVendorOs !== undefined) {
-    return { onboard: "ssh", vendor_os: knownVendorOs };
-  }
-
-  const identity: DmiIdentity = {
-    manufacturer: (hw.manufacturer ?? "").toLowerCase(),
-    product: (hw.product ?? "").toLowerCase(),
-    board: (hw.board ?? "").toLowerCase(),
-  };
-
-  for (const rule of VENDOR_OS_RULES) {
-    if (rule.matches(identity)) {
-      return { onboard: "ssh", vendor_os: rule.vendorOs };
-    }
-  }
-
-  return { onboard: "pxe" };
-}
-
-/** Human-readable reason a vendor-OS machine must not be reinstalled. */
-export function vendorOsDescription(vendorOs: string | undefined): string {
-  const rule = VENDOR_OS_RULES.find((r) => r.vendorOs === vendorOs);
-  return rule?.description ?? "a vendor OS with no image in our pipeline";
-}

bastion/src/shared/src/index.ts

@@ -1,8 +1,6 @@
 export type {
   OsId,
   Arch,
-  OnboardMethod,
-  RootCandidate,
   Role,
   HardwareInfo,
   InstallConfig,
@@ -10,20 +8,15 @@ export type {
   DebugConfig,
   BastionState,
   BastionConfig,
+  VyosVlanSpec,
+  VyosInstallSpec,
+  VyosBundle,
+  VyosBundleSetOp,
 } from "./types/index.js";
 
-export {
-  SUPPORTED_ARCHES,
-  normalizeArch,
-  fedoraMirrorFor,
-  osSupportsArch,
-  archesForOs,
-  classifyOnboard,
-  vendorOsDescription,
-} from "./hardware/index.js";
-
-export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./types/index.js";
-export type { RoleInfo } from "./types/index.js";
+export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId,
+  roleInfo, runsK3s, isK3sServer, isManaged, hostsLonghorn, isRole } from "./types/index.js";
+export type { RoleInfo, Lifecycle, K3sMode } from "./types/index.js";
 
 export { APP_NAME, APP_VERSION } from "./constants/index.js";
 
@@ -56,3 +49,8 @@ export {
   AgentNotConnectedError,
   isLabError,
 } from "./errors/index.js";
+
+export { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB,
+  SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion,
+  validateDataDisks, DATA_DISK_LABELS } from "./types/index.js";
+export type { LvSpec, SchemaVersion, DataDisks, DataDiskName } from "./types/index.js";

bastion/src/shared/src/protocol/index.ts

@@ -1,6 +1,7 @@
 // Protocol types for agent-labd WebSocket communication.
 
 import { randomUUID } from "node:crypto";
+import type { VyosInstallSpec } from "../types/state.js";
 
 // --- Agent -> labd messages ---
 
@@ -108,12 +109,12 @@ export type BastionMessage =
 export type LabdBastionMessage =
   | { type: "bastion-enrolled"; bastionId: string }
   | { type: "bastion-heartbeat-ack"; serverTime: string }
-  | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string }
+  | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string; schemaVersion?: 1 | 2; dataDisks?: import("../types/layout.js").DataDisks; vyos?: VyosInstallSpec }
   | { type: "command-forget"; requestId: string; mac: string }
   | { type: "command-role-update"; requestId: string; mac: string; role: string }
   | { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean }
   | { type: "command-register"; requestId: string; mac: string; hostname: string; role: string; ip: string }
-  | { type: "command-discover"; requestId: string; mac: string; product?: string; board?: string; serial?: string; manufacturer?: string; cpu_model?: string; cpu_cores?: number; memory_gb?: number; arch?: string; disks?: Array<{ name: string; size_gb: number; model: string }>; nics?: Array<{ name: string; mac: string; state: string }>; root_device?: string; root_args?: string }
+  | { type: "command-discover"; requestId: string; mac: string; product?: string; board?: string; serial?: string; manufacturer?: string; cpu_model?: string; cpu_cores?: number; memory_gb?: number; arch?: string; disks?: Array<{ name: string; size_gb: number; model: string }>; nics?: Array<{ name: string; mac: string; state: string }> }
   | { type: "server-shutdown"; reconnectAfter: number };
 
 export type BastionMessageType = BastionMessage["type"];

bastion/src/shared/src/types/config.ts

@@ -14,11 +14,19 @@ export interface BastionConfig {
   // Ubuntu support
   ubuntuVersion: string;
   ubuntuMirror: string;
+  // VyOS support — netboot artifacts are extracted from the ISO at startup.
+  // LTS ISOs are subscription-only, so this defaults to a rolling release.
+  vyosIsoUrl: string;
+  vyosDefaultPassword: string;
   // Syslog listener for install logs (Anaconda logging --host)
   syslogPort: number;
   // Flags
   skipDnsmasq?: boolean | undefined;
   skipArtifacts?: boolean | undefined;
+  // Post-provision k3s auto-install. Defaults on. Set false to suppress it --
+  // required by multi-node tests, which must sequence cluster formation
+  // themselves rather than have each node auto-install into its own cluster.
+  autoK3s?: boolean | undefined;
   // Labd registration (optional — if unset, bastion runs standalone)
   labdUrl?: string | undefined;
   bastionJoinToken?: string | undefined;

bastion/src/shared/src/types/index.ts

@@ -1,17 +1,25 @@
 export type {
   OsId,
   Arch,
-  OnboardMethod,
-  RootCandidate,
   Role,
   HardwareInfo,
   InstallConfig,
   InstalledInfo,
   DebugConfig,
   BastionState,
+  VyosVlanSpec,
+  VyosInstallSpec,
+  VyosBundle,
+  VyosBundleSetOp,
 } from "./state.js";
 
-export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./state.js";
-export type { RoleInfo } from "./state.js";
+export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId,
+  roleInfo, runsK3s, isK3sServer, isManaged, hostsLonghorn, isRole } from "./state.js";
+export type { RoleInfo, Lifecycle, K3sMode } from "./state.js";
 
 export type { BastionConfig } from "./config.js";
+
+export { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB,
+  SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion,
+  validateDataDisks, DATA_DISK_LABELS } from "./layout.js";
+export type { LvSpec, SchemaVersion, DataDisks, DataDiskName } from "./layout.js";

bastion/src/shared/src/types/layout.ts

@@ -0,0 +1,210 @@
+import { roleInfo, type Role } from "./state.js";
+
+/**
+ * Disk layout, schema v2.
+ *
+ * WHY THIS FILE EXISTS
+ * --------------------
+ * Three generators partition machines, and before this file they disagreed:
+ *
+ *   install.ks.ts        rancher 120G (infra|worker), longhorn --grow (worker)
+ *   ubuntu-autoinstall   rancher  20G (infra only),   longhorn grow   (worker)
+ *   asahi-firstboot.sh   rancher  20G (server only),  longhorn 100%FREE (worker|server)
+ *
+ * asahi-firstboot even carries the comment "sizes match install.ks.ts" while
+ * being 100 GB smaller — which is where worker1's undersized image store came
+ * from, and why a 2026-08 fix to one file never reached the fleet.
+ *
+ * v2 is defined ONCE, here, and all three generators render from it. v1 stays
+ * frozen inline in each generator, because every existing machine was built by
+ * it and reprovision must reproduce it byte-for-byte.
+ *
+ * SIZING
+ * ------
+ * Must fit a 1 TB disk (~926 GiB VG after EFI+/boot) with headroom left
+ * UNALLOCATED on purpose: LVM can grow into free extents online, and the
+ * fleet's recurring failure has been over-allocation, not under-allocation.
+ * `--grow` is deliberately never used — it is what left every node at VFree=0
+ * and made later fixes impossible without a rebuild.
+ */
+
+/**
+ * Roles that predate the v2 layout. Only these have a v1 layout to reproduce;
+ * everything else is v2-only, because no machine was ever installed as one.
+ */
+export const LEGACY_ROLES = ["vanilla", "worker", "infra", "labcontroller"] as const;
+
+/**
+ * True when this role has no v1 layout at all.
+ *
+ * Rendering one at v1 would silently produce base volumes with no rancher and
+ * no longhorn -- a machine that looks installed and cannot run k3s properly.
+ * Callers must reject this rather than guess.
+ */
+export function requiresSchemaV2(role: Role): boolean {
+  return !(LEGACY_ROLES as readonly string[]).includes(role);
+}
+
+/**
+ * The schema a FRESH install of this role should use.
+ *
+ * A role with no v1 layout has exactly one possible answer, so defaulting to it
+ * is unambiguous -- there is no other layout it could mean. Rejecting the
+ * request instead was a mistake: a reinstall legitimately passes no schema
+ * (the disk decides via the lab-schema-v2 tag), and refusing that made it
+ * impossible to reinstall a v2-only machine at all.
+ *
+ * An EXPLICIT v1 for such a role is still a contradiction and must be refused
+ * by the caller -- see requiresSchemaV2.
+ */
+export function effectiveSchemaVersion(role: Role, requested?: 1 | 2): SchemaVersion {
+  if (requested) return requested;
+  return requiresSchemaV2(role) ? 2 : 1;
+}
+
+export const SCHEMA_VERSIONS = [1, 2] as const;
+export type SchemaVersion = (typeof SCHEMA_VERSIONS)[number];
+
+export interface LvSpec {
+  /** LV name inside the volume group. */
+  name: string;
+  /** Mount point, or null for swap. */
+  mount: string | null;
+  /** Size in MiB. Never "grow". */
+  sizeMib: number;
+  fs: "xfs" | "swap";
+}
+
+/** Outside the VG: the EFI system partition and /boot. */
+export const V2_BOOT = {
+  efiMib: 1024,   // was 600 - firmware and shim have grown
+  bootMib: 4096,  // was 3072 - several kernels + initramfs
+} as const;
+
+const V2_BASE: readonly LvSpec[] = [
+  { name: "swap",   mount: null,       sizeMib: 16384, fs: "swap" },
+  { name: "root",   mount: "/",        sizeMib: 32768, fs: "xfs" },
+  { name: "var",    mount: "/var",     sizeMib: 49152, fs: "xfs" },
+  { name: "varlog", mount: "/var/log", sizeMib: 16384, fs: "xfs" },
+  { name: "home",   mount: "/home",    sizeMib:  8192, fs: "xfs" },
+  { name: "srv",    mount: "/srv",     sizeMib: 16384, fs: "xfs" },
+] as const;
+
+/** Image store AND local-path PVCs both live here - size for both. */
+export const V2_RANCHER_MIB = 256000;  // 250 GiB
+export const V2_LONGHORN_MIB = 307200; // 300 GiB
+
+/**
+ * Dedicated whole-disk data volumes (v2 only).
+ *
+ * The fixed 250G/300G LVs exist to FIT a 1 TB OS disk. A machine with spare
+ * NVMe bays (thebeast: 1 TB OS + 2 TB + 2 TB) wants the opposite trade: give
+ * each data mount a whole disk and keep the OS VG small. Each value is a
+ * stable /dev/disk/by-id/ path -- NEVER a kernel name, because nvme0n1/nvme1n1
+ * enumeration order is not stable across boots on multi-NVMe hosts, and a
+ * swapped name here formats the wrong disk.
+ *
+ * The disk gets one partition spanning it, xfs, labelled (DATA_DISK_LABELS)
+ * so a reprovision can recognise and preserve it the same way the LVs are
+ * recognised by name inside the VG. The corresponding LV is simply not
+ * created -- v2Layout() drops it when a dedicated disk supplies that mount.
+ */
+export interface DataDisks {
+  /** Whole disk for /var/lib/longhorn (replica store). */
+  longhorn?: string;
+  /** Whole disk for /var/lib/rancher (image store + local-path PVCs). */
+  rancher?: string;
+}
+
+/**
+ * Filesystem labels that mark a dedicated data disk's partition. They are the
+ * reprovision contract: %pre finds these by label and mounts them --noformat,
+ * exactly as it finds LVs by name. Changing a label orphans every disk
+ * already carrying the old one.
+ */
+export const DATA_DISK_LABELS = {
+  longhorn: "lab-longhorn",
+  rancher: "lab-rancher",
+} as const;
+
+export type DataDiskName = keyof typeof DATA_DISK_LABELS;
+
+const BY_ID_PREFIX = "/dev/disk/by-id/";
+
+/**
+ * Why a DataDisks spec is unusable for this role, or null when it is fine.
+ * Shared by the HTTP API, the websocket command handler and the CLI so all
+ * three reject with the same words instead of three drifting checks.
+ */
+export function validateDataDisks(role: Role, disks: DataDisks): string | null {
+  const info = roleInfo(role);
+  for (const name of Object.keys(DATA_DISK_LABELS) as DataDiskName[]) {
+    const device = disks[name];
+    if (device === undefined) continue;
+    if (!device.startsWith(BY_ID_PREFIX)) {
+      return `${name} disk must be a stable ${BY_ID_PREFIX} path (got '${device}'). ` +
+        `Kernel names like nvme0n1 change across boots on multi-NVMe hosts.`;
+    }
+  }
+  if (disks.rancher !== undefined && info.k3sMode === null) {
+    return `role '${role}' runs no k3s -- a dedicated rancher disk makes no sense for it`;
+  }
+  if (disks.longhorn !== undefined && (info.k3sMode === null || info.controlPlaneOnly)) {
+    return `role '${role}' holds no Longhorn replicas -- a dedicated longhorn disk makes no sense for it`;
+  }
+  if (disks.longhorn !== undefined && disks.longhorn === disks.rancher) {
+    return `longhorn and rancher point at the same disk (${disks.longhorn}) -- they need one each`;
+  }
+  return null;
+}
+
+/**
+ * The v2 volumes for a role, gated by CAPABILITY rather than role name.
+ *
+ * `omitRoot` is for Asahi, where the vendor installer already created a btrfs
+ * root partition outside our VG and we must not make a second one.
+ *
+ * `dataDisks` drops the longhorn/rancher LV whose mount a dedicated disk
+ * supplies -- two things mounting /var/lib/longhorn is an invalid layout.
+ */
+export function v2Layout(role: Role, opts: { omitRoot?: boolean; dataDisks?: DataDisks } = {}): LvSpec[] {
+  const info = roleInfo(role);
+  const lvs: LvSpec[] = V2_BASE.filter(
+    (lv) => !(opts.omitRoot && lv.name === "root"),
+  ).map((lv) => ({ ...lv }));
+
+  // Anything running k3s needs a dedicated image store. This is the check that
+  // `role === "infra" || role === "worker"` kept getting wrong: labcontroller
+  // runs k3s and was silently excluded.
+  if (info.k3sMode !== null && opts.dataDisks?.rancher === undefined) {
+    lvs.push({ name: "rancher", mount: "/var/lib/rancher", sizeMib: V2_RANCHER_MIB, fs: "xfs" });
+  }
+
+  // Fixed, never --grow. A control-plane-only node holds no replicas.
+  if (info.k3sMode !== null && !info.controlPlaneOnly && opts.dataDisks?.longhorn === undefined) {
+    lvs.push({ name: "longhorn", mount: "/var/lib/longhorn", sizeMib: V2_LONGHORN_MIB, fs: "xfs" });
+  }
+
+  return lvs;
+}
+
+/** Total MiB a role's v2 layout consumes inside the VG. */
+export function v2AllocatedMib(role: Role, opts: { omitRoot?: boolean; dataDisks?: DataDisks } = {}): number {
+  return v2Layout(role, opts).reduce((n, lv) => n + lv.sizeMib, 0);
+}
+
+/**
+ * Bind-mounting containerd onto the Longhorn LV.
+ *
+ * Codifies what was already done BY HAND on worker1 and exists in no template:
+ * its 20 GiB rancher LV could not hold the image store, so containerd was
+ * pointed at a directory on the big Longhorn LV. It survives reboot only
+ * because someone wrote a correct fstab line; a reinstall would lose it.
+ *
+ * v2 sizes rancher properly so this is not needed on new boxes - it is kept for
+ * rescuing v1 machines that cannot be repartitioned yet.
+ */
+export const CONTAINERD_BIND_FSTAB =
+  "/var/lib/longhorn/k3s-containerd /var/lib/rancher/k3s/agent/containerd none " +
+  "bind,x-systemd.requires-mounts-for=/var/lib/longhorn," +
+  "x-systemd.requires-mounts-for=/var/lib/rancher 0 0";

bastion/src/shared/src/types/state.ts

@@ -2,25 +2,15 @@
 
 export type ProvisionStackType = "dhcpproxy" | "iso" | "cloud-init";
 
-export type OsId = "fedora-43" | "ubuntu-26.04";
+export type OsId = "fedora-43" | "ubuntu-26.04" | "vyos-rolling";
 export type Arch = "x86_64" | "aarch64";
 
-export const SUPPORTED_OS: readonly OsId[] = ["fedora-43", "ubuntu-26.04"] as const;
+export const SUPPORTED_OS: readonly OsId[] = ["fedora-43", "ubuntu-26.04", "vyos-rolling"] as const;
 
 export function isValidOsId(value: string): value is OsId {
   return (SUPPORTED_OS as readonly string[]).includes(value);
 }
 
-/**
- * How a machine joins the lab.
- *
- * "pxe"  -- bare metal we install over the network (the default).
- * "ssh"  -- the machine already runs a vendor OS we cannot reproduce, so we onboard
- *           over SSH and manage userspace only. Installing would destroy that OS.
- *           See classifyOnboard() and os-install-research.md.
- */
-export type OnboardMethod = "pxe" | "ssh";
-
 export interface HardwareInfo {
   mac: string;
   product: string;
@@ -36,27 +26,34 @@ export interface HardwareInfo {
   first_seen: string;
   last_seen: string;
   bastionId?: string;  // set when aggregated through labd
-  // Onboarding classification -- absent means "pxe" (see classifyOnboard)
-  onboard?: OnboardMethod;
-  vendor_os?: string;   // e.g. "dgx-os": the OS this machine must keep running
-  // Root filesystem, for booting the installed system over PXE (--pxe-boot).
-  // Observed from the machine, never assumed.
-  root_device?: string;                 // e.g. "/dev/mapper/labvg-root"
-  root_args?: string;                   // e.g. "rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap"
-  root_candidates?: RootCandidate[];    // reported from a rescue shell when unknown
 }
 
-/** A possible root filesystem found while probing an unreachable machine. */
-export interface RootCandidate {
-  device: string;       // e.g. "/dev/mapper/labvg-root"
-  args?: string;        // extra dracut args needed to assemble it
-  fstype?: string;
-  size_gb?: number;
-  os_release?: string;  // PRETTY_NAME from /etc/os-release, if mountable
-}
+export type Role =
+  // Legacy roles. Every machine in the fleet today is one of these, and they
+  // default to the v1 disk layout so a reinstall reproduces what is on disk.
+  | "vanilla" | "worker" | "infra" | "labcontroller"
+  // Topology-named profiles. New machines only, so they are v2-only: there is
+  // no v1 layout to be compatible with, and none of them can reach the frozen
+  // v1 branch in the templates.
+  | "standalone-cluster" | "k8s-controller" | "k8s-worker" | "k8s-combo";
 
-export type Role = "vanilla" | "worker" | "infra" | "labcontroller";
-export const SUPPORTED_ROLES: readonly Role[] = ["vanilla", "worker", "infra", "labcontroller"] as const;
+export const SUPPORTED_ROLES: readonly Role[] = [
+  "vanilla", "worker", "infra", "labcontroller",
+  "standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo",
+] as const;
+
+/**
+ * How a machine comes to exist, which decides whether we partition it at all.
+ *
+ *  - "managed"   we PXE-install it with our kickstart; our disk layout applies.
+ *  - "appliance" we install it, but the vendor installer owns the disk (VyOS).
+ *  - "onboard"   we never install it; we adopt a running machine (DGX Spark,
+ *                Apple Silicon under Asahi). No partitioning, ever.
+ */
+export type Lifecycle = "managed" | "appliance" | "onboard";
+
+/** null = no k3s at all. */
+export type K3sMode = "server" | "agent" | null;
 
 export interface RoleInfo {
   name: Role;
@@ -64,6 +61,18 @@ export interface RoleInfo {
   parent?: Role;       // inherits from this role
   k3s: boolean;        // installs k3s
   apps: string[];      // apps auto-deployed after k3s
+
+  // --- capabilities -------------------------------------------------------
+  // These exist so decisions are made by asking what a machine DOES, not by
+  // comparing its role name. Every `role === "infra" || role === "labcontroller"`
+  // in this codebase is a place that forgot a role: `labcontroller` runs k3s and
+  // still got no rancher LV, because install.ks.ts asked `role === "infra"`.
+  lifecycle: Lifecycle;
+  k3sMode: K3sMode;
+  /** Holds Longhorn replicas, i.e. wants a dedicated /var/lib/longhorn LV. */
+  hostsLonghorn: boolean;
+  /** Control plane that deliberately runs no workloads and no storage. */
+  controlPlaneOnly: boolean;
 }
 
 export const ROLE_REGISTRY: readonly RoleInfo[] = [
@@ -72,6 +81,10 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [
     description: "OS only — no k3s, no cluster services",
     k3s: false,
     apps: [],
+    lifecycle: "managed",
+    k3sMode: null,
+    hostsLonghorn: false,
+    controlPlaneOnly: false,
   },
   {
     name: "worker",
@@ -79,6 +92,10 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [
     parent: "vanilla",
     k3s: true,
     apps: [],
+    lifecycle: "managed",
+    k3sMode: "agent",
+    hostsLonghorn: true,
+    controlPlaneOnly: false,
   },
   {
     name: "infra",
@@ -86,6 +103,15 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [
     parent: "vanilla",
     k3s: true,
     apps: [],
+    lifecycle: "managed",
+    k3sMode: "server",
+    // NOTE: worker0/1/2 are `infra` and DO hold every Longhorn replica in the
+    // homelab cluster, but the v1 kickstart only ever gave a longhorn LV to
+    // `worker`. On worker2 that means Longhorn data sits on /var alongside the
+    // image store. Left false so the v1 layout is unchanged; v2 sizes it
+    // properly from this flag.
+    hostsLonghorn: false,
+    controlPlaneOnly: false,
   },
   {
     name: "labcontroller",
@@ -93,22 +119,258 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [
     parent: "infra",
     k3s: true,
     apps: ["cockroachdb", "labd", "bastion"],
+    lifecycle: "managed",
+    k3sMode: "server",
+    hostsLonghorn: false,
+    controlPlaneOnly: false,
+  },
+  {
+    name: "standalone-cluster",
+    description: "Single node that IS its own cluster — k3s server + workloads + storage",
+    parent: "vanilla",
+    k3s: true,
+    apps: [],
+    lifecycle: "managed",
+    k3sMode: "server",
+    // Longhorn yes, at replica 1. A cluster of one cannot satisfy a 3-replica
+    // policy, but keeping the `longhorn` StorageClass present means manifests
+    // are portable across every cluster instead of needing a local-path
+    // variant. Replica count is a property of the cluster, not the manifest.
+    hostsLonghorn: true,
+    controlPlaneOnly: false,
+  },
+  {
+    name: "k8s-controller",
+    description: "Control plane + etcd only — runs no workloads and holds no storage",
+    parent: "vanilla",
+    k3s: true,
+    apps: [],
+    lifecycle: "managed",
+    k3sMode: "server",
+    hostsLonghorn: false,
+    controlPlaneOnly: true,
+  },
+  {
+    name: "k8s-worker",
+    description: "k3s agent + Longhorn storage — joins an existing cluster",
+    parent: "vanilla",
+    k3s: true,
+    apps: [],
+    lifecycle: "managed",
+    k3sMode: "agent",
+    hostsLonghorn: true,
+    controlPlaneOnly: false,
+  },
+  {
+    name: "k8s-combo",
+    description: "Control plane + etcd AND workloads + storage — hyperconverged cluster member",
+    parent: "vanilla",
+    k3s: true,
+    apps: [],
+    lifecycle: "managed",
+    k3sMode: "server",
+    hostsLonghorn: true,
+    controlPlaneOnly: false,
   },
 ] as const;
 
+/**
+ * Capability lookups. Use these instead of comparing role names.
+ *
+ * Every call site that asks `role === "infra" || role === "labcontroller"` is a
+ * place that has to be edited again when a role is added — and the one that was
+ * missed (install.ks.ts asking `role === "infra"`) is why `labcontroller` boxes
+ * get no dedicated image-store LV.
+ */
+/**
+ * Narrow an untrusted string (state file, HTTP body) to a Role.
+ * State is JSON on disk and can contain anything, including a role that was
+ * removed in a later version -- so this is a guard, not a cast.
+ */
+export function isRole(value: string): value is Role {
+  return (SUPPORTED_ROLES as readonly string[]).includes(value);
+}
+
+export function roleInfo(role: Role): RoleInfo {
+  const info = ROLE_REGISTRY.find((r) => r.name === role);
+  if (!info) throw new Error(`unknown role: ${role}`);
+  return info;
+}
+
+/** Runs k3s in any capacity. */
+export function runsK3s(role: Role): boolean {
+  return roleInfo(role).k3sMode !== null;
+}
+
+/** Runs k3s as a server (control plane + etcd) rather than an agent. */
+export function isK3sServer(role: Role): boolean {
+  return roleInfo(role).k3sMode === "server";
+}
+
+/** We PXE-install this machine, so our disk layout applies to it. */
+export function isManaged(role: Role): boolean {
+  return roleInfo(role).lifecycle === "managed";
+}
+
+/** Wants a dedicated /var/lib/longhorn LV. */
+export function hostsLonghorn(role: Role): boolean {
+  return roleInfo(role).hostsLonghorn;
+}
+
 export interface ProgressLogEntry {
   stage: string;
   detail: string;
   timestamp: string;
 }
 
+/** A tagged VLAN sub-interface on the bond (or on the mgmt NIC when unbonded). */
+export interface VyosVlanSpec {
+  id: number;
+  address: string;         // CIDR, e.g. "10.0.10.1/24"
+  description?: string;
+  /**
+   * VRRP virtual address (CIDR) floated on this VLAN. Emitted as a
+   * high-availability vrrp group with vrid = VLAN id, so the same spec on both
+   * HA peers (with different priorities) produces a matching group pair.
+   */
+  vrrp?: string;
+}
+
+/** One config node in a rendered bundle. Mirrors VyosSetOp on the bastion side. */
+export interface VyosBundleSetOp {
+  path: string[];
+  value?: string;
+  /** false appends to a multi-value node (e.g. bond members) instead of replacing. */
+  replace?: boolean;
+}
+
+/**
+ * A router's complete desired config, rendered from the Pulumi model.
+ *
+ * Produced by `kubernetes-deployment/scripts/vyos-render-bundle.ts` from the
+ * same subtree model `pulumi up` applies. The point is that labctl never
+ * authors VyOS config: bring-up replays what Pulumi already declares, so a
+ * freshly installed router and a `pulumi up` cannot disagree.
+ *
+ * Secret values arrive as `@secret:<key>` sentinels and are DROPPED at install
+ * time -- the bundle is committed to git and must stay safe to read. The router
+ * comes up on the LAN without its PPPoE credential; the first `pulumi up`
+ * supplies it. That handoff is deliberate.
+ */
+export interface VyosBundle {
+  sets: VyosBundleSetOp[];
+  /** Paths that are VyOS tag nodes — the installer's ConfigTree needs them marked. */
+  tags: string[][];
+}
+
+/**
+ * VyOS-specific install parameters. Rendered into the config.boot that the
+ * installer adopts, so the router comes up already configured.
+ *
+ * NOTE: bondMembers must NOT include the interface PXE booted from. Firmware
+ * PXE cannot run over LACP, so the install-time NIC has to stay unbonded.
+ */
+export interface VyosInstallSpec {
+  /**
+   * A complete rendered config for this router. When present it REPLACES the
+   * derived interface/VLAN/VRRP config below -- the bundle already describes
+   * all of it, and deriving a second opinion is exactly the drift this exists
+   * to prevent. The remaining install parameters (password, disk, console) are
+   * still honoured because they are installer inputs, not router config.
+   */
+  bundle?: VyosBundle;
+  /**
+   * Key for the VyOS HTTP API, enabled at install so the router is manageable
+   * from the moment it boots.
+   *
+   * Without this the box comes up reachable only over SSH, and enabling the API
+   * later is a hand-run config change on a live firewall -- which is exactly the
+   * gap that left vyos001/vyos002 unmanageable by Pulumi after their cutover.
+   * The API is deliberately NOT part of the Pulumi model: a provider that
+   * manages its own transport can revoke its own access.
+   */
+  apiKey?: string;
+  /**
+   * Address the API listens on. Defaults to the management address when static.
+   * Never left unbound: an unrestricted listener puts a config-write endpoint on
+   * every segment the router touches, including the WAN.
+   */
+  apiListenAddress?: string;
+  /** Interfaces aggregated into bond0 with LACP (802.3ad). Omit for no bond. */
+  bondMembers?: string[];
+  /** CIDR address on bond0 itself — the switch trunk's native/untagged VLAN. */
+  bondAddress?: string;
+  /** VRRP virtual address (CIDR) floated on the untagged bond (vrid 1). */
+  bondVrrp?: string;
+  /**
+   * VRRP priority for every group on this box. Higher wins mastership.
+   * The HA pair differs ONLY here (e.g. 200 on the primary, 100 on the
+   * standby) — addresses differ per box, VIPs and vrids match.
+   */
+  vrrpPriority?: number;
+  /** Tagged VLAN sub-interfaces, created on bond0 when bonded, else on mgmtInterface. */
+  vlans?: VyosVlanSpec[];
+  /** Untagged interface the machine PXE booted from. Defaults to "eth0". */
+  mgmtInterface?: string;
+  /** CIDR address for mgmtInterface, or "dhcp". Defaults to "dhcp". */
+  mgmtAddress?: string;
+  /**
+   * Tagged management VLAN on mgmtInterface, separate from the routed VLANs
+   * carried by the bond.
+   *
+   * Needed when the PXE port is a trunk: it boots untagged on the VLAN the
+   * bastion's proxy DHCP serves, and carries the management VLAN tagged so the
+   * router stays reachable there without giving up reinstallability.
+   */
+  mgmtVlan?: VyosVlanSpec;
+  /** Password for the "vyos" user. Falls back to the bastion default. */
+  password?: string;
+  /**
+   * On reinstall the VyOS installer carries the previous on-disk config (and
+   * SSH host keys) forward -- the "reinstall without losing data" default.
+   * Set true to make the bastion-generated config win instead: after install
+   * the driver overwrites the installed image's config.boot.
+   */
+  freshConfig?: boolean;
+  /**
+   * VyOS interface name -> MAC, emitted as `hw-id` so names bind deterministically.
+   *
+   * Discovery runs under Fedora and reports predictable names (enp2s0,
+   * enp1s0f0np0), but VyOS enumerates its own eth<N> names, so a name observed
+   * during discovery cannot be used directly. Pinning by MAC removes the guess
+   * about which physical port a given eth<N> is.
+   */
+  hwIds?: Record<string, string>;
+}
+
 export interface InstallConfig {
   hostname: string;
   disk: string;
   role: Role;
   os?: OsId;          // defaults to "fedora-43" for backward compat
+  vyos?: VyosInstallSpec;  // only consulted when os is "vyos-rolling"
   arch?: Arch;         // detected from HardwareInfo or overridden
+  /**
+   * Disk layout for a FRESH install. Defaults to 1.
+   * Ignored on a reprovision: %pre reads the lab-schema-v2 LVM tag off the
+   * disk instead, because reinstalling a machine must never change its layout.
+   */
+  schemaVersion?: 1 | 2;
+  /**
+   * Dedicated whole-disk data volumes (v2 + fedora only), by stable
+   * /dev/disk/by-id path. See DataDisks in layout.ts. On a reprovision these
+   * are optional: %pre rediscovers the disks by filesystem label and preserves
+   * them whether or not the queue entry names them.
+   */
+  dataDisks?: import("./layout.js").DataDisks;
   queued_at: string;
+  /**
+   * When dispatch last served this machine an install boot script. Progress
+   * callbacks only start once the installer environment is up, so a machine
+   * dispatched long ago with no progress is wedged before that point (bad
+   * kernel/initrd, no network in the initramfs, wrong NIC picked...).
+   */
+  dispatched_at?: string;
   progress?: string;
   progress_at?: string;
   progress_detail?: string;
@@ -119,6 +381,12 @@ export interface InstallConfig {
 export interface InstalledInfo {
   hostname: string;
   role: string;
+  /**
+   * Disk layout this machine was installed with. Advisory only -- the
+   * authoritative answer is the `lab-schema-v2` LVM tag on the machine's own
+   * volume group, because that is what a reinstall actually reads.
+   */
+  schemaVersion?: 1 | 2;
   os?: OsId;
   ip: string;
   installed_at: string;
@@ -130,11 +398,6 @@ export interface InstalledInfo {
   cpu_cores?: number;
   memory_gb?: number;
   arch?: string;
-  onboard?: OnboardMethod;
-  vendor_os?: string;
-  root_device?: string;
-  root_args?: string;
-  root_candidates?: RootCandidate[];
 }
 
 export interface DebugConfig {

bastion/src/shared/tests/capabilities.test.ts

@@ -0,0 +1,129 @@
+import { describe, it, expect } from "vitest";
+import {
+  SUPPORTED_ROLES,
+  ROLE_REGISTRY,
+  roleInfo,
+  runsK3s,
+  isK3sServer,
+  isManaged,
+  hostsLonghorn,
+} from "../src/types/state.js";
+import type { Role } from "../src/types/state.js";
+
+/**
+ * The capability lookups exist to replace role-name comparisons scattered across
+ * the codebase. Introducing them must change NO behaviour — so these tests pin
+ * each helper against the literal expression it replaces, copied verbatim from
+ * the call site it came from.
+ *
+ * If one of these fails, the refactor changed behaviour somewhere. That is the
+ * whole point of the file.
+ */
+/**
+ * Only the FOUR legacy roles have behaviour to preserve. The topology profiles
+ * (standalone-cluster, k8s-*) are new, v2-only, and have no historical
+ * expression to be compatible with -- pinning them against `role === "worker"`
+ * would assert nonsense.
+ */
+const LEGACY_ROLES = ["vanilla", "worker", "infra", "labcontroller"] as const;
+
+describe("capability helpers reproduce the expressions they replace", () => {
+  // Verbatim from the call sites, as of the commit that introduced this file.
+  const legacy = {
+    // k3s-install.ts:7, k3s-config.ts:7, render-config.ts:25,
+    // k3s-module.ts:65, health/k3s-service.ts:5, post-provision.ts:153
+    isServer: (r: Role) => r === "infra" || r === "labcontroller",
+    // ROLE_REGISTRY.k3s — the field that already existed and nobody read
+    runsK3s: (r: Role) => ROLE_REGISTRY.find((x) => x.name === r)!.k3s,
+    // install.ks.ts:41, ubuntu-autoinstall.ts:34
+    hasLonghorn: (r: Role) => r === "worker",
+    // install.ks.ts:48
+    isVanilla: (r: Role) => r === "vanilla",
+  };
+
+  it.each(LEGACY_ROLES)("isK3sServer matches the legacy expression for %s", (role) => {
+    expect(isK3sServer(role)).toBe(legacy.isServer(role));
+  });
+
+  it.each(LEGACY_ROLES)("runsK3s matches ROLE_REGISTRY.k3s for %s", (role) => {
+    expect(runsK3s(role)).toBe(legacy.runsK3s(role));
+  });
+
+  it.each(LEGACY_ROLES)("hostsLonghorn matches the legacy expression for %s", (role) => {
+    expect(hostsLonghorn(role)).toBe(legacy.hasLonghorn(role));
+  });
+
+  it.each(LEGACY_ROLES)("k3sMode === null matches isVanilla for %s", (role) => {
+    expect(roleInfo(role).k3sMode === null).toBe(legacy.isVanilla(role));
+  });
+
+  it("every supported role is managed today (nothing is onboarded yet)", () => {
+    for (const role of SUPPORTED_ROLES) expect(isManaged(role)).toBe(true);
+  });
+});
+
+/**
+ * The topology profiles. These are asserted directly rather than against a
+ * legacy expression, because they describe cluster SHAPE -- which is the thing
+ * role names were supposed to convey and did not.
+ */
+describe("topology profiles", () => {
+  it("standalone-cluster is a server that also holds storage", () => {
+    expect(isK3sServer("standalone-cluster")).toBe(true);
+    // Longhorn at replica 1: a cluster of one cannot do 3, but keeping the
+    // StorageClass present keeps manifests portable across clusters.
+    expect(hostsLonghorn("standalone-cluster")).toBe(true);
+    expect(roleInfo("standalone-cluster").controlPlaneOnly).toBe(false);
+  });
+
+  it("k8s-controller is the only role that runs k3s and holds no storage", () => {
+    expect(isK3sServer("k8s-controller")).toBe(true);
+    expect(hostsLonghorn("k8s-controller")).toBe(false);
+    expect(roleInfo("k8s-controller").controlPlaneOnly).toBe(true);
+  });
+
+  it("k8s-worker is an agent with storage", () => {
+    expect(isK3sServer("k8s-worker")).toBe(false);
+    expect(runsK3s("k8s-worker")).toBe(true);
+    expect(hostsLonghorn("k8s-worker")).toBe(true);
+  });
+
+  it("k8s-combo is both control plane and storage — unlike k8s-controller", () => {
+    expect(isK3sServer("k8s-combo")).toBe(true);
+    expect(hostsLonghorn("k8s-combo")).toBe(true);
+    expect(roleInfo("k8s-combo").controlPlaneOnly).toBe(false);
+  });
+
+  it("k8s-combo and k8s-controller differ ONLY in storage duty", () => {
+    const combo = roleInfo("k8s-combo");
+    const ctrl = roleInfo("k8s-controller");
+    expect(combo.k3sMode).toBe(ctrl.k3sMode);
+    expect(combo.hostsLonghorn).not.toBe(ctrl.hostsLonghorn);
+  });
+});
+
+describe("registry integrity", () => {
+  it("every SUPPORTED_ROLE has a registry entry", () => {
+    for (const role of SUPPORTED_ROLES) expect(() => roleInfo(role)).not.toThrow();
+  });
+
+  it("registry has no entries outside SUPPORTED_ROLES", () => {
+    for (const info of ROLE_REGISTRY) expect(SUPPORTED_ROLES).toContain(info.name);
+  });
+
+  it("k3s boolean and k3sMode never disagree", () => {
+    for (const info of ROLE_REGISTRY) {
+      expect(info.k3s).toBe(info.k3sMode !== null);
+    }
+  });
+
+  it("an unknown role throws rather than silently defaulting", () => {
+    expect(() => roleInfo("does-not-exist" as Role)).toThrow(/unknown role/);
+  });
+
+  it("a control-plane-only role never hosts Longhorn", () => {
+    for (const info of ROLE_REGISTRY) {
+      if (info.controlPlaneOnly) expect(info.hostsLonghorn).toBe(false);
+    }
+  });
+});

bastion/src/shared/tests/layout.test.ts

@@ -0,0 +1,92 @@
+import { describe, it, expect } from "vitest";
+import { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB } from "../src/types/layout.js";
+import { SUPPORTED_ROLES, roleInfo } from "../src/types/state.js";
+
+/** A 1 TB disk is the smallest we will ever buy. 1e12 bytes in MiB. */
+const ONE_TB_MIB = Math.floor(1_000_000_000_000 / (1024 * 1024)); // 953674
+const VG_MIB_ON_1TB = ONE_TB_MIB - V2_BOOT.efiMib - V2_BOOT.bootMib;
+
+describe("v2 layout fits the smallest disk we will buy", () => {
+  it.each(SUPPORTED_ROLES)("%s fits in a 1 TB disk with headroom left over", (role) => {
+    const used = v2AllocatedMib(role);
+    expect(used).toBeLessThan(VG_MIB_ON_1TB);
+    // The entire point of v2 is to STOP consuming the disk. Demand real slack.
+    const freeMib = VG_MIB_ON_1TB - used;
+    expect(freeMib).toBeGreaterThan(100 * 1024); // >100 GiB unallocated
+  });
+
+  it("the heaviest role still leaves >200 GiB free on 1 TB", () => {
+    const worst = Math.max(...SUPPORTED_ROLES.map((r) => v2AllocatedMib(r)));
+    expect(VG_MIB_ON_1TB - worst).toBeGreaterThan(200 * 1024);
+  });
+});
+
+describe("v2 gates volumes by capability, not role name", () => {
+  it.each(SUPPORTED_ROLES)("%s gets a rancher LV iff it runs k3s", (role) => {
+    const hasRancher = v2Layout(role).some((lv) => lv.name === "rancher");
+    expect(hasRancher).toBe(roleInfo(role).k3sMode !== null);
+  });
+
+  it("labcontroller gets a rancher LV — the v1 bug this schema fixes", () => {
+    const lvs = v2Layout("labcontroller");
+    const rancher = lvs.find((lv) => lv.name === "rancher");
+    expect(rancher).toBeDefined();
+    expect(rancher!.sizeMib).toBe(V2_RANCHER_MIB);
+  });
+
+  it("vanilla gets neither rancher nor longhorn", () => {
+    const names = v2Layout("vanilla").map((lv) => lv.name);
+    expect(names).not.toContain("rancher");
+    expect(names).not.toContain("longhorn");
+  });
+
+  it("a control-plane-only role would get no longhorn LV", () => {
+    for (const role of SUPPORTED_ROLES) {
+      if (roleInfo(role).controlPlaneOnly) {
+        expect(v2Layout(role).map((l) => l.name)).not.toContain("longhorn");
+      }
+    }
+  });
+});
+
+describe("v2 never over-allocates", () => {
+  it("no volume is grow/unbounded — every size is a concrete number", () => {
+    for (const role of SUPPORTED_ROLES) {
+      for (const lv of v2Layout(role)) {
+        expect(Number.isFinite(lv.sizeMib)).toBe(true);
+        expect(lv.sizeMib).toBeGreaterThan(0);
+      }
+    }
+  });
+
+  it("longhorn is fixed at 300 GiB, not grown", () => {
+    const lh = v2Layout("worker").find((lv) => lv.name === "longhorn");
+    expect(lh?.sizeMib).toBe(V2_LONGHORN_MIB);
+    expect(V2_LONGHORN_MIB).toBe(300 * 1024);
+  });
+
+  it("rancher is sized for an image store AND local-path PVCs", () => {
+    // A single ROCm vLLM image measured 62 GiB extracted on 2026-09-17.
+    expect(V2_RANCHER_MIB).toBeGreaterThan(200 * 1024);
+  });
+});
+
+describe("Asahi variant", () => {
+  it("omitRoot drops the root LV — the vendor installer already made one", () => {
+    const names = v2Layout("infra", { omitRoot: true }).map((lv) => lv.name);
+    expect(names).not.toContain("root");
+    expect(names).toContain("var");
+  });
+
+  it("omitRoot frees the root allocation and nothing else", () => {
+    const delta = v2AllocatedMib("infra") - v2AllocatedMib("infra", { omitRoot: true });
+    expect(delta).toBe(32768);
+  });
+});
+
+describe("layout is a pure value, safe to render three times", () => {
+  it("mutating a returned spec does not affect the next call", () => {
+    v2Layout("worker")[0]!.sizeMib = 999;
+    expect(v2Layout("worker")[0]!.sizeMib).not.toBe(999);
+  });
+});

bastion/tests/integration/arm-pxe-provision.test.ts

@@ -1,537 +0,0 @@
-// Integration test: aarch64 network PXE boot.
-//
-// The boot-ISO path already covered ARM (arm-iso-provision.test.ts). This covers the
-// network path: DHCP option 93 handing an arm64 client an arm64 iPXE binary, dispatch
-// serving an aarch64 kernel, and `provision debug` reaching a rescue shell -- which is
-// what the DGX Sparks actually need and could not do.
-//
-// Two suites, because they cost very different amounts of time:
-//
-//   "ARM PXE rescue"   NBP handoff -> rescue with SSH.                    ~25-30 min
-//   "ARM PXE install"  discover -> install -> installed.                  ~75-95 min
-//
-// The rescue suite seeds the machine into state as an already-known aarch64 box rather
-// than discovering it first. That is the DGX Spark situation exactly -- SSH-onboarded,
-// never PXE-discovered, architecture known only from its record -- and it holds the test
-// to one emulated boot. Each boot spends ~15 of its ~18 minutes downloading Anaconda's
-// stage2 under TCG, so discovering first would double the runtime without touching any
-// code path the rescue boot does not already exercise.
-//
-// The install suite only runs with ARM_PXE_FULL=1. No ARM machine in the lab is ever
-// PXE-installed except the MS-R1, and an hour-plus test that runs by default is a test
-// nobody runs.
-//
-// IMPORTANT: aarch64 has no KVM on an x86_64 host, so all of this is emulated and
-// roughly 10x slower than native.
-//
-// A note for whoever debugs a failure here: if the VM panics with
-//   VFS: Unable to mount root fs on unknown-block(0,0)
-// that is very likely iPXE silently dropping the initrd because the build lacks
-// EFI_LOAD_FILE2_PROTOCOL -- on arm64 the kernel EFI stub fetches the initrd over
-// LoadFile2, and an iPXE without it accepts the `initrd` line and does nothing. It is
-// NOT a reproduction of the DGX Spark kernel bug that motivated this work, despite
-// being the identical message. assertIpxeSupportsLoadFile2() below checks the build up
-// front so that failure names itself; to check by hand:
-//   node -e 'const b=require("fs").readFileSync("/usr/share/ipxe/arm64-efi/snponly.efi");
-//            console.log(b.indexOf(Buffer.from("c1c00640b3fc3e40996d4a6c8724e06d","hex")))'
-// Fedora's ipxe-bootimgs-aarch64-20240119 has it at 0x3bbf0.
-//
-// Prerequisites:
-//   - qemu-system-aarch64        (sudo dnf install qemu-system-aarch64)
-//   - edk2-aarch64               (sudo dnf install edk2-aarch64)
-//   - ipxe-bootimgs-aarch64      (sudo dnf install ipxe-bootimgs-aarch64)
-//   - libvirtd, sudo, internet access
-//
-// Run: sudo ./scripts/test-provision.sh arm-pxe
-
-import { describe, it, expect, beforeAll, afterAll } from "vitest";
-import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, writeFileSync } from "node:fs";
-import { execSync } from "node:child_process";
-import { join } from "node:path";
-import { homedir, tmpdir } from "node:os";
-import { log, waitForSsh } from "./helpers/libvirt.js";
-import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js";
-import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./helpers/pxe-vm.js";
-import { sshExec } from "./helpers/ssh.js";
-
-const IPXE_ARM64 = "/usr/share/ipxe/arm64-efi/snponly.efi";
-const AAVMF = "/usr/share/edk2/aarch64/QEMU_EFI.fd";
-
-const VM_MEMORY = 4096;
-const VM_VCPUS = 2;
-const VM_DISK_GB = 250;
-const SSH_USER = "lab";
-const BASTION_IP = PXE_GATEWAY;
-const DHCP_RANGE_START = `${PXE_SUBNET}.100`;
-const DHCP_RANGE_END = `${PXE_SUBNET}.200`;
-const SERIAL_PORT = 4555;
-
-// Emulated aarch64 -- generous timeouts throughout. Measured on an x86_64 host with no
-// KVM for aarch64: a single PXE boot to a running Anaconda takes ~18 minutes, almost all
-// of it downloading inst.stage2 over the network under TCG. Budget well above that;
-// timing out just short of success wastes a whole run.
-const LEASE_TIMEOUT_MS = 10 * 60_000;
-const DISCOVERY_TIMEOUT_MS = 35 * 60_000;
-const INSTALL_TIMEOUT_MS = 75 * 60_000;
-const SSH_TIMEOUT_MS = 35 * 60_000;
-
-const RUN_FULL_INSTALL = process.env["ARM_PXE_FULL"] === "1";
-
-function sleep(ms: number): Promise<void> {
-  return new Promise((r) => setTimeout(r, ms));
-}
-
-function findSshKey(): { pubKey: string; keyPath: string } {
-  const candidates: string[] = [];
-  if (process.env["SSH_KEY_PATH"]) candidates.push(process.env["SSH_KEY_PATH"]);
-  const homes = [homedir()];
-  const sudoUser = process.env["SUDO_USER"];
-  if (sudoUser) homes.push(join("/home", sudoUser));
-  for (const home of homes) {
-    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
-      candidates.push(join(home, ".ssh", name));
-    }
-  }
-  for (const keyPath of candidates) {
-    if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
-      return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
-    }
-  }
-  throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/");
-}
-
-async function pollApi<T>(
-  url: string,
-  check: (data: T) => boolean,
-  timeoutMs: number,
-  intervalMs = 10_000,
-): Promise<T> {
-  const start = Date.now();
-  while (Date.now() - start < timeoutMs) {
-    try {
-      const res = await fetch(url);
-      if (res.ok) {
-        const data = (await res.json()) as T;
-        if (check(data)) return data;
-      }
-    } catch { /* bastion not up yet, or a network hiccup */ }
-    await sleep(intervalMs);
-  }
-  throw new Error(`Timeout after ${timeoutMs}ms polling ${url}`);
-}
-
-function requirePrerequisites(): void {
-  if (!existsSync("/usr/bin/qemu-system-aarch64")) {
-    throw new Error("qemu-system-aarch64 not installed. Run: sudo dnf install qemu-system-aarch64");
-  }
-  if (!existsSync(AAVMF)) {
-    throw new Error(`AAVMF firmware not found at ${AAVMF}. Run: sudo dnf install edk2-aarch64`);
-  }
-  if (!existsSync(IPXE_ARM64)) {
-    throw new Error(`arm64 iPXE not found at ${IPXE_ARM64}. Run: sudo dnf install ipxe-bootimgs-aarch64`);
-  }
-}
-
-/**
- * Confirm the arm64 iPXE binary implements EFI_LOAD_FILE2_PROTOCOL.
- *
- * Without it the `initrd` line is accepted and silently ignored, and the kernel panics
- * with unknown-block(0,0). Checking here turns a confusing 30-minute boot failure into
- * an immediate, explanatory one.
- *
- * GUID 4006c0c1-fcb3-403e-996d-4a6c8724e06d, little-endian in the binary's GUID table.
- */
-function assertIpxeSupportsLoadFile2(): void {
-  const LOAD_FILE2_GUID = Buffer.from("c1c00640b3fc3e40996d4a6c8724e06d", "hex");
-  const binary = readFileSync(IPXE_ARM64);
-  if (binary.indexOf(LOAD_FILE2_GUID) < 0) {
-    throw new Error(
-      `${IPXE_ARM64} does not reference EFI_LOAD_FILE2_PROTOCOL. On arm64 the kernel ` +
-      `EFI stub fetches the initrd over LoadFile2; without it iPXE drops the initrd ` +
-      `silently and the kernel panics with "unknown-block(0,0)". Rebuild iPXE with ` +
-      `LoadFile2, or chainload grubaa64.efi for aarch64 instead.`,
-    );
-  }
-  log(`iPXE arm64 implements LoadFile2 — initrd will be delivered to the EFI stub`);
-}
-
-interface Harness {
-  testDir: string;
-  app: { close: () => Promise<void> };
-  stopDnsmasq: () => void;
-  state: { update: (fn: (s: BastionStateLike) => void) => void };
-  vmMac: string;
-  httpPort: number;
-}
-
-/** Just the parts of BastionState this test seeds. */
-interface BastionStateLike {
-  discovered: Record<string, Record<string, unknown>>;
-  installed: Record<string, Record<string, unknown>>;
-  install_queue: Record<string, Record<string, unknown>>;
-  debug: Record<string, Record<string, unknown>>;
-}
-
-/** Bring up an isolated network, a bastion with both arch payloads, and an arm64 VM. */
-async function startHarness(vmName: string, httpPort: number, pubKey: string): Promise<Harness> {
-  requirePrerequisites();
-  assertIpxeSupportsLoadFile2();
-
-  log("Setting up PXE test network...");
-  ensurePxeNetwork();
-
-  const testDir = join(tmpdir(), `lab-arm-pxe-test-${Date.now()}`);
-  for (const sub of ["tftp", "http", "logs"]) {
-    mkdirSync(join(testDir, sub), { recursive: true });
-  }
-
-  const { createApp } = await import("../../src/bastion/src/server.js");
-  const { loadConfig } = await import("../../src/bastion/src/config.js");
-  const { generateDnsmasqConf, startDnsmasq, stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
-  const { generateDiscoverKickstart } = await import("../../src/bastion/src/services/kickstart-generator.js");
-  const { renderBootIpxe, kernelPath, initrdPath } = await import("../../src/bastion/src/templates/boot.ipxe.js");
-  // Relative, not "@lab/shared": these tests run from the repo root against sources,
-  // where the workspace package alias is not resolvable.
-  const { SUPPORTED_ARCHES, fedoraMirrorFor } = await import("../../src/shared/src/hardware/index.js");
-
-  const config = loadConfig({
-    bastionDir: testDir,
-    httpPort,
-    iface: "virbr-pxe",
-    serverIp: BASTION_IP,
-    network: `${PXE_SUBNET}.0`,
-    gateway: BASTION_IP,
-    dhcpMode: "full",
-    dhcpRangeStart: DHCP_RANGE_START,
-    dhcpRangeEnd: DHCP_RANGE_END,
-    domain: "arm-pxe-test.local",
-    sshKeys: [pubKey],
-    adminUser: SSH_USER,
-  });
-
-  // iPXE binaries. The arm64 one is the whole point: dnsmasq hands it out on DHCP
-  // option 93 -- 11 for UEFI PXE (TFTP) and 19 for UEFI HTTP Boot.
-  //
-  // They go in BOTH directories, exactly as main.ts stages them. AAVMF prefers HTTP
-  // Boot, so it is served an http:// URL and fetches from httpDir; a firmware that
-  // takes the TFTP path reads the same file from tftpDir. Staging only tftpDir gives a
-  // 404 and "No bootable option or device was found" on the console.
-  log("Staging iPXE binaries...");
-  const ipxeX86 = "/usr/share/ipxe/ipxe-snponly-x86_64.efi";
-  copyFileSync(IPXE_ARM64, join(config.tftpDir, "ipxe-arm64.efi"));
-  copyFileSync(IPXE_ARM64, join(config.httpDir, "ipxe-arm64.efi"));
-  if (existsSync(ipxeX86)) {
-    copyFileSync(ipxeX86, join(config.tftpDir, "ipxe.efi"));
-    copyFileSync(ipxeX86, join(config.httpDir, "ipxe.efi"));
-  }
-
-  // Fedora kernel + initrd for both architectures, cached across runs.
-  const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache";
-  execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" });
-
-  for (const arch of SUPPORTED_ARCHES) {
-    const mirror = fedoraMirrorFor(config.fedoraVersion, arch);
-    const kernelCache = join(cacheDir, `vmlinuz-${arch}`);
-    const initrdCache = join(cacheDir, `initrd-${arch}.img`);
-
-    if (!existsSync(kernelCache)) {
-      log(`Downloading Fedora ${config.fedoraVersion} ${arch} kernel...`);
-      execSync(`curl -# -L -f -o "${kernelCache}" "${mirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 600_000 });
-    }
-    if (!existsSync(initrdCache)) {
-      log(`Downloading Fedora ${config.fedoraVersion} ${arch} initrd...`);
-      execSync(`curl -# -L -f -o "${initrdCache}" "${mirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 600_000 });
-    }
-
-    // Staged under the exact names the iPXE templates will ask for.
-    copyFileSync(kernelCache, join(config.httpDir, kernelPath(arch)));
-    copyFileSync(initrdCache, join(config.httpDir, initrdPath(arch)));
-    log(`Staged ${arch}: ${kernelPath(arch)} + ${initrdPath(arch)}`);
-  }
-
-  writeFileSync(join(config.httpDir, "discover.ks"), generateDiscoverKickstart(config));
-  writeFileSync(
-    join(config.httpDir, "boot.ipxe"),
-    renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }),
-  );
-  generateDnsmasqConf(config);
-
-  const { app, state, syslog } = createApp(config);
-  await app.listen({ port: config.httpPort, host: "0.0.0.0" });
-  syslog.start();
-  log(`Bastion HTTP listening on :${config.httpPort}`);
-
-  log("Starting dnsmasq (full DHCP)...");
-  startDnsmasq(config).catch((err) => {
-    log(`dnsmasq failed: ${err instanceof Error ? err.message : String(err)}`);
-  });
-  await sleep(1500);
-
-  log("Creating aarch64 PXE VM (emulated — this is slow)...");
-  createPxeVm({
-    name: vmName,
-    memory: VM_MEMORY,
-    vcpus: VM_VCPUS,
-    diskSize: VM_DISK_GB,
-    network: PXE_NETWORK_NAME,
-    arch: "aarch64",
-  });
-
-  const vmMac = getVmMac(vmName);
-  if (!vmMac) throw new Error("Could not determine VM MAC address");
-  log(`ARM VM MAC: ${vmMac}`);
-
-  return {
-    testDir,
-    app,
-    stopDnsmasq,
-    state: state as unknown as Harness["state"],
-    vmMac,
-    httpPort: config.httpPort,
-  };
-}
-
-async function stopHarness(vmName: string, harness: Harness | undefined): Promise<void> {
-  // KEEP_VM=1 leaves the VM, network and bastion up so a failure can be inspected on
-  // the console. Emulated aarch64 runs cost half an hour; tearing the evidence down
-  // automatically means paying that again to see what happened.
-  if (process.env["KEEP_VM"] === "1") {
-    log(`KEEP_VM=1 — leaving ${vmName} running for inspection.`);
-    log(`  console:  sudo virsh screenshot ${vmName} /tmp/vm.ppm`);
-    log(`  serial:   socat - TCP:127.0.0.1:${SERIAL_PORT}`);
-    if (harness) log(`  bastion:  ${harness.testDir} (still serving on :${harness.httpPort})`);
-    log(`  cleanup:  sudo virsh destroy ${vmName}; sudo virsh undefine ${vmName} --remove-all-storage --nvram`);
-    return;
-  }
-
-  log("Cleaning up...");
-  if (harness) {
-    await harness.app.close().catch(() => {});
-    harness.stopDnsmasq();
-  }
-  destroyPxeVm(vmName);
-  destroyPxeNetwork();
-  if (harness) rmSync(harness.testDir, { recursive: true, force: true });
-}
-
-/** Read the DHCP lease the bastion handed a MAC. Rescue mode reports no IP itself. */
-function leaseIpFor(testDir: string, mac: string): string | null {
-  const leaseFile = join(testDir, "dnsmasq.leases");
-  if (!existsSync(leaseFile)) return null;
-  for (const line of readFileSync(leaseFile, "utf-8").split("\n")) {
-    // <expiry> <mac> <ip> <hostname> <clientid>
-    const parts = line.trim().split(/\s+/);
-    if (parts.length >= 3 && parts[1]?.toLowerCase() === mac.toLowerCase()) {
-      return parts[2] ?? null;
-    }
-  }
-  return null;
-}
-
-async function waitForLease(testDir: string, mac: string, timeoutMs: number): Promise<string> {
-  const start = Date.now();
-  while (Date.now() - start < timeoutMs) {
-    const ip = leaseIpFor(testDir, mac);
-    if (ip !== null) return ip;
-    await sleep(5000);
-  }
-  throw new Error(`No DHCP lease for ${mac} within ${timeoutMs}ms`);
-}
-
-// ---------------------------------------------------------------------------
-// Rescue path -- what the DGX Sparks need.
-// ---------------------------------------------------------------------------
-
-describe("ARM PXE rescue", () => {
-  const VM_NAME = "lab-arm-pxe-rescue";
-  const HTTP_PORT = 8096;
-  let harness: Harness | undefined;
-  let sshKeyPath: string;
-  let rescueIp: string;
-
-  beforeAll(async () => {
-    const { pubKey, keyPath } = findSshKey();
-    sshKeyPath = keyPath;
-    harness = await startHarness(VM_NAME, HTTP_PORT, pubKey);
-    const { testDir, vmMac, state } = harness;
-
-    // Seed the machine as an already-known aarch64 box queued for rescue. This is the
-    // DGX Spark situation exactly: SSH-onboarded, never PXE-discovered, architecture
-    // known only from its record -- and it also keeps the test to a SINGLE emulated
-    // boot. Each boot spends ~15 minutes pulling Anaconda's stage2 over the network
-    // under TCG, so discovering first and rescuing second doubles the runtime for no
-    // extra coverage of the path being tested. Discovery is covered by the full suite.
-    log(`Seeding ${vmMac} as a known aarch64 machine queued for rescue...`);
-    state.update((s) => {
-      s.discovered[vmMac] = {
-        mac: vmMac,
-        product: "Test ARM64 Machine",
-        board: "virt",
-        serial: "SN-ARM64",
-        manufacturer: "QEMU",
-        cpu_model: "cortex-a57",
-        cpu_cores: VM_VCPUS,
-        memory_gb: 4,
-        arch: "aarch64",
-        disks: [],
-        nics: [],
-        first_seen: new Date().toISOString(),
-        last_seen: new Date().toISOString(),
-      };
-      s.debug[vmMac] = { hostname: "arm-rescue-test", queued_at: new Date().toISOString() };
-    });
-
-    // Restart so the VM boots against the seeded state. createPxeVm already started it.
-    rebootPxeVm(VM_NAME);
-    await sleep(5_000);
-    deleteNftablesRejectRules();
-
-    // The whole chain now runs once: DHCP option 93 -> arm64 iPXE -> /boot.ipxe ->
-    // /dispatch (architecture from the record, not the query) -> aarch64 kernel +
-    // initrd -> Anaconda rescue -> sshd. Reaching a shell at all proves iPXE handed
-    // the initrd to the EFI stub over LoadFile2; without it the kernel panics first.
-    log("Waiting for the rescue environment's DHCP lease...");
-    rescueIp = await waitForLease(testDir, vmMac, LEASE_TIMEOUT_MS);
-    log(`Rescue IP: ${rescueIp}`);
-
-    log("Waiting for SSH into the rescue shell (started by inst.sshd)...");
-    log("(emulated aarch64 — Anaconda's stage2 download dominates; be patient)");
-    await waitForSsh(rescueIp, "root", SSH_TIMEOUT_MS, sshKeyPath).catch(async (err) => {
-      log("Rescue SSH timed out. Serial console:");
-      try {
-        log(await readSerialLog(SERIAL_PORT, { lastLines: 100, timeoutMs: 15_000 }));
-      } catch { /* console unavailable */ }
-      throw err;
-    });
-    log("ARM PXE rescue reached.");
-  }, LEASE_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000);
-
-  afterAll(async () => { await stopHarness(VM_NAME, harness); });
-
-  it("resolved the architecture from the machine record", async () => {
-    const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`);
-    const data = (await res.json()) as { discovered: Record<string, { arch: string }> };
-    expect(data.discovered[harness!.vmMac]?.arch).toBe("aarch64");
-  });
-
-  it("rescue shell is reachable over SSH and is aarch64", () => {
-    const result = sshExec(rescueIp, "root", "uname -m", { keyPath: sshKeyPath, timeout: 60_000 });
-    expect(result.exitCode).toBe(0);
-    expect(result.stdout.trim()).toBe("aarch64");
-  });
-
-  it("booted an initramfs — the LoadFile2 path worked", () => {
-    // If iPXE had dropped the initrd the kernel would never have reached userspace at
-    // all, but assert it explicitly so a regression names itself.
-    const result = sshExec(rescueIp, "root", "cat /proc/cmdline; ls /run/install", {
-      keyPath: sshKeyPath, timeout: 60_000,
-    });
-    expect(result.exitCode).toBe(0);
-    expect(result.stdout).toContain("inst.rescue");
-  });
-
-  it("rescue kernel came from the bastion over HTTP", () => {
-    const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 });
-    expect(result.stdout).toContain(`${BASTION_IP}:${HTTP_PORT}`);
-    // arm64 gets serial console arguments, never nomodeset.
-    expect(result.stdout).toContain("console=ttyAMA0");
-    expect(result.stdout).not.toContain("nomodeset");
-  });
-
-  it("has LVM tools available for inspecting an installed system", () => {
-    const result = sshExec(rescueIp, "root", "command -v vgchange && command -v lsblk", {
-      keyPath: sshKeyPath, timeout: 60_000,
-    });
-    expect(result.exitCode).toBe(0);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// Full install -- opt-in, ~60-90 minutes emulated.
-// ---------------------------------------------------------------------------
-
-describe.runIf(RUN_FULL_INSTALL)("ARM PXE install", () => {
-  const VM_NAME = "lab-arm-pxe-install";
-  const HTTP_PORT = 8095;
-  let harness: Harness | undefined;
-  let sshKeyPath: string;
-  let vmIp: string;
-
-  beforeAll(async () => {
-    const { pubKey, keyPath } = findSshKey();
-    sshKeyPath = keyPath;
-    harness = await startHarness(VM_NAME, HTTP_PORT, pubKey);
-    const { vmMac } = harness;
-
-    log("Waiting for aarch64 discovery...");
-    await pollApi<{ discovered: Record<string, unknown> }>(
-      `http://${BASTION_IP}:${HTTP_PORT}/api/machines`,
-      (data) => vmMac in data.discovered,
-      DISCOVERY_TIMEOUT_MS,
-    );
-    log("Discovered. Queueing install...");
-
-    const installRes = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/install`, {
-      method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ mac: vmMac, hostname: VM_NAME, disk: "", role: "vanilla" }),
-    });
-    expect(installRes.status).toBe(200);
-
-    await sleep(30_000);
-    rebootPxeVm(VM_NAME);
-
-    log("Waiting for the emulated aarch64 install (60-90 min)...");
-    type LogsResponse = { status: string; progress: string; ip?: string };
-    const final = await pollApi<LogsResponse>(
-      `http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`,
-      (d) => d.status === "installed" || d.progress === "error",
-      INSTALL_TIMEOUT_MS,
-      30_000,
-    );
-
-    if (final.progress === "error") {
-      const logs = await (await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`)).json();
-      log(`ARM install FAILED: ${JSON.stringify(logs, null, 2)}`);
-      throw new Error("ARM PXE install failed — see logs above");
-    }
-
-    vmIp = final.ip ?? "";
-    log(`ARM install complete. IP: ${vmIp}`);
-
-    await sleep(30_000);
-    rebootPxeVm(VM_NAME);
-    await sleep(5_000);
-    deleteNftablesRejectRules();
-    await waitForSsh(vmIp, SSH_USER, SSH_TIMEOUT_MS, sshKeyPath);
-  }, DISCOVERY_TIMEOUT_MS + INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 600_000);
-
-  afterAll(async () => { await stopHarness(VM_NAME, harness); });
-
-  it("machine reached installed state", async () => {
-    const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`);
-    const data = (await res.json()) as { installed: Record<string, { hostname: string }> };
-    expect(data.installed[harness!.vmMac]?.hostname).toBe(VM_NAME);
-  });
-
-  it("installed system is aarch64", () => {
-    const result = sshExec(vmIp, SSH_USER, "uname -m", { keyPath: sshKeyPath, timeout: 60_000 });
-    expect(result.stdout.trim()).toBe("aarch64");
-  });
-
-  it("SSH works with the admin user", () => {
-    const result = sshExec(vmIp, SSH_USER, "whoami", { keyPath: sshKeyPath, timeout: 60_000 });
-    expect(result.stdout.trim()).toBe(SSH_USER);
-  });
-
-  it("LVM layout is correct", () => {
-    const result = sshExec(vmIp, SSH_USER, "sudo lvs labvg --noheadings -o lv_name", {
-      keyPath: sshKeyPath, timeout: 60_000,
-    });
-    expect(result.exitCode).toBe(0);
-    const lvs = result.stdout.trim().split("\n").map((l) => l.trim());
-    for (const expected of ["root", "var", "varlog", "swap", "home", "srv"]) {
-      expect(lvs).toContain(expected);
-    }
-  });
-});

bastion/tests/integration/cluster-smoke.test.ts

@@ -0,0 +1,164 @@
+// Integration test: form a real 3-server k3s cluster from PXE-installed VMs.
+//
+// WHY THIS EXISTS SEPARATELY FROM THE DRILL
+// -----------------------------------------
+// The rolling drill is a ~3 hour test. Almost everything that can go wrong in
+// its first thirty minutes goes wrong here instead, in forty-five: the token
+// handoff, the cluster-init/join discriminator, the etcd bootstrap race, root
+// SSH for the module, and whether post-provision was actually suppressed.
+// Debugging those inside a three-hour test is how you spend a day per attempt.
+//
+// It asserts on ETCD MEMBERSHIP, not just `kubectl get nodes`. A node object
+// and an etcd member are different things, and the whole hazard in a rolling
+// reinstall is that they can disagree.
+//
+// Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo. Set FEDORA_MIRROR to
+// a local mirror or each install pulls the full package set over the internet.
+//
+//   sudo -E npx vitest run -c tests/integration/vitest.config.ts -t "cluster smoke"
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import {
+  startTestBastion, provisionVms, destroyVms,
+  type Harness, type ProvisionedVm,
+} from "./helpers/provision-harness.js";
+import {
+  formCluster, clusterSnapshot, printSnapshot, etcdMembers, etcdHealthy,
+  allServersReady, pickJoinTarget, kubectlOn, k8sNodeName,
+} from "./helpers/k3s-cluster.js";
+import { destroyPxeNetwork } from "./helpers/pxe-network.js";
+
+const HTTP_PORT = 8097;
+const SSH_USER = "lab";
+const DISK_GB = 1000;
+const NODES = ["smoke-a", "smoke-b", "smoke-c"];
+
+function findSshKey(): { pubKey: string; keyPath: string } {
+  const homes = [homedir()];
+  const sudoUser = process.env["SUDO_USER"];
+  if (sudoUser) homes.push(join("/home", sudoUser));
+  for (const home of homes) {
+    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
+      const keyPath = join(home, ".ssh", name);
+      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
+        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
+      }
+    }
+  }
+  throw new Error("no SSH keypair found");
+}
+
+describe("cluster smoke: three servers, one cluster", () => {
+  let h: Harness;
+  let vms: ProvisionedVm[] = [];
+
+  /**
+   * Kubernetes node names, NOT libvirt domain names.
+   *
+   * k3s registers a node by FQDN, so the VM `smoke-a` is the node
+   * `smoke-a.pxe-test.local`. Asserting against the VM names would compare two
+   * different namespaces and fail for a reason that has nothing to do with the
+   * cluster.
+   */
+  const nodeNames = (): string[] => vms.map((v) => k8sNodeName(h, v)).sort();
+
+  beforeAll(async () => {
+    const { pubKey, keyPath } = findSshKey();
+    destroyVms(...NODES);
+    h = await startTestBastion({
+      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey, sshKeyPath: keyPath,
+      label: "cluster-smoke",
+      // Mandatory. Left on, post-provision installs each node into its OWN
+      // cluster and this test would be measuring three clusters of one.
+      autoK3s: false,
+    });
+  }, 20 * 60_000);
+
+  afterAll(async () => {
+    // Dump state before anything is destroyed — the VMs are the only place
+    // this information exists, and they are about to stop existing.
+    try {
+      if (vms.length > 0) printSnapshot("final", clusterSnapshot(h, vms[0]!));
+    } catch { /* cluster may already be gone */ }
+    destroyVms(...NODES);
+    try { destroyPxeNetwork(); } catch { /* shared */ }
+    await h?.close();
+  }, 10 * 60_000);
+
+  it("provisions three v2 combo nodes", async () => {
+    vms = await provisionVms(h, NODES.map((name) => ({
+      name, role: "k8s-combo" as const, schemaVersion: 2 as const,
+      diskGb: DISK_GB, memory: 6144, vcpus: 4,
+    })));
+    expect(vms).toHaveLength(3);
+    for (const vm of vms) expect(vm.ip).toMatch(/^\d+\.\d+\.\d+\.\d+$/);
+  }, 60 * 60_000);
+
+  it("forms ONE cluster, not three", async () => {
+    await formCluster(h, vms, { role: "k8s-combo" });
+
+    const snap = clusterSnapshot(h, vms[0]!);
+    printSnapshot("after formation", snap);
+
+    // The precise failure this guards: three separate single-node clusters
+    // each report 1/1 Ready and look perfectly healthy in isolation.
+    expect(snap.nodes).toHaveLength(3);
+    expect(snap.readyCount).toBe(3);
+    expect(snap.nodes.map((n) => n.name).sort()).toEqual(nodeNames());
+  }, 90 * 60_000);
+
+  it("has exactly three healthy etcd members", () => {
+    const members = etcdMembers(h, vms[0]!);
+    // .node, not .name: etcd names a member "<node>-<8 hex>", so comparing
+    // raw names against node names never matches.
+    expect(members.map((m) => m.node).sort()).toEqual(nodeNames());
+    expect(members).toHaveLength(3);
+    expect(members.every((m) => m.started)).toBe(true);
+    expect(etcdHealthy(h, vms[0]!)).toBe(true);
+  }, 5 * 60_000);
+
+  it("agrees about membership from EVERY node, not just the first", () => {
+    // Asked of each server independently: a split brain is three nodes that
+    // each believe they are correct.
+    for (const vm of vms) {
+      const members = etcdMembers(h, vm).map((m) => m.node).sort();
+      expect(members, `etcd membership as seen by ${vm.name}`).toEqual(nodeNames());
+      const nodes = clusterSnapshot(h, vm).nodes.map((n) => n.name).sort();
+      expect(nodes, `node list as seen by ${vm.name}`).toEqual(nodeNames());
+    }
+  }, 10 * 60_000);
+
+  it("reports every server ready", () => {
+    const ready = allServersReady(h, vms);
+    expect(ready.ok, ready.detail).toBe(true);
+  }, 5 * 60_000);
+
+  it("makes all three control-plane and etcd", () => {
+    const snap = clusterSnapshot(h, vms[0]!);
+    for (const n of snap.nodes) {
+      expect(n.roles, `${n.name} roles`).toContain("control-plane");
+      expect(n.roles, `${n.name} roles`).toContain("etcd");
+    }
+  }, 5 * 60_000);
+
+  it("can pick a join target other than a given node", () => {
+    // The primitive the rolling drill depends on: after the first server is
+    // wiped it must rejoin against a survivor.
+    const target = pickJoinTarget(h, vms, vms[0]!);
+    expect(target.name).not.toBe(vms[0]!.name);
+  }, 5 * 60_000);
+
+  it("carries the Longhorn disk label on every node", () => {
+    // Set via k3s node-label. Without it Longhorn creates no disk on the node,
+    // and the node then hosts zero replicas while looking entirely healthy.
+    const out = kubectlOn(h, vms[0]!,
+      `get nodes -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.labels.node\\.longhorn\\.io/create-default-disk}{"\\n"}{end}'`);
+    for (const vm of vms) {
+      expect(out, `${vm.name} create-default-disk label`)
+        .toContain(`${k8sNodeName(h, vm)}=config`);
+    }
+  }, 5 * 60_000);
+});

bastion/tests/integration/data-disks.test.ts

@@ -0,0 +1,173 @@
+// Integration test: dedicated data disks, on a real VM.
+//
+// The unit tests prove the kickstart RENDERS the right %pre and part lines.
+// This proves anaconda ACTUALLY PRODUCES the layout thebeast will get: OS on
+// one disk, /var/lib/longhorn and /var/lib/rancher each a whole-disk labelled
+// xfs partition on their own disks -- and that a reprovision with NO flags
+// rediscovers both by label and remounts them without reformatting.
+//
+// The VM mirrors thebeast's topology (1 OS disk + 2 data disks addressed by
+// /dev/disk/by-id/ serial); virtio serials stand in for NVMe serials, which is
+// exactly the aliasing the feature is specified over (any by-id path).
+//
+// Run (on a libvirt host):
+//   sudo -E npx vitest run -c tests/integration/vitest.config.ts tests/integration/data-disks.test.ts
+//
+// Set FEDORA_MIRROR to a local mirror or each install pulls packages from the
+// internet (~12-15 min per install either way; two installs in this file).
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { v2Layout, DATA_DISK_LABELS } from "../../src/shared/src/types/layout.js";
+import {
+  startTestBastion, provisionVm, reprovisionVm, readLvs, readSchemaTag,
+  writeMarkers, readMarkers, destroyVms, virtioById,
+  type Harness, type ProvisionedVm,
+} from "./helpers/provision-harness.js";
+import { destroyPxeNetwork } from "./helpers/pxe-network.js";
+import { sshExec } from "./helpers/ssh.js";
+
+const HTTP_PORT = 8094;
+const SSH_USER = "lab";
+const DISK_GB = 1000; // same sparse OS disk as every other layout test
+
+// Short virtio serials (virtio-blk truncates at 20 chars), echoing thebeast's
+// real disk serial fragments so the drill log reads like the runbook.
+const LH = { serial: "LHDRILL7F600711", sizeGb: 20 };
+const RN = { serial: "RNDRILL7FD4070C", sizeGb: 20 };
+
+const PRESERVED = ["/home", "/srv", "/var/lib/longhorn", "/var/lib/rancher"];
+
+function findSshKey(): { pubKey: string; keyPath: string } {
+  // A passphrase-protected default key fails SILENTLY here: BatchMode ssh
+  // never offers it without an agent, the server logs only "Connection closed
+  // [preauth]", and the test reads as "installed machine never came up" when
+  // the machine was fine. LAB_TEST_SSH_KEY names a dedicated passphrase-less
+  // keypair and sidesteps the whole class.
+  const envKey = process.env["LAB_TEST_SSH_KEY"];
+  if (envKey) {
+    if (!existsSync(envKey) || !existsSync(`${envKey}.pub`)) {
+      throw new Error(`LAB_TEST_SSH_KEY=${envKey}: key or .pub missing`);
+    }
+    return { pubKey: readFileSync(`${envKey}.pub`, "utf-8").trim(), keyPath: envKey };
+  }
+  const homes = [homedir()];
+  const sudoUser = process.env["SUDO_USER"];
+  if (sudoUser) homes.push(join("/home", sudoUser));
+  for (const home of homes) {
+    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
+      const keyPath = join(home, ".ssh", name);
+      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
+        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
+      }
+    }
+  }
+  throw new Error("no SSH keypair found");
+}
+
+/** Run a command on the VM and insist it succeeds. */
+function ssh(h: Harness, vm: ProvisionedVm, cmd: string): string {
+  const r = sshExec(vm.ip, h.sshUser, cmd, { keyPath: h.sshKeyPath });
+  if (r.exitCode !== 0) throw new Error(`[${vm.name}] \`${cmd}\` failed: ${r.stderr || r.stdout}`);
+  return r.stdout.trim();
+}
+
+/**
+ * What actually backs a mount point, and whether it is the whole-disk labelled
+ * partition the feature promises.
+ */
+function inspectMount(h: Harness, vm: ProvisionedVm, mount: string, serial: string) {
+  const source = ssh(h, vm, `findmnt -no SOURCE ${mount}`);
+  const fstype = ssh(h, vm, `findmnt -no FSTYPE ${mount}`);
+  const label = ssh(h, vm, `sudo blkid -s LABEL -o value ${source}`);
+  const uuid = ssh(h, vm, `sudo blkid -s UUID -o value ${source}`);
+  // The partition must live on the disk with OUR serial, not wherever the
+  // kernel happened to enumerate something.
+  const disk = ssh(h, vm, `readlink -f ${virtioById(serial)}`);
+  const parent = "/dev/" + ssh(h, vm, `lsblk -ndo pkname ${source}`);
+  return { source, fstype, label, uuid, disk, parent };
+}
+
+describe("dedicated data disks on real hardware", () => {
+  let h: Harness;
+  const created: string[] = [];
+
+  beforeAll(async () => {
+    const { pubKey, keyPath } = findSshKey();
+    h = await startTestBastion({
+      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey,
+      sshKeyPath: keyPath, label: "datadisks",
+    });
+  }, 20 * 60_000);
+
+  afterAll(async () => {
+    // KEEP_VMS=1 leaves the VM behind for a post-mortem (virsh console,
+    // screenshot, mounting the qcow2). Without it a failed boot is destroyed
+    // along with every clue about why it failed.
+    if (!process.env["KEEP_VMS"]) destroyVms(...created);
+    try { await h?.close(); } catch { /* closed */ }
+    try { destroyPxeNetwork(); } catch { /* gone */ }
+  }, 5 * 60_000);
+
+  it("k8s-worker with longhorn+rancher disks: install, then flagless reprovision rediscovers both", async () => {
+    const name = "dd-worker";
+    created.push(name);
+    const dataDisks = { longhorn: virtioById(LH.serial), rancher: virtioById(RN.serial) };
+
+    // --- Fresh install, thebeast-style -----------------------------------
+    let vm = await provisionVm(h, {
+      name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB,
+      dataDisks: { longhorn: LH, rancher: RN },
+    });
+
+    // The VG must hold exactly the v2 set MINUS the LVs the disks replace.
+    const lvs = readLvs(h, vm);
+    const expected = v2Layout("k8s-worker", { dataDisks }).map((lv) => lv.name).sort();
+    expect(Object.keys(lvs).sort(), "volume set").toEqual(expected);
+    expect(lvs["longhorn"]).toBeUndefined();
+    expect(lvs["rancher"]).toBeUndefined();
+    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");
+
+    // Each mount is a whole-disk xfs partition, labelled, on the RIGHT disk.
+    const lhBefore = inspectMount(h, vm, "/var/lib/longhorn", LH.serial);
+    expect(lhBefore.fstype).toBe("xfs");
+    expect(lhBefore.label).toBe(DATA_DISK_LABELS.longhorn);
+    expect(lhBefore.parent, "/var/lib/longhorn sits on the longhorn disk").toBe(lhBefore.disk);
+
+    const rnBefore = inspectMount(h, vm, "/var/lib/rancher", RN.serial);
+    expect(rnBefore.fstype).toBe("xfs");
+    expect(rnBefore.label).toBe(DATA_DISK_LABELS.rancher);
+    expect(rnBefore.parent, "/var/lib/rancher sits on the rancher disk").toBe(rnBefore.disk);
+
+    const token = writeMarkers(h, vm, PRESERVED);
+
+    // --- Reprovision with NO dataDisks flags ------------------------------
+    // This is the claim that matters for day 2: the labels on disk are the
+    // truth, and a plain `labctl provision reprovision` must find and keep
+    // them. Passing the flags again here would hide a regression in exactly
+    // that rediscovery.
+    vm = await reprovisionVm(h, vm, "k8s-worker");
+
+    const markers = readMarkers(h, vm, PRESERVED);
+    for (const mount of PRESERVED) {
+      expect(markers[mount], `${mount} must survive a reinstall`).toBe(token);
+    }
+
+    // Still LV-less for longhorn/rancher, still v2.
+    const lvsAfter = readLvs(h, vm);
+    expect(Object.keys(lvsAfter).sort()).toEqual(expected);
+    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");
+
+    // Same partitions, same filesystems. An unchanged UUID is the proof that
+    // --noformat really happened -- a remade xfs with the same label would
+    // pass every other check here.
+    const lhAfter = inspectMount(h, vm, "/var/lib/longhorn", LH.serial);
+    expect(lhAfter.uuid, "longhorn fs was NOT reformatted").toBe(lhBefore.uuid);
+    expect(lhAfter.parent).toBe(lhAfter.disk);
+    const rnAfter = inspectMount(h, vm, "/var/lib/rancher", RN.serial);
+    expect(rnAfter.uuid, "rancher fs was NOT reformatted").toBe(rnBefore.uuid);
+    expect(rnAfter.parent).toBe(rnAfter.disk);
+  }, 150 * 60_000);
+});

bastion/tests/integration/helpers/k3s-cluster.ts

@@ -0,0 +1,427 @@
+// Form and inspect a real multi-node k3s cluster from an integration test.
+//
+// WHY THIS EXISTS
+// ---------------
+// Nothing in the repo decides which node bootstraps a cluster and which nodes
+// join it. The production generator has the right discriminator --
+// generateServerConfig emits `cluster-init: true` when no server URL is set and
+// `server:`+`token:` when one is -- but the choice is an INPUT, and the only
+// caller that ever supplied it was a hand-run labsim shell script.
+//
+// So a test that wants three servers in one cluster has to sequence it. That
+// sequencing is here rather than in a test file because the rolling drill is
+// not the only thing that will want it: an etcd-recovery test (recoverEtcdMember
+// still has no caller), a dual-stack conversion and any HA upgrade test all
+// need exactly these primitives.
+//
+// Everything drives K3sModule, not shell. That is deliberate and is the same
+// rule labsim follows by shelling out to the compiled render-config binary: a
+// test that reimplements the install proves the test works, not the product.
+
+import { execSync } from "node:child_process";
+import type { Role } from "../../../src/shared/src/types/state.js";
+import { K3sModule } from "../../../src/modules/modules/k3s/src/k3s-module.js";
+import type { ModuleContext } from "../../../src/modules/src/types.js";
+import { log } from "./libvirt.js";
+import { parseEtcdMembers, type EtcdMember } from "../../../src/modules/modules/k3s/src/operations/etcd-members.js";
+import { sshExec } from "./ssh.js";
+import type { Harness, ProvisionedVm } from "./provision-harness.js";
+
+export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * A pre-shared cluster token, the way labsim does it.
+ *
+ * The alternative is scraping /var/lib/rancher/k3s/server/node-token off the
+ * first server, and nothing in the repo does that. It is also the wrong shape
+ * for a rolling migration: the first server gets wiped and rejoins, so the
+ * token has to outlive it and be knowable before it exists.
+ */
+export const DRILL_TOKEN = "lab-drill-shared-token";
+
+/**
+ * k3s operations run unprefixed `mkdir -p /etc/rancher/k3s` and
+ * `systemctl restart k3s`, so they need root. install.ks.ts provisions
+ * `sshkey --username=root` and sets PermitRootLogin prohibit-password, so root
+ * SSH works on every machine this harness builds. The harness's own `lab` user
+ * stays for the sudo-based reads (readLvs, markers).
+ */
+export const MODULE_SSH_USER = "root";
+
+/**
+ * The k3s version the drill runs, pinned to what the fleet runs.
+ *
+ * get.k3s.io installs LATEST, and latest is currently two minor versions ahead
+ * of production (v1.36.4 vs v1.34.6). Two things follow, and both matter:
+ *
+ *  - a rehearsal on a different minor version is rehearsing a different
+ *    system, and Longhorn in particular is only qualified against a range of
+ *    Kubernetes versions;
+ *  - it is the same trap production would hit. A node reinstalled mid-migration
+ *    comes back on latest while its peers stay put, and Kubernetes supports
+ *    only ONE minor of skew between control-plane members.
+ *
+ * Override with LAB_K3S_VERSION to test an upgrade deliberately.
+ */
+export const PINNED_K3S_VERSION = process.env["LAB_K3S_VERSION"] ?? "v1.34.6+k3s1";
+
+/** Build the ModuleContext K3sModule expects for one VM. */
+export function moduleCtx(
+  h: Harness,
+  vm: ProvisionedVm,
+  role: Role,
+  config: Record<string, unknown> = {},
+): ModuleContext {
+  return {
+    hostname: vm.name,
+    ip: vm.ip,
+    role,
+    os: "fedora-43",
+    arch: "x86_64",
+    sshUser: MODULE_SSH_USER,
+    sshKeyPath: h.sshKeyPath,
+    config,
+  };
+}
+
+/** Run a command on a node as root, throwing with context on failure. */
+export function nodeExec(h: Harness, vm: ProvisionedVm, cmd: string, timeoutMs = 120_000): string {
+  const r = sshExec(vm.ip, MODULE_SSH_USER, cmd, { keyPath: h.sshKeyPath, timeout: timeoutMs });
+  if (r.exitCode !== 0) {
+    throw new Error(`[${vm.name}] ${cmd} failed (${r.exitCode}): ${r.stderr.trim() || r.stdout.trim()}`);
+  }
+  return r.stdout;
+}
+
+/** Same, but returns the result instead of throwing. */
+export function tryNodeExec(h: Harness, vm: ProvisionedVm, cmd: string, timeoutMs = 120_000) {
+  return sshExec(vm.ip, MODULE_SSH_USER, cmd, { keyPath: h.sshKeyPath, timeout: timeoutMs });
+}
+
+/**
+ * The name Kubernetes knows this VM by.
+ *
+ * NOT the libvirt domain name. k3s registers a node under its FQDN, so a VM
+ * called `smoke-a` appears as `smoke-a.pxe-test.local`. Passing the VM name to
+ * `kubectl get node` silently finds nothing, and every wait built on it times
+ * out instead of failing usefully. Read from the node itself, the same way the
+ * production operations do (`hostname -f`), so the two can never disagree.
+ */
+const nodeNameCache = new Map<string, string>();
+export function k8sNodeName(h: Harness, vm: ProvisionedVm): string {
+  const cached = nodeNameCache.get(vm.name);
+  if (cached !== undefined) return cached;
+  const name = nodeExec(h, vm, "hostname -f 2>/dev/null || hostname").trim();
+  if (name === "") throw new Error(`[${vm.name}] could not resolve node name`);
+  nodeNameCache.set(vm.name, name);
+  return name;
+}
+
+/** Forget cached names — a reinstalled machine re-registers. */
+export function forgetNodeName(vm: ProvisionedVm): void {
+  nodeNameCache.delete(vm.name);
+}
+
+/** kubectl against a server node. */
+export function kubectlOn(h: Harness, server: ProvisionedVm, args: string, timeoutMs = 120_000): string {
+  return nodeExec(h, server, `k3s kubectl ${args}`, timeoutMs);
+}
+
+export function kubectlJson<T>(h: Harness, server: ProvisionedVm, args: string): T {
+  return JSON.parse(kubectlOn(h, server, `${args} -o json`)) as T;
+}
+
+/**
+ * Wait until a server's API answers /ping.
+ *
+ * Straight from labsim-k8s-etcd.sh, whose comment is the whole justification:
+ * a joiner that starts before the first server has finished bootstrapping etcd
+ * races it, and the failure looks like etcd instability rather than a startup
+ * order problem. Probed from the HOST, so it does not depend on the joining
+ * node being reachable yet.
+ */
+export async function waitForApiPing(vm: ProvisionedVm, timeoutMs = 450_000): Promise<void> {
+  const start = Date.now();
+  let lastErr = "";
+  while (Date.now() - start < timeoutMs) {
+    try {
+      const out = execSync(
+        `curl -sk --max-time 5 https://${vm.ip}:6443/ping || true`,
+        { encoding: "utf-8", timeout: 15_000 },
+      ).trim();
+      if (out === "pong") {
+        log(`[${vm.name}] API answering after ${Math.round((Date.now() - start) / 1000)}s`);
+        return;
+      }
+      lastErr = out;
+    } catch (e) {
+      lastErr = e instanceof Error ? e.message : String(e);
+    }
+    await sleep(5_000);
+  }
+  throw new Error(`[${vm.name}] API never answered /ping in ${timeoutMs / 1000}s (last: ${lastErr || "no response"})`);
+}
+
+/** Wait for a node to appear Ready in the cluster, as seen from `server`. */
+export async function waitForNodeReady(
+  h: Harness,
+  server: ProvisionedVm,
+  /** The VM to wait for; its Kubernetes name is resolved from the machine. */
+  target: ProvisionedVm,
+  // Generous: a node is NotReady until its CNI is up, and on the cluster-init
+  // server that means waiting for Cilium to install and pull its images. Five
+  // minutes is enough on a warm host and not enough on a cold one, and the
+  // cost of being wrong is a wasted multi-hour run.
+  timeoutMs = 900_000,
+): Promise<void> {
+  const nodeName = k8sNodeName(h, target);
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    const r = tryNodeExec(h, server,
+      `k3s kubectl get node ${nodeName} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'`);
+    last = r.stdout.trim();
+    if (last === "True") {
+      log(`[${nodeName}] Ready after ${Math.round((Date.now() - start) / 1000)}s`);
+      return;
+    }
+    await sleep(5_000);
+  }
+  throw new Error(`[${nodeName}] never became Ready in ${timeoutMs / 1000}s (last status: ${last || "absent"})`);
+}
+
+/**
+ * etcdctl invocation.
+ *
+ * NOT `k3s etcdctl` -- that is not a subcommand ("No help topic for 'etcdctl'",
+ * verified on a live v1.34 node). The binary comes from the `etcd` rpm, which
+ * our kickstart does NOT install, so ensureEtcdctl() has to put it there.
+ * Cert paths verified against a real node.
+ */
+const ETCD_BASE = "/var/lib/rancher/k3s/server/tls/etcd";
+const ETCDCTL =
+  `ETCDCTL_API=3 etcdctl ` +
+  `--cacert=${ETCD_BASE}/server-ca.crt ` +
+  `--cert=${ETCD_BASE}/server-client.crt ` +
+  `--key=${ETCD_BASE}/server-client.key ` +
+  `--endpoints=https://127.0.0.1:2379`;
+
+/** Install etcdctl if the node does not have it. Idempotent, ~30s once. */
+export function ensureEtcdctl(h: Harness, server: ProvisionedVm): void {
+  const has = tryNodeExec(h, server, "command -v etcdctl");
+  if (has.exitCode === 0) return;
+  log(`[${server.name}] installing etcdctl (the etcd rpm; kickstart does not include it)`);
+  nodeExec(h, server, "dnf install -y etcd >/dev/null 2>&1 || dnf install -y etcd-ctl >/dev/null 2>&1", 300_000);
+  nodeExec(h, server, "command -v etcdctl");
+}
+
+
+/**
+ * etcd membership, read with etcdctl against k3s's own certs.
+ *
+ * Read rather than inferred from `kubectl get nodes`: a node object can be
+ * deleted while its etcd member survives, and that stale member is exactly the
+ * failure a rolling reinstall creates. Counting nodes would not see it.
+ */
+export function etcdMembers(h: Harness, server: ProvisionedVm): EtcdMember[] {
+  // NOT `-w json`. Two reasons, and the second is the dangerous one:
+  //
+  //  1. `member remove` requires the ID in HEX; the JSON writer emits decimal.
+  //  2. etcd member IDs are uint64. JSON.parse turns them into JS numbers,
+  //     which silently lose precision above 2^53 -- the ID that comes back is
+  //     an APPROXIMATION of the real one. Passing that to `member remove`
+  //     fails with "value out of range" if you are lucky, and in principle
+  //     could match some other member if you are not. This happens at the one
+  //     moment it must not: the node is already stopped and about to be wiped.
+  //
+  // The default writer prints the ID as hex text, so it never becomes a
+  // number at all:
+  //   825852d3683bc406, started, worker2-...-5f7ad0c9, https://...:2380, ...
+  return parseEtcdMembers(nodeExec(h, server, `${ETCDCTL} member list`));
+}
+
+
+/** Are all etcd endpoints healthy? Answers the question quorum actually asks. */
+export function etcdHealthy(h: Harness, server: ProvisionedVm): boolean {
+  const r = tryNodeExec(h, server, `${ETCDCTL} endpoint health`);
+  return r.exitCode === 0 && r.stdout.includes("is healthy");
+}
+
+/**
+ * Remove an etcd member by node name.
+ *
+ * Must happen BEFORE the node is wiped. k3s-install.ts removes the local
+ * datastore when a server joins, so a reinstalled node rejoins as a NEW member
+ * while the old record persists: 3 members becomes 4, quorum rises from 2 to 3,
+ * and the cluster that tolerated one failure now tolerates none.
+ */
+export function removeEtcdMember(h: Harness, peer: ProvisionedVm, nodeName: string): boolean {
+  const members = etcdMembers(h, peer);
+  const target = members.find((m) => m.node === nodeName || m.name === nodeName);
+  if (!target) {
+    log(`[etcd] no member for ${nodeName} (have: ${members.map((m) => m.node).join(",")})`);
+    return false;
+  }
+  nodeExec(h, peer, `${ETCDCTL} member remove ${target.id}`);
+  log(`[etcd] removed member ${target.name} (${target.id})`);
+  return true;
+}
+
+export interface ClusterSnapshot {
+  nodes: { name: string; ready: boolean; roles: string }[];
+  readyCount: number;
+  etcdMembers: number;
+  etcdHealthy: boolean;
+}
+
+/** One read of cluster state, in the shape labsim's cmd_status prints. */
+export function clusterSnapshot(h: Harness, server: ProvisionedVm): ClusterSnapshot {
+  const raw = kubectlJson<{ items: {
+    metadata: { name: string; labels?: Record<string, string> };
+    status: { conditions?: { type: string; status: string }[] };
+  }[] }>(h, server, "get nodes");
+
+  const nodes = raw.items.map((n) => ({
+    name: n.metadata.name,
+    ready: (n.status.conditions ?? []).some((c) => c.type === "Ready" && c.status === "True"),
+    roles: Object.keys(n.metadata.labels ?? {})
+      .filter((k) => k.startsWith("node-role.kubernetes.io/"))
+      .map((k) => k.replace("node-role.kubernetes.io/", ""))
+      .sort().join(",") || "<none>",
+  }));
+
+  let members = 0;
+  let healthy = false;
+  try {
+    members = etcdMembers(h, server).length;
+    healthy = etcdHealthy(h, server);
+  } catch { /* an agent, or the API is down — report zero rather than throw */ }
+
+  return { nodes, readyCount: nodes.filter((n) => n.ready).length, etcdMembers: members, etcdHealthy: healthy };
+}
+
+/**
+ * Print a snapshot. Called liberally during a drill: the VMs are destroyed at
+ * the end, so this transcript is the only surviving record of what happened.
+ */
+export function printSnapshot(label: string, s: ClusterSnapshot): void {
+  log(`[snapshot] ${label}: ${s.readyCount}/${s.nodes.length} Ready, ` +
+      `etcd members=${s.etcdMembers} healthy=${s.etcdHealthy}`);
+  for (const n of s.nodes) log(`             ${n.ready ? "Ready   " : "NotReady"} ${n.name} (${n.roles})`);
+}
+
+/**
+ * Install k3s on one node via the production module.
+ *
+ * Pass no serverUrl to bootstrap the cluster (the token still seeds), or a
+ * serverUrl to join an existing one.
+ */
+export async function installK3sNode(h: Harness, vm: ProvisionedVm, opts: {
+  role: Role;
+  serverUrl?: string;
+  token?: string;
+}): Promise<void> {
+  const mod = new K3sModule();
+  const config: Record<string, unknown> = {
+    k3sToken: opts.token ?? DRILL_TOKEN,
+    k3sVersion: PINNED_K3S_VERSION,
+    ...(opts.serverUrl === undefined ? {} : { k3sServerUrl: opts.serverUrl }),
+  };
+
+  log(`[${vm.name}] installing k3s (${opts.role}${opts.serverUrl ? ` -> ${opts.serverUrl}` : " — cluster-init"})`);
+  const install = await mod.install(moduleCtx(h, vm, opts.role, config));
+  for (const line of install.output) log(`  [${vm.name}] ${line}`);
+  if (!install.success) {
+    throw new Error(`[${vm.name}] k3s install failed: ${install.errors.join("; ")}`);
+  }
+
+  const configure = await mod.configure(moduleCtx(h, vm, opts.role, config));
+  for (const line of configure.output) log(`  [${vm.name}] ${line}`);
+  if (!configure.success) {
+    throw new Error(`[${vm.name}] k3s configure failed: ${configure.errors.join("; ")}`);
+  }
+}
+
+export function serverUrlOf(vm: ProvisionedVm): string {
+  return `https://${vm.ip}:6443`;
+}
+
+/**
+ * Form a cluster from a list of server VMs, strictly in order.
+ *
+ * The first bootstraps etcd and installs the CNI (k3s-module runs networking
+ * only on the cluster-init node -- joiners get Cilium via its DaemonSet). The
+ * rest join one at a time, each gated on the previous node being Ready.
+ *
+ * Serial on purpose. labsim gets away with starting joiners together because
+ * its nodes are cloud-init VMs that poll /ping themselves; here the few extra
+ * minutes buy determinism in the one place where etcd bootstrap genuinely
+ * races.
+ */
+export async function formCluster(h: Harness, servers: ProvisionedVm[], opts: {
+  role?: Role;
+  token?: string;
+} = {}): Promise<void> {
+  if (servers.length === 0) throw new Error("formCluster: no servers");
+  const role = opts.role ?? "k8s-combo";
+  const token = opts.token ?? DRILL_TOKEN;
+  const first = servers[0]!;
+
+  await installK3sNode(h, first, { role, token });
+  await waitForApiPing(first);
+  await waitForNodeReady(h, first, first);
+  ensureEtcdctl(h, first);
+  printSnapshot("after cluster-init", clusterSnapshot(h, first));
+
+  for (const vm of servers.slice(1)) {
+    await waitForApiPing(first);
+    await installK3sNode(h, vm, { role, token, serverUrl: serverUrlOf(first) });
+    await waitForNodeReady(h, first, vm);
+    ensureEtcdctl(h, vm);
+    printSnapshot(`after joining ${vm.name}`, clusterSnapshot(h, first));
+  }
+}
+
+/** Join an agent. Agents cannot start without a server URL and token. */
+export async function joinAgent(h: Harness, vm: ProvisionedVm, opts: {
+  server: ProvisionedVm;
+  role?: Role;
+  token?: string;
+}): Promise<void> {
+  await waitForApiPing(opts.server);
+  await installK3sNode(h, vm, {
+    role: opts.role ?? "k8s-worker",
+    token: opts.token ?? DRILL_TOKEN,
+    serverUrl: serverUrlOf(opts.server),
+  });
+  await waitForNodeReady(h, opts.server, vm);
+}
+
+/**
+ * A healthy server that is NOT the node being worked on.
+ *
+ * Needed because the first server is itself rolled during a migration: after
+ * it is wiped it must rejoin against a survivor, so "the first node" cannot be
+ * the join target and cannot be where cluster state is read from.
+ */
+export function pickJoinTarget(
+  h: Harness,
+  servers: ProvisionedVm[],
+  exclude: ProvisionedVm,
+): ProvisionedVm {
+  for (const vm of servers) {
+    if (vm.name === exclude.name) continue;
+    const r = tryNodeExec(h, vm, "k3s kubectl get --raw /readyz");
+    if (r.exitCode === 0 && r.stdout.trim() === "ok") return vm;
+  }
+  throw new Error(`No healthy server other than ${exclude.name} — cluster has no quorum to join against`);
+}
+
+/** Every server reports /readyz ok. Asserted positively, not inferred. */
+export function allServersReady(h: Harness, servers: ProvisionedVm[]): { ok: boolean; detail: string } {
+  const results = servers.map((vm) => {
+    const r = tryNodeExec(h, vm, "k3s kubectl get --raw /readyz");
+    return `${vm.name}=${r.exitCode === 0 ? r.stdout.trim() : `ERR(${r.exitCode})`}`;
+  });
+  return { ok: results.every((r) => r.endsWith("=ok")), detail: results.join(" ") };
+}

bastion/tests/integration/helpers/longhorn.ts

@@ -0,0 +1,666 @@
+// Install Longhorn on a test cluster and put verifiable data in it.
+//
+// WHAT THIS IS FOR
+// ----------------
+// The rolling drill's whole claim is "the bytes survived". That needs data
+// whose integrity can be checked independently of Longhorn's own opinion of
+// itself -- Longhorn reporting `robustness: healthy` is not evidence that the
+// file is intact, it is evidence that Longhorn thinks it is.
+//
+// So: write a known blob, record its sha256, and re-verify THROUGH A POD
+// RESTART every time. The restart is not incidental. It forces a real
+// detach/attach, frequently onto a different node, which is the only way to
+// show the data followed the volume rather than sitting in a page cache on the
+// node that wrote it.
+//
+// Longhorn version is pinned to the fleet's (1.8.1) so the drill rehearses what
+// production runs, not what happens to be latest.
+
+import type { Harness, ProvisionedVm } from "./provision-harness.js";
+import { log } from "./libvirt.js";
+import { nodeExec, tryNodeExec, kubectlOn, kubectlJson, sleep, k8sNodeName } from "./k3s-cluster.js";
+
+export const LONGHORN_VERSION = "1.8.1";
+export const LONGHORN_NS = "longhorn-system";
+
+/**
+ * Install Longhorn via k3s's built-in helm controller.
+ *
+ * A HelmChart manifest dropped in the server manifests directory is how k3s
+ * installs charts natively, so this needs no helm binary on the host and no
+ * kubeconfig plumbing.
+ *
+ * The namespace is created FIRST, with privileged Pod Security labels. Longhorn
+ * needs hostPath and privileged containers; under the default `restricted`
+ * enforcement its DaemonSets are rejected and the install fails in a way that
+ * looks like a chart problem.
+ *
+ * `createDefaultDiskLabeledNodes: true` matches production, and is the setting
+ * that makes the node label and disk annotation load-bearing: without both, a
+ * node gets no disk and silently hosts no replicas.
+ */
+/**
+ * S3 backup target for the drill, from the environment.
+ *
+ * Absent means the backup/restore phase is skipped rather than silently
+ * "passing" -- a backup test that quietly does nothing is worse than no test.
+ * The run script exports these; see DRILL_S3_* in run-drill.sh.
+ */
+export interface BackupTargetOpts {
+  /** e.g. s3://longhorn-drill-backup@us-east-1/ */
+  target: string;
+  endpoint: string;
+  accessKey: string;
+  secretKey: string;
+}
+
+export function backupTargetFromEnv(): BackupTargetOpts | undefined {
+  const target = process.env["DRILL_S3_TARGET"];
+  const endpoint = process.env["DRILL_S3_ENDPOINT"];
+  const accessKey = process.env["DRILL_S3_ACCESS_KEY"];
+  const secretKey = process.env["DRILL_S3_SECRET_KEY"];
+  if (target === undefined || endpoint === undefined
+    || accessKey === undefined || secretKey === undefined) return undefined;
+  return { target, endpoint, accessKey, secretKey };
+}
+
+export const BACKUP_SECRET = "longhorn-backup-credentials";
+
+export async function installLonghorn(h: Harness, server: ProvisionedVm, opts: {
+  defaultReplicas?: number;
+  backup?: BackupTargetOpts;
+} = {}): Promise<void> {
+  const replicas = opts.defaultReplicas ?? 3;
+
+  log(`[longhorn] creating ${LONGHORN_NS} with privileged PSA labels`);
+  nodeExec(h, server, `k3s kubectl apply -f - <<'EOF'
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ${LONGHORN_NS}
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged
+EOF`);
+
+  if (opts.backup !== undefined) {
+    // Secret BEFORE the chart: Longhorn reads it by name from a defaultSetting,
+    // and a missing secret makes the first backup fail with a credential error
+    // that reads like a network problem.
+    //
+    // AWS_ENDPOINTS is what makes this work against a non-AWS gateway; without
+    // it the driver resolves real AWS endpoints.
+    log(`[longhorn] configuring backup target ${opts.backup.target}`);
+    // tryNodeExec, not nodeExec: nodeExec throws with the whole command in the
+    // message, and this command contains the S3 secret. A credential landing in
+    // a log that then gets pasted into drill evidence is not acceptable, so the
+    // failure is reported without the payload.
+    const secretApplied = tryNodeExec(h, server, `k3s kubectl -n ${LONGHORN_NS} apply -f - <<'EOF'
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ${BACKUP_SECRET}
+  namespace: ${LONGHORN_NS}
+stringData:
+  AWS_ACCESS_KEY_ID: ${opts.backup.accessKey}
+  AWS_SECRET_ACCESS_KEY: ${opts.backup.secretKey}
+  AWS_ENDPOINTS: ${opts.backup.endpoint}
+EOF`);
+    if (secretApplied.exitCode !== 0) {
+      throw new Error(
+        `[${server.name}] could not create the ${BACKUP_SECRET} secret ` +
+        `(exit ${secretApplied.exitCode}). Command withheld: it contains credentials. ` +
+        `stderr: ${secretApplied.stderr.trim().slice(0, 200)}`);
+    }
+  }
+
+  log(`[longhorn] installing chart ${LONGHORN_VERSION} via k3s helm controller`);
+  nodeExec(h, server, `cat > /var/lib/rancher/k3s/server/manifests/longhorn.yaml <<'EOF'
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: longhorn
+  namespace: kube-system
+spec:
+  chart: longhorn
+  repo: https://charts.longhorn.io
+  version: ${LONGHORN_VERSION}
+  targetNamespace: ${LONGHORN_NS}
+  valuesContent: |-
+    defaultSettings:
+      defaultDataPath: /var/lib/longhorn
+      defaultReplicaCount: ${replicas}
+      createDefaultDiskLabeledNodes: true
+      defaultDataLocality: best-effort
+      replicaSoftAntiAffinity: false
+      replicaReplenishmentWaitInterval: 600
+      orphanAutoDeletion: true
+    persistence:
+      defaultClass: true
+      defaultClassReplicaCount: ${replicas}
+      defaultFsType: ext4
+    longhornUI:
+      replicas: 1
+EOF`);
+}
+
+/**
+ * Every expected node has a Longhorn disk that is actually schedulable.
+ *
+ * Takes the VMs, not names, and resolves each one's Kubernetes name itself.
+ * Longhorn keys its node records by the FQDN the kubelet registered
+ * (`drill-s1.pxe-test.local`), never the libvirt domain name (`drill-s1`), so
+ * a caller passing VM names matches nothing and waits out the full timeout
+ * while the cluster is in fact perfectly healthy.
+ */
+export async function waitForLonghornReady(
+  h: Harness,
+  server: ProvisionedVm,
+  vms: ProvisionedVm[],
+  timeoutMs = 20 * 60_000,
+): Promise<void> {
+  const expectedNodes = vms.map((vm) => k8sNodeName(h, vm));
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    const r = tryNodeExec(h, server,
+      `k3s kubectl -n ${LONGHORN_NS} get nodes.longhorn.io -o json`, 60_000);
+    if (r.exitCode === 0) {
+      try {
+        const parsed = JSON.parse(r.stdout) as { items: {
+          metadata: { name: string };
+          status?: { diskStatus?: Record<string, { conditions?: { type: string; status: string }[] }> };
+        }[] };
+        const ready = parsed.items.filter((n) => {
+          const disks = Object.values(n.status?.diskStatus ?? {});
+          return disks.length > 0 && disks.some((d) =>
+            (d.conditions ?? []).some((c) => c.type === "Schedulable" && c.status === "True"));
+        }).map((n) => n.metadata.name);
+
+        last = `${ready.length}/${expectedNodes.length} with schedulable disk: ${ready.join(",") || "none"}`;
+        if (expectedNodes.every((n) => ready.includes(n))) {
+          log(`[longhorn] ready after ${Math.round((Date.now() - start) / 1000)}s — ${last}`);
+          return;
+        }
+      } catch { /* API not serving the CRD yet */ }
+    }
+    await sleep(15_000);
+  }
+
+  // Say WHY, per node, instead of just "not ready". A node with no disk has a
+  // specific cause and the record knows it; without this the only way to find
+  // out is to catch the cluster before teardown, which is a race you lose.
+  const detail: string[] = [];
+  try {
+    const raw = kubectlJson<{ items: {
+      metadata: { name: string; annotations?: Record<string, string>; labels?: Record<string, string> };
+      spec?: { disks?: Record<string, unknown>; allowScheduling?: boolean };
+      status?: {
+        conditions?: { type: string; status: string; reason?: string; message?: string }[];
+        diskStatus?: Record<string, { conditions?: { type: string; status: string; reason?: string; message?: string }[] }>;
+      };
+    }[] }>(h, server, `-n ${LONGHORN_NS} get nodes.longhorn.io`);
+
+    for (const n of raw.items) {
+      const disks = Object.keys(n.spec?.disks ?? {});
+      const conds = (n.status?.conditions ?? [])
+        .filter((c) => c.status !== "True")
+        .map((c) => `${c.type}=${c.status}(${c.reason ?? ""}: ${(c.message ?? "").slice(0, 80)})`);
+      const diskConds = Object.entries(n.status?.diskStatus ?? {}).flatMap(([dn, d]) =>
+        (d.conditions ?? []).filter((c) => c.status !== "True")
+          .map((c) => `${dn}:${c.type}=${c.status}(${c.reason ?? ""}: ${(c.message ?? "").slice(0, 80)})`));
+      const ann = n.metadata.annotations?.["node.longhorn.io/default-disks-config"] ?? "<absent>";
+      const lbl = n.metadata.labels?.["node.longhorn.io/create-default-disk"] ?? "<absent>";
+      detail.push(
+        `  ${n.metadata.name}: disks=[${disks.join(",")}] label=${lbl} ann=${ann} ` +
+        `${conds.length > 0 ? `nodeConds=${conds.join(";")} ` : ""}` +
+        `${diskConds.length > 0 ? `diskConds=${diskConds.join(";")}` : ""}`);
+    }
+  } catch (e) {
+    detail.push(`  (could not read Longhorn nodes: ${e instanceof Error ? e.message : String(e)})`);
+  }
+
+  throw new Error(
+    `Longhorn not ready in ${timeoutMs / 60000} min (${last}).\n` +
+    `A node with no disk usually means the create-default-disk label or the\n` +
+    `default-disks-config annotation is missing, or the path is not mounted.\n` +
+    detail.join("\n"));
+}
+
+/** Create a PVC plus a Deployment that mounts it at /data. */
+export function createDataWorkload(h: Harness, server: ProvisionedVm, opts: {
+  name: string;
+  sizeGi: number;
+  replicas: number;
+}): void {
+  log(`[longhorn] creating ${opts.name} (${opts.sizeGi}Gi, ${opts.replicas} replicas)`);
+  nodeExec(h, server, `k3s kubectl apply -f - <<'EOF'
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: lh-${opts.name}
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+parameters:
+  numberOfReplicas: "${opts.replicas}"
+  staleReplicaTimeout: "30"
+  fsType: ext4
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ${opts.name}
+spec:
+  accessModes: [ReadWriteOnce]
+  storageClassName: lh-${opts.name}
+  resources:
+    requests:
+      storage: ${opts.sizeGi}Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ${opts.name}
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels: {app: ${opts.name}}
+  template:
+    metadata:
+      labels: {app: ${opts.name}}
+    spec:
+      # Pod Security Admission enforces "restricted" on this namespace (the k3s
+      # hardening group applies the label), so a bare pod spec is REJECTED
+      # outright -- the Deployment reports 0/1 with no pod and no obvious
+      # reason unless you read the ReplicaSet events. Every one of these
+      # fields is required by "restricted"; dropping any brings it back.
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        # fsGroup so the non-root user can actually write to the Longhorn
+        # volume, which is otherwise owned by root.
+        fsGroup: 1000
+        seccompProfile: {type: RuntimeDefault}
+      containers:
+        - name: shell
+          image: busybox:1.36
+          command: ["sh", "-c", "sleep infinity"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: ["ALL"]}
+          volumeMounts:
+            - {name: data, mountPath: /data}
+      volumes:
+        - name: data
+          persistentVolumeClaim: {claimName: ${opts.name}}
+EOF`);
+}
+
+/** Wait for the workload's pod to be Running and its volume attached. */
+export async function waitForWorkload(
+  h: Harness, server: ProvisionedVm, name: string, timeoutMs = 10 * 60_000,
+): Promise<void> {
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    const r = tryNodeExec(h, server,
+      `k3s kubectl get pods -l app=${name} -o jsonpath='{.items[*].status.phase}'`);
+    last = r.stdout.trim();
+    if (last === "Running") {
+      // Ready, not merely scheduled: the volume mount happens before Ready.
+      const ready = tryNodeExec(h, server,
+        `k3s kubectl get pods -l app=${name} -o jsonpath='{.items[*].status.containerStatuses[*].ready}'`);
+      if (ready.stdout.trim() === "true") return;
+    }
+    await sleep(5_000);
+  }
+  throw new Error(`workload ${name} not running in ${timeoutMs / 60000} min (last phase: ${last || "none"})`);
+}
+
+function podOf(h: Harness, server: ProvisionedVm, name: string): string {
+  const pod = kubectlOn(h, server, `get pods -l app=${name} -o jsonpath='{.items[0].metadata.name}'`).trim();
+  if (!pod) throw new Error(`no pod for workload ${name}`);
+  return pod;
+}
+
+/**
+ * Write a random blob, fsync it, and record its digest.
+ *
+ * conv=fsync matters: without it the write can still be in the guest's page
+ * cache, and a checksum taken immediately afterwards proves nothing about what
+ * reached the replicas.
+ */
+export async function writeBlob(h: Harness, server: ProvisionedVm, opts: {
+  name: string; mib: number;
+}): Promise<string> {
+  const pod = podOf(h, server, opts.name);
+  log(`[longhorn] writing ${opts.mib} MiB to ${opts.name}`);
+  nodeExec(h, server,
+    `k3s kubectl exec ${pod} -- sh -c ` +
+    `"dd if=/dev/urandom of=/data/blob bs=1M count=${opts.mib} conv=fsync 2>/dev/null && ` +
+    `cd /data && sha256sum blob > blob.sha256 && sync"`,
+    15 * 60_000);
+
+  const digest = kubectlOn(h, server, `exec ${pod} -- cat /data/blob.sha256`).trim().split(/\s+/)[0] ?? "";
+  if (!/^[0-9a-f]{64}$/.test(digest)) throw new Error(`bad digest for ${opts.name}: ${digest}`);
+  log(`[longhorn] ${opts.name} digest ${digest.slice(0, 16)}...`);
+  return digest;
+}
+
+/**
+ * Restart the pod, then verify the blob.
+ *
+ * The restart is the point. It detaches and reattaches the volume, so a pass
+ * means the bytes came back from Longhorn rather than from wherever the writer
+ * left them.
+ */
+export async function verifyBlob(h: Harness, server: ProvisionedVm, opts: {
+  name: string; expectDigest: string;
+}): Promise<void> {
+  const before = podOf(h, server, opts.name);
+  kubectlOn(h, server, `delete pod ${before} --wait=true --timeout=5m`, 6 * 60_000);
+  await waitForWorkload(h, server, opts.name);
+  const after = podOf(h, server, opts.name);
+  if (after === before) throw new Error(`${opts.name}: pod was not actually replaced`);
+
+  // sha256sum -c proves the file matches what was recorded IN the volume...
+  const check = tryNodeExec(h, server,
+    `k3s kubectl exec ${after} -- sh -c "cd /data && sha256sum -c blob.sha256"`, 15 * 60_000);
+  if (check.exitCode !== 0) {
+    throw new Error(`${opts.name}: DATA CORRUPTION — sha256sum -c failed: ${check.stdout}${check.stderr}`);
+  }
+
+  // ...and this proves the recorded digest is still the one we started with,
+  // which -c alone would not catch if both file and checksum were replaced.
+  const actual = kubectlOn(h, server,
+    `exec ${after} -- sh -c "cd /data && sha256sum blob"`, 15 * 60_000).trim().split(/\s+/)[0];
+  if (actual !== opts.expectDigest) {
+    throw new Error(`${opts.name}: DIGEST CHANGED — expected ${opts.expectDigest}, got ${actual}`);
+  }
+  log(`[longhorn] ${opts.name} verified intact across a detach/attach (pod ${before} -> ${after})`);
+}
+
+export interface VolumeState {
+  pv: string;
+  state: string;
+  robustness: string;
+  wanted: number;
+  replicaNodes: string[];
+  healthyReplicas: number;
+}
+
+/** Longhorn's view of the volume backing a PVC. */
+export function volumeState(h: Harness, server: ProvisionedVm, pvcName: string): VolumeState {
+  const pv = kubectlOn(h, server, `get pvc ${pvcName} -o jsonpath='{.spec.volumeName}'`).trim();
+  if (!pv) throw new Error(`PVC ${pvcName} has no bound volume`);
+
+  const vol = kubectlJson<{
+    spec: { numberOfReplicas?: number };
+    status?: { state?: string; robustness?: string };
+  }>(h, server, `-n ${LONGHORN_NS} get volumes.longhorn.io ${pv}`);
+
+  const reps = kubectlJson<{ items: {
+    spec: { nodeID?: string; volumeName?: string };
+    status?: { currentState?: string; started?: boolean };
+  }[] }>(h, server, `-n ${LONGHORN_NS} get replicas.longhorn.io`);
+
+  const mine = reps.items.filter((r) => r.spec.volumeName === pv);
+  return {
+    pv,
+    state: vol.status?.state ?? "unknown",
+    robustness: vol.status?.robustness ?? "unknown",
+    wanted: vol.spec.numberOfReplicas ?? 0,
+    replicaNodes: mine.map((r) => r.spec.nodeID ?? "").filter(Boolean).sort(),
+    healthyReplicas: mine.filter((r) => r.status?.currentState === "running").length,
+  };
+}
+
+/**
+ * Wait for a volume to be healthy on the expected number of distinct nodes.
+ *
+ * Logs rebuild progress while waiting, so a slow rebuild reads as progress
+ * rather than as an opaque hour of silence.
+ */
+export async function waitForVolumeHealthy(h: Harness, server: ProvisionedVm, opts: {
+  pvc: string; expectReplicas: number; timeoutMs?: number;
+}): Promise<VolumeState> {
+  const timeoutMs = opts.timeoutMs ?? 60 * 60_000;
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    let st: VolumeState | null = null;
+    try { st = volumeState(h, server, opts.pvc); } catch { /* transient */ }
+
+    if (st) {
+      const distinct = new Set(st.replicaNodes).size;
+      last = `${st.state}/${st.robustness} on ${distinct} node(s): ${st.replicaNodes.join(",")}`;
+      if (st.robustness === "healthy" && distinct >= opts.expectReplicas) {
+        log(`[longhorn] ${opts.pvc} healthy on ${distinct} nodes after ${Math.round((Date.now() - start) / 1000)}s`);
+        return st;
+      }
+      const prog = tryNodeExec(h, server,
+        `k3s kubectl -n ${LONGHORN_NS} get engines.longhorn.io -o jsonpath=` +
+        `'{range .items[*]}{.spec.volumeName}{" "}{.status.rebuildStatus}{"\\n"}{end}'`);
+      const line = prog.stdout.split("\n").find((l) => l.includes(st!.pv) && l.trim() !== st!.pv);
+      log(`[longhorn] ${opts.pvc}: ${last}${line ? ` | rebuild ${line.replace(st.pv, "").trim()}` : ""}`);
+    }
+    await sleep(20_000);
+  }
+  throw new Error(`${opts.pvc} not healthy on ${opts.expectReplicas} nodes in ${timeoutMs / 60000} min (last: ${last})`);
+}
+
+/**
+ * Wait until Longhorn reports the backup target usable.
+ *
+ * `available: true` is the appliance saying it could reach the endpoint AND
+ * authenticate. Backing up without checking this first produces a failure that
+ * looks like a volume problem.
+ */
+export async function setBackupTarget(
+  h: Harness, server: ProvisionedVm, backup: BackupTargetOpts, timeoutMs = 5 * 60_000,
+): Promise<void> {
+  // PATCH the CR; do NOT set defaultSettings.backupTarget.
+  //
+  // In Longhorn 1.8 the backup target moved out of settings and onto
+  // `backuptargets.longhorn.io/default`. The helm value is SILENTLY IGNORED --
+  // no `backup-target` Setting object is created at all, the default
+  // BackupTarget keeps an empty URL, and everything reports success. Observed
+  // on a fresh 1.8.1 install here and confirmed against the live fleet.
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const exists = tryNodeExec(h, server,
+      `k3s kubectl -n ${LONGHORN_NS} get backuptargets.longhorn.io default -o name`);
+    if (exists.exitCode === 0 && exists.stdout.trim() !== "") break;
+    await sleep(5_000);
+  }
+
+  const patch = JSON.stringify({
+    spec: { backupTargetURL: backup.target, credentialSecret: BACKUP_SECRET, pollInterval: "5m0s" },
+  });
+  nodeExec(h, server,
+    `k3s kubectl -n ${LONGHORN_NS} patch backuptargets.longhorn.io default --type=merge -p '${patch}'`);
+  log(`[longhorn] backup target set to ${backup.target}`);
+}
+
+export async function waitForBackupTarget(
+  h: Harness, server: ProvisionedVm, timeoutMs = 5 * 60_000,
+): Promise<void> {
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    const r = tryNodeExec(h, server,
+      `k3s kubectl -n ${LONGHORN_NS} get backuptargets.longhorn.io default ` +
+      `-o jsonpath='{.spec.backupTargetURL}|{.status.available}|{.status.conditions[*].message}'`);
+    last = r.stdout.trim();
+    if (r.exitCode === 0 && last.includes("|true|")) {
+      log(`[longhorn] backup target available after ${Math.round((Date.now() - start) / 1000)}s`);
+      return;
+    }
+    await sleep(10_000);
+  }
+  throw new Error(
+    `Backup target not available in ${timeoutMs / 60000} min (url|available|conditions = ${last}). ` +
+    `Longhorn could not reach or authenticate to the endpoint.`);
+}
+
+/**
+ * Snapshot a volume and back it up, returning the backup's name.
+ *
+ * Longhorn's own CRs, not the CSI VolumeSnapshot API -- this cluster has no
+ * snapshot.storage.k8s.io resources at all, so VolumeSnapshot/VolumeSnapshotClass
+ * simply do not exist here. Any backup tooling has to use these.
+ */
+export async function backupVolume(
+  h: Harness, server: ProvisionedVm, pvc: string, timeoutMs = 20 * 60_000,
+): Promise<{ backup: string; volume: string }> {
+  const pv = volumeState(h, server, pvc).pv;
+  const snapName = `drill-snap-${Date.now().toString(36)}`;
+
+  log(`[longhorn] snapshotting ${pvc} (${pv})`);
+  nodeExec(h, server, `k3s kubectl -n ${LONGHORN_NS} apply -f - <<'EOF'
+apiVersion: longhorn.io/v1beta2
+kind: Snapshot
+metadata:
+  name: ${snapName}
+  namespace: ${LONGHORN_NS}
+spec:
+  volume: ${pv}
+  createSnapshot: true
+EOF`);
+
+  // The snapshot has to exist before a backup can reference it.
+  const snapDeadline = Date.now() + 5 * 60_000;
+  while (Date.now() < snapDeadline) {
+    const r = tryNodeExec(h, server,
+      `k3s kubectl -n ${LONGHORN_NS} get snapshots.longhorn.io ${snapName} -o jsonpath='{.status.readyToUse}'`);
+    if (r.stdout.trim() === "true") break;
+    await sleep(5_000);
+  }
+
+  const backupName = `drill-bak-${Date.now().toString(36)}`;
+  log(`[longhorn] backing up ${pvc} to the backup target`);
+  nodeExec(h, server, `k3s kubectl -n ${LONGHORN_NS} apply -f - <<'EOF'
+apiVersion: longhorn.io/v1beta2
+kind: Backup
+metadata:
+  name: ${backupName}
+  namespace: ${LONGHORN_NS}
+  labels:
+    backup-volume: ${pv}
+spec:
+  snapshotName: ${snapName}
+EOF`);
+
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    const r = tryNodeExec(h, server,
+      `k3s kubectl -n ${LONGHORN_NS} get backups.longhorn.io ${backupName} ` +
+      `-o jsonpath='{.status.state}|{.status.progress}|{.status.error}'`);
+    last = r.stdout.trim();
+    if (last.startsWith("Completed")) {
+      log(`[longhorn] backup ${backupName} completed after ${Math.round((Date.now() - start) / 1000)}s`);
+      return { backup: backupName, volume: pv };
+    }
+    if (last.includes("Error")) {
+      throw new Error(`Backup ${backupName} failed: ${last}`);
+    }
+    await sleep(10_000);
+  }
+  throw new Error(`Backup ${backupName} did not complete in ${timeoutMs / 60000} min (state|progress|error = ${last})`);
+}
+
+/**
+ * Restore a backup into a brand-new PVC and workload.
+ *
+ * Uses the `fromBackup` StorageClass parameter, which is the supported route
+ * when the CSI VolumeSnapshot API is unavailable. The restored volume is a
+ * separate volume with its own replicas -- nothing about the original is
+ * touched, so a failed restore cannot damage the source.
+ */
+export async function restoreFromBackup(h: Harness, server: ProvisionedVm, opts: {
+  backup: string;
+  name: string;
+  sizeGi: number;
+}): Promise<void> {
+  const url = nodeExec(h, server,
+    `k3s kubectl -n ${LONGHORN_NS} get backups.longhorn.io ${opts.backup} -o jsonpath='{.status.url}'`).trim();
+  if (url === "") throw new Error(`Backup ${opts.backup} has no status.url; cannot restore from it`);
+  log(`[longhorn] restoring ${opts.backup} into ${opts.name}`);
+
+  nodeExec(h, server, `k3s kubectl apply -f - <<'EOF'
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: lh-${opts.name}
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+parameters:
+  numberOfReplicas: "2"
+  staleReplicaTimeout: "30"
+  fsType: ext4
+  fromBackup: "${url}"
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ${opts.name}
+spec:
+  accessModes: [ReadWriteOnce]
+  storageClassName: lh-${opts.name}
+  resources:
+    requests:
+      storage: ${opts.sizeGi}Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ${opts.name}
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels: {app: ${opts.name}}
+  template:
+    metadata:
+      labels: {app: ${opts.name}}
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
+        seccompProfile: {type: RuntimeDefault}
+      containers:
+        - name: shell
+          image: busybox:1.36
+          command: ["sh", "-c", "sleep infinity"]
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: ["ALL"]}
+          volumeMounts:
+            - {name: data, mountPath: /data}
+      volumes:
+        - name: data
+          persistentVolumeClaim: {claimName: ${opts.name}}
+EOF`);
+}
+
+/**
+ * Read a blob's digest from a workload, without restarting it.
+ *
+ * Used on a RESTORED volume, where the point is to compare bytes against the
+ * original's recorded digest rather than to prove durability again.
+ */
+export function readBlobDigest(h: Harness, server: ProvisionedVm, name: string): string {
+  const pod = kubectlOn(h, server, `get pods -l app=${name} -o jsonpath='{.items[0].metadata.name}'`).trim();
+  if (pod === "") throw new Error(`no pod for workload ${name}`);
+  const out = kubectlOn(h, server,
+    `exec ${pod} -- sh -c "cd /data && sha256sum blob"`, 15 * 60_000).trim();
+  return out.split(/\s+/)[0] ?? "";
+}

bastion/tests/integration/helpers/provision-harness.ts

@@ -0,0 +1,583 @@
+// Reusable bastion + VM provisioning harness for partition-layout tests.
+//
+// pxe-provision.test.ts grew a ~200-line beforeAll that stands up a real
+// bastion, caches Fedora boot artifacts, and drives one VM through discovery
+// and install. Testing a partition MATRIX needs that same sequence for several
+// VMs and several roles, so it lives here instead of being copy-pasted per
+// topology.
+
+import { existsSync, mkdirSync, copyFileSync, symlinkSync, writeFileSync } from "node:fs";
+import { execSync } from "node:child_process";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import type { BastionConfig } from "../../../src/shared/src/types/config.js";
+import type { Role } from "../../../src/shared/src/types/state.js";
+import { log, waitForSsh } from "./libvirt.js";
+import { ensurePxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_BRIDGE, PXE_GATEWAY, PXE_SUBNET } from "./pxe-network.js";
+import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./pxe-vm.js";
+import { sshExec } from "./ssh.js";
+
+export const DISCOVERY_TIMEOUT_MS = 6 * 60_000;
+export const INSTALL_TIMEOUT_MS = 60 * 60_000;
+export const SSH_TIMEOUT_MS = 12 * 60_000;
+
+export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+export interface Harness {
+  config: BastionConfig;
+  close: () => Promise<void>;
+  httpPort: number;
+  sshUser: string;
+  sshKeyPath: string;
+}
+
+async function pollUntil<T>(url: string, check: (d: T) => boolean, timeoutMs: number, intervalMs = 5000): Promise<T> {
+  const start = Date.now();
+  let lastErr = "";
+  while (Date.now() - start < timeoutMs) {
+    try {
+      const res = await fetch(url);
+      if (res.ok) {
+        const data = (await res.json()) as T;
+        if (check(data)) return data;
+      }
+    } catch (e) { lastErr = e instanceof Error ? e.message : String(e); }
+    await sleep(intervalMs);
+  }
+  throw new Error(`Timeout after ${timeoutMs}ms polling ${url}${lastErr ? ` (last error: ${lastErr})` : ""}`);
+}
+
+/**
+ * Stand up a real bastion on the isolated PXE network.
+ *
+ * FEDORA_MIRROR is honoured (config.ts now reads it) -- point it at a local
+ * mirror before running a multi-VM matrix, or every install pulls the full
+ * package set from download.fedoraproject.org.
+ */
+export async function startTestBastion(opts: {
+  httpPort: number;
+  sshUser: string;
+  sshPubKey: string;
+  sshKeyPath: string;
+  label: string;
+  /**
+   * Leave the post-provision k3s auto-install ON (the default) or suppress it.
+   *
+   * Any test that forms a cluster ITSELF must pass false. The auto-install runs
+   * a bare `INSTALL_K3S_EXEC=server|agent` with no server URL and no token, so
+   * three combo nodes become three separate single-node clusters -- and it
+   * stops k3s and deletes CNI state minutes after an install lands, landing on
+   * top of whatever the test is doing by then.
+   */
+  autoK3s?: boolean;
+}): Promise<Harness> {
+  ensurePxeNetwork();
+
+  const testDir = join(tmpdir(), `lab-${opts.label}-${Date.now()}`);
+  for (const sub of ["tftp", "http", "logs"]) mkdirSync(join(testDir, sub), { recursive: true });
+
+  const { createApp } = await import("../../../src/bastion/src/server.js");
+  const { loadConfig } = await import("../../../src/bastion/src/config.js");
+  const { generateDnsmasqConf, startDnsmasq, stopDnsmasq } = await import("../../../src/bastion/src/services/dnsmasq.js");
+  const { generateDiscoverKickstart } = await import("../../../src/bastion/src/services/kickstart-generator.js");
+  const { renderBootIpxe } = await import("../../../src/bastion/src/templates/boot.ipxe.js");
+
+  const config = loadConfig({
+    bastionDir: testDir,
+    httpPort: opts.httpPort,
+    iface: "virbr-pxe",
+    serverIp: PXE_GATEWAY,
+    network: `${PXE_SUBNET}.0`,
+    gateway: PXE_GATEWAY,
+    dhcpMode: "full",
+    dhcpRangeStart: `${PXE_SUBNET}.100`,
+    dhcpRangeEnd: `${PXE_SUBNET}.200`,
+    domain: "pxe-test.local",
+    sshKeys: [opts.sshPubKey],
+    adminUser: opts.sshUser,
+    ...(opts.autoK3s === undefined ? {} : { autoK3s: opts.autoK3s }),
+  });
+
+  const ipxeSrc = "/usr/share/ipxe/ipxe-snponly-x86_64.efi";
+  if (!existsSync(ipxeSrc)) throw new Error(`iPXE not found: ${ipxeSrc}. Install: sudo dnf install ipxe-bootimgs-x86`);
+  const ipxeDest = join(config.tftpDir, "ipxe.efi");
+  copyFileSync(ipxeSrc, ipxeDest);
+
+  const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache";
+  execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" });
+  const kernel = join(cacheDir, `vmlinuz-${config.fedoraVersion}`);
+  const initrd = join(cacheDir, `initrd-${config.fedoraVersion}.img`);
+  if (!existsSync(kernel)) {
+    log(`Downloading Fedora ${config.fedoraVersion} kernel from ${config.fedoraMirror}...`);
+    execSync(`curl -sS -L -f -o "${kernel}" "${config.fedoraMirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 300_000 });
+  }
+  if (!existsSync(initrd)) {
+    log(`Downloading Fedora ${config.fedoraVersion} initrd...`);
+    execSync(`curl -sS -L -f -o "${initrd}" "${config.fedoraMirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 300_000 });
+  }
+  copyFileSync(kernel, join(config.httpDir, "vmlinuz"));
+  copyFileSync(initrd, join(config.httpDir, "initrd.img"));
+  try { symlinkSync(ipxeDest, join(config.httpDir, "ipxe.efi")); } catch { /* exists */ }
+
+  writeFileSync(join(config.httpDir, "discover.ks"), generateDiscoverKickstart(config));
+  writeFileSync(join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }));
+  generateDnsmasqConf(config);
+
+  // Kill dnsmasq processes left behind by EARLIER TEST RUNS.
+  //
+  // startTestBastion's close() shuts down the HTTP app but never stopped
+  // dnsmasq, so every run leaked one. A leaked dnsmasq keeps :67 on the bridge
+  // and keeps answering DHCP -- with the PREVIOUS run's config, pointing PXE
+  // clients at a bastion port that is no longer listening. The next run then
+  // sees perfect DHCP (offer, request, ack, even the right bootfile name) and
+  // no machine ever reaches it. That is not a failure mode anyone guesses from
+  // "discovery timed out".
+  killOrphanedDnsmasq();
+
+  const { app, syslog } = createApp(config);
+  await app.listen({ port: config.httpPort, host: "0.0.0.0" });
+  syslog.start();
+  // NOT awaited, deliberately.
+  //
+  // startDnsmasq returns the execa child handle, and because the function is
+  // `async` the promise it returns ADOPTS that thenable -- which settles only
+  // when dnsmasq EXITS. With --no-daemon that is never, so `await` here hangs
+  // the harness silently and forever, with the log frozen right after
+  // "TFTP root is ...". (Learned by doing exactly that.) The rejection still
+  // has to be handled or an early bind failure becomes an unhandled rejection.
+  startDnsmasq(config).catch((err) => log(`dnsmasq: ${err instanceof Error ? err.message : String(err)}`));
+  await sleep(1000);
+
+  // Prove DHCP is actually listening before handing the harness back.
+  //
+  // startDnsmasq's rejection is swallowed into a log line, so a dnsmasq that
+  // failed to bind looks exactly like one that started. The symptom then
+  // arrives six minutes later as four VMs timing out in discovery, which
+  // reads as a PXE or kickstart problem rather than "there was never any
+  // DHCP". Fail here, immediately, with the real reason.
+  await assertDhcpListening(join(testDir, "dnsmasq.conf"));
+  log(`Bastion up on :${opts.httpPort} (mirror: ${config.fedoraMirror}, autoK3s=${config.autoK3s})`);
+
+  return {
+    config,
+    httpPort: opts.httpPort,
+    sshUser: opts.sshUser,
+    sshKeyPath: opts.sshKeyPath,
+    close: async () => {
+      // Stop OUR dnsmasq. close() never used to, which is how a leaked one
+      // ended up owning :67 and serving the PREVIOUS run's TFTP root and
+      // bastion port to the NEXT run's VMs -- they booted toward a dead port
+      // and simply never appeared, while DHCP looked perfect.
+      try { stopDnsmasq(); } catch { /* already gone */ }
+      await app.close();
+      // Belt and braces: if that did not stop it, sweep by config path.
+      killOrphanedDnsmasq();
+    },
+  };
+}
+
+/**
+ * Kill any dnsmasq started by a previous (or current) lab test run.
+ *
+ * Matched on the config path, which is always /tmp/lab-<label>-<ts>/dnsmasq.conf
+ * -- so this never touches libvirt's own dnsmasq for the default network, or
+ * anything else on the host.
+ */
+function killOrphanedDnsmasq(): void {
+  try {
+    // Match on the process NAME first (pgrep -x), then inspect each one's
+    // arguments. A `pgrep -f 'dnsmasq.*/tmp/lab-'` matches the shell running
+    // that very pattern, so the naive version can kill its own shell -- and
+    // libvirt's dnsmasq for the default network must never be touched either.
+    const pids = execSync("pgrep -x dnsmasq 2>/dev/null || true", { encoding: "utf-8" })
+      .trim().split("\n").filter((x) => /^\d+$/.test(x.trim()));
+    let killed = 0;
+    for (const pid of pids) {
+      let args = "";
+      try {
+        args = execSync(`tr '\\0' ' ' < /proc/${pid.trim()}/cmdline 2>/dev/null || true`, { encoding: "utf-8" });
+      } catch { continue; }
+      if (!args.includes("--conf-file=/tmp/lab-")) continue;
+      log(`Killing orphaned test dnsmasq (pid ${pid.trim()}) — it would answer DHCP with stale config`);
+      try { execSync(`sudo kill -9 ${pid.trim()} 2>/dev/null || kill -9 ${pid.trim()} 2>/dev/null || true`); } catch { /* gone */ }
+      killed++;
+    }
+    // Give the socket a moment to be released before we bind it.
+    if (killed > 0) execSync("sleep 1");
+  } catch { /* nothing to kill */ }
+}
+
+/**
+ * Is something serving DHCP on the PXE bridge?
+ *
+ * `ss -lunp` shows UDP listeners; dnsmasq binds port 67. Checked on the bridge
+ * address specifically, because a dnsmasq bound to the wrong interface is the
+ * failure being guarded against.
+ */
+async function assertDhcpListening(confPath: string, timeoutMs = 20_000): Promise<void> {
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    try {
+      // Check for OUR dnsmasq by its config path, not merely for something on
+      // :67. A leaked dnsmasq from a previous run also holds :67 and also
+      // answers DHCP -- with the wrong bastion port -- so a port check alone
+      // reports success for precisely the situation it is meant to catch.
+      const pids = execSync("pgrep -x dnsmasq 2>/dev/null || true", { encoding: "utf-8" })
+        .trim().split("\n").filter((x) => /^\d+$/.test(x.trim()));
+      last = pids.filter((pid) => {
+        try {
+          return execSync(`tr '\\0' ' ' < /proc/${pid.trim()}/cmdline 2>/dev/null || true`, { encoding: "utf-8" })
+            .includes(confPath);
+        } catch { return false; }
+      }).join(",");
+      if (last !== "") {
+        const ports = execSync("ss -lun 2>/dev/null || true", { encoding: "utf-8" });
+        if (/:67\b/.test(ports)) {
+          log("dnsmasq (ours) is serving DHCP on :67");
+          return;
+        }
+      }
+    } catch { /* retry */ }
+    await sleep(1000);
+  }
+  throw new Error(
+    `dnsmasq for ${confPath} is not serving DHCP after ${timeoutMs / 1000}s. ` +
+    `It most likely failed to bind to ${PXE_BRIDGE} -- check the bastion log for ` +
+    `"error binding DHCP socket", and check for a leaked dnsmasq from an earlier run ` +
+    `(pgrep -af 'dnsmasq.*--conf-file=/tmp/lab-'). Without this every VM sits in ` +
+    `discovery until it times out, six minutes later, looking like a PXE problem.`,
+  );
+}
+
+/**
+ * Wait for the installer to actually start, rebooting into PXE if it does not.
+ *
+ * "Started" means the bastion has seen anything at all for this MAC beyond the
+ * queued state -- a dispatch, a progress line, a status. A VM that never gets
+ * that far is stuck before Anaconda, and only a reboot will shift it.
+ */
+async function waitForInstallStart(
+  base: string,
+  mac: string,
+  name: string,
+  attemptMs = 6 * 60_000,
+  attempts = 3,
+): Promise<void> {
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    try {
+      // NOT `status === "installing"`. A machine that has merely been QUEUED
+      // already reports that -- the bastion sets it the moment the install is
+      // dispatched, before anything has booted. Using it would match instantly
+      // and this retry would never fire.
+      //
+      // The honest signals that Anaconda is alive are `progress` leaving
+      // "queued", or log lines arriving over syslog.
+      await pollUntil<{ status?: string; progress?: string; log_total?: number }>(
+        `${base}/api/logs/${encodeURIComponent(mac)}`,
+        (d) => d.status === "installed"
+          || (d.progress !== undefined && d.progress !== "queued")
+          || (d.log_total ?? 0) > 0,
+        attemptMs, 5_000);
+      if (attempt > 1) log(`[${name}] PXE boot took on attempt ${attempt}`);
+      return;
+    } catch {
+      if (attempt === attempts) {
+        throw new Error(
+          `[${name}] installer never started after ${attempts} PXE boots ` +
+          `(${attemptMs / 60000} min each). The VM is stuck before Anaconda — ` +
+          `check TFTP and the iPXE chainload.`);
+      }
+      log(`[${name}] no installer activity in ${attemptMs / 60000} min; rebooting into PXE (attempt ${attempt + 1}/${attempts})`);
+      rebootPxeVm(name);
+      deleteNftablesRejectRules();
+    }
+  }
+}
+
+export interface ProvisionedVm { name: string; mac: string; ip: string; serialPort: number }
+
+/**
+ * "SSH never came up" on its own is undiagnosable once afterAll has destroyed
+ * the VM -- the machine may be sitting in emergency mode with the exact error
+ * on its serial console. Capture that console into the failure.
+ */
+async function sshFailurePostMortem(name: string, serialPort: number, cause: unknown): Promise<never> {
+  let state = "unknown";
+  try { state = execSync(`sudo virsh domstate ${name}`, { encoding: "utf-8" }).trim(); } catch { /* gone */ }
+  let serial = "(no serial output)";
+  try { serial = await readSerialLog(serialPort, { lastLines: 60 }); } catch { /* no console */ }
+  throw new Error(
+    `[${name}] ${cause instanceof Error ? cause.message : String(cause)}\n` +
+    `domstate: ${state}\n--- serial console (last lines) ---\n${serial}`,
+  );
+}
+
+/**
+ * Serial ports are a global resource on the host, so hand out a distinct one
+ * per VM. Without this only one VM can run at a time -- qemu refuses to start
+ * the second with "Address already in use".
+ */
+let nextSerialPort = 4600;
+
+/**
+ * The stable path a virtio data disk gets inside the guest. This is what the
+ * install request must carry: the feature refuses anything outside
+ * /dev/disk/by-id/, exactly as it does for real NVMe serials.
+ */
+export function virtioById(serial: string): string {
+  return `/dev/disk/by-id/virtio-${serial}`;
+}
+
+/** Create a blank VM, let it discover, queue the install, wait for SSH. */
+export async function provisionVm(h: Harness, spec: {
+  name: string;
+  role: Role;
+  schemaVersion?: 1 | 2;
+  diskGb: number;
+  memory?: number;
+  vcpus?: number;
+  /**
+   * Dedicated data disks: attached to the VM with the given virtio serial AND
+   * passed to /api/install as /dev/disk/by-id/virtio-<serial>. Mirrors
+   * thebeast's layout (OS disk + separate longhorn + rancher disks).
+   */
+  dataDisks?: { longhorn?: { serial: string; sizeGb: number }; rancher?: { serial: string; sizeGb: number } };
+}): Promise<ProvisionedVm> {
+  const base = `http://${PXE_GATEWAY}:${h.httpPort}`;
+  const vmDataDisks = [spec.dataDisks?.longhorn, spec.dataDisks?.rancher]
+    .filter((d): d is { serial: string; sizeGb: number } => d !== undefined);
+  log(`[${spec.name}] creating VM (${spec.diskGb}GB disk + ${vmDataDisks.length} data disk(s), role=${spec.role}, schema=v${spec.schemaVersion ?? 1})`);
+  const serialPort = nextSerialPort++;
+  createPxeVm({
+    name: spec.name,
+    memory: spec.memory ?? 4096,
+    vcpus: spec.vcpus ?? 4,
+    diskSize: spec.diskGb,
+    network: PXE_NETWORK_NAME,
+    serialPort,
+    ...(vmDataDisks.length > 0 ? { dataDisks: vmDataDisks } : {}),
+  });
+  deleteNftablesRejectRules();
+
+  const mac = getVmMac(spec.name);
+  if (!mac) throw new Error(`[${spec.name}] no MAC`);
+
+  log(`[${spec.name}] waiting for discovery (${mac})...`);
+  await pollUntil<{ discovered: Record<string, unknown> }>(
+    `${base}/api/machines`, (d) => mac in (d.discovered ?? {}), DISCOVERY_TIMEOUT_MS);
+
+  log(`[${spec.name}] queueing install...`);
+  const res = await fetch(`${base}/api/install`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      mac, hostname: spec.name, disk: "", role: spec.role,
+      ...(spec.schemaVersion ? { schemaVersion: spec.schemaVersion } : {}),
+      ...(spec.dataDisks ? {
+        dataDisks: {
+          ...(spec.dataDisks.longhorn ? { longhorn: virtioById(spec.dataDisks.longhorn.serial) } : {}),
+          ...(spec.dataDisks.rancher ? { rancher: virtioById(spec.dataDisks.rancher.serial) } : {}),
+        },
+      } : {}),
+    }),
+  });
+  if (!res.ok) throw new Error(`[${spec.name}] install queue rejected: ${res.status} ${await res.text()}`);
+
+  rebootPxeVm(spec.name);
+  deleteNftablesRejectRules();
+
+  // Make sure the PXE boot actually TOOK before settling in for the install.
+  //
+  // A UEFI VM that fails to chainload just retries DHCP quietly, for ever.
+  // Observed: one VM of four sat in a DHCP loop for an HOUR with no TFTP
+  // fetch, then began installing seconds after the 60-minute install timeout
+  // had already fired -- so the run failed on a machine that was, by then,
+  // working. TFTP is UDP and four VMs contend for it, so this is a question of
+  // when rather than if.
+  //
+  // Rebooting is cheap and a boot that has not started within a few minutes is
+  // not going to start on its own.
+  await waitForInstallStart(base, mac, spec.name);
+
+  log(`[${spec.name}] installing (up to ${INSTALL_TIMEOUT_MS / 60000} min)...`);
+  await pollUntil<{ status?: string }>(
+    `${base}/api/logs/${encodeURIComponent(mac)}`, (d) => d.status === "installed", INSTALL_TIMEOUT_MS, 10_000);
+
+  // The installed record carries the DHCP address the machine ended up with.
+  const machines = await pollUntil<{ installed: Record<string, { ip?: string }> }>(
+    `${base}/api/machines`, (d) => Boolean(d.installed?.[mac]?.ip), 2 * 60_000);
+  const ip = machines.installed[mac]!.ip!;
+
+  try {
+    if (execSync(`sudo virsh domstate ${spec.name}`, { encoding: "utf-8" }).trim() === "shut off") {
+      execSync(`sudo virsh start ${spec.name}`, { stdio: "pipe" });
+    }
+  } catch { /* already running */ }
+  deleteNftablesRejectRules();
+
+  log(`[${spec.name}] waiting for SSH at ${ip}...`);
+  try {
+    await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath);
+  } catch (e) {
+    await sshFailurePostMortem(spec.name, serialPort, e);
+  }
+  log(`[${spec.name}] up at ${ip}`);
+  return { name: spec.name, mac, ip, serialPort };
+}
+
+/**
+ * Provision several VMs at once.
+ *
+ * Installs are ~12 minutes each and almost entirely I/O and network wait, so
+ * four in sequence is an hour that four in parallel is twenty minutes. They are
+ * staggered because the expensive, contended part is the start: PXE, DHCP and
+ * the first package fetches all land at once otherwise.
+ *
+ * Rejects only after every VM has settled, so a single failure does not leave
+ * the others half-built and unreferenced -- the caller still gets the names it
+ * needs to tear down.
+ */
+export async function provisionVms(
+  h: Harness,
+  specs: Parameters<typeof provisionVm>[1][],
+  opts: { staggerMs?: number } = {},
+): Promise<ProvisionedVm[]> {
+  const stagger = opts.staggerMs ?? 60_000;
+  const settled = await Promise.allSettled(
+    specs.map(async (spec, i) => {
+      await sleep(i * stagger);
+      return provisionVm(h, spec);
+    }),
+  );
+
+  const failures = settled
+    .map((r, i) => (r.status === "rejected" ? `${specs[i]?.name}: ${String(r.reason)}` : null))
+    .filter((x): x is string => x !== null);
+  if (failures.length > 0) {
+    throw new Error(`provisionVms: ${failures.length}/${specs.length} failed\n  ${failures.join("\n  ")}`);
+  }
+  return settled.map((r) => (r as PromiseFulfilledResult<ProvisionedVm>).value);
+}
+
+/** LV name -> size in MiB, read off the installed machine. */
+export function readLvs(h: Harness, vm: ProvisionedVm): Record<string, number> {
+  const res = sshExec(vm.ip, h.sshUser,
+    "sudo lvs labvg --noheadings --units m --nosuffix -o lv_name,lv_size --separator=,",
+    { keyPath: h.sshKeyPath });
+  if (res.exitCode !== 0) throw new Error(`[${vm.name}] lvs failed: ${res.stderr}`);
+  const map: Record<string, number> = {};
+  for (const line of res.stdout.split("\n")) {
+    const [name, size] = line.trim().split(",");
+    if (name && size) map[name] = Math.round(Number(size));
+  }
+  // Log what actually landed on disk. The VMs are destroyed in afterAll, so
+  // without this the only record of the real layout is the pass/fail bit.
+  const summary = Object.entries(map)
+    .sort(([a], [b]) => a.localeCompare(b))
+    .map(([n, m]) => `${n}=${(m / 1024).toFixed(0)}G`)
+    .join(" ");
+  log(`[${vm.name}] on-disk LVs: ${summary}`);
+  return map;
+}
+
+/** The VG tag that records which schema this disk was built with. */
+export function readSchemaTag(h: Harness, vm: ProvisionedVm): string {
+  return sshExec(vm.ip, h.sshUser, "sudo vgs --noheadings -o vg_tags labvg",
+    { keyPath: h.sshKeyPath }).stdout.trim();
+}
+
+export function destroyVms(...names: string[]): void {
+  for (const n of names) { try { destroyPxeVm(n); } catch { /* already gone */ } }
+}
+
+/**
+ * Reinstall a machine that is already installed, the way
+ * `labctl provision reprovision` does: queue another install for the same MAC
+ * and reboot it into PXE.
+ *
+ * The schema is deliberately NOT passed. A reinstall must reproduce whatever
+ * layout is already on the disk -- %pre reads the lab-schema-v2 LVM tag rather
+ * than trusting the caller -- and passing one here would hide a regression in
+ * exactly that logic.
+ */
+export async function reprovisionVm(
+  h: Harness,
+  vm: ProvisionedVm,
+  role: Role,
+  /**
+   * Only set this to prove it is IGNORED. A reinstall takes its layout from the
+   * disk, not from the request; passing a schema here is how you test that.
+   */
+  requestSchemaVersion?: 1 | 2,
+): Promise<ProvisionedVm> {
+  const base = `http://${PXE_GATEWAY}:${h.httpPort}`;
+  log(`[${vm.name}] reprovisioning (role=${role}, schema: whatever is on disk)...`);
+
+  const res = await fetch(`${base}/api/install`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      mac: vm.mac, hostname: vm.name, disk: "", role,
+      ...(requestSchemaVersion ? { schemaVersion: requestSchemaVersion } : {}),
+    }),
+  });
+  if (!res.ok) throw new Error(`[${vm.name}] reinstall queue rejected: ${res.status} ${await res.text()}`);
+
+  rebootPxeVm(vm.name);
+  deleteNftablesRejectRules();
+
+  log(`[${vm.name}] reinstalling...`);
+  await pollUntil<{ status?: string }>(
+    `${base}/api/logs/${encodeURIComponent(vm.mac)}`,
+    (d) => d.status === "installed", INSTALL_TIMEOUT_MS, 10_000);
+
+  const machines = await pollUntil<{ installed: Record<string, { ip?: string }> }>(
+    `${base}/api/machines`, (d) => Boolean(d.installed?.[vm.mac]?.ip), 2 * 60_000);
+  const ip = machines.installed[vm.mac]!.ip!;
+
+  try {
+    if (execSync(`sudo virsh domstate ${vm.name}`, { encoding: "utf-8" }).trim() === "shut off") {
+      execSync(`sudo virsh start ${vm.name}`, { stdio: "pipe" });
+    }
+  } catch { /* already running */ }
+  deleteNftablesRejectRules();
+
+  try {
+    await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath);
+  } catch (e) {
+    await sshFailurePostMortem(vm.name, vm.serialPort, e);
+  }
+  log(`[${vm.name}] back up at ${ip} after reinstall`);
+  return { ...vm, ip };
+}
+
+/**
+ * Drop a marker file into each mount point that a reinstall is supposed to
+ * preserve. Ported from test-reprovision.sh, which proved the idea but only
+ * ever ran by hand.
+ */
+export function writeMarkers(h: Harness, vm: ProvisionedVm, mounts: string[]): string {
+  const token = `preserve-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+  for (const m of mounts) {
+    const r = sshExec(vm.ip, h.sshUser,
+      `sudo sh -c 'echo ${token} > ${m}/LAB_PRESERVE_TEST.txt && sync'`,
+      { keyPath: h.sshKeyPath });
+    if (r.exitCode !== 0) throw new Error(`[${vm.name}] could not write marker in ${m}: ${r.stderr}`);
+  }
+  log(`[${vm.name}] wrote marker ${token} into ${mounts.join(", ")}`);
+  return token;
+}
+
+/** Which mount points still carry the marker. */
+export function readMarkers(h: Harness, vm: ProvisionedVm, mounts: string[]): Record<string, string | null> {
+  const out: Record<string, string | null> = {};
+  for (const m of mounts) {
+    const r = sshExec(vm.ip, h.sshUser, `sudo cat ${m}/LAB_PRESERVE_TEST.txt 2>/dev/null || true`,
+      { keyPath: h.sshKeyPath });
+    const v = r.stdout.trim();
+    out[m] = v === "" ? null : v;
+  }
+  log(`[${vm.name}] markers after reinstall: ${JSON.stringify(out)}`);
+  return out;
+}

bastion/tests/integration/helpers/pxe-network.ts

@@ -56,6 +56,17 @@ export function ensurePxeNetwork(): void {
 
     try { unlinkSync(xmlPath); } catch { /* ignore */ }
 
+    // `virsh net-start` returns before the bridge is usable. The bastion's
+    // dnsmasq starts about a second later and binds to this device by name; if
+    // the interface is not up and addressed yet it fails with "error binding
+    // DHCP socket to device virbr-pxe", and because that failure is
+    // asynchronous the harness carries on with NO DHCP AT ALL -- every VM then
+    // sits in discovery until it times out, six minutes later, looking like a
+    // PXE problem rather than a startup-order one.
+    //
+    // Cheap to wait, expensive not to.
+    waitForBridge();
+
     log(`Network ${PXE_NETWORK_NAME} created and active`);
   }
 
@@ -64,6 +75,32 @@ export function ensurePxeNetwork(): void {
   deleteNftablesRejectRules();
 }
 
+/**
+ * Block until the bridge exists AND carries the gateway address.
+ *
+ * Checking only that the device exists is not enough: it appears before the
+ * address is assigned, and dnsmasq needs the address to bind.
+ */
+export function waitForBridge(timeoutMs = 30_000): void {
+  const start = Date.now();
+  let last = "";
+  while (Date.now() - start < timeoutMs) {
+    const r = spawnSync("ip", ["-br", "addr", "show", PXE_BRIDGE], { encoding: "utf-8", stdio: "pipe" });
+    last = (r.stdout ?? "").trim();
+    if (r.status === 0 && last.includes(`${PXE_GATEWAY}/`)) {
+      log(`Bridge ${PXE_BRIDGE} ready with ${PXE_GATEWAY} after ${Date.now() - start}ms`);
+      return;
+    }
+    // Busy-wait briefly; this normally resolves in well under a second.
+    const until = Date.now() + 250;
+    while (Date.now() < until) { /* spin */ }
+  }
+  throw new Error(
+    `Bridge ${PXE_BRIDGE} did not come up with ${PXE_GATEWAY} within ${timeoutMs}ms ` +
+    `(last: ${last || "device absent"}). dnsmasq cannot bind, so PXE would silently not work.`,
+  );
+}
+
 /** Delete libvirt's nftables reject rules for our bridge so host→VM traffic works.
  *  Must be called after every VM start/restart — libvirt recreates them. */
 export function deleteNftablesRejectRules(): void {

bastion/tests/integration/helpers/pxe-vm.ts

@@ -28,7 +28,34 @@ export interface PxeVmConfig {
   vcpus: number;
   diskSize: number;     // GB
   network: string;      // libvirt network name
+  /**
+   * Serial console TCP port. Defaults to 4555 for backwards compatibility with
+   * the single-VM tests, but it MUST be unique per concurrently-running VM:
+   * qemu fails with "Failed to find an available port: Address already in use"
+   * and the domain never starts.
+   */
+  serialPort?: number;
   arch?: "x86_64" | "aarch64";
+  /**
+   * Extra NICs enumerated BEFORE the PXE NIC, on a network with no route to
+   * the bastion (defaults to libvirt's "default").
+   *
+   * Real multi-NIC boxes expose a class of bug a single-NIC VM cannot: an
+   * initramfs that picks "the first connected interface" grabs one of these
+   * instead of the NIC that PXE booted, and then cannot reach the bastion.
+   * Defaults to 0 (single NIC).
+   */
+  decoyNics?: number;
+  decoyNetwork?: string;
+  /**
+   * Extra blank data disks, attached after the OS disk. Each gets a fixed
+   * virtio serial, so inside the guest it has a stable
+   * /dev/disk/by-id/virtio-<serial> path -- the moral equivalent of the
+   * nvme-..._<serial> links the dedicated-data-disk feature requires on real
+   * hardware. virtio-blk truncates serials at 20 characters, so keep them
+   * short or the by-id link quietly won't match.
+   */
+  dataDisks?: { serial: string; sizeGb: number }[];
 }
 
 /** Create a blank UEFI VM that PXE boots from the network. */
@@ -42,6 +69,14 @@ export function createPxeVm(config: PxeVmConfig): void {
   const diskPath = join(IMAGE_DIR, `${config.name}.qcow2`);
   run(`qemu-img create -f qcow2 "${diskPath}" ${config.diskSize}G`);
 
+  const dataDiskArgs: string[] = [];
+  for (const dd of config.dataDisks ?? []) {
+    if (dd.serial.length > 20) throw new Error(`virtio serial '${dd.serial}' exceeds 20 chars -- the guest by-id link would be truncated`);
+    const p = join(IMAGE_DIR, `${config.name}-${dd.serial}.qcow2`);
+    run(`qemu-img create -f qcow2 "${p}" ${dd.sizeGb}G`);
+    dataDiskArgs.push(`--disk=path=${p},format=qcow2,bus=virtio,serial=${dd.serial}`);
+  }
+
   // UEFI firmware paths (Fedora)
   if (arch === "aarch64") {
     const aavmf = "/usr/share/edk2/aarch64/QEMU_EFI.fd";
@@ -61,6 +96,11 @@ export function createPxeVm(config: PxeVmConfig): void {
     `--memory=${config.memory}`,
     `--vcpus=${config.vcpus}`,
     `--disk=path=${diskPath},format=qcow2,bus=virtio`,
+    ...dataDiskArgs,
+    // Decoys first so they enumerate ahead of the PXE NIC. They are up and
+    // carry a lease, but have no route to the bastion.
+    ...Array.from({ length: config.decoyNics ?? 0 }, () =>
+      `--network=network=${config.decoyNetwork ?? "default"},model=virtio`),
     `--network=network=${config.network},model=virtio`,
     // UEFI firmware — required for PXE boot in modern mode
     `--boot=uefi,network,hd`,
@@ -71,8 +111,8 @@ export function createPxeVm(config: PxeVmConfig): void {
     // Graphics for debugging (VNC, connect with virt-viewer if needed)
     "--graphics=vnc,listen=127.0.0.1",
     // Serial console via TCP — allows exec without network/SSH
-    // Connect: socat - TCP:127.0.0.1:4555
-    "--serial=tcp,host=127.0.0.1:4555,mode=bind,protocol=telnet",
+    // Connect: socat - TCP:127.0.0.1:<serialPort>
+    `--serial=tcp,host=127.0.0.1:${config.serialPort ?? 4555},mode=bind,protocol=telnet`,
   ];
 
   if (arch === "aarch64") {
@@ -81,7 +121,7 @@ export function createPxeVm(config: PxeVmConfig): void {
 
   log(`Running: virt-install --name=${config.name} --boot=uefi,network ...`);
   run(virtInstallArgs.join(" "), { timeout: 30_000 });
-  log(`PXE VM ${config.name} created (serial: telnet 127.0.0.1 4555)`);
+  log(`PXE VM ${config.name} created (serial: telnet 127.0.0.1 ${config.serialPort ?? 4555})`);
 }
 
 /** Destroy a PXE VM and clean up its disk. */
@@ -95,12 +135,21 @@ export function destroyPxeVm(name: string): void {
 }
 
 /** Get the MAC address of a VM's first NIC. */
-export function getVmMac(name: string): string | null {
+export function getVmMac(name: string, network?: string): string | null {
   const result = virsh("domiflist", name);
   if (result.status !== 0) return null;
   // Output format: Interface  Type  Source  Model  MAC
-  const match = result.stdout.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i);
-  return match ? match[1].toLowerCase() : null;
+  // With decoy NICs present, match the line for the PXE network so we return
+  // the NIC that actually boots rather than whichever is listed first.
+  const lines = result.stdout.split("\n");
+  const candidates = network === undefined
+    ? lines
+    : lines.filter((l) => l.split(/\s+/).includes(network));
+  for (const line of candidates.length > 0 ? candidates : lines) {
+    const m = line.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i);
+    if (m) return m[1].toLowerCase();
+  }
+  return null;
 }
 
 /** Reboot a VM (force off + start). */

bastion/tests/integration/k3s-single-node.test.ts

@@ -161,7 +161,7 @@ EOF'
       CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
       curl -L --fail --silent "https://github.com/cilium/cilium-cli/releases/download/\${CILIUM_CLI_VERSION}/cilium-linux-amd64.tar.gz" | sudo tar xz -C /usr/local/bin
       DEFAULT_DEV=$(ip -4 route show default | awk '{print $5}' | head -1)
-      sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install --set kubeProxyReplacement=true --set ipam.mode=kubernetes --set devices=$DEFAULT_DEV --set nodePort.directRoutingDevice=$DEFAULT_DEV
+      sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install --set kubeProxyReplacement=true --set ipam.mode=cluster-pool --set ipam.operator.clusterPoolIPv4PodCIDRList='{10.42.0.0/16}' --set ipam.operator.clusterPoolIPv4MaskSize=24 --set devices=$DEFAULT_DEV --set nodePort.directRoutingDevice=$DEFAULT_DEV
     `.trim(), "cilium install", { keyPath: sshKeyPath, timeout: 120_000 });
 
     log("Waiting for Cilium to be ready...");

bastion/tests/integration/partition-layout.test.ts

@@ -0,0 +1,145 @@
+// Integration tests: the v2 disk layout, on real VMs.
+//
+// Unit tests prove the kickstart RENDERS the right layout. These prove anaconda
+// ACTUALLY PRODUCES it -- which is a different claim, and the one that matters:
+// the labmaster incident was caused by what landed on disk, not by what was
+// written into a template.
+//
+// Every VM gets a 1 TB sparse disk, because "fits a 1 TB disk with headroom" is
+// the constraint v2 was designed around. qcow2 is sparse, so a 1 TB disk costs
+// ~10-15 GB of real space.
+//
+// Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo, and a Fedora mirror.
+// Set FEDORA_MIRROR to a local mirror -- otherwise each install pulls the full
+// package set over the internet and the matrix takes hours.
+//
+// Run one scenario at a time:
+//   sudo -E npx vitest run -c tests/integration/vitest.config.ts -t "vanilla"
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { v2Layout } from "../../src/shared/src/types/layout.js";
+import type { Role } from "../../src/shared/src/types/state.js";
+import {
+  startTestBastion, provisionVm, readLvs, readSchemaTag, destroyVms,
+  type Harness, type ProvisionedVm,
+} from "./helpers/provision-harness.js";
+import { destroyPxeNetwork } from "./helpers/pxe-network.js";
+
+const HTTP_PORT = 8096;
+const SSH_USER = "lab";
+const DISK_GB = 1000; // the smallest disk we will ever buy
+
+function findSshKey(): { pubKey: string; keyPath: string } {
+  const homes = [homedir()];
+  const sudoUser = process.env["SUDO_USER"];
+  if (sudoUser) homes.push(join("/home", sudoUser));
+  for (const home of homes) {
+    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
+      const keyPath = join(home, ".ssh", name);
+      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
+        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
+      }
+    }
+  }
+  throw new Error("no SSH keypair found");
+}
+
+/** Assert the on-disk volumes match the shared v2 definition for this role. */
+function expectV2Layout(h: Harness, vm: ProvisionedVm, role: Role): void {
+  const actual = readLvs(h, vm);
+  const expected = v2Layout(role);
+
+  expect(Object.keys(actual).sort(), `${vm.name} volume set`).toEqual(
+    expected.map((lv) => lv.name).sort(),
+  );
+  for (const spec of expected) {
+    // LVM rounds to extent boundaries (4 MiB), so allow a small delta.
+    expect(actual[spec.name], `${vm.name}/${spec.name} size`).toBeGreaterThanOrEqual(spec.sizeMib - 4);
+    expect(actual[spec.name], `${vm.name}/${spec.name} size`).toBeLessThanOrEqual(spec.sizeMib + 4);
+  }
+  expect(readSchemaTag(h, vm), `${vm.name} schema tag`).toContain("lab-schema-v2");
+}
+
+describe("v2 partition layout on real hardware", () => {
+  let h: Harness;
+  const created: string[] = [];
+
+  beforeAll(async () => {
+    const { pubKey, keyPath } = findSshKey();
+    h = await startTestBastion({
+      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey,
+      sshKeyPath: keyPath, label: "partition",
+    });
+  }, 20 * 60_000);
+
+  afterAll(async () => {
+    destroyVms(...created);
+    try { await h?.close(); } catch { /* already closed */ }
+    try { destroyPxeNetwork(); } catch { /* already gone */ }
+  }, 5 * 60_000);
+
+  // --- Topology 1: vanilla -------------------------------------------------
+  it("vanilla: base volumes only, no rancher, no longhorn", async () => {
+    const name = "v2-vanilla";
+    created.push(name);
+    const vm = await provisionVm(h, { name, role: "vanilla", schemaVersion: 2, diskGb: DISK_GB });
+    expectV2Layout(h, vm, "vanilla");
+
+    const lvs = readLvs(h, vm);
+    expect(lvs["rancher"]).toBeUndefined();
+    expect(lvs["longhorn"]).toBeUndefined();
+  }, 60 * 60_000);
+
+  // --- Topology 2: standalone cluster --------------------------------------
+  it("standalone-cluster: single node with its own control plane and storage", async () => {
+    const name = "v2-standalone";
+    created.push(name);
+    const vm = await provisionVm(h, { name, role: "standalone-cluster", schemaVersion: 2, diskGb: DISK_GB, memory: 6144 });
+    expectV2Layout(h, vm, "standalone-cluster");
+
+    const lvs = readLvs(h, vm);
+    // A cluster of one still gets a Longhorn volume, so the `longhorn`
+    // StorageClass exists and manifests stay portable across clusters.
+    expect(lvs["longhorn"]).toBeDefined();
+    expect(lvs["rancher"]).toBeDefined();
+  }, 60 * 60_000);
+
+  // --- Topology 3: 3x combo (control plane AND worker on every node) -------
+  it("k8s-combo x3: every node is control plane and storage", async () => {
+    const names = ["v2-combo-1", "v2-combo-2", "v2-combo-3"];
+    created.push(...names);
+    const vms: ProvisionedVm[] = [];
+    for (const name of names) {
+      vms.push(await provisionVm(h, { name, role: "k8s-combo", schemaVersion: 2, diskGb: DISK_GB }));
+    }
+    for (const vm of vms) {
+      expectV2Layout(h, vm, "k8s-combo");
+      const lvs = readLvs(h, vm);
+      expect(lvs["longhorn"], `${vm.name} holds storage`).toBeDefined();
+      expect(lvs["rancher"], `${vm.name} has an image store`).toBeDefined();
+    }
+  }, 180 * 60_000);
+
+  // --- Topology 4: 3 controllers + 2 workers -------------------------------
+  it("3x k8s-controller + 2x k8s-worker: storage only on the workers", async () => {
+    const ctrls = ["v2-ctrl-1", "v2-ctrl-2", "v2-ctrl-3"];
+    const workers = ["v2-wrk-1", "v2-wrk-2"];
+    created.push(...ctrls, ...workers);
+
+    for (const name of ctrls) {
+      const vm = await provisionVm(h, { name, role: "k8s-controller", schemaVersion: 2, diskGb: DISK_GB });
+      expectV2Layout(h, vm, "k8s-controller");
+      // The distinguishing property of a controller: image store, no storage.
+      expect(readLvs(h, vm)["longhorn"], `${name} must not hold replicas`).toBeUndefined();
+      expect(readLvs(h, vm)["rancher"]).toBeDefined();
+    }
+    for (const name of workers) {
+      const vm = await provisionVm(h, { name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB });
+      expectV2Layout(h, vm, "k8s-worker");
+      expect(readLvs(h, vm)["longhorn"], `${name} must hold replicas`).toBeDefined();
+    }
+  }, 300 * 60_000);
+});

bastion/tests/integration/pxe-rescue.test.ts

@@ -1,210 +0,0 @@
-// Integration test: `labctl provision debug` -> Anaconda rescue with SSH, on x86_64.
-//
-// The rescue path had no test coverage on any architecture, which matters because it is
-// the lab's recovery tool of last resort -- the thing you reach for when a machine will
-// not boot. It runs here on x86_64 with KVM so it completes in minutes; the aarch64
-// equivalent is the same code path with a different kernel, but is emulated and far too
-// slow to iterate on.
-//
-// Run: sudo ./scripts/test-provision.sh rescue
-
-import { describe, it, expect, beforeAll, afterAll } from "vitest";
-import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, writeFileSync } from "node:fs";
-import { execSync } from "node:child_process";
-import { join } from "node:path";
-import { homedir, tmpdir } from "node:os";
-import { log, waitForSsh } from "./helpers/libvirt.js";
-import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js";
-import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./helpers/pxe-vm.js";
-import { sshExec } from "./helpers/ssh.js";
-
-const VM_NAME = "lab-pxe-rescue-test";
-const HTTP_PORT = 8094;
-const VM_MEMORY = 4096;
-const VM_VCPUS = 4;
-const VM_DISK_GB = 20;
-const BASTION_IP = PXE_GATEWAY;
-const SERIAL_PORT = 4555;
-
-const LEASE_TIMEOUT_MS = 8 * 60_000;
-const SSH_TIMEOUT_MS = 15 * 60_000;
-
-function sleep(ms: number): Promise<void> {
-  return new Promise((r) => setTimeout(r, ms));
-}
-
-function findSshKey(): { pubKey: string; keyPath: string } {
-  const candidates: string[] = [];
-  if (process.env["SSH_KEY_PATH"]) candidates.push(process.env["SSH_KEY_PATH"]);
-  const homes = [homedir()];
-  const sudoUser = process.env["SUDO_USER"];
-  if (sudoUser) homes.push(join("/home", sudoUser));
-  for (const home of homes) {
-    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) candidates.push(join(home, ".ssh", name));
-  }
-  for (const keyPath of candidates) {
-    if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
-      return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
-    }
-  }
-  throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/");
-}
-
-function leaseIpFor(testDir: string, mac: string): string | null {
-  const leaseFile = join(testDir, "dnsmasq.leases");
-  if (!existsSync(leaseFile)) return null;
-  for (const line of readFileSync(leaseFile, "utf-8").split("\n")) {
-    const parts = line.trim().split(/\s+/);
-    if (parts.length >= 3 && parts[1]?.toLowerCase() === mac.toLowerCase()) return parts[2] ?? null;
-  }
-  return null;
-}
-
-async function waitForLease(testDir: string, mac: string, timeoutMs: number): Promise<string> {
-  const start = Date.now();
-  while (Date.now() - start < timeoutMs) {
-    const ip = leaseIpFor(testDir, mac);
-    if (ip !== null) return ip;
-    await sleep(5000);
-  }
-  throw new Error(`No DHCP lease for ${mac} within ${timeoutMs}ms`);
-}
-
-// Suite name must not be a substring of "ARM PXE rescue" -- vitest -t matches
-// substrings, so a looser name here would drag the emulated aarch64 suite in with it.
-describe("x86 rescue boot", () => {
-  let app: { close: () => Promise<void> };
-  let stopDnsmasqFn: () => void;
-  let testDir: string;
-  let vmMac: string;
-  let rescueIp: string;
-  let sshKeyPath: string;
-
-  beforeAll(async () => {
-    const { pubKey, keyPath } = findSshKey();
-    sshKeyPath = keyPath;
-
-    log("Setting up PXE test network...");
-    ensurePxeNetwork();
-
-    testDir = join(tmpdir(), `lab-pxe-rescue-${Date.now()}`);
-    for (const sub of ["tftp", "http", "logs"]) mkdirSync(join(testDir, sub), { recursive: true });
-
-    const { createApp } = await import("../../src/bastion/src/server.js");
-    const { loadConfig } = await import("../../src/bastion/src/config.js");
-    const { generateDnsmasqConf, startDnsmasq, stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
-    const { renderBootIpxe, kernelPath, initrdPath } = await import("../../src/bastion/src/templates/boot.ipxe.js");
-    stopDnsmasqFn = stopDnsmasq;
-
-    const config = loadConfig({
-      bastionDir: testDir,
-      httpPort: HTTP_PORT,
-      iface: "virbr-pxe",
-      serverIp: BASTION_IP,
-      network: `${PXE_SUBNET}.0`,
-      gateway: BASTION_IP,
-      dhcpMode: "full",
-      dhcpRangeStart: `${PXE_SUBNET}.100`,
-      dhcpRangeEnd: `${PXE_SUBNET}.200`,
-      domain: "rescue-test.local",
-      sshKeys: [pubKey],
-      adminUser: "lab",
-    });
-
-    // iPXE in both dirs: TFTP PXE and UEFI HTTP Boot are both possible, and OVMF picks.
-    const ipxeX86 = "/usr/share/ipxe/ipxe-snponly-x86_64.efi";
-    if (!existsSync(ipxeX86)) throw new Error(`iPXE not found: ${ipxeX86}`);
-    copyFileSync(ipxeX86, join(config.tftpDir, "ipxe.efi"));
-    copyFileSync(ipxeX86, join(config.httpDir, "ipxe.efi"));
-
-    const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache";
-    execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" });
-    const kernelCache = join(cacheDir, "vmlinuz-x86_64");
-    const initrdCache = join(cacheDir, "initrd-x86_64.img");
-    if (!existsSync(kernelCache)) {
-      log("Downloading Fedora x86_64 kernel...");
-      execSync(`curl -# -L -f -o "${kernelCache}" "${config.fedoraMirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 600_000 });
-    }
-    if (!existsSync(initrdCache)) {
-      log("Downloading Fedora x86_64 initrd...");
-      execSync(`curl -# -L -f -o "${initrdCache}" "${config.fedoraMirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 600_000 });
-    }
-    copyFileSync(kernelCache, join(config.httpDir, kernelPath("x86_64")));
-    copyFileSync(initrdCache, join(config.httpDir, initrdPath("x86_64")));
-
-    writeFileSync(join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }));
-    generateDnsmasqConf(config);
-
-    const { app: fastify, state, syslog } = createApp(config);
-    app = fastify;
-    await fastify.listen({ port: config.httpPort, host: "0.0.0.0" });
-    syslog.start();
-    log(`Bastion HTTP listening on :${HTTP_PORT}`);
-
-    startDnsmasq(config).catch((err) => log(`dnsmasq failed: ${err instanceof Error ? err.message : String(err)}`));
-    await sleep(1500);
-
-    log("Creating x86_64 PXE VM (KVM)...");
-    createPxeVm({ name: VM_NAME, memory: VM_MEMORY, vcpus: VM_VCPUS, diskSize: VM_DISK_GB, network: PXE_NETWORK_NAME });
-    const mac = getVmMac(VM_NAME);
-    if (!mac) throw new Error("Could not determine VM MAC");
-    vmMac = mac;
-    log(`VM MAC: ${vmMac}`);
-
-    // Queue rescue directly, as `labctl provision debug` does.
-    log("Queueing debug/rescue mode...");
-    state.update((s) => {
-      s.debug[vmMac] = { hostname: "rescue-test", queued_at: new Date().toISOString() };
-    });
-
-    rebootPxeVm(VM_NAME);
-    await sleep(5_000);
-    deleteNftablesRejectRules();
-
-    rescueIp = await waitForLease(testDir, vmMac, LEASE_TIMEOUT_MS);
-    log(`Rescue IP: ${rescueIp}`);
-
-    log("Waiting for SSH into the rescue shell (inst.sshd)...");
-    await waitForSsh(rescueIp, "root", SSH_TIMEOUT_MS, sshKeyPath).catch(async (err) => {
-      log("Rescue SSH timed out. Serial console:");
-      try { log(await readSerialLog(SERIAL_PORT, { lastLines: 120, timeoutMs: 20_000 })); } catch { /* none */ }
-      throw err;
-    });
-    log("Rescue shell reachable.");
-  }, LEASE_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000);
-
-  afterAll(async () => {
-    if (process.env["KEEP_VM"] === "1") {
-      log(`KEEP_VM=1 — leaving ${VM_NAME} up (serial: socat - TCP:127.0.0.1:${SERIAL_PORT})`);
-      return;
-    }
-    log("Cleaning up...");
-    if (app) await app.close().catch(() => {});
-    if (stopDnsmasqFn) stopDnsmasqFn();
-    destroyPxeVm(VM_NAME);
-    destroyPxeNetwork();
-    if (testDir) rmSync(testDir, { recursive: true, force: true });
-  });
-
-  it("rescue shell is reachable over SSH as root", () => {
-    const result = sshExec(rescueIp, "root", "whoami", { keyPath: sshKeyPath, timeout: 60_000 });
-    expect(result.exitCode).toBe(0);
-    expect(result.stdout.trim()).toBe("root");
-  });
-
-  it("is the Anaconda rescue environment", () => {
-    const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 });
-    expect(result.stdout).toContain("inst.rescue");
-    expect(result.stdout).toContain("inst.sshd");
-  });
-
-  it("kernel and initrd came from the bastion", () => {
-    const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 });
-    expect(result.stdout).toContain(`${BASTION_IP}:${HTTP_PORT}`);
-  });
-
-  it("has LVM tools for inspecting an installed system", () => {
-    const result = sshExec(rescueIp, "root", "command -v vgchange && command -v lsblk", { keyPath: sshKeyPath, timeout: 60_000 });
-    expect(result.exitCode).toBe(0);
-  });
-});