docs(bastion): state the rescue-SSH evidence at its actual strength

The previous wording claimed the failure was "reproduced on x86_64", which
overstates it. The aarch64 observation is direct -- port 22 probed for 30
minutes with the Anaconda environment demonstrably up. The x86_64 run only
shows SSH not becoming available inside a 15-minute budget; that VM was
never observed reaching the rescue environment, because vitest's reporter
discards the streamed log.

The distinction decides where the next person looks, so it should not rest
on an inference presented as an observation.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015nRFZXpKwUVE4SRSHw6GjF

.gitea/workflows/ci.yml

@@ -28,8 +28,6 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
-        with:
-          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -49,8 +47,6 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
-        with:
-          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -58,9 +54,6 @@ jobs:
 
       - run: pnpm install --frozen-lockfile
 
-      - name: Generate Prisma client (labd typecheck needs it)
-        run: pnpm -C src/labd run db:generate
-
       - name: Typecheck
         run: pnpm typecheck
 
@@ -73,8 +66,6 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
-        with:
-          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -82,15 +73,6 @@ jobs:
 
       - run: pnpm install --frozen-lockfile
 
-      - name: Generate Prisma client (labd typecheck needs it)
-        run: pnpm -C src/labd run db:generate
-
-      - name: Install test validators (ubuntu-autoinstall parses YAML via python, kickstart.test runs ksvalidator, the bastion smoke start runs `ip route`)
-        # pykickstart is not in the runner image's apt sources -- pip it is.
-        run: |
-          sudo apt-get update -qq && sudo apt-get install -y -qq python3-yaml python3-pip iproute2
-          sudo pip3 install --quiet --break-system-packages pykickstart
-
       - name: Build (needed by completions check)
         run: pnpm build
 
@@ -109,8 +91,6 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
-        with:
-          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -119,9 +99,6 @@ jobs:
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
-      - name: Generate Prisma client (labd typecheck needs it)
-        run: pnpm -C src/labd run db:generate
-
       - name: Build all packages
         run: pnpm build
 
@@ -146,13 +123,13 @@ jobs:
 
       - name: Package x86_64 RPM + DEB
         run: |
-          sed -e 's|^arch:.*|arch: amd64|' -e 's|src: ./dist/labctl$|src: ./dist/lab-x86_64|' nfpm.yaml > /tmp/nfpm-x86_64.yaml
+          sed -e 's|^arch:.*|arch: amd64|' -e 's|src: ./dist/lab$|src: ./dist/lab-x86_64|' nfpm.yaml > /tmp/nfpm-x86_64.yaml
           nfpm pkg --config /tmp/nfpm-x86_64.yaml --packager rpm --target dist/
           nfpm pkg --config /tmp/nfpm-x86_64.yaml --packager deb --target dist/
 
       - name: Package arm64 RPM + DEB
         run: |
-          sed -e 's|^arch:.*|arch: arm64|' -e 's|src: ./dist/labctl$|src: ./dist/lab-arm64|' nfpm.yaml > /tmp/nfpm-arm64.yaml
+          sed -e 's|^arch:.*|arch: arm64|' -e 's|src: ./dist/lab$|src: ./dist/lab-arm64|' nfpm.yaml > /tmp/nfpm-arm64.yaml
           nfpm pkg --config /tmp/nfpm-arm64.yaml --packager rpm --target dist/
           nfpm pkg --config /tmp/nfpm-arm64.yaml --packager deb --target dist/
 
@@ -160,14 +137,14 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: rpm-packages
-          path: bastion/dist/labctl-*.rpm
+          path: bastion/dist/lab-*.rpm
           retention-days: 7
 
       - name: Upload DEB artifacts
         uses: actions/upload-artifact@v3
         with:
           name: deb-packages
-          path: bastion/dist/labctl*.deb
+          path: bastion/dist/lab*.deb
           retention-days: 7
 
   # -- Release pipeline (main branch push only) --------------
@@ -198,7 +175,7 @@ jobs:
           GITEA_OWNER: ${{ env.GITEA_OWNER }}
           GITEA_REPO: lab
         run: |
-          for RPM_FILE in dist/labctl-*.rpm; do
+          for RPM_FILE in dist/lab-*.rpm; do
             [ -f "$RPM_FILE" ] || continue
             RPM_VERSION=$(rpm -qp --queryformat '%{VERSION}-%{RELEASE}' "$RPM_FILE")
             RPM_ARCH=$(rpm -qp --queryformat '%{ARCH}' "$RPM_FILE")
@@ -255,7 +232,7 @@ jobs:
           # Publish to each supported distribution
           DISTRIBUTIONS="trixie forky noble plucky"
 
-          for DEB_FILE in dist/labctl*.deb; do
+          for DEB_FILE in dist/lab*.deb; do
             [ -f "$DEB_FILE" ] || continue
             DEB_VERSION=$(dpkg-deb --field "$DEB_FILE" Version)
             DEB_ARCH=$(dpkg-deb --field "$DEB_FILE" Architecture)

.gitignore

@@ -31,9 +31,3 @@ node_modules/
 # Asahi build artifacts (large)
 bastion/.asahi-cache/
 bastion/asahi-repo/*.zip
-
-# Regenerated by labsim/dualstack-lab.sh; derived state, not source.
-labsim/dualstack-evidence/
-
-# Runtime snapshots from labsim/cilium-ipam-switch.sh
-labsim/.ipam-switch-state/

bastion/Dockerfile.bastion

@@ -29,13 +29,11 @@ COPY src/modules/modules/ src/modules/modules/
 RUN pnpm build
 
 # ── Stage 1b: Build iPXE snp.efi (uses UEFI SNP protocol for ISO boot) ──
-# Pinned to the build platform: this stage cross-compiles BOTH arch outputs
-# itself, and the iPXE toolchain does not survive qemu-user emulation.
-FROM --platform=$BUILDPLATFORM fedora:43 AS ipxe-builder
+FROM fedora:43 AS ipxe-builder
 
 RUN dnf install -y git gcc make perl-interpreter xz-devel gcc-aarch64-linux-gnu && dnf clean all
 RUN git clone --depth=1 https://github.com/ipxe/ipxe.git /tmp/ipxe
-RUN cd /tmp/ipxe/src && make bin-x86_64-efi/snp.efi bin-x86_64-efi/ipxe.efi && \
+RUN cd /tmp/ipxe/src && make bin-x86_64-efi/snp.efi && \
     make CROSS_COMPILE=aarch64-linux-gnu- bin-arm64-efi/snp.efi
 
 # ── Stage 2: Production runtime (Fedora -- needs dnsmasq) ───────
@@ -58,15 +56,6 @@ RUN dnf install -y \
 # boot from CD-ROM/USB -- it requires PXE chainloading)
 COPY --from=ipxe-builder /tmp/ipxe/src/bin-x86_64-efi/snp.efi /usr/share/ipxe/ipxe-snp-x86_64.efi
 COPY --from=ipxe-builder /tmp/ipxe/src/bin-arm64-efi/snp.efi /usr/share/ipxe/arm64-efi/ipxe-snp.efi
-# Full iPXE build (native NIC drivers + snp + nii fallback), served as the
-# default UEFI x86_64 PXE binary. snponly rides the firmware's NII/UNDI stack,
-# and on some NICs that stack cannot RECEIVE UNICAST frames: DHCP (broadcast)
-# works, then every ARP reply is lost and HTTP times out. Observed on
-# thebeast's onboard NIC 2026-10-19 (tcpdump: labmaster answered every ARP
-# request within microseconds, iPXE kept re-asking). A native driver bypasses
-# the firmware stack entirely; NICs without one still fall back to snp/nii,
-# so machines that worked before are unaffected.
-COPY --from=ipxe-builder /tmp/ipxe/src/bin-x86_64-efi/ipxe.efi /usr/share/ipxe/ipxe-full-x86_64.efi
 
 # Install pnpm
 RUN npm install -g pnpm@9

bastion/completions/labctl.bash

@@ -40,14 +40,8 @@ _labctl() {
     "app labcontroller status")
       COMPREPLY=($(compgen -W "--user -h --help" -- "$cur"))
       return ;;
-    "app k3s evacuate")
-      COMPREPLY=($(compgen -W "--user --allow-single-replica -h --help" -- "$cur"))
-      return ;;
-    "app k3s admit")
-      COMPREPLY=($(compgen -W "--user -h --help" -- "$cur"))
-      return ;;
     "app k3s install")
-      COMPREPLY=($(compgen -W "--role --user --k3s-server --k3s-token --k3s-version -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "--role --user --k3s-server --k3s-token -h --help" -- "$cur"))
       return ;;
     "app k3s health")
       COMPREPLY=($(compgen -W "--user -h --help" -- "$cur"))
@@ -65,10 +59,10 @@ _labctl() {
       COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
       return ;;
     "provision install")
-      COMPREPLY=($(compgen -W "--role --os --disk --schema-version --longhorn-disk --rancher-disk --vyos-mgmt --vyos-mgmt-address --vyos-bond --vyos-bond-address --vyos-bond-vrrp --vlan-vip --vyos-vrrp-priority --vyos-mgmt-vlan --vlan --vyos-password --vyos-hwid --vyos-fresh-config --vyos-bundle --vyos-api-key --vyos-api-listen -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur"))
       return ;;
     "provision reprovision")
-      COMPREPLY=($(compgen -W "--role --os --disk --user -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur"))
       return ;;
     "provision debug")
       COMPREPLY=($(compgen -W "--pxe-boot -h --help" -- "$cur"))
@@ -107,7 +101,7 @@ _labctl() {
       COMPREPLY=($(compgen -W "deploy status -h --help" -- "$cur"))
       return ;;
     "app k3s")
-      COMPREPLY=($(compgen -W "evacuate admit install health list kubeconfig -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "install health list kubeconfig -h --help" -- "$cur"))
       return ;;
     "version")
       COMPREPLY=($(compgen -W "-h --help" -- "$cur"))

bastion/completions/labctl.fish

@@ -131,39 +131,20 @@ complete -c labctl -n "__labctl_using_cmd provision" -a makeiso -d 'Generate a U
 complete -c labctl -n "__labctl_using_cmd provision" -a recheck -d 'Refresh hardware info for all installed machines via SSH'
 
 # provision install options
-complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo'
-complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling'
-complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target OS disk: kernel name or stable /dev/disk/by-id/ path (auto-detect if omitted)' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l schema-version -d 'Disk layout for a FRESH install (default 1). Ignored on reprovision: the layout already on the disk wins.' -xa '1 2'
-complete -c labctl -n "__labctl_in_cmd provision install" -l longhorn-disk -d 'Dedicated whole disk for /var/lib/longhorn, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l rancher-disk -d 'Dedicated whole disk for /var/lib/rancher, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt -d 'VyOS: untagged interface the machine PXE boots from (default eth0)' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-address -d 'VyOS: CIDR for the management interface, or \'dhcp\' (default dhcp)' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond -d 'VyOS: comma-separated LACP bond members (must exclude the PXE NIC)' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond-address -d 'VyOS: address on the untagged bond (trunk native VLAN)' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond-vrrp -d 'VyOS: VRRP VIP floated on the untagged bond' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l vlan-vip -d 'VyOS: VRRP VIP for a --vlan entry (repeatable)' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-vrrp-priority -d 'VyOS: VRRP priority for all groups on this box (higher = master)' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-vlan -d 'VyOS: tagged management VLAN on the PXE port' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l vlan -d 'VyOS: tagged VLAN sub-interface on the bond (repeatable)' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-password -d 'VyOS: password for the \'vyos\' user' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-hwid -d 'VyOS: pin an interface name to a MAC via hw-id (repeatable)' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-fresh-config -d 'VyOS: on reinstall, overwrite the preserved config with the generated one'
-complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bundle -d 'VyOS: apply a Pulumi-rendered bundle verbatim (kubernetes-deployment/infra/vyos/vyos-bundle.json). Replaces the derived --vyos-bond/--vlan/... config; secret values are left unset for `pulumi up`.' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-api-key -d 'VyOS: enable the HTTP API with this key so Pulumi can manage the router from first boot' -x
-complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-api-listen -d 'VyOS: address the HTTP API binds to (default: the static management address). Required when management is DHCP; the API is never bound to all interfaces.' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
+complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04'
+complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target disk device (auto-detect if omitted)' -x
 
 # provision reprovision options
-complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo'
-complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling'
+complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
+complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04'
 complete -c labctl -n "__labctl_in_cmd provision reprovision" -l disk -d 'Target disk device (auto-detect if omitted)' -x
-complete -c labctl -n "__labctl_in_cmd provision reprovision" -l user -d 'SSH user for the reboot (default: vyos for VyOS machines, else current user)' -x
 
 # provision debug options
 complete -c labctl -n "__labctl_in_cmd provision debug" -l pxe-boot -d 'Boot installed system via PXE (kernel+initrd from network, root from NVMe)'
 
 # provision register options
-complete -c labctl -n "__labctl_in_cmd provision register" -l role -d 'Machine role' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo'
+complete -c labctl -n "__labctl_in_cmd provision register" -l role -d 'Machine role' -xa 'vanilla worker infra labcontroller'
 complete -c labctl -n "__labctl_in_cmd provision register" -l ip -d 'Machine IP address' -x
 
 # provision logs options
@@ -206,26 +187,16 @@ complete -c labctl -n "__labctl_in_cmd app labcontroller deploy" -l crdb-replica
 complete -c labctl -n "__labctl_in_cmd app labcontroller status" -l user -d 'SSH user' -x
 
 # app k3s subcommands
-complete -c labctl -n "__labctl_using_cmd app k3s" -a evacuate -d 'Cordon, evict Longhorn replicas and drain so a node can be safely wiped'
-complete -c labctl -n "__labctl_using_cmd app k3s" -a admit -d 'Return an evacuated node to service: uncordon, clear eviction, restore its Longhorn disk'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a install -d 'Install k3s on a target machine (hostname, IP, or MAC)'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a health -d 'Check k3s health (all hosts if no target given)'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a list -d 'List installed machines and their k3s status'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a kubeconfig -d 'Fetch kubeconfig from a target and merge into ~/.kube/config'
 
-# app k3s evacuate options
-complete -c labctl -n "__labctl_in_cmd app k3s evacuate" -l user -d 'SSH user' -x
-complete -c labctl -n "__labctl_in_cmd app k3s evacuate" -l allow-single-replica -d 'permit evacuating volumes that have only one replica (rebuilds from the only copy; there is no backup target)'
-
-# app k3s admit options
-complete -c labctl -n "__labctl_in_cmd app k3s admit" -l user -d 'SSH user' -x
-
 # app k3s install options
-complete -c labctl -n "__labctl_in_cmd app k3s install" -l role -d 'k3s role: infra, labcontroller, standalone-cluster, k8s-controller, k8s-combo (servers) or worker, k8s-worker (agents)' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l role -d 'k3s role: infra (server) or worker (agent)' -x
 complete -c labctl -n "__labctl_in_cmd app k3s install" -l user -d 'SSH user' -x
-complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-server -d 'k3s server URL (required for agent roles, and to JOIN as a server)' -x
-complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-token -d 'k3s join token; on a first server it SEEDS the cluster token' -x
-complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-version -d 'exact k3s version, e.g. v1.34.6+k3s1. Unset installs LATEST, which on a rejoining node can be several minor versions ahead of the cluster' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-server -d 'k3s server URL (required for worker role)' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-token -d 'k3s join token (required for worker role)' -x
 
 # app k3s health options
 complete -c labctl -n "__labctl_in_cmd app k3s health" -l user -d 'SSH user' -x

bastion/docs/ARCHITECTURE.md

@@ -89,6 +89,83 @@ Side paths:
 
 ---
 
+## Multi-architecture PXE
+
+The bastion serves both `x86_64` and `aarch64` over the network. Nothing about this is
+operator-configured -- there is no `--arch` flag, by design.
+
+### How a client's architecture is decided
+
+1. **DHCP option 93** (Client System Architecture) picks the *bootloader*. dnsmasq matches
+   it and hands out a matching iPXE binary:
+
+   | Option 93 | Client | Served |
+   |---|---|---|
+   | `0` | x86 BIOS | `undionly.kpxe` (TFTP) |
+   | `7`, `9` | x64 UEFI | `ipxe.efi` (TFTP) |
+   | `11` | **ARM64 UEFI** | `ipxe-arm64.efi` (TFTP) |
+   | `16` | x64 UEFI HTTP Boot | `http://…/ipxe.efi` |
+   | `19` | **ARM64 UEFI HTTP Boot** | `http://…/ipxe-arm64.efi` |
+
+   Values come from the IANA Processor Architecture Types registry. Note `19`, not `20` --
+   `20` is *pc/at bios boot from http*. EDK2/AAVMF prefers HTTP Boot over TFTP PXE, so the
+   iPXE binaries are staged in **both** `tftpDir` and `httpDir` (symlinked by `main.ts`).
+
+2. **`/dispatch` picks the kernel.** Option 93 never reaches the HTTP endpoint, so
+   `boot.ipxe` passes iPXE's own `${buildarch}` as `?arch=`. `resolveArch()` prefers, in
+   order: the tracked machine record → the reported `?arch=` → the configured default.
+   The record wins because it is what we observed on the machine itself.
+
+### Artifact naming
+
+`x86_64` keeps the original unsuffixed paths so its rendered iPXE scripts are unchanged;
+everything else is suffixed. `kernelPath()` / `initrdPath()` in `templates/boot.ipxe.ts`
+are the single source of truth, used by both the templates and `main.ts` staging.
+
+| arch | kernel | initrd |
+|---|---|---|
+| `x86_64` | `/vmlinuz` | `/initrd.img` |
+| `aarch64` | `/vmlinuz-aarch64` | `/initrd-aarch64.img` |
+
+`tests/ipxe-x86-regression.test.ts` pins the x86_64 output against a golden fixture.
+
+### arm64 gotchas
+
+- **LoadFile2 is mandatory.** arm64 has no `HdrS` boot protocol; the kernel's EFI stub
+  fetches the initrd over the UEFI `EFI_LOAD_FILE2_PROTOCOL`. An iPXE build without it
+  accepts the `initrd` line, silently drops it, and the kernel panics with
+  `VFS: Unable to mount root fs on unknown-block(0,0)`. Fedora's
+  `ipxe-bootimgs-aarch64` implements it; the integration test asserts this up front so
+  the failure names itself instead of looking like a disk problem.
+- **`nomodeset` is x86-only.** On arm64 there is no VGA path to fall back to. aarch64 gets
+  `console=tty0 console=ttyAMA0,115200` instead — the last `console=` wins for
+  `/dev/console`, so serial is the interactive one.
+- **Ubuntu is x86_64-only.** `releases.ubuntu.com` publishes no arm64 netboot artifacts.
+  `osSupportsArch()` encodes this, and both the install guard and `/dispatch` refuse the
+  combination rather than serving an x86 kernel to an ARM machine.
+
+---
+
+## Onboarding classification (vendor OS)
+
+Machines carry an `onboard` field: `"pxe"` (default) or `"ssh"`, plus `vendor_os` naming
+what they run. `classifyOnboard()` in `@lab/shared` sets it from DMI identity, with known
+hardware also matched by MAC — a machine can sit in state for a long time with no DMI, and
+a DMI-only rule would fail open exactly where it matters.
+
+`onboard: "ssh"` means *we cannot rebuild this machine's OS*. Installs are refused at both
+entry points (`/api/install` and the labd `command-install` handler) with an error naming
+the machine and pointing at `provision debug`. **Rescue is never guarded** — being unable
+to reinstall a machine is precisely when a rescue shell is needed.
+
+This is a fact about the machine, not a blocklist. The refusal follows from "no image in
+our pipeline restores `vendor_os`", so adding a DGX OS image to the pipeline is what
+unblocks the DGX Sparks — no entry needs deleting.
+
+Current classifications: NVIDIA DGX Spark (`spark-2935`, `spark-3a1c`) → `dgx-os`.
+
+---
+
 ## Packages
 
 ### Monorepo Structure
@@ -404,6 +481,36 @@ Hardcoded `/dev/sda` default broke NVMe-only machines. Fix: default to empty str
 ### Anaconda Rescue Mode Limitations
 `%pre` and `%post` sections do not execute in `inst.rescue` mode. SSH in rescue mode is provided by Anaconda's `inst.sshd` kernel parameter + `sshpw` kickstart directive. Manual setup via `curl bastion:8080/debug-setup.sh | bash` for nc listener.
 
+**Unresolved (2026-08-11): rescue SSH has never been observed working.** Adding the first
+integration coverage for `provision debug` (`tests/integration/pxe-rescue.test.ts`) showed the
+rescue environment coming up correctly — the bastion serves the kernel and initrd, Anaconda
+boots, fetches `debug.ks`, and reaches its installer environment — but **nothing ever listens on
+port 22**.
+
+Strength of the evidence, stated precisely because it decides where to look next:
+- **aarch64 — direct.** Port 22 probed every 20s for 30 minutes while the Anaconda installer
+  environment was demonstrably running (NetworkManager, polkitd, rsyslog on the console). Never
+  opened.
+- **x86_64 — corroborating, not conclusive.** One clean KVM run (943s) where SSH never became
+  available inside a 15-minute budget. That VM's progress into the rescue environment was *not*
+  observed — vitest's final reporter discards the streamed log — so it is consistent with the
+  aarch64 result but does not independently prove it. Re-run with `KEEP_VM=1` and probe port 22
+  directly to settle it.
+
+If the x86_64 result holds up, this is orthogonal to the multi-architecture work, since x86_64 is
+untouched by it. Leads worth checking, in order:
+- Does `inst.sshd` actually start `sshd` in `inst.rescue` mode, or only in install mode? The
+  port never opens, so this is the prime suspect — an auth problem would still show an open port.
+- `sshkey` may apply only to the *installed* system, leaving the installer environment
+  password-only via `sshpw`. That would matter once sshd does listen: the test authenticates
+  key-only (`BatchMode=yes`).
+- The `%anaconda`-context directives in `debug.ks` may be skipped entirely when a kickstart is
+  supplied alongside `inst.rescue`.
+
+Until this is resolved, `provision debug` gets you a booted rescue environment on the console
+(including on arm64), but not an SSH shell. The `debug-setup.sh` nc-listener path is the
+documented workaround and is unaffected.
+
 ---
 
 ## Planned Work (Taskmaster)

bastion/package.json

@@ -21,10 +21,14 @@
     "test:integration:pxe:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'PXE boot'",
     "test:integration:iso": "vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'",
     "test:integration:iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'",
-    "test:integration:vyos": "vitest run -c tests/integration/vitest.config.ts -t 'VyOS provisioning'",
-    "test:integration:vyos:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'VyOS provisioning'",
     "test:integration:arm-iso": "vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'",
     "test:integration:arm-iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'",
+    "test:integration:rescue": "vitest run -c tests/integration/vitest.config.ts -t 'x86 rescue boot'",
+    "test:integration:rescue:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'x86 rescue boot'",
+    "test:integration:arm-pxe": "vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE rescue'",
+    "test:integration:arm-pxe:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE rescue'",
+    "test:integration:arm-pxe-full": "ARM_PXE_FULL=1 vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE'",
+    "test:integration:arm-pxe-full:host": "sudo -E ARM_PXE_FULL=1 $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE'",
     "test:integration:asahi": "vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'",
     "test:integration:asahi:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'",
     "test:integration:asahi-validate": "vitest run -c tests/integration/vitest.config.ts -t 'asahi.*validation'",

bastion/scripts/test-provision.sh

@@ -2,16 +2,19 @@
 # Run PXE and/or ISO boot integration tests.
 #
 # Usage:
-#   sudo ./scripts/test-provision.sh          # run PXE + ISO (x86_64)
-#   sudo ./scripts/test-provision.sh pxe      # PXE only
-#   sudo ./scripts/test-provision.sh iso      # ISO only (x86_64)
-#   sudo ./scripts/test-provision.sh arm      # ARM ISO boot (emulated, SLOW ~60min)
-#   sudo ./scripts/test-provision.sh all      # all tests including ARM
+#   sudo ./scripts/test-provision.sh              # run PXE + ISO (x86_64)
+#   sudo ./scripts/test-provision.sh pxe          # PXE only
+#   sudo ./scripts/test-provision.sh iso          # ISO only (x86_64)
+#   sudo ./scripts/test-provision.sh rescue       # x86_64 Anaconda rescue boot + SSH (~15min)
+#   sudo ./scripts/test-provision.sh arm          # ARM ISO boot (emulated, SLOW ~60min)
+#   sudo ./scripts/test-provision.sh arm-pxe      # ARM network PXE rescue: NBP + rescue over SSH (~25-30min)
+#   sudo ./scripts/test-provision.sh arm-pxe-full # ARM network PXE incl. discover + full install (~75-95min)
+#   sudo ./scripts/test-provision.sh all          # all tests including ARM
 #
 # Prerequisites:
 #   libvirtd, OVMF (edk2-ovmf), iPXE (ipxe-bootimgs-x86),
 #   dnsmasq, xorriso, mtools, virt-install, qemu-img
-#   ARM: qemu-system-aarch64, edk2-aarch64
+#   ARM: qemu-system-aarch64, edk2-aarch64, ipxe-bootimgs-aarch64
 set -e
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
@@ -58,6 +61,10 @@ if [ ! -f /usr/share/edk2/ovmf/OVMF_CODE.fd ]; then
     exit 1
 fi
 
+MODE="${1:-both}"
+
+# iPXE binaries are per-architecture. x86_64 is always required (the dnsmasq config
+# references it); arm64 only for the ARM network-PXE modes.
 IPXE_EFI=""
 for f in /usr/share/ipxe/ipxe-snponly-x86_64.efi /usr/share/ipxe/ipxe-snp-x86_64.efi /usr/share/ipxe/ipxe-x86_64.efi; do
     [ -f "$f" ] && IPXE_EFI="$f" && break
@@ -67,6 +74,20 @@ if [ -z "$IPXE_EFI" ]; then
     exit 1
 fi
 
+IPXE_EFI_ARM64=""
+for f in /usr/share/ipxe/arm64-efi/snponly.efi /usr/share/ipxe/arm64-efi/ipxe.efi; do
+    [ -f "$f" ] && IPXE_EFI_ARM64="$f" && break
+done
+
+case "$MODE" in
+    arm-pxe|arm-pxe-full|all)
+        if [ -z "$IPXE_EFI_ARM64" ] && [ "$MODE" != "all" ]; then
+            echo -e "${RED}arm64 iPXE binary not found.${RESET} Install: sudo dnf install ipxe-bootimgs-aarch64"
+            exit 1
+        fi
+        ;;
+esac
+
 # Find SSH key
 SSH_KEY=""
 for name in id_ed25519 id_ecdsa id_rsa; do
@@ -83,10 +104,19 @@ fi
 echo -e "  User:    ${BOLD}$REAL_USER${RESET}"
 echo -e "  SSH key: ${BOLD}$SSH_KEY${RESET}"
 echo -e "  iPXE:    ${BOLD}$IPXE_EFI${RESET}"
+echo -e "  iPXE a64:${BOLD} ${IPXE_EFI_ARM64:-not installed}${RESET}"
 echo ""
 
-# --- Determine which tests to run ---
-MODE="${1:-both}"
+require_arm_emulation() {
+    if ! command -v qemu-system-aarch64 &>/dev/null; then
+        echo -e "${RED}qemu-system-aarch64 not found.${RESET} Install: sudo dnf install qemu-system-aarch64 edk2-aarch64"
+        exit 1
+    fi
+    if [ ! -f /usr/share/edk2/aarch64/QEMU_EFI.fd ]; then
+        echo -e "${RED}AAVMF firmware not found.${RESET} Install: sudo dnf install edk2-aarch64"
+        exit 1
+    fi
+}
 
 run_test() {
     local name="$1" pattern="$2"
@@ -116,13 +146,26 @@ case "$MODE" in
         run_test "ISO boot" "ISO boot" || FAILED=1
         ;;
     arm|arm-iso)
-        if ! command -v qemu-system-aarch64 &>/dev/null; then
-            echo -e "${RED}qemu-system-aarch64 not found.${RESET} Install: sudo dnf install qemu-system-aarch64 edk2-aarch64"
-            exit 1
-        fi
+        require_arm_emulation
         echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 30-60 minutes.${RESET}"
         run_test "ARM ISO boot" "ARM ISO" || FAILED=1
         ;;
+    rescue)
+        echo -e "${YELLOW}x86_64 rescue boot (KVM). Expect ~15 minutes.${RESET}"
+        run_test "x86 rescue boot" "x86 rescue boot" || FAILED=1
+        ;;
+    arm-pxe)
+        require_arm_emulation
+        echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 25-30 minutes.${RESET}"
+        echo -e "${YELLOW}Covers option 93 -> arm64 NBP, arch resolution, and rescue over SSH.${RESET}"
+        echo -e "${YELLOW}For the full install too, use: $0 arm-pxe-full${RESET}"
+        run_test "ARM PXE rescue" "ARM PXE rescue" || FAILED=1
+        ;;
+    arm-pxe-full)
+        require_arm_emulation
+        echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 75-95 minutes.${RESET}"
+        ARM_PXE_FULL=1 run_test "ARM PXE (rescue + install)" "ARM PXE" || FAILED=1
+        ;;
     both)
         run_test "PXE boot" "PXE boot" || FAILED=1
         run_test "ISO boot" "ISO boot" || FAILED=1
@@ -133,12 +176,17 @@ case "$MODE" in
         if command -v qemu-system-aarch64 &>/dev/null; then
             echo -e "${YELLOW}ARM emulation is ~10x slower than native.${RESET}"
             run_test "ARM ISO boot" "ARM ISO" || FAILED=1
+            if [ -n "$IPXE_EFI_ARM64" ]; then
+                run_test "ARM PXE rescue" "ARM PXE rescue" || FAILED=1
+            else
+                echo -e "${YELLOW}Skipping ARM PXE test (ipxe-bootimgs-aarch64 not installed)${RESET}"
+            fi
         else
-            echo -e "${YELLOW}Skipping ARM test (qemu-system-aarch64 not installed)${RESET}"
+            echo -e "${YELLOW}Skipping ARM tests (qemu-system-aarch64 not installed)${RESET}"
         fi
         ;;
     *)
-        echo "Usage: $0 [pxe|iso|arm|both|all]"
+        echo "Usage: $0 [pxe|iso|rescue|arm|arm-pxe|arm-pxe-full|both|all]"
         exit 1
         ;;
 esac

bastion/src/bastion/src/config.ts

@@ -20,22 +20,7 @@ export function loadConfig(overrides: Partial<BastionConfig> = {}): BastionConfi
   const ubuntuMirror = overrides.ubuntuMirror ?? process.env["UBUNTU_MIRROR"]
     ?? `https://releases.ubuntu.com/${ubuntuVersion}`;
 
-  // "latest" resolves the newest nightly ISO from the vyos-nightly-build GitHub
-  // releases at startup. downloads.vyos.io no longer serves direct rolling ISOs
-  // (it returns the vyos.io site, and nightly builds sit behind a signup form);
-  // GitHub releases are the remaining free, unauthenticated direct source.
-  // LTS ISOs are subscription-only. Set VYOS_ISO_URL to pin a specific build.
-  const vyosIsoUrl = overrides.vyosIsoUrl ?? process.env["VYOS_ISO_URL"] ?? "latest";
-  const vyosDefaultPassword = overrides.vyosDefaultPassword
-    ?? process.env["VYOS_DEFAULT_PASSWORD"] ?? "vyos";
-
-  // Overridable like ubuntuMirror and vyosIsoUrl above -- it was the only one
-  // of the three hardcoded. Without this, every integration-test install pulls
-  // the full package set from download.fedoraproject.org, which makes a
-  // multi-VM provisioning matrix impractical (~25 min per install, serialised).
-  // Point FEDORA_MIRROR at a local mirror or caching proxy for test runs.
-  const fedoraMirror = overrides.fedoraMirror ?? process.env["FEDORA_MIRROR"]
-    ?? `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`;
+  const fedoraMirror = `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`;
   const tftpDir = `${bastionDir}/tftp`;
   const httpDir = `${bastionDir}/http`;
   const stateFile = `${bastionDir}/state.json`;
@@ -53,8 +38,6 @@ export function loadConfig(overrides: Partial<BastionConfig> = {}): BastionConfi
     dhcpRangeEnd,
     ubuntuVersion,
     ubuntuMirror,
-    vyosIsoUrl,
-    vyosDefaultPassword,
     // These are populated at runtime by the network service
     iface: overrides.iface ?? "",
     serverIp: overrides.serverIp ?? "",
@@ -65,7 +48,6 @@ export function loadConfig(overrides: Partial<BastionConfig> = {}): BastionConfi
     syslogPort,
     skipDnsmasq: overrides.skipDnsmasq,
     skipArtifacts: overrides.skipArtifacts,
-    autoK3s: overrides.autoK3s ?? process.env["AUTO_K3S"] !== "false",
     labdUrl: overrides.labdUrl ?? process.env["LABD_URL"],
     bastionJoinToken: overrides.bastionJoinToken ?? process.env["BASTION_JOIN_TOKEN"],
     fedoraMirror,

bastion/src/bastion/src/main.ts

@@ -3,8 +3,9 @@
 
 import { mkdirSync, writeFileSync, readFileSync, existsSync, copyFileSync, symlinkSync, unlinkSync } from "node:fs";
 import { execSync } from "node:child_process";
-import type { BastionConfig } from "@lab/shared";
-import { isRole, validateDataDisks } from "@lab/shared";
+import type { Arch, BastionConfig } from "@lab/shared";
+import { SUPPORTED_ARCHES, fedoraMirrorFor, classifyOnboard } from "@lab/shared";
+import { kernelPath, initrdPath } from "./templates/boot.ipxe.js";
 import { loadConfig } from "./config.js";
 import { populateNetworkConfig } from "./services/network.js";
 import { createApp } from "./server.js";
@@ -14,21 +15,19 @@ import { renderBootIpxe } from "./templates/boot.ipxe.js";
 import { logger } from "./services/logger.js";
 import { BastionConnection } from "./services/labd-connection.js";
 import { progressBus } from "./services/progress-events.js";
+import { checkInstallAllowed } from "./services/install-guard.js";
 import { ensureBootIso } from "./routes/boot-iso.js";
 
-// For binaries that ship WITH the image: the data dir is a persistent volume,
-// so a copy-if-missing would keep serving whatever build landed there first,
-// forever, across image upgrades. Compare content and replace on change.
-function copyIfChanged(src: string, dest: string, label: string): void {
+function copyIfMissing(src: string, dest: string, label: string): void {
+  if (existsSync(dest)) {
+    logger.info(`  ${label} -- cached`);
+    return;
+  }
   if (!existsSync(src)) {
     throw new Error(`${label}: source not found at ${src}`);
   }
-  if (existsSync(dest) && readFileSync(dest).equals(readFileSync(src))) {
-    logger.info(`  ${label} -- up to date`);
-    return;
-  }
   copyFileSync(src, dest);
-  logger.info(`  ${label} -- installed from ${src}`);
+  logger.info(`  ${label} -- copied from ${src}`);
 }
 
 function download(url: string, dest: string, label: string): void {
@@ -44,125 +43,6 @@ function download(url: string, dest: string, label: string): void {
   }
 }
 
-/**
- * Pick the largest regular-file initrd from an `xorriso -lsl` listing.
- *
- * /live carries decoys: a 0-byte initrd.img placeholder on some images, or an
- * initrd.img SYMLINK to the real version-suffixed file on others. Parsing is
- * field-based (ls -l layout: perms links uid gid size month day time 'name')
- * and considers only lines whose mode string marks a regular file — symlinks
- * report their link size, not the target's, and must not win.
- */
-export function pickLargestInitrd(
-  listing: string,
-): { name: string; size: number } | undefined {
-  let best: { name: string; size: number } | undefined;
-  for (const line of listing.split("\n")) {
-    if (!line.startsWith("-")) continue;          // regular files only
-    const quoted = /'([^']+)'/.exec(line);
-    const fields = line.trim().split(/\s+/);
-    const size = parseInt(fields[4] ?? "", 10);
-    const name = quoted?.[1] ?? "";
-    if (!name.startsWith("initrd")) continue;
-    if (!Number.isFinite(size) || size <= 0) continue;
-    if (best === undefined || size > best.size) {
-      best = { name, size };
-    }
-  }
-  return best;
-}
-
-const VYOS_NIGHTLY_RELEASES =
-  "https://api.github.com/repos/vyos/vyos-nightly-build/releases/latest";
-
-/**
- * Resolve the configured VyOS ISO URL, expanding the "latest" sentinel.
- *
- * The nightly asset filename embeds a build date, so there is no stable
- * "latest.iso" path to hardcode — the newest release has to be looked up.
- * Any other value is used verbatim, which is how VYOS_ISO_URL pins a build
- * or points at a locally mirrored copy.
- */
-function resolveVyosIsoUrl(configured: string): string {
-  if (configured !== "latest") return configured;
-
-  const body = execSync(`curl -sSfL "${VYOS_NIGHTLY_RELEASES}"`, {
-    encoding: "utf-8",
-    stdio: ["pipe", "pipe", "pipe"],
-  });
-  const release = JSON.parse(body) as {
-    tag_name?: string;
-    assets?: Array<{ name: string; browser_download_url: string }>;
-  };
-
-  const asset = (release.assets ?? []).find((a) =>
-    /generic-amd64\.iso$/.test(a.name),
-  );
-  if (!asset) {
-    throw new Error(
-      `No generic-amd64 ISO asset in VyOS nightly release ${release.tag_name ?? "?"}`,
-    );
-  }
-
-  logger.info(`  VyOS ISO resolved to ${asset.name} (${release.tag_name ?? "?"})`);
-  return asset.browser_download_url;
-}
-
-/**
- * Extract VyOS netboot artifacts from the release ISO.
- *
- * VyOS publishes no netboot bundle, so kernel/initrd/squashfs have to come out
- * of the ISO. xorriso is already in the bastion image (used for boot.iso) and
- * extracts without root or a loop mount.
- *
- * The initrd needs care: /live contains an empty initrd.img placeholder
- * alongside the real one, which carries a version-suffixed name. Booting the
- * 0-byte file fails with no useful diagnostic, so pick the largest initrd*.
- */
-export function prepareVyosArtifacts(config: BastionConfig): void {
-  const kernel = `${config.httpDir}/vyos-vmlinuz`;
-  const initrd = `${config.httpDir}/vyos-initrd`;
-  const squashfs = `${config.httpDir}/vyos-filesystem.squashfs`;
-
-  if (existsSync(kernel) && existsSync(initrd) && existsSync(squashfs)) {
-    logger.info("  VyOS netboot artifacts -- cached");
-    return;
-  }
-
-  const iso = `${config.bastionDir}/vyos.iso`;
-  download(resolveVyosIsoUrl(config.vyosIsoUrl), iso, "VyOS ISO");
-
-  const extract = (isoPath: string, dest: string, label: string): void => {
-    execSync(
-      `xorriso -osirrox on -indev "${iso}" -extract "${isoPath}" "${dest}"`,
-      { stdio: "pipe" },
-    );
-    logger.info(`  ${label} -- extracted from ${isoPath}`);
-  };
-
-  extract("/live/vmlinuz", kernel, "VyOS kernel");
-  extract("/live/filesystem.squashfs", squashfs, "VyOS squashfs");
-
-  // Pick the real initrd by size from the ISO's own directory listing.
-  const listing = execSync(`xorriso -indev "${iso}" -lsl /live/ --`, {
-    encoding: "utf-8",
-    stdio: ["pipe", "pipe", "pipe"],
-  });
-
-  const best = pickLargestInitrd(listing);
-  if (best === undefined) {
-    throw new Error("No non-empty initrd found in /live on the VyOS ISO");
-  }
-  extract(`/live/${best.name}`, initrd, `VyOS initrd (${best.name}, ${best.size} bytes)`);
-
-  // The ISO is only needed to produce the three artifacts above.
-  try {
-    unlinkSync(iso);
-  } catch {
-    // Non-fatal: leaving it costs disk but nothing else.
-  }
-}
-
 function symlinkSafe(target: string, linkPath: string): void {
   try {
     symlinkSync(target, linkPath);
@@ -253,43 +133,66 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
   mkdirSync(config.tftpDir, { recursive: true });
   mkdirSync(config.httpDir, { recursive: true });
 
+  // Architectures we can actually network boot, reported in the banner so a missing
+  // arm64 payload is visible at startup instead of at 2am when a rescue is needed.
+  const bootArches: Arch[] = [];
+  let ipxeArm64Ready = false;
+
   // Prepare boot artifacts
   if (config.skipArtifacts !== true) {
-    logger.info(`Preparing boot artifacts (Fedora ${config.fedoraVersion} ${config.arch})...`);
+    logger.info(`Preparing boot artifacts (Fedora ${config.fedoraVersion}, ${SUPPORTED_ARCHES.join(" + ")})...`);
 
-    copyIfChanged(
+    copyIfMissing(
       "/usr/share/ipxe/undionly.kpxe",
       `${config.tftpDir}/undionly.kpxe`,
       "iPXE BIOS",
     );
-    // Full build (native drivers + snp/nii fallback), NOT snponly: the
-    // firmware NII stack snponly rides cannot receive unicast on some NICs
-    // (thebeast 2026-10-19: DHCP fine, every ARP reply lost, HTTP timeout).
-    copyIfChanged(
-      "/usr/share/ipxe/ipxe-full-x86_64.efi",
+    copyIfMissing(
+      "/usr/share/ipxe/ipxe-snponly-x86_64.efi",
       `${config.tftpDir}/ipxe.efi`,
       "iPXE UEFI x86_64",
     );
     try {
-      copyIfChanged(
+      copyIfMissing(
         "/usr/share/ipxe/arm64-efi/snponly.efi",
         `${config.tftpDir}/ipxe-arm64.efi`,
         "iPXE UEFI arm64",
       );
+      ipxeArm64Ready = true;
     } catch {
-      logger.warn("arm64 iPXE not available -- skipping");
+      logger.warn("arm64 iPXE not available -- arm64 machines cannot network boot.");
+      logger.warn("  Install with: sudo dnf install ipxe-bootimgs-aarch64");
     }
 
-    download(
-      `${config.fedoraMirror}/images/pxeboot/vmlinuz`,
-      `${config.httpDir}/vmlinuz`,
-      "Fedora kernel",
-    );
-    download(
-      `${config.fedoraMirror}/images/pxeboot/initrd.img`,
-      `${config.httpDir}/initrd.img`,
-      "Fedora initrd",
-    );
+    // Fedora pxeboot kernel + initrd per architecture. x86_64 keeps the unsuffixed
+    // names it has always used; other architectures are suffixed. The iPXE templates
+    // resolve the same paths via kernelPath()/initrdPath().
+    for (const arch of SUPPORTED_ARCHES) {
+      const mirror = fedoraMirrorFor(config.fedoraVersion, arch);
+      try {
+        download(
+          `${mirror}/images/pxeboot/vmlinuz`,
+          `${config.httpDir}${kernelPath(arch)}`,
+          `Fedora ${arch} kernel`,
+        );
+        download(
+          `${mirror}/images/pxeboot/initrd.img`,
+          `${config.httpDir}${initrdPath(arch)}`,
+          `Fedora ${arch} initrd`,
+        );
+        bootArches.push(arch);
+      } catch (err) {
+        // Non-fatal: a bastion with no arm64 artifacts still serves x86_64 fine.
+        // Failing startup over an unreachable mirror for an architecture that may not
+        // even be present on this network would be worse.
+        logger.warn(`Fedora ${arch} kernel/initrd unavailable -- ${arch} PXE disabled`);
+        logger.warn(`  ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+
+    if (!bootArches.includes("x86_64")) {
+      throw new Error("Fedora x86_64 kernel/initrd could not be staged -- cannot serve PXE");
+    }
 
     // Ubuntu netboot artifacts (non-fatal — Ubuntu version may not be released yet)
     try {
@@ -308,17 +211,6 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
       logger.warn(`Ubuntu ${config.ubuntuVersion} artifacts not available -- Ubuntu provisioning disabled`);
     }
 
-    // VyOS netboot artifacts (non-fatal — same policy as Ubuntu)
-    try {
-      logger.info("Preparing VyOS netboot artifacts...");
-      prepareVyosArtifacts(config);
-    } catch (err) {
-      logger.warn(
-        `VyOS artifacts not available -- VyOS provisioning disabled ` +
-        `(${err instanceof Error ? err.message : String(err)})`,
-      );
-    }
-
     // Symlink iPXE binaries into HTTP dir for UEFI HTTP Boot
     for (const name of ["ipxe.efi", "ipxe-arm64.efi"]) {
       const src = `${config.tftpDir}/${name}`;
@@ -391,21 +283,12 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
     // Wire up command handlers so labd can send install/forget/role commands
     labdConn.onCommand("command-install", async (msg) => {
       if (msg.type !== "command-install") throw new Error("unexpected");
-      // Same guard as POST /api/install: disks named by an operator must
-      // never silently degrade into LVs because they arrived over the socket.
-      const hasDataDisks = msg.dataDisks !== undefined
-        && Object.values(msg.dataDisks).some((d) => d !== undefined);
-      if (hasDataDisks) {
-        if ((msg.os ?? "fedora-43") !== "fedora-43") {
-          return { status: "error", error: `dataDisks are only supported for fedora installs (got os '${msg.os}')` };
-        }
-        if (!isRole(msg.role)) {
-          return { status: "error", error: `invalid role: '${msg.role}'` };
-        }
-        const problem = validateDataDisks(msg.role, msg.dataDisks!);
-        if (problem !== null) {
-          return { status: "error", error: `invalid dataDisks: ${problem}` };
-        }
+      const installMac = msg.mac.toLowerCase().replace(/-/g, ":");
+      const osId = (msg.os as import("@lab/shared").OsId | undefined) ?? "fedora-43";
+      const check = checkInstallAllowed(state.load(), installMac, osId);
+      if (check.allowed === false) {
+        logger.warn(`INSTALL REFUSED: ${installMac} -- ${check.error}`);
+        return { status: "error", error: check.error };
       }
       state.update((s) => {
         s.install_queue[msg.mac] = {
@@ -414,9 +297,6 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
           role: msg.role as import("@lab/shared").Role,
           os: msg.os as import("@lab/shared").OsId,
           queued_at: new Date().toISOString(),
-          ...(msg.schemaVersion ? { schemaVersion: msg.schemaVersion } : {}),
-          ...(hasDataDisks ? { dataDisks: msg.dataDisks } : {}),
-          ...(msg.vyos ? { vyos: msg.vyos } : {}),
         };
       });
       return { status: "ok", data: { mac: msg.mac, hostname: msg.hostname } };
@@ -470,13 +350,24 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
       const mac = (msg.mac as string).toLowerCase();
       const now = new Date().toISOString();
       const existing = state.load().discovered[mac];
+      const identity = {
+        mac,
+        manufacturer: (msg.manufacturer as string) ?? "unknown",
+        product: (msg.product as string) ?? "unknown",
+        board: (msg.board as string) ?? "unknown",
+        ...(existing?.onboard !== undefined ? { onboard: existing.onboard } : {}),
+        ...(existing?.vendor_os !== undefined ? { vendor_os: existing.vendor_os } : {}),
+      };
+      const onboarding = classifyOnboard(identity);
+      const rootDevice = msg.root_device ?? existing?.root_device;
+      const rootArgs = msg.root_args ?? existing?.root_args;
       state.update((s) => {
         s.discovered[mac] = {
           mac,
-          product: (msg.product as string) ?? "unknown",
-          board: (msg.board as string) ?? "unknown",
+          product: identity.product,
+          board: identity.board,
           serial: (msg.serial as string) ?? "unknown",
-          manufacturer: (msg.manufacturer as string) ?? "unknown",
+          manufacturer: identity.manufacturer,
           cpu_model: (msg.cpu_model as string) ?? "unknown",
           cpu_cores: (msg.cpu_cores as number) ?? 0,
           memory_gb: (msg.memory_gb as number) ?? 0,
@@ -485,7 +376,20 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
           nics: (msg.nics as Array<{ name: string; mac: string; state: string }>) ?? [],
           first_seen: existing?.first_seen ?? now,
           last_seen: now,
+          onboard: onboarding.onboard,
+          ...(onboarding.vendor_os !== undefined ? { vendor_os: onboarding.vendor_os } : {}),
+          ...(rootDevice !== undefined ? { root_device: rootDevice } : {}),
+          ...(rootArgs !== undefined ? { root_args: rootArgs } : {}),
         };
+        // Keep the installed record in step -- the guard and --pxe-boot both read it.
+        const inst = s.installed[mac];
+        if (inst) {
+          inst.arch = (msg.arch as string) ?? inst.arch;
+          inst.onboard = onboarding.onboard;
+          if (onboarding.vendor_os !== undefined) inst.vendor_os = onboarding.vendor_os;
+          if (rootDevice !== undefined) inst.root_device = rootDevice;
+          if (rootArgs !== undefined) inst.root_args = rootArgs;
+        }
       });
       logger.info(`HARDWARE UPDATED: ${mac} -- ${msg.manufacturer ?? "?"} ${msg.product ?? "?"} (${msg.cpu_model ?? "?"}, ${msg.cpu_cores ?? "?"} cores, ${msg.memory_gb ?? "?"}GB RAM)`);
       return { status: "ok", data: { mac } };
@@ -520,7 +424,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
   }
 
   // Print banner
-  printBanner(config);
+  printBanner(config, bootArches, ipxeArm64Ready);
 
   // Graceful shutdown
   const shutdown = async (): Promise<void> => {
@@ -542,11 +446,22 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
   await new Promise(() => {});
 }
 
-function printBanner(config: BastionConfig): void {
+function printBanner(config: BastionConfig, bootArches: Arch[], ipxeArm64Ready: boolean): void {
   const dhcpInfo = config.dhcpMode === "full"
     ? `full (${config.dhcpRangeStart}-${config.dhcpRangeEnd})`
     : "proxy (alongside existing DHCP)";
 
+  // arm64 needs both an iPXE binary (DHCP hands it out on option 93 = 0x0b) and a
+  // kernel/initrd pair. Report the combination, since either missing breaks it.
+  const archInfo = config.skipArtifacts === true
+    ? "(artifacts skipped)"
+    : SUPPORTED_ARCHES
+        .map((a) => {
+          const ready = bootArches.includes(a) && (a !== "aarch64" || ipxeArm64Ready);
+          return ready ? a : `${a} (unavailable)`;
+        })
+        .join(", ");
+
   console.log("");
   console.log("\x1b[36m\x1b[1m" + "=".repeat(60) + "\x1b[0m");
   console.log("\x1b[36m\x1b[1m  Lab PXE Bastion -- Discovery Mode\x1b[0m");
@@ -555,7 +470,8 @@ function printBanner(config: BastionConfig): void {
   console.log(`  Network:   \x1b[1m${config.network}/24\x1b[0m via \x1b[1m${config.iface}\x1b[0m`);
   console.log(`  DHCP:      \x1b[1m${dhcpInfo}\x1b[0m`);
   console.log(`  HTTP:      \x1b[1mhttp://${config.serverIp}:${config.httpPort}/\x1b[0m`);
-  console.log(`  OS:        \x1b[1mFedora ${config.fedoraVersion} (${config.arch})\x1b[0m`);
+  console.log(`  OS:        \x1b[1mFedora ${config.fedoraVersion}\x1b[0m`);
+  console.log(`  Net boot:  \x1b[1m${archInfo}\x1b[0m`);
   console.log(`  Domain:    \x1b[1m${config.domain}\x1b[0m`);
   console.log(`  State:     \x1b[1m${config.stateFile}\x1b[0m`);
   console.log("");

bastion/src/bastion/src/routes/api.ts

@@ -5,27 +5,19 @@
 // /api/discover  - receive hardware discovery reports from PXE-booted machines
 
 import type { FastifyInstance } from "fastify";
-import type { BastionConfig, HardwareInfo, InstalledInfo, Role, VyosInstallSpec } from "@lab/shared";
-import { isValidOsId, SUPPORTED_ROLES, SUPPORTED_OS } from "@lab/shared";
+import type { HardwareInfo, InstalledInfo, Role } from "@lab/shared";
+import { isValidOsId, SUPPORTED_ROLES, classifyOnboard } from "@lab/shared";
 import type { StateManager } from "../services/state.js";
 import { logger } from "../services/logger.js";
 import { triggerPostProvisionK3s } from "../services/post-provision.js";
+import { checkInstallAllowed } from "../services/install-guard.js";
 import { progressBus } from "../services/progress-events.js";
 import type { ProgressEvent } from "../services/progress-events.js";
 import type { InstallLogBuffer } from "../services/install-log.js";
 import type { SyslogListener } from "../services/syslog-listener.js";
-import { isRole, requiresSchemaV2, validateDataDisks, type DataDisks } from "@lab/shared";
-
-/**
- * Seconds after dispatch with zero progress before a machine is called stalled.
- * Generous: the slowest legitimate gap is fetching a ~600MB VyOS squashfs over
- * HTTP before the hook can report anything.
- */
-const STALL_THRESHOLD_S = 8 * 60;
 
 export function registerApiRoutes(
   app: FastifyInstance,
-  config: BastionConfig,
   state: StateManager,
   installLog: InstallLogBuffer,
   syslog: SyslogListener,
@@ -43,12 +35,9 @@ export function registerApiRoutes(
       disk?: string;
       role?: string;
       os?: string;
-      vyos?: VyosInstallSpec;
-      schemaVersion?: number;
-      dataDisks?: DataDisks;
     };
   }>("/api/install", async (request, reply) => {
-    const { mac: rawMac, hostname, disk, role, os, vyos, schemaVersion, dataDisks } = request.body ?? {};
+    const { mac: rawMac, hostname, disk, role, os } = request.body ?? {};
     const mac = (rawMac ?? "").toLowerCase().replace(/-/g, ":");
 
     if (mac === "") {
@@ -60,54 +49,15 @@ export function registerApiRoutes(
       return reply.status(400).send({ error: `invalid role: '${validRole}'. Supported: ${SUPPORTED_ROLES.join(", ")}` });
     }
 
-    // Reject an unknown schema rather than silently installing v1: someone
-    // asking for v3 wants a layout we do not have, and quietly giving them the
-    // old one is how a fleet ends up with layouts nobody chose.
-    if (schemaVersion !== undefined && schemaVersion !== 1 && schemaVersion !== 2) {
-      return reply.status(400).send({ error: `invalid schemaVersion: ${schemaVersion}. Supported: 1, 2` });
-    }
-
-    // Only an EXPLICIT v1 is a contradiction. Omitting the schema is normal --
-    // a reinstall never passes one, because the layout already on the disk wins.
-    if (schemaVersion === 1 && isRole(validRole) && requiresSchemaV2(validRole)) {
-      return reply.status(400).send({
-        error: `role '${validRole}' requires schemaVersion 2 (it has no v1 disk layout)`,
-      });
-    }
-
     const osId = os ?? "fedora-43";
     if (!isValidOsId(osId)) {
-      return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: ${SUPPORTED_OS.join(", ")}` });
+      return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: fedora-43, ubuntu-26.04` });
     }
 
-    // Dedicated data disks are rendered by the kickstart generator only, and
-    // only in the v2 layout. Rejected here rather than silently dropped at
-    // render time -- an operator who named disks must not get LVs instead.
-    const hasDataDisks = dataDisks !== undefined
-      && Object.values(dataDisks).some((d) => d !== undefined);
-    if (hasDataDisks) {
-      if (osId !== "fedora-43") {
-        return reply.status(400).send({
-          error: `dataDisks are only supported for fedora installs (got os '${osId}')`,
-        });
-      }
-      if (schemaVersion === 1) {
-        return reply.status(400).send({
-          error: "dataDisks require schemaVersion 2 -- the v1 layout is frozen and single-disk",
-        });
-      }
-      if (!isRole(validRole)) {
-        return reply.status(400).send({ error: `invalid role: '${validRole}'` });
-      }
-      if (!requiresSchemaV2(validRole) && schemaVersion === undefined) {
-        return reply.status(400).send({
-          error: `role '${validRole}' defaults to the v1 layout -- pass schemaVersion 2 to use dataDisks`,
-        });
-      }
-      const problem = validateDataDisks(validRole, dataDisks as DataDisks);
-      if (problem !== null) {
-        return reply.status(400).send({ error: `invalid dataDisks: ${problem}` });
-      }
+    const check = checkInstallAllowed(state.load(), mac, osId);
+    if (check.allowed === false) {
+      logger.warn(`INSTALL REFUSED: ${mac} -- ${check.error}`);
+      return reply.status(409).send({ error: check.error });
     }
 
     state.update((s) => {
@@ -117,9 +67,6 @@ export function registerApiRoutes(
         role: validRole as Role,
         os: osId,
         queued_at: new Date().toISOString(),
-        ...(schemaVersion ? { schemaVersion: schemaVersion as 1 | 2 } : {}),
-        ...(hasDataDisks ? { dataDisks } : {}),
-        ...(vyos ? { vyos } : {}),
       };
     });
 
@@ -218,31 +165,12 @@ export function registerApiRoutes(
           };
           s.installed[mac] = installedInfo;
 
-          // VyOS: the only login user is "vyos", and a router never runs k3s —
-          // without this guard a non-vanilla role + recorded IP would trigger
-          // the k3s post-provision against a VyOS box.
-          const isVyos = (installedInfo.os ?? "").startsWith("vyos");
-          const admin = isVyos
-            ? "vyos"
-            : installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "lab" : "root";
+          const admin = installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "lab" : "root";
           console.log(`\n  \x1b[0;32m\x1b[1m  ssh ${admin}@${ip}\x1b[0m\n`);  // eslint-disable-line no-console
 
-          // Auto-install k3s for non-vanilla roles.
-          // autoK3s=false suppresses this entirely. Multi-node tests need that:
-          // post-provision installs each node into its OWN cluster, so leaving it
-          // on turns three combo nodes into three single-node clusters, and it
-          // also stops k3s and deletes CNI state minutes after install lands --
-          // right on top of a test that is forming a cluster itself.
-          if (config.autoK3s === false) {
-            logger.info(`[api] autoK3s disabled; skipping k3s post-provision for ${mac}`);
-          } else if (!isVyos && installedInfo.role !== "vanilla" && ip !== "") {
-            // State is JSON on disk; a role written by an older build may no
-            // longer exist. Skip loudly rather than crash the install callback.
-            if (isRole(installedInfo.role)) {
-              void triggerPostProvisionK3s(installedInfo.hostname, ip, installedInfo.role, admin, mac);
-            } else {
-              console.warn(`[api] unknown role ${installedInfo.role} for ${mac}; skipping k3s post-provision`);  // eslint-disable-line no-console
-            }
+          // Auto-install k3s for non-vanilla roles
+          if (installedInfo.role !== "vanilla" && ip !== "") {
+            void triggerPostProvisionK3s(installedInfo.hostname, ip, installedInfo.role, admin, mac);
           }
         }
       }
@@ -363,6 +291,10 @@ export function registerApiRoutes(
       arch?: string;
       disks?: Array<{ name: string; size_gb: number; model: string }>;
       nics?: Array<{ name: string; mac: string; state: string }>;
+      // Root filesystem, when the reporter could observe it (recheck over SSH, or the
+      // probe script run from a rescue shell). Used by --pxe-boot.
+      root_device?: string;
+      root_args?: string;
     };
   }>("/api/discover", async (request, reply) => {
     const data = request.body;
@@ -377,22 +309,53 @@ export function registerApiRoutes(
 
     state.update((s) => {
       const existing = s.discovered[mac];
+      // Classify onboarding from the DMI identity we just received. An explicit
+      // classification already on the record wins (see classifyOnboard).
+      const onboarding = classifyOnboard({
+        mac,
+        manufacturer: data.manufacturer ?? existing?.manufacturer ?? "unknown",
+        product: data.product ?? existing?.product ?? "unknown",
+        board: data.board ?? existing?.board ?? "unknown",
+        ...(existing?.onboard !== undefined ? { onboard: existing.onboard } : {}),
+        ...(existing?.vendor_os !== undefined ? { vendor_os: existing.vendor_os } : {}),
+      });
+      const rootDevice = data.root_device ?? existing?.root_device;
+      const rootArgs = data.root_args ?? existing?.root_args;
+
+      // Absent fields keep whatever we already knew. Reporters are not all the full
+      // discovery kickstart: the rescue-shell probe posts only a root device, and
+      // blanking a machine's hardware inventory as a side effect of that would be
+      // silent data loss.
       const hwInfo: HardwareInfo = {
         mac,
-        product: data.product ?? "unknown",
-        board: data.board ?? "unknown",
-        serial: data.serial ?? "unknown",
-        manufacturer: data.manufacturer ?? "unknown",
-        cpu_model: data.cpu_model ?? "unknown",
-        cpu_cores: data.cpu_cores ?? 0,
-        memory_gb: data.memory_gb ?? 0,
-        arch: data.arch ?? "unknown",
-        disks: data.disks ?? [],
-        nics: data.nics ?? [],
+        product: data.product ?? existing?.product ?? "unknown",
+        board: data.board ?? existing?.board ?? "unknown",
+        serial: data.serial ?? existing?.serial ?? "unknown",
+        manufacturer: data.manufacturer ?? existing?.manufacturer ?? "unknown",
+        cpu_model: data.cpu_model ?? existing?.cpu_model ?? "unknown",
+        cpu_cores: data.cpu_cores ?? existing?.cpu_cores ?? 0,
+        memory_gb: data.memory_gb ?? existing?.memory_gb ?? 0,
+        arch: data.arch ?? existing?.arch ?? "unknown",
+        disks: data.disks ?? existing?.disks ?? [],
+        nics: data.nics ?? existing?.nics ?? [],
         first_seen: existing?.first_seen ?? now,
         last_seen: now,
+        onboard: onboarding.onboard,
+        ...(onboarding.vendor_os !== undefined ? { vendor_os: onboarding.vendor_os } : {}),
+        ...(rootDevice !== undefined ? { root_device: rootDevice } : {}),
+        ...(rootArgs !== undefined ? { root_args: rootArgs } : {}),
       };
       s.discovered[mac] = hwInfo;
+
+      // Keep the installed record in step -- the install guard and --pxe-boot read it.
+      const inst = s.installed[mac];
+      if (inst) {
+        if (data.arch !== undefined) inst.arch = data.arch;
+        inst.onboard = onboarding.onboard;
+        if (onboarding.vendor_os !== undefined) inst.vendor_os = onboarding.vendor_os;
+        if (rootDevice !== undefined) inst.root_device = rootDevice;
+        if (rootArgs !== undefined) inst.root_args = rootArgs;
+      }
     });
 
     const label = isNew ? "NEW MACHINE DISCOVERED" : "MACHINE RE-DISCOVERED";
@@ -513,15 +476,6 @@ export function registerApiRoutes(
     const installedEntry = currentState.installed[mac];
 
     if (queueEntry) {
-      // A machine that was handed an install script but has reported nothing
-      // since is wedged BEFORE the installer environment came up — a bad
-      // kernel/initrd, no network in the initramfs, or the wrong NIC picked.
-      // Surfacing it here is what makes that diagnosable without a console.
-      const since = queueEntry.progress_at ?? queueEntry.dispatched_at;
-      const stalledForS = since !== undefined && queueEntry.progress === undefined
-        ? Math.floor((Date.now() - new Date(since).getTime()) / 1000)
-        : 0;
-
       return reply.send({
         mac,
         hostname: queueEntry.hostname,
@@ -529,9 +483,6 @@ export function registerApiRoutes(
         progress: queueEntry.progress ?? "queued",
         progress_detail: queueEntry.progress_detail ?? "",
         progress_at: queueEntry.progress_at ?? queueEntry.queued_at,
-        dispatched_at: queueEntry.dispatched_at,
-        stalled_for_s: stalledForS,
-        stalled: stalledForS > STALL_THRESHOLD_S,
         role: queueEntry.role,
         os: queueEntry.os,
         stages: queueEntry.log ?? [],

bastion/src/bastion/src/routes/dispatch.ts

@@ -5,7 +5,8 @@
 //   - unknown       -> discovery mode (collect hardware, POST to bastion)
 
 import type { FastifyInstance } from "fastify";
-import type { BastionConfig } from "@lab/shared";
+import type { Arch, BastionConfig, BastionState, OsId } from "@lab/shared";
+import { normalizeArch, fedoraMirrorFor, osSupportsArch } from "@lab/shared";
 import type { StateManager } from "../services/state.js";
 import {
   renderDiscoverIpxe,
@@ -13,12 +14,51 @@ import {
   renderDebugIpxe,
   renderPxeBootDebugIpxe,
   renderLocalBootIpxe,
+  renderUnsupportedIpxe,
 } from "../templates/boot.ipxe.js";
 import { renderUbuntuInstallIpxe } from "../templates/ubuntu-boot.ipxe.js";
-import { renderVyosInstallIpxe } from "../templates/vyos-boot.ipxe.js";
 import { renderDebugKickstart } from "../templates/debug.ks.js";
 import { logger } from "../services/logger.js";
 
+/**
+ * Resolve a booting machine's architecture.
+ *
+ * Order matters. The tracked record is what we actually observed on the machine, so it
+ * wins. `reported` is iPXE's ${buildarch}, which is only as good as the binary DHCP
+ * handed the client -- correct in practice, but a misconfigured option 93 mapping would
+ * make it lie. The configured default is the last resort.
+ *
+ * There is deliberately no operator-supplied architecture anywhere in this path.
+ */
+export function resolveArch(
+  state: BastionState,
+  mac: string,
+  reported: string | undefined,
+  config: BastionConfig,
+): Arch {
+  return normalizeArch(state.installed[mac]?.arch)
+    ?? normalizeArch(state.install_queue[mac]?.arch)
+    ?? normalizeArch(state.discovered[mac]?.arch)
+    ?? normalizeArch(reported)
+    ?? normalizeArch(config.arch)
+    ?? "x86_64";
+}
+
+/** The root filesystem to boot for --pxe-boot, if the machine's record carries one. */
+function resolveRoot(
+  state: BastionState,
+  mac: string,
+): { rootDevice: string; rootArgs?: string } | null {
+  const installed = state.installed[mac];
+  const discovered = state.discovered[mac];
+  const rootDevice = installed?.root_device ?? discovered?.root_device;
+  if (rootDevice === undefined || rootDevice === "") return null;
+  const rootArgs = installed?.root_args ?? discovered?.root_args;
+  return rootArgs !== undefined && rootArgs !== ""
+    ? { rootDevice, rootArgs }
+    : { rootDevice };
+}
+
 export function registerDispatchRoutes(
   app: FastifyInstance,
   config: BastionConfig,
@@ -53,18 +93,68 @@ curl -sf -X POST "http://${config.serverIp}:${config.httpPort}/api/progress" \\
     -H "Content-Type: application/json" \\
     -d "{\\"mac\\":\\"$MAC_ADDR\\",\\"stage\\":\\"debug-ready\\",\\"detail\\":\\"nc $IP_ADDR 2323\\"}" 2>/dev/null || true
 
+# --- Find the installed root filesystem and report it ---
+# This is what 'labctl provision debug --pxe-boot' needs. The rescue image cannot
+# report it by itself: %pre/%post do not run in rescue mode, so it happens here.
+vgchange -ay >/dev/null 2>&1 || true
+
+ROOT_DEVICE=""
+ROOT_ARGS=""
+PROBE_MNT=/tmp/lab-rootprobe
+mkdir -p "$PROBE_MNT"
+
+# Candidates: every LVM logical volume plus every non-LVM partition with a filesystem.
+for CAND in $(lvs --noheadings -o lv_path 2>/dev/null) \\
+            $(blkid -o device 2>/dev/null | grep -v '^/dev/mapper/'); do
+    [ -b "$CAND" ] || continue
+    mount -o ro "$CAND" "$PROBE_MNT" >/dev/null 2>&1 || continue
+    # A root filesystem has both of these; /boot and /home do not.
+    if [ -f "$PROBE_MNT/etc/fstab" ] && [ -d "$PROBE_MNT/usr" ]; then
+        ROOT_DEVICE="$CAND"
+        PRETTY=$(. "$PROBE_MNT/etc/os-release" 2>/dev/null && echo "$PRETTY_NAME")
+        echo "  found root: $CAND \${PRETTY:+($PRETTY)}"
+        if [ "$(lsblk -no TYPE "$CAND" 2>/dev/null | head -1)" = "lvm" ]; then
+            VGLV=$(lvs --noheadings -o vg_name,lv_name "$CAND" 2>/dev/null | awk '{print $1"/"$2}')
+            [ -n "$VGLV" ] && ROOT_ARGS="rd.lvm.lv=$VGLV"
+            # Swap comes from fstab here — /proc/swaps is the rescue image's, not the host's.
+            SWLV=$(awk '$3=="swap" && $1 ~ /^\\/dev\\// {print $1; exit}' "$PROBE_MNT/etc/fstab" 2>/dev/null)
+            if [ -n "$SWLV" ]; then
+                SWVGLV=$(lvs --noheadings -o vg_name,lv_name "$SWLV" 2>/dev/null | awk '{print $1"/"$2}')
+                [ -n "$SWVGLV" ] && [ "$SWVGLV" != "$VGLV" ] && ROOT_ARGS="$ROOT_ARGS rd.lvm.lv=$SWVGLV"
+            fi
+        fi
+        umount "$PROBE_MNT" >/dev/null 2>&1 || true
+        break
+    fi
+    umount "$PROBE_MNT" >/dev/null 2>&1 || true
+done
+
+if [ -n "$ROOT_DEVICE" ]; then
+    curl -sf -X POST "http://${config.serverIp}:${config.httpPort}/api/discover" \\
+        -H "Content-Type: application/json" \\
+        -d "{\\"mac\\":\\"$MAC_ADDR\\",\\"root_device\\":\\"$ROOT_DEVICE\\",\\"root_args\\":\\"$ROOT_ARGS\\"}" 2>/dev/null \\
+        && echo "  reported to bastion — 'labctl provision debug --pxe-boot' will work now"
+else
+    echo "  no root filesystem found — --pxe-boot cannot be used on this machine"
+fi
+
 echo ""
 echo "=== Debug environment ready ==="
 echo "  nc $IP_ADDR 2323    (remote shell)"
 echo "  ssh root@$IP_ADDR   (password: debug)"
+if [ -n "$ROOT_DEVICE" ]; then
+    echo "  root: $ROOT_DEVICE $ROOT_ARGS"
+fi
 echo "==============================="
 `;
     return reply.type("text/plain").send(script);
   });
 
-  app.get<{ Querystring: { mac?: string } }>("/dispatch", async (request, reply) => {
+  app.get<{ Querystring: { mac?: string; arch?: string } }>("/dispatch", async (request, reply) => {
     const mac = (request.query.mac ?? "").toLowerCase().replace(/-/g, ":");
     const currentState = state.load();
+    const arch = resolveArch(currentState, mac, request.query.arch, config);
+    const fedoraMirror = fedoraMirrorFor(config.fedoraVersion, arch);
 
     // Debug mode takes highest priority — auto-clear after serving once
     const debugEntry = currentState.debug[mac];
@@ -73,22 +163,48 @@ echo "==============================="
       state.update((s) => { delete s.debug[mac]; });
 
       let script: string;
-      if (debugEntry.pxeBoot) {
-        logger.info(`PXE BOOT DEBUG: ${mac} -> ${hostname} (kernel+initrd from PXE, root from NVMe)`);
+      const wantsPxeBoot = debugEntry.pxeBoot === true;
+      const root = wantsPxeBoot ? resolveRoot(currentState, mac) : null;
+
+      if (root !== null) {
+        logger.info(`PXE BOOT DEBUG: ${mac} -> ${hostname} (${arch}, root=${root.rootDevice})`);
         script = renderPxeBootDebugIpxe({
           mac,
           hostname,
           serverIp: config.serverIp,
           httpPort: config.httpPort,
+          arch,
+          ...root,
         });
       } else {
-        logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (rescue mode)`);
+        // --pxe-boot without a known root device falls back to rescue rather than
+        // guessing. A wrong root= leaves the machine unbootable, and rescue is where
+        // the operator can find the real one (curl /debug-setup.sh reports it back).
+        const notice = wantsPxeBoot
+          ? [
+              "",
+              "NOTE: --pxe-boot requested, but no root device is recorded",
+              "      for this machine. Booting rescue instead.",
+              "      From the rescue shell, run:",
+              // No pipe or && here: iPXE treats || and && as command separators, so keep
+              // the printed command free of anything its parser might claim.
+              `        curl -s http://${config.serverIp}:${config.httpPort}/debug-setup.sh -o /tmp/s.sh ; sh /tmp/s.sh`,
+              "      then retry --pxe-boot.",
+            ]
+          : undefined;
+        if (wantsPxeBoot) {
+          logger.warn(`PXE BOOT DEBUG: ${mac} -> ${hostname} has no recorded root device -- serving rescue instead`);
+        } else {
+          logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (${arch}, rescue mode)`);
+        }
         script = renderDebugIpxe({
           mac,
           hostname,
           serverIp: config.serverIp,
           httpPort: config.httpPort,
-          fedoraMirror: config.fedoraMirror,
+          fedoraMirror,
+          arch,
+          ...(notice ? { notice } : {}),
         });
       }
       return reply.type("text/plain").send(script);
@@ -98,24 +214,24 @@ echo "==============================="
     if (queueEntry) {
       const hostname = queueEntry.hostname ?? "lab-node";
       const os = queueEntry.os ?? "fedora-43";
-      logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os})`);
-
-      // Stamp the handoff so a machine that boots the installer but never
-      // reports can be spotted without a console.
-      state.update((s) => {
-        const entry = s.install_queue[mac];
-        if (entry) entry.dispatched_at = new Date().toISOString();
-      });
+      logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os}, ${arch})`);
 
       let script: string;
-      if (os.startsWith("vyos")) {
-        script = renderVyosInstallIpxe({
-          mac,
-          hostname,
-          serverIp: config.serverIp,
-          httpPort: config.httpPort,
-        });
-      } else if (os.startsWith("ubuntu")) {
+      if (os.startsWith("ubuntu")) {
+        // Last line of defence. The install guard refuses this combination when the
+        // machine's architecture is already known, but a machine queued before it was
+        // discovered can reach here. Serving the x86-only Ubuntu kernel to an arm64
+        // client is precisely the bug this work exists to fix, so stop instead.
+        if (!osSupportsArch(os as OsId, arch)) {
+          logger.error(`INSTALL BLOCKED: ${mac} -> ${hostname} -- ${os} has no ${arch} artifacts`);
+          script = renderUnsupportedIpxe({
+            hostname,
+            mac,
+            reason: `${os} publishes no ${arch} netboot artifacts`,
+            action: `labctl provision install ${mac} ${hostname} --os fedora-43`,
+          });
+          return reply.type("text/plain").send(script);
+        }
         script = renderUbuntuInstallIpxe({
           mac,
           hostname,
@@ -130,7 +246,8 @@ echo "==============================="
           serverIp: config.serverIp,
           httpPort: config.httpPort,
           fedoraVersion: config.fedoraVersion,
-          fedoraMirror: config.fedoraMirror,
+          fedoraMirror,
+          arch,
         });
       }
 
@@ -147,13 +264,14 @@ echo "==============================="
     }
 
     // Unknown MAC -> discovery mode
-    logger.info(`PXE request from ${mac} -> discovery mode`);
+    logger.info(`PXE request from ${mac} (${arch}) -> discovery mode`);
 
     const script = renderDiscoverIpxe({
       mac,
       serverIp: config.serverIp,
       httpPort: config.httpPort,
-      fedoraMirror: config.fedoraMirror,
+      fedoraMirror,
+      arch,
     });
 
     return reply.type("text/plain").send(script);

bastion/src/bastion/src/routes/kickstart.ts

@@ -30,11 +30,6 @@ export function registerKickstartRoutes(
       hostname: queueEntry?.hostname ?? "lab-node",
       disk: queueEntry?.disk ?? "",
       role: queueEntry?.role ?? "worker",
-      // Progress callbacks must report the mac this install is QUEUED under,
-      // not whatever NIC enumerates first on the installed system.
-      ...(mac !== "" ? { mac } : {}),
-      ...(queueEntry?.schemaVersion ? { schemaVersion: queueEntry.schemaVersion } : {}),
-      ...(queueEntry?.dataDisks ? { dataDisks: queueEntry.dataDisks } : {}),
     });
 
     return reply.type("text/plain").send(ks);
@@ -200,7 +195,6 @@ esac
       hostname: queueEntry?.hostname ?? "lab-node",
       disk: queueEntry?.disk ?? "",
       role: queueEntry?.role ?? "worker",
-      ...(mac !== "" ? { mac } : {}),
       domain: config.domain,
       ubuntuVersion: config.ubuntuVersion,
       timezone: config.timezone,

bastion/src/bastion/src/routes/vyos.ts

@@ -1,71 +0,0 @@
-// VyOS network install routes.
-//
-// VyOS has no unattended installer, so the automation is injected via
-// live-config's `hooks` component: the iPXE script passes
-// live-config.hooks=<.../vyos/autoinstall.sh>, live-config wgets it and runs it
-// as root, and that script fetches and executes the generated install driver.
-
-import type { FastifyInstance } from "fastify";
-import type { BastionConfig } from "@lab/shared";
-import type { StateManager } from "../services/state.js";
-import { buildVyosConfigSpec } from "../templates/vyos-config-spec.js";
-import { renderVyosInstallPy } from "../templates/vyos-install.py.js";
-import { logger } from "../services/logger.js";
-
-function normalizeMac(value: string | undefined): string {
-  return (value ?? "").toLowerCase().replace(/-/g, ":");
-}
-
-export function registerVyosRoutes(
-  app: FastifyInstance,
-  config: BastionConfig,
-  state: StateManager,
-): void {
-  // live-config hook. Kept minimal: everything version-specific lives in the
-  // generated Python. wget is guaranteed present -- live-config used it to
-  // fetch this very script.
-  app.get<{ Querystring: { mac?: string } }>("/vyos/autoinstall.sh", async (request, reply) => {
-    const mac = normalizeMac(request.query.mac);
-    const base = `http://${config.serverIp}:${config.httpPort}`;
-
-    logger.info(`VYOS AUTOINSTALL HOOK served to ${mac || "unknown MAC"}`);
-
-    const script = `#!/bin/sh
-# Lab PXE Bastion -- VyOS unattended install hook (run by live-config as root)
-set -eu
-
-wget -q "${base}/vyos/install.py?mac=${mac}" -O /tmp/vyos-install.py
-exec python3 /tmp/vyos-install.py
-`;
-    return reply.type("text/plain").send(script);
-  });
-
-  // Per-MAC install driver, with the machine's config spec baked in.
-  app.get<{ Querystring: { mac?: string } }>("/vyos/install.py", async (request, reply) => {
-    const mac = normalizeMac(request.query.mac);
-    const queueEntry = state.load().install_queue[mac];
-
-    const spec = buildVyosConfigSpec({
-      hostname: queueEntry?.hostname ?? "vyos",
-      spec: queueEntry?.vyos,
-      defaultPassword: config.vyosDefaultPassword,
-      sshKeys: config.sshKeys,
-      disk: queueEntry?.disk,
-    });
-
-    logger.info(
-      `VYOS INSTALL DRIVER served to ${mac} (${spec.hostname}, ` +
-      `${spec.sets.length} config ops, disk="${spec.disk || "auto"}")`,
-    );
-
-    const script = renderVyosInstallPy({
-      spec,
-      mac,
-      serverIp: config.serverIp,
-      httpPort: config.httpPort,
-      role: queueEntry?.role ?? "vanilla",
-    });
-
-    return reply.type("text/plain").send(script);
-  });
-}

bastion/src/bastion/src/server.ts

@@ -12,7 +12,6 @@ import { registerDispatchRoutes } from "./routes/dispatch.js";
 import { registerKickstartRoutes } from "./routes/kickstart.js";
 import { registerApiRoutes } from "./routes/api.js";
 import { registerAsahiRoutes } from "./routes/asahi.js";
-import { registerVyosRoutes } from "./routes/vyos.js";
 
 
 export function createApp(config: BastionConfig): { app: ReturnType<typeof Fastify>; state: StateManager; installLog: InstallLogBuffer; syslog: SyslogListener } {
@@ -46,9 +45,8 @@ export function createApp(config: BastionConfig): { app: ReturnType<typeof Fasti
   // Register route handlers
   registerDispatchRoutes(app, config, state);
   registerKickstartRoutes(app, config, state, syslog);
-  registerApiRoutes(app, config, state, installLog, syslog);
+  registerApiRoutes(app, state, installLog, syslog);
   registerAsahiRoutes(app, config);
-  registerVyosRoutes(app, config, state);
   // boot.iso is generated at startup and served as a static file from httpDir
   // (static serving supports HTTP Range requests, required by JetKVM streaming)
 

bastion/src/bastion/src/services/install-guard.ts

@@ -0,0 +1,94 @@
+// Pre-flight checks for queuing an OS install.
+//
+// Both entry points -- the HTTP /api/install route and the labd command-install handler
+// -- run this, so `labctl provision install` and `provision reprovision` are covered
+// whichever way the request arrives.
+//
+// Rescue/debug is deliberately NOT guarded. Being unable to reinstall a machine is
+// exactly when you most need to boot it into a rescue shell.
+
+import type { Arch, BastionState, OsId } from "@lab/shared";
+import { classifyOnboard, normalizeArch, osSupportsArch, vendorOsDescription, archesForOs } from "@lab/shared";
+
+export type InstallCheck =
+  | { allowed: true }
+  | { allowed: false; error: string };
+
+interface MachineIdentity {
+  hostname: string;
+  arch: Arch | undefined;
+  identity: Parameters<typeof classifyOnboard>[0];
+}
+
+/** Best-known identity for a MAC, merged across the three state maps. */
+function identify(state: BastionState, mac: string): MachineIdentity {
+  const discovered = state.discovered[mac];
+  const installed = state.installed[mac];
+  const queued = state.install_queue[mac];
+
+  const manufacturer = discovered?.manufacturer ?? installed?.manufacturer;
+  const product = discovered?.product ?? installed?.product;
+  const board = discovered?.board;
+  const onboard = installed?.onboard ?? discovered?.onboard;
+  const vendorOs = installed?.vendor_os ?? discovered?.vendor_os;
+
+  return {
+    hostname: installed?.hostname ?? queued?.hostname ?? discovered?.product ?? mac,
+    arch: normalizeArch(installed?.arch ?? queued?.arch ?? discovered?.arch),
+    identity: {
+      mac,
+      ...(manufacturer !== undefined ? { manufacturer } : {}),
+      ...(product !== undefined ? { product } : {}),
+      ...(board !== undefined ? { board } : {}),
+      ...(onboard !== undefined ? { onboard } : {}),
+      ...(vendorOs !== undefined ? { vendor_os: vendorOs } : {}),
+    },
+  };
+}
+
+/**
+ * Decide whether `mac` may be queued for an install of `os`.
+ *
+ * Refusals name the machine and the reason, and point at the action that is available
+ * instead. An operator hitting this at 2am should not have to read the source to work
+ * out what happened.
+ */
+export function checkInstallAllowed(
+  state: BastionState,
+  mac: string,
+  os: OsId,
+): InstallCheck {
+  const machine = identify(state, mac);
+  const { onboard, vendor_os } = classifyOnboard(machine.identity);
+
+  // 1. Machines running a vendor OS we cannot rebuild.
+  if (onboard === "ssh") {
+    const what = vendorOsDescription(vendor_os);
+    return {
+      allowed: false,
+      error:
+        `Refusing to install ${machine.hostname} (${mac}): it runs ${what}. ` +
+        `No image in our pipeline can restore it, so installing ${os} would destroy that ` +
+        `driver and firmware stack permanently. This machine is SSH-onboard: we manage its ` +
+        `userspace, not its OS. ` +
+        `To boot it into a rescue shell instead, run: labctl provision debug ${machine.hostname}`,
+      // TODO: when a DGX OS / SparkOS image joins the pipeline, an install targeting a
+      // machine whose vendor_os matches that image should be allowed through here.
+    };
+  }
+
+  // 2. Architecture the OS has no netboot artifacts for.
+  if (machine.arch !== undefined && !osSupportsArch(os, machine.arch)) {
+    const supported = archesForOs(os);
+    return {
+      allowed: false,
+      error:
+        `Refusing to install ${os} on ${machine.hostname} (${mac}): ` +
+        `${os} has no ${machine.arch} netboot artifacts` +
+        (supported.length > 0 ? ` (only ${supported.join(", ")})` : "") +
+        `. Use an OS that supports ${machine.arch}.`,
+    };
+  }
+
+  return { allowed: true };
+}

bastion/src/bastion/src/services/kickstart-generator.ts

@@ -24,19 +24,12 @@ export function generateInstallKickstart(
     hostname: string;
     disk: string;
     role: Role;
-    /** PXE MAC the install is keyed under; baked into progress callbacks. */
-    mac?: string;
-    schemaVersion?: 1 | 2;
-    dataDisks?: import("@lab/shared").DataDisks;
   },
 ): string {
   const ksParams: InstallKickstartParams = {
     hostname: params.hostname,
     disk: params.disk,
     role: params.role,
-    ...(params.mac !== undefined && params.mac !== "" ? { mac: params.mac } : {}),
-    ...(params.schemaVersion ? { schemaVersion: params.schemaVersion } : {}),
-    ...(params.dataDisks ? { dataDisks: params.dataDisks } : {}),
     domain: config.domain,
     fedoraVersion: config.fedoraVersion,
     timezone: config.timezone,

bastion/src/bastion/src/services/post-provision.ts

@@ -7,7 +7,6 @@ import { homedir } from "node:os";
 import { join } from "node:path";
 import { logger } from "./logger.js";
 import { progressBus } from "./progress-events.js";
-import { isK3sServer, type Role } from "@lab/shared";
 
 function findSshKey(): string | undefined {
   const sudoUser = process.env["SUDO_USER"];
@@ -85,36 +84,6 @@ function sshRunStreaming(ip: string, user: string, command: string, keyPath: str
   });
 }
 
-/**
- * Roles this bare post-provision install can legitimately handle.
- *
- * The install below is a plain `INSTALL_K3S_EXEC=server|agent` with no K3S_URL
- * and no K3S_TOKEN, and it knows about exactly one machine. That is fine for a
- * node that is its own cluster, and wrong for anything that has to JOIN one:
- *
- *   - a cluster member installed this way gets `cluster-init` semantics and
- *     becomes its own separate cluster, so provisioning three of them yields
- *     three clusters instead of one (split brain);
- *   - an AGENT cannot even start — `INSTALL_K3S_EXEC=agent` with nothing to
- *     join fails with "Job for k3s-agent.service failed because the control
- *     process exited with error code", which is exactly what a k8s-worker
- *     install does today.
- *
- * So cluster members are not auto-installed. They are handed the command that
- * supplies the missing server URL and token.
- */
-export function shouldAutoInstallK3s(role: Role): boolean {
-  switch (role) {
-    case "k8s-combo":
-    case "k8s-controller":
-    case "k8s-worker":
-    case "worker":
-      return false;
-    default:
-      return true;
-  }
-}
-
 /**
  * Trigger k3s installation on a freshly provisioned machine.
  * Runs in the background — logs progress to bastion console and progressBus.
@@ -122,7 +91,7 @@ export function shouldAutoInstallK3s(role: Role): boolean {
 export async function triggerPostProvisionK3s(
   hostname: string,
   ip: string,
-  role: Role,
+  role: string,
   sshUser: string,
   mac?: string,
 ): Promise<void> {
@@ -135,21 +104,6 @@ export async function triggerPostProvisionK3s(
     }
   };
 
-  // Gate before ANY work: the prerequisite steps below stop k3s and delete CNI
-  // state, so they must not run for a role we are declining to install either.
-  if (!shouldAutoInstallK3s(role)) {
-    emitStage(
-      "post-provision",
-      `${role} is a cluster member; not auto-installing k3s (it would form its own cluster)`,
-    );
-    const verb = isK3sServer(role) ? "--k3s-server https://<existing-server>:6443" : "--k3s-server https://<server>:6443";
-    logger.info(
-      `[k3s] Join it to the cluster with:\n` +
-      `        labctl app k3s install ${hostname} --role ${role} ${verb} --k3s-token <token>`,
-    );
-    return;
-  }
-
   emitStage("post-provision", `auto-installing k3s on ${hostname} (${ip}) role=${role}`);
   emitStage("post-provision", "waiting for SSH (machine may still be rebooting)");
 
@@ -196,7 +150,7 @@ sysctl --system > /dev/null'`, keyPath, hostname, mac);
 
   // Step 4: Install k3s
   // labcontroller extends infra — both are k3s servers
-  const k3sRole = isK3sServer(role) ? "server" : "agent";
+  const k3sRole = (role === "infra" || role === "labcontroller") ? "server" : "agent";
   emitStage("post-provision", `installing k3s ${k3sRole}`);
   const code = await sshRunStreaming(ip, sshUser,
     `curl -sfL https://get.k3s.io | sudo INSTALL_K3S_EXEC="${k3sRole}" INSTALL_K3S_SKIP_SELINUX_RPM=true sh -`,

bastion/src/bastion/src/templates/asahi-firstboot.sh.ts

@@ -3,7 +3,6 @@
 // Creates the standard lab LVM layout on the data partition, matching install.ks.ts.
 
 import type { Role } from "@lab/shared";
-import { isK3sServer, v2Layout, type LvSpec } from "@lab/shared";
 
 export interface AsahiFirstbootParams {
   hostname: string;
@@ -13,42 +12,13 @@ export interface AsahiFirstbootParams {
   sshKeys: string[];
   adminUser: string;
   mac: string;
-  /** Fresh-setup layout. Defaults to 1 so existing callers are unchanged. */
-  schemaVersion?: 1 | 2;
 }
 
 export function renderFirstbootScript(params: AsahiFirstbootParams): string {
   const { hostname, role, serverIp, httpPort, sshKeys, adminUser, mac } = params;
-  const schemaVersion = params.schemaVersion ?? 1;
-
-  // ---------------------------------------------------------------------------
-  // v2 LAYOUT, from the SAME definition Fedora and Ubuntu render
-  // (shared/src/types/layout.ts).
-  //
-  // omitRoot: on Apple Silicon the Asahi installer has already created a btrfs
-  // root partition OUTSIDE our volume group -- worker1 runs a 4.6 GB one. We
-  // must not create a second root LV.
-  //
-  // The v1 path below hardcodes rancher at 20480M while claiming "sizes match
-  // install.ks.ts", which uses 122880M. That 100 GB gap is what left worker1
-  // with an image store too small to hold a single large container image.
-  // ---------------------------------------------------------------------------
-  const v2 = v2Layout(role, { omitRoot: true });
-  const v2LvLines = v2.map((lv: LvSpec) => `lvcreate -L ${lv.sizeMib}M -n ${lv.name} labvg -y`).join("\n");
-  const v2FormatLines = v2
-    .map((lv: LvSpec) => (lv.fs === "swap" ? `mkswap /dev/labvg/${lv.name}` : `mkfs.xfs /dev/labvg/${lv.name}`))
-    .join("\n");
-  const v2MountLines = v2
-    .filter((lv: LvSpec) => lv.fs !== "swap")
-    .map((lv: LvSpec) => `mount_lv ${lv.name} ${lv.mount}`)
-    .join("\n");
-  const v2FstabLines = v2
-    .filter((lv: LvSpec) => lv.fs !== "swap")
-    .map((lv: LvSpec) => `echo "/dev/labvg/${lv.name} ${lv.mount} xfs defaults 0 0" >> /etc/fstab`)
-    .join("\n");
 
   const isWorker = role === "worker";
-  const isInfra = isK3sServer(role);
+  const isInfra = role === "infra" || role === "labcontroller";
 
   // Role-specific LV creation commands
   const roleLvLines: string[] = [];
@@ -157,27 +127,24 @@ write_lab_fstab() {
     # Add fresh LVM entries
     echo "# Lab LVM volumes" >> /etc/fstab
     echo "/dev/labvg/swap    none         swap defaults 0 0" >> /etc/fstab
-${schemaVersion === 2 ? v2FstabLines.split("\n").map((l: string) => "    " + l).join("\n") : `    echo "/dev/labvg/var     /var         xfs  defaults 0 0" >> /etc/fstab
+    echo "/dev/labvg/var     /var         xfs  defaults 0 0" >> /etc/fstab
     echo "/dev/labvg/varlog  /var/log     xfs  defaults 0 0" >> /etc/fstab
     echo "/dev/labvg/home    /home        xfs  defaults 0 0" >> /etc/fstab
     echo "/dev/labvg/srv     /srv         xfs  defaults 0 0" >> /etc/fstab
-    ${roleFstabLines.join('\n    ')}`}
+    ${roleFstabLines.join('\n    ')}
 }
 
 # ── Check for existing VG ────────────────────────────────────────
 if vgs labvg &>/dev/null; then
     echo "Volume group 'labvg' already exists — reprovision detected."
-    DISK_SCHEMA=v1
-    vgs --noheadings -o vg_tags labvg 2>/dev/null | grep -q lab-schema-v2 && DISK_SCHEMA=v2
-    echo "On-disk schema: $DISK_SCHEMA"
     echo "Activating existing volumes..."
     vgchange -ay labvg
 
-${schemaVersion === 2 ? v2MountLines.split("\n").map((l: string) => "    " + l).join("\n") : `    mount_lv var /var
+    mount_lv var /var
     mount_lv varlog /var/log
     mount_lv home /home
     mount_lv srv /srv
-${roleMountLines.map(l => `    ${l}`).join('\n')}`}
+${roleMountLines.map(l => `    ${l}`).join('\n')}
 
     # Enable swap
     if lvs labvg/swap &>/dev/null; then
@@ -195,28 +162,24 @@ echo "Creating LVM on $DATA_PART..."
 
 pvcreate "$DATA_PART"
 vgcreate labvg "$DATA_PART"
-${schemaVersion === 2 ? `
-# Tag so a later run can tell which layout is on this disk without trusting a
-# caller-supplied flag -- same mechanism install.ks.ts uses.
-vgchange --addtag lab-schema-v2 labvg >/dev/null 2>&1 || true` : ""}
 
 # Create LVs — sizes match install.ks.ts (in MiB)
 echo "Creating logical volumes..."
-${schemaVersion === 2 ? v2LvLines : `lvcreate -L 27648M  -n swap   labvg -y   # 27GB swap
+lvcreate -L 27648M  -n swap   labvg -y   # 27GB swap
 lvcreate -L 102400M -n var    labvg -y   # 100GB /var
 lvcreate -L 10240M  -n varlog labvg -y   # 10GB /var/log
 lvcreate -L 10240M  -n home   labvg -y   # 10GB /home
 lvcreate -L 20480M  -n srv    labvg -y   # 20GB /srv
-${roleLvLines.join('\n')}`}
+${roleLvLines.join('\n')}
 
 # Format
 echo "Formatting volumes..."
-${schemaVersion === 2 ? v2FormatLines : `mkswap /dev/labvg/swap
+mkswap /dev/labvg/swap
 mkfs.xfs /dev/labvg/var
 mkfs.xfs /dev/labvg/varlog
 mkfs.xfs /dev/labvg/home
 mkfs.xfs /dev/labvg/srv
-${roleFormatLines.join('\n')}`}
+${roleFormatLines.join('\n')}
 
 # Migrate and mount volumes that can be switched live.
 # Copy existing content first so we don't shadow files (e.g. /home/user/.ssh).

bastion/src/bastion/src/templates/boot.ipxe.ts

@@ -1,4 +1,55 @@
 // iPXE boot script templates for dispatch routing.
+//
+// Architecture handling: the bastion serves one kernel/initrd pair per architecture.
+// x86_64 keeps the original unsuffixed paths so its output is unchanged; every other
+// architecture gets an arch-suffixed pair. See stageBootArtifacts() in main.ts for the
+// matching staging side, and boot-iso.ts for the same scheme on the ISO path.
+
+import type { Arch } from "@lab/shared";
+
+/** Kernel/initrd URL paths, keyed by architecture. */
+export function kernelPath(arch: Arch): string {
+  return arch === "x86_64" ? "/vmlinuz" : `/vmlinuz-${arch}`;
+}
+
+export function initrdPath(arch: Arch): string {
+  return arch === "x86_64" ? "/initrd.img" : `/initrd-${arch}.img`;
+}
+
+/**
+ * Console arguments per architecture.
+ *
+ * arm64 has no VGA text console: a headless machine only talks over the SoC UART, so
+ * ttyAMA0 must be listed as well. The last console= wins for /dev/console, so serial
+ * is the interactive one while tty0 still receives boot output on machines with a
+ * display attached.
+ */
+const CONSOLE_ARGS: Record<Arch, string> = {
+  x86_64: "console=tty0",
+  aarch64: "console=tty0 console=ttyAMA0,115200",
+};
+
+/**
+ * Anaconda arguments for the graphical-suppression / console setup.
+ *
+ * `nomodeset` disables kernel mode setting, which on x86 forces the generic VGA path
+ * and makes flaky GPU drivers survive the installer. On arm64 it does not mean the
+ * same thing -- there is no VGA fallback to drop back to, and it can leave the machine
+ * with no usable console at all -- so arm64 gets explicit console arguments instead.
+ */
+function installerArgs(arch: Arch): string {
+  return arch === "x86_64" ? "inst.text nomodeset" : `inst.text ${CONSOLE_ARGS[arch]}`;
+}
+
+/** Extra console arguments appended to templates that don't already set them. */
+function extraConsoleArgs(arch: Arch): string {
+  return arch === "x86_64" ? "" : ` ${CONSOLE_ARGS[arch]}`;
+}
+
+/** Join kernel arguments, dropping empties so callers can pass optional groups. */
+function joinArgs(...parts: Array<string | undefined>): string {
+  return parts.filter((p) => p !== undefined && p !== "").join(" ");
+}
 
 export interface BootIpxeParams {
   serverIp: string;
@@ -8,6 +59,11 @@ export interface BootIpxeParams {
 /**
  * Initial iPXE boot script that chains to the dispatch endpoint.
  * This is what dnsmasq serves to iPXE clients via HTTP.
+ *
+ * `${buildarch}` is iPXE's own build architecture ("x86_64" or "arm64"), which is the
+ * one architecture signal available on every path -- network PXE, UEFI HTTP boot and
+ * the boot ISO alike. DHCP option 93 only reaches dnsmasq, never this HTTP endpoint.
+ * dispatch prefers the tracked machine record and falls back to this.
  */
 export function renderBootIpxe(params: BootIpxeParams): string {
   return `#!ipxe
@@ -19,7 +75,7 @@ echo   Contacting server for instructions...
 echo ============================================
 echo
 
-chain http://${params.serverIp}:${params.httpPort}/dispatch?mac=\${net0/mac}
+chain http://${params.serverIp}:${params.httpPort}/dispatch?mac=\${net0/mac}&arch=\${buildarch}
 `;
 }
 
@@ -31,7 +87,9 @@ export function renderDiscoverIpxe(params: {
   serverIp: string;
   httpPort: number;
   fedoraMirror: string;
+  arch: Arch;
 }): string {
+  const base = `http://${params.serverIp}:${params.httpPort}`;
   return `#!ipxe
 
 echo
@@ -42,8 +100,8 @@ echo   Collecting hardware info...
 echo =============================================
 echo
 
-kernel http://${params.serverIp}:${params.httpPort}/vmlinuz inst.ks=http://${params.serverIp}:${params.httpPort}/discover.ks inst.stage2=${params.fedoraMirror} inst.text nomodeset
-initrd http://${params.serverIp}:${params.httpPort}/initrd.img
+kernel ${base}${kernelPath(params.arch)} inst.ks=${base}/discover.ks inst.stage2=${params.fedoraMirror} ${installerArgs(params.arch)}
+initrd ${base}${initrdPath(params.arch)}
 boot
 `;
 }
@@ -58,7 +116,9 @@ export function renderInstallIpxe(params: {
   httpPort: number;
   fedoraVersion: string;
   fedoraMirror: string;
+  arch: Arch;
 }): string {
+  const base = `http://${params.serverIp}:${params.httpPort}`;
   return `#!ipxe
 
 echo
@@ -69,8 +129,8 @@ echo   MAC:    ${params.mac}
 echo =============================================
 echo
 
-kernel http://${params.serverIp}:${params.httpPort}/vmlinuz inst.ks=http://${params.serverIp}:${params.httpPort}/ks?mac=${params.mac} inst.repo=${params.fedoraMirror} inst.text nomodeset
-initrd http://${params.serverIp}:${params.httpPort}/initrd.img
+kernel ${base}${kernelPath(params.arch)} inst.ks=${base}/ks?mac=${params.mac} inst.repo=${params.fedoraMirror} ${installerArgs(params.arch)}
+initrd ${base}${initrdPath(params.arch)}
 boot
 `;
 }
@@ -78,6 +138,9 @@ boot
 /**
  * iPXE script for debug/rescue mode -- boots Fedora installer in rescue mode.
  * Provides a shell with LVM tools, network, and SSH for inspecting installed systems.
+ *
+ * `notice` is shown before the boot line. dispatch uses it to explain why a requested
+ * --pxe-boot fell back to rescue.
  */
 export function renderDebugIpxe(params: {
   mac: string;
@@ -85,7 +148,11 @@ export function renderDebugIpxe(params: {
   serverIp: string;
   httpPort: number;
   fedoraMirror: string;
+  arch: Arch;
+  notice?: string[];
 }): string {
+  const base = `http://${params.serverIp}:${params.httpPort}`;
+  const notice = (params.notice ?? []).map((line) => `echo   ${line}\n`).join("");
   return `#!ipxe
 
 echo
@@ -93,11 +160,11 @@ echo =============================================
 echo   Lab PXE Bastion - DEBUG/RESCUE MODE
 echo   Target: ${params.hostname}
 echo   MAC:    ${params.mac}
-echo =============================================
+${notice}echo =============================================
 echo
 
-kernel http://${params.serverIp}:${params.httpPort}/vmlinuz inst.rescue inst.text inst.sshd inst.ks=http://${params.serverIp}:${params.httpPort}/debug.ks?mac=${params.mac} inst.stage2=${params.fedoraMirror}
-initrd http://${params.serverIp}:${params.httpPort}/initrd.img
+kernel ${base}${kernelPath(params.arch)} inst.rescue inst.text inst.sshd inst.ks=${base}/debug.ks?mac=${params.mac} inst.stage2=${params.fedoraMirror}${extraConsoleArgs(params.arch)}
+initrd ${base}${initrdPath(params.arch)}
 boot
 `;
 }
@@ -106,13 +173,28 @@ boot
  * iPXE script for PXE-boot debug mode -- boots the installed system's root
  * filesystem using the bastion's PXE kernel+initrd instead of local GRUB.
  * Workaround for UEFI firmware bugs that make local disk boot slow.
+ *
+ * rootDevice/rootArgs come from the machine's record -- they are not assumed. Our
+ * Fedora installs use an LVM layout, but nothing guarantees any given machine does,
+ * and a wrong root= here means an unbootable machine. dispatch refuses to render this
+ * script without them.
  */
 export function renderPxeBootDebugIpxe(params: {
   mac: string;
   hostname: string;
   serverIp: string;
   httpPort: number;
+  arch: Arch;
+  rootDevice: string;
+  rootArgs?: string;
 }): string {
+  const base = `http://${params.serverIp}:${params.httpPort}`;
+  const cmdline = joinArgs(
+    `root=${params.rootDevice}`,
+    "ro",
+    params.rootArgs,
+    CONSOLE_ARGS[params.arch],
+  );
   return `#!ipxe
 
 echo
@@ -124,12 +206,40 @@ echo   Kernel+initrd from PXE, root from NVMe
 echo =============================================
 echo
 
-kernel http://${params.serverIp}:${params.httpPort}/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0
-initrd http://${params.serverIp}:${params.httpPort}/initrd.img
+kernel ${base}${kernelPath(params.arch)} ${cmdline}
+initrd ${base}${initrdPath(params.arch)}
 boot
 `;
 }
 
+/**
+ * iPXE script for a request we refuse to serve.
+ *
+ * Better a machine that stops with a legible reason on its console than one handed a
+ * kernel it cannot execute, which fails much later and much less clearly.
+ */
+export function renderUnsupportedIpxe(params: {
+  mac: string;
+  hostname: string;
+  reason: string;
+  action?: string;
+}): string {
+  return `#!ipxe
+
+echo
+echo =============================================
+echo   Lab PXE Bastion - CANNOT BOOT THIS MACHINE
+echo   Target: ${params.hostname}
+echo   MAC:    ${params.mac}
+echo
+echo   ${params.reason}
+${params.action !== undefined ? `echo\necho   Try: ${params.action}\n` : ""}echo =============================================
+echo
+sleep 10
+exit 1
+`;
+}
+
 /**
  * iPXE script for already-installed machines -- exits to boot from local disk.
  */

bastion/src/bastion/src/templates/dnsmasq.conf.ts

@@ -48,15 +48,20 @@ enable-tftp
 tftp-root=${tftpDir}
 tftp-no-blocksize
 
-# Detect client architecture -- PXE (TFTP) clients
+# Detect client architecture -- PXE (TFTP) clients.
+# Values are DHCP option 93 (Client System Architecture), IANA "Processor Architecture
+# Types". Getting these wrong means the machine is handed a bootloader its firmware
+# cannot execute, and it loops or hangs with no console output.
 dhcp-match=set:bios,option:client-arch,0
 dhcp-match=set:efi-x86_64,option:client-arch,7
 dhcp-match=set:efi-x86_64,option:client-arch,9
 dhcp-match=set:efi-arm64,option:client-arch,11
 
-# Detect client architecture -- UEFI HTTP Boot clients (no TFTP size limit)
+# Detect client architecture -- UEFI HTTP Boot clients (no TFTP size limit).
+# 16 = x64 uefi boot from http, 19 = arm uefi 64 boot from http.
+# (20 is pc/at bios boot from http -- not arm64.)
 dhcp-match=set:httpboot-x86_64,option:client-arch,16
-dhcp-match=set:httpboot-arm64,option:client-arch,20
+dhcp-match=set:httpboot-arm64,option:client-arch,19
 
 # Detect iPXE clients (already chainloaded)
 dhcp-userclass=set:ipxe,iPXE

bastion/src/bastion/src/templates/install.ks.ts

@@ -2,25 +2,12 @@
 // Full Fedora server install with LVM partitioning, %pre for reprovision detection,
 // packages, and %post with SSH keys, user creation, k3s prereqs, progress callbacks.
 
-import type { Role, DataDisks, DataDiskName } from "@lab/shared";
-import {
-  v2Layout, V2_BOOT, requiresSchemaV2, effectiveSchemaVersion, isK3sServer,
-  validateDataDisks, DATA_DISK_LABELS, type LvSpec,
-} from "@lab/shared";
+import type { Role } from "@lab/shared";
 
 export interface InstallKickstartParams {
   hostname: string;
   disk: string;
   role: Role;
-  /**
-   * The PXE MAC the bastion keys this install under (install_queue /
-   * installed). When set, progress callbacks report THIS mac verbatim.
-   * Runtime detection (`ip link ... exit` = first NIC) is only a fallback:
-   * on a multi-NIC board the first NIC need not be the PXE NIC, and a
-   * completion reported under the wrong mac is a silent no-op — the queue
-   * entry survives and the machine reinstalls forever (thebeast 2026-10-20).
-   */
-  mac?: string;
   domain: string;
   fedoraVersion: string;
   timezone: string;
@@ -30,22 +17,6 @@ export interface InstallKickstartParams {
   syslogPort: number;
   sshKeys: string[];
   adminUser: string;
-  /**
-   * Disk layout to use for a FRESH install. Defaults to 1 so every existing
-   * caller keeps today's layout untouched.
-   *
-   * On a REPROVISION this is ignored: the schema already on the disk wins,
-   * read from an LVM tag on the volume group. Reinstalling a machine must
-   * never silently change its layout -- that is how data gets destroyed.
-   */
-  schemaVersion?: 1 | 2;
-  /**
-   * Dedicated whole-disk data volumes, by stable /dev/disk/by-id path (v2
-   * only). The matching LV is not created; the disk gets one labelled xfs
-   * partition instead. On a reprovision %pre rediscovers these by label, so a
-   * reinstall preserves them whether or not the queue entry names them.
-   */
-  dataDisks?: DataDisks;
 }
 
 export function renderInstallKickstart(params: InstallKickstartParams): string {
@@ -53,7 +24,6 @@ export function renderInstallKickstart(params: InstallKickstartParams): string {
     hostname,
     disk,
     role,
-    mac,
     domain,
     fedoraVersion,
     timezone,
@@ -65,87 +35,12 @@ export function renderInstallKickstart(params: InstallKickstartParams): string {
     adminUser,
   } = params;
 
-  // Shell line defining `mac` inside the bastion_progress helpers.
-  const macLine = mac !== undefined && mac !== ""
-    ? `local mac="${mac}"`
-    : `local mac=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')`;
-
   const fqdn = domain ? `${hostname}.${domain}` : hostname;
   const vg = "labvg";
   const now = new Date().toISOString();
-  // ---------------------------------------------------------------------------
-  // v1 LAYOUT -- DO NOT REPLACE THESE WITH CAPABILITY LOOKUPS.
-  //
-  // These literals are deliberately frozen. Every machine already in the fleet
-  // was partitioned by them, and `labctl provision reprovision` must reproduce
-  // that same layout byte-for-byte or it destroys data. Yes, they are wrong:
-  // `labcontroller` runs k3s and gets no rancher LV, and `infra` nodes hold
-  // every Longhorn replica in the homelab cluster while only `worker` gets a
-  // longhorn LV. Those are fixed in the v2 layout, which new boxes get; see
-  // schemaVersion. Changing them here rewrites history for existing hardware.
-  // ---------------------------------------------------------------------------
-  // ---------------------------------------------------------------------------
-  // v2 LAYOUT. One definition, shared with ubuntu-autoinstall and asahi-firstboot
-  // (shared/src/types/layout.ts) so the three generators cannot drift apart
-  // again. Capability-gated, and nothing uses --grow.
-  // ---------------------------------------------------------------------------
-  const schemaVersion = effectiveSchemaVersion(role, params.schemaVersion);
-  if (params.schemaVersion === 1 && requiresSchemaV2(role)) {
-    // Fail here rather than render a machine with no rancher/longhorn LV.
-    throw new Error(
-      `role '${role}' has no v1 disk layout - it was introduced with schema v2. ` +
-      `Pass schemaVersion: 2 (labctl provision install --schema-version 2).`,
-    );
-  }
-
-  const dataDisks = params.dataDisks ?? {};
-  const hasDataDisks = Object.values(dataDisks).some((d) => d !== undefined);
-  if (hasDataDisks) {
-    // The v1 branches are frozen and know nothing about dedicated disks, so a
-    // v1 render with them would silently create BOTH the LV and the disk.
-    if (schemaVersion !== 2) {
-      throw new Error(
-        `dedicated data disks require the v2 layout - pass schemaVersion: 2 ` +
-        `(labctl provision install --schema-version 2)`,
-      );
-    }
-    const problem = validateDataDisks(role, dataDisks);
-    if (problem !== null) throw new Error(problem);
-  }
-
-  const v2RootMib = v2Layout(role).find((lv: LvSpec) => lv.name === "root")?.sizeMib ?? 32768;
-  const v2 = v2Layout(role, { dataDisks });
-  const v2LogvolLines = v2
-    .map((lv: LvSpec) =>
-      lv.fs === "swap"
-        ? `logvol swap --vgname=${vg} --name=${lv.name} --fstype=swap --size=${lv.sizeMib}`
-        : `logvol ${lv.mount} --vgname=${vg} --name=${lv.name} --fstype=xfs --size=${lv.sizeMib}`,
-    )
-    .join("\n");
-  /** OS volumes only - the ones a reprovision recreates rather than preserves. */
-  const v2OsLogvolLines = v2
-    .filter((lv: LvSpec) => ["swap", "root", "var", "varlog"].includes(lv.name))
-    .map((lv: LvSpec) =>
-      lv.fs === "swap"
-        ? `logvol swap --vgname=${vg} --name=${lv.name} --fstype=swap --size=${lv.sizeMib}`
-        : `logvol ${lv.mount} --vgname=${vg} --name=${lv.name} --fstype=xfs --size=${lv.sizeMib}`,
-    )
-    .join("\n");
-
   const hasLonghorn = role === "worker";
   const hasRancher = role === "infra";
-  // k8s roles get a dedicated 120G image-store LV. 2026-08 incident: the old
-  // 20G LV idled at 85% used, so a single ~5G image pull tripped imagefs
-  // eviction. Must be sized here — longhorn's --grow consumes all remaining
-  // VG space, making post-install lvextend impossible on worker nodes.
-  const hasRancherLv = role === "infra" || role === "worker";
   const isVanilla = role === "vanilla";
-  // etcd members need etcdctl available OFFLINE. The recovery procedure for a
-  // lost or stale member is `etcdctl member list/remove`, and `k3s etcdctl` is
-  // not a command -- the binary comes from the `etcd` rpm. Telling an operator
-  // to `dnf install etcd` in the middle of a quorum failure assumes working
-  // DNS, routing and a mirror, which is exactly what may be broken.
-  const needsEtcdctl = isK3sServer(role);
 
   // -- Auth section --
   // Always set a root password (for serial console debugging) + SSH keys
@@ -190,29 +85,14 @@ chmod 440 /etc/sudoers.d/${adminUser}`;
   }
 
   // -- Disk detection --
-  // An explicit disk may be a stable /dev/disk/by-id path: kernel names are
-  // not stable across boots on multi-NVMe hosts, so a pinned install must be
-  // able to name the disk by serial. Anaconda directives want the kernel name,
-  // so %pre resolves the symlink at install time.
   const diskLine = disk
-    ? disk.startsWith("/dev/")
-      ? `# OS disk pinned by stable path -- resolve to the kernel name anaconda wants
-DISK=""
-for _wait in $(seq 1 10); do
-    _os_target=$(readlink -f "${disk}" 2>/dev/null || true)
-    if [ -n "$_os_target" ] && [ -b "$_os_target" ]; then DISK=$(basename "$_os_target"); break; fi
-    sleep 1
-done
-[ -z "$DISK" ] && { bastion_progress "error" "OS disk not found: ${disk}"; echo "ERROR: OS disk not found: ${disk}"; exit 1; }`
-      : `DISK="${disk}"`
+    ? `DISK="${disk}"`
     : `DISK=""
 # Wait up to 10s for NVMe/SCSI disks to appear (they init async in initrd)
 for _wait in $(seq 1 10); do
     for d in /dev/nvme0n1 /dev/nvme1n1 /dev/sda /dev/sdb /dev/vda; do
         [ -b "$d" ] || continue
         _bname=$(basename "$d")
-        # Never auto-pick a dedicated data disk as the OS disk
-        case "$_bname" in "$LONGHORN_DISK"|"$RANCHER_DISK") continue ;; esac
         # Skip removable disks (USB, CD-ROM, JetKVM virtual media)
         [ -f "/sys/block/$_bname/removable" ] && [ "$(cat /sys/block/$_bname/removable)" = "1" ] && continue
         # Skip USB-attached disks (JetKVM virtual media shows as SCSI over USB)
@@ -228,88 +108,14 @@ for _wait in $(seq 1 10); do
 done
 [ -z "$DISK" ] && { echo "ERROR: no disk found"; exit 1; }`;
 
-  // -- Dedicated data disks: %pre resolution --
-  // Resolved BEFORE the OS disk so auto-detect can refuse to pick one of them.
-  const dataDiskResolveBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[])
-    .map((name) => {
-      const device = dataDisks[name];
-      const VAR = name.toUpperCase();
-      if (device === undefined) return `${VAR}_DISK=""`;
-      return `# ${name} gets a whole disk of its own, pinned by stable path
-${VAR}_DISK_ID="${device}"
-${VAR}_DISK=""
-for _wait in $(seq 1 10); do
-    _dd_target=$(readlink -f "$${VAR}_DISK_ID" 2>/dev/null || true)
-    if [ -n "$_dd_target" ] && [ -b "$_dd_target" ]; then ${VAR}_DISK=$(basename "$_dd_target"); break; fi
-    sleep 1
-done
-[ -z "$${VAR}_DISK" ] && { bastion_progress "error" "${name} disk not found: $${VAR}_DISK_ID"; echo "ERROR: ${name} disk not found: $${VAR}_DISK_ID"; exit 1; }`;
-    })
-    .join("\n");
-
-  // -- Dedicated data disks: discovery and preservation decisions --
-  // A labelled data partition is on-disk truth, exactly like the lab-schema-v2
-  // LVM tag: it is preserved whether or not this install names the disk, and it
-  // is never silently wiped or shadowed. blkid finds it wherever tonight's NVMe
-  // enumeration put it.
-  const dataDiskDecisionBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[])
-    .map((name) => {
-      const VAR = name.toUpperCase();
-      const label = DATA_DISK_LABELS[name];
-      const mount = name === "longhorn" ? "/var/lib/longhorn" : "/var/lib/rancher";
-      return `${VAR}_PART=$(blkid -t LABEL=${label} -o device 2>/dev/null | head -1)
-${VAR}_REUSE=no
-if [ -n "$${VAR}_PART" ]; then
-    _dd_parent=$(lsblk -ndo pkname "$${VAR}_PART" 2>/dev/null)
-    if [ -n "$${VAR}_DISK" ] && [ "$_dd_parent" != "$${VAR}_DISK" ]; then
-        bastion_progress "error" "${label} label is on $_dd_parent, not on the configured ${name} disk $${VAR}_DISK"
-        echo "ERROR: ${label} label found on $_dd_parent but the ${name} disk is configured as $${VAR}_DISK -- refusing to guess"
-        exit 1
-    fi
-    if [ "$REPROVISION" = "no" ] && [ -z "$${VAR}_DISK" ]; then
-        bastion_progress "error" "found ${label} data on $_dd_parent but this install configures no ${name} disk"
-        echo "ERROR: ${label} labelled data exists on $_dd_parent. Refusing to wipe or shadow it -- queue the install with a ${name} data disk, or wipe the label first"
-        exit 1
-    fi
-    ${VAR}_DISK="$_dd_parent"
-    ${VAR}_REUSE=yes
-fi
-if [ "$PRESERVE_${VAR}" = "yes" ] && [ -n "$${VAR}_DISK" ]; then
-    bastion_progress "error" "both a ${name} LV and a dedicated ${name} disk exist"
-    echo "ERROR: VG $VG has a ${name} LV AND a dedicated ${name} disk is present -- two things cannot mount ${mount}. Migrate the LV off first"
-    exit 1
-fi`;
-    })
-    .join("\n\n");
-
-  // -- Dedicated data disks: partition directives --
-  // Appended to whichever /tmp/part.ks the branches above wrote. A reused
-  // partition is mounted --noformat; a fresh disk gets ONE partition spanning
-  // it. --grow is fine HERE: the whole point of a dedicated disk is to consume
-  // it, unlike LVs competing inside a shared VG.
-  const dataDiskPartBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[])
-    .map((name) => {
-      const VAR = name.toUpperCase();
-      const label = DATA_DISK_LABELS[name];
-      const mount = name === "longhorn" ? "/var/lib/longhorn" : "/var/lib/rancher";
-      return `if [ -n "$${VAR}_DISK" ]; then
-    if [ "$${VAR}_REUSE" = "yes" ]; then
-        echo "part ${mount} --onpart=$${VAR}_PART --noformat" >> /tmp/part.ks
-    else
-        echo "part ${mount} --fstype=xfs --ondisk=$${VAR}_DISK --grow --size=1 --label=${label}" >> /tmp/part.ks
-    fi
-fi`;
-    })
-    .join("\n");
-
   // -- Longhorn LV for fresh install --
   const longhornFreshLine = hasLonghorn
     ? `logvol /var/lib/longhorn --vgname=${vg} --name=longhorn --fstype=xfs --grow --size=1`
     : "";
 
-  // -- Rancher LV for fresh install (k8s roles: worker + infra) --
-  const rancherFreshLine = hasRancherLv
-    ? `logvol /var/lib/rancher --vgname=${vg} --name=rancher --fstype=xfs --size=122880`
+  // -- Rancher LV for fresh install (infra role) --
+  const rancherFreshLine = hasRancher
+    ? `logvol /var/lib/rancher --vgname=${vg} --name=rancher --fstype=xfs --size=20480`
     : "";
 
   return `# Lab Bastion -- Fedora ${fedoraVersion} server install
@@ -323,13 +129,7 @@ lang ${locale}
 keyboard uk
 timezone ${timezone} --utc
 
-# --ipv6=auto, not =dhcp: "auto" follows the router advertisement, so the RA's
-# managed-flag is what steers the node to DHCPv6, and a VLAN with no DHCPv6 yet
-# still installs instead of blocking on a lease that will never come. Failing an
-# OS install because IPv6 was not ready would be a worse trade than a node that
-# briefly has no v6. The address itself comes from a kea DHCPv6 reservation
-# keyed on MAC -- the same source of truth as the v4 address.
-network --bootproto=dhcp --ipv6=auto --activate --hostname=${fqdn}
+network --bootproto=dhcp --activate --hostname=${fqdn}
 
 ${auth}
 ${userDirective}
@@ -350,7 +150,7 @@ set -x
 # Progress callback helper
 bastion_progress() {
     local stage="$1" detail="\${2:-}"
-    ${macLine}
+    local mac=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')
     curl -sf -X POST "http://${serverIp}:${httpPort}/api/progress" \\
         -H "Content-Type: application/json" \\
         -d "{\\"mac\\":\\"$mac\\",\\"stage\\":\\"$stage\\",\\"detail\\":\\"$detail\\"}" 2>/dev/null || true
@@ -359,9 +159,6 @@ bastion_progress() {
 bastion_progress "partitioning" "detecting disk"
 
 VG="${vg}"
-
-${dataDiskResolveBlocks}
-
 ${diskLine}
 
 REPROVISION=no
@@ -371,16 +168,6 @@ if vgs $VG &>/dev/null; then
     echo "=== Existing VG found - reprovision mode ==="
     REPROVISION=yes
 
-    # Which layout is ALREADY on this disk? The LVM tag is authoritative -- the
-    # schemaVersion the caller asked for is irrelevant here, because silently
-    # re-laying-out an existing machine destroys its data. A v1 box reinstalls
-    # as v1 forever, until it is deliberately migrated.
-    DISK_SCHEMA=v1
-    if vgs --noheadings -o vg_tags $VG 2>/dev/null | grep -q lab-schema-v2; then
-        DISK_SCHEMA=v2
-    fi
-    echo "On-disk schema: $DISK_SCHEMA"
-
     # Detect which data LVs to preserve
     PRESERVE_LONGHORN=no; PRESERVE_SRV=no; PRESERVE_HOME=no; PRESERVE_RANCHER=no
     lvs $VG/longhorn &>/dev/null && PRESERVE_LONGHORN=yes
@@ -398,19 +185,6 @@ else
     bastion_progress "partitioning" "fresh install on $DISK"
 fi
 
-${dataDiskDecisionBlocks}
-
-# Every disk anaconda may touch; everything else stays invisible to it.
-KS_DISKS="$DISK"
-[ -n "$LONGHORN_DISK" ] && KS_DISKS="$KS_DISKS,$LONGHORN_DISK"
-[ -n "$RANCHER_DISK" ] && [ "$RANCHER_DISK" != "$LONGHORN_DISK" ] && KS_DISKS="$KS_DISKS,$RANCHER_DISK"
-
-# Fresh installs wipe the OS disk plus any data disk being formatted for the
-# first time -- NEVER one carrying a lab-* label (those are reused --noformat).
-CLEARPART_DRIVES="$DISK"
-[ -n "$LONGHORN_DISK" ] && [ "$LONGHORN_REUSE" = "no" ] && CLEARPART_DRIVES="$CLEARPART_DRIVES,$LONGHORN_DISK"
-[ -n "$RANCHER_DISK" ] && [ "$RANCHER_REUSE" = "no" ] && [ "$RANCHER_DISK" != "$LONGHORN_DISK" ] && CLEARPART_DRIVES="$CLEARPART_DRIVES,$RANCHER_DISK"
-
 if [ "$REPROVISION" = "yes" ]; then
     # Find existing boot partitions by type
     EFI_PART=$(blkid -t TYPE=vfat -o device /dev/\${DISK}* 2>/dev/null | head -1)
@@ -421,26 +195,16 @@ if [ "$REPROVISION" = "yes" ]; then
 
     # Build partition config reusing existing PV/VG
     cat > /tmp/part.ks << PARTEOF
-ignoredisk --only-use=$KS_DISKS
+ignoredisk --only-use=$DISK
 clearpart --none
 part /boot/efi --onpart=$EFI_PART --fstype=efi
 part /boot --onpart=$BOOT_PART --fstype=ext4
 volgroup ${vg} --useexisting --noformat
-PARTEOF
-
-    # OS volumes are recreated, so they must match the layout already on disk.
-    if [ "$DISK_SCHEMA" = "v2" ]; then
-        cat >> /tmp/part.ks << PARTEOF
-${v2OsLogvolLines}
-PARTEOF
-    else
-        cat >> /tmp/part.ks << PARTEOF
 logvol swap --vgname=${vg} --name=swap --fstype=swap --size=27648
 logvol / --vgname=${vg} --name=root --fstype=xfs --size=33792
 logvol /var --vgname=${vg} --name=var --fstype=xfs --size=102400
 logvol /var/log --vgname=${vg} --name=varlog --fstype=xfs --size=10240
 PARTEOF
-    fi
 
     # Preserve or recreate data LVs
     if [ "$PRESERVE_HOME" = "yes" ]; then
@@ -465,19 +229,7 @@ PARTEOF
 
 else
     # Fresh install
-${schemaVersion === 2 ? `    cat > /tmp/part.ks << PARTEOF
-ignoredisk --only-use=$KS_DISKS
-clearpart --all --initlabel --drives=$CLEARPART_DRIVES
-part /boot/efi --fstype=efi --size=${V2_BOOT.efiMib} --ondisk=$DISK
-part /boot --fstype=ext4 --size=${V2_BOOT.bootMib} --ondisk=$DISK
-part pv.01 --size=1 --grow --ondisk=$DISK
-volgroup ${vg} pv.01
-${v2LogvolLines}
-PARTEOF
-    # Tag the VG so a later reprovision knows this disk is v2 WITHOUT trusting
-    # whatever schemaVersion the caller happens to pass. The tag lives on the
-    # VG, which a reprovision preserves; a marker file would not survive.
-    SCHEMA_TAG_PENDING=v2` : `    cat > /tmp/part.ks << PARTEOF
+    cat > /tmp/part.ks << PARTEOF
 ignoredisk --only-use=$DISK
 clearpart --all --initlabel --drives=$DISK
 part /boot/efi --fstype=efi --size=600 --ondisk=$DISK
@@ -492,11 +244,9 @@ logvol /home --vgname=${vg} --name=home --fstype=xfs --size=10240
 logvol /srv --vgname=${vg} --name=srv --fstype=xfs --size=20480
 ${longhornFreshLine}
 ${rancherFreshLine}
-PARTEOF`}
+PARTEOF
 fi
 
-${dataDiskPartBlocks}
-
 echo "=== Generated partition config ==="
 cat /tmp/part.ks
 echo "==================================="
@@ -541,9 +291,6 @@ tar
 socat
 conntrack-tools
 ethtool`}
-${needsEtcdctl ? `
-# etcdctl, for recovering etcd membership without internet access
-etcd` : ""}
 
 # Boot management
 efibootmgr
@@ -568,7 +315,7 @@ set -x
 # Progress callback helper
 bastion_progress() {
     local stage="$1" detail="\${2:-}"
-    ${macLine}
+    local mac=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')
     curl -sf -X POST "http://${serverIp}:${httpPort}/api/progress" \\
         -H "Content-Type: application/json" \\
         -d "{\\"mac\\":\\"$mac\\",\\"stage\\":\\"$stage\\",\\"detail\\":\\"$detail\\"}" 2>/dev/null || true
@@ -614,21 +361,6 @@ fs.inotify.max_user_watches         = 1048576
 SYSCTL
 sysctl --system || true
 
-# -- IPv6 link-local address generation: EUI-64, fleet-wide --
-# A cluster node takes its IPv6 from a MAC-keyed DHCPv6 reservation. kea can only
-# match that reservation when it can recover the node's MAC, and for a modern
-# client that sends a DUID-UUID (no MAC in it) the only place kea can find one is
-# an EUI-64 link-local. NetworkManager's default is stable-privacy (RFC 7217),
-# whose link-local hides the MAC -- so a node on the default silently never gets
-# its reserved address, and with a reservations-only subnet it gets nothing at
-# all. Proven on 2026-09-06: worker0/worker2 (eui64) bound; worker1/spark
-# (default) did not, until flipped. Setting it here means a new node is correct
-# from first boot, before its connection is ever activated.
-cat > /etc/NetworkManager/conf.d/10-ipv6-eui64.conf << 'NMEUI64'
-[connection]
-ipv6.addr-gen-mode=eui64
-NMEUI64
-
 # -- Disable firewalld permanently (k3s/Cilium manage iptables directly) --
 # Note: no '--now' — systemd is not running in the Anaconda chroot
 systemctl disable firewalld || true
@@ -656,30 +388,12 @@ bastion_progress "post-install" "3-bootorder done"
 # -- Enable SysRq magic keys (for emergency reboot via Alt+SysRq+REISUB) --
 echo "kernel.sysrq=1" > /etc/sysctl.d/90-sysrq.conf
 
-# -- Disk schema tag --
-# Tag the VG so a future reprovision can tell which layout is on this disk
-# WITHOUT trusting whatever the caller passes. Deliberately derived from what
-# actually landed on disk rather than from render-time intent: a v1 machine
-# reinstalled with --schema-version 2 keeps its v1 layout (see %pre), and must
-# not end up wearing a v2 tag it does not deserve.
-V2_ROOT_MIB=${v2RootMib}
-ROOT_MIB=$(lvs --noheadings --units m --nosuffix -o lv_size ${vg}/root 2>/dev/null | tr -d ' ' | cut -d. -f1)
-if [ "$ROOT_MIB" = "$V2_ROOT_MIB" ]; then
-    vgchange --addtag lab-schema-v2 ${vg} >/dev/null 2>&1 || true
-    echo "Tagged ${vg} as lab-schema-v2 (root=\${ROOT_MIB}M)"
-    DISK_SCHEMA_ON_DISK=2
-else
-    DISK_SCHEMA_ON_DISK=1
-    echo "Layout is v1 (root=\${ROOT_MIB}M, v2 would be \${V2_ROOT_MIB}M)"
-fi
-
 # -- Provisioning metadata --
 cat > /etc/lab-provisioned << PROVEOF
 hostname: ${fqdn}
 role: ${role}
 provisioned: $(date -Iseconds)
 bastion: ${serverIp}
-schema: v$DISK_SCHEMA_ON_DISK
 PROVEOF
 
 cat > /root/README << 'README'

bastion/src/bastion/src/templates/ubuntu-autoinstall.ts

@@ -2,17 +2,10 @@
 // Equivalent of the Fedora kickstart: LVM partitioning, packages,
 // SSH keys, k3s prereqs, progress callbacks.
 
-import { v2Layout, type LvSpec } from "@lab/shared";
-import type { Role } from "@lab/shared";
-
 export interface UbuntuAutoinstallParams {
   hostname: string;
   disk: string;
-  role: Role;
-  /** PXE MAC the install is keyed under; baked into the progress callback. */
-  mac?: string;
-  /** Fresh-install layout. Defaults to 1 so existing callers are unchanged. */
-  schemaVersion?: 1 | 2;
+  role: string;  // "vanilla" | "worker" | "infra"
   domain: string;
   ubuntuVersion: string;
   timezone: string;
@@ -28,7 +21,6 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
     hostname,
     disk,
     role,
-    mac,
     domain,
     timezone,
     serverIp,
@@ -37,56 +29,8 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
     adminUser,
   } = params;
 
-  // MAC reported by the completion callback: the queue key when known;
-  // first-NIC runtime detection only as fallback (wrong on multi-NIC boards).
-  const reportMac = mac !== undefined && mac !== ""
-    ? mac
-    : `$(ip link show | awk "/ether/ && !/00:00:00:00/ {print \\$2; exit}")`;
-
   const fqdn = domain ? `${hostname}.${domain}` : hostname;
   const vg = "labvg";
-  // ---------------------------------------------------------------------------
-  // v1 LAYOUT -- DO NOT REPLACE THESE WITH CAPABILITY LOOKUPS.
-  //
-  // These literals are deliberately frozen. Every machine already in the fleet
-  // was partitioned by them, and `labctl provision reprovision` must reproduce
-  // that same layout byte-for-byte or it destroys data. Yes, they are wrong:
-  // `labcontroller` runs k3s and gets no rancher LV, and `infra` nodes hold
-  // every Longhorn replica in the homelab cluster while only `worker` gets a
-  // longhorn LV. Those are fixed in the v2 layout, which new boxes get; see
-  // schemaVersion. Changing them here rewrites history for existing hardware.
-  // ---------------------------------------------------------------------------
-  // ---------------------------------------------------------------------------
-  // v2 LAYOUT, rendered from the SAME definition Fedora and Asahi use
-  // (shared/src/types/layout.ts). Before this, Ubuntu sized rancher at 20G
-  // while the kickstart used 120G -- the drift that the 2026-08 imagefs
-  // eviction incident was caused by.
-  //
-  // Indentation is load-bearing: list items sit at 6 spaces, their keys at 8.
-  // At any other depth curtin's YAML does not parse. See the v1 comment below.
-  // ---------------------------------------------------------------------------
-  const schemaVersion = params.schemaVersion ?? 1;
-  const renderV2Lv = (lv: LvSpec): string => {
-    const gib = Math.round(lv.sizeMib / 1024);
-    const mountPath = lv.fs === "swap" ? "none" : lv.mount;
-    return [
-      `      - id: lv-${lv.name}`,
-      `        name: ${lv.name}`,
-      `        type: lvm_partition`,
-      `        volgroup: vg0`,
-      `        size: ${gib}G`,
-      `      - id: fs-${lv.name}`,
-      `        type: format`,
-      `        volume: lv-${lv.name}`,
-      `        fstype: ${lv.fs}`,
-      `      - id: mount-${lv.name}`,
-      `        type: mount`,
-      `        device: fs-${lv.name}`,
-      `        path: ${mountPath}`,
-    ].join("\n");
-  };
-  const v2LvBlock = v2Layout(role).map(renderV2Lv).join("\n");
-
   const hasLonghorn = role === "worker";
   const hasRancher = role === "infra";
 
@@ -96,39 +40,34 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
   // Build the LVM layout to match Fedora kickstart sizes
   const extraLvs: string[] = [];
   if (hasLonghorn) {
-    // 6 spaces for the list item, 8 for its keys -- these are siblings of the
-    // lv-home/lv-srv entries in storage.config, which sit at 6. At 8 the
-    // rendered document is not valid YAML at all ("expected <block end>, but
-    // found '-'"), so an Ubuntu node with the longhorn role could never have
-    // installed. Ubuntu + longhorn is exactly the worker shape.
-    extraLvs.push(`      - id: lv-longhorn
-        name: longhorn
-        type: lvm_partition
-        volgroup: vg0
-        size: -1
-      - id: fs-longhorn
-        type: format
-        volume: lv-longhorn
-        fstype: xfs
-      - id: mount-longhorn
-        type: mount
-        device: fs-longhorn
-        path: /var/lib/longhorn`);
+    extraLvs.push(`        - id: lv-longhorn
+          name: longhorn
+          type: lvm_partition
+          volgroup: vg0
+          size: -1
+        - id: fs-longhorn
+          type: format
+          volume: lv-longhorn
+          fstype: xfs
+        - id: mount-longhorn
+          type: mount
+          device: fs-longhorn
+          path: /var/lib/longhorn`);
   }
   if (hasRancher) {
-    extraLvs.push(`      - id: lv-rancher
-        name: rancher
-        type: lvm_partition
-        volgroup: vg0
-        size: 20G
-      - id: fs-rancher
-        type: format
-        volume: lv-rancher
-        fstype: xfs
-      - id: mount-rancher
-        type: mount
-        device: fs-rancher
-        path: /var/lib/rancher`);
+    extraLvs.push(`        - id: lv-rancher
+          name: rancher
+          type: lvm_partition
+          volgroup: vg0
+          size: 20G
+        - id: fs-rancher
+          type: format
+          volume: lv-rancher
+          fstype: xfs
+        - id: mount-rancher
+          type: mount
+          device: fs-rancher
+          path: /var/lib/rancher`);
   }
 
   const extraLvsBlock = extraLvs.length > 0 ? "\n" + extraLvs.join("\n") : "";
@@ -142,11 +81,6 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
     `curtin in-target -- bash -c 'cat > /etc/modules-load.d/k3s.conf << EOF\nbr_netfilter\noverlay\nip_conntrack\nEOF'`,
     // Sysctl for k3s networking
     `curtin in-target -- bash -c 'cat > /etc/sysctl.d/90-k3s.conf << EOF\nnet.bridge.bridge-nf-call-iptables  = 1\nnet.bridge.bridge-nf-call-ip6tables = 1\nnet.ipv4.ip_forward                 = 1\nnet.ipv6.conf.all.forwarding        = 1\nfs.inotify.max_user_instances       = 524288\nfs.inotify.max_user_watches         = 1048576\nEOF'`,
-    // IPv6 link-local = EUI-64, so a MAC-keyed DHCPv6 reservation can match: kea
-    // recovers the node's MAC from an EUI-64 link-local when the client sends a
-    // DUID-UUID (no MAC in it). NM's stable-privacy default hides the MAC and the
-    // node silently never gets its reserved address. Proven 2026-09-06.
-    `curtin in-target -- bash -c 'cat > /etc/NetworkManager/conf.d/10-ipv6-eui64.conf << EOF\n[connection]\nipv6.addr-gen-mode=eui64\nEOF'`,
     // Disable ufw firewall
     `curtin in-target -- systemctl disable ufw || true`,
     // Enable chrony/ntp
@@ -184,23 +118,10 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
 
   // Progress callback (complete)
   lateCommands.push(
-    `curtin in-target -- bash -c 'IP_ADDR=$(ip -4 addr show | awk "/inet / && !/127.0.0/ {split(\\$2,a,\\"/\\"); print a[1]; exit}"); curl -sf -X POST "http://${serverIp}:${httpPort}/api/progress" -H "Content-Type: application/json" -d "{\\"mac\\":\\"${reportMac}\\",\\"stage\\":\\"complete\\",\\"detail\\":\\"ready at $IP_ADDR\\"}" || true'`,
+    `curtin in-target -- bash -c 'IP_ADDR=$(ip -4 addr show | awk "/inet / && !/127.0.0/ {split(\\$2,a,\\"/\\"); print a[1]; exit}"); curl -sf -X POST "http://${serverIp}:${httpPort}/api/progress" -H "Content-Type: application/json" -d "{\\"mac\\":\\"$(ip link show | awk "/ether/ && !/00:00:00:00/ {print \\$2; exit}")\\",\\"stage\\":\\"complete\\",\\"detail\\":\\"ready at $IP_ADDR\\"}" || true'`,
   );
 
-  // JSON.stringify, not `"${c}"`. JSON is a subset of YAML, so this produces a
-  // correctly escaped double-quoted scalar for free -- and the naive version
-  // was broken in two ways at once, for every role:
-  //
-  //   * embedded double quotes ended the scalar early
-  //     (`echo "tmpfs /tmp ..." >> /etc/fstab` -> "expected <block end>")
-  //   * the heredocs contain REAL newlines, and YAML folds newlines inside a
-  //     double-quoted scalar into spaces -- so even where it parsed, the
-  //     heredoc arrived at the target as one long line and wrote a file with
-  //     no line breaks.
-  //
-  // JSON escaping turns the newlines into \n, which YAML unescapes back to
-  // real newlines on parse, so the heredoc survives intact.
-  const lateCommandsYaml = lateCommands.map((c) => `        - ${JSON.stringify(c)}`).join("\n");
+  const lateCommandsYaml = lateCommands.map((c) => `        - "${c}"`).join("\n");
 
   return `#cloud-config
 autoinstall:
@@ -218,30 +139,6 @@ autoinstall:
     allow-pw: false
     authorized-keys:
 ${sshKeysYaml}
-  # Both address families. Without dhcp6 the installer's default is IPv4-only,
-  # so a node provisioned into a dual-stack cluster comes up with no IPv6, k3s
-  # has no v6 node-ip to bind, and it joins as an IPv4-only member of a
-  # dual-stack cluster -- which surfaces later as pods on that node being
-  # unreachable over v6 while the node itself reads Ready.
-  #
-  # The address itself comes from a kea DHCPv6 reservation keyed on MAC, the
-  # same source of truth as the v4 address, so nothing here needs to know it.
-  #
-  # optional: true matters -- it lets the install proceed if the v6 lease is
-  # slow or the VLAN has no DHCPv6 yet, rather than blocking on a timeout. The
-  # node still needs the address before k3s starts, but that is a later step's
-  # problem and failing the OS install over it would be worse.
-  # (No backticks in this comment: it lives inside a TS template literal, and a
-  # backtick here ends the literal and breaks the build.)
-  network:
-    version: 2
-    ethernets:
-      primary:
-        match:
-          name: "en*"
-        dhcp4: true
-        dhcp6: true
-        optional: true
   storage:
     config:
       - id: disk0
@@ -285,7 +182,7 @@ ${sshKeysYaml}
         name: ${vg}
         devices:
           - part-pv
-${schemaVersion === 2 ? v2LvBlock : `      - id: lv-swap
+      - id: lv-swap
         name: swap
         type: lvm_partition
         volgroup: vg0
@@ -362,7 +259,7 @@ ${schemaVersion === 2 ? v2LvBlock : `      - id: lv-swap
       - id: mount-srv
         type: mount
         device: fs-srv
-        path: /srv${extraLvsBlock}`}
+        path: /srv${extraLvsBlock}
   packages:
     - openssh-server
     - curl

bastion/src/bastion/src/templates/vyos-boot.ipxe.ts

@@ -1,53 +0,0 @@
-// iPXE boot script template for VyOS network install.
-//
-// VyOS ships no unattended installer: `install image` is unconditionally
-// interactive (image_installer.py's install action takes no arguments, and
-// --no-prompt is wired only to `add`). So PXE boots the *live* system and the
-// automation is injected through live-config's `hooks` component, which fetches
-// a script over HTTP and runs it as root late in live boot.
-//
-// Unlike the Fedora/Ubuntu paths this boots a live image rather than an
-// installer, so there is no kickstart/autoinstall equivalent — see
-// routes/vyos.ts for the hook that actually drives the install.
-
-export function renderVyosInstallIpxe(params: {
-  mac: string;
-  hostname: string;
-  serverIp: string;
-  httpPort: number;
-}): string {
-  const base = `http://${params.serverIp}:${params.httpPort}`;
-
-  // Pin the boot NIC by MAC. live-boot otherwise scans for the first
-  // *connected* interface, and on a multi-NIC box that race is lost by
-  // whichever port negotiates slowest: on the Protectli VP2440 the SFP+
-  // pair links first, so live-boot picked the fiber ports (which have no
-  // DHCP), burned 15s per port, and gave up with "Unable to find a live
-  // file system on the network" -- while the copper port that actually PXE
-  // booted came up at 4.6s and was never tried.
-  //
-  // live-boot's Device_from_bootif() strips the "01-" and matches the MAC
-  // against /sys/class/net/*. params.mac is the dispatch key, i.e. exactly
-  // the NIC that PXE booted -- more reliable than iPXE's ${net0} on a box
-  // where the booting NIC may not be net0.
-  const bootif = `01-${params.mac.toLowerCase().replace(/:/g, "-")}`;
-
-  // Deliberately NOT passing `nonetworking` (present in VyOS's own PXE docs):
-  // live-config's hook component needs networking up to fetch the hook over
-  // HTTP. Also no `console=ttyS0` — on hardware without a physical UART that
-  // costs 30s at every systemd boot phase.
-  return `#!ipxe
-
-echo
-echo =============================================
-echo   Lab PXE Bastion - INSTALLING VyOS
-echo   Target: ${params.hostname}
-echo   MAC:    ${params.mac}
-echo =============================================
-echo
-
-kernel ${base}/vyos-vmlinuz boot=live nopersistence noautologin BOOTIF=${bootif} fetch=${base}/vyos-filesystem.squashfs live-config.hooks=${base}/vyos/autoinstall.sh?mac=${params.mac}
-initrd ${base}/vyos-initrd
-boot
-`;
-}

bastion/src/bastion/src/templates/vyos-config-spec.ts

@@ -1,352 +0,0 @@
-// Builds the VyOS configuration spec applied by the autoinstall hook.
-//
-// We deliberately do NOT emit a config.boot file as text. A config.boot carries a
-// `vyos-config-version` trailer; without a trailer matching the running image,
-// VyOS runs its migration scripts from version 0 on first boot. Instead the hook
-// loads the image's own /opt/vyatta/etc/config.boot.default through vyos.configtree
-// and applies these set operations on top, so syntax and version trailer always
-// match the exact image being installed.
-
-import type { VyosInstallSpec } from "@lab/shared";
-
-export interface VyosSetOp {
-  path: string[];
-  value?: string;
-  /** false appends to a multi-value node (e.g. bond members) instead of replacing. */
-  replace?: boolean;
-}
-
-export interface VyosConfigSpec {
-  hostname: string;
-  /** "" means accept the installer default (the running image's version string). */
-  imageName: string;
-  password: string;
-  console: "K" | "S";
-  /** Target disk name (e.g. "nvme0n1"); "" accepts the installer's first-disk default. */
-  disk: string;
-  /**
-   * IP the driver should report in the "complete" callback ("ready at <ip>" —
-   * the exact format routes/api.ts parses installed.ip from). The mgmt
-   * address when static; "" means detect the live DHCP address at runtime.
-   */
-  reportAddress: string;
-  /** Whether to accept RAID-1 when the installer finds more than one disk. */
-  raid: boolean;
-  /** Overwrite the installed config.boot with the generated one on reinstall. */
-  freshConfig: boolean;
-  sets: VyosSetOp[];
-  /** Paths that are VyOS tag nodes — must be marked as such in the ConfigTree. */
-  tags: string[][];
-}
-
-/**
- * Normalise a target disk to the form the installer expects.
- *
- * find_disks() enumerates via `lsblk -Jbp` (-p = full paths), so its valid
- * responses are "/dev/mmcblk0"-style. A bare "mmcblk0" is rejected by
- * ask_input()'s valid_responses check and re-prompts forever.
- */
-function normalizeDiskPath(value: string | undefined): string {
-  const raw = (value ?? "").trim();
-  if (raw === "") return "";
-  return raw.startsWith("/dev/") ? raw : `/dev/${raw}`;
-}
-
-/** Sentinel marking a value that lives in Pulumi config, not in the bundle. */
-const SECRET_PREFIX = "@secret:";
-
-/**
- * Enable the VyOS HTTP API so the router is manageable the moment it boots.
- *
- * This belongs at install time rather than in the Pulumi model: the model is
- * applied THROUGH this API, so a router that lacks it cannot be brought under
- * management without a hand-run change on a live firewall. It is also why the
- * model excludes `service https` outright -- a provider able to rewrite its own
- * transport can lock itself out permanently.
- *
- * `listen-address` is always set. Leaving it unbound would expose a
- * config-write endpoint on every segment the router touches, the WAN included.
- */
-function apiSets(apiKey: string, listenAddress: string): VyosSetOp[] {
-  const sets: VyosSetOp[] = [
-    { path: ["service", "https", "api", "keys", "id", "pulumi", "key"], value: apiKey },
-    { path: ["service", "https", "api", "rest"] },
-  ];
-  if (listenAddress !== "") {
-    sets.push({ path: ["service", "https", "listen-address"], value: listenAddress });
-  }
-  return sets;
-}
-
-/** Tag nodes introduced by the API config, needed by the installer's ConfigTree. */
-const API_TAGS: string[][] = [["service", "https", "api", "keys", "id"]];
-
-/**
- * The address to bind the API to: an explicit choice, else the management
- * address with its prefix length stripped. Under DHCP there is no address to
- * bind at build time, so the caller must pass one or the listener stays unbound
- * and the API is not enabled at all.
- */
-function apiListenAddress(spec: VyosInstallSpec, mgmtAddress: string): string {
-  if (spec.apiListenAddress !== undefined && spec.apiListenAddress !== "") {
-    return spec.apiListenAddress;
-  }
-  return mgmtAddress.includes("/") ? (mgmtAddress.split("/")[0] ?? "") : "";
-}
-
-/**
- * Use a Pulumi-rendered bundle as the router's config verbatim.
- *
- * Secret-valued nodes are dropped rather than installed with their sentinel
- * text: writing `@secret:pppoePassword` into config.boot would look configured
- * while being wrong, which is worse than being absent. The router comes up
- * without those values and the first `pulumi up` fills them in.
- *
- * `system host-name` is forced to the hostname the install was asked for. The
- * bundle carries the name of whichever router it was exported from, and
- * installing vyos001's hostname onto vyos002 would collide on the network.
- */
-function buildFromBundle(
-  params: { hostname: string; defaultPassword: string; disk?: string | undefined },
-  spec: VyosInstallSpec,
-  bundle: NonNullable<VyosInstallSpec["bundle"]>,
-  mgmtAddress: string,
-): VyosConfigSpec {
-  const sets: VyosSetOp[] = [];
-  const dropped: string[] = [];
-  for (const op of bundle.sets) {
-    if (op.value !== undefined && op.value.startsWith(SECRET_PREFIX)) {
-      dropped.push(op.path.join(" "));
-      continue;
-    }
-    if (op.path.length === 2 && op.path[0] === "system" && op.path[1] === "host-name") {
-      continue;
-    }
-    sets.push({
-      path: op.path,
-      ...(op.value === undefined ? {} : { value: op.value }),
-      ...(op.replace === undefined ? {} : { replace: op.replace }),
-    });
-  }
-  sets.unshift({ path: ["system", "host-name"], value: params.hostname });
-
-  if (dropped.length > 0) {
-    console.warn(
-      `vyos ${params.hostname}: ${dropped.length} secret-valued node(s) left unset by the ` +
-        `bundle; run \`pulumi up\` to supply them: ${dropped.join(", ")}`,
-    );
-  }
-
-  const tags = [...bundle.tags];
-  const api = enableApi(spec, params.hostname, mgmtAddress);
-  if (api.length > 0) {
-    sets.push(...api);
-    tags.push(...API_TAGS);
-  }
-
-  return {
-    hostname: params.hostname,
-    imageName: "",
-    password: spec.password ?? params.defaultPassword,
-    console: "K",
-    disk: normalizeDiskPath(params.disk),
-    reportAddress: mgmtAddress.includes("/") ? (mgmtAddress.split("/")[0] ?? "") : "",
-    raid: false,
-    freshConfig: spec.freshConfig ?? false,
-    sets,
-    tags,
-  };
-}
-
-/**
- * The API config for this install, or nothing when it cannot be enabled safely.
- *
- * Refusing to enable it unbound is deliberate. Under DHCP there is no address
- * known at build time, and the alternative -- binding to every interface --
- * would publish a config-write endpoint on the WAN. Better to leave the router
- * SSH-only and say so than to open it everywhere.
- */
-function enableApi(spec: VyosInstallSpec, hostname: string, mgmtAddress: string): VyosSetOp[] {
-  if (spec.apiKey === undefined || spec.apiKey === "") return [];
-  const listen = apiListenAddress(spec, mgmtAddress);
-  if (listen === "") {
-    console.warn(
-      `vyos ${hostname}: --vyos-api-key given but no address to bind to ` +
-        `(management is "${mgmtAddress}"). Pass --vyos-api-listen <addr>; the HTTP API ` +
-        `has NOT been enabled, so Pulumi cannot manage this router yet.`,
-    );
-    return [];
-  }
-  return apiSets(spec.apiKey, listen);
-}
-
-export function buildVyosConfigSpec(params: {
-  hostname: string;
-  spec?: VyosInstallSpec | undefined;
-  defaultPassword: string;
-  sshKeys?: string[] | undefined;
-  disk?: string | undefined;
-}): VyosConfigSpec {
-  const spec = params.spec ?? {};
-  const mgmt = spec.mgmtInterface ?? "eth0";
-  const mgmtAddress = spec.mgmtAddress ?? "dhcp";
-
-  // A rendered bundle replaces the derived config entirely. Deriving a second
-  // opinion alongside it is the drift the bundle exists to prevent: Pulumi and
-  // labctl would each believe they knew the router's config, and the box would
-  // end up with whichever ran last.
-  if (spec.bundle !== undefined) {
-    return buildFromBundle(params, spec, spec.bundle, mgmtAddress);
-  }
-  const bondMembers = spec.bondMembers ?? [];
-  const vlans = spec.vlans ?? [];
-
-  const sets: VyosSetOp[] = [];
-  const tags: string[][] = [
-    ["interfaces", "ethernet"],
-    ["system", "login", "user"],
-  ];
-
-  const hwIds = spec.hwIds ?? {};
-  const pinHwId = (iface: string): void => {
-    const mac = hwIds[iface];
-    if (mac !== undefined && mac !== "") {
-      sets.push({ path: ["interfaces", "ethernet", iface, "hw-id"], value: mac });
-    }
-  };
-
-  sets.push({ path: ["system", "host-name"], value: params.hostname });
-
-  // Management interface — the NIC that PXE booted, left untagged and unbonded.
-  sets.push({ path: ["interfaces", "ethernet", mgmt, "address"], value: mgmtAddress });
-  pinHwId(mgmt);
-
-  // Tagged management VLAN on the PXE port. Emitted regardless of bonding, so
-  // the box stays reachable on the management VLAN while still booting untagged
-  // on whichever VLAN the bastion's proxy DHCP serves.
-  const mgmtVlan = spec.mgmtVlan;
-  if (mgmtVlan !== undefined) {
-    tags.push(["interfaces", "ethernet", mgmt, "vif"]);
-    const vif = ["interfaces", "ethernet", mgmt, "vif", String(mgmtVlan.id)];
-    sets.push({ path: [...vif, "address"], value: mgmtVlan.address });
-    if (mgmtVlan.description !== undefined && mgmtVlan.description !== "") {
-      sets.push({ path: [...vif, "description"], value: mgmtVlan.description });
-    }
-  }
-
-  // LACP bond. Members must exclude the PXE NIC; firmware PXE cannot run over LACP.
-  const bonded = bondMembers.length > 0;
-  if (bonded) {
-    tags.push(["interfaces", "bonding"]);
-    sets.push({ path: ["interfaces", "bonding", "bond0", "mode"], value: "802.3ad" });
-    sets.push({ path: ["interfaces", "bonding", "bond0", "hash-policy"], value: "layer2+3" });
-    for (const member of bondMembers) {
-      sets.push({
-        path: ["interfaces", "bonding", "bond0", "member", "interface"],
-        value: member,
-        replace: false,
-      });
-      pinHwId(member);
-    }
-    // Address on the trunk's native/untagged VLAN.
-    if (spec.bondAddress !== undefined && spec.bondAddress !== "") {
-      sets.push({ path: ["interfaces", "bonding", "bond0", "address"], value: spec.bondAddress });
-    }
-  }
-
-  // VRRP groups accumulate here; emitted (plus a sync group) after the VLANs.
-  // interface accepts dotted vifs (constraint regex `[0-9]+(.\d+)?`), address
-  // is a tag node (the VIP is the tag value itself), vrid range is 1-255.
-  const vrrpGroups: Array<{ name: string; iface: string; vrid: number; vip: string }> = [];
-  if (bonded && spec.bondVrrp !== undefined && spec.bondVrrp !== "") {
-    // vrid 1 for the untagged group: the native VLAN is never a vif, so this
-    // cannot collide with a vlan-id-derived vrid.
-    vrrpGroups.push({ name: "native", iface: "bond0", vrid: 1, vip: spec.bondVrrp });
-  }
-
-  // Tagged VLAN sub-interfaces hang off the bond when there is one, else off mgmt.
-  const parent = bonded
-    ? ["interfaces", "bonding", "bond0"]
-    : ["interfaces", "ethernet", mgmt];
-  if (vlans.length > 0) {
-    tags.push([...parent, "vif"]);
-    const parentName = bonded ? "bond0" : mgmt;
-    for (const vlan of vlans) {
-      const vif = [...parent, "vif", String(vlan.id)];
-      sets.push({ path: [...vif, "address"], value: vlan.address });
-      if (vlan.description !== undefined && vlan.description !== "") {
-        sets.push({ path: [...vif, "description"], value: vlan.description });
-      }
-      if (vlan.vrrp !== undefined && vlan.vrrp !== "") {
-        vrrpGroups.push({
-          name: `vlan${vlan.id}`,
-          iface: `${parentName}.${vlan.id}`,
-          vrid: vlan.id,
-          vip: vlan.vrrp,
-        });
-      }
-    }
-  }
-
-  // Emit VRRP groups plus one sync group so all VLANs fail over together —
-  // without it a single-link event could split mastership across the pair.
-  if (vrrpGroups.length > 0) {
-    tags.push(["high-availability", "vrrp", "group"]);
-    tags.push(["high-availability", "vrrp", "sync-group"]);
-    const priority = String(spec.vrrpPriority ?? 100);
-    for (const g of vrrpGroups) {
-      const base = ["high-availability", "vrrp", "group", g.name];
-      sets.push({ path: [...base, "interface"], value: g.iface });
-      sets.push({ path: [...base, "vrid"], value: String(g.vrid) });
-      sets.push({ path: [...base, "priority"], value: priority });
-      // address is a tag node: the VIP is the path's final segment, no value.
-      sets.push({ path: [...base, "address", g.vip] });
-      tags.push([...base, "address"]);
-      sets.push({
-        path: ["high-availability", "vrrp", "sync-group", "MAIN", "member"],
-        value: g.name,
-        replace: false,
-      });
-    }
-  }
-
-  sets.push({ path: ["service", "ssh", "port"], value: "22" });
-
-  const sshKeys = params.sshKeys ?? [];
-  if (sshKeys.length > 0) {
-    tags.push(["system", "login", "user", "vyos", "authentication", "public-keys"]);
-    sshKeys.forEach((entry, index) => {
-      const parts = entry.trim().split(/\s+/);
-      const type = parts[0] ?? "";
-      const key = parts[1] ?? "";
-      if (!type.startsWith("ssh-") && !type.startsWith("ecdsa-")) return;
-      if (!key) return;
-      const name = parts[2] ?? `lab-key-${index}`;
-      const base = ["system", "login", "user", "vyos", "authentication", "public-keys", name];
-      sets.push({ path: [...base, "type"], value: type });
-      sets.push({ path: [...base, "key"], value: key });
-    });
-  }
-
-  // Enabled here too, not just for bundle installs: every VyOS this bastion
-  // provisions should be manageable from first boot.
-  const api = enableApi(spec, params.hostname, mgmtAddress);
-  if (api.length > 0) {
-    sets.push(...api);
-    tags.push(...API_TAGS);
-  }
-
-  return {
-    hostname: params.hostname,
-    imageName: "",
-    password: spec.password ?? params.defaultPassword,
-    console: "K",
-    disk: normalizeDiskPath(params.disk),
-    // Static mgmt address wins; under DHCP the driver detects the live IP.
-    reportAddress: mgmtAddress.includes("/") ? (mgmtAddress.split("/")[0] ?? "") : "",
-    raid: false,
-    freshConfig: spec.freshConfig ?? false,
-    sets,
-    tags,
-  };
-}

bastion/src/bastion/src/templates/vyos-install.py.ts

@@ -1,514 +0,0 @@
-// Renders the Python program that performs the unattended VyOS install.
-//
-// It runs as root inside the live system, fetched and executed by live-config's
-// `hooks` component (see vyos-boot.ipxe.ts). It does three things:
-//   1. builds config.boot from the image's own default via vyos.configtree
-//   2. drives the interactive `install image` through a pty
-//   3. reports progress back to the bastion, then reboots
-//
-// A pty is used rather than piping stdin because the installer reads the
-// password through getpass(), which opens /dev/tty directly and would ignore a
-// pipe. Prompts are matched by text rather than replayed positionally: the
-// installer skips the boot-config question when it finds a previous
-// installation, so a fixed answer sequence desyncs on reinstall.
-
-import type { VyosConfigSpec } from "./vyos-config-spec.js";
-
-export function renderVyosInstallPy(params: {
-  spec: VyosConfigSpec;
-  mac: string;
-  serverIp: string;
-  httpPort: number;
-  role: string;
-}): string {
-  // Base64 so arbitrary values (passwords, descriptions, SSH keys) can never
-  // terminate the Python string literal that carries them.
-  const specB64 = Buffer.from(JSON.stringify(params.spec), "utf-8").toString("base64");
-
-  return `#!/usr/bin/env python3
-"""Unattended VyOS install driver -- generated by the lab PXE bastion."""
-
-import base64
-import json
-import os
-import pty
-import re
-import select
-import subprocess
-import sys
-import time
-import urllib.request
-
-SPEC = json.loads(base64.b64decode("${specB64}").decode("utf-8"))
-BASTION = "http://${params.serverIp}:${params.httpPort}"
-MAC = "${params.mac}"
-ROLE = ${JSON.stringify(params.role ?? "vanilla")}
-
-INSTALLER = "/usr/libexec/vyos/op_mode/image_installer.py"
-CONFIG_DIR = "/opt/vyatta/etc/config"
-# The installer copies the rootfs from the boot MEDIUM path -- which only a
-# CD/USB boot provides. With fetch= (HTTP netboot) nothing is mounted there
-# (verified in VM: Errno 2), so the squashfs must be linked or re-fetched into
-# place before 'install image' runs.
-ROOTFS_EXPECTED = "/usr/lib/live/mount/medium/live/filesystem.squashfs"
-SQUASHFS_URL = "http://${params.serverIp}:${params.httpPort}/vyos-filesystem.squashfs"
-# The live-config hook runs BEFORE vyos-router creates the /opt/vyatta compat
-# path, so the squashfs's own location must be tried too (verified in VM: only
-# /usr/share/vyos/config.boot.default exists at hook time).
-DEFAULT_CONFIG_CANDIDATES = [
-    "/opt/vyatta/etc/config.boot.default",
-    "/usr/share/vyos/config.boot.default",
-]
-STALL_TIMEOUT = 900  # seconds without installer output before giving up
-
-
-def detect_ip():
-    """Best-effort local IP as seen on the route toward the bastion.
-
-    Matches Fedora's semantics (IP captured during install): under DHCP the
-    installed system will renew on the same NIC/subnet the live env used.
-    """
-    import socket
-    try:
-        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        s.connect(("${params.serverIp}", ${params.httpPort}))
-        ip = s.getsockname()[0]
-        s.close()
-        return ip
-    except Exception:
-        return ""
-
-
-class LogStreamer:
-    """Stream install output to the bastion's /api/log so 'labctl provision
-    logs -f' works live for VyOS, like Anaconda's syslog does for Fedora.
-
-    Strictly best-effort: a failed POST drops the batch and must never stall
-    the pty read loop or fail the install.
-    """
-
-    ANSI = re.compile(rb"\\x1b\\[[0-9;?]*[a-zA-Z]|\\x1b[=>]|\\r")
-
-    def __init__(self):
-        self.partial = b""
-        self.pending = []
-        self.last_flush = time.time()
-
-    def feed(self, chunk):
-        """Raw pty bytes: split into lines, strip ANSI noise, queue."""
-        self.partial += chunk
-        while b"\\n" in self.partial:
-            raw, self.partial = self.partial.split(b"\\n", 1)
-            text = self.ANSI.sub(b"", raw).decode("utf-8", "replace").rstrip()
-            if text:
-                self.pending.append(text)
-        self.maybe_flush()
-
-    def line(self, text):
-        """A driver-originated message (already a clean string)."""
-        self.pending.append(text)
-        self.maybe_flush()
-
-    def maybe_flush(self):
-        if len(self.pending) >= 20 or (self.pending and time.time() - self.last_flush >= 2):
-            self.flush()
-
-    def flush(self):
-        if not self.pending:
-            return
-        batch, self.pending = self.pending[:200], self.pending[200:]
-        self.last_flush = time.time()
-        try:
-            body = json.dumps({"mac": MAC, "lines": batch}).encode()
-            req = urllib.request.Request(
-                BASTION + "/api/log",
-                data=body,
-                headers={"Content-Type": "application/json"},
-            )
-            urllib.request.urlopen(req, timeout=5).read()
-        except Exception:
-            pass
-
-
-STREAM = LogStreamer()
-
-
-def say(msg):
-    """Print locally and stream to the bastion log buffer."""
-    print(msg)
-    STREAM.line(str(msg))
-
-
-def report(stage, detail=""):
-    """Best-effort progress callback; never fatal."""
-    STREAM.flush()
-    try:
-        body = json.dumps({"mac": MAC, "stage": stage, "detail": detail}).encode()
-        req = urllib.request.Request(
-            BASTION + "/api/progress",
-            data=body,
-            headers={"Content-Type": "application/json"},
-        )
-        urllib.request.urlopen(req, timeout=5).read()
-    except Exception:
-        pass
-
-
-def build_config():
-    """Apply our set operations onto the image's own default config.
-
-    Using config.boot.default as the base keeps the vyos-config-version trailer
-    consistent with the running image, so first boot does not run migrations.
-    """
-    from vyos.configtree import ConfigTree
-
-    default_config = next(
-        (p for p in DEFAULT_CONFIG_CANDIDATES if os.path.exists(p)), None)
-    if default_config is None:
-        raise FileNotFoundError(
-            "no config.boot.default found (tried %s)" % ", ".join(DEFAULT_CONFIG_CANDIDATES))
-    say("base config: %s" % default_config)
-
-    with open(default_config) as handle:
-        config = ConfigTree(handle.read())
-
-    for op in SPEC["sets"]:
-        replace = op.get("replace", True)
-        if "value" in op and op["value"] is not None:
-            config.set(op["path"], value=op["value"], replace=replace)
-        else:
-            config.set(op["path"])
-
-    # Tag nodes must be marked after the nodes exist, as the installer itself does.
-    for tag in SPEC["tags"]:
-        try:
-            config.set_tag(tag)
-        except Exception as err:
-            say("warning: set_tag %s failed: %s" % (tag, err))
-
-    os.makedirs(CONFIG_DIR, exist_ok=True)
-    target = os.path.join(CONFIG_DIR, "config.boot")
-
-    # Re-attach the vyos-config-version footer: ConfigTree.to_string() emits
-    # only the config body, and a config without the footer is treated as
-    # ancient -- the boot migrator then runs every migration over it and (as
-    # observed in the VM test) crashes in system/31-to-32. Building the footer
-    # from the running system pins it to the exact image being installed.
-    body = config.to_string()
-    try:
-        from vyos.component_version import version_info_from_system
-        info = version_info_from_system()
-        info.update_config_body(body)
-        info.write(target)
-        say("wrote %s (footer: %s)" % (target, info.release))
-    except Exception as err:
-        say("warning: version footer failed (%s); writing bare config" % err)
-        with open(target, "w") as handle:
-            handle.write(body)
-    return target
-
-
-def find_live_squashfs():
-    """Locate the squashfs live-boot fetched, without walking into the mounted
-    rootfs or overlay (each would mean traversing the entire OS tree)."""
-    explicit = [
-        "/run/live/medium/live/filesystem.squashfs",
-        "/lib/live/mount/medium/live/filesystem.squashfs",
-    ]
-    for path in explicit:
-        if os.path.isfile(path) and os.path.getsize(path) > 0:
-            return path
-    for root in ("/run/live", "/lib/live/mount", "/usr/lib/live/mount"):
-        for dirpath, dirs, files in os.walk(root):
-            depth = dirpath.count(os.sep) - root.count(os.sep)
-            dirs[:] = [d for d in dirs
-                       if d not in ("rootfs", "overlay")
-                       and not d.endswith(".squashfs")
-                       and depth < 3]
-            if "filesystem.squashfs" in files:
-                path = os.path.join(dirpath, "filesystem.squashfs")
-                if os.path.isfile(path) and os.path.getsize(path) > 0:
-                    return path
-    return None
-
-
-def ensure_rootfs():
-    """Make FILE_ROOTFS_SRC exist so the installer can copy the system image."""
-    if os.path.isfile(ROOTFS_EXPECTED) and os.path.getsize(ROOTFS_EXPECTED) > 0:
-        return
-    src = find_live_squashfs()
-    if src is None:
-        say("squashfs not in live mounts; re-fetching %s" % SQUASHFS_URL)
-        src = "/tmp/filesystem.squashfs"
-        urllib.request.urlretrieve(SQUASHFS_URL, src)
-    os.makedirs(os.path.dirname(ROOTFS_EXPECTED), exist_ok=True)
-    if os.path.lexists(ROOTFS_EXPECTED):
-        os.remove(ROOTFS_EXPECTED)
-    os.symlink(src, ROOTFS_EXPECTED)
-    say("rootfs source: %s -> %s" % (ROOTFS_EXPECTED, src))
-
-
-def build_rules():
-    """Prompt -> response table for the interactive installer."""
-    password = SPEC["password"].encode() + b"\\n"
-    image_name = SPEC["imageName"].encode() + b"\\n"
-    disk = SPEC["disk"].encode() + b"\\n"
-    console = SPEC["console"].encode() + b"\\n"
-    raid = (b"yes\\n" if SPEC["raid"] else b"no\\n")
-
-    return [
-        (re.compile(rb"Would you like to continue\\?"), b"yes\\n"),
-        (re.compile(rb"What would you like to name this image\\?"), image_name),
-        (re.compile(rb"Please confirm password for the .vyos. user:"), password),
-        (re.compile(rb"Please enter a password for the .vyos. user:"), password),
-        (re.compile(rb"What console should be used by default"), console),
-        # Three RAID variants: "configure RAID-1 mirroring?", "...on them?",
-        # and "choose two disks for RAID-1 mirroring?" -- all default to YES,
-        # so a missed one both hangs the install and risks an unwanted mirror.
-        (re.compile(rb"Would you like to [^?]*RAID-1 mirroring"), raid),
-        (re.compile(rb"Installation will delete all data on (?:the drive|both drives)\\. Continue\\?"), b"yes\\n"),
-        (re.compile(rb"Which one should be used for installation\\?"), disk),
-        (re.compile(rb"Would you like to use all the free space on the drive\\?"), b"yes\\n"),
-        (re.compile(rb"Which file would you like as boot config\\?"), b"1\\n"),
-        # Reinstall path only (search_previous_installation): carrying the old
-        # /config and SSH host keys forward is VyOS's "reinstall without losing
-        # data". Always yes -- freshConfig replaces config.boot afterwards, so
-        # answering no here would also discard non-config data under /config.
-        (re.compile(rb"Would you like to copy data to the new image\\?"), b"yes\\n"),
-        (re.compile(rb"Would you like to copy the encrypted config to the new image\\?"), b"yes\\n"),
-        # More than one previous image found -- take the first offered.
-        (re.compile(rb"From which image would you like to save config information\\?"), b"1\\n"),
-        (re.compile(rb"From which image would you like to copy the encrypted config\\?"), b"1\\n"),
-    ]
-
-
-def run_installer():
-    """Drive image_installer.py over a pty, answering prompts as they appear."""
-    rules = build_rules()
-    master, slave = pty.openpty()
-
-    proc = subprocess.Popen(
-        [INSTALLER, "--action", "install"],
-        stdin=slave,
-        stdout=slave,
-        stderr=slave,
-        close_fds=True,
-        preexec_fn=os.setsid,
-    )
-    os.close(slave)
-
-    buf = b""
-    transcript = b""   # rolling tail of everything the installer printed
-    last_output = time.time()
-
-    while True:
-        ready, _, _ = select.select([master], [], [], 1.0)
-
-        if ready:
-            try:
-                chunk = os.read(master, 4096)
-            except OSError:
-                break
-            if not chunk:
-                break
-
-            sys.stdout.buffer.write(chunk)
-            sys.stdout.buffer.flush()
-            buf += chunk
-            transcript = (transcript + chunk)[-8000:]
-            STREAM.feed(chunk)
-            last_output = time.time()
-
-            # Answer every prompt currently in the buffer, earliest first, so
-            # ordering is preserved even when the installer skips questions --
-            # and so a single chunk carrying two prompts gets both answers.
-            while True:
-                best = None
-                for pattern, response in rules:
-                    found = pattern.search(buf)
-                    if found and (best is None or found.start() < best[0].start()):
-                        best = (found, response)
-                if best is None:
-                    break
-                found, response = best
-                os.write(master, response)
-                transcript = (transcript + b"\\n>>> answered: " + response)[-8000:]
-                STREAM.line(">>> answered: " + response.decode("utf-8", "replace").strip())
-                buf = buf[found.end():]
-
-            # Bound memory if the installer emits a lot without prompting.
-            if len(buf) > 65536:
-                buf = buf[-8192:]
-
-        elif proc.poll() is not None:
-            break
-
-        STREAM.maybe_flush()
-
-        if time.time() - last_output > STALL_TIMEOUT:
-            proc.kill()
-            raise SystemExit("installer produced no output for %ds" % STALL_TIMEOUT)
-
-    os.close(master)
-    return proc.wait(), transcript.decode("utf-8", "replace")
-
-
-def ensure_network_boot_first():
-    """Keep network boot first so the bastion intercepts every reboot.
-
-    Port of the Fedora kickstart's %post efibootmgr step (install.ks.ts) --
-    what makes reprovision-by-reboot work. Best-effort: skipped on BIOS boots
-    or when efibootmgr is absent. Runs from the live env after the installer;
-    efibootmgr edits NVRAM, not the disk, so installer cleanup is irrelevant.
-    """
-    import shutil
-    if not os.path.isdir("/sys/firmware/efi") or shutil.which("efibootmgr") is None:
-        say("boot order: skipped (BIOS boot or efibootmgr missing)")
-        return
-    try:
-        out = subprocess.run(["efibootmgr"], capture_output=True, text=True, timeout=30).stdout
-        order = []
-        network_entry = None
-        for line in out.splitlines():
-            m = re.match(r"^BootOrder:\\s*(.*)$", line)
-            if m:
-                order = [x.strip() for x in m.group(1).split(",") if x.strip()]
-                continue
-            m = re.match(r"^Boot([0-9A-Fa-f]{4})\\*?\\s+(.*)$", line)
-            if m and network_entry is None:
-                if re.search(r"network|pxe|ipv4|ipv6|http", m.group(2), re.IGNORECASE):
-                    network_entry = m.group(1).upper()
-        if network_entry is None or not order:
-            say("boot order: no network boot entry found; leaving as is")
-            return
-        new_order = [network_entry] + [x for x in order if x.upper() != network_entry]
-        if [x.upper() for x in order] == [x.upper() for x in new_order]:
-            say("boot order: network entry Boot%s already first" % network_entry)
-            return
-        subprocess.run(["efibootmgr", "-o", ",".join(new_order)],
-                       capture_output=True, timeout=30)
-        say("boot order: moved network entry Boot%s first" % network_entry)
-    except Exception as err:
-        say("warning: boot order adjustment failed: %s" % err)
-
-
-def with_target_mounted(fn):
-    """Mount the installed root partition, call fn(rw_dir), always unmount.
-
-    The installer has unmounted and cleaned the target by the time this runs,
-    so the block device is free. The partition holding boot/<image>/rw is the
-    VyOS root; the glob also yields the installed image's rw dir directly.
-    """
-    import glob
-    disk = SPEC["disk"]
-    if not disk:
-        # No pinned disk (installer picked the default) -- enumerate all disks.
-        candidates = ["/dev/" + b for b in os.listdir("/sys/block")
-                      if not b.startswith(("loop", "ram", "zram", "sr"))]
-    else:
-        candidates = [disk]
-
-    mnt = "/mnt/lab-target"
-    os.makedirs(mnt, exist_ok=True)
-    for dev in candidates:
-        name = os.path.basename(dev)
-        parts = sorted(p for p in os.listdir("/sys/block/%s" % name)
-                       if p.startswith(name)) if os.path.isdir("/sys/block/%s" % name) else []
-        for part in parts:
-            pdev = "/dev/" + part
-            if subprocess.run(["mount", pdev, mnt], capture_output=True).returncode != 0:
-                continue
-            try:
-                rw_dirs = glob.glob(os.path.join(mnt, "boot", "*", "rw"))
-                if rw_dirs:
-                    fn(rw_dirs[0])
-                    return True
-            finally:
-                subprocess.run(["umount", mnt], capture_output=True)
-    return False
-
-
-def post_install_target_steps():
-    """Metadata + optional fresh-config overwrite inside the installed image."""
-    def apply(rw_dir):
-        config_dir = os.path.join(rw_dir, "opt/vyatta/etc/config")
-        os.makedirs(config_dir, exist_ok=True)
-
-        # /config/lab-provisioned -- survives VyOS image upgrades. Mirrors the
-        # Fedora kickstart's /etc/lab-provisioned.
-        try:
-            with open(os.path.join(config_dir, "lab-provisioned"), "w") as handle:
-                handle.write("hostname=%s\\n" % SPEC["hostname"])
-                handle.write("role=%s\\n" % ROLE)
-                handle.write("provisioned=%s\\n" % time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
-                handle.write("bastion=%s\\n" % BASTION)
-            say("wrote /config/lab-provisioned")
-        except Exception as err:
-            say("warning: lab-provisioned metadata failed: %s" % err)
-
-        # freshConfig: make the bastion-generated config win over the previous
-        # installation's carried-forward config. Explicit intent -- failure is
-        # fatal (raised out of with_target_mounted).
-        if SPEC.get("freshConfig"):
-            import shutil
-            shutil.copyfile(os.path.join(CONFIG_DIR, "config.boot"),
-                            os.path.join(config_dir, "config.boot"))
-            say("freshConfig: replaced installed config.boot with generated config")
-
-    mounted = with_target_mounted(apply)
-    if not mounted:
-        if SPEC.get("freshConfig"):
-            raise RuntimeError("freshConfig requested but installed root partition not found")
-        say("warning: installed root partition not found; skipping metadata")
-
-
-def main():
-    report("vyos-install", "building config.boot")
-    try:
-        build_config()
-    except Exception as err:
-        report("error", "config generation failed: %s" % err)
-        raise
-
-    report("vyos-install", "staging rootfs for installer")
-    try:
-        ensure_rootfs()
-    except Exception as err:
-        report("error", "rootfs staging failed: %s" % err)
-        raise
-
-    report("vyos-install", "running install image")
-    code, transcript = run_installer()
-
-    if code != 0:
-        # Surface the installer's last words in bastion progress -- the console
-        # they were printed on is usually invisible during unattended installs.
-        report("error", "install image exited %d | tail: %s" % (code, transcript[-4000:]))
-        raise SystemExit(code)
-
-    report("post-install", "boot order + metadata")
-    ensure_network_boot_first()
-    try:
-        post_install_target_steps()
-    except Exception as err:
-        report("error", "post-install target steps failed: %s" % err)
-        raise
-
-    # "complete" is the stage the bastion uses to move a machine out of the
-    # install queue into installed state, and "ready at <ip>" is the exact
-    # detail format it parses installed.ip from -- see routes/api.ts.
-    ip = SPEC.get("reportAddress") or detect_ip()
-    report("complete", "ready at %s" % ip if ip else "VyOS installed, rebooting")
-    os.system("sync")
-    # --force: this driver is a child of live-config.service, whose start job is
-    # still running -- a normal reboot deadlocks waiting for it (verified in VM:
-    # shutdown blocked >1min on "start job is running for live-config"). The
-    # installer has already unmounted and cleaned the target, so an immediate
-    # reboot is safe.
-    os.system("systemctl reboot --force")
-
-
-if __name__ == "__main__":
-    main()
-`;
-}

bastion/src/bastion/tests/arch-dispatch.test.ts

@@ -0,0 +1,291 @@
+// aarch64 support in the PXE dispatch path.
+//
+// The x86_64 side is pinned separately by ipxe-x86-regression.test.ts.
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import type { BastionConfig, BastionState, HardwareInfo } from "@lab/shared";
+import { createApp } from "../src/server.js";
+import { resolveArch } from "../src/routes/dispatch.js";
+import { renderDnsmasqConf } from "../src/templates/dnsmasq.conf.js";
+import type { FastifyInstance } from "fastify";
+import type { StateManager } from "../src/services/state.js";
+
+function createTestConfig(testDir: string): BastionConfig {
+  return {
+    fedoraVersion: "43",
+    arch: "x86_64",
+    httpPort: 0,
+    timezone: "Europe/London",
+    locale: "en_GB.UTF-8",
+    bastionDir: testDir,
+    domain: "test.local",
+    dhcpMode: "proxy",
+    dhcpRangeStart: "",
+    dhcpRangeEnd: "",
+    ubuntuVersion: "26.04",
+    ubuntuMirror: "https://releases.ubuntu.com/26.04",
+    iface: "eth0",
+    serverIp: "10.0.0.1",
+    network: "10.0.0.0",
+    gateway: "10.0.0.1",
+    sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"],
+    adminUser: "testadmin",
+    syslogPort: 15514,
+    skipDnsmasq: true,
+    skipArtifacts: true,
+    fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
+    tftpDir: join(testDir, "tftp"),
+    httpDir: join(testDir, "http"),
+    stateFile: join(testDir, "state.json"),
+  };
+}
+
+function hardware(mac: string, over: Partial<HardwareInfo> = {}): HardwareInfo {
+  return {
+    mac,
+    product: "TestBox",
+    board: "TestBoard",
+    serial: "SN123",
+    manufacturer: "TestCorp",
+    cpu_model: "Test CPU",
+    cpu_cores: 4,
+    memory_gb: 16,
+    arch: "x86_64",
+    disks: [],
+    nics: [],
+    first_seen: new Date().toISOString(),
+    last_seen: new Date().toISOString(),
+    ...over,
+  };
+}
+
+const emptyState = (): BastionState => ({
+  discovered: {}, install_queue: {}, installed: {}, debug: {},
+});
+
+describe("architecture resolution", () => {
+  const config = createTestConfig("/tmp/unused");
+  const mac = "aa:bb:cc:dd:ee:ff";
+
+  it("prefers the tracked record over what the client reports", () => {
+    const state = emptyState();
+    state.discovered[mac] = hardware(mac, { arch: "aarch64" });
+    // Client claims x86_64; the machine record says otherwise and wins.
+    expect(resolveArch(state, mac, "x86_64", config)).toBe("aarch64");
+  });
+
+  it("falls back to the architecture reported at boot", () => {
+    expect(resolveArch(emptyState(), mac, "arm64", config)).toBe("aarch64");
+  });
+
+  it("normalises iPXE's arm64 spelling to aarch64", () => {
+    expect(resolveArch(emptyState(), mac, "arm64", config)).toBe("aarch64");
+    expect(resolveArch(emptyState(), mac, "x86_64", config)).toBe("x86_64");
+  });
+
+  it("falls back to the configured default for unknown architectures", () => {
+    expect(resolveArch(emptyState(), mac, "riscv64", config)).toBe("x86_64");
+    expect(resolveArch(emptyState(), mac, undefined, config)).toBe("x86_64");
+  });
+
+  it("reads arch from the installed record for already-provisioned machines", () => {
+    const state = emptyState();
+    state.installed[mac] = {
+      hostname: "spark", role: "worker", ip: "10.0.0.5",
+      installed_at: new Date().toISOString(), arch: "aarch64",
+    };
+    expect(resolveArch(state, mac, undefined, config)).toBe("aarch64");
+  });
+});
+
+describe("aarch64 dispatch", () => {
+  let testDir: string;
+  let app: FastifyInstance;
+  let state: StateManager;
+  const mac = "aa:bb:cc:dd:ee:ff";
+
+  beforeEach(() => {
+    testDir = join(tmpdir(), `bastion-arch-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+    mkdirSync(join(testDir, "http"), { recursive: true });
+    mkdirSync(join(testDir, "tftp"), { recursive: true });
+    const result = createApp(createTestConfig(testDir));
+    app = result.app;
+    state = result.state;
+  });
+
+  afterEach(async () => {
+    await app.close();
+    rmSync(testDir, { recursive: true, force: true });
+  });
+
+  it("serves the aarch64 kernel and initrd to an arm64 client", async () => {
+    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}&arch=arm64` });
+    expect(res.statusCode).toBe(200);
+    expect(res.body).toContain("/vmlinuz-aarch64");
+    expect(res.body).toContain("/initrd-aarch64.img");
+    expect(res.body).not.toContain("/vmlinuz ");
+  });
+
+  it("points an arm64 client at the aarch64 Fedora mirror", async () => {
+    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}&arch=arm64` });
+    expect(res.body).toContain("Everything/aarch64/os");
+    expect(res.body).not.toContain("Everything/x86_64/os");
+  });
+
+  it("uses serial console arguments and not nomodeset on arm64", async () => {
+    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}&arch=arm64` });
+    expect(res.body).toContain("console=ttyAMA0,115200");
+    expect(res.body).not.toContain("nomodeset");
+  });
+
+  it("refuses to serve the x86-only Ubuntu kernel to an arm64 client", async () => {
+    // A machine queued for Ubuntu before it was discovered as aarch64 reaches dispatch
+    // with no guard having run. Serving it /ubuntu-vmlinuz is the original bug.
+    state.update((s) => {
+      s.install_queue[mac] = {
+        hostname: "arm-node", disk: "", role: "worker",
+        os: "ubuntu-26.04", queued_at: new Date().toISOString(),
+      };
+    });
+
+    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}&arch=arm64` });
+    expect(res.statusCode).toBe(200);
+    expect(res.body).toContain("CANNOT BOOT THIS MACHINE");
+    expect(res.body).toContain("no aarch64 netboot artifacts");
+    expect(res.body).not.toContain("ubuntu-vmlinuz");
+  });
+
+  it("still serves Ubuntu to an x86_64 client", async () => {
+    state.update((s) => {
+      s.install_queue[mac] = {
+        hostname: "x86-node", disk: "", role: "worker",
+        os: "ubuntu-26.04", queued_at: new Date().toISOString(),
+      };
+    });
+
+    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}&arch=x86_64` });
+    expect(res.body).toContain("ubuntu-vmlinuz");
+    expect(res.body).not.toContain("CANNOT BOOT");
+  });
+
+  it("serves a rescue kernel for the recorded architecture, not the requester's", async () => {
+    // The Spark case: machine known to be aarch64, queued for rescue.
+    state.update((s) => {
+      s.discovered[mac] = hardware(mac, { arch: "aarch64" });
+      s.debug[mac] = { hostname: "spark-2935", queued_at: new Date().toISOString() };
+    });
+
+    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}` });
+    expect(res.body).toContain("DEBUG/RESCUE MODE");
+    expect(res.body).toContain("/vmlinuz-aarch64");
+    expect(res.body).toContain("inst.rescue");
+    expect(res.body).toContain("inst.sshd");
+  });
+});
+
+describe("--pxe-boot root device", () => {
+  let testDir: string;
+  let app: FastifyInstance;
+  let state: StateManager;
+  const mac = "aa:bb:cc:dd:ee:ff";
+
+  beforeEach(() => {
+    testDir = join(tmpdir(), `bastion-root-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+    mkdirSync(join(testDir, "http"), { recursive: true });
+    mkdirSync(join(testDir, "tftp"), { recursive: true });
+    const result = createApp(createTestConfig(testDir));
+    app = result.app;
+    state = result.state;
+  });
+
+  afterEach(async () => {
+    await app.close();
+    rmSync(testDir, { recursive: true, force: true });
+  });
+
+  it("uses the root device recorded on the machine", async () => {
+    state.update((s) => {
+      s.installed[mac] = {
+        hostname: "worker-1", role: "worker", ip: "10.0.0.50",
+        installed_at: new Date().toISOString(),
+        root_device: "/dev/mapper/otherVG-root",
+        root_args: "rd.lvm.lv=otherVG/root",
+      };
+      s.debug[mac] = { hostname: "worker-1", queued_at: new Date().toISOString(), pxeBoot: true };
+    });
+
+    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}` });
+    expect(res.body).toContain("PXE BOOT (debug)");
+    expect(res.body).toContain("root=/dev/mapper/otherVG-root");
+    expect(res.body).toContain("rd.lvm.lv=otherVG/root");
+    // The old hardcoded layout must not leak back in.
+    expect(res.body).not.toContain("labvg");
+  });
+
+  it("falls back to rescue rather than guessing when no root device is known", async () => {
+    state.update((s) => {
+      s.installed[mac] = {
+        hostname: "spark-2935", role: "worker", ip: "192.168.8.12",
+        installed_at: new Date().toISOString(), arch: "aarch64",
+      };
+      s.debug[mac] = { hostname: "spark-2935", queued_at: new Date().toISOString(), pxeBoot: true };
+    });
+
+    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}` });
+    expect(res.body).toContain("DEBUG/RESCUE MODE");
+    expect(res.body).toContain("no root device is recorded");
+    expect(res.body).toContain("debug-setup.sh");
+    expect(res.body).not.toContain("root=");
+    // And it is still the right architecture.
+    expect(res.body).toContain("/vmlinuz-aarch64");
+  });
+
+  it("records a root device reported from a rescue shell without erasing hardware info", async () => {
+    state.update((s) => {
+      s.discovered[mac] = hardware(mac, { product: "DGX Spark", manufacturer: "NVIDIA", arch: "aarch64" });
+    });
+
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/discover",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ mac, root_device: "/dev/nvme0n1p2" }),
+    });
+    expect(res.statusCode).toBe(200);
+
+    const hw = state.load().discovered[mac];
+    expect(hw?.root_device).toBe("/dev/nvme0n1p2");
+    // The partial report must not blank what we already knew.
+    expect(hw?.product).toBe("DGX Spark");
+    expect(hw?.cpu_cores).toBe(4);
+    expect(hw?.arch).toBe("aarch64");
+  });
+});
+
+describe("dnsmasq architecture detection", () => {
+  const conf = renderDnsmasqConf(createTestConfig("/tmp/unused"));
+
+  it("maps DHCP option 93 values to per-architecture bootloaders", () => {
+    // 11 = ARM 64-bit UEFI
+    expect(conf).toContain("dhcp-match=set:efi-arm64,option:client-arch,11");
+    expect(conf).toContain("dhcp-boot=tag:efi-arm64,tag:!ipxe,ipxe-arm64.efi");
+    // 7 / 9 = x64 UEFI, 0 = x86 BIOS
+    expect(conf).toContain("dhcp-match=set:efi-x86_64,option:client-arch,7");
+    expect(conf).toContain("dhcp-match=set:efi-x86_64,option:client-arch,9");
+    expect(conf).toContain("dhcp-match=set:bios,option:client-arch,0");
+  });
+
+  it("matches arm64 UEFI HTTP boot on 19, not 20", () => {
+    // IANA: 19 = arm uefi 64 boot from http, 20 = pc/at bios boot from http.
+    expect(conf).toContain("dhcp-match=set:httpboot-arm64,option:client-arch,19");
+    expect(conf).not.toContain("dhcp-match=set:httpboot-arm64,option:client-arch,20");
+    expect(conf).toContain("dhcp-match=set:httpboot-x86_64,option:client-arch,16");
+  });
+
+  it("offers an arm64 PXE service directive in proxy mode", () => {
+    expect(conf).toContain('pxe-service=tag:!ipxe,ARM64_EFI,"PXE Boot",ipxe-arm64.efi');
+  });
+});

bastion/src/bastion/tests/asahi.test.ts

@@ -223,67 +223,3 @@ describe("renderFirstbootUnit", () => {
     expect(unit).toContain("ConditionPathExists=!/etc/lab-lvm-setup-done");
   });
 });
-
-// ---------------------------------------------------------------------------
-// v2 layout. Asahi was the THIRD generator and the most drifted: it hardcoded
-// rancher at 20480M under a comment claiming "sizes match install.ks.ts", which
-// uses 122880M. worker1 still runs that 20G image store today.
-// ---------------------------------------------------------------------------
-describe("v2 layout", () => {
-  const roles = ["vanilla", "worker", "infra", "labcontroller"] as const;
-  const p = (role: (typeof roles)[number], schemaVersion?: 1 | 2) =>
-    renderFirstbootScript({
-      hostname: "mac1", role, serverIp: "10.0.0.1", httpPort: 8080,
-      sshKeys: ["ssh-ed25519 AAAA t"], adminUser: "michal", mac: "aa:bb:cc:dd:ee:ff",
-      ...(schemaVersion ? { schemaVersion } : {}),
-    } as any);
-
-  function lvSizes(script: string): Record<string, number> {
-    const out: Record<string, number> = {};
-    for (const m of script.matchAll(/lvcreate -L (\d+)M\s+-n (\w+)\s+labvg/g)) {
-      out[m[2]!] = Number(m[1]);
-    }
-    return out;
-  }
-
-  it("v2 never creates a root LV — the Asahi installer already made a btrfs root", () => {
-    for (const role of roles) {
-      expect(Object.keys(lvSizes(p(role, 2)))).not.toContain("root");
-    }
-  });
-
-  it("v2 sizes rancher at 250G, not the 20G that undersized worker1", () => {
-    const lvs = lvSizes(p("infra", 2));
-    expect(lvs["rancher"]).toBe(256000);
-    expect(lvs["var"]).toBe(49152);
-    expect(lvs["swap"]).toBe(16384);
-  });
-
-  it("v2 gives a worker a rancher LV; v1 gave it only to servers", () => {
-    expect(lvSizes(p("worker", 2))["rancher"]).toBe(256000);
-    expect(lvSizes(p("worker"))["rancher"]).toBeUndefined();
-  });
-
-  it("v2 never uses 100%FREE", () => {
-    for (const role of roles) {
-      expect(p(role, 2)).not.toContain("100%FREE");
-    }
-  });
-
-  it("v1 still uses 100%FREE for longhorn and is otherwise unchanged", () => {
-    const v1 = p("worker");
-    expect(v1).toContain("lvcreate -l 100%FREE -n longhorn");
-    const lvs = lvSizes(v1);
-    expect(lvs["swap"]).toBe(27648);
-    expect(lvs["var"]).toBe(102400);
-  });
-
-  it("v2 tags the volume group so a later run can detect the layout", () => {
-    expect(p("worker", 2)).toContain("vgchange --addtag lab-schema-v2 labvg");
-    expect(p("worker")).not.toContain("--addtag lab-schema-v2");
-  });
-
-  it("reprovision reports the on-disk schema", () => {
-    expect(p("worker", 2)).toMatch(/vgs --noheadings -o vg_tags labvg.*grep -q lab-schema-v2/);
-  });
-});

bastion/src/bastion/tests/auto-k3s.test.ts

@@ -1,137 +0,0 @@
-// Tests for the post-provision k3s auto-install gate.
-//
-// Two independent switches, for two different problems:
-//
-//  1. shouldAutoInstallK3s(role) — a ROLE gate. The post-provision installer
-//     runs a bare `INSTALL_K3S_EXEC=server|agent` with no K3S_URL and no
-//     K3S_TOKEN and knows about exactly one machine. For a node that is its
-//     own cluster that is fine. For a cluster MEMBER it is wrong: servers get
-//     cluster-init semantics and form their own separate cluster (so three
-//     k8s-combo boxes yield three clusters), and agents cannot start at all
-//     ("Job for k3s-agent.service failed").
-//
-//  2. config.autoK3s — a global off switch, needed because the prerequisite
-//     steps stop k3s and delete CNI state several minutes after an install
-//     lands, which lands on top of a test forming its own cluster.
-
-import { describe, it, expect } from "vitest";
-import { shouldAutoInstallK3s } from "../src/services/post-provision.js";
-import { loadConfig } from "../src/config.js";
-import { SUPPORTED_ROLES, isK3sServer, roleInfo } from "@lab/shared";
-import type { Role } from "@lab/shared";
-
-describe("shouldAutoInstallK3s", () => {
-  // These join a cluster someone else owns, so they cannot be auto-installed
-  // without the server URL and token that post-provision does not have.
-  const CLUSTER_MEMBERS: Role[] = ["k8s-combo", "k8s-controller", "k8s-worker", "worker"];
-
-  // These are each their own cluster (or no k3s at all), so a bare install is
-  // a correct install.
-  const SELF_CONTAINED: Role[] = ["standalone-cluster", "infra", "labcontroller", "vanilla"];
-
-  it.each(CLUSTER_MEMBERS)("refuses to auto-install %s", (role) => {
-    expect(shouldAutoInstallK3s(role)).toBe(false);
-  });
-
-  it.each(SELF_CONTAINED)("still auto-installs %s", (role) => {
-    expect(shouldAutoInstallK3s(role)).toBe(true);
-  });
-
-  it("covers every supported role — a new role must make a deliberate choice", () => {
-    for (const role of SUPPORTED_ROLES) {
-      expect(
-        CLUSTER_MEMBERS.includes(role) || SELF_CONTAINED.includes(role),
-        `role ${role} is in neither list; decide whether post-provision can install it`,
-      ).toBe(true);
-      expect(typeof shouldAutoInstallK3s(role)).toBe("boolean");
-    }
-  });
-
-  it("refuses every k3s AGENT role — an agent with nothing to join cannot start", () => {
-    for (const role of SUPPORTED_ROLES) {
-      if (roleInfo(role).k3sMode === "agent") {
-        expect(shouldAutoInstallK3s(role), `${role} is an agent`).toBe(false);
-      }
-    }
-  });
-
-  it("does not refuse purely on the basis of being a server", () => {
-    // Guard against the gate degenerating into isK3sServer(): labcontroller
-    // and infra are servers AND auto-installable.
-    const servers = SUPPORTED_ROLES.filter(isK3sServer);
-    expect(servers.some((r) => shouldAutoInstallK3s(r))).toBe(true);
-    expect(servers.some((r) => !shouldAutoInstallK3s(r))).toBe(true);
-  });
-});
-
-describe("config.autoK3s", () => {
-  it("defaults to enabled", () => {
-    const config = loadConfig({});
-    expect(config.autoK3s).toBe(true);
-  });
-
-  it("is disabled by an explicit override", () => {
-    const config = loadConfig({ autoK3s: false });
-    expect(config.autoK3s).toBe(false);
-  });
-
-  it("an explicit true override wins over nothing", () => {
-    const config = loadConfig({ autoK3s: true });
-    expect(config.autoK3s).toBe(true);
-  });
-
-  it("AUTO_K3S=false in the environment disables it", () => {
-    const prev = process.env["AUTO_K3S"];
-    process.env["AUTO_K3S"] = "false";
-    try {
-      expect(loadConfig({}).autoK3s).toBe(false);
-    } finally {
-      if (prev === undefined) delete process.env["AUTO_K3S"]; else process.env["AUTO_K3S"] = prev;
-    }
-  });
-
-  it("only the exact string 'false' disables it", () => {
-    const prev = process.env["AUTO_K3S"];
-    process.env["AUTO_K3S"] = "0";
-    try {
-      expect(loadConfig({}).autoK3s).toBe(true);
-    } finally {
-      if (prev === undefined) delete process.env["AUTO_K3S"]; else process.env["AUTO_K3S"] = prev;
-    }
-  });
-});
-
-describe("kickstart packages", () => {
-  // etcd membership recovery is `etcdctl member list/remove`, and `k3s etcdctl`
-  // is NOT a command (verified on a live node: "No help topic for 'etcdctl'").
-  // The binary ships in the `etcd` rpm. The recovery card is written for a
-  // situation with no internet, so telling the operator to dnf-install it there
-  // is not a plan.
-  it("installs etcd on k3s servers and nowhere else", async () => {
-    const { renderInstallKickstart } = await import("../src/templates/install.ks.js");
-    const render = (role: Role): string =>
-      renderInstallKickstart({
-        hostname: "test-node", disk: "nvme0n1", role, domain: "ad.itaz.eu",
-        fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8",
-        serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514,
-        sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal",
-        ...(role === "vanilla" || role === "worker" || role === "infra" || role === "labcontroller"
-          ? {} : { schemaVersion: 2 as const }),
-      });
-    // Scope to %packages: "etcd" appears in paths and comments elsewhere, and a
-    // whole-document match would pass for the wrong reason.
-    const pkgSection = (ks: string): string => {
-      const start = ks.indexOf("%packages");
-      return ks.slice(start, ks.indexOf("%end", start));
-    };
-
-    for (const role of ["infra", "labcontroller", "k8s-combo", "k8s-controller", "standalone-cluster"] as const) {
-      expect(pkgSection(render(role)), `${role} is an etcd member and needs etcdctl offline`)
-        .toMatch(/^etcd$/m);
-    }
-    for (const role of ["vanilla", "worker", "k8s-worker"] as const) {
-      expect(pkgSection(render(role)), `${role} runs no etcd`)
-        .not.toMatch(/^etcd$/m);
-    }
-  });
-});

bastion/src/bastion/tests/data-disks.test.ts

@@ -1,274 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { mkdirSync, rmSync } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import type { FastifyInstance } from "fastify";
-import { renderInstallKickstart } from "../src/templates/install.ks.js";
-import type { InstallKickstartParams } from "../src/templates/install.ks.js";
-import { createApp } from "../src/server.js";
-import { validateDataDisks, v2Layout, DATA_DISK_LABELS, type BastionConfig, type DataDisks } from "@lab/shared";
-
-/**
- * Dedicated whole-disk data volumes (thebeast: 1 TB OS + 2 TB longhorn +
- * 2 TB rancher). The layout contract these tests pin down:
- *
- *   - naming a data disk DROPS the matching LV -- never both.
- *   - disks are named by /dev/disk/by-id only; kernel names are rejected,
- *     because nvme enumeration order is not stable across boots.
- *   - a lab-longhorn / lab-rancher labelled partition is on-disk truth, like
- *     the lab-schema-v2 VG tag: reused --noformat, never wiped, never
- *     silently shadowed.
- */
-
-const BY_ID = "/dev/disk/by-id";
-const LONGHORN_ID = `${BY_ID}/nvme-Sabrent_Rocket_4.0_2TB_7F600711179D89261778`;
-const RANCHER_ID = `${BY_ID}/nvme-Sabrent_Rocket_4.0_2TB_7FD4070C1EDA92197301`;
-const OS_ID = `${BY_ID}/nvme-Samsung_SSD_970_EVO_Plus_1TB_S4EWNJ0N138234M`;
-
-function baseParams(over: Partial<InstallKickstartParams> = {}): InstallKickstartParams {
-  return {
-    hostname: "thebeast", disk: "", role: "k8s-worker", domain: "ad.itaz.eu",
-    fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8",
-    serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514,
-    sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal",
-    dataDisks: { longhorn: LONGHORN_ID, rancher: RANCHER_ID },
-    ...over,
-  };
-}
-
-/** The fresh-install part.ks heredoc (the one that creates the PV). */
-function freshBlock(ks: string): string {
-  const blocks = [...ks.matchAll(/cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g)]
-    .map((m) => m[1]!);
-  const fresh = blocks.find((b) => b.includes("part pv.01"));
-  if (!fresh) throw new Error("no fresh-install part.ks block found");
-  return fresh;
-}
-
-describe("validateDataDisks", () => {
-  const ok: DataDisks = { longhorn: LONGHORN_ID, rancher: RANCHER_ID };
-
-  it("accepts a sane spec for a role that hosts longhorn", () => {
-    expect(validateDataDisks("k8s-worker", ok)).toBeNull();
-    expect(validateDataDisks("k8s-combo", ok)).toBeNull();
-    expect(validateDataDisks("worker", ok)).toBeNull();
-  });
-
-  it("rejects kernel names -- they are not stable across boots", () => {
-    expect(validateDataDisks("k8s-worker", { longhorn: "nvme1n1" })).toMatch(/by-id/);
-    expect(validateDataDisks("k8s-worker", { rancher: "/dev/nvme2n1" })).toMatch(/by-id/);
-  });
-
-  it("rejects a longhorn disk for a role that holds no replicas", () => {
-    expect(validateDataDisks("k8s-controller", { longhorn: LONGHORN_ID })).toMatch(/no Longhorn replicas/);
-    expect(validateDataDisks("vanilla", { longhorn: LONGHORN_ID })).toMatch(/no Longhorn replicas/);
-  });
-
-  it("rejects a rancher disk for a role that runs no k3s", () => {
-    expect(validateDataDisks("vanilla", { rancher: RANCHER_ID })).toMatch(/no k3s/);
-  });
-
-  it("a control-plane-only node may still take a rancher disk", () => {
-    expect(validateDataDisks("k8s-controller", { rancher: RANCHER_ID })).toBeNull();
-  });
-
-  it("rejects the same disk doing both jobs", () => {
-    expect(validateDataDisks("k8s-worker", { longhorn: LONGHORN_ID, rancher: LONGHORN_ID }))
-      .toMatch(/same disk/);
-  });
-});
-
-describe("v2Layout with dedicated disks", () => {
-  it("drops exactly the LV a dedicated disk replaces", () => {
-    const names = (d: DataDisks) => v2Layout("k8s-worker", { dataDisks: d }).map((l) => l.name);
-    expect(names({})).toContain("longhorn");
-    expect(names({ longhorn: LONGHORN_ID })).not.toContain("longhorn");
-    expect(names({ longhorn: LONGHORN_ID })).toContain("rancher");
-    expect(names({ longhorn: LONGHORN_ID, rancher: RANCHER_ID })).not.toContain("rancher");
-    // base volumes untouched
-    expect(names({ longhorn: LONGHORN_ID, rancher: RANCHER_ID }))
-      .toEqual(expect.arrayContaining(["swap", "root", "var", "varlog", "home", "srv"]));
-  });
-});
-
-describe("renderInstallKickstart with data disks", () => {
-  it("refuses the v1 layout -- it is frozen and single-disk", () => {
-    expect(() => renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 1 })))
-      .toThrow(/v2 layout/);
-    // a legacy role with no explicit schema defaults to v1 -> same refusal
-    expect(() => renderInstallKickstart(baseParams({ role: "worker" })))
-      .toThrow(/v2 layout/);
-  });
-
-  it("refuses a kernel-name data disk", () => {
-    expect(() => renderInstallKickstart(baseParams({ dataDisks: { longhorn: "nvme1n1" } })))
-      .toThrow(/by-id/);
-  });
-
-  it("creates no longhorn/rancher LV when both mounts come from dedicated disks", () => {
-    const fresh = freshBlock(renderInstallKickstart(baseParams()));
-    expect(fresh).not.toContain("--name=longhorn");
-    expect(fresh).not.toContain("--name=rancher");
-    // base LVs still there
-    expect(fresh).toContain("--name=root");
-  });
-
-  it("anaconda only sees the named disks, and only wipes unlabelled ones", () => {
-    const ks = renderInstallKickstart(baseParams());
-    const fresh = freshBlock(ks);
-    expect(fresh).toContain("ignoredisk --only-use=$KS_DISKS");
-    expect(fresh).toContain("clearpart --all --initlabel --drives=$CLEARPART_DRIVES");
-    // a disk whose partition carries our label is excluded from clearpart
-    expect(ks).toContain('[ "$LONGHORN_REUSE" = "no" ] && CLEARPART_DRIVES=');
-  });
-
-  it("%pre resolves each by-id path and fails loudly when a disk is missing", () => {
-    const ks = renderInstallKickstart(baseParams());
-    expect(ks).toContain(`LONGHORN_DISK_ID="${LONGHORN_ID}"`);
-    expect(ks).toContain(`RANCHER_DISK_ID="${RANCHER_ID}"`);
-    expect(ks).toContain('echo "ERROR: longhorn disk not found: $LONGHORN_DISK_ID"; exit 1');
-  });
-
-  it("a labelled partition is reused --noformat; a fresh disk gets one labelled partition", () => {
-    const ks = renderInstallKickstart(baseParams());
-    expect(ks).toContain("part /var/lib/longhorn --onpart=$LONGHORN_PART --noformat");
-    expect(ks).toContain(
-      `part /var/lib/longhorn --fstype=xfs --ondisk=$LONGHORN_DISK --grow --size=1 --label=${DATA_DISK_LABELS.longhorn}`,
-    );
-    expect(ks).toContain("part /var/lib/rancher --onpart=$RANCHER_PART --noformat");
-    expect(ks).toContain(
-      `part /var/lib/rancher --fstype=xfs --ondisk=$RANCHER_DISK --grow --size=1 --label=${DATA_DISK_LABELS.rancher}`,
-    );
-  });
-
-  it("refuses to run when a data LV and a dedicated disk both exist", () => {
-    const ks = renderInstallKickstart(baseParams());
-    expect(ks).toContain('[ "$PRESERVE_LONGHORN" = "yes" ] && [ -n "$LONGHORN_DISK" ]');
-    expect(ks).toContain('[ "$PRESERVE_RANCHER" = "yes" ] && [ -n "$RANCHER_DISK" ]');
-  });
-
-  it("a fresh install never wipes or shadows a labelled disk it was not told about", () => {
-    // Rendered even WITHOUT dataDisks: the label is on-disk truth and the
-    // check protects against a queue entry that forgot the disks.
-    const ks = renderInstallKickstart(baseParams({ role: "k8s-worker", dataDisks: {} }));
-    expect(ks).toContain(`blkid -t LABEL=${DATA_DISK_LABELS.longhorn}`);
-    expect(ks).toContain("Refusing to wipe or shadow it");
-  });
-
-  it("an OS disk given as a by-id path is resolved in %pre", () => {
-    const ks = renderInstallKickstart(baseParams({ disk: OS_ID }));
-    expect(ks).toContain(`readlink -f "${OS_ID}"`);
-    expect(ks).not.toContain(`DISK="${OS_ID}"`); // never fed to anaconda unresolved
-  });
-
-  it("auto-detect never picks a data disk as the OS disk", () => {
-    const ks = renderInstallKickstart(baseParams({ disk: "" }));
-    expect(ks).toContain('case "$_bname" in "$LONGHORN_DISK"|"$RANCHER_DISK") continue ;; esac');
-  });
-
-  it("a plain kernel-name OS disk still renders exactly as before", () => {
-    const ks = renderInstallKickstart(baseParams({ disk: "vda", dataDisks: {} }));
-    expect(ks).toContain('DISK="vda"');
-  });
-
-  it("the %pre block still survives the /ks-auto extractor", () => {
-    const ks = renderInstallKickstart(baseParams());
-    const blocks = [...ks.matchAll(/%pre[^\n]*\n([\s\S]*?)%end/g)];
-    expect(blocks.length).toBeGreaterThan(0);
-    expect(blocks[0]![1]).toContain("/tmp/part.ks");
-  });
-});
-
-describe("dataDisks plumbing: /api/install -> /ks", () => {
-  let app: FastifyInstance;
-  let testDir: string;
-
-  function testConfig(dir: string): BastionConfig {
-    return {
-      fedoraVersion: "43", arch: "x86_64", httpPort: 0,
-      timezone: "Europe/London", locale: "en_GB.UTF-8",
-      bastionDir: dir, domain: "test.local", dhcpMode: "proxy",
-      dhcpRangeStart: "", dhcpRangeEnd: "",
-      ubuntuVersion: "26.04", ubuntuMirror: "https://releases.ubuntu.com/26.04",
-      vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso",
-      vyosDefaultPassword: "vyos", iface: "eth0",
-      serverIp: "10.0.0.1", network: "10.0.0.0", gateway: "10.0.0.1",
-      sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"],
-      adminUser: "testadmin", syslogPort: 15514,
-      skipDnsmasq: true, skipArtifacts: true,
-      fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
-      tftpDir: join(dir, "tftp"), httpDir: join(dir, "http"), stateFile: join(dir, "state.json"),
-    };
-  }
-
-  beforeEach(async () => {
-    testDir = join(tmpdir(), `datadisks-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
-    mkdirSync(testDir, { recursive: true });
-    const built = await createApp(testConfig(testDir));
-    app = built.app;
-    await app.ready();
-  });
-
-  afterEach(async () => {
-    await app.close();
-    rmSync(testDir, { recursive: true, force: true });
-  });
-
-  const queue = (body: Record<string, unknown>) =>
-    app.inject({ method: "POST", url: "/api/install", payload: { mac: "aa:bb:cc:dd:ee:03", hostname: "thebeast", ...body } });
-
-  const ks = async () => (await app.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:03" })).body;
-
-  it("named disks reach the rendered kickstart, and their LVs are gone", async () => {
-    const res = await queue({ role: "k8s-worker", dataDisks: { longhorn: LONGHORN_ID, rancher: RANCHER_ID } });
-    expect(res.statusCode).toBe(200);
-    const body = await ks();
-    expect(body).toContain(LONGHORN_ID);
-    expect(body).toContain(RANCHER_ID);
-    expect(freshBlock(body)).not.toContain("--name=longhorn");
-    expect(freshBlock(body)).not.toContain("--name=rancher");
-  });
-
-  it("rejects dataDisks on a non-fedora install instead of silently dropping them", async () => {
-    const res = await queue({ role: "k8s-worker", os: "ubuntu-26.04", dataDisks: { longhorn: LONGHORN_ID } });
-    expect(res.statusCode).toBe(400);
-    expect(res.json().error).toMatch(/fedora/);
-  });
-
-  it("rejects dataDisks with an explicit v1 schema", async () => {
-    const res = await queue({ role: "worker", schemaVersion: 1, dataDisks: { longhorn: LONGHORN_ID } });
-    expect(res.statusCode).toBe(400);
-    expect(res.json().error).toMatch(/schemaVersion 2/);
-  });
-
-  it("rejects dataDisks for a legacy role that would default to v1", async () => {
-    const res = await queue({ role: "worker", dataDisks: { longhorn: LONGHORN_ID } });
-    expect(res.statusCode).toBe(400);
-    expect(res.json().error).toMatch(/schemaVersion 2/);
-  });
-
-  it("legacy role + explicit v2 + dataDisks is accepted", async () => {
-    const res = await queue({ role: "worker", schemaVersion: 2, dataDisks: { longhorn: LONGHORN_ID } });
-    expect(res.statusCode).toBe(200);
-    expect(await ks()).toContain(LONGHORN_ID);
-  });
-
-  it("rejects a kernel-name disk with the shared message", async () => {
-    const res = await queue({ role: "k8s-worker", dataDisks: { longhorn: "nvme1n1" } });
-    expect(res.statusCode).toBe(400);
-    expect(res.json().error).toMatch(/by-id/);
-  });
-
-  it("rejects a capability mismatch (longhorn disk on a control-plane-only role)", async () => {
-    const res = await queue({ role: "k8s-controller", dataDisks: { longhorn: LONGHORN_ID } });
-    expect(res.statusCode).toBe(400);
-    expect(res.json().error).toMatch(/no Longhorn replicas/);
-  });
-
-  it("an install without dataDisks is untouched by all of this", async () => {
-    expect((await queue({ role: "k8s-worker" })).statusCode).toBe(200);
-    const body = await ks();
-    expect(freshBlock(body)).toContain("--name=longhorn --fstype=xfs --size=307200");
-    expect(freshBlock(body)).toContain("--name=rancher --fstype=xfs --size=256000");
-  });
-});

bastion/src/bastion/tests/dispatch.test.ts

@@ -22,8 +22,6 @@ function createTestConfig(testDir: string): BastionConfig {
     dhcpRangeEnd: "",
     ubuntuVersion: "26.04",
     ubuntuMirror: "https://releases.ubuntu.com/26.04",
-    vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso",
-    vyosDefaultPassword: "vyos",
     iface: "eth0",
     serverIp: "10.0.0.1",
     network: "10.0.0.0",

bastion/src/bastion/tests/fixtures/ipxe-x86_64-golden.json

@@ -0,0 +1,8 @@
+{
+  "boot": "#!ipxe\n\necho\necho ============================================\necho   Lab PXE Bastion\necho   Contacting server for instructions...\necho ============================================\necho\n\nchain http://10.0.0.1:8080/dispatch?mac=${net0/mac}\n",
+  "discover": "#!ipxe\n\necho\necho =============================================\necho   Lab PXE Bastion - DISCOVERY MODE\necho   MAC: aa:bb:cc:dd:ee:ff\necho   Collecting hardware info...\necho =============================================\necho\n\nkernel http://10.0.0.1:8080/vmlinuz inst.ks=http://10.0.0.1:8080/discover.ks inst.stage2=https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os inst.text nomodeset\ninitrd http://10.0.0.1:8080/initrd.img\nboot\n",
+  "install": "#!ipxe\n\necho\necho =============================================\necho   Lab PXE Bastion - INSTALLING Fedora 43\necho   Target: worker-1\necho   MAC:    aa:bb:cc:dd:ee:ff\necho =============================================\necho\n\nkernel http://10.0.0.1:8080/vmlinuz inst.ks=http://10.0.0.1:8080/ks?mac=aa:bb:cc:dd:ee:ff inst.repo=https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os inst.text nomodeset\ninitrd http://10.0.0.1:8080/initrd.img\nboot\n",
+  "debug": "#!ipxe\n\necho\necho =============================================\necho   Lab PXE Bastion - DEBUG/RESCUE MODE\necho   Target: worker-1\necho   MAC:    aa:bb:cc:dd:ee:ff\necho =============================================\necho\n\nkernel http://10.0.0.1:8080/vmlinuz inst.rescue inst.text inst.sshd inst.ks=http://10.0.0.1:8080/debug.ks?mac=aa:bb:cc:dd:ee:ff inst.stage2=https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os\ninitrd http://10.0.0.1:8080/initrd.img\nboot\n",
+  "pxeBoot": "#!ipxe\n\necho\necho =============================================\necho   Lab PXE Bastion - PXE BOOT (debug)\necho   Target: worker-1\necho   MAC:    aa:bb:cc:dd:ee:ff\necho   Kernel+initrd from PXE, root from NVMe\necho =============================================\necho\n\nkernel http://10.0.0.1:8080/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0\ninitrd http://10.0.0.1:8080/initrd.img\nboot\n",
+  "localBoot": "#!ipxe\n\necho\necho =============================================\necho   Lab PXE Bastion - worker-1\necho   Already installed, booting from local disk\necho =============================================\necho\nsleep 3\nexit 1\n"
+}

bastion/src/bastion/tests/install-guard.test.ts

@@ -0,0 +1,194 @@
+// Installs must never reach a machine running a vendor OS we cannot restore.
+//
+// This is the guardrail that stops someone reinstalling a DGX Spark at 2am. Rescue is
+// deliberately still allowed for the same machines -- that is the whole point.
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import type { BastionConfig, BastionState, HardwareInfo } from "@lab/shared";
+import { classifyOnboard } from "@lab/shared";
+import { createApp } from "../src/server.js";
+import { checkInstallAllowed } from "../src/services/install-guard.js";
+import type { FastifyInstance } from "fastify";
+import type { StateManager } from "../src/services/state.js";
+
+// The real machines this exists to protect.
+const SPARK_2935 = "4c:bb:47:7f:29:35";
+const SPARK_3A1C = "48:21:0b:96:3a:1c";
+const ORDINARY = "aa:bb:cc:dd:ee:ff";
+
+function createTestConfig(testDir: string): BastionConfig {
+  return {
+    fedoraVersion: "43", arch: "x86_64", httpPort: 0,
+    timezone: "Europe/London", locale: "en_GB.UTF-8", bastionDir: testDir,
+    domain: "test.local", dhcpMode: "proxy", dhcpRangeStart: "", dhcpRangeEnd: "",
+    ubuntuVersion: "26.04", ubuntuMirror: "https://releases.ubuntu.com/26.04",
+    iface: "eth0", serverIp: "10.0.0.1", network: "10.0.0.0", gateway: "10.0.0.1",
+    sshKeys: [], adminUser: "testadmin", syslogPort: 15514,
+    skipDnsmasq: true, skipArtifacts: true,
+    fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
+    tftpDir: join(testDir, "tftp"), httpDir: join(testDir, "http"),
+    stateFile: join(testDir, "state.json"),
+  };
+}
+
+function hardware(mac: string, over: Partial<HardwareInfo> = {}): HardwareInfo {
+  return {
+    mac, product: "TestBox", board: "TestBoard", serial: "SN1",
+    manufacturer: "TestCorp", cpu_model: "Test CPU", cpu_cores: 4, memory_gb: 16,
+    arch: "x86_64", disks: [], nics: [],
+    first_seen: new Date().toISOString(), last_seen: new Date().toISOString(),
+    ...over,
+  };
+}
+
+const emptyState = (): BastionState => ({
+  discovered: {}, install_queue: {}, installed: {}, debug: {},
+});
+
+describe("classifyOnboard", () => {
+  it("recognises a DGX Spark from its DMI identity", () => {
+    expect(classifyOnboard({
+      mac: ORDINARY, manufacturer: "NVIDIA", product: "NVIDIA DGX Spark", board: "GB10",
+    })).toEqual({ onboard: "ssh", vendor_os: "dgx-os" });
+  });
+
+  it("recognises the known Sparks even with no DMI recorded", () => {
+    // Neither Spark has hardware info in bastion state today. A DMI-only rule would
+    // fail open on exactly the machines this protects.
+    expect(classifyOnboard({ mac: SPARK_2935 }).onboard).toBe("ssh");
+    expect(classifyOnboard({ mac: SPARK_3A1C }).onboard).toBe("ssh");
+  });
+
+  it("treats ordinary hardware as PXE-installable", () => {
+    expect(classifyOnboard({
+      mac: ORDINARY, manufacturer: "Beelink", product: "SER9", board: "SER9",
+    })).toEqual({ onboard: "pxe" });
+  });
+
+  it("does not override an explicit classification already on the record", () => {
+    expect(classifyOnboard({
+      mac: SPARK_2935, onboard: "pxe",
+    })).toEqual({ onboard: "pxe" });
+  });
+});
+
+describe("checkInstallAllowed", () => {
+  it("refuses a DGX Spark and explains why", () => {
+    const state = emptyState();
+    state.installed[SPARK_2935] = {
+      hostname: "spark-2935", role: "worker", ip: "192.168.8.12",
+      installed_at: new Date().toISOString(), arch: "aarch64",
+    };
+
+    const result = checkInstallAllowed(state, SPARK_2935, "fedora-43");
+    expect(result.allowed).toBe(false);
+    if (result.allowed === false) {
+      expect(result.error).toContain("spark-2935");
+      expect(result.error).toContain("DGX OS");
+      expect(result.error).toContain("provision debug");
+    }
+  });
+
+  it("refuses a Spark that is only known by MAC", () => {
+    expect(checkInstallAllowed(emptyState(), SPARK_3A1C, "fedora-43").allowed).toBe(false);
+  });
+
+  it("allows an ordinary discovered machine", () => {
+    const state = emptyState();
+    state.discovered[ORDINARY] = hardware(ORDINARY);
+    expect(checkInstallAllowed(state, ORDINARY, "fedora-43").allowed).toBe(true);
+  });
+
+  it("allows Fedora on aarch64", () => {
+    const state = emptyState();
+    state.discovered[ORDINARY] = hardware(ORDINARY, { arch: "aarch64" });
+    expect(checkInstallAllowed(state, ORDINARY, "fedora-43").allowed).toBe(true);
+  });
+
+  it("refuses Ubuntu on aarch64 -- no netboot artifacts are published", () => {
+    const state = emptyState();
+    state.discovered[ORDINARY] = hardware(ORDINARY, { arch: "aarch64" });
+    const result = checkInstallAllowed(state, ORDINARY, "ubuntu-26.04");
+    expect(result.allowed).toBe(false);
+    if (result.allowed === false) {
+      expect(result.error).toContain("aarch64");
+    }
+  });
+
+  it("allows Ubuntu on x86_64", () => {
+    const state = emptyState();
+    state.discovered[ORDINARY] = hardware(ORDINARY, { arch: "x86_64" });
+    expect(checkInstallAllowed(state, ORDINARY, "ubuntu-26.04").allowed).toBe(true);
+  });
+});
+
+describe("install route enforces the guard", () => {
+  let testDir: string;
+  let app: FastifyInstance;
+  let state: StateManager;
+
+  beforeEach(() => {
+    testDir = join(tmpdir(), `bastion-guard-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+    mkdirSync(join(testDir, "http"), { recursive: true });
+    mkdirSync(join(testDir, "tftp"), { recursive: true });
+    const result = createApp(createTestConfig(testDir));
+    app = result.app;
+    state = result.state;
+  });
+
+  afterEach(async () => {
+    await app.close();
+    rmSync(testDir, { recursive: true, force: true });
+  });
+
+  it("rejects POST /api/install for a Spark and queues nothing", async () => {
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/install",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ mac: SPARK_2935, hostname: "spark-2935", role: "worker" }),
+    });
+
+    expect(res.statusCode).toBe(409);
+    expect(JSON.parse(res.body).error).toContain("Refusing to install");
+    expect(state.load().install_queue[SPARK_2935]).toBeUndefined();
+  });
+
+  it("still serves rescue to a Spark -- debug is never guarded", async () => {
+    state.update((s) => {
+      s.installed[SPARK_2935] = {
+        hostname: "spark-2935", role: "worker", ip: "192.168.8.12",
+        installed_at: new Date().toISOString(), arch: "aarch64",
+      };
+      s.debug[SPARK_2935] = { hostname: "spark-2935", queued_at: new Date().toISOString() };
+    });
+
+    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${SPARK_2935}` });
+    expect(res.statusCode).toBe(200);
+    expect(res.body).toContain("DEBUG/RESCUE MODE");
+    expect(res.body).toContain("/vmlinuz-aarch64");
+  });
+
+  it("a Spark that PXE boots unqueued gets discovery, never an install", async () => {
+    const res = await app.inject({ method: "GET", url: `/dispatch?mac=${SPARK_2935}&arch=arm64` });
+    expect(res.body).toContain("DISCOVERY MODE");
+    expect(res.body).not.toContain("INSTALLING");
+  });
+
+  it("still accepts an ordinary machine", async () => {
+    state.update((s) => { s.discovered[ORDINARY] = hardware(ORDINARY); });
+
+    const res = await app.inject({
+      method: "POST",
+      url: "/api/install",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ mac: ORDINARY, hostname: "worker-1", role: "worker" }),
+    });
+
+    expect(res.statusCode).toBe(200);
+    expect(state.load().install_queue[ORDINARY]).toBeDefined();
+  });
+});

bastion/src/bastion/tests/ipxe-x86-regression.test.ts

@@ -0,0 +1,89 @@
+// x86_64 iPXE output regression gate.
+//
+// The aarch64 PXE work must not change what an x86_64 machine is served. The golden
+// fixture was dumped from the templates as they stood before that work started, so
+// any diff here is a regression, not an improvement.
+//
+// The one deliberate exception is renderBootIpxe: its chain URL gained
+// `&arch=${buildarch}` so the dispatch endpoint can observe the client's
+// architecture at boot time. That single change is asserted explicitly below
+// rather than being allowed to slip through the byte-for-byte comparison.
+
+import { describe, it, expect } from "vitest";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { dirname } from "node:path";
+import {
+  renderBootIpxe,
+  renderDiscoverIpxe,
+  renderInstallIpxe,
+  renderDebugIpxe,
+  renderPxeBootDebugIpxe,
+  renderLocalBootIpxe,
+} from "../src/templates/boot.ipxe.js";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const golden = JSON.parse(
+  readFileSync(join(here, "fixtures", "ipxe-x86_64-golden.json"), "utf-8"),
+) as Record<string, string>;
+
+// Exactly the parameters used to dump the fixture.
+const serverIp = "10.0.0.1";
+const httpPort = 8080;
+const mac = "aa:bb:cc:dd:ee:ff";
+const hostname = "worker-1";
+const fedoraVersion = "43";
+const fedoraMirror =
+  "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os";
+
+// The x86_64 LVM layout the fixture was captured with. Before this work the values
+// were hardcoded in the template; they are now supplied by the caller from machine
+// state, so the fixture pins the rendering, not the defaults.
+const x86Root = {
+  rootDevice: "/dev/mapper/labvg-root",
+  rootArgs: "rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap",
+};
+
+describe("x86_64 iPXE output is unchanged", () => {
+  it("discover script is byte-identical", () => {
+    const rendered = renderDiscoverIpxe({
+      mac, serverIp, httpPort, fedoraMirror, arch: "x86_64",
+    });
+    expect(rendered).toBe(golden["discover"]);
+  });
+
+  it("install script is byte-identical", () => {
+    const rendered = renderInstallIpxe({
+      mac, hostname, serverIp, httpPort, fedoraVersion, fedoraMirror, arch: "x86_64",
+    });
+    expect(rendered).toBe(golden["install"]);
+  });
+
+  it("debug/rescue script is byte-identical", () => {
+    const rendered = renderDebugIpxe({
+      mac, hostname, serverIp, httpPort, fedoraMirror, arch: "x86_64",
+    });
+    expect(rendered).toBe(golden["debug"]);
+  });
+
+  it("--pxe-boot script is byte-identical when state carries the Fedora LVM layout", () => {
+    const rendered = renderPxeBootDebugIpxe({
+      mac, hostname, serverIp, httpPort, arch: "x86_64", ...x86Root,
+    });
+    expect(rendered).toBe(golden["pxeBoot"]);
+  });
+
+  it("local boot script is byte-identical", () => {
+    expect(renderLocalBootIpxe(hostname)).toBe(golden["localBoot"]);
+  });
+
+  it("boot.ipxe differs only by the &arch= chain parameter", () => {
+    const rendered = renderBootIpxe({ serverIp, httpPort });
+    // The sole intended difference.
+    expect(rendered).toBe(golden["boot"].replace(
+      "/dispatch?mac=${net0/mac}",
+      "/dispatch?mac=${net0/mac}&arch=${buildarch}",
+    ));
+  });
+});

bastion/src/bastion/tests/kickstart-layout.test.ts

@@ -1,186 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { renderInstallKickstart } from "../src/templates/install.ks.js";
-import type { InstallKickstartParams } from "../src/templates/install.ks.js";
-import { SUPPORTED_ROLES, LEGACY_ROLES, v2Layout, type Role } from "@lab/shared";
-
-/**
- * Partition-layout tests.
- *
- * The existing ksvalidator test only sees the MAIN kickstart body. The actual
- * layout is written by %pre into /tmp/part.ks inside a heredoc, so until now
- * nothing validated it at all -- a broken layout rendered, shipped and was only
- * discovered on the installed machine.
- *
- * These parse the heredocs. Following tests/ubuntu-autoinstall.test.ts:
- * "A `toContain` assertion would not have caught that; only parsing does."
- */
-
-function baseParams(over: Partial<InstallKickstartParams> = {}): InstallKickstartParams {
-  return {
-    hostname: "test-node", disk: "nvme0n1", role: "worker", domain: "ad.itaz.eu",
-    fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8",
-    serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514,
-    sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal",
-    ...over,
-  };
-}
-
-/** Pull every `cat >[>] /tmp/part.ks << PARTEOF ... PARTEOF` body out of the script. */
-function partKsBlocks(ks: string): string[] {
-  const out: string[] = [];
-  const re = /cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g;
-  let m: RegExpExecArray | null;
-  while ((m = re.exec(ks)) !== null) out.push(m[1]!);
-  return out;
-}
-
-interface Logvol { mount: string; name: string; fs: string; size?: number; useExisting: boolean }
-
-function parseLogvols(block: string): Logvol[] {
-  return block.split("\n")
-    .filter((l) => l.trim().startsWith("logvol "))
-    .map((line) => {
-      const parts = line.trim().split(/\s+/);
-      const get = (flag: string) => {
-        const p = parts.find((x) => x.startsWith(`${flag}=`));
-        return p ? p.slice(flag.length + 1) : undefined;
-      };
-      return {
-        mount: parts[1]!,
-        name: get("--name") ?? "",
-        fs: get("--fstype") ?? "",
-        ...(get("--size") ? { size: Number(get("--size")) } : {}),
-        useExisting: parts.includes("--useexisting"),
-      };
-    });
-}
-
-/** The fresh-install block is the one that also creates the PV. */
-function freshBlock(ks: string): string {
-  const b = partKsBlocks(ks).find((x) => x.includes("part pv.01"));
-  if (!b) throw new Error("no fresh-install part.ks block found");
-  return b;
-}
-
-describe("v1 is frozen — existing machines must reinstall identically", () => {
-  it.each(LEGACY_ROLES)("%s v1 fresh layout is unchanged", (role) => {
-    const lvs = parseLogvols(freshBlock(renderInstallKickstart(baseParams({ role }))));
-    const byName = Object.fromEntries(lvs.map((l) => [l.name, l]));
-
-    // The exact v1 sizes, in MiB. If one of these changes, every existing box
-    // in the fleet would be re-laid-out on its next reinstall.
-    expect(byName["swap"]!.size).toBe(27648);
-    expect(byName["root"]!.size).toBe(33792);
-    expect(byName["var"]!.size).toBe(102400);
-    expect(byName["varlog"]!.size).toBe(10240);
-    expect(byName["home"]!.size).toBe(10240);
-    expect(byName["srv"]!.size).toBe(20480);
-  });
-
-  it("v1 gives rancher to infra and worker only — the historical bug, preserved", () => {
-    const has = (r: Role) => freshBlock(renderInstallKickstart(baseParams({ role: r }))).includes("--name=rancher");
-    expect(has("infra")).toBe(true);
-    expect(has("worker")).toBe(true);
-    expect(has("labcontroller")).toBe(false); // the bug, deliberately kept in v1
-    expect(has("vanilla")).toBe(false);
-  });
-
-  it("v1 is the default — no caller has to opt out", () => {
-    const ks = renderInstallKickstart(baseParams({ role: "worker" }));
-    expect(parseLogvols(freshBlock(ks)).find((l) => l.name === "root")!.size).toBe(33792);
-  });
-});
-
-describe("v2-only roles have no v1 layout", () => {
-  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"] as const)(
-    "%s defaults to v2 when no schema is given", (role) => {
-      const lvs = parseLogvols(freshBlock(renderInstallKickstart(baseParams({ role }))));
-      expect(lvs.find((l) => l.name === "root")!.size).toBe(32768);
-      expect(lvs.some((l) => l.name === "rancher")).toBe(true);
-    });
-
-  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"] as const)(
-    "%s throws on an EXPLICIT v1 rather than silently dropping its volumes", (role) => {
-      expect(() => renderInstallKickstart(baseParams({ role, schemaVersion: 1 }))).toThrow(/no v1 disk layout/);
-    });
-});
-
-describe("v2 renders the shared layout", () => {
-  it.each(SUPPORTED_ROLES)("%s v2 matches v2Layout() exactly", (role) => {
-    const ks = renderInstallKickstart(baseParams({ role, schemaVersion: 2 }));
-    const rendered = parseLogvols(freshBlock(ks));
-    const expected = v2Layout(role);
-
-    expect(rendered.map((l) => l.name).sort()).toEqual(expected.map((l) => l.name).sort());
-    for (const spec of expected) {
-      const got = rendered.find((l) => l.name === spec.name)!;
-      expect(got.size).toBe(spec.sizeMib);
-      expect(got.fs).toBe(spec.fs);
-    }
-  });
-
-  it("v2 gives labcontroller a rancher LV — the v1 bug, fixed", () => {
-    const ks = renderInstallKickstart(baseParams({ role: "labcontroller", schemaVersion: 2 }));
-    const rancher = parseLogvols(freshBlock(ks)).find((l) => l.name === "rancher");
-    expect(rancher).toBeDefined();
-    expect(rancher!.size).toBe(256000);
-  });
-
-  it("v2 never uses --grow", () => {
-    for (const role of SUPPORTED_ROLES) {
-      const block = freshBlock(renderInstallKickstart(baseParams({ role, schemaVersion: 2 })));
-      expect(block).not.toMatch(/logvol[^\n]*--grow/);
-    }
-  });
-
-  it("v2 uses the larger EFI and /boot partitions", () => {
-    const block = freshBlock(renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 2 })));
-    expect(block).toMatch(/part \/boot\/efi --fstype=efi --size=1024/);
-    expect(block).toMatch(/part \/boot --fstype=ext4 --size=4096/);
-  });
-});
-
-describe("reprovision protects existing data", () => {
-  it("reprovision reads the on-disk schema instead of trusting the caller", () => {
-    const ks = renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 2 }));
-    expect(ks).toContain("DISK_SCHEMA=v1");
-    expect(ks).toMatch(/vgs --noheadings -o vg_tags .*grep -q lab-schema-v2/);
-  });
-
-  it("the reprovision branch never wipes the disk", () => {
-    const ks = renderInstallKickstart(baseParams({ role: "worker" }));
-    const repro = partKsBlocks(ks).find((b) => b.includes("--useexisting"))!;
-    expect(repro).toContain("clearpart --none");
-    expect(repro).not.toContain("clearpart --all");
-  });
-
-  it("preserved data volumes are never reformatted", () => {
-    const ks = renderInstallKickstart(baseParams({ role: "worker" }));
-    for (const lv of ["longhorn", "rancher", "home", "srv"]) {
-      const re = new RegExp(`--name=${lv} --useexisting --noformat`);
-      expect(ks).toMatch(re);
-    }
-  });
-});
-
-describe("the %pre block survives the /ks-auto extractor", () => {
-  it("matches the same regex ks-auto uses to re-run %pre", () => {
-    // routes/kickstart.ts re-extracts %pre with python:
-    //   re.findall(r'%pre[^\n]*\n(.*?)%end', text, re.DOTALL)
-    // A change that breaks this breaks ISO installs silently.
-    for (const role of SUPPORTED_ROLES) {
-      // v2-only roles throw at v1 by design, so only render the combinations
-      // that are actually legal.
-      // An explicit v1 is illegal for a v2-only role; everything else renders.
-      const versions = (LEGACY_ROLES as readonly string[]).includes(role)
-        ? ([1, 2] as const)
-        : ([2] as const);
-      for (const schemaVersion of versions) {
-        const ks = renderInstallKickstart(baseParams({ role, schemaVersion }));
-        const blocks = [...ks.matchAll(/%pre[^\n]*\n([\s\S]*?)%end/g)];
-        expect(blocks.length).toBeGreaterThan(0);
-        expect(blocks[0]![1]).toContain("/tmp/part.ks");
-      }
-    }
-  });
-});

bastion/src/bastion/tests/kickstart.test.ts

@@ -96,9 +96,9 @@ describe("renderInstallKickstart", () => {
     expect(ks).toContain("/api/progress");
   });
 
-  it("infra role has 120G /var/lib/rancher partition", () => {
+  it("infra role has /var/lib/rancher partition", () => {
     const ks = renderInstallKickstart(baseParams({ role: "infra" }));
-    expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=122880");
+    expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=20480");
   });
 
   it("infra role has k3s install", () => {
@@ -106,14 +106,10 @@ describe("renderInstallKickstart", () => {
     expect(ks).toContain("curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true sh -");
   });
 
-  it("worker role has 120G /var/lib/rancher partition (imageFs must be sized before longhorn --grow)", () => {
+  it("worker role does NOT have /var/lib/rancher partition in fresh install", () => {
     const ks = renderInstallKickstart(baseParams({ role: "worker" }));
-    expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=122880");
-  });
-
-  it("vanilla role does NOT have /var/lib/rancher partition in fresh install", () => {
-    const ks = renderInstallKickstart(baseParams({ role: "vanilla" }));
-    expect(ks).not.toContain("--name=rancher --fstype=xfs");
+    // Worker should not have the fresh-install rancher partition line
+    expect(ks).not.toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=20480");
   });
 
   it("worker role does NOT have k3s install", () => {
@@ -215,21 +211,3 @@ describe("renderInstallKickstart", () => {
     expect(ks).not.toContain("ttyS0");
   });
 });
-
-describe("progress callback mac", () => {
-  it("bakes the queue mac into every bastion_progress helper when given", () => {
-    const ks = renderInstallKickstart(baseParams({ mac: "34:97:f6:32:76:e9" }));
-    const baked = ks.match(/local mac="34:97:f6:32:76:e9"/g) ?? [];
-    expect(baked.length).toBeGreaterThanOrEqual(2); // %pre and %post helpers
-    // No runtime first-NIC detection left: on multi-NIC boards the first NIC
-    // need not be the PXE NIC, and a completion under the wrong mac is a
-    // silent no-op that reinstalls the machine forever.
-    expect(ks).not.toContain("local mac=$(ip link show");
-  });
-
-  it("falls back to runtime detection when no mac is known", () => {
-    const ks = renderInstallKickstart(baseParams());
-    expect(ks).toContain("local mac=$(ip link show");
-    expect(ks).not.toContain('local mac=""');
-  });
-});

bastion/src/bastion/tests/layout-consistency.test.ts

@@ -1,105 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { execFileSync } from "node:child_process";
-import { writeFileSync } from "node:fs";
-import { renderInstallKickstart } from "../src/templates/install.ks.js";
-import { renderUbuntuAutoinstall } from "../src/templates/ubuntu-autoinstall.js";
-import { renderFirstbootScript } from "../src/templates/asahi-firstboot.sh.js";
-import { v2Layout, SUPPORTED_ROLES, type Role } from "@lab/shared";
-
-/**
- * THE test this whole refactor exists for.
- *
- * Three generators partition machines. Before the shared layout they disagreed,
- * and nothing noticed:
- *
- *   install.ks.ts       rancher 122880M
- *   ubuntu-autoinstall  rancher  20480M
- *   asahi-firstboot     rancher  20480M   <- under a comment claiming it
- *                                            matched install.ks.ts
- *
- * That 100 GB gap is why worker1 runs an image store too small for one large
- * container image. This test fails the moment any generator drifts again.
- */
-
-const KS = (role: Role) => renderInstallKickstart({
-  hostname: "n", disk: "nvme0n1", role, domain: "d", fedoraVersion: "43",
-  timezone: "UTC", locale: "en_US.UTF-8", serverIp: "10.0.0.1", httpPort: 8080,
-  syslogPort: 5514, sshKeys: [], adminUser: "a", schemaVersion: 2,
-});
-
-const UB = (role: Role) => renderUbuntuAutoinstall({
-  hostname: "n", disk: "/dev/sda", role, domain: "d", ubuntuVersion: "24.04",
-  timezone: "UTC", locale: "en_US.UTF-8", serverIp: "10.0.0.1", httpPort: 8080,
-  sshKeys: [], adminUser: "a", schemaVersion: 2,
-} as never);
-
-const AS = (role: Role) => renderFirstbootScript({
-  hostname: "n", role, serverIp: "10.0.0.1", httpPort: 8080,
-  sshKeys: [], adminUser: "a", mac: "aa:bb:cc:dd:ee:ff", schemaVersion: 2,
-} as never);
-
-/** MiB per LV name, from the Fedora kickstart's part.ks heredoc. */
-function fedoraSizes(role: Role): Record<string, number> {
-  const ks = KS(role);
-  const block = [...ks.matchAll(/cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g)]
-    .map((m) => m[1]!).find((b) => b.includes("part pv.01"))!;
-  const out: Record<string, number> = {};
-  for (const line of block.split("\n").filter((l) => l.trim().startsWith("logvol "))) {
-    const name = /--name=(\S+)/.exec(line)?.[1];
-    const size = /--size=(\d+)/.exec(line)?.[1];
-    if (name && size) out[name] = Number(size);
-  }
-  return out;
-}
-
-/** MiB per LV name, from the Ubuntu curtin document (declared in GiB). */
-function ubuntuSizes(role: Role): Record<string, number> {
-  const tmp = `/tmp/consistency-${role}.yaml`;
-  writeFileSync(tmp, UB(role));
-  const json = execFileSync("python3",
-    ["-c", "import sys,yaml,json; json.dump(yaml.safe_load(open(sys.argv[1])), sys.stdout)", tmp],
-    { encoding: "utf-8" });
-  const cfg = JSON.parse(json).autoinstall.storage.config as Array<Record<string, string>>;
-  return Object.fromEntries(
-    cfg.filter((c) => c["type"] === "lvm_partition")
-       .map((c) => [c["name"]!, Number(String(c["size"]).replace("G", "")) * 1024]),
-  );
-}
-
-/** MiB per LV name, from the Asahi firstboot lvcreate calls. */
-function asahiSizes(role: Role): Record<string, number> {
-  const out: Record<string, number> = {};
-  for (const m of AS(role).matchAll(/lvcreate -L (\d+)M\s+-n (\w+)\s+labvg/g)) out[m[2]!] = Number(m[1]);
-  return out;
-}
-
-describe("all three generators render the same v2 layout", () => {
-  it.each(SUPPORTED_ROLES)("%s: Fedora matches the shared definition", (role) => {
-    const expected = Object.fromEntries(v2Layout(role).map((lv) => [lv.name, lv.sizeMib]));
-    expect(fedoraSizes(role)).toEqual(expected);
-  });
-
-  it.each(SUPPORTED_ROLES)("%s: Ubuntu matches the shared definition", (role) => {
-    const expected = Object.fromEntries(v2Layout(role).map((lv) => [lv.name, lv.sizeMib]));
-    expect(ubuntuSizes(role)).toEqual(expected);
-  });
-
-  it.each(SUPPORTED_ROLES)("%s: Asahi matches, minus the vendor-created root", (role) => {
-    const expected = Object.fromEntries(
-      v2Layout(role, { omitRoot: true }).map((lv) => [lv.name, lv.sizeMib]),
-    );
-    expect(asahiSizes(role)).toEqual(expected);
-  });
-
-  it.each(SUPPORTED_ROLES)("%s: Fedora and Ubuntu are byte-identical in sizing", (role) => {
-    expect(ubuntuSizes(role)).toEqual(fedoraSizes(role));
-  });
-
-  it("rancher is 250G everywhere — the specific drift that bit worker1", () => {
-    for (const role of ["worker", "infra", "labcontroller"] as const) {
-      expect(fedoraSizes(role)["rancher"]).toBe(256000);
-      expect(ubuntuSizes(role)["rancher"]).toBe(256000);
-      expect(asahiSizes(role)["rancher"]).toBe(256000);
-    }
-  });
-});

bastion/src/bastion/tests/schema-version.test.ts

@@ -1,166 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { mkdirSync, rmSync } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import type { BastionConfig } from "@lab/shared";
-import { createApp } from "../src/server.js";
-import type { FastifyInstance } from "fastify";
-import type { StateManager } from "../src/services/state.js";
-import type { InstallLogBuffer } from "../src/services/install-log.js";
-
-function createTestConfig(testDir: string): BastionConfig {
-  return {
-    fedoraVersion: "43",
-    arch: "x86_64",
-    httpPort: 0,
-    timezone: "Europe/London",
-    locale: "en_GB.UTF-8",
-    bastionDir: testDir,
-    domain: "test.local",
-    dhcpMode: "proxy",
-    dhcpRangeStart: "",
-    dhcpRangeEnd: "",
-    ubuntuVersion: "26.04",
-    ubuntuMirror: "https://releases.ubuntu.com/26.04",
-    vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso",
-    vyosDefaultPassword: "vyos",
-    iface: "eth0",
-    serverIp: "10.0.0.1",
-    network: "10.0.0.0",
-    gateway: "10.0.0.1",
-    sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"],
-    adminUser: "testadmin",
-    syslogPort: 15514,
-    skipDnsmasq: true,
-    skipArtifacts: true,
-    fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
-    tftpDir: join(testDir, "tftp"),
-    httpDir: join(testDir, "http"),
-    stateFile: join(testDir, "state.json"),
-  };
-}
-
-describe("schemaVersion plumbing: /api/install -> /ks", () => {
-  let app: FastifyInstance;
-  let testDir: string;
-
-  beforeEach(async () => {
-    testDir = join(tmpdir(), `schema-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
-    mkdirSync(testDir, { recursive: true });
-    const built = await createApp(createTestConfig(testDir));
-    app = built.app;
-    await app.ready();
-  });
-
-  afterEach(async () => {
-    await app.close();
-    rmSync(testDir, { recursive: true, force: true });
-  });
-
-  const queue = (body: Record<string, unknown>) =>
-    app.inject({ method: "POST", url: "/api/install", payload: { mac: "aa:bb:cc:dd:ee:01", hostname: "n1", ...body } });
-
-  const ks = () => app.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:01" });
-
-  it("defaults to v1 when no schemaVersion is given", async () => {
-    expect((await queue({ role: "worker" })).statusCode).toBe(200);
-    const body = (await ks()).body;
-    expect(body).toContain("--name=root --fstype=xfs --size=33792");
-  });
-
-  it("schemaVersion 2 reaches the rendered kickstart", async () => {
-    expect((await queue({ role: "worker", schemaVersion: 2 })).statusCode).toBe(200);
-    const body = (await ks()).body;
-    expect(body).toContain("--name=root --fstype=xfs --size=32768");
-    expect(body).toContain("--name=rancher --fstype=xfs --size=256000");
-  });
-
-  it("v2 gives labcontroller a rancher LV through the full path", async () => {
-    expect((await queue({ role: "labcontroller", schemaVersion: 2 })).statusCode).toBe(200);
-    expect((await ks()).body).toContain("--name=rancher");
-  });
-
-  it("rejects an unknown schemaVersion instead of silently installing v1", async () => {
-    const res = await queue({ role: "worker", schemaVersion: 3 });
-    expect(res.statusCode).toBe(400);
-    expect(res.json().error).toMatch(/invalid schemaVersion/);
-  });
-
-  it("still rejects an unknown role", async () => {
-    expect((await queue({ role: "nope" })).statusCode).toBe(400);
-  });
-});
-
-describe("v2-only roles cannot be installed at v1", () => {
-  let app2: FastifyInstance;
-  let dir2: string;
-
-  beforeEach(async () => {
-    dir2 = join(tmpdir(), `schema-v2only-${Date.now()}-${Math.random().toString(36).slice(2)}`);
-    mkdirSync(dir2, { recursive: true });
-    const built = await createApp(createTestConfig(dir2));
-    app2 = built.app;
-    await app2.ready();
-  });
-  afterEach(async () => { await app2.close(); rmSync(dir2, { recursive: true, force: true }); });
-
-  const q = (body: Record<string, unknown>) =>
-    app2.inject({ method: "POST", url: "/api/install", payload: { mac: "aa:bb:cc:dd:ee:02", hostname: "n2", ...body } });
-
-  /**
-   * Omitting the schema must WORK for a v2-only role: a reinstall never passes
-   * one (the lab-schema-v2 tag on the disk decides), and the first version of
-   * this guard rejected that, making it impossible to reinstall a v2-only
-   * machine at all. Found by the reinstall integration test.
-   */
-  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"])(
-    "%s is accepted without a schemaVersion and defaults to v2", async (role) => {
-      const res = await q({ role });
-      expect(res.statusCode).toBe(200);
-      const ks = (await app2.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:02" })).body;
-      expect(ks).toContain("--name=root --fstype=xfs --size=32768");
-    });
-
-  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"])(
-    "%s is rejected on an EXPLICIT schemaVersion 1 — that is a contradiction", async (role) => {
-      const res = await q({ role, schemaVersion: 1 });
-      expect(res.statusCode).toBe(400);
-      expect(res.json().error).toMatch(/requires schemaVersion 2/);
-    });
-
-  it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"])(
-    "%s is accepted with schemaVersion 2", async (role) => {
-      expect((await q({ role, schemaVersion: 2 })).statusCode).toBe(200);
-    });
-
-  it("legacy roles are still accepted without a schemaVersion", async () => {
-    for (const role of ["vanilla", "worker", "infra", "labcontroller"]) {
-      expect((await q({ role })).statusCode).toBe(200);
-    }
-  });
-
-  /**
-   * Assert on the FRESH-INSTALL part.ks block specifically. The whole kickstart
-   * also contains the reprovision branch, whose shell emits `--name=longhorn`
-   * inside a `PRESERVE_LONGHORN` guard -- so a naive whole-file `not.toContain`
-   * matches that and fails for reasons that have nothing to do with the layout.
-   */
-  function freshBlockOf(ks: string): string {
-    const blocks = [...ks.matchAll(/cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g)]
-      .map((m) => m[1]!);
-    const fresh = blocks.find((b) => b.includes("part pv.01"));
-    if (!fresh) throw new Error("no fresh-install block in rendered kickstart");
-    return fresh;
-  }
-
-  it("k8s-controller gets no longhorn LV; k8s-combo does", async () => {
-    await q({ role: "k8s-controller", schemaVersion: 2 });
-    const ctrl = freshBlockOf((await app2.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:02" })).body);
-    expect(ctrl).toContain("--name=rancher");
-    expect(ctrl).not.toContain("--name=longhorn");
-
-    await q({ role: "k8s-combo", schemaVersion: 2 });
-    const combo = freshBlockOf((await app2.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:02" })).body);
-    expect(combo).toContain("--name=longhorn --fstype=xfs --size=307200");
-  });
-});

bastion/src/bastion/tests/ubuntu-autoinstall.test.ts

@@ -1,197 +0,0 @@
-// The Ubuntu autoinstall document must be valid YAML for EVERY role.
-//
-// There was no test here, and the template shipped a document that did not
-// parse: the longhorn/rancher LVM entries were indented 8 spaces while their
-// siblings in storage.config sit at 6, giving "expected <block end>, but found
-// '-'". Every role that gets a longhorn volume -- which is the worker shape --
-// rendered an uninstallable document. A `toContain` assertion would not have
-// caught that; only parsing does.
-//
-// Parsed with python3's yaml rather than a new npm dependency, mirroring how
-// kickstart.test.ts shells out to `ksvalidator`: the point is to check the
-// artefact with a real parser, not to grow the dependency tree.
-
-import { describe, it, expect } from "vitest";
-import { execFileSync } from "node:child_process";
-import { writeFileSync, unlinkSync } from "node:fs";
-import { renderUbuntuAutoinstall } from "../src/templates/ubuntu-autoinstall.js";
-
-const base = {
-  hostname: "n6",
-  disk: "/dev/sda",
-  domain: "ad.itaz.eu",
-  ubuntuVersion: "24.04",
-  timezone: "Europe/London",
-  locale: "en_GB.UTF-8",
-  serverIp: "10.0.0.1",
-  httpPort: 8080,
-  sshKeys: ["ssh-ed25519 AAAAtest test@lab"],
-  adminUser: "root",
-};
-
-/** Parse with python3's yaml and return the document as JSON. */
-function parseYaml(text: string, label: string): Record<string, any> {
-  const tmp = `/tmp/autoinstall-test-${label}.yaml`;
-  writeFileSync(tmp, text);
-  try {
-    const out = execFileSync(
-      "python3",
-      ["-c", "import sys,yaml,json; json.dump(yaml.safe_load(open(sys.argv[1])), sys.stdout)", tmp],
-      { encoding: "utf-8" },
-    );
-    return JSON.parse(out);
-  } catch (err: unknown) {
-    const msg = err instanceof Error ? (err as { stderr?: string }).stderr ?? err.message : String(err);
-    throw new Error(`autoinstall YAML did not parse for ${label}: ${msg}`);
-  } finally {
-    try { unlinkSync(tmp); } catch { /* ignore */ }
-  }
-}
-
-describe("renderUbuntuAutoinstall", () => {
-  for (const role of ["vanilla", "worker", "infra"]) {
-    it(`renders parseable YAML for role=${role}`, () => {
-      const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role }), role);
-      expect(doc.autoinstall).toBeDefined();
-      expect(doc.autoinstall.version).toBe(1);
-      // storage.config must be a flat list; the indentation bug produced a
-      // nested map here, which is how it went unnoticed.
-      expect(Array.isArray(doc.autoinstall.storage.config)).toBe(true);
-    });
-  }
-
-  it("gives the longhorn role its volume as a sibling entry, not a nested map", () => {
-    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" }), "longhorn");
-    const ids = doc.autoinstall.storage.config.map((e: { id: string }) => e.id);
-    expect(ids).toContain("lv-longhorn");
-    expect(ids).toContain("mount-longhorn");
-  });
-
-  it("sets EUI-64 link-local so MAC-keyed DHCPv6 reservations can match", () => {
-    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" }), "eui64");
-    const late = (doc.autoinstall["late-commands"] as string[]).join("\n");
-    expect(late).toContain("10-ipv6-eui64.conf");
-    expect(late).toContain("ipv6.addr-gen-mode=eui64");
-  });
-
-  it("requests both address families on the primary NIC", () => {
-    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" }), "net");
-    const eth = doc.autoinstall.network.ethernets.primary;
-    expect(eth.dhcp4).toBe(true);
-    // Without this a node provisioned into a dual-stack cluster comes up with
-    // no IPv6 and joins as an IPv4-only member.
-    expect(eth.dhcp6).toBe(true);
-    // The install must not block waiting for a v6 lease that may never come.
-    expect(eth.optional).toBe(true);
-  });
-});
-
-// ---------------------------------------------------------------------------
-// v2 layout. Ubuntu previously sized rancher at 20G while the Fedora kickstart
-// used 120G, from a template whose comment claimed the sizes matched. Both now
-// render from shared/src/types/layout.ts, so this checks the rendering, and
-// layout.test.ts checks the definition.
-// ---------------------------------------------------------------------------
-describe("v2 storage layout", () => {
-  const roles = ["vanilla", "worker", "infra", "labcontroller"] as const;
-
-  function lvmPartitions(doc: Record<string, any>): Record<string, string> {
-    const cfg = doc["autoinstall"]["storage"]["config"] as Array<Record<string, any>>;
-    return Object.fromEntries(
-      cfg.filter((c) => c["type"] === "lvm_partition").map((c) => [c["name"], c["size"]]),
-    );
-  }
-
-  it.each(roles)("%s renders valid YAML at schemaVersion 2", (role) => {
-    const doc = parseYaml(
-      renderUbuntuAutoinstall({ ...base, role, schemaVersion: 2 } as any),
-      `v2-${role}`,
-    );
-    expect(Array.isArray(doc["autoinstall"]["storage"]["config"])).toBe(true);
-  });
-
-  it("v2 sizes match the shared definition, not the old 20G rancher", () => {
-    const doc = parseYaml(
-      renderUbuntuAutoinstall({ ...base, role: "worker", schemaVersion: 2 } as any),
-      "v2-sizes",
-    );
-    const lvs = lvmPartitions(doc);
-    expect(lvs["rancher"]).toBe("250G");   // was 20G here and 120G in kickstart
-    expect(lvs["longhorn"]).toBe("300G");  // was grow
-    expect(lvs["root"]).toBe("32G");
-    expect(lvs["var"]).toBe("48G");
-  });
-
-  it("v2 gives labcontroller a rancher LV; v1 does not", () => {
-    const v2 = lvmPartitions(parseYaml(
-      renderUbuntuAutoinstall({ ...base, role: "labcontroller", schemaVersion: 2 } as any), "v2-lc"));
-    const v1 = lvmPartitions(parseYaml(
-      renderUbuntuAutoinstall({ ...base, role: "labcontroller" } as any), "v1-lc"));
-    expect(v2["rancher"]).toBe("250G");
-    expect(v1["rancher"]).toBeUndefined();
-  });
-
-  it("v1 remains the default and is unchanged", () => {
-    const lvs = lvmPartitions(parseYaml(
-      renderUbuntuAutoinstall({ ...base, role: "worker" } as any), "v1-default"));
-    expect(lvs["root"]).toBe("33G");
-    expect(lvs["var"]).toBe("100G");
-    expect(lvs["swap"]).toBe("27G");
-  });
-
-  it("no v2 volume is unbounded", () => {
-    for (const role of roles) {
-      const lvs = lvmPartitions(parseYaml(
-        renderUbuntuAutoinstall({ ...base, role, schemaVersion: 2 } as any), `v2-grow-${role}`));
-      for (const [name, size] of Object.entries(lvs)) {
-        expect(String(size), `${role}/${name}`).toMatch(/^\d+G$/);
-      }
-    }
-  });
-});
-
-/**
- * KNOWN GAP, asserted so it cannot be assumed away.
- *
- * The Fedora kickstart preserves /home, /srv, longhorn and rancher across a
- * reinstall (%pre probes for an existing labvg). The Ubuntu autoinstall has no
- * equivalent: `disk0` is wiped unconditionally, so EVERY Ubuntu reinstall is
- * destructive.
- *
- * These tests pin that behaviour rather than pretend it is fine. If someone
- * adds curtin `preserve: true` handling, these fail and should be rewritten --
- * which is the point: the change becomes visible instead of silent.
- */
-describe("Ubuntu reinstall is destructive (known gap)", () => {
-  it("wipes the disk unconditionally — there is no reprovision branch", () => {
-    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" } as any), "destructive");
-    const cfg = doc["autoinstall"]["storage"]["config"] as Array<Record<string, any>>;
-    const disk = cfg.find((c) => c["type"] === "disk");
-    expect(disk?.["wipe"]).toBe("superblock-recursive");
-  });
-
-  it("no volume is marked preserve — nothing survives a reinstall", () => {
-    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" } as any), "no-preserve");
-    const cfg = doc["autoinstall"]["storage"]["config"] as Array<Record<string, any>>;
-    expect(cfg.some((c) => c["preserve"] === true)).toBe(false);
-  });
-
-  it("the same is true at schemaVersion 2 — v2 did not fix this", () => {
-    const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker", schemaVersion: 2 } as any), "v2-destructive");
-    const cfg = doc["autoinstall"]["storage"]["config"] as Array<Record<string, any>>;
-    expect(cfg.find((c) => c["type"] === "disk")?.["wipe"]).toBe("superblock-recursive");
-  });
-});
-
-describe("progress callback mac", () => {
-  it("bakes the queue mac into the completion callback when given", () => {
-    const doc = renderUbuntuAutoinstall({ ...base, role: "worker", mac: "aa:bb:cc:dd:ee:ff" });
-    expect(doc).toContain("aa:bb:cc:dd:ee:ff");
-    expect(doc).not.toContain("ip link show");
-  });
-
-  it("falls back to runtime detection when no mac is known", () => {
-    const doc = renderUbuntuAutoinstall({ ...base, role: "worker" });
-    expect(doc).toContain("ip link show");
-  });
-});

bastion/src/bastion/tests/vyos-bundle.test.ts

@@ -1,155 +0,0 @@
-import { describe, it, expect, vi } from "vitest";
-import type { VyosBundle } from "@lab/shared";
-import { buildVyosConfigSpec } from "../src/templates/vyos-config-spec.js";
-
-/**
- * A bundle is what makes "one config, two apply paths" true rather than
- * aspirational: `pulumi up` POSTs the subtree model to a running router, labctl
- * writes the same model into config.boot during a PXE install. These tests pin
- * the properties that keep the two honest.
- */
-const bundle: VyosBundle = {
-  sets: [
-    { path: ["system", "host-name"], value: "vyos001" },
-    { path: ["interfaces", "bonding", "bond0", "address"], value: "192.168.1.252/24" },
-    { path: ["interfaces", "bonding", "bond0", "member", "interface"], value: "eth1", replace: false },
-    { path: ["interfaces", "bonding", "bond0", "vif", "53", "disable"] },
-    { path: ["interfaces", "pppoe", "pppoe0", "authentication", "password"], value: "@secret:pppoePassword" },
-    { path: ["interfaces", "pppoe", "pppoe0", "mtu"], value: "1492" },
-  ],
-  tags: [["interfaces", "bonding", "bond0"], ["interfaces", "ethernet"]],
-};
-
-const build = (hostname: string, extra: Record<string, unknown> = {}) =>
-  buildVyosConfigSpec({
-    hostname,
-    spec: { bundle, ...extra },
-    defaultPassword: "changeme",
-  });
-
-describe("vyos config spec from a Pulumi bundle", () => {
-  it("applies non-secret nodes verbatim, preserving valuelessness and replace:false", () => {
-    const spec = build("vyos001");
-
-    expect(spec.sets).toContainEqual({
-      path: ["interfaces", "bonding", "bond0", "address"],
-      value: "192.168.1.252/24",
-    });
-    // A multi-value node must keep replace:false or the second bond member
-    // overwrites the first.
-    expect(spec.sets).toContainEqual({
-      path: ["interfaces", "bonding", "bond0", "member", "interface"],
-      value: "eth1",
-      replace: false,
-    });
-    // A valueless node must not acquire a value on the way through.
-    expect(spec.sets).toContainEqual({
-      path: ["interfaces", "bonding", "bond0", "vif", "53", "disable"],
-    });
-    expect(spec.tags).toEqual(bundle.tags);
-  });
-
-  it("drops secret-valued nodes instead of installing the sentinel text", () => {
-    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
-    const spec = build("vyos001");
-
-    const values = spec.sets.map((s) => s.value ?? "");
-    expect(values.some((v) => v.startsWith("@secret:"))).toBe(false);
-    expect(spec.sets.some((s) => s.path.includes("authentication"))).toBe(false);
-    // Silently dropping the WAN credential would leave someone debugging a dead
-    // PPPoE link, so it has to be said out loud.
-    expect(warn).toHaveBeenCalledWith(expect.stringContaining("pulumi up"));
-    warn.mockRestore();
-  });
-
-  it("forces the hostname the install was asked for, not the bundle's", () => {
-    // The bundle is exported from one router and reused for its peer; taking the
-    // hostname from it would put two vyos001s on the network.
-    const spec = build("vyos002");
-    const hostnames = spec.sets.filter(
-      (s) => s.path.length === 2 && s.path[0] === "system" && s.path[1] === "host-name",
-    );
-    expect(hostnames).toEqual([{ path: ["system", "host-name"], value: "vyos002" }]);
-  });
-
-  it("still honours installer inputs, which are not router config", () => {
-    const spec = buildVyosConfigSpec({
-      hostname: "vyos001",
-      spec: { bundle, password: "s3cret", freshConfig: true },
-      defaultPassword: "changeme",
-      disk: "nvme0n1",
-    });
-    expect(spec.password).toBe("s3cret");
-    expect(spec.freshConfig).toBe(true);
-    expect(spec.disk).toBe("/dev/nvme0n1");
-  });
-
-  it("enables the HTTP API at install so Pulumi can manage the router from first boot", () => {
-    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
-    const spec = buildVyosConfigSpec({
-      hostname: "vyos001",
-      spec: { bundle, apiKey: "k3y", apiListenAddress: "10.0.1.252" },
-      defaultPassword: "changeme",
-    });
-
-    expect(spec.sets).toContainEqual({
-      path: ["service", "https", "api", "keys", "id", "pulumi", "key"],
-      value: "k3y",
-    });
-    expect(spec.sets).toContainEqual({ path: ["service", "https", "api", "rest"] });
-    expect(spec.sets).toContainEqual({
-      path: ["service", "https", "listen-address"],
-      value: "10.0.1.252",
-    });
-    // The key id is a tag node; without this the installer's ConfigTree rejects it.
-    expect(spec.tags).toContainEqual(["service", "https", "api", "keys", "id"]);
-    warn.mockRestore();
-  });
-
-  it("binds the API to the static management address when none is given", () => {
-    const spec = buildVyosConfigSpec({
-      hostname: "vyos001",
-      spec: { apiKey: "k3y", mgmtAddress: "192.168.1.252/24" },
-      defaultPassword: "changeme",
-    });
-    expect(spec.sets).toContainEqual({
-      path: ["service", "https", "listen-address"],
-      value: "192.168.1.252",
-    });
-  });
-
-  it("refuses to enable the API unbound rather than exposing it on the WAN", () => {
-    const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
-    // Management is DHCP, so there is no address to bind at build time. Binding
-    // to everything would put a config-write endpoint on the WAN.
-    const spec = buildVyosConfigSpec({
-      hostname: "vyos001",
-      spec: { apiKey: "k3y", mgmtAddress: "dhcp" },
-      defaultPassword: "changeme",
-    });
-    expect(spec.sets.some((s) => s.path[0] === "service" && s.path[1] === "https")).toBe(false);
-    expect(warn).toHaveBeenCalledWith(expect.stringContaining("has NOT been enabled"));
-    warn.mockRestore();
-  });
-
-  it("does not enable the API when no key is supplied", () => {
-    const spec = buildVyosConfigSpec({
-      hostname: "vyos001",
-      spec: { mgmtAddress: "192.168.1.252/24" },
-      defaultPassword: "changeme",
-    });
-    expect(spec.sets.some((s) => s.path[0] === "service" && s.path[1] === "https")).toBe(false);
-  });
-
-  it("ignores the derived path entirely when a bundle is present", () => {
-    // Belt and braces: even if topology flags reach this far (the CLI rejects
-    // them), the bundle must win rather than merge.
-    const spec = buildVyosConfigSpec({
-      hostname: "vyos001",
-      spec: { bundle, bondMembers: ["eth2", "eth3"], vlans: [{ id: 99, address: "10.9.9.1/24" }] },
-      defaultPassword: "changeme",
-    });
-    expect(spec.sets.some((s) => s.path.includes("99"))).toBe(false);
-    expect(spec.sets.filter((s) => s.value === "eth2" || s.value === "eth3")).toEqual([]);
-  });
-});

bastion/src/bastion/tests/vyos.test.ts

@@ -1,548 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { mkdirSync, rmSync } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import type { BastionConfig } from "@lab/shared";
-import type { FastifyInstance } from "fastify";
-import { createApp } from "../src/server.js";
-import type { StateManager } from "../src/services/state.js";
-import { buildVyosConfigSpec } from "../src/templates/vyos-config-spec.js";
-import { renderVyosInstallPy } from "../src/templates/vyos-install.py.js";
-
-function createTestConfig(testDir: string): BastionConfig {
-  return {
-    fedoraVersion: "43",
-    arch: "x86_64",
-    httpPort: 0,
-    timezone: "Europe/London",
-    locale: "en_GB.UTF-8",
-    bastionDir: testDir,
-    domain: "test.local",
-    dhcpMode: "proxy",
-    dhcpRangeStart: "",
-    dhcpRangeEnd: "",
-    ubuntuVersion: "26.04",
-    ubuntuMirror: "https://releases.ubuntu.com/26.04",
-    vyosIsoUrl: "https://example.invalid/vyos.iso",
-    vyosDefaultPassword: "test-pw",
-    iface: "eth0",
-    serverIp: "10.0.0.1",
-    network: "10.0.0.0",
-    gateway: "10.0.0.1",
-    sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST lab@test"],
-    adminUser: "testadmin",
-    syslogPort: 15515,
-    skipDnsmasq: true,
-    skipArtifacts: true,
-    fedoraMirror: "https://example.invalid/fedora",
-    tftpDir: join(testDir, "tftp"),
-    httpDir: join(testDir, "http"),
-    stateFile: join(testDir, "state.json"),
-  };
-}
-
-/** Pull the base64 spec back out of the generated Python driver. */
-function decodeSpecFrom(python: string): Record<string, unknown> {
-  const match = /base64\.b64decode\("([^"]+)"\)/.exec(python);
-  if (!match?.[1]) throw new Error("no base64 spec found in generated driver");
-  return JSON.parse(Buffer.from(match[1], "base64").toString("utf-8"));
-}
-
-describe("vyos config spec", () => {
-  it("puts VLANs on the bond when members are given", () => {
-    const spec = buildVyosConfigSpec({
-      hostname: "fw1",
-      defaultPassword: "pw",
-      spec: {
-        mgmtInterface: "eth0",
-        mgmtAddress: "10.0.8.2/24",
-        bondMembers: ["eth2", "eth3"],
-        vlans: [{ id: 10, address: "10.0.10.1/24", description: "k8s" }],
-      },
-    });
-
-    const paths = spec.sets.map((s) => s.path.join(" "));
-    expect(paths).toContain("interfaces bonding bond0 mode");
-    expect(paths).toContain("interfaces bonding bond0 vif 10 address");
-    // VLANs must hang off the bond, not the management NIC.
-    expect(paths).not.toContain("interfaces ethernet eth0 vif 10 address");
-
-    // Bond members are a multi-value node — appending, not replacing, is what
-    // keeps the second member from overwriting the first.
-    const members = spec.sets.filter(
-      (s) => s.path.join(" ") === "interfaces bonding bond0 member interface",
-    );
-    expect(members.map((m) => m.value)).toEqual(["eth2", "eth3"]);
-    expect(members.every((m) => m.replace === false)).toBe(true);
-  });
-
-  it("falls back to VLANs on the management NIC when unbonded", () => {
-    const spec = buildVyosConfigSpec({
-      hostname: "fw2",
-      defaultPassword: "pw",
-      spec: { mgmtInterface: "eth1", vlans: [{ id: 20, address: "10.0.20.1/24" }] },
-    });
-
-    const paths = spec.sets.map((s) => s.path.join(" "));
-    expect(paths).toContain("interfaces ethernet eth1 vif 20 address");
-  });
-
-  it("normalises the target disk to a full /dev path", () => {
-    // find_disks() enumerates with `lsblk -Jbp`, so valid responses are full
-    // paths; a bare name fails valid_responses and re-prompts forever.
-    expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw", disk: "/dev/mmcblk0" }).disk)
-      .toBe("/dev/mmcblk0");
-    expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw", disk: "mmcblk0" }).disk)
-      .toBe("/dev/mmcblk0");
-    expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw" }).disk).toBe("");
-  });
-
-  it("defaults to dhcp on eth0 and never opts into RAID", () => {
-    const spec = buildVyosConfigSpec({ hostname: "fw4", defaultPassword: "pw" });
-    const address = spec.sets.find(
-      (s) => s.path.join(" ") === "interfaces ethernet eth0 address",
-    );
-    expect(address?.value).toBe("dhcp");
-    // The installer's RAID prompt defaults to yes; a second disk must not
-    // silently produce a mirror.
-    expect(spec.raid).toBe(false);
-  });
-});
-
-describe("vyos routes", () => {
-  let testDir: string;
-  let app: FastifyInstance;
-  let state: StateManager;
-  const mac = "aa:bb:cc:11:22:33";
-
-  beforeEach(() => {
-    testDir = join(tmpdir(), `bastion-vyos-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
-    mkdirSync(join(testDir, "http"), { recursive: true });
-    mkdirSync(join(testDir, "tftp"), { recursive: true });
-
-    const result = createApp(createTestConfig(testDir));
-    app = result.app;
-    state = result.state;
-  });
-
-  afterEach(async () => {
-    await app.close();
-    rmSync(testDir, { recursive: true, force: true });
-  });
-
-  it("dispatches a queued vyos machine to the live-boot script", async () => {
-    state.update((s) => {
-      s.install_queue[mac] = {
-        hostname: "fw1",
-        disk: "/dev/nvme0n1",
-        role: "worker",
-        os: "vyos-rolling",
-        queued_at: new Date().toISOString(),
-      };
-    });
-
-    const response = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}` });
-    expect(response.statusCode).toBe(200);
-
-    expect(response.body).toContain("/vyos-vmlinuz");
-    expect(response.body).toContain("fetch=http://10.0.0.1:0/vyos-filesystem.squashfs");
-    expect(response.body).toContain(`live-config.hooks=http://10.0.0.1:0/vyos/autoinstall.sh?mac=${mac}`);
-
-    // `nonetworking` appears in VyOS's own PXE docs but breaks the hook fetch,
-    // and console=ttyS0 costs 30s per systemd phase on boards with no UART.
-    expect(response.body).not.toContain("nonetworking");
-    expect(response.body).not.toContain("console=ttyS0");
-  });
-
-  it("serves a hook that fetches and executes the install driver", async () => {
-    const response = await app.inject({ method: "GET", url: `/vyos/autoinstall.sh?mac=${mac}` });
-    expect(response.statusCode).toBe(200);
-    expect(response.body).toContain(`/vyos/install.py?mac=${mac}`);
-    expect(response.body).toContain("python3 /tmp/vyos-install.py");
-  });
-
-  it("bakes the machine's config into the generated install driver", async () => {
-    state.update((s) => {
-      s.install_queue[mac] = {
-        hostname: "fw1",
-        disk: "/dev/nvme0n1",
-        role: "worker",
-        os: "vyos-rolling",
-        queued_at: new Date().toISOString(),
-        vyos: {
-          mgmtInterface: "eth0",
-          mgmtAddress: "10.0.8.2/24",
-          bondMembers: ["eth2", "eth3"],
-          vlans: [{ id: 10, address: "10.0.10.1/24" }],
-          password: "s3cret",
-        },
-      };
-    });
-
-    const response = await app.inject({ method: "GET", url: `/vyos/install.py?mac=${mac}` });
-    expect(response.statusCode).toBe(200);
-
-    // Builds config from the image's own default so the vyos-config-version
-    // trailer matches and first boot skips migrations.
-    expect(response.body).toContain("/opt/vyatta/etc/config.boot.default");
-    expect(response.body).toContain("/usr/libexec/vyos/op_mode/image_installer.py");
-    // "complete" is what moves the machine out of the install queue.
-    expect(response.body).toContain('report("complete"');
-
-    const spec = decodeSpecFrom(response.body);
-    expect(spec["hostname"]).toBe("fw1");
-    expect(spec["password"]).toBe("s3cret");
-    expect(spec["disk"]).toBe("/dev/nvme0n1");
-
-    const paths = (spec["sets"] as Array<{ path: string[] }>).map((s) => s.path.join(" "));
-    expect(paths).toContain("interfaces bonding bond0 vif 10 address");
-    expect(paths).toContain("system host-name");
-  });
-
-  it("falls back to the bastion default password when none is set", async () => {
-    state.update((s) => {
-      s.install_queue[mac] = {
-        hostname: "fw9",
-        disk: "",
-        role: "worker",
-        os: "vyos-rolling",
-        queued_at: new Date().toISOString(),
-      };
-    });
-
-    const response = await app.inject({ method: "GET", url: `/vyos/install.py?mac=${mac}` });
-    const spec = decodeSpecFrom(response.body);
-    expect(spec["password"]).toBe("test-pw");
-    // Empty disk means "accept the installer's first-disk default".
-    expect(spec["disk"]).toBe("");
-  });
-});
-
-describe("vyos hw-id pinning", () => {
-  it("emits hw-id for the mgmt interface and each bond member", () => {
-    // Discovery sees enp2s0/enp1s0f0np0 under Fedora, but VyOS enumerates its
-    // own eth<N>. Pinning by MAC is what makes the mapping deterministic.
-    const spec = buildVyosConfigSpec({
-      hostname: "fw1",
-      defaultPassword: "pw",
-      spec: {
-        mgmtInterface: "eth2",
-        bondMembers: ["eth0", "eth1"],
-        hwIds: {
-          eth2: "64:62:66:25:96:47",
-          eth0: "64:62:66:25:96:45",
-          eth1: "64:62:66:25:96:46",
-        },
-      },
-    });
-
-    const hw = spec.sets.filter((s) => s.path[s.path.length - 1] === "hw-id");
-    expect(hw.map((s) => [s.path[2], s.value])).toEqual([
-      ["eth2", "64:62:66:25:96:47"],
-      ["eth0", "64:62:66:25:96:45"],
-      ["eth1", "64:62:66:25:96:46"],
-    ]);
-  });
-
-  it("omits hw-id entirely when no mapping is given", () => {
-    const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" });
-    expect(spec.sets.some((s) => s.path.includes("hw-id"))).toBe(false);
-  });
-});
-
-describe("vyos management VLAN", () => {
-  it("puts the mgmt VLAN on the PXE port while the bond carries routed VLANs", () => {
-    // Trunked PXE port: boots untagged on the VLAN the bastion serves, stays
-    // reachable on the tagged management VLAN.
-    const spec = buildVyosConfigSpec({
-      hostname: "vyos001",
-      defaultPassword: "pw",
-      spec: {
-        mgmtInterface: "eth2",
-        mgmtAddress: "dhcp",
-        mgmtVlan: { id: 3, address: "192.168.3.4/24", description: "kvm" },
-        bondMembers: ["eth0", "eth1"],
-        vlans: [{ id: 2, address: "192.168.8.2/23" }],
-      },
-    });
-
-    const paths = spec.sets.map((s) => s.path.join(" "));
-    expect(paths).toContain("interfaces ethernet eth2 vif 3 address");
-    expect(paths).toContain("interfaces bonding bond0 vif 2 address");
-    // The mgmt VLAN must not land on the bond.
-    expect(paths).not.toContain("interfaces bonding bond0 vif 3 address");
-    expect(spec.tags.map((t) => t.join(" "))).toContain("interfaces ethernet eth2 vif");
-  });
-});
-
-describe("vyos VRRP HA", () => {
-  const haSpec = {
-    mgmtInterface: "eth2",
-    mgmtAddress: "dhcp",
-    bondMembers: ["eth0", "eth1"],
-    bondAddress: "192.168.1.252/24",
-    bondVrrp: "192.168.1.254/24",
-    vrrpPriority: 200,
-    vlans: [
-      { id: 3, address: "192.168.3.4/24", vrrp: "192.168.3.254/24" },
-      { id: 200, address: "192.168.2.252/24" },   // no VIP on this one
-    ],
-  };
-
-  it("emits a vrrp group per VIP with vrid = VLAN id and dotted vif interface", () => {
-    const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw", spec: haSpec });
-    const paths = spec.sets.map((s) => `${s.path.join(" ")}${s.value !== undefined ? "=" + s.value : ""}`);
-
-    expect(paths).toContain("interfaces bonding bond0 address=192.168.1.252/24");
-    // untagged bond group: vrid 1, interface bond0 itself
-    expect(paths).toContain("high-availability vrrp group native interface=bond0");
-    expect(paths).toContain("high-availability vrrp group native vrid=1");
-    // address is a tag node -- VIP is the final path segment, no value
-    expect(paths).toContain("high-availability vrrp group native address 192.168.1.254/24");
-    // VLAN group: vrid = VLAN id, dotted vif
-    expect(paths).toContain("high-availability vrrp group vlan3 interface=bond0.3");
-    expect(paths).toContain("high-availability vrrp group vlan3 vrid=3");
-    expect(paths).toContain("high-availability vrrp group vlan3 address 192.168.3.254/24");
-    // VLAN without a VIP gets no group
-    expect(paths.some((p) => p.includes("group vlan200"))).toBe(false);
-  });
-
-  it("applies the box-wide priority and one sync group over all groups", () => {
-    const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw", spec: haSpec });
-    const prio = spec.sets.filter((s) => s.path[s.path.length - 1] === "priority"
-      && s.path[0] === "high-availability");
-    expect(prio).toHaveLength(2);
-    expect(prio.every((s) => s.value === "200")).toBe(true);
-
-    // sync group binds the pair: all groups fail over together
-    const members = spec.sets.filter(
-      (s) => s.path.join(" ") === "high-availability vrrp sync-group MAIN member",
-    );
-    expect(members.map((m) => m.value)).toEqual(["native", "vlan3"]);
-    expect(members.every((m) => m.replace === false)).toBe(true);
-  });
-
-  it("emits no high-availability nodes when no VIPs are given", () => {
-    const spec = buildVyosConfigSpec({
-      hostname: "fw1",
-      defaultPassword: "pw",
-      spec: { bondMembers: ["eth0", "eth1"], vlans: [{ id: 3, address: "192.168.3.4/24" }] },
-    });
-    expect(spec.sets.some((s) => s.path[0] === "high-availability")).toBe(false);
-  });
-});
-
-describe("pickLargestInitrd", async () => {
-  const { pickLargestInitrd } = await import("../src/main.js");
-
-  // Verbatim from `xorriso -lsl /live/` on vyos-2026.08.05-0033-rolling.
-  const realListing = `total 8
--r--r--r--    1 0        0           22255 Aug  5 01:33 'filesystem.packages'
--r--r--r--    1 0        0               6 Aug  5 01:33 'filesystem.packages-remove'
--r--r--r--    1 0        0        541192192 Aug  5 01:33 'filesystem.squashfs'
--r--r--r--    1 0        0        50352547 Aug  5 01:33 'initrd.img'
--r--r--r--    1 0        0        50352547 Aug  5 01:33 'initrd.img-6.18.41-vyos'
--r--r--r--    1 0        0              20 Aug  5 01:33 'packages.txt'
--r--r--r--    1 0        0         9135104 Aug  2 19:54 'vmlinuz'
--r--r--r--    1 0        0         9135104 Aug  2 19:54 'vmlinuz-6.18.41-vyos'
-`;
-
-  it("picks a full-size initrd from a real nightly listing", () => {
-    expect(pickLargestInitrd(realListing)).toEqual({ name: "initrd.img", size: 50352547 });
-  });
-
-  it("ignores 0-byte decoys and symlinks (which report link size, not target size)", () => {
-    const listing = `total 8
--r--r--r--    1 0        0               0 Aug  5 01:33 'initrd.img'
-lrwxrwxrwx    1 0        0              24 Aug  5 01:33 'initrd.img-link' -> 'initrd.img-6.18.41-vyos'
--r--r--r--    1 0        0        50352547 Aug  5 01:33 'initrd.img-6.18.41-vyos'
-`;
-    expect(pickLargestInitrd(listing)).toEqual({ name: "initrd.img-6.18.41-vyos", size: 50352547 });
-  });
-
-  it("returns undefined when only decoys exist", () => {
-    expect(pickLargestInitrd("-r--r--r-- 1 0 0 0 Aug 5 01:33 'initrd.img'\n")).toBeUndefined();
-  });
-});
-
-describe("vyos fedora-parity features", () => {
-  it("computes reportAddress from a static mgmt address, empty for dhcp", () => {
-    const staticSpec = buildVyosConfigSpec({
-      hostname: "fw1", defaultPassword: "pw",
-      spec: { mgmtAddress: "192.168.8.2/23" },
-    });
-    expect(staticSpec.reportAddress).toBe("192.168.8.2");
-
-    const dhcpSpec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" });
-    expect(dhcpSpec.reportAddress).toBe("");
-  });
-
-  it("defaults freshConfig off (reinstall preserves the on-disk config)", () => {
-    expect(buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" }).freshConfig).toBe(false);
-    expect(buildVyosConfigSpec({
-      hostname: "fw1", defaultPassword: "pw", spec: { freshConfig: true },
-    }).freshConfig).toBe(true);
-  });
-
-  it("driver streams logs to /api/log and reports 'ready at' on completion", async () => {
-    const testDir = join(tmpdir(), `bastion-vyos-parity-${Date.now()}`);
-    mkdirSync(join(testDir, "http"), { recursive: true });
-    mkdirSync(join(testDir, "tftp"), { recursive: true });
-    const { app: parityApp, state: parityState } = createApp(createTestConfig(testDir));
-    try {
-      parityState.update((s) => {
-        s.install_queue["aa:bb:cc:44:55:66"] = {
-          hostname: "fw9", disk: "/dev/vda", role: "vanilla",
-          os: "vyos-rolling", queued_at: new Date().toISOString(),
-        };
-      });
-      const response = await parityApp.inject({
-        method: "GET", url: "/vyos/install.py?mac=aa:bb:cc:44:55:66",
-      });
-      expect(response.body).toContain("/api/log");
-      expect(response.body).toContain('"lines": batch');
-      expect(response.body).toContain('report("complete", "ready at %s"');
-      expect(response.body).toContain("ensure_network_boot_first");
-      expect(response.body).toContain("lab-provisioned");
-      expect(response.body).toContain('ROLE = "vanilla"');
-    } finally {
-      await parityApp.close();
-      rmSync(testDir, { recursive: true, force: true });
-    }
-  });
-
-  it("complete with 'ready at' records installed.ip for a vyos machine", async () => {
-    const testDir = join(tmpdir(), `bastion-vyos-complete-${Date.now()}`);
-    mkdirSync(join(testDir, "http"), { recursive: true });
-    mkdirSync(join(testDir, "tftp"), { recursive: true });
-    const { app: cApp, state: cState } = createApp(createTestConfig(testDir));
-    try {
-      const mac2 = "aa:bb:cc:77:88:99";
-      cState.update((s) => {
-        s.install_queue[mac2] = {
-          hostname: "fw1", disk: "/dev/vda", role: "vanilla",
-          os: "vyos-rolling", queued_at: new Date().toISOString(),
-        };
-      });
-      const response = await cApp.inject({
-        method: "POST", url: "/api/progress",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ mac: mac2, stage: "complete", detail: "ready at 192.168.8.2" }),
-      });
-      expect(response.statusCode).toBe(200);
-      const installed = cState.load().installed[mac2];
-      expect(installed?.ip).toBe("192.168.8.2");
-      expect(installed?.os).toBe("vyos-rolling");
-    } finally {
-      await cApp.close();
-      rmSync(testDir, { recursive: true, force: true });
-    }
-  });
-});
-
-describe("vyos installer prompt coverage", () => {
-  // Every interactive prompt image_installer.py can emit, copied verbatim from
-  // the MSG_* constants (including the reinstall-only search_previous_installation
-  // ones). An unanswered prompt does not fail loudly -- the installer simply
-  // blocks on stdin until the driver's stall timeout, which is how the reinstall
-  // path silently hung for 15 minutes in the VM test.
-  const PROMPTS: Record<string, string> = {
-    continue: "Would you like to continue? [y/N] ",
-    imageName: "What would you like to name this image? (Default: 1.5-rolling) ",
-    password: 'Please enter a password for the "vyos" user: ',
-    passwordConfirm: 'Please confirm password for the "vyos" user: ',
-    console: "What console should be used by default? (K: KVM, S: Serial)? (Default: K) ",
-    raidConfigure: "Would you like to configure RAID-1 mirroring? [Y/n] ",
-    raidFoundDisks: "Would you like to configure RAID-1 mirroring on them? [Y/n] ",
-    raidChooseDisks: "Would you like to choose two disks for RAID-1 mirroring? [Y/n] ",
-    diskSelect: "Which one should be used for installation? (Default: /dev/vda) ",
-    diskConfirm: "Installation will delete all data on the drive. Continue? [y/N] ",
-    raidConfirm: "Installation will delete all data on both drives. Continue? [y/N] ",
-    rootSizeAll: "Would you like to use all the free space on the drive? [Y/n] ",
-    bootConfig: "Which file would you like as boot config? ",
-    copyData: "Would you like to copy data to the new image? [Y/n] ",
-    chooseCopyData: "From which image would you like to save config information? ",
-    copyEncData: "Would you like to copy the encrypted config to the new image? [Y/n] ",
-    chooseCopyEncData: "From which image would you like to copy the encrypted config? ",
-  };
-
-  it("answers every installer prompt exactly once", () => {
-    const { execFileSync } = require("node:child_process") as typeof import("node:child_process");
-    const { writeFileSync, unlinkSync, mkdtempSync } = require("node:fs") as typeof import("node:fs");
-
-    // Skip cleanly where python3 is unavailable (same spirit as the
-    // ksvalidator-backed kickstart test).
-    try {
-      execFileSync("python3", ["--version"], { stdio: "pipe" });
-    } catch {
-      return;
-    }
-
-    const spec = buildVyosConfigSpec({
-      hostname: "fw1", defaultPassword: "pw", disk: "/dev/vda",
-    });
-    const driver = renderVyosInstallPy({
-      spec, mac: "aa:bb:cc:11:22:33", serverIp: "10.0.0.1", httpPort: 8080, role: "vanilla",
-    });
-
-    const dir = mkdtempSync(join(tmpdir(), "vyos-rules-"));
-    const driverPath = join(dir, "driver.py");
-    const checkPath = join(dir, "check.py");
-    writeFileSync(driverPath, driver);
-    writeFileSync(checkPath, `
-import importlib.util, json, sys
-spec = importlib.util.spec_from_file_location("drv", ${JSON.stringify(driverPath)})
-drv = importlib.util.module_from_spec(spec); spec.loader.exec_module(drv)
-rules = drv.build_rules()
-prompts = json.loads(sys.argv[1])
-out = {}
-for label, text in prompts.items():
-    out[label] = len([r for p, r in rules if p.search(text.encode())])
-print(json.dumps(out))
-`);
-
-    try {
-      const stdout = execFileSync("python3", [checkPath, JSON.stringify(PROMPTS)], {
-        encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"],
-      });
-      const counts = JSON.parse(stdout) as Record<string, number>;
-      const unanswered = Object.entries(counts).filter(([, n]) => n !== 1);
-      expect(unanswered).toEqual([]);
-    } finally {
-      try { unlinkSync(driverPath); unlinkSync(checkPath); } catch { /* best effort */ }
-      try { rmSync(dir, { recursive: true, force: true }); } catch { /* best effort */ }
-    }
-  });
-});
-
-describe("vyos boot NIC pinning", () => {
-  it("pins the boot interface by MAC via BOOTIF", async () => {
-    // Without this, live-boot picks the first *connected* NIC. On the VP2440
-    // the SFP+ pair links before the copper PXE port, so live-boot tried the
-    // fiber ports (no DHCP), timed out 15s each, and failed with "Unable to
-    // find a live file system on the network".
-    const testDir = join(tmpdir(), `bastion-vyos-bootif-${Date.now()}`);
-    mkdirSync(join(testDir, "http"), { recursive: true });
-    mkdirSync(join(testDir, "tftp"), { recursive: true });
-    const { app: a, state: st } = createApp(createTestConfig(testDir));
-    try {
-      const m = "64:62:66:25:96:47";
-      st.update((s) => {
-        s.install_queue[m] = {
-          hostname: "vyos001", disk: "/dev/mmcblk0", role: "vanilla",
-          os: "vyos-rolling", queued_at: new Date().toISOString(),
-        };
-      });
-      const res = await a.inject({ method: "GET", url: `/dispatch?mac=${m}` });
-      // live-boot's Device_from_bootif() expects 01-<mac with dashes>
-      expect(res.body).toContain("BOOTIF=01-64-62-66-25-96-47");
-      // and it must be on the kernel line, before fetch= is attempted
-      const kernelLine = res.body.split("\n").find((l) => l.startsWith("kernel "));
-      expect(kernelLine).toContain("BOOTIF=01-64-62-66-25-96-47");
-      expect(kernelLine).toContain("fetch=");
-    } finally {
-      await a.close();
-      rmSync(testDir, { recursive: true, force: true });
-    }
-  });
-});

bastion/src/cli/src/api/client.ts

@@ -90,9 +90,6 @@ export class LabdClient {
 
   async installMachine(opts: {
     mac: string; hostname: string; disk?: string; role?: string; os?: string;
-    schemaVersion?: 1 | 2;
-    dataDisks?: import("@lab/shared").DataDisks;
-    vyos?: import("@lab/shared").VyosInstallSpec;
   }): Promise<{ status: string; data?: unknown; error?: string }> {
     return this.request("POST", "/api/machines/install", { body: opts });
   }
@@ -113,6 +110,7 @@ export class LabdClient {
     memory_gb?: number; arch?: string;
     disks?: Array<{ name: string; size_gb: number; model: string }>;
     nics?: Array<{ name: string; mac: string; state: string }>;
+    root_device?: string; root_args?: string;
   }): Promise<{ status: string; error?: string }> {
     return this.request("POST", "/api/machines/discover", { body: data });
   }

bastion/src/cli/src/commands/app.ts

@@ -57,7 +57,6 @@ async function fetchState(): Promise<BastionState | null> {
 }
 
 import { registerLabcontrollerCommands } from "./labcontroller.js";
-import { isRole, SUPPORTED_ROLES, type Role } from "@lab/shared";
 
 export function registerAppCommand(program: Command): void {
   const appCmd = program.command("app").description("Application management");
@@ -67,104 +66,18 @@ export function registerAppCommand(program: Command): void {
 
   const k3sCmd = appCmd.command("k3s").description("k3s cluster management");
 
-  k3sCmd
-    .command("evacuate <target>")
-    .description("Cordon, evict Longhorn replicas and drain so a node can be safely wiped")
-    .option("--user <user>", "SSH user", "root")
-    .option(
-      "--allow-single-replica",
-      "permit evacuating volumes that have only one replica (rebuilds from the only copy; there is no backup target)",
-    )
-    .action(async (target: string, opts: { user: string; allowSingleReplica?: boolean }) => {
-      const state = await fetchState();
-      const resolved = resolveTarget(target, state);
-      if (!resolved) {
-        console.error(`Cannot resolve target: ${target}`);
-        process.exit(1);
-      }
-      const sshKey = findSshKey();
-
-      console.log(`Evacuating ${resolved.hostname} (${resolved.ip})...`);
-      console.log("This waits for Longhorn to rebuild replicas elsewhere and can take a while.\n");
-
-      const k3sMod = new K3sModule();
-      const result = await k3sMod.evacuate({
-        hostname: resolved.hostname,
-        ip: resolved.ip,
-        role: "infra",
-        os: "fedora-43" as const,
-        arch: "x86_64" as const,
-        sshUser: opts.user,
-        ...(sshKey ? { sshKeyPath: sshKey } : {}),
-        config: opts.allowSingleReplica ? { evictionAllowSingleReplica: true } : {},
-      });
-
-      for (const line of result.output) console.log(`  ${line}`);
-      if (!result.success) {
-        console.error(`\nNode is NOT safe to wipe (${result.errors.join(", ")}). Do not reprovision it.`);
-        console.error(`If an eviction was started and left pending, clear it with:`);
-        console.error(`  labctl app k3s admit ${resolved.hostname}`);
-        process.exit(1);
-      }
-      console.log(`\n${resolved.hostname} is evacuated. Safe to reprovision.`);
-    });
-
-  k3sCmd
-    .command("admit <target>")
-    .description("Return an evacuated node to service: uncordon, clear eviction, restore its Longhorn disk")
-    .option("--user <user>", "SSH user", "root")
-    .action(async (target: string, opts: { user: string }) => {
-      const state = await fetchState();
-      const resolved = resolveTarget(target, state);
-      if (!resolved) {
-        console.error(`Cannot resolve target: ${target}`);
-        process.exit(1);
-      }
-      const sshKey = findSshKey();
-
-      console.log(`Returning ${resolved.hostname} (${resolved.ip}) to service...`);
-
-      const k3sMod = new K3sModule();
-      const result = await k3sMod.admit({
-        hostname: resolved.hostname,
-        ip: resolved.ip,
-        role: isRole(resolved.role) ? resolved.role : "infra",
-        os: "fedora-43" as const,
-        arch: "x86_64" as const,
-        sshUser: opts.user,
-        ...(sshKey ? { sshKeyPath: sshKey } : {}),
-        config: {},
-      });
-
-      for (const line of result.output) console.log(`  ${line}`);
-      if (!result.success) {
-        console.error(`\n${resolved.hostname} is NOT fully back in service (${result.errors.join(", ")}).`);
-        process.exit(1);
-      }
-      console.log(`\n${resolved.hostname} is back in service.`);
-    });
-
   k3sCmd
     .command("install <target>")
     .description("Install k3s on a target machine (hostname, IP, or MAC)")
-    .option(
-      "--role <role>",
-      "k3s role: infra, labcontroller, standalone-cluster, k8s-controller, k8s-combo (servers) or worker, k8s-worker (agents)",
-      "infra",
-    )
+    .option("--role <role>", "k3s role: infra (server) or worker (agent)", "infra")
     .option("--user <user>", "SSH user", "root")
-    .option("--k3s-server <url>", "k3s server URL (required for agent roles, and to JOIN as a server)")
-    .option("--k3s-token <token>", "k3s join token; on a first server it SEEDS the cluster token")
-    .option(
-      "--k3s-version <version>",
-      "exact k3s version, e.g. v1.34.6+k3s1. Unset installs LATEST, which on a rejoining node can be several minor versions ahead of the cluster",
-    )
+    .option("--k3s-server <url>", "k3s server URL (required for worker role)")
+    .option("--k3s-token <token>", "k3s join token (required for worker role)")
     .action(async (target: string, opts: {
       role: string;
       user: string;
       k3sServer?: string;
       k3sToken?: string;
-      k3sVersion?: string;
     }) => {
       const state = await fetchState();
       const resolved = resolveTarget(target, state);
@@ -175,16 +88,7 @@ export function registerAppCommand(program: Command): void {
         process.exit(1);
       }
 
-      // Every supported role, not just infra/worker. The old mapping silently
-      // turned anything that was not "worker" into "infra" -- so
-      // `--role k8s-combo` installed a DIFFERENT role than the one asked for,
-      // with different disk expectations and no warning.
-      if (!isRole(opts.role)) {
-        console.error(`Unknown role: ${opts.role}`);
-        console.error(`Valid roles: ${SUPPORTED_ROLES.join(", ")}`);
-        process.exit(1);
-      }
-      const role: Role = opts.role;
+      const role = opts.role === "worker" ? "worker" : "infra";
       const sshKey = findSshKey();
 
       console.log(`Installing k3s on ${resolved.hostname} (${resolved.ip}) as ${role}...`);
@@ -202,22 +106,9 @@ export function registerAppCommand(program: Command): void {
         config: {
           ...(opts.k3sServer ? { k3sServerUrl: opts.k3sServer } : {}),
           ...(opts.k3sToken ? { k3sToken: opts.k3sToken } : {}),
-          ...(opts.k3sVersion ? { k3sVersion: opts.k3sVersion } : {}),
         },
       };
 
-      // Only warn when it matters: a node JOINING an existing cluster. On a
-      // first server there is nothing to be skewed against, and a warning that
-      // fires when it is irrelevant is a warning people learn to skip.
-      if (!opts.k3sVersion && opts.k3sServer) {
-        console.warn(
-          "  WARNING: no --k3s-version given, so get.k3s.io installs LATEST.\n" +
-          "  This node is JOINING an existing cluster — check what that cluster runs\n" +
-          "  (kubectl get nodes -o wide) and pin it. Kubernetes supports only ONE minor\n" +
-          "  version of skew between control-plane members.",
-        );
-      }
-
       const installResult = await k3s.install(moduleCtx);
       for (const line of installResult.output) {
         console.log(`  ${line}`);
@@ -390,8 +281,7 @@ export function registerAppCommand(program: Command): void {
       const healthResult = await k3s.health({
         hostname: resolved.hostname,
         ip: resolved.ip,
-        // State is JSON on disk; narrow it rather than trusting the string.
-        role: isRole(resolved.role) ? resolved.role : "vanilla",
+        role: resolved.role,
         os: "fedora-43" as const,
         arch: "x86_64" as const,
         sshUser: opts.user,

bastion/src/cli/src/commands/debug.ts

@@ -8,6 +8,7 @@ import { join } from "node:path";
 import { Command } from "commander";
 import type { BastionState } from "@lab/shared";
 import { getLabdClient } from "../api/config.js";
+import { ROOT_DEVICE_PROBE, parseRootProbe } from "../utils/hardware-probe.js";
 
 /** Resolve a target (hostname, MAC, or IP) to {mac, hostname, ip} from state. */
 function resolveTarget(
@@ -44,6 +45,54 @@ function resolveTarget(
   return null;
 }
 
+/** The local admin account to SSH as (root is not usable — it has no key here). */
+function sshUser(): string {
+  const adminUser = process.env["SUDO_USER"] ?? process.env["USER"] ?? "";
+  return adminUser === "root" ? "" : adminUser;
+}
+
+/** Common ssh arguments, ending with user@host. Null when there is no usable user. */
+function sshBaseArgs(ip: string): string[] | null {
+  const user = sshUser();
+  if (user === "") return null;
+
+  const sudoUser = process.env["SUDO_USER"];
+  const realHome = sudoUser !== undefined ? join("/home", sudoUser) : homedir();
+  const sshKey = ["id_ed25519", "id_rsa", "id_ecdsa"]
+    .map((name) => join(realHome, ".ssh", name))
+    .find((k) => existsSync(k));
+
+  return [
+    "-o", "StrictHostKeyChecking=no",
+    "-o", "UserKnownHostsFile=/dev/null",
+    "-o", "ConnectTimeout=10",
+    ...(sshKey !== undefined ? ["-i", sshKey] : []),
+    `${user}@${ip}`,
+  ];
+}
+
+/**
+ * Run a shell script on the target as root and return its stdout, or null.
+ *
+ * The script goes over stdin rather than the command line so it can contain quotes
+ * without a second round of shell escaping. `sudo -n` fails fast instead of hanging on
+ * a password prompt that would then eat the script.
+ */
+function sshCapture(ip: string, script: string): string | null {
+  const base = sshBaseArgs(ip);
+  if (base === null) return null;
+  try {
+    return execFileSync("ssh", [...base, "sudo", "-n", "sh", "-s"], {
+      input: script,
+      encoding: "utf-8",
+      stdio: ["pipe", "pipe", "pipe"],
+      timeout: 30_000,
+    });
+  } catch {
+    return null;
+  }
+}
+
 export function registerDebugCommand(parent: Command): void {
   parent
     .command("debug <target>")
@@ -71,6 +120,31 @@ export function registerDebugCommand(parent: Command): void {
       }
 
       const { mac, hostname, ip } = resolved;
+
+      // --pxe-boot needs a root= for the installed system. If the machine is still
+      // reachable, observe it now rather than assuming a disk layout: a wrong root=
+      // leaves the machine unbootable. If it isn't reachable, dispatch falls back to
+      // rescue and the operator reports the real one from there.
+      if (opts.pxeBoot === true && ip !== "") {
+        const known = state.installed[mac]?.root_device ?? state.discovered[mac]?.root_device;
+        if (known === undefined || known === "") {
+          console.log(`No root device recorded for ${hostname}. Probing over SSH...`);
+          const probe = sshCapture(ip, ROOT_DEVICE_PROBE);
+          const root = probe === null ? {} : parseRootProbe(probe);
+          if (root.root_device !== undefined) {
+            console.log(`  root=${root.root_device}${root.root_args !== undefined ? ` ${root.root_args}` : ""}`);
+            try {
+              await client.discoverMachine({ mac, ...root });
+            } catch (err) {
+              console.error(`  Could not record it: ${err instanceof Error ? err.message : String(err)}`);
+            }
+          } else {
+            console.log("  Probe failed. Booting rescue instead; report the root device with:");
+            console.log("    curl http://<bastion>:8080/debug-setup.sh | bash");
+          }
+        }
+      }
+
       console.log(`Queuing debug mode for ${hostname} (${mac})...`);
 
       try {
@@ -86,32 +160,15 @@ export function registerDebugCommand(parent: Command): void {
 
       // Try SSH reboot into PXE
       if (ip !== "") {
-        const adminUser = process.env["SUDO_USER"] ?? process.env["USER"] ?? "";
-        const effectiveUser = adminUser === "root" ? "" : adminUser;
-
-        if (effectiveUser !== "") {
-          console.log(`\nAttempting SSH reboot into PXE (${effectiveUser}@${ip})...`);
-
-          const sudoUser = process.env["SUDO_USER"];
-          const realHome = sudoUser !== undefined ? join("/home", sudoUser) : homedir();
-          const keyPaths = [
-            join(realHome, ".ssh", "id_ed25519"),
-            join(realHome, ".ssh", "id_rsa"),
-            join(realHome, ".ssh", "id_ecdsa"),
-          ];
-          const sshKey = keyPaths.find(k => existsSync(k));
-
-          const sshArgs = [
-            "-o", "StrictHostKeyChecking=no",
-            "-o", "UserKnownHostsFile=/dev/null",
-            "-o", "ConnectTimeout=10",
-            ...(sshKey !== undefined ? ["-i", sshKey] : []),
-            `${effectiveUser}@${ip}`,
-            'PXE_ENTRY=$(sudo efibootmgr | grep -iE "pxe|network|ipv4" | head -1 | grep -oP "Boot\\K[0-9A-F]+"); if [ -n "$PXE_ENTRY" ]; then sudo efibootmgr --bootnext "$PXE_ENTRY" && echo "PXE set as next boot" && sudo reboot; else echo "No PXE boot entry found, rebooting anyway..." && sudo reboot; fi',
-          ];
+        const base = sshBaseArgs(ip);
+        if (base !== null) {
+          console.log(`\nAttempting SSH reboot into PXE (${sshUser()}@${ip})...`);
 
           try {
-            execFileSync("ssh", sshArgs, { stdio: "inherit" });
+            execFileSync("ssh", [
+              ...base,
+              'PXE_ENTRY=$(sudo efibootmgr | grep -iE "pxe|network|ipv4" | head -1 | grep -oP "Boot\\K[0-9A-F]+"); if [ -n "$PXE_ENTRY" ]; then sudo efibootmgr --bootnext "$PXE_ENTRY" && echo "PXE set as next boot" && sudo reboot; else echo "No PXE boot entry found, rebooting anyway..." && sudo reboot; fi',
+            ], { stdio: "inherit" });
           } catch {
             // SSH connection closing during reboot is expected
           }

bastion/src/cli/src/commands/install.ts

@@ -1,61 +1,10 @@
 // CLI command: provision install
 // Queue a discovered machine for OS installation via labd.
 
-import { readFileSync } from "node:fs";
-import { Command, Option, InvalidArgumentError } from "commander";
-import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isRole, requiresSchemaV2, validateDataDisks } from "@lab/shared";
-import type { DataDisks, VyosBundle, VyosInstallSpec, VyosVlanSpec } from "@lab/shared";
+import { Command, Option } from "commander";
+import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY } from "@lab/shared";
 import { getLabdClient } from "../api/config.js";
 
-/**
- * Load one router's config out of a Pulumi-rendered bundle.
- *
- * The bundle is produced by `kubernetes-deployment` (npm run vyos:bundle) and
- * holds every router it manages, keyed by name. Selecting by hostname here is
- * what keeps bring-up and `pulumi up` describing the same box: labctl replays
- * the declared config rather than deriving its own.
- */
-export function loadVyosBundle(path: string, hostname: string): VyosBundle {
-  let parsed: { version?: number; routers?: Record<string, VyosBundle> };
-  try {
-    parsed = JSON.parse(readFileSync(path, "utf8"));
-  } catch (e) {
-    throw new InvalidArgumentError(`Cannot read VyOS bundle ${path}: ${(e as Error).message}`);
-  }
-  if (parsed.version !== 1) {
-    throw new InvalidArgumentError(
-      `VyOS bundle ${path} has version ${parsed.version ?? "<none>"}; this labctl understands 1`,
-    );
-  }
-  const router = parsed.routers?.[hostname];
-  if (router === undefined) {
-    const known = Object.keys(parsed.routers ?? {}).join(", ") || "<none>";
-    throw new InvalidArgumentError(
-      `VyOS bundle ${path} has no entry for "${hostname}" (has: ${known})`,
-    );
-  }
-  return router;
-}
-
-/** Parse a repeated --vlan flag: "<id>:<cidr>[:<description>]". */
-export function parseVlan(value: string, previous: VyosVlanSpec[] = []): VyosVlanSpec[] {
-  const parts = value.split(":");
-  const id = Number(parts[0]);
-  const address = parts[1] ?? "";
-  // InvalidArgumentError makes commander print a clean message instead of
-  // dumping a stack trace at the operator.
-  if (!Number.isInteger(id) || id < 1 || id > 4094) {
-    throw new InvalidArgumentError(`Invalid VLAN id in "${value}" (expected 1-4094)`);
-  }
-  if (!address.includes("/")) {
-    throw new InvalidArgumentError(
-      `Invalid VLAN address in "${value}" (expected CIDR, e.g. 10.0.10.1/24)`,
-    );
-  }
-  const description = parts.slice(2).join(":");
-  return [...previous, { id, address, ...(description ? { description } : {}) }];
-}
-
 function roleTable(): string {
   const lines: string[] = ["", "Available roles:"];
   for (const r of ROLE_REGISTRY) {
@@ -66,38 +15,6 @@ function roleTable(): string {
   return lines.join("\n");
 }
 
-/** Parse a repeated --vlan-vip flag: "<id>:<cidr>" — VRRP VIP for a --vlan entry. */
-export function parseVlanVip(
-  value: string,
-  previous: Record<number, string> = {},
-): Record<number, string> {
-  const index = value.indexOf(":");
-  const id = Number(index === -1 ? Number.NaN : value.slice(0, index));
-  const cidr = index === -1 ? "" : value.slice(index + 1).trim();
-  if (!Number.isInteger(id) || id < 1 || id > 4094 || !cidr.includes("/")) {
-    throw new InvalidArgumentError(
-      `Invalid VLAN VIP "${value}" (expected <id>:<cidr>, e.g. 3:192.168.3.254/24)`,
-    );
-  }
-  return { ...previous, [id]: cidr };
-}
-
-/** Parse a repeated --vyos-hwid flag: "<iface>=<mac>". */
-export function parseHwId(
-  value: string,
-  previous: Record<string, string> = {},
-): Record<string, string> {
-  const index = value.indexOf("=");
-  const iface = index === -1 ? "" : value.slice(0, index).trim();
-  const mac = index === -1 ? "" : value.slice(index + 1).trim().toLowerCase();
-  if (iface === "" || !/^([0-9a-f]{2}:){5}[0-9a-f]{2}$/.test(mac)) {
-    throw new InvalidArgumentError(
-      `Invalid hw-id "${value}" (expected <iface>=<mac>, e.g. eth2=64:62:66:25:96:47)`,
-    );
-  }
-  return { ...previous, [iface]: mac };
-}
-
 export function registerInstallCommand(parent: Command): void {
   parent
     .command("install <mac> <hostname>")
@@ -106,65 +23,11 @@ export function registerInstallCommand(parent: Command): void {
     .addHelpText("after", roleTable())
     .addOption(new Option("--role <role>", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker"))
     .addOption(new Option("--os <os>", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43"))
-    .option("--disk <device>", "Target OS disk: kernel name or stable /dev/disk/by-id/ path (auto-detect if omitted)")
-    .addOption(new Option("--schema-version <n>", "Disk layout for a FRESH install (default 1). Ignored on reprovision: the layout already on the disk wins.").choices(["1", "2"]))
-    .option(
-      "--longhorn-disk <by-id>",
-      "Dedicated whole disk for /var/lib/longhorn, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)",
-    )
-    .option(
-      "--rancher-disk <by-id>",
-      "Dedicated whole disk for /var/lib/rancher, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)",
-    )
-    .option("--vyos-mgmt <iface>", "VyOS: untagged interface the machine PXE boots from (default eth0)")
-    .option("--vyos-mgmt-address <addr>", "VyOS: CIDR for the management interface, or 'dhcp' (default dhcp)")
-    .option("--vyos-bond <ifaces>", "VyOS: comma-separated LACP bond members (must exclude the PXE NIC)")
-    .option("--vyos-bond-address <cidr>", "VyOS: address on the untagged bond (trunk native VLAN)")
-    .option("--vyos-bond-vrrp <cidr>", "VyOS: VRRP VIP floated on the untagged bond")
-    .option("--vlan-vip <id:cidr>", "VyOS: VRRP VIP for a --vlan entry (repeatable)", parseVlanVip)
-    .option("--vyos-vrrp-priority <n>", "VyOS: VRRP priority for all groups on this box (higher = master)")
-    .option("--vyos-mgmt-vlan <id:cidr[:desc]>", "VyOS: tagged management VLAN on the PXE port")
-    .option("--vlan <id:cidr[:desc]>", "VyOS: tagged VLAN sub-interface on the bond (repeatable)", parseVlan)
-    .option("--vyos-password <password>", "VyOS: password for the 'vyos' user")
-    .option("--vyos-hwid <iface=mac>", "VyOS: pin an interface name to a MAC via hw-id (repeatable)", parseHwId)
-    .option("--vyos-fresh-config", "VyOS: on reinstall, overwrite the preserved config with the generated one")
-    .option(
-      "--vyos-bundle <path>",
-      "VyOS: apply a Pulumi-rendered bundle verbatim (kubernetes-deployment/infra/vyos/vyos-bundle.json). " +
-        "Replaces the derived --vyos-bond/--vlan/... config; secret values are left unset for `pulumi up`.",
-    )
-    .option(
-      "--vyos-api-key <key>",
-      "VyOS: enable the HTTP API with this key so Pulumi can manage the router from first boot",
-    )
-    .option(
-      "--vyos-api-listen <addr>",
-      "VyOS: address the HTTP API binds to (default: the static management address). " +
-        "Required when management is DHCP; the API is never bound to all interfaces.",
-    )
+    .option("--disk <device>", "Target disk device (auto-detect if omitted)")
     .action(async (mac: string, hostname: string, opts: {
       role: string;
       os: string;
-      /** commander gives us the string form of the --schema-version choice. */
-      schemaVersion?: string;
       disk?: string;
-      longhornDisk?: string;
-      rancherDisk?: string;
-      vyosMgmt?: string;
-      vyosMgmtAddress?: string;
-      vyosBond?: string;
-      vyosBondAddress?: string;
-      vyosBondVrrp?: string;
-      vlan?: VyosVlanSpec[];
-      vlanVip?: Record<number, string>;
-      vyosVrrpPriority?: string;
-      vyosMgmtVlan?: string;
-      vyosPassword?: string;
-      vyosHwid?: Record<string, string>;
-      vyosFreshConfig?: boolean;
-      vyosBundle?: string;
-      vyosApiKey?: string;
-      vyosApiListen?: string;
     }) => {
       if (!isValidOsId(opts.os)) {
         console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`);
@@ -176,132 +39,18 @@ export function registerInstallCommand(parent: Command): void {
         process.exit(1);
       }
 
-      // Dedicated data disks -- validated here for fast feedback, and again on
-      // the bastion, which is the enforcement point.
-      const dataDisks: DataDisks = {
-        ...(opts.longhornDisk !== undefined && opts.longhornDisk !== ""
-          ? { longhorn: opts.longhornDisk } : {}),
-        ...(opts.rancherDisk !== undefined && opts.rancherDisk !== ""
-          ? { rancher: opts.rancherDisk } : {}),
-      };
-      const hasDataDisks = Object.keys(dataDisks).length > 0;
-      if (hasDataDisks) {
-        if (opts.os !== "fedora-43") {
-          console.error(`--longhorn-disk/--rancher-disk only work with --os fedora-43 (got --os ${opts.os})`);
-          process.exit(1);
-        }
-        if (opts.schemaVersion === "1"
-            || (opts.schemaVersion === undefined && isRole(opts.role) && !requiresSchemaV2(opts.role))) {
-          console.error("--longhorn-disk/--rancher-disk need the v2 layout -- add --schema-version 2");
-          process.exit(1);
-        }
-        const problem = isRole(opts.role) ? validateDataDisks(opts.role, dataDisks) : null;
-        if (problem !== null) {
-          console.error(`Invalid data disks: ${problem}`);
-          process.exit(1);
-        }
-      }
-
-      const bondMembers = opts.vyosBond !== undefined && opts.vyosBond !== ""
-        ? opts.vyosBond.split(",").map((s) => s.trim()).filter((s) => s.length > 0)
-        : [];
-
-      // Attach --vlan-vip entries to their --vlan definitions. A VIP for a VLAN
-      // that was never defined is a typo that would otherwise vanish silently.
-      const vips = opts.vlanVip ?? {};
-      const vlans = (opts.vlan ?? []).map((v) =>
-        vips[v.id] !== undefined ? { ...v, vrrp: vips[v.id] as string } : v,
-      );
-      for (const id of Object.keys(vips)) {
-        if (!vlans.some((v) => String(v.id) === id)) {
-          console.error(`--vlan-vip ${id}:... has no matching --vlan ${id}:... entry`);
-          process.exit(1);
-        }
-      }
-
-      const vrrpPriority = opts.vyosVrrpPriority !== undefined && opts.vyosVrrpPriority !== ""
-        ? Number(opts.vyosVrrpPriority)
-        : undefined;
-      if (vrrpPriority !== undefined
-          && (!Number.isInteger(vrrpPriority) || vrrpPriority < 1 || vrrpPriority > 255)) {
-        console.error(`--vyos-vrrp-priority must be an integer 1-255 (got ${opts.vyosVrrpPriority})`);
-        process.exit(1);
-      }
-      const vyos: VyosInstallSpec = {
-        ...(opts.vyosMgmt !== undefined && opts.vyosMgmt !== ""
-          ? { mgmtInterface: opts.vyosMgmt } : {}),
-        ...(opts.vyosMgmtAddress !== undefined && opts.vyosMgmtAddress !== ""
-          ? { mgmtAddress: opts.vyosMgmtAddress } : {}),
-        ...(bondMembers.length > 0 ? { bondMembers } : {}),
-        ...(opts.vyosBondAddress !== undefined && opts.vyosBondAddress !== ""
-          ? { bondAddress: opts.vyosBondAddress } : {}),
-        ...(opts.vyosBondVrrp !== undefined && opts.vyosBondVrrp !== ""
-          ? { bondVrrp: opts.vyosBondVrrp } : {}),
-        ...(vrrpPriority !== undefined ? { vrrpPriority } : {}),
-        ...(vlans.length > 0 ? { vlans } : {}),
-        ...(opts.vyosPassword !== undefined && opts.vyosPassword !== ""
-          ? { password: opts.vyosPassword } : {}),
-        ...(opts.vyosHwid !== undefined && Object.keys(opts.vyosHwid).length > 0
-          ? { hwIds: opts.vyosHwid } : {}),
-        ...(opts.vyosMgmtVlan !== undefined && opts.vyosMgmtVlan !== ""
-          ? { mgmtVlan: parseVlan(opts.vyosMgmtVlan)[0] as VyosVlanSpec } : {}),
-        ...(opts.vyosFreshConfig === true ? { freshConfig: true } : {}),
-        ...(opts.vyosBundle !== undefined && opts.vyosBundle !== ""
-          ? { bundle: loadVyosBundle(opts.vyosBundle, hostname) } : {}),
-        ...(opts.vyosApiKey !== undefined && opts.vyosApiKey !== ""
-          ? { apiKey: opts.vyosApiKey } : {}),
-        ...(opts.vyosApiListen !== undefined && opts.vyosApiListen !== ""
-          ? { apiListenAddress: opts.vyosApiListen } : {}),
-      };
-      const hasVyosOptions = Object.keys(vyos).length > 0;
-
-      // A bundle already describes the whole router. Accepting derived topology
-      // flags alongside it would silently discard them (the bundle wins in
-      // buildVyosConfigSpec), so say so rather than appear to honour both.
-      if (vyos.bundle !== undefined) {
-        const derived = ["mgmtInterface", "mgmtAddress", "bondMembers", "bondAddress",
-                         "bondVrrp", "vrrpPriority", "vlans", "mgmtVlan"] as const;
-        const conflicting = derived.filter((k) => vyos[k] !== undefined);
-        if (conflicting.length > 0) {
-          console.error(
-            `--vyos-bundle describes the whole router; these would be ignored: ${conflicting.join(", ")}`,
-          );
-          console.error("Remove them, or change the bundle in kubernetes-deployment and re-render.");
-          process.exit(1);
-        }
-      }
-
-      if (hasVyosOptions && !opts.os.startsWith("vyos")) {
-        console.error(`VyOS options require --os vyos-rolling (got --os ${opts.os})`);
-        process.exit(1);
-      }
-
-      // Firmware PXE cannot run over LACP, so the NIC that boots the installer
-      // must stay out of the bond — otherwise the next reinstall has no path in.
-      const mgmt = vyos.mgmtInterface ?? "eth0";
-      if (bondMembers.includes(mgmt)) {
-        console.error(`--vyos-bond must not include the PXE/management interface "${mgmt}"`);
-        console.error("PXE cannot boot over an LACP bond; keep that NIC unbonded.");
-        process.exit(1);
-      }
-
       try {
         const result = await getLabdClient().installMachine({
-          ...(opts.schemaVersion ? { schemaVersion: Number(opts.schemaVersion) as 1 | 2 } : {}),
           mac,
           hostname,
           role: opts.role,
           os: opts.os,
           ...(opts.disk ? { disk: opts.disk } : {}),
-          ...(hasDataDisks ? { dataDisks } : {}),
-          ...(hasVyosOptions ? { vyos } : {}),
         });
 
         console.log(JSON.stringify(result, null, 2));
         console.log("");
-        const osLabel = opts.os.startsWith("ubuntu")
-          ? "Ubuntu"
-          : opts.os.startsWith("vyos") ? "VyOS" : "Fedora";
+        const osLabel = opts.os.startsWith("ubuntu") ? "Ubuntu" : "Fedora";
         console.log(`Power on the machine to start ${osLabel} installation.`);
 
         const roleInfo = ROLE_REGISTRY.find(r => r.name === opts.role);

bastion/src/cli/src/commands/recheck.ts

@@ -4,6 +4,7 @@
 import type { Command } from "commander";
 import { sshExec } from "@lab/modules";
 import { getLabdClient } from "../api/config.js";
+import { ROOT_DEVICE_PROBE } from "../utils/hardware-probe.js";
 
 const BOLD = "\x1b[1m";
 const GREEN = "\x1b[0;32m";
@@ -24,7 +25,9 @@ const HW_COLLECT_SCRIPT = [
   'N=$(grep -c "^processor" /proc/cpuinfo 2>/dev/null || echo 0)',
   'R=$(awk "/MemTotal/ {printf \\"%d\\", \\$2/1024/1024}" /proc/meminfo 2>/dev/null || echo 0)',
   'A=$(uname -m)',
-  'printf \'{"product":"%s","board":"%s","serial":"%s","manufacturer":"%s","cpu_model":"%s","cpu_cores":%s,"memory_gb":%s,"arch":"%s"}\\n\' "$P" "$B" "$S" "$M" "$C" "$N" "$R" "$A"',
+  // Root filesystem, so --pxe-boot has a root= to use instead of assuming our layout.
+  ROOT_DEVICE_PROBE,
+  'printf \'{"product":"%s","board":"%s","serial":"%s","manufacturer":"%s","cpu_model":"%s","cpu_cores":%s,"memory_gb":%s,"arch":"%s","root_device":"%s","root_args":"%s"}\\n\' "$P" "$B" "$S" "$M" "$C" "$N" "$R" "$A" "$RD" "$RA"',
 ].join("; ");
 
 export function registerRecheckCommand(parent: Command): void {
@@ -44,14 +47,11 @@ export function registerRecheckCommand(parent: Command): void {
       }
 
       // Build list of machines to check
-      const targets: Array<{ mac: string; hostname: string; ip: string; sshUser: string }> = [];
-      const userIsDefault = opts.user === "root";
+      const targets: Array<{ mac: string; hostname: string; ip: string }> = [];
       for (const [mac, info] of Object.entries(state.installed)) {
         if (!info.ip) continue;
         if (opts.target && info.hostname !== opts.target && mac !== opts.target) continue;
-        // VyOS boxes only have the "vyos" login; honor an explicit --user.
-        const sshUser = userIsDefault && (info.os ?? "").startsWith("vyos") ? "vyos" : opts.user;
-        targets.push({ mac, hostname: info.hostname, ip: info.ip, sshUser });
+        targets.push({ mac, hostname: info.hostname, ip: info.ip });
       }
 
       if (targets.length === 0) {
@@ -64,12 +64,12 @@ export function registerRecheckCommand(parent: Command): void {
       let updated = 0;
       let failed = 0;
 
-      for (const { mac, hostname, ip, sshUser } of targets) {
+      for (const { mac, hostname, ip } of targets) {
         process.stdout.write(`  ${hostname.padEnd(24)} ${DIM}(${ip})${RESET}  `);
 
         try {
           const t0 = Date.now();
-          const result = await sshExec(ip, sshUser, HW_COLLECT_SCRIPT, SSH_OPTS);
+          const result = await sshExec(ip, opts.user, HW_COLLECT_SCRIPT, SSH_OPTS);
           const elapsed = Date.now() - t0;
           if (result.exitCode !== 0) {
             console.log(`${RED}SSH failed (exit ${result.exitCode}, ${elapsed}ms)${RESET}`);
@@ -84,7 +84,10 @@ export function registerRecheckCommand(parent: Command): void {
           const cpu = hwData.cpu_model || "?";
           const cores = hwData.cpu_cores || "?";
           const mem = hwData.memory_gb || "?";
-          console.log(`${GREEN}OK${RESET} ${DIM}${cpu}, ${cores} cores, ${mem}GB${RESET}`);
+          const root = typeof hwData.root_device === "string" && hwData.root_device !== ""
+            ? `, root=${hwData.root_device}`
+            : "";
+          console.log(`${GREEN}OK${RESET} ${DIM}${cpu}, ${cores} cores, ${mem}GB${root}${RESET}`);
           updated++;
         } catch (err) {
           console.log(`${RED}FAIL${RESET} ${DIM}${err instanceof Error ? err.message : String(err)}${RESET}`);

bastion/src/cli/src/commands/reprovision.ts

@@ -24,12 +24,12 @@ function roleTable(): string {
 function resolveTarget(
   target: string,
   state: BastionState,
-): { mac: string; hostname: string; ip: string; os?: string } | null {
+): { mac: string; hostname: string; ip: string } | null {
   const normalized = target.toLowerCase().replace(/-/g, ":");
 
   if (state.installed[normalized]) {
     const info = state.installed[normalized];
-    return { mac: normalized, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
+    return { mac: normalized, hostname: info.hostname, ip: info.ip };
   }
 
   if (state.discovered[normalized]) {
@@ -38,13 +38,13 @@ function resolveTarget(
 
   for (const [mac, info] of Object.entries(state.installed)) {
     if (info.hostname === target || info.hostname.startsWith(target + ".")) {
-      return { mac, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
+      return { mac, hostname: info.hostname, ip: info.ip };
     }
   }
 
   for (const [mac, info] of Object.entries(state.installed)) {
     if (info.ip === target) {
-      return { mac, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
+      return { mac, hostname: info.hostname, ip: info.ip };
     }
   }
 
@@ -60,12 +60,10 @@ export function registerReprovisionCommand(parent: Command): void {
     .addOption(new Option("--role <role>", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker"))
     .addOption(new Option("--os <os>", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43"))
     .option("--disk <device>", "Target disk device (auto-detect if omitted)")
-    .option("--user <user>", "SSH user for the reboot (default: vyos for VyOS machines, else current user)")
     .action(async (target: string, hostnameOverride: string | undefined, opts: {
       role: string;
       os: string;
       disk?: string;
-      user?: string;
     }) => {
       if (!isValidOsId(opts.os)) {
         console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`);
@@ -125,11 +123,7 @@ export function registerReprovisionCommand(parent: Command): void {
         return;
       }
 
-      // SSH user: explicit flag > the machine's current OS (VyOS boxes only
-      // have the "vyos" login) > the invoking user.
-      const currentOsIsVyos = (resolved.os ?? "").startsWith("vyos");
-      const adminUser = opts.user
-        ?? (currentOsIsVyos ? "vyos" : (process.env["SUDO_USER"] ?? process.env["USER"] ?? ""));
+      const adminUser = process.env["SUDO_USER"] ?? process.env["USER"] ?? "";
       const effectiveUser = adminUser === "root" ? "" : adminUser;
 
       if (effectiveUser === "") {

bastion/src/cli/src/utils/hardware-probe.ts

@@ -0,0 +1,59 @@
+// Shell snippets for observing a machine's hardware over SSH.
+//
+// Pure shell + awk, no Python: these run on whatever the target happens to be,
+// including a minimal rescue environment.
+
+/**
+ * Report the root filesystem and any dracut arguments needed to assemble it.
+ *
+ * Emits two lines:
+ *   ROOT_DEVICE=<device>
+ *   ROOT_ARGS=<args>
+ *
+ * Used by `--pxe-boot`, which boots the installed system with a kernel and initrd from
+ * the network. Getting root= wrong there leaves the machine unbootable, so this observes
+ * the machine rather than assuming our Fedora LVM layout.
+ *
+ * Device form is chosen for stability across reboots: LVM logical volumes keep their
+ * /dev/mapper path, anything else is reported by UUID, which survives device renumbering.
+ */
+export const ROOT_DEVICE_PROBE = [
+  'RD=$(findmnt -no SOURCE / 2>/dev/null | head -1)',
+  'RA=""',
+  'RT=$(lsblk -no TYPE "$RD" 2>/dev/null | head -1)',
+  'if [ "$RT" = "lvm" ]; then',
+  '  VGLV=$(lvs --noheadings -o vg_name,lv_name "$RD" 2>/dev/null | awk \'{print $1"/"$2}\')',
+  '  [ -n "$VGLV" ] && RA="rd.lvm.lv=$VGLV"',
+  // Swap must be assembled too or resume= stalls the boot waiting for it.
+  '  SW=$(awk \'NR>1 {print $1; exit}\' /proc/swaps 2>/dev/null)',
+  '  if [ -n "$SW" ] && [ "$(lsblk -no TYPE "$SW" 2>/dev/null | head -1)" = "lvm" ]; then',
+  '    SWVGLV=$(lvs --noheadings -o vg_name,lv_name "$SW" 2>/dev/null | awk \'{print $1"/"$2}\')',
+  '    [ -n "$SWVGLV" ] && [ "$SWVGLV" != "$VGLV" ] && RA="$RA rd.lvm.lv=$SWVGLV"',
+  '  fi',
+  'elif [ -n "$RD" ]; then',
+  '  U=$(findmnt -no UUID / 2>/dev/null | head -1)',
+  '  [ -n "$U" ] && RD="UUID=$U"',
+  'fi',
+  'printf \'ROOT_DEVICE=%s\\nROOT_ARGS=%s\\n\' "$RD" "$RA"',
+].join("; ");
+
+export interface RootInfo {
+  root_device?: string;
+  root_args?: string;
+}
+
+/** Parse the ROOT_DEVICE/ROOT_ARGS lines emitted by ROOT_DEVICE_PROBE. */
+export function parseRootProbe(stdout: string): RootInfo {
+  const out: RootInfo = {};
+  for (const line of stdout.split("\n")) {
+    const trimmed = line.trim();
+    if (trimmed.startsWith("ROOT_DEVICE=")) {
+      const v = trimmed.slice("ROOT_DEVICE=".length).trim();
+      if (v !== "") out.root_device = v;
+    } else if (trimmed.startsWith("ROOT_ARGS=")) {
+      const v = trimmed.slice("ROOT_ARGS=".length).trim();
+      if (v !== "") out.root_args = v;
+    }
+  }
+  return out;
+}

bastion/src/cli/tests/install-vyos.test.ts

@@ -1,35 +0,0 @@
-// Tests for VyOS install option parsing.
-
-import { describe, it, expect } from "vitest";
-import { parseVlan } from "../src/commands/install.js";
-
-describe("parseVlan", () => {
-  it("parses id and CIDR", () => {
-    expect(parseVlan("10:10.0.10.1/24")).toEqual([{ id: 10, address: "10.0.10.1/24" }]);
-  });
-
-  it("accumulates across repeated flags", () => {
-    const first = parseVlan("10:10.0.10.1/24");
-    const both = parseVlan("20:10.0.20.1/24", first);
-    expect(both).toHaveLength(2);
-    expect(both[1]).toEqual({ id: 20, address: "10.0.20.1/24" });
-  });
-
-  it("keeps a description, including one containing colons", () => {
-    expect(parseVlan("30:10.0.30.1/24:mgmt:secondary")).toEqual([
-      { id: 30, address: "10.0.30.1/24", description: "mgmt:secondary" },
-    ]);
-  });
-
-  it("rejects an address that is not CIDR", () => {
-    // A bare address would produce a VyOS config that fails to commit on first
-    // boot, long after the operator has stopped watching.
-    expect(() => parseVlan("10:10.0.10.1")).toThrow(/CIDR/);
-  });
-
-  it("rejects out-of-range and non-numeric VLAN ids", () => {
-    expect(() => parseVlan("0:10.0.10.1/24")).toThrow(/1-4094/);
-    expect(() => parseVlan("4095:10.0.10.1/24")).toThrow(/1-4094/);
-    expect(() => parseVlan("abc:10.0.10.1/24")).toThrow(/1-4094/);
-  });
-});

bastion/src/cli/tests/smoke-bastion.test.ts

@@ -143,10 +143,7 @@ describe("bastion smoke tests", () => {
       await sleep(200);
     }
 
-    // On failure, show what the process actually said -- an empty stdout with
-    // a hidden stderr is undiagnosable from a CI log.
-    expect(stdout, `bastion never printed its banner; stderr was:\n${stderr}`)
-      .toContain("Waiting for PXE boot requests");
+    expect(stdout).toContain("Waiting for PXE boot requests");
     expect(stdout).toContain("HTTP server listening");
 
     // Verify the process is still alive after startup

bastion/src/labd/src/routes/bastions.ts

@@ -10,7 +10,6 @@ import type { FastifyInstance } from "fastify";
 import type { DbClient } from "../server.js";
 import { bastionRegistry } from "../services/bastion-registry.js";
 import { generateRequestId } from "@lab/shared";
-import type { VyosInstallSpec } from "@lab/shared";
 
 const COMMAND_TIMEOUT_MS = 15_000;
 
@@ -164,9 +163,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
 
   // Queue install — route to correct bastion by MAC
   app.post<{
-    Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string; schemaVersion?: 1 | 2; dataDisks?: import("@lab/shared").DataDisks; vyos?: VyosInstallSpec };
+    Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string };
   }>("/api/machines/install", async (request, reply) => {
-    const { mac, hostname, disk, role, os, schemaVersion, dataDisks, vyos } = request.body ?? {};
+    const { mac, hostname, disk, role, os } = request.body ?? {};
     if (!mac || !hostname) {
       return reply.code(400).send({ error: "mac and hostname are required" });
     }
@@ -184,9 +183,6 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
           const result = await sendCommand(all[0]!.bastionId, {
             type: "command-install",
             mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
-            ...(schemaVersion ? { schemaVersion } : {}),
-            ...(dataDisks ? { dataDisks } : {}),
-            ...(vyos ? { vyos } : {}),
           });
           return reply.code(result.status === "ok" ? 200 : 500).send(result);
         } catch (err) {
@@ -200,9 +196,6 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
       const result = await sendCommand(bastion.bastionId, {
         type: "command-install",
         mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
-        ...(schemaVersion ? { schemaVersion } : {}),
-        ...(dataDisks ? { dataDisks } : {}),
-        ...(vyos ? { vyos } : {}),
       });
       return reply.code(result.status === "ok" ? 200 : 500).send(result);
     } catch (err) {
@@ -306,6 +299,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
       memory_gb?: number; arch?: string;
       disks?: Array<{ name: string; size_gb: number; model: string }>;
       nics?: Array<{ name: string; mac: string; state: string }>;
+      root_device?: string; root_args?: string;
     };
   }>("/api/machines/discover", async (request, reply) => {
     const data = request.body ?? {};

bastion/src/modules/modules/k3s/bin/render-config.ts

@@ -1,42 +0,0 @@
-#!/usr/bin/env node
-// Render /etc/rancher/k3s/config.yaml using the PRODUCTION generator.
-//
-// Exists so labsim (and anything else) can produce the exact config.yaml a node
-// would get from `labctl install`, without an SSH/OperationContext. Driving the
-// rehearsal through this means the sim tests the same code path production runs
-// -- if generateServerConfig ever changes shape, the sim moves with it.
-//
-// All input via env, so a sim's cloud-init or a shell can call it plainly:
-//
-//   ROLE=infra HOSTNAME=k8s1 IP=172.31.2.11 render-config.ts        # cluster-init server
-//   ROLE=infra HOSTNAME=k8s2 IP=172.31.2.12 \
-//     K3S_SERVER_URL=https://172.31.2.11:6443 K3S_TOKEN=... render-config.ts   # joining server
-//   ROLE=worker HOSTNAME=k8s4 IP=172.31.2.14 K3S_SERVER_URL=... K3S_TOKEN=... render-config.ts  # agent
-//
-// Dual-stack is opt-in and matches K3sConfig exactly: set IPV6, CLUSTER_CIDR,
-// SERVICE_CIDR (comma-separated families) and the generator emits the dual
-// node-ip + CIDRs. Omit them and the output is byte-identical to a v4-only node.
-import { generateServerConfig, generateAgentConfig } from "../src/operations/k3s-config.js";
-import type { K3sConfig } from "../src/types.js";
-import type { Role } from "@lab/shared";
-
-const env = process.env;
-const role = (env.ROLE ?? "worker") as Role;
-const isServer = role === "infra" || role === "labcontroller";
-
-const splitCsv = (v: string | undefined): string[] | undefined =>
-  v ? v.split(",").map((s) => s.trim()).filter(Boolean) : undefined;
-
-const cfg: K3sConfig = {
-  hostname: env.HOSTNAME ?? "node",
-  ip: env.IP ?? (() => { throw new Error("IP is required"); })(),
-  role,
-  k3sServerUrl: env.K3S_SERVER_URL,
-  k3sToken: env.K3S_TOKEN,
-  tlsSans: splitCsv(env.TLS_SANS),
-  ipv6: env.IPV6,
-  clusterCidr: splitCsv(env.CLUSTER_CIDR),
-  serviceCidr: splitCsv(env.SERVICE_CIDR),
-};
-
-process.stdout.write(isServer ? generateServerConfig(cfg) : generateAgentConfig(cfg));

bastion/src/modules/modules/k3s/src/groups/admit.ts

@@ -1,31 +0,0 @@
-// Return an evacuated node to service. The inverse of the evacuate group.
-//
-// Needed for two different journeys that end in the same place:
-//   - a node that was wiped, reinstalled and has just rejoined the cluster;
-//   - a node whose evacuation was aborted and must simply be put back.
-//
-// Order matters here too. configureLonghornDisk runs AFTER eviction is cleared,
-// because a node that is still marked for eviction will have any disk we give
-// it immediately drained again.
-
-import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
-import { runSequential } from "../utils.js";
-import { uncordonNode } from "../operations/node-evacuate.js";
-import { clearLonghornEviction, assertNodeSchedulable } from "../operations/node-admit.js";
-import { configureLonghornDisk } from "../operations/longhorn-disk.js";
-
-export const admitGroup: OperationGroup = {
-  name: "admit",
-  description: "Uncordon a node, clear its eviction request and restore its Longhorn disk",
-  operations: [
-    { name: "Uncordon node", fn: uncordonNode },
-    { name: "Clear Longhorn eviction", fn: clearLonghornEviction },
-    { name: "Restore Longhorn default disk", fn: configureLonghornDisk },
-    { name: "Verify node is back in service", fn: assertNodeSchedulable },
-  ],
-};
-
-export async function runAdmit(ctx: OperationContext): Promise<OperationResult[]> {
-  ctx.log("Returning node to service...");
-  return runSequential(ctx, admitGroup.operations);
-}

bastion/src/modules/modules/k3s/src/groups/evacuate.ts

@@ -1,54 +0,0 @@
-// Evacuate a node so it can be wiped and reinstalled with a different layout.
-//
-// The ORDER is the operation:
-//
-//   1. Cordon      stop new work landing on a node that is about to leave.
-//   2. Check       can eviction even finish? Seconds to answer, an hour to
-//                  discover the hard way.
-//   3. Evict       move the replicas and WAIT for them to rebuild elsewhere.
-//   4. Drain       now move the workloads.
-//   5. Verify      re-read the cluster; do not trust that 1-4 worked.
-//
-// EVICT BEFORE DRAIN, which is the opposite of the obvious order and the one
-// thing here most likely to be "tidied" back into a bug. `drain --force` does
-// not override PodDisruptionBudgets, and longhorn-manager holds a PDB on the
-// instance-manager pod for as long as volumes are attached or degraded. Drain
-// first and you stop the instance-manager that serves the replicas you are
-// about to evict: the drain blocks for its full 15-minute timeout, and if it
-// did succeed it would have removed the only source a single-replica volume
-// could rebuild from.
-//
-// Removing the etcd member is NOT part of this group -- it costs you the API
-// server the rest of the procedure needs, so it belongs immediately before the
-// wipe, not here.
-//
-// This group deliberately does not reinstall anything. It leaves the node
-// cordoned, drained and empty, and a human (or `labctl provision reprovision`)
-// takes it from there.
-
-import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
-import { runSequential } from "../utils.js";
-import {
-  cordonNode,
-  drainNode,
-  evictLonghornReplicas,
-  assertEvictionCompletable,
-  assertNodeEvacuated,
-} from "../operations/node-evacuate.js";
-
-export const evacuateGroup: OperationGroup = {
-  name: "evacuate",
-  description: "Cordon, evict storage from and drain a node so it can be safely wiped",
-  operations: [
-    { name: "Cordon node", fn: cordonNode },
-    { name: "Check eviction can complete", fn: assertEvictionCompletable },
-    { name: "Evict Longhorn replicas (waits for rebuild)", fn: evictLonghornReplicas },
-    { name: "Drain workloads", fn: drainNode },
-    { name: "Verify node is empty", fn: assertNodeEvacuated },
-  ],
-};
-
-export async function runEvacuate(ctx: OperationContext): Promise<OperationResult[]> {
-  ctx.log("Evacuating node before wipe...");
-  return runSequential(ctx, evacuateGroup.operations);
-}

bastion/src/modules/modules/k3s/src/groups/host-prep.ts

@@ -1,23 +1,21 @@
-// Host preparation: kernel modules, sysctl, swap, storage, firewall, SELinux.
+// Host preparation: kernel modules, sysctl, swap, firewall, SELinux.
 
 import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
 import { runSequential } from "../utils.js";
 import { loadKernelModules } from "../operations/kernel-modules.js";
 import { applyCisHardening } from "../operations/sysctl.js";
-import { enableSwap } from "../operations/swap.js";
-import { growRancherLv } from "../operations/rancher-storage.js";
+import { disableSwap } from "../operations/swap.js";
 import { disableFirewall } from "../operations/firewall.js";
 import { setSelinuxPermissive } from "../operations/selinux.js";
 import { enableIscsi } from "../operations/iscsi.js";
 
 export const hostPrepGroup: OperationGroup = {
   name: "host-prep",
-  description: "Prepare host for k3s: kernel modules, sysctl, swap, imageFs sizing, firewall, SELinux, iSCSI",
+  description: "Prepare host for k3s: kernel modules, sysctl, swap, firewall, SELinux, iSCSI",
   operations: [
     { name: "Load kernel modules", fn: loadKernelModules },
     { name: "Apply CIS sysctl", fn: applyCisHardening },
-    { name: "Enable swap", fn: enableSwap },
-    { name: "Grow rancher LV", fn: growRancherLv },
+    { name: "Disable swap", fn: disableSwap },
     { name: "Disable firewall", fn: disableFirewall },
     { name: "Set SELinux permissive", fn: setSelinuxPermissive },
     { name: "Enable iSCSI", fn: enableIscsi },

bastion/src/modules/modules/k3s/src/groups/k3s-agent.ts

@@ -4,7 +4,6 @@ import type { OperationContext, OperationResult, OperationGroup } from "../types
 import { runSequential } from "../utils.js";
 import { writeK3sConfig } from "../operations/k3s-config.js";
 import { installK3sBinary } from "../operations/k3s-install.js";
-import { labelWorkerRole } from "../operations/worker-label.js";
 
 export const k3sAgentGroup: OperationGroup = {
   name: "k3s-agent",
@@ -12,8 +11,6 @@ export const k3sAgentGroup: OperationGroup = {
   operations: [
     { name: "Write k3s config", fn: writeK3sConfig },
     { name: "Install k3s agent", fn: installK3sBinary },
-    // After registration, never before: the kubelet may not set this itself.
-    { name: "Label node as worker", fn: labelWorkerRole },
   ],
 };
 

bastion/src/modules/modules/k3s/src/health/k3s-service.ts

@@ -1,9 +1,8 @@
 import type { Operation, OperationResult } from "../types.js";
 import { isServiceActive } from "../utils.js";
-import { isK3sServer } from "@lab/shared";
 
 export const checkK3sService: Operation = async (ctx): Promise<OperationResult> => {
-  const isServer = isK3sServer(ctx.config.role);
+  const isServer = ctx.config.role === "infra" || ctx.config.role === "labcontroller";
   const service = isServer ? "k3s" : "k3s-agent";
   const active = await isServiceActive(ctx, service);
   return { success: active, changed: false, message: active ? `${service} is active` : `${service} is not active` };

bastion/src/modules/modules/k3s/src/install.ts

@@ -1,21 +1,17 @@
 // k3s module — install phase.
 // Installs k3s with CIS-hardened configuration and Cilium CNI.
 
-import { isK3sServer, type Role } from "@lab/shared";
-
 export interface K3sInstallContext {
   hostname: string;
   ip: string;
-  role: Role;     // server vs agent is derived via isK3sServer(), never by name
+  role: string;   // "infra" = server, "worker" = agent
   k3sServerUrl?: string;  // Required for agent role
   k3sToken?: string;       // Required for agent role
 }
 
 /** Generate the shell script that installs k3s on a target machine. */
 export function generateInstallScript(ctx: K3sInstallContext): string {
-  // Was `ctx.role === "infra"`, which silently treated a `labcontroller` -- a
-  // role that runs a k3s SERVER -- as an agent. Derived from the registry now.
-  const isServer = isK3sServer(ctx.role);
+  const isServer = ctx.role === "infra";
 
   return `#!/bin/bash
 set -euo pipefail
@@ -219,9 +215,7 @@ echo "  Using network device: \$DEFAULT_DEV"
 
 KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install \\
   --set kubeProxyReplacement=true \\
-  --set ipam.mode=cluster-pool \\
-  --set ipam.operator.clusterPoolIPv4PodCIDRList='{10.42.0.0/16}' \\
-  --set ipam.operator.clusterPoolIPv4MaskSize=24 \\
+  --set ipam.mode=kubernetes \\
   --set devices="\$DEFAULT_DEV" \\
   --set nodePort.directRoutingDevice="\$DEFAULT_DEV"
 

bastion/src/modules/modules/k3s/src/k3s-module.ts

@@ -6,35 +6,21 @@ import type { OperationContext, K3sConfig, OperationResult } from "./types.js";
 import { sshExec } from "../../../src/ssh.js";
 import { aggregateResults } from "./utils.js";
 import { runHostPrep } from "./groups/host-prep.js";
-import { runEvacuate } from "./groups/evacuate.js";
-import { runAdmit } from "./groups/admit.js";
 import { runK3sServer } from "./groups/k3s-server.js";
 import { runK3sAgent } from "./groups/k3s-agent.js";
 import { runNetworking } from "./groups/networking.js";
 import { runHardening } from "./groups/hardening.js";
 import { runSequential } from "./utils.js";
 import * as health from "./health/index.js";
-import { isK3sServer } from "@lab/shared";
 
 function toOpContext(ctx: ModuleContext): OperationContext {
-  // Pass the WHOLE config bag through, then pin the fields that come from the
-  // ModuleContext itself.
-  //
-  // This used to enumerate three keys -- k3sServerUrl, k3sToken, tlsSans -- and
-  // silently drop every other one. Anything a caller set beyond those three
-  // vanished with no error and no warning: k3sVersion (so a pinned version was
-  // ignored and get.k3s.io installed LATEST), evictionAllowSingleReplica (so
-  // `labctl app k3s evacuate --allow-single-replica` was a no-op that claimed
-  // to override a data-loss guard), and the dual-stack fields ipv6 /
-  // clusterCidr / serviceCidr.
-  //
-  // Enumerating is the bug: every field added to K3sConfig has to be
-  // remembered here, and forgetting produces silence rather than a type error.
   const config: K3sConfig = {
-    ...(ctx.config as Partial<K3sConfig>),
     hostname: ctx.hostname,
     ip: ctx.ip,
     role: ctx.role as K3sConfig["role"],
+    k3sServerUrl: ctx.config["k3sServerUrl"] as string | undefined,
+    k3sToken: ctx.config["k3sToken"] as string | undefined,
+    tlsSans: ctx.config["tlsSans"] as string[] | undefined,
   };
   return {
     config,
@@ -49,16 +35,7 @@ function toOpContext(ctx: ModuleContext): OperationContext {
     },
     os: ctx.os,
     arch: ctx.arch,
-    // Emit progress LIVE, on stderr.
-    //
-    // This used to be a no-op "collected via results", but results only carry
-    // an operation's final message. Anything an operation reports WHILE it
-    // runs was therefore thrown away -- including evictLonghornReplicas's
-    // per-poll "N replica(s) still on <node>", which is the only visibility
-    // into an eviction that can legitimately take an hour. When one stalled,
-    // there was no way to tell which volumes were stuck or whether progress
-    // was being made at all.
-    log: (msg) => { console.error(`  [${ctx.hostname}] ${msg}`); },  // eslint-disable-line no-console
+    log: (_msg) => { /* collected via results */ },
   };
 }
 
@@ -85,7 +62,7 @@ export class K3sModule implements Module {
   async install(ctx: ModuleContext): Promise<ModuleResult> {
     const start = performance.now();
     const opCtx = toOpContext(ctx);
-    const isServer = isK3sServer(ctx.role);
+    const isServer = ctx.role === "infra" || ctx.role === "labcontroller";
 
     // Phase 1: Host preparation
     const prepResults = await runHostPrep(opCtx);
@@ -118,31 +95,6 @@ export class K3sModule implements Module {
     return toModuleResult("configure", results, start);
   }
 
-  /**
-   * Evacuate the node so it can be wiped. Does NOT reinstall anything --
-   * it leaves the node cordoned, drained and empty.
-   */
-  async evacuate(ctx: ModuleContext): Promise<ModuleResult> {
-    const start = performance.now();
-    const opCtx = toOpContext(ctx);
-    const results = await runEvacuate(opCtx);
-    return toModuleResult("evacuate", results, start);
-  }
-
-  /**
-   * Return an evacuated node to service: uncordon, clear Longhorn's eviction
-   * request, restore its default disk.
-   *
-   * The counterpart to evacuate(), and the only supported way out of an
-   * evacuation that was aborted partway.
-   */
-  async admit(ctx: ModuleContext): Promise<ModuleResult> {
-    const start = performance.now();
-    const opCtx = toOpContext(ctx);
-    const results = await runAdmit(opCtx);
-    return toModuleResult("admit", results, start);
-  }
-
   async health(ctx: ModuleContext): Promise<ModuleResult> {
     const start = performance.now();
     const opCtx = toOpContext(ctx);

bastion/src/modules/modules/k3s/src/operations/cilium.ts

@@ -47,9 +47,7 @@ export const installCilium: Operation = async (ctx): Promise<OperationResult> =>
   const installResult = await ctx.ssh.exec(
     `KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install \
       --set kubeProxyReplacement=true \
-      --set ipam.mode=cluster-pool \
-      --set ipam.operator.clusterPoolIPv4PodCIDRList='{10.42.0.0/16}' \
-      --set ipam.operator.clusterPoolIPv4MaskSize=24 \
+      --set ipam.mode=kubernetes \
       --set k8sServiceHost=127.0.0.1 \
       --set k8sServicePort=6444 \
       --set cni.exclusive=false \

bastion/src/modules/modules/k3s/src/operations/etcd-members.ts

@@ -1,60 +0,0 @@
-// Parsing `etcdctl member list`.
-//
-// WHY NOT `-w json`
-// -----------------
-// Two reasons, and the second is the dangerous one:
-//
-//  1. `member remove` requires the id in HEX. The JSON writer emits decimal.
-//  2. etcd member ids are uint64. JSON.parse turns them into JS numbers, which
-//     lose precision above 2^53 -- so the id that comes back is an
-//     APPROXIMATION of the real one.
-//
-// Observed for real: `member remove 13323819137391910000` ->
-// "bad member ID arg (strconv.ParseUint: value out of range), expecting ID in
-// Hex". That happens at the worst possible moment, with the node already
-// stopped and about to be wiped. Failing loudly is the lucky outcome; a
-// mangled id that happened to match another member would remove the WRONG one.
-//
-// The default writer prints the id as hex TEXT, so it never becomes a number.
-
-export interface EtcdMember {
-  /** Raw member name: "<node>-<8 hex>", e.g. "worker0.lab-e26aacc5". */
-  name: string;
-  /** The node name with etcd's random suffix stripped. */
-  node: string;
-  /** Hex id, exactly as etcd prints it — never round-tripped through a number. */
-  id: string;
-  started: boolean;
-}
-
-/**
- * etcd names a member `<node>-<8 hex>`. Comparing a raw member name against a
- * node name without stripping that suffix silently never matches.
- */
-export function etcdNodeName(memberName: string): string {
-  return memberName.replace(/-[0-9a-f]{8}$/, "");
-}
-
-/**
- * Parse the default `etcdctl member list` writer:
- *
- *   825852d3683bc406, started, worker2.lab-5f7ad0c9, https://..:2380, https://..:2379, false
- *   <hex id>        , status , <node>-<8 hex>      , peer           , client         , learner
- */
-export function parseEtcdMembers(out: string): EtcdMember[] {
-  return out.split("\n")
-    .map((l) => l.trim())
-    .filter((l) => l !== "")
-    .map((line) => {
-      const cols = line.split(",").map((c) => c.trim());
-      const name = cols[2] ?? "";
-      return {
-        name,
-        node: etcdNodeName(name),
-        id: cols[0] ?? "",
-        started: (cols[1] ?? "") === "started",
-      };
-    })
-    // Drop anything without a hex id: blank lines, warnings, stray output.
-    .filter((m) => /^[0-9a-f]+$/i.test(m.id));
-}

bastion/src/modules/modules/k3s/src/operations/index.ts

@@ -1,11 +1,10 @@
 export { loadKernelModules } from "./kernel-modules.js";
 export { applyCisHardening } from "./sysctl.js";
-export { enableSwap } from "./swap.js";
-export { growRancherLv } from "./rancher-storage.js";
+export { disableSwap } from "./swap.js";
 export { enableIscsi } from "./iscsi.js";
 export { disableFirewall } from "./firewall.js";
 export { setSelinuxPermissive } from "./selinux.js";
-export { writeK3sConfig, generateServerConfig, generateAgentConfig } from "./k3s-config.js";
+export { writeK3sConfig } from "./k3s-config.js";
 export { writeAuditPolicy } from "./audit-policy.js";
 export { cleanupStaleCni } from "./cni-cleanup.js";
 export { installK3sBinary } from "./k3s-install.js";
@@ -24,10 +23,3 @@ export type {
   RecoverEtcdMemberOptions,
   RecoverEtcdMemberResult,
 } from "./etcd-recover.js";
-export {
-  cordonNode, uncordonNode, drainNode, evictLonghornReplicas,
-  assertEvictionCompletable, assertNodeEvacuated, checkEvictionCompletable,
-} from "./node-evacuate.js";
-export type { EvictionFeasibility } from "./node-evacuate.js";
-export { clearLonghornEviction, assertNodeSchedulable } from "./node-admit.js";
-export { labelWorkerRole } from "./worker-label.js";

bastion/src/modules/modules/k3s/src/operations/k3s-config.ts

@@ -2,54 +2,13 @@
 
 import type { Operation, OperationResult, K3sConfig } from "../types.js";
 import { sshOpts, writeRemoteFile } from "../utils.js";
-import { isK3sServer } from "@lab/shared";
 
-/**
- * The address-family block: `cluster-cidr`, `service-cidr` and `node-ip`.
- *
- * Emitted ONLY when the corresponding config is supplied, and that is
- * deliberate. With no `ipv6` and no CIDRs this returns "", so the generated
- * file is byte-identical to what every existing node already has -- no diff,
- * so `writeRemoteFile` reports unchanged and nothing restarts k3s. Dual-stack
- * is therefore opt-in per node rather than a flag day.
- *
- * `node-ip` is only written once there is a second family to name. k3s
- * auto-detects a sensible IPv4 on its own, and writing it out unconditionally
- * would rewrite the config of five healthy nodes to tell them what they had
- * already worked out.
- */
-function addressFamilyLines(config: K3sConfig, opts: { cidrs: boolean }): string {
-  const lines: string[] = [];
-  if (opts.cidrs && config.clusterCidr?.length) {
-    lines.push(`cluster-cidr: "${config.clusterCidr.join(",")}"`);
-  }
-  if (opts.cidrs && config.serviceCidr?.length) {
-    lines.push(`service-cidr: "${config.serviceCidr.join(",")}"`);
-  }
-  if (config.ipv6) {
-    // Order matters to k3s: the FIRST entry is the primary family, and the
-    // supported single-to-dual-stack conversion is the one that preserves it.
-    // IPv4 stays primary so existing Services keep their ClusterIP.
-    lines.push(`node-ip: "${config.ip},${config.ipv6}"`);
-  }
-  return lines.length ? `${lines.join("\n")}\n` : "";
+function isServerRole(role: string): boolean {
+  return role === "infra" || role === "labcontroller";
 }
 
-// Exported so the exact production config.yaml can be rendered outside an SSH
-// context -- notably by the labsim 3-server-etcd rehearsal, which must drive its
-// nodes through THIS generator rather than a parallel set of INSTALL_K3S_EXEC
-// flags, or it proves a mechanism production does not run.
-export function generateServerConfig(config: K3sConfig): string {
-  // The IPv6 address goes in the cert too. Without it, anything that reaches
-  // this apiserver over v6 -- a peer server joining, or kubectl against the v6
-  // address -- fails TLS verification, and the error names the certificate
-  // rather than the missing SAN, which is a long way from the cause.
-  const tlsSans = [
-    config.hostname,
-    config.ip,
-    ...(config.ipv6 ? [config.ipv6] : []),
-    ...(config.tlsSans ?? []),
-  ];
+function generateServerConfig(config: K3sConfig): string {
+  const tlsSans = [config.hostname, config.ip, ...(config.tlsSans ?? [])];
   const isJoining = !!config.k3sServerUrl;
   const clusterLines = isJoining
     ? `server: "${config.k3sServerUrl}"\ntoken: "${config.k3sToken}"`
@@ -62,7 +21,7 @@ export function generateServerConfig(config: K3sConfig): string {
   // and never expire.
   return `# k3s server configuration — CIS hardened, etcd HA
 ${clusterLines}
-${addressFamilyLines(config, { cidrs: true })}protect-kernel-defaults: true
+protect-kernel-defaults: true
 secrets-encryption: true
 write-kubeconfig-mode: "0640"
 
@@ -92,34 +51,10 @@ ${tlsSans.map((s) => `  - "${s}"`).join("\n")}
 `;
 }
 
-// Takes the config now: an agent needs its own dual `node-ip` just as much as a
-// server does. Without one it joins as an IPv4-only node into a dual-stack
-// cluster, gets no IPv6 pod CIDR, and the failure surfaces later as pods on that
-// node being unreachable over v6 while the node itself reads Ready.
-// It takes no cluster/service CIDRs -- those are server-side only.
-export function generateAgentConfig(config: K3sConfig): string {
-  // NO node-role.kubernetes.io/worker LABEL HERE, deliberately.
-  //
-  // A kubelet is not permitted to assign itself labels in the
-  // `node-role.kubernetes.io/` namespace -- the servers run with
-  // `enable-admission-plugins=NodeRestriction`, which rejects the registration
-  // outright. The agent then never finishes registering, k3s-agent.service
-  // never signals readiness, and systemd reports:
-  //
-  //   Job for k3s-agent.service failed because the service did not take the
-  //   steps required by its unit configuration.
-  //
-  // which says nothing about labels and sends you looking at the network, the
-  // token, or the kubelet's sysctls. Every agent installed through this module
-  // failed this way; the estate's existing workers only escaped it because
-  // post-provision installs them with no config.yaml at all and their role
-  // label was applied separately, server-side.
-  //
-  // `node.longhorn.io/...` is fine: it is outside the kubernetes.io namespace,
-  // so the kubelet may set it. The worker role label is applied after the node
-  // registers, by labelWorkerRole below.
-  return `${addressFamilyLines(config, { cidrs: false })}protect-kernel-defaults: true
+function generateAgentConfig(): string {
+  return `protect-kernel-defaults: true
 node-label:
+  - "node-role.kubernetes.io/worker=true"
   - "node.longhorn.io/create-default-disk=config"
 kubelet-arg:
   - "protect-kernel-defaults=true"
@@ -129,41 +64,11 @@ kubelet-arg:
 }
 
 export const writeK3sConfig: Operation = async (ctx): Promise<OperationResult> => {
-  // Refuse to name an address the node does not have.
-  //
-  // Most of this estate is SSH-onboard, not PXE-provisioned: Asahi cannot PXE
-  // at all, and the DGX Sparks run NVIDIA's own OS and must never be
-  // reinstalled. For those nodes the install templates govern nothing and this
-  // module is the ONLY labctl touchpoint, so nothing upstream can guarantee the
-  // vendor OS actually took a DHCPv6 lease.
-  //
-  // Writing node-ip for a missing address does not fail here -- it fails later,
-  // when k3s will not start, with an error about binding rather than about
-  // addressing. Checking costs one ssh round trip and turns a confusing
-  // start-up failure into a sentence naming the address and the node.
-  if (ctx.config.ipv6) {
-    const probe = await ctx.ssh.exec(
-      `ip -6 -o addr show 2>/dev/null | grep -qF " ${ctx.config.ipv6}/" && echo present || true`,
-      sshOpts(ctx),
-    );
-    if (!probe.stdout.includes("present")) {
-      return {
-        success: false,
-        changed: false,
-        message: `Node does not have IPv6 ${ctx.config.ipv6}`,
-        error:
-          `k3s config would set node-ip to ${ctx.config.ipv6}, but that address is not on any ` +
-          `interface. k3s resolves node-ip at start-up and would fail to bind. Check the node ` +
-          `took its DHCPv6 lease (a kea reservation keyed on its MAC) before retrying.`,
-      };
-    }
-  }
-
   await ctx.ssh.exec("mkdir -p /etc/rancher/k3s", sshOpts(ctx));
 
-  const content = isK3sServer(ctx.config.role)
+  const content = isServerRole(ctx.config.role)
     ? generateServerConfig(ctx.config)
-    : generateAgentConfig(ctx.config);
+    : generateAgentConfig();
 
   const changed = await writeRemoteFile(ctx, "/etc/rancher/k3s/config.yaml", content);
 

bastion/src/modules/modules/k3s/src/operations/k3s-install.ts

@@ -2,19 +2,13 @@
 
 import type { Operation, OperationResult } from "../types.js";
 import { sshOpts } from "../utils.js";
-import { isK3sServer } from "@lab/shared";
+
+function isServerRole(role: string): boolean {
+  return role === "infra" || role === "labcontroller";
+}
 
 export const installK3sBinary: Operation = async (ctx): Promise<OperationResult> => {
-  const isServer = isK3sServer(ctx.config.role);
-
-  // Pin the version when asked. get.k3s.io installs LATEST by default, so a
-  // node reinstalled during a rolling migration comes back on whatever shipped
-  // since -- potentially several minor versions ahead of the cluster it is
-  // about to rejoin, which Kubernetes does not support between control-plane
-  // members.
-  const versionEnv = ctx.config.k3sVersion !== undefined && ctx.config.k3sVersion !== ""
-    ? `INSTALL_K3S_VERSION="${ctx.config.k3sVersion}" `
-    : "";
+  const isServer = isServerRole(ctx.config.role);
 
   // Check if already installed
   const version = await ctx.ssh.exec("k3s --version 2>/dev/null", sshOpts(ctx));
@@ -30,21 +24,12 @@ export const installK3sBinary: Operation = async (ctx): Promise<OperationResult>
       );
     }
 
-    // Token and URL are independent. A token WITHOUT a URL seeds the cluster
-    // token on the cluster-init node, so every later joiner can be given the
-    // same pre-shared value instead of scraping node-token off the first
-    // server -- which matters here because that first server gets wiped and
-    // rejoined during a rolling migration. URL only counts when a token comes
-    // with it; a URL alone cannot authenticate a join.
-    //
-    // The token deliberately does NOT go in config.yaml: writeRemoteFile diffs
-    // that file, so a token appearing there would restart k3s on every node.
-    const joinEnv = [
-      ctx.config.k3sServerUrl && ctx.config.k3sToken ? `K3S_URL="${ctx.config.k3sServerUrl}"` : "",
-      ctx.config.k3sToken ? `K3S_TOKEN="${ctx.config.k3sToken}"` : "",
-    ].filter(Boolean).join(" ");
+    // If joining an existing cluster, pass K3S_URL and K3S_TOKEN
+    const joinEnv = ctx.config.k3sServerUrl && ctx.config.k3sToken
+      ? `K3S_URL="${ctx.config.k3sServerUrl}" K3S_TOKEN="${ctx.config.k3sToken}"`
+      : "";
     const result = await ctx.ssh.exec(
-      `curl -sfL https://get.k3s.io | ${versionEnv}INSTALL_K3S_EXEC="server" INSTALL_K3S_SKIP_SELINUX_RPM=true ${joinEnv} sh -`,
+      `curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server" INSTALL_K3S_SKIP_SELINUX_RPM=true ${joinEnv} sh -`,
       { timeoutMs: 300_000 },
     );
     if (result.exitCode !== 0) {
@@ -65,7 +50,7 @@ export const installK3sBinary: Operation = async (ctx): Promise<OperationResult>
       };
     }
     const result = await ctx.ssh.exec(
-      `curl -sfL https://get.k3s.io | ${versionEnv}INSTALL_K3S_EXEC="agent" K3S_URL="${ctx.config.k3sServerUrl}" K3S_TOKEN="${ctx.config.k3sToken}" sh -`,
+      `curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="agent" K3S_URL="${ctx.config.k3sServerUrl}" K3S_TOKEN="${ctx.config.k3sToken}" sh -`,
       { timeoutMs: 300_000 },
     );
     if (result.exitCode !== 0) {

bastion/src/modules/modules/k3s/src/operations/longhorn-disk.ts

@@ -1,206 +1,50 @@
-// Give a node a Longhorn disk at /var/lib/longhorn.
-//
-// The cluster runs `createDefaultDiskLabeledNodes: true`, which means Longhorn
-// creates a disk ONLY on nodes carrying the label
-// `node.longhorn.io/create-default-disk=config` (set via k3s node-label) AND
-// the annotation `node.longhorn.io/default-disks-config` (set here). Miss
-// either and Longhorn gives the node no disk at all: it joins, goes Ready,
-// accepts pods, and silently hosts zero replicas.
-//
-// WHY THERE IS A REPAIR PATH
-// --------------------------
-// Longhorn reads that annotation when it FIRST creates the node's
-// `nodes.longhorn.io` record. This operation runs during configure(), which on
-// a rejoining node is after the record already exists -- so the annotation
-// lands too late and is never read. That is invisible until a rebuild is
-// expected and never arrives, roughly forty minutes into a migration.
-//
-// So: annotate, then read back, and if there is still no disk, write it
-// directly into spec.disks. When Longhorn does honour the annotation the
-// repair is a no-op.
+// Annotate nodes with Longhorn default disk config when /var/lib/longhorn exists.
+// The label is set in k3s config (node-label), but the annotation must be applied via kubectl.
 
-import type { Operation, OperationContext, OperationResult } from "../types.js";
+import type { Operation, OperationResult } from "../types.js";
 import { sshOpts } from "../utils.js";
 import { sshExec as remoteSshExec } from "../../../../src/ssh.js";
 
-const DISK_PATH = "/var/lib/longhorn";
-/** Fixed key so a repeated repair updates one disk rather than adding more. */
-const DISK_KEY = "default-disk-lab";
-const VERIFY_INTERVAL_MS = 5_000;
-/** How long to wait for Longhorn to create the node's record after it joins. */
-const REGISTER_TIMEOUT_MS = 180_000;
-/** How long to then wait for Longhorn to give that record a disk. */
-const DISK_TIMEOUT_MS = 60_000;
-
-const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
-
-/**
- * Run kubectl from wherever it can be reached: locally on a server, otherwise
- * over SSH to the configured server (an agent has no local kubectl).
- */
-async function kubectlAnywhere(
-  ctx: OperationContext,
-  args: string,
-): Promise<{ ok: boolean; out: string }> {
-  const local = await ctx.ssh.exec(`k3s kubectl ${args} 2>&1`, sshOpts(ctx));
-  if (local.exitCode === 0) return { ok: true, out: local.stdout };
-
-  if (ctx.config.k3sServerUrl) {
-    try {
-      const serverHost = new URL(ctx.config.k3sServerUrl).hostname;
-      const remote = await remoteSshExec(serverHost, "root", `k3s kubectl ${args}`, {
-        ...(ctx.ssh.keyPath ? { keyPath: ctx.ssh.keyPath } : {}),
-        timeoutMs: 15_000,
-      });
-      if (remote.exitCode === 0) return { ok: true, out: remote.stdout };
-      return { ok: false, out: remote.stdout || remote.stderr };
-    } catch {
-      /* fall through */
-    }
-  }
-  return { ok: false, out: local.stdout || local.stderr };
-}
-
-/** Does the node's Longhorn record exist yet? */
-async function nodeRecordExists(ctx: OperationContext, nodeName: string): Promise<boolean> {
-  const res = await kubectlAnywhere(ctx, `-n longhorn-system get nodes.longhorn.io "${nodeName}" -o name`);
-  return res.ok && res.out.trim() !== "";
-}
-
-/** Does the node's Longhorn record carry a disk at DISK_PATH? */
-async function hasDisk(ctx: OperationContext, nodeName: string): Promise<boolean> {
-  const res = await kubectlAnywhere(
-    ctx,
-    `-n longhorn-system get nodes.longhorn.io "${nodeName}" ` +
-      `-o jsonpath='{range .spec.disks.*}{.path}{"\\n"}{end}'`,
-  );
-  if (!res.ok) return false;
-  return res.out.split("\n").map((l) => l.trim()).includes(DISK_PATH);
-}
-
 export const configureLonghornDisk: Operation = async (ctx): Promise<OperationResult> => {
-  const check = await ctx.ssh.exec(`test -d ${DISK_PATH} && echo yes || echo no`, sshOpts(ctx));
+  // Check if /var/lib/longhorn exists on this node
+  const check = await ctx.ssh.exec("test -d /var/lib/longhorn && echo yes || echo no", sshOpts(ctx));
   if (check.stdout.trim() !== "yes") {
-    return { success: true, changed: false, message: `No ${DISK_PATH} directory — skipping Longhorn disk config` };
+    return { success: true, changed: false, message: "No /var/lib/longhorn directory — skipping Longhorn disk config" };
   }
 
-  // Longhorn writes replicas into <path>/replicas and creates that directory
-  // only when it FIRST initialises the disk. If the directory later goes away
-  // while longhorn-disk.cfg remains -- someone clearing space, a partial
-  // restore, a wipe that preserved the LV -- Longhorn considers the disk
-  // already initialised and never recreates it. The replica process then dies
-  // on a non-recursive mkdir:
-  //
-  //   level=fatal msg="Error running start replica command"
-  //     error="mkdir /host/var/lib/longhorn/replicas/pvc-...: ..."
-  //
-  // every single time, for ever, while the node reports Ready and Schedulable
-  // and its disk looks healthy. Cheap to guarantee, expensive to diagnose.
-  await ctx.ssh.exec(`mkdir -p ${DISK_PATH}/replicas`, sshOpts(ctx));
-
+  // Find the node name (hostname as registered in k3s)
   const nodeNameResult = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
   const nodeName = nodeNameResult.stdout.trim();
 
-  // SINGLE quotes around the whole key=value, not double.
-  //
-  // The value is JSON and therefore full of double quotes. Wrapping it in
-  // double quotes ends the quoted string at the first inner one, and the shell
-  // strips the rest, so the node gets
-  //     [{path:/var/lib/longhorn,allowScheduling:true}]
-  // instead of
-  //     [{"path":"/var/lib/longhorn","allowScheduling":true}]
-  // -- valid-looking, unparseable, and silent. Longhorn then creates NO disk on
-  // the node, which surfaces much later as "why does nothing schedule here".
-  // The JSON contains no single quotes, so single-quoting is safe.
-  const annotation = JSON.stringify([{ path: DISK_PATH, allowScheduling: true }]);
-  const annotated = await kubectlAnywhere(
-    ctx,
-    `annotate node "${nodeName}" 'node.longhorn.io/default-disks-config=${annotation}' --overwrite`,
+  const annotation = JSON.stringify([{ path: "/var/lib/longhorn", allowScheduling: true }]);
+
+  // Try kubectl locally first (works on server nodes)
+  const result = await ctx.ssh.exec(
+    `k3s kubectl annotate node "${nodeName}" "node.longhorn.io/default-disks-config=${annotation}" --overwrite 2>&1 || true`,
+    sshOpts(ctx),
   );
 
-  // No Longhorn at all (or no way to reach the API) — nothing more to do.
-  const lh = await kubectlAnywhere(ctx, "get ns longhorn-system -o name");
-  if (!lh.ok) {
-    return annotated.ok
-      ? { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName} (Longhorn not installed yet)` }
-      : { success: true, changed: false, message: "Longhorn disk label set (annotation requires server kubectl)" };
+  if (result.stdout.includes("annotated") || result.stdout.includes("unchanged")) {
+    return { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName}` };
   }
 
-  // Longhorn registration is ASYNCHRONOUS. On a node that has just joined --
-  // which is exactly the case after a wipe-and-rejoin -- the nodes.longhorn.io
-  // record does not exist yet, so there is nothing to inspect and nothing to
-  // repair. Waiting a fixed 60s and then giving up meant the repair never ran
-  // on the one path that needs it most, and the caller's schedulability gate
-  // then failed immediately with "Longhorn reports no disk".
-  //
-  // So: wait for the RECORD first, generously, and only then worry about the
-  // disk on it.
-  const registerDeadline = Date.now() + REGISTER_TIMEOUT_MS;
-  let registered = false;
-  while (Date.now() < registerDeadline) {
-    if (await hasDisk(ctx, nodeName)) {
-      return { success: true, changed: annotated.ok, message: `${nodeName} has a Longhorn disk at ${DISK_PATH}` };
+  // For worker/agent nodes without local kubectl: apply via the server
+  if (ctx.config.k3sServerUrl) {
+    // The CLI has SSH access to the server — use sshExec from there
+    const serverHost = new URL(ctx.config.k3sServerUrl).hostname;
+    try {
+      const remoteResult = await remoteSshExec(
+        serverHost, "root",
+        `k3s kubectl annotate node "${nodeName}" "node.longhorn.io/default-disks-config=${annotation}" --overwrite`,
+        { ...(ctx.ssh.keyPath ? { keyPath: ctx.ssh.keyPath } : {}), timeoutMs: 15_000 },
+      );
+      if (remoteResult.stdout.includes("annotated") || remoteResult.stdout.includes("unchanged")) {
+        return { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName} (via server)` };
+      }
+    } catch {
+      // Fall through to manual instruction
     }
-    if (await nodeRecordExists(ctx, nodeName)) { registered = true; break; }
-    await sleep(VERIFY_INTERVAL_MS);
   }
 
-  if (!registered) {
-    return {
-      success: true,
-      changed: annotated.ok,
-      message: `${nodeName} not registered with Longhorn within ${REGISTER_TIMEOUT_MS / 1000}s; ` +
-        `annotation is in place for when it is`,
-    };
-  }
-
-  // Registered. Give Longhorn a chance to act on the annotation by itself
-  // before overriding it.
-  const diskDeadline = Date.now() + DISK_TIMEOUT_MS;
-  while (Date.now() < diskDeadline) {
-    if (await hasDisk(ctx, nodeName)) {
-      return { success: true, changed: annotated.ok, message: `${nodeName} has a Longhorn disk at ${DISK_PATH}` };
-    }
-    await sleep(VERIFY_INTERVAL_MS);
-  }
-
-  // The record exists and still has no disk: the annotation was applied too
-  // late to be read. Write it in.
-  ctx.log(`${nodeName} has no Longhorn disk after annotating; writing spec.disks directly`);
-  const patch = JSON.stringify({
-    spec: {
-      disks: {
-        [DISK_KEY]: {
-          path: DISK_PATH,
-          allowScheduling: true,
-          evictionRequested: false,
-          storageReserved: 0,
-          tags: [],
-          diskType: "filesystem",
-        },
-      },
-    },
-  });
-  const repaired = await kubectlAnywhere(
-    ctx,
-    `-n longhorn-system patch nodes.longhorn.io "${nodeName}" --type=merge -p '${patch}'`,
-  );
-  if (!repaired.ok) {
-    return {
-      success: false,
-      changed: false,
-      message: `${nodeName} has no Longhorn disk and could not be repaired`,
-      error: `${repaired.out}. Without a disk this node will never host a replica.`,
-    };
-  }
-
-  if (await hasDisk(ctx, nodeName)) {
-    return { success: true, changed: true, message: `Longhorn disk repaired on ${nodeName} (${DISK_PATH})` };
-  }
-  return {
-    success: false,
-    changed: true,
-    message: `Patched spec.disks on ${nodeName} but Longhorn still reports no disk at ${DISK_PATH}`,
-    error: "Check longhorn-manager logs; the node will not host replicas until this is resolved.",
-  };
+  return { success: true, changed: false, message: "Longhorn disk label set (annotation requires server kubectl)" };
 };

bastion/src/modules/modules/k3s/src/operations/node-admit.ts

@@ -1,136 +0,0 @@
-// Bring a node back into service. The inverse of node-evacuate.ts.
-//
-// WHY THIS EXISTS
-// ---------------
-// Evacuation is not symmetric with itself. It leaves three marks on a node --
-// a kubernetes cordon, Longhorn's `allowScheduling:false` + `evictionRequested`,
-// and (on a node that was wiped and rejoined) a MISSING default Longhorn disk.
-// Undoing a cordon undoes only the first. Without the other two the node comes
-// back Ready, accepts pods, and quietly hosts no replicas at all -- which looks
-// like success right up until you notice storage never rebalanced onto it.
-//
-// This is also the only supported way out of an aborted evacuation.
-
-import type { Operation, OperationContext, OperationResult } from "../types.js";
-import { sshOpts } from "../utils.js";
-
-async function nodeName(ctx: OperationContext): Promise<string> {
-  const r = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
-  return r.stdout.trim();
-}
-
-async function kubectl(ctx: OperationContext, args: string): Promise<{ ok: boolean; out: string }> {
-  const res = await ctx.ssh.exec(`k3s kubectl ${args} 2>&1`, sshOpts(ctx));
-  return { ok: res.exitCode === 0, out: res.exitCode === 0 ? res.stdout : (res.stdout || res.stderr) };
-}
-
-/**
- * Clear Longhorn's eviction request and re-enable scheduling.
- *
- * Both fields in one merge patch, mirroring how eviction sets them: Longhorn's
- * validating webhook rejects `evictionRequested:true` on a schedulable node, so
- * the pair has to move together in both directions.
- */
-export const clearLonghornEviction: Operation = async (ctx): Promise<OperationResult> => {
-  const name = await nodeName(ctx);
-
-  const lh = await kubectl(ctx, "get ns longhorn-system -o name");
-  if (!lh.ok) {
-    return { success: true, changed: false, message: "Longhorn not installed — nothing to clear" };
-  }
-
-  const before = await kubectl(
-    ctx,
-    `-n longhorn-system get nodes.longhorn.io "${name}" -o jsonpath='{.spec.allowScheduling},{.spec.evictionRequested}'`,
-  );
-  if (!before.ok) {
-    return { success: true, changed: false, message: `${name} has no Longhorn node record yet` };
-  }
-  if (before.out.trim() === "true,false") {
-    return { success: true, changed: false, message: `${name} already accepts replicas` };
-  }
-
-  const patch = JSON.stringify({ spec: { allowScheduling: true, evictionRequested: false } });
-  const res = await kubectl(
-    ctx,
-    `-n longhorn-system patch nodes.longhorn.io "${name}" --type=merge -p '${patch}'`,
-  );
-  return res.ok
-    ? { success: true, changed: true, message: `${name} accepts replicas again` }
-    : { success: false, changed: false, message: `Could not clear eviction on ${name}`, error: res.out };
-};
-
-/**
- * Confirm the node can actually host replicas again.
- *
- * Reads back from Longhorn rather than trusting the patches above, because the
- * failure this catches is a node that is Ready and schedulable in kubernetes
- * while having no Longhorn disk at all -- the exact state a rejoined node lands
- * in when its default-disk annotation was applied too late.
- */
-const SCHEDULABLE_TIMEOUT_MS = 120_000;
-const SCHEDULABLE_INTERVAL_MS = 5_000;
-const sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
-
-export const assertNodeSchedulable: Operation = async (ctx): Promise<OperationResult> => {
-  // Poll rather than check once.
-  //
-  // Longhorn registers a node and attaches its disk ASYNCHRONOUSLY, and this
-  // runs moments after a wipe-and-rejoin. Checking instantly reports "Longhorn
-  // reports no disk on this node" for a node that is simply a few seconds from
-  // being fine -- which is a false alarm on the exact path the gate exists to
-  // protect. Still fails if it never settles.
-  const deadline = Date.now() + SCHEDULABLE_TIMEOUT_MS;
-  let problems: string[] = [];
-  for (;;) {
-    problems = await schedulableProblems(ctx);
-    if (problems.length === 0 || Date.now() >= deadline) break;
-    await sleep(SCHEDULABLE_INTERVAL_MS);
-  }
-
-  const name = await nodeName(ctx);
-  return problems.length === 0
-    ? { success: true, changed: false, message: `${name} is back in service` }
-    : {
-        success: false,
-        changed: false,
-        message: `${name} is NOT fully back in service after ${SCHEDULABLE_TIMEOUT_MS / 1000}s`,
-        error: problems.join("; "),
-      };
-};
-
-/** One pass of the schedulability checks. Returns the reasons it is not. */
-async function schedulableProblems(ctx: OperationContext): Promise<string[]> {
-  const name = await nodeName(ctx);
-  const problems: string[] = [];
-
-  const sched = await kubectl(ctx, `get node "${name}" -o jsonpath='{.spec.unschedulable}'`);
-  if (sched.ok && sched.out.trim() === "true") problems.push("node is still cordoned");
-
-  const lh = await kubectl(ctx, "get ns longhorn-system -o name");
-  if (lh.ok) {
-    const spec = await kubectl(
-      ctx,
-      `-n longhorn-system get nodes.longhorn.io "${name}" -o jsonpath='{.spec.allowScheduling},{.spec.evictionRequested}'`,
-    );
-    if (!spec.ok) {
-      problems.push("no Longhorn node record");
-    } else {
-      const [allow, evicting] = spec.out.trim().split(",");
-      if (allow !== "true") problems.push("Longhorn scheduling is still disabled");
-      if (evicting === "true") problems.push("Longhorn eviction is still requested");
-    }
-
-    // A node with no disk hosts no replicas, however healthy it otherwise looks.
-    const disks = await kubectl(
-      ctx,
-      `-n longhorn-system get nodes.longhorn.io "${name}" ` +
-        `-o jsonpath='{range .status.diskStatus.*}{.conditions[?(@.type=="Schedulable")].status}{"\\n"}{end}'`,
-    );
-    const states = disks.ok ? disks.out.split("\n").map((l) => l.trim()).filter(Boolean) : [];
-    if (states.length === 0) problems.push("Longhorn reports no disk on this node");
-    else if (!states.includes("True")) problems.push(`no schedulable disk (states: ${states.join(",")})`);
-  }
-
-  return problems;
-}

bastion/src/modules/modules/k3s/src/operations/node-evacuate.ts

@@ -1,547 +0,0 @@
-// Evacuate a node before it is wiped and reinstalled.
-//
-// WHY THIS EXISTS
-// ---------------
-// The kickstart's %pre can preserve data across a reinstall, but ONLY when the
-// disk layout is unchanged: it keys off `labvg` already existing with the
-// expected LV names. Migrating a machine to the v2 layout means recreating the
-// volume group, which takes the `clearpart --all` branch and destroys every
-// Longhorn replica on that node.
-//
-// So a layout migration cannot rely on disk-level preservation. The node has to
-// be EVACUATED first -- replicas rebuilt elsewhere, workloads moved, etcd
-// membership surrendered -- and then wiped. These operations are the missing
-// half; before this file the repo had no drain, no cordon, and no replica
-// eviction anywhere.
-//
-// Ordering matters and is not interchangeable:
-//   cordon -> check -> evict replicas (WAIT for rebuild) -> drain -> remove etcd member
-//
-// Two ordering facts, both learned the hard way:
-//   - Removing the etcd member first costs you the API server you need for the
-//     rest of the procedure.
-//   - EVICT BEFORE DRAIN. `drain --force` does not override PodDisruptionBudgets,
-//     and longhorn-manager keeps a PDB on the instance-manager pod while volumes
-//     are attached or degraded. Draining a storage node first therefore stops the
-//     instance-manager that serves the very replicas being evicted: the drain
-//     blocks for its full timeout, and if it did succeed it would remove the only
-//     source a single-replica volume could rebuild from.
-
-import type { Operation, OperationContext, OperationResult } from "../types.js";
-import { sshOpts } from "../utils.js";
-
-/** Longhorn's own default; a rebuild of a large replica set can exceed it. */
-const REPLICA_REBUILD_TIMEOUT_MS = 60 * 60_000;
-const POLL_INTERVAL_MS = 15_000;
-/** How often to report "still waiting" when the replica count is not moving. */
-const HEARTBEAT_INTERVAL_MS = 3 * 60_000;
-
-const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
-
-async function nodeName(ctx: OperationContext): Promise<string> {
-  const r = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
-  return r.stdout.trim();
-}
-
-/**
- * kubectl, from wherever it is reachable.
- *
- * An agent has no local kubectl, so the caller must be pointed at a server.
- * Returns null when there is no way to reach the API, so callers can report
- * "could not do this" instead of silently continuing.
- */
-async function kubectl(
-  ctx: OperationContext,
-  args: string,
-  /**
-   * Override the SSH timeout for this call.
-   *
-   * sshOpts() is a flat 30 seconds, which is right for a status read and
-   * completely wrong for anything that does work. A `kubectl drain` asking for
-   * --timeout=15m gets killed by the transport at 30s and returns non-zero
-   * having ALREADY evicted most of the pods -- so it looks like a drain
-   * failure, with a transcript that reads like success, on every real node.
-   */
-  timeoutMs?: number,
-): Promise<{ ok: boolean; out: string }> {
-  const opts = timeoutMs === undefined ? sshOpts(ctx) : { timeoutMs };
-  const local = await ctx.ssh.exec(`k3s kubectl ${args} 2>&1`, opts);
-  if (local.exitCode === 0) return { ok: true, out: local.stdout };
-  return { ok: false, out: local.stdout || local.stderr };
-}
-
-/** Mark the node unschedulable. Safe and instantly reversible. */
-export const cordonNode: Operation = async (ctx): Promise<OperationResult> => {
-  const name = await nodeName(ctx);
-  const res = await kubectl(ctx, `cordon "${name}"`);
-  if (!res.ok) {
-    return { success: false, changed: false, message: `Could not cordon ${name}`, error: res.out };
-  }
-  const already = res.out.includes("already cordoned");
-  return {
-    success: true,
-    changed: !already,
-    message: already ? `${name} was already cordoned` : `Cordoned ${name}`,
-  };
-};
-
-export const uncordonNode: Operation = async (ctx): Promise<OperationResult> => {
-  const name = await nodeName(ctx);
-  const res = await kubectl(ctx, `uncordon "${name}"`);
-  return res.ok
-    ? { success: true, changed: true, message: `Uncordoned ${name}` }
-    : { success: false, changed: false, message: `Could not uncordon ${name}`, error: res.out };
-};
-
-/**
- * Move workloads off the node.
- *
- * --ignore-daemonsets because a DaemonSet pod cannot be evicted by definition;
- * --delete-emptydir-data because refusing to delete emptyDir is what makes an
- * unattended drain hang forever, and a node about to be wiped is going to lose
- * that data regardless.
- */
-export const drainNode: Operation = async (ctx): Promise<OperationResult> => {
-  const name = await nodeName(ctx);
-  // The SSH timeout must outlive kubectl's own --timeout, or the transport
-  // kills a drain that is working and the result is indistinguishable from a
-  // drain that failed.
-  const DRAIN_TIMEOUT_MS = 15 * 60_000;
-  const res = await kubectl(
-    ctx,
-    `drain "${name}" --ignore-daemonsets --delete-emptydir-data --force --timeout=15m`,
-    DRAIN_TIMEOUT_MS + 60_000,
-  );
-  if (res.ok) return { success: true, changed: true, message: `Drained ${name}` };
-
-  // Surface the END of the transcript. The first line of a drain is always
-  // "node/x already cordoned", which tells you nothing and hides the reason.
-  const tail = res.out.split("\n").map((l) => l.trim()).filter(Boolean).slice(-6).join(" | ");
-  return {
-    success: false,
-    changed: false,
-    message: `Drain failed on ${name}`,
-    error: tail === "" ? res.out : `...${tail}`,
-  };
-};
-
-/** kubectl -o json, parsed. Returns null when the read or the parse failed. */
-async function kubectlJson<T>(ctx: OperationContext, args: string): Promise<T | null> {
-  const res = await kubectl(ctx, `${args} -o json`);
-  if (!res.ok) return null;
-  try {
-    return JSON.parse(res.out) as T;
-  } catch {
-    return null;
-  }
-}
-
-/** A Longhorn setting's value, or the supplied default when it is unreadable. */
-async function longhornSetting(ctx: OperationContext, name: string, fallback: string): Promise<string> {
-  const res = await kubectl(ctx, `-n longhorn-system get settings.longhorn.io ${name} -o jsonpath='{.value}'`);
-  const v = res.ok ? res.out.trim() : "";
-  return v === "" ? fallback : v;
-}
-
-interface LhNode {
-  metadata: { name: string };
-  spec: { allowScheduling?: boolean };
-  status?: { diskStatus?: Record<string, {
-    storageAvailable?: number; storageMaximum?: number;
-    storageScheduled?: number; storageReserved?: number;
-  }> };
-}
-interface LhReplica {
-  metadata: { name: string };
-  spec: { nodeID?: string; volumeName?: string };
-}
-interface LhVolume {
-  metadata: { name: string };
-  spec: { numberOfReplicas?: number; size?: number | string };
-  status?: { state?: string; robustness?: string };
-}
-
-export interface EvictionFeasibility {
-  completable: boolean;
-  /** Reasons eviction can never finish. Each names the offending volume. */
-  blockers: string[];
-  /** Things that will work but are dangerous. */
-  warnings: string[];
-  replicasMoving: number;
-  bytesMoving: number;
-}
-
-/**
- * Can eviction of this node actually COMPLETE?
- *
- * evictLonghornReplicas waits for the node's replica count to reach zero. If
- * there is nowhere for a replica to go, that wait can never end -- it burns the
- * full hour-long timeout and then fails, which is an expensive way to discover
- * a fact that is knowable in a couple of seconds.
- *
- * It is knowable because Longhorn's placement rules are simple:
- *
- *   - With strict anti-affinity (replica-soft-anti-affinity=false, the default
- *     and what this fleet runs) a volume may put at most ONE replica on a node.
- *     So a volume wanting N replicas needs N schedulable nodes OTHER than the
- *     one being evacuated. A 3-replica volume on a 3-node cluster is therefore
- *     unevictable -- the two survivors already hold a copy each and cannot take
- *     the third.
- *   - The scheduler admits a replica against over-provisioned space, but the
- *     rebuild writes real bytes to a real disk. Those are different numbers,
- *     and at 500% over-provisioning the first can pass while the second fails.
- *
- * Read-only, a handful of API calls, finishes in seconds.
- */
-export async function checkEvictionCompletable(
-  ctx: OperationContext,
-  name: string,
-): Promise<EvictionFeasibility> {
-  const blockers: string[] = [];
-  const warnings: string[] = [];
-
-  const nodes = await kubectlJson<{ items: LhNode[] }>(ctx, "-n longhorn-system get nodes.longhorn.io");
-  const replicas = await kubectlJson<{ items: LhReplica[] }>(ctx, "-n longhorn-system get replicas.longhorn.io");
-  const volumes = await kubectlJson<{ items: LhVolume[] }>(ctx, "-n longhorn-system get volumes.longhorn.io");
-  if (!nodes || !replicas || !volumes) {
-    return {
-      completable: false,
-      blockers: ["Could not read Longhorn nodes/replicas/volumes — refusing to guess"],
-      warnings, replicasMoving: 0, bytesMoving: 0,
-    };
-  }
-
-  const softAntiAffinity = (await longhornSetting(ctx, "replica-soft-anti-affinity", "false")) === "true";
-  const overProvisionPct = Number(await longhornSetting(ctx, "storage-over-provisioning-percentage", "100")) || 100;
-
-  const mine = replicas.items.filter((r) => r.spec.nodeID === name);
-  if (mine.length === 0) {
-    return { completable: true, blockers, warnings, replicasMoving: 0, bytesMoving: 0 };
-  }
-
-  const volumeByName = new Map(volumes.items.map((v) => [v.metadata.name, v]));
-  // Candidates are schedulable Longhorn nodes that are not the one leaving.
-  const candidates = nodes.items.filter(
-    (n) => n.metadata.name !== name && n.spec.allowScheduling !== false,
-  );
-
-  const diskFree = (n: LhNode): number =>
-    Object.values(n.status?.diskStatus ?? {}).reduce((a, d) => a + (d.storageAvailable ?? 0), 0);
-  const schedulableFree = (n: LhNode): number =>
-    Object.values(n.status?.diskStatus ?? {}).reduce((a, d) => {
-      const usable = ((d.storageMaximum ?? 0) - (d.storageReserved ?? 0)) * (overProvisionPct / 100);
-      return a + Math.max(0, usable - (d.storageScheduled ?? 0));
-    }, 0);
-
-  let bytesMoving = 0;
-
-  for (const rep of mine) {
-    const volName = rep.spec.volumeName ?? "";
-    const vol = volumeByName.get(volName);
-    if (!vol) {
-      warnings.push(`replica ${rep.metadata.name} has no volume ${volName} — orphan?`);
-      continue;
-    }
-    const want = vol.spec.numberOfReplicas ?? 1;
-    const size = Number(vol.spec.size ?? 0);
-    bytesMoving += size;
-
-    // (1) Placement. Strict anti-affinity means one replica per node per volume.
-    if (!softAntiAffinity && candidates.length < want) {
-      blockers.push(
-        `volume ${volName} wants ${want} replicas but only ${candidates.length} other schedulable ` +
-        `node(s) exist — with strict anti-affinity its replica cannot be placed. Add a schedulable node.`,
-      );
-      continue;
-    }
-
-    // (2) Scheduler capacity, at the configured over-provisioning percentage.
-    const schedulableFit = candidates.filter((n) => schedulableFree(n) >= size);
-    if (schedulableFit.length === 0) {
-      blockers.push(
-        `volume ${volName} (${gib(size)}) does not fit the scheduler on any candidate node ` +
-        `even at ${overProvisionPct}% over-provisioning.`,
-      );
-      continue;
-    }
-
-    // (3) Real capacity. The scheduler counts over-provisioned space; a rebuild
-    // writes real bytes. This is the check that catches a small disk hiding
-    // behind a generous over-provisioning percentage.
-    const realFit = candidates.filter((n) => diskFree(n) >= size);
-    if (realFit.length === 0) {
-      const best = Math.max(0, ...candidates.map(diskFree));
-      blockers.push(
-        `volume ${volName} needs ${gib(size)} but the roomiest candidate node has ${gib(best)} ` +
-        `actually free. storage-over-provisioning-percentage=${overProvisionPct} lets the scheduler ` +
-        `accept this; the disk does not.`,
-      );
-      continue;
-    }
-
-    // (4) Volumes with no second copy, and volumes that cannot rebuild at all.
-    if (want === 1) {
-      warnings.push(
-        `volume ${volName} has a single replica — evicting it rebuilds from the only copy that exists`,
-      );
-    }
-    if (vol.status?.state === "detached") {
-      warnings.push(`volume ${volName} is detached — Longhorn must attach it to rebuild`);
-    }
-    if (vol.status?.robustness === "degraded") {
-      warnings.push(`volume ${volName} is already degraded before eviction starts`);
-    }
-  }
-
-  return { completable: blockers.length === 0, blockers, warnings, replicasMoving: mine.length, bytesMoving };
-}
-
-function gib(bytes: number): string {
-  return `${(bytes / 2 ** 30).toFixed(1)} GiB`;
-}
-
-/**
- * Standalone gate: refuse fast and loudly rather than hang for an hour.
- *
- * Exposed as its own Operation so it can be run on its own -- "could I evacuate
- * this node if I wanted to?" is a question worth answering without starting.
- */
-export const assertEvictionCompletable: Operation = async (ctx): Promise<OperationResult> => {
-  const name = await nodeName(ctx);
-  const lh = await kubectl(ctx, "get ns longhorn-system -o name");
-  if (!lh.ok) {
-    return { success: true, changed: false, message: "Longhorn not installed — nothing to evict" };
-  }
-
-  const f = await checkEvictionCompletable(ctx, name);
-  for (const w of f.warnings) ctx.log(`  WARNING: ${w}`);
-
-  if (!f.completable) {
-    return {
-      success: false,
-      changed: false,
-      message: `Eviction of ${name} cannot complete`,
-      error: f.blockers.join("; "),
-    };
-  }
-  return {
-    success: true,
-    changed: false,
-    message: f.replicasMoving === 0
-      ? `${name} holds no replicas`
-      : `${f.replicasMoving} replica(s) (${gib(f.bytesMoving)}) can be placed elsewhere`,
-  };
-};
-
-/**
- * Ask Longhorn to move this node's replicas elsewhere, then WAIT.
- *
- * The wait is the entire point. Setting evictionRequested and immediately
- * wiping the disk destroys the replicas mid-rebuild, which on a 3-replica
- * volume takes you to 1 and on the 2-replica Prometheus volume takes you to 0.
- *
- * Returns changed:false with an explanation when Longhorn is not installed,
- * so a standalone or storage-free node is a no-op rather than an error.
- */
-export const evictLonghornReplicas: Operation = async (ctx): Promise<OperationResult> => {
-  const name = await nodeName(ctx);
-
-  const hasLonghorn = await kubectl(ctx, "get ns longhorn-system -o name");
-  if (!hasLonghorn.ok) {
-    return { success: true, changed: false, message: "Longhorn not installed — nothing to evict" };
-  }
-
-  // Refuse in seconds rather than discover it after an hour of polling.
-  // Inside the operation, not only in the group, so `labctl app k3s evacuate`
-  // gets the same protection as the drill does.
-  const feasible = await checkEvictionCompletable(ctx, name);
-  for (const w of feasible.warnings) ctx.log(`  WARNING: ${w}`);
-  if (!feasible.completable) {
-    return {
-      success: false,
-      changed: false,
-      message: `Eviction of ${name} cannot complete — not starting it`,
-      error: feasible.blockers.join("; "),
-    };
-  }
-  // A single-replica volume rebuilds from the only copy in existence, and this
-  // fleet has no backup target. That is a decision for a human, not a default.
-  const singles = feasible.warnings.filter((w) => w.includes("single replica"));
-  if (singles.length > 0 && ctx.config.evictionAllowSingleReplica !== true) {
-    return {
-      success: false,
-      changed: false,
-      message: `${name} holds single-replica volume(s); refusing to evict`,
-      error: `${singles.join("; ")}. Raise them to 3 replicas and wait for healthy, ` +
-        `or pass evictionAllowSingleReplica to accept the risk.`,
-    };
-  }
-
-  const patch = JSON.stringify({
-    spec: { allowScheduling: false, evictionRequested: true },
-  });
-  const req = await kubectl(
-    ctx,
-    `-n longhorn-system patch nodes.longhorn.io "${name}" --type=merge -p '${patch}'`,
-  );
-  if (!req.ok) {
-    return { success: false, changed: false, message: `Could not request eviction for ${name}`, error: req.out };
-  }
-
-  ctx.log(`Eviction requested for ${name}; waiting for replicas to rebuild elsewhere...`);
-  const start = Date.now();
-  let last = -1;
-  let lastHeartbeat = Date.now();
-  try {
-  while (Date.now() - start < REPLICA_REBUILD_TIMEOUT_MS) {
-    const res = await kubectl(
-      ctx,
-      `-n longhorn-system get replicas.longhorn.io -o jsonpath='{range .items[?(@.spec.nodeID=="${name}")]}{.metadata.name}{"\\n"}{end}'`,
-    );
-    const remaining = res.ok ? res.out.split("\n").map((l) => l.trim()).filter(Boolean).length : -1;
-    if (remaining === 0) {
-      return {
-        success: true,
-        changed: true,
-        message: `All replicas evicted from ${name} after ${Math.round((Date.now() - start) / 1000)}s`,
-      };
-    }
-    // Log on change, and otherwise every few minutes. A silent hour is
-    // indistinguishable from a hung process; a heartbeat that carries the
-    // elapsed time makes "stalled" visible while it is happening rather than
-    // only in the timeout message.
-    const elapsedS = Math.round((Date.now() - start) / 1000);
-    if (remaining !== last) {
-      ctx.log(`  ${remaining} replica(s) still on ${name} (${elapsedS}s elapsed)`);
-      last = remaining;
-      lastHeartbeat = Date.now();
-    } else if (Date.now() - lastHeartbeat >= HEARTBEAT_INTERVAL_MS) {
-      ctx.log(`  still waiting: ${remaining} replica(s) on ${name}, no change in ` +
-        `${Math.round((Date.now() - lastHeartbeat) / 60_000)} min (${elapsedS}s elapsed)`);
-      lastHeartbeat = Date.now();
-    }
-    await sleep(POLL_INTERVAL_MS);
-  }
-  } catch (err) {
-    // Never leave the node eviction-pending because of a transient failure.
-    await cancelEviction(ctx, name);
-    throw err;
-  }
-
-  // Timed out. Say exactly what is stuck before restoring the node, because
-  // after the cancel the eviction flags are gone and the evidence with them.
-  const stuck = await describeStuckReplicas(ctx, name);
-
-  // Put the node back the way we found it: an abandoned eviction leaves it
-  // unschedulable with Longhorn still trying to drain it, which is safe but
-  // sticky, and nothing else would ever clear it.
-  const cancelled = await cancelEviction(ctx, name);
-  return {
-    success: false,
-    changed: true,
-    message: `Timed out waiting for replica eviction from ${name}`,
-    error: `${last} replica(s) still present after ${REPLICA_REBUILD_TIMEOUT_MS / 60000} min. ` +
-      `Wiping now would destroy them.\nStill on ${name}:\n${stuck}\n` +
-      (cancelled
-        ? `Eviction request has been cancelled and scheduling restored.`
-        : `WARNING: could not cancel the eviction request — clear it with ` +
-          `\`labctl app k3s admit ${name}\`.`),
-  };
-};
-
-/**
- * Describe the replicas still sitting on a node, and why they are not moving.
- *
- * An eviction that times out after an hour used to report only a count. That
- * is not enough to act on: the question is always WHICH volume is stuck and
- * what state its replica is in. A replica with `desireState=running` and
- * `currentState=stopped` and no `failedAt` is Longhorn wanting to start it and
- * not doing so -- a very different problem from a volume that cannot be placed.
- */
-async function describeStuckReplicas(ctx: OperationContext, name: string): Promise<string> {
-  const reps = await kubectlJson<{ items: {
-    metadata: { name: string };
-    spec: { nodeID?: string; volumeName?: string; failedAt?: string; desireState?: string };
-    status?: { currentState?: string };
-  }[] }>(ctx, "-n longhorn-system get replicas.longhorn.io");
-  if (!reps) return "  (could not read replicas)";
-
-  const mine = reps.items.filter((r) => r.spec.nodeID === name);
-  if (mine.length === 0) return "  (none)";
-
-  const vols = await kubectlJson<{ items: {
-    metadata: { name: string };
-    spec: { numberOfReplicas?: number };
-    status?: { state?: string; robustness?: string };
-  }[] }>(ctx, "-n longhorn-system get volumes.longhorn.io");
-  const byName = new Map((vols?.items ?? []).map((v) => [v.metadata.name, v]));
-
-  return mine.map((r) => {
-    const v = byName.get(r.spec.volumeName ?? "");
-    return `  ${r.spec.volumeName ?? "?"}: replica ${r.metadata.name.slice(-12)} ` +
-      `currentState=${r.status?.currentState ?? "?"} desireState=${r.spec.desireState ?? "?"} ` +
-      `failedAt=${r.spec.failedAt === undefined || r.spec.failedAt === "" ? "(none)" : r.spec.failedAt} | ` +
-      `volume ${v?.status?.state ?? "?"}/${v?.status?.robustness ?? "?"} ` +
-      `wants ${v?.spec.numberOfReplicas ?? "?"} replicas`;
-  }).join("\n");
-}
-
-/**
- * Undo the eviction request. Returns false if it could not be undone, which the
- * caller must surface -- silently failing here leaves a node nobody can use.
- */
-async function cancelEviction(ctx: OperationContext, name: string): Promise<boolean> {
-  const patch = JSON.stringify({ spec: { allowScheduling: true, evictionRequested: false } });
-  const res = await kubectl(
-    ctx,
-    `-n longhorn-system patch nodes.longhorn.io "${name}" --type=merge -p '${patch}'`,
-  );
-  if (res.ok) ctx.log(`Cancelled eviction request on ${name} and restored scheduling`);
-  return res.ok;
-}
-
-/**
- * Verify the node holds nothing before it is wiped.
- *
- * A separate, cheap operation on purpose: it is the gate that turns "we ran the
- * eviction steps" into "the node is actually empty", and it is the one thing
- * worth asserting twice. Reading back from the thing that does the work, not
- * from the request we sent it.
- */
-export const assertNodeEvacuated: Operation = async (ctx): Promise<OperationResult> => {
-  const name = await nodeName(ctx);
-  const problems: string[] = [];
-
-  const sched = await kubectl(ctx, `get node "${name}" -o jsonpath='{.spec.unschedulable}'`);
-  if (sched.ok && sched.out.trim() !== "true") problems.push("node is still schedulable");
-
-  const pods = await kubectl(
-    ctx,
-    `get pods -A --field-selector spec.nodeName="${name}",status.phase=Running ` +
-      `-o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\\n"}{end}'`,
-  );
-  if (pods.ok) {
-    const running = pods.out.split("\n").map((l) => l.trim()).filter(Boolean);
-    // DaemonSet pods legitimately remain; anything else means the drain lied.
-    if (running.length > 0) ctx.log(`  ${running.length} pod(s) remain (expected: DaemonSets only)`);
-  }
-
-  const lh = await kubectl(ctx, "get ns longhorn-system -o name");
-  if (lh.ok) {
-    const reps = await kubectl(
-      ctx,
-      `-n longhorn-system get replicas.longhorn.io -o jsonpath='{range .items[?(@.spec.nodeID=="${name}")]}{.metadata.name}{"\\n"}{end}'`,
-    );
-    const n = reps.ok ? reps.out.split("\n").map((l) => l.trim()).filter(Boolean).length : 0;
-    if (n > 0) problems.push(`${n} Longhorn replica(s) still on this node`);
-  }
-
-  return problems.length === 0
-    ? { success: true, changed: false, message: `${name} is evacuated and safe to wipe` }
-    : {
-        success: false,
-        changed: false,
-        message: `${name} is NOT safe to wipe`,
-        error: problems.join("; "),
-      };
-};

bastion/src/modules/modules/k3s/src/operations/rancher-storage.ts

@@ -1,48 +0,0 @@
-// Grow the labvg/rancher LV (k3s image store / imageFs) to 120G.
-// 2026-08 incident: the original 20G LV sat at 85% used from steady-state
-// images alone, so one ~5G image pull tripped imagefs eviction and evicted
-// unrelated pods. Fresh installs are sized at 120G by the kickstart; this op
-// covers nodes installed before that change and vanilla nodes converted to
-// k8s later. Never removes or shrinks anything — if the VG lacks free space
-// (e.g. a longhorn --grow LV consumed it), it reports and moves on.
-
-import type { Operation, OperationResult } from "../types.js";
-import { sshOpts } from "../utils.js";
-
-const RANCHER_LV = "labvg/rancher";
-const TARGET_MIB = 122880; // 120G
-
-export const growRancherLv: Operation = async (ctx): Promise<OperationResult> => {
-  const lv = await ctx.ssh.exec(
-    `lvs --noheadings --units m --nosuffix -o lv_size ${RANCHER_LV} 2>/dev/null || true`,
-    sshOpts(ctx),
-  );
-  const sizeMib = Number.parseFloat(lv.stdout.trim());
-  if (Number.isNaN(sizeMib)) {
-    return { success: true, changed: false, message: "No labvg/rancher LV — imageFs shares /var, skipping" };
-  }
-  if (sizeMib >= TARGET_MIB) {
-    return { success: true, changed: false, message: `rancher LV already ${Math.round(sizeMib / 1024)}G` };
-  }
-
-  const vg = await ctx.ssh.exec(`vgs --noheadings --units m --nosuffix -o vg_free labvg`, sshOpts(ctx));
-  const freeMib = Number.parseFloat(vg.stdout.trim());
-  const neededMib = TARGET_MIB - sizeMib;
-  if (Number.isNaN(freeMib) || freeMib < neededMib) {
-    return {
-      success: true,
-      changed: false,
-      message: `VG labvg has ${Math.floor((Number.isNaN(freeMib) ? 0 : freeMib) / 1024)}G free — ` +
-        `need ${Math.ceil(neededMib / 1024)}G to grow rancher LV to 120G (manual LV rebuild required)`,
-    };
-  }
-
-  await ctx.ssh.exec(`lvextend -L ${TARGET_MIB}m /dev/${RANCHER_LV}`, sshOpts(ctx));
-  await ctx.ssh.exec(`xfs_growfs /var/lib/rancher`, sshOpts(ctx));
-
-  return {
-    success: true,
-    changed: true,
-    message: `rancher LV grown ${Math.round(sizeMib / 1024)}G → 120G`,
-  };
-};

bastion/src/modules/modules/k3s/src/operations/swap.ts

@@ -1,40 +1,22 @@
-// Enable swap so memory pressure spills to disk instead of OOM-killing.
-// kubelet runs with failSwapOn=false (k3s default); zram stays the fast tier,
-// the labvg-swap LV is the overflow tier. Replaces the old CIS-style
-// disableSwap op — a kernel OOM kill of a node daemon is worse than slow swap.
+// Disable swap (CIS requirement for k3s).
 
 import type { Operation, OperationResult } from "../types.js";
 import { sshOpts } from "../utils.js";
 
-const SWAP_DEV = "/dev/mapper/labvg-swap";
+export const disableSwap: Operation = async (ctx): Promise<OperationResult> => {
+  const check = await ctx.ssh.exec("swapon --show --noheadings", sshOpts(ctx));
+  const active = check.stdout.trim().length > 0;
 
-export const enableSwap: Operation = async (ctx): Promise<OperationResult> => {
-  const lv = await ctx.ssh.exec(`test -b ${SWAP_DEV} && echo yes || echo no`, sshOpts(ctx));
-  if (lv.stdout.trim() !== "yes") {
-    return { success: true, changed: false, message: "No labvg-swap LV — skipping swap enable" };
+  if (active) {
+    await ctx.ssh.exec("swapoff -a", sshOpts(ctx));
   }
 
-  const active = await ctx.ssh.exec(
-    `grep -q "^$(readlink -f ${SWAP_DEV}) " /proc/swaps && echo on || echo off`,
-    sshOpts(ctx),
-  );
-  const wasOff = active.stdout.trim() !== "on";
-
-  if (wasOff) {
-    // Format if the LV was never (or wrongly) initialised, then activate
-    await ctx.ssh.exec(`blkid ${SWAP_DEV} | grep -q 'TYPE="swap"' || mkswap ${SWAP_DEV}`, sshOpts(ctx));
-    await ctx.ssh.exec(`swapon ${SWAP_DEV}`, sshOpts(ctx));
-  }
-
-  // Persist across reboots (idempotent)
-  await ctx.ssh.exec(
-    `grep -q "labvg-swap" /etc/fstab || echo "${SWAP_DEV} none swap defaults 0 0" >> /etc/fstab`,
-    sshOpts(ctx),
-  );
+  // Remove swap entries from fstab permanently
+  await ctx.ssh.exec("sed -i '/\\sswap\\s/d' /etc/fstab", sshOpts(ctx));
 
   return {
     success: true,
-    changed: wasOff,
-    message: wasOff ? "LV swap enabled" : "LV swap already active",
+    changed: active,
+    message: active ? "Swap disabled" : "Swap already disabled",
   };
 };

bastion/src/modules/modules/k3s/src/operations/worker-label.ts

@@ -1,66 +0,0 @@
-// Apply the worker role label from the SERVER side, after the node registers.
-//
-// WHY THIS IS NOT A NODE-LABEL
-// ----------------------------
-// `node-role.kubernetes.io/worker=true` cannot be set by the kubelet itself.
-// The servers enable the NodeRestriction admission plugin, which forbids a
-// kubelet from assigning itself any label in the `node-role.kubernetes.io/`
-// namespace. Putting it in the agent's `node-label:` list makes registration
-// fail, and the resulting message --
-//
-//   Job for k3s-agent.service failed because the service did not take the
-//   steps required by its unit configuration.
-//
-// names neither labels nor admission, so it reads as a networking, token or
-// sysctl problem. It is none of those.
-//
-// The label is cosmetic (it populates the ROLES column in `kubectl get nodes`)
-// so failing to apply it is a warning, never an error.
-
-import type { Operation, OperationContext, OperationResult } from "../types.js";
-import { sshOpts } from "../utils.js";
-import { sshExec as remoteSshExec } from "../../../../src/ssh.js";
-import { isK3sServer } from "@lab/shared";
-
-const WORKER_LABEL = "node-role.kubernetes.io/worker=true";
-
-async function nodeName(ctx: OperationContext): Promise<string> {
-  const r = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
-  return r.stdout.trim();
-}
-
-export const labelWorkerRole: Operation = async (ctx): Promise<OperationResult> => {
-  if (isK3sServer(ctx.config.role)) {
-    return { success: true, changed: false, message: "Server node — k3s sets its own roles" };
-  }
-
-  const name = await nodeName(ctx);
-
-  // An agent has no local kubectl, so this has to run on a server.
-  const local = await ctx.ssh.exec(`k3s kubectl label node "${name}" ${WORKER_LABEL} --overwrite 2>&1`, sshOpts(ctx));
-  if (local.exitCode === 0) {
-    return { success: true, changed: true, message: `Labelled ${name} as worker` };
-  }
-
-  if (ctx.config.k3sServerUrl !== undefined && ctx.config.k3sServerUrl !== "") {
-    try {
-      const serverHost = new URL(ctx.config.k3sServerUrl).hostname;
-      const remote = await remoteSshExec(
-        serverHost, "root",
-        `k3s kubectl label node "${name}" ${WORKER_LABEL} --overwrite`,
-        { ...(ctx.ssh.keyPath ? { keyPath: ctx.ssh.keyPath } : {}), timeoutMs: 30_000 },
-      );
-      if (remote.exitCode === 0) {
-        return { success: true, changed: true, message: `Labelled ${name} as worker (via server)` };
-      }
-    } catch { /* fall through to the warning */ }
-  }
-
-  // Cosmetic: the ROLES column will read <none>. Not worth failing an install.
-  return {
-    success: true,
-    changed: false,
-    message: `Could not apply the worker role label to ${name} (cosmetic; set it with ` +
-      `\`kubectl label node ${name} ${WORKER_LABEL}\`)`,
-  };
-};

bastion/src/modules/modules/k3s/src/types.ts

@@ -16,56 +16,6 @@ export interface K3sConfig {
 
   // Additional TLS SANs for API server certificate
   tlsSans?: string[] | undefined;
-
-  /**
-   * Exact k3s version to install, e.g. "v1.34.6+k3s1".
-   *
-   * Unset means whatever get.k3s.io currently calls latest, which is how a
-   * reinstalled node silently ends up AHEAD of the cluster it is rejoining.
-   * Kubernetes supports only one minor version of skew between control-plane
-   * members; a node reinstalled two minors ahead of its peers is an
-   * unsupported cluster, and a rolling migration reinstalls nodes one at a
-   * time by design. Pin it to the version the cluster already runs.
-   */
-  k3sVersion?: string | undefined;
-
-  /**
-   * Permit evacuating a node that holds single-replica Longhorn volumes.
-   *
-   * Off by default and deliberately awkward. Evicting the only replica of a
-   * volume rebuilds it from the sole copy in existence; with no backup target
-   * configured, a failure during that rebuild is permanent data loss. Raising
-   * the volume to 3 replicas and waiting for it to go healthy is almost always
-   * the right move instead.
-   */
-  evictionAllowSingleReplica?: boolean | undefined;
-
-  /**
-   * IPv6 address of this node on the cluster VLAN. Its PRESENCE is what makes a
-   * node dual-stack: supply it and `node-ip` is emitted as `<v4>,<v6>`; omit it
-   * and the generated config is byte-identical to the IPv4-only one.
-   *
-   * It must be a real address on the node before k3s starts. k3s resolves
-   * node-ip at boot, so a SLAAC or DHCPv6 lease that has not landed yet leaves
-   * the node with no usable v6 identity. The estate takes it from a kea DHCPv6
-   * reservation keyed on MAC, the same way it takes its IPv4 address.
-   */
-  ipv6?: string | undefined;
-
-  /**
-   * Pod and Service ranges, one entry per address family. Passed rather than
-   * hardcoded so the ranges stay configuration -- and so labsim can rehearse
-   * with its own addresses through this same generator, instead of a parallel
-   * set of INSTALL_K3S_EXEC flags that would prove a different mechanism.
-   *
-   * k3s validates the two together and crash-loops on a mismatch (a dual
-   * cluster-cidr with a single-family service-cidr, say), which is a safe
-   * failure but an avoidable one: set both or neither.
-   *
-   * Servers only -- agents take neither.
-   */
-  clusterCidr?: string[] | undefined;
-  serviceCidr?: string[] | undefined;
 }
 
 /** SSH execution interface injected into operations. */

bastion/src/modules/modules/k3s/tests/agent-node-label.test.ts

@@ -1,75 +0,0 @@
-// The agent must NOT try to self-assign a node-role label.
-//
-// A kubelet may not set labels in the `node-role.kubernetes.io/` namespace;
-// the servers run NodeRestriction, which rejects the registration. The agent
-// then never finishes registering and systemd reports only:
-//
-//   Job for k3s-agent.service failed because the service did not take the
-//   steps required by its unit configuration.
-//
-// That message names neither labels nor admission control, so the cost of
-// reintroducing this is hours of looking at the network and the join token.
-// Observed twice for real: on v2-wrk-1, and on the drill's storage agent.
-
-import { describe, it, expect } from "vitest";
-import { generateAgentConfig, generateServerConfig } from "../src/operations/k3s-config.js";
-import { k3sAgentGroup } from "../src/groups/k3s-agent.js";
-import { mockCtx, OK, stdout, expectCommand } from "./helpers.js";
-import { labelWorkerRole } from "../src/operations/worker-label.js";
-
-const base = { hostname: "w1.lab", ip: "10.0.0.9", role: "k8s-worker" as const };
-
-describe("agent config", () => {
-  it("does NOT put a node-role label in node-label", () => {
-    const cfg = generateAgentConfig({ ...base });
-    expect(cfg).not.toContain("node-role.kubernetes.io");
-  });
-
-  it("keeps the Longhorn label, which IS allowed (outside kubernetes.io)", () => {
-    expect(generateAgentConfig({ ...base })).toContain("node.longhorn.io/create-default-disk=config");
-  });
-
-  it("server config does not set node-role either — k3s assigns those itself", () => {
-    const cfg = generateServerConfig({ ...base, role: "k8s-combo", hostname: "s1.lab" });
-    expect(cfg).not.toContain("node-role.kubernetes.io");
-  });
-
-  it("the agent group applies the worker label AFTER installing", () => {
-    const names = k3sAgentGroup.operations.map((o) => o.name);
-    const install = names.findIndex((n) => /Install k3s agent/i.test(n));
-    const label = names.findIndex((n) => /Label node as worker/i.test(n));
-    expect(install).toBeGreaterThanOrEqual(0);
-    expect(label).toBeGreaterThan(install);
-  });
-});
-
-describe("labelWorkerRole", () => {
-  it("labels an agent from the server side", async () => {
-    const ctx = mockCtx({ role: "k8s-worker" });
-    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
-      if (cmd.startsWith("hostname")) return stdout("w1.lab\n");
-      return OK;
-    });
-    const r = await labelWorkerRole(ctx);
-    expect(r.success).toBe(true);
-    expectCommand(ctx.ssh, /label node "w1\.lab" node-role\.kubernetes\.io\/worker=true/);
-  });
-
-  it("is a no-op on a server — k3s sets control-plane/etcd itself", async () => {
-    const r = await labelWorkerRole(mockCtx({ role: "k8s-combo" }));
-    expect(r.success).toBe(true);
-    expect(r.changed).toBe(false);
-  });
-
-  it("never fails an install over a cosmetic label", async () => {
-    const ctx = mockCtx({ role: "k8s-worker" });
-    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
-      if (cmd.startsWith("hostname")) return stdout("w1.lab\n");
-      return { exitCode: 1, stdout: "", stderr: "no kubectl here" };
-    });
-    const r = await labelWorkerRole(ctx);
-    expect(r.success, "a missing ROLES column must not fail a node install").toBe(true);
-    expect(r.changed).toBe(false);
-    expect(r.message).toMatch(/cosmetic/);
-  });
-});

bastion/src/modules/modules/k3s/tests/etcd-member-parse.test.ts

@@ -1,58 +0,0 @@
-// Parsing `etcdctl member list`.
-//
-// WHY NOT -w json
-// ---------------
-// etcd member IDs are uint64. JSON.parse turns them into JS numbers, which
-// lose precision above 2^53, so the id that comes back is an APPROXIMATION of
-// the real one -- and `member remove` wants HEX anyway, while the JSON writer
-// emits decimal. Observed for real: `member remove 13323819137391910000`
-// -> "bad member ID arg (value out of range), expecting ID in Hex".
-//
-// That happens at the worst possible moment: the node is already stopped and
-// about to be wiped. A mangled id fails loudly if you are lucky; in principle
-// it could match a different member, which would remove the wrong one.
-//
-// The default writer prints the id as hex TEXT, so it never becomes a number.
-// The fixture below is real output from a production node.
-
-import { describe, it, expect } from "vitest";
-import { parseEtcdMembers } from "../src/operations/etcd-members.js";
-
-const REAL = `825852d3683bc406, started, worker2-k8s0.ad.itaz.eu-5f7ad0c9, https://192.168.8.25:2380, https://192.168.8.25:2379, false
-a6170c0755307cea, started, worker1-k8s0.ad.itaz.eu-82b4fe24, https://192.168.8.13:2380, https://192.168.8.13:2379, false
-ad1fe97d38022a78, started, worker0-k8s0.ad.itaz.eu-e26aacc5, https://192.168.8.23:2380, https://192.168.8.23:2379, false`;
-
-describe("parseEtcdMembers", () => {
-  it("keeps ids as exact hex text, never as numbers", () => {
-    const m = parseEtcdMembers(REAL);
-    expect(m.map((x) => x.id)).toEqual([
-      "825852d3683bc406", "a6170c0755307cea", "ad1fe97d38022a78",
-    ]);
-    // The precision trap, stated as an assertion: every one of these exceeds
-    // Number.MAX_SAFE_INTEGER, so a numeric round-trip would corrupt them.
-    for (const x of m) {
-      expect(Number(BigInt(`0x${x.id}`)) > Number.MAX_SAFE_INTEGER).toBe(true);
-      expect(BigInt(`0x${x.id}`).toString(16)).toBe(x.id);
-    }
-  });
-
-  it("strips etcd's random name suffix to give the node name", () => {
-    expect(parseEtcdMembers(REAL).map((x) => x.node)).toEqual([
-      "worker2-k8s0.ad.itaz.eu", "worker1-k8s0.ad.itaz.eu", "worker0-k8s0.ad.itaz.eu",
-    ]);
-  });
-
-  it("reports started status", () => {
-    expect(parseEtcdMembers(REAL).every((x) => x.started)).toBe(true);
-    const unstarted = parseEtcdMembers("abc123, unstarted, , https://x:2380, , false");
-    expect(unstarted[0]?.started).toBe(false);
-  });
-
-  it("ignores blank lines and anything without a hex id", () => {
-    expect(parseEtcdMembers(`\n  \nWARNING: something\n${REAL}\n`)).toHaveLength(3);
-  });
-
-  it("returns nothing for empty output rather than a bogus member", () => {
-    expect(parseEtcdMembers("")).toEqual([]);
-  });
-});

bastion/src/modules/modules/k3s/tests/join-env.test.ts

@@ -1,158 +0,0 @@
-// Tests for how K3S_URL / K3S_TOKEN reach the k3s installer.
-//
-// Token and URL are independent, and the distinction is load-bearing:
-//
-//   token only      -> SEEDS the cluster token on the cluster-init node, so
-//                      every later joiner can use the same pre-shared value.
-//   url + token     -> JOINS an existing cluster.
-//   url only        -> cannot authenticate; must not be passed.
-//
-// Before this was fixed the installer emitted join env ONLY when both were
-// present, so a cluster-init node always self-generated its token and there
-// was no way to pre-share one. A `toContain("K3S_TOKEN")` assertion would pass
-// against the broken version for the url+token case, so each case is asserted
-// on the exact command.
-
-import { describe, it, expect } from "vitest";
-import { mockCtx, stdout, OK } from "./helpers.js";
-import { installK3sBinary } from "../src/operations/k3s-install.js";
-
-/** The `curl ... | sh -` line the operation ran. */
-function installCommand(ssh: ReturnType<typeof mockCtx>["ssh"]): string {
-  const calls = ssh.exec.mock.calls.map((c: [string, unknown?]) => c[0]);
-  const found = calls.find((c: string) => c.includes("get.k3s.io"));
-  if (!found) throw new Error(`No install command issued. Commands:\n${calls.join("\n")}`);
-  return found;
-}
-
-/** k3s --version fails (not installed), everything after succeeds. */
-function primeServerInstall(ctx: ReturnType<typeof mockCtx>): void {
-  ctx.ssh.exec.mockResolvedValue(OK);
-  ctx.ssh.exec.mockResolvedValueOnce({ exitCode: 1, stdout: "", stderr: "" }); // k3s --version
-}
-
-describe("installK3sBinary — server join environment", () => {
-  it("token alone seeds the cluster token and does NOT pass a server URL", async () => {
-    const ctx = mockCtx({ role: "k8s-combo", k3sToken: "pre-shared-token" });
-    primeServerInstall(ctx);
-    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
-
-    await installK3sBinary(ctx);
-    const cmd = installCommand(ctx.ssh);
-
-    expect(cmd).toContain('K3S_TOKEN="pre-shared-token"');
-    expect(cmd).not.toContain("K3S_URL");
-    expect(cmd).toContain('INSTALL_K3S_EXEC="server"');
-  });
-
-  it("url + token joins an existing cluster", async () => {
-    const ctx = mockCtx({
-      role: "k8s-combo",
-      k3sServerUrl: "https://10.0.0.11:6443",
-      k3sToken: "pre-shared-token",
-    });
-    primeServerInstall(ctx);
-    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
-
-    await installK3sBinary(ctx);
-    const cmd = installCommand(ctx.ssh);
-
-    expect(cmd).toContain('K3S_URL="https://10.0.0.11:6443"');
-    expect(cmd).toContain('K3S_TOKEN="pre-shared-token"');
-  });
-
-  it("neither url nor token passes no join environment at all", async () => {
-    const ctx = mockCtx({ role: "k8s-combo" });
-    primeServerInstall(ctx);
-    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
-
-    await installK3sBinary(ctx);
-    const cmd = installCommand(ctx.ssh);
-
-    expect(cmd).not.toContain("K3S_URL");
-    expect(cmd).not.toContain("K3S_TOKEN");
-  });
-
-  it("a URL without a token is ignored — it cannot authenticate a join", async () => {
-    const ctx = mockCtx({ role: "k8s-combo", k3sServerUrl: "https://10.0.0.11:6443" });
-    primeServerInstall(ctx);
-    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
-
-    await installK3sBinary(ctx);
-    expect(installCommand(ctx.ssh)).not.toContain("K3S_URL");
-  });
-});
-
-describe("installK3sBinary — datastore wipe guard", () => {
-  // Wiping server/{tls,cred,db} is correct when JOINING (stale certs are fatal:
-  // "newer than datastore") and catastrophic on a cluster-init node, where it
-  // would destroy the etcd database the node is about to bootstrap from.
-  const WIPE = "rm -rf /var/lib/rancher/k3s/server/tls";
-
-  it("wipes stale server state when joining", async () => {
-    const ctx = mockCtx({
-      role: "k8s-combo",
-      k3sServerUrl: "https://10.0.0.11:6443",
-      k3sToken: "t",
-    });
-    primeServerInstall(ctx);
-    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
-
-    await installK3sBinary(ctx);
-    const calls = ctx.ssh.exec.mock.calls.map((c: [string, unknown?]) => c[0]);
-    expect(calls.some((c: string) => c.includes(WIPE))).toBe(true);
-  });
-
-  it("does NOT wipe the datastore when only seeding a token (cluster-init)", async () => {
-    const ctx = mockCtx({ role: "k8s-combo", k3sToken: "pre-shared-token" });
-    primeServerInstall(ctx);
-    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
-
-    await installK3sBinary(ctx);
-    const calls = ctx.ssh.exec.mock.calls.map((c: [string, unknown?]) => c[0]);
-    expect(calls.some((c: string) => c.includes(WIPE))).toBe(false);
-  });
-});
-
-describe("installK3sBinary — agent", () => {
-  it("refuses to install an agent with no server URL or token", async () => {
-    const ctx = mockCtx({ role: "k8s-worker" });
-    primeServerInstall(ctx);
-
-    const result = await installK3sBinary(ctx);
-    expect(result.success).toBe(false);
-    expect(result.message).toContain("Agent requires");
-  });
-});
-
-describe("installK3sBinary — version pinning", () => {
-  // get.k3s.io installs LATEST. During a rolling migration that means a
-  // reinstalled node rejoins on a newer k3s than its peers; Kubernetes supports
-  // only one minor of skew between control-plane members, and at the time of
-  // writing latest was TWO ahead of the fleet (v1.36.4 vs v1.34.6).
-  it("pins the version when one is given", async () => {
-    const ctx = mockCtx({ role: "k8s-combo", k3sToken: "t", k3sVersion: "v1.34.6+k3s1" });
-    primeServerInstall(ctx);
-    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
-    await installK3sBinary(ctx);
-    expect(installCommand(ctx.ssh)).toContain('INSTALL_K3S_VERSION="v1.34.6+k3s1"');
-  });
-
-  it("pins the agent install too", async () => {
-    const ctx = mockCtx({
-      role: "k8s-worker", k3sServerUrl: "https://10.0.0.11:6443",
-      k3sToken: "t", k3sVersion: "v1.34.6+k3s1",
-    });
-    primeServerInstall(ctx);
-    await installK3sBinary(ctx);
-    expect(installCommand(ctx.ssh)).toContain('INSTALL_K3S_VERSION="v1.34.6+k3s1"');
-  });
-
-  it("omits the variable entirely when unset, preserving today's behaviour", async () => {
-    const ctx = mockCtx({ role: "k8s-combo", k3sToken: "t" });
-    primeServerInstall(ctx);
-    ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready"));
-    await installK3sBinary(ctx);
-    expect(installCommand(ctx.ssh)).not.toContain("INSTALL_K3S_VERSION");
-  });
-});

bastion/src/modules/modules/k3s/tests/longhorn-disk.test.ts

@@ -1,188 +0,0 @@
-// Tests for giving a node its Longhorn disk.
-//
-// The cluster runs createDefaultDiskLabeledNodes:true, so a node gets a disk
-// only if it carries BOTH the create-default-disk label and the
-// default-disks-config annotation. Longhorn reads that annotation when it
-// first creates the node's record — which means on a node that is REJOINING,
-// the record already exists and the annotation is never read. The node comes
-// back Ready, accepts pods, and hosts no replicas at all.
-//
-// That failure is invisible until a rebuild is expected and does not arrive,
-// so these tests pin the read-back and the repair, not just the annotate.
-
-import { describe, it, expect, vi, afterEach } from "vitest";
-import { mockCtx, OK, FAIL, stdout, expectCommand, expectNoCommand } from "./helpers.js";
-import { configureLonghornDisk } from "../src/operations/longhorn-disk.js";
-
-const HOST = stdout("node1.lab\n");
-const DISK = "/var/lib/longhorn";
-
-interface Opts {
-  /** Does Longhorn create the disk on its own (i.e. it read the annotation)? */
-  longhornCreatesDisk?: boolean;
-  /** Has longhorn-manager registered this node yet? */
-  recordExists?: boolean;
-  hasDir?: boolean;
-  noLonghorn?: boolean;
-  patchFails?: boolean;
-  /** Does the disk show up after a direct spec.disks patch? */
-  repairWorks?: boolean;
-}
-
-function ctxFor(o: Opts = {}) {
-  // Model the real object rather than a script of reads: a disk exists if
-  // Longhorn made one, or if our repair patch landed and took effect.
-  let patched = false;
-  const ctx = mockCtx();
-  ctx.ssh.exec.mockImplementation(async (cmd: string) => {
-    if (cmd.startsWith("test -d")) return stdout(o.hasDir === false ? "no" : "yes");
-    if (cmd.startsWith("hostname")) return HOST;
-    if (cmd.includes("get ns longhorn-system")) return o.noLonghorn ? FAIL : stdout("namespace/longhorn-system");
-    // Existence probe (-o name) is a DIFFERENT question from "has a disk",
-    // and must be matched before the jsonpath reads.
-    if (cmd.includes("get nodes.longhorn.io") && cmd.includes("-o name")) {
-      return o.recordExists === false ? FAIL : stdout("node.longhorn.io/node1.lab");
-    }
-    if (cmd.includes("get nodes.longhorn.io") && cmd.includes("jsonpath")) {
-      const has = (o.longhornCreatesDisk ?? false) || (patched && (o.repairWorks ?? true));
-      return stdout(has ? DISK : "");
-    }
-    if (cmd.includes("patch nodes.longhorn.io")) {
-      if (o.patchFails === true) return FAIL;
-      patched = true;
-      return OK;
-    }
-    if (cmd.includes("annotate node")) return stdout("node/node1.lab annotated");
-    return OK;
-  });
-  return ctx;
-}
-
-/** Run the operation with the 60s verify window collapsed. */
-async function runFast(ctx: ReturnType<typeof ctxFor>) {
-  vi.useFakeTimers();
-  try {
-    const p = configureLonghornDisk(ctx);
-    // Long enough to cover the registration wait (180s) AND the disk wait (60s).
-    await vi.advanceTimersByTimeAsync(300_000);
-    return await p;
-  } finally {
-    vi.useRealTimers();
-  }
-}
-
-afterEach(() => { vi.useRealTimers(); });
-
-describe("the annotation survives the shell", () => {
-  // The value is JSON, so it is full of double quotes. Wrapping it in double
-  // quotes truncates it at the first inner quote and the node ends up with
-  //     [{path:/var/lib/longhorn,allowScheduling:true}]
-  // which Longhorn cannot parse, so it creates no disk at all -- silently.
-  // Asserting on the command string alone would NOT catch this; the command
-  // looks fine until a shell reads it. So run it through a real shell.
-  it("arrives as valid JSON after shell parsing", async () => {
-    const ctx = ctxFor({ longhornCreatesDisk: true });
-    await runFast(ctx);
-    const cmd = ctx.ssh.exec.mock.calls
-      .map((c: [string, unknown?]) => c[0])
-      .find((c: string) => c.includes("default-disks-config")) ?? "";
-    expect(cmd, "annotate command").not.toBe("");
-
-    // Let a real shell word-split it exactly as sshd would, and recover the
-    // argument that carries the annotation.
-    const { execFileSync } = await import("node:child_process");
-    const printed = execFileSync("bash", ["-c", `printf '%s\n' ${cmd.replace(/^k3s kubectl /, "")}`], {
-      encoding: "utf-8",
-    });
-    const arg = printed.split("\n").find((l) => l.startsWith("node.longhorn.io/default-disks-config=")) ?? "";
-    expect(arg, "annotation argument after shell parsing").not.toBe("");
-
-    const json = arg.slice("node.longhorn.io/default-disks-config=".length);
-    const parsed = JSON.parse(json) as { path: string; allowScheduling: boolean }[];
-    expect(parsed).toEqual([{ path: DISK, allowScheduling: true }]);
-  });
-});
-
-describe("configureLonghornDisk", () => {
-  it("guarantees the replicas directory exists", async () => {
-    // Longhorn creates <path>/replicas only when it first initialises the
-    // disk. If it disappears while longhorn-disk.cfg remains, Longhorn will
-    // not recreate it and every replica placed on the node dies on a
-    // non-recursive mkdir -- permanently, and invisibly, because the node
-    // still reports Ready and Schedulable.
-    const ctx = ctxFor({ longhornCreatesDisk: true });
-    await runFast(ctx);
-    expectCommand(ctx.ssh, /mkdir -p \/var\/lib\/longhorn\/replicas/);
-  });
-
-  it("skips a node with no /var/lib/longhorn", async () => {
-    const r = await configureLonghornDisk(ctxFor({ hasDir: false }));
-    expect(r.success).toBe(true);
-    expect(r.changed).toBe(false);
-    expect(r.message).toMatch(/skipping/);
-  });
-
-  it("annotates and is satisfied when Longhorn creates the disk itself", async () => {
-    const ctx = ctxFor({ longhornCreatesDisk: true });
-    const r = await runFast(ctx);
-    expect(r.success).toBe(true);
-    expectCommand(ctx.ssh, /default-disks-config/);
-    // Longhorn honoured the annotation, so no direct spec.disks write.
-    expectNoCommand(ctx.ssh, /patch nodes\.longhorn\.io/);
-  });
-
-  it("REPAIRS a rejoining node whose annotation was read too late", async () => {
-    // No disk for the whole verify window, then present after the patch.
-    const ctx = ctxFor({ longhornCreatesDisk: false, repairWorks: true });
-    const r = await runFast(ctx);
-    expect(r.success).toBe(true);
-    expect(r.changed).toBe(true);
-    expect(r.message).toMatch(/repaired/i);
-    expectCommand(ctx.ssh, /patch nodes\.longhorn\.io/);
-    expectCommand(ctx.ssh, new RegExp(`"path":"${DISK.replace(/\//g, "\\/")}"`.replace(/\\\//g, "/")));
-  });
-
-  it("FAILS loudly when the disk cannot be created — a node with no disk is useless", async () => {
-    const ctx = ctxFor({ longhornCreatesDisk: false, repairWorks: false });
-    const r = await runFast(ctx);
-    expect(r.success).toBe(false);
-    expect(r.error).toMatch(/will not host replicas|never host a replica/);
-  });
-
-  it("FAILS when the repair patch itself is rejected", async () => {
-    const ctx = ctxFor({ longhornCreatesDisk: false, patchFails: true });
-    const r = await runFast(ctx);
-    expect(r.success).toBe(false);
-    expect(r.message).toMatch(/could not be repaired/);
-  });
-
-  it("does NOT fail a node Longhorn has not registered yet", async () => {
-    // A node that has only just rejoined has no nodes.longhorn.io record. The
-    // annotation is already in place, so Longhorn will read it when it creates
-    // one. Failing here would abort a rejoin over something self-resolving.
-    const ctx = ctxFor({ longhornCreatesDisk: false, recordExists: false });
-    const r = await runFast(ctx);
-    expect(r.success).toBe(true);
-    expect(r.message).toMatch(/not registered/);
-    expectNoCommand(ctx.ssh, /patch nodes\.longhorn\.io/);
-  });
-
-  it("annotates and stops when Longhorn is not installed yet", async () => {
-    // Install-time ordering: the annotation is set so Longhorn honours it when
-    // it does arrive, but there is nothing to read back from.
-    const ctx = ctxFor({ noLonghorn: true });
-    const r = await configureLonghornDisk(ctx);
-    expect(r.success).toBe(true);
-    expectCommand(ctx.ssh, /default-disks-config/);
-    expectNoCommand(ctx.ssh, /patch nodes\.longhorn\.io/);
-  });
-
-  it("uses one fixed disk key, so repeated repairs update rather than multiply", async () => {
-    const ctx = ctxFor({ longhornCreatesDisk: false, repairWorks: true });
-    await runFast(ctx);
-    const patch = ctx.ssh.exec.mock.calls
-      .map((c: [string, unknown?]) => c[0])
-      .find((c: string) => c.includes("patch nodes.longhorn.io")) ?? "";
-    expect(patch).toContain("default-disk-lab");
-  });
-});

bastion/src/modules/modules/k3s/tests/module-config.test.ts

@@ -1,79 +0,0 @@
-// The ModuleContext -> OperationContext config hand-off.
-//
-// toOpContext used to enumerate three keys and silently drop the rest, so a
-// caller could set a field, get no error, and have it ignored. That produced
-// two live bugs: a pinned k3sVersion was discarded (get.k3s.io installed
-// LATEST, two minor versions ahead of the fleet), and
-// `--allow-single-replica` was a no-op that claimed to override a data-loss
-// guard.
-//
-// These tests assert the pass-through per field, and the last one fails for
-// ANY future K3sConfig field that goes missing — which is the property that
-// was actually absent.
-
-import { describe, it, expect, vi } from "vitest";
-import { K3sModule } from "../src/k3s-module.js";
-import type { ModuleContext } from "../../../src/types.js";
-
-vi.mock("../../../src/ssh.js", () => ({
-  sshExec: vi.fn(async (_ip: string, _user: string, cmd: string) => {
-    // Record every command so the test can assert on what was sent.
-    issued.push(cmd);
-    if (cmd.includes("k3s --version")) return { exitCode: 1, stdout: "", stderr: "" };
-    if (cmd.includes("hostname")) return { exitCode: 0, stdout: "n1.lab\n", stderr: "" };
-    if (cmd.includes("get nodes")) return { exitCode: 0, stdout: "NAME\nn1 Ready", stderr: "" };
-    return { exitCode: 0, stdout: "", stderr: "" };
-  }),
-}));
-
-const issued: string[] = [];
-
-function ctxWith(config: Record<string, unknown>): ModuleContext {
-  return {
-    hostname: "n1.lab", ip: "10.0.0.5", role: "k8s-combo",
-    os: "fedora-43", arch: "x86_64", sshUser: "root", config,
-  };
-}
-
-describe("ModuleContext.config reaches the operations", () => {
-  it("passes k3sVersion through to the installer", async () => {
-    issued.length = 0;
-    await new K3sModule().install(ctxWith({
-      k3sToken: "t", k3sVersion: "v1.34.6+k3s1",
-    }));
-    const install = issued.find((c) => c.includes("get.k3s.io")) ?? "";
-    expect(install, "install command").toContain('INSTALL_K3S_VERSION="v1.34.6+k3s1"');
-  });
-
-  it("still passes the three fields it always did", async () => {
-    issued.length = 0;
-    await new K3sModule().install(ctxWith({
-      k3sServerUrl: "https://10.0.0.1:6443", k3sToken: "tok",
-    }));
-    const install = issued.find((c) => c.includes("get.k3s.io")) ?? "";
-    expect(install).toContain('K3S_URL="https://10.0.0.1:6443"');
-    expect(install).toContain('K3S_TOKEN="tok"');
-  });
-
-  it("drops nothing: every key given is visible to the operations", async () => {
-    // The generic guard. Enumerating keys in toOpContext is the bug; this
-    // fails if anyone reintroduces it for a field added later.
-    const { K3sModule: Mod } = await import("../src/k3s-module.js");
-    const mod = new Mod();
-    const given = {
-      k3sServerUrl: "https://10.0.0.1:6443",
-      k3sToken: "tok",
-      k3sVersion: "v1.34.6+k3s1",
-      evictionAllowSingleReplica: true,
-      tlsSans: ["a.lab"],
-    };
-    issued.length = 0;
-    await mod.install(ctxWith(given));
-    // k3sVersion and the join env are the observable ones at install time;
-    // assert them explicitly rather than trusting the shape.
-    const install = issued.find((c) => c.includes("get.k3s.io")) ?? "";
-    expect(install).toContain("INSTALL_K3S_VERSION=");
-    expect(install).toContain("K3S_URL=");
-    expect(install).toContain("K3S_TOKEN=");
-  });
-});

bastion/src/modules/modules/k3s/tests/node-admit.test.ts

@@ -1,211 +0,0 @@
-// Tests for returning an evacuated node to service.
-//
-// Evacuation leaves three separate marks on a node — a kubernetes cordon,
-// Longhorn's allowScheduling/evictionRequested pair, and (after a wipe and
-// rejoin) a MISSING Longhorn disk. Undoing only the cordon gives you a node
-// that is Ready, accepts pods and hosts no replicas at all, which reads as
-// success until storage never rebalances onto it. These tests pin all three.
-
-import { describe, it, expect, vi, afterEach } from "vitest";
-import { mockCtx, OK, FAIL, stdout, expectCommand } from "./helpers.js";
-import { clearLonghornEviction, assertNodeSchedulable } from "../src/operations/node-admit.js";
-import { evictLonghornReplicas } from "../src/operations/node-evacuate.js";
-import { evacuateGroup } from "../src/groups/evacuate.js";
-import { admitGroup } from "../src/groups/admit.js";
-
-const HOST = stdout("node1.lab\n");
-
-/**
- * Run assertNodeSchedulable with its 120s poll collapsed.
- *
- * It polls because Longhorn registers a node asynchronously and this runs
- * moments after a rejoin; the negative cases therefore wait out the whole
- * window before they can honestly fail.
- */
-async function runGate(ctx: ReturnType<typeof mockCtx>) {
-  vi.useFakeTimers();
-  try {
-    const p = assertNodeSchedulable(ctx);
-    await vi.advanceTimersByTimeAsync(150_000);
-    return await p;
-  } finally {
-    vi.useRealTimers();
-  }
-}
-
-afterEach(() => { vi.useRealTimers(); });
-
-/** Command-matching mock; see node-evacuate.test.ts for why not call order. */
-function routed(handlers: Record<string, string | null>, opts: { noLonghorn?: boolean } = {}) {
-  const ctx = mockCtx();
-  ctx.ssh.exec.mockImplementation(async (cmd: string) => {
-    if (cmd.startsWith("hostname")) return HOST;
-    if (cmd.includes("get ns longhorn-system")) return opts.noLonghorn ? FAIL : stdout("namespace/longhorn-system");
-    for (const [needle, out] of Object.entries(handlers)) {
-      if (cmd.includes(needle)) return out === null ? FAIL : stdout(out);
-    }
-    return OK;
-  });
-  return ctx;
-}
-
-describe("clearLonghornEviction", () => {
-  it("clears scheduling and eviction together in one patch", async () => {
-    const ctx = routed({ "get nodes.longhorn.io": "false,true" });
-    const r = await clearLonghornEviction(ctx);
-    expect(r.success).toBe(true);
-    expect(r.changed).toBe(true);
-    // Both fields must move together: Longhorn's webhook rejects
-    // evictionRequested:true on a schedulable node, so the inverse pairing
-    // has to be atomic as well.
-    expectCommand(ctx.ssh, /"allowScheduling":true/);
-    expectCommand(ctx.ssh, /"evictionRequested":false/);
-  });
-
-  it("is idempotent when the node already accepts replicas", async () => {
-    const ctx = routed({ "get nodes.longhorn.io": "true,false" });
-    const r = await clearLonghornEviction(ctx);
-    expect(r.success).toBe(true);
-    expect(r.changed).toBe(false);
-  });
-
-  it("is a no-op when Longhorn is not installed", async () => {
-    const ctx = routed({}, { noLonghorn: true });
-    const r = await clearLonghornEviction(ctx);
-    expect(r.success).toBe(true);
-    expect(r.changed).toBe(false);
-  });
-
-  it("reports failure rather than claiming the node is back", async () => {
-    const ctx = routed({ "get nodes.longhorn.io": "false,true", "patch nodes.longhorn.io": null });
-    const r = await clearLonghornEviction(ctx);
-    expect(r.success).toBe(false);
-  });
-});
-
-describe("assertNodeSchedulable — the gate after a rejoin", () => {
-  it("passes when uncordoned, not evicting, and holding a schedulable disk", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
-      if (cmd.startsWith("hostname")) return HOST;
-      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
-      if (cmd.includes("diskStatus")) return stdout("True");
-      if (cmd.includes("get nodes.longhorn.io")) return stdout("true,false");
-      if (cmd.includes("get node ")) return stdout("");
-      return OK;
-    });
-    const r = await assertNodeSchedulable(ctx);
-    expect(r.success).toBe(true);
-  });
-
-  it("REFUSES a node that has no Longhorn disk — the silent rejoin failure", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
-      if (cmd.startsWith("hostname")) return HOST;
-      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
-      if (cmd.includes("diskStatus")) return stdout("");   // no disks at all
-      if (cmd.includes("get nodes.longhorn.io")) return stdout("true,false");
-      if (cmd.includes("get node ")) return stdout("");
-      return OK;
-    });
-    const r = await runGate(ctx);
-    expect(r.success).toBe(false);
-    expect(r.error).toMatch(/no disk/);
-  });
-
-  it("REFUSES a node still marked for eviction", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
-      if (cmd.startsWith("hostname")) return HOST;
-      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
-      if (cmd.includes("diskStatus")) return stdout("True");
-      if (cmd.includes("get nodes.longhorn.io")) return stdout("false,true");
-      if (cmd.includes("get node ")) return stdout("");
-      return OK;
-    });
-    const r = await runGate(ctx);
-    expect(r.success).toBe(false);
-    expect(r.error).toMatch(/eviction is still requested|scheduling is still disabled/);
-  });
-
-  it("REFUSES a node that is still cordoned", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
-      if (cmd.startsWith("hostname")) return HOST;
-      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
-      if (cmd.includes("diskStatus")) return stdout("True");
-      if (cmd.includes("get nodes.longhorn.io")) return stdout("true,false");
-      if (cmd.includes("get node ")) return stdout("true");   // unschedulable
-      return OK;
-    });
-    const r = await runGate(ctx);
-    expect(r.success).toBe(false);
-    expect(r.error).toMatch(/cordoned/);
-  });
-});
-
-describe("evictLonghornReplicas — never leaves a node eviction-pending", () => {
-  it("cancels the eviction request when the wait blows up", async () => {
-    const ctx = mockCtx();
-    const patched: string[] = [];
-    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
-      if (cmd.startsWith("hostname")) return HOST;
-      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
-      if (cmd.includes("jsonpath")) {
-        // Blow up during the rebuild wait, after eviction was requested.
-        if (cmd.includes("replicas.longhorn.io")) throw new Error("API server went away");
-        return stdout("");
-      }
-      if (cmd.includes("-o json")) return stdout(JSON.stringify({ items: [] }));
-      if (cmd.includes("patch nodes.longhorn.io")) { patched.push(cmd); return OK; }
-      return OK;
-    });
-
-    await expect(evictLonghornReplicas(ctx)).rejects.toThrow("API server went away");
-
-    // Two patches: the request, then the rollback. Leaving only the first
-    // behind would strand the node unschedulable with Longhorn still draining.
-    expect(patched.length).toBe(2);
-    expect(patched[0]).toContain('"evictionRequested":true');
-    expect(patched[1]).toContain('"evictionRequested":false');
-    expect(patched[1]).toContain('"allowScheduling":true');
-  });
-});
-
-describe("group ordering", () => {
-  it("evacuate evicts BEFORE it drains", () => {
-    // drain --force does not override PodDisruptionBudgets, and Longhorn holds
-    // one on the instance-manager pod. Draining first stops the process that
-    // serves the replicas being evicted: the drain blocks for its full timeout
-    // and would remove the only rebuild source for a single-replica volume.
-    const names = evacuateGroup.operations.map((o) => o.name);
-    const evict = names.findIndex((n) => /Evict/i.test(n));
-    const drain = names.findIndex((n) => /Drain/i.test(n));
-    expect(evict).toBeGreaterThanOrEqual(0);
-    expect(drain).toBeGreaterThanOrEqual(0);
-    expect(evict).toBeLessThan(drain);
-  });
-
-  it("evacuate checks feasibility before it mutates anything", () => {
-    const names = evacuateGroup.operations.map((o) => o.name);
-    expect(names.findIndex((n) => /Check eviction/i.test(n)))
-      .toBeLessThan(names.findIndex((n) => /Evict Longhorn/i.test(n)));
-  });
-
-  it("evacuate verifies last", () => {
-    const names = evacuateGroup.operations.map((o) => o.name);
-    expect(names[names.length - 1]).toMatch(/Verify/i);
-  });
-
-  it("admit clears eviction before restoring the disk", () => {
-    // A node still marked for eviction has any disk it is given drained again.
-    const names = admitGroup.operations.map((o) => o.name);
-    expect(names.findIndex((n) => /Clear Longhorn eviction/i.test(n)))
-      .toBeLessThan(names.findIndex((n) => /Restore Longhorn default disk/i.test(n)));
-  });
-
-  it("admit verifies last", () => {
-    const names = admitGroup.operations.map((o) => o.name);
-    expect(names[names.length - 1]).toMatch(/Verify/i);
-  });
-});

bastion/src/modules/modules/k3s/tests/node-evacuate.test.ts

@@ -1,413 +0,0 @@
-import { describe, it, expect, vi } from "vitest";
-import { mockCtx, OK, FAIL, stdout, expectCommand, expectNoCommand } from "./helpers.js";
-import {
-  cordonNode, uncordonNode, drainNode, evictLonghornReplicas, assertNodeEvacuated,
-  assertEvictionCompletable, checkEvictionCompletable,
-} from "../src/operations/node-evacuate.js";
-import type { K3sConfig } from "../src/types.js";
-
-const HOST = stdout("node1.lab\n");
-const GIB = 2 ** 30;
-
-interface RoutedOpts {
-  nodes?: string[];
-  replicas?: { volume: string }[];
-  volumes?: { name: string; replicas: number; size: number }[];
-  softAntiAffinity?: boolean;
-  overProvisionPct?: number;
-  diskFreeBytes?: number;
-  diskMaxBytes?: number;
-  patchFails?: boolean;
-}
-
-/**
- * A context whose SSH mock answers by MATCHING the command rather than by call
- * order. The operations under test read several API objects before they do
- * anything, and an ordered mock has to be re-counted every time one is added —
- * which makes the test fail for reasons that have nothing to do with the
- * behaviour it is meant to pin.
- */
-function routed(o: RoutedOpts = {}, config?: Partial<K3sConfig>) {
-  const nodeNames = o.nodes ?? ["node1.lab", "node2.lab", "node3.lab", "node4.lab"];
-  const free = o.diskFreeBytes ?? 500 * GIB;
-  const max = o.diskMaxBytes ?? 700 * GIB;
-  const lhNodes = {
-    items: nodeNames.map((name) => ({
-      metadata: { name },
-      spec: { allowScheduling: true },
-      status: { diskStatus: { "default-disk": {
-        storageAvailable: free, storageMaximum: max, storageScheduled: 0, storageReserved: 0,
-      } } },
-    })),
-  };
-  const replicas = {
-    items: (o.replicas ?? []).map((r, i) => ({
-      metadata: { name: `replica-${i}` },
-      spec: { nodeID: "node1.lab", volumeName: r.volume },
-    })),
-  };
-  const volumes = {
-    items: (o.volumes ?? []).map((v) => ({
-      metadata: { name: v.name },
-      spec: { numberOfReplicas: v.replicas, size: v.size },
-      status: { state: "attached", robustness: "healthy" },
-    })),
-  };
-
-  const ctx = mockCtx(config);
-  ctx.ssh.exec.mockImplementation(async (cmd: string) => {
-    if (cmd.startsWith("hostname")) return HOST;
-    if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
-    // NOTE: "-o jsonpath=" CONTAINS "-o json", so jsonpath must be matched
-    // first or the eviction poll below is handed a whole JSON document and
-    // never sees zero replicas. (This mock got that wrong once already.)
-    if (cmd.includes("jsonpath")) {
-      // The eviction wait polls replicas on the node — report none remaining.
-      if (cmd.includes("replicas.longhorn.io")) return stdout("");
-      if (cmd.includes("replica-soft-anti-affinity")) return stdout(String(o.softAntiAffinity ?? false));
-      if (cmd.includes("storage-over-provisioning-percentage")) return stdout(String(o.overProvisionPct ?? 100));
-      return stdout("");
-    }
-    if (cmd.includes("get nodes.longhorn.io") && cmd.includes("-o json")) return stdout(JSON.stringify(lhNodes));
-    if (cmd.includes("get replicas.longhorn.io") && cmd.includes("-o json")) return stdout(JSON.stringify(replicas));
-    if (cmd.includes("get volumes.longhorn.io") && cmd.includes("-o json")) return stdout(JSON.stringify(volumes));
-    if (cmd.includes("patch nodes.longhorn.io")) return o.patchFails ? FAIL : OK;
-    return OK;
-  });
-  return ctx;
-}
-
-describe("cordonNode", () => {
-  it("cordons and reports a change", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(stdout('node/node1.lab cordoned'));
-    const r = await cordonNode(ctx);
-    expect(r.success).toBe(true);
-    expect(r.changed).toBe(true);
-    expectCommand(ctx.ssh, /cordon "node1\.lab"/);
-  });
-
-  it("is idempotent — already cordoned is changed:false", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(stdout("node/node1.lab already cordoned"));
-    const r = await cordonNode(ctx);
-    expect(r.success).toBe(true);
-    expect(r.changed).toBe(false);
-  });
-
-  it("fails loudly when the API is unreachable", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(FAIL);
-    const r = await cordonNode(ctx);
-    expect(r.success).toBe(false);
-  });
-});
-
-describe("drainNode", () => {
-  it("ignores daemonsets and deletes emptyDir, or it hangs forever", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(OK);
-    const r = await drainNode(ctx);
-    expect(r.success).toBe(true);
-    expectCommand(ctx.ssh, /--ignore-daemonsets/);
-    expectCommand(ctx.ssh, /--delete-emptydir-data/);
-  });
-
-  it("gives the SSH transport longer than kubectl's own --timeout", async () => {
-    // drainNode asks kubectl for --timeout=15m. sshOpts() is a flat 30s, so
-    // without an explicit override the transport kills a drain that is working
-    // -- after it has already evicted most of the pods. The result is
-    // non-zero with a transcript that reads like success, on every node with
-    // real workloads. `labctl app k3s evacuate` could never have drained one.
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(OK);
-    await drainNode(ctx);
-    const call = ctx.ssh.exec.mock.calls.find((c: [string, { timeoutMs?: number }?]) =>
-      c[0].includes("drain "));
-    expect(call, "drain command").toBeDefined();
-    const timeoutMs = call?.[1]?.timeoutMs ?? 0;
-    expect(timeoutMs, "SSH timeout must exceed kubectl's 15m --timeout")
-      .toBeGreaterThan(15 * 60_000);
-  });
-
-  it("reports the END of the drain transcript, not the useless first line", async () => {
-    // Every drain starts with "node/x already cordoned". Surfacing the whole
-    // transcript means the error you see is that line, which explains nothing.
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce({
-      exitCode: 1,
-      stdout: "node/n1 already cordoned\nevicting pod a\nevicting pod b\n" +
-        "error: cannot evict pod as it would violate the pod's disruption budget",
-      stderr: "",
-    });
-    const r = await drainNode(ctx);
-    expect(r.success).toBe(false);
-    expect(r.error).toContain("disruption budget");
-  });
-
-  it("reports failure rather than pretending the node is empty", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(FAIL);
-    const r = await drainNode(ctx);
-    expect(r.success).toBe(false);
-  });
-});
-
-describe("evictLonghornReplicas", () => {
-  it("is a no-op when Longhorn is not installed", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(FAIL); // no longhorn-system ns
-    const r = await evictLonghornReplicas(ctx);
-    expect(r.success).toBe(true);
-    expect(r.changed).toBe(false);
-    expect(r.message).toMatch(/not installed/);
-  });
-
-  it("requests eviction and waits until zero replicas remain", async () => {
-    // Node holds no replicas, so the pre-check clears it immediately.
-    const ctx = routed({ replicas: [] });
-    const r = await evictLonghornReplicas(ctx);
-    expect(r.success).toBe(true);
-    expect(r.changed).toBe(true);
-    expectCommand(ctx.ssh, /evictionRequested/);
-    expectCommand(ctx.ssh, /allowScheduling/);
-  });
-
-  it("fails when eviction cannot even be requested", async () => {
-    const ctx = routed({ replicas: [], patchFails: true });
-    const r = await evictLonghornReplicas(ctx);
-    expect(r.success).toBe(false);
-  });
-
-  it("does not request eviction at all when it could never complete", async () => {
-    // 3-replica volume, and only 2 other schedulable nodes: the survivors
-    // already hold a copy each and strict anti-affinity forbids a third.
-    const ctx = routed({
-      nodes: ["node1.lab", "node2.lab", "node3.lab"],
-      replicas: [{ volume: "pvc-a" }],
-      volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }],
-    });
-    const r = await evictLonghornReplicas(ctx);
-    expect(r.success).toBe(false);
-    expect(r.error).toMatch(/cannot be placed|wants 3 replicas/);
-    // The important half: nothing was mutated.
-    expectNoCommand(ctx.ssh, /evictionRequested/);
-  });
-
-  it("refuses a single-replica volume unless explicitly allowed", async () => {
-    const opts = {
-      nodes: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"],
-      replicas: [{ volume: "pvc-solo" }],
-      volumes: [{ name: "pvc-solo", replicas: 1, size: 1 * GIB }],
-    };
-    const r = await evictLonghornReplicas(routed(opts));
-    expect(r.success).toBe(false);
-    expect(r.error).toMatch(/single replica/);
-
-    const allowed = await evictLonghornReplicas(
-      routed(opts, { evictionAllowSingleReplica: true }),
-    );
-    expect(allowed.success).toBe(true);
-  });
-});
-
-describe("checkEvictionCompletable — refuse in seconds, not in an hour", () => {
-  it("passes when a spare schedulable node exists", async () => {
-    const ctx = routed({
-      nodes: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"],
-      replicas: [{ volume: "pvc-a" }],
-      volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }],
-    });
-    const f = await checkEvictionCompletable(ctx, "node1.lab");
-    expect(f.completable).toBe(true);
-    expect(f.replicasMoving).toBe(1);
-  });
-
-  it("blocks when replica count equals the surviving node count", async () => {
-    const ctx = routed({
-      nodes: ["node1.lab", "node2.lab", "node3.lab"],
-      replicas: [{ volume: "pvc-a" }],
-      volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }],
-    });
-    const f = await checkEvictionCompletable(ctx, "node1.lab");
-    expect(f.completable).toBe(false);
-    expect(f.blockers.join(" ")).toContain("pvc-a");
-  });
-
-  it("allows it when soft anti-affinity is on — placement stops being the limit", async () => {
-    const ctx = routed({
-      nodes: ["node1.lab", "node2.lab", "node3.lab"],
-      replicas: [{ volume: "pvc-a" }],
-      volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }],
-      softAntiAffinity: true,
-    });
-    const f = await checkEvictionCompletable(ctx, "node1.lab");
-    expect(f.completable).toBe(true);
-  });
-
-  it("catches a small disk hiding behind over-provisioning", async () => {
-    // The scheduler is happy at 500%; the disk has 49 GiB and the volume is 92.
-    const ctx = routed({
-      nodes: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"],
-      replicas: [{ volume: "pvc-big" }],
-      volumes: [{ name: "pvc-big", replicas: 3, size: 92 * GIB }],
-      overProvisionPct: 500,
-      diskFreeBytes: 49 * GIB,
-      diskMaxBytes: 100 * GIB,
-    });
-    const f = await checkEvictionCompletable(ctx, "node1.lab");
-    expect(f.completable).toBe(false);
-    expect(f.blockers.join(" ")).toMatch(/actually free/);
-    expect(f.blockers.join(" ")).toMatch(/over-provisioning-percentage=500/);
-  });
-
-  it("warns, but does not block, on a single-replica volume", async () => {
-    const ctx = routed({
-      nodes: ["node1.lab", "node2.lab", "node3.lab"],
-      replicas: [{ volume: "pvc-solo" }],
-      volumes: [{ name: "pvc-solo", replicas: 1, size: 1 * GIB }],
-    });
-    const f = await checkEvictionCompletable(ctx, "node1.lab");
-    expect(f.completable).toBe(true);
-    expect(f.warnings.join(" ")).toMatch(/single replica/);
-  });
-
-  it("is trivially completable when the node holds nothing", async () => {
-    const f = await checkEvictionCompletable(routed({ replicas: [] }), "node1.lab");
-    expect(f.completable).toBe(true);
-    expect(f.replicasMoving).toBe(0);
-  });
-
-  it("refuses to guess when Longhorn cannot be read", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockResolvedValue(FAIL);
-    const f = await checkEvictionCompletable(ctx, "node1.lab");
-    expect(f.completable).toBe(false);
-    expect(f.blockers.join(" ")).toMatch(/refusing to guess/);
-  });
-});
-
-describe("assertNodeEvacuated — the gate before a wipe", () => {
-  it("passes only when cordoned with no replicas left", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec
-      .mockResolvedValueOnce(HOST)
-      .mockResolvedValueOnce(stdout("true"))   // unschedulable
-      .mockResolvedValueOnce(stdout(""))       // no running pods
-      .mockResolvedValueOnce(stdout("namespace/longhorn-system"))
-      .mockResolvedValueOnce(stdout(""));      // no replicas
-    const r = await assertNodeEvacuated(ctx);
-    expect(r.success).toBe(true);
-    expect(r.message).toMatch(/safe to wipe/);
-  });
-
-  it("REFUSES when replicas are still on the node", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec
-      .mockResolvedValueOnce(HOST)
-      .mockResolvedValueOnce(stdout("true"))
-      .mockResolvedValueOnce(stdout(""))
-      .mockResolvedValueOnce(stdout("namespace/longhorn-system"))
-      .mockResolvedValueOnce(stdout("replica-abc\nreplica-def"));
-    const r = await assertNodeEvacuated(ctx);
-    expect(r.success).toBe(false);
-    expect(r.error).toMatch(/2 Longhorn replica/);
-  });
-
-  it("REFUSES when the node is still schedulable", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec
-      .mockResolvedValueOnce(HOST)
-      .mockResolvedValueOnce(stdout(""))       // not unschedulable
-      .mockResolvedValueOnce(stdout(""))
-      .mockResolvedValueOnce(FAIL);            // no longhorn
-    const r = await assertNodeEvacuated(ctx);
-    expect(r.success).toBe(false);
-    expect(r.error).toMatch(/still schedulable/);
-  });
-});
-
-describe("uncordonNode", () => {
-  it("returns the node to service", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(OK);
-    const r = await uncordonNode(ctx);
-    expect(r.success).toBe(true);
-    expectCommand(ctx.ssh, /uncordon "node1\.lab"/);
-  });
-});
-
-describe("eviction timeout diagnostics", () => {
-  // An eviction can legitimately run for an hour. When it times out, a bare
-  // count ("2 replica(s) still present") is not actionable -- the question is
-  // always WHICH volume is stuck and in what state. A replica with
-  // desireState=running, currentState=stopped and no failedAt is Longhorn
-  // wanting to start it and not doing so, which is a completely different
-  // problem from a volume that cannot be placed. Observed for real on a
-  // freshly wiped-and-rejoined node.
-  it("names the stuck volumes and their replica states, then restores the node", async () => {
-    const ctx = mockCtx();
-    const patches: string[] = [];
-    ctx.ssh.exec.mockImplementation(async (cmd: string) => {
-      if (cmd.startsWith("hostname")) return HOST;
-      if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system");
-      if (cmd.includes("patch nodes.longhorn.io")) { patches.push(cmd); return OK; }
-      if (cmd.includes("jsonpath")) {
-        // The poll never reaches zero: one replica remains for the whole run.
-        if (cmd.includes("replicas.longhorn.io")) return stdout("replica-stuck");
-        if (cmd.includes("replica-soft-anti-affinity")) return stdout("false");
-        if (cmd.includes("storage-over-provisioning-percentage")) return stdout("100");
-        return stdout("");
-      }
-      if (cmd.includes("get replicas.longhorn.io") && cmd.includes("-o json")) {
-        return stdout(JSON.stringify({ items: [{
-          metadata: { name: "pvc-abc-r-deadbeef99" },
-          spec: { nodeID: "node1.lab", volumeName: "pvc-abc", failedAt: "", desireState: "running" },
-          status: { currentState: "stopped" },
-        }] }));
-      }
-      if (cmd.includes("get volumes.longhorn.io") && cmd.includes("-o json")) {
-        return stdout(JSON.stringify({ items: [{
-          metadata: { name: "pvc-abc" },
-          spec: { numberOfReplicas: 3, size: 2 * 2 ** 30 },
-          status: { state: "attached", robustness: "degraded" },
-        }] }));
-      }
-      if (cmd.includes("get nodes.longhorn.io") && cmd.includes("-o json")) {
-        return stdout(JSON.stringify({ items: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"].map((n) => ({
-          metadata: { name: n },
-          spec: { allowScheduling: true },
-          status: { diskStatus: { d: {
-            storageAvailable: 500 * 2 ** 30, storageMaximum: 700 * 2 ** 30,
-            storageScheduled: 0, storageReserved: 0,
-          } } },
-        })) }));
-      }
-      return OK;
-    });
-
-    vi.useFakeTimers();
-    let result;
-    try {
-      const p = evictLonghornReplicas(ctx);
-      await vi.advanceTimersByTimeAsync(61 * 60_000);
-      result = await p;
-    } finally {
-      vi.useRealTimers();
-    }
-
-    expect(result.success).toBe(false);
-    // The actionable part: which volume, and what state its replica is in.
-    expect(result.error).toContain("pvc-abc");
-    expect(result.error).toContain("currentState=stopped");
-    expect(result.error).toContain("desireState=running");
-    expect(result.error).toContain("failedAt=(none)");
-    expect(result.error).toContain("attached/degraded");
-
-    // And the node must be handed back usable, not left eviction-pending.
-    expect(result.error).toMatch(/cancelled and scheduling restored/);
-    expect(patches).toHaveLength(2);
-    expect(patches[1]).toContain('"evictionRequested":false');
-    expect(patches[1]).toContain('"allowScheduling":true');
-  });
-});

bastion/src/modules/modules/k3s/tests/operations.test.ts

@@ -72,97 +72,31 @@ describe("applyCisHardening", () => {
 
 // --- Swap ---
 
-import { enableSwap } from "../src/operations/swap.js";
+import { disableSwap } from "../src/operations/swap.js";
 
-describe("enableSwap", () => {
-  it("activates LV swap when present but off", async () => {
+describe("disableSwap", () => {
+  it("disables active swap", async () => {
     const ctx = mockCtx();
     ctx.ssh.exec
-      .mockResolvedValueOnce(stdout("yes"))  // LV exists
-      .mockResolvedValueOnce(stdout("off"))  // not in /proc/swaps
-      .mockResolvedValueOnce(OK)             // blkid || mkswap
-      .mockResolvedValueOnce(OK)             // swapon
-      .mockResolvedValueOnce(OK);            // fstab entry
+      .mockResolvedValueOnce(stdout("/dev/sda2 partition 2G"))  // swap active
+      .mockResolvedValueOnce(OK)   // swapoff
+      .mockResolvedValueOnce(OK);  // sed fstab
 
-    const result = await enableSwap(ctx);
+    const result = await disableSwap(ctx);
     expect(result.success).toBe(true);
     expect(result.changed).toBe(true);
-    expectCommand(ctx.ssh, "swapon /dev/mapper/labvg-swap");
+    expectCommand(ctx.ssh, "swapoff -a");
   });
 
-  it("is idempotent when LV swap already active", async () => {
+  it("is idempotent when swap already off", async () => {
     const ctx = mockCtx();
     ctx.ssh.exec
-      .mockResolvedValueOnce(stdout("yes"))  // LV exists
-      .mockResolvedValueOnce(stdout("on"))   // already in /proc/swaps
-      .mockResolvedValueOnce(OK);            // fstab entry (always ensured)
+      .mockResolvedValueOnce(stdout(""))  // no swap
+      .mockResolvedValueOnce(OK);          // sed fstab (always runs)
 
-    const result = await enableSwap(ctx);
+    const result = await disableSwap(ctx);
     expect(result.changed).toBe(false);
-    expectNoCommand(ctx.ssh, "swapon /dev/mapper/labvg-swap");
-  });
-
-  it("skips when no labvg-swap LV exists", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockResolvedValueOnce(stdout("no")); // LV missing
-
-    const result = await enableSwap(ctx);
-    expect(result.success).toBe(true);
-    expect(result.changed).toBe(false);
-    expectNoCommand(ctx.ssh, "swapon");
-  });
-});
-
-// --- Rancher LV (imageFs sizing) ---
-
-import { growRancherLv } from "../src/operations/rancher-storage.js";
-
-describe("growRancherLv", () => {
-  it("grows a 20G LV to 120G when the VG has space", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec
-      .mockResolvedValueOnce(stdout("  20480.00"))   // lv_size
-      .mockResolvedValueOnce(stdout("  747807.00"))  // vg_free
-      .mockResolvedValueOnce(OK)                     // lvextend
-      .mockResolvedValueOnce(OK);                    // xfs_growfs
-
-    const result = await growRancherLv(ctx);
-    expect(result.success).toBe(true);
-    expect(result.changed).toBe(true);
-    expectCommand(ctx.ssh, "lvextend -L 122880m /dev/labvg/rancher");
-    expectCommand(ctx.ssh, "xfs_growfs /var/lib/rancher");
-  });
-
-  it("is idempotent when the LV is already 120G", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockResolvedValueOnce(stdout("  122880.00")); // lv_size
-
-    const result = await growRancherLv(ctx);
-    expect(result.changed).toBe(false);
-    expectNoCommand(ctx.ssh, "lvextend");
-  });
-
-  it("reports without failing when the VG has no free space", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec
-      .mockResolvedValueOnce(stdout("  20480.00"))  // lv_size
-      .mockResolvedValueOnce(stdout("  0.00"));     // vg_free
-
-    const result = await growRancherLv(ctx);
-    expect(result.success).toBe(true);
-    expect(result.changed).toBe(false);
-    expect(result.message).toContain("free");
-    expectNoCommand(ctx.ssh, "lvextend");
-  });
-
-  it("skips when there is no rancher LV", async () => {
-    const ctx = mockCtx();
-    ctx.ssh.exec.mockResolvedValueOnce(stdout("")); // lvs empty
-
-    const result = await growRancherLv(ctx);
-    expect(result.success).toBe(true);
-    expect(result.changed).toBe(false);
-    expectNoCommand(ctx.ssh, "lvextend");
+    expectNoCommand(ctx.ssh, "swapoff");
   });
 });
 
@@ -268,108 +202,6 @@ describe("writeK3sConfig", () => {
     expect(writeCall).toContain("protect-kernel-defaults: true");
     expect(writeCall).not.toContain("secrets-encryption");
   });
-
-  // --- dual-stack ---
-  //
-  // The property that matters most is the NEGATIVE one: with no ipv6 and no
-  // CIDRs the output must be byte-identical to what the five existing nodes
-  // already have on disk. If it is not, rolling this out rewrites every node's
-  // config.yaml and restarts a healthy cluster to tell it what it already knew.
-
-  // With an ipv6 configured there is an extra ssh round trip up front -- the
-  // probe that checks the address is really on the node -- so the write lands
-  // one call later.
-  const writtenBy = async (config: Parameters<typeof mockCtx>[0]) => {
-    const ctx = mockCtx(config);
-    const hasV6 = !!(config as { ipv6?: string }).ipv6;
-    if (hasV6) ctx.ssh.exec.mockResolvedValueOnce(stdout("present"));
-    ctx.ssh.exec
-      .mockResolvedValueOnce(OK)
-      .mockResolvedValueOnce(stdout("__LABCTL_NOT_FOUND__"))
-      .mockResolvedValueOnce(OK);
-    await writeK3sConfig(ctx);
-    return ctx.ssh.exec.mock.calls[hasV6 ? 3 : 2]![0] as string;
-  };
-
-  it("emits no address-family lines at all when single-stack", async () => {
-    const server = await writtenBy({ hostname: "n1.lab", ip: "10.0.1.1", role: "infra" });
-    expect(server).not.toContain("node-ip");
-    expect(server).not.toContain("cluster-cidr");
-    expect(server).not.toContain("service-cidr");
-
-    const agent = await writtenBy({ role: "worker" });
-    expect(agent).not.toContain("node-ip");
-    // Nothing inserted ahead of, or between, the lines the agent config has
-    // always opened with.
-    expect(agent).toContain("protect-kernel-defaults: true\nnode-label:");
-  });
-
-  it("emits dual node-ip and CIDRs on a server, IPv4 first", async () => {
-    const out = await writtenBy({
-      hostname: "n1.lab",
-      ip: "192.168.8.23",
-      role: "infra",
-      ipv6: "2001:470:187e:2::23",
-      clusterCidr: ["10.42.0.0/16", "2001:470:187e:1000::/56"],
-      serviceCidr: ["10.43.0.0/16", "2001:470:187e:1fff::/112"],
-    });
-    expect(out).toContain('node-ip: "192.168.8.23,2001:470:187e:2::23"');
-    expect(out).toContain('cluster-cidr: "10.42.0.0/16,2001:470:187e:1000::/56"');
-    expect(out).toContain('service-cidr: "10.43.0.0/16,2001:470:187e:1fff::/112"');
-    // IPv4 must stay the primary family -- that is the supported conversion
-    // path and what lets existing Services keep their ClusterIP.
-    expect(out.indexOf("10.42.0.0/16")).toBeLessThan(out.indexOf("2001:470:187e:1000::/56"));
-    // still a valid server config
-    expect(out).toContain("cluster-init: true");
-    expect(out).toContain("secrets-encryption: true");
-    // the v6 address must be a TLS SAN, or a peer joining over v6 fails
-    // verification with an error that names the cert, not the missing SAN
-    expect(out).toContain('  - "2001:470:187e:2::23"');
-  });
-
-  it("gives an AGENT its own dual node-ip but no CIDRs", async () => {
-    const out = await writtenBy({
-      ip: "192.168.8.12",
-      role: "worker",
-      ipv6: "2001:470:187e:2::12",
-      // deliberately supplied: an agent must ignore them
-      clusterCidr: ["10.42.0.0/16", "2001:470:187e:1000::/56"],
-      serviceCidr: ["10.43.0.0/16", "2001:470:187e:1fff::/112"],
-    });
-    expect(out).toContain('node-ip: "192.168.8.12,2001:470:187e:2::12"');
-    expect(out).not.toContain("cluster-cidr");
-    expect(out).not.toContain("service-cidr");
-  });
-
-  it("refuses to write a config naming an IPv6 the node does not have", async () => {
-    // The SSH-onboard case: Asahi and the DGX Sparks run an OS labctl never
-    // installed, so nothing upstream guarantees a DHCPv6 lease was taken.
-    // Writing the config anyway defers the failure to k3s start-up, where it
-    // reads as a bind error rather than a missing address.
-    const ctx = mockCtx({ ip: "192.168.8.12", role: "worker", ipv6: "2001:470:187e:2::12" });
-    ctx.ssh.exec.mockResolvedValueOnce(stdout("")); // probe: address absent
-
-    const result = await writeK3sConfig(ctx);
-    expect(result.success).toBe(false);
-    expect(result.changed).toBe(false);
-    expect(result.error).toContain("2001:470:187e:2::12");
-    // and it must not have written anything
-    expect(ctx.ssh.exec).toHaveBeenCalledTimes(1);
-  });
-
-  it("does not write node-ip for a v4-only node even when CIDRs are given", async () => {
-    // Guards the flag-day risk: supplying ranges alone must not start rewriting
-    // node identity on nodes that have no IPv6 yet.
-    const out = await writtenBy({
-      hostname: "n1.lab",
-      ip: "10.0.1.1",
-      role: "infra",
-      clusterCidr: ["10.42.0.0/16"],
-      serviceCidr: ["10.43.0.0/16"],
-    });
-    expect(out).toContain('cluster-cidr: "10.42.0.0/16"');
-    expect(out).not.toContain("node-ip");
-  });
 });
 
 // --- CNI Cleanup ---

bastion/src/modules/modules/k3s/tests/smoke.test.ts

@@ -16,7 +16,7 @@ describe("smoke: full server install pipeline", () => {
     const pipeline: NamedOperation[] = [
       { name: "Kernel modules", fn: ops.loadKernelModules },
       { name: "Sysctl hardening", fn: ops.applyCisHardening },
-      { name: "Enable swap", fn: ops.enableSwap },
+      { name: "Disable swap", fn: ops.disableSwap },
       { name: "Disable firewall", fn: ops.disableFirewall },
       { name: "SELinux permissive", fn: ops.setSelinuxPermissive },
       { name: "Write k3s config", fn: ops.writeK3sConfig },
@@ -73,7 +73,7 @@ describe("smoke: pipeline stops on failure", () => {
     };
 
     const results = await runSequential(ctx, [
-      { name: "OK op", fn: ops.enableSwap },
+      { name: "OK op", fn: ops.disableSwap },
       { name: "Failing op", fn: failingOp },
       { name: "Never called", fn: neverCalled },
     ]);
@@ -98,12 +98,11 @@ describe("smoke: agent install rejects missing config", () => {
 });
 
 describe("smoke: all operations are exported", () => {
-  it("exports all 16 operations", () => {
+  it("exports all 15 operations", () => {
     const exported = [
       ops.loadKernelModules,
       ops.applyCisHardening,
-      ops.enableSwap,
-      ops.growRancherLv,
+      ops.disableSwap,
       ops.disableFirewall,
       ops.setSelinuxPermissive,
       ops.writeK3sConfig,
@@ -118,7 +117,7 @@ describe("smoke: all operations are exported", () => {
       ops.checkCertExpiry,
     ];
 
-    expect(exported).toHaveLength(16);
+    expect(exported).toHaveLength(15);
     for (const op of exported) {
       expect(typeof op).toBe("function");
     }

bastion/src/modules/src/types.ts

@@ -1,4 +1,4 @@
-import type { OsId, Arch, Role } from "@lab/shared";
+import type { OsId, Arch } from "@lab/shared";
 
 export interface ModuleMetadata {
   name: string;
@@ -14,10 +14,7 @@ export interface ModuleMetadata {
 export interface ModuleContext {
   hostname: string;
   ip: string;
-  // Role, not string. A literal `string` let `role === "infra"` comparisons
-  // compile while silently missing every other server role -- which is how
-  // labcontroller ended up installing as an agent.
-  role: Role;
+  role: string;
   os: OsId;
   arch: Arch;
   sshUser: string;
@@ -27,7 +24,7 @@ export interface ModuleContext {
 
 export interface ModuleResult {
   success: boolean;
-  phase: "install" | "configure" | "health" | "evacuate" | "admit";
+  phase: "install" | "configure" | "health";
   duration: number;
   output: string[];
   errors: string[];

bastion/src/shared/src/hardware/index.ts

@@ -0,0 +1,154 @@
+// Architecture normalisation and machine classification.
+//
+// Both are derived from what the system already observes about a machine -- never from
+// an operator-supplied flag.
+
+import type { Arch, HardwareInfo, OnboardMethod, OsId } from "../types/index.js";
+
+export const SUPPORTED_ARCHES: readonly Arch[] = ["x86_64", "aarch64"] as const;
+
+/**
+ * Normalise an architecture string to one we serve boot artifacts for.
+ *
+ * Sources and their spellings:
+ *   uname -m         -> "x86_64" / "aarch64"
+ *   iPXE ${buildarch}-> "x86_64" / "arm64"
+ *   dpkg/Debian      -> "amd64"  / "arm64"
+ *
+ * Returns undefined for anything we don't serve, so callers fall back rather than
+ * inventing a kernel path that would 404.
+ */
+export function normalizeArch(value: string | undefined | null): Arch | undefined {
+  switch ((value ?? "").trim().toLowerCase()) {
+    case "x86_64":
+    case "x86-64":
+    case "amd64":
+      return "x86_64";
+    case "aarch64":
+    case "arm64":
+      return "aarch64";
+    default:
+      return undefined;
+  }
+}
+
+/** Fedora pxeboot artifact base URL for an architecture. */
+export function fedoraMirrorFor(fedoraVersion: string, arch: Arch): string {
+  return `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`;
+}
+
+/**
+ * Which architectures each OS in the pipeline can actually be installed on.
+ *
+ * Fedora publishes pxeboot vmlinuz/initrd for both. Ubuntu does not: as of 26.04,
+ * releases.ubuntu.com publishes amd64 artifacts only, so there is nothing to netboot an
+ * arm64 machine with. Claiming support would fail at download time with a 404 instead
+ * of a useful message.
+ */
+const OS_ARCH_SUPPORT: Record<OsId, readonly Arch[]> = {
+  "fedora-43": ["x86_64", "aarch64"],
+  "ubuntu-26.04": ["x86_64"],
+};
+
+export function osSupportsArch(os: OsId, arch: Arch): boolean {
+  return (OS_ARCH_SUPPORT[os] ?? []).includes(arch);
+}
+
+export function archesForOs(os: OsId): readonly Arch[] {
+  return OS_ARCH_SUPPORT[os] ?? [];
+}
+
+/**
+ * Machines that run a vendor OS we have no image for.
+ *
+ * These are SSH-onboard: we manage userspace, but reinstalling destroys a driver and
+ * firmware stack our pipeline cannot rebuild. Matched on DMI identity, which is what
+ * discovery and `provision recheck` both collect.
+ *
+ * This is deliberately a property of the machine ("it runs DGX OS"), not a blocklist
+ * ("never install this MAC"). When a DGX OS image joins the pipeline, teaching the
+ * installer about vendor_os "dgx-os" is what unblocks these machines -- no entry here
+ * needs deleting.
+ */
+interface VendorOsRule {
+  vendorOs: string;
+  description: string;
+  matches: (hw: DmiIdentity) => boolean;
+}
+
+interface DmiIdentity {
+  manufacturer: string;
+  product: string;
+  board: string;
+}
+
+const VENDOR_OS_RULES: readonly VendorOsRule[] = [
+  {
+    vendorOs: "dgx-os",
+    description: "NVIDIA DGX OS (proprietary driver + firmware stack, no image in our pipeline)",
+    matches: ({ manufacturer, product, board }) =>
+      (manufacturer.includes("nvidia") || product.includes("nvidia")) &&
+      (product.includes("dgx") || product.includes("spark") ||
+       board.includes("gb10") || product.includes("gb10")),
+  },
+];
+
+/**
+ * Machines known to run a vendor OS, by MAC.
+ *
+ * The DMI rules above only fire once discovery or `provision recheck` has populated a
+ * hardware record. Machines onboarded over SSH may sit in state for a long time with no
+ * DMI at all -- which is exactly the state both DGX Sparks are in today -- so a
+ * DMI-only classifier would fail open on the machines this guard exists to protect.
+ *
+ * This is a statement of fact about known hardware ("this box runs DGX OS"), not an
+ * install policy. Whether that means "refuse" is decided by whether the pipeline has an
+ * image for that vendor OS.
+ */
+const KNOWN_VENDOR_OS_MACS: Record<string, string> = {
+  "4c:bb:47:7f:29:35": "dgx-os",  // spark-2935
+  "48:21:0b:96:3a:1c": "dgx-os",  // spark-3a1c
+};
+
+/**
+ * Classify how a machine should be onboarded, from its hardware record.
+ *
+ * An explicit `onboard` already on the record wins: it may have been set by an operator
+ * or by a rule that has since changed, and silently overriding it would be worse than
+ * leaving it.
+ */
+export function classifyOnboard(
+  hw: Partial<Pick<HardwareInfo, "mac" | "manufacturer" | "product" | "board">>
+    & { onboard?: OnboardMethod; vendor_os?: string },
+): { onboard: OnboardMethod; vendor_os?: string } {
+  if (hw.onboard !== undefined) {
+    return hw.vendor_os !== undefined
+      ? { onboard: hw.onboard, vendor_os: hw.vendor_os }
+      : { onboard: hw.onboard };
+  }
+
+  const knownVendorOs = KNOWN_VENDOR_OS_MACS[(hw.mac ?? "").toLowerCase().replace(/-/g, ":")];
+  if (knownVendorOs !== undefined) {
+    return { onboard: "ssh", vendor_os: knownVendorOs };
+  }
+
+  const identity: DmiIdentity = {
+    manufacturer: (hw.manufacturer ?? "").toLowerCase(),
+    product: (hw.product ?? "").toLowerCase(),
+    board: (hw.board ?? "").toLowerCase(),
+  };
+
+  for (const rule of VENDOR_OS_RULES) {
+    if (rule.matches(identity)) {
+      return { onboard: "ssh", vendor_os: rule.vendorOs };
+    }
+  }
+
+  return { onboard: "pxe" };
+}
+
+/** Human-readable reason a vendor-OS machine must not be reinstalled. */
+export function vendorOsDescription(vendorOs: string | undefined): string {
+  const rule = VENDOR_OS_RULES.find((r) => r.vendorOs === vendorOs);
+  return rule?.description ?? "a vendor OS with no image in our pipeline";
+}

bastion/src/shared/src/index.ts

@@ -1,6 +1,8 @@
 export type {
   OsId,
   Arch,
+  OnboardMethod,
+  RootCandidate,
   Role,
   HardwareInfo,
   InstallConfig,
@@ -8,15 +10,20 @@ export type {
   DebugConfig,
   BastionState,
   BastionConfig,
-  VyosVlanSpec,
-  VyosInstallSpec,
-  VyosBundle,
-  VyosBundleSetOp,
 } from "./types/index.js";
 
-export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId,
-  roleInfo, runsK3s, isK3sServer, isManaged, hostsLonghorn, isRole } from "./types/index.js";
-export type { RoleInfo, Lifecycle, K3sMode } from "./types/index.js";
+export {
+  SUPPORTED_ARCHES,
+  normalizeArch,
+  fedoraMirrorFor,
+  osSupportsArch,
+  archesForOs,
+  classifyOnboard,
+  vendorOsDescription,
+} from "./hardware/index.js";
+
+export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./types/index.js";
+export type { RoleInfo } from "./types/index.js";
 
 export { APP_NAME, APP_VERSION } from "./constants/index.js";
 
@@ -49,8 +56,3 @@ export {
   AgentNotConnectedError,
   isLabError,
 } from "./errors/index.js";
-
-export { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB,
-  SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion,
-  validateDataDisks, DATA_DISK_LABELS } from "./types/index.js";
-export type { LvSpec, SchemaVersion, DataDisks, DataDiskName } from "./types/index.js";

bastion/src/shared/src/protocol/index.ts

@@ -1,7 +1,6 @@
 // Protocol types for agent-labd WebSocket communication.
 
 import { randomUUID } from "node:crypto";
-import type { VyosInstallSpec } from "../types/state.js";
 
 // --- Agent -> labd messages ---
 
@@ -109,12 +108,12 @@ export type BastionMessage =
 export type LabdBastionMessage =
   | { type: "bastion-enrolled"; bastionId: string }
   | { type: "bastion-heartbeat-ack"; serverTime: string }
-  | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string; schemaVersion?: 1 | 2; dataDisks?: import("../types/layout.js").DataDisks; vyos?: VyosInstallSpec }
+  | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string }
   | { type: "command-forget"; requestId: string; mac: string }
   | { type: "command-role-update"; requestId: string; mac: string; role: string }
   | { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean }
   | { type: "command-register"; requestId: string; mac: string; hostname: string; role: string; ip: string }
-  | { type: "command-discover"; requestId: string; mac: string; product?: string; board?: string; serial?: string; manufacturer?: string; cpu_model?: string; cpu_cores?: number; memory_gb?: number; arch?: string; disks?: Array<{ name: string; size_gb: number; model: string }>; nics?: Array<{ name: string; mac: string; state: string }> }
+  | { type: "command-discover"; requestId: string; mac: string; product?: string; board?: string; serial?: string; manufacturer?: string; cpu_model?: string; cpu_cores?: number; memory_gb?: number; arch?: string; disks?: Array<{ name: string; size_gb: number; model: string }>; nics?: Array<{ name: string; mac: string; state: string }>; root_device?: string; root_args?: string }
   | { type: "server-shutdown"; reconnectAfter: number };
 
 export type BastionMessageType = BastionMessage["type"];

bastion/src/shared/src/types/config.ts

@@ -14,19 +14,11 @@ export interface BastionConfig {
   // Ubuntu support
   ubuntuVersion: string;
   ubuntuMirror: string;
-  // VyOS support — netboot artifacts are extracted from the ISO at startup.
-  // LTS ISOs are subscription-only, so this defaults to a rolling release.
-  vyosIsoUrl: string;
-  vyosDefaultPassword: string;
   // Syslog listener for install logs (Anaconda logging --host)
   syslogPort: number;
   // Flags
   skipDnsmasq?: boolean | undefined;
   skipArtifacts?: boolean | undefined;
-  // Post-provision k3s auto-install. Defaults on. Set false to suppress it --
-  // required by multi-node tests, which must sequence cluster formation
-  // themselves rather than have each node auto-install into its own cluster.
-  autoK3s?: boolean | undefined;
   // Labd registration (optional — if unset, bastion runs standalone)
   labdUrl?: string | undefined;
   bastionJoinToken?: string | undefined;

bastion/src/shared/src/types/index.ts

@@ -1,25 +1,17 @@
 export type {
   OsId,
   Arch,
+  OnboardMethod,
+  RootCandidate,
   Role,
   HardwareInfo,
   InstallConfig,
   InstalledInfo,
   DebugConfig,
   BastionState,
-  VyosVlanSpec,
-  VyosInstallSpec,
-  VyosBundle,
-  VyosBundleSetOp,
 } from "./state.js";
 
-export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId,
-  roleInfo, runsK3s, isK3sServer, isManaged, hostsLonghorn, isRole } from "./state.js";
-export type { RoleInfo, Lifecycle, K3sMode } from "./state.js";
+export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./state.js";
+export type { RoleInfo } from "./state.js";
 
 export type { BastionConfig } from "./config.js";
-
-export { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB,
-  SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion,
-  validateDataDisks, DATA_DISK_LABELS } from "./layout.js";
-export type { LvSpec, SchemaVersion, DataDisks, DataDiskName } from "./layout.js";

bastion/src/shared/src/types/layout.ts

@@ -1,210 +0,0 @@
-import { roleInfo, type Role } from "./state.js";
-
-/**
- * Disk layout, schema v2.
- *
- * WHY THIS FILE EXISTS
- * --------------------
- * Three generators partition machines, and before this file they disagreed:
- *
- *   install.ks.ts        rancher 120G (infra|worker), longhorn --grow (worker)
- *   ubuntu-autoinstall   rancher  20G (infra only),   longhorn grow   (worker)
- *   asahi-firstboot.sh   rancher  20G (server only),  longhorn 100%FREE (worker|server)
- *
- * asahi-firstboot even carries the comment "sizes match install.ks.ts" while
- * being 100 GB smaller — which is where worker1's undersized image store came
- * from, and why a 2026-08 fix to one file never reached the fleet.
- *
- * v2 is defined ONCE, here, and all three generators render from it. v1 stays
- * frozen inline in each generator, because every existing machine was built by
- * it and reprovision must reproduce it byte-for-byte.
- *
- * SIZING
- * ------
- * Must fit a 1 TB disk (~926 GiB VG after EFI+/boot) with headroom left
- * UNALLOCATED on purpose: LVM can grow into free extents online, and the
- * fleet's recurring failure has been over-allocation, not under-allocation.
- * `--grow` is deliberately never used — it is what left every node at VFree=0
- * and made later fixes impossible without a rebuild.
- */
-
-/**
- * Roles that predate the v2 layout. Only these have a v1 layout to reproduce;
- * everything else is v2-only, because no machine was ever installed as one.
- */
-export const LEGACY_ROLES = ["vanilla", "worker", "infra", "labcontroller"] as const;
-
-/**
- * True when this role has no v1 layout at all.
- *
- * Rendering one at v1 would silently produce base volumes with no rancher and
- * no longhorn -- a machine that looks installed and cannot run k3s properly.
- * Callers must reject this rather than guess.
- */
-export function requiresSchemaV2(role: Role): boolean {
-  return !(LEGACY_ROLES as readonly string[]).includes(role);
-}
-
-/**
- * The schema a FRESH install of this role should use.
- *
- * A role with no v1 layout has exactly one possible answer, so defaulting to it
- * is unambiguous -- there is no other layout it could mean. Rejecting the
- * request instead was a mistake: a reinstall legitimately passes no schema
- * (the disk decides via the lab-schema-v2 tag), and refusing that made it
- * impossible to reinstall a v2-only machine at all.
- *
- * An EXPLICIT v1 for such a role is still a contradiction and must be refused
- * by the caller -- see requiresSchemaV2.
- */
-export function effectiveSchemaVersion(role: Role, requested?: 1 | 2): SchemaVersion {
-  if (requested) return requested;
-  return requiresSchemaV2(role) ? 2 : 1;
-}
-
-export const SCHEMA_VERSIONS = [1, 2] as const;
-export type SchemaVersion = (typeof SCHEMA_VERSIONS)[number];
-
-export interface LvSpec {
-  /** LV name inside the volume group. */
-  name: string;
-  /** Mount point, or null for swap. */
-  mount: string | null;
-  /** Size in MiB. Never "grow". */
-  sizeMib: number;
-  fs: "xfs" | "swap";
-}
-
-/** Outside the VG: the EFI system partition and /boot. */
-export const V2_BOOT = {
-  efiMib: 1024,   // was 600 - firmware and shim have grown
-  bootMib: 4096,  // was 3072 - several kernels + initramfs
-} as const;
-
-const V2_BASE: readonly LvSpec[] = [
-  { name: "swap",   mount: null,       sizeMib: 16384, fs: "swap" },
-  { name: "root",   mount: "/",        sizeMib: 32768, fs: "xfs" },
-  { name: "var",    mount: "/var",     sizeMib: 49152, fs: "xfs" },
-  { name: "varlog", mount: "/var/log", sizeMib: 16384, fs: "xfs" },
-  { name: "home",   mount: "/home",    sizeMib:  8192, fs: "xfs" },
-  { name: "srv",    mount: "/srv",     sizeMib: 16384, fs: "xfs" },
-] as const;
-
-/** Image store AND local-path PVCs both live here - size for both. */
-export const V2_RANCHER_MIB = 256000;  // 250 GiB
-export const V2_LONGHORN_MIB = 307200; // 300 GiB
-
-/**
- * Dedicated whole-disk data volumes (v2 only).
- *
- * The fixed 250G/300G LVs exist to FIT a 1 TB OS disk. A machine with spare
- * NVMe bays (thebeast: 1 TB OS + 2 TB + 2 TB) wants the opposite trade: give
- * each data mount a whole disk and keep the OS VG small. Each value is a
- * stable /dev/disk/by-id/ path -- NEVER a kernel name, because nvme0n1/nvme1n1
- * enumeration order is not stable across boots on multi-NVMe hosts, and a
- * swapped name here formats the wrong disk.
- *
- * The disk gets one partition spanning it, xfs, labelled (DATA_DISK_LABELS)
- * so a reprovision can recognise and preserve it the same way the LVs are
- * recognised by name inside the VG. The corresponding LV is simply not
- * created -- v2Layout() drops it when a dedicated disk supplies that mount.
- */
-export interface DataDisks {
-  /** Whole disk for /var/lib/longhorn (replica store). */
-  longhorn?: string;
-  /** Whole disk for /var/lib/rancher (image store + local-path PVCs). */
-  rancher?: string;
-}
-
-/**
- * Filesystem labels that mark a dedicated data disk's partition. They are the
- * reprovision contract: %pre finds these by label and mounts them --noformat,
- * exactly as it finds LVs by name. Changing a label orphans every disk
- * already carrying the old one.
- */
-export const DATA_DISK_LABELS = {
-  longhorn: "lab-longhorn",
-  rancher: "lab-rancher",
-} as const;
-
-export type DataDiskName = keyof typeof DATA_DISK_LABELS;
-
-const BY_ID_PREFIX = "/dev/disk/by-id/";
-
-/**
- * Why a DataDisks spec is unusable for this role, or null when it is fine.
- * Shared by the HTTP API, the websocket command handler and the CLI so all
- * three reject with the same words instead of three drifting checks.
- */
-export function validateDataDisks(role: Role, disks: DataDisks): string | null {
-  const info = roleInfo(role);
-  for (const name of Object.keys(DATA_DISK_LABELS) as DataDiskName[]) {
-    const device = disks[name];
-    if (device === undefined) continue;
-    if (!device.startsWith(BY_ID_PREFIX)) {
-      return `${name} disk must be a stable ${BY_ID_PREFIX} path (got '${device}'). ` +
-        `Kernel names like nvme0n1 change across boots on multi-NVMe hosts.`;
-    }
-  }
-  if (disks.rancher !== undefined && info.k3sMode === null) {
-    return `role '${role}' runs no k3s -- a dedicated rancher disk makes no sense for it`;
-  }
-  if (disks.longhorn !== undefined && (info.k3sMode === null || info.controlPlaneOnly)) {
-    return `role '${role}' holds no Longhorn replicas -- a dedicated longhorn disk makes no sense for it`;
-  }
-  if (disks.longhorn !== undefined && disks.longhorn === disks.rancher) {
-    return `longhorn and rancher point at the same disk (${disks.longhorn}) -- they need one each`;
-  }
-  return null;
-}
-
-/**
- * The v2 volumes for a role, gated by CAPABILITY rather than role name.
- *
- * `omitRoot` is for Asahi, where the vendor installer already created a btrfs
- * root partition outside our VG and we must not make a second one.
- *
- * `dataDisks` drops the longhorn/rancher LV whose mount a dedicated disk
- * supplies -- two things mounting /var/lib/longhorn is an invalid layout.
- */
-export function v2Layout(role: Role, opts: { omitRoot?: boolean; dataDisks?: DataDisks } = {}): LvSpec[] {
-  const info = roleInfo(role);
-  const lvs: LvSpec[] = V2_BASE.filter(
-    (lv) => !(opts.omitRoot && lv.name === "root"),
-  ).map((lv) => ({ ...lv }));
-
-  // Anything running k3s needs a dedicated image store. This is the check that
-  // `role === "infra" || role === "worker"` kept getting wrong: labcontroller
-  // runs k3s and was silently excluded.
-  if (info.k3sMode !== null && opts.dataDisks?.rancher === undefined) {
-    lvs.push({ name: "rancher", mount: "/var/lib/rancher", sizeMib: V2_RANCHER_MIB, fs: "xfs" });
-  }
-
-  // Fixed, never --grow. A control-plane-only node holds no replicas.
-  if (info.k3sMode !== null && !info.controlPlaneOnly && opts.dataDisks?.longhorn === undefined) {
-    lvs.push({ name: "longhorn", mount: "/var/lib/longhorn", sizeMib: V2_LONGHORN_MIB, fs: "xfs" });
-  }
-
-  return lvs;
-}
-
-/** Total MiB a role's v2 layout consumes inside the VG. */
-export function v2AllocatedMib(role: Role, opts: { omitRoot?: boolean; dataDisks?: DataDisks } = {}): number {
-  return v2Layout(role, opts).reduce((n, lv) => n + lv.sizeMib, 0);
-}
-
-/**
- * Bind-mounting containerd onto the Longhorn LV.
- *
- * Codifies what was already done BY HAND on worker1 and exists in no template:
- * its 20 GiB rancher LV could not hold the image store, so containerd was
- * pointed at a directory on the big Longhorn LV. It survives reboot only
- * because someone wrote a correct fstab line; a reinstall would lose it.
- *
- * v2 sizes rancher properly so this is not needed on new boxes - it is kept for
- * rescuing v1 machines that cannot be repartitioned yet.
- */
-export const CONTAINERD_BIND_FSTAB =
-  "/var/lib/longhorn/k3s-containerd /var/lib/rancher/k3s/agent/containerd none " +
-  "bind,x-systemd.requires-mounts-for=/var/lib/longhorn," +
-  "x-systemd.requires-mounts-for=/var/lib/rancher 0 0";

bastion/src/shared/src/types/state.ts

@@ -2,15 +2,25 @@
 
 export type ProvisionStackType = "dhcpproxy" | "iso" | "cloud-init";
 
-export type OsId = "fedora-43" | "ubuntu-26.04" | "vyos-rolling";
+export type OsId = "fedora-43" | "ubuntu-26.04";
 export type Arch = "x86_64" | "aarch64";
 
-export const SUPPORTED_OS: readonly OsId[] = ["fedora-43", "ubuntu-26.04", "vyos-rolling"] as const;
+export const SUPPORTED_OS: readonly OsId[] = ["fedora-43", "ubuntu-26.04"] as const;
 
 export function isValidOsId(value: string): value is OsId {
   return (SUPPORTED_OS as readonly string[]).includes(value);
 }
 
+/**
+ * How a machine joins the lab.
+ *
+ * "pxe"  -- bare metal we install over the network (the default).
+ * "ssh"  -- the machine already runs a vendor OS we cannot reproduce, so we onboard
+ *           over SSH and manage userspace only. Installing would destroy that OS.
+ *           See classifyOnboard() and os-install-research.md.
+ */
+export type OnboardMethod = "pxe" | "ssh";
+
 export interface HardwareInfo {
   mac: string;
   product: string;
@@ -26,34 +36,27 @@ export interface HardwareInfo {
   first_seen: string;
   last_seen: string;
   bastionId?: string;  // set when aggregated through labd
+  // Onboarding classification -- absent means "pxe" (see classifyOnboard)
+  onboard?: OnboardMethod;
+  vendor_os?: string;   // e.g. "dgx-os": the OS this machine must keep running
+  // Root filesystem, for booting the installed system over PXE (--pxe-boot).
+  // Observed from the machine, never assumed.
+  root_device?: string;                 // e.g. "/dev/mapper/labvg-root"
+  root_args?: string;                   // e.g. "rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap"
+  root_candidates?: RootCandidate[];    // reported from a rescue shell when unknown
 }
 
-export type Role =
-  // Legacy roles. Every machine in the fleet today is one of these, and they
-  // default to the v1 disk layout so a reinstall reproduces what is on disk.
-  | "vanilla" | "worker" | "infra" | "labcontroller"
-  // Topology-named profiles. New machines only, so they are v2-only: there is
-  // no v1 layout to be compatible with, and none of them can reach the frozen
-  // v1 branch in the templates.
-  | "standalone-cluster" | "k8s-controller" | "k8s-worker" | "k8s-combo";
+/** A possible root filesystem found while probing an unreachable machine. */
+export interface RootCandidate {
+  device: string;       // e.g. "/dev/mapper/labvg-root"
+  args?: string;        // extra dracut args needed to assemble it
+  fstype?: string;
+  size_gb?: number;
+  os_release?: string;  // PRETTY_NAME from /etc/os-release, if mountable
+}
 
-export const SUPPORTED_ROLES: readonly Role[] = [
-  "vanilla", "worker", "infra", "labcontroller",
-  "standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo",
-] as const;
-
-/**
- * How a machine comes to exist, which decides whether we partition it at all.
- *
- *  - "managed"   we PXE-install it with our kickstart; our disk layout applies.
- *  - "appliance" we install it, but the vendor installer owns the disk (VyOS).
- *  - "onboard"   we never install it; we adopt a running machine (DGX Spark,
- *                Apple Silicon under Asahi). No partitioning, ever.
- */
-export type Lifecycle = "managed" | "appliance" | "onboard";
-
-/** null = no k3s at all. */
-export type K3sMode = "server" | "agent" | null;
+export type Role = "vanilla" | "worker" | "infra" | "labcontroller";
+export const SUPPORTED_ROLES: readonly Role[] = ["vanilla", "worker", "infra", "labcontroller"] as const;
 
 export interface RoleInfo {
   name: Role;
@@ -61,18 +64,6 @@ export interface RoleInfo {
   parent?: Role;       // inherits from this role
   k3s: boolean;        // installs k3s
   apps: string[];      // apps auto-deployed after k3s
-
-  // --- capabilities -------------------------------------------------------
-  // These exist so decisions are made by asking what a machine DOES, not by
-  // comparing its role name. Every `role === "infra" || role === "labcontroller"`
-  // in this codebase is a place that forgot a role: `labcontroller` runs k3s and
-  // still got no rancher LV, because install.ks.ts asked `role === "infra"`.
-  lifecycle: Lifecycle;
-  k3sMode: K3sMode;
-  /** Holds Longhorn replicas, i.e. wants a dedicated /var/lib/longhorn LV. */
-  hostsLonghorn: boolean;
-  /** Control plane that deliberately runs no workloads and no storage. */
-  controlPlaneOnly: boolean;
 }
 
 export const ROLE_REGISTRY: readonly RoleInfo[] = [
@@ -81,10 +72,6 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [
     description: "OS only — no k3s, no cluster services",
     k3s: false,
     apps: [],
-    lifecycle: "managed",
-    k3sMode: null,
-    hostsLonghorn: false,
-    controlPlaneOnly: false,
   },
   {
     name: "worker",
@@ -92,10 +79,6 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [
     parent: "vanilla",
     k3s: true,
     apps: [],
-    lifecycle: "managed",
-    k3sMode: "agent",
-    hostsLonghorn: true,
-    controlPlaneOnly: false,
   },
   {
     name: "infra",
@@ -103,15 +86,6 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [
     parent: "vanilla",
     k3s: true,
     apps: [],
-    lifecycle: "managed",
-    k3sMode: "server",
-    // NOTE: worker0/1/2 are `infra` and DO hold every Longhorn replica in the
-    // homelab cluster, but the v1 kickstart only ever gave a longhorn LV to
-    // `worker`. On worker2 that means Longhorn data sits on /var alongside the
-    // image store. Left false so the v1 layout is unchanged; v2 sizes it
-    // properly from this flag.
-    hostsLonghorn: false,
-    controlPlaneOnly: false,
   },
   {
     name: "labcontroller",
@@ -119,258 +93,22 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [
     parent: "infra",
     k3s: true,
     apps: ["cockroachdb", "labd", "bastion"],
-    lifecycle: "managed",
-    k3sMode: "server",
-    hostsLonghorn: false,
-    controlPlaneOnly: false,
-  },
-  {
-    name: "standalone-cluster",
-    description: "Single node that IS its own cluster — k3s server + workloads + storage",
-    parent: "vanilla",
-    k3s: true,
-    apps: [],
-    lifecycle: "managed",
-    k3sMode: "server",
-    // Longhorn yes, at replica 1. A cluster of one cannot satisfy a 3-replica
-    // policy, but keeping the `longhorn` StorageClass present means manifests
-    // are portable across every cluster instead of needing a local-path
-    // variant. Replica count is a property of the cluster, not the manifest.
-    hostsLonghorn: true,
-    controlPlaneOnly: false,
-  },
-  {
-    name: "k8s-controller",
-    description: "Control plane + etcd only — runs no workloads and holds no storage",
-    parent: "vanilla",
-    k3s: true,
-    apps: [],
-    lifecycle: "managed",
-    k3sMode: "server",
-    hostsLonghorn: false,
-    controlPlaneOnly: true,
-  },
-  {
-    name: "k8s-worker",
-    description: "k3s agent + Longhorn storage — joins an existing cluster",
-    parent: "vanilla",
-    k3s: true,
-    apps: [],
-    lifecycle: "managed",
-    k3sMode: "agent",
-    hostsLonghorn: true,
-    controlPlaneOnly: false,
-  },
-  {
-    name: "k8s-combo",
-    description: "Control plane + etcd AND workloads + storage — hyperconverged cluster member",
-    parent: "vanilla",
-    k3s: true,
-    apps: [],
-    lifecycle: "managed",
-    k3sMode: "server",
-    hostsLonghorn: true,
-    controlPlaneOnly: false,
   },
 ] as const;
 
-/**
- * Capability lookups. Use these instead of comparing role names.
- *
- * Every call site that asks `role === "infra" || role === "labcontroller"` is a
- * place that has to be edited again when a role is added — and the one that was
- * missed (install.ks.ts asking `role === "infra"`) is why `labcontroller` boxes
- * get no dedicated image-store LV.
- */
-/**
- * Narrow an untrusted string (state file, HTTP body) to a Role.
- * State is JSON on disk and can contain anything, including a role that was
- * removed in a later version -- so this is a guard, not a cast.
- */
-export function isRole(value: string): value is Role {
-  return (SUPPORTED_ROLES as readonly string[]).includes(value);
-}
-
-export function roleInfo(role: Role): RoleInfo {
-  const info = ROLE_REGISTRY.find((r) => r.name === role);
-  if (!info) throw new Error(`unknown role: ${role}`);
-  return info;
-}
-
-/** Runs k3s in any capacity. */
-export function runsK3s(role: Role): boolean {
-  return roleInfo(role).k3sMode !== null;
-}
-
-/** Runs k3s as a server (control plane + etcd) rather than an agent. */
-export function isK3sServer(role: Role): boolean {
-  return roleInfo(role).k3sMode === "server";
-}
-
-/** We PXE-install this machine, so our disk layout applies to it. */
-export function isManaged(role: Role): boolean {
-  return roleInfo(role).lifecycle === "managed";
-}
-
-/** Wants a dedicated /var/lib/longhorn LV. */
-export function hostsLonghorn(role: Role): boolean {
-  return roleInfo(role).hostsLonghorn;
-}
-
 export interface ProgressLogEntry {
   stage: string;
   detail: string;
   timestamp: string;
 }
 
-/** A tagged VLAN sub-interface on the bond (or on the mgmt NIC when unbonded). */
-export interface VyosVlanSpec {
-  id: number;
-  address: string;         // CIDR, e.g. "10.0.10.1/24"
-  description?: string;
-  /**
-   * VRRP virtual address (CIDR) floated on this VLAN. Emitted as a
-   * high-availability vrrp group with vrid = VLAN id, so the same spec on both
-   * HA peers (with different priorities) produces a matching group pair.
-   */
-  vrrp?: string;
-}
-
-/** One config node in a rendered bundle. Mirrors VyosSetOp on the bastion side. */
-export interface VyosBundleSetOp {
-  path: string[];
-  value?: string;
-  /** false appends to a multi-value node (e.g. bond members) instead of replacing. */
-  replace?: boolean;
-}
-
-/**
- * A router's complete desired config, rendered from the Pulumi model.
- *
- * Produced by `kubernetes-deployment/scripts/vyos-render-bundle.ts` from the
- * same subtree model `pulumi up` applies. The point is that labctl never
- * authors VyOS config: bring-up replays what Pulumi already declares, so a
- * freshly installed router and a `pulumi up` cannot disagree.
- *
- * Secret values arrive as `@secret:<key>` sentinels and are DROPPED at install
- * time -- the bundle is committed to git and must stay safe to read. The router
- * comes up on the LAN without its PPPoE credential; the first `pulumi up`
- * supplies it. That handoff is deliberate.
- */
-export interface VyosBundle {
-  sets: VyosBundleSetOp[];
-  /** Paths that are VyOS tag nodes — the installer's ConfigTree needs them marked. */
-  tags: string[][];
-}
-
-/**
- * VyOS-specific install parameters. Rendered into the config.boot that the
- * installer adopts, so the router comes up already configured.
- *
- * NOTE: bondMembers must NOT include the interface PXE booted from. Firmware
- * PXE cannot run over LACP, so the install-time NIC has to stay unbonded.
- */
-export interface VyosInstallSpec {
-  /**
-   * A complete rendered config for this router. When present it REPLACES the
-   * derived interface/VLAN/VRRP config below -- the bundle already describes
-   * all of it, and deriving a second opinion is exactly the drift this exists
-   * to prevent. The remaining install parameters (password, disk, console) are
-   * still honoured because they are installer inputs, not router config.
-   */
-  bundle?: VyosBundle;
-  /**
-   * Key for the VyOS HTTP API, enabled at install so the router is manageable
-   * from the moment it boots.
-   *
-   * Without this the box comes up reachable only over SSH, and enabling the API
-   * later is a hand-run config change on a live firewall -- which is exactly the
-   * gap that left vyos001/vyos002 unmanageable by Pulumi after their cutover.
-   * The API is deliberately NOT part of the Pulumi model: a provider that
-   * manages its own transport can revoke its own access.
-   */
-  apiKey?: string;
-  /**
-   * Address the API listens on. Defaults to the management address when static.
-   * Never left unbound: an unrestricted listener puts a config-write endpoint on
-   * every segment the router touches, including the WAN.
-   */
-  apiListenAddress?: string;
-  /** Interfaces aggregated into bond0 with LACP (802.3ad). Omit for no bond. */
-  bondMembers?: string[];
-  /** CIDR address on bond0 itself — the switch trunk's native/untagged VLAN. */
-  bondAddress?: string;
-  /** VRRP virtual address (CIDR) floated on the untagged bond (vrid 1). */
-  bondVrrp?: string;
-  /**
-   * VRRP priority for every group on this box. Higher wins mastership.
-   * The HA pair differs ONLY here (e.g. 200 on the primary, 100 on the
-   * standby) — addresses differ per box, VIPs and vrids match.
-   */
-  vrrpPriority?: number;
-  /** Tagged VLAN sub-interfaces, created on bond0 when bonded, else on mgmtInterface. */
-  vlans?: VyosVlanSpec[];
-  /** Untagged interface the machine PXE booted from. Defaults to "eth0". */
-  mgmtInterface?: string;
-  /** CIDR address for mgmtInterface, or "dhcp". Defaults to "dhcp". */
-  mgmtAddress?: string;
-  /**
-   * Tagged management VLAN on mgmtInterface, separate from the routed VLANs
-   * carried by the bond.
-   *
-   * Needed when the PXE port is a trunk: it boots untagged on the VLAN the
-   * bastion's proxy DHCP serves, and carries the management VLAN tagged so the
-   * router stays reachable there without giving up reinstallability.
-   */
-  mgmtVlan?: VyosVlanSpec;
-  /** Password for the "vyos" user. Falls back to the bastion default. */
-  password?: string;
-  /**
-   * On reinstall the VyOS installer carries the previous on-disk config (and
-   * SSH host keys) forward -- the "reinstall without losing data" default.
-   * Set true to make the bastion-generated config win instead: after install
-   * the driver overwrites the installed image's config.boot.
-   */
-  freshConfig?: boolean;
-  /**
-   * VyOS interface name -> MAC, emitted as `hw-id` so names bind deterministically.
-   *
-   * Discovery runs under Fedora and reports predictable names (enp2s0,
-   * enp1s0f0np0), but VyOS enumerates its own eth<N> names, so a name observed
-   * during discovery cannot be used directly. Pinning by MAC removes the guess
-   * about which physical port a given eth<N> is.
-   */
-  hwIds?: Record<string, string>;
-}
-
 export interface InstallConfig {
   hostname: string;
   disk: string;
   role: Role;
   os?: OsId;          // defaults to "fedora-43" for backward compat
-  vyos?: VyosInstallSpec;  // only consulted when os is "vyos-rolling"
   arch?: Arch;         // detected from HardwareInfo or overridden
-  /**
-   * Disk layout for a FRESH install. Defaults to 1.
-   * Ignored on a reprovision: %pre reads the lab-schema-v2 LVM tag off the
-   * disk instead, because reinstalling a machine must never change its layout.
-   */
-  schemaVersion?: 1 | 2;
-  /**
-   * Dedicated whole-disk data volumes (v2 + fedora only), by stable
-   * /dev/disk/by-id path. See DataDisks in layout.ts. On a reprovision these
-   * are optional: %pre rediscovers the disks by filesystem label and preserves
-   * them whether or not the queue entry names them.
-   */
-  dataDisks?: import("./layout.js").DataDisks;
   queued_at: string;
-  /**
-   * When dispatch last served this machine an install boot script. Progress
-   * callbacks only start once the installer environment is up, so a machine
-   * dispatched long ago with no progress is wedged before that point (bad
-   * kernel/initrd, no network in the initramfs, wrong NIC picked...).
-   */
-  dispatched_at?: string;
   progress?: string;
   progress_at?: string;
   progress_detail?: string;
@@ -381,12 +119,6 @@ export interface InstallConfig {
 export interface InstalledInfo {
   hostname: string;
   role: string;
-  /**
-   * Disk layout this machine was installed with. Advisory only -- the
-   * authoritative answer is the `lab-schema-v2` LVM tag on the machine's own
-   * volume group, because that is what a reinstall actually reads.
-   */
-  schemaVersion?: 1 | 2;
   os?: OsId;
   ip: string;
   installed_at: string;
@@ -398,6 +130,11 @@ export interface InstalledInfo {
   cpu_cores?: number;
   memory_gb?: number;
   arch?: string;
+  onboard?: OnboardMethod;
+  vendor_os?: string;
+  root_device?: string;
+  root_args?: string;
+  root_candidates?: RootCandidate[];
 }
 
 export interface DebugConfig {

bastion/src/shared/tests/capabilities.test.ts

@@ -1,129 +0,0 @@
-import { describe, it, expect } from "vitest";
-import {
-  SUPPORTED_ROLES,
-  ROLE_REGISTRY,
-  roleInfo,
-  runsK3s,
-  isK3sServer,
-  isManaged,
-  hostsLonghorn,
-} from "../src/types/state.js";
-import type { Role } from "../src/types/state.js";
-
-/**
- * The capability lookups exist to replace role-name comparisons scattered across
- * the codebase. Introducing them must change NO behaviour — so these tests pin
- * each helper against the literal expression it replaces, copied verbatim from
- * the call site it came from.
- *
- * If one of these fails, the refactor changed behaviour somewhere. That is the
- * whole point of the file.
- */
-/**
- * Only the FOUR legacy roles have behaviour to preserve. The topology profiles
- * (standalone-cluster, k8s-*) are new, v2-only, and have no historical
- * expression to be compatible with -- pinning them against `role === "worker"`
- * would assert nonsense.
- */
-const LEGACY_ROLES = ["vanilla", "worker", "infra", "labcontroller"] as const;
-
-describe("capability helpers reproduce the expressions they replace", () => {
-  // Verbatim from the call sites, as of the commit that introduced this file.
-  const legacy = {
-    // k3s-install.ts:7, k3s-config.ts:7, render-config.ts:25,
-    // k3s-module.ts:65, health/k3s-service.ts:5, post-provision.ts:153
-    isServer: (r: Role) => r === "infra" || r === "labcontroller",
-    // ROLE_REGISTRY.k3s — the field that already existed and nobody read
-    runsK3s: (r: Role) => ROLE_REGISTRY.find((x) => x.name === r)!.k3s,
-    // install.ks.ts:41, ubuntu-autoinstall.ts:34
-    hasLonghorn: (r: Role) => r === "worker",
-    // install.ks.ts:48
-    isVanilla: (r: Role) => r === "vanilla",
-  };
-
-  it.each(LEGACY_ROLES)("isK3sServer matches the legacy expression for %s", (role) => {
-    expect(isK3sServer(role)).toBe(legacy.isServer(role));
-  });
-
-  it.each(LEGACY_ROLES)("runsK3s matches ROLE_REGISTRY.k3s for %s", (role) => {
-    expect(runsK3s(role)).toBe(legacy.runsK3s(role));
-  });
-
-  it.each(LEGACY_ROLES)("hostsLonghorn matches the legacy expression for %s", (role) => {
-    expect(hostsLonghorn(role)).toBe(legacy.hasLonghorn(role));
-  });
-
-  it.each(LEGACY_ROLES)("k3sMode === null matches isVanilla for %s", (role) => {
-    expect(roleInfo(role).k3sMode === null).toBe(legacy.isVanilla(role));
-  });
-
-  it("every supported role is managed today (nothing is onboarded yet)", () => {
-    for (const role of SUPPORTED_ROLES) expect(isManaged(role)).toBe(true);
-  });
-});
-
-/**
- * The topology profiles. These are asserted directly rather than against a
- * legacy expression, because they describe cluster SHAPE -- which is the thing
- * role names were supposed to convey and did not.
- */
-describe("topology profiles", () => {
-  it("standalone-cluster is a server that also holds storage", () => {
-    expect(isK3sServer("standalone-cluster")).toBe(true);
-    // Longhorn at replica 1: a cluster of one cannot do 3, but keeping the
-    // StorageClass present keeps manifests portable across clusters.
-    expect(hostsLonghorn("standalone-cluster")).toBe(true);
-    expect(roleInfo("standalone-cluster").controlPlaneOnly).toBe(false);
-  });
-
-  it("k8s-controller is the only role that runs k3s and holds no storage", () => {
-    expect(isK3sServer("k8s-controller")).toBe(true);
-    expect(hostsLonghorn("k8s-controller")).toBe(false);
-    expect(roleInfo("k8s-controller").controlPlaneOnly).toBe(true);
-  });
-
-  it("k8s-worker is an agent with storage", () => {
-    expect(isK3sServer("k8s-worker")).toBe(false);
-    expect(runsK3s("k8s-worker")).toBe(true);
-    expect(hostsLonghorn("k8s-worker")).toBe(true);
-  });
-
-  it("k8s-combo is both control plane and storage — unlike k8s-controller", () => {
-    expect(isK3sServer("k8s-combo")).toBe(true);
-    expect(hostsLonghorn("k8s-combo")).toBe(true);
-    expect(roleInfo("k8s-combo").controlPlaneOnly).toBe(false);
-  });
-
-  it("k8s-combo and k8s-controller differ ONLY in storage duty", () => {
-    const combo = roleInfo("k8s-combo");
-    const ctrl = roleInfo("k8s-controller");
-    expect(combo.k3sMode).toBe(ctrl.k3sMode);
-    expect(combo.hostsLonghorn).not.toBe(ctrl.hostsLonghorn);
-  });
-});
-
-describe("registry integrity", () => {
-  it("every SUPPORTED_ROLE has a registry entry", () => {
-    for (const role of SUPPORTED_ROLES) expect(() => roleInfo(role)).not.toThrow();
-  });
-
-  it("registry has no entries outside SUPPORTED_ROLES", () => {
-    for (const info of ROLE_REGISTRY) expect(SUPPORTED_ROLES).toContain(info.name);
-  });
-
-  it("k3s boolean and k3sMode never disagree", () => {
-    for (const info of ROLE_REGISTRY) {
-      expect(info.k3s).toBe(info.k3sMode !== null);
-    }
-  });
-
-  it("an unknown role throws rather than silently defaulting", () => {
-    expect(() => roleInfo("does-not-exist" as Role)).toThrow(/unknown role/);
-  });
-
-  it("a control-plane-only role never hosts Longhorn", () => {
-    for (const info of ROLE_REGISTRY) {
-      if (info.controlPlaneOnly) expect(info.hostsLonghorn).toBe(false);
-    }
-  });
-});

bastion/src/shared/tests/layout.test.ts

@@ -1,92 +0,0 @@
-import { describe, it, expect } from "vitest";
-import { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB } from "../src/types/layout.js";
-import { SUPPORTED_ROLES, roleInfo } from "../src/types/state.js";
-
-/** A 1 TB disk is the smallest we will ever buy. 1e12 bytes in MiB. */
-const ONE_TB_MIB = Math.floor(1_000_000_000_000 / (1024 * 1024)); // 953674
-const VG_MIB_ON_1TB = ONE_TB_MIB - V2_BOOT.efiMib - V2_BOOT.bootMib;
-
-describe("v2 layout fits the smallest disk we will buy", () => {
-  it.each(SUPPORTED_ROLES)("%s fits in a 1 TB disk with headroom left over", (role) => {
-    const used = v2AllocatedMib(role);
-    expect(used).toBeLessThan(VG_MIB_ON_1TB);
-    // The entire point of v2 is to STOP consuming the disk. Demand real slack.
-    const freeMib = VG_MIB_ON_1TB - used;
-    expect(freeMib).toBeGreaterThan(100 * 1024); // >100 GiB unallocated
-  });
-
-  it("the heaviest role still leaves >200 GiB free on 1 TB", () => {
-    const worst = Math.max(...SUPPORTED_ROLES.map((r) => v2AllocatedMib(r)));
-    expect(VG_MIB_ON_1TB - worst).toBeGreaterThan(200 * 1024);
-  });
-});
-
-describe("v2 gates volumes by capability, not role name", () => {
-  it.each(SUPPORTED_ROLES)("%s gets a rancher LV iff it runs k3s", (role) => {
-    const hasRancher = v2Layout(role).some((lv) => lv.name === "rancher");
-    expect(hasRancher).toBe(roleInfo(role).k3sMode !== null);
-  });
-
-  it("labcontroller gets a rancher LV — the v1 bug this schema fixes", () => {
-    const lvs = v2Layout("labcontroller");
-    const rancher = lvs.find((lv) => lv.name === "rancher");
-    expect(rancher).toBeDefined();
-    expect(rancher!.sizeMib).toBe(V2_RANCHER_MIB);
-  });
-
-  it("vanilla gets neither rancher nor longhorn", () => {
-    const names = v2Layout("vanilla").map((lv) => lv.name);
-    expect(names).not.toContain("rancher");
-    expect(names).not.toContain("longhorn");
-  });
-
-  it("a control-plane-only role would get no longhorn LV", () => {
-    for (const role of SUPPORTED_ROLES) {
-      if (roleInfo(role).controlPlaneOnly) {
-        expect(v2Layout(role).map((l) => l.name)).not.toContain("longhorn");
-      }
-    }
-  });
-});
-
-describe("v2 never over-allocates", () => {
-  it("no volume is grow/unbounded — every size is a concrete number", () => {
-    for (const role of SUPPORTED_ROLES) {
-      for (const lv of v2Layout(role)) {
-        expect(Number.isFinite(lv.sizeMib)).toBe(true);
-        expect(lv.sizeMib).toBeGreaterThan(0);
-      }
-    }
-  });
-
-  it("longhorn is fixed at 300 GiB, not grown", () => {
-    const lh = v2Layout("worker").find((lv) => lv.name === "longhorn");
-    expect(lh?.sizeMib).toBe(V2_LONGHORN_MIB);
-    expect(V2_LONGHORN_MIB).toBe(300 * 1024);
-  });
-
-  it("rancher is sized for an image store AND local-path PVCs", () => {
-    // A single ROCm vLLM image measured 62 GiB extracted on 2026-09-17.
-    expect(V2_RANCHER_MIB).toBeGreaterThan(200 * 1024);
-  });
-});
-
-describe("Asahi variant", () => {
-  it("omitRoot drops the root LV — the vendor installer already made one", () => {
-    const names = v2Layout("infra", { omitRoot: true }).map((lv) => lv.name);
-    expect(names).not.toContain("root");
-    expect(names).toContain("var");
-  });
-
-  it("omitRoot frees the root allocation and nothing else", () => {
-    const delta = v2AllocatedMib("infra") - v2AllocatedMib("infra", { omitRoot: true });
-    expect(delta).toBe(32768);
-  });
-});
-
-describe("layout is a pure value, safe to render three times", () => {
-  it("mutating a returned spec does not affect the next call", () => {
-    v2Layout("worker")[0]!.sizeMib = 999;
-    expect(v2Layout("worker")[0]!.sizeMib).not.toBe(999);
-  });
-});

bastion/tests/integration/arm-pxe-provision.test.ts

@@ -0,0 +1,537 @@
+// Integration test: aarch64 network PXE boot.
+//
+// The boot-ISO path already covered ARM (arm-iso-provision.test.ts). This covers the
+// network path: DHCP option 93 handing an arm64 client an arm64 iPXE binary, dispatch
+// serving an aarch64 kernel, and `provision debug` reaching a rescue shell -- which is
+// what the DGX Sparks actually need and could not do.
+//
+// Two suites, because they cost very different amounts of time:
+//
+//   "ARM PXE rescue"   NBP handoff -> rescue with SSH.                    ~25-30 min
+//   "ARM PXE install"  discover -> install -> installed.                  ~75-95 min
+//
+// The rescue suite seeds the machine into state as an already-known aarch64 box rather
+// than discovering it first. That is the DGX Spark situation exactly -- SSH-onboarded,
+// never PXE-discovered, architecture known only from its record -- and it holds the test
+// to one emulated boot. Each boot spends ~15 of its ~18 minutes downloading Anaconda's
+// stage2 under TCG, so discovering first would double the runtime without touching any
+// code path the rescue boot does not already exercise.
+//
+// The install suite only runs with ARM_PXE_FULL=1. No ARM machine in the lab is ever
+// PXE-installed except the MS-R1, and an hour-plus test that runs by default is a test
+// nobody runs.
+//
+// IMPORTANT: aarch64 has no KVM on an x86_64 host, so all of this is emulated and
+// roughly 10x slower than native.
+//
+// A note for whoever debugs a failure here: if the VM panics with
+//   VFS: Unable to mount root fs on unknown-block(0,0)
+// that is very likely iPXE silently dropping the initrd because the build lacks
+// EFI_LOAD_FILE2_PROTOCOL -- on arm64 the kernel EFI stub fetches the initrd over
+// LoadFile2, and an iPXE without it accepts the `initrd` line and does nothing. It is
+// NOT a reproduction of the DGX Spark kernel bug that motivated this work, despite
+// being the identical message. assertIpxeSupportsLoadFile2() below checks the build up
+// front so that failure names itself; to check by hand:
+//   node -e 'const b=require("fs").readFileSync("/usr/share/ipxe/arm64-efi/snponly.efi");
+//            console.log(b.indexOf(Buffer.from("c1c00640b3fc3e40996d4a6c8724e06d","hex")))'
+// Fedora's ipxe-bootimgs-aarch64-20240119 has it at 0x3bbf0.
+//
+// Prerequisites:
+//   - qemu-system-aarch64        (sudo dnf install qemu-system-aarch64)
+//   - edk2-aarch64               (sudo dnf install edk2-aarch64)
+//   - ipxe-bootimgs-aarch64      (sudo dnf install ipxe-bootimgs-aarch64)
+//   - libvirtd, sudo, internet access
+//
+// Run: sudo ./scripts/test-provision.sh arm-pxe
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, writeFileSync } from "node:fs";
+import { execSync } from "node:child_process";
+import { join } from "node:path";
+import { homedir, tmpdir } from "node:os";
+import { log, waitForSsh } from "./helpers/libvirt.js";
+import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js";
+import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./helpers/pxe-vm.js";
+import { sshExec } from "./helpers/ssh.js";
+
+const IPXE_ARM64 = "/usr/share/ipxe/arm64-efi/snponly.efi";
+const AAVMF = "/usr/share/edk2/aarch64/QEMU_EFI.fd";
+
+const VM_MEMORY = 4096;
+const VM_VCPUS = 2;
+const VM_DISK_GB = 250;
+const SSH_USER = "lab";
+const BASTION_IP = PXE_GATEWAY;
+const DHCP_RANGE_START = `${PXE_SUBNET}.100`;
+const DHCP_RANGE_END = `${PXE_SUBNET}.200`;
+const SERIAL_PORT = 4555;
+
+// Emulated aarch64 -- generous timeouts throughout. Measured on an x86_64 host with no
+// KVM for aarch64: a single PXE boot to a running Anaconda takes ~18 minutes, almost all
+// of it downloading inst.stage2 over the network under TCG. Budget well above that;
+// timing out just short of success wastes a whole run.
+const LEASE_TIMEOUT_MS = 10 * 60_000;
+const DISCOVERY_TIMEOUT_MS = 35 * 60_000;
+const INSTALL_TIMEOUT_MS = 75 * 60_000;
+const SSH_TIMEOUT_MS = 35 * 60_000;
+
+const RUN_FULL_INSTALL = process.env["ARM_PXE_FULL"] === "1";
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+function findSshKey(): { pubKey: string; keyPath: string } {
+  const candidates: string[] = [];
+  if (process.env["SSH_KEY_PATH"]) candidates.push(process.env["SSH_KEY_PATH"]);
+  const homes = [homedir()];
+  const sudoUser = process.env["SUDO_USER"];
+  if (sudoUser) homes.push(join("/home", sudoUser));
+  for (const home of homes) {
+    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
+      candidates.push(join(home, ".ssh", name));
+    }
+  }
+  for (const keyPath of candidates) {
+    if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
+      return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
+    }
+  }
+  throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/");
+}
+
+async function pollApi<T>(
+  url: string,
+  check: (data: T) => boolean,
+  timeoutMs: number,
+  intervalMs = 10_000,
+): Promise<T> {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    try {
+      const res = await fetch(url);
+      if (res.ok) {
+        const data = (await res.json()) as T;
+        if (check(data)) return data;
+      }
+    } catch { /* bastion not up yet, or a network hiccup */ }
+    await sleep(intervalMs);
+  }
+  throw new Error(`Timeout after ${timeoutMs}ms polling ${url}`);
+}
+
+function requirePrerequisites(): void {
+  if (!existsSync("/usr/bin/qemu-system-aarch64")) {
+    throw new Error("qemu-system-aarch64 not installed. Run: sudo dnf install qemu-system-aarch64");
+  }
+  if (!existsSync(AAVMF)) {
+    throw new Error(`AAVMF firmware not found at ${AAVMF}. Run: sudo dnf install edk2-aarch64`);
+  }
+  if (!existsSync(IPXE_ARM64)) {
+    throw new Error(`arm64 iPXE not found at ${IPXE_ARM64}. Run: sudo dnf install ipxe-bootimgs-aarch64`);
+  }
+}
+
+/**
+ * Confirm the arm64 iPXE binary implements EFI_LOAD_FILE2_PROTOCOL.
+ *
+ * Without it the `initrd` line is accepted and silently ignored, and the kernel panics
+ * with unknown-block(0,0). Checking here turns a confusing 30-minute boot failure into
+ * an immediate, explanatory one.
+ *
+ * GUID 4006c0c1-fcb3-403e-996d-4a6c8724e06d, little-endian in the binary's GUID table.
+ */
+function assertIpxeSupportsLoadFile2(): void {
+  const LOAD_FILE2_GUID = Buffer.from("c1c00640b3fc3e40996d4a6c8724e06d", "hex");
+  const binary = readFileSync(IPXE_ARM64);
+  if (binary.indexOf(LOAD_FILE2_GUID) < 0) {
+    throw new Error(
+      `${IPXE_ARM64} does not reference EFI_LOAD_FILE2_PROTOCOL. On arm64 the kernel ` +
+      `EFI stub fetches the initrd over LoadFile2; without it iPXE drops the initrd ` +
+      `silently and the kernel panics with "unknown-block(0,0)". Rebuild iPXE with ` +
+      `LoadFile2, or chainload grubaa64.efi for aarch64 instead.`,
+    );
+  }
+  log(`iPXE arm64 implements LoadFile2 — initrd will be delivered to the EFI stub`);
+}
+
+interface Harness {
+  testDir: string;
+  app: { close: () => Promise<void> };
+  stopDnsmasq: () => void;
+  state: { update: (fn: (s: BastionStateLike) => void) => void };
+  vmMac: string;
+  httpPort: number;
+}
+
+/** Just the parts of BastionState this test seeds. */
+interface BastionStateLike {
+  discovered: Record<string, Record<string, unknown>>;
+  installed: Record<string, Record<string, unknown>>;
+  install_queue: Record<string, Record<string, unknown>>;
+  debug: Record<string, Record<string, unknown>>;
+}
+
+/** Bring up an isolated network, a bastion with both arch payloads, and an arm64 VM. */
+async function startHarness(vmName: string, httpPort: number, pubKey: string): Promise<Harness> {
+  requirePrerequisites();
+  assertIpxeSupportsLoadFile2();
+
+  log("Setting up PXE test network...");
+  ensurePxeNetwork();
+
+  const testDir = join(tmpdir(), `lab-arm-pxe-test-${Date.now()}`);
+  for (const sub of ["tftp", "http", "logs"]) {
+    mkdirSync(join(testDir, sub), { recursive: true });
+  }
+
+  const { createApp } = await import("../../src/bastion/src/server.js");
+  const { loadConfig } = await import("../../src/bastion/src/config.js");
+  const { generateDnsmasqConf, startDnsmasq, stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
+  const { generateDiscoverKickstart } = await import("../../src/bastion/src/services/kickstart-generator.js");
+  const { renderBootIpxe, kernelPath, initrdPath } = await import("../../src/bastion/src/templates/boot.ipxe.js");
+  // Relative, not "@lab/shared": these tests run from the repo root against sources,
+  // where the workspace package alias is not resolvable.
+  const { SUPPORTED_ARCHES, fedoraMirrorFor } = await import("../../src/shared/src/hardware/index.js");
+
+  const config = loadConfig({
+    bastionDir: testDir,
+    httpPort,
+    iface: "virbr-pxe",
+    serverIp: BASTION_IP,
+    network: `${PXE_SUBNET}.0`,
+    gateway: BASTION_IP,
+    dhcpMode: "full",
+    dhcpRangeStart: DHCP_RANGE_START,
+    dhcpRangeEnd: DHCP_RANGE_END,
+    domain: "arm-pxe-test.local",
+    sshKeys: [pubKey],
+    adminUser: SSH_USER,
+  });
+
+  // iPXE binaries. The arm64 one is the whole point: dnsmasq hands it out on DHCP
+  // option 93 -- 11 for UEFI PXE (TFTP) and 19 for UEFI HTTP Boot.
+  //
+  // They go in BOTH directories, exactly as main.ts stages them. AAVMF prefers HTTP
+  // Boot, so it is served an http:// URL and fetches from httpDir; a firmware that
+  // takes the TFTP path reads the same file from tftpDir. Staging only tftpDir gives a
+  // 404 and "No bootable option or device was found" on the console.
+  log("Staging iPXE binaries...");
+  const ipxeX86 = "/usr/share/ipxe/ipxe-snponly-x86_64.efi";
+  copyFileSync(IPXE_ARM64, join(config.tftpDir, "ipxe-arm64.efi"));
+  copyFileSync(IPXE_ARM64, join(config.httpDir, "ipxe-arm64.efi"));
+  if (existsSync(ipxeX86)) {
+    copyFileSync(ipxeX86, join(config.tftpDir, "ipxe.efi"));
+    copyFileSync(ipxeX86, join(config.httpDir, "ipxe.efi"));
+  }
+
+  // Fedora kernel + initrd for both architectures, cached across runs.
+  const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache";
+  execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" });
+
+  for (const arch of SUPPORTED_ARCHES) {
+    const mirror = fedoraMirrorFor(config.fedoraVersion, arch);
+    const kernelCache = join(cacheDir, `vmlinuz-${arch}`);
+    const initrdCache = join(cacheDir, `initrd-${arch}.img`);
+
+    if (!existsSync(kernelCache)) {
+      log(`Downloading Fedora ${config.fedoraVersion} ${arch} kernel...`);
+      execSync(`curl -# -L -f -o "${kernelCache}" "${mirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 600_000 });
+    }
+    if (!existsSync(initrdCache)) {
+      log(`Downloading Fedora ${config.fedoraVersion} ${arch} initrd...`);
+      execSync(`curl -# -L -f -o "${initrdCache}" "${mirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 600_000 });
+    }
+
+    // Staged under the exact names the iPXE templates will ask for.
+    copyFileSync(kernelCache, join(config.httpDir, kernelPath(arch)));
+    copyFileSync(initrdCache, join(config.httpDir, initrdPath(arch)));
+    log(`Staged ${arch}: ${kernelPath(arch)} + ${initrdPath(arch)}`);
+  }
+
+  writeFileSync(join(config.httpDir, "discover.ks"), generateDiscoverKickstart(config));
+  writeFileSync(
+    join(config.httpDir, "boot.ipxe"),
+    renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }),
+  );
+  generateDnsmasqConf(config);
+
+  const { app, state, syslog } = createApp(config);
+  await app.listen({ port: config.httpPort, host: "0.0.0.0" });
+  syslog.start();
+  log(`Bastion HTTP listening on :${config.httpPort}`);
+
+  log("Starting dnsmasq (full DHCP)...");
+  startDnsmasq(config).catch((err) => {
+    log(`dnsmasq failed: ${err instanceof Error ? err.message : String(err)}`);
+  });
+  await sleep(1500);
+
+  log("Creating aarch64 PXE VM (emulated — this is slow)...");
+  createPxeVm({
+    name: vmName,
+    memory: VM_MEMORY,
+    vcpus: VM_VCPUS,
+    diskSize: VM_DISK_GB,
+    network: PXE_NETWORK_NAME,
+    arch: "aarch64",
+  });
+
+  const vmMac = getVmMac(vmName);
+  if (!vmMac) throw new Error("Could not determine VM MAC address");
+  log(`ARM VM MAC: ${vmMac}`);
+
+  return {
+    testDir,
+    app,
+    stopDnsmasq,
+    state: state as unknown as Harness["state"],
+    vmMac,
+    httpPort: config.httpPort,
+  };
+}
+
+async function stopHarness(vmName: string, harness: Harness | undefined): Promise<void> {
+  // KEEP_VM=1 leaves the VM, network and bastion up so a failure can be inspected on
+  // the console. Emulated aarch64 runs cost half an hour; tearing the evidence down
+  // automatically means paying that again to see what happened.
+  if (process.env["KEEP_VM"] === "1") {
+    log(`KEEP_VM=1 — leaving ${vmName} running for inspection.`);
+    log(`  console:  sudo virsh screenshot ${vmName} /tmp/vm.ppm`);
+    log(`  serial:   socat - TCP:127.0.0.1:${SERIAL_PORT}`);
+    if (harness) log(`  bastion:  ${harness.testDir} (still serving on :${harness.httpPort})`);
+    log(`  cleanup:  sudo virsh destroy ${vmName}; sudo virsh undefine ${vmName} --remove-all-storage --nvram`);
+    return;
+  }
+
+  log("Cleaning up...");
+  if (harness) {
+    await harness.app.close().catch(() => {});
+    harness.stopDnsmasq();
+  }
+  destroyPxeVm(vmName);
+  destroyPxeNetwork();
+  if (harness) rmSync(harness.testDir, { recursive: true, force: true });
+}
+
+/** Read the DHCP lease the bastion handed a MAC. Rescue mode reports no IP itself. */
+function leaseIpFor(testDir: string, mac: string): string | null {
+  const leaseFile = join(testDir, "dnsmasq.leases");
+  if (!existsSync(leaseFile)) return null;
+  for (const line of readFileSync(leaseFile, "utf-8").split("\n")) {
+    // <expiry> <mac> <ip> <hostname> <clientid>
+    const parts = line.trim().split(/\s+/);
+    if (parts.length >= 3 && parts[1]?.toLowerCase() === mac.toLowerCase()) {
+      return parts[2] ?? null;
+    }
+  }
+  return null;
+}
+
+async function waitForLease(testDir: string, mac: string, timeoutMs: number): Promise<string> {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    const ip = leaseIpFor(testDir, mac);
+    if (ip !== null) return ip;
+    await sleep(5000);
+  }
+  throw new Error(`No DHCP lease for ${mac} within ${timeoutMs}ms`);
+}
+
+// ---------------------------------------------------------------------------
+// Rescue path -- what the DGX Sparks need.
+// ---------------------------------------------------------------------------
+
+describe("ARM PXE rescue", () => {
+  const VM_NAME = "lab-arm-pxe-rescue";
+  const HTTP_PORT = 8096;
+  let harness: Harness | undefined;
+  let sshKeyPath: string;
+  let rescueIp: string;
+
+  beforeAll(async () => {
+    const { pubKey, keyPath } = findSshKey();
+    sshKeyPath = keyPath;
+    harness = await startHarness(VM_NAME, HTTP_PORT, pubKey);
+    const { testDir, vmMac, state } = harness;
+
+    // Seed the machine as an already-known aarch64 box queued for rescue. This is the
+    // DGX Spark situation exactly: SSH-onboarded, never PXE-discovered, architecture
+    // known only from its record -- and it also keeps the test to a SINGLE emulated
+    // boot. Each boot spends ~15 minutes pulling Anaconda's stage2 over the network
+    // under TCG, so discovering first and rescuing second doubles the runtime for no
+    // extra coverage of the path being tested. Discovery is covered by the full suite.
+    log(`Seeding ${vmMac} as a known aarch64 machine queued for rescue...`);
+    state.update((s) => {
+      s.discovered[vmMac] = {
+        mac: vmMac,
+        product: "Test ARM64 Machine",
+        board: "virt",
+        serial: "SN-ARM64",
+        manufacturer: "QEMU",
+        cpu_model: "cortex-a57",
+        cpu_cores: VM_VCPUS,
+        memory_gb: 4,
+        arch: "aarch64",
+        disks: [],
+        nics: [],
+        first_seen: new Date().toISOString(),
+        last_seen: new Date().toISOString(),
+      };
+      s.debug[vmMac] = { hostname: "arm-rescue-test", queued_at: new Date().toISOString() };
+    });
+
+    // Restart so the VM boots against the seeded state. createPxeVm already started it.
+    rebootPxeVm(VM_NAME);
+    await sleep(5_000);
+    deleteNftablesRejectRules();
+
+    // The whole chain now runs once: DHCP option 93 -> arm64 iPXE -> /boot.ipxe ->
+    // /dispatch (architecture from the record, not the query) -> aarch64 kernel +
+    // initrd -> Anaconda rescue -> sshd. Reaching a shell at all proves iPXE handed
+    // the initrd to the EFI stub over LoadFile2; without it the kernel panics first.
+    log("Waiting for the rescue environment's DHCP lease...");
+    rescueIp = await waitForLease(testDir, vmMac, LEASE_TIMEOUT_MS);
+    log(`Rescue IP: ${rescueIp}`);
+
+    log("Waiting for SSH into the rescue shell (started by inst.sshd)...");
+    log("(emulated aarch64 — Anaconda's stage2 download dominates; be patient)");
+    await waitForSsh(rescueIp, "root", SSH_TIMEOUT_MS, sshKeyPath).catch(async (err) => {
+      log("Rescue SSH timed out. Serial console:");
+      try {
+        log(await readSerialLog(SERIAL_PORT, { lastLines: 100, timeoutMs: 15_000 }));
+      } catch { /* console unavailable */ }
+      throw err;
+    });
+    log("ARM PXE rescue reached.");
+  }, LEASE_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000);
+
+  afterAll(async () => { await stopHarness(VM_NAME, harness); });
+
+  it("resolved the architecture from the machine record", async () => {
+    const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`);
+    const data = (await res.json()) as { discovered: Record<string, { arch: string }> };
+    expect(data.discovered[harness!.vmMac]?.arch).toBe("aarch64");
+  });
+
+  it("rescue shell is reachable over SSH and is aarch64", () => {
+    const result = sshExec(rescueIp, "root", "uname -m", { keyPath: sshKeyPath, timeout: 60_000 });
+    expect(result.exitCode).toBe(0);
+    expect(result.stdout.trim()).toBe("aarch64");
+  });
+
+  it("booted an initramfs — the LoadFile2 path worked", () => {
+    // If iPXE had dropped the initrd the kernel would never have reached userspace at
+    // all, but assert it explicitly so a regression names itself.
+    const result = sshExec(rescueIp, "root", "cat /proc/cmdline; ls /run/install", {
+      keyPath: sshKeyPath, timeout: 60_000,
+    });
+    expect(result.exitCode).toBe(0);
+    expect(result.stdout).toContain("inst.rescue");
+  });
+
+  it("rescue kernel came from the bastion over HTTP", () => {
+    const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 });
+    expect(result.stdout).toContain(`${BASTION_IP}:${HTTP_PORT}`);
+    // arm64 gets serial console arguments, never nomodeset.
+    expect(result.stdout).toContain("console=ttyAMA0");
+    expect(result.stdout).not.toContain("nomodeset");
+  });
+
+  it("has LVM tools available for inspecting an installed system", () => {
+    const result = sshExec(rescueIp, "root", "command -v vgchange && command -v lsblk", {
+      keyPath: sshKeyPath, timeout: 60_000,
+    });
+    expect(result.exitCode).toBe(0);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Full install -- opt-in, ~60-90 minutes emulated.
+// ---------------------------------------------------------------------------
+
+describe.runIf(RUN_FULL_INSTALL)("ARM PXE install", () => {
+  const VM_NAME = "lab-arm-pxe-install";
+  const HTTP_PORT = 8095;
+  let harness: Harness | undefined;
+  let sshKeyPath: string;
+  let vmIp: string;
+
+  beforeAll(async () => {
+    const { pubKey, keyPath } = findSshKey();
+    sshKeyPath = keyPath;
+    harness = await startHarness(VM_NAME, HTTP_PORT, pubKey);
+    const { vmMac } = harness;
+
+    log("Waiting for aarch64 discovery...");
+    await pollApi<{ discovered: Record<string, unknown> }>(
+      `http://${BASTION_IP}:${HTTP_PORT}/api/machines`,
+      (data) => vmMac in data.discovered,
+      DISCOVERY_TIMEOUT_MS,
+    );
+    log("Discovered. Queueing install...");
+
+    const installRes = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/install`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ mac: vmMac, hostname: VM_NAME, disk: "", role: "vanilla" }),
+    });
+    expect(installRes.status).toBe(200);
+
+    await sleep(30_000);
+    rebootPxeVm(VM_NAME);
+
+    log("Waiting for the emulated aarch64 install (60-90 min)...");
+    type LogsResponse = { status: string; progress: string; ip?: string };
+    const final = await pollApi<LogsResponse>(
+      `http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`,
+      (d) => d.status === "installed" || d.progress === "error",
+      INSTALL_TIMEOUT_MS,
+      30_000,
+    );
+
+    if (final.progress === "error") {
+      const logs = await (await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`)).json();
+      log(`ARM install FAILED: ${JSON.stringify(logs, null, 2)}`);
+      throw new Error("ARM PXE install failed — see logs above");
+    }
+
+    vmIp = final.ip ?? "";
+    log(`ARM install complete. IP: ${vmIp}`);
+
+    await sleep(30_000);
+    rebootPxeVm(VM_NAME);
+    await sleep(5_000);
+    deleteNftablesRejectRules();
+    await waitForSsh(vmIp, SSH_USER, SSH_TIMEOUT_MS, sshKeyPath);
+  }, DISCOVERY_TIMEOUT_MS + INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 600_000);
+
+  afterAll(async () => { await stopHarness(VM_NAME, harness); });
+
+  it("machine reached installed state", async () => {
+    const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`);
+    const data = (await res.json()) as { installed: Record<string, { hostname: string }> };
+    expect(data.installed[harness!.vmMac]?.hostname).toBe(VM_NAME);
+  });
+
+  it("installed system is aarch64", () => {
+    const result = sshExec(vmIp, SSH_USER, "uname -m", { keyPath: sshKeyPath, timeout: 60_000 });
+    expect(result.stdout.trim()).toBe("aarch64");
+  });
+
+  it("SSH works with the admin user", () => {
+    const result = sshExec(vmIp, SSH_USER, "whoami", { keyPath: sshKeyPath, timeout: 60_000 });
+    expect(result.stdout.trim()).toBe(SSH_USER);
+  });
+
+  it("LVM layout is correct", () => {
+    const result = sshExec(vmIp, SSH_USER, "sudo lvs labvg --noheadings -o lv_name", {
+      keyPath: sshKeyPath, timeout: 60_000,
+    });
+    expect(result.exitCode).toBe(0);
+    const lvs = result.stdout.trim().split("\n").map((l) => l.trim());
+    for (const expected of ["root", "var", "varlog", "swap", "home", "srv"]) {
+      expect(lvs).toContain(expected);
+    }
+  });
+});

bastion/tests/integration/cluster-smoke.test.ts

@@ -1,164 +0,0 @@
-// Integration test: form a real 3-server k3s cluster from PXE-installed VMs.
-//
-// WHY THIS EXISTS SEPARATELY FROM THE DRILL
-// -----------------------------------------
-// The rolling drill is a ~3 hour test. Almost everything that can go wrong in
-// its first thirty minutes goes wrong here instead, in forty-five: the token
-// handoff, the cluster-init/join discriminator, the etcd bootstrap race, root
-// SSH for the module, and whether post-provision was actually suppressed.
-// Debugging those inside a three-hour test is how you spend a day per attempt.
-//
-// It asserts on ETCD MEMBERSHIP, not just `kubectl get nodes`. A node object
-// and an etcd member are different things, and the whole hazard in a rolling
-// reinstall is that they can disagree.
-//
-// Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo. Set FEDORA_MIRROR to
-// a local mirror or each install pulls the full package set over the internet.
-//
-//   sudo -E npx vitest run -c tests/integration/vitest.config.ts -t "cluster smoke"
-
-import { describe, it, expect, beforeAll, afterAll } from "vitest";
-import { existsSync, readFileSync } from "node:fs";
-import { homedir } from "node:os";
-import { join } from "node:path";
-import {
-  startTestBastion, provisionVms, destroyVms,
-  type Harness, type ProvisionedVm,
-} from "./helpers/provision-harness.js";
-import {
-  formCluster, clusterSnapshot, printSnapshot, etcdMembers, etcdHealthy,
-  allServersReady, pickJoinTarget, kubectlOn, k8sNodeName,
-} from "./helpers/k3s-cluster.js";
-import { destroyPxeNetwork } from "./helpers/pxe-network.js";
-
-const HTTP_PORT = 8097;
-const SSH_USER = "lab";
-const DISK_GB = 1000;
-const NODES = ["smoke-a", "smoke-b", "smoke-c"];
-
-function findSshKey(): { pubKey: string; keyPath: string } {
-  const homes = [homedir()];
-  const sudoUser = process.env["SUDO_USER"];
-  if (sudoUser) homes.push(join("/home", sudoUser));
-  for (const home of homes) {
-    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
-      const keyPath = join(home, ".ssh", name);
-      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
-        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
-      }
-    }
-  }
-  throw new Error("no SSH keypair found");
-}
-
-describe("cluster smoke: three servers, one cluster", () => {
-  let h: Harness;
-  let vms: ProvisionedVm[] = [];
-
-  /**
-   * Kubernetes node names, NOT libvirt domain names.
-   *
-   * k3s registers a node by FQDN, so the VM `smoke-a` is the node
-   * `smoke-a.pxe-test.local`. Asserting against the VM names would compare two
-   * different namespaces and fail for a reason that has nothing to do with the
-   * cluster.
-   */
-  const nodeNames = (): string[] => vms.map((v) => k8sNodeName(h, v)).sort();
-
-  beforeAll(async () => {
-    const { pubKey, keyPath } = findSshKey();
-    destroyVms(...NODES);
-    h = await startTestBastion({
-      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey, sshKeyPath: keyPath,
-      label: "cluster-smoke",
-      // Mandatory. Left on, post-provision installs each node into its OWN
-      // cluster and this test would be measuring three clusters of one.
-      autoK3s: false,
-    });
-  }, 20 * 60_000);
-
-  afterAll(async () => {
-    // Dump state before anything is destroyed — the VMs are the only place
-    // this information exists, and they are about to stop existing.
-    try {
-      if (vms.length > 0) printSnapshot("final", clusterSnapshot(h, vms[0]!));
-    } catch { /* cluster may already be gone */ }
-    destroyVms(...NODES);
-    try { destroyPxeNetwork(); } catch { /* shared */ }
-    await h?.close();
-  }, 10 * 60_000);
-
-  it("provisions three v2 combo nodes", async () => {
-    vms = await provisionVms(h, NODES.map((name) => ({
-      name, role: "k8s-combo" as const, schemaVersion: 2 as const,
-      diskGb: DISK_GB, memory: 6144, vcpus: 4,
-    })));
-    expect(vms).toHaveLength(3);
-    for (const vm of vms) expect(vm.ip).toMatch(/^\d+\.\d+\.\d+\.\d+$/);
-  }, 60 * 60_000);
-
-  it("forms ONE cluster, not three", async () => {
-    await formCluster(h, vms, { role: "k8s-combo" });
-
-    const snap = clusterSnapshot(h, vms[0]!);
-    printSnapshot("after formation", snap);
-
-    // The precise failure this guards: three separate single-node clusters
-    // each report 1/1 Ready and look perfectly healthy in isolation.
-    expect(snap.nodes).toHaveLength(3);
-    expect(snap.readyCount).toBe(3);
-    expect(snap.nodes.map((n) => n.name).sort()).toEqual(nodeNames());
-  }, 90 * 60_000);
-
-  it("has exactly three healthy etcd members", () => {
-    const members = etcdMembers(h, vms[0]!);
-    // .node, not .name: etcd names a member "<node>-<8 hex>", so comparing
-    // raw names against node names never matches.
-    expect(members.map((m) => m.node).sort()).toEqual(nodeNames());
-    expect(members).toHaveLength(3);
-    expect(members.every((m) => m.started)).toBe(true);
-    expect(etcdHealthy(h, vms[0]!)).toBe(true);
-  }, 5 * 60_000);
-
-  it("agrees about membership from EVERY node, not just the first", () => {
-    // Asked of each server independently: a split brain is three nodes that
-    // each believe they are correct.
-    for (const vm of vms) {
-      const members = etcdMembers(h, vm).map((m) => m.node).sort();
-      expect(members, `etcd membership as seen by ${vm.name}`).toEqual(nodeNames());
-      const nodes = clusterSnapshot(h, vm).nodes.map((n) => n.name).sort();
-      expect(nodes, `node list as seen by ${vm.name}`).toEqual(nodeNames());
-    }
-  }, 10 * 60_000);
-
-  it("reports every server ready", () => {
-    const ready = allServersReady(h, vms);
-    expect(ready.ok, ready.detail).toBe(true);
-  }, 5 * 60_000);
-
-  it("makes all three control-plane and etcd", () => {
-    const snap = clusterSnapshot(h, vms[0]!);
-    for (const n of snap.nodes) {
-      expect(n.roles, `${n.name} roles`).toContain("control-plane");
-      expect(n.roles, `${n.name} roles`).toContain("etcd");
-    }
-  }, 5 * 60_000);
-
-  it("can pick a join target other than a given node", () => {
-    // The primitive the rolling drill depends on: after the first server is
-    // wiped it must rejoin against a survivor.
-    const target = pickJoinTarget(h, vms, vms[0]!);
-    expect(target.name).not.toBe(vms[0]!.name);
-  }, 5 * 60_000);
-
-  it("carries the Longhorn disk label on every node", () => {
-    // Set via k3s node-label. Without it Longhorn creates no disk on the node,
-    // and the node then hosts zero replicas while looking entirely healthy.
-    const out = kubectlOn(h, vms[0]!,
-      `get nodes -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.labels.node\\.longhorn\\.io/create-default-disk}{"\\n"}{end}'`);
-    for (const vm of vms) {
-      expect(out, `${vm.name} create-default-disk label`)
-        .toContain(`${k8sNodeName(h, vm)}=config`);
-    }
-  }, 5 * 60_000);
-});

bastion/tests/integration/data-disks.test.ts

@@ -1,173 +0,0 @@
-// Integration test: dedicated data disks, on a real VM.
-//
-// The unit tests prove the kickstart RENDERS the right %pre and part lines.
-// This proves anaconda ACTUALLY PRODUCES the layout thebeast will get: OS on
-// one disk, /var/lib/longhorn and /var/lib/rancher each a whole-disk labelled
-// xfs partition on their own disks -- and that a reprovision with NO flags
-// rediscovers both by label and remounts them without reformatting.
-//
-// The VM mirrors thebeast's topology (1 OS disk + 2 data disks addressed by
-// /dev/disk/by-id/ serial); virtio serials stand in for NVMe serials, which is
-// exactly the aliasing the feature is specified over (any by-id path).
-//
-// Run (on a libvirt host):
-//   sudo -E npx vitest run -c tests/integration/vitest.config.ts tests/integration/data-disks.test.ts
-//
-// Set FEDORA_MIRROR to a local mirror or each install pulls packages from the
-// internet (~12-15 min per install either way; two installs in this file).
-
-import { describe, it, expect, beforeAll, afterAll } from "vitest";
-import { existsSync, readFileSync } from "node:fs";
-import { homedir } from "node:os";
-import { join } from "node:path";
-import { v2Layout, DATA_DISK_LABELS } from "../../src/shared/src/types/layout.js";
-import {
-  startTestBastion, provisionVm, reprovisionVm, readLvs, readSchemaTag,
-  writeMarkers, readMarkers, destroyVms, virtioById,
-  type Harness, type ProvisionedVm,
-} from "./helpers/provision-harness.js";
-import { destroyPxeNetwork } from "./helpers/pxe-network.js";
-import { sshExec } from "./helpers/ssh.js";
-
-const HTTP_PORT = 8094;
-const SSH_USER = "lab";
-const DISK_GB = 1000; // same sparse OS disk as every other layout test
-
-// Short virtio serials (virtio-blk truncates at 20 chars), echoing thebeast's
-// real disk serial fragments so the drill log reads like the runbook.
-const LH = { serial: "LHDRILL7F600711", sizeGb: 20 };
-const RN = { serial: "RNDRILL7FD4070C", sizeGb: 20 };
-
-const PRESERVED = ["/home", "/srv", "/var/lib/longhorn", "/var/lib/rancher"];
-
-function findSshKey(): { pubKey: string; keyPath: string } {
-  // A passphrase-protected default key fails SILENTLY here: BatchMode ssh
-  // never offers it without an agent, the server logs only "Connection closed
-  // [preauth]", and the test reads as "installed machine never came up" when
-  // the machine was fine. LAB_TEST_SSH_KEY names a dedicated passphrase-less
-  // keypair and sidesteps the whole class.
-  const envKey = process.env["LAB_TEST_SSH_KEY"];
-  if (envKey) {
-    if (!existsSync(envKey) || !existsSync(`${envKey}.pub`)) {
-      throw new Error(`LAB_TEST_SSH_KEY=${envKey}: key or .pub missing`);
-    }
-    return { pubKey: readFileSync(`${envKey}.pub`, "utf-8").trim(), keyPath: envKey };
-  }
-  const homes = [homedir()];
-  const sudoUser = process.env["SUDO_USER"];
-  if (sudoUser) homes.push(join("/home", sudoUser));
-  for (const home of homes) {
-    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
-      const keyPath = join(home, ".ssh", name);
-      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
-        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
-      }
-    }
-  }
-  throw new Error("no SSH keypair found");
-}
-
-/** Run a command on the VM and insist it succeeds. */
-function ssh(h: Harness, vm: ProvisionedVm, cmd: string): string {
-  const r = sshExec(vm.ip, h.sshUser, cmd, { keyPath: h.sshKeyPath });
-  if (r.exitCode !== 0) throw new Error(`[${vm.name}] \`${cmd}\` failed: ${r.stderr || r.stdout}`);
-  return r.stdout.trim();
-}
-
-/**
- * What actually backs a mount point, and whether it is the whole-disk labelled
- * partition the feature promises.
- */
-function inspectMount(h: Harness, vm: ProvisionedVm, mount: string, serial: string) {
-  const source = ssh(h, vm, `findmnt -no SOURCE ${mount}`);
-  const fstype = ssh(h, vm, `findmnt -no FSTYPE ${mount}`);
-  const label = ssh(h, vm, `sudo blkid -s LABEL -o value ${source}`);
-  const uuid = ssh(h, vm, `sudo blkid -s UUID -o value ${source}`);
-  // The partition must live on the disk with OUR serial, not wherever the
-  // kernel happened to enumerate something.
-  const disk = ssh(h, vm, `readlink -f ${virtioById(serial)}`);
-  const parent = "/dev/" + ssh(h, vm, `lsblk -ndo pkname ${source}`);
-  return { source, fstype, label, uuid, disk, parent };
-}
-
-describe("dedicated data disks on real hardware", () => {
-  let h: Harness;
-  const created: string[] = [];
-
-  beforeAll(async () => {
-    const { pubKey, keyPath } = findSshKey();
-    h = await startTestBastion({
-      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey,
-      sshKeyPath: keyPath, label: "datadisks",
-    });
-  }, 20 * 60_000);
-
-  afterAll(async () => {
-    // KEEP_VMS=1 leaves the VM behind for a post-mortem (virsh console,
-    // screenshot, mounting the qcow2). Without it a failed boot is destroyed
-    // along with every clue about why it failed.
-    if (!process.env["KEEP_VMS"]) destroyVms(...created);
-    try { await h?.close(); } catch { /* closed */ }
-    try { destroyPxeNetwork(); } catch { /* gone */ }
-  }, 5 * 60_000);
-
-  it("k8s-worker with longhorn+rancher disks: install, then flagless reprovision rediscovers both", async () => {
-    const name = "dd-worker";
-    created.push(name);
-    const dataDisks = { longhorn: virtioById(LH.serial), rancher: virtioById(RN.serial) };
-
-    // --- Fresh install, thebeast-style -----------------------------------
-    let vm = await provisionVm(h, {
-      name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB,
-      dataDisks: { longhorn: LH, rancher: RN },
-    });
-
-    // The VG must hold exactly the v2 set MINUS the LVs the disks replace.
-    const lvs = readLvs(h, vm);
-    const expected = v2Layout("k8s-worker", { dataDisks }).map((lv) => lv.name).sort();
-    expect(Object.keys(lvs).sort(), "volume set").toEqual(expected);
-    expect(lvs["longhorn"]).toBeUndefined();
-    expect(lvs["rancher"]).toBeUndefined();
-    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");
-
-    // Each mount is a whole-disk xfs partition, labelled, on the RIGHT disk.
-    const lhBefore = inspectMount(h, vm, "/var/lib/longhorn", LH.serial);
-    expect(lhBefore.fstype).toBe("xfs");
-    expect(lhBefore.label).toBe(DATA_DISK_LABELS.longhorn);
-    expect(lhBefore.parent, "/var/lib/longhorn sits on the longhorn disk").toBe(lhBefore.disk);
-
-    const rnBefore = inspectMount(h, vm, "/var/lib/rancher", RN.serial);
-    expect(rnBefore.fstype).toBe("xfs");
-    expect(rnBefore.label).toBe(DATA_DISK_LABELS.rancher);
-    expect(rnBefore.parent, "/var/lib/rancher sits on the rancher disk").toBe(rnBefore.disk);
-
-    const token = writeMarkers(h, vm, PRESERVED);
-
-    // --- Reprovision with NO dataDisks flags ------------------------------
-    // This is the claim that matters for day 2: the labels on disk are the
-    // truth, and a plain `labctl provision reprovision` must find and keep
-    // them. Passing the flags again here would hide a regression in exactly
-    // that rediscovery.
-    vm = await reprovisionVm(h, vm, "k8s-worker");
-
-    const markers = readMarkers(h, vm, PRESERVED);
-    for (const mount of PRESERVED) {
-      expect(markers[mount], `${mount} must survive a reinstall`).toBe(token);
-    }
-
-    // Still LV-less for longhorn/rancher, still v2.
-    const lvsAfter = readLvs(h, vm);
-    expect(Object.keys(lvsAfter).sort()).toEqual(expected);
-    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");
-
-    // Same partitions, same filesystems. An unchanged UUID is the proof that
-    // --noformat really happened -- a remade xfs with the same label would
-    // pass every other check here.
-    const lhAfter = inspectMount(h, vm, "/var/lib/longhorn", LH.serial);
-    expect(lhAfter.uuid, "longhorn fs was NOT reformatted").toBe(lhBefore.uuid);
-    expect(lhAfter.parent).toBe(lhAfter.disk);
-    const rnAfter = inspectMount(h, vm, "/var/lib/rancher", RN.serial);
-    expect(rnAfter.uuid, "rancher fs was NOT reformatted").toBe(rnBefore.uuid);
-    expect(rnAfter.parent).toBe(rnAfter.disk);
-  }, 150 * 60_000);
-});

bastion/tests/integration/helpers/k3s-cluster.ts

@@ -1,427 +0,0 @@
-// Form and inspect a real multi-node k3s cluster from an integration test.
-//
-// WHY THIS EXISTS
-// ---------------
-// Nothing in the repo decides which node bootstraps a cluster and which nodes
-// join it. The production generator has the right discriminator --
-// generateServerConfig emits `cluster-init: true` when no server URL is set and
-// `server:`+`token:` when one is -- but the choice is an INPUT, and the only
-// caller that ever supplied it was a hand-run labsim shell script.
-//
-// So a test that wants three servers in one cluster has to sequence it. That
-// sequencing is here rather than in a test file because the rolling drill is
-// not the only thing that will want it: an etcd-recovery test (recoverEtcdMember
-// still has no caller), a dual-stack conversion and any HA upgrade test all
-// need exactly these primitives.
-//
-// Everything drives K3sModule, not shell. That is deliberate and is the same
-// rule labsim follows by shelling out to the compiled render-config binary: a
-// test that reimplements the install proves the test works, not the product.
-
-import { execSync } from "node:child_process";
-import type { Role } from "../../../src/shared/src/types/state.js";
-import { K3sModule } from "../../../src/modules/modules/k3s/src/k3s-module.js";
-import type { ModuleContext } from "../../../src/modules/src/types.js";
-import { log } from "./libvirt.js";
-import { parseEtcdMembers, type EtcdMember } from "../../../src/modules/modules/k3s/src/operations/etcd-members.js";
-import { sshExec } from "./ssh.js";
-import type { Harness, ProvisionedVm } from "./provision-harness.js";
-
-export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
-
-/**
- * A pre-shared cluster token, the way labsim does it.
- *
- * The alternative is scraping /var/lib/rancher/k3s/server/node-token off the
- * first server, and nothing in the repo does that. It is also the wrong shape
- * for a rolling migration: the first server gets wiped and rejoins, so the
- * token has to outlive it and be knowable before it exists.
- */
-export const DRILL_TOKEN = "lab-drill-shared-token";
-
-/**
- * k3s operations run unprefixed `mkdir -p /etc/rancher/k3s` and
- * `systemctl restart k3s`, so they need root. install.ks.ts provisions
- * `sshkey --username=root` and sets PermitRootLogin prohibit-password, so root
- * SSH works on every machine this harness builds. The harness's own `lab` user
- * stays for the sudo-based reads (readLvs, markers).
- */
-export const MODULE_SSH_USER = "root";
-
-/**
- * The k3s version the drill runs, pinned to what the fleet runs.
- *
- * get.k3s.io installs LATEST, and latest is currently two minor versions ahead
- * of production (v1.36.4 vs v1.34.6). Two things follow, and both matter:
- *
- *  - a rehearsal on a different minor version is rehearsing a different
- *    system, and Longhorn in particular is only qualified against a range of
- *    Kubernetes versions;
- *  - it is the same trap production would hit. A node reinstalled mid-migration
- *    comes back on latest while its peers stay put, and Kubernetes supports
- *    only ONE minor of skew between control-plane members.
- *
- * Override with LAB_K3S_VERSION to test an upgrade deliberately.
- */
-export const PINNED_K3S_VERSION = process.env["LAB_K3S_VERSION"] ?? "v1.34.6+k3s1";
-
-/** Build the ModuleContext K3sModule expects for one VM. */
-export function moduleCtx(
-  h: Harness,
-  vm: ProvisionedVm,
-  role: Role,
-  config: Record<string, unknown> = {},
-): ModuleContext {
-  return {
-    hostname: vm.name,
-    ip: vm.ip,
-    role,
-    os: "fedora-43",
-    arch: "x86_64",
-    sshUser: MODULE_SSH_USER,
-    sshKeyPath: h.sshKeyPath,
-    config,
-  };
-}
-
-/** Run a command on a node as root, throwing with context on failure. */
-export function nodeExec(h: Harness, vm: ProvisionedVm, cmd: string, timeoutMs = 120_000): string {
-  const r = sshExec(vm.ip, MODULE_SSH_USER, cmd, { keyPath: h.sshKeyPath, timeout: timeoutMs });
-  if (r.exitCode !== 0) {
-    throw new Error(`[${vm.name}] ${cmd} failed (${r.exitCode}): ${r.stderr.trim() || r.stdout.trim()}`);
-  }
-  return r.stdout;
-}
-
-/** Same, but returns the result instead of throwing. */
-export function tryNodeExec(h: Harness, vm: ProvisionedVm, cmd: string, timeoutMs = 120_000) {
-  return sshExec(vm.ip, MODULE_SSH_USER, cmd, { keyPath: h.sshKeyPath, timeout: timeoutMs });
-}
-
-/**
- * The name Kubernetes knows this VM by.
- *
- * NOT the libvirt domain name. k3s registers a node under its FQDN, so a VM
- * called `smoke-a` appears as `smoke-a.pxe-test.local`. Passing the VM name to
- * `kubectl get node` silently finds nothing, and every wait built on it times
- * out instead of failing usefully. Read from the node itself, the same way the
- * production operations do (`hostname -f`), so the two can never disagree.
- */
-const nodeNameCache = new Map<string, string>();
-export function k8sNodeName(h: Harness, vm: ProvisionedVm): string {
-  const cached = nodeNameCache.get(vm.name);
-  if (cached !== undefined) return cached;
-  const name = nodeExec(h, vm, "hostname -f 2>/dev/null || hostname").trim();
-  if (name === "") throw new Error(`[${vm.name}] could not resolve node name`);
-  nodeNameCache.set(vm.name, name);
-  return name;
-}
-
-/** Forget cached names — a reinstalled machine re-registers. */
-export function forgetNodeName(vm: ProvisionedVm): void {
-  nodeNameCache.delete(vm.name);
-}
-
-/** kubectl against a server node. */
-export function kubectlOn(h: Harness, server: ProvisionedVm, args: string, timeoutMs = 120_000): string {
-  return nodeExec(h, server, `k3s kubectl ${args}`, timeoutMs);
-}
-
-export function kubectlJson<T>(h: Harness, server: ProvisionedVm, args: string): T {
-  return JSON.parse(kubectlOn(h, server, `${args} -o json`)) as T;
-}
-
-/**
- * Wait until a server's API answers /ping.
- *
- * Straight from labsim-k8s-etcd.sh, whose comment is the whole justification:
- * a joiner that starts before the first server has finished bootstrapping etcd
- * races it, and the failure looks like etcd instability rather than a startup
- * order problem. Probed from the HOST, so it does not depend on the joining
- * node being reachable yet.
- */
-export async function waitForApiPing(vm: ProvisionedVm, timeoutMs = 450_000): Promise<void> {
-  const start = Date.now();
-  let lastErr = "";
-  while (Date.now() - start < timeoutMs) {
-    try {
-      const out = execSync(
-        `curl -sk --max-time 5 https://${vm.ip}:6443/ping || true`,
-        { encoding: "utf-8", timeout: 15_000 },
-      ).trim();
-      if (out === "pong") {
-        log(`[${vm.name}] API answering after ${Math.round((Date.now() - start) / 1000)}s`);
-        return;
-      }
-      lastErr = out;
-    } catch (e) {
-      lastErr = e instanceof Error ? e.message : String(e);
-    }
-    await sleep(5_000);
-  }
-  throw new Error(`[${vm.name}] API never answered /ping in ${timeoutMs / 1000}s (last: ${lastErr || "no response"})`);
-}
-
-/** Wait for a node to appear Ready in the cluster, as seen from `server`. */
-export async function waitForNodeReady(
-  h: Harness,
-  server: ProvisionedVm,
-  /** The VM to wait for; its Kubernetes name is resolved from the machine. */
-  target: ProvisionedVm,
-  // Generous: a node is NotReady until its CNI is up, and on the cluster-init
-  // server that means waiting for Cilium to install and pull its images. Five
-  // minutes is enough on a warm host and not enough on a cold one, and the
-  // cost of being wrong is a wasted multi-hour run.
-  timeoutMs = 900_000,
-): Promise<void> {
-  const nodeName = k8sNodeName(h, target);
-  const start = Date.now();
-  let last = "";
-  while (Date.now() - start < timeoutMs) {
-    const r = tryNodeExec(h, server,
-      `k3s kubectl get node ${nodeName} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'`);
-    last = r.stdout.trim();
-    if (last === "True") {
-      log(`[${nodeName}] Ready after ${Math.round((Date.now() - start) / 1000)}s`);
-      return;
-    }
-    await sleep(5_000);
-  }
-  throw new Error(`[${nodeName}] never became Ready in ${timeoutMs / 1000}s (last status: ${last || "absent"})`);
-}
-
-/**
- * etcdctl invocation.
- *
- * NOT `k3s etcdctl` -- that is not a subcommand ("No help topic for 'etcdctl'",
- * verified on a live v1.34 node). The binary comes from the `etcd` rpm, which
- * our kickstart does NOT install, so ensureEtcdctl() has to put it there.
- * Cert paths verified against a real node.
- */
-const ETCD_BASE = "/var/lib/rancher/k3s/server/tls/etcd";
-const ETCDCTL =
-  `ETCDCTL_API=3 etcdctl ` +
-  `--cacert=${ETCD_BASE}/server-ca.crt ` +
-  `--cert=${ETCD_BASE}/server-client.crt ` +
-  `--key=${ETCD_BASE}/server-client.key ` +
-  `--endpoints=https://127.0.0.1:2379`;
-
-/** Install etcdctl if the node does not have it. Idempotent, ~30s once. */
-export function ensureEtcdctl(h: Harness, server: ProvisionedVm): void {
-  const has = tryNodeExec(h, server, "command -v etcdctl");
-  if (has.exitCode === 0) return;
-  log(`[${server.name}] installing etcdctl (the etcd rpm; kickstart does not include it)`);
-  nodeExec(h, server, "dnf install -y etcd >/dev/null 2>&1 || dnf install -y etcd-ctl >/dev/null 2>&1", 300_000);
-  nodeExec(h, server, "command -v etcdctl");
-}
-
-
-/**
- * etcd membership, read with etcdctl against k3s's own certs.
- *
- * Read rather than inferred from `kubectl get nodes`: a node object can be
- * deleted while its etcd member survives, and that stale member is exactly the
- * failure a rolling reinstall creates. Counting nodes would not see it.
- */
-export function etcdMembers(h: Harness, server: ProvisionedVm): EtcdMember[] {
-  // NOT `-w json`. Two reasons, and the second is the dangerous one:
-  //
-  //  1. `member remove` requires the ID in HEX; the JSON writer emits decimal.
-  //  2. etcd member IDs are uint64. JSON.parse turns them into JS numbers,
-  //     which silently lose precision above 2^53 -- the ID that comes back is
-  //     an APPROXIMATION of the real one. Passing that to `member remove`
-  //     fails with "value out of range" if you are lucky, and in principle
-  //     could match some other member if you are not. This happens at the one
-  //     moment it must not: the node is already stopped and about to be wiped.
-  //
-  // The default writer prints the ID as hex text, so it never becomes a
-  // number at all:
-  //   825852d3683bc406, started, worker2-...-5f7ad0c9, https://...:2380, ...
-  return parseEtcdMembers(nodeExec(h, server, `${ETCDCTL} member list`));
-}
-
-
-/** Are all etcd endpoints healthy? Answers the question quorum actually asks. */
-export function etcdHealthy(h: Harness, server: ProvisionedVm): boolean {
-  const r = tryNodeExec(h, server, `${ETCDCTL} endpoint health`);
-  return r.exitCode === 0 && r.stdout.includes("is healthy");
-}
-
-/**
- * Remove an etcd member by node name.
- *
- * Must happen BEFORE the node is wiped. k3s-install.ts removes the local
- * datastore when a server joins, so a reinstalled node rejoins as a NEW member
- * while the old record persists: 3 members becomes 4, quorum rises from 2 to 3,
- * and the cluster that tolerated one failure now tolerates none.
- */
-export function removeEtcdMember(h: Harness, peer: ProvisionedVm, nodeName: string): boolean {
-  const members = etcdMembers(h, peer);
-  const target = members.find((m) => m.node === nodeName || m.name === nodeName);
-  if (!target) {
-    log(`[etcd] no member for ${nodeName} (have: ${members.map((m) => m.node).join(",")})`);
-    return false;
-  }
-  nodeExec(h, peer, `${ETCDCTL} member remove ${target.id}`);
-  log(`[etcd] removed member ${target.name} (${target.id})`);
-  return true;
-}
-
-export interface ClusterSnapshot {
-  nodes: { name: string; ready: boolean; roles: string }[];
-  readyCount: number;
-  etcdMembers: number;
-  etcdHealthy: boolean;
-}
-
-/** One read of cluster state, in the shape labsim's cmd_status prints. */
-export function clusterSnapshot(h: Harness, server: ProvisionedVm): ClusterSnapshot {
-  const raw = kubectlJson<{ items: {
-    metadata: { name: string; labels?: Record<string, string> };
-    status: { conditions?: { type: string; status: string }[] };
-  }[] }>(h, server, "get nodes");
-
-  const nodes = raw.items.map((n) => ({
-    name: n.metadata.name,
-    ready: (n.status.conditions ?? []).some((c) => c.type === "Ready" && c.status === "True"),
-    roles: Object.keys(n.metadata.labels ?? {})
-      .filter((k) => k.startsWith("node-role.kubernetes.io/"))
-      .map((k) => k.replace("node-role.kubernetes.io/", ""))
-      .sort().join(",") || "<none>",
-  }));
-
-  let members = 0;
-  let healthy = false;
-  try {
-    members = etcdMembers(h, server).length;
-    healthy = etcdHealthy(h, server);
-  } catch { /* an agent, or the API is down — report zero rather than throw */ }
-
-  return { nodes, readyCount: nodes.filter((n) => n.ready).length, etcdMembers: members, etcdHealthy: healthy };
-}
-
-/**
- * Print a snapshot. Called liberally during a drill: the VMs are destroyed at
- * the end, so this transcript is the only surviving record of what happened.
- */
-export function printSnapshot(label: string, s: ClusterSnapshot): void {
-  log(`[snapshot] ${label}: ${s.readyCount}/${s.nodes.length} Ready, ` +
-      `etcd members=${s.etcdMembers} healthy=${s.etcdHealthy}`);
-  for (const n of s.nodes) log(`             ${n.ready ? "Ready   " : "NotReady"} ${n.name} (${n.roles})`);
-}
-
-/**
- * Install k3s on one node via the production module.
- *
- * Pass no serverUrl to bootstrap the cluster (the token still seeds), or a
- * serverUrl to join an existing one.
- */
-export async function installK3sNode(h: Harness, vm: ProvisionedVm, opts: {
-  role: Role;
-  serverUrl?: string;
-  token?: string;
-}): Promise<void> {
-  const mod = new K3sModule();
-  const config: Record<string, unknown> = {
-    k3sToken: opts.token ?? DRILL_TOKEN,
-    k3sVersion: PINNED_K3S_VERSION,
-    ...(opts.serverUrl === undefined ? {} : { k3sServerUrl: opts.serverUrl }),
-  };
-
-  log(`[${vm.name}] installing k3s (${opts.role}${opts.serverUrl ? ` -> ${opts.serverUrl}` : " — cluster-init"})`);
-  const install = await mod.install(moduleCtx(h, vm, opts.role, config));
-  for (const line of install.output) log(`  [${vm.name}] ${line}`);
-  if (!install.success) {
-    throw new Error(`[${vm.name}] k3s install failed: ${install.errors.join("; ")}`);
-  }
-
-  const configure = await mod.configure(moduleCtx(h, vm, opts.role, config));
-  for (const line of configure.output) log(`  [${vm.name}] ${line}`);
-  if (!configure.success) {
-    throw new Error(`[${vm.name}] k3s configure failed: ${configure.errors.join("; ")}`);
-  }
-}
-
-export function serverUrlOf(vm: ProvisionedVm): string {
-  return `https://${vm.ip}:6443`;
-}
-
-/**
- * Form a cluster from a list of server VMs, strictly in order.
- *
- * The first bootstraps etcd and installs the CNI (k3s-module runs networking
- * only on the cluster-init node -- joiners get Cilium via its DaemonSet). The
- * rest join one at a time, each gated on the previous node being Ready.
- *
- * Serial on purpose. labsim gets away with starting joiners together because
- * its nodes are cloud-init VMs that poll /ping themselves; here the few extra
- * minutes buy determinism in the one place where etcd bootstrap genuinely
- * races.
- */
-export async function formCluster(h: Harness, servers: ProvisionedVm[], opts: {
-  role?: Role;
-  token?: string;
-} = {}): Promise<void> {
-  if (servers.length === 0) throw new Error("formCluster: no servers");
-  const role = opts.role ?? "k8s-combo";
-  const token = opts.token ?? DRILL_TOKEN;
-  const first = servers[0]!;
-
-  await installK3sNode(h, first, { role, token });
-  await waitForApiPing(first);
-  await waitForNodeReady(h, first, first);
-  ensureEtcdctl(h, first);
-  printSnapshot("after cluster-init", clusterSnapshot(h, first));
-
-  for (const vm of servers.slice(1)) {
-    await waitForApiPing(first);
-    await installK3sNode(h, vm, { role, token, serverUrl: serverUrlOf(first) });
-    await waitForNodeReady(h, first, vm);
-    ensureEtcdctl(h, vm);
-    printSnapshot(`after joining ${vm.name}`, clusterSnapshot(h, first));
-  }
-}
-
-/** Join an agent. Agents cannot start without a server URL and token. */
-export async function joinAgent(h: Harness, vm: ProvisionedVm, opts: {
-  server: ProvisionedVm;
-  role?: Role;
-  token?: string;
-}): Promise<void> {
-  await waitForApiPing(opts.server);
-  await installK3sNode(h, vm, {
-    role: opts.role ?? "k8s-worker",
-    token: opts.token ?? DRILL_TOKEN,
-    serverUrl: serverUrlOf(opts.server),
-  });
-  await waitForNodeReady(h, opts.server, vm);
-}
-
-/**
- * A healthy server that is NOT the node being worked on.
- *
- * Needed because the first server is itself rolled during a migration: after
- * it is wiped it must rejoin against a survivor, so "the first node" cannot be
- * the join target and cannot be where cluster state is read from.
- */
-export function pickJoinTarget(
-  h: Harness,
-  servers: ProvisionedVm[],
-  exclude: ProvisionedVm,
-): ProvisionedVm {
-  for (const vm of servers) {
-    if (vm.name === exclude.name) continue;
-    const r = tryNodeExec(h, vm, "k3s kubectl get --raw /readyz");
-    if (r.exitCode === 0 && r.stdout.trim() === "ok") return vm;
-  }
-  throw new Error(`No healthy server other than ${exclude.name} — cluster has no quorum to join against`);
-}
-
-/** Every server reports /readyz ok. Asserted positively, not inferred. */
-export function allServersReady(h: Harness, servers: ProvisionedVm[]): { ok: boolean; detail: string } {
-  const results = servers.map((vm) => {
-    const r = tryNodeExec(h, vm, "k3s kubectl get --raw /readyz");
-    return `${vm.name}=${r.exitCode === 0 ? r.stdout.trim() : `ERR(${r.exitCode})`}`;
-  });
-  return { ok: results.every((r) => r.endsWith("=ok")), detail: results.join(" ") };
-}

bastion/tests/integration/helpers/longhorn.ts

@@ -1,666 +0,0 @@
-// Install Longhorn on a test cluster and put verifiable data in it.
-//
-// WHAT THIS IS FOR
-// ----------------
-// The rolling drill's whole claim is "the bytes survived". That needs data
-// whose integrity can be checked independently of Longhorn's own opinion of
-// itself -- Longhorn reporting `robustness: healthy` is not evidence that the
-// file is intact, it is evidence that Longhorn thinks it is.
-//
-// So: write a known blob, record its sha256, and re-verify THROUGH A POD
-// RESTART every time. The restart is not incidental. It forces a real
-// detach/attach, frequently onto a different node, which is the only way to
-// show the data followed the volume rather than sitting in a page cache on the
-// node that wrote it.
-//
-// Longhorn version is pinned to the fleet's (1.8.1) so the drill rehearses what
-// production runs, not what happens to be latest.
-
-import type { Harness, ProvisionedVm } from "./provision-harness.js";
-import { log } from "./libvirt.js";
-import { nodeExec, tryNodeExec, kubectlOn, kubectlJson, sleep, k8sNodeName } from "./k3s-cluster.js";
-
-export const LONGHORN_VERSION = "1.8.1";
-export const LONGHORN_NS = "longhorn-system";
-
-/**
- * Install Longhorn via k3s's built-in helm controller.
- *
- * A HelmChart manifest dropped in the server manifests directory is how k3s
- * installs charts natively, so this needs no helm binary on the host and no
- * kubeconfig plumbing.
- *
- * The namespace is created FIRST, with privileged Pod Security labels. Longhorn
- * needs hostPath and privileged containers; under the default `restricted`
- * enforcement its DaemonSets are rejected and the install fails in a way that
- * looks like a chart problem.
- *
- * `createDefaultDiskLabeledNodes: true` matches production, and is the setting
- * that makes the node label and disk annotation load-bearing: without both, a
- * node gets no disk and silently hosts no replicas.
- */
-/**
- * S3 backup target for the drill, from the environment.
- *
- * Absent means the backup/restore phase is skipped rather than silently
- * "passing" -- a backup test that quietly does nothing is worse than no test.
- * The run script exports these; see DRILL_S3_* in run-drill.sh.
- */
-export interface BackupTargetOpts {
-  /** e.g. s3://longhorn-drill-backup@us-east-1/ */
-  target: string;
-  endpoint: string;
-  accessKey: string;
-  secretKey: string;
-}
-
-export function backupTargetFromEnv(): BackupTargetOpts | undefined {
-  const target = process.env["DRILL_S3_TARGET"];
-  const endpoint = process.env["DRILL_S3_ENDPOINT"];
-  const accessKey = process.env["DRILL_S3_ACCESS_KEY"];
-  const secretKey = process.env["DRILL_S3_SECRET_KEY"];
-  if (target === undefined || endpoint === undefined
-    || accessKey === undefined || secretKey === undefined) return undefined;
-  return { target, endpoint, accessKey, secretKey };
-}
-
-export const BACKUP_SECRET = "longhorn-backup-credentials";
-
-export async function installLonghorn(h: Harness, server: ProvisionedVm, opts: {
-  defaultReplicas?: number;
-  backup?: BackupTargetOpts;
-} = {}): Promise<void> {
-  const replicas = opts.defaultReplicas ?? 3;
-
-  log(`[longhorn] creating ${LONGHORN_NS} with privileged PSA labels`);
-  nodeExec(h, server, `k3s kubectl apply -f - <<'EOF'
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: ${LONGHORN_NS}
-  labels:
-    pod-security.kubernetes.io/enforce: privileged
-    pod-security.kubernetes.io/audit: privileged
-    pod-security.kubernetes.io/warn: privileged
-EOF`);
-
-  if (opts.backup !== undefined) {
-    // Secret BEFORE the chart: Longhorn reads it by name from a defaultSetting,
-    // and a missing secret makes the first backup fail with a credential error
-    // that reads like a network problem.
-    //
-    // AWS_ENDPOINTS is what makes this work against a non-AWS gateway; without
-    // it the driver resolves real AWS endpoints.
-    log(`[longhorn] configuring backup target ${opts.backup.target}`);
-    // tryNodeExec, not nodeExec: nodeExec throws with the whole command in the
-    // message, and this command contains the S3 secret. A credential landing in
-    // a log that then gets pasted into drill evidence is not acceptable, so the
-    // failure is reported without the payload.
-    const secretApplied = tryNodeExec(h, server, `k3s kubectl -n ${LONGHORN_NS} apply -f - <<'EOF'
-apiVersion: v1
-kind: Secret
-metadata:
-  name: ${BACKUP_SECRET}
-  namespace: ${LONGHORN_NS}
-stringData:
-  AWS_ACCESS_KEY_ID: ${opts.backup.accessKey}
-  AWS_SECRET_ACCESS_KEY: ${opts.backup.secretKey}
-  AWS_ENDPOINTS: ${opts.backup.endpoint}
-EOF`);
-    if (secretApplied.exitCode !== 0) {
-      throw new Error(
-        `[${server.name}] could not create the ${BACKUP_SECRET} secret ` +
-        `(exit ${secretApplied.exitCode}). Command withheld: it contains credentials. ` +
-        `stderr: ${secretApplied.stderr.trim().slice(0, 200)}`);
-    }
-  }
-
-  log(`[longhorn] installing chart ${LONGHORN_VERSION} via k3s helm controller`);
-  nodeExec(h, server, `cat > /var/lib/rancher/k3s/server/manifests/longhorn.yaml <<'EOF'
-apiVersion: helm.cattle.io/v1
-kind: HelmChart
-metadata:
-  name: longhorn
-  namespace: kube-system
-spec:
-  chart: longhorn
-  repo: https://charts.longhorn.io
-  version: ${LONGHORN_VERSION}
-  targetNamespace: ${LONGHORN_NS}
-  valuesContent: |-
-    defaultSettings:
-      defaultDataPath: /var/lib/longhorn
-      defaultReplicaCount: ${replicas}
-      createDefaultDiskLabeledNodes: true
-      defaultDataLocality: best-effort
-      replicaSoftAntiAffinity: false
-      replicaReplenishmentWaitInterval: 600
-      orphanAutoDeletion: true
-    persistence:
-      defaultClass: true
-      defaultClassReplicaCount: ${replicas}
-      defaultFsType: ext4
-    longhornUI:
-      replicas: 1
-EOF`);
-}
-
-/**
- * Every expected node has a Longhorn disk that is actually schedulable.
- *
- * Takes the VMs, not names, and resolves each one's Kubernetes name itself.
- * Longhorn keys its node records by the FQDN the kubelet registered
- * (`drill-s1.pxe-test.local`), never the libvirt domain name (`drill-s1`), so
- * a caller passing VM names matches nothing and waits out the full timeout
- * while the cluster is in fact perfectly healthy.
- */
-export async function waitForLonghornReady(
-  h: Harness,
-  server: ProvisionedVm,
-  vms: ProvisionedVm[],
-  timeoutMs = 20 * 60_000,
-): Promise<void> {
-  const expectedNodes = vms.map((vm) => k8sNodeName(h, vm));
-  const start = Date.now();
-  let last = "";
-  while (Date.now() - start < timeoutMs) {
-    const r = tryNodeExec(h, server,
-      `k3s kubectl -n ${LONGHORN_NS} get nodes.longhorn.io -o json`, 60_000);
-    if (r.exitCode === 0) {
-      try {
-        const parsed = JSON.parse(r.stdout) as { items: {
-          metadata: { name: string };
-          status?: { diskStatus?: Record<string, { conditions?: { type: string; status: string }[] }> };
-        }[] };
-        const ready = parsed.items.filter((n) => {
-          const disks = Object.values(n.status?.diskStatus ?? {});
-          return disks.length > 0 && disks.some((d) =>
-            (d.conditions ?? []).some((c) => c.type === "Schedulable" && c.status === "True"));
-        }).map((n) => n.metadata.name);
-
-        last = `${ready.length}/${expectedNodes.length} with schedulable disk: ${ready.join(",") || "none"}`;
-        if (expectedNodes.every((n) => ready.includes(n))) {
-          log(`[longhorn] ready after ${Math.round((Date.now() - start) / 1000)}s — ${last}`);
-          return;
-        }
-      } catch { /* API not serving the CRD yet */ }
-    }
-    await sleep(15_000);
-  }
-
-  // Say WHY, per node, instead of just "not ready". A node with no disk has a
-  // specific cause and the record knows it; without this the only way to find
-  // out is to catch the cluster before teardown, which is a race you lose.
-  const detail: string[] = [];
-  try {
-    const raw = kubectlJson<{ items: {
-      metadata: { name: string; annotations?: Record<string, string>; labels?: Record<string, string> };
-      spec?: { disks?: Record<string, unknown>; allowScheduling?: boolean };
-      status?: {
-        conditions?: { type: string; status: string; reason?: string; message?: string }[];
-        diskStatus?: Record<string, { conditions?: { type: string; status: string; reason?: string; message?: string }[] }>;
-      };
-    }[] }>(h, server, `-n ${LONGHORN_NS} get nodes.longhorn.io`);
-
-    for (const n of raw.items) {
-      const disks = Object.keys(n.spec?.disks ?? {});
-      const conds = (n.status?.conditions ?? [])
-        .filter((c) => c.status !== "True")
-        .map((c) => `${c.type}=${c.status}(${c.reason ?? ""}: ${(c.message ?? "").slice(0, 80)})`);
-      const diskConds = Object.entries(n.status?.diskStatus ?? {}).flatMap(([dn, d]) =>
-        (d.conditions ?? []).filter((c) => c.status !== "True")
-          .map((c) => `${dn}:${c.type}=${c.status}(${c.reason ?? ""}: ${(c.message ?? "").slice(0, 80)})`));
-      const ann = n.metadata.annotations?.["node.longhorn.io/default-disks-config"] ?? "<absent>";
-      const lbl = n.metadata.labels?.["node.longhorn.io/create-default-disk"] ?? "<absent>";
-      detail.push(
-        `  ${n.metadata.name}: disks=[${disks.join(",")}] label=${lbl} ann=${ann} ` +
-        `${conds.length > 0 ? `nodeConds=${conds.join(";")} ` : ""}` +
-        `${diskConds.length > 0 ? `diskConds=${diskConds.join(";")}` : ""}`);
-    }
-  } catch (e) {
-    detail.push(`  (could not read Longhorn nodes: ${e instanceof Error ? e.message : String(e)})`);
-  }
-
-  throw new Error(
-    `Longhorn not ready in ${timeoutMs / 60000} min (${last}).\n` +
-    `A node with no disk usually means the create-default-disk label or the\n` +
-    `default-disks-config annotation is missing, or the path is not mounted.\n` +
-    detail.join("\n"));
-}
-
-/** Create a PVC plus a Deployment that mounts it at /data. */
-export function createDataWorkload(h: Harness, server: ProvisionedVm, opts: {
-  name: string;
-  sizeGi: number;
-  replicas: number;
-}): void {
-  log(`[longhorn] creating ${opts.name} (${opts.sizeGi}Gi, ${opts.replicas} replicas)`);
-  nodeExec(h, server, `k3s kubectl apply -f - <<'EOF'
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: lh-${opts.name}
-provisioner: driver.longhorn.io
-allowVolumeExpansion: true
-parameters:
-  numberOfReplicas: "${opts.replicas}"
-  staleReplicaTimeout: "30"
-  fsType: ext4
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: ${opts.name}
-spec:
-  accessModes: [ReadWriteOnce]
-  storageClassName: lh-${opts.name}
-  resources:
-    requests:
-      storage: ${opts.sizeGi}Gi
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: ${opts.name}
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels: {app: ${opts.name}}
-  template:
-    metadata:
-      labels: {app: ${opts.name}}
-    spec:
-      # Pod Security Admission enforces "restricted" on this namespace (the k3s
-      # hardening group applies the label), so a bare pod spec is REJECTED
-      # outright -- the Deployment reports 0/1 with no pod and no obvious
-      # reason unless you read the ReplicaSet events. Every one of these
-      # fields is required by "restricted"; dropping any brings it back.
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 1000
-        runAsGroup: 1000
-        # fsGroup so the non-root user can actually write to the Longhorn
-        # volume, which is otherwise owned by root.
-        fsGroup: 1000
-        seccompProfile: {type: RuntimeDefault}
-      containers:
-        - name: shell
-          image: busybox:1.36
-          command: ["sh", "-c", "sleep infinity"]
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities: {drop: ["ALL"]}
-          volumeMounts:
-            - {name: data, mountPath: /data}
-      volumes:
-        - name: data
-          persistentVolumeClaim: {claimName: ${opts.name}}
-EOF`);
-}
-
-/** Wait for the workload's pod to be Running and its volume attached. */
-export async function waitForWorkload(
-  h: Harness, server: ProvisionedVm, name: string, timeoutMs = 10 * 60_000,
-): Promise<void> {
-  const start = Date.now();
-  let last = "";
-  while (Date.now() - start < timeoutMs) {
-    const r = tryNodeExec(h, server,
-      `k3s kubectl get pods -l app=${name} -o jsonpath='{.items[*].status.phase}'`);
-    last = r.stdout.trim();
-    if (last === "Running") {
-      // Ready, not merely scheduled: the volume mount happens before Ready.
-      const ready = tryNodeExec(h, server,
-        `k3s kubectl get pods -l app=${name} -o jsonpath='{.items[*].status.containerStatuses[*].ready}'`);
-      if (ready.stdout.trim() === "true") return;
-    }
-    await sleep(5_000);
-  }
-  throw new Error(`workload ${name} not running in ${timeoutMs / 60000} min (last phase: ${last || "none"})`);
-}
-
-function podOf(h: Harness, server: ProvisionedVm, name: string): string {
-  const pod = kubectlOn(h, server, `get pods -l app=${name} -o jsonpath='{.items[0].metadata.name}'`).trim();
-  if (!pod) throw new Error(`no pod for workload ${name}`);
-  return pod;
-}
-
-/**
- * Write a random blob, fsync it, and record its digest.
- *
- * conv=fsync matters: without it the write can still be in the guest's page
- * cache, and a checksum taken immediately afterwards proves nothing about what
- * reached the replicas.
- */
-export async function writeBlob(h: Harness, server: ProvisionedVm, opts: {
-  name: string; mib: number;
-}): Promise<string> {
-  const pod = podOf(h, server, opts.name);
-  log(`[longhorn] writing ${opts.mib} MiB to ${opts.name}`);
-  nodeExec(h, server,
-    `k3s kubectl exec ${pod} -- sh -c ` +
-    `"dd if=/dev/urandom of=/data/blob bs=1M count=${opts.mib} conv=fsync 2>/dev/null && ` +
-    `cd /data && sha256sum blob > blob.sha256 && sync"`,
-    15 * 60_000);
-
-  const digest = kubectlOn(h, server, `exec ${pod} -- cat /data/blob.sha256`).trim().split(/\s+/)[0] ?? "";
-  if (!/^[0-9a-f]{64}$/.test(digest)) throw new Error(`bad digest for ${opts.name}: ${digest}`);
-  log(`[longhorn] ${opts.name} digest ${digest.slice(0, 16)}...`);
-  return digest;
-}
-
-/**
- * Restart the pod, then verify the blob.
- *
- * The restart is the point. It detaches and reattaches the volume, so a pass
- * means the bytes came back from Longhorn rather than from wherever the writer
- * left them.
- */
-export async function verifyBlob(h: Harness, server: ProvisionedVm, opts: {
-  name: string; expectDigest: string;
-}): Promise<void> {
-  const before = podOf(h, server, opts.name);
-  kubectlOn(h, server, `delete pod ${before} --wait=true --timeout=5m`, 6 * 60_000);
-  await waitForWorkload(h, server, opts.name);
-  const after = podOf(h, server, opts.name);
-  if (after === before) throw new Error(`${opts.name}: pod was not actually replaced`);
-
-  // sha256sum -c proves the file matches what was recorded IN the volume...
-  const check = tryNodeExec(h, server,
-    `k3s kubectl exec ${after} -- sh -c "cd /data && sha256sum -c blob.sha256"`, 15 * 60_000);
-  if (check.exitCode !== 0) {
-    throw new Error(`${opts.name}: DATA CORRUPTION — sha256sum -c failed: ${check.stdout}${check.stderr}`);
-  }
-
-  // ...and this proves the recorded digest is still the one we started with,
-  // which -c alone would not catch if both file and checksum were replaced.
-  const actual = kubectlOn(h, server,
-    `exec ${after} -- sh -c "cd /data && sha256sum blob"`, 15 * 60_000).trim().split(/\s+/)[0];
-  if (actual !== opts.expectDigest) {
-    throw new Error(`${opts.name}: DIGEST CHANGED — expected ${opts.expectDigest}, got ${actual}`);
-  }
-  log(`[longhorn] ${opts.name} verified intact across a detach/attach (pod ${before} -> ${after})`);
-}
-
-export interface VolumeState {
-  pv: string;
-  state: string;
-  robustness: string;
-  wanted: number;
-  replicaNodes: string[];
-  healthyReplicas: number;
-}
-
-/** Longhorn's view of the volume backing a PVC. */
-export function volumeState(h: Harness, server: ProvisionedVm, pvcName: string): VolumeState {
-  const pv = kubectlOn(h, server, `get pvc ${pvcName} -o jsonpath='{.spec.volumeName}'`).trim();
-  if (!pv) throw new Error(`PVC ${pvcName} has no bound volume`);
-
-  const vol = kubectlJson<{
-    spec: { numberOfReplicas?: number };
-    status?: { state?: string; robustness?: string };
-  }>(h, server, `-n ${LONGHORN_NS} get volumes.longhorn.io ${pv}`);
-
-  const reps = kubectlJson<{ items: {
-    spec: { nodeID?: string; volumeName?: string };
-    status?: { currentState?: string; started?: boolean };
-  }[] }>(h, server, `-n ${LONGHORN_NS} get replicas.longhorn.io`);
-
-  const mine = reps.items.filter((r) => r.spec.volumeName === pv);
-  return {
-    pv,
-    state: vol.status?.state ?? "unknown",
-    robustness: vol.status?.robustness ?? "unknown",
-    wanted: vol.spec.numberOfReplicas ?? 0,
-    replicaNodes: mine.map((r) => r.spec.nodeID ?? "").filter(Boolean).sort(),
-    healthyReplicas: mine.filter((r) => r.status?.currentState === "running").length,
-  };
-}
-
-/**
- * Wait for a volume to be healthy on the expected number of distinct nodes.
- *
- * Logs rebuild progress while waiting, so a slow rebuild reads as progress
- * rather than as an opaque hour of silence.
- */
-export async function waitForVolumeHealthy(h: Harness, server: ProvisionedVm, opts: {
-  pvc: string; expectReplicas: number; timeoutMs?: number;
-}): Promise<VolumeState> {
-  const timeoutMs = opts.timeoutMs ?? 60 * 60_000;
-  const start = Date.now();
-  let last = "";
-  while (Date.now() - start < timeoutMs) {
-    let st: VolumeState | null = null;
-    try { st = volumeState(h, server, opts.pvc); } catch { /* transient */ }
-
-    if (st) {
-      const distinct = new Set(st.replicaNodes).size;
-      last = `${st.state}/${st.robustness} on ${distinct} node(s): ${st.replicaNodes.join(",")}`;
-      if (st.robustness === "healthy" && distinct >= opts.expectReplicas) {
-        log(`[longhorn] ${opts.pvc} healthy on ${distinct} nodes after ${Math.round((Date.now() - start) / 1000)}s`);
-        return st;
-      }
-      const prog = tryNodeExec(h, server,
-        `k3s kubectl -n ${LONGHORN_NS} get engines.longhorn.io -o jsonpath=` +
-        `'{range .items[*]}{.spec.volumeName}{" "}{.status.rebuildStatus}{"\\n"}{end}'`);
-      const line = prog.stdout.split("\n").find((l) => l.includes(st!.pv) && l.trim() !== st!.pv);
-      log(`[longhorn] ${opts.pvc}: ${last}${line ? ` | rebuild ${line.replace(st.pv, "").trim()}` : ""}`);
-    }
-    await sleep(20_000);
-  }
-  throw new Error(`${opts.pvc} not healthy on ${opts.expectReplicas} nodes in ${timeoutMs / 60000} min (last: ${last})`);
-}
-
-/**
- * Wait until Longhorn reports the backup target usable.
- *
- * `available: true` is the appliance saying it could reach the endpoint AND
- * authenticate. Backing up without checking this first produces a failure that
- * looks like a volume problem.
- */
-export async function setBackupTarget(
-  h: Harness, server: ProvisionedVm, backup: BackupTargetOpts, timeoutMs = 5 * 60_000,
-): Promise<void> {
-  // PATCH the CR; do NOT set defaultSettings.backupTarget.
-  //
-  // In Longhorn 1.8 the backup target moved out of settings and onto
-  // `backuptargets.longhorn.io/default`. The helm value is SILENTLY IGNORED --
-  // no `backup-target` Setting object is created at all, the default
-  // BackupTarget keeps an empty URL, and everything reports success. Observed
-  // on a fresh 1.8.1 install here and confirmed against the live fleet.
-  const deadline = Date.now() + timeoutMs;
-  while (Date.now() < deadline) {
-    const exists = tryNodeExec(h, server,
-      `k3s kubectl -n ${LONGHORN_NS} get backuptargets.longhorn.io default -o name`);
-    if (exists.exitCode === 0 && exists.stdout.trim() !== "") break;
-    await sleep(5_000);
-  }
-
-  const patch = JSON.stringify({
-    spec: { backupTargetURL: backup.target, credentialSecret: BACKUP_SECRET, pollInterval: "5m0s" },
-  });
-  nodeExec(h, server,
-    `k3s kubectl -n ${LONGHORN_NS} patch backuptargets.longhorn.io default --type=merge -p '${patch}'`);
-  log(`[longhorn] backup target set to ${backup.target}`);
-}
-
-export async function waitForBackupTarget(
-  h: Harness, server: ProvisionedVm, timeoutMs = 5 * 60_000,
-): Promise<void> {
-  const start = Date.now();
-  let last = "";
-  while (Date.now() - start < timeoutMs) {
-    const r = tryNodeExec(h, server,
-      `k3s kubectl -n ${LONGHORN_NS} get backuptargets.longhorn.io default ` +
-      `-o jsonpath='{.spec.backupTargetURL}|{.status.available}|{.status.conditions[*].message}'`);
-    last = r.stdout.trim();
-    if (r.exitCode === 0 && last.includes("|true|")) {
-      log(`[longhorn] backup target available after ${Math.round((Date.now() - start) / 1000)}s`);
-      return;
-    }
-    await sleep(10_000);
-  }
-  throw new Error(
-    `Backup target not available in ${timeoutMs / 60000} min (url|available|conditions = ${last}). ` +
-    `Longhorn could not reach or authenticate to the endpoint.`);
-}
-
-/**
- * Snapshot a volume and back it up, returning the backup's name.
- *
- * Longhorn's own CRs, not the CSI VolumeSnapshot API -- this cluster has no
- * snapshot.storage.k8s.io resources at all, so VolumeSnapshot/VolumeSnapshotClass
- * simply do not exist here. Any backup tooling has to use these.
- */
-export async function backupVolume(
-  h: Harness, server: ProvisionedVm, pvc: string, timeoutMs = 20 * 60_000,
-): Promise<{ backup: string; volume: string }> {
-  const pv = volumeState(h, server, pvc).pv;
-  const snapName = `drill-snap-${Date.now().toString(36)}`;
-
-  log(`[longhorn] snapshotting ${pvc} (${pv})`);
-  nodeExec(h, server, `k3s kubectl -n ${LONGHORN_NS} apply -f - <<'EOF'
-apiVersion: longhorn.io/v1beta2
-kind: Snapshot
-metadata:
-  name: ${snapName}
-  namespace: ${LONGHORN_NS}
-spec:
-  volume: ${pv}
-  createSnapshot: true
-EOF`);
-
-  // The snapshot has to exist before a backup can reference it.
-  const snapDeadline = Date.now() + 5 * 60_000;
-  while (Date.now() < snapDeadline) {
-    const r = tryNodeExec(h, server,
-      `k3s kubectl -n ${LONGHORN_NS} get snapshots.longhorn.io ${snapName} -o jsonpath='{.status.readyToUse}'`);
-    if (r.stdout.trim() === "true") break;
-    await sleep(5_000);
-  }
-
-  const backupName = `drill-bak-${Date.now().toString(36)}`;
-  log(`[longhorn] backing up ${pvc} to the backup target`);
-  nodeExec(h, server, `k3s kubectl -n ${LONGHORN_NS} apply -f - <<'EOF'
-apiVersion: longhorn.io/v1beta2
-kind: Backup
-metadata:
-  name: ${backupName}
-  namespace: ${LONGHORN_NS}
-  labels:
-    backup-volume: ${pv}
-spec:
-  snapshotName: ${snapName}
-EOF`);
-
-  const start = Date.now();
-  let last = "";
-  while (Date.now() - start < timeoutMs) {
-    const r = tryNodeExec(h, server,
-      `k3s kubectl -n ${LONGHORN_NS} get backups.longhorn.io ${backupName} ` +
-      `-o jsonpath='{.status.state}|{.status.progress}|{.status.error}'`);
-    last = r.stdout.trim();
-    if (last.startsWith("Completed")) {
-      log(`[longhorn] backup ${backupName} completed after ${Math.round((Date.now() - start) / 1000)}s`);
-      return { backup: backupName, volume: pv };
-    }
-    if (last.includes("Error")) {
-      throw new Error(`Backup ${backupName} failed: ${last}`);
-    }
-    await sleep(10_000);
-  }
-  throw new Error(`Backup ${backupName} did not complete in ${timeoutMs / 60000} min (state|progress|error = ${last})`);
-}
-
-/**
- * Restore a backup into a brand-new PVC and workload.
- *
- * Uses the `fromBackup` StorageClass parameter, which is the supported route
- * when the CSI VolumeSnapshot API is unavailable. The restored volume is a
- * separate volume with its own replicas -- nothing about the original is
- * touched, so a failed restore cannot damage the source.
- */
-export async function restoreFromBackup(h: Harness, server: ProvisionedVm, opts: {
-  backup: string;
-  name: string;
-  sizeGi: number;
-}): Promise<void> {
-  const url = nodeExec(h, server,
-    `k3s kubectl -n ${LONGHORN_NS} get backups.longhorn.io ${opts.backup} -o jsonpath='{.status.url}'`).trim();
-  if (url === "") throw new Error(`Backup ${opts.backup} has no status.url; cannot restore from it`);
-  log(`[longhorn] restoring ${opts.backup} into ${opts.name}`);
-
-  nodeExec(h, server, `k3s kubectl apply -f - <<'EOF'
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: lh-${opts.name}
-provisioner: driver.longhorn.io
-allowVolumeExpansion: true
-parameters:
-  numberOfReplicas: "2"
-  staleReplicaTimeout: "30"
-  fsType: ext4
-  fromBackup: "${url}"
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: ${opts.name}
-spec:
-  accessModes: [ReadWriteOnce]
-  storageClassName: lh-${opts.name}
-  resources:
-    requests:
-      storage: ${opts.sizeGi}Gi
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: ${opts.name}
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels: {app: ${opts.name}}
-  template:
-    metadata:
-      labels: {app: ${opts.name}}
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 1000
-        runAsGroup: 1000
-        fsGroup: 1000
-        seccompProfile: {type: RuntimeDefault}
-      containers:
-        - name: shell
-          image: busybox:1.36
-          command: ["sh", "-c", "sleep infinity"]
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities: {drop: ["ALL"]}
-          volumeMounts:
-            - {name: data, mountPath: /data}
-      volumes:
-        - name: data
-          persistentVolumeClaim: {claimName: ${opts.name}}
-EOF`);
-}
-
-/**
- * Read a blob's digest from a workload, without restarting it.
- *
- * Used on a RESTORED volume, where the point is to compare bytes against the
- * original's recorded digest rather than to prove durability again.
- */
-export function readBlobDigest(h: Harness, server: ProvisionedVm, name: string): string {
-  const pod = kubectlOn(h, server, `get pods -l app=${name} -o jsonpath='{.items[0].metadata.name}'`).trim();
-  if (pod === "") throw new Error(`no pod for workload ${name}`);
-  const out = kubectlOn(h, server,
-    `exec ${pod} -- sh -c "cd /data && sha256sum blob"`, 15 * 60_000).trim();
-  return out.split(/\s+/)[0] ?? "";
-}

bastion/tests/integration/helpers/provision-harness.ts

@@ -1,583 +0,0 @@
-// Reusable bastion + VM provisioning harness for partition-layout tests.
-//
-// pxe-provision.test.ts grew a ~200-line beforeAll that stands up a real
-// bastion, caches Fedora boot artifacts, and drives one VM through discovery
-// and install. Testing a partition MATRIX needs that same sequence for several
-// VMs and several roles, so it lives here instead of being copy-pasted per
-// topology.
-
-import { existsSync, mkdirSync, copyFileSync, symlinkSync, writeFileSync } from "node:fs";
-import { execSync } from "node:child_process";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import type { BastionConfig } from "../../../src/shared/src/types/config.js";
-import type { Role } from "../../../src/shared/src/types/state.js";
-import { log, waitForSsh } from "./libvirt.js";
-import { ensurePxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_BRIDGE, PXE_GATEWAY, PXE_SUBNET } from "./pxe-network.js";
-import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./pxe-vm.js";
-import { sshExec } from "./ssh.js";
-
-export const DISCOVERY_TIMEOUT_MS = 6 * 60_000;
-export const INSTALL_TIMEOUT_MS = 60 * 60_000;
-export const SSH_TIMEOUT_MS = 12 * 60_000;
-
-export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
-
-export interface Harness {
-  config: BastionConfig;
-  close: () => Promise<void>;
-  httpPort: number;
-  sshUser: string;
-  sshKeyPath: string;
-}
-
-async function pollUntil<T>(url: string, check: (d: T) => boolean, timeoutMs: number, intervalMs = 5000): Promise<T> {
-  const start = Date.now();
-  let lastErr = "";
-  while (Date.now() - start < timeoutMs) {
-    try {
-      const res = await fetch(url);
-      if (res.ok) {
-        const data = (await res.json()) as T;
-        if (check(data)) return data;
-      }
-    } catch (e) { lastErr = e instanceof Error ? e.message : String(e); }
-    await sleep(intervalMs);
-  }
-  throw new Error(`Timeout after ${timeoutMs}ms polling ${url}${lastErr ? ` (last error: ${lastErr})` : ""}`);
-}
-
-/**
- * Stand up a real bastion on the isolated PXE network.
- *
- * FEDORA_MIRROR is honoured (config.ts now reads it) -- point it at a local
- * mirror before running a multi-VM matrix, or every install pulls the full
- * package set from download.fedoraproject.org.
- */
-export async function startTestBastion(opts: {
-  httpPort: number;
-  sshUser: string;
-  sshPubKey: string;
-  sshKeyPath: string;
-  label: string;
-  /**
-   * Leave the post-provision k3s auto-install ON (the default) or suppress it.
-   *
-   * Any test that forms a cluster ITSELF must pass false. The auto-install runs
-   * a bare `INSTALL_K3S_EXEC=server|agent` with no server URL and no token, so
-   * three combo nodes become three separate single-node clusters -- and it
-   * stops k3s and deletes CNI state minutes after an install lands, landing on
-   * top of whatever the test is doing by then.
-   */
-  autoK3s?: boolean;
-}): Promise<Harness> {
-  ensurePxeNetwork();
-
-  const testDir = join(tmpdir(), `lab-${opts.label}-${Date.now()}`);
-  for (const sub of ["tftp", "http", "logs"]) mkdirSync(join(testDir, sub), { recursive: true });
-
-  const { createApp } = await import("../../../src/bastion/src/server.js");
-  const { loadConfig } = await import("../../../src/bastion/src/config.js");
-  const { generateDnsmasqConf, startDnsmasq, stopDnsmasq } = await import("../../../src/bastion/src/services/dnsmasq.js");
-  const { generateDiscoverKickstart } = await import("../../../src/bastion/src/services/kickstart-generator.js");
-  const { renderBootIpxe } = await import("../../../src/bastion/src/templates/boot.ipxe.js");
-
-  const config = loadConfig({
-    bastionDir: testDir,
-    httpPort: opts.httpPort,
-    iface: "virbr-pxe",
-    serverIp: PXE_GATEWAY,
-    network: `${PXE_SUBNET}.0`,
-    gateway: PXE_GATEWAY,
-    dhcpMode: "full",
-    dhcpRangeStart: `${PXE_SUBNET}.100`,
-    dhcpRangeEnd: `${PXE_SUBNET}.200`,
-    domain: "pxe-test.local",
-    sshKeys: [opts.sshPubKey],
-    adminUser: opts.sshUser,
-    ...(opts.autoK3s === undefined ? {} : { autoK3s: opts.autoK3s }),
-  });
-
-  const ipxeSrc = "/usr/share/ipxe/ipxe-snponly-x86_64.efi";
-  if (!existsSync(ipxeSrc)) throw new Error(`iPXE not found: ${ipxeSrc}. Install: sudo dnf install ipxe-bootimgs-x86`);
-  const ipxeDest = join(config.tftpDir, "ipxe.efi");
-  copyFileSync(ipxeSrc, ipxeDest);
-
-  const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache";
-  execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" });
-  const kernel = join(cacheDir, `vmlinuz-${config.fedoraVersion}`);
-  const initrd = join(cacheDir, `initrd-${config.fedoraVersion}.img`);
-  if (!existsSync(kernel)) {
-    log(`Downloading Fedora ${config.fedoraVersion} kernel from ${config.fedoraMirror}...`);
-    execSync(`curl -sS -L -f -o "${kernel}" "${config.fedoraMirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 300_000 });
-  }
-  if (!existsSync(initrd)) {
-    log(`Downloading Fedora ${config.fedoraVersion} initrd...`);
-    execSync(`curl -sS -L -f -o "${initrd}" "${config.fedoraMirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 300_000 });
-  }
-  copyFileSync(kernel, join(config.httpDir, "vmlinuz"));
-  copyFileSync(initrd, join(config.httpDir, "initrd.img"));
-  try { symlinkSync(ipxeDest, join(config.httpDir, "ipxe.efi")); } catch { /* exists */ }
-
-  writeFileSync(join(config.httpDir, "discover.ks"), generateDiscoverKickstart(config));
-  writeFileSync(join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }));
-  generateDnsmasqConf(config);
-
-  // Kill dnsmasq processes left behind by EARLIER TEST RUNS.
-  //
-  // startTestBastion's close() shuts down the HTTP app but never stopped
-  // dnsmasq, so every run leaked one. A leaked dnsmasq keeps :67 on the bridge
-  // and keeps answering DHCP -- with the PREVIOUS run's config, pointing PXE
-  // clients at a bastion port that is no longer listening. The next run then
-  // sees perfect DHCP (offer, request, ack, even the right bootfile name) and
-  // no machine ever reaches it. That is not a failure mode anyone guesses from
-  // "discovery timed out".
-  killOrphanedDnsmasq();
-
-  const { app, syslog } = createApp(config);
-  await app.listen({ port: config.httpPort, host: "0.0.0.0" });
-  syslog.start();
-  // NOT awaited, deliberately.
-  //
-  // startDnsmasq returns the execa child handle, and because the function is
-  // `async` the promise it returns ADOPTS that thenable -- which settles only
-  // when dnsmasq EXITS. With --no-daemon that is never, so `await` here hangs
-  // the harness silently and forever, with the log frozen right after
-  // "TFTP root is ...". (Learned by doing exactly that.) The rejection still
-  // has to be handled or an early bind failure becomes an unhandled rejection.
-  startDnsmasq(config).catch((err) => log(`dnsmasq: ${err instanceof Error ? err.message : String(err)}`));
-  await sleep(1000);
-
-  // Prove DHCP is actually listening before handing the harness back.
-  //
-  // startDnsmasq's rejection is swallowed into a log line, so a dnsmasq that
-  // failed to bind looks exactly like one that started. The symptom then
-  // arrives six minutes later as four VMs timing out in discovery, which
-  // reads as a PXE or kickstart problem rather than "there was never any
-  // DHCP". Fail here, immediately, with the real reason.
-  await assertDhcpListening(join(testDir, "dnsmasq.conf"));
-  log(`Bastion up on :${opts.httpPort} (mirror: ${config.fedoraMirror}, autoK3s=${config.autoK3s})`);
-
-  return {
-    config,
-    httpPort: opts.httpPort,
-    sshUser: opts.sshUser,
-    sshKeyPath: opts.sshKeyPath,
-    close: async () => {
-      // Stop OUR dnsmasq. close() never used to, which is how a leaked one
-      // ended up owning :67 and serving the PREVIOUS run's TFTP root and
-      // bastion port to the NEXT run's VMs -- they booted toward a dead port
-      // and simply never appeared, while DHCP looked perfect.
-      try { stopDnsmasq(); } catch { /* already gone */ }
-      await app.close();
-      // Belt and braces: if that did not stop it, sweep by config path.
-      killOrphanedDnsmasq();
-    },
-  };
-}
-
-/**
- * Kill any dnsmasq started by a previous (or current) lab test run.
- *
- * Matched on the config path, which is always /tmp/lab-<label>-<ts>/dnsmasq.conf
- * -- so this never touches libvirt's own dnsmasq for the default network, or
- * anything else on the host.
- */
-function killOrphanedDnsmasq(): void {
-  try {
-    // Match on the process NAME first (pgrep -x), then inspect each one's
-    // arguments. A `pgrep -f 'dnsmasq.*/tmp/lab-'` matches the shell running
-    // that very pattern, so the naive version can kill its own shell -- and
-    // libvirt's dnsmasq for the default network must never be touched either.
-    const pids = execSync("pgrep -x dnsmasq 2>/dev/null || true", { encoding: "utf-8" })
-      .trim().split("\n").filter((x) => /^\d+$/.test(x.trim()));
-    let killed = 0;
-    for (const pid of pids) {
-      let args = "";
-      try {
-        args = execSync(`tr '\\0' ' ' < /proc/${pid.trim()}/cmdline 2>/dev/null || true`, { encoding: "utf-8" });
-      } catch { continue; }
-      if (!args.includes("--conf-file=/tmp/lab-")) continue;
-      log(`Killing orphaned test dnsmasq (pid ${pid.trim()}) — it would answer DHCP with stale config`);
-      try { execSync(`sudo kill -9 ${pid.trim()} 2>/dev/null || kill -9 ${pid.trim()} 2>/dev/null || true`); } catch { /* gone */ }
-      killed++;
-    }
-    // Give the socket a moment to be released before we bind it.
-    if (killed > 0) execSync("sleep 1");
-  } catch { /* nothing to kill */ }
-}
-
-/**
- * Is something serving DHCP on the PXE bridge?
- *
- * `ss -lunp` shows UDP listeners; dnsmasq binds port 67. Checked on the bridge
- * address specifically, because a dnsmasq bound to the wrong interface is the
- * failure being guarded against.
- */
-async function assertDhcpListening(confPath: string, timeoutMs = 20_000): Promise<void> {
-  const start = Date.now();
-  let last = "";
-  while (Date.now() - start < timeoutMs) {
-    try {
-      // Check for OUR dnsmasq by its config path, not merely for something on
-      // :67. A leaked dnsmasq from a previous run also holds :67 and also
-      // answers DHCP -- with the wrong bastion port -- so a port check alone
-      // reports success for precisely the situation it is meant to catch.
-      const pids = execSync("pgrep -x dnsmasq 2>/dev/null || true", { encoding: "utf-8" })
-        .trim().split("\n").filter((x) => /^\d+$/.test(x.trim()));
-      last = pids.filter((pid) => {
-        try {
-          return execSync(`tr '\\0' ' ' < /proc/${pid.trim()}/cmdline 2>/dev/null || true`, { encoding: "utf-8" })
-            .includes(confPath);
-        } catch { return false; }
-      }).join(",");
-      if (last !== "") {
-        const ports = execSync("ss -lun 2>/dev/null || true", { encoding: "utf-8" });
-        if (/:67\b/.test(ports)) {
-          log("dnsmasq (ours) is serving DHCP on :67");
-          return;
-        }
-      }
-    } catch { /* retry */ }
-    await sleep(1000);
-  }
-  throw new Error(
-    `dnsmasq for ${confPath} is not serving DHCP after ${timeoutMs / 1000}s. ` +
-    `It most likely failed to bind to ${PXE_BRIDGE} -- check the bastion log for ` +
-    `"error binding DHCP socket", and check for a leaked dnsmasq from an earlier run ` +
-    `(pgrep -af 'dnsmasq.*--conf-file=/tmp/lab-'). Without this every VM sits in ` +
-    `discovery until it times out, six minutes later, looking like a PXE problem.`,
-  );
-}
-
-/**
- * Wait for the installer to actually start, rebooting into PXE if it does not.
- *
- * "Started" means the bastion has seen anything at all for this MAC beyond the
- * queued state -- a dispatch, a progress line, a status. A VM that never gets
- * that far is stuck before Anaconda, and only a reboot will shift it.
- */
-async function waitForInstallStart(
-  base: string,
-  mac: string,
-  name: string,
-  attemptMs = 6 * 60_000,
-  attempts = 3,
-): Promise<void> {
-  for (let attempt = 1; attempt <= attempts; attempt++) {
-    try {
-      // NOT `status === "installing"`. A machine that has merely been QUEUED
-      // already reports that -- the bastion sets it the moment the install is
-      // dispatched, before anything has booted. Using it would match instantly
-      // and this retry would never fire.
-      //
-      // The honest signals that Anaconda is alive are `progress` leaving
-      // "queued", or log lines arriving over syslog.
-      await pollUntil<{ status?: string; progress?: string; log_total?: number }>(
-        `${base}/api/logs/${encodeURIComponent(mac)}`,
-        (d) => d.status === "installed"
-          || (d.progress !== undefined && d.progress !== "queued")
-          || (d.log_total ?? 0) > 0,
-        attemptMs, 5_000);
-      if (attempt > 1) log(`[${name}] PXE boot took on attempt ${attempt}`);
-      return;
-    } catch {
-      if (attempt === attempts) {
-        throw new Error(
-          `[${name}] installer never started after ${attempts} PXE boots ` +
-          `(${attemptMs / 60000} min each). The VM is stuck before Anaconda — ` +
-          `check TFTP and the iPXE chainload.`);
-      }
-      log(`[${name}] no installer activity in ${attemptMs / 60000} min; rebooting into PXE (attempt ${attempt + 1}/${attempts})`);
-      rebootPxeVm(name);
-      deleteNftablesRejectRules();
-    }
-  }
-}
-
-export interface ProvisionedVm { name: string; mac: string; ip: string; serialPort: number }
-
-/**
- * "SSH never came up" on its own is undiagnosable once afterAll has destroyed
- * the VM -- the machine may be sitting in emergency mode with the exact error
- * on its serial console. Capture that console into the failure.
- */
-async function sshFailurePostMortem(name: string, serialPort: number, cause: unknown): Promise<never> {
-  let state = "unknown";
-  try { state = execSync(`sudo virsh domstate ${name}`, { encoding: "utf-8" }).trim(); } catch { /* gone */ }
-  let serial = "(no serial output)";
-  try { serial = await readSerialLog(serialPort, { lastLines: 60 }); } catch { /* no console */ }
-  throw new Error(
-    `[${name}] ${cause instanceof Error ? cause.message : String(cause)}\n` +
-    `domstate: ${state}\n--- serial console (last lines) ---\n${serial}`,
-  );
-}
-
-/**
- * Serial ports are a global resource on the host, so hand out a distinct one
- * per VM. Without this only one VM can run at a time -- qemu refuses to start
- * the second with "Address already in use".
- */
-let nextSerialPort = 4600;
-
-/**
- * The stable path a virtio data disk gets inside the guest. This is what the
- * install request must carry: the feature refuses anything outside
- * /dev/disk/by-id/, exactly as it does for real NVMe serials.
- */
-export function virtioById(serial: string): string {
-  return `/dev/disk/by-id/virtio-${serial}`;
-}
-
-/** Create a blank VM, let it discover, queue the install, wait for SSH. */
-export async function provisionVm(h: Harness, spec: {
-  name: string;
-  role: Role;
-  schemaVersion?: 1 | 2;
-  diskGb: number;
-  memory?: number;
-  vcpus?: number;
-  /**
-   * Dedicated data disks: attached to the VM with the given virtio serial AND
-   * passed to /api/install as /dev/disk/by-id/virtio-<serial>. Mirrors
-   * thebeast's layout (OS disk + separate longhorn + rancher disks).
-   */
-  dataDisks?: { longhorn?: { serial: string; sizeGb: number }; rancher?: { serial: string; sizeGb: number } };
-}): Promise<ProvisionedVm> {
-  const base = `http://${PXE_GATEWAY}:${h.httpPort}`;
-  const vmDataDisks = [spec.dataDisks?.longhorn, spec.dataDisks?.rancher]
-    .filter((d): d is { serial: string; sizeGb: number } => d !== undefined);
-  log(`[${spec.name}] creating VM (${spec.diskGb}GB disk + ${vmDataDisks.length} data disk(s), role=${spec.role}, schema=v${spec.schemaVersion ?? 1})`);
-  const serialPort = nextSerialPort++;
-  createPxeVm({
-    name: spec.name,
-    memory: spec.memory ?? 4096,
-    vcpus: spec.vcpus ?? 4,
-    diskSize: spec.diskGb,
-    network: PXE_NETWORK_NAME,
-    serialPort,
-    ...(vmDataDisks.length > 0 ? { dataDisks: vmDataDisks } : {}),
-  });
-  deleteNftablesRejectRules();
-
-  const mac = getVmMac(spec.name);
-  if (!mac) throw new Error(`[${spec.name}] no MAC`);
-
-  log(`[${spec.name}] waiting for discovery (${mac})...`);
-  await pollUntil<{ discovered: Record<string, unknown> }>(
-    `${base}/api/machines`, (d) => mac in (d.discovered ?? {}), DISCOVERY_TIMEOUT_MS);
-
-  log(`[${spec.name}] queueing install...`);
-  const res = await fetch(`${base}/api/install`, {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({
-      mac, hostname: spec.name, disk: "", role: spec.role,
-      ...(spec.schemaVersion ? { schemaVersion: spec.schemaVersion } : {}),
-      ...(spec.dataDisks ? {
-        dataDisks: {
-          ...(spec.dataDisks.longhorn ? { longhorn: virtioById(spec.dataDisks.longhorn.serial) } : {}),
-          ...(spec.dataDisks.rancher ? { rancher: virtioById(spec.dataDisks.rancher.serial) } : {}),
-        },
-      } : {}),
-    }),
-  });
-  if (!res.ok) throw new Error(`[${spec.name}] install queue rejected: ${res.status} ${await res.text()}`);
-
-  rebootPxeVm(spec.name);
-  deleteNftablesRejectRules();
-
-  // Make sure the PXE boot actually TOOK before settling in for the install.
-  //
-  // A UEFI VM that fails to chainload just retries DHCP quietly, for ever.
-  // Observed: one VM of four sat in a DHCP loop for an HOUR with no TFTP
-  // fetch, then began installing seconds after the 60-minute install timeout
-  // had already fired -- so the run failed on a machine that was, by then,
-  // working. TFTP is UDP and four VMs contend for it, so this is a question of
-  // when rather than if.
-  //
-  // Rebooting is cheap and a boot that has not started within a few minutes is
-  // not going to start on its own.
-  await waitForInstallStart(base, mac, spec.name);
-
-  log(`[${spec.name}] installing (up to ${INSTALL_TIMEOUT_MS / 60000} min)...`);
-  await pollUntil<{ status?: string }>(
-    `${base}/api/logs/${encodeURIComponent(mac)}`, (d) => d.status === "installed", INSTALL_TIMEOUT_MS, 10_000);
-
-  // The installed record carries the DHCP address the machine ended up with.
-  const machines = await pollUntil<{ installed: Record<string, { ip?: string }> }>(
-    `${base}/api/machines`, (d) => Boolean(d.installed?.[mac]?.ip), 2 * 60_000);
-  const ip = machines.installed[mac]!.ip!;
-
-  try {
-    if (execSync(`sudo virsh domstate ${spec.name}`, { encoding: "utf-8" }).trim() === "shut off") {
-      execSync(`sudo virsh start ${spec.name}`, { stdio: "pipe" });
-    }
-  } catch { /* already running */ }
-  deleteNftablesRejectRules();
-
-  log(`[${spec.name}] waiting for SSH at ${ip}...`);
-  try {
-    await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath);
-  } catch (e) {
-    await sshFailurePostMortem(spec.name, serialPort, e);
-  }
-  log(`[${spec.name}] up at ${ip}`);
-  return { name: spec.name, mac, ip, serialPort };
-}
-
-/**
- * Provision several VMs at once.
- *
- * Installs are ~12 minutes each and almost entirely I/O and network wait, so
- * four in sequence is an hour that four in parallel is twenty minutes. They are
- * staggered because the expensive, contended part is the start: PXE, DHCP and
- * the first package fetches all land at once otherwise.
- *
- * Rejects only after every VM has settled, so a single failure does not leave
- * the others half-built and unreferenced -- the caller still gets the names it
- * needs to tear down.
- */
-export async function provisionVms(
-  h: Harness,
-  specs: Parameters<typeof provisionVm>[1][],
-  opts: { staggerMs?: number } = {},
-): Promise<ProvisionedVm[]> {
-  const stagger = opts.staggerMs ?? 60_000;
-  const settled = await Promise.allSettled(
-    specs.map(async (spec, i) => {
-      await sleep(i * stagger);
-      return provisionVm(h, spec);
-    }),
-  );
-
-  const failures = settled
-    .map((r, i) => (r.status === "rejected" ? `${specs[i]?.name}: ${String(r.reason)}` : null))
-    .filter((x): x is string => x !== null);
-  if (failures.length > 0) {
-    throw new Error(`provisionVms: ${failures.length}/${specs.length} failed\n  ${failures.join("\n  ")}`);
-  }
-  return settled.map((r) => (r as PromiseFulfilledResult<ProvisionedVm>).value);
-}
-
-/** LV name -> size in MiB, read off the installed machine. */
-export function readLvs(h: Harness, vm: ProvisionedVm): Record<string, number> {
-  const res = sshExec(vm.ip, h.sshUser,
-    "sudo lvs labvg --noheadings --units m --nosuffix -o lv_name,lv_size --separator=,",
-    { keyPath: h.sshKeyPath });
-  if (res.exitCode !== 0) throw new Error(`[${vm.name}] lvs failed: ${res.stderr}`);
-  const map: Record<string, number> = {};
-  for (const line of res.stdout.split("\n")) {
-    const [name, size] = line.trim().split(",");
-    if (name && size) map[name] = Math.round(Number(size));
-  }
-  // Log what actually landed on disk. The VMs are destroyed in afterAll, so
-  // without this the only record of the real layout is the pass/fail bit.
-  const summary = Object.entries(map)
-    .sort(([a], [b]) => a.localeCompare(b))
-    .map(([n, m]) => `${n}=${(m / 1024).toFixed(0)}G`)
-    .join(" ");
-  log(`[${vm.name}] on-disk LVs: ${summary}`);
-  return map;
-}
-
-/** The VG tag that records which schema this disk was built with. */
-export function readSchemaTag(h: Harness, vm: ProvisionedVm): string {
-  return sshExec(vm.ip, h.sshUser, "sudo vgs --noheadings -o vg_tags labvg",
-    { keyPath: h.sshKeyPath }).stdout.trim();
-}
-
-export function destroyVms(...names: string[]): void {
-  for (const n of names) { try { destroyPxeVm(n); } catch { /* already gone */ } }
-}
-
-/**
- * Reinstall a machine that is already installed, the way
- * `labctl provision reprovision` does: queue another install for the same MAC
- * and reboot it into PXE.
- *
- * The schema is deliberately NOT passed. A reinstall must reproduce whatever
- * layout is already on the disk -- %pre reads the lab-schema-v2 LVM tag rather
- * than trusting the caller -- and passing one here would hide a regression in
- * exactly that logic.
- */
-export async function reprovisionVm(
-  h: Harness,
-  vm: ProvisionedVm,
-  role: Role,
-  /**
-   * Only set this to prove it is IGNORED. A reinstall takes its layout from the
-   * disk, not from the request; passing a schema here is how you test that.
-   */
-  requestSchemaVersion?: 1 | 2,
-): Promise<ProvisionedVm> {
-  const base = `http://${PXE_GATEWAY}:${h.httpPort}`;
-  log(`[${vm.name}] reprovisioning (role=${role}, schema: whatever is on disk)...`);
-
-  const res = await fetch(`${base}/api/install`, {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify({
-      mac: vm.mac, hostname: vm.name, disk: "", role,
-      ...(requestSchemaVersion ? { schemaVersion: requestSchemaVersion } : {}),
-    }),
-  });
-  if (!res.ok) throw new Error(`[${vm.name}] reinstall queue rejected: ${res.status} ${await res.text()}`);
-
-  rebootPxeVm(vm.name);
-  deleteNftablesRejectRules();
-
-  log(`[${vm.name}] reinstalling...`);
-  await pollUntil<{ status?: string }>(
-    `${base}/api/logs/${encodeURIComponent(vm.mac)}`,
-    (d) => d.status === "installed", INSTALL_TIMEOUT_MS, 10_000);
-
-  const machines = await pollUntil<{ installed: Record<string, { ip?: string }> }>(
-    `${base}/api/machines`, (d) => Boolean(d.installed?.[vm.mac]?.ip), 2 * 60_000);
-  const ip = machines.installed[vm.mac]!.ip!;
-
-  try {
-    if (execSync(`sudo virsh domstate ${vm.name}`, { encoding: "utf-8" }).trim() === "shut off") {
-      execSync(`sudo virsh start ${vm.name}`, { stdio: "pipe" });
-    }
-  } catch { /* already running */ }
-  deleteNftablesRejectRules();
-
-  try {
-    await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath);
-  } catch (e) {
-    await sshFailurePostMortem(vm.name, vm.serialPort, e);
-  }
-  log(`[${vm.name}] back up at ${ip} after reinstall`);
-  return { ...vm, ip };
-}
-
-/**
- * Drop a marker file into each mount point that a reinstall is supposed to
- * preserve. Ported from test-reprovision.sh, which proved the idea but only
- * ever ran by hand.
- */
-export function writeMarkers(h: Harness, vm: ProvisionedVm, mounts: string[]): string {
-  const token = `preserve-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
-  for (const m of mounts) {
-    const r = sshExec(vm.ip, h.sshUser,
-      `sudo sh -c 'echo ${token} > ${m}/LAB_PRESERVE_TEST.txt && sync'`,
-      { keyPath: h.sshKeyPath });
-    if (r.exitCode !== 0) throw new Error(`[${vm.name}] could not write marker in ${m}: ${r.stderr}`);
-  }
-  log(`[${vm.name}] wrote marker ${token} into ${mounts.join(", ")}`);
-  return token;
-}
-
-/** Which mount points still carry the marker. */
-export function readMarkers(h: Harness, vm: ProvisionedVm, mounts: string[]): Record<string, string | null> {
-  const out: Record<string, string | null> = {};
-  for (const m of mounts) {
-    const r = sshExec(vm.ip, h.sshUser, `sudo cat ${m}/LAB_PRESERVE_TEST.txt 2>/dev/null || true`,
-      { keyPath: h.sshKeyPath });
-    const v = r.stdout.trim();
-    out[m] = v === "" ? null : v;
-  }
-  log(`[${vm.name}] markers after reinstall: ${JSON.stringify(out)}`);
-  return out;
-}

bastion/tests/integration/helpers/pxe-network.ts

@@ -56,17 +56,6 @@ export function ensurePxeNetwork(): void {
 
     try { unlinkSync(xmlPath); } catch { /* ignore */ }
 
-    // `virsh net-start` returns before the bridge is usable. The bastion's
-    // dnsmasq starts about a second later and binds to this device by name; if
-    // the interface is not up and addressed yet it fails with "error binding
-    // DHCP socket to device virbr-pxe", and because that failure is
-    // asynchronous the harness carries on with NO DHCP AT ALL -- every VM then
-    // sits in discovery until it times out, six minutes later, looking like a
-    // PXE problem rather than a startup-order one.
-    //
-    // Cheap to wait, expensive not to.
-    waitForBridge();
-
     log(`Network ${PXE_NETWORK_NAME} created and active`);
   }
 
@@ -75,32 +64,6 @@ export function ensurePxeNetwork(): void {
   deleteNftablesRejectRules();
 }
 
-/**
- * Block until the bridge exists AND carries the gateway address.
- *
- * Checking only that the device exists is not enough: it appears before the
- * address is assigned, and dnsmasq needs the address to bind.
- */
-export function waitForBridge(timeoutMs = 30_000): void {
-  const start = Date.now();
-  let last = "";
-  while (Date.now() - start < timeoutMs) {
-    const r = spawnSync("ip", ["-br", "addr", "show", PXE_BRIDGE], { encoding: "utf-8", stdio: "pipe" });
-    last = (r.stdout ?? "").trim();
-    if (r.status === 0 && last.includes(`${PXE_GATEWAY}/`)) {
-      log(`Bridge ${PXE_BRIDGE} ready with ${PXE_GATEWAY} after ${Date.now() - start}ms`);
-      return;
-    }
-    // Busy-wait briefly; this normally resolves in well under a second.
-    const until = Date.now() + 250;
-    while (Date.now() < until) { /* spin */ }
-  }
-  throw new Error(
-    `Bridge ${PXE_BRIDGE} did not come up with ${PXE_GATEWAY} within ${timeoutMs}ms ` +
-    `(last: ${last || "device absent"}). dnsmasq cannot bind, so PXE would silently not work.`,
-  );
-}
-
 /** Delete libvirt's nftables reject rules for our bridge so host→VM traffic works.
  *  Must be called after every VM start/restart — libvirt recreates them. */
 export function deleteNftablesRejectRules(): void {

bastion/tests/integration/helpers/pxe-vm.ts

@@ -28,34 +28,7 @@ export interface PxeVmConfig {
   vcpus: number;
   diskSize: number;     // GB
   network: string;      // libvirt network name
-  /**
-   * Serial console TCP port. Defaults to 4555 for backwards compatibility with
-   * the single-VM tests, but it MUST be unique per concurrently-running VM:
-   * qemu fails with "Failed to find an available port: Address already in use"
-   * and the domain never starts.
-   */
-  serialPort?: number;
   arch?: "x86_64" | "aarch64";
-  /**
-   * Extra NICs enumerated BEFORE the PXE NIC, on a network with no route to
-   * the bastion (defaults to libvirt's "default").
-   *
-   * Real multi-NIC boxes expose a class of bug a single-NIC VM cannot: an
-   * initramfs that picks "the first connected interface" grabs one of these
-   * instead of the NIC that PXE booted, and then cannot reach the bastion.
-   * Defaults to 0 (single NIC).
-   */
-  decoyNics?: number;
-  decoyNetwork?: string;
-  /**
-   * Extra blank data disks, attached after the OS disk. Each gets a fixed
-   * virtio serial, so inside the guest it has a stable
-   * /dev/disk/by-id/virtio-<serial> path -- the moral equivalent of the
-   * nvme-..._<serial> links the dedicated-data-disk feature requires on real
-   * hardware. virtio-blk truncates serials at 20 characters, so keep them
-   * short or the by-id link quietly won't match.
-   */
-  dataDisks?: { serial: string; sizeGb: number }[];
 }
 
 /** Create a blank UEFI VM that PXE boots from the network. */
@@ -69,14 +42,6 @@ export function createPxeVm(config: PxeVmConfig): void {
   const diskPath = join(IMAGE_DIR, `${config.name}.qcow2`);
   run(`qemu-img create -f qcow2 "${diskPath}" ${config.diskSize}G`);
 
-  const dataDiskArgs: string[] = [];
-  for (const dd of config.dataDisks ?? []) {
-    if (dd.serial.length > 20) throw new Error(`virtio serial '${dd.serial}' exceeds 20 chars -- the guest by-id link would be truncated`);
-    const p = join(IMAGE_DIR, `${config.name}-${dd.serial}.qcow2`);
-    run(`qemu-img create -f qcow2 "${p}" ${dd.sizeGb}G`);
-    dataDiskArgs.push(`--disk=path=${p},format=qcow2,bus=virtio,serial=${dd.serial}`);
-  }
-
   // UEFI firmware paths (Fedora)
   if (arch === "aarch64") {
     const aavmf = "/usr/share/edk2/aarch64/QEMU_EFI.fd";
@@ -96,11 +61,6 @@ export function createPxeVm(config: PxeVmConfig): void {
     `--memory=${config.memory}`,
     `--vcpus=${config.vcpus}`,
     `--disk=path=${diskPath},format=qcow2,bus=virtio`,
-    ...dataDiskArgs,
-    // Decoys first so they enumerate ahead of the PXE NIC. They are up and
-    // carry a lease, but have no route to the bastion.
-    ...Array.from({ length: config.decoyNics ?? 0 }, () =>
-      `--network=network=${config.decoyNetwork ?? "default"},model=virtio`),
     `--network=network=${config.network},model=virtio`,
     // UEFI firmware — required for PXE boot in modern mode
     `--boot=uefi,network,hd`,
@@ -111,8 +71,8 @@ export function createPxeVm(config: PxeVmConfig): void {
     // Graphics for debugging (VNC, connect with virt-viewer if needed)
     "--graphics=vnc,listen=127.0.0.1",
     // Serial console via TCP — allows exec without network/SSH
-    // Connect: socat - TCP:127.0.0.1:<serialPort>
-    `--serial=tcp,host=127.0.0.1:${config.serialPort ?? 4555},mode=bind,protocol=telnet`,
+    // Connect: socat - TCP:127.0.0.1:4555
+    "--serial=tcp,host=127.0.0.1:4555,mode=bind,protocol=telnet",
   ];
 
   if (arch === "aarch64") {
@@ -121,7 +81,7 @@ export function createPxeVm(config: PxeVmConfig): void {
 
   log(`Running: virt-install --name=${config.name} --boot=uefi,network ...`);
   run(virtInstallArgs.join(" "), { timeout: 30_000 });
-  log(`PXE VM ${config.name} created (serial: telnet 127.0.0.1 ${config.serialPort ?? 4555})`);
+  log(`PXE VM ${config.name} created (serial: telnet 127.0.0.1 4555)`);
 }
 
 /** Destroy a PXE VM and clean up its disk. */
@@ -135,21 +95,12 @@ export function destroyPxeVm(name: string): void {
 }
 
 /** Get the MAC address of a VM's first NIC. */
-export function getVmMac(name: string, network?: string): string | null {
+export function getVmMac(name: string): string | null {
   const result = virsh("domiflist", name);
   if (result.status !== 0) return null;
   // Output format: Interface  Type  Source  Model  MAC
-  // With decoy NICs present, match the line for the PXE network so we return
-  // the NIC that actually boots rather than whichever is listed first.
-  const lines = result.stdout.split("\n");
-  const candidates = network === undefined
-    ? lines
-    : lines.filter((l) => l.split(/\s+/).includes(network));
-  for (const line of candidates.length > 0 ? candidates : lines) {
-    const m = line.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i);
-    if (m) return m[1].toLowerCase();
-  }
-  return null;
+  const match = result.stdout.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i);
+  return match ? match[1].toLowerCase() : null;
 }
 
 /** Reboot a VM (force off + start). */

bastion/tests/integration/k3s-single-node.test.ts

@@ -161,7 +161,7 @@ EOF'
       CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
       curl -L --fail --silent "https://github.com/cilium/cilium-cli/releases/download/\${CILIUM_CLI_VERSION}/cilium-linux-amd64.tar.gz" | sudo tar xz -C /usr/local/bin
       DEFAULT_DEV=$(ip -4 route show default | awk '{print $5}' | head -1)
-      sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install --set kubeProxyReplacement=true --set ipam.mode=cluster-pool --set ipam.operator.clusterPoolIPv4PodCIDRList='{10.42.0.0/16}' --set ipam.operator.clusterPoolIPv4MaskSize=24 --set devices=$DEFAULT_DEV --set nodePort.directRoutingDevice=$DEFAULT_DEV
+      sudo KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install --set kubeProxyReplacement=true --set ipam.mode=kubernetes --set devices=$DEFAULT_DEV --set nodePort.directRoutingDevice=$DEFAULT_DEV
     `.trim(), "cilium install", { keyPath: sshKeyPath, timeout: 120_000 });
 
     log("Waiting for Cilium to be ready...");

bastion/tests/integration/partition-layout.test.ts

@@ -1,145 +0,0 @@
-// Integration tests: the v2 disk layout, on real VMs.
-//
-// Unit tests prove the kickstart RENDERS the right layout. These prove anaconda
-// ACTUALLY PRODUCES it -- which is a different claim, and the one that matters:
-// the labmaster incident was caused by what landed on disk, not by what was
-// written into a template.
-//
-// Every VM gets a 1 TB sparse disk, because "fits a 1 TB disk with headroom" is
-// the constraint v2 was designed around. qcow2 is sparse, so a 1 TB disk costs
-// ~10-15 GB of real space.
-//
-// Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo, and a Fedora mirror.
-// Set FEDORA_MIRROR to a local mirror -- otherwise each install pulls the full
-// package set over the internet and the matrix takes hours.
-//
-// Run one scenario at a time:
-//   sudo -E npx vitest run -c tests/integration/vitest.config.ts -t "vanilla"
-
-import { describe, it, expect, beforeAll, afterAll } from "vitest";
-import { existsSync, readFileSync } from "node:fs";
-import { homedir } from "node:os";
-import { join } from "node:path";
-import { v2Layout } from "../../src/shared/src/types/layout.js";
-import type { Role } from "../../src/shared/src/types/state.js";
-import {
-  startTestBastion, provisionVm, readLvs, readSchemaTag, destroyVms,
-  type Harness, type ProvisionedVm,
-} from "./helpers/provision-harness.js";
-import { destroyPxeNetwork } from "./helpers/pxe-network.js";
-
-const HTTP_PORT = 8096;
-const SSH_USER = "lab";
-const DISK_GB = 1000; // the smallest disk we will ever buy
-
-function findSshKey(): { pubKey: string; keyPath: string } {
-  const homes = [homedir()];
-  const sudoUser = process.env["SUDO_USER"];
-  if (sudoUser) homes.push(join("/home", sudoUser));
-  for (const home of homes) {
-    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
-      const keyPath = join(home, ".ssh", name);
-      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
-        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
-      }
-    }
-  }
-  throw new Error("no SSH keypair found");
-}
-
-/** Assert the on-disk volumes match the shared v2 definition for this role. */
-function expectV2Layout(h: Harness, vm: ProvisionedVm, role: Role): void {
-  const actual = readLvs(h, vm);
-  const expected = v2Layout(role);
-
-  expect(Object.keys(actual).sort(), `${vm.name} volume set`).toEqual(
-    expected.map((lv) => lv.name).sort(),
-  );
-  for (const spec of expected) {
-    // LVM rounds to extent boundaries (4 MiB), so allow a small delta.
-    expect(actual[spec.name], `${vm.name}/${spec.name} size`).toBeGreaterThanOrEqual(spec.sizeMib - 4);
-    expect(actual[spec.name], `${vm.name}/${spec.name} size`).toBeLessThanOrEqual(spec.sizeMib + 4);
-  }
-  expect(readSchemaTag(h, vm), `${vm.name} schema tag`).toContain("lab-schema-v2");
-}
-
-describe("v2 partition layout on real hardware", () => {
-  let h: Harness;
-  const created: string[] = [];
-
-  beforeAll(async () => {
-    const { pubKey, keyPath } = findSshKey();
-    h = await startTestBastion({
-      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey,
-      sshKeyPath: keyPath, label: "partition",
-    });
-  }, 20 * 60_000);
-
-  afterAll(async () => {
-    destroyVms(...created);
-    try { await h?.close(); } catch { /* already closed */ }
-    try { destroyPxeNetwork(); } catch { /* already gone */ }
-  }, 5 * 60_000);
-
-  // --- Topology 1: vanilla -------------------------------------------------
-  it("vanilla: base volumes only, no rancher, no longhorn", async () => {
-    const name = "v2-vanilla";
-    created.push(name);
-    const vm = await provisionVm(h, { name, role: "vanilla", schemaVersion: 2, diskGb: DISK_GB });
-    expectV2Layout(h, vm, "vanilla");
-
-    const lvs = readLvs(h, vm);
-    expect(lvs["rancher"]).toBeUndefined();
-    expect(lvs["longhorn"]).toBeUndefined();
-  }, 60 * 60_000);
-
-  // --- Topology 2: standalone cluster --------------------------------------
-  it("standalone-cluster: single node with its own control plane and storage", async () => {
-    const name = "v2-standalone";
-    created.push(name);
-    const vm = await provisionVm(h, { name, role: "standalone-cluster", schemaVersion: 2, diskGb: DISK_GB, memory: 6144 });
-    expectV2Layout(h, vm, "standalone-cluster");
-
-    const lvs = readLvs(h, vm);
-    // A cluster of one still gets a Longhorn volume, so the `longhorn`
-    // StorageClass exists and manifests stay portable across clusters.
-    expect(lvs["longhorn"]).toBeDefined();
-    expect(lvs["rancher"]).toBeDefined();
-  }, 60 * 60_000);
-
-  // --- Topology 3: 3x combo (control plane AND worker on every node) -------
-  it("k8s-combo x3: every node is control plane and storage", async () => {
-    const names = ["v2-combo-1", "v2-combo-2", "v2-combo-3"];
-    created.push(...names);
-    const vms: ProvisionedVm[] = [];
-    for (const name of names) {
-      vms.push(await provisionVm(h, { name, role: "k8s-combo", schemaVersion: 2, diskGb: DISK_GB }));
-    }
-    for (const vm of vms) {
-      expectV2Layout(h, vm, "k8s-combo");
-      const lvs = readLvs(h, vm);
-      expect(lvs["longhorn"], `${vm.name} holds storage`).toBeDefined();
-      expect(lvs["rancher"], `${vm.name} has an image store`).toBeDefined();
-    }
-  }, 180 * 60_000);
-
-  // --- Topology 4: 3 controllers + 2 workers -------------------------------
-  it("3x k8s-controller + 2x k8s-worker: storage only on the workers", async () => {
-    const ctrls = ["v2-ctrl-1", "v2-ctrl-2", "v2-ctrl-3"];
-    const workers = ["v2-wrk-1", "v2-wrk-2"];
-    created.push(...ctrls, ...workers);
-
-    for (const name of ctrls) {
-      const vm = await provisionVm(h, { name, role: "k8s-controller", schemaVersion: 2, diskGb: DISK_GB });
-      expectV2Layout(h, vm, "k8s-controller");
-      // The distinguishing property of a controller: image store, no storage.
-      expect(readLvs(h, vm)["longhorn"], `${name} must not hold replicas`).toBeUndefined();
-      expect(readLvs(h, vm)["rancher"]).toBeDefined();
-    }
-    for (const name of workers) {
-      const vm = await provisionVm(h, { name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB });
-      expectV2Layout(h, vm, "k8s-worker");
-      expect(readLvs(h, vm)["longhorn"], `${name} must hold replicas`).toBeDefined();
-    }
-  }, 300 * 60_000);
-});