thebeast runbook: VM drill passed, data copies done 2026-10-14

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V6HNx6cyr4J4CiMGCy1wkF

.gitea/workflows/ci.yml

@@ -28,6 +28,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
+        with:
+          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -47,6 +49,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
+        with:
+          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -54,6 +58,9 @@ jobs:
 
       - run: pnpm install --frozen-lockfile
 
+      - name: Generate Prisma client (labd typecheck needs it)
+        run: pnpm -C src/labd run db:generate
+
       - name: Typecheck
         run: pnpm typecheck
 
@@ -66,6 +73,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
+        with:
+          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -73,6 +82,15 @@ jobs:
 
       - run: pnpm install --frozen-lockfile
 
+      - name: Generate Prisma client (labd typecheck needs it)
+        run: pnpm -C src/labd run db:generate
+
+      - name: Install test validators (ubuntu-autoinstall parses YAML via python, kickstart.test runs ksvalidator, the bastion smoke start runs `ip route`)
+        # pykickstart is not in the runner image's apt sources -- pip it is.
+        run: |
+          sudo apt-get update -qq && sudo apt-get install -y -qq python3-yaml python3-pip iproute2
+          sudo pip3 install --quiet --break-system-packages pykickstart
+
       - name: Build (needed by completions check)
         run: pnpm build
 
@@ -91,6 +109,8 @@ jobs:
       - uses: actions/checkout@v4
 
       - uses: pnpm/action-setup@v4
+        with:
+          package_json_file: bastion/package.json
 
       - uses: actions/setup-node@v4
         with:
@@ -99,6 +119,9 @@ jobs:
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
+      - name: Generate Prisma client (labd typecheck needs it)
+        run: pnpm -C src/labd run db:generate
+
       - name: Build all packages
         run: pnpm build
 
@@ -123,13 +146,13 @@ jobs:
 
       - name: Package x86_64 RPM + DEB
         run: |
-          sed -e 's|^arch:.*|arch: amd64|' -e 's|src: ./dist/lab$|src: ./dist/lab-x86_64|' nfpm.yaml > /tmp/nfpm-x86_64.yaml
+          sed -e 's|^arch:.*|arch: amd64|' -e 's|src: ./dist/labctl$|src: ./dist/lab-x86_64|' nfpm.yaml > /tmp/nfpm-x86_64.yaml
           nfpm pkg --config /tmp/nfpm-x86_64.yaml --packager rpm --target dist/
           nfpm pkg --config /tmp/nfpm-x86_64.yaml --packager deb --target dist/
 
       - name: Package arm64 RPM + DEB
         run: |
-          sed -e 's|^arch:.*|arch: arm64|' -e 's|src: ./dist/lab$|src: ./dist/lab-arm64|' nfpm.yaml > /tmp/nfpm-arm64.yaml
+          sed -e 's|^arch:.*|arch: arm64|' -e 's|src: ./dist/labctl$|src: ./dist/lab-arm64|' nfpm.yaml > /tmp/nfpm-arm64.yaml
           nfpm pkg --config /tmp/nfpm-arm64.yaml --packager rpm --target dist/
           nfpm pkg --config /tmp/nfpm-arm64.yaml --packager deb --target dist/
 

bastion/completions/labctl.bash

@@ -40,8 +40,14 @@ _labctl() {
     "app labcontroller status")
       COMPREPLY=($(compgen -W "--user -h --help" -- "$cur"))
       return ;;
+    "app k3s evacuate")
+      COMPREPLY=($(compgen -W "--user --allow-single-replica -h --help" -- "$cur"))
+      return ;;
+    "app k3s admit")
+      COMPREPLY=($(compgen -W "--user -h --help" -- "$cur"))
+      return ;;
     "app k3s install")
-      COMPREPLY=($(compgen -W "--role --user --k3s-server --k3s-token -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "--role --user --k3s-server --k3s-token --k3s-version -h --help" -- "$cur"))
       return ;;
     "app k3s health")
       COMPREPLY=($(compgen -W "--user -h --help" -- "$cur"))
@@ -59,7 +65,7 @@ _labctl() {
       COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
       return ;;
     "provision install")
-      COMPREPLY=($(compgen -W "--role --os --disk --vyos-mgmt --vyos-mgmt-address --vyos-bond --vyos-bond-address --vyos-bond-vrrp --vlan-vip --vyos-vrrp-priority --vyos-mgmt-vlan --vlan --vyos-password --vyos-hwid --vyos-fresh-config -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "--role --os --disk --schema-version --longhorn-disk --rancher-disk --vyos-mgmt --vyos-mgmt-address --vyos-bond --vyos-bond-address --vyos-bond-vrrp --vlan-vip --vyos-vrrp-priority --vyos-mgmt-vlan --vlan --vyos-password --vyos-hwid --vyos-fresh-config --vyos-bundle --vyos-api-key --vyos-api-listen -h --help" -- "$cur"))
       return ;;
     "provision reprovision")
       COMPREPLY=($(compgen -W "--role --os --disk --user -h --help" -- "$cur"))
@@ -101,7 +107,7 @@ _labctl() {
       COMPREPLY=($(compgen -W "deploy status -h --help" -- "$cur"))
       return ;;
     "app k3s")
-      COMPREPLY=($(compgen -W "install health list kubeconfig -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "evacuate admit install health list kubeconfig -h --help" -- "$cur"))
       return ;;
     "version")
       COMPREPLY=($(compgen -W "-h --help" -- "$cur"))

bastion/completions/labctl.fish

@@ -131,9 +131,12 @@ complete -c labctl -n "__labctl_using_cmd provision" -a makeiso -d 'Generate a U
 complete -c labctl -n "__labctl_using_cmd provision" -a recheck -d 'Refresh hardware info for all installed machines via SSH'
 
 # provision install options
-complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
+complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo'
 complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling'
-complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target disk device (auto-detect if omitted)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target OS disk: kernel name or stable /dev/disk/by-id/ path (auto-detect if omitted)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l schema-version -d 'Disk layout for a FRESH install (default 1). Ignored on reprovision: the layout already on the disk wins.' -xa '1 2'
+complete -c labctl -n "__labctl_in_cmd provision install" -l longhorn-disk -d 'Dedicated whole disk for /var/lib/longhorn, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l rancher-disk -d 'Dedicated whole disk for /var/lib/rancher, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)' -x
 complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt -d 'VyOS: untagged interface the machine PXE boots from (default eth0)' -x
 complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-address -d 'VyOS: CIDR for the management interface, or \'dhcp\' (default dhcp)' -x
 complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond -d 'VyOS: comma-separated LACP bond members (must exclude the PXE NIC)' -x
@@ -146,9 +149,12 @@ complete -c labctl -n "__labctl_in_cmd provision install" -l vlan -d 'VyOS: tagg
 complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-password -d 'VyOS: password for the \'vyos\' user' -x
 complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-hwid -d 'VyOS: pin an interface name to a MAC via hw-id (repeatable)' -x
 complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-fresh-config -d 'VyOS: on reinstall, overwrite the preserved config with the generated one'
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bundle -d 'VyOS: apply a Pulumi-rendered bundle verbatim (kubernetes-deployment/infra/vyos/vyos-bundle.json). Replaces the derived --vyos-bond/--vlan/... config; secret values are left unset for `pulumi up`.' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-api-key -d 'VyOS: enable the HTTP API with this key so Pulumi can manage the router from first boot' -x
+complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-api-listen -d 'VyOS: address the HTTP API binds to (default: the static management address). Required when management is DHCP; the API is never bound to all interfaces.' -x
 
 # provision reprovision options
-complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
+complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo'
 complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling'
 complete -c labctl -n "__labctl_in_cmd provision reprovision" -l disk -d 'Target disk device (auto-detect if omitted)' -x
 complete -c labctl -n "__labctl_in_cmd provision reprovision" -l user -d 'SSH user for the reboot (default: vyos for VyOS machines, else current user)' -x
@@ -157,7 +163,7 @@ complete -c labctl -n "__labctl_in_cmd provision reprovision" -l user -d 'SSH us
 complete -c labctl -n "__labctl_in_cmd provision debug" -l pxe-boot -d 'Boot installed system via PXE (kernel+initrd from network, root from NVMe)'
 
 # provision register options
-complete -c labctl -n "__labctl_in_cmd provision register" -l role -d 'Machine role' -xa 'vanilla worker infra labcontroller'
+complete -c labctl -n "__labctl_in_cmd provision register" -l role -d 'Machine role' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo'
 complete -c labctl -n "__labctl_in_cmd provision register" -l ip -d 'Machine IP address' -x
 
 # provision logs options
@@ -200,16 +206,26 @@ complete -c labctl -n "__labctl_in_cmd app labcontroller deploy" -l crdb-replica
 complete -c labctl -n "__labctl_in_cmd app labcontroller status" -l user -d 'SSH user' -x
 
 # app k3s subcommands
+complete -c labctl -n "__labctl_using_cmd app k3s" -a evacuate -d 'Cordon, evict Longhorn replicas and drain so a node can be safely wiped'
+complete -c labctl -n "__labctl_using_cmd app k3s" -a admit -d 'Return an evacuated node to service: uncordon, clear eviction, restore its Longhorn disk'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a install -d 'Install k3s on a target machine (hostname, IP, or MAC)'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a health -d 'Check k3s health (all hosts if no target given)'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a list -d 'List installed machines and their k3s status'
 complete -c labctl -n "__labctl_using_cmd app k3s" -a kubeconfig -d 'Fetch kubeconfig from a target and merge into ~/.kube/config'
 
+# app k3s evacuate options
+complete -c labctl -n "__labctl_in_cmd app k3s evacuate" -l user -d 'SSH user' -x
+complete -c labctl -n "__labctl_in_cmd app k3s evacuate" -l allow-single-replica -d 'permit evacuating volumes that have only one replica (rebuilds from the only copy; there is no backup target)'
+
+# app k3s admit options
+complete -c labctl -n "__labctl_in_cmd app k3s admit" -l user -d 'SSH user' -x
+
 # app k3s install options
-complete -c labctl -n "__labctl_in_cmd app k3s install" -l role -d 'k3s role: infra (server) or worker (agent)' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l role -d 'k3s role: infra, labcontroller, standalone-cluster, k8s-controller, k8s-combo (servers) or worker, k8s-worker (agents)' -x
 complete -c labctl -n "__labctl_in_cmd app k3s install" -l user -d 'SSH user' -x
-complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-server -d 'k3s server URL (required for worker role)' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-server -d 'k3s server URL (required for agent roles, and to JOIN as a server)' -x
-complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-token -d 'k3s join token (required for worker role)' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-token -d 'k3s join token; on a first server it SEEDS the cluster token' -x
+complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-version -d 'exact k3s version, e.g. v1.34.6+k3s1. Unset installs LATEST, which on a rejoining node can be several minor versions ahead of the cluster' -x
 
 # app k3s health options
 complete -c labctl -n "__labctl_in_cmd app k3s health" -l user -d 'SSH user' -x

bastion/src/bastion/src/main.ts

@@ -4,6 +4,7 @@
 import { mkdirSync, writeFileSync, readFileSync, existsSync, copyFileSync, symlinkSync, unlinkSync } from "node:fs";
 import { execSync } from "node:child_process";
 import type { BastionConfig } from "@lab/shared";
+import { isRole, validateDataDisks } from "@lab/shared";
 import { loadConfig } from "./config.js";
 import { populateNetworkConfig } from "./services/network.js";
 import { createApp } from "./server.js";
@@ -384,6 +385,22 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
     // Wire up command handlers so labd can send install/forget/role commands
     labdConn.onCommand("command-install", async (msg) => {
       if (msg.type !== "command-install") throw new Error("unexpected");
+      // Same guard as POST /api/install: disks named by an operator must
+      // never silently degrade into LVs because they arrived over the socket.
+      const hasDataDisks = msg.dataDisks !== undefined
+        && Object.values(msg.dataDisks).some((d) => d !== undefined);
+      if (hasDataDisks) {
+        if ((msg.os ?? "fedora-43") !== "fedora-43") {
+          return { status: "error", error: `dataDisks are only supported for fedora installs (got os '${msg.os}')` };
+        }
+        if (!isRole(msg.role)) {
+          return { status: "error", error: `invalid role: '${msg.role}'` };
+        }
+        const problem = validateDataDisks(msg.role, msg.dataDisks!);
+        if (problem !== null) {
+          return { status: "error", error: `invalid dataDisks: ${problem}` };
+        }
+      }
       state.update((s) => {
         s.install_queue[msg.mac] = {
           hostname: msg.hostname,
@@ -391,6 +408,8 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
           role: msg.role as import("@lab/shared").Role,
           os: msg.os as import("@lab/shared").OsId,
           queued_at: new Date().toISOString(),
+          ...(msg.schemaVersion ? { schemaVersion: msg.schemaVersion } : {}),
+          ...(hasDataDisks ? { dataDisks: msg.dataDisks } : {}),
           ...(msg.vyos ? { vyos: msg.vyos } : {}),
         };
       });

bastion/src/bastion/src/routes/api.ts

@@ -14,7 +14,7 @@ import { progressBus } from "../services/progress-events.js";
 import type { ProgressEvent } from "../services/progress-events.js";
 import type { InstallLogBuffer } from "../services/install-log.js";
 import type { SyslogListener } from "../services/syslog-listener.js";
-import { isRole, requiresSchemaV2 } from "@lab/shared";
+import { isRole, requiresSchemaV2, validateDataDisks, type DataDisks } from "@lab/shared";
 
 /**
  * Seconds after dispatch with zero progress before a machine is called stalled.
@@ -45,9 +45,10 @@ export function registerApiRoutes(
       os?: string;
       vyos?: VyosInstallSpec;
       schemaVersion?: number;
+      dataDisks?: DataDisks;
     };
   }>("/api/install", async (request, reply) => {
-    const { mac: rawMac, hostname, disk, role, os, vyos, schemaVersion } = request.body ?? {};
+    const { mac: rawMac, hostname, disk, role, os, vyos, schemaVersion, dataDisks } = request.body ?? {};
     const mac = (rawMac ?? "").toLowerCase().replace(/-/g, ":");
 
     if (mac === "") {
@@ -79,6 +80,36 @@ export function registerApiRoutes(
       return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: ${SUPPORTED_OS.join(", ")}` });
     }
 
+    // Dedicated data disks are rendered by the kickstart generator only, and
+    // only in the v2 layout. Rejected here rather than silently dropped at
+    // render time -- an operator who named disks must not get LVs instead.
+    const hasDataDisks = dataDisks !== undefined
+      && Object.values(dataDisks).some((d) => d !== undefined);
+    if (hasDataDisks) {
+      if (osId !== "fedora-43") {
+        return reply.status(400).send({
+          error: `dataDisks are only supported for fedora installs (got os '${osId}')`,
+        });
+      }
+      if (schemaVersion === 1) {
+        return reply.status(400).send({
+          error: "dataDisks require schemaVersion 2 -- the v1 layout is frozen and single-disk",
+        });
+      }
+      if (!isRole(validRole)) {
+        return reply.status(400).send({ error: `invalid role: '${validRole}'` });
+      }
+      if (!requiresSchemaV2(validRole) && schemaVersion === undefined) {
+        return reply.status(400).send({
+          error: `role '${validRole}' defaults to the v1 layout -- pass schemaVersion 2 to use dataDisks`,
+        });
+      }
+      const problem = validateDataDisks(validRole, dataDisks as DataDisks);
+      if (problem !== null) {
+        return reply.status(400).send({ error: `invalid dataDisks: ${problem}` });
+      }
+    }
+
     state.update((s) => {
       s.install_queue[mac] = {
         hostname: hostname ?? "lab-node",
@@ -87,6 +118,7 @@ export function registerApiRoutes(
         os: osId,
         queued_at: new Date().toISOString(),
         ...(schemaVersion ? { schemaVersion: schemaVersion as 1 | 2 } : {}),
+        ...(hasDataDisks ? { dataDisks } : {}),
         ...(vyos ? { vyos } : {}),
       };
     });

bastion/src/bastion/src/routes/kickstart.ts

@@ -31,6 +31,7 @@ export function registerKickstartRoutes(
       disk: queueEntry?.disk ?? "",
       role: queueEntry?.role ?? "worker",
       ...(queueEntry?.schemaVersion ? { schemaVersion: queueEntry.schemaVersion } : {}),
+      ...(queueEntry?.dataDisks ? { dataDisks: queueEntry.dataDisks } : {}),
     });
 
     return reply.type("text/plain").send(ks);

bastion/src/bastion/src/services/kickstart-generator.ts

@@ -25,6 +25,7 @@ export function generateInstallKickstart(
     disk: string;
     role: Role;
     schemaVersion?: 1 | 2;
+    dataDisks?: import("@lab/shared").DataDisks;
   },
 ): string {
   const ksParams: InstallKickstartParams = {
@@ -32,6 +33,7 @@ export function generateInstallKickstart(
     disk: params.disk,
     role: params.role,
     ...(params.schemaVersion ? { schemaVersion: params.schemaVersion } : {}),
+    ...(params.dataDisks ? { dataDisks: params.dataDisks } : {}),
     domain: config.domain,
     fedoraVersion: config.fedoraVersion,
     timezone: config.timezone,

bastion/src/bastion/src/templates/install.ks.ts

@@ -2,8 +2,11 @@
 // Full Fedora server install with LVM partitioning, %pre for reprovision detection,
 // packages, and %post with SSH keys, user creation, k3s prereqs, progress callbacks.
 
-import type { Role } from "@lab/shared";
+import type { Role, DataDisks, DataDiskName } from "@lab/shared";
-import { v2Layout, V2_BOOT, requiresSchemaV2, effectiveSchemaVersion, isK3sServer, type LvSpec } from "@lab/shared";
+import {
+  v2Layout, V2_BOOT, requiresSchemaV2, effectiveSchemaVersion, isK3sServer,
+  validateDataDisks, DATA_DISK_LABELS, type LvSpec,
+} from "@lab/shared";
 
 export interface InstallKickstartParams {
   hostname: string;
@@ -27,6 +30,13 @@ export interface InstallKickstartParams {
    * never silently change its layout -- that is how data gets destroyed.
    */
   schemaVersion?: 1 | 2;
+  /**
+   * Dedicated whole-disk data volumes, by stable /dev/disk/by-id path (v2
+   * only). The matching LV is not created; the disk gets one labelled xfs
+   * partition instead. On a reprovision %pre rediscovers these by label, so a
+   * reinstall preserves them whether or not the queue entry names them.
+   */
+  dataDisks?: DataDisks;
 }
 
 export function renderInstallKickstart(params: InstallKickstartParams): string {
@@ -72,8 +82,24 @@ export function renderInstallKickstart(params: InstallKickstartParams): string {
       `Pass schemaVersion: 2 (labctl provision install --schema-version 2).`,
     );
   }
+
+  const dataDisks = params.dataDisks ?? {};
+  const hasDataDisks = Object.values(dataDisks).some((d) => d !== undefined);
+  if (hasDataDisks) {
+    // The v1 branches are frozen and know nothing about dedicated disks, so a
+    // v1 render with them would silently create BOTH the LV and the disk.
+    if (schemaVersion !== 2) {
+      throw new Error(
+        `dedicated data disks require the v2 layout - pass schemaVersion: 2 ` +
+        `(labctl provision install --schema-version 2)`,
+      );
+    }
+    const problem = validateDataDisks(role, dataDisks);
+    if (problem !== null) throw new Error(problem);
+  }
+
   const v2RootMib = v2Layout(role).find((lv: LvSpec) => lv.name === "root")?.sizeMib ?? 32768;
-  const v2 = v2Layout(role);
+  const v2 = v2Layout(role, { dataDisks });
   const v2LogvolLines = v2
     .map((lv: LvSpec) =>
       lv.fs === "swap"
@@ -149,14 +175,29 @@ chmod 440 /etc/sudoers.d/${adminUser}`;
   }
 
   // -- Disk detection --
+  // An explicit disk may be a stable /dev/disk/by-id path: kernel names are
+  // not stable across boots on multi-NVMe hosts, so a pinned install must be
+  // able to name the disk by serial. Anaconda directives want the kernel name,
+  // so %pre resolves the symlink at install time.
   const diskLine = disk
-    ? `DISK="${disk}"`
+    ? disk.startsWith("/dev/")
+      ? `# OS disk pinned by stable path -- resolve to the kernel name anaconda wants
+DISK=""
+for _wait in $(seq 1 10); do
+    _os_target=$(readlink -f "${disk}" 2>/dev/null || true)
+    if [ -n "$_os_target" ] && [ -b "$_os_target" ]; then DISK=$(basename "$_os_target"); break; fi
+    sleep 1
+done
+[ -z "$DISK" ] && { bastion_progress "error" "OS disk not found: ${disk}"; echo "ERROR: OS disk not found: ${disk}"; exit 1; }`
+      : `DISK="${disk}"`
     : `DISK=""
 # Wait up to 10s for NVMe/SCSI disks to appear (they init async in initrd)
 for _wait in $(seq 1 10); do
     for d in /dev/nvme0n1 /dev/nvme1n1 /dev/sda /dev/sdb /dev/vda; do
         [ -b "$d" ] || continue
         _bname=$(basename "$d")
+        # Never auto-pick a dedicated data disk as the OS disk
+        case "$_bname" in "$LONGHORN_DISK"|"$RANCHER_DISK") continue ;; esac
         # Skip removable disks (USB, CD-ROM, JetKVM virtual media)
         [ -f "/sys/block/$_bname/removable" ] && [ "$(cat /sys/block/$_bname/removable)" = "1" ] && continue
         # Skip USB-attached disks (JetKVM virtual media shows as SCSI over USB)
@@ -172,6 +213,80 @@ for _wait in $(seq 1 10); do
 done
 [ -z "$DISK" ] && { echo "ERROR: no disk found"; exit 1; }`;
 
+  // -- Dedicated data disks: %pre resolution --
+  // Resolved BEFORE the OS disk so auto-detect can refuse to pick one of them.
+  const dataDiskResolveBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[])
+    .map((name) => {
+      const device = dataDisks[name];
+      const VAR = name.toUpperCase();
+      if (device === undefined) return `${VAR}_DISK=""`;
+      return `# ${name} gets a whole disk of its own, pinned by stable path
+${VAR}_DISK_ID="${device}"
+${VAR}_DISK=""
+for _wait in $(seq 1 10); do
+    _dd_target=$(readlink -f "$${VAR}_DISK_ID" 2>/dev/null || true)
+    if [ -n "$_dd_target" ] && [ -b "$_dd_target" ]; then ${VAR}_DISK=$(basename "$_dd_target"); break; fi
+    sleep 1
+done
+[ -z "$${VAR}_DISK" ] && { bastion_progress "error" "${name} disk not found: $${VAR}_DISK_ID"; echo "ERROR: ${name} disk not found: $${VAR}_DISK_ID"; exit 1; }`;
+    })
+    .join("\n");
+
+  // -- Dedicated data disks: discovery and preservation decisions --
+  // A labelled data partition is on-disk truth, exactly like the lab-schema-v2
+  // LVM tag: it is preserved whether or not this install names the disk, and it
+  // is never silently wiped or shadowed. blkid finds it wherever tonight's NVMe
+  // enumeration put it.
+  const dataDiskDecisionBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[])
+    .map((name) => {
+      const VAR = name.toUpperCase();
+      const label = DATA_DISK_LABELS[name];
+      const mount = name === "longhorn" ? "/var/lib/longhorn" : "/var/lib/rancher";
+      return `${VAR}_PART=$(blkid -t LABEL=${label} -o device 2>/dev/null | head -1)
+${VAR}_REUSE=no
+if [ -n "$${VAR}_PART" ]; then
+    _dd_parent=$(lsblk -ndo pkname "$${VAR}_PART" 2>/dev/null)
+    if [ -n "$${VAR}_DISK" ] && [ "$_dd_parent" != "$${VAR}_DISK" ]; then
+        bastion_progress "error" "${label} label is on $_dd_parent, not on the configured ${name} disk $${VAR}_DISK"
+        echo "ERROR: ${label} label found on $_dd_parent but the ${name} disk is configured as $${VAR}_DISK -- refusing to guess"
+        exit 1
+    fi
+    if [ "$REPROVISION" = "no" ] && [ -z "$${VAR}_DISK" ]; then
+        bastion_progress "error" "found ${label} data on $_dd_parent but this install configures no ${name} disk"
+        echo "ERROR: ${label} labelled data exists on $_dd_parent. Refusing to wipe or shadow it -- queue the install with a ${name} data disk, or wipe the label first"
+        exit 1
+    fi
+    ${VAR}_DISK="$_dd_parent"
+    ${VAR}_REUSE=yes
+fi
+if [ "$PRESERVE_${VAR}" = "yes" ] && [ -n "$${VAR}_DISK" ]; then
+    bastion_progress "error" "both a ${name} LV and a dedicated ${name} disk exist"
+    echo "ERROR: VG $VG has a ${name} LV AND a dedicated ${name} disk is present -- two things cannot mount ${mount}. Migrate the LV off first"
+    exit 1
+fi`;
+    })
+    .join("\n\n");
+
+  // -- Dedicated data disks: partition directives --
+  // Appended to whichever /tmp/part.ks the branches above wrote. A reused
+  // partition is mounted --noformat; a fresh disk gets ONE partition spanning
+  // it. --grow is fine HERE: the whole point of a dedicated disk is to consume
+  // it, unlike LVs competing inside a shared VG.
+  const dataDiskPartBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[])
+    .map((name) => {
+      const VAR = name.toUpperCase();
+      const label = DATA_DISK_LABELS[name];
+      const mount = name === "longhorn" ? "/var/lib/longhorn" : "/var/lib/rancher";
+      return `if [ -n "$${VAR}_DISK" ]; then
+    if [ "$${VAR}_REUSE" = "yes" ]; then
+        echo "part ${mount} --onpart=$${VAR}_PART --noformat" >> /tmp/part.ks
+    else
+        echo "part ${mount} --fstype=xfs --ondisk=$${VAR}_DISK --grow --size=1 --label=${label}" >> /tmp/part.ks
+    fi
+fi`;
+    })
+    .join("\n");
+
   // -- Longhorn LV for fresh install --
   const longhornFreshLine = hasLonghorn
     ? `logvol /var/lib/longhorn --vgname=${vg} --name=longhorn --fstype=xfs --grow --size=1`
@@ -229,6 +344,9 @@ bastion_progress() {
 bastion_progress "partitioning" "detecting disk"
 
 VG="${vg}"
+
+${dataDiskResolveBlocks}
+
 ${diskLine}
 
 REPROVISION=no
@@ -265,6 +383,19 @@ else
     bastion_progress "partitioning" "fresh install on $DISK"
 fi
 
+${dataDiskDecisionBlocks}
+
+# Every disk anaconda may touch; everything else stays invisible to it.
+KS_DISKS="$DISK"
+[ -n "$LONGHORN_DISK" ] && KS_DISKS="$KS_DISKS,$LONGHORN_DISK"
+[ -n "$RANCHER_DISK" ] && [ "$RANCHER_DISK" != "$LONGHORN_DISK" ] && KS_DISKS="$KS_DISKS,$RANCHER_DISK"
+
+# Fresh installs wipe the OS disk plus any data disk being formatted for the
+# first time -- NEVER one carrying a lab-* label (those are reused --noformat).
+CLEARPART_DRIVES="$DISK"
+[ -n "$LONGHORN_DISK" ] && [ "$LONGHORN_REUSE" = "no" ] && CLEARPART_DRIVES="$CLEARPART_DRIVES,$LONGHORN_DISK"
+[ -n "$RANCHER_DISK" ] && [ "$RANCHER_REUSE" = "no" ] && [ "$RANCHER_DISK" != "$LONGHORN_DISK" ] && CLEARPART_DRIVES="$CLEARPART_DRIVES,$RANCHER_DISK"
+
 if [ "$REPROVISION" = "yes" ]; then
     # Find existing boot partitions by type
     EFI_PART=$(blkid -t TYPE=vfat -o device /dev/\${DISK}* 2>/dev/null | head -1)
@@ -275,7 +406,7 @@ if [ "$REPROVISION" = "yes" ]; then
 
     # Build partition config reusing existing PV/VG
     cat > /tmp/part.ks << PARTEOF
-ignoredisk --only-use=$DISK
+ignoredisk --only-use=$KS_DISKS
 clearpart --none
 part /boot/efi --onpart=$EFI_PART --fstype=efi
 part /boot --onpart=$BOOT_PART --fstype=ext4
@@ -320,8 +451,8 @@ PARTEOF
 else
     # Fresh install
 ${schemaVersion === 2 ? `    cat > /tmp/part.ks << PARTEOF
-ignoredisk --only-use=$DISK
+ignoredisk --only-use=$KS_DISKS
-clearpart --all --initlabel --drives=$DISK
+clearpart --all --initlabel --drives=$CLEARPART_DRIVES
 part /boot/efi --fstype=efi --size=${V2_BOOT.efiMib} --ondisk=$DISK
 part /boot --fstype=ext4 --size=${V2_BOOT.bootMib} --ondisk=$DISK
 part pv.01 --size=1 --grow --ondisk=$DISK
@@ -349,6 +480,8 @@ ${rancherFreshLine}
 PARTEOF`}
 fi
 
+${dataDiskPartBlocks}
+
 echo "=== Generated partition config ==="
 cat /tmp/part.ks
 echo "==================================="

bastion/src/bastion/tests/data-disks.test.ts

@@ -0,0 +1,274 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { mkdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import type { FastifyInstance } from "fastify";
+import { renderInstallKickstart } from "../src/templates/install.ks.js";
+import type { InstallKickstartParams } from "../src/templates/install.ks.js";
+import { createApp } from "../src/server.js";
+import { validateDataDisks, v2Layout, DATA_DISK_LABELS, type BastionConfig, type DataDisks } from "@lab/shared";
+
+/**
+ * Dedicated whole-disk data volumes (thebeast: 1 TB OS + 2 TB longhorn +
+ * 2 TB rancher). The layout contract these tests pin down:
+ *
+ *   - naming a data disk DROPS the matching LV -- never both.
+ *   - disks are named by /dev/disk/by-id only; kernel names are rejected,
+ *     because nvme enumeration order is not stable across boots.
+ *   - a lab-longhorn / lab-rancher labelled partition is on-disk truth, like
+ *     the lab-schema-v2 VG tag: reused --noformat, never wiped, never
+ *     silently shadowed.
+ */
+
+const BY_ID = "/dev/disk/by-id";
+const LONGHORN_ID = `${BY_ID}/nvme-Sabrent_Rocket_4.0_2TB_7F600711179D89261778`;
+const RANCHER_ID = `${BY_ID}/nvme-Sabrent_Rocket_4.0_2TB_7FD4070C1EDA92197301`;
+const OS_ID = `${BY_ID}/nvme-Samsung_SSD_970_EVO_Plus_1TB_S4EWNJ0N138234M`;
+
+function baseParams(over: Partial<InstallKickstartParams> = {}): InstallKickstartParams {
+  return {
+    hostname: "thebeast", disk: "", role: "k8s-worker", domain: "ad.itaz.eu",
+    fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8",
+    serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514,
+    sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal",
+    dataDisks: { longhorn: LONGHORN_ID, rancher: RANCHER_ID },
+    ...over,
+  };
+}
+
+/** The fresh-install part.ks heredoc (the one that creates the PV). */
+function freshBlock(ks: string): string {
+  const blocks = [...ks.matchAll(/cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g)]
+    .map((m) => m[1]!);
+  const fresh = blocks.find((b) => b.includes("part pv.01"));
+  if (!fresh) throw new Error("no fresh-install part.ks block found");
+  return fresh;
+}
+
+describe("validateDataDisks", () => {
+  const ok: DataDisks = { longhorn: LONGHORN_ID, rancher: RANCHER_ID };
+
+  it("accepts a sane spec for a role that hosts longhorn", () => {
+    expect(validateDataDisks("k8s-worker", ok)).toBeNull();
+    expect(validateDataDisks("k8s-combo", ok)).toBeNull();
+    expect(validateDataDisks("worker", ok)).toBeNull();
+  });
+
+  it("rejects kernel names -- they are not stable across boots", () => {
+    expect(validateDataDisks("k8s-worker", { longhorn: "nvme1n1" })).toMatch(/by-id/);
+    expect(validateDataDisks("k8s-worker", { rancher: "/dev/nvme2n1" })).toMatch(/by-id/);
+  });
+
+  it("rejects a longhorn disk for a role that holds no replicas", () => {
+    expect(validateDataDisks("k8s-controller", { longhorn: LONGHORN_ID })).toMatch(/no Longhorn replicas/);
+    expect(validateDataDisks("vanilla", { longhorn: LONGHORN_ID })).toMatch(/no Longhorn replicas/);
+  });
+
+  it("rejects a rancher disk for a role that runs no k3s", () => {
+    expect(validateDataDisks("vanilla", { rancher: RANCHER_ID })).toMatch(/no k3s/);
+  });
+
+  it("a control-plane-only node may still take a rancher disk", () => {
+    expect(validateDataDisks("k8s-controller", { rancher: RANCHER_ID })).toBeNull();
+  });
+
+  it("rejects the same disk doing both jobs", () => {
+    expect(validateDataDisks("k8s-worker", { longhorn: LONGHORN_ID, rancher: LONGHORN_ID }))
+      .toMatch(/same disk/);
+  });
+});
+
+describe("v2Layout with dedicated disks", () => {
+  it("drops exactly the LV a dedicated disk replaces", () => {
+    const names = (d: DataDisks) => v2Layout("k8s-worker", { dataDisks: d }).map((l) => l.name);
+    expect(names({})).toContain("longhorn");
+    expect(names({ longhorn: LONGHORN_ID })).not.toContain("longhorn");
+    expect(names({ longhorn: LONGHORN_ID })).toContain("rancher");
+    expect(names({ longhorn: LONGHORN_ID, rancher: RANCHER_ID })).not.toContain("rancher");
+    // base volumes untouched
+    expect(names({ longhorn: LONGHORN_ID, rancher: RANCHER_ID }))
+      .toEqual(expect.arrayContaining(["swap", "root", "var", "varlog", "home", "srv"]));
+  });
+});
+
+describe("renderInstallKickstart with data disks", () => {
+  it("refuses the v1 layout -- it is frozen and single-disk", () => {
+    expect(() => renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 1 })))
+      .toThrow(/v2 layout/);
+    // a legacy role with no explicit schema defaults to v1 -> same refusal
+    expect(() => renderInstallKickstart(baseParams({ role: "worker" })))
+      .toThrow(/v2 layout/);
+  });
+
+  it("refuses a kernel-name data disk", () => {
+    expect(() => renderInstallKickstart(baseParams({ dataDisks: { longhorn: "nvme1n1" } })))
+      .toThrow(/by-id/);
+  });
+
+  it("creates no longhorn/rancher LV when both mounts come from dedicated disks", () => {
+    const fresh = freshBlock(renderInstallKickstart(baseParams()));
+    expect(fresh).not.toContain("--name=longhorn");
+    expect(fresh).not.toContain("--name=rancher");
+    // base LVs still there
+    expect(fresh).toContain("--name=root");
+  });
+
+  it("anaconda only sees the named disks, and only wipes unlabelled ones", () => {
+    const ks = renderInstallKickstart(baseParams());
+    const fresh = freshBlock(ks);
+    expect(fresh).toContain("ignoredisk --only-use=$KS_DISKS");
+    expect(fresh).toContain("clearpart --all --initlabel --drives=$CLEARPART_DRIVES");
+    // a disk whose partition carries our label is excluded from clearpart
+    expect(ks).toContain('[ "$LONGHORN_REUSE" = "no" ] && CLEARPART_DRIVES=');
+  });
+
+  it("%pre resolves each by-id path and fails loudly when a disk is missing", () => {
+    const ks = renderInstallKickstart(baseParams());
+    expect(ks).toContain(`LONGHORN_DISK_ID="${LONGHORN_ID}"`);
+    expect(ks).toContain(`RANCHER_DISK_ID="${RANCHER_ID}"`);
+    expect(ks).toContain('echo "ERROR: longhorn disk not found: $LONGHORN_DISK_ID"; exit 1');
+  });
+
+  it("a labelled partition is reused --noformat; a fresh disk gets one labelled partition", () => {
+    const ks = renderInstallKickstart(baseParams());
+    expect(ks).toContain("part /var/lib/longhorn --onpart=$LONGHORN_PART --noformat");
+    expect(ks).toContain(
+      `part /var/lib/longhorn --fstype=xfs --ondisk=$LONGHORN_DISK --grow --size=1 --label=${DATA_DISK_LABELS.longhorn}`,
+    );
+    expect(ks).toContain("part /var/lib/rancher --onpart=$RANCHER_PART --noformat");
+    expect(ks).toContain(
+      `part /var/lib/rancher --fstype=xfs --ondisk=$RANCHER_DISK --grow --size=1 --label=${DATA_DISK_LABELS.rancher}`,
+    );
+  });
+
+  it("refuses to run when a data LV and a dedicated disk both exist", () => {
+    const ks = renderInstallKickstart(baseParams());
+    expect(ks).toContain('[ "$PRESERVE_LONGHORN" = "yes" ] && [ -n "$LONGHORN_DISK" ]');
+    expect(ks).toContain('[ "$PRESERVE_RANCHER" = "yes" ] && [ -n "$RANCHER_DISK" ]');
+  });
+
+  it("a fresh install never wipes or shadows a labelled disk it was not told about", () => {
+    // Rendered even WITHOUT dataDisks: the label is on-disk truth and the
+    // check protects against a queue entry that forgot the disks.
+    const ks = renderInstallKickstart(baseParams({ role: "k8s-worker", dataDisks: {} }));
+    expect(ks).toContain(`blkid -t LABEL=${DATA_DISK_LABELS.longhorn}`);
+    expect(ks).toContain("Refusing to wipe or shadow it");
+  });
+
+  it("an OS disk given as a by-id path is resolved in %pre", () => {
+    const ks = renderInstallKickstart(baseParams({ disk: OS_ID }));
+    expect(ks).toContain(`readlink -f "${OS_ID}"`);
+    expect(ks).not.toContain(`DISK="${OS_ID}"`); // never fed to anaconda unresolved
+  });
+
+  it("auto-detect never picks a data disk as the OS disk", () => {
+    const ks = renderInstallKickstart(baseParams({ disk: "" }));
+    expect(ks).toContain('case "$_bname" in "$LONGHORN_DISK"|"$RANCHER_DISK") continue ;; esac');
+  });
+
+  it("a plain kernel-name OS disk still renders exactly as before", () => {
+    const ks = renderInstallKickstart(baseParams({ disk: "vda", dataDisks: {} }));
+    expect(ks).toContain('DISK="vda"');
+  });
+
+  it("the %pre block still survives the /ks-auto extractor", () => {
+    const ks = renderInstallKickstart(baseParams());
+    const blocks = [...ks.matchAll(/%pre[^\n]*\n([\s\S]*?)%end/g)];
+    expect(blocks.length).toBeGreaterThan(0);
+    expect(blocks[0]![1]).toContain("/tmp/part.ks");
+  });
+});
+
+describe("dataDisks plumbing: /api/install -> /ks", () => {
+  let app: FastifyInstance;
+  let testDir: string;
+
+  function testConfig(dir: string): BastionConfig {
+    return {
+      fedoraVersion: "43", arch: "x86_64", httpPort: 0,
+      timezone: "Europe/London", locale: "en_GB.UTF-8",
+      bastionDir: dir, domain: "test.local", dhcpMode: "proxy",
+      dhcpRangeStart: "", dhcpRangeEnd: "",
+      ubuntuVersion: "26.04", ubuntuMirror: "https://releases.ubuntu.com/26.04",
+      vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso",
+      vyosDefaultPassword: "vyos", iface: "eth0",
+      serverIp: "10.0.0.1", network: "10.0.0.0", gateway: "10.0.0.1",
+      sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"],
+      adminUser: "testadmin", syslogPort: 15514,
+      skipDnsmasq: true, skipArtifacts: true,
+      fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
+      tftpDir: join(dir, "tftp"), httpDir: join(dir, "http"), stateFile: join(dir, "state.json"),
+    };
+  }
+
+  beforeEach(async () => {
+    testDir = join(tmpdir(), `datadisks-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+    mkdirSync(testDir, { recursive: true });
+    const built = await createApp(testConfig(testDir));
+    app = built.app;
+    await app.ready();
+  });
+
+  afterEach(async () => {
+    await app.close();
+    rmSync(testDir, { recursive: true, force: true });
+  });
+
+  const queue = (body: Record<string, unknown>) =>
+    app.inject({ method: "POST", url: "/api/install", payload: { mac: "aa:bb:cc:dd:ee:03", hostname: "thebeast", ...body } });
+
+  const ks = async () => (await app.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:03" })).body;
+
+  it("named disks reach the rendered kickstart, and their LVs are gone", async () => {
+    const res = await queue({ role: "k8s-worker", dataDisks: { longhorn: LONGHORN_ID, rancher: RANCHER_ID } });
+    expect(res.statusCode).toBe(200);
+    const body = await ks();
+    expect(body).toContain(LONGHORN_ID);
+    expect(body).toContain(RANCHER_ID);
+    expect(freshBlock(body)).not.toContain("--name=longhorn");
+    expect(freshBlock(body)).not.toContain("--name=rancher");
+  });
+
+  it("rejects dataDisks on a non-fedora install instead of silently dropping them", async () => {
+    const res = await queue({ role: "k8s-worker", os: "ubuntu-26.04", dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/fedora/);
+  });
+
+  it("rejects dataDisks with an explicit v1 schema", async () => {
+    const res = await queue({ role: "worker", schemaVersion: 1, dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/schemaVersion 2/);
+  });
+
+  it("rejects dataDisks for a legacy role that would default to v1", async () => {
+    const res = await queue({ role: "worker", dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/schemaVersion 2/);
+  });
+
+  it("legacy role + explicit v2 + dataDisks is accepted", async () => {
+    const res = await queue({ role: "worker", schemaVersion: 2, dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(200);
+    expect(await ks()).toContain(LONGHORN_ID);
+  });
+
+  it("rejects a kernel-name disk with the shared message", async () => {
+    const res = await queue({ role: "k8s-worker", dataDisks: { longhorn: "nvme1n1" } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/by-id/);
+  });
+
+  it("rejects a capability mismatch (longhorn disk on a control-plane-only role)", async () => {
+    const res = await queue({ role: "k8s-controller", dataDisks: { longhorn: LONGHORN_ID } });
+    expect(res.statusCode).toBe(400);
+    expect(res.json().error).toMatch(/no Longhorn replicas/);
+  });
+
+  it("an install without dataDisks is untouched by all of this", async () => {
+    expect((await queue({ role: "k8s-worker" })).statusCode).toBe(200);
+    const body = await ks();
+    expect(freshBlock(body)).toContain("--name=longhorn --fstype=xfs --size=307200");
+    expect(freshBlock(body)).toContain("--name=rancher --fstype=xfs --size=256000");
+  });
+});

bastion/src/cli/src/api/client.ts

@@ -91,6 +91,7 @@ export class LabdClient {
   async installMachine(opts: {
     mac: string; hostname: string; disk?: string; role?: string; os?: string;
     schemaVersion?: 1 | 2;
+    dataDisks?: import("@lab/shared").DataDisks;
     vyos?: import("@lab/shared").VyosInstallSpec;
   }): Promise<{ status: string; data?: unknown; error?: string }> {
     return this.request("POST", "/api/machines/install", { body: opts });

bastion/src/cli/src/commands/install.ts

@@ -3,8 +3,8 @@
 
 import { readFileSync } from "node:fs";
 import { Command, Option, InvalidArgumentError } from "commander";
-import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY } from "@lab/shared";
+import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isRole, requiresSchemaV2, validateDataDisks } from "@lab/shared";
-import type { VyosBundle, VyosInstallSpec, VyosVlanSpec } from "@lab/shared";
+import type { DataDisks, VyosBundle, VyosInstallSpec, VyosVlanSpec } from "@lab/shared";
 import { getLabdClient } from "../api/config.js";
 
 /**
@@ -106,8 +106,16 @@ export function registerInstallCommand(parent: Command): void {
     .addHelpText("after", roleTable())
     .addOption(new Option("--role <role>", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker"))
     .addOption(new Option("--os <os>", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43"))
-    .option("--disk <device>", "Target disk device (auto-detect if omitted)")
+    .option("--disk <device>", "Target OS disk: kernel name or stable /dev/disk/by-id/ path (auto-detect if omitted)")
     .addOption(new Option("--schema-version <n>", "Disk layout for a FRESH install (default 1). Ignored on reprovision: the layout already on the disk wins.").choices(["1", "2"]))
+    .option(
+      "--longhorn-disk <by-id>",
+      "Dedicated whole disk for /var/lib/longhorn, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)",
+    )
+    .option(
+      "--rancher-disk <by-id>",
+      "Dedicated whole disk for /var/lib/rancher, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)",
+    )
     .option("--vyos-mgmt <iface>", "VyOS: untagged interface the machine PXE boots from (default eth0)")
     .option("--vyos-mgmt-address <addr>", "VyOS: CIDR for the management interface, or 'dhcp' (default dhcp)")
     .option("--vyos-bond <ifaces>", "VyOS: comma-separated LACP bond members (must exclude the PXE NIC)")
@@ -140,6 +148,8 @@ export function registerInstallCommand(parent: Command): void {
       /** commander gives us the string form of the --schema-version choice. */
       schemaVersion?: string;
       disk?: string;
+      longhornDisk?: string;
+      rancherDisk?: string;
       vyosMgmt?: string;
       vyosMgmtAddress?: string;
       vyosBond?: string;
@@ -166,6 +176,32 @@ export function registerInstallCommand(parent: Command): void {
         process.exit(1);
       }
 
+      // Dedicated data disks -- validated here for fast feedback, and again on
+      // the bastion, which is the enforcement point.
+      const dataDisks: DataDisks = {
+        ...(opts.longhornDisk !== undefined && opts.longhornDisk !== ""
+          ? { longhorn: opts.longhornDisk } : {}),
+        ...(opts.rancherDisk !== undefined && opts.rancherDisk !== ""
+          ? { rancher: opts.rancherDisk } : {}),
+      };
+      const hasDataDisks = Object.keys(dataDisks).length > 0;
+      if (hasDataDisks) {
+        if (opts.os !== "fedora-43") {
+          console.error(`--longhorn-disk/--rancher-disk only work with --os fedora-43 (got --os ${opts.os})`);
+          process.exit(1);
+        }
+        if (opts.schemaVersion === "1"
+            || (opts.schemaVersion === undefined && isRole(opts.role) && !requiresSchemaV2(opts.role))) {
+          console.error("--longhorn-disk/--rancher-disk need the v2 layout -- add --schema-version 2");
+          process.exit(1);
+        }
+        const problem = isRole(opts.role) ? validateDataDisks(opts.role, dataDisks) : null;
+        if (problem !== null) {
+          console.error(`Invalid data disks: ${problem}`);
+          process.exit(1);
+        }
+      }
+
       const bondMembers = opts.vyosBond !== undefined && opts.vyosBond !== ""
         ? opts.vyosBond.split(",").map((s) => s.trim()).filter((s) => s.length > 0)
         : [];
@@ -257,6 +293,7 @@ export function registerInstallCommand(parent: Command): void {
           role: opts.role,
           os: opts.os,
           ...(opts.disk ? { disk: opts.disk } : {}),
+          ...(hasDataDisks ? { dataDisks } : {}),
           ...(hasVyosOptions ? { vyos } : {}),
         });
 

bastion/src/cli/tests/smoke-bastion.test.ts

@@ -143,7 +143,10 @@ describe("bastion smoke tests", () => {
       await sleep(200);
     }
 
-    expect(stdout).toContain("Waiting for PXE boot requests");
+    // On failure, show what the process actually said -- an empty stdout with
+    // a hidden stderr is undiagnosable from a CI log.
+    expect(stdout, `bastion never printed its banner; stderr was:\n${stderr}`)
+      .toContain("Waiting for PXE boot requests");
     expect(stdout).toContain("HTTP server listening");
 
     // Verify the process is still alive after startup

bastion/src/labd/src/routes/bastions.ts

@@ -164,9 +164,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
 
   // Queue install — route to correct bastion by MAC
   app.post<{
-    Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string; vyos?: VyosInstallSpec };
+    Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string; schemaVersion?: 1 | 2; dataDisks?: import("@lab/shared").DataDisks; vyos?: VyosInstallSpec };
   }>("/api/machines/install", async (request, reply) => {
-    const { mac, hostname, disk, role, os, vyos } = request.body ?? {};
+    const { mac, hostname, disk, role, os, schemaVersion, dataDisks, vyos } = request.body ?? {};
     if (!mac || !hostname) {
       return reply.code(400).send({ error: "mac and hostname are required" });
     }
@@ -184,6 +184,8 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
           const result = await sendCommand(all[0]!.bastionId, {
             type: "command-install",
             mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
+            ...(schemaVersion ? { schemaVersion } : {}),
+            ...(dataDisks ? { dataDisks } : {}),
             ...(vyos ? { vyos } : {}),
           });
           return reply.code(result.status === "ok" ? 200 : 500).send(result);
@@ -198,6 +200,8 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
       const result = await sendCommand(bastion.bastionId, {
         type: "command-install",
         mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
+        ...(schemaVersion ? { schemaVersion } : {}),
+        ...(dataDisks ? { dataDisks } : {}),
         ...(vyos ? { vyos } : {}),
       });
       return reply.code(result.status === "ok" ? 200 : 500).send(result);

bastion/src/shared/src/index.ts

@@ -51,5 +51,6 @@ export {
 } from "./errors/index.js";
 
 export { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB,
-  SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion } from "./types/index.js";
+  SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion,
-export type { LvSpec, SchemaVersion } from "./types/index.js";
+  validateDataDisks, DATA_DISK_LABELS } from "./types/index.js";
+export type { LvSpec, SchemaVersion, DataDisks, DataDiskName } from "./types/index.js";

bastion/src/shared/src/protocol/index.ts

@@ -109,7 +109,7 @@ export type BastionMessage =
 export type LabdBastionMessage =
   | { type: "bastion-enrolled"; bastionId: string }
   | { type: "bastion-heartbeat-ack"; serverTime: string }
-  | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string; vyos?: VyosInstallSpec }
+  | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string; schemaVersion?: 1 | 2; dataDisks?: import("../types/layout.js").DataDisks; vyos?: VyosInstallSpec }
   | { type: "command-forget"; requestId: string; mac: string }
   | { type: "command-role-update"; requestId: string; mac: string; role: string }
   | { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean }

bastion/src/shared/src/types/index.ts

@@ -20,5 +20,6 @@ export type { RoleInfo, Lifecycle, K3sMode } from "./state.js";
 export type { BastionConfig } from "./config.js";
 
 export { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB,
-  SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion } from "./layout.js";
+  SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion,
-export type { LvSpec, SchemaVersion } from "./layout.js";
+  validateDataDisks, DATA_DISK_LABELS } from "./layout.js";
+export type { LvSpec, SchemaVersion, DataDisks, DataDiskName } from "./layout.js";

bastion/src/shared/src/types/layout.ts

@@ -94,13 +94,80 @@ const V2_BASE: readonly LvSpec[] = [
 export const V2_RANCHER_MIB = 256000;  // 250 GiB
 export const V2_LONGHORN_MIB = 307200; // 300 GiB
 
+/**
+ * Dedicated whole-disk data volumes (v2 only).
+ *
+ * The fixed 250G/300G LVs exist to FIT a 1 TB OS disk. A machine with spare
+ * NVMe bays (thebeast: 1 TB OS + 2 TB + 2 TB) wants the opposite trade: give
+ * each data mount a whole disk and keep the OS VG small. Each value is a
+ * stable /dev/disk/by-id/ path -- NEVER a kernel name, because nvme0n1/nvme1n1
+ * enumeration order is not stable across boots on multi-NVMe hosts, and a
+ * swapped name here formats the wrong disk.
+ *
+ * The disk gets one partition spanning it, xfs, labelled (DATA_DISK_LABELS)
+ * so a reprovision can recognise and preserve it the same way the LVs are
+ * recognised by name inside the VG. The corresponding LV is simply not
+ * created -- v2Layout() drops it when a dedicated disk supplies that mount.
+ */
+export interface DataDisks {
+  /** Whole disk for /var/lib/longhorn (replica store). */
+  longhorn?: string;
+  /** Whole disk for /var/lib/rancher (image store + local-path PVCs). */
+  rancher?: string;
+}
+
+/**
+ * Filesystem labels that mark a dedicated data disk's partition. They are the
+ * reprovision contract: %pre finds these by label and mounts them --noformat,
+ * exactly as it finds LVs by name. Changing a label orphans every disk
+ * already carrying the old one.
+ */
+export const DATA_DISK_LABELS = {
+  longhorn: "lab-longhorn",
+  rancher: "lab-rancher",
+} as const;
+
+export type DataDiskName = keyof typeof DATA_DISK_LABELS;
+
+const BY_ID_PREFIX = "/dev/disk/by-id/";
+
+/**
+ * Why a DataDisks spec is unusable for this role, or null when it is fine.
+ * Shared by the HTTP API, the websocket command handler and the CLI so all
+ * three reject with the same words instead of three drifting checks.
+ */
+export function validateDataDisks(role: Role, disks: DataDisks): string | null {
+  const info = roleInfo(role);
+  for (const name of Object.keys(DATA_DISK_LABELS) as DataDiskName[]) {
+    const device = disks[name];
+    if (device === undefined) continue;
+    if (!device.startsWith(BY_ID_PREFIX)) {
+      return `${name} disk must be a stable ${BY_ID_PREFIX} path (got '${device}'). ` +
+        `Kernel names like nvme0n1 change across boots on multi-NVMe hosts.`;
+    }
+  }
+  if (disks.rancher !== undefined && info.k3sMode === null) {
+    return `role '${role}' runs no k3s -- a dedicated rancher disk makes no sense for it`;
+  }
+  if (disks.longhorn !== undefined && (info.k3sMode === null || info.controlPlaneOnly)) {
+    return `role '${role}' holds no Longhorn replicas -- a dedicated longhorn disk makes no sense for it`;
+  }
+  if (disks.longhorn !== undefined && disks.longhorn === disks.rancher) {
+    return `longhorn and rancher point at the same disk (${disks.longhorn}) -- they need one each`;
+  }
+  return null;
+}
+
 /**
  * The v2 volumes for a role, gated by CAPABILITY rather than role name.
  *
  * `omitRoot` is for Asahi, where the vendor installer already created a btrfs
  * root partition outside our VG and we must not make a second one.
+ *
+ * `dataDisks` drops the longhorn/rancher LV whose mount a dedicated disk
+ * supplies -- two things mounting /var/lib/longhorn is an invalid layout.
  */
-export function v2Layout(role: Role, opts: { omitRoot?: boolean } = {}): LvSpec[] {
+export function v2Layout(role: Role, opts: { omitRoot?: boolean; dataDisks?: DataDisks } = {}): LvSpec[] {
   const info = roleInfo(role);
   const lvs: LvSpec[] = V2_BASE.filter(
     (lv) => !(opts.omitRoot && lv.name === "root"),
@@ -109,12 +176,12 @@ export function v2Layout(role: Role, opts: { omitRoot?: boolean } = {}): LvSpec[
   // Anything running k3s needs a dedicated image store. This is the check that
   // `role === "infra" || role === "worker"` kept getting wrong: labcontroller
   // runs k3s and was silently excluded.
-  if (info.k3sMode !== null) {
+  if (info.k3sMode !== null && opts.dataDisks?.rancher === undefined) {
     lvs.push({ name: "rancher", mount: "/var/lib/rancher", sizeMib: V2_RANCHER_MIB, fs: "xfs" });
   }
 
   // Fixed, never --grow. A control-plane-only node holds no replicas.
-  if (info.k3sMode !== null && !info.controlPlaneOnly) {
+  if (info.k3sMode !== null && !info.controlPlaneOnly && opts.dataDisks?.longhorn === undefined) {
     lvs.push({ name: "longhorn", mount: "/var/lib/longhorn", sizeMib: V2_LONGHORN_MIB, fs: "xfs" });
   }
 
@@ -122,7 +189,7 @@ export function v2Layout(role: Role, opts: { omitRoot?: boolean } = {}): LvSpec[
 }
 
 /** Total MiB a role's v2 layout consumes inside the VG. */
-export function v2AllocatedMib(role: Role, opts: { omitRoot?: boolean } = {}): number {
+export function v2AllocatedMib(role: Role, opts: { omitRoot?: boolean; dataDisks?: DataDisks } = {}): number {
   return v2Layout(role, opts).reduce((n, lv) => n + lv.sizeMib, 0);
 }
 

bastion/src/shared/src/types/state.ts

@@ -356,6 +356,13 @@ export interface InstallConfig {
    * disk instead, because reinstalling a machine must never change its layout.
    */
   schemaVersion?: 1 | 2;
+  /**
+   * Dedicated whole-disk data volumes (v2 + fedora only), by stable
+   * /dev/disk/by-id path. See DataDisks in layout.ts. On a reprovision these
+   * are optional: %pre rediscovers the disks by filesystem label and preserves
+   * them whether or not the queue entry names them.
+   */
+  dataDisks?: import("./layout.js").DataDisks;
   queued_at: string;
   /**
    * When dispatch last served this machine an install boot script. Progress

bastion/tests/integration/data-disks.test.ts

@@ -0,0 +1,173 @@
+// Integration test: dedicated data disks, on a real VM.
+//
+// The unit tests prove the kickstart RENDERS the right %pre and part lines.
+// This proves anaconda ACTUALLY PRODUCES the layout thebeast will get: OS on
+// one disk, /var/lib/longhorn and /var/lib/rancher each a whole-disk labelled
+// xfs partition on their own disks -- and that a reprovision with NO flags
+// rediscovers both by label and remounts them without reformatting.
+//
+// The VM mirrors thebeast's topology (1 OS disk + 2 data disks addressed by
+// /dev/disk/by-id/ serial); virtio serials stand in for NVMe serials, which is
+// exactly the aliasing the feature is specified over (any by-id path).
+//
+// Run (on a libvirt host):
+//   sudo -E npx vitest run -c tests/integration/vitest.config.ts tests/integration/data-disks.test.ts
+//
+// Set FEDORA_MIRROR to a local mirror or each install pulls packages from the
+// internet (~12-15 min per install either way; two installs in this file).
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { v2Layout, DATA_DISK_LABELS } from "../../src/shared/src/types/layout.js";
+import {
+  startTestBastion, provisionVm, reprovisionVm, readLvs, readSchemaTag,
+  writeMarkers, readMarkers, destroyVms, virtioById,
+  type Harness, type ProvisionedVm,
+} from "./helpers/provision-harness.js";
+import { destroyPxeNetwork } from "./helpers/pxe-network.js";
+import { sshExec } from "./helpers/ssh.js";
+
+const HTTP_PORT = 8094;
+const SSH_USER = "lab";
+const DISK_GB = 1000; // same sparse OS disk as every other layout test
+
+// Short virtio serials (virtio-blk truncates at 20 chars), echoing thebeast's
+// real disk serial fragments so the drill log reads like the runbook.
+const LH = { serial: "LHDRILL7F600711", sizeGb: 20 };
+const RN = { serial: "RNDRILL7FD4070C", sizeGb: 20 };
+
+const PRESERVED = ["/home", "/srv", "/var/lib/longhorn", "/var/lib/rancher"];
+
+function findSshKey(): { pubKey: string; keyPath: string } {
+  // A passphrase-protected default key fails SILENTLY here: BatchMode ssh
+  // never offers it without an agent, the server logs only "Connection closed
+  // [preauth]", and the test reads as "installed machine never came up" when
+  // the machine was fine. LAB_TEST_SSH_KEY names a dedicated passphrase-less
+  // keypair and sidesteps the whole class.
+  const envKey = process.env["LAB_TEST_SSH_KEY"];
+  if (envKey) {
+    if (!existsSync(envKey) || !existsSync(`${envKey}.pub`)) {
+      throw new Error(`LAB_TEST_SSH_KEY=${envKey}: key or .pub missing`);
+    }
+    return { pubKey: readFileSync(`${envKey}.pub`, "utf-8").trim(), keyPath: envKey };
+  }
+  const homes = [homedir()];
+  const sudoUser = process.env["SUDO_USER"];
+  if (sudoUser) homes.push(join("/home", sudoUser));
+  for (const home of homes) {
+    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
+      const keyPath = join(home, ".ssh", name);
+      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
+        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
+      }
+    }
+  }
+  throw new Error("no SSH keypair found");
+}
+
+/** Run a command on the VM and insist it succeeds. */
+function ssh(h: Harness, vm: ProvisionedVm, cmd: string): string {
+  const r = sshExec(vm.ip, h.sshUser, cmd, { keyPath: h.sshKeyPath });
+  if (r.exitCode !== 0) throw new Error(`[${vm.name}] \`${cmd}\` failed: ${r.stderr || r.stdout}`);
+  return r.stdout.trim();
+}
+
+/**
+ * What actually backs a mount point, and whether it is the whole-disk labelled
+ * partition the feature promises.
+ */
+function inspectMount(h: Harness, vm: ProvisionedVm, mount: string, serial: string) {
+  const source = ssh(h, vm, `findmnt -no SOURCE ${mount}`);
+  const fstype = ssh(h, vm, `findmnt -no FSTYPE ${mount}`);
+  const label = ssh(h, vm, `sudo blkid -s LABEL -o value ${source}`);
+  const uuid = ssh(h, vm, `sudo blkid -s UUID -o value ${source}`);
+  // The partition must live on the disk with OUR serial, not wherever the
+  // kernel happened to enumerate something.
+  const disk = ssh(h, vm, `readlink -f ${virtioById(serial)}`);
+  const parent = "/dev/" + ssh(h, vm, `lsblk -ndo pkname ${source}`);
+  return { source, fstype, label, uuid, disk, parent };
+}
+
+describe("dedicated data disks on real hardware", () => {
+  let h: Harness;
+  const created: string[] = [];
+
+  beforeAll(async () => {
+    const { pubKey, keyPath } = findSshKey();
+    h = await startTestBastion({
+      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey,
+      sshKeyPath: keyPath, label: "datadisks",
+    });
+  }, 20 * 60_000);
+
+  afterAll(async () => {
+    // KEEP_VMS=1 leaves the VM behind for a post-mortem (virsh console,
+    // screenshot, mounting the qcow2). Without it a failed boot is destroyed
+    // along with every clue about why it failed.
+    if (!process.env["KEEP_VMS"]) destroyVms(...created);
+    try { await h?.close(); } catch { /* closed */ }
+    try { destroyPxeNetwork(); } catch { /* gone */ }
+  }, 5 * 60_000);
+
+  it("k8s-worker with longhorn+rancher disks: install, then flagless reprovision rediscovers both", async () => {
+    const name = "dd-worker";
+    created.push(name);
+    const dataDisks = { longhorn: virtioById(LH.serial), rancher: virtioById(RN.serial) };
+
+    // --- Fresh install, thebeast-style -----------------------------------
+    let vm = await provisionVm(h, {
+      name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB,
+      dataDisks: { longhorn: LH, rancher: RN },
+    });
+
+    // The VG must hold exactly the v2 set MINUS the LVs the disks replace.
+    const lvs = readLvs(h, vm);
+    const expected = v2Layout("k8s-worker", { dataDisks }).map((lv) => lv.name).sort();
+    expect(Object.keys(lvs).sort(), "volume set").toEqual(expected);
+    expect(lvs["longhorn"]).toBeUndefined();
+    expect(lvs["rancher"]).toBeUndefined();
+    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");
+
+    // Each mount is a whole-disk xfs partition, labelled, on the RIGHT disk.
+    const lhBefore = inspectMount(h, vm, "/var/lib/longhorn", LH.serial);
+    expect(lhBefore.fstype).toBe("xfs");
+    expect(lhBefore.label).toBe(DATA_DISK_LABELS.longhorn);
+    expect(lhBefore.parent, "/var/lib/longhorn sits on the longhorn disk").toBe(lhBefore.disk);
+
+    const rnBefore = inspectMount(h, vm, "/var/lib/rancher", RN.serial);
+    expect(rnBefore.fstype).toBe("xfs");
+    expect(rnBefore.label).toBe(DATA_DISK_LABELS.rancher);
+    expect(rnBefore.parent, "/var/lib/rancher sits on the rancher disk").toBe(rnBefore.disk);
+
+    const token = writeMarkers(h, vm, PRESERVED);
+
+    // --- Reprovision with NO dataDisks flags ------------------------------
+    // This is the claim that matters for day 2: the labels on disk are the
+    // truth, and a plain `labctl provision reprovision` must find and keep
+    // them. Passing the flags again here would hide a regression in exactly
+    // that rediscovery.
+    vm = await reprovisionVm(h, vm, "k8s-worker");
+
+    const markers = readMarkers(h, vm, PRESERVED);
+    for (const mount of PRESERVED) {
+      expect(markers[mount], `${mount} must survive a reinstall`).toBe(token);
+    }
+
+    // Still LV-less for longhorn/rancher, still v2.
+    const lvsAfter = readLvs(h, vm);
+    expect(Object.keys(lvsAfter).sort()).toEqual(expected);
+    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");
+
+    // Same partitions, same filesystems. An unchanged UUID is the proof that
+    // --noformat really happened -- a remade xfs with the same label would
+    // pass every other check here.
+    const lhAfter = inspectMount(h, vm, "/var/lib/longhorn", LH.serial);
+    expect(lhAfter.uuid, "longhorn fs was NOT reformatted").toBe(lhBefore.uuid);
+    expect(lhAfter.parent).toBe(lhAfter.disk);
+    const rnAfter = inspectMount(h, vm, "/var/lib/rancher", RN.serial);
+    expect(rnAfter.uuid, "rancher fs was NOT reformatted").toBe(rnBefore.uuid);
+    expect(rnAfter.parent).toBe(rnAfter.disk);
+  }, 150 * 60_000);
+});

bastion/tests/integration/helpers/provision-harness.ts

@@ -14,7 +14,7 @@ import type { BastionConfig } from "../../../src/shared/src/types/config.js";
 import type { Role } from "../../../src/shared/src/types/state.js";
 import { log, waitForSsh } from "./libvirt.js";
 import { ensurePxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_BRIDGE, PXE_GATEWAY, PXE_SUBNET } from "./pxe-network.js";
-import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm } from "./pxe-vm.js";
+import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./pxe-vm.js";
 import { sshExec } from "./ssh.js";
 
 export const DISCOVERY_TIMEOUT_MS = 6 * 60_000;
@@ -295,7 +295,23 @@ async function waitForInstallStart(
   }
 }
 
-export interface ProvisionedVm { name: string; mac: string; ip: string }
+export interface ProvisionedVm { name: string; mac: string; ip: string; serialPort: number }
+
+/**
+ * "SSH never came up" on its own is undiagnosable once afterAll has destroyed
+ * the VM -- the machine may be sitting in emergency mode with the exact error
+ * on its serial console. Capture that console into the failure.
+ */
+async function sshFailurePostMortem(name: string, serialPort: number, cause: unknown): Promise<never> {
+  let state = "unknown";
+  try { state = execSync(`sudo virsh domstate ${name}`, { encoding: "utf-8" }).trim(); } catch { /* gone */ }
+  let serial = "(no serial output)";
+  try { serial = await readSerialLog(serialPort, { lastLines: 60 }); } catch { /* no console */ }
+  throw new Error(
+    `[${name}] ${cause instanceof Error ? cause.message : String(cause)}\n` +
+    `domstate: ${state}\n--- serial console (last lines) ---\n${serial}`,
+  );
+}
 
 /**
  * Serial ports are a global resource on the host, so hand out a distinct one
@@ -304,6 +320,15 @@ export interface ProvisionedVm { name: string; mac: string; ip: string }
  */
 let nextSerialPort = 4600;
 
+/**
+ * The stable path a virtio data disk gets inside the guest. This is what the
+ * install request must carry: the feature refuses anything outside
+ * /dev/disk/by-id/, exactly as it does for real NVMe serials.
+ */
+export function virtioById(serial: string): string {
+  return `/dev/disk/by-id/virtio-${serial}`;
+}
+
 /** Create a blank VM, let it discover, queue the install, wait for SSH. */
 export async function provisionVm(h: Harness, spec: {
   name: string;
@@ -312,16 +337,26 @@ export async function provisionVm(h: Harness, spec: {
   diskGb: number;
   memory?: number;
   vcpus?: number;
+  /**
+   * Dedicated data disks: attached to the VM with the given virtio serial AND
+   * passed to /api/install as /dev/disk/by-id/virtio-<serial>. Mirrors
+   * thebeast's layout (OS disk + separate longhorn + rancher disks).
+   */
+  dataDisks?: { longhorn?: { serial: string; sizeGb: number }; rancher?: { serial: string; sizeGb: number } };
 }): Promise<ProvisionedVm> {
   const base = `http://${PXE_GATEWAY}:${h.httpPort}`;
-  log(`[${spec.name}] creating VM (${spec.diskGb}GB disk, role=${spec.role}, schema=v${spec.schemaVersion ?? 1})`);
+  const vmDataDisks = [spec.dataDisks?.longhorn, spec.dataDisks?.rancher]
+    .filter((d): d is { serial: string; sizeGb: number } => d !== undefined);
+  log(`[${spec.name}] creating VM (${spec.diskGb}GB disk + ${vmDataDisks.length} data disk(s), role=${spec.role}, schema=v${spec.schemaVersion ?? 1})`);
+  const serialPort = nextSerialPort++;
   createPxeVm({
     name: spec.name,
     memory: spec.memory ?? 4096,
     vcpus: spec.vcpus ?? 4,
     diskSize: spec.diskGb,
     network: PXE_NETWORK_NAME,
-    serialPort: nextSerialPort++,
+    serialPort,
+    ...(vmDataDisks.length > 0 ? { dataDisks: vmDataDisks } : {}),
   });
   deleteNftablesRejectRules();
 
@@ -339,6 +374,12 @@ export async function provisionVm(h: Harness, spec: {
     body: JSON.stringify({
       mac, hostname: spec.name, disk: "", role: spec.role,
       ...(spec.schemaVersion ? { schemaVersion: spec.schemaVersion } : {}),
+      ...(spec.dataDisks ? {
+        dataDisks: {
+          ...(spec.dataDisks.longhorn ? { longhorn: virtioById(spec.dataDisks.longhorn.serial) } : {}),
+          ...(spec.dataDisks.rancher ? { rancher: virtioById(spec.dataDisks.rancher.serial) } : {}),
+        },
+      } : {}),
     }),
   });
   if (!res.ok) throw new Error(`[${spec.name}] install queue rejected: ${res.status} ${await res.text()}`);
@@ -376,9 +417,13 @@ export async function provisionVm(h: Harness, spec: {
   deleteNftablesRejectRules();
 
   log(`[${spec.name}] waiting for SSH at ${ip}...`);
-  await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath);
+  try {
+    await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath);
+  } catch (e) {
+    await sshFailurePostMortem(spec.name, serialPort, e);
+  }
   log(`[${spec.name}] up at ${ip}`);
-  return { name: spec.name, mac, ip };
+  return { name: spec.name, mac, ip, serialPort };
 }
 
 /**
@@ -498,7 +543,11 @@ export async function reprovisionVm(
   } catch { /* already running */ }
   deleteNftablesRejectRules();
 
-  await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath);
+  try {
+    await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath);
+  } catch (e) {
+    await sshFailurePostMortem(vm.name, vm.serialPort, e);
+  }
   log(`[${vm.name}] back up at ${ip} after reinstall`);
   return { ...vm, ip };
 }

bastion/tests/integration/helpers/pxe-vm.ts

@@ -47,6 +47,15 @@ export interface PxeVmConfig {
    */
   decoyNics?: number;
   decoyNetwork?: string;
+  /**
+   * Extra blank data disks, attached after the OS disk. Each gets a fixed
+   * virtio serial, so inside the guest it has a stable
+   * /dev/disk/by-id/virtio-<serial> path -- the moral equivalent of the
+   * nvme-..._<serial> links the dedicated-data-disk feature requires on real
+   * hardware. virtio-blk truncates serials at 20 characters, so keep them
+   * short or the by-id link quietly won't match.
+   */
+  dataDisks?: { serial: string; sizeGb: number }[];
 }
 
 /** Create a blank UEFI VM that PXE boots from the network. */
@@ -60,6 +69,14 @@ export function createPxeVm(config: PxeVmConfig): void {
   const diskPath = join(IMAGE_DIR, `${config.name}.qcow2`);
   run(`qemu-img create -f qcow2 "${diskPath}" ${config.diskSize}G`);
 
+  const dataDiskArgs: string[] = [];
+  for (const dd of config.dataDisks ?? []) {
+    if (dd.serial.length > 20) throw new Error(`virtio serial '${dd.serial}' exceeds 20 chars -- the guest by-id link would be truncated`);
+    const p = join(IMAGE_DIR, `${config.name}-${dd.serial}.qcow2`);
+    run(`qemu-img create -f qcow2 "${p}" ${dd.sizeGb}G`);
+    dataDiskArgs.push(`--disk=path=${p},format=qcow2,bus=virtio,serial=${dd.serial}`);
+  }
+
   // UEFI firmware paths (Fedora)
   if (arch === "aarch64") {
     const aavmf = "/usr/share/edk2/aarch64/QEMU_EFI.fd";
@@ -79,6 +96,7 @@ export function createPxeVm(config: PxeVmConfig): void {
     `--memory=${config.memory}`,
     `--vcpus=${config.vcpus}`,
     `--disk=path=${diskPath},format=qcow2,bus=virtio`,
+    ...dataDiskArgs,
     // Decoys first so they enumerate ahead of the PXE NIC. They are up and
     // carry a lease, but have no route to the bastion.
     ...Array.from({ length: config.decoyNics ?? 0 }, () =>

migration/THEBEAST-K8S-WORKER.md

@@ -0,0 +1,113 @@
+# thebeast → prod k8s worker — conversion runbook
+
+Converting the old desktop **thebeast** (currently Fedora/ZFS desktop at
+`10.0.1.22`) into a `k8s-worker` node of the **prod cluster (k8s0)** via the
+PXE bastion, using the dedicated-data-disk layout (`--longhorn-disk` /
+`--rancher-disk`, introduced with this runbook).
+
+Decisions already taken (2026-10-14): Windows disk is sacrificed (unused for
+years, no backup wanted) · cluster = prod k8s0 · OS = Fedora `k8s-worker`
+(thebeast is mainly an AMD-GPU machine: 2× AMD + 1× RTX 3080 Ti, no unified
+memory) · the node will carry a dedicated taint and also serve CI/CD.
+
+**VM-drilled 2026-10-14** (on thebeast itself, evidence:
+`drill-evidence/data-disks-2026-10-14.txt`): fresh install lands the exact
+layout below, and a flagless reprovision rediscovers both data disks by label
+and remounts them without reformatting (UUIDs unchanged, marker files intact).
+
+## Disk map — ALWAYS by-id, never nvme0n1
+
+Kernel NVMe names are **not stable across boots on this machine** (observed).
+Every command below uses `/dev/disk/by-id`.
+
+| target | disk | by-id |
+|---|---|---|
+| OS (wipes Windows) | Samsung 970 EVO Plus 1TB | `nvme-Samsung_SSD_970_EVO_Plus_1TB_S4EWNJ0N138234M` |
+| /var/lib/longhorn | Sabrent Rocket 4.0 2TB | `nvme-Sabrent_Rocket_4.0_2TB_7F600711179D89261778` |
+| /var/lib/rancher | Sabrent Rocket 4.0 2TB | `nvme-Sabrent_Rocket_4.0_2TB_7FD4070C1EDA92197301` |
+| DISCONNECT after final shutdown | Samsung 970 EVO Plus 2TB `S4J4NF0NC00809J` | sits on its own motherboard add-on card, PCIe x4, model discontinued |
+
+NICs: `enp69s0` `34:97:f6:32:76:e9` (current uplink — this MAC is what the
+bastion will see), `enp70s0` `34:97:f6:31:b3:99` (down).
+
+## 0. Before the final shutdown — data
+
+**rpool is a 4-wide raidz1 already DEGRADED** (member
+`S4J4NF0R102776W` removed since 2026-09-14). There is **zero redundancy
+left**: one more failure loses the pool, and a disk pulled out of it is NOT
+importable alone. Finish the copies first, then never boot the desktop again.
+
+Already safe (verified 2026-10-14): `~/developer` (all 113 repos) + the
+credential/dotfile set → bazzite (Oct 12 rsync, 1 file of drift).
+
+Copied to bazzite and rsync-verified 2026-10-14: `~/keyssecrets`,
+`~/dicesecrets`, `~/Videos/canaries` (24G), `~/Pictures`, `~/Documents`.
+`~/opt/vaultbackup` and `~/migration-backup` declared done/not-needed (user,
+2026-10-14).
+
+Still open before shutdown:
+- `~/Videos/{Webcam,subscribescreen,Screencasts,music}` (~370M) and
+  `~/backups/sogo-final-2026-09-13` — small, not yet copied
+- Verify the Syncthing replicas on synology0-nas for `~/Logseq`,
+  `~/DICESharedBoostNote`, `~/Boostnote`, `~/MangaReader_backup` — thebeast's
+  syncthing shows 0 connected peers, so the NAS copy is **unverified**
+- Final re-sync of `~/.claude` + atuin (activity postdates the Oct 12 copy)
+
+## 1. Physical + network
+
+1. Final desktop shutdown.
+2. Disconnect the Samsung 2TB `S4J4NF0NC00809J` (the add-on card).
+3. Move thebeast's switch port to the `192.168.8.0/24` VLAN (same as k8s0 and
+   labmaster). The bastion's proxy-DHCP (`dhcp-range=192.168.8.0,proxy`)
+   coexists with the real DHCP there — no DHCP changes needed.
+4. Make sure network boot is enabled/first in firmware (UEFI PXE, IPv4).
+
+## 2. Discovery
+
+PXE boot. Unknown MAC → discovery image → the machine self-reports:
+
+```
+labctl provision list        # thebeast appears under discovered (MAC 34:97:f6:32:76:e9)
+```
+
+## 3. Install
+
+```
+labctl provision install 34:97:f6:32:76:e9 thebeast \
+  --role k8s-worker \
+  --disk /dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_1TB_S4EWNJ0N138234M \
+  --longhorn-disk /dev/disk/by-id/nvme-Sabrent_Rocket_4.0_2TB_7F600711179D89261778 \
+  --rancher-disk /dev/disk/by-id/nvme-Sabrent_Rocket_4.0_2TB_7FD4070C1EDA92197301
+```
+
+(`k8s-worker` is v2-only, so no `--schema-version` needed.)
+
+What the layout does: OS disk gets EFI + /boot + `labvg` with the v2 base LVs
+only — **no** longhorn/rancher LVs. Each 2 TB disk gets ONE xfs partition
+spanning it, labelled `lab-longhorn` / `lab-rancher`, mounted at
+`/var/lib/longhorn` / `/var/lib/rancher`. The ZFS signatures on those disks
+are wiped by the install — which is the point of section 0.
+
+Reinstall semantics from then on: the labels are on-disk truth. A reprovision
+rediscovers both data disks by label (no flags needed) and mounts them
+`--noformat`; a fresh install refuses to wipe or shadow a labelled disk it was
+not told about.
+
+## 4. Join k8s0
+
+As on every k8s-worker, k3s is installed with `INSTALL_K3S_SKIP_START`; the
+join is deliberate (see `/root/README` on the node):
+
+```
+curl -sfL https://get.k3s.io | K3S_URL=https://<k8s0-server>:6443 K3S_TOKEN=<token> sh -
+```
+
+Add the agreed CI/CD taint/labels at join time (k3s agent flags), e.g.:
+
+```
+--node-taint dedicated=cicd:NoSchedule --node-label itaz.eu/cicd=true
+```
+
+(exact taint TBD with the Pulumi model; GPU bits — ROCm for the 2 AMD cards,
+nvidia for the 3080 Ti — are cluster-side follow-ups in
+thelab-kubernetes-pulumi, not part of the install.)

migration/drill-evidence/data-disks-2026-10-14.txt

@@ -0,0 +1,40 @@
+Dedicated data disks drill — fresh install + flagless reprovision, real VM
+run finished: 2026-10-14T23:30+01:00   host: thebeast
+test: bastion/tests/integration/data-disks.test.ts (1 passed, 1164s)
+topology: 1 VM, role k8s-worker schema v2, 1000G OS disk + 2x 20G data disks
+addressed as /dev/disk/by-id/virtio-LHDRILL7F600711 (longhorn) and
+/dev/disk/by-id/virtio-RNDRILL7FD4070C (rancher) — the same by-id contract
+thebeast's NVMe serials use.
+
+Proven:
+- fresh install: labvg = home/root/srv/swap/var/varlog ONLY (no longhorn,
+  no rancher LV); /var/lib/longhorn and /var/lib/rancher are whole-disk xfs
+  partitions labelled lab-longhorn / lab-rancher, each verified to sit on
+  the disk with the matching serial; lab-schema-v2 tag present; fstab by UUID.
+- reprovision with NO dataDisks flags: both disks rediscovered by label,
+  remounted --noformat (filesystem UUIDs identical before/after), marker
+  files intact in /home /srv /var/lib/longhorn /var/lib/rancher, LV set and
+  schema tag unchanged.
+
+=== checkpoints ===
+[22:10:40] [dd-worker] creating VM (1000GB disk + 2 data disk(s), role=k8s-worker, schema=v2)
+[22:13:02] [dd-worker] queueing install...
+[22:15:25] [dd-worker] installing (up to 60 min)...
+[22:21:24] [dd-worker] up at 192.168.251.194
+[22:21:24] [dd-worker] on-disk LVs: home=8G root=32G srv=16G swap=16G var=48G varlog=16G
+[22:21:27] [dd-worker] wrote marker preserve-1792016486955-iu1rih into /home, /srv, /var/lib/longhorn, /var/lib/rancher
+[22:21:27] [dd-worker] reprovisioning (role=k8s-worker, schema: whatever is on disk)...
+[22:29:58] [dd-worker] back up at 192.168.251.194 after reinstall
+[22:29:59] [dd-worker] markers after reinstall: all four mounts carry preserve-1792016486955-iu1rih
+[22:29:59] [dd-worker] on-disk LVs: home=8G root=32G srv=16G swap=16G var=48G varlog=16G
+✓ k8s-worker with longhorn+rancher disks: install, then flagless reprovision rediscovers both (1161s)
+
+Run notes (cost of learning, so nobody pays it twice):
+- the drill machine's bastion imports resolve @lab/shared to dist/ -- run
+  `pnpm build` after switching branches or the API answers with the OLD role
+  registry ("invalid role: 'k8s-worker'").
+- a passphrase-protected default ~/.ssh/id_rsa fails as "SSH not available"
+  with a perfectly healthy machine (BatchMode never offers it without an
+  agent). Use LAB_TEST_SSH_KEY with a dedicated passphrase-less key.
+- run: cd bastion && LAB_TEST_SSH_KEY=~/.ssh/lab-integration_ed25519 \
+    sudo -E npx vitest run -c tests/integration/vitest.config.ts tests/integration/data-disks.test.ts