integration: VM drill for dedicated data disks

One real-VM test mirroring thebeast's topology: OS disk + two data disks
addressed by /dev/disk/by-id/ (virtio serials standing in for NVMe
serials). Fresh k8s-worker v2 install must put labelled whole-disk xfs
on each data disk and drop the matching LVs; a flagless reprovision must
rediscover both by label and remount them unformatted -- proven by
marker files AND unchanged filesystem UUIDs.

Harness: createPxeVm grows dataDisks (extra virtio disks with fixed
serials, <=20 chars or the guest by-id link truncates), provisionVm
passes the by-id paths to /api/install, virtioById exported.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V6HNx6cyr4J4CiMGCy1wkF

bastion/tests/integration/data-disks.test.ts

@@ -0,0 +1,158 @@
+// Integration test: dedicated data disks, on a real VM.
+//
+// The unit tests prove the kickstart RENDERS the right %pre and part lines.
+// This proves anaconda ACTUALLY PRODUCES the layout thebeast will get: OS on
+// one disk, /var/lib/longhorn and /var/lib/rancher each a whole-disk labelled
+// xfs partition on their own disks -- and that a reprovision with NO flags
+// rediscovers both by label and remounts them without reformatting.
+//
+// The VM mirrors thebeast's topology (1 OS disk + 2 data disks addressed by
+// /dev/disk/by-id/ serial); virtio serials stand in for NVMe serials, which is
+// exactly the aliasing the feature is specified over (any by-id path).
+//
+// Run (on a libvirt host):
+//   sudo -E npx vitest run -c tests/integration/vitest.config.ts tests/integration/data-disks.test.ts
+//
+// Set FEDORA_MIRROR to a local mirror or each install pulls packages from the
+// internet (~12-15 min per install either way; two installs in this file).
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { v2Layout, DATA_DISK_LABELS } from "../../src/shared/src/types/layout.js";
+import {
+  startTestBastion, provisionVm, reprovisionVm, readLvs, readSchemaTag,
+  writeMarkers, readMarkers, destroyVms, virtioById,
+  type Harness, type ProvisionedVm,
+} from "./helpers/provision-harness.js";
+import { destroyPxeNetwork } from "./helpers/pxe-network.js";
+import { sshExec } from "./helpers/ssh.js";
+
+const HTTP_PORT = 8094;
+const SSH_USER = "lab";
+const DISK_GB = 1000; // same sparse OS disk as every other layout test
+
+// Short virtio serials (virtio-blk truncates at 20 chars), echoing thebeast's
+// real disk serial fragments so the drill log reads like the runbook.
+const LH = { serial: "LHDRILL7F600711", sizeGb: 20 };
+const RN = { serial: "RNDRILL7FD4070C", sizeGb: 20 };
+
+const PRESERVED = ["/home", "/srv", "/var/lib/longhorn", "/var/lib/rancher"];
+
+function findSshKey(): { pubKey: string; keyPath: string } {
+  const homes = [homedir()];
+  const sudoUser = process.env["SUDO_USER"];
+  if (sudoUser) homes.push(join("/home", sudoUser));
+  for (const home of homes) {
+    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
+      const keyPath = join(home, ".ssh", name);
+      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
+        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
+      }
+    }
+  }
+  throw new Error("no SSH keypair found");
+}
+
+/** Run a command on the VM and insist it succeeds. */
+function ssh(h: Harness, vm: ProvisionedVm, cmd: string): string {
+  const r = sshExec(vm.ip, h.sshUser, cmd, { keyPath: h.sshKeyPath });
+  if (r.exitCode !== 0) throw new Error(`[${vm.name}] \`${cmd}\` failed: ${r.stderr || r.stdout}`);
+  return r.stdout.trim();
+}
+
+/**
+ * What actually backs a mount point, and whether it is the whole-disk labelled
+ * partition the feature promises.
+ */
+function inspectMount(h: Harness, vm: ProvisionedVm, mount: string, serial: string) {
+  const source = ssh(h, vm, `findmnt -no SOURCE ${mount}`);
+  const fstype = ssh(h, vm, `findmnt -no FSTYPE ${mount}`);
+  const label = ssh(h, vm, `sudo blkid -s LABEL -o value ${source}`);
+  const uuid = ssh(h, vm, `sudo blkid -s UUID -o value ${source}`);
+  // The partition must live on the disk with OUR serial, not wherever the
+  // kernel happened to enumerate something.
+  const disk = ssh(h, vm, `readlink -f ${virtioById(serial)}`);
+  const parent = "/dev/" + ssh(h, vm, `lsblk -ndo pkname ${source}`);
+  return { source, fstype, label, uuid, disk, parent };
+}
+
+describe("dedicated data disks on real hardware", () => {
+  let h: Harness;
+  const created: string[] = [];
+
+  beforeAll(async () => {
+    const { pubKey, keyPath } = findSshKey();
+    h = await startTestBastion({
+      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey,
+      sshKeyPath: keyPath, label: "datadisks",
+    });
+  }, 20 * 60_000);
+
+  afterAll(async () => {
+    destroyVms(...created);
+    try { await h?.close(); } catch { /* closed */ }
+    try { destroyPxeNetwork(); } catch { /* gone */ }
+  }, 5 * 60_000);
+
+  it("k8s-worker with longhorn+rancher disks: install, then flagless reprovision rediscovers both", async () => {
+    const name = "dd-worker";
+    created.push(name);
+    const dataDisks = { longhorn: virtioById(LH.serial), rancher: virtioById(RN.serial) };
+
+    // --- Fresh install, thebeast-style -----------------------------------
+    let vm = await provisionVm(h, {
+      name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB,
+      dataDisks: { longhorn: LH, rancher: RN },
+    });
+
+    // The VG must hold exactly the v2 set MINUS the LVs the disks replace.
+    const lvs = readLvs(h, vm);
+    const expected = v2Layout("k8s-worker", { dataDisks }).map((lv) => lv.name).sort();
+    expect(Object.keys(lvs).sort(), "volume set").toEqual(expected);
+    expect(lvs["longhorn"]).toBeUndefined();
+    expect(lvs["rancher"]).toBeUndefined();
+    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");
+
+    // Each mount is a whole-disk xfs partition, labelled, on the RIGHT disk.
+    const lhBefore = inspectMount(h, vm, "/var/lib/longhorn", LH.serial);
+    expect(lhBefore.fstype).toBe("xfs");
+    expect(lhBefore.label).toBe(DATA_DISK_LABELS.longhorn);
+    expect(lhBefore.parent, "/var/lib/longhorn sits on the longhorn disk").toBe(lhBefore.disk);
+
+    const rnBefore = inspectMount(h, vm, "/var/lib/rancher", RN.serial);
+    expect(rnBefore.fstype).toBe("xfs");
+    expect(rnBefore.label).toBe(DATA_DISK_LABELS.rancher);
+    expect(rnBefore.parent, "/var/lib/rancher sits on the rancher disk").toBe(rnBefore.disk);
+
+    const token = writeMarkers(h, vm, PRESERVED);
+
+    // --- Reprovision with NO dataDisks flags ------------------------------
+    // This is the claim that matters for day 2: the labels on disk are the
+    // truth, and a plain `labctl provision reprovision` must find and keep
+    // them. Passing the flags again here would hide a regression in exactly
+    // that rediscovery.
+    vm = await reprovisionVm(h, vm, "k8s-worker");
+
+    const markers = readMarkers(h, vm, PRESERVED);
+    for (const mount of PRESERVED) {
+      expect(markers[mount], `${mount} must survive a reinstall`).toBe(token);
+    }
+
+    // Still LV-less for longhorn/rancher, still v2.
+    const lvsAfter = readLvs(h, vm);
+    expect(Object.keys(lvsAfter).sort()).toEqual(expected);
+    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");
+
+    // Same partitions, same filesystems. An unchanged UUID is the proof that
+    // --noformat really happened -- a remade xfs with the same label would
+    // pass every other check here.
+    const lhAfter = inspectMount(h, vm, "/var/lib/longhorn", LH.serial);
+    expect(lhAfter.uuid, "longhorn fs was NOT reformatted").toBe(lhBefore.uuid);
+    expect(lhAfter.parent).toBe(lhAfter.disk);
+    const rnAfter = inspectMount(h, vm, "/var/lib/rancher", RN.serial);
+    expect(rnAfter.uuid, "rancher fs was NOT reformatted").toBe(rnBefore.uuid);
+    expect(rnAfter.parent).toBe(rnAfter.disk);
+  }, 150 * 60_000);
+});

bastion/tests/integration/helpers/provision-harness.ts

@@ -304,6 +304,15 @@ export interface ProvisionedVm { name: string; mac: string; ip: string }
  */
 let nextSerialPort = 4600;
 
+/**
+ * The stable path a virtio data disk gets inside the guest. This is what the
+ * install request must carry: the feature refuses anything outside
+ * /dev/disk/by-id/, exactly as it does for real NVMe serials.
+ */
+export function virtioById(serial: string): string {
+  return `/dev/disk/by-id/virtio-${serial}`;
+}
+
 /** Create a blank VM, let it discover, queue the install, wait for SSH. */
 export async function provisionVm(h: Harness, spec: {
   name: string;
@@ -312,9 +321,17 @@ export async function provisionVm(h: Harness, spec: {
   diskGb: number;
   memory?: number;
   vcpus?: number;
+  /**
+   * Dedicated data disks: attached to the VM with the given virtio serial AND
+   * passed to /api/install as /dev/disk/by-id/virtio-<serial>. Mirrors
+   * thebeast's layout (OS disk + separate longhorn + rancher disks).
+   */
+  dataDisks?: { longhorn?: { serial: string; sizeGb: number }; rancher?: { serial: string; sizeGb: number } };
 }): Promise<ProvisionedVm> {
   const base = `http://${PXE_GATEWAY}:${h.httpPort}`;
-  log(`[${spec.name}] creating VM (${spec.diskGb}GB disk, role=${spec.role}, schema=v${spec.schemaVersion ?? 1})`);
+  const vmDataDisks = [spec.dataDisks?.longhorn, spec.dataDisks?.rancher]
+    .filter((d): d is { serial: string; sizeGb: number } => d !== undefined);
+  log(`[${spec.name}] creating VM (${spec.diskGb}GB disk + ${vmDataDisks.length} data disk(s), role=${spec.role}, schema=v${spec.schemaVersion ?? 1})`);
   createPxeVm({
     name: spec.name,
     memory: spec.memory ?? 4096,
@@ -322,6 +339,7 @@ export async function provisionVm(h: Harness, spec: {
     diskSize: spec.diskGb,
     network: PXE_NETWORK_NAME,
     serialPort: nextSerialPort++,
+    ...(vmDataDisks.length > 0 ? { dataDisks: vmDataDisks } : {}),
   });
   deleteNftablesRejectRules();
 
@@ -339,6 +357,12 @@ export async function provisionVm(h: Harness, spec: {
     body: JSON.stringify({
       mac, hostname: spec.name, disk: "", role: spec.role,
       ...(spec.schemaVersion ? { schemaVersion: spec.schemaVersion } : {}),
+      ...(spec.dataDisks ? {
+        dataDisks: {
+          ...(spec.dataDisks.longhorn ? { longhorn: virtioById(spec.dataDisks.longhorn.serial) } : {}),
+          ...(spec.dataDisks.rancher ? { rancher: virtioById(spec.dataDisks.rancher.serial) } : {}),
+        },
+      } : {}),
     }),
   });
   if (!res.ok) throw new Error(`[${spec.name}] install queue rejected: ${res.status} ${await res.text()}`);

bastion/tests/integration/helpers/pxe-vm.ts

@@ -47,6 +47,15 @@ export interface PxeVmConfig {
    */
   decoyNics?: number;
   decoyNetwork?: string;
+  /**
+   * Extra blank data disks, attached after the OS disk. Each gets a fixed
+   * virtio serial, so inside the guest it has a stable
+   * /dev/disk/by-id/virtio-<serial> path -- the moral equivalent of the
+   * nvme-..._<serial> links the dedicated-data-disk feature requires on real
+   * hardware. virtio-blk truncates serials at 20 characters, so keep them
+   * short or the by-id link quietly won't match.
+   */
+  dataDisks?: { serial: string; sizeGb: number }[];
 }
 
 /** Create a blank UEFI VM that PXE boots from the network. */
@@ -60,6 +69,14 @@ export function createPxeVm(config: PxeVmConfig): void {
   const diskPath = join(IMAGE_DIR, `${config.name}.qcow2`);
   run(`qemu-img create -f qcow2 "${diskPath}" ${config.diskSize}G`);
 
+  const dataDiskArgs: string[] = [];
+  for (const dd of config.dataDisks ?? []) {
+    if (dd.serial.length > 20) throw new Error(`virtio serial '${dd.serial}' exceeds 20 chars -- the guest by-id link would be truncated`);
+    const p = join(IMAGE_DIR, `${config.name}-${dd.serial}.qcow2`);
+    run(`qemu-img create -f qcow2 "${p}" ${dd.sizeGb}G`);
+    dataDiskArgs.push(`--disk=path=${p},format=qcow2,bus=virtio,serial=${dd.serial}`);
+  }
+
   // UEFI firmware paths (Fedora)
   if (arch === "aarch64") {
     const aavmf = "/usr/share/edk2/aarch64/QEMU_EFI.fd";
@@ -79,6 +96,7 @@ export function createPxeVm(config: PxeVmConfig): void {
     `--memory=${config.memory}`,
     `--vcpus=${config.vcpus}`,
     `--disk=path=${diskPath},format=qcow2,bus=virtio`,
+    ...dataDiskArgs,
     // Decoys first so they enumerate ahead of the PXE NIC. They are up and
     // carry a lease, but have no route to the bastion.
     ...Array.from({ length: config.decoyNics ?? 0 }, () =>