From 16659236f2ec80d202cdd8dd5f1343704fb2260c Mon Sep 17 00:00:00 2001 From: Michal Date: Wed, 14 Oct 2026 22:20:56 +0100 Subject: [PATCH] integration: VM drill for dedicated data disks One real-VM test mirroring thebeast's topology: OS disk + two data disks addressed by /dev/disk/by-id/ (virtio serials standing in for NVMe serials). Fresh k8s-worker v2 install must put labelled whole-disk xfs on each data disk and drop the matching LVs; a flagless reprovision must rediscover both by label and remount them unformatted -- proven by marker files AND unchanged filesystem UUIDs. Harness: createPxeVm grows dataDisks (extra virtio disks with fixed serials, <=20 chars or the guest by-id link truncates), provisionVm passes the by-id paths to /api/install, virtioById exported. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01V6HNx6cyr4J4CiMGCy1wkF --- bastion/tests/integration/data-disks.test.ts | 158 ++++++++++++++++++ .../integration/helpers/provision-harness.ts | 26 ++- bastion/tests/integration/helpers/pxe-vm.ts | 18 ++ 3 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 bastion/tests/integration/data-disks.test.ts diff --git a/bastion/tests/integration/data-disks.test.ts b/bastion/tests/integration/data-disks.test.ts new file mode 100644 index 0000000..ac48462 --- /dev/null +++ b/bastion/tests/integration/data-disks.test.ts @@ -0,0 +1,158 @@ +// Integration test: dedicated data disks, on a real VM. +// +// The unit tests prove the kickstart RENDERS the right %pre and part lines. +// This proves anaconda ACTUALLY PRODUCES the layout thebeast will get: OS on +// one disk, /var/lib/longhorn and /var/lib/rancher each a whole-disk labelled +// xfs partition on their own disks -- and that a reprovision with NO flags +// rediscovers both by label and remounts them without reformatting. +// +// The VM mirrors thebeast's topology (1 OS disk + 2 data disks addressed by +// /dev/disk/by-id/ serial); virtio serials stand in for NVMe serials, which is +// exactly the aliasing the feature is specified over (any by-id path). +// +// Run (on a libvirt host): +// sudo -E npx vitest run -c tests/integration/vitest.config.ts tests/integration/data-disks.test.ts +// +// Set FEDORA_MIRROR to a local mirror or each install pulls packages from the +// internet (~12-15 min per install either way; two installs in this file). + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { existsSync, readFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { v2Layout, DATA_DISK_LABELS } from "../../src/shared/src/types/layout.js"; +import { + startTestBastion, provisionVm, reprovisionVm, readLvs, readSchemaTag, + writeMarkers, readMarkers, destroyVms, virtioById, + type Harness, type ProvisionedVm, +} from "./helpers/provision-harness.js"; +import { destroyPxeNetwork } from "./helpers/pxe-network.js"; +import { sshExec } from "./helpers/ssh.js"; + +const HTTP_PORT = 8094; +const SSH_USER = "lab"; +const DISK_GB = 1000; // same sparse OS disk as every other layout test + +// Short virtio serials (virtio-blk truncates at 20 chars), echoing thebeast's +// real disk serial fragments so the drill log reads like the runbook. +const LH = { serial: "LHDRILL7F600711", sizeGb: 20 }; +const RN = { serial: "RNDRILL7FD4070C", sizeGb: 20 }; + +const PRESERVED = ["/home", "/srv", "/var/lib/longhorn", "/var/lib/rancher"]; + +function findSshKey(): { pubKey: string; keyPath: string } { + const homes = [homedir()]; + const sudoUser = process.env["SUDO_USER"]; + if (sudoUser) homes.push(join("/home", sudoUser)); + for (const home of homes) { + for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) { + const keyPath = join(home, ".ssh", name); + if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) { + return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath }; + } + } + } + throw new Error("no SSH keypair found"); +} + +/** Run a command on the VM and insist it succeeds. */ +function ssh(h: Harness, vm: ProvisionedVm, cmd: string): string { + const r = sshExec(vm.ip, h.sshUser, cmd, { keyPath: h.sshKeyPath }); + if (r.exitCode !== 0) throw new Error(`[${vm.name}] \`${cmd}\` failed: ${r.stderr || r.stdout}`); + return r.stdout.trim(); +} + +/** + * What actually backs a mount point, and whether it is the whole-disk labelled + * partition the feature promises. + */ +function inspectMount(h: Harness, vm: ProvisionedVm, mount: string, serial: string) { + const source = ssh(h, vm, `findmnt -no SOURCE ${mount}`); + const fstype = ssh(h, vm, `findmnt -no FSTYPE ${mount}`); + const label = ssh(h, vm, `sudo blkid -s LABEL -o value ${source}`); + const uuid = ssh(h, vm, `sudo blkid -s UUID -o value ${source}`); + // The partition must live on the disk with OUR serial, not wherever the + // kernel happened to enumerate something. + const disk = ssh(h, vm, `readlink -f ${virtioById(serial)}`); + const parent = "/dev/" + ssh(h, vm, `lsblk -ndo pkname ${source}`); + return { source, fstype, label, uuid, disk, parent }; +} + +describe("dedicated data disks on real hardware", () => { + let h: Harness; + const created: string[] = []; + + beforeAll(async () => { + const { pubKey, keyPath } = findSshKey(); + h = await startTestBastion({ + httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey, + sshKeyPath: keyPath, label: "datadisks", + }); + }, 20 * 60_000); + + afterAll(async () => { + destroyVms(...created); + try { await h?.close(); } catch { /* closed */ } + try { destroyPxeNetwork(); } catch { /* gone */ } + }, 5 * 60_000); + + it("k8s-worker with longhorn+rancher disks: install, then flagless reprovision rediscovers both", async () => { + const name = "dd-worker"; + created.push(name); + const dataDisks = { longhorn: virtioById(LH.serial), rancher: virtioById(RN.serial) }; + + // --- Fresh install, thebeast-style ----------------------------------- + let vm = await provisionVm(h, { + name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB, + dataDisks: { longhorn: LH, rancher: RN }, + }); + + // The VG must hold exactly the v2 set MINUS the LVs the disks replace. + const lvs = readLvs(h, vm); + const expected = v2Layout("k8s-worker", { dataDisks }).map((lv) => lv.name).sort(); + expect(Object.keys(lvs).sort(), "volume set").toEqual(expected); + expect(lvs["longhorn"]).toBeUndefined(); + expect(lvs["rancher"]).toBeUndefined(); + expect(readSchemaTag(h, vm)).toContain("lab-schema-v2"); + + // Each mount is a whole-disk xfs partition, labelled, on the RIGHT disk. + const lhBefore = inspectMount(h, vm, "/var/lib/longhorn", LH.serial); + expect(lhBefore.fstype).toBe("xfs"); + expect(lhBefore.label).toBe(DATA_DISK_LABELS.longhorn); + expect(lhBefore.parent, "/var/lib/longhorn sits on the longhorn disk").toBe(lhBefore.disk); + + const rnBefore = inspectMount(h, vm, "/var/lib/rancher", RN.serial); + expect(rnBefore.fstype).toBe("xfs"); + expect(rnBefore.label).toBe(DATA_DISK_LABELS.rancher); + expect(rnBefore.parent, "/var/lib/rancher sits on the rancher disk").toBe(rnBefore.disk); + + const token = writeMarkers(h, vm, PRESERVED); + + // --- Reprovision with NO dataDisks flags ------------------------------ + // This is the claim that matters for day 2: the labels on disk are the + // truth, and a plain `labctl provision reprovision` must find and keep + // them. Passing the flags again here would hide a regression in exactly + // that rediscovery. + vm = await reprovisionVm(h, vm, "k8s-worker"); + + const markers = readMarkers(h, vm, PRESERVED); + for (const mount of PRESERVED) { + expect(markers[mount], `${mount} must survive a reinstall`).toBe(token); + } + + // Still LV-less for longhorn/rancher, still v2. + const lvsAfter = readLvs(h, vm); + expect(Object.keys(lvsAfter).sort()).toEqual(expected); + expect(readSchemaTag(h, vm)).toContain("lab-schema-v2"); + + // Same partitions, same filesystems. An unchanged UUID is the proof that + // --noformat really happened -- a remade xfs with the same label would + // pass every other check here. + const lhAfter = inspectMount(h, vm, "/var/lib/longhorn", LH.serial); + expect(lhAfter.uuid, "longhorn fs was NOT reformatted").toBe(lhBefore.uuid); + expect(lhAfter.parent).toBe(lhAfter.disk); + const rnAfter = inspectMount(h, vm, "/var/lib/rancher", RN.serial); + expect(rnAfter.uuid, "rancher fs was NOT reformatted").toBe(rnBefore.uuid); + expect(rnAfter.parent).toBe(rnAfter.disk); + }, 150 * 60_000); +}); diff --git a/bastion/tests/integration/helpers/provision-harness.ts b/bastion/tests/integration/helpers/provision-harness.ts index 2a535b6..a912cf1 100644 --- a/bastion/tests/integration/helpers/provision-harness.ts +++ b/bastion/tests/integration/helpers/provision-harness.ts @@ -304,6 +304,15 @@ export interface ProvisionedVm { name: string; mac: string; ip: string } */ let nextSerialPort = 4600; +/** + * The stable path a virtio data disk gets inside the guest. This is what the + * install request must carry: the feature refuses anything outside + * /dev/disk/by-id/, exactly as it does for real NVMe serials. + */ +export function virtioById(serial: string): string { + return `/dev/disk/by-id/virtio-${serial}`; +} + /** Create a blank VM, let it discover, queue the install, wait for SSH. */ export async function provisionVm(h: Harness, spec: { name: string; @@ -312,9 +321,17 @@ export async function provisionVm(h: Harness, spec: { diskGb: number; memory?: number; vcpus?: number; + /** + * Dedicated data disks: attached to the VM with the given virtio serial AND + * passed to /api/install as /dev/disk/by-id/virtio-. Mirrors + * thebeast's layout (OS disk + separate longhorn + rancher disks). + */ + dataDisks?: { longhorn?: { serial: string; sizeGb: number }; rancher?: { serial: string; sizeGb: number } }; }): Promise { const base = `http://${PXE_GATEWAY}:${h.httpPort}`; - log(`[${spec.name}] creating VM (${spec.diskGb}GB disk, role=${spec.role}, schema=v${spec.schemaVersion ?? 1})`); + const vmDataDisks = [spec.dataDisks?.longhorn, spec.dataDisks?.rancher] + .filter((d): d is { serial: string; sizeGb: number } => d !== undefined); + log(`[${spec.name}] creating VM (${spec.diskGb}GB disk + ${vmDataDisks.length} data disk(s), role=${spec.role}, schema=v${spec.schemaVersion ?? 1})`); createPxeVm({ name: spec.name, memory: spec.memory ?? 4096, @@ -322,6 +339,7 @@ export async function provisionVm(h: Harness, spec: { diskSize: spec.diskGb, network: PXE_NETWORK_NAME, serialPort: nextSerialPort++, + ...(vmDataDisks.length > 0 ? { dataDisks: vmDataDisks } : {}), }); deleteNftablesRejectRules(); @@ -339,6 +357,12 @@ export async function provisionVm(h: Harness, spec: { body: JSON.stringify({ mac, hostname: spec.name, disk: "", role: spec.role, ...(spec.schemaVersion ? { schemaVersion: spec.schemaVersion } : {}), + ...(spec.dataDisks ? { + dataDisks: { + ...(spec.dataDisks.longhorn ? { longhorn: virtioById(spec.dataDisks.longhorn.serial) } : {}), + ...(spec.dataDisks.rancher ? { rancher: virtioById(spec.dataDisks.rancher.serial) } : {}), + }, + } : {}), }), }); if (!res.ok) throw new Error(`[${spec.name}] install queue rejected: ${res.status} ${await res.text()}`); diff --git a/bastion/tests/integration/helpers/pxe-vm.ts b/bastion/tests/integration/helpers/pxe-vm.ts index 2972200..13dea2d 100644 --- a/bastion/tests/integration/helpers/pxe-vm.ts +++ b/bastion/tests/integration/helpers/pxe-vm.ts @@ -47,6 +47,15 @@ export interface PxeVmConfig { */ decoyNics?: number; decoyNetwork?: string; + /** + * Extra blank data disks, attached after the OS disk. Each gets a fixed + * virtio serial, so inside the guest it has a stable + * /dev/disk/by-id/virtio- path -- the moral equivalent of the + * nvme-..._ links the dedicated-data-disk feature requires on real + * hardware. virtio-blk truncates serials at 20 characters, so keep them + * short or the by-id link quietly won't match. + */ + dataDisks?: { serial: string; sizeGb: number }[]; } /** Create a blank UEFI VM that PXE boots from the network. */ @@ -60,6 +69,14 @@ export function createPxeVm(config: PxeVmConfig): void { const diskPath = join(IMAGE_DIR, `${config.name}.qcow2`); run(`qemu-img create -f qcow2 "${diskPath}" ${config.diskSize}G`); + const dataDiskArgs: string[] = []; + for (const dd of config.dataDisks ?? []) { + if (dd.serial.length > 20) throw new Error(`virtio serial '${dd.serial}' exceeds 20 chars -- the guest by-id link would be truncated`); + const p = join(IMAGE_DIR, `${config.name}-${dd.serial}.qcow2`); + run(`qemu-img create -f qcow2 "${p}" ${dd.sizeGb}G`); + dataDiskArgs.push(`--disk=path=${p},format=qcow2,bus=virtio,serial=${dd.serial}`); + } + // UEFI firmware paths (Fedora) if (arch === "aarch64") { const aavmf = "/usr/share/edk2/aarch64/QEMU_EFI.fd"; @@ -79,6 +96,7 @@ export function createPxeVm(config: PxeVmConfig): void { `--memory=${config.memory}`, `--vcpus=${config.vcpus}`, `--disk=path=${diskPath},format=qcow2,bus=virtio`, + ...dataDiskArgs, // Decoys first so they enumerate ahead of the PXE NIC. They are up and // carry a lease, but have no route to the bastion. ...Array.from({ length: config.decoyNics ?? 0 }, () =>