lab/bastion/tests/integration/data-disks.test.ts

// Integration test: dedicated data disks, on a real VM.
//
// The unit tests prove the kickstart RENDERS the right %pre and part lines.
// This proves anaconda ACTUALLY PRODUCES the layout thebeast will get: OS on
// one disk, /var/lib/longhorn and /var/lib/rancher each a whole-disk labelled
// xfs partition on their own disks -- and that a reprovision with NO flags
// rediscovers both by label and remounts them without reformatting.
//
// The VM mirrors thebeast's topology (1 OS disk + 2 data disks addressed by
// /dev/disk/by-id/ serial); virtio serials stand in for NVMe serials, which is
// exactly the aliasing the feature is specified over (any by-id path).
//
// Run (on a libvirt host):
//   sudo -E npx vitest run -c tests/integration/vitest.config.ts tests/integration/data-disks.test.ts
//
// Set FEDORA_MIRROR to a local mirror or each install pulls packages from the
// internet (~12-15 min per install either way; two installs in this file).

import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { existsSync, readFileSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import { v2Layout, DATA_DISK_LABELS } from "../../src/shared/src/types/layout.js";
import {
  startTestBastion, provisionVm, reprovisionVm, readLvs, readSchemaTag,
  writeMarkers, readMarkers, destroyVms, virtioById,
  type Harness, type ProvisionedVm,
} from "./helpers/provision-harness.js";
import { destroyPxeNetwork } from "./helpers/pxe-network.js";
import { sshExec } from "./helpers/ssh.js";

const HTTP_PORT = 8094;
const SSH_USER = "lab";
const DISK_GB = 1000; // same sparse OS disk as every other layout test

// Short virtio serials (virtio-blk truncates at 20 chars), echoing thebeast's
// real disk serial fragments so the drill log reads like the runbook.
const LH = { serial: "LHDRILL7F600711", sizeGb: 20 };
const RN = { serial: "RNDRILL7FD4070C", sizeGb: 20 };

const PRESERVED = ["/home", "/srv", "/var/lib/longhorn", "/var/lib/rancher"];

function findSshKey(): { pubKey: string; keyPath: string } {
  // A passphrase-protected default key fails SILENTLY here: BatchMode ssh
  // never offers it without an agent, the server logs only "Connection closed
  // [preauth]", and the test reads as "installed machine never came up" when
  // the machine was fine. LAB_TEST_SSH_KEY names a dedicated passphrase-less
  // keypair and sidesteps the whole class.
  const envKey = process.env["LAB_TEST_SSH_KEY"];
  if (envKey) {
    if (!existsSync(envKey) || !existsSync(`${envKey}.pub`)) {
      throw new Error(`LAB_TEST_SSH_KEY=${envKey}: key or .pub missing`);
    }
    return { pubKey: readFileSync(`${envKey}.pub`, "utf-8").trim(), keyPath: envKey };
  }
  const homes = [homedir()];
  const sudoUser = process.env["SUDO_USER"];
  if (sudoUser) homes.push(join("/home", sudoUser));
  for (const home of homes) {
    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
      const keyPath = join(home, ".ssh", name);
      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
      }
    }
  }
  throw new Error("no SSH keypair found");
}

/** Run a command on the VM and insist it succeeds. */
function ssh(h: Harness, vm: ProvisionedVm, cmd: string): string {
  const r = sshExec(vm.ip, h.sshUser, cmd, { keyPath: h.sshKeyPath });
  if (r.exitCode !== 0) throw new Error(`[${vm.name}] \`${cmd}\` failed: ${r.stderr || r.stdout}`);
  return r.stdout.trim();
}

/**
 * What actually backs a mount point, and whether it is the whole-disk labelled
 * partition the feature promises.
 */
function inspectMount(h: Harness, vm: ProvisionedVm, mount: string, serial: string) {
  const source = ssh(h, vm, `findmnt -no SOURCE ${mount}`);
  const fstype = ssh(h, vm, `findmnt -no FSTYPE ${mount}`);
  const label = ssh(h, vm, `sudo blkid -s LABEL -o value ${source}`);
  const uuid = ssh(h, vm, `sudo blkid -s UUID -o value ${source}`);
  // The partition must live on the disk with OUR serial, not wherever the
  // kernel happened to enumerate something.
  const disk = ssh(h, vm, `readlink -f ${virtioById(serial)}`);
  const parent = "/dev/" + ssh(h, vm, `lsblk -ndo pkname ${source}`);
  return { source, fstype, label, uuid, disk, parent };
}

describe("dedicated data disks on real hardware", () => {
  let h: Harness;
  const created: string[] = [];

  beforeAll(async () => {
    const { pubKey, keyPath } = findSshKey();
    h = await startTestBastion({
      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey,
      sshKeyPath: keyPath, label: "datadisks",
    });
  }, 20 * 60_000);

  afterAll(async () => {
    // KEEP_VMS=1 leaves the VM behind for a post-mortem (virsh console,
    // screenshot, mounting the qcow2). Without it a failed boot is destroyed
    // along with every clue about why it failed.
    if (!process.env["KEEP_VMS"]) destroyVms(...created);
    try { await h?.close(); } catch { /* closed */ }
    try { destroyPxeNetwork(); } catch { /* gone */ }
  }, 5 * 60_000);

  it("k8s-worker with longhorn+rancher disks: install, then flagless reprovision rediscovers both", async () => {
    const name = "dd-worker";
    created.push(name);
    const dataDisks = { longhorn: virtioById(LH.serial), rancher: virtioById(RN.serial) };

    // --- Fresh install, thebeast-style -----------------------------------
    let vm = await provisionVm(h, {
      name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB,
      dataDisks: { longhorn: LH, rancher: RN },
    });

    // The VG must hold exactly the v2 set MINUS the LVs the disks replace.
    const lvs = readLvs(h, vm);
    const expected = v2Layout("k8s-worker", { dataDisks }).map((lv) => lv.name).sort();
    expect(Object.keys(lvs).sort(), "volume set").toEqual(expected);
    expect(lvs["longhorn"]).toBeUndefined();
    expect(lvs["rancher"]).toBeUndefined();
    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");

    // Each mount is a whole-disk xfs partition, labelled, on the RIGHT disk.
    const lhBefore = inspectMount(h, vm, "/var/lib/longhorn", LH.serial);
    expect(lhBefore.fstype).toBe("xfs");
    expect(lhBefore.label).toBe(DATA_DISK_LABELS.longhorn);
    expect(lhBefore.parent, "/var/lib/longhorn sits on the longhorn disk").toBe(lhBefore.disk);

    const rnBefore = inspectMount(h, vm, "/var/lib/rancher", RN.serial);
    expect(rnBefore.fstype).toBe("xfs");
    expect(rnBefore.label).toBe(DATA_DISK_LABELS.rancher);
    expect(rnBefore.parent, "/var/lib/rancher sits on the rancher disk").toBe(rnBefore.disk);

    const token = writeMarkers(h, vm, PRESERVED);

    // --- Reprovision with NO dataDisks flags ------------------------------
    // This is the claim that matters for day 2: the labels on disk are the
    // truth, and a plain `labctl provision reprovision` must find and keep
    // them. Passing the flags again here would hide a regression in exactly
    // that rediscovery.
    vm = await reprovisionVm(h, vm, "k8s-worker");

    const markers = readMarkers(h, vm, PRESERVED);
    for (const mount of PRESERVED) {
      expect(markers[mount], `${mount} must survive a reinstall`).toBe(token);
    }

    // Still LV-less for longhorn/rancher, still v2.
    const lvsAfter = readLvs(h, vm);
    expect(Object.keys(lvsAfter).sort()).toEqual(expected);
    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");

    // Same partitions, same filesystems. An unchanged UUID is the proof that
    // --noformat really happened -- a remade xfs with the same label would
    // pass every other check here.
    const lhAfter = inspectMount(h, vm, "/var/lib/longhorn", LH.serial);
    expect(lhAfter.uuid, "longhorn fs was NOT reformatted").toBe(lhBefore.uuid);
    expect(lhAfter.parent).toBe(lhAfter.disk);
    const rnAfter = inspectMount(h, vm, "/var/lib/rancher", RN.serial);
    expect(rnAfter.uuid, "rancher fs was NOT reformatted").toBe(rnBefore.uuid);
    expect(rnAfter.parent).toBe(rnAfter.disk);
  }, 150 * 60_000);
});