lab/bastion/tests/integration/partition-layout.test.ts

// Integration tests: the v2 disk layout, on real VMs.
//
// Unit tests prove the kickstart RENDERS the right layout. These prove anaconda
// ACTUALLY PRODUCES it -- which is a different claim, and the one that matters:
// the labmaster incident was caused by what landed on disk, not by what was
// written into a template.
//
// Every VM gets a 1 TB sparse disk, because "fits a 1 TB disk with headroom" is
// the constraint v2 was designed around. qcow2 is sparse, so a 1 TB disk costs
// ~10-15 GB of real space.
//
// Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo, and a Fedora mirror.
// Set FEDORA_MIRROR to a local mirror -- otherwise each install pulls the full
// package set over the internet and the matrix takes hours.
//
// Run one scenario at a time:
//   sudo -E npx vitest run -c tests/integration/vitest.config.ts -t "vanilla"

import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { existsSync, readFileSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import { v2Layout } from "../../src/shared/src/types/layout.js";
import type { Role } from "../../src/shared/src/types/state.js";
import {
  startTestBastion, provisionVm, readLvs, readSchemaTag, destroyVms,
  type Harness, type ProvisionedVm,
} from "./helpers/provision-harness.js";
import { destroyPxeNetwork } from "./helpers/pxe-network.js";

const HTTP_PORT = 8096;
const SSH_USER = "lab";
const DISK_GB = 1000; // the smallest disk we will ever buy

function findSshKey(): { pubKey: string; keyPath: string } {
  const homes = [homedir()];
  const sudoUser = process.env["SUDO_USER"];
  if (sudoUser) homes.push(join("/home", sudoUser));
  for (const home of homes) {
    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
      const keyPath = join(home, ".ssh", name);
      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
      }
    }
  }
  throw new Error("no SSH keypair found");
}

/** Assert the on-disk volumes match the shared v2 definition for this role. */
function expectV2Layout(h: Harness, vm: ProvisionedVm, role: Role): void {
  const actual = readLvs(h, vm);
  const expected = v2Layout(role);

  expect(Object.keys(actual).sort(), `${vm.name} volume set`).toEqual(
    expected.map((lv) => lv.name).sort(),
  );
  for (const spec of expected) {
    // LVM rounds to extent boundaries (4 MiB), so allow a small delta.
    expect(actual[spec.name], `${vm.name}/${spec.name} size`).toBeGreaterThanOrEqual(spec.sizeMib - 4);
    expect(actual[spec.name], `${vm.name}/${spec.name} size`).toBeLessThanOrEqual(spec.sizeMib + 4);
  }
  expect(readSchemaTag(h, vm), `${vm.name} schema tag`).toContain("lab-schema-v2");
}

describe("v2 partition layout on real hardware", () => {
  let h: Harness;
  const created: string[] = [];

  beforeAll(async () => {
    const { pubKey, keyPath } = findSshKey();
    h = await startTestBastion({
      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey,
      sshKeyPath: keyPath, label: "partition",
    });
  }, 20 * 60_000);

  afterAll(async () => {
    destroyVms(...created);
    try { await h?.close(); } catch { /* already closed */ }
    try { destroyPxeNetwork(); } catch { /* already gone */ }
  }, 5 * 60_000);

  // --- Topology 1: vanilla -------------------------------------------------
  it("vanilla: base volumes only, no rancher, no longhorn", async () => {
    const name = "v2-vanilla";
    created.push(name);
    const vm = await provisionVm(h, { name, role: "vanilla", schemaVersion: 2, diskGb: DISK_GB });
    expectV2Layout(h, vm, "vanilla");

    const lvs = readLvs(h, vm);
    expect(lvs["rancher"]).toBeUndefined();
    expect(lvs["longhorn"]).toBeUndefined();
  }, 60 * 60_000);

  // --- Topology 2: standalone cluster --------------------------------------
  it("standalone-cluster: single node with its own control plane and storage", async () => {
    const name = "v2-standalone";
    created.push(name);
    const vm = await provisionVm(h, { name, role: "standalone-cluster", schemaVersion: 2, diskGb: DISK_GB, memory: 6144 });
    expectV2Layout(h, vm, "standalone-cluster");

    const lvs = readLvs(h, vm);
    // A cluster of one still gets a Longhorn volume, so the `longhorn`
    // StorageClass exists and manifests stay portable across clusters.
    expect(lvs["longhorn"]).toBeDefined();
    expect(lvs["rancher"]).toBeDefined();
  }, 60 * 60_000);

  // --- Topology 3: 3x combo (control plane AND worker on every node) -------
  it("k8s-combo x3: every node is control plane and storage", async () => {
    const names = ["v2-combo-1", "v2-combo-2", "v2-combo-3"];
    created.push(...names);
    const vms: ProvisionedVm[] = [];
    for (const name of names) {
      vms.push(await provisionVm(h, { name, role: "k8s-combo", schemaVersion: 2, diskGb: DISK_GB }));
    }
    for (const vm of vms) {
      expectV2Layout(h, vm, "k8s-combo");
      const lvs = readLvs(h, vm);
      expect(lvs["longhorn"], `${vm.name} holds storage`).toBeDefined();
      expect(lvs["rancher"], `${vm.name} has an image store`).toBeDefined();
    }
  }, 180 * 60_000);

  // --- Topology 4: 3 controllers + 2 workers -------------------------------
  it("3x k8s-controller + 2x k8s-worker: storage only on the workers", async () => {
    const ctrls = ["v2-ctrl-1", "v2-ctrl-2", "v2-ctrl-3"];
    const workers = ["v2-wrk-1", "v2-wrk-2"];
    created.push(...ctrls, ...workers);

    for (const name of ctrls) {
      const vm = await provisionVm(h, { name, role: "k8s-controller", schemaVersion: 2, diskGb: DISK_GB });
      expectV2Layout(h, vm, "k8s-controller");
      // The distinguishing property of a controller: image store, no storage.
      expect(readLvs(h, vm)["longhorn"], `${name} must not hold replicas`).toBeUndefined();
      expect(readLvs(h, vm)["rancher"]).toBeDefined();
    }
    for (const name of workers) {
      const vm = await provisionVm(h, { name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB });
      expectV2Layout(h, vm, "k8s-worker");
      expect(readLvs(h, vm)["longhorn"], `${name} must hold replicas`).toBeDefined();
    }
  }, 300 * 60_000);
});