// Integration tests: the v2 disk layout, on real VMs. // // Unit tests prove the kickstart RENDERS the right layout. These prove anaconda // ACTUALLY PRODUCES it -- which is a different claim, and the one that matters: // the labmaster incident was caused by what landed on disk, not by what was // written into a template. // // Every VM gets a 1 TB sparse disk, because "fits a 1 TB disk with headroom" is // the constraint v2 was designed around. qcow2 is sparse, so a 1 TB disk costs // ~10-15 GB of real space. // // Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo, and a Fedora mirror. // Set FEDORA_MIRROR to a local mirror -- otherwise each install pulls the full // package set over the internet and the matrix takes hours. // // Run one scenario at a time: // sudo -E npx vitest run -c tests/integration/vitest.config.ts -t "vanilla" import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { existsSync, readFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { v2Layout } from "../../src/shared/src/types/layout.js"; import type { Role } from "../../src/shared/src/types/state.js"; import { startTestBastion, provisionVm, readLvs, readSchemaTag, destroyVms, type Harness, type ProvisionedVm, } from "./helpers/provision-harness.js"; import { destroyPxeNetwork } from "./helpers/pxe-network.js"; const HTTP_PORT = 8096; const SSH_USER = "lab"; const DISK_GB = 1000; // the smallest disk we will ever buy function findSshKey(): { pubKey: string; keyPath: string } { const homes = [homedir()]; const sudoUser = process.env["SUDO_USER"]; if (sudoUser) homes.push(join("/home", sudoUser)); for (const home of homes) { for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) { const keyPath = join(home, ".ssh", name); if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) { return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath }; } } } throw new Error("no SSH keypair found"); } /** Assert the on-disk volumes match the shared v2 definition for this role. */ function expectV2Layout(h: Harness, vm: ProvisionedVm, role: Role): void { const actual = readLvs(h, vm); const expected = v2Layout(role); expect(Object.keys(actual).sort(), `${vm.name} volume set`).toEqual( expected.map((lv) => lv.name).sort(), ); for (const spec of expected) { // LVM rounds to extent boundaries (4 MiB), so allow a small delta. expect(actual[spec.name], `${vm.name}/${spec.name} size`).toBeGreaterThanOrEqual(spec.sizeMib - 4); expect(actual[spec.name], `${vm.name}/${spec.name} size`).toBeLessThanOrEqual(spec.sizeMib + 4); } expect(readSchemaTag(h, vm), `${vm.name} schema tag`).toContain("lab-schema-v2"); } describe("v2 partition layout on real hardware", () => { let h: Harness; const created: string[] = []; beforeAll(async () => { const { pubKey, keyPath } = findSshKey(); h = await startTestBastion({ httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey, sshKeyPath: keyPath, label: "partition", }); }, 20 * 60_000); afterAll(async () => { destroyVms(...created); try { await h?.close(); } catch { /* already closed */ } try { destroyPxeNetwork(); } catch { /* already gone */ } }, 5 * 60_000); // --- Topology 1: vanilla ------------------------------------------------- it("vanilla: base volumes only, no rancher, no longhorn", async () => { const name = "v2-vanilla"; created.push(name); const vm = await provisionVm(h, { name, role: "vanilla", schemaVersion: 2, diskGb: DISK_GB }); expectV2Layout(h, vm, "vanilla"); const lvs = readLvs(h, vm); expect(lvs["rancher"]).toBeUndefined(); expect(lvs["longhorn"]).toBeUndefined(); }, 60 * 60_000); // --- Topology 2: standalone cluster -------------------------------------- it("standalone-cluster: single node with its own control plane and storage", async () => { const name = "v2-standalone"; created.push(name); const vm = await provisionVm(h, { name, role: "standalone-cluster", schemaVersion: 2, diskGb: DISK_GB, memory: 6144 }); expectV2Layout(h, vm, "standalone-cluster"); const lvs = readLvs(h, vm); // A cluster of one still gets a Longhorn volume, so the `longhorn` // StorageClass exists and manifests stay portable across clusters. expect(lvs["longhorn"]).toBeDefined(); expect(lvs["rancher"]).toBeDefined(); }, 60 * 60_000); // --- Topology 3: 3x combo (control plane AND worker on every node) ------- it("k8s-combo x3: every node is control plane and storage", async () => { const names = ["v2-combo-1", "v2-combo-2", "v2-combo-3"]; created.push(...names); const vms: ProvisionedVm[] = []; for (const name of names) { vms.push(await provisionVm(h, { name, role: "k8s-combo", schemaVersion: 2, diskGb: DISK_GB })); } for (const vm of vms) { expectV2Layout(h, vm, "k8s-combo"); const lvs = readLvs(h, vm); expect(lvs["longhorn"], `${vm.name} holds storage`).toBeDefined(); expect(lvs["rancher"], `${vm.name} has an image store`).toBeDefined(); } }, 180 * 60_000); // --- Topology 4: 3 controllers + 2 workers ------------------------------- it("3x k8s-controller + 2x k8s-worker: storage only on the workers", async () => { const ctrls = ["v2-ctrl-1", "v2-ctrl-2", "v2-ctrl-3"]; const workers = ["v2-wrk-1", "v2-wrk-2"]; created.push(...ctrls, ...workers); for (const name of ctrls) { const vm = await provisionVm(h, { name, role: "k8s-controller", schemaVersion: 2, diskGb: DISK_GB }); expectV2Layout(h, vm, "k8s-controller"); // The distinguishing property of a controller: image store, no storage. expect(readLvs(h, vm)["longhorn"], `${name} must not hold replicas`).toBeUndefined(); expect(readLvs(h, vm)["rancher"]).toBeDefined(); } for (const name of workers) { const vm = await provisionVm(h, { name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB }); expectV2Layout(h, vm, "k8s-worker"); expect(readLvs(h, vm)["longhorn"], `${name} must hold replicas`).toBeDefined(); } }, 300 * 60_000); });