feat: ARM ISO boot integration test, OVMF boot fixes

ARM integration test: - arm-iso-provision.test.ts: aarch64 VM boots from bastion-generated ISO - Uses QEMU aarch64 emulation (slow but validates the R1 scenario) - Generous timeouts for emulated boot (15min discovery, 60min install) - test-provision.sh updated: `sudo ./scripts/test-provision.sh arm` VM boot fixes: - setBootDisk() preserves UEFI loader/nvram when switching to disk boot - /boot/efi mount gets nofail in fstab (prevents emergency mode in VMs) - chronyd enable uses || true (fails in kickstart chroot) - createIsoVm supports arch parameter for ARM VMs Note: SSH-after-reboot in OVMF VMs still fails — OVMF doesn't respect efibootmgr changes and loops PXE/HTTP Boot. Real hardware works fine. The install flow itself (discovery → kickstart → complete) is validated. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-27 00:26:12 +00:00
parent 46b017d77e
commit 7446d669c1
7 changed files with 423 additions and 18 deletions
--- a/bastion/tests/integration/arm-iso-provision.test.ts
+++ b/bastion/tests/integration/arm-iso-provision.test.ts
@@ -0,0 +1,334 @@
+// Integration test: ARM (aarch64) boot ISO provisioning.
+//
+// Simulates the MinisForum R1 scenario: ARM machine without PXE support,
+// booting from the bastion-generated ISO via QEMU aarch64 emulation.
+//
+// IMPORTANT: This runs under emulation (not KVM), so it's SLOW (~30-60 min).
+// The ISO contains an embedded aarch64 kernel+initrd for the fallback path.
+//
+// Prerequisites:
+//   - qemu-system-aarch64 installed (sudo dnf install qemu-system-aarch64)
+//   - edk2-aarch64 installed (sudo dnf install edk2-aarch64)
+//   - xorriso, mtools for ISO generation
+//   - sudo access, libvirtd running
+//
+// Run: sudo pnpm run test:integration:arm-iso
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, symlinkSync, writeFileSync } from "node:fs";
+import { execSync } from "node:child_process";
+import { join } from "node:path";
+import { homedir, tmpdir } from "node:os";
+import { log, waitForSsh } from "./helpers/libvirt.js";
+import { ensurePxeNetwork, destroyPxeNetwork, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js";
+import { createIsoVm, destroyPxeVm, getVmMac, rebootPxeVm, setBootDisk } from "./helpers/pxe-vm.js";
+import { sshExec } from "./helpers/ssh.js";
+
+const VM_NAME = "lab-arm-iso-test";
+const VM_MEMORY = 4096;
+const VM_VCPUS = 2;
+const VM_DISK_GB = 250;       // LVM layout needs ~204GB, QCOW2 is sparse
+const HTTP_PORT = 8097;       // different from x86 tests
+const SSH_USER = "michal";
+const BASTION_IP = PXE_GATEWAY;
+const DHCP_RANGE_START = `${PXE_SUBNET}.100`;
+const DHCP_RANGE_END = `${PXE_SUBNET}.200`;
+
+// ARM emulation is SLOW — generous timeouts
+const DISCOVERY_TIMEOUT_MS = 15 * 60_000;   // 15 min for emulated boot + discovery
+const INSTALL_TIMEOUT_MS = 60 * 60_000;     // 60 min for emulated Fedora install
+const SSH_TIMEOUT_MS = 15 * 60_000;         // 15 min for emulated reboot + SSH
+
+function findSshKey(): { pubKey: string; keyPath: string } {
+  const homes = [homedir()];
+  const sudoUser = process.env["SUDO_USER"];
+  if (sudoUser) homes.push(join("/home", sudoUser));
+  if (process.env["SSH_KEY_PATH"]) {
+    const keyPath = process.env["SSH_KEY_PATH"];
+    const pubPath = `${keyPath}.pub`;
+    if (existsSync(keyPath) && existsSync(pubPath)) {
+      return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath };
+    }
+  }
+  for (const home of homes) {
+    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
+      const keyPath = join(home, ".ssh", name);
+      const pubPath = `${keyPath}.pub`;
+      if (existsSync(keyPath) && existsSync(pubPath)) {
+        return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath };
+      }
+    }
+  }
+  throw new Error("No SSH key found");
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+async function pollApi<T>(
+  url: string,
+  check: (data: T) => boolean,
+  timeoutMs: number,
+  intervalMs = 10000,
+): Promise<T> {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    try {
+      const res = await fetch(url);
+      if (res.ok) {
+        const data = (await res.json()) as T;
+        if (check(data)) return data;
+      }
+    } catch { /* bastion not ready */ }
+    await sleep(intervalMs);
+  }
+  throw new Error(`Timeout after ${timeoutMs}ms polling ${url}`);
+}
+
+describe("ARM ISO boot provisioning", () => {
+  let bastionApp: { close: () => Promise<void> };
+  let testDir: string;
+  let vmMac: string;
+  let vmIp: string;
+  let sshKeyPath: string;
+
+  beforeAll(async () => {
+    // Check ARM prerequisites
+    if (!existsSync("/usr/bin/qemu-system-aarch64")) {
+      throw new Error("qemu-system-aarch64 not installed. Run: sudo dnf install qemu-system-aarch64");
+    }
+    if (!existsSync("/usr/share/edk2/aarch64/QEMU_EFI.fd")) {
+      throw new Error("AAVMF not installed. Run: sudo dnf install edk2-aarch64");
+    }
+
+    const { pubKey, keyPath } = findSshKey();
+    sshKeyPath = keyPath;
+
+    // 1. Network
+    log("Setting up PXE test network (for ARM ISO test)...");
+    ensurePxeNetwork();
+
+    // 2. Bastion dirs
+    testDir = join(tmpdir(), `lab-arm-iso-test-${Date.now()}`);
+    mkdirSync(testDir, { recursive: true });
+    mkdirSync(join(testDir, "tftp"), { recursive: true });
+    mkdirSync(join(testDir, "http"), { recursive: true });
+    mkdirSync(join(testDir, "logs"), { recursive: true });
+
+    // 3. Start bastion with ISO generation (needs both x86_64 and aarch64 artifacts)
+    log("Starting bastion with ARM boot ISO generation...");
+
+    const { createApp } = await import("../../src/bastion/src/server.js");
+    const { loadConfig } = await import("../../src/bastion/src/config.js");
+    const { generateDnsmasqConf, startDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
+    const { generateDiscoverKickstart } = await import("../../src/bastion/src/services/kickstart-generator.js");
+    const { renderBootIpxe } = await import("../../src/bastion/src/templates/boot.ipxe.js");
+    const { ensureBootIso } = await import("../../src/bastion/src/routes/boot-iso.js");
+
+    const config = loadConfig({
+      bastionDir: testDir,
+      httpPort: HTTP_PORT,
+      iface: "virbr-pxe",
+      serverIp: BASTION_IP,
+      network: `${PXE_SUBNET}.0`,
+      gateway: BASTION_IP,
+      dhcpMode: "full",
+      dhcpRangeStart: DHCP_RANGE_START,
+      dhcpRangeEnd: DHCP_RANGE_END,
+      domain: "arm-test.local",
+      sshKeys: [pubKey],
+      adminUser: SSH_USER,
+    });
+
+    // iPXE for TFTP (x86_64 — still needed for dnsmasq config)
+    const ipxeSrc = "/usr/share/ipxe/ipxe-snponly-x86_64.efi";
+    if (existsSync(ipxeSrc)) {
+      copyFileSync(ipxeSrc, join(config.tftpDir, "ipxe.efi"));
+    }
+
+    // Fedora kernel + initrd for BOTH architectures
+    const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache";
+    execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" });
+
+    // x86_64 (for the bastion HTTP serving)
+    for (const [arch, prefix] of [["x86_64", ""], ["aarch64", "aarch64-"]] as const) {
+      const mirror = `https://download.fedoraproject.org/pub/fedora/linux/releases/${config.fedoraVersion}/Everything/${arch}/os`;
+      const kernel = join(cacheDir, `vmlinuz-${arch}`);
+      const initrd = join(cacheDir, `initrd-${arch}.img`);
+
+      if (!existsSync(kernel)) {
+        log(`Downloading Fedora ${config.fedoraVersion} ${arch} kernel...`);
+        execSync(`curl -# -L -f -o "${kernel}" "${mirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 300_000 });
+      } else {
+        log(`Fedora ${arch} kernel cached`);
+      }
+      if (!existsSync(initrd)) {
+        log(`Downloading Fedora ${config.fedoraVersion} ${arch} initrd...`);
+        execSync(`curl -# -L -f -o "${initrd}" "${mirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 300_000 });
+      } else {
+        log(`Fedora ${arch} initrd cached`);
+      }
+
+      // x86_64 artifacts go to httpDir for normal PXE serving
+      if (arch === "x86_64") {
+        copyFileSync(kernel, join(config.httpDir, "vmlinuz"));
+        copyFileSync(initrd, join(config.httpDir, "initrd.img"));
+      }
+    }
+
+    try { symlinkSync(join(config.tftpDir, "ipxe.efi"), join(config.httpDir, "ipxe.efi")); } catch { /* exists */ }
+
+    // Generate boot scripts
+    const discoverKs = generateDiscoverKickstart(config);
+    writeFileSync(join(config.httpDir, "discover.ks"), discoverKs);
+    const bootIpxe = renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort });
+    writeFileSync(join(config.httpDir, "boot.ipxe"), bootIpxe);
+
+    // Generate the boot ISO — includes both x86_64 and aarch64 payloads
+    log("Generating boot ISO with ARM support...");
+    ensureBootIso(config);
+
+    const isoPath = join(config.httpDir, "boot.iso");
+    if (!existsSync(isoPath)) {
+      throw new Error("Boot ISO was not generated");
+    }
+    const { statSync } = await import("node:fs");
+    const isoSize = statSync(isoPath).size;
+    log(`Boot ISO generated: ${(isoSize / 1024 / 1024).toFixed(1)}MB`);
+
+    // dnsmasq + HTTP server
+    generateDnsmasqConf(config);
+    const { app } = createApp(config);
+    bastionApp = app;
+    await app.listen({ port: config.httpPort, host: "0.0.0.0" });
+    log(`Bastion HTTP listening on :${HTTP_PORT}`);
+
+    void startDnsmasq(config);
+    await sleep(1000);
+
+    // 4. Create ARM VM booting from ISO
+    log("Creating ARM ISO boot VM (aarch64 emulation — this will be SLOW)...");
+    createIsoVm({
+      name: VM_NAME,
+      memory: VM_MEMORY,
+      vcpus: VM_VCPUS,
+      diskSize: VM_DISK_GB,
+      network: PXE_NETWORK_NAME,
+      isoPath,
+      arch: "aarch64",
+    });
+
+    const mac = getVmMac(VM_NAME);
+    if (!mac) throw new Error("Could not determine VM MAC address");
+    vmMac = mac;
+    log(`ARM VM MAC: ${vmMac}`);
+
+    // 5. Wait for discovery (ARM emulation is slow)
+    log("Waiting for ARM VM to boot ISO -> iPXE -> DHCP -> bastion -> discover...");
+    log("(ARM emulation is ~10x slower than native — be patient)");
+    type MachinesResponse = { discovered: Record<string, unknown> };
+    await pollApi<MachinesResponse>(
+      `http://${BASTION_IP}:${HTTP_PORT}/api/machines`,
+      (data) => vmMac in data.discovered,
+      DISCOVERY_TIMEOUT_MS,
+      15_000,
+    );
+    log("ARM VM discovered via ISO boot!");
+
+    // 6. Queue install
+    log("Queueing ARM machine for install...");
+    await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/install`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ mac: vmMac, hostname: VM_NAME, disk: "", role: "vanilla" }),
+    });
+
+    // 7. Reboot for install
+    log("Waiting for discovery reboot...");
+    await sleep(30_000); // ARM is slow
+    rebootPxeVm(VM_NAME);
+
+    // 8. Wait for install (ARM emulation makes this very slow)
+    log("Waiting for ARM install (emulated — expect 30-60 minutes)...");
+    type LogsResponse = { status: string; progress: string; ip?: string };
+    const finalState = await pollApi<LogsResponse>(
+      `http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`,
+      (data) => data.status === "installed" || data.progress === "error",
+      INSTALL_TIMEOUT_MS,
+      30_000,
+    );
+
+    if (finalState.progress === "error") {
+      const logsRes = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`);
+      const logs = await logsRes.json();
+      log(`ARM INSTALL FAILED. State: ${JSON.stringify(logs, null, 2)}`);
+      throw new Error("ARM install failed — check logs above");
+    }
+
+    vmIp = finalState.ip ?? "";
+    log(`ARM install complete! VM IP: ${vmIp}`);
+
+    // 9. Switch boot to disk
+    log("Switching ARM VM boot order to disk...");
+    await sleep(15_000);
+    setBootDisk(VM_NAME);
+
+    // 10. Wait for SSH (ARM reboot is slow)
+    log("Waiting for SSH on ARM VM...");
+    await waitForSsh(vmIp, SSH_USER, SSH_TIMEOUT_MS, sshKeyPath);
+    log("ARM ISO boot provision test setup complete.");
+
+  }, DISCOVERY_TIMEOUT_MS + INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000);
+
+  afterAll(async () => {
+    log("Cleaning up ARM test...");
+    if (bastionApp) await bastionApp.close().catch(() => {});
+    const { stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
+    stopDnsmasq();
+    destroyPxeVm(VM_NAME);
+    destroyPxeNetwork();
+    if (testDir) rmSync(testDir, { recursive: true, force: true });
+  });
+
+  it("ARM machine was discovered and installed", async () => {
+    const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`);
+    const data = (await res.json()) as { installed: Record<string, { hostname: string }> };
+    expect(data.installed[vmMac]).toBeDefined();
+    expect(data.installed[vmMac].hostname).toBe(VM_NAME);
+  });
+
+  it("architecture is aarch64", async () => {
+    const result = sshExec(vmIp, SSH_USER, "uname -m", { keyPath: sshKeyPath, timeout: 30_000 });
+    expect(result.exitCode).toBe(0);
+    expect(result.stdout.trim()).toBe("aarch64");
+  });
+
+  it("SSH works", () => {
+    const result = sshExec(vmIp, SSH_USER, "whoami", { keyPath: sshKeyPath, timeout: 30_000 });
+    expect(result.exitCode).toBe(0);
+    expect(result.stdout.trim()).toBe(SSH_USER);
+  });
+
+  it("hostname is correct", () => {
+    const result = sshExec(vmIp, SSH_USER, "hostname -f", { keyPath: sshKeyPath, timeout: 30_000 });
+    expect(result.exitCode).toBe(0);
+    expect(result.stdout.trim()).toContain(VM_NAME);
+  });
+
+  it("provisioning metadata exists", () => {
+    const result = sshExec(vmIp, SSH_USER, "cat /etc/lab-provisioned", { keyPath: sshKeyPath, timeout: 30_000 });
+    expect(result.exitCode).toBe(0);
+    expect(result.stdout).toContain(`hostname: ${VM_NAME}`);
+    expect(result.stdout).toContain("role: vanilla");
+  });
+
+  it("LVM layout is correct", () => {
+    const result = sshExec(vmIp, SSH_USER, "sudo lvs labvg --noheadings -o lv_name", { keyPath: sshKeyPath, timeout: 30_000 });
+    expect(result.exitCode).toBe(0);
+    const lvs = result.stdout.trim().split("\n").map((l: string) => l.trim());
+    for (const expected of ["root", "var", "varlog", "swap", "home", "srv"]) {
+      expect(lvs).toContain(expected);
+    }
+  });
+});