// Integration test: ARM (aarch64) boot ISO provisioning. // // Simulates the MinisForum R1 scenario: ARM machine without PXE support, // booting from the bastion-generated ISO via QEMU aarch64 emulation. // // IMPORTANT: This runs under emulation (not KVM), so it's SLOW (~30-60 min). // The ISO contains an embedded aarch64 kernel+initrd for the fallback path. // // Prerequisites: // - qemu-system-aarch64 installed (sudo dnf install qemu-system-aarch64) // - edk2-aarch64 installed (sudo dnf install edk2-aarch64) // - xorriso, mtools for ISO generation // - sudo access, libvirtd running // // Run: sudo pnpm run test:integration:arm-iso import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, symlinkSync, writeFileSync } from "node:fs"; import { execSync } from "node:child_process"; import { join } from "node:path"; import { homedir, tmpdir } from "node:os"; import { log, waitForSsh } from "./helpers/libvirt.js"; import { ensurePxeNetwork, destroyPxeNetwork, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js"; import { createIsoVm, destroyPxeVm, getVmMac, rebootPxeVm } from "./helpers/pxe-vm.js"; import { sshExec } from "./helpers/ssh.js"; const VM_NAME = "lab-arm-iso-test"; const VM_MEMORY = 4096; const VM_VCPUS = 2; const VM_DISK_GB = 250; // LVM layout needs ~204GB, QCOW2 is sparse const HTTP_PORT = 8097; // different from x86 tests const SSH_USER = "michal"; const BASTION_IP = PXE_GATEWAY; const DHCP_RANGE_START = `${PXE_SUBNET}.100`; const DHCP_RANGE_END = `${PXE_SUBNET}.200`; // ARM emulation is SLOW — generous timeouts const DISCOVERY_TIMEOUT_MS = 15 * 60_000; // 15 min for emulated boot + discovery const INSTALL_TIMEOUT_MS = 60 * 60_000; // 60 min for emulated Fedora install const SSH_TIMEOUT_MS = 15 * 60_000; // 15 min for emulated reboot + SSH function findSshKey(): { pubKey: string; keyPath: string } { const homes = [homedir()]; const sudoUser = process.env["SUDO_USER"]; if (sudoUser) homes.push(join("/home", sudoUser)); if (process.env["SSH_KEY_PATH"]) { const keyPath = process.env["SSH_KEY_PATH"]; const pubPath = `${keyPath}.pub`; if (existsSync(keyPath) && existsSync(pubPath)) { return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath }; } } for (const home of homes) { for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) { const keyPath = join(home, ".ssh", name); const pubPath = `${keyPath}.pub`; if (existsSync(keyPath) && existsSync(pubPath)) { return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath }; } } } throw new Error("No SSH key found"); } function sleep(ms: number): Promise { return new Promise((r) => setTimeout(r, ms)); } async function pollApi( url: string, check: (data: T) => boolean, timeoutMs: number, intervalMs = 10000, ): Promise { const start = Date.now(); while (Date.now() - start < timeoutMs) { try { const res = await fetch(url); if (res.ok) { const data = (await res.json()) as T; if (check(data)) return data; } } catch { /* bastion not ready */ } await sleep(intervalMs); } throw new Error(`Timeout after ${timeoutMs}ms polling ${url}`); } describe("ARM ISO boot provisioning", () => { let bastionApp: { close: () => Promise }; let testDir: string; let vmMac: string; let vmIp: string; let sshKeyPath: string; beforeAll(async () => { // Check ARM prerequisites if (!existsSync("/usr/bin/qemu-system-aarch64")) { throw new Error("qemu-system-aarch64 not installed. Run: sudo dnf install qemu-system-aarch64"); } if (!existsSync("/usr/share/edk2/aarch64/QEMU_EFI.fd")) { throw new Error("AAVMF not installed. Run: sudo dnf install edk2-aarch64"); } const { pubKey, keyPath } = findSshKey(); sshKeyPath = keyPath; // 1. Network log("Setting up PXE test network (for ARM ISO test)..."); ensurePxeNetwork(); // 2. Bastion dirs testDir = join(tmpdir(), `lab-arm-iso-test-${Date.now()}`); mkdirSync(testDir, { recursive: true }); mkdirSync(join(testDir, "tftp"), { recursive: true }); mkdirSync(join(testDir, "http"), { recursive: true }); mkdirSync(join(testDir, "logs"), { recursive: true }); // 3. Start bastion with ISO generation (needs both x86_64 and aarch64 artifacts) log("Starting bastion with ARM boot ISO generation..."); const { createApp } = await import("../../src/bastion/src/server.js"); const { loadConfig } = await import("../../src/bastion/src/config.js"); const { generateDnsmasqConf, startDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js"); const { generateDiscoverKickstart } = await import("../../src/bastion/src/services/kickstart-generator.js"); const { renderBootIpxe } = await import("../../src/bastion/src/templates/boot.ipxe.js"); const { ensureBootIso } = await import("../../src/bastion/src/routes/boot-iso.js"); const config = loadConfig({ bastionDir: testDir, httpPort: HTTP_PORT, iface: "virbr-pxe", serverIp: BASTION_IP, network: `${PXE_SUBNET}.0`, gateway: BASTION_IP, dhcpMode: "full", dhcpRangeStart: DHCP_RANGE_START, dhcpRangeEnd: DHCP_RANGE_END, domain: "arm-test.local", sshKeys: [pubKey], adminUser: SSH_USER, }); // iPXE for TFTP (x86_64 — still needed for dnsmasq config) const ipxeSrc = "/usr/share/ipxe/ipxe-snponly-x86_64.efi"; if (existsSync(ipxeSrc)) { copyFileSync(ipxeSrc, join(config.tftpDir, "ipxe.efi")); } // Fedora kernel + initrd for BOTH architectures const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache"; execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" }); // x86_64 (for the bastion HTTP serving) for (const [arch, prefix] of [["x86_64", ""], ["aarch64", "aarch64-"]] as const) { const mirror = `https://download.fedoraproject.org/pub/fedora/linux/releases/${config.fedoraVersion}/Everything/${arch}/os`; const kernel = join(cacheDir, `vmlinuz-${arch}`); const initrd = join(cacheDir, `initrd-${arch}.img`); if (!existsSync(kernel)) { log(`Downloading Fedora ${config.fedoraVersion} ${arch} kernel...`); execSync(`curl -# -L -f -o "${kernel}" "${mirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 300_000 }); } else { log(`Fedora ${arch} kernel cached`); } if (!existsSync(initrd)) { log(`Downloading Fedora ${config.fedoraVersion} ${arch} initrd...`); execSync(`curl -# -L -f -o "${initrd}" "${mirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 300_000 }); } else { log(`Fedora ${arch} initrd cached`); } // x86_64 artifacts go to httpDir for normal PXE serving if (arch === "x86_64") { copyFileSync(kernel, join(config.httpDir, "vmlinuz")); copyFileSync(initrd, join(config.httpDir, "initrd.img")); } } try { symlinkSync(join(config.tftpDir, "ipxe.efi"), join(config.httpDir, "ipxe.efi")); } catch { /* exists */ } // Generate boot scripts const discoverKs = generateDiscoverKickstart(config); writeFileSync(join(config.httpDir, "discover.ks"), discoverKs); const bootIpxe = renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }); writeFileSync(join(config.httpDir, "boot.ipxe"), bootIpxe); // Generate the boot ISO — includes both x86_64 and aarch64 payloads log("Generating boot ISO with ARM support..."); ensureBootIso(config); const isoPath = join(config.httpDir, "boot.iso"); if (!existsSync(isoPath)) { throw new Error("Boot ISO was not generated"); } const { statSync } = await import("node:fs"); const isoSize = statSync(isoPath).size; log(`Boot ISO generated: ${(isoSize / 1024 / 1024).toFixed(1)}MB`); // dnsmasq + HTTP server generateDnsmasqConf(config); const { app } = createApp(config); bastionApp = app; await app.listen({ port: config.httpPort, host: "0.0.0.0" }); log(`Bastion HTTP listening on :${HTTP_PORT}`); void startDnsmasq(config); await sleep(1000); // 4. Create ARM VM booting from ISO log("Creating ARM ISO boot VM (aarch64 emulation — this will be SLOW)..."); createIsoVm({ name: VM_NAME, memory: VM_MEMORY, vcpus: VM_VCPUS, diskSize: VM_DISK_GB, network: PXE_NETWORK_NAME, isoPath, arch: "aarch64", }); const mac = getVmMac(VM_NAME); if (!mac) throw new Error("Could not determine VM MAC address"); vmMac = mac; log(`ARM VM MAC: ${vmMac}`); // 5. Wait for discovery (ARM emulation is slow) log("Waiting for ARM VM to boot ISO -> iPXE -> DHCP -> bastion -> discover..."); log("(ARM emulation is ~10x slower than native — be patient)"); type MachinesResponse = { discovered: Record }; await pollApi( `http://${BASTION_IP}:${HTTP_PORT}/api/machines`, (data) => vmMac in data.discovered, DISCOVERY_TIMEOUT_MS, 15_000, ); log("ARM VM discovered via ISO boot!"); // 6. Queue install log("Queueing ARM machine for install..."); await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/install`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ mac: vmMac, hostname: VM_NAME, disk: "", role: "vanilla" }), }); // 7. Reboot for install log("Waiting for discovery reboot..."); await sleep(30_000); // ARM is slow rebootPxeVm(VM_NAME); // 8. Wait for install (ARM emulation makes this very slow) log("Waiting for ARM install (emulated — expect 30-60 minutes)..."); type LogsResponse = { status: string; progress: string; ip?: string }; const finalState = await pollApi( `http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`, (data) => data.status === "installed" || data.progress === "error", INSTALL_TIMEOUT_MS, 30_000, ); if (finalState.progress === "error") { const logsRes = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`); const logs = await logsRes.json(); log(`ARM INSTALL FAILED. State: ${JSON.stringify(logs, null, 2)}`); throw new Error("ARM install failed — check logs above"); } vmIp = finalState.ip ?? ""; log(`ARM install complete! VM IP: ${vmIp}`); // 9. Ensure VM is running after kickstart reboot/poweroff log("Waiting for kickstart reboot/poweroff..."); await sleep(30_000); // ARM is slow const { spawnSync: spSync } = await import("node:child_process"); const stateResult = spSync("sudo", ["virsh", "domstate", VM_NAME], { encoding: "utf-8", stdio: "pipe" }); if (stateResult.stdout?.trim() === "shut off") { log("ARM VM shut off after install. Restarting..."); rebootPxeVm(VM_NAME); } // 10. Wait for SSH (ARM reboot is slow) log("Waiting for SSH on ARM VM..."); await waitForSsh(vmIp, SSH_USER, SSH_TIMEOUT_MS, sshKeyPath); log("ARM ISO boot provision test setup complete."); }, DISCOVERY_TIMEOUT_MS + INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000); afterAll(async () => { log("Cleaning up ARM test..."); if (bastionApp) await bastionApp.close().catch(() => {}); const { stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js"); stopDnsmasq(); destroyPxeVm(VM_NAME); destroyPxeNetwork(); if (testDir) rmSync(testDir, { recursive: true, force: true }); }); it("ARM machine was discovered and installed", async () => { const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`); const data = (await res.json()) as { installed: Record }; expect(data.installed[vmMac]).toBeDefined(); expect(data.installed[vmMac].hostname).toBe(VM_NAME); }); it("architecture is aarch64", async () => { const result = sshExec(vmIp, SSH_USER, "uname -m", { keyPath: sshKeyPath, timeout: 30_000 }); expect(result.exitCode).toBe(0); expect(result.stdout.trim()).toBe("aarch64"); }); it("SSH works", () => { const result = sshExec(vmIp, SSH_USER, "whoami", { keyPath: sshKeyPath, timeout: 30_000 }); expect(result.exitCode).toBe(0); expect(result.stdout.trim()).toBe(SSH_USER); }); it("hostname is correct", () => { const result = sshExec(vmIp, SSH_USER, "hostname -f", { keyPath: sshKeyPath, timeout: 30_000 }); expect(result.exitCode).toBe(0); expect(result.stdout.trim()).toContain(VM_NAME); }); it("provisioning metadata exists", () => { const result = sshExec(vmIp, SSH_USER, "cat /etc/lab-provisioned", { keyPath: sshKeyPath, timeout: 30_000 }); expect(result.exitCode).toBe(0); expect(result.stdout).toContain(`hostname: ${VM_NAME}`); expect(result.stdout).toContain("role: vanilla"); }); it("LVM layout is correct", () => { const result = sshExec(vmIp, SSH_USER, "sudo lvs labvg --noheadings -o lv_name", { keyPath: sshKeyPath, timeout: 30_000 }); expect(result.exitCode).toBe(0); const lvs = result.stdout.trim().split("\n").map((l: string) => l.trim()); for (const expected of ["root", "var", "varlog", "swap", "home", "srv"]) { expect(lvs).toContain(expected); } }); });