// Integration test: aarch64 network PXE boot. // // The boot-ISO path already covered ARM (arm-iso-provision.test.ts). This covers the // network path: DHCP option 93 handing an arm64 client an arm64 iPXE binary, dispatch // serving an aarch64 kernel, and `provision debug` reaching a rescue shell -- which is // what the DGX Sparks actually need and could not do. // // Two suites, because they cost very different amounts of time: // // "ARM PXE rescue" NBP handoff -> rescue with SSH. ~25-30 min // "ARM PXE install" discover -> install -> installed. ~75-95 min // // The rescue suite seeds the machine into state as an already-known aarch64 box rather // than discovering it first. That is the DGX Spark situation exactly -- SSH-onboarded, // never PXE-discovered, architecture known only from its record -- and it holds the test // to one emulated boot. Each boot spends ~15 of its ~18 minutes downloading Anaconda's // stage2 under TCG, so discovering first would double the runtime without touching any // code path the rescue boot does not already exercise. // // The install suite only runs with ARM_PXE_FULL=1. No ARM machine in the lab is ever // PXE-installed except the MS-R1, and an hour-plus test that runs by default is a test // nobody runs. // // IMPORTANT: aarch64 has no KVM on an x86_64 host, so all of this is emulated and // roughly 10x slower than native. // // A note for whoever debugs a failure here: if the VM panics with // VFS: Unable to mount root fs on unknown-block(0,0) // that is very likely iPXE silently dropping the initrd because the build lacks // EFI_LOAD_FILE2_PROTOCOL -- on arm64 the kernel EFI stub fetches the initrd over // LoadFile2, and an iPXE without it accepts the `initrd` line and does nothing. It is // NOT a reproduction of the DGX Spark kernel bug that motivated this work, despite // being the identical message. assertIpxeSupportsLoadFile2() below checks the build up // front so that failure names itself; to check by hand: // node -e 'const b=require("fs").readFileSync("/usr/share/ipxe/arm64-efi/snponly.efi"); // console.log(b.indexOf(Buffer.from("c1c00640b3fc3e40996d4a6c8724e06d","hex")))' // Fedora's ipxe-bootimgs-aarch64-20240119 has it at 0x3bbf0. // // Prerequisites: // - qemu-system-aarch64 (sudo dnf install qemu-system-aarch64) // - edk2-aarch64 (sudo dnf install edk2-aarch64) // - ipxe-bootimgs-aarch64 (sudo dnf install ipxe-bootimgs-aarch64) // - libvirtd, sudo, internet access // // Run: sudo ./scripts/test-provision.sh arm-pxe import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, writeFileSync } from "node:fs"; import { execSync } from "node:child_process"; import { join } from "node:path"; import { homedir, tmpdir } from "node:os"; import { log, waitForSsh } from "./helpers/libvirt.js"; import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js"; import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./helpers/pxe-vm.js"; import { sshExec } from "./helpers/ssh.js"; const IPXE_ARM64 = "/usr/share/ipxe/arm64-efi/snponly.efi"; const AAVMF = "/usr/share/edk2/aarch64/QEMU_EFI.fd"; const VM_MEMORY = 4096; const VM_VCPUS = 2; const VM_DISK_GB = 250; const SSH_USER = "lab"; const BASTION_IP = PXE_GATEWAY; const DHCP_RANGE_START = `${PXE_SUBNET}.100`; const DHCP_RANGE_END = `${PXE_SUBNET}.200`; const SERIAL_PORT = 4555; // Emulated aarch64 -- generous timeouts throughout. Measured on an x86_64 host with no // KVM for aarch64: a single PXE boot to a running Anaconda takes ~18 minutes, almost all // of it downloading inst.stage2 over the network under TCG. Budget well above that; // timing out just short of success wastes a whole run. const LEASE_TIMEOUT_MS = 10 * 60_000; const DISCOVERY_TIMEOUT_MS = 35 * 60_000; const INSTALL_TIMEOUT_MS = 75 * 60_000; const SSH_TIMEOUT_MS = 35 * 60_000; const RUN_FULL_INSTALL = process.env["ARM_PXE_FULL"] === "1"; function sleep(ms: number): Promise { return new Promise((r) => setTimeout(r, ms)); } function findSshKey(): { pubKey: string; keyPath: string } { const candidates: string[] = []; if (process.env["SSH_KEY_PATH"]) candidates.push(process.env["SSH_KEY_PATH"]); const homes = [homedir()]; const sudoUser = process.env["SUDO_USER"]; if (sudoUser) homes.push(join("/home", sudoUser)); for (const home of homes) { for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) { candidates.push(join(home, ".ssh", name)); } } for (const keyPath of candidates) { if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) { return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath }; } } throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/"); } async function pollApi( url: string, check: (data: T) => boolean, timeoutMs: number, intervalMs = 10_000, ): Promise { const start = Date.now(); while (Date.now() - start < timeoutMs) { try { const res = await fetch(url); if (res.ok) { const data = (await res.json()) as T; if (check(data)) return data; } } catch { /* bastion not up yet, or a network hiccup */ } await sleep(intervalMs); } throw new Error(`Timeout after ${timeoutMs}ms polling ${url}`); } function requirePrerequisites(): void { if (!existsSync("/usr/bin/qemu-system-aarch64")) { throw new Error("qemu-system-aarch64 not installed. Run: sudo dnf install qemu-system-aarch64"); } if (!existsSync(AAVMF)) { throw new Error(`AAVMF firmware not found at ${AAVMF}. Run: sudo dnf install edk2-aarch64`); } if (!existsSync(IPXE_ARM64)) { throw new Error(`arm64 iPXE not found at ${IPXE_ARM64}. Run: sudo dnf install ipxe-bootimgs-aarch64`); } } /** * Confirm the arm64 iPXE binary implements EFI_LOAD_FILE2_PROTOCOL. * * Without it the `initrd` line is accepted and silently ignored, and the kernel panics * with unknown-block(0,0). Checking here turns a confusing 30-minute boot failure into * an immediate, explanatory one. * * GUID 4006c0c1-fcb3-403e-996d-4a6c8724e06d, little-endian in the binary's GUID table. */ function assertIpxeSupportsLoadFile2(): void { const LOAD_FILE2_GUID = Buffer.from("c1c00640b3fc3e40996d4a6c8724e06d", "hex"); const binary = readFileSync(IPXE_ARM64); if (binary.indexOf(LOAD_FILE2_GUID) < 0) { throw new Error( `${IPXE_ARM64} does not reference EFI_LOAD_FILE2_PROTOCOL. On arm64 the kernel ` + `EFI stub fetches the initrd over LoadFile2; without it iPXE drops the initrd ` + `silently and the kernel panics with "unknown-block(0,0)". Rebuild iPXE with ` + `LoadFile2, or chainload grubaa64.efi for aarch64 instead.`, ); } log(`iPXE arm64 implements LoadFile2 — initrd will be delivered to the EFI stub`); } interface Harness { testDir: string; app: { close: () => Promise }; stopDnsmasq: () => void; state: { update: (fn: (s: BastionStateLike) => void) => void }; vmMac: string; httpPort: number; } /** Just the parts of BastionState this test seeds. */ interface BastionStateLike { discovered: Record>; installed: Record>; install_queue: Record>; debug: Record>; } /** Bring up an isolated network, a bastion with both arch payloads, and an arm64 VM. */ async function startHarness(vmName: string, httpPort: number, pubKey: string): Promise { requirePrerequisites(); assertIpxeSupportsLoadFile2(); log("Setting up PXE test network..."); ensurePxeNetwork(); const testDir = join(tmpdir(), `lab-arm-pxe-test-${Date.now()}`); for (const sub of ["tftp", "http", "logs"]) { mkdirSync(join(testDir, sub), { recursive: true }); } const { createApp } = await import("../../src/bastion/src/server.js"); const { loadConfig } = await import("../../src/bastion/src/config.js"); const { generateDnsmasqConf, startDnsmasq, stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js"); const { generateDiscoverKickstart } = await import("../../src/bastion/src/services/kickstart-generator.js"); const { renderBootIpxe, kernelPath, initrdPath } = await import("../../src/bastion/src/templates/boot.ipxe.js"); // Relative, not "@lab/shared": these tests run from the repo root against sources, // where the workspace package alias is not resolvable. const { SUPPORTED_ARCHES, fedoraMirrorFor } = await import("../../src/shared/src/hardware/index.js"); const config = loadConfig({ bastionDir: testDir, httpPort, iface: "virbr-pxe", serverIp: BASTION_IP, network: `${PXE_SUBNET}.0`, gateway: BASTION_IP, dhcpMode: "full", dhcpRangeStart: DHCP_RANGE_START, dhcpRangeEnd: DHCP_RANGE_END, domain: "arm-pxe-test.local", sshKeys: [pubKey], adminUser: SSH_USER, }); // iPXE binaries. The arm64 one is the whole point: dnsmasq hands it out on DHCP // option 93 -- 11 for UEFI PXE (TFTP) and 19 for UEFI HTTP Boot. // // They go in BOTH directories, exactly as main.ts stages them. AAVMF prefers HTTP // Boot, so it is served an http:// URL and fetches from httpDir; a firmware that // takes the TFTP path reads the same file from tftpDir. Staging only tftpDir gives a // 404 and "No bootable option or device was found" on the console. log("Staging iPXE binaries..."); const ipxeX86 = "/usr/share/ipxe/ipxe-snponly-x86_64.efi"; copyFileSync(IPXE_ARM64, join(config.tftpDir, "ipxe-arm64.efi")); copyFileSync(IPXE_ARM64, join(config.httpDir, "ipxe-arm64.efi")); if (existsSync(ipxeX86)) { copyFileSync(ipxeX86, join(config.tftpDir, "ipxe.efi")); copyFileSync(ipxeX86, join(config.httpDir, "ipxe.efi")); } // Fedora kernel + initrd for both architectures, cached across runs. const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache"; execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" }); for (const arch of SUPPORTED_ARCHES) { const mirror = fedoraMirrorFor(config.fedoraVersion, arch); const kernelCache = join(cacheDir, `vmlinuz-${arch}`); const initrdCache = join(cacheDir, `initrd-${arch}.img`); if (!existsSync(kernelCache)) { log(`Downloading Fedora ${config.fedoraVersion} ${arch} kernel...`); execSync(`curl -# -L -f -o "${kernelCache}" "${mirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 600_000 }); } if (!existsSync(initrdCache)) { log(`Downloading Fedora ${config.fedoraVersion} ${arch} initrd...`); execSync(`curl -# -L -f -o "${initrdCache}" "${mirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 600_000 }); } // Staged under the exact names the iPXE templates will ask for. copyFileSync(kernelCache, join(config.httpDir, kernelPath(arch))); copyFileSync(initrdCache, join(config.httpDir, initrdPath(arch))); log(`Staged ${arch}: ${kernelPath(arch)} + ${initrdPath(arch)}`); } writeFileSync(join(config.httpDir, "discover.ks"), generateDiscoverKickstart(config)); writeFileSync( join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }), ); generateDnsmasqConf(config); const { app, state, syslog } = createApp(config); await app.listen({ port: config.httpPort, host: "0.0.0.0" }); syslog.start(); log(`Bastion HTTP listening on :${config.httpPort}`); log("Starting dnsmasq (full DHCP)..."); startDnsmasq(config).catch((err) => { log(`dnsmasq failed: ${err instanceof Error ? err.message : String(err)}`); }); await sleep(1500); log("Creating aarch64 PXE VM (emulated — this is slow)..."); createPxeVm({ name: vmName, memory: VM_MEMORY, vcpus: VM_VCPUS, diskSize: VM_DISK_GB, network: PXE_NETWORK_NAME, arch: "aarch64", }); const vmMac = getVmMac(vmName); if (!vmMac) throw new Error("Could not determine VM MAC address"); log(`ARM VM MAC: ${vmMac}`); return { testDir, app, stopDnsmasq, state: state as unknown as Harness["state"], vmMac, httpPort: config.httpPort, }; } async function stopHarness(vmName: string, harness: Harness | undefined): Promise { // KEEP_VM=1 leaves the VM, network and bastion up so a failure can be inspected on // the console. Emulated aarch64 runs cost half an hour; tearing the evidence down // automatically means paying that again to see what happened. if (process.env["KEEP_VM"] === "1") { log(`KEEP_VM=1 — leaving ${vmName} running for inspection.`); log(` console: sudo virsh screenshot ${vmName} /tmp/vm.ppm`); log(` serial: socat - TCP:127.0.0.1:${SERIAL_PORT}`); if (harness) log(` bastion: ${harness.testDir} (still serving on :${harness.httpPort})`); log(` cleanup: sudo virsh destroy ${vmName}; sudo virsh undefine ${vmName} --remove-all-storage --nvram`); return; } log("Cleaning up..."); if (harness) { await harness.app.close().catch(() => {}); harness.stopDnsmasq(); } destroyPxeVm(vmName); destroyPxeNetwork(); if (harness) rmSync(harness.testDir, { recursive: true, force: true }); } /** Read the DHCP lease the bastion handed a MAC. Rescue mode reports no IP itself. */ function leaseIpFor(testDir: string, mac: string): string | null { const leaseFile = join(testDir, "dnsmasq.leases"); if (!existsSync(leaseFile)) return null; for (const line of readFileSync(leaseFile, "utf-8").split("\n")) { // const parts = line.trim().split(/\s+/); if (parts.length >= 3 && parts[1]?.toLowerCase() === mac.toLowerCase()) { return parts[2] ?? null; } } return null; } async function waitForLease(testDir: string, mac: string, timeoutMs: number): Promise { const start = Date.now(); while (Date.now() - start < timeoutMs) { const ip = leaseIpFor(testDir, mac); if (ip !== null) return ip; await sleep(5000); } throw new Error(`No DHCP lease for ${mac} within ${timeoutMs}ms`); } // --------------------------------------------------------------------------- // Rescue path -- what the DGX Sparks need. // --------------------------------------------------------------------------- describe("ARM PXE rescue", () => { const VM_NAME = "lab-arm-pxe-rescue"; const HTTP_PORT = 8096; let harness: Harness | undefined; let sshKeyPath: string; let rescueIp: string; beforeAll(async () => { const { pubKey, keyPath } = findSshKey(); sshKeyPath = keyPath; harness = await startHarness(VM_NAME, HTTP_PORT, pubKey); const { testDir, vmMac, state } = harness; // Seed the machine as an already-known aarch64 box queued for rescue. This is the // DGX Spark situation exactly: SSH-onboarded, never PXE-discovered, architecture // known only from its record -- and it also keeps the test to a SINGLE emulated // boot. Each boot spends ~15 minutes pulling Anaconda's stage2 over the network // under TCG, so discovering first and rescuing second doubles the runtime for no // extra coverage of the path being tested. Discovery is covered by the full suite. log(`Seeding ${vmMac} as a known aarch64 machine queued for rescue...`); state.update((s) => { s.discovered[vmMac] = { mac: vmMac, product: "Test ARM64 Machine", board: "virt", serial: "SN-ARM64", manufacturer: "QEMU", cpu_model: "cortex-a57", cpu_cores: VM_VCPUS, memory_gb: 4, arch: "aarch64", disks: [], nics: [], first_seen: new Date().toISOString(), last_seen: new Date().toISOString(), }; s.debug[vmMac] = { hostname: "arm-rescue-test", queued_at: new Date().toISOString() }; }); // Restart so the VM boots against the seeded state. createPxeVm already started it. rebootPxeVm(VM_NAME); await sleep(5_000); deleteNftablesRejectRules(); // The whole chain now runs once: DHCP option 93 -> arm64 iPXE -> /boot.ipxe -> // /dispatch (architecture from the record, not the query) -> aarch64 kernel + // initrd -> Anaconda rescue -> sshd. Reaching a shell at all proves iPXE handed // the initrd to the EFI stub over LoadFile2; without it the kernel panics first. log("Waiting for the rescue environment's DHCP lease..."); rescueIp = await waitForLease(testDir, vmMac, LEASE_TIMEOUT_MS); log(`Rescue IP: ${rescueIp}`); log("Waiting for SSH into the rescue shell (started by inst.sshd)..."); log("(emulated aarch64 — Anaconda's stage2 download dominates; be patient)"); await waitForSsh(rescueIp, "root", SSH_TIMEOUT_MS, sshKeyPath).catch(async (err) => { log("Rescue SSH timed out. Serial console:"); try { log(await readSerialLog(SERIAL_PORT, { lastLines: 100, timeoutMs: 15_000 })); } catch { /* console unavailable */ } throw err; }); log("ARM PXE rescue reached."); }, LEASE_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000); afterAll(async () => { await stopHarness(VM_NAME, harness); }); it("resolved the architecture from the machine record", async () => { const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`); const data = (await res.json()) as { discovered: Record }; expect(data.discovered[harness!.vmMac]?.arch).toBe("aarch64"); }); it("rescue shell is reachable over SSH and is aarch64", () => { const result = sshExec(rescueIp, "root", "uname -m", { keyPath: sshKeyPath, timeout: 60_000 }); expect(result.exitCode).toBe(0); expect(result.stdout.trim()).toBe("aarch64"); }); it("booted an initramfs — the LoadFile2 path worked", () => { // If iPXE had dropped the initrd the kernel would never have reached userspace at // all, but assert it explicitly so a regression names itself. const result = sshExec(rescueIp, "root", "cat /proc/cmdline; ls /run/install", { keyPath: sshKeyPath, timeout: 60_000, }); expect(result.exitCode).toBe(0); expect(result.stdout).toContain("inst.rescue"); }); it("rescue kernel came from the bastion over HTTP", () => { const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 }); expect(result.stdout).toContain(`${BASTION_IP}:${HTTP_PORT}`); // arm64 gets serial console arguments, never nomodeset. expect(result.stdout).toContain("console=ttyAMA0"); expect(result.stdout).not.toContain("nomodeset"); }); it("has LVM tools available for inspecting an installed system", () => { const result = sshExec(rescueIp, "root", "command -v vgchange && command -v lsblk", { keyPath: sshKeyPath, timeout: 60_000, }); expect(result.exitCode).toBe(0); }); }); // --------------------------------------------------------------------------- // Full install -- opt-in, ~60-90 minutes emulated. // --------------------------------------------------------------------------- describe.runIf(RUN_FULL_INSTALL)("ARM PXE install", () => { const VM_NAME = "lab-arm-pxe-install"; const HTTP_PORT = 8095; let harness: Harness | undefined; let sshKeyPath: string; let vmIp: string; beforeAll(async () => { const { pubKey, keyPath } = findSshKey(); sshKeyPath = keyPath; harness = await startHarness(VM_NAME, HTTP_PORT, pubKey); const { vmMac } = harness; log("Waiting for aarch64 discovery..."); await pollApi<{ discovered: Record }>( `http://${BASTION_IP}:${HTTP_PORT}/api/machines`, (data) => vmMac in data.discovered, DISCOVERY_TIMEOUT_MS, ); log("Discovered. Queueing install..."); const installRes = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/install`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ mac: vmMac, hostname: VM_NAME, disk: "", role: "vanilla" }), }); expect(installRes.status).toBe(200); await sleep(30_000); rebootPxeVm(VM_NAME); log("Waiting for the emulated aarch64 install (60-90 min)..."); type LogsResponse = { status: string; progress: string; ip?: string }; const final = await pollApi( `http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`, (d) => d.status === "installed" || d.progress === "error", INSTALL_TIMEOUT_MS, 30_000, ); if (final.progress === "error") { const logs = await (await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`)).json(); log(`ARM install FAILED: ${JSON.stringify(logs, null, 2)}`); throw new Error("ARM PXE install failed — see logs above"); } vmIp = final.ip ?? ""; log(`ARM install complete. IP: ${vmIp}`); await sleep(30_000); rebootPxeVm(VM_NAME); await sleep(5_000); deleteNftablesRejectRules(); await waitForSsh(vmIp, SSH_USER, SSH_TIMEOUT_MS, sshKeyPath); }, DISCOVERY_TIMEOUT_MS + INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 600_000); afterAll(async () => { await stopHarness(VM_NAME, harness); }); it("machine reached installed state", async () => { const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`); const data = (await res.json()) as { installed: Record }; expect(data.installed[harness!.vmMac]?.hostname).toBe(VM_NAME); }); it("installed system is aarch64", () => { const result = sshExec(vmIp, SSH_USER, "uname -m", { keyPath: sshKeyPath, timeout: 60_000 }); expect(result.stdout.trim()).toBe("aarch64"); }); it("SSH works with the admin user", () => { const result = sshExec(vmIp, SSH_USER, "whoami", { keyPath: sshKeyPath, timeout: 60_000 }); expect(result.stdout.trim()).toBe(SSH_USER); }); it("LVM layout is correct", () => { const result = sshExec(vmIp, SSH_USER, "sudo lvs labvg --noheadings -o lv_name", { keyPath: sshKeyPath, timeout: 60_000, }); expect(result.exitCode).toBe(0); const lvs = result.stdout.trim().split("\n").map((l) => l.trim()); for (const expected of ["root", "var", "varlog", "swap", "home", "srv"]) { expect(lvs).toContain(expected); } }); });