diff --git a/bastion/package.json b/bastion/package.json index fc1b5a3..62f2d8e 100644 --- a/bastion/package.json +++ b/bastion/package.json @@ -23,6 +23,10 @@ "test:integration:iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'", "test:integration:arm-iso": "vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'", "test:integration:arm-iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'", + "test:integration:arm-pxe": "vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE rescue'", + "test:integration:arm-pxe:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE rescue'", + "test:integration:arm-pxe-full": "ARM_PXE_FULL=1 vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE'", + "test:integration:arm-pxe-full:host": "sudo -E ARM_PXE_FULL=1 $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE'", "test:integration:asahi": "vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'", "test:integration:asahi:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'", "test:integration:asahi-validate": "vitest run -c tests/integration/vitest.config.ts -t 'asahi.*validation'", diff --git a/bastion/scripts/test-provision.sh b/bastion/scripts/test-provision.sh index 4bc20c6..c9e372f 100755 --- a/bastion/scripts/test-provision.sh +++ b/bastion/scripts/test-provision.sh @@ -2,16 +2,18 @@ # Run PXE and/or ISO boot integration tests. # # Usage: -# sudo ./scripts/test-provision.sh # run PXE + ISO (x86_64) -# sudo ./scripts/test-provision.sh pxe # PXE only -# sudo ./scripts/test-provision.sh iso # ISO only (x86_64) -# sudo ./scripts/test-provision.sh arm # ARM ISO boot (emulated, SLOW ~60min) -# sudo ./scripts/test-provision.sh all # all tests including ARM +# sudo ./scripts/test-provision.sh # run PXE + ISO (x86_64) +# sudo ./scripts/test-provision.sh pxe # PXE only +# sudo ./scripts/test-provision.sh iso # ISO only (x86_64) +# sudo ./scripts/test-provision.sh arm # ARM ISO boot (emulated, SLOW ~60min) +# sudo ./scripts/test-provision.sh arm-pxe # ARM network PXE: NBP + discover + rescue (~20-30min) +# sudo ./scripts/test-provision.sh arm-pxe-full # ARM network PXE incl. full install (~60-90min) +# sudo ./scripts/test-provision.sh all # all tests including ARM # # Prerequisites: # libvirtd, OVMF (edk2-ovmf), iPXE (ipxe-bootimgs-x86), # dnsmasq, xorriso, mtools, virt-install, qemu-img -# ARM: qemu-system-aarch64, edk2-aarch64 +# ARM: qemu-system-aarch64, edk2-aarch64, ipxe-bootimgs-aarch64 set -e SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -58,6 +60,10 @@ if [ ! -f /usr/share/edk2/ovmf/OVMF_CODE.fd ]; then exit 1 fi +MODE="${1:-both}" + +# iPXE binaries are per-architecture. x86_64 is always required (the dnsmasq config +# references it); arm64 only for the ARM network-PXE modes. IPXE_EFI="" for f in /usr/share/ipxe/ipxe-snponly-x86_64.efi /usr/share/ipxe/ipxe-snp-x86_64.efi /usr/share/ipxe/ipxe-x86_64.efi; do [ -f "$f" ] && IPXE_EFI="$f" && break @@ -67,6 +73,20 @@ if [ -z "$IPXE_EFI" ]; then exit 1 fi +IPXE_EFI_ARM64="" +for f in /usr/share/ipxe/arm64-efi/snponly.efi /usr/share/ipxe/arm64-efi/ipxe.efi; do + [ -f "$f" ] && IPXE_EFI_ARM64="$f" && break +done + +case "$MODE" in + arm-pxe|arm-pxe-full|all) + if [ -z "$IPXE_EFI_ARM64" ] && [ "$MODE" != "all" ]; then + echo -e "${RED}arm64 iPXE binary not found.${RESET} Install: sudo dnf install ipxe-bootimgs-aarch64" + exit 1 + fi + ;; +esac + # Find SSH key SSH_KEY="" for name in id_ed25519 id_ecdsa id_rsa; do @@ -83,10 +103,19 @@ fi echo -e " User: ${BOLD}$REAL_USER${RESET}" echo -e " SSH key: ${BOLD}$SSH_KEY${RESET}" echo -e " iPXE: ${BOLD}$IPXE_EFI${RESET}" +echo -e " iPXE a64:${BOLD} ${IPXE_EFI_ARM64:-not installed}${RESET}" echo "" -# --- Determine which tests to run --- -MODE="${1:-both}" +require_arm_emulation() { + if ! command -v qemu-system-aarch64 &>/dev/null; then + echo -e "${RED}qemu-system-aarch64 not found.${RESET} Install: sudo dnf install qemu-system-aarch64 edk2-aarch64" + exit 1 + fi + if [ ! -f /usr/share/edk2/aarch64/QEMU_EFI.fd ]; then + echo -e "${RED}AAVMF firmware not found.${RESET} Install: sudo dnf install edk2-aarch64" + exit 1 + fi +} run_test() { local name="$1" pattern="$2" @@ -116,13 +145,22 @@ case "$MODE" in run_test "ISO boot" "ISO boot" || FAILED=1 ;; arm|arm-iso) - if ! command -v qemu-system-aarch64 &>/dev/null; then - echo -e "${RED}qemu-system-aarch64 not found.${RESET} Install: sudo dnf install qemu-system-aarch64 edk2-aarch64" - exit 1 - fi + require_arm_emulation echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 30-60 minutes.${RESET}" run_test "ARM ISO boot" "ARM ISO" || FAILED=1 ;; + arm-pxe) + require_arm_emulation + echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 20-30 minutes.${RESET}" + echo -e "${YELLOW}Covers option 93 -> arm64 NBP, discovery, and rescue over SSH.${RESET}" + echo -e "${YELLOW}For the full install too, use: $0 arm-pxe-full${RESET}" + run_test "ARM PXE rescue" "ARM PXE rescue" || FAILED=1 + ;; + arm-pxe-full) + require_arm_emulation + echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 60-90 minutes.${RESET}" + ARM_PXE_FULL=1 run_test "ARM PXE (rescue + install)" "ARM PXE" || FAILED=1 + ;; both) run_test "PXE boot" "PXE boot" || FAILED=1 run_test "ISO boot" "ISO boot" || FAILED=1 @@ -133,12 +171,17 @@ case "$MODE" in if command -v qemu-system-aarch64 &>/dev/null; then echo -e "${YELLOW}ARM emulation is ~10x slower than native.${RESET}" run_test "ARM ISO boot" "ARM ISO" || FAILED=1 + if [ -n "$IPXE_EFI_ARM64" ]; then + run_test "ARM PXE rescue" "ARM PXE rescue" || FAILED=1 + else + echo -e "${YELLOW}Skipping ARM PXE test (ipxe-bootimgs-aarch64 not installed)${RESET}" + fi else - echo -e "${YELLOW}Skipping ARM test (qemu-system-aarch64 not installed)${RESET}" + echo -e "${YELLOW}Skipping ARM tests (qemu-system-aarch64 not installed)${RESET}" fi ;; *) - echo "Usage: $0 [pxe|iso|arm|both|all]" + echo "Usage: $0 [pxe|iso|arm|arm-pxe|arm-pxe-full|both|all]" exit 1 ;; esac diff --git a/bastion/src/bastion/src/routes/dispatch.ts b/bastion/src/bastion/src/routes/dispatch.ts index 2a03921..7392c68 100644 --- a/bastion/src/bastion/src/routes/dispatch.ts +++ b/bastion/src/bastion/src/routes/dispatch.ts @@ -162,9 +162,10 @@ echo "===============================" state.update((s) => { delete s.debug[mac]; }); let script: string; - const root = debugEntry.pxeBoot ? resolveRoot(currentState, mac) : null; + const wantsPxeBoot = debugEntry.pxeBoot === true; + const root = wantsPxeBoot ? resolveRoot(currentState, mac) : null; - if (debugEntry.pxeBoot && root) { + if (root !== null) { logger.info(`PXE BOOT DEBUG: ${mac} -> ${hostname} (${arch}, root=${root.rootDevice})`); script = renderPxeBootDebugIpxe({ mac, @@ -178,7 +179,7 @@ echo "===============================" // --pxe-boot without a known root device falls back to rescue rather than // guessing. A wrong root= leaves the machine unbootable, and rescue is where // the operator can find the real one (curl /debug-setup.sh reports it back). - const notice = debugEntry.pxeBoot + const notice = wantsPxeBoot ? [ "", "NOTE: --pxe-boot requested, but no root device is recorded", @@ -188,7 +189,7 @@ echo "===============================" " then retry --pxe-boot.", ] : undefined; - if (debugEntry.pxeBoot) { + if (wantsPxeBoot) { logger.warn(`PXE BOOT DEBUG: ${mac} -> ${hostname} has no recorded root device -- serving rescue instead`); } else { logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (${arch}, rescue mode)`); diff --git a/bastion/src/bastion/src/services/install-guard.ts b/bastion/src/bastion/src/services/install-guard.ts index 670e866..9dbb517 100644 --- a/bastion/src/bastion/src/services/install-guard.ts +++ b/bastion/src/bastion/src/services/install-guard.ts @@ -7,15 +7,21 @@ // Rescue/debug is deliberately NOT guarded. Being unable to reinstall a machine is // exactly when you most need to boot it into a rescue shell. -import type { BastionState, OsId } from "@lab/shared"; +import type { Arch, BastionState, OsId } from "@lab/shared"; import { classifyOnboard, normalizeArch, osSupportsArch, vendorOsDescription, archesForOs } from "@lab/shared"; export type InstallCheck = | { allowed: true } | { allowed: false; error: string }; +interface MachineIdentity { + hostname: string; + arch: Arch | undefined; + identity: Parameters[0]; +} + /** Best-known identity for a MAC, merged across the three state maps. */ -function identify(state: BastionState, mac: string) { +function identify(state: BastionState, mac: string): MachineIdentity { const discovered = state.discovered[mac]; const installed = state.installed[mac]; const queued = state.install_queue[mac]; diff --git a/bastion/src/cli/src/commands/debug.ts b/bastion/src/cli/src/commands/debug.ts index 4c54dde..6569efb 100644 --- a/bastion/src/cli/src/commands/debug.ts +++ b/bastion/src/cli/src/commands/debug.ts @@ -132,7 +132,7 @@ export function registerDebugCommand(parent: Command): void { const probe = sshCapture(ip, ROOT_DEVICE_PROBE); const root = probe === null ? {} : parseRootProbe(probe); if (root.root_device !== undefined) { - console.log(` root=${root.root_device}${root.root_args ? ` ${root.root_args}` : ""}`); + console.log(` root=${root.root_device}${root.root_args !== undefined ? ` ${root.root_args}` : ""}`); try { await client.discoverMachine({ mac, ...root }); } catch (err) { diff --git a/bastion/src/cli/src/commands/recheck.ts b/bastion/src/cli/src/commands/recheck.ts index 5695382..183440e 100644 --- a/bastion/src/cli/src/commands/recheck.ts +++ b/bastion/src/cli/src/commands/recheck.ts @@ -84,7 +84,9 @@ export function registerRecheckCommand(parent: Command): void { const cpu = hwData.cpu_model || "?"; const cores = hwData.cpu_cores || "?"; const mem = hwData.memory_gb || "?"; - const root = hwData.root_device ? `, root=${hwData.root_device}` : ""; + const root = typeof hwData.root_device === "string" && hwData.root_device !== "" + ? `, root=${hwData.root_device}` + : ""; console.log(`${GREEN}OK${RESET} ${DIM}${cpu}, ${cores} cores, ${mem}GB${root}${RESET}`); updated++; } catch (err) { diff --git a/bastion/tests/integration/arm-pxe-provision.test.ts b/bastion/tests/integration/arm-pxe-provision.test.ts new file mode 100644 index 0000000..f0be42f --- /dev/null +++ b/bastion/tests/integration/arm-pxe-provision.test.ts @@ -0,0 +1,490 @@ +// Integration test: aarch64 network PXE boot. +// +// The boot-ISO path already covered ARM (arm-iso-provision.test.ts). This covers the +// network path: DHCP option 93 handing an arm64 client an arm64 iPXE binary, dispatch +// serving an aarch64 kernel, and `provision debug` reaching a rescue shell -- which is +// what the DGX Sparks actually need and could not do. +// +// Two suites, because they cost very different amounts of time: +// +// "ARM PXE rescue" NBP handoff -> discovery -> rescue with SSH. ~20-30 min +// "ARM PXE install" full discover -> install -> installed. ~60-90 min +// +// The install suite only runs with ARM_PXE_FULL=1. No ARM machine in the lab is ever +// PXE-installed except the MS-R1, and an hour-plus test that runs by default is a test +// nobody runs. +// +// IMPORTANT: aarch64 has no KVM on an x86_64 host, so all of this is emulated and +// roughly 10x slower than native. +// +// A note for whoever debugs a failure here: if the VM panics with +// VFS: Unable to mount root fs on unknown-block(0,0) +// that is very likely iPXE silently dropping the initrd because the build lacks +// EFI_LOAD_FILE2_PROTOCOL -- on arm64 the kernel EFI stub fetches the initrd over +// LoadFile2, and an iPXE without it accepts the `initrd` line and does nothing. It is +// NOT a reproduction of the DGX Spark kernel bug that motivated this work, despite +// being the identical message. assertIpxeSupportsLoadFile2() below checks the build up +// front so that failure names itself; to check by hand: +// node -e 'const b=require("fs").readFileSync("/usr/share/ipxe/arm64-efi/snponly.efi"); +// console.log(b.indexOf(Buffer.from("c1c00640b3fc3e40996d4a6c8724e06d","hex")))' +// Fedora's ipxe-bootimgs-aarch64-20240119 has it at 0x3bbf0. +// +// Prerequisites: +// - qemu-system-aarch64 (sudo dnf install qemu-system-aarch64) +// - edk2-aarch64 (sudo dnf install edk2-aarch64) +// - ipxe-bootimgs-aarch64 (sudo dnf install ipxe-bootimgs-aarch64) +// - libvirtd, sudo, internet access +// +// Run: sudo ./scripts/test-provision.sh arm-pxe + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, writeFileSync } from "node:fs"; +import { execSync } from "node:child_process"; +import { join } from "node:path"; +import { homedir, tmpdir } from "node:os"; +import { log, waitForSsh } from "./helpers/libvirt.js"; +import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js"; +import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./helpers/pxe-vm.js"; +import { sshExec } from "./helpers/ssh.js"; + +const IPXE_ARM64 = "/usr/share/ipxe/arm64-efi/snponly.efi"; +const AAVMF = "/usr/share/edk2/aarch64/QEMU_EFI.fd"; + +const VM_MEMORY = 4096; +const VM_VCPUS = 2; +const VM_DISK_GB = 250; +const SSH_USER = "lab"; +const BASTION_IP = PXE_GATEWAY; +const DHCP_RANGE_START = `${PXE_SUBNET}.100`; +const DHCP_RANGE_END = `${PXE_SUBNET}.200`; +const SERIAL_PORT = 4555; + +// Emulated aarch64 -- generous timeouts throughout. +const DISCOVERY_TIMEOUT_MS = 20 * 60_000; +const RESCUE_TIMEOUT_MS = 20 * 60_000; +const INSTALL_TIMEOUT_MS = 75 * 60_000; +const SSH_TIMEOUT_MS = 20 * 60_000; + +const RUN_FULL_INSTALL = process.env["ARM_PXE_FULL"] === "1"; + +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +function findSshKey(): { pubKey: string; keyPath: string } { + const candidates: string[] = []; + if (process.env["SSH_KEY_PATH"]) candidates.push(process.env["SSH_KEY_PATH"]); + const homes = [homedir()]; + const sudoUser = process.env["SUDO_USER"]; + if (sudoUser) homes.push(join("/home", sudoUser)); + for (const home of homes) { + for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) { + candidates.push(join(home, ".ssh", name)); + } + } + for (const keyPath of candidates) { + if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) { + return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath }; + } + } + throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/"); +} + +async function pollApi( + url: string, + check: (data: T) => boolean, + timeoutMs: number, + intervalMs = 10_000, +): Promise { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + try { + const res = await fetch(url); + if (res.ok) { + const data = (await res.json()) as T; + if (check(data)) return data; + } + } catch { /* bastion not up yet, or a network hiccup */ } + await sleep(intervalMs); + } + throw new Error(`Timeout after ${timeoutMs}ms polling ${url}`); +} + +function requirePrerequisites(): void { + if (!existsSync("/usr/bin/qemu-system-aarch64")) { + throw new Error("qemu-system-aarch64 not installed. Run: sudo dnf install qemu-system-aarch64"); + } + if (!existsSync(AAVMF)) { + throw new Error(`AAVMF firmware not found at ${AAVMF}. Run: sudo dnf install edk2-aarch64`); + } + if (!existsSync(IPXE_ARM64)) { + throw new Error(`arm64 iPXE not found at ${IPXE_ARM64}. Run: sudo dnf install ipxe-bootimgs-aarch64`); + } +} + +/** + * Confirm the arm64 iPXE binary implements EFI_LOAD_FILE2_PROTOCOL. + * + * Without it the `initrd` line is accepted and silently ignored, and the kernel panics + * with unknown-block(0,0). Checking here turns a confusing 30-minute boot failure into + * an immediate, explanatory one. + * + * GUID 4006c0c1-fcb3-403e-996d-4a6c8724e06d, little-endian in the binary's GUID table. + */ +function assertIpxeSupportsLoadFile2(): void { + const LOAD_FILE2_GUID = Buffer.from("c1c00640b3fc3e40996d4a6c8724e06d", "hex"); + const binary = readFileSync(IPXE_ARM64); + if (binary.indexOf(LOAD_FILE2_GUID) < 0) { + throw new Error( + `${IPXE_ARM64} does not reference EFI_LOAD_FILE2_PROTOCOL. On arm64 the kernel ` + + `EFI stub fetches the initrd over LoadFile2; without it iPXE drops the initrd ` + + `silently and the kernel panics with "unknown-block(0,0)". Rebuild iPXE with ` + + `LoadFile2, or chainload grubaa64.efi for aarch64 instead.`, + ); + } + log(`iPXE arm64 implements LoadFile2 — initrd will be delivered to the EFI stub`); +} + +interface Harness { + testDir: string; + app: { close: () => Promise }; + stopDnsmasq: () => void; + vmMac: string; + httpPort: number; +} + +/** Bring up an isolated network, a bastion with both arch payloads, and an arm64 VM. */ +async function startHarness(vmName: string, httpPort: number, pubKey: string): Promise { + requirePrerequisites(); + assertIpxeSupportsLoadFile2(); + + log("Setting up PXE test network..."); + ensurePxeNetwork(); + + const testDir = join(tmpdir(), `lab-arm-pxe-test-${Date.now()}`); + for (const sub of ["tftp", "http", "logs"]) { + mkdirSync(join(testDir, sub), { recursive: true }); + } + + const { createApp } = await import("../../src/bastion/src/server.js"); + const { loadConfig } = await import("../../src/bastion/src/config.js"); + const { generateDnsmasqConf, startDnsmasq, stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js"); + const { generateDiscoverKickstart } = await import("../../src/bastion/src/services/kickstart-generator.js"); + const { renderBootIpxe } = await import("../../src/bastion/src/templates/boot.ipxe.js"); + const { kernelPath, initrdPath } = await import("../../src/bastion/src/templates/boot.ipxe.js"); + const { SUPPORTED_ARCHES, fedoraMirrorFor } = await import("@lab/shared"); + + const config = loadConfig({ + bastionDir: testDir, + httpPort, + iface: "virbr-pxe", + serverIp: BASTION_IP, + network: `${PXE_SUBNET}.0`, + gateway: BASTION_IP, + dhcpMode: "full", + dhcpRangeStart: DHCP_RANGE_START, + dhcpRangeEnd: DHCP_RANGE_END, + domain: "arm-pxe-test.local", + sshKeys: [pubKey], + adminUser: SSH_USER, + }); + + // iPXE binaries. The arm64 one is the whole point: dnsmasq hands it out on DHCP + // option 93 = 11, and x86_64 is staged too so the config renders as it does in + // production. + log("Staging iPXE binaries..."); + copyFileSync(IPXE_ARM64, join(config.tftpDir, "ipxe-arm64.efi")); + const ipxeX86 = "/usr/share/ipxe/ipxe-snponly-x86_64.efi"; + if (existsSync(ipxeX86)) copyFileSync(ipxeX86, join(config.tftpDir, "ipxe.efi")); + + // Fedora kernel + initrd for both architectures, cached across runs. + const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache"; + execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" }); + + for (const arch of SUPPORTED_ARCHES) { + const mirror = fedoraMirrorFor(config.fedoraVersion, arch); + const kernelCache = join(cacheDir, `vmlinuz-${arch}`); + const initrdCache = join(cacheDir, `initrd-${arch}.img`); + + if (!existsSync(kernelCache)) { + log(`Downloading Fedora ${config.fedoraVersion} ${arch} kernel...`); + execSync(`curl -# -L -f -o "${kernelCache}" "${mirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 600_000 }); + } + if (!existsSync(initrdCache)) { + log(`Downloading Fedora ${config.fedoraVersion} ${arch} initrd...`); + execSync(`curl -# -L -f -o "${initrdCache}" "${mirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 600_000 }); + } + + // Staged under the exact names the iPXE templates will ask for. + copyFileSync(kernelCache, join(config.httpDir, kernelPath(arch))); + copyFileSync(initrdCache, join(config.httpDir, initrdPath(arch))); + log(`Staged ${arch}: ${kernelPath(arch)} + ${initrdPath(arch)}`); + } + + writeFileSync(join(config.httpDir, "discover.ks"), generateDiscoverKickstart(config)); + writeFileSync( + join(config.httpDir, "boot.ipxe"), + renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }), + ); + generateDnsmasqConf(config); + + const { app, syslog } = createApp(config); + await app.listen({ port: config.httpPort, host: "0.0.0.0" }); + syslog.start(); + log(`Bastion HTTP listening on :${config.httpPort}`); + + log("Starting dnsmasq (full DHCP)..."); + startDnsmasq(config).catch((err) => { + log(`dnsmasq failed: ${err instanceof Error ? err.message : String(err)}`); + }); + await sleep(1500); + + log("Creating aarch64 PXE VM (emulated — this is slow)..."); + createPxeVm({ + name: vmName, + memory: VM_MEMORY, + vcpus: VM_VCPUS, + diskSize: VM_DISK_GB, + network: PXE_NETWORK_NAME, + arch: "aarch64", + }); + + const vmMac = getVmMac(vmName); + if (!vmMac) throw new Error("Could not determine VM MAC address"); + log(`ARM VM MAC: ${vmMac}`); + + return { testDir, app, stopDnsmasq, vmMac, httpPort: config.httpPort }; +} + +async function stopHarness(vmName: string, harness: Harness | undefined): Promise { + log("Cleaning up..."); + if (harness) { + await harness.app.close().catch(() => {}); + harness.stopDnsmasq(); + } + destroyPxeVm(vmName); + destroyPxeNetwork(); + if (harness) rmSync(harness.testDir, { recursive: true, force: true }); +} + +/** Read the DHCP lease the bastion handed a MAC. Rescue mode reports no IP itself. */ +function leaseIpFor(testDir: string, mac: string): string | null { + const leaseFile = join(testDir, "dnsmasq.leases"); + if (!existsSync(leaseFile)) return null; + for (const line of readFileSync(leaseFile, "utf-8").split("\n")) { + // + const parts = line.trim().split(/\s+/); + if (parts.length >= 3 && parts[1]?.toLowerCase() === mac.toLowerCase()) { + return parts[2] ?? null; + } + } + return null; +} + +async function waitForLease(testDir: string, mac: string, timeoutMs: number): Promise { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const ip = leaseIpFor(testDir, mac); + if (ip !== null) return ip; + await sleep(5000); + } + throw new Error(`No DHCP lease for ${mac} within ${timeoutMs}ms`); +} + +// --------------------------------------------------------------------------- +// Rescue path -- what the DGX Sparks need. +// --------------------------------------------------------------------------- + +describe("ARM PXE rescue", () => { + const VM_NAME = "lab-arm-pxe-rescue"; + const HTTP_PORT = 8096; + let harness: Harness | undefined; + let sshKeyPath: string; + let rescueIp: string; + + beforeAll(async () => { + const { pubKey, keyPath } = findSshKey(); + sshKeyPath = keyPath; + harness = await startHarness(VM_NAME, HTTP_PORT, pubKey); + const { testDir, vmMac } = harness; + + // 1. The VM PXE boots. This exercises the whole chain: DHCP option 93 -> arm64 + // iPXE -> /boot.ipxe -> /dispatch?arch=arm64 -> aarch64 kernel + initrd. + // Reaching discovery at all proves the initrd was delivered over LoadFile2 -- + // without it the kernel panics before it can run the kickstart. + log("Waiting for aarch64 VM to PXE boot and report discovery..."); + log("(emulated aarch64 — be patient)"); + type MachinesResponse = { discovered: Record }; + const discovered = await pollApi( + `http://${BASTION_IP}:${HTTP_PORT}/api/machines`, + (data) => vmMac in data.discovered, + DISCOVERY_TIMEOUT_MS, + ).catch(async (err) => { + log("Discovery timed out. Serial console:"); + try { + log(await readSerialLog(SERIAL_PORT, { lastLines: 100, timeoutMs: 15_000 })); + } catch { /* console unavailable */ } + throw err; + }); + log(`aarch64 VM discovered as ${discovered.discovered[vmMac]?.arch}`); + + // 2. Queue rescue, exactly as `labctl provision debug` does. + log("Queueing debug/rescue mode..."); + const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/debug`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ mac: vmMac }), + }); + expect(res.ok).toBe(true); + + // 3. Reboot into rescue. Rescue mode runs no %pre/%post, so the machine never + // reports its own IP — read it from the DHCP lease instead. + await sleep(20_000); + rebootPxeVm(VM_NAME); + await sleep(5_000); + deleteNftablesRejectRules(); + + log("Waiting for rescue environment DHCP lease..."); + rescueIp = await waitForLease(testDir, vmMac, RESCUE_TIMEOUT_MS); + log(`Rescue IP: ${rescueIp}`); + + log("Waiting for SSH into the rescue shell (started by inst.sshd)..."); + await waitForSsh(rescueIp, "root", SSH_TIMEOUT_MS, sshKeyPath).catch(async (err) => { + log("Rescue SSH timed out. Serial console:"); + try { + log(await readSerialLog(SERIAL_PORT, { lastLines: 100, timeoutMs: 15_000 })); + } catch { /* console unavailable */ } + throw err; + }); + log("ARM PXE rescue reached."); + }, DISCOVERY_TIMEOUT_MS + RESCUE_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000); + + afterAll(async () => { await stopHarness(VM_NAME, harness); }); + + it("discovered the machine as aarch64", async () => { + const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`); + const data = (await res.json()) as { discovered: Record }; + expect(data.discovered[harness!.vmMac]?.arch).toBe("aarch64"); + }); + + it("rescue shell is reachable over SSH and is aarch64", () => { + const result = sshExec(rescueIp, "root", "uname -m", { keyPath: sshKeyPath, timeout: 60_000 }); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe("aarch64"); + }); + + it("booted an initramfs — the LoadFile2 path worked", () => { + // If iPXE had dropped the initrd the kernel would never have reached userspace at + // all, but assert it explicitly so a regression names itself. + const result = sshExec(rescueIp, "root", "cat /proc/cmdline; ls /run/install", { + keyPath: sshKeyPath, timeout: 60_000, + }); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("inst.rescue"); + }); + + it("rescue kernel came from the bastion over HTTP", () => { + const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 }); + expect(result.stdout).toContain(`${BASTION_IP}:${HTTP_PORT}`); + // arm64 gets serial console arguments, never nomodeset. + expect(result.stdout).toContain("console=ttyAMA0"); + expect(result.stdout).not.toContain("nomodeset"); + }); + + it("has LVM tools available for inspecting an installed system", () => { + const result = sshExec(rescueIp, "root", "command -v vgchange && command -v lsblk", { + keyPath: sshKeyPath, timeout: 60_000, + }); + expect(result.exitCode).toBe(0); + }); +}); + +// --------------------------------------------------------------------------- +// Full install -- opt-in, ~60-90 minutes emulated. +// --------------------------------------------------------------------------- + +describe.runIf(RUN_FULL_INSTALL)("ARM PXE install", () => { + const VM_NAME = "lab-arm-pxe-install"; + const HTTP_PORT = 8095; + let harness: Harness | undefined; + let sshKeyPath: string; + let vmIp: string; + + beforeAll(async () => { + const { pubKey, keyPath } = findSshKey(); + sshKeyPath = keyPath; + harness = await startHarness(VM_NAME, HTTP_PORT, pubKey); + const { vmMac } = harness; + + log("Waiting for aarch64 discovery..."); + await pollApi<{ discovered: Record }>( + `http://${BASTION_IP}:${HTTP_PORT}/api/machines`, + (data) => vmMac in data.discovered, + DISCOVERY_TIMEOUT_MS, + ); + log("Discovered. Queueing install..."); + + const installRes = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/install`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ mac: vmMac, hostname: VM_NAME, disk: "", role: "vanilla" }), + }); + expect(installRes.status).toBe(200); + + await sleep(30_000); + rebootPxeVm(VM_NAME); + + log("Waiting for the emulated aarch64 install (60-90 min)..."); + type LogsResponse = { status: string; progress: string; ip?: string }; + const final = await pollApi( + `http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`, + (d) => d.status === "installed" || d.progress === "error", + INSTALL_TIMEOUT_MS, + 30_000, + ); + + if (final.progress === "error") { + const logs = await (await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`)).json(); + log(`ARM install FAILED: ${JSON.stringify(logs, null, 2)}`); + throw new Error("ARM PXE install failed — see logs above"); + } + + vmIp = final.ip ?? ""; + log(`ARM install complete. IP: ${vmIp}`); + + await sleep(30_000); + rebootPxeVm(VM_NAME); + await sleep(5_000); + deleteNftablesRejectRules(); + await waitForSsh(vmIp, SSH_USER, SSH_TIMEOUT_MS, sshKeyPath); + }, DISCOVERY_TIMEOUT_MS + INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 600_000); + + afterAll(async () => { await stopHarness(VM_NAME, harness); }); + + it("machine reached installed state", async () => { + const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`); + const data = (await res.json()) as { installed: Record }; + expect(data.installed[harness!.vmMac]?.hostname).toBe(VM_NAME); + }); + + it("installed system is aarch64", () => { + const result = sshExec(vmIp, SSH_USER, "uname -m", { keyPath: sshKeyPath, timeout: 60_000 }); + expect(result.stdout.trim()).toBe("aarch64"); + }); + + it("SSH works with the admin user", () => { + const result = sshExec(vmIp, SSH_USER, "whoami", { keyPath: sshKeyPath, timeout: 60_000 }); + expect(result.stdout.trim()).toBe(SSH_USER); + }); + + it("LVM layout is correct", () => { + const result = sshExec(vmIp, SSH_USER, "sudo lvs labvg --noheadings -o lv_name", { + keyPath: sshKeyPath, timeout: 60_000, + }); + expect(result.exitCode).toBe(0); + const lvs = result.stdout.trim().split("\n").map((l) => l.trim()); + for (const expected of ["root", "var", "varlog", "swap", "home", "srv"]) { + expect(lvs).toContain(expected); + } + }); +});