michal/lab
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases Wiki Activity

feat(bastion): first-class aarch64 support in the network-PXE path #17

Open
michal wants to merge 11 commits from feat/arm64-pxe-support into main
pull from: feat/arm64-pxe-support
merge into: michal:main
Conversation 0 Commits 11 Files Changed 25 +568 -22
7 changed files with 568 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 3fab400a96 - Show all commits

4
bastion/package.json
View File

@@ -23,6 +23,10 @@
"test:integration:iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'",
"test:integration:arm-iso": "vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'",
"test:integration:arm-iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'",
"test:integration:arm-pxe": "vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE rescue'",
"test:integration:arm-pxe:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE rescue'",
"test:integration:arm-pxe-full": "ARM_PXE_FULL=1 vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE'",
"test:integration:arm-pxe-full:host": "sudo -E ARM_PXE_FULL=1 $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE'",
"test:integration:asahi": "vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'",
"test:integration:asahi:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'",
"test:integration:asahi-validate": "vitest run -c tests/integration/vitest.config.ts -t 'asahi.*validation'",

71
bastion/scripts/test-provision.sh
View File

@@ -2,16 +2,18 @@
# Run PXE and/or ISO boot integration tests.
#
# Usage:
# sudo ./scripts/test-provision.sh # run PXE + ISO (x86_64)
# sudo ./scripts/test-provision.sh pxe # PXE only
# sudo ./scripts/test-provision.sh iso # ISO only (x86_64)
# sudo ./scripts/test-provision.sh arm # ARM ISO boot (emulated, SLOW ~60min)
# sudo ./scripts/test-provision.sh all # all tests including ARM
# sudo ./scripts/test-provision.sh # run PXE + ISO (x86_64)
# sudo ./scripts/test-provision.sh pxe # PXE only
# sudo ./scripts/test-provision.sh iso # ISO only (x86_64)
# sudo ./scripts/test-provision.sh arm # ARM ISO boot (emulated, SLOW ~60min)
# sudo ./scripts/test-provision.sh arm-pxe # ARM network PXE: NBP + discover + rescue (~20-30min)
# sudo ./scripts/test-provision.sh arm-pxe-full # ARM network PXE incl. full install (~60-90min)
# sudo ./scripts/test-provision.sh all # all tests including ARM
#
# Prerequisites:
# libvirtd, OVMF (edk2-ovmf), iPXE (ipxe-bootimgs-x86),
# dnsmasq, xorriso, mtools, virt-install, qemu-img
# ARM: qemu-system-aarch64, edk2-aarch64
# ARM: qemu-system-aarch64, edk2-aarch64, ipxe-bootimgs-aarch64
set -e
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
@@ -58,6 +60,10 @@ if [ ! -f /usr/share/edk2/ovmf/OVMF_CODE.fd ]; then
exit 1
fi
MODE="${1:-both}"
# iPXE binaries are per-architecture. x86_64 is always required (the dnsmasq config
# references it); arm64 only for the ARM network-PXE modes.
IPXE_EFI=""
for f in /usr/share/ipxe/ipxe-snponly-x86_64.efi /usr/share/ipxe/ipxe-snp-x86_64.efi /usr/share/ipxe/ipxe-x86_64.efi; do
[ -f "$f" ] && IPXE_EFI="$f" && break
@@ -67,6 +73,20 @@ if [ -z "$IPXE_EFI" ]; then
exit 1
fi
IPXE_EFI_ARM64=""
for f in /usr/share/ipxe/arm64-efi/snponly.efi /usr/share/ipxe/arm64-efi/ipxe.efi; do
[ -f "$f" ] && IPXE_EFI_ARM64="$f" && break
done
case "$MODE" in
arm-pxe|arm-pxe-full|all)
if [ -z "$IPXE_EFI_ARM64" ] && [ "$MODE" != "all" ]; then
echo -e "${RED}arm64 iPXE binary not found.${RESET} Install: sudo dnf install ipxe-bootimgs-aarch64"
exit 1
fi
;;
esac
# Find SSH key
SSH_KEY=""
for name in id_ed25519 id_ecdsa id_rsa; do
@@ -83,10 +103,19 @@ fi
echo -e " User: ${BOLD}$REAL_USER${RESET}"
echo -e " SSH key: ${BOLD}$SSH_KEY${RESET}"
echo -e " iPXE: ${BOLD}$IPXE_EFI${RESET}"
echo -e " iPXE a64:${BOLD} ${IPXE_EFI_ARM64:-not installed}${RESET}"
echo ""
# --- Determine which tests to run ---
MODE="${1:-both}"
require_arm_emulation() {
if ! command -v qemu-system-aarch64 &>/dev/null; then
echo -e "${RED}qemu-system-aarch64 not found.${RESET} Install: sudo dnf install qemu-system-aarch64 edk2-aarch64"
exit 1
fi
if [ ! -f /usr/share/edk2/aarch64/QEMU_EFI.fd ]; then
echo -e "${RED}AAVMF firmware not found.${RESET} Install: sudo dnf install edk2-aarch64"
exit 1
fi
}
run_test() {
local name="$1" pattern="$2"
@@ -116,13 +145,22 @@ case "$MODE" in
run_test "ISO boot" "ISO boot" || FAILED=1
;;
arm|arm-iso)
if ! command -v qemu-system-aarch64 &>/dev/null; then
echo -e "${RED}qemu-system-aarch64 not found.${RESET} Install: sudo dnf install qemu-system-aarch64 edk2-aarch64"
exit 1
fi
require_arm_emulation
echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 30-60 minutes.${RESET}"
run_test "ARM ISO boot" "ARM ISO" || FAILED=1
;;
arm-pxe)
require_arm_emulation
echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 20-30 minutes.${RESET}"
echo -e "${YELLOW}Covers option 93 -> arm64 NBP, discovery, and rescue over SSH.${RESET}"
echo -e "${YELLOW}For the full install too, use: $0 arm-pxe-full${RESET}"
run_test "ARM PXE rescue" "ARM PXE rescue" || FAILED=1
;;
arm-pxe-full)
require_arm_emulation
echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 60-90 minutes.${RESET}"
ARM_PXE_FULL=1 run_test "ARM PXE (rescue + install)" "ARM PXE" || FAILED=1
;;
both)
run_test "PXE boot" "PXE boot" || FAILED=1
run_test "ISO boot" "ISO boot" || FAILED=1
@@ -133,12 +171,17 @@ case "$MODE" in
if command -v qemu-system-aarch64 &>/dev/null; then
echo -e "${YELLOW}ARM emulation is ~10x slower than native.${RESET}"
run_test "ARM ISO boot" "ARM ISO" || FAILED=1
if [ -n "$IPXE_EFI_ARM64" ]; then
run_test "ARM PXE rescue" "ARM PXE rescue" || FAILED=1
else
echo -e "${YELLOW}Skipping ARM PXE test (ipxe-bootimgs-aarch64 not installed)${RESET}"
fi
else
echo -e "${YELLOW}Skipping ARM test (qemu-system-aarch64 not installed)${RESET}"
echo -e "${YELLOW}Skipping ARM tests (qemu-system-aarch64 not installed)${RESET}"
fi
;;
*)
echo "Usage: $0 [pxe|iso|arm|both|all]"
echo "Usage: $0 [pxe|iso|arm|arm-pxe|arm-pxe-full|both|all]"
exit 1
;;
esac

9
bastion/src/bastion/src/routes/dispatch.ts
View File

@@ -162,9 +162,10 @@ echo "==============================="
state.update((s) => { delete s.debug[mac]; });
let script: string;
const root = debugEntry.pxeBoot ? resolveRoot(currentState, mac) : null;
const wantsPxeBoot = debugEntry.pxeBoot === true;
const root = wantsPxeBoot ? resolveRoot(currentState, mac) : null;
if (debugEntry.pxeBoot && root) {
if (root !== null) {
logger.info(`PXE BOOT DEBUG: ${mac} -> ${hostname} (${arch}, root=${root.rootDevice})`);
script = renderPxeBootDebugIpxe({
mac,
@@ -178,7 +179,7 @@ echo "==============================="
// --pxe-boot without a known root device falls back to rescue rather than
// guessing. A wrong root= leaves the machine unbootable, and rescue is where
// the operator can find the real one (curl /debug-setup.sh reports it back).
const notice = debugEntry.pxeBoot
const notice = wantsPxeBoot
? [
"",
"NOTE: --pxe-boot requested, but no root device is recorded",
@@ -188,7 +189,7 @@ echo "==============================="
" then retry --pxe-boot.",
]
: undefined;
if (debugEntry.pxeBoot) {
if (wantsPxeBoot) {
logger.warn(`PXE BOOT DEBUG: ${mac} -> ${hostname} has no recorded root device -- serving rescue instead`);
} else {
logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (${arch}, rescue mode)`);

10
bastion/src/bastion/src/services/install-guard.ts
View File

@@ -7,15 +7,21 @@
// Rescue/debug is deliberately NOT guarded. Being unable to reinstall a machine is
// exactly when you most need to boot it into a rescue shell.
import type { BastionState, OsId } from "@lab/shared";
import type { Arch, BastionState, OsId } from "@lab/shared";
import { classifyOnboard, normalizeArch, osSupportsArch, vendorOsDescription, archesForOs } from "@lab/shared";
export type InstallCheck =
| { allowed: true }
| { allowed: false; error: string };
interface MachineIdentity {
hostname: string;
arch: Arch | undefined;
identity: Parameters<typeof classifyOnboard>[0];
}
/** Best-known identity for a MAC, merged across the three state maps. */
function identify(state: BastionState, mac: string) {
function identify(state: BastionState, mac: string): MachineIdentity {
const discovered = state.discovered[mac];
const installed = state.installed[mac];
const queued = state.install_queue[mac];

2
bastion/src/cli/src/commands/debug.ts
View File

@@ -132,7 +132,7 @@ export function registerDebugCommand(parent: Command): void {
const probe = sshCapture(ip, ROOT_DEVICE_PROBE);
const root = probe === null ? {} : parseRootProbe(probe);
if (root.root_device !== undefined) {
console.log(` root=${root.root_device}${root.root_args ? ` ${root.root_args}` : ""}`);
console.log(` root=${root.root_device}${root.root_args !== undefined ? ` ${root.root_args}` : ""}`);
try {
await client.discoverMachine({ mac, ...root });
} catch (err) {

4
bastion/src/cli/src/commands/recheck.ts
View File

@@ -84,7 +84,9 @@ export function registerRecheckCommand(parent: Command): void {
const cpu = hwData.cpu_model || "?";
const cores = hwData.cpu_cores || "?";
const mem = hwData.memory_gb || "?";
const root = hwData.root_device ? `, root=${hwData.root_device}` : "";
const root = typeof hwData.root_device === "string" && hwData.root_device !== ""
? `, root=${hwData.root_device}`
: "";
console.log(`${GREEN}OK${RESET} ${DIM}${cpu}, ${cores} cores, ${mem}GB${root}${RESET}`);
updated++;
} catch (err) {

490
bastion/tests/integration/arm-pxe-provision.test.ts Normal file
View File

@@ -0,0 +1,490 @@
// Integration test: aarch64 network PXE boot.
//
// The boot-ISO path already covered ARM (arm-iso-provision.test.ts). This covers the
// network path: DHCP option 93 handing an arm64 client an arm64 iPXE binary, dispatch
// serving an aarch64 kernel, and `provision debug` reaching a rescue shell -- which is
// what the DGX Sparks actually need and could not do.
//
// Two suites, because they cost very different amounts of time:
//
// "ARM PXE rescue" NBP handoff -> discovery -> rescue with SSH. ~20-30 min
// "ARM PXE install" full discover -> install -> installed. ~60-90 min
//
// The install suite only runs with ARM_PXE_FULL=1. No ARM machine in the lab is ever
// PXE-installed except the MS-R1, and an hour-plus test that runs by default is a test
// nobody runs.
//
// IMPORTANT: aarch64 has no KVM on an x86_64 host, so all of this is emulated and
// roughly 10x slower than native.
//
// A note for whoever debugs a failure here: if the VM panics with
// VFS: Unable to mount root fs on unknown-block(0,0)
// that is very likely iPXE silently dropping the initrd because the build lacks
// EFI_LOAD_FILE2_PROTOCOL -- on arm64 the kernel EFI stub fetches the initrd over
// LoadFile2, and an iPXE without it accepts the `initrd` line and does nothing. It is
// NOT a reproduction of the DGX Spark kernel bug that motivated this work, despite
// being the identical message. assertIpxeSupportsLoadFile2() below checks the build up
// front so that failure names itself; to check by hand:
// node -e 'const b=require("fs").readFileSync("/usr/share/ipxe/arm64-efi/snponly.efi");
// console.log(b.indexOf(Buffer.from("c1c00640b3fc3e40996d4a6c8724e06d","hex")))'
// Fedora's ipxe-bootimgs-aarch64-20240119 has it at 0x3bbf0.
//
// Prerequisites:
// - qemu-system-aarch64 (sudo dnf install qemu-system-aarch64)
// - edk2-aarch64 (sudo dnf install edk2-aarch64)
// - ipxe-bootimgs-aarch64 (sudo dnf install ipxe-bootimgs-aarch64)
// - libvirtd, sudo, internet access
//
// Run: sudo ./scripts/test-provision.sh arm-pxe
import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, writeFileSync } from "node:fs";
import { execSync } from "node:child_process";
import { join } from "node:path";
import { homedir, tmpdir } from "node:os";
import { log, waitForSsh } from "./helpers/libvirt.js";
import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js";
import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./helpers/pxe-vm.js";
import { sshExec } from "./helpers/ssh.js";
const IPXE_ARM64 = "/usr/share/ipxe/arm64-efi/snponly.efi";
const AAVMF = "/usr/share/edk2/aarch64/QEMU_EFI.fd";
const VM_MEMORY = 4096;
const VM_VCPUS = 2;
const VM_DISK_GB = 250;
const SSH_USER = "lab";
const BASTION_IP = PXE_GATEWAY;
const DHCP_RANGE_START = `${PXE_SUBNET}.100`;
const DHCP_RANGE_END = `${PXE_SUBNET}.200`;
const SERIAL_PORT = 4555;
// Emulated aarch64 -- generous timeouts throughout.
const DISCOVERY_TIMEOUT_MS = 20 * 60_000;
const RESCUE_TIMEOUT_MS = 20 * 60_000;
const INSTALL_TIMEOUT_MS = 75 * 60_000;
const SSH_TIMEOUT_MS = 20 * 60_000;
const RUN_FULL_INSTALL = process.env["ARM_PXE_FULL"] === "1";
function sleep(ms: number): Promise<void> {
return new Promise((r) => setTimeout(r, ms));
}
function findSshKey(): { pubKey: string; keyPath: string } {
const candidates: string[] = [];
if (process.env["SSH_KEY_PATH"]) candidates.push(process.env["SSH_KEY_PATH"]);
const homes = [homedir()];
const sudoUser = process.env["SUDO_USER"];
if (sudoUser) homes.push(join("/home", sudoUser));
for (const home of homes) {
for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
candidates.push(join(home, ".ssh", name));
}
}
for (const keyPath of candidates) {
if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
}
}
throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/");
}
async function pollApi<T>(
url: string,
check: (data: T) => boolean,
timeoutMs: number,
intervalMs = 10_000,
): Promise<T> {
const start = Date.now();
while (Date.now() - start < timeoutMs) {
try {
const res = await fetch(url);
if (res.ok) {
const data = (await res.json()) as T;
if (check(data)) return data;
}
} catch { /* bastion not up yet, or a network hiccup */ }
await sleep(intervalMs);
}
throw new Error(`Timeout after ${timeoutMs}ms polling ${url}`);
}
function requirePrerequisites(): void {
if (!existsSync("/usr/bin/qemu-system-aarch64")) {
throw new Error("qemu-system-aarch64 not installed. Run: sudo dnf install qemu-system-aarch64");
}
if (!existsSync(AAVMF)) {
throw new Error(`AAVMF firmware not found at ${AAVMF}. Run: sudo dnf install edk2-aarch64`);
}
if (!existsSync(IPXE_ARM64)) {
throw new Error(`arm64 iPXE not found at ${IPXE_ARM64}. Run: sudo dnf install ipxe-bootimgs-aarch64`);
}
}
/**
* Confirm the arm64 iPXE binary implements EFI_LOAD_FILE2_PROTOCOL.
*
* Without it the `initrd` line is accepted and silently ignored, and the kernel panics
* with unknown-block(0,0). Checking here turns a confusing 30-minute boot failure into
* an immediate, explanatory one.
*
* GUID 4006c0c1-fcb3-403e-996d-4a6c8724e06d, little-endian in the binary's GUID table.
*/
function assertIpxeSupportsLoadFile2(): void {
const LOAD_FILE2_GUID = Buffer.from("c1c00640b3fc3e40996d4a6c8724e06d", "hex");
const binary = readFileSync(IPXE_ARM64);
if (binary.indexOf(LOAD_FILE2_GUID) < 0) {
throw new Error(
`${IPXE_ARM64} does not reference EFI_LOAD_FILE2_PROTOCOL. On arm64 the kernel ` +
`EFI stub fetches the initrd over LoadFile2; without it iPXE drops the initrd ` +
`silently and the kernel panics with "unknown-block(0,0)". Rebuild iPXE with ` +
`LoadFile2, or chainload grubaa64.efi for aarch64 instead.`,
);
}
log(`iPXE arm64 implements LoadFile2 — initrd will be delivered to the EFI stub`);
}
interface Harness {
testDir: string;
app: { close: () => Promise<void> };
stopDnsmasq: () => void;
vmMac: string;
httpPort: number;
}
/** Bring up an isolated network, a bastion with both arch payloads, and an arm64 VM. */
async function startHarness(vmName: string, httpPort: number, pubKey: string): Promise<Harness> {
requirePrerequisites();
assertIpxeSupportsLoadFile2();
log("Setting up PXE test network...");
ensurePxeNetwork();
const testDir = join(tmpdir(), `lab-arm-pxe-test-${Date.now()}`);
for (const sub of ["tftp", "http", "logs"]) {
mkdirSync(join(testDir, sub), { recursive: true });
}
const { createApp } = await import("../../src/bastion/src/server.js");
const { loadConfig } = await import("../../src/bastion/src/config.js");
const { generateDnsmasqConf, startDnsmasq, stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
const { generateDiscoverKickstart } = await import("../../src/bastion/src/services/kickstart-generator.js");
const { renderBootIpxe } = await import("../../src/bastion/src/templates/boot.ipxe.js");
const { kernelPath, initrdPath } = await import("../../src/bastion/src/templates/boot.ipxe.js");
const { SUPPORTED_ARCHES, fedoraMirrorFor } = await import("@lab/shared");
const config = loadConfig({
bastionDir: testDir,
httpPort,
iface: "virbr-pxe",
serverIp: BASTION_IP,
network: `${PXE_SUBNET}.0`,
gateway: BASTION_IP,
dhcpMode: "full",
dhcpRangeStart: DHCP_RANGE_START,
dhcpRangeEnd: DHCP_RANGE_END,
domain: "arm-pxe-test.local",
sshKeys: [pubKey],
adminUser: SSH_USER,
});
// iPXE binaries. The arm64 one is the whole point: dnsmasq hands it out on DHCP
// option 93 = 11, and x86_64 is staged too so the config renders as it does in
// production.
log("Staging iPXE binaries...");
copyFileSync(IPXE_ARM64, join(config.tftpDir, "ipxe-arm64.efi"));
const ipxeX86 = "/usr/share/ipxe/ipxe-snponly-x86_64.efi";
if (existsSync(ipxeX86)) copyFileSync(ipxeX86, join(config.tftpDir, "ipxe.efi"));
// Fedora kernel + initrd for both architectures, cached across runs.
const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache";
execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" });
for (const arch of SUPPORTED_ARCHES) {
const mirror = fedoraMirrorFor(config.fedoraVersion, arch);
const kernelCache = join(cacheDir, `vmlinuz-${arch}`);
const initrdCache = join(cacheDir, `initrd-${arch}.img`);
if (!existsSync(kernelCache)) {
log(`Downloading Fedora ${config.fedoraVersion} ${arch} kernel...`);
execSync(`curl -# -L -f -o "${kernelCache}" "${mirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 600_000 });
}
if (!existsSync(initrdCache)) {
log(`Downloading Fedora ${config.fedoraVersion} ${arch} initrd...`);
execSync(`curl -# -L -f -o "${initrdCache}" "${mirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 600_000 });
}
// Staged under the exact names the iPXE templates will ask for.
copyFileSync(kernelCache, join(config.httpDir, kernelPath(arch)));
copyFileSync(initrdCache, join(config.httpDir, initrdPath(arch)));
log(`Staged ${arch}: ${kernelPath(arch)} + ${initrdPath(arch)}`);
}
writeFileSync(join(config.httpDir, "discover.ks"), generateDiscoverKickstart(config));
writeFileSync(
join(config.httpDir, "boot.ipxe"),
renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }),
);
generateDnsmasqConf(config);
const { app, syslog } = createApp(config);
await app.listen({ port: config.httpPort, host: "0.0.0.0" });
syslog.start();
log(`Bastion HTTP listening on :${config.httpPort}`);
log("Starting dnsmasq (full DHCP)...");
startDnsmasq(config).catch((err) => {
log(`dnsmasq failed: ${err instanceof Error ? err.message : String(err)}`);
});
await sleep(1500);
log("Creating aarch64 PXE VM (emulated — this is slow)...");
createPxeVm({
name: vmName,
memory: VM_MEMORY,
vcpus: VM_VCPUS,
diskSize: VM_DISK_GB,
network: PXE_NETWORK_NAME,
arch: "aarch64",
});
const vmMac = getVmMac(vmName);
if (!vmMac) throw new Error("Could not determine VM MAC address");
log(`ARM VM MAC: ${vmMac}`);
return { testDir, app, stopDnsmasq, vmMac, httpPort: config.httpPort };
}
async function stopHarness(vmName: string, harness: Harness | undefined): Promise<void> {
log("Cleaning up...");
if (harness) {
await harness.app.close().catch(() => {});
harness.stopDnsmasq();
}
destroyPxeVm(vmName);
destroyPxeNetwork();
if (harness) rmSync(harness.testDir, { recursive: true, force: true });
}
/** Read the DHCP lease the bastion handed a MAC. Rescue mode reports no IP itself. */
function leaseIpFor(testDir: string, mac: string): string | null {
const leaseFile = join(testDir, "dnsmasq.leases");
if (!existsSync(leaseFile)) return null;
for (const line of readFileSync(leaseFile, "utf-8").split("\n")) {
// <expiry> <mac> <ip> <hostname> <clientid>
const parts = line.trim().split(/\s+/);
if (parts.length >= 3 && parts[1]?.toLowerCase() === mac.toLowerCase()) {
return parts[2] ?? null;
}
}
return null;
}
async function waitForLease(testDir: string, mac: string, timeoutMs: number): Promise<string> {
const start = Date.now();
while (Date.now() - start < timeoutMs) {
const ip = leaseIpFor(testDir, mac);
if (ip !== null) return ip;
await sleep(5000);
}
throw new Error(`No DHCP lease for ${mac} within ${timeoutMs}ms`);
}
// ---------------------------------------------------------------------------
// Rescue path -- what the DGX Sparks need.
// ---------------------------------------------------------------------------
describe("ARM PXE rescue", () => {
const VM_NAME = "lab-arm-pxe-rescue";
const HTTP_PORT = 8096;
let harness: Harness | undefined;
let sshKeyPath: string;
let rescueIp: string;
beforeAll(async () => {
const { pubKey, keyPath } = findSshKey();
sshKeyPath = keyPath;
harness = await startHarness(VM_NAME, HTTP_PORT, pubKey);
const { testDir, vmMac } = harness;
// 1. The VM PXE boots. This exercises the whole chain: DHCP option 93 -> arm64
// iPXE -> /boot.ipxe -> /dispatch?arch=arm64 -> aarch64 kernel + initrd.
// Reaching discovery at all proves the initrd was delivered over LoadFile2 --
// without it the kernel panics before it can run the kickstart.
log("Waiting for aarch64 VM to PXE boot and report discovery...");
log("(emulated aarch64 — be patient)");
type MachinesResponse = { discovered: Record<string, { arch: string }> };
const discovered = await pollApi<MachinesResponse>(
`http://${BASTION_IP}:${HTTP_PORT}/api/machines`,
(data) => vmMac in data.discovered,
DISCOVERY_TIMEOUT_MS,
).catch(async (err) => {
log("Discovery timed out. Serial console:");
try {
log(await readSerialLog(SERIAL_PORT, { lastLines: 100, timeoutMs: 15_000 }));
} catch { /* console unavailable */ }
throw err;
});
log(`aarch64 VM discovered as ${discovered.discovered[vmMac]?.arch}`);
// 2. Queue rescue, exactly as `labctl provision debug` does.
log("Queueing debug/rescue mode...");
const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/debug`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ mac: vmMac }),
});
expect(res.ok).toBe(true);
// 3. Reboot into rescue. Rescue mode runs no %pre/%post, so the machine never
// reports its own IP — read it from the DHCP lease instead.
await sleep(20_000);
rebootPxeVm(VM_NAME);
await sleep(5_000);
deleteNftablesRejectRules();
log("Waiting for rescue environment DHCP lease...");
rescueIp = await waitForLease(testDir, vmMac, RESCUE_TIMEOUT_MS);
log(`Rescue IP: ${rescueIp}`);
log("Waiting for SSH into the rescue shell (started by inst.sshd)...");
await waitForSsh(rescueIp, "root", SSH_TIMEOUT_MS, sshKeyPath).catch(async (err) => {
log("Rescue SSH timed out. Serial console:");
try {
log(await readSerialLog(SERIAL_PORT, { lastLines: 100, timeoutMs: 15_000 }));
} catch { /* console unavailable */ }
throw err;
});
log("ARM PXE rescue reached.");
}, DISCOVERY_TIMEOUT_MS + RESCUE_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000);
afterAll(async () => { await stopHarness(VM_NAME, harness); });
it("discovered the machine as aarch64", async () => {
const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`);
const data = (await res.json()) as { discovered: Record<string, { arch: string }> };
expect(data.discovered[harness!.vmMac]?.arch).toBe("aarch64");
});
it("rescue shell is reachable over SSH and is aarch64", () => {
const result = sshExec(rescueIp, "root", "uname -m", { keyPath: sshKeyPath, timeout: 60_000 });
expect(result.exitCode).toBe(0);
expect(result.stdout.trim()).toBe("aarch64");
});
it("booted an initramfs — the LoadFile2 path worked", () => {
// If iPXE had dropped the initrd the kernel would never have reached userspace at
// all, but assert it explicitly so a regression names itself.
const result = sshExec(rescueIp, "root", "cat /proc/cmdline; ls /run/install", {
keyPath: sshKeyPath, timeout: 60_000,
});
expect(result.exitCode).toBe(0);
expect(result.stdout).toContain("inst.rescue");
});
it("rescue kernel came from the bastion over HTTP", () => {
const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 });
expect(result.stdout).toContain(`${BASTION_IP}:${HTTP_PORT}`);
// arm64 gets serial console arguments, never nomodeset.
expect(result.stdout).toContain("console=ttyAMA0");
expect(result.stdout).not.toContain("nomodeset");
});
it("has LVM tools available for inspecting an installed system", () => {
const result = sshExec(rescueIp, "root", "command -v vgchange && command -v lsblk", {
keyPath: sshKeyPath, timeout: 60_000,
});
expect(result.exitCode).toBe(0);
});
});
// ---------------------------------------------------------------------------
// Full install -- opt-in, ~60-90 minutes emulated.
// ---------------------------------------------------------------------------
describe.runIf(RUN_FULL_INSTALL)("ARM PXE install", () => {
const VM_NAME = "lab-arm-pxe-install";
const HTTP_PORT = 8095;
let harness: Harness | undefined;
let sshKeyPath: string;
let vmIp: string;
beforeAll(async () => {
const { pubKey, keyPath } = findSshKey();
sshKeyPath = keyPath;
harness = await startHarness(VM_NAME, HTTP_PORT, pubKey);
const { vmMac } = harness;
log("Waiting for aarch64 discovery...");
await pollApi<{ discovered: Record<string, unknown> }>(
`http://${BASTION_IP}:${HTTP_PORT}/api/machines`,
(data) => vmMac in data.discovered,
DISCOVERY_TIMEOUT_MS,
);
log("Discovered. Queueing install...");
const installRes = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/install`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ mac: vmMac, hostname: VM_NAME, disk: "", role: "vanilla" }),
});
expect(installRes.status).toBe(200);
await sleep(30_000);
rebootPxeVm(VM_NAME);
log("Waiting for the emulated aarch64 install (60-90 min)...");
type LogsResponse = { status: string; progress: string; ip?: string };
const final = await pollApi<LogsResponse>(
`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`,
(d) => d.status === "installed" || d.progress === "error",
INSTALL_TIMEOUT_MS,
30_000,
);
if (final.progress === "error") {
const logs = await (await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`)).json();
log(`ARM install FAILED: ${JSON.stringify(logs, null, 2)}`);
throw new Error("ARM PXE install failed — see logs above");
}
vmIp = final.ip ?? "";
log(`ARM install complete. IP: ${vmIp}`);
await sleep(30_000);
rebootPxeVm(VM_NAME);
await sleep(5_000);
deleteNftablesRejectRules();
await waitForSsh(vmIp, SSH_USER, SSH_TIMEOUT_MS, sshKeyPath);
}, DISCOVERY_TIMEOUT_MS + INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 600_000);
afterAll(async () => { await stopHarness(VM_NAME, harness); });
it("machine reached installed state", async () => {
const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`);
const data = (await res.json()) as { installed: Record<string, { hostname: string }> };
expect(data.installed[harness!.vmMac]?.hostname).toBe(VM_NAME);
});
it("installed system is aarch64", () => {
const result = sshExec(vmIp, SSH_USER, "uname -m", { keyPath: sshKeyPath, timeout: 60_000 });
expect(result.stdout.trim()).toBe("aarch64");
});
it("SSH works with the admin user", () => {
const result = sshExec(vmIp, SSH_USER, "whoami", { keyPath: sshKeyPath, timeout: 60_000 });
expect(result.stdout.trim()).toBe(SSH_USER);
});
it("LVM layout is correct", () => {
const result = sshExec(vmIp, SSH_USER, "sudo lvs labvg --noheadings -o lv_name", {
keyPath: sshKeyPath, timeout: 60_000,
});
expect(result.exitCode).toBe(0);
const lvs = result.stdout.trim().split("\n").map((l) => l.trim());
for (const expected of ["root", "var", "varlog", "swap", "home", "srv"]) {
expect(lvs).toContain(expected);
}
});
});