// Integration test: `labctl provision debug` -> Anaconda rescue with SSH, on x86_64. // // The rescue path had no test coverage on any architecture, which matters because it is // the lab's recovery tool of last resort -- the thing you reach for when a machine will // not boot. It runs here on x86_64 with KVM so it completes in minutes; the aarch64 // equivalent is the same code path with a different kernel, but is emulated and far too // slow to iterate on. // // Run: sudo ./scripts/test-provision.sh rescue import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, writeFileSync } from "node:fs"; import { execSync } from "node:child_process"; import { join } from "node:path"; import { homedir, tmpdir } from "node:os"; import { log, waitForSsh } from "./helpers/libvirt.js"; import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js"; import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./helpers/pxe-vm.js"; import { sshExec } from "./helpers/ssh.js"; const VM_NAME = "lab-pxe-rescue-test"; const HTTP_PORT = 8094; const VM_MEMORY = 4096; const VM_VCPUS = 4; const VM_DISK_GB = 20; const BASTION_IP = PXE_GATEWAY; const SERIAL_PORT = 4555; const LEASE_TIMEOUT_MS = 8 * 60_000; const SSH_TIMEOUT_MS = 15 * 60_000; function sleep(ms: number): Promise { return new Promise((r) => setTimeout(r, ms)); } function findSshKey(): { pubKey: string; keyPath: string } { const candidates: string[] = []; if (process.env["SSH_KEY_PATH"]) candidates.push(process.env["SSH_KEY_PATH"]); const homes = [homedir()]; const sudoUser = process.env["SUDO_USER"]; if (sudoUser) homes.push(join("/home", sudoUser)); for (const home of homes) { for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) candidates.push(join(home, ".ssh", name)); } for (const keyPath of candidates) { if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) { return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath }; } } throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/"); } function leaseIpFor(testDir: string, mac: string): string | null { const leaseFile = join(testDir, "dnsmasq.leases"); if (!existsSync(leaseFile)) return null; for (const line of readFileSync(leaseFile, "utf-8").split("\n")) { const parts = line.trim().split(/\s+/); if (parts.length >= 3 && parts[1]?.toLowerCase() === mac.toLowerCase()) return parts[2] ?? null; } return null; } async function waitForLease(testDir: string, mac: string, timeoutMs: number): Promise { const start = Date.now(); while (Date.now() - start < timeoutMs) { const ip = leaseIpFor(testDir, mac); if (ip !== null) return ip; await sleep(5000); } throw new Error(`No DHCP lease for ${mac} within ${timeoutMs}ms`); } // Suite name must not be a substring of "ARM PXE rescue" -- vitest -t matches // substrings, so a looser name here would drag the emulated aarch64 suite in with it. describe("x86 rescue boot", () => { let app: { close: () => Promise }; let stopDnsmasqFn: () => void; let testDir: string; let vmMac: string; let rescueIp: string; let sshKeyPath: string; beforeAll(async () => { const { pubKey, keyPath } = findSshKey(); sshKeyPath = keyPath; log("Setting up PXE test network..."); ensurePxeNetwork(); testDir = join(tmpdir(), `lab-pxe-rescue-${Date.now()}`); for (const sub of ["tftp", "http", "logs"]) mkdirSync(join(testDir, sub), { recursive: true }); const { createApp } = await import("../../src/bastion/src/server.js"); const { loadConfig } = await import("../../src/bastion/src/config.js"); const { generateDnsmasqConf, startDnsmasq, stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js"); const { renderBootIpxe, kernelPath, initrdPath } = await import("../../src/bastion/src/templates/boot.ipxe.js"); stopDnsmasqFn = stopDnsmasq; const config = loadConfig({ bastionDir: testDir, httpPort: HTTP_PORT, iface: "virbr-pxe", serverIp: BASTION_IP, network: `${PXE_SUBNET}.0`, gateway: BASTION_IP, dhcpMode: "full", dhcpRangeStart: `${PXE_SUBNET}.100`, dhcpRangeEnd: `${PXE_SUBNET}.200`, domain: "rescue-test.local", sshKeys: [pubKey], adminUser: "lab", }); // iPXE in both dirs: TFTP PXE and UEFI HTTP Boot are both possible, and OVMF picks. const ipxeX86 = "/usr/share/ipxe/ipxe-snponly-x86_64.efi"; if (!existsSync(ipxeX86)) throw new Error(`iPXE not found: ${ipxeX86}`); copyFileSync(ipxeX86, join(config.tftpDir, "ipxe.efi")); copyFileSync(ipxeX86, join(config.httpDir, "ipxe.efi")); const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache"; execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" }); const kernelCache = join(cacheDir, "vmlinuz-x86_64"); const initrdCache = join(cacheDir, "initrd-x86_64.img"); if (!existsSync(kernelCache)) { log("Downloading Fedora x86_64 kernel..."); execSync(`curl -# -L -f -o "${kernelCache}" "${config.fedoraMirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 600_000 }); } if (!existsSync(initrdCache)) { log("Downloading Fedora x86_64 initrd..."); execSync(`curl -# -L -f -o "${initrdCache}" "${config.fedoraMirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 600_000 }); } copyFileSync(kernelCache, join(config.httpDir, kernelPath("x86_64"))); copyFileSync(initrdCache, join(config.httpDir, initrdPath("x86_64"))); writeFileSync(join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort })); generateDnsmasqConf(config); const { app: fastify, state, syslog } = createApp(config); app = fastify; await fastify.listen({ port: config.httpPort, host: "0.0.0.0" }); syslog.start(); log(`Bastion HTTP listening on :${HTTP_PORT}`); startDnsmasq(config).catch((err) => log(`dnsmasq failed: ${err instanceof Error ? err.message : String(err)}`)); await sleep(1500); log("Creating x86_64 PXE VM (KVM)..."); createPxeVm({ name: VM_NAME, memory: VM_MEMORY, vcpus: VM_VCPUS, diskSize: VM_DISK_GB, network: PXE_NETWORK_NAME }); const mac = getVmMac(VM_NAME); if (!mac) throw new Error("Could not determine VM MAC"); vmMac = mac; log(`VM MAC: ${vmMac}`); // Queue rescue directly, as `labctl provision debug` does. log("Queueing debug/rescue mode..."); state.update((s) => { s.debug[vmMac] = { hostname: "rescue-test", queued_at: new Date().toISOString() }; }); rebootPxeVm(VM_NAME); await sleep(5_000); deleteNftablesRejectRules(); rescueIp = await waitForLease(testDir, vmMac, LEASE_TIMEOUT_MS); log(`Rescue IP: ${rescueIp}`); log("Waiting for SSH into the rescue shell (inst.sshd)..."); await waitForSsh(rescueIp, "root", SSH_TIMEOUT_MS, sshKeyPath).catch(async (err) => { log("Rescue SSH timed out. Serial console:"); try { log(await readSerialLog(SERIAL_PORT, { lastLines: 120, timeoutMs: 20_000 })); } catch { /* none */ } throw err; }); log("Rescue shell reachable."); }, LEASE_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000); afterAll(async () => { if (process.env["KEEP_VM"] === "1") { log(`KEEP_VM=1 — leaving ${VM_NAME} up (serial: socat - TCP:127.0.0.1:${SERIAL_PORT})`); return; } log("Cleaning up..."); if (app) await app.close().catch(() => {}); if (stopDnsmasqFn) stopDnsmasqFn(); destroyPxeVm(VM_NAME); destroyPxeNetwork(); if (testDir) rmSync(testDir, { recursive: true, force: true }); }); it("rescue shell is reachable over SSH as root", () => { const result = sshExec(rescueIp, "root", "whoami", { keyPath: sshKeyPath, timeout: 60_000 }); expect(result.exitCode).toBe(0); expect(result.stdout.trim()).toBe("root"); }); it("is the Anaconda rescue environment", () => { const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 }); expect(result.stdout).toContain("inst.rescue"); expect(result.stdout).toContain("inst.sshd"); }); it("kernel and initrd came from the bastion", () => { const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 }); expect(result.stdout).toContain(`${BASTION_IP}:${HTTP_PORT}`); }); it("has LVM tools for inspecting an installed system", () => { const result = sshExec(rescueIp, "root", "command -v vgchange && command -v lsblk", { keyPath: sshKeyPath, timeout: 60_000 }); expect(result.exitCode).toBe(0); }); });