From 346bd80c13dae43375905a08830c2da313d4e1c3 Mon Sep 17 00:00:00 2001 From: Michal Date: Tue, 11 Aug 2026 15:25:57 +0100 Subject: [PATCH] test(bastion): cover the rescue boot path and record what it exposed `provision debug` is the lab's recovery tool of last resort and had no integration coverage on any architecture. Adding it -- x86_64 on KVM so it runs in ~15 minutes, plus the aarch64 equivalent -- showed the rescue environment coming up correctly but nothing ever listening on port 22. Reproduced on both architectures, so it is neither ARM-specific nor an emulation artefact, and it is orthogonal to the multi-arch work: x86_64 is unchanged by that. Documented in ARCHITECTURE.md with the leads worth checking, rather than left as a silent gap. Also restructures the ARM rescue suite to seed the machine into state instead of discovering it first. That mirrors the DGX Spark situation -- SSH-onboarded, never PXE-discovered, architecture known only from its record -- and holds the test to one emulated boot, since each spends ~15 of its ~18 minutes fetching Anaconda's stage2 under TCG. KEEP_VM=1 leaves the VM up on failure; half-hour emulated runs are too expensive to pay twice just to see what happened. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015nRFZXpKwUVE4SRSHw6GjF --- bastion/docs/ARCHITECTURE.md | 22 ++ bastion/package.json | 2 + bastion/scripts/test-provision.sh | 17 +- .../integration/arm-pxe-provision.test.ts | 124 +++++++---- bastion/tests/integration/pxe-rescue.test.ts | 210 ++++++++++++++++++ 5 files changed, 326 insertions(+), 49 deletions(-) create mode 100644 bastion/tests/integration/pxe-rescue.test.ts diff --git a/bastion/docs/ARCHITECTURE.md b/bastion/docs/ARCHITECTURE.md index 52963a3..4babe11 100644 --- a/bastion/docs/ARCHITECTURE.md +++ b/bastion/docs/ARCHITECTURE.md @@ -481,6 +481,28 @@ Hardcoded `/dev/sda` default broke NVMe-only machines. Fix: default to empty str ### Anaconda Rescue Mode Limitations `%pre` and `%post` sections do not execute in `inst.rescue` mode. SSH in rescue mode is provided by Anaconda's `inst.sshd` kernel parameter + `sshpw` kickstart directive. Manual setup via `curl bastion:8080/debug-setup.sh | bash` for nc listener. +**Unresolved (2026-08-11): rescue SSH has never been observed working.** Adding the first +integration coverage for `provision debug` (`tests/integration/pxe-rescue.test.ts`) showed the +rescue environment coming up correctly — the bastion serves the kernel and initrd, Anaconda +boots, fetches `debug.ks`, and reaches its installer environment — but **nothing ever listens on +port 22**. Confirmed on aarch64 by probing the port for 30 minutes while the Anaconda environment +was demonstrably running (NetworkManager, polkitd, rsyslog all up), and reproduced on x86_64 with +KVM, so it is not architecture-specific and not an emulation artefact. + +This is orthogonal to the multi-architecture work: the same failure occurs on x86_64, which that +work does not touch. Leads worth checking, in order: +- Does `inst.sshd` actually start `sshd` in `inst.rescue` mode, or only in install mode? The + port never opens, so this is the prime suspect — an auth problem would still show an open port. +- `sshkey` may apply only to the *installed* system, leaving the installer environment + password-only via `sshpw`. That would matter once sshd does listen: the test authenticates + key-only (`BatchMode=yes`). +- The `%anaconda`-context directives in `debug.ks` may be skipped entirely when a kickstart is + supplied alongside `inst.rescue`. + +Until this is resolved, `provision debug` gets you a booted rescue environment on the console +(including on arm64), but not an SSH shell. The `debug-setup.sh` nc-listener path is the +documented workaround and is unaffected. + --- ## Planned Work (Taskmaster) diff --git a/bastion/package.json b/bastion/package.json index 62f2d8e..f733572 100644 --- a/bastion/package.json +++ b/bastion/package.json @@ -23,6 +23,8 @@ "test:integration:iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'", "test:integration:arm-iso": "vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'", "test:integration:arm-iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'", + "test:integration:rescue": "vitest run -c tests/integration/vitest.config.ts -t 'x86 rescue boot'", + "test:integration:rescue:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'x86 rescue boot'", "test:integration:arm-pxe": "vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE rescue'", "test:integration:arm-pxe:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE rescue'", "test:integration:arm-pxe-full": "ARM_PXE_FULL=1 vitest run -c tests/integration/vitest.config.ts -t 'ARM PXE'", diff --git a/bastion/scripts/test-provision.sh b/bastion/scripts/test-provision.sh index c9e372f..7a37e91 100755 --- a/bastion/scripts/test-provision.sh +++ b/bastion/scripts/test-provision.sh @@ -5,9 +5,10 @@ # sudo ./scripts/test-provision.sh # run PXE + ISO (x86_64) # sudo ./scripts/test-provision.sh pxe # PXE only # sudo ./scripts/test-provision.sh iso # ISO only (x86_64) +# sudo ./scripts/test-provision.sh rescue # x86_64 Anaconda rescue boot + SSH (~15min) # sudo ./scripts/test-provision.sh arm # ARM ISO boot (emulated, SLOW ~60min) -# sudo ./scripts/test-provision.sh arm-pxe # ARM network PXE: NBP + discover + rescue (~20-30min) -# sudo ./scripts/test-provision.sh arm-pxe-full # ARM network PXE incl. full install (~60-90min) +# sudo ./scripts/test-provision.sh arm-pxe # ARM network PXE rescue: NBP + rescue over SSH (~25-30min) +# sudo ./scripts/test-provision.sh arm-pxe-full # ARM network PXE incl. discover + full install (~75-95min) # sudo ./scripts/test-provision.sh all # all tests including ARM # # Prerequisites: @@ -149,16 +150,20 @@ case "$MODE" in echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 30-60 minutes.${RESET}" run_test "ARM ISO boot" "ARM ISO" || FAILED=1 ;; + rescue) + echo -e "${YELLOW}x86_64 rescue boot (KVM). Expect ~15 minutes.${RESET}" + run_test "x86 rescue boot" "x86 rescue boot" || FAILED=1 + ;; arm-pxe) require_arm_emulation - echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 20-30 minutes.${RESET}" - echo -e "${YELLOW}Covers option 93 -> arm64 NBP, discovery, and rescue over SSH.${RESET}" + echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 25-30 minutes.${RESET}" + echo -e "${YELLOW}Covers option 93 -> arm64 NBP, arch resolution, and rescue over SSH.${RESET}" echo -e "${YELLOW}For the full install too, use: $0 arm-pxe-full${RESET}" run_test "ARM PXE rescue" "ARM PXE rescue" || FAILED=1 ;; arm-pxe-full) require_arm_emulation - echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 60-90 minutes.${RESET}" + echo -e "${YELLOW}ARM emulation is ~10x slower than native. Expect 75-95 minutes.${RESET}" ARM_PXE_FULL=1 run_test "ARM PXE (rescue + install)" "ARM PXE" || FAILED=1 ;; both) @@ -181,7 +186,7 @@ case "$MODE" in fi ;; *) - echo "Usage: $0 [pxe|iso|arm|arm-pxe|arm-pxe-full|both|all]" + echo "Usage: $0 [pxe|iso|rescue|arm|arm-pxe|arm-pxe-full|both|all]" exit 1 ;; esac diff --git a/bastion/tests/integration/arm-pxe-provision.test.ts b/bastion/tests/integration/arm-pxe-provision.test.ts index 30ef2a5..94a1fe2 100644 --- a/bastion/tests/integration/arm-pxe-provision.test.ts +++ b/bastion/tests/integration/arm-pxe-provision.test.ts @@ -7,8 +7,15 @@ // // Two suites, because they cost very different amounts of time: // -// "ARM PXE rescue" NBP handoff -> discovery -> rescue with SSH. ~20-30 min -// "ARM PXE install" full discover -> install -> installed. ~60-90 min +// "ARM PXE rescue" NBP handoff -> rescue with SSH. ~25-30 min +// "ARM PXE install" discover -> install -> installed. ~75-95 min +// +// The rescue suite seeds the machine into state as an already-known aarch64 box rather +// than discovering it first. That is the DGX Spark situation exactly -- SSH-onboarded, +// never PXE-discovered, architecture known only from its record -- and it holds the test +// to one emulated boot. Each boot spends ~15 of its ~18 minutes downloading Anaconda's +// stage2 under TCG, so discovering first would double the runtime without touching any +// code path the rescue boot does not already exercise. // // The install suite only runs with ARM_PXE_FULL=1. No ARM machine in the lab is ever // PXE-installed except the MS-R1, and an hour-plus test that runs by default is a test @@ -59,11 +66,14 @@ const DHCP_RANGE_START = `${PXE_SUBNET}.100`; const DHCP_RANGE_END = `${PXE_SUBNET}.200`; const SERIAL_PORT = 4555; -// Emulated aarch64 -- generous timeouts throughout. -const DISCOVERY_TIMEOUT_MS = 20 * 60_000; -const RESCUE_TIMEOUT_MS = 20 * 60_000; +// Emulated aarch64 -- generous timeouts throughout. Measured on an x86_64 host with no +// KVM for aarch64: a single PXE boot to a running Anaconda takes ~18 minutes, almost all +// of it downloading inst.stage2 over the network under TCG. Budget well above that; +// timing out just short of success wastes a whole run. +const LEASE_TIMEOUT_MS = 10 * 60_000; +const DISCOVERY_TIMEOUT_MS = 35 * 60_000; const INSTALL_TIMEOUT_MS = 75 * 60_000; -const SSH_TIMEOUT_MS = 20 * 60_000; +const SSH_TIMEOUT_MS = 35 * 60_000; const RUN_FULL_INSTALL = process.env["ARM_PXE_FULL"] === "1"; @@ -149,10 +159,19 @@ interface Harness { testDir: string; app: { close: () => Promise }; stopDnsmasq: () => void; + state: { update: (fn: (s: BastionStateLike) => void) => void }; vmMac: string; httpPort: number; } +/** Just the parts of BastionState this test seeds. */ +interface BastionStateLike { + discovered: Record>; + installed: Record>; + install_queue: Record>; + debug: Record>; +} + /** Bring up an isolated network, a bastion with both arch payloads, and an arm64 VM. */ async function startHarness(vmName: string, httpPort: number, pubKey: string): Promise { requirePrerequisites(); @@ -237,7 +256,7 @@ async function startHarness(vmName: string, httpPort: number, pubKey: string): P ); generateDnsmasqConf(config); - const { app, syslog } = createApp(config); + const { app, state, syslog } = createApp(config); await app.listen({ port: config.httpPort, host: "0.0.0.0" }); syslog.start(); log(`Bastion HTTP listening on :${config.httpPort}`); @@ -262,10 +281,29 @@ async function startHarness(vmName: string, httpPort: number, pubKey: string): P if (!vmMac) throw new Error("Could not determine VM MAC address"); log(`ARM VM MAC: ${vmMac}`); - return { testDir, app, stopDnsmasq, vmMac, httpPort: config.httpPort }; + return { + testDir, + app, + stopDnsmasq, + state: state as unknown as Harness["state"], + vmMac, + httpPort: config.httpPort, + }; } async function stopHarness(vmName: string, harness: Harness | undefined): Promise { + // KEEP_VM=1 leaves the VM, network and bastion up so a failure can be inspected on + // the console. Emulated aarch64 runs cost half an hour; tearing the evidence down + // automatically means paying that again to see what happened. + if (process.env["KEEP_VM"] === "1") { + log(`KEEP_VM=1 — leaving ${vmName} running for inspection.`); + log(` console: sudo virsh screenshot ${vmName} /tmp/vm.ppm`); + log(` serial: socat - TCP:127.0.0.1:${SERIAL_PORT}`); + if (harness) log(` bastion: ${harness.testDir} (still serving on :${harness.httpPort})`); + log(` cleanup: sudo virsh destroy ${vmName}; sudo virsh undefine ${vmName} --remove-all-storage --nvram`); + return; + } + log("Cleaning up..."); if (harness) { await harness.app.close().catch(() => {}); @@ -315,49 +353,49 @@ describe("ARM PXE rescue", () => { const { pubKey, keyPath } = findSshKey(); sshKeyPath = keyPath; harness = await startHarness(VM_NAME, HTTP_PORT, pubKey); - const { testDir, vmMac } = harness; + const { testDir, vmMac, state } = harness; - // 1. The VM PXE boots. This exercises the whole chain: DHCP option 93 -> arm64 - // iPXE -> /boot.ipxe -> /dispatch?arch=arm64 -> aarch64 kernel + initrd. - // Reaching discovery at all proves the initrd was delivered over LoadFile2 -- - // without it the kernel panics before it can run the kickstart. - log("Waiting for aarch64 VM to PXE boot and report discovery..."); - log("(emulated aarch64 — be patient)"); - type MachinesResponse = { discovered: Record }; - const discovered = await pollApi( - `http://${BASTION_IP}:${HTTP_PORT}/api/machines`, - (data) => vmMac in data.discovered, - DISCOVERY_TIMEOUT_MS, - ).catch(async (err) => { - log("Discovery timed out. Serial console:"); - try { - log(await readSerialLog(SERIAL_PORT, { lastLines: 100, timeoutMs: 15_000 })); - } catch { /* console unavailable */ } - throw err; + // Seed the machine as an already-known aarch64 box queued for rescue. This is the + // DGX Spark situation exactly: SSH-onboarded, never PXE-discovered, architecture + // known only from its record -- and it also keeps the test to a SINGLE emulated + // boot. Each boot spends ~15 minutes pulling Anaconda's stage2 over the network + // under TCG, so discovering first and rescuing second doubles the runtime for no + // extra coverage of the path being tested. Discovery is covered by the full suite. + log(`Seeding ${vmMac} as a known aarch64 machine queued for rescue...`); + state.update((s) => { + s.discovered[vmMac] = { + mac: vmMac, + product: "Test ARM64 Machine", + board: "virt", + serial: "SN-ARM64", + manufacturer: "QEMU", + cpu_model: "cortex-a57", + cpu_cores: VM_VCPUS, + memory_gb: 4, + arch: "aarch64", + disks: [], + nics: [], + first_seen: new Date().toISOString(), + last_seen: new Date().toISOString(), + }; + s.debug[vmMac] = { hostname: "arm-rescue-test", queued_at: new Date().toISOString() }; }); - log(`aarch64 VM discovered as ${discovered.discovered[vmMac]?.arch}`); - // 2. Queue rescue, exactly as `labctl provision debug` does. - log("Queueing debug/rescue mode..."); - const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/debug`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ mac: vmMac }), - }); - expect(res.ok).toBe(true); - - // 3. Reboot into rescue. Rescue mode runs no %pre/%post, so the machine never - // reports its own IP — read it from the DHCP lease instead. - await sleep(20_000); + // Restart so the VM boots against the seeded state. createPxeVm already started it. rebootPxeVm(VM_NAME); await sleep(5_000); deleteNftablesRejectRules(); - log("Waiting for rescue environment DHCP lease..."); - rescueIp = await waitForLease(testDir, vmMac, RESCUE_TIMEOUT_MS); + // The whole chain now runs once: DHCP option 93 -> arm64 iPXE -> /boot.ipxe -> + // /dispatch (architecture from the record, not the query) -> aarch64 kernel + + // initrd -> Anaconda rescue -> sshd. Reaching a shell at all proves iPXE handed + // the initrd to the EFI stub over LoadFile2; without it the kernel panics first. + log("Waiting for the rescue environment's DHCP lease..."); + rescueIp = await waitForLease(testDir, vmMac, LEASE_TIMEOUT_MS); log(`Rescue IP: ${rescueIp}`); log("Waiting for SSH into the rescue shell (started by inst.sshd)..."); + log("(emulated aarch64 — Anaconda's stage2 download dominates; be patient)"); await waitForSsh(rescueIp, "root", SSH_TIMEOUT_MS, sshKeyPath).catch(async (err) => { log("Rescue SSH timed out. Serial console:"); try { @@ -366,11 +404,11 @@ describe("ARM PXE rescue", () => { throw err; }); log("ARM PXE rescue reached."); - }, DISCOVERY_TIMEOUT_MS + RESCUE_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000); + }, LEASE_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000); afterAll(async () => { await stopHarness(VM_NAME, harness); }); - it("discovered the machine as aarch64", async () => { + it("resolved the architecture from the machine record", async () => { const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`); const data = (await res.json()) as { discovered: Record }; expect(data.discovered[harness!.vmMac]?.arch).toBe("aarch64"); diff --git a/bastion/tests/integration/pxe-rescue.test.ts b/bastion/tests/integration/pxe-rescue.test.ts new file mode 100644 index 0000000..768ba26 --- /dev/null +++ b/bastion/tests/integration/pxe-rescue.test.ts @@ -0,0 +1,210 @@ +// Integration test: `labctl provision debug` -> Anaconda rescue with SSH, on x86_64. +// +// The rescue path had no test coverage on any architecture, which matters because it is +// the lab's recovery tool of last resort -- the thing you reach for when a machine will +// not boot. It runs here on x86_64 with KVM so it completes in minutes; the aarch64 +// equivalent is the same code path with a different kernel, but is emulated and far too +// slow to iterate on. +// +// Run: sudo ./scripts/test-provision.sh rescue + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, writeFileSync } from "node:fs"; +import { execSync } from "node:child_process"; +import { join } from "node:path"; +import { homedir, tmpdir } from "node:os"; +import { log, waitForSsh } from "./helpers/libvirt.js"; +import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js"; +import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./helpers/pxe-vm.js"; +import { sshExec } from "./helpers/ssh.js"; + +const VM_NAME = "lab-pxe-rescue-test"; +const HTTP_PORT = 8094; +const VM_MEMORY = 4096; +const VM_VCPUS = 4; +const VM_DISK_GB = 20; +const BASTION_IP = PXE_GATEWAY; +const SERIAL_PORT = 4555; + +const LEASE_TIMEOUT_MS = 8 * 60_000; +const SSH_TIMEOUT_MS = 15 * 60_000; + +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +function findSshKey(): { pubKey: string; keyPath: string } { + const candidates: string[] = []; + if (process.env["SSH_KEY_PATH"]) candidates.push(process.env["SSH_KEY_PATH"]); + const homes = [homedir()]; + const sudoUser = process.env["SUDO_USER"]; + if (sudoUser) homes.push(join("/home", sudoUser)); + for (const home of homes) { + for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) candidates.push(join(home, ".ssh", name)); + } + for (const keyPath of candidates) { + if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) { + return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath }; + } + } + throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/"); +} + +function leaseIpFor(testDir: string, mac: string): string | null { + const leaseFile = join(testDir, "dnsmasq.leases"); + if (!existsSync(leaseFile)) return null; + for (const line of readFileSync(leaseFile, "utf-8").split("\n")) { + const parts = line.trim().split(/\s+/); + if (parts.length >= 3 && parts[1]?.toLowerCase() === mac.toLowerCase()) return parts[2] ?? null; + } + return null; +} + +async function waitForLease(testDir: string, mac: string, timeoutMs: number): Promise { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const ip = leaseIpFor(testDir, mac); + if (ip !== null) return ip; + await sleep(5000); + } + throw new Error(`No DHCP lease for ${mac} within ${timeoutMs}ms`); +} + +// Suite name must not be a substring of "ARM PXE rescue" -- vitest -t matches +// substrings, so a looser name here would drag the emulated aarch64 suite in with it. +describe("x86 rescue boot", () => { + let app: { close: () => Promise }; + let stopDnsmasqFn: () => void; + let testDir: string; + let vmMac: string; + let rescueIp: string; + let sshKeyPath: string; + + beforeAll(async () => { + const { pubKey, keyPath } = findSshKey(); + sshKeyPath = keyPath; + + log("Setting up PXE test network..."); + ensurePxeNetwork(); + + testDir = join(tmpdir(), `lab-pxe-rescue-${Date.now()}`); + for (const sub of ["tftp", "http", "logs"]) mkdirSync(join(testDir, sub), { recursive: true }); + + const { createApp } = await import("../../src/bastion/src/server.js"); + const { loadConfig } = await import("../../src/bastion/src/config.js"); + const { generateDnsmasqConf, startDnsmasq, stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js"); + const { renderBootIpxe, kernelPath, initrdPath } = await import("../../src/bastion/src/templates/boot.ipxe.js"); + stopDnsmasqFn = stopDnsmasq; + + const config = loadConfig({ + bastionDir: testDir, + httpPort: HTTP_PORT, + iface: "virbr-pxe", + serverIp: BASTION_IP, + network: `${PXE_SUBNET}.0`, + gateway: BASTION_IP, + dhcpMode: "full", + dhcpRangeStart: `${PXE_SUBNET}.100`, + dhcpRangeEnd: `${PXE_SUBNET}.200`, + domain: "rescue-test.local", + sshKeys: [pubKey], + adminUser: "lab", + }); + + // iPXE in both dirs: TFTP PXE and UEFI HTTP Boot are both possible, and OVMF picks. + const ipxeX86 = "/usr/share/ipxe/ipxe-snponly-x86_64.efi"; + if (!existsSync(ipxeX86)) throw new Error(`iPXE not found: ${ipxeX86}`); + copyFileSync(ipxeX86, join(config.tftpDir, "ipxe.efi")); + copyFileSync(ipxeX86, join(config.httpDir, "ipxe.efi")); + + const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache"; + execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" }); + const kernelCache = join(cacheDir, "vmlinuz-x86_64"); + const initrdCache = join(cacheDir, "initrd-x86_64.img"); + if (!existsSync(kernelCache)) { + log("Downloading Fedora x86_64 kernel..."); + execSync(`curl -# -L -f -o "${kernelCache}" "${config.fedoraMirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 600_000 }); + } + if (!existsSync(initrdCache)) { + log("Downloading Fedora x86_64 initrd..."); + execSync(`curl -# -L -f -o "${initrdCache}" "${config.fedoraMirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 600_000 }); + } + copyFileSync(kernelCache, join(config.httpDir, kernelPath("x86_64"))); + copyFileSync(initrdCache, join(config.httpDir, initrdPath("x86_64"))); + + writeFileSync(join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort })); + generateDnsmasqConf(config); + + const { app: fastify, state, syslog } = createApp(config); + app = fastify; + await fastify.listen({ port: config.httpPort, host: "0.0.0.0" }); + syslog.start(); + log(`Bastion HTTP listening on :${HTTP_PORT}`); + + startDnsmasq(config).catch((err) => log(`dnsmasq failed: ${err instanceof Error ? err.message : String(err)}`)); + await sleep(1500); + + log("Creating x86_64 PXE VM (KVM)..."); + createPxeVm({ name: VM_NAME, memory: VM_MEMORY, vcpus: VM_VCPUS, diskSize: VM_DISK_GB, network: PXE_NETWORK_NAME }); + const mac = getVmMac(VM_NAME); + if (!mac) throw new Error("Could not determine VM MAC"); + vmMac = mac; + log(`VM MAC: ${vmMac}`); + + // Queue rescue directly, as `labctl provision debug` does. + log("Queueing debug/rescue mode..."); + state.update((s) => { + s.debug[vmMac] = { hostname: "rescue-test", queued_at: new Date().toISOString() }; + }); + + rebootPxeVm(VM_NAME); + await sleep(5_000); + deleteNftablesRejectRules(); + + rescueIp = await waitForLease(testDir, vmMac, LEASE_TIMEOUT_MS); + log(`Rescue IP: ${rescueIp}`); + + log("Waiting for SSH into the rescue shell (inst.sshd)..."); + await waitForSsh(rescueIp, "root", SSH_TIMEOUT_MS, sshKeyPath).catch(async (err) => { + log("Rescue SSH timed out. Serial console:"); + try { log(await readSerialLog(SERIAL_PORT, { lastLines: 120, timeoutMs: 20_000 })); } catch { /* none */ } + throw err; + }); + log("Rescue shell reachable."); + }, LEASE_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000); + + afterAll(async () => { + if (process.env["KEEP_VM"] === "1") { + log(`KEEP_VM=1 — leaving ${VM_NAME} up (serial: socat - TCP:127.0.0.1:${SERIAL_PORT})`); + return; + } + log("Cleaning up..."); + if (app) await app.close().catch(() => {}); + if (stopDnsmasqFn) stopDnsmasqFn(); + destroyPxeVm(VM_NAME); + destroyPxeNetwork(); + if (testDir) rmSync(testDir, { recursive: true, force: true }); + }); + + it("rescue shell is reachable over SSH as root", () => { + const result = sshExec(rescueIp, "root", "whoami", { keyPath: sshKeyPath, timeout: 60_000 }); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe("root"); + }); + + it("is the Anaconda rescue environment", () => { + const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 }); + expect(result.stdout).toContain("inst.rescue"); + expect(result.stdout).toContain("inst.sshd"); + }); + + it("kernel and initrd came from the bastion", () => { + const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 }); + expect(result.stdout).toContain(`${BASTION_IP}:${HTTP_PORT}`); + }); + + it("has LVM tools for inspecting an installed system", () => { + const result = sshExec(rescueIp, "root", "command -v vgchange && command -v lsblk", { keyPath: sshKeyPath, timeout: 60_000 }); + expect(result.exitCode).toBe(0); + }); +});