test(bastion): cover the rescue boot path and record what it exposed

`provision debug` is the lab's recovery tool of last resort and had no
integration coverage on any architecture. Adding it -- x86_64 on KVM so
it runs in ~15 minutes, plus the aarch64 equivalent -- showed the rescue
environment coming up correctly but nothing ever listening on port 22.

Reproduced on both architectures, so it is neither ARM-specific nor an
emulation artefact, and it is orthogonal to the multi-arch work: x86_64
is unchanged by that. Documented in ARCHITECTURE.md with the leads worth
checking, rather than left as a silent gap.

Also restructures the ARM rescue suite to seed the machine into state
instead of discovering it first. That mirrors the DGX Spark situation --
SSH-onboarded, never PXE-discovered, architecture known only from its
record -- and holds the test to one emulated boot, since each spends
~15 of its ~18 minutes fetching Anaconda's stage2 under TCG.

KEEP_VM=1 leaves the VM up on failure; half-hour emulated runs are too
expensive to pay twice just to see what happened.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015nRFZXpKwUVE4SRSHw6GjF

bastion/tests/integration/pxe-rescue.test.ts

@@ -0,0 +1,210 @@
+// Integration test: `labctl provision debug` -> Anaconda rescue with SSH, on x86_64.
+//
+// The rescue path had no test coverage on any architecture, which matters because it is
+// the lab's recovery tool of last resort -- the thing you reach for when a machine will
+// not boot. It runs here on x86_64 with KVM so it completes in minutes; the aarch64
+// equivalent is the same code path with a different kernel, but is emulated and far too
+// slow to iterate on.
+//
+// Run: sudo ./scripts/test-provision.sh rescue
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, writeFileSync } from "node:fs";
+import { execSync } from "node:child_process";
+import { join } from "node:path";
+import { homedir, tmpdir } from "node:os";
+import { log, waitForSsh } from "./helpers/libvirt.js";
+import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js";
+import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./helpers/pxe-vm.js";
+import { sshExec } from "./helpers/ssh.js";
+
+const VM_NAME = "lab-pxe-rescue-test";
+const HTTP_PORT = 8094;
+const VM_MEMORY = 4096;
+const VM_VCPUS = 4;
+const VM_DISK_GB = 20;
+const BASTION_IP = PXE_GATEWAY;
+const SERIAL_PORT = 4555;
+
+const LEASE_TIMEOUT_MS = 8 * 60_000;
+const SSH_TIMEOUT_MS = 15 * 60_000;
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+function findSshKey(): { pubKey: string; keyPath: string } {
+  const candidates: string[] = [];
+  if (process.env["SSH_KEY_PATH"]) candidates.push(process.env["SSH_KEY_PATH"]);
+  const homes = [homedir()];
+  const sudoUser = process.env["SUDO_USER"];
+  if (sudoUser) homes.push(join("/home", sudoUser));
+  for (const home of homes) {
+    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) candidates.push(join(home, ".ssh", name));
+  }
+  for (const keyPath of candidates) {
+    if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
+      return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
+    }
+  }
+  throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/");
+}
+
+function leaseIpFor(testDir: string, mac: string): string | null {
+  const leaseFile = join(testDir, "dnsmasq.leases");
+  if (!existsSync(leaseFile)) return null;
+  for (const line of readFileSync(leaseFile, "utf-8").split("\n")) {
+    const parts = line.trim().split(/\s+/);
+    if (parts.length >= 3 && parts[1]?.toLowerCase() === mac.toLowerCase()) return parts[2] ?? null;
+  }
+  return null;
+}
+
+async function waitForLease(testDir: string, mac: string, timeoutMs: number): Promise<string> {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    const ip = leaseIpFor(testDir, mac);
+    if (ip !== null) return ip;
+    await sleep(5000);
+  }
+  throw new Error(`No DHCP lease for ${mac} within ${timeoutMs}ms`);
+}
+
+// Suite name must not be a substring of "ARM PXE rescue" -- vitest -t matches
+// substrings, so a looser name here would drag the emulated aarch64 suite in with it.
+describe("x86 rescue boot", () => {
+  let app: { close: () => Promise<void> };
+  let stopDnsmasqFn: () => void;
+  let testDir: string;
+  let vmMac: string;
+  let rescueIp: string;
+  let sshKeyPath: string;
+
+  beforeAll(async () => {
+    const { pubKey, keyPath } = findSshKey();
+    sshKeyPath = keyPath;
+
+    log("Setting up PXE test network...");
+    ensurePxeNetwork();
+
+    testDir = join(tmpdir(), `lab-pxe-rescue-${Date.now()}`);
+    for (const sub of ["tftp", "http", "logs"]) mkdirSync(join(testDir, sub), { recursive: true });
+
+    const { createApp } = await import("../../src/bastion/src/server.js");
+    const { loadConfig } = await import("../../src/bastion/src/config.js");
+    const { generateDnsmasqConf, startDnsmasq, stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
+    const { renderBootIpxe, kernelPath, initrdPath } = await import("../../src/bastion/src/templates/boot.ipxe.js");
+    stopDnsmasqFn = stopDnsmasq;
+
+    const config = loadConfig({
+      bastionDir: testDir,
+      httpPort: HTTP_PORT,
+      iface: "virbr-pxe",
+      serverIp: BASTION_IP,
+      network: `${PXE_SUBNET}.0`,
+      gateway: BASTION_IP,
+      dhcpMode: "full",
+      dhcpRangeStart: `${PXE_SUBNET}.100`,
+      dhcpRangeEnd: `${PXE_SUBNET}.200`,
+      domain: "rescue-test.local",
+      sshKeys: [pubKey],
+      adminUser: "lab",
+    });
+
+    // iPXE in both dirs: TFTP PXE and UEFI HTTP Boot are both possible, and OVMF picks.
+    const ipxeX86 = "/usr/share/ipxe/ipxe-snponly-x86_64.efi";
+    if (!existsSync(ipxeX86)) throw new Error(`iPXE not found: ${ipxeX86}`);
+    copyFileSync(ipxeX86, join(config.tftpDir, "ipxe.efi"));
+    copyFileSync(ipxeX86, join(config.httpDir, "ipxe.efi"));
+
+    const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache";
+    execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" });
+    const kernelCache = join(cacheDir, "vmlinuz-x86_64");
+    const initrdCache = join(cacheDir, "initrd-x86_64.img");
+    if (!existsSync(kernelCache)) {
+      log("Downloading Fedora x86_64 kernel...");
+      execSync(`curl -# -L -f -o "${kernelCache}" "${config.fedoraMirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 600_000 });
+    }
+    if (!existsSync(initrdCache)) {
+      log("Downloading Fedora x86_64 initrd...");
+      execSync(`curl -# -L -f -o "${initrdCache}" "${config.fedoraMirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 600_000 });
+    }
+    copyFileSync(kernelCache, join(config.httpDir, kernelPath("x86_64")));
+    copyFileSync(initrdCache, join(config.httpDir, initrdPath("x86_64")));
+
+    writeFileSync(join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }));
+    generateDnsmasqConf(config);
+
+    const { app: fastify, state, syslog } = createApp(config);
+    app = fastify;
+    await fastify.listen({ port: config.httpPort, host: "0.0.0.0" });
+    syslog.start();
+    log(`Bastion HTTP listening on :${HTTP_PORT}`);
+
+    startDnsmasq(config).catch((err) => log(`dnsmasq failed: ${err instanceof Error ? err.message : String(err)}`));
+    await sleep(1500);
+
+    log("Creating x86_64 PXE VM (KVM)...");
+    createPxeVm({ name: VM_NAME, memory: VM_MEMORY, vcpus: VM_VCPUS, diskSize: VM_DISK_GB, network: PXE_NETWORK_NAME });
+    const mac = getVmMac(VM_NAME);
+    if (!mac) throw new Error("Could not determine VM MAC");
+    vmMac = mac;
+    log(`VM MAC: ${vmMac}`);
+
+    // Queue rescue directly, as `labctl provision debug` does.
+    log("Queueing debug/rescue mode...");
+    state.update((s) => {
+      s.debug[vmMac] = { hostname: "rescue-test", queued_at: new Date().toISOString() };
+    });
+
+    rebootPxeVm(VM_NAME);
+    await sleep(5_000);
+    deleteNftablesRejectRules();
+
+    rescueIp = await waitForLease(testDir, vmMac, LEASE_TIMEOUT_MS);
+    log(`Rescue IP: ${rescueIp}`);
+
+    log("Waiting for SSH into the rescue shell (inst.sshd)...");
+    await waitForSsh(rescueIp, "root", SSH_TIMEOUT_MS, sshKeyPath).catch(async (err) => {
+      log("Rescue SSH timed out. Serial console:");
+      try { log(await readSerialLog(SERIAL_PORT, { lastLines: 120, timeoutMs: 20_000 })); } catch { /* none */ }
+      throw err;
+    });
+    log("Rescue shell reachable.");
+  }, LEASE_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000);
+
+  afterAll(async () => {
+    if (process.env["KEEP_VM"] === "1") {
+      log(`KEEP_VM=1 — leaving ${VM_NAME} up (serial: socat - TCP:127.0.0.1:${SERIAL_PORT})`);
+      return;
+    }
+    log("Cleaning up...");
+    if (app) await app.close().catch(() => {});
+    if (stopDnsmasqFn) stopDnsmasqFn();
+    destroyPxeVm(VM_NAME);
+    destroyPxeNetwork();
+    if (testDir) rmSync(testDir, { recursive: true, force: true });
+  });
+
+  it("rescue shell is reachable over SSH as root", () => {
+    const result = sshExec(rescueIp, "root", "whoami", { keyPath: sshKeyPath, timeout: 60_000 });
+    expect(result.exitCode).toBe(0);
+    expect(result.stdout.trim()).toBe("root");
+  });
+
+  it("is the Anaconda rescue environment", () => {
+    const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 });
+    expect(result.stdout).toContain("inst.rescue");
+    expect(result.stdout).toContain("inst.sshd");
+  });
+
+  it("kernel and initrd came from the bastion", () => {
+    const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 });
+    expect(result.stdout).toContain(`${BASTION_IP}:${HTTP_PORT}`);
+  });
+
+  it("has LVM tools for inspecting an installed system", () => {
+    const result = sshExec(rescueIp, "root", "command -v vgchange && command -v lsblk", { keyPath: sshKeyPath, timeout: 60_000 });
+    expect(result.exitCode).toBe(0);
+  });
+});