lab/bastion/tests/integration/pxe-rescue.test.ts

// Integration test: `labctl provision debug` -> Anaconda rescue with SSH, on x86_64.
//
// The rescue path had no test coverage on any architecture, which matters because it is
// the lab's recovery tool of last resort -- the thing you reach for when a machine will
// not boot. It runs here on x86_64 with KVM so it completes in minutes; the aarch64
// equivalent is the same code path with a different kernel, but is emulated and far too
// slow to iterate on.
//
// Run: sudo ./scripts/test-provision.sh rescue

import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, writeFileSync } from "node:fs";
import { execSync } from "node:child_process";
import { join } from "node:path";
import { homedir, tmpdir } from "node:os";
import { log, waitForSsh } from "./helpers/libvirt.js";
import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js";
import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./helpers/pxe-vm.js";
import { sshExec } from "./helpers/ssh.js";

const VM_NAME = "lab-pxe-rescue-test";
const HTTP_PORT = 8094;
const VM_MEMORY = 4096;
const VM_VCPUS = 4;
const VM_DISK_GB = 20;
const BASTION_IP = PXE_GATEWAY;
const SERIAL_PORT = 4555;

const LEASE_TIMEOUT_MS = 8 * 60_000;
const SSH_TIMEOUT_MS = 15 * 60_000;

function sleep(ms: number): Promise<void> {
  return new Promise((r) => setTimeout(r, ms));
}

function findSshKey(): { pubKey: string; keyPath: string } {
  const candidates: string[] = [];
  if (process.env["SSH_KEY_PATH"]) candidates.push(process.env["SSH_KEY_PATH"]);
  const homes = [homedir()];
  const sudoUser = process.env["SUDO_USER"];
  if (sudoUser) homes.push(join("/home", sudoUser));
  for (const home of homes) {
    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) candidates.push(join(home, ".ssh", name));
  }
  for (const keyPath of candidates) {
    if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
      return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
    }
  }
  throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/");
}

function leaseIpFor(testDir: string, mac: string): string | null {
  const leaseFile = join(testDir, "dnsmasq.leases");
  if (!existsSync(leaseFile)) return null;
  for (const line of readFileSync(leaseFile, "utf-8").split("\n")) {
    const parts = line.trim().split(/\s+/);
    if (parts.length >= 3 && parts[1]?.toLowerCase() === mac.toLowerCase()) return parts[2] ?? null;
  }
  return null;
}

async function waitForLease(testDir: string, mac: string, timeoutMs: number): Promise<string> {
  const start = Date.now();
  while (Date.now() - start < timeoutMs) {
    const ip = leaseIpFor(testDir, mac);
    if (ip !== null) return ip;
    await sleep(5000);
  }
  throw new Error(`No DHCP lease for ${mac} within ${timeoutMs}ms`);
}

// Suite name must not be a substring of "ARM PXE rescue" -- vitest -t matches
// substrings, so a looser name here would drag the emulated aarch64 suite in with it.
describe("x86 rescue boot", () => {
  let app: { close: () => Promise<void> };
  let stopDnsmasqFn: () => void;
  let testDir: string;
  let vmMac: string;
  let rescueIp: string;
  let sshKeyPath: string;

  beforeAll(async () => {
    const { pubKey, keyPath } = findSshKey();
    sshKeyPath = keyPath;

    log("Setting up PXE test network...");
    ensurePxeNetwork();

    testDir = join(tmpdir(), `lab-pxe-rescue-${Date.now()}`);
    for (const sub of ["tftp", "http", "logs"]) mkdirSync(join(testDir, sub), { recursive: true });

    const { createApp } = await import("../../src/bastion/src/server.js");
    const { loadConfig } = await import("../../src/bastion/src/config.js");
    const { generateDnsmasqConf, startDnsmasq, stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
    const { renderBootIpxe, kernelPath, initrdPath } = await import("../../src/bastion/src/templates/boot.ipxe.js");
    stopDnsmasqFn = stopDnsmasq;

    const config = loadConfig({
      bastionDir: testDir,
      httpPort: HTTP_PORT,
      iface: "virbr-pxe",
      serverIp: BASTION_IP,
      network: `${PXE_SUBNET}.0`,
      gateway: BASTION_IP,
      dhcpMode: "full",
      dhcpRangeStart: `${PXE_SUBNET}.100`,
      dhcpRangeEnd: `${PXE_SUBNET}.200`,
      domain: "rescue-test.local",
      sshKeys: [pubKey],
      adminUser: "lab",
    });

    // iPXE in both dirs: TFTP PXE and UEFI HTTP Boot are both possible, and OVMF picks.
    const ipxeX86 = "/usr/share/ipxe/ipxe-snponly-x86_64.efi";
    if (!existsSync(ipxeX86)) throw new Error(`iPXE not found: ${ipxeX86}`);
    copyFileSync(ipxeX86, join(config.tftpDir, "ipxe.efi"));
    copyFileSync(ipxeX86, join(config.httpDir, "ipxe.efi"));

    const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache";
    execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" });
    const kernelCache = join(cacheDir, "vmlinuz-x86_64");
    const initrdCache = join(cacheDir, "initrd-x86_64.img");
    if (!existsSync(kernelCache)) {
      log("Downloading Fedora x86_64 kernel...");
      execSync(`curl -# -L -f -o "${kernelCache}" "${config.fedoraMirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 600_000 });
    }
    if (!existsSync(initrdCache)) {
      log("Downloading Fedora x86_64 initrd...");
      execSync(`curl -# -L -f -o "${initrdCache}" "${config.fedoraMirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 600_000 });
    }
    copyFileSync(kernelCache, join(config.httpDir, kernelPath("x86_64")));
    copyFileSync(initrdCache, join(config.httpDir, initrdPath("x86_64")));

    writeFileSync(join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }));
    generateDnsmasqConf(config);

    const { app: fastify, state, syslog } = createApp(config);
    app = fastify;
    await fastify.listen({ port: config.httpPort, host: "0.0.0.0" });
    syslog.start();
    log(`Bastion HTTP listening on :${HTTP_PORT}`);

    startDnsmasq(config).catch((err) => log(`dnsmasq failed: ${err instanceof Error ? err.message : String(err)}`));
    await sleep(1500);

    log("Creating x86_64 PXE VM (KVM)...");
    createPxeVm({ name: VM_NAME, memory: VM_MEMORY, vcpus: VM_VCPUS, diskSize: VM_DISK_GB, network: PXE_NETWORK_NAME });
    const mac = getVmMac(VM_NAME);
    if (!mac) throw new Error("Could not determine VM MAC");
    vmMac = mac;
    log(`VM MAC: ${vmMac}`);

    // Queue rescue directly, as `labctl provision debug` does.
    log("Queueing debug/rescue mode...");
    state.update((s) => {
      s.debug[vmMac] = { hostname: "rescue-test", queued_at: new Date().toISOString() };
    });

    rebootPxeVm(VM_NAME);
    await sleep(5_000);
    deleteNftablesRejectRules();

    rescueIp = await waitForLease(testDir, vmMac, LEASE_TIMEOUT_MS);
    log(`Rescue IP: ${rescueIp}`);

    log("Waiting for SSH into the rescue shell (inst.sshd)...");
    await waitForSsh(rescueIp, "root", SSH_TIMEOUT_MS, sshKeyPath).catch(async (err) => {
      log("Rescue SSH timed out. Serial console:");
      try { log(await readSerialLog(SERIAL_PORT, { lastLines: 120, timeoutMs: 20_000 })); } catch { /* none */ }
      throw err;
    });
    log("Rescue shell reachable.");
  }, LEASE_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000);

  afterAll(async () => {
    if (process.env["KEEP_VM"] === "1") {
      log(`KEEP_VM=1 — leaving ${VM_NAME} up (serial: socat - TCP:127.0.0.1:${SERIAL_PORT})`);
      return;
    }
    log("Cleaning up...");
    if (app) await app.close().catch(() => {});
    if (stopDnsmasqFn) stopDnsmasqFn();
    destroyPxeVm(VM_NAME);
    destroyPxeNetwork();
    if (testDir) rmSync(testDir, { recursive: true, force: true });
  });

  it("rescue shell is reachable over SSH as root", () => {
    const result = sshExec(rescueIp, "root", "whoami", { keyPath: sshKeyPath, timeout: 60_000 });
    expect(result.exitCode).toBe(0);
    expect(result.stdout.trim()).toBe("root");
  });

  it("is the Anaconda rescue environment", () => {
    const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 });
    expect(result.stdout).toContain("inst.rescue");
    expect(result.stdout).toContain("inst.sshd");
  });

  it("kernel and initrd came from the bastion", () => {
    const result = sshExec(rescueIp, "root", "cat /proc/cmdline", { keyPath: sshKeyPath, timeout: 60_000 });
    expect(result.stdout).toContain(`${BASTION_IP}:${HTTP_PORT}`);
  });

  it("has LVM tools for inspecting an installed system", () => {
    const result = sshExec(rescueIp, "root", "command -v vgchange && command -v lsblk", { keyPath: sshKeyPath, timeout: 60_000 });
    expect(result.exitCode).toBe(0);
  });
});