feat: install logging, error trapping, PXE/ISO integration tests

Kickstart installs on real hardware failed silently — no error reporting,
only 3 progress callbacks, zero log streaming. This overhaul makes every
install fully observable.

Kickstart improvements:
- Error trapping in %pre and %post (trap ERR sends failure details to bastion)
- 12+ granular progress stages (was 3): SSH, hostname, k3s prep, EFI boot, metadata
- Background log streamer: tails %post output and batch-sends to /api/log
- bastion_log() function for explicit log lines from kickstart scripts

Bastion API:
- POST /api/log — receives raw log lines from kickstart (single or batch)
- InstallLogBuffer — per-MAC ring buffer (2000 lines) + file persistence
- GET /api/logs/:mac — now returns log_lines + log_total alongside stages
- SSE /api/logs/:mac/follow — uses named events (event: stage vs event: log)
- Progress events forwarded to labd via bastion-progress WebSocket message
- Post-provision k3s logs routed through progressBus (was console-only)

dnsmasq fixes found during VM testing:
- HTTP Boot filename: ipxe-real.efi → ipxe.efi (leftover from old 2-stage approach)
- pxe-service directives: only in proxy mode (breaks OVMF PXE in full mode)
- PXEClient vendor class echo for UEFI firmware compatibility

Integration tests:
- PXE boot test: blank UEFI VM → dnsmasq → HTTP Boot → iPXE → bastion → install
- ISO boot test: blank VM boots from bastion-generated ISO → same flow
- Shared helpers: pxe-network (no DHCP, nftables fix), pxe-vm (UEFI + ISO boot)
- test-provision.sh: runs both PXE + ISO tests with prerequisite checks
- 250GB sparse QCOW2 disk (LVM layout needs ~204GB)

201 unit tests passing (11 new).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

bastion/tests/integration/helpers/libvirt.ts

@@ -0,0 +1,219 @@
+// Libvirt VM lifecycle management for integration tests.
+
+import { execSync, spawnSync, type SpawnSyncReturns } from "node:child_process";
+import { existsSync, mkdirSync, writeFileSync, unlinkSync } from "node:fs";
+import { join } from "node:path";
+
+const IMAGE_DIR = "/var/lib/libvirt/images";
+// Cloud-init ISOs must be in a path accessible to the host's libvirtd,
+// so we use the shared images directory (not /tmp which may be container-only).
+const CLOUD_INIT_DIR = "/var/lib/libvirt/images/lab-cloud-init";
+
+// When running as root (inside container or via sudo), don't prefix with sudo.
+const IS_ROOT = process.getuid?.() === 0;
+
+/** Run a shell command, prefixing with sudo if not root. */
+function run(cmd: string, opts?: { timeout?: number }): string {
+  const full = IS_ROOT ? cmd : `sudo ${cmd}`;
+  return execSync(full, { encoding: "utf-8", stdio: "pipe", timeout: opts?.timeout ?? 60_000 });
+}
+
+/** Spawn a command, prefixing args with sudo if not root. */
+function virsh(...args: string[]): SpawnSyncReturns<string> {
+  const cmd = IS_ROOT ? "virsh" : "sudo";
+  const finalArgs = IS_ROOT ? args : ["virsh", ...args];
+  return spawnSync(cmd, finalArgs, { encoding: "utf-8", stdio: "pipe", timeout: 30_000 });
+}
+
+export interface VmConfig {
+  name: string;
+  memory: number;       // MB
+  vcpus: number;
+  diskSize: number;     // GB
+  network: string;      // libvirt network name
+  cloudImageUrl: string;
+  sshPubKey: string;    // content of authorized key
+  userData?: string;     // custom cloud-init user-data
+}
+
+export function log(msg: string): void {
+  const ts = new Date().toISOString().slice(11, 19);
+  console.log(`  [${ts}] ${msg}`);
+}
+
+/** Download a cloud image if not already cached. */
+export function ensureCloudImage(url: string, name: string): string {
+  const dest = join(IMAGE_DIR, `${name}.qcow2`);
+  if (existsSync(dest)) {
+    log(`Cloud image cached: ${dest}`);
+    return dest;
+  }
+  log(`Downloading cloud image: ${url}`);
+  run(`curl -L -f -o "${dest}" "${url}"`, { timeout: 300_000 });
+  return dest;
+}
+
+/** Create a cloud-init ISO for a VM. */
+export function createCloudInitIso(vmName: string, config: VmConfig): string {
+  mkdirSync(CLOUD_INIT_DIR, { recursive: true });
+  const dir = join(CLOUD_INIT_DIR, vmName);
+  mkdirSync(dir, { recursive: true });
+
+  const userData = config.userData ?? `#cloud-config
+hostname: ${vmName}
+manage_etc_hosts: true
+users:
+  - default
+  - name: fedora
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    shell: /bin/bash
+    ssh_authorized_keys:
+      - ${config.sshPubKey}
+ssh_pwauth: false
+package_update: false
+packages:
+  - curl
+  - socat
+  - conntrack-tools
+  - ethtool
+  - iptables-nft
+runcmd:
+  - modprobe br_netfilter
+  - modprobe overlay
+  - |
+    cat > /etc/sysctl.d/90-k3s.conf << 'EOF'
+    net.bridge.bridge-nf-call-iptables = 1
+    net.bridge.bridge-nf-call-ip6tables = 1
+    net.ipv4.ip_forward = 1
+    vm.panic_on_oom = 0
+    vm.overcommit_memory = 1
+    kernel.panic = 10
+    kernel.panic_on_oops = 1
+    EOF
+  - sysctl --system
+  - swapoff -a
+  - systemctl disable --now firewalld 2>/dev/null || true
+  - systemctl disable --now ufw 2>/dev/null || true
+`;
+
+  const metaData = `instance-id: ${vmName}\nlocal-hostname: ${vmName}\n`;
+
+  writeFileSync(join(dir, "user-data"), userData);
+  writeFileSync(join(dir, "meta-data"), metaData);
+
+  const isoPath = join(CLOUD_INIT_DIR, `${vmName}-cloud-init.iso`);
+  execSync(
+    `genisoimage -output "${isoPath}" -volid cidata -joliet -rock "${dir}/user-data" "${dir}/meta-data" 2>/dev/null || ` +
+    `mkisofs -output "${isoPath}" -volid cidata -joliet -rock "${dir}/user-data" "${dir}/meta-data" 2>/dev/null || ` +
+    `xorrisofs -output "${isoPath}" -volid cidata -joliet -rock "${dir}/user-data" "${dir}/meta-data"`,
+    { stdio: "pipe" },
+  );
+
+  return isoPath;
+}
+
+/** Create and start a VM from a cloud image with cloud-init. */
+export function createVm(config: VmConfig): void {
+  destroyVm(config.name);
+
+  log(`Creating VM: ${config.name} (${config.memory}MB RAM, ${config.vcpus} vCPU, ${config.diskSize}GB disk)`);
+
+  const baseImage = ensureCloudImage(config.cloudImageUrl, `${config.name}-base`);
+  const diskPath = join(IMAGE_DIR, `${config.name}.qcow2`);
+
+  run(`cp "${baseImage}" "${diskPath}"`);
+  run(`qemu-img resize "${diskPath}" ${config.diskSize}G`);
+
+  const cloudInitIso = createCloudInitIso(config.name, config);
+
+  const virtInstallArgs = [
+    "virt-install",
+    `--name=${config.name}`,
+    `--memory=${config.memory}`,
+    `--vcpus=${config.vcpus}`,
+    `--disk=path=${diskPath},format=qcow2`,
+    `--disk=path=${cloudInitIso},device=cdrom`,
+    `--network=network=${config.network},model=virtio`,
+    "--os-variant=generic",
+    "--import",
+    "--noautoconsole",
+    "--wait=0",
+  ];
+
+  log(`Running: virt-install --name=${config.name} ...`);
+  run(virtInstallArgs.join(" "));
+  log(`VM ${config.name} created and starting`);
+}
+
+/** Destroy a VM and remove its storage. */
+export function destroyVm(name: string): void {
+  const result = virsh("dominfo", name);
+  if (result.status !== 0) return;
+
+  log(`Destroying VM: ${name}`);
+  virsh("destroy", name);
+  virsh("undefine", name, "--remove-all-storage");
+
+  const isoPath = join(CLOUD_INIT_DIR, `${name}-cloud-init.iso`);
+  try { unlinkSync(isoPath); } catch { /* ignore */ }
+}
+
+/** Get the IP address of a running VM. */
+export function getVmIp(name: string): string | null {
+  try {
+    // Try with agent first, then without
+    let output = virsh("domifaddr", name, "--source", "agent").stdout;
+    if (!output || !output.includes(".")) {
+      output = virsh("domifaddr", name).stdout;
+    }
+    const match = output.match(/(\d+\.\d+\.\d+\.\d+)/);
+    return match ? match[1] : null;
+  } catch {
+    return null;
+  }
+}
+
+/** Wait for a VM to get an IP address. */
+export async function waitForVmIp(name: string, timeoutMs: number): Promise<string> {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    const ip = getVmIp(name);
+    if (ip) {
+      log(`VM ${name} got IP: ${ip}`);
+      return ip;
+    }
+    await sleep(2000);
+  }
+  throw new Error(`VM ${name} did not get an IP within ${timeoutMs}ms`);
+}
+
+/** Wait for SSH to become available on a host. */
+export async function waitForSsh(
+  ip: string,
+  user: string,
+  timeoutMs: number,
+  keyPath?: string,
+): Promise<void> {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    const result = spawnSync("ssh", [
+      "-o", "StrictHostKeyChecking=no",
+      "-o", "ConnectTimeout=3",
+      "-o", "BatchMode=yes",
+      ...(keyPath ? ["-i", keyPath] : []),
+      `${user}@${ip}`,
+      "echo ok",
+    ], { encoding: "utf-8", stdio: "pipe", timeout: 10_000 });
+
+    if (result.status === 0 && result.stdout.includes("ok")) {
+      log(`SSH ready on ${ip}`);
+      return;
+    }
+    await sleep(3000);
+  }
+  throw new Error(`SSH not available on ${ip} within ${timeoutMs}ms`);
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}

bastion/tests/integration/helpers/network.ts

@@ -0,0 +1,68 @@
+// Libvirt network management for integration tests.
+
+import { execSync, spawnSync } from "node:child_process";
+import { writeFileSync, unlinkSync } from "node:fs";
+import { log } from "./libvirt.js";
+
+export const TEST_NETWORK_NAME = "lab-test";
+export const TEST_NETWORK_BRIDGE = "virbr-lab";
+export const TEST_NETWORK_SUBNET = "192.168.250";
+export const TEST_NETWORK_GATEWAY = `${TEST_NETWORK_SUBNET}.1`;
+
+const IS_ROOT = process.getuid?.() === 0;
+
+function run(cmd: string): string {
+  const full = IS_ROOT ? cmd : `sudo ${cmd}`;
+  return execSync(full, { encoding: "utf-8", stdio: "pipe" });
+}
+
+function virsh(...args: string[]): { status: number; stdout: string } {
+  const cmd = IS_ROOT ? "virsh" : "sudo";
+  const finalArgs = IS_ROOT ? args : ["virsh", ...args];
+  const result = spawnSync(cmd, finalArgs, { encoding: "utf-8", stdio: "pipe" });
+  return { status: result.status ?? 1, stdout: result.stdout ?? "" };
+}
+
+const NETWORK_XML = `<network>
+  <name>${TEST_NETWORK_NAME}</name>
+  <forward mode='nat'/>
+  <bridge name='${TEST_NETWORK_BRIDGE}' stp='on' delay='0'/>
+  <ip address='${TEST_NETWORK_GATEWAY}' netmask='255.255.255.0'>
+    <dhcp>
+      <range start='${TEST_NETWORK_SUBNET}.100' end='${TEST_NETWORK_SUBNET}.200'/>
+    </dhcp>
+  </ip>
+</network>`;
+
+/** Ensure the test libvirt network exists and is active. */
+export function ensureTestNetwork(): void {
+  const result = virsh("net-info", TEST_NETWORK_NAME);
+
+  if (result.status === 0 && result.stdout.includes("Active:         yes")) {
+    log(`Network ${TEST_NETWORK_NAME} already active`);
+    return;
+  }
+
+  // Destroy existing if present but inactive
+  if (result.status === 0) {
+    virsh("net-destroy", TEST_NETWORK_NAME);
+    virsh("net-undefine", TEST_NETWORK_NAME);
+  }
+
+  const xmlPath = `/tmp/lab-test-network.xml`;
+  writeFileSync(xmlPath, NETWORK_XML);
+
+  log(`Creating libvirt network: ${TEST_NETWORK_NAME} (${TEST_NETWORK_SUBNET}.0/24)`);
+  run(`virsh net-define "${xmlPath}"`);
+  run(`virsh net-start "${TEST_NETWORK_NAME}"`);
+
+  try { unlinkSync(xmlPath); } catch { /* ignore */ }
+  log(`Network ${TEST_NETWORK_NAME} created and active`);
+}
+
+/** Destroy the test network. */
+export function destroyTestNetwork(): void {
+  log(`Destroying network: ${TEST_NETWORK_NAME}`);
+  virsh("net-destroy", TEST_NETWORK_NAME);
+  virsh("net-undefine", TEST_NETWORK_NAME);
+}

bastion/tests/integration/helpers/pxe-network.ts

@@ -0,0 +1,95 @@
+// Libvirt network for PXE boot testing.
+// Unlike the regular test network, this one has NO DHCP —
+// the bastion provides full DHCP + PXE on this network.
+
+import { execSync, spawnSync } from "node:child_process";
+import { writeFileSync, unlinkSync } from "node:fs";
+import { log } from "./libvirt.js";
+
+export const PXE_NETWORK_NAME = "lab-pxe-test";
+export const PXE_BRIDGE = "virbr-pxe";
+export const PXE_SUBNET = "192.168.251";
+export const PXE_GATEWAY = `${PXE_SUBNET}.1`;
+
+const IS_ROOT = process.getuid?.() === 0;
+
+function run(cmd: string): string {
+  const full = IS_ROOT ? cmd : `sudo ${cmd}`;
+  return execSync(full, { encoding: "utf-8", stdio: "pipe" });
+}
+
+function virsh(...args: string[]): { status: number; stdout: string } {
+  const cmd = IS_ROOT ? "virsh" : "sudo";
+  const finalArgs = IS_ROOT ? args : ["virsh", ...args];
+  const result = spawnSync(cmd, finalArgs, { encoding: "utf-8", stdio: "pipe" });
+  return { status: result.status ?? 1, stdout: result.stdout ?? "" };
+}
+
+// No <dhcp> section — bastion dnsmasq provides full DHCP + PXE
+const NETWORK_XML = `<network>
+  <name>${PXE_NETWORK_NAME}</name>
+  <forward mode='nat'/>
+  <bridge name='${PXE_BRIDGE}' stp='on' delay='0'/>
+  <ip address='${PXE_GATEWAY}' netmask='255.255.255.0'>
+  </ip>
+</network>`;
+
+/** Ensure the PXE test network exists and is active (no DHCP). */
+export function ensurePxeNetwork(): void {
+  const result = virsh("net-info", PXE_NETWORK_NAME);
+
+  if (result.status === 0 && result.stdout.includes("Active:         yes")) {
+    log(`Network ${PXE_NETWORK_NAME} already active`);
+    return;
+  }
+
+  // Destroy existing if present but inactive
+  if (result.status === 0) {
+    virsh("net-destroy", PXE_NETWORK_NAME);
+    virsh("net-undefine", PXE_NETWORK_NAME);
+  }
+
+  const xmlPath = "/tmp/lab-pxe-test-network.xml";
+  writeFileSync(xmlPath, NETWORK_XML);
+
+  log(`Creating PXE libvirt network: ${PXE_NETWORK_NAME} (${PXE_SUBNET}.0/24, no DHCP)`);
+  run(`virsh net-define "${xmlPath}"`);
+  run(`virsh net-start "${PXE_NETWORK_NAME}"`);
+
+  try { unlinkSync(xmlPath); } catch { /* ignore */ }
+
+  // Libvirt creates nftables rules that reject traffic on the bridge.
+  // DHCP works (dnsmasq uses raw sockets) but TFTP/HTTP from VM->host gets blocked.
+  // Delete the reject rules so VM traffic can reach the bastion.
+  try {
+    // Delete the reject rules that libvirt added for our bridge.
+    // We find and delete each rule by its handle number.
+    const deleteRejectRules = (chain: string): void => {
+      const output = run(`nft -a list chain inet libvirt ${chain} 2>/dev/null || true`);
+      const lines = output.split("\n");
+      for (const line of lines) {
+        if (line.includes(PXE_BRIDGE) && line.includes("reject")) {
+          const handleMatch = line.match(/# handle (\d+)/);
+          if (handleMatch) {
+            run(`nft delete rule inet libvirt ${chain} handle ${handleMatch[1]}`);
+          }
+        }
+      }
+    };
+    deleteRejectRules("guest_input");
+    deleteRejectRules("guest_output");
+    log(`Removed nftables reject rules for ${PXE_BRIDGE}`);
+  } catch {
+    log(`Could not update nftables rules (may need manual firewall config)`);
+  }
+
+  log(`Network ${PXE_NETWORK_NAME} created and active`);
+}
+
+/** Destroy the PXE test network. */
+export function destroyPxeNetwork(): void {
+  log(`Destroying PXE network: ${PXE_NETWORK_NAME}`);
+  // nftables rules are cleaned up when the network is destroyed (libvirt removes them)
+  virsh("net-destroy", PXE_NETWORK_NAME);
+  virsh("net-undefine", PXE_NETWORK_NAME);
+}

bastion/tests/integration/helpers/pxe-vm.ts

@@ -0,0 +1,146 @@
+// Create a blank UEFI VM for PXE boot testing.
+// Unlike cloud image VMs, these have an empty disk and boot from network.
+
+import { execSync, spawnSync, type SpawnSyncReturns } from "node:child_process";
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { log } from "./libvirt.js";
+
+const IMAGE_DIR = "/var/lib/libvirt/images";
+const IS_ROOT = process.getuid?.() === 0;
+
+function run(cmd: string, opts?: { timeout?: number }): string {
+  const full = IS_ROOT ? cmd : `sudo ${cmd}`;
+  return execSync(full, { encoding: "utf-8", stdio: "pipe", timeout: opts?.timeout ?? 60_000 });
+}
+
+function virsh(...args: string[]): SpawnSyncReturns<string> {
+  const cmd = IS_ROOT ? "virsh" : "sudo";
+  const finalArgs = IS_ROOT ? args : ["virsh", ...args];
+  return spawnSync(cmd, finalArgs, { encoding: "utf-8", stdio: "pipe", timeout: 30_000 });
+}
+
+export interface PxeVmConfig {
+  name: string;
+  memory: number;       // MB
+  vcpus: number;
+  diskSize: number;     // GB
+  network: string;      // libvirt network name
+  arch?: "x86_64" | "aarch64";
+}
+
+/** Create a blank UEFI VM that PXE boots from the network. */
+export function createPxeVm(config: PxeVmConfig): void {
+  destroyPxeVm(config.name);
+
+  const arch = config.arch ?? "x86_64";
+  log(`Creating PXE VM: ${config.name} (${arch}, ${config.memory}MB RAM, ${config.vcpus} vCPU, ${config.diskSize}GB disk)`);
+
+  // Create blank disk
+  const diskPath = join(IMAGE_DIR, `${config.name}.qcow2`);
+  run(`qemu-img create -f qcow2 "${diskPath}" ${config.diskSize}G`);
+
+  // OVMF firmware paths (Fedora)
+  const ovmfCode = arch === "aarch64"
+    ? "/usr/share/edk2/aarch64/QEMU_EFI-pflash.raw"
+    : "/usr/share/edk2/ovmf/OVMF_CODE.fd";
+
+  if (!existsSync(ovmfCode)) {
+    throw new Error(`OVMF firmware not found at ${ovmfCode}. Install: sudo dnf install edk2-ovmf`);
+  }
+
+  const virtInstallArgs = [
+    "virt-install",
+    `--name=${config.name}`,
+    `--memory=${config.memory}`,
+    `--vcpus=${config.vcpus}`,
+    `--disk=path=${diskPath},format=qcow2,bus=virtio`,
+    `--network=network=${config.network},model=virtio`,
+    // UEFI firmware — required for PXE boot in modern mode
+    `--boot=uefi,network`,
+    // No OS to install — PXE provides everything
+    "--os-variant=generic",
+    "--noautoconsole",
+    "--wait=0",
+    // Graphics for debugging (VNC, connect with virt-viewer if needed)
+    "--graphics=vnc,listen=127.0.0.1",
+  ];
+
+  if (arch === "aarch64") {
+    virtInstallArgs.push("--arch=aarch64", "--machine=virt");
+  }
+
+  log(`Running: virt-install --name=${config.name} --boot=uefi,network ...`);
+  run(virtInstallArgs.join(" "), { timeout: 30_000 });
+  log(`PXE VM ${config.name} created and booting from network`);
+}
+
+/** Destroy a PXE VM and clean up its disk. */
+export function destroyPxeVm(name: string): void {
+  const result = virsh("dominfo", name);
+  if (result.status !== 0) return;
+
+  log(`Destroying PXE VM: ${name}`);
+  virsh("destroy", name);
+  virsh("undefine", name, "--remove-all-storage", "--nvram");
+}
+
+/** Get the MAC address of a VM's first NIC. */
+export function getVmMac(name: string): string | null {
+  const result = virsh("domiflist", name);
+  if (result.status !== 0) return null;
+  // Output format: Interface  Type  Source  Model  MAC
+  const match = result.stdout.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i);
+  return match ? match[1].toLowerCase() : null;
+}
+
+/** Reboot a VM (force off + start). */
+export function rebootPxeVm(name: string): void {
+  log(`Rebooting PXE VM: ${name}`);
+  virsh("destroy", name);
+  // Brief pause to let resources release
+  spawnSync("sleep", ["2"]);
+  virsh("start", name);
+  log(`PXE VM ${name} restarted`);
+}
+
+export interface IsoVmConfig {
+  name: string;
+  memory: number;       // MB
+  vcpus: number;
+  diskSize: number;     // GB
+  network: string;      // libvirt network name
+  isoPath: string;      // path to boot ISO
+}
+
+/** Create a UEFI VM that boots from a CD-ROM ISO (not PXE). */
+export function createIsoVm(config: IsoVmConfig): void {
+  destroyPxeVm(config.name);
+
+  log(`Creating ISO boot VM: ${config.name} (${config.memory}MB RAM, ${config.vcpus} vCPU, ${config.diskSize}GB disk)`);
+
+  // Create blank disk
+  const diskPath = join(IMAGE_DIR, `${config.name}.qcow2`);
+  run(`qemu-img create -f qcow2 "${diskPath}" ${config.diskSize}G`);
+
+  const virtInstallArgs = [
+    "virt-install",
+    `--name=${config.name}`,
+    `--memory=${config.memory}`,
+    `--vcpus=${config.vcpus}`,
+    `--disk=path=${diskPath},format=qcow2,bus=virtio`,
+    // Boot ISO as CD-ROM
+    `--disk=path=${config.isoPath},device=cdrom,readonly=on`,
+    `--network=network=${config.network},model=virtio`,
+    // UEFI firmware, boot from cdrom (not network)
+    "--boot=uefi,cdrom",
+    "--os-variant=generic",
+    "--noautoconsole",
+    "--wait=0",
+    "--graphics=vnc,listen=127.0.0.1",
+  ];
+
+  log(`Running: virt-install --name=${config.name} --boot=uefi,cdrom ...`);
+  run(virtInstallArgs.join(" "), { timeout: 30_000 });
+  log(`ISO boot VM ${config.name} created and booting from ISO`);
+}

bastion/tests/integration/helpers/ssh.ts

@@ -0,0 +1,106 @@
+// SSH execution helper for integration tests.
+
+import { spawn, spawnSync } from "node:child_process";
+import { log } from "./libvirt.js";
+
+export interface SshResult {
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+}
+
+/** Execute a command via SSH and return the result. */
+export function sshExec(
+  ip: string,
+  user: string,
+  command: string,
+  options?: { keyPath?: string; timeout?: number },
+): SshResult {
+  const args = [
+    "-o", "StrictHostKeyChecking=no",
+    "-o", "ConnectTimeout=10",
+    "-o", "BatchMode=yes",
+    ...(options?.keyPath ? ["-i", options.keyPath] : []),
+    `${user}@${ip}`,
+    command,
+  ];
+
+  const result = spawnSync("ssh", args, {
+    stdio: "pipe",
+    timeout: options?.timeout ?? 120_000,
+    encoding: "utf-8",
+  });
+
+  return {
+    exitCode: result.status ?? 1,
+    stdout: result.stdout ?? "",
+    stderr: result.stderr ?? "",
+  };
+}
+
+/** Execute a command via SSH with live streaming output. */
+export async function sshExecStreaming(
+  ip: string,
+  user: string,
+  command: string,
+  onLine: (line: string) => void,
+  options?: { keyPath?: string; timeout?: number },
+): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const args = [
+      "-o", "StrictHostKeyChecking=no",
+      "-o", "ConnectTimeout=10",
+      "-o", "BatchMode=yes",
+      ...(options?.keyPath ? ["-i", options.keyPath] : []),
+      `${user}@${ip}`,
+      command,
+    ];
+
+    const proc = spawn("ssh", args, {
+      stdio: ["ignore", "pipe", "pipe"],
+      timeout: options?.timeout ?? 600_000,
+    });
+
+    let buffer = "";
+    proc.stdout.on("data", (data: Buffer) => {
+      buffer += data.toString();
+      const lines = buffer.split("\n");
+      buffer = lines.pop() ?? "";
+      for (const line of lines) {
+        onLine(line);
+      }
+    });
+
+    proc.stderr.on("data", (data: Buffer) => {
+      const text = data.toString().trim();
+      if (text) onLine(`[stderr] ${text}`);
+    });
+
+    proc.on("close", (code) => {
+      if (buffer.trim()) onLine(buffer.trim());
+      resolve(code ?? 1);
+    });
+
+    proc.on("error", (err) => {
+      reject(err);
+    });
+  });
+}
+
+/** Run a command on the VM via SSH and log it with a prefix. */
+export async function sshRun(
+  ip: string,
+  user: string,
+  command: string,
+  label: string,
+  options?: { keyPath?: string; timeout?: number },
+): Promise<number> {
+  log(`${label}: ${command.slice(0, 80)}${command.length > 80 ? "..." : ""}`);
+  const code = await sshExecStreaming(ip, user, command, (line) => {
+    console.log(`    ${line}`);
+  }, options);
+  if (code !== 0) {
+    log(`${label}: FAILED (exit ${code})`);
+  }
+  return code;
+}