integration: post-mortem on SSH timeout (serial console + domstate), KEEP_VMS escape hatch

A VM that installs fine but never reaches SSH was destroyed by afterAll
with the only evidence -- its console -- inside it. Now the failure
carries the last serial lines and domstate, and KEEP_VMS=1 preserves
the machine for inspection.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V6HNx6cyr4J4CiMGCy1wkF

bastion/tests/integration/data-disks.test.ts

@@ -91,7 +91,10 @@ describe("dedicated data disks on real hardware", () => {
   }, 20 * 60_000);
 
   afterAll(async () => {
-    destroyVms(...created);
+    // KEEP_VMS=1 leaves the VM behind for a post-mortem (virsh console,
+    // screenshot, mounting the qcow2). Without it a failed boot is destroyed
+    // along with every clue about why it failed.
+    if (!process.env["KEEP_VMS"]) destroyVms(...created);
     try { await h?.close(); } catch { /* closed */ }
     try { destroyPxeNetwork(); } catch { /* gone */ }
   }, 5 * 60_000);

bastion/tests/integration/helpers/provision-harness.ts

@@ -14,7 +14,7 @@ import type { BastionConfig } from "../../../src/shared/src/types/config.js";
 import type { Role } from "../../../src/shared/src/types/state.js";
 import { log, waitForSsh } from "./libvirt.js";
 import { ensurePxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_BRIDGE, PXE_GATEWAY, PXE_SUBNET } from "./pxe-network.js";
-import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm } from "./pxe-vm.js";
+import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./pxe-vm.js";
 import { sshExec } from "./ssh.js";
 
 export const DISCOVERY_TIMEOUT_MS = 6 * 60_000;
@@ -295,7 +295,23 @@ async function waitForInstallStart(
   }
 }
 
-export interface ProvisionedVm { name: string; mac: string; ip: string }
+export interface ProvisionedVm { name: string; mac: string; ip: string; serialPort: number }
+
+/**
+ * "SSH never came up" on its own is undiagnosable once afterAll has destroyed
+ * the VM -- the machine may be sitting in emergency mode with the exact error
+ * on its serial console. Capture that console into the failure.
+ */
+async function sshFailurePostMortem(name: string, serialPort: number, cause: unknown): Promise<never> {
+  let state = "unknown";
+  try { state = execSync(`sudo virsh domstate ${name}`, { encoding: "utf-8" }).trim(); } catch { /* gone */ }
+  let serial = "(no serial output)";
+  try { serial = await readSerialLog(serialPort, { lastLines: 60 }); } catch { /* no console */ }
+  throw new Error(
+    `[${name}] ${cause instanceof Error ? cause.message : String(cause)}\n` +
+    `domstate: ${state}\n--- serial console (last lines) ---\n${serial}`,
+  );
+}
 
 /**
  * Serial ports are a global resource on the host, so hand out a distinct one
@@ -332,13 +348,14 @@ export async function provisionVm(h: Harness, spec: {
   const vmDataDisks = [spec.dataDisks?.longhorn, spec.dataDisks?.rancher]
     .filter((d): d is { serial: string; sizeGb: number } => d !== undefined);
   log(`[${spec.name}] creating VM (${spec.diskGb}GB disk + ${vmDataDisks.length} data disk(s), role=${spec.role}, schema=v${spec.schemaVersion ?? 1})`);
+  const serialPort = nextSerialPort++;
   createPxeVm({
     name: spec.name,
     memory: spec.memory ?? 4096,
     vcpus: spec.vcpus ?? 4,
     diskSize: spec.diskGb,
     network: PXE_NETWORK_NAME,
-    serialPort: nextSerialPort++,
+    serialPort,
     ...(vmDataDisks.length > 0 ? { dataDisks: vmDataDisks } : {}),
   });
   deleteNftablesRejectRules();
@@ -400,9 +417,13 @@ export async function provisionVm(h: Harness, spec: {
   deleteNftablesRejectRules();
 
   log(`[${spec.name}] waiting for SSH at ${ip}...`);
+  try {
     await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath);
+  } catch (e) {
+    await sshFailurePostMortem(spec.name, serialPort, e);
+  }
   log(`[${spec.name}] up at ${ip}`);
-  return { name: spec.name, mac, ip };
+  return { name: spec.name, mac, ip, serialPort };
 }
 
 /**
@@ -522,7 +543,11 @@ export async function reprovisionVm(
   } catch { /* already running */ }
   deleteNftablesRejectRules();
 
+  try {
     await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath);
+  } catch (e) {
+    await sshFailurePostMortem(vm.name, vm.serialPort, e);
+  }
   log(`[${vm.name}] back up at ${ip} after reinstall`);
   return { ...vm, ip };
 }