wip: save current ks debugging state before bisect revert

All accumulated changes to kickstart template, test infrastructure,
and dnsmasq config. None of these produce a clean boot yet — saving
state before reverting to baseline for bisection.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

bastion/tests/integration/helpers/pxe-vm.ts

@@ -63,7 +63,7 @@ export function createPxeVm(config: PxeVmConfig): void {
     `--disk=path=${diskPath},format=qcow2,bus=virtio`,
     `--network=network=${config.network},model=virtio`,
     // UEFI firmware — required for PXE boot in modern mode
-    `--boot=uefi,network`,
+    `--boot=uefi,network,hd`,
     // No OS to install — PXE provides everything
     "--os-variant=generic",
     "--noautoconsole",
@@ -113,29 +113,54 @@ export function rebootPxeVm(name: string): void {
   log(`PXE VM ${name} restarted`);
 }
 
-/** Change VM boot order to disk first (skip PXE on next boot). */
-export function setBootDisk(name: string): void {
-  log(`Setting ${name} boot order to disk first`);
-  virsh("destroy", name);
-  spawnSync("sleep", ["2"]);
-  // Get current XML, replace boot dev='network' with boot dev='hd'
-  // This preserves UEFI loader/nvram settings (virt-xml --boot hd can break them)
-  const dumpXml = virsh("dumpxml", name);
-  if (dumpXml.status !== 0) throw new Error("Failed to dump VM XML");
-  let xml = dumpXml.stdout;
-  // Replace any <boot dev='...' /> entries with hd
-  xml = xml.replace(/<boot dev='[^']*'\/>/g, "<boot dev='hd'/>");
-  // If no boot dev entry, add one before </os>
-  if (!xml.includes("<boot dev=")) {
-    xml = xml.replace("</os>", "  <boot dev='hd'/>\n  </os>");
-  }
-  const xmlPath = `/tmp/${name}-bootfix.xml`;
-  const { writeFileSync: writeFs, unlinkSync: unlinkFs } = require("node:fs") as typeof import("node:fs");
-  writeFs(xmlPath, xml);
-  run(`virsh define "${xmlPath}"`);
-  try { unlinkFs(xmlPath); } catch { /* ignore */ }
-  virsh("start", name);
-  log(`${name} restarted with disk boot (UEFI preserved)`);
+/**
+ * Read raw output from the VM's serial console (telnet TCP port).
+ * Returns the last N lines. Useful for diagnostics when SSH isn't available.
+ */
+export async function readSerialLog(
+  port: number,
+  opts: { lastLines?: number; timeoutMs?: number } = {},
+): Promise<string> {
+  const { lastLines = 50, timeoutMs = 10_000 } = opts;
+  return new Promise((resolve) => {
+    const sock = createConnection({ host: "127.0.0.1", port });
+    let buf = "";
+    const timer = setTimeout(() => { sock.destroy(); resolve(buf); }, timeoutMs);
+    sock.on("data", (d: Buffer) => { buf += d.toString(); });
+    sock.on("error", () => { clearTimeout(timer); resolve(`(connection error) ${buf}`); });
+    sock.on("close", () => { clearTimeout(timer); resolve(buf); });
+    // Send a newline to trigger any buffered output / prompt
+    setTimeout(() => sock.write("\r\n"), 500);
+  }).then((raw: unknown) => {
+    const lines = (raw as string).split("\n").map(l => l.trimEnd()).filter(Boolean);
+    return lines.slice(-lastLines).join("\n");
+  });
+}
+
+/**
+ * Execute a command on the VM's serial console via socat.
+ * Requires auto-login root shell on the serial port.
+ */
+export function serialExec(
+  port: number,
+  command: string,
+  timeoutMs = 15_000,
+): string {
+  const marker = `__END_${Date.now()}__`;
+  // Use socat to handle telnet negotiation properly
+  const input = `\r\n${command}; echo '${marker}'\r\n`;
+  const result = spawnSync("bash", ["-c",
+    `echo -e '${input.replace(/'/g, "\\'")}' | socat -T${Math.ceil(timeoutMs / 1000)} - TCP:127.0.0.1:${port} 2>/dev/null`
+  ], { encoding: "utf-8", stdio: "pipe", timeout: timeoutMs + 5000 });
+  const output = result.stdout ?? "";
+  const markerIdx = output.indexOf(marker);
+  if (markerIdx < 0) return `(no marker) ${output.slice(-500)}`;
+  // Get lines between command echo and marker
+  const before = output.substring(0, markerIdx);
+  const lines = before.split("\n");
+  // Skip everything up to and including the command echo line
+  const cmdIdx = lines.findIndex(l => l.includes(command.substring(0, 20)));
+  return lines.slice(cmdIdx >= 0 ? cmdIdx + 1 : 1).join("\n").trim();
 }
 
 export interface IsoVmConfig {
@@ -187,69 +212,3 @@ export function createIsoVm(config: IsoVmConfig): void {
   log(`ISO boot VM ${config.name} created (serial: telnet 127.0.0.1 4556)`);
 }
 
-/**
- * Execute a command on a VM via its serial console (telnet).
- * Works even when the VM has no network/SSH.
- * Returns the output after the command's echo.
- */
-export async function serialExec(
-  port: number,
-  command: string,
-  timeoutMs = 10_000,
-): Promise<string> {
-  return new Promise((resolve, reject) => {
-    const timer = setTimeout(() => {
-      sock.destroy();
-      reject(new Error(`Serial exec timeout after ${timeoutMs}ms`));
-    }, timeoutMs);
-
-    const sock = createConnection({ host: "127.0.0.1", port });
-    let buffer = "";
-    let sentCommand = false;
-    // Random marker to delimit command output
-    const marker = `__SERIAL_END_${Date.now()}__`;
-
-    sock.on("connect", () => {
-      // Wait for login prompt or shell prompt, then send command
-      setTimeout(() => {
-        // Send a newline first to get a prompt
-        sock.write("\r\n");
-      }, 500);
-    });
-
-    sock.on("data", (data: Buffer) => {
-      buffer += data.toString();
-
-      if (!sentCommand && (buffer.includes("login:") || buffer.includes("# ") || buffer.includes("$ "))) {
-        if (buffer.includes("login:")) {
-          // Auto-login as root
-          sock.write("root\r\n");
-          sentCommand = false; // wait for shell prompt after login
-          buffer = "";
-          return;
-        }
-        // At shell prompt — send command with marker
-        sentCommand = true;
-        buffer = "";
-        sock.write(`${command}; echo "${marker}"\r\n`);
-      }
-
-      if (sentCommand && buffer.includes(marker)) {
-        clearTimeout(timer);
-        // Extract output between command echo and marker
-        const markerIdx = buffer.indexOf(marker);
-        const output = buffer.substring(0, markerIdx).trim();
-        // Remove the command echo (first line)
-        const lines = output.split("\n");
-        const result = lines.slice(1).join("\n").trim();
-        sock.destroy();
-        resolve(result);
-      }
-    });
-
-    sock.on("error", (err) => {
-      clearTimeout(timer);
-      reject(new Error(`Serial connection failed: ${err.message}`));
-    });
-  });
-}