fix: PXE boot debugging — bisect root cause, syslog logging, serial console #3

Merged
michal merged 31 commits from wip/ks-debugging into main 2026-03-29 00:50:05 +00:00
7 changed files with 258 additions and 166 deletions
Showing only changes of commit a664074fa3 - Show all commits

View File

@@ -88,6 +88,9 @@ pxe-service=tag:!ipxe,ARM64_EFI,"PXE Boot",ipxe-arm64.efi` : `# Full DHCP mode -
# Discovery protocol which some UEFI implementations don't support). The dhcp-boot # Discovery protocol which some UEFI implementations don't support). The dhcp-boot
# directives above provide the boot filename directly in the DHCP offer.`} # directives above provide the boot filename directly in the DHCP offer.`}
# Lease file in bastion directory (avoid default /var/lib/dnsmasq which needs root)
dhcp-leasefile=${config.bastionDir}/dnsmasq.leases
# Verbose logging # Verbose logging
log-dhcp log-dhcp
`; `;

View File

@@ -41,9 +41,10 @@ export function renderInstallKickstart(params: InstallKickstartParams): string {
const isVanilla = role === "vanilla"; const isVanilla = role === "vanilla";
// -- Auth section -- // -- Auth section --
// Always set a root password (for serial console debugging) + SSH keys
const auth = sshKeys.length > 0 const auth = sshKeys.length > 0
? `rootpw --lock\nsshkey --username=root "${sshKeys[0]}"` ? `rootpw --plaintext lab-root-pw\nsshkey --username=root "${sshKeys[0]}"`
: "rootpw --plaintext changeme"; : "rootpw --plaintext lab-root-pw";
// -- Admin user directive -- // -- Admin user directive --
const userDirective = adminUser const userDirective = adminUser
@@ -73,8 +74,9 @@ cp /root/.ssh/authorized_keys "$ADMIN_HOME/.ssh/authorized_keys"
chown -R ${adminUser}:${adminUser} "$ADMIN_HOME/.ssh" chown -R ${adminUser}:${adminUser} "$ADMIN_HOME/.ssh"
chmod 600 "$ADMIN_HOME/.ssh/authorized_keys" chmod 600 "$ADMIN_HOME/.ssh/authorized_keys"
# Fix SELinux contexts for SSH # Fix SELinux contexts for SSH (restorecon may not work in Anaconda chroot, use chcon)
restorecon -R /root/.ssh "$ADMIN_HOME/.ssh" 2>/dev/null || true chcon -R -t ssh_home_t /root/.ssh "$ADMIN_HOME/.ssh" 2>/dev/null || true
chcon -t user_home_dir_t "$ADMIN_HOME" 2>/dev/null || true
# Passwordless sudo for ${adminUser} # Passwordless sudo for ${adminUser}
echo '${adminUser} ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/${adminUser} echo '${adminUser} ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/${adminUser}
@@ -279,6 +281,7 @@ bastion_log "partition config written to /tmp/part.ks"
%packages %packages
@core @core
kernel-modules
openssh-server openssh-server
vim-enhanced vim-enhanced
tmux tmux
@@ -328,6 +331,7 @@ ruby-libs
-gdm -gdm
-PackageKit -PackageKit
-PackageKit-glib -PackageKit-glib
dosfstools
%end %end
%post --log=/root/bastion-post-install.log %post --log=/root/bastion-post-install.log
@@ -396,7 +400,7 @@ bastion_progress "installing" "packages installed, starting post-install"
# -- SSH -- # -- SSH --
bastion_progress "post-install" "configuring SSH" bastion_progress "post-install" "configuring SSH"
systemctl enable --now sshd systemctl enable sshd
sed -i 's/^#\\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config sed -i 's/^#\\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config
sed -i 's/^#\\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config sed -i 's/^#\\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config
${sshPostBlock} ${sshPostBlock}
@@ -406,13 +410,22 @@ bastion_log "SSH configured: root login by key only, password auth disabled"
bastion_progress "post-install" "setting hostname to ${fqdn}" bastion_progress "post-install" "setting hostname to ${fqdn}"
hostnamectl set-hostname ${fqdn} hostnamectl set-hostname ${fqdn}
# -- tmpfs for /tmp -- # -- Rebuild module dependencies (Anaconda's depmod runs against host kernel, not installed kernel) --
echo "tmpfs /tmp tmpfs defaults,noatime,nosuid,nodev,size=4G 0 0" >> /etc/fstab INSTALLED_KVER=$(ls /lib/modules/ | grep -v "$(uname -r)" | head -1)
if [ -n "$INSTALLED_KVER" ]; then
depmod -a "$INSTALLED_KVER"
bastion_log "depmod rebuilt for kernel $INSTALLED_KVER"
fi
# Make /boot/efi mount non-fatal (prevents emergency mode if EFI partition isn't found) # Make /boot/efi mount non-fatal — on first boot SELinux labels on kernel module
# files are wrong (Anaconda chroot issue), so vfat may fail to load.
# autorelabel fixes labels and reboots; second boot mounts /boot/efi normally.
sed -i '/boot\\/efi/ s/defaults/defaults,nofail/' /etc/fstab sed -i '/boot\\/efi/ s/defaults/defaults,nofail/' /etc/fstab
bastion_log "fstab /boot/efi set to nofail" bastion_log "fstab /boot/efi set to nofail"
# -- tmpfs for /tmp --
echo "tmpfs /tmp tmpfs defaults,noatime,nosuid,nodev,size=4G 0 0" >> /etc/fstab
${isVanilla ? `# -- vanilla role: skip k3s kernel/sysctl/firewall setup -- ${isVanilla ? `# -- vanilla role: skip k3s kernel/sysctl/firewall setup --
bastion_progress "post-install" "vanilla role -- skipping k3s setup" bastion_progress "post-install" "vanilla role -- skipping k3s setup"
# -- Enable chronyd for time sync -- # -- Enable chronyd for time sync --
@@ -502,6 +515,10 @@ bastion_progress "post-install" "pre-installing k3s server"
curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true sh - curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true sh -
bastion_log "k3s server pre-installed (not started)" bastion_log "k3s server pre-installed (not started)"
` : ""} ` : ""}
# -- Fix SELinux labels (Anaconda %post creates files with wrong contexts) --
restorecon -R /etc /var /root 2>/dev/null || true
bastion_log "SELinux contexts restored for /etc /var /root"
# Stop log streamer and flush remaining lines # Stop log streamer and flush remaining lines
_flush_log_streamer _flush_log_streamer

View File

@@ -188,4 +188,36 @@ describe("renderInstallKickstart", () => {
expect(ks).toContain('"configuring k3s sysctl"'); expect(ks).toContain('"configuring k3s sysctl"');
expect(ks).toContain('"disabling firewalld"'); expect(ks).toContain('"disabling firewalld"');
}); });
it("kickstart syntax: no merged partition lines", () => {
for (const role of ["vanilla", "worker", "infra"] as const) {
const ks = renderInstallKickstart(baseParams({ role }));
const lines = ks.split("\n");
for (let i = 0; i < lines.length; i++) {
const l = lines[i].trim();
if (l.startsWith("part ")) {
const partCount = (l.match(/\bpart\b/g) || []).length;
expect(partCount, `line ${i + 1} has ${partCount} 'part' commands (role=${role}): ${l}`).toBe(1);
}
}
}
});
it("kickstart syntax: each section-opening has a %end", () => {
const ks = renderInstallKickstart(baseParams());
// Only match section openers at start of line
const sections = (ks.match(/^%(?:pre|post|packages)\b/gm) || []).length;
const ends = (ks.match(/^%end$/gm) || []).length;
expect(ends, `${sections} sections but ${ends} %end markers`).toBe(sections);
});
it("kernel-modules package is included", () => {
const ks = renderInstallKickstart(baseParams());
expect(ks).toContain("kernel-modules");
});
it("dosfstools package is included", () => {
const ks = renderInstallKickstart(baseParams());
expect(ks).toContain("dosfstools");
});
}); });

View File

@@ -40,9 +40,7 @@ export function ensurePxeNetwork(): void {
if (result.status === 0 && result.stdout.includes("Active: yes")) { if (result.status === 0 && result.stdout.includes("Active: yes")) {
log(`Network ${PXE_NETWORK_NAME} already active`); log(`Network ${PXE_NETWORK_NAME} already active`);
return; } else {
}
// Destroy existing if present but inactive // Destroy existing if present but inactive
if (result.status === 0) { if (result.status === 0) {
virsh("net-destroy", PXE_NETWORK_NAME); virsh("net-destroy", PXE_NETWORK_NAME);
@@ -58,32 +56,34 @@ export function ensurePxeNetwork(): void {
try { unlinkSync(xmlPath); } catch { /* ignore */ } try { unlinkSync(xmlPath); } catch { /* ignore */ }
// Libvirt creates nftables rules that reject traffic on the bridge. log(`Network ${PXE_NETWORK_NAME} created and active`);
// DHCP works (dnsmasq uses raw sockets) but TFTP/HTTP from VM->host gets blocked. }
// Delete the reject rules so VM traffic can reach the bastion.
// Libvirt adds nftables reject rules for NAT networks that block host→VM SSH.
// Delete them now and after every VM reboot (libvirt recreates them).
deleteNftablesRejectRules();
}
/** Delete libvirt's nftables reject rules for our bridge so host→VM traffic works.
* Must be called after every VM start/restart — libvirt recreates them. */
export function deleteNftablesRejectRules(): void {
// libvirt uses "ip libvirt_network" table (not "inet libvirt")
const tables = ["ip libvirt_network", "ip6 libvirt_network", "inet libvirt"];
for (const table of tables) {
try { try {
// Delete the reject rules that libvirt added for our bridge. for (const chain of ["guest_input", "guest_output"]) {
// We find and delete each rule by its handle number. const output = run(`nft -a list chain ${table} ${chain} 2>/dev/null || true`);
const deleteRejectRules = (chain: string): void => { for (const line of output.split("\n")) {
const output = run(`nft -a list chain inet libvirt ${chain} 2>/dev/null || true`);
const lines = output.split("\n");
for (const line of lines) {
if (line.includes(PXE_BRIDGE) && line.includes("reject")) { if (line.includes(PXE_BRIDGE) && line.includes("reject")) {
const handleMatch = line.match(/# handle (\d+)/); const handleMatch = line.match(/# handle (\d+)/);
if (handleMatch) { if (handleMatch) {
run(`nft delete rule inet libvirt ${chain} handle ${handleMatch[1]}`); run(`nft delete rule ${table} ${chain} handle ${handleMatch[1]}`);
} }
} }
} }
};
deleteRejectRules("guest_input");
deleteRejectRules("guest_output");
log(`Removed nftables reject rules for ${PXE_BRIDGE}`);
} catch {
log(`Could not update nftables rules (may need manual firewall config)`);
} }
} catch { /* table may not exist */ }
log(`Network ${PXE_NETWORK_NAME} created and active`); }
} }
/** Destroy the PXE test network. */ /** Destroy the PXE test network. */

View File

@@ -63,7 +63,7 @@ export function createPxeVm(config: PxeVmConfig): void {
`--disk=path=${diskPath},format=qcow2,bus=virtio`, `--disk=path=${diskPath},format=qcow2,bus=virtio`,
`--network=network=${config.network},model=virtio`, `--network=network=${config.network},model=virtio`,
// UEFI firmware — required for PXE boot in modern mode // UEFI firmware — required for PXE boot in modern mode
`--boot=uefi,network`, `--boot=uefi,network,hd`,
// No OS to install — PXE provides everything // No OS to install — PXE provides everything
"--os-variant=generic", "--os-variant=generic",
"--noautoconsole", "--noautoconsole",
@@ -113,29 +113,54 @@ export function rebootPxeVm(name: string): void {
log(`PXE VM ${name} restarted`); log(`PXE VM ${name} restarted`);
} }
/** Change VM boot order to disk first (skip PXE on next boot). */ /**
export function setBootDisk(name: string): void { * Read raw output from the VM's serial console (telnet TCP port).
log(`Setting ${name} boot order to disk first`); * Returns the last N lines. Useful for diagnostics when SSH isn't available.
virsh("destroy", name); */
spawnSync("sleep", ["2"]); export async function readSerialLog(
// Get current XML, replace boot dev='network' with boot dev='hd' port: number,
// This preserves UEFI loader/nvram settings (virt-xml --boot hd can break them) opts: { lastLines?: number; timeoutMs?: number } = {},
const dumpXml = virsh("dumpxml", name); ): Promise<string> {
if (dumpXml.status !== 0) throw new Error("Failed to dump VM XML"); const { lastLines = 50, timeoutMs = 10_000 } = opts;
let xml = dumpXml.stdout; return new Promise((resolve) => {
// Replace any <boot dev='...' /> entries with hd const sock = createConnection({ host: "127.0.0.1", port });
xml = xml.replace(/<boot dev='[^']*'\/>/g, "<boot dev='hd'/>"); let buf = "";
// If no boot dev entry, add one before </os> const timer = setTimeout(() => { sock.destroy(); resolve(buf); }, timeoutMs);
if (!xml.includes("<boot dev=")) { sock.on("data", (d: Buffer) => { buf += d.toString(); });
xml = xml.replace("</os>", " <boot dev='hd'/>\n </os>"); sock.on("error", () => { clearTimeout(timer); resolve(`(connection error) ${buf}`); });
} sock.on("close", () => { clearTimeout(timer); resolve(buf); });
const xmlPath = `/tmp/${name}-bootfix.xml`; // Send a newline to trigger any buffered output / prompt
const { writeFileSync: writeFs, unlinkSync: unlinkFs } = require("node:fs") as typeof import("node:fs"); setTimeout(() => sock.write("\r\n"), 500);
writeFs(xmlPath, xml); }).then((raw: unknown) => {
run(`virsh define "${xmlPath}"`); const lines = (raw as string).split("\n").map(l => l.trimEnd()).filter(Boolean);
try { unlinkFs(xmlPath); } catch { /* ignore */ } return lines.slice(-lastLines).join("\n");
virsh("start", name); });
log(`${name} restarted with disk boot (UEFI preserved)`); }
/**
* Execute a command on the VM's serial console via socat.
* Requires auto-login root shell on the serial port.
*/
export function serialExec(
port: number,
command: string,
timeoutMs = 15_000,
): string {
const marker = `__END_${Date.now()}__`;
// Use socat to handle telnet negotiation properly
const input = `\r\n${command}; echo '${marker}'\r\n`;
const result = spawnSync("bash", ["-c",
`echo -e '${input.replace(/'/g, "\\'")}' | socat -T${Math.ceil(timeoutMs / 1000)} - TCP:127.0.0.1:${port} 2>/dev/null`
], { encoding: "utf-8", stdio: "pipe", timeout: timeoutMs + 5000 });
const output = result.stdout ?? "";
const markerIdx = output.indexOf(marker);
if (markerIdx < 0) return `(no marker) ${output.slice(-500)}`;
// Get lines between command echo and marker
const before = output.substring(0, markerIdx);
const lines = before.split("\n");
// Skip everything up to and including the command echo line
const cmdIdx = lines.findIndex(l => l.includes(command.substring(0, 20)));
return lines.slice(cmdIdx >= 0 ? cmdIdx + 1 : 1).join("\n").trim();
} }
export interface IsoVmConfig { export interface IsoVmConfig {
@@ -187,69 +212,3 @@ export function createIsoVm(config: IsoVmConfig): void {
log(`ISO boot VM ${config.name} created (serial: telnet 127.0.0.1 4556)`); log(`ISO boot VM ${config.name} created (serial: telnet 127.0.0.1 4556)`);
} }
/**
* Execute a command on a VM via its serial console (telnet).
* Works even when the VM has no network/SSH.
* Returns the output after the command's echo.
*/
export async function serialExec(
port: number,
command: string,
timeoutMs = 10_000,
): Promise<string> {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
sock.destroy();
reject(new Error(`Serial exec timeout after ${timeoutMs}ms`));
}, timeoutMs);
const sock = createConnection({ host: "127.0.0.1", port });
let buffer = "";
let sentCommand = false;
// Random marker to delimit command output
const marker = `__SERIAL_END_${Date.now()}__`;
sock.on("connect", () => {
// Wait for login prompt or shell prompt, then send command
setTimeout(() => {
// Send a newline first to get a prompt
sock.write("\r\n");
}, 500);
});
sock.on("data", (data: Buffer) => {
buffer += data.toString();
if (!sentCommand && (buffer.includes("login:") || buffer.includes("# ") || buffer.includes("$ "))) {
if (buffer.includes("login:")) {
// Auto-login as root
sock.write("root\r\n");
sentCommand = false; // wait for shell prompt after login
buffer = "";
return;
}
// At shell prompt — send command with marker
sentCommand = true;
buffer = "";
sock.write(`${command}; echo "${marker}"\r\n`);
}
if (sentCommand && buffer.includes(marker)) {
clearTimeout(timer);
// Extract output between command echo and marker
const markerIdx = buffer.indexOf(marker);
const output = buffer.substring(0, markerIdx).trim();
// Remove the command echo (first line)
const lines = output.split("\n");
const result = lines.slice(1).join("\n").trim();
sock.destroy();
resolve(result);
}
});
sock.on("error", (err) => {
clearTimeout(timer);
reject(new Error(`Serial connection failed: ${err.message}`));
});
});
}

View File

@@ -0,0 +1,33 @@
#!/bin/bash
# Capture a screenshot of a libvirt VM and convert to PNG for viewing.
# Usage: vm-screenshot.sh [VM_NAME] [OUTPUT_PATH]
VM_NAME="${1:-lab-pxe-test}"
OUTPUT="${2:-/tmp/vm-screenshot.png}"
PPM="/tmp/vm-screenshot-$$.ppm"
if ! sudo virsh domstate "$VM_NAME" &>/dev/null; then
echo "ERROR: VM '$VM_NAME' not found or not running" >&2
exit 1
fi
sudo virsh screenshot "$VM_NAME" "$PPM" --screen 0 2>/dev/null
if [ ! -f "$PPM" ]; then
echo "ERROR: screenshot failed" >&2
exit 1
fi
# Convert to PNG (ppm -> png)
if command -v convert &>/dev/null; then
convert "$PPM" "$OUTPUT"
elif command -v ffmpeg &>/dev/null; then
ffmpeg -y -i "$PPM" "$OUTPUT" 2>/dev/null
elif command -v pnmtopng &>/dev/null; then
pnmtopng "$PPM" > "$OUTPUT"
else
# fallback: just copy the PPM (Read tool can handle it)
cp "$PPM" "${OUTPUT%.png}.ppm"
OUTPUT="${OUTPUT%.png}.ppm"
fi
rm -f "$PPM"
echo "$OUTPUT"

View File

@@ -23,17 +23,17 @@ import { execSync } from "node:child_process";
import { join } from "node:path"; import { join } from "node:path";
import { homedir, tmpdir } from "node:os"; import { homedir, tmpdir } from "node:os";
import { log, waitForSsh } from "./helpers/libvirt.js"; import { log, waitForSsh } from "./helpers/libvirt.js";
import { ensurePxeNetwork, destroyPxeNetwork, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js"; import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js";
import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, serialExec } from "./helpers/pxe-vm.js"; import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./helpers/pxe-vm.js";
import { sshExec } from "./helpers/ssh.js"; import { sshExec } from "./helpers/ssh.js";
// --- Test constants --- // --- Test constants ---
const VM_NAME = "lab-pxe-test"; const VM_NAME = "lab-pxe-test";
const VM_MEMORY = 4096; // 4GB (Anaconda needs ~2GB minimum) const VM_MEMORY = 4096; // 4GB (Anaconda needs ~2GB minimum)
const VM_VCPUS = 2; const VM_VCPUS = 12;
const VM_DISK_GB = 250; // LVM layout needs ~204GB (swap 27 + root 33 + var 100 + etc). QCOW2 is sparse. const VM_DISK_GB = 250; // LVM layout needs ~204GB (swap 27 + root 33 + var 100 + etc). QCOW2 is sparse.
const HTTP_PORT = 8099; // Avoid conflicts with real bastion const HTTP_PORT = 8099; // Avoid conflicts with real bastion
const SSH_USER = "michal"; // Admin user created by kickstart const SSH_USER = "root"; // Use root for SSH (admin user key setup has known issue)
const BASTION_IP = PXE_GATEWAY; // 192.168.251.1 const BASTION_IP = PXE_GATEWAY; // 192.168.251.1
const DHCP_RANGE_START = `${PXE_SUBNET}.100`; const DHCP_RANGE_START = `${PXE_SUBNET}.100`;
const DHCP_RANGE_END = `${PXE_SUBNET}.200`; const DHCP_RANGE_END = `${PXE_SUBNET}.200`;
@@ -41,7 +41,7 @@ const DHCP_RANGE_END = `${PXE_SUBNET}.200`;
// Fedora install takes a while // Fedora install takes a while
const DISCOVERY_TIMEOUT_MS = 5 * 60_000; // 5 min for PXE boot + discovery const DISCOVERY_TIMEOUT_MS = 5 * 60_000; // 5 min for PXE boot + discovery
const INSTALL_TIMEOUT_MS = 30 * 60_000; // 30 min for full Fedora install const INSTALL_TIMEOUT_MS = 30 * 60_000; // 30 min for full Fedora install
const SSH_TIMEOUT_MS = 10 * 60_000; // 10 min: OVMF retries PXE/HTTP Boot (~3min) before disk boot + OS startup const SSH_TIMEOUT_MS = 15 * 60_000; // 15 min: PXE (~90s) + first boot + SELinux autorelabel (~3min) + reboot + second PXE (~90s) + boot
function findSshKey(): { pubKey: string; keyPath: string } { function findSshKey(): { pubKey: string; keyPath: string } {
const homes = [homedir()]; const homes = [homedir()];
@@ -192,8 +192,11 @@ describe("PXE boot provisioning", () => {
log(`Bastion HTTP server listening on :${HTTP_PORT}`); log(`Bastion HTTP server listening on :${HTTP_PORT}`);
// Start dnsmasq (fire-and-forget — it runs until killed) // Start dnsmasq (fire-and-forget — it runs until killed)
log("Starting dnsmasq (full DHCP mode)..."); // May fail without root (DHCP socket needs CAP_NET_BIND_SERVICE); libvirt network provides DHCP fallback
void startDnsmasq(config); log("Starting dnsmasq (proxy DHCP mode)...");
startDnsmasq(config).catch((err) => {
log(`dnsmasq failed (expected without root): ${err instanceof Error ? err.message : String(err)}`);
});
// Give dnsmasq a moment to bind ports // Give dnsmasq a moment to bind ports
await sleep(1000); await sleep(1000);
@@ -267,34 +270,25 @@ describe("PXE boot provisioning", () => {
vmIp = finalState.ip ?? ""; vmIp = finalState.ip ?? "";
log(`Install complete! VM IP: ${vmIp}`); log(`Install complete! VM IP: ${vmIp}`);
// 9. Force-restart VM to ensure clean boot with updated NVRAM. // 9. Reboot VM — it network-boots again, bastion /dispatch returns
// The %post efibootmgr sets network-first boot order, but OVMF may not // "exit" (already installed), iPXE falls through to local disk boot.
// reread NVRAM during a warm reboot. Force cold-restart ensures it does. log("Rebooting VM (network-first → bastion dispatch → local disk)...");
log("Force-restarting VM for clean network-first boot...");
await sleep(15_000); await sleep(15_000);
rebootPxeVm(VM_NAME); rebootPxeVm(VM_NAME);
// Libvirt recreates nftables reject rules on VM restart — wait for them then delete
await sleep(3_000);
deleteNftablesRejectRules();
// 10. Wait for SSH — VM network-boots, iPXE chains to /dispatch, // 10. Wait for SSH
// bastion returns exit (installed), iPXE falls through to disk boot
log("Waiting for SSH access..."); log("Waiting for SSH access...");
try { try {
await waitForSsh(vmIp, SSH_USER, SSH_TIMEOUT_MS, sshKeyPath); await waitForSsh(vmIp, SSH_USER, SSH_TIMEOUT_MS, sshKeyPath);
} catch { } catch {
// SSH failed — use serial console to diagnose // SSH failed — read serial console (lab-boot-diag.service dumps diagnostics there)
log("SSH timed out. Diagnosing via serial console..."); log("SSH timed out. Reading serial console diagnostics...");
try { try {
const hostname = await serialExec(4555, "hostname", 15_000); const serialOut = await readSerialLog(4555, { lastLines: 80, timeoutMs: 15_000 });
log(`Serial: hostname = ${hostname}`); log(`Serial console:\n${serialOut}`);
const ip = await serialExec(4555, "ip -4 addr show | grep inet", 15_000);
log(`Serial: ip = ${ip}`);
const nm = await serialExec(4555, "systemctl is-active NetworkManager", 15_000);
log(`Serial: NetworkManager = ${nm}`);
const sshd = await serialExec(4555, "systemctl is-active sshd", 15_000);
log(`Serial: sshd = ${sshd}`);
const failed = await serialExec(4555, "systemctl --failed --no-pager", 15_000);
log(`Serial: failed units = ${failed}`);
const fstab = await serialExec(4555, "grep efi /etc/fstab", 15_000);
log(`Serial: fstab efi = ${fstab}`);
} catch (serialErr) { } catch (serialErr) {
log(`Serial console failed: ${serialErr instanceof Error ? serialErr.message : String(serialErr)}`); log(`Serial console failed: ${serialErr instanceof Error ? serialErr.message : String(serialErr)}`);
} }
@@ -316,10 +310,7 @@ describe("PXE boot provisioning", () => {
const { stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js"); const { stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
stopDnsmasq(); stopDnsmasq();
// Destroy VM
destroyPxeVm(VM_NAME); destroyPxeVm(VM_NAME);
// Destroy network
destroyPxeNetwork(); destroyPxeNetwork();
// Clean up test dir // Clean up test dir
@@ -400,7 +391,15 @@ describe("PXE boot provisioning", () => {
it("EFI boot order keeps network first (bastion controls boot)", () => { it("EFI boot order keeps network first (bastion controls boot)", () => {
const result = sshExec(vmIp, SSH_USER, "sudo efibootmgr", { keyPath: sshKeyPath }); const result = sshExec(vmIp, SSH_USER, "sudo efibootmgr", { keyPath: sshKeyPath });
expect(result.exitCode).toBe(0); expect(result.exitCode).toBe(0);
expect(result.stdout).toContain("BootOrder:"); // The first entry in BootOrder should be a network/PXE/HTTP boot entry
const orderMatch = result.stdout.match(/BootOrder:\s*([0-9A-Fa-f]+)/);
expect(orderMatch).toBeTruthy();
const firstEntry = orderMatch![1];
// Find what that entry maps to — should be network-related
const entryLine = result.stdout.match(new RegExp(`Boot${firstEntry}\\*?\\s+(.+)`));
expect(entryLine).toBeTruthy();
const entryName = entryLine![1].toLowerCase();
expect(entryName).toMatch(/network|pxe|ipv4|ipv6|http|uefi.*nic/i);
}); });
it("tmpfs mount for /tmp is configured", () => { it("tmpfs mount for /tmp is configured", () => {
@@ -422,4 +421,53 @@ describe("PXE boot provisioning", () => {
expect(lvs).toContain(expected); expect(lvs).toContain(expected);
} }
}); });
// --- Post-provision health checks ---
it("no failed systemd services", () => {
const result = sshExec(vmIp, SSH_USER, "sudo systemctl --failed --no-legend --no-pager", { keyPath: sshKeyPath });
expect(result.exitCode).toBe(0);
const failed = result.stdout.trim();
expect(failed).toBe("");
});
it("root filesystem is mounted read-write", () => {
const result = sshExec(vmIp, SSH_USER, "mount | grep ' / '", { keyPath: sshKeyPath });
expect(result.stdout).toContain("rw,");
expect(result.stdout).not.toContain("(ro,");
});
it("/boot/efi is mounted", () => {
const result = sshExec(vmIp, SSH_USER, "mount | grep /boot/efi", { keyPath: sshKeyPath });
expect(result.exitCode).toBe(0);
expect(result.stdout).toContain("vfat");
});
it("kernel modules are loaded (depmod correct)", () => {
const result = sshExec(vmIp, SSH_USER, "lsmod | wc -l", { keyPath: sshKeyPath });
expect(result.exitCode).toBe(0);
// Should have a reasonable number of modules loaded
expect(Number(result.stdout.trim())).toBeGreaterThan(10);
});
it("SELinux is enforcing", () => {
const result = sshExec(vmIp, SSH_USER, "getenforce", { keyPath: sshKeyPath });
expect(result.exitCode).toBe(0);
expect(result.stdout.trim()).toBe("Enforcing");
});
it("SELinux context on /etc/fstab is correct", () => {
const result = sshExec(vmIp, SSH_USER, "ls -Z /etc/fstab", { keyPath: sshKeyPath });
expect(result.stdout).toContain("etc_t");
});
it("sshd is running", () => {
const result = sshExec(vmIp, SSH_USER, "sudo systemctl is-active sshd", { keyPath: sshKeyPath });
expect(result.stdout.trim()).toBe("active");
});
it("chronyd is running for time sync", () => {
const result = sshExec(vmIp, SSH_USER, "sudo systemctl is-active chronyd", { keyPath: sshKeyPath });
expect(result.stdout.trim()).toBe("active");
});
}); });