feat: PXE debug boot mode for rescue/diagnostics #4

Merged
michal merged 16 commits from wip/ks-debugging into main 2026-03-30 02:59:35 +00:00
21 changed files with 305 additions and 129 deletions
Showing only changes of commit 0a4916d3c9 - Show all commits

View File

@@ -62,13 +62,13 @@ _labctl() {
COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur")) COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur"))
return ;; return ;;
"provision debug") "provision debug")
COMPREPLY=($(compgen -W "--sshd -h --help" -- "$cur")) COMPREPLY=($(compgen -W "--pxe-boot -h --help" -- "$cur"))
return ;; return ;;
"provision forget") "provision forget")
COMPREPLY=($(compgen -W "-h --help" -- "$cur")) COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
return ;; return ;;
"provision logs") "provision logs")
COMPREPLY=($(compgen -W "-h --help" -- "$cur")) COMPREPLY=($(compgen -W "-f --follow -h --help" -- "$cur"))
return ;; return ;;
"provision makeiso") "provision makeiso")
COMPREPLY=($(compgen -W "--arch --local --out -h --help" -- "$cur")) COMPREPLY=($(compgen -W "--arch --local --out -h --help" -- "$cur"))

View File

@@ -138,7 +138,10 @@ complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operatin
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l disk -d 'Target disk device (auto-detect if omitted)' -x complete -c labctl -n "__labctl_in_cmd provision reprovision" -l disk -d 'Target disk device (auto-detect if omitted)' -x
# provision debug options # provision debug options
complete -c labctl -n "__labctl_in_cmd provision debug" -l sshd -d 'Start SSH + nc listener automatically, report IP to bastion' complete -c labctl -n "__labctl_in_cmd provision debug" -l pxe-boot -d 'Boot installed system via PXE (kernel+initrd from network, root from NVMe)'
# provision logs options
complete -c labctl -n "__labctl_in_cmd provision logs" -s f -l follow -d 'Follow log output in real-time'
# provision makeiso options # provision makeiso options
complete -c labctl -n "__labctl_in_cmd provision makeiso" -l arch -d 'Target architecture(s)' -xa 'x86_64 aarch64' complete -c labctl -n "__labctl_in_cmd provision makeiso" -l arch -d 'Target architecture(s)' -xa 'x86_64 aarch64'

View File

@@ -257,7 +257,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
state.update((s) => { state.update((s) => {
s.install_queue[msg.mac] = { s.install_queue[msg.mac] = {
hostname: msg.hostname, hostname: msg.hostname,
disk: msg.disk ?? "/dev/sda", disk: msg.disk ?? "",
role: msg.role as import("@lab/shared").Role, role: msg.role as import("@lab/shared").Role,
os: msg.os as import("@lab/shared").OsId, os: msg.os as import("@lab/shared").OsId,
queued_at: new Date().toISOString(), queued_at: new Date().toISOString(),
@@ -269,7 +269,6 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
labdConn.onCommand("command-debug", async (msg) => { labdConn.onCommand("command-debug", async (msg) => {
if (msg.type !== "command-debug") throw new Error("unexpected"); if (msg.type !== "command-debug") throw new Error("unexpected");
const mac = msg.mac.toLowerCase(); const mac = msg.mac.toLowerCase();
const sshd = msg.sshd ?? false;
const pxeBoot = msg.pxeBoot ?? false; const pxeBoot = msg.pxeBoot ?? false;
const currentState = state.load(); const currentState = state.load();
const hostname = const hostname =
@@ -278,7 +277,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
currentState.discovered[mac]?.product ?? currentState.discovered[mac]?.product ??
mac; mac;
state.update((s) => { state.update((s) => {
s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd, pxeBoot }; s.debug[mac] = { hostname, queued_at: new Date().toISOString(), pxeBoot };
}); });
return { status: "ok", data: { mac, hostname } }; return { status: "ok", data: { mac, hostname } };
}); });

View File

@@ -13,11 +13,13 @@ import { triggerPostProvisionK3s } from "../services/post-provision.js";
import { progressBus } from "../services/progress-events.js"; import { progressBus } from "../services/progress-events.js";
import type { ProgressEvent } from "../services/progress-events.js"; import type { ProgressEvent } from "../services/progress-events.js";
import type { InstallLogBuffer } from "../services/install-log.js"; import type { InstallLogBuffer } from "../services/install-log.js";
import type { SyslogListener } from "../services/syslog-listener.js";
export function registerApiRoutes( export function registerApiRoutes(
app: FastifyInstance, app: FastifyInstance,
state: StateManager, state: StateManager,
installLog: InstallLogBuffer, installLog: InstallLogBuffer,
syslog: SyslogListener,
): void { ): void {
// List all machines // List all machines
app.get("/api/machines", async (_request, reply) => { app.get("/api/machines", async (_request, reply) => {
@@ -84,6 +86,11 @@ export function registerApiRoutes(
const { mac: rawMac, stage, detail } = request.body ?? {}; const { mac: rawMac, stage, detail } = request.body ?? {};
const mac = (rawMac ?? "unknown").toLowerCase(); const mac = (rawMac ?? "unknown").toLowerCase();
const stageName = stage ?? "unknown"; const stageName = stage ?? "unknown";
// Register IP → MAC for syslog routing
if (mac !== "unknown") {
syslog.registerIp(request.ip, mac);
}
const detailStr = detail ?? ""; const detailStr = detail ?? "";
const GREEN = "\x1b[0;32m"; const GREEN = "\x1b[0;32m";
@@ -191,10 +198,9 @@ export function registerApiRoutes(
// Queue debug/rescue mode for a machine // Queue debug/rescue mode for a machine
app.post<{ app.post<{
Body: { mac?: string; sshd?: boolean; pxeBoot?: boolean }; Body: { mac?: string; pxeBoot?: boolean };
}>("/api/debug", async (request, reply) => { }>("/api/debug", async (request, reply) => {
const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":"); const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":");
const sshd = request.body?.sshd ?? false;
const pxeBoot = request.body?.pxeBoot ?? false; const pxeBoot = request.body?.pxeBoot ?? false;
if (mac === "") { if (mac === "") {
return reply.status(400).send({ error: "mac is required" }); return reply.status(400).send({ error: "mac is required" });
@@ -209,7 +215,7 @@ export function registerApiRoutes(
mac; mac;
state.update((s) => { state.update((s) => {
s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd, pxeBoot }; s.debug[mac] = { hostname, queued_at: new Date().toISOString(), pxeBoot };
}); });
logger.info(`DEBUG QUEUED: ${mac} -> ${hostname}`); logger.info(`DEBUG QUEUED: ${mac} -> ${hostname}`);

View File

@@ -23,21 +23,44 @@ export function registerDispatchRoutes(
config: BastionConfig, config: BastionConfig,
state: StateManager, state: StateManager,
): void { ): void {
// Serve debug/rescue kickstart (minimal: SSH keys + network) // Serve debug/rescue kickstart (minimal: SSH keys + network for inst.sshd)
app.get<{ Querystring: { mac?: string; sshd?: string } }>("/debug.ks", async (request, reply) => { app.get<{ Querystring: { mac?: string } }>("/debug.ks", async (_request, reply) => {
const mac = (request.query.mac ?? "").toLowerCase().replace(/-/g, ":");
const currentState = state.load();
const wantSshd = request.query.sshd === "1" || currentState.debug[mac]?.sshd === true;
const ks = renderDebugKickstart({ const ks = renderDebugKickstart({
sshKeys: config.sshKeys ?? [], sshKeys: config.sshKeys ?? [],
sshd: wantSshd,
serverIp: config.serverIp, serverIp: config.serverIp,
httpPort: config.httpPort, httpPort: config.httpPort,
}); });
return reply.type("text/plain").send(ks); return reply.type("text/plain").send(ks);
}); });
// Shell script for manual debug setup (nc listener + IP reporting)
// Usage from rescue shell: curl http://bastion:port/debug-setup.sh | bash
app.get("/debug-setup.sh", async (_request, reply) => {
const script = `#!/bin/bash
# Lab Bastion debug setup — run from rescue shell
set -x
IP_ADDR=$(ip -4 addr show | awk '/inet / && !/127.0.0/ {split($2,a,"/"); print a[1]; exit}')
MAC_ADDR=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')
# Start persistent nc listener for remote shell
(while true; do nc -l -p 2323 -e /bin/bash 2>/dev/null; done) &
echo "nc shell listener on port 2323"
# Report IP to bastion
curl -sf -X POST "http://${config.serverIp}:${config.httpPort}/api/progress" \\
-H "Content-Type: application/json" \\
-d "{\\"mac\\":\\"$MAC_ADDR\\",\\"stage\\":\\"debug-ready\\",\\"detail\\":\\"nc $IP_ADDR 2323\\"}" 2>/dev/null || true
echo ""
echo "=== Debug environment ready ==="
echo " nc $IP_ADDR 2323 (remote shell)"
echo " ssh root@$IP_ADDR (password: debug)"
echo "==============================="
`;
return reply.type("text/plain").send(script);
});
app.get<{ Querystring: { mac?: string } }>("/dispatch", async (request, reply) => { app.get<{ Querystring: { mac?: string } }>("/dispatch", async (request, reply) => {
const mac = (request.query.mac ?? "").toLowerCase().replace(/-/g, ":"); const mac = (request.query.mac ?? "").toLowerCase().replace(/-/g, ":");
const currentState = state.load(); const currentState = state.load();

View File

@@ -5,6 +5,7 @@
import type { FastifyInstance } from "fastify"; import type { FastifyInstance } from "fastify";
import type { BastionConfig } from "@lab/shared"; import type { BastionConfig } from "@lab/shared";
import type { StateManager } from "../services/state.js"; import type { StateManager } from "../services/state.js";
import type { SyslogListener } from "../services/syslog-listener.js";
import { generateInstallKickstart, generateDiscoverKickstart } from "../services/kickstart-generator.js"; import { generateInstallKickstart, generateDiscoverKickstart } from "../services/kickstart-generator.js";
import { renderUbuntuAutoinstall, renderUbuntuMetaData, type UbuntuAutoinstallParams } from "../templates/ubuntu-autoinstall.js"; import { renderUbuntuAutoinstall, renderUbuntuMetaData, type UbuntuAutoinstallParams } from "../templates/ubuntu-autoinstall.js";
@@ -12,6 +13,7 @@ export function registerKickstartRoutes(
app: FastifyInstance, app: FastifyInstance,
config: BastionConfig, config: BastionConfig,
state: StateManager, state: StateManager,
syslog: SyslogListener,
): void { ): void {
// Per-MAC install kickstart // Per-MAC install kickstart
app.get<{ Querystring: { mac?: string } }>("/ks", async (request, reply) => { app.get<{ Querystring: { mac?: string } }>("/ks", async (request, reply) => {
@@ -19,6 +21,11 @@ export function registerKickstartRoutes(
const currentState = state.load(); const currentState = state.load();
const queueEntry = currentState.install_queue[mac]; const queueEntry = currentState.install_queue[mac];
// Register IP → MAC so syslog listener can route Anaconda logs
if (mac) {
syslog.registerIp(request.ip, mac);
}
const ks = generateInstallKickstart(config, { const ks = generateInstallKickstart(config, {
hostname: queueEntry?.hostname ?? "lab-node", hostname: queueEntry?.hostname ?? "lab-node",
disk: queueEntry?.disk ?? "", disk: queueEntry?.disk ?? "",

View File

@@ -43,8 +43,8 @@ export function createApp(config: BastionConfig): { app: ReturnType<typeof Fasti
// Register route handlers // Register route handlers
registerDispatchRoutes(app, config, state); registerDispatchRoutes(app, config, state);
registerKickstartRoutes(app, config, state); registerKickstartRoutes(app, config, state, syslog);
registerApiRoutes(app, state, installLog); registerApiRoutes(app, state, installLog, syslog);
// boot.iso is generated at startup and served as a static file from httpDir // boot.iso is generated at startup and served as a static file from httpDir
// (static serving supports HTTP Range requests, required by JetKVM streaming) // (static serving supports HTTP Range requests, required by JetKVM streaming)

View File

@@ -30,6 +30,8 @@ export class SyslogListener {
private port: number; private port: number;
private installLog: InstallLogBuffer; private installLog: InstallLogBuffer;
private state: StateManager; private state: StateManager;
/** Explicit IP → MAC mapping registered from kickstart/progress requests. */
private ipToMac = new Map<string, string>();
constructor(port: number, installLog: InstallLogBuffer, state: StateManager) { constructor(port: number, installLog: InstallLogBuffer, state: StateManager) {
this.port = port; this.port = port;
@@ -37,14 +39,21 @@ export class SyslogListener {
this.state = state; this.state = state;
} }
/** Resolve a source IP to a MAC address using the install queue. */ /** Register an IP → MAC mapping (called when we learn a machine's IP). */
registerIp(ip: string, mac: string): void {
this.ipToMac.set(ip, mac.toLowerCase());
}
/** Resolve a source IP to a MAC address. */
private resolveIpToMac(ip: string): string | null { private resolveIpToMac(ip: string): string | null {
// Check explicit mapping first (most reliable)
const explicit = this.ipToMac.get(ip);
if (explicit) return explicit;
const currentState = this.state.load(); const currentState = this.state.load();
// Check install queue — machines being installed have an IP from DHCP // Check install queue — machines being installed have an IP from DHCP
for (const [mac, entry] of Object.entries(currentState.install_queue)) { for (const [mac, entry] of Object.entries(currentState.install_queue)) {
// The progress callback sends IP in "complete" detail, but during install
// we need to match by what we know. Check if any progress mentions this IP.
if (entry.progress_detail?.includes(ip)) return mac; if (entry.progress_detail?.includes(ip)) return mac;
} }

View File

@@ -124,7 +124,7 @@ echo Kernel+initrd from PXE, root from NVMe
echo ============================================= echo =============================================
echo echo
kernel http://${params.serverIp}:${params.httpPort}/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0 console=ttyS0,115200n8 modprobe.blacklist=amdgpu kernel http://${params.serverIp}:${params.httpPort}/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0
initrd http://${params.serverIp}:${params.httpPort}/initrd.img initrd http://${params.serverIp}:${params.httpPort}/initrd.img
boot boot
`; `;

View File

@@ -1,81 +1,33 @@
// Debug/rescue kickstart template. // Debug/rescue kickstart template.
// Minimal kickstart for Anaconda rescue mode. // Minimal kickstart for Anaconda rescue mode.
// When sshd=true: generates host keys, starts sshd, reports IP to bastion. //
// No dependency on mounted filesystems — fully self-contained. // SSH access: Anaconda's inst.sshd starts sshd automatically.
// The sshpw directive sets the password, sshkey adds authorized keys.
// %pre/%post do NOT run in rescue mode — don't put setup code there.
export interface DebugKickstartParams { export interface DebugKickstartParams {
sshKeys: string[]; sshKeys: string[];
sshd?: boolean;
serverIp?: string; serverIp?: string;
httpPort?: number; httpPort?: number;
} }
export function renderDebugKickstart(params: DebugKickstartParams): string { export function renderDebugKickstart(params: DebugKickstartParams): string {
const sshpw = "sshpw --username=root --plaintext lab-root-pw";
const sshkeyLine = params.sshKeys.length > 0 const sshkeyLine = params.sshKeys.length > 0
? `sshkey --username=root "${params.sshKeys[0]}"` ? `sshkey --username=root "${params.sshKeys[0]}"`
: ""; : "";
const sshdSetup = params.sshd ? `
%pre --log=/tmp/debug-sshd.log
#!/bin/bash
set -x
# Wait for network to come up
for i in $(seq 1 30); do
IP_ADDR=$(ip -4 addr show | awk '/inet / && !/127.0.0/ {split($2,a,"/"); print a[1]; exit}')
[ -n "$IP_ADDR" ] && break
sleep 1
done
# Generate host keys (self-contained, no mounted FS needed)
ssh-keygen -t ed25519 -f /tmp/ssh_host_ed25519_key -N "" -q
ssh-keygen -t rsa -f /tmp/ssh_host_rsa_key -N "" -q
# Write minimal sshd config
cat > /tmp/sshd_config << 'SSHCFG'
HostKey /tmp/ssh_host_ed25519_key
HostKey /tmp/ssh_host_rsa_key
PermitRootLogin yes
PasswordAuthentication yes
PubkeyAuthentication yes
AuthorizedKeysFile /root/.ssh/authorized_keys
SSHCFG
# Set root password for SSH access
echo "root:debug" | chpasswd
# Set up SSH authorized keys
mkdir -p /root/.ssh && chmod 700 /root/.ssh
${params.sshKeys.map(k => `echo '${k}' >> /root/.ssh/authorized_keys`).join("\n")}
chmod 600 /root/.ssh/authorized_keys 2>/dev/null || true
# Start sshd
/usr/sbin/sshd -f /tmp/sshd_config -p 22
echo "sshd started on port 22"
# Start persistent nc listener for remote shell
(while true; do nc -l -p 2323 -e /bin/bash 2>/dev/null; done) &
echo "nc shell listener on port 2323"
# Report IP to bastion
MAC_ADDR=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')
curl -sf -X POST "http://${params.serverIp}:${params.httpPort}/api/progress" \\
-H "Content-Type: application/json" \\
-d "{\\"mac\\":\\"$MAC_ADDR\\",\\"stage\\":\\"debug-ready\\",\\"detail\\":\\"ssh root@$IP_ADDR (pw: debug) | nc $IP_ADDR 2323\\"}" 2>/dev/null || true
echo "Debug environment ready: ssh root@$IP_ADDR or nc $IP_ADDR 2323"
%end
` : "";
return `# Lab Bastion -- Debug/Rescue Kickstart return `# Lab Bastion -- Debug/Rescue Kickstart
# Minimal: SSH + network for Anaconda rescue mode # Minimal: SSH + network for Anaconda rescue mode
#
# SSH is started by Anaconda (inst.sshd kernel param).
# Password: debug | SSH keys from bastion config.
# %pre/%post do NOT run in rescue mode.
lang en_US.UTF-8 lang en_US.UTF-8
keyboard uk keyboard uk
network --bootproto=dhcp --activate network --bootproto=dhcp --activate
${sshpw} sshpw --username=root --plaintext debug
${sshkeyLine} ${sshkeyLine}
${sshdSetup}`; `;
} }

View File

@@ -134,10 +134,9 @@ network --bootproto=dhcp --activate --hostname=${fqdn}
${auth} ${auth}
${userDirective} ${userDirective}
bootloader --append="console=tty0 console=ttyS0,115200n8" bootloader --append="console=tty0"
# logging --host=${serverIp} --port=${syslogPort} logging --host=${serverIp} --port=${syslogPort}
# Disabled: syslog UDP port needs to be exposed in k3s service/hostPort first
url --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-$releasever&arch=$basearch url --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-$releasever&arch=$basearch
@@ -342,17 +341,7 @@ echo "tmpfs /tmp tmpfs defaults,noatime,nosuid,nodev,size=4G 0 0" >> /etc/fstab
${isVanilla ? `# -- vanilla role: skip k3s kernel/sysctl/firewall setup -- ${isVanilla ? `# -- vanilla role: skip k3s kernel/sysctl/firewall setup --
# -- Enable chronyd for time sync -- # -- Enable chronyd for time sync --
systemctl enable chronyd || true systemctl enable chronyd || true` : `# -- Kernel modules for k3s --
# -- Serial console (for debugging — auto-login as root on ttyS0) --
# AWS EC2 compatible: ttyS0 @ 115200n8
systemctl enable serial-getty@ttyS0.service || true
# -- Forward all system logs to serial console --
cat > /etc/rsyslog.d/serial-console.conf << 'RSYSLOG'
*.* /dev/ttyS0
RSYSLOG
systemctl enable rsyslog || true` : `# -- Kernel modules for k3s --
cat > /etc/modules-load.d/k3s.conf << 'MODULES' cat > /etc/modules-load.d/k3s.conf << 'MODULES'
br_netfilter br_netfilter
overlay overlay
@@ -396,6 +385,9 @@ fi
bastion_progress "post-install" "3-bootorder done" bastion_progress "post-install" "3-bootorder done"
# -- Enable SysRq magic keys (for emergency reboot via Alt+SysRq+REISUB) --
echo "kernel.sysrq=1" > /etc/sysctl.d/90-sysrq.conf
# -- Provisioning metadata -- # -- Provisioning metadata --
cat > /etc/lab-provisioned << PROVEOF cat > /etc/lab-provisioned << PROVEOF
hostname: ${fqdn} hostname: ${fqdn}

View File

@@ -28,6 +28,7 @@ function createTestConfig(testDir: string): BastionConfig {
gateway: "10.0.0.1", gateway: "10.0.0.1",
sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"], sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"],
adminUser: "testadmin", adminUser: "testadmin",
syslogPort: 15514,
skipDnsmasq: true, skipDnsmasq: true,
skipArtifacts: true, skipArtifacts: true,
fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os", fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",

View File

@@ -206,10 +206,8 @@ describe("renderInstallKickstart", () => {
} }
}); });
it("forwards system logs to serial console", () => { it("does not include serial console (causes 30s boot timeout on hardware without UART)", () => {
const ks = renderInstallKickstart(baseParams({ role: "vanilla" })); const ks = renderInstallKickstart(baseParams({ role: "vanilla" }));
expect(ks).toContain("serial-console.conf"); expect(ks).not.toContain("ttyS0");
expect(ks).toContain("/dev/ttyS0");
expect(ks).toContain("rsyslog");
}); });
}); });

View File

@@ -0,0 +1,121 @@
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { createSocket } from "node:dgram";
import { mkdtempSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { SyslogListener } from "../src/services/syslog-listener.js";
import { InstallLogBuffer } from "../src/services/install-log.js";
import { StateManager } from "../src/services/state.js";
function sendUdpSyslog(port: number, message: string): Promise<void> {
return new Promise((resolve, reject) => {
const client = createSocket("udp4");
const buf = Buffer.from(message);
client.send(buf, 0, buf.length, port, "127.0.0.1", (err) => {
client.close();
if (err) reject(err);
else resolve();
});
});
}
describe("SyslogListener", () => {
let tmpDir: string;
let state: StateManager;
let installLog: InstallLogBuffer;
let syslog: SyslogListener;
const PORT = 15514; // use non-privileged port for testing
beforeEach(() => {
tmpDir = mkdtempSync(join(tmpdir(), "syslog-test-"));
state = new StateManager(join(tmpDir, "state.json"));
state.init();
installLog = new InstallLogBuffer(tmpDir);
syslog = new SyslogListener(PORT, installLog, state);
syslog.start();
});
afterEach(() => {
syslog.stop();
rmSync(tmpDir, { recursive: true, force: true });
});
it("receives and stores syslog messages for registered IP", async () => {
const mac = "aa:bb:cc:dd:ee:ff";
// Queue a machine so hostname can be resolved
state.update((s) => {
s.install_queue[mac] = {
hostname: "testnode",
disk: "/dev/sda",
role: "worker",
os: "fedora-43",
queued_at: new Date().toISOString(),
};
});
// Register IP → MAC mapping
syslog.registerIp("127.0.0.1", mac);
// Send a syslog message (RFC 3164 format)
await sendUdpSyslog(PORT, "<13>Mar 30 01:30:00 localhost anaconda[1234]: Installing package vim-enhanced");
// Wait for UDP delivery
await new Promise((r) => setTimeout(r, 200));
const lines = installLog.getLines(mac);
expect(lines.length).toBeGreaterThan(0);
expect(lines[0]!.line).toContain("anaconda");
expect(lines[0]!.line).toContain("Installing package vim-enhanced");
});
it("ignores messages from unknown IPs", async () => {
// Don't register any IP mapping
await sendUdpSyslog(PORT, "<13>Mar 30 01:30:00 localhost anaconda[1234]: test message");
await new Promise((r) => setTimeout(r, 200));
// No MAC to check, but the listener should not crash
// and no logs should be stored for any MAC
expect(installLog.lineCount("unknown")).toBe(0);
});
it("resolves IP from installed machines state", async () => {
const mac = "11:22:33:44:55:66";
state.update((s) => {
s.installed[mac] = {
hostname: "installed-node",
role: "worker",
ip: "127.0.0.1",
installed_at: new Date().toISOString(),
};
});
await sendUdpSyslog(PORT, "<14>Mar 30 02:00:00 installed-node sshd[5678]: Accepted publickey for root");
await new Promise((r) => setTimeout(r, 200));
const lines = installLog.getLines(mac);
expect(lines.length).toBeGreaterThan(0);
expect(lines[0]!.line).toContain("sshd");
});
it("parses various syslog formats", async () => {
const mac = "aa:bb:cc:dd:ee:ff";
syslog.registerIp("127.0.0.1", mac);
state.update((s) => {
s.install_queue[mac] = {
hostname: "testnode",
disk: "/dev/sda",
role: "worker",
os: "fedora-43",
queued_at: new Date().toISOString(),
};
});
// Message without PID
await sendUdpSyslog(PORT, "<13>Mar 30 01:30:00 localhost kernel: NVMe device ready");
await new Promise((r) => setTimeout(r, 200));
const lines = installLog.getLines(mac);
expect(lines.length).toBeGreaterThan(0);
expect(lines[0]!.line).toContain("kernel");
});
});

View File

@@ -94,8 +94,8 @@ export class LabdClient {
return this.request("POST", "/api/machines/install", { body: opts }); return this.request("POST", "/api/machines/install", { body: opts });
} }
async debugMachine(mac: string, opts?: { sshd?: boolean; pxeBoot?: boolean }): Promise<{ status: string; data?: { mac: string; hostname: string }; error?: string }> { async debugMachine(mac: string, opts?: { pxeBoot?: boolean }): Promise<{ status: string; data?: { mac: string; hostname: string }; error?: string }> {
return this.request("POST", "/api/machines/debug", { body: { mac, sshd: opts?.sshd, pxeBoot: opts?.pxeBoot } }); return this.request("POST", "/api/machines/debug", { body: { mac, pxeBoot: opts?.pxeBoot } });
} }
async forgetMachine(mac: string): Promise<{ status: string }> { async forgetMachine(mac: string): Promise<{ status: string }> {

View File

@@ -48,10 +48,9 @@ export function registerDebugCommand(parent: Command): void {
parent parent
.command("debug <target>") .command("debug <target>")
.description("PXE boot into Fedora rescue mode for debugging (target: hostname, MAC, or IP)") .description("PXE boot into Fedora rescue mode for debugging (target: hostname, MAC, or IP)")
.option("--sshd", "Start SSH + nc listener automatically, report IP to bastion")
.option("--pxe-boot", "Boot installed system via PXE (kernel+initrd from network, root from NVMe)") .option("--pxe-boot", "Boot installed system via PXE (kernel+initrd from network, root from NVMe)")
.showHelpAfterError(true) .showHelpAfterError(true)
.action(async (target: string, opts: { sshd?: boolean; pxeBoot?: boolean }) => { .action(async (target: string, opts: { pxeBoot?: boolean }) => {
const client = getLabdClient(); const client = getLabdClient();
// Resolve target from labd aggregated state // Resolve target from labd aggregated state
@@ -75,7 +74,7 @@ export function registerDebugCommand(parent: Command): void {
console.log(`Queuing debug mode for ${hostname} (${mac})...`); console.log(`Queuing debug mode for ${hostname} (${mac})...`);
try { try {
const result = await client.debugMachine(mac, { sshd: opts.sshd === true, pxeBoot: opts.pxeBoot === true }); const result = await client.debugMachine(mac, { pxeBoot: opts.pxeBoot === true });
if (result.error) { if (result.error) {
console.error(`Failed: ${result.error}`); console.error(`Failed: ${result.error}`);
process.exit(1); process.exit(1);
@@ -118,15 +117,21 @@ export function registerDebugCommand(parent: Command): void {
} }
} }
const sshdNote = opts.sshd // Determine bastion URL from labd config for the setup script URL
? `\nSSH + nc listener will start automatically. Watch bastion logs for the IP callback. const bastionUrl = process.env["LABD_URL"]
Password: debug | nc <ip> 2323 for raw shell\n` ? process.env["LABD_URL"].replace(/\/ws\/bastion$/, "").replace(/^wss?:/, "http:")
: ""; : "http://<bastion-ip>:8080";
console.log(` console.log(`
Debug mode queued for ${hostname} (${mac}). Debug mode queued for ${hostname} (${mac}).
Reboot the machine to enter Fedora rescue mode. Reboot the machine to enter Fedora rescue mode.
${sshdNote}
SSH access (started by Anaconda):
ssh root@<ip> (password: debug)
For nc remote shell, run from rescue shell:
curl ${bastionUrl}/debug-setup.sh | bash
Once in rescue shell: Once in rescue shell:
# Activate LVM and mount installed system # Activate LVM and mount installed system

View File

@@ -39,12 +39,10 @@ export function registerLogsCommand(parent: Command): void {
parent parent
.command("logs <target>") .command("logs <target>")
.description("Show provisioning logs for a machine (hostname, MAC, or IP)") .description("Show provisioning logs for a machine (hostname, MAC, or IP)")
.action(async (target: string) => { .option("-f, --follow", "Follow log output in real-time")
.action(async (target: string, opts: { follow?: boolean }) => {
const mac = await resolveToMac(target); const mac = await resolveToMac(target);
try {
const data = await getLabdClient().getMachineLogs(mac);
const BOLD = "\x1b[1m"; const BOLD = "\x1b[1m";
const GREEN = "\x1b[32m"; const GREEN = "\x1b[32m";
const YELLOW = "\x1b[33m"; const YELLOW = "\x1b[33m";
@@ -52,6 +50,14 @@ export function registerLogsCommand(parent: Command): void {
const DIM = "\x1b[2m"; const DIM = "\x1b[2m";
const RESET = "\x1b[0m"; const RESET = "\x1b[0m";
if (opts.follow) {
await followLogs(mac, { BOLD, GREEN, YELLOW, RED, DIM, RESET });
return;
}
try {
const data = await getLabdClient().getMachineLogs(mac);
console.log(`${BOLD}${data["hostname"]}${RESET} (${mac})`); console.log(`${BOLD}${data["hostname"]}${RESET} (${mac})`);
console.log(` Status: ${data["status"] === "installed" ? GREEN : YELLOW}${data["status"]}${RESET}`); console.log(` Status: ${data["status"] === "installed" ? GREEN : YELLOW}${data["status"]}${RESET}`);
console.log(` Role: ${data["role"]}`); console.log(` Role: ${data["role"]}`);
@@ -83,3 +89,58 @@ export function registerLogsCommand(parent: Command): void {
} }
}); });
} }
/** Follow logs by polling labd. */
async function followLogs(
mac: string,
colors: { BOLD: string; GREEN: string; YELLOW: string; RED: string; DIM: string; RESET: string },
): Promise<void> {
const { BOLD, GREEN, YELLOW, RED, DIM, RESET } = colors;
const client = getLabdClient();
console.log(`${DIM}Following logs for ${mac} (Ctrl+C to stop)${RESET}`);
console.log("");
let lastStageCount = 0;
let lastStatus = "";
while (true) {
try {
const data = await client.getMachineLogs(mac);
const status = String(data["status"] ?? "");
const log = data["log"] as Array<{ stage: string; detail: string; timestamp: string }> | undefined;
// Print header once or on status change
if (status !== lastStatus) {
const hostname = String(data["hostname"] ?? mac);
const statusColor = status === "installed" ? GREEN : YELLOW;
console.log(` ${BOLD}${hostname}${RESET} ${statusColor}${status}${RESET}`);
lastStatus = status;
}
// Print new stages
if (log && log.length > lastStageCount) {
for (let i = lastStageCount; i < log.length; i++) {
const entry = log[i]!;
const time = entry.timestamp.slice(11, 19);
const color = entry.stage === "complete" ? GREEN : entry.stage === "error" ? RED : YELLOW;
const detail = entry.detail ? ` ${DIM}-- ${entry.detail}${RESET}` : "";
console.log(` ${DIM}${time}${RESET} ${color}${entry.stage}${RESET}${detail}`);
}
lastStageCount = log.length;
}
// Done
if (status === "installed") {
const ip = data["ip"] ?? "";
console.log("");
console.log(` ${GREEN}${BOLD}Install complete!${RESET}${ip ? ` ${DIM}ssh lab@${ip}${RESET}` : ""}`);
process.exit(0);
}
} catch {
// Machine may not be in logs yet (still queued)
}
await new Promise((r) => setTimeout(r, 5000));
}
}

View File

@@ -151,7 +151,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
try { try {
const result = await sendCommand(all[0]!.bastionId, { const result = await sendCommand(all[0]!.bastionId, {
type: "command-install", type: "command-install",
mac, hostname, disk: disk ?? "/dev/sda", role: role ?? "infra", os: os ?? "fedora-43", mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
}); });
return reply.code(result.status === "ok" ? 200 : 500).send(result); return reply.code(result.status === "ok" ? 200 : 500).send(result);
} catch (err) { } catch (err) {
@@ -164,7 +164,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
try { try {
const result = await sendCommand(bastion.bastionId, { const result = await sendCommand(bastion.bastionId, {
type: "command-install", type: "command-install",
mac, hostname, disk: disk ?? "/dev/sda", role: role ?? "infra", os: os ?? "fedora-43", mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
}); });
return reply.code(result.status === "ok" ? 200 : 500).send(result); return reply.code(result.status === "ok" ? 200 : 500).send(result);
} catch (err) { } catch (err) {
@@ -174,10 +174,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
// Queue debug/rescue mode — route to correct bastion by MAC // Queue debug/rescue mode — route to correct bastion by MAC
app.post<{ app.post<{
Body: { mac?: string; sshd?: boolean; pxeBoot?: boolean }; Body: { mac?: string; pxeBoot?: boolean };
}>("/api/machines/debug", async (request, reply) => { }>("/api/machines/debug", async (request, reply) => {
const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":"); const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":");
const sshd = request.body?.sshd ?? false;
const pxeBoot = request.body?.pxeBoot ?? false; const pxeBoot = request.body?.pxeBoot ?? false;
if (!mac) { if (!mac) {
return reply.code(400).send({ error: "mac is required" }); return reply.code(400).send({ error: "mac is required" });
@@ -191,7 +190,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
} }
if (all.length === 1) { if (all.length === 1) {
try { try {
const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac, sshd, pxeBoot }); const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac, pxeBoot });
return reply.code(result.status === "ok" ? 200 : 500).send(result); return reply.code(result.status === "ok" ? 200 : 500).send(result);
} catch (err) { } catch (err) {
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) }); return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
@@ -201,7 +200,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
} }
try { try {
const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac, sshd, pxeBoot }); const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac, pxeBoot });
return reply.code(result.status === "ok" ? 200 : 500).send(result); return reply.code(result.status === "ok" ? 200 : 500).send(result);
} catch (err) { } catch (err) {
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) }); return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });

View File

@@ -111,7 +111,7 @@ export type LabdBastionMessage =
| { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string } | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string }
| { type: "command-forget"; requestId: string; mac: string } | { type: "command-forget"; requestId: string; mac: string }
| { type: "command-role-update"; requestId: string; mac: string; role: string } | { type: "command-role-update"; requestId: string; mac: string; role: string }
| { type: "command-debug"; requestId: string; mac: string; sshd?: boolean; pxeBoot?: boolean } | { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean }
| { type: "server-shutdown"; reconnectAfter: number }; | { type: "server-shutdown"; reconnectAfter: number };
export type BastionMessageType = BastionMessage["type"]; export type BastionMessageType = BastionMessage["type"];

View File

@@ -101,7 +101,6 @@ export interface InstalledInfo {
export interface DebugConfig { export interface DebugConfig {
hostname: string; hostname: string;
queued_at: string; queued_at: string;
sshd?: boolean;
pxeBoot?: boolean; pxeBoot?: boolean;
} }

View File

@@ -224,11 +224,12 @@ describe("PXE boot provisioning", () => {
// Generate dnsmasq config // Generate dnsmasq config
generateDnsmasqConf(config); generateDnsmasqConf(config);
// Start HTTP server // Start HTTP server + syslog listener
const { app, state } = createApp(config); const { app, state, syslog } = createApp(config);
bastionApp = app; bastionApp = app;
await app.listen({ port: config.httpPort, host: "0.0.0.0" }); await app.listen({ port: config.httpPort, host: "0.0.0.0" });
log(`Bastion HTTP server listening on :${HTTP_PORT}`); syslog.start();
log(`Bastion HTTP server listening on :${HTTP_PORT}, syslog on UDP :${config.syslogPort}`);
// Start dnsmasq (fire-and-forget — it runs until killed) // Start dnsmasq (fire-and-forget — it runs until killed)
// May fail without root (DHCP socket needs CAP_NET_BIND_SERVICE); libvirt network provides DHCP fallback // May fail without root (DHCP socket needs CAP_NET_BIND_SERVICE); libvirt network provides DHCP fallback
@@ -387,8 +388,8 @@ describe("PXE boot provisioning", () => {
expect(data.progress).toBe("complete"); expect(data.progress).toBe("complete");
}); });
it.skip("log lines were captured", async () => { it("syslog install logs were captured", async () => {
// Requires log streamer in %post — skipped until re-added // Anaconda forwards logs via syslog (logging --host directive in kickstart)
const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`); const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`);
const data = (await res.json()) as { log_total?: number; log_lines?: Array<{ line: string }> }; const data = (await res.json()) as { log_total?: number; log_lines?: Array<{ line: string }> };
expect(data.log_total).toBeGreaterThan(0); expect(data.log_total).toBeGreaterThan(0);