From 0a4916d3c9ce9984efbd735ed4be6a09b7d92a07 Mon Sep 17 00:00:00 2001 From: Michal Date: Mon, 30 Mar 2026 03:58:51 +0100 Subject: [PATCH] fix: remove serial console (root cause of 30s boot delay), enable syslog logging, disk auto-detect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause found: console=ttyS0,115200n8 causes 30-second timeout at every systemd boot phase on hardware without a physical serial UART. Each phase transition blocks waiting for the non-existent UART. Changes: - Remove console=ttyS0 from kickstart bootloader args and %post setup - Enable Anaconda syslog forwarding (logging --host --port) for install visibility - Improve syslog IP→MAC resolution (register from kickstart fetch + progress) - Fix disk auto-detect: default to empty string (not /dev/sda) for NVMe support - Enable SysRq magic keys (kernel.sysrq=1) for emergency reboot via JetKVM - Simplify debug command: remove --sshd flag (inst.sshd always available), add /debug-setup.sh HTTP endpoint for nc listener setup - Add labctl provision logs -f (follow mode with polling) - Add syslog listener unit tests - Enable syslog log capture test in integration suite Co-Authored-By: Claude Opus 4.6 (1M context) --- bastion/completions/labctl.bash | 4 +- bastion/completions/labctl.fish | 5 +- bastion/src/bastion/src/main.ts | 5 +- bastion/src/bastion/src/routes/api.ts | 12 +- bastion/src/bastion/src/routes/dispatch.ts | 37 +++++- bastion/src/bastion/src/routes/kickstart.ts | 7 + bastion/src/bastion/src/server.ts | 4 +- .../bastion/src/services/syslog-listener.ts | 15 ++- .../src/bastion/src/templates/boot.ipxe.ts | 2 +- bastion/src/bastion/src/templates/debug.ks.ts | 68 ++-------- .../src/bastion/src/templates/install.ks.ts | 20 +-- bastion/src/bastion/tests/dispatch.test.ts | 1 + bastion/src/bastion/tests/kickstart.test.ts | 6 +- .../src/bastion/tests/syslog-listener.test.ts | 121 ++++++++++++++++++ bastion/src/cli/src/api/client.ts | 4 +- bastion/src/cli/src/commands/debug.ts | 21 +-- bastion/src/cli/src/commands/logs.ts | 77 +++++++++-- bastion/src/labd/src/routes/bastions.ts | 11 +- bastion/src/shared/src/protocol/index.ts | 2 +- bastion/src/shared/src/types/state.ts | 1 - .../tests/integration/pxe-provision.test.ts | 11 +- 21 files changed, 305 insertions(+), 129 deletions(-) create mode 100644 bastion/src/bastion/tests/syslog-listener.test.ts diff --git a/bastion/completions/labctl.bash b/bastion/completions/labctl.bash index 9f7b49b..86a58fc 100644 --- a/bastion/completions/labctl.bash +++ b/bastion/completions/labctl.bash @@ -62,13 +62,13 @@ _labctl() { COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur")) return ;; "provision debug") - COMPREPLY=($(compgen -W "--sshd -h --help" -- "$cur")) + COMPREPLY=($(compgen -W "--pxe-boot -h --help" -- "$cur")) return ;; "provision forget") COMPREPLY=($(compgen -W "-h --help" -- "$cur")) return ;; "provision logs") - COMPREPLY=($(compgen -W "-h --help" -- "$cur")) + COMPREPLY=($(compgen -W "-f --follow -h --help" -- "$cur")) return ;; "provision makeiso") COMPREPLY=($(compgen -W "--arch --local --out -h --help" -- "$cur")) diff --git a/bastion/completions/labctl.fish b/bastion/completions/labctl.fish index 3a14103..a63ae32 100644 --- a/bastion/completions/labctl.fish +++ b/bastion/completions/labctl.fish @@ -138,7 +138,10 @@ complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operatin complete -c labctl -n "__labctl_in_cmd provision reprovision" -l disk -d 'Target disk device (auto-detect if omitted)' -x # provision debug options -complete -c labctl -n "__labctl_in_cmd provision debug" -l sshd -d 'Start SSH + nc listener automatically, report IP to bastion' +complete -c labctl -n "__labctl_in_cmd provision debug" -l pxe-boot -d 'Boot installed system via PXE (kernel+initrd from network, root from NVMe)' + +# provision logs options +complete -c labctl -n "__labctl_in_cmd provision logs" -s f -l follow -d 'Follow log output in real-time' # provision makeiso options complete -c labctl -n "__labctl_in_cmd provision makeiso" -l arch -d 'Target architecture(s)' -xa 'x86_64 aarch64' diff --git a/bastion/src/bastion/src/main.ts b/bastion/src/bastion/src/main.ts index 289551e..8c6d066 100644 --- a/bastion/src/bastion/src/main.ts +++ b/bastion/src/bastion/src/main.ts @@ -257,7 +257,7 @@ export async function startBastion(overrides: Partial = {}): Prom state.update((s) => { s.install_queue[msg.mac] = { hostname: msg.hostname, - disk: msg.disk ?? "/dev/sda", + disk: msg.disk ?? "", role: msg.role as import("@lab/shared").Role, os: msg.os as import("@lab/shared").OsId, queued_at: new Date().toISOString(), @@ -269,7 +269,6 @@ export async function startBastion(overrides: Partial = {}): Prom labdConn.onCommand("command-debug", async (msg) => { if (msg.type !== "command-debug") throw new Error("unexpected"); const mac = msg.mac.toLowerCase(); - const sshd = msg.sshd ?? false; const pxeBoot = msg.pxeBoot ?? false; const currentState = state.load(); const hostname = @@ -278,7 +277,7 @@ export async function startBastion(overrides: Partial = {}): Prom currentState.discovered[mac]?.product ?? mac; state.update((s) => { - s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd, pxeBoot }; + s.debug[mac] = { hostname, queued_at: new Date().toISOString(), pxeBoot }; }); return { status: "ok", data: { mac, hostname } }; }); diff --git a/bastion/src/bastion/src/routes/api.ts b/bastion/src/bastion/src/routes/api.ts index 5b9fe83..b178b43 100644 --- a/bastion/src/bastion/src/routes/api.ts +++ b/bastion/src/bastion/src/routes/api.ts @@ -13,11 +13,13 @@ import { triggerPostProvisionK3s } from "../services/post-provision.js"; import { progressBus } from "../services/progress-events.js"; import type { ProgressEvent } from "../services/progress-events.js"; import type { InstallLogBuffer } from "../services/install-log.js"; +import type { SyslogListener } from "../services/syslog-listener.js"; export function registerApiRoutes( app: FastifyInstance, state: StateManager, installLog: InstallLogBuffer, + syslog: SyslogListener, ): void { // List all machines app.get("/api/machines", async (_request, reply) => { @@ -84,6 +86,11 @@ export function registerApiRoutes( const { mac: rawMac, stage, detail } = request.body ?? {}; const mac = (rawMac ?? "unknown").toLowerCase(); const stageName = stage ?? "unknown"; + + // Register IP → MAC for syslog routing + if (mac !== "unknown") { + syslog.registerIp(request.ip, mac); + } const detailStr = detail ?? ""; const GREEN = "\x1b[0;32m"; @@ -191,10 +198,9 @@ export function registerApiRoutes( // Queue debug/rescue mode for a machine app.post<{ - Body: { mac?: string; sshd?: boolean; pxeBoot?: boolean }; + Body: { mac?: string; pxeBoot?: boolean }; }>("/api/debug", async (request, reply) => { const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":"); - const sshd = request.body?.sshd ?? false; const pxeBoot = request.body?.pxeBoot ?? false; if (mac === "") { return reply.status(400).send({ error: "mac is required" }); @@ -209,7 +215,7 @@ export function registerApiRoutes( mac; state.update((s) => { - s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd, pxeBoot }; + s.debug[mac] = { hostname, queued_at: new Date().toISOString(), pxeBoot }; }); logger.info(`DEBUG QUEUED: ${mac} -> ${hostname}`); diff --git a/bastion/src/bastion/src/routes/dispatch.ts b/bastion/src/bastion/src/routes/dispatch.ts index 1954d4d..0ecc1c4 100644 --- a/bastion/src/bastion/src/routes/dispatch.ts +++ b/bastion/src/bastion/src/routes/dispatch.ts @@ -23,21 +23,44 @@ export function registerDispatchRoutes( config: BastionConfig, state: StateManager, ): void { - // Serve debug/rescue kickstart (minimal: SSH keys + network) - app.get<{ Querystring: { mac?: string; sshd?: string } }>("/debug.ks", async (request, reply) => { - const mac = (request.query.mac ?? "").toLowerCase().replace(/-/g, ":"); - const currentState = state.load(); - const wantSshd = request.query.sshd === "1" || currentState.debug[mac]?.sshd === true; - + // Serve debug/rescue kickstart (minimal: SSH keys + network for inst.sshd) + app.get<{ Querystring: { mac?: string } }>("/debug.ks", async (_request, reply) => { const ks = renderDebugKickstart({ sshKeys: config.sshKeys ?? [], - sshd: wantSshd, serverIp: config.serverIp, httpPort: config.httpPort, }); return reply.type("text/plain").send(ks); }); + // Shell script for manual debug setup (nc listener + IP reporting) + // Usage from rescue shell: curl http://bastion:port/debug-setup.sh | bash + app.get("/debug-setup.sh", async (_request, reply) => { + const script = `#!/bin/bash +# Lab Bastion debug setup — run from rescue shell +set -x + +IP_ADDR=$(ip -4 addr show | awk '/inet / && !/127.0.0/ {split($2,a,"/"); print a[1]; exit}') +MAC_ADDR=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}') + +# Start persistent nc listener for remote shell +(while true; do nc -l -p 2323 -e /bin/bash 2>/dev/null; done) & +echo "nc shell listener on port 2323" + +# Report IP to bastion +curl -sf -X POST "http://${config.serverIp}:${config.httpPort}/api/progress" \\ + -H "Content-Type: application/json" \\ + -d "{\\"mac\\":\\"$MAC_ADDR\\",\\"stage\\":\\"debug-ready\\",\\"detail\\":\\"nc $IP_ADDR 2323\\"}" 2>/dev/null || true + +echo "" +echo "=== Debug environment ready ===" +echo " nc $IP_ADDR 2323 (remote shell)" +echo " ssh root@$IP_ADDR (password: debug)" +echo "===============================" +`; + return reply.type("text/plain").send(script); + }); + app.get<{ Querystring: { mac?: string } }>("/dispatch", async (request, reply) => { const mac = (request.query.mac ?? "").toLowerCase().replace(/-/g, ":"); const currentState = state.load(); diff --git a/bastion/src/bastion/src/routes/kickstart.ts b/bastion/src/bastion/src/routes/kickstart.ts index bce0e04..49ca90a 100644 --- a/bastion/src/bastion/src/routes/kickstart.ts +++ b/bastion/src/bastion/src/routes/kickstart.ts @@ -5,6 +5,7 @@ import type { FastifyInstance } from "fastify"; import type { BastionConfig } from "@lab/shared"; import type { StateManager } from "../services/state.js"; +import type { SyslogListener } from "../services/syslog-listener.js"; import { generateInstallKickstart, generateDiscoverKickstart } from "../services/kickstart-generator.js"; import { renderUbuntuAutoinstall, renderUbuntuMetaData, type UbuntuAutoinstallParams } from "../templates/ubuntu-autoinstall.js"; @@ -12,6 +13,7 @@ export function registerKickstartRoutes( app: FastifyInstance, config: BastionConfig, state: StateManager, + syslog: SyslogListener, ): void { // Per-MAC install kickstart app.get<{ Querystring: { mac?: string } }>("/ks", async (request, reply) => { @@ -19,6 +21,11 @@ export function registerKickstartRoutes( const currentState = state.load(); const queueEntry = currentState.install_queue[mac]; + // Register IP → MAC so syslog listener can route Anaconda logs + if (mac) { + syslog.registerIp(request.ip, mac); + } + const ks = generateInstallKickstart(config, { hostname: queueEntry?.hostname ?? "lab-node", disk: queueEntry?.disk ?? "", diff --git a/bastion/src/bastion/src/server.ts b/bastion/src/bastion/src/server.ts index 8bdaf6d..9a2979a 100644 --- a/bastion/src/bastion/src/server.ts +++ b/bastion/src/bastion/src/server.ts @@ -43,8 +43,8 @@ export function createApp(config: BastionConfig): { app: ReturnType(); constructor(port: number, installLog: InstallLogBuffer, state: StateManager) { this.port = port; @@ -37,14 +39,21 @@ export class SyslogListener { this.state = state; } - /** Resolve a source IP to a MAC address using the install queue. */ + /** Register an IP → MAC mapping (called when we learn a machine's IP). */ + registerIp(ip: string, mac: string): void { + this.ipToMac.set(ip, mac.toLowerCase()); + } + + /** Resolve a source IP to a MAC address. */ private resolveIpToMac(ip: string): string | null { + // Check explicit mapping first (most reliable) + const explicit = this.ipToMac.get(ip); + if (explicit) return explicit; + const currentState = this.state.load(); // Check install queue — machines being installed have an IP from DHCP for (const [mac, entry] of Object.entries(currentState.install_queue)) { - // The progress callback sends IP in "complete" detail, but during install - // we need to match by what we know. Check if any progress mentions this IP. if (entry.progress_detail?.includes(ip)) return mac; } diff --git a/bastion/src/bastion/src/templates/boot.ipxe.ts b/bastion/src/bastion/src/templates/boot.ipxe.ts index f56e815..95f36d2 100644 --- a/bastion/src/bastion/src/templates/boot.ipxe.ts +++ b/bastion/src/bastion/src/templates/boot.ipxe.ts @@ -124,7 +124,7 @@ echo Kernel+initrd from PXE, root from NVMe echo ============================================= echo -kernel http://${params.serverIp}:${params.httpPort}/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0 console=ttyS0,115200n8 modprobe.blacklist=amdgpu +kernel http://${params.serverIp}:${params.httpPort}/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0 initrd http://${params.serverIp}:${params.httpPort}/initrd.img boot `; diff --git a/bastion/src/bastion/src/templates/debug.ks.ts b/bastion/src/bastion/src/templates/debug.ks.ts index 083cc19..29a7e35 100644 --- a/bastion/src/bastion/src/templates/debug.ks.ts +++ b/bastion/src/bastion/src/templates/debug.ks.ts @@ -1,81 +1,33 @@ // Debug/rescue kickstart template. // Minimal kickstart for Anaconda rescue mode. -// When sshd=true: generates host keys, starts sshd, reports IP to bastion. -// No dependency on mounted filesystems — fully self-contained. +// +// SSH access: Anaconda's inst.sshd starts sshd automatically. +// The sshpw directive sets the password, sshkey adds authorized keys. +// %pre/%post do NOT run in rescue mode — don't put setup code there. export interface DebugKickstartParams { sshKeys: string[]; - sshd?: boolean; serverIp?: string; httpPort?: number; } export function renderDebugKickstart(params: DebugKickstartParams): string { - const sshpw = "sshpw --username=root --plaintext lab-root-pw"; const sshkeyLine = params.sshKeys.length > 0 ? `sshkey --username=root "${params.sshKeys[0]}"` : ""; - const sshdSetup = params.sshd ? ` -%pre --log=/tmp/debug-sshd.log -#!/bin/bash -set -x - -# Wait for network to come up -for i in $(seq 1 30); do - IP_ADDR=$(ip -4 addr show | awk '/inet / && !/127.0.0/ {split($2,a,"/"); print a[1]; exit}') - [ -n "$IP_ADDR" ] && break - sleep 1 -done - -# Generate host keys (self-contained, no mounted FS needed) -ssh-keygen -t ed25519 -f /tmp/ssh_host_ed25519_key -N "" -q -ssh-keygen -t rsa -f /tmp/ssh_host_rsa_key -N "" -q - -# Write minimal sshd config -cat > /tmp/sshd_config << 'SSHCFG' -HostKey /tmp/ssh_host_ed25519_key -HostKey /tmp/ssh_host_rsa_key -PermitRootLogin yes -PasswordAuthentication yes -PubkeyAuthentication yes -AuthorizedKeysFile /root/.ssh/authorized_keys -SSHCFG - -# Set root password for SSH access -echo "root:debug" | chpasswd - -# Set up SSH authorized keys -mkdir -p /root/.ssh && chmod 700 /root/.ssh -${params.sshKeys.map(k => `echo '${k}' >> /root/.ssh/authorized_keys`).join("\n")} -chmod 600 /root/.ssh/authorized_keys 2>/dev/null || true - -# Start sshd -/usr/sbin/sshd -f /tmp/sshd_config -p 22 -echo "sshd started on port 22" - -# Start persistent nc listener for remote shell -(while true; do nc -l -p 2323 -e /bin/bash 2>/dev/null; done) & -echo "nc shell listener on port 2323" - -# Report IP to bastion -MAC_ADDR=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}') -curl -sf -X POST "http://${params.serverIp}:${params.httpPort}/api/progress" \\ - -H "Content-Type: application/json" \\ - -d "{\\"mac\\":\\"$MAC_ADDR\\",\\"stage\\":\\"debug-ready\\",\\"detail\\":\\"ssh root@$IP_ADDR (pw: debug) | nc $IP_ADDR 2323\\"}" 2>/dev/null || true - -echo "Debug environment ready: ssh root@$IP_ADDR or nc $IP_ADDR 2323" -%end -` : ""; - return `# Lab Bastion -- Debug/Rescue Kickstart # Minimal: SSH + network for Anaconda rescue mode +# +# SSH is started by Anaconda (inst.sshd kernel param). +# Password: debug | SSH keys from bastion config. +# %pre/%post do NOT run in rescue mode. lang en_US.UTF-8 keyboard uk network --bootproto=dhcp --activate -${sshpw} +sshpw --username=root --plaintext debug ${sshkeyLine} -${sshdSetup}`; +`; } diff --git a/bastion/src/bastion/src/templates/install.ks.ts b/bastion/src/bastion/src/templates/install.ks.ts index cf5ef73..94af999 100644 --- a/bastion/src/bastion/src/templates/install.ks.ts +++ b/bastion/src/bastion/src/templates/install.ks.ts @@ -134,10 +134,9 @@ network --bootproto=dhcp --activate --hostname=${fqdn} ${auth} ${userDirective} -bootloader --append="console=tty0 console=ttyS0,115200n8" +bootloader --append="console=tty0" -# logging --host=${serverIp} --port=${syslogPort} -# Disabled: syslog UDP port needs to be exposed in k3s service/hostPort first +logging --host=${serverIp} --port=${syslogPort} url --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-$releasever&arch=$basearch @@ -342,17 +341,7 @@ echo "tmpfs /tmp tmpfs defaults,noatime,nosuid,nodev,size=4G 0 0" >> /etc/fstab ${isVanilla ? `# -- vanilla role: skip k3s kernel/sysctl/firewall setup -- # -- Enable chronyd for time sync -- -systemctl enable chronyd || true - -# -- Serial console (for debugging — auto-login as root on ttyS0) -- -# AWS EC2 compatible: ttyS0 @ 115200n8 -systemctl enable serial-getty@ttyS0.service || true - -# -- Forward all system logs to serial console -- -cat > /etc/rsyslog.d/serial-console.conf << 'RSYSLOG' -*.* /dev/ttyS0 -RSYSLOG -systemctl enable rsyslog || true` : `# -- Kernel modules for k3s -- +systemctl enable chronyd || true` : `# -- Kernel modules for k3s -- cat > /etc/modules-load.d/k3s.conf << 'MODULES' br_netfilter overlay @@ -396,6 +385,9 @@ fi bastion_progress "post-install" "3-bootorder done" +# -- Enable SysRq magic keys (for emergency reboot via Alt+SysRq+REISUB) -- +echo "kernel.sysrq=1" > /etc/sysctl.d/90-sysrq.conf + # -- Provisioning metadata -- cat > /etc/lab-provisioned << PROVEOF hostname: ${fqdn} diff --git a/bastion/src/bastion/tests/dispatch.test.ts b/bastion/src/bastion/tests/dispatch.test.ts index 0b9572b..3d07ac4 100644 --- a/bastion/src/bastion/tests/dispatch.test.ts +++ b/bastion/src/bastion/tests/dispatch.test.ts @@ -28,6 +28,7 @@ function createTestConfig(testDir: string): BastionConfig { gateway: "10.0.0.1", sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"], adminUser: "testadmin", + syslogPort: 15514, skipDnsmasq: true, skipArtifacts: true, fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os", diff --git a/bastion/src/bastion/tests/kickstart.test.ts b/bastion/src/bastion/tests/kickstart.test.ts index 2629877..771f5d1 100644 --- a/bastion/src/bastion/tests/kickstart.test.ts +++ b/bastion/src/bastion/tests/kickstart.test.ts @@ -206,10 +206,8 @@ describe("renderInstallKickstart", () => { } }); - it("forwards system logs to serial console", () => { + it("does not include serial console (causes 30s boot timeout on hardware without UART)", () => { const ks = renderInstallKickstart(baseParams({ role: "vanilla" })); - expect(ks).toContain("serial-console.conf"); - expect(ks).toContain("/dev/ttyS0"); - expect(ks).toContain("rsyslog"); + expect(ks).not.toContain("ttyS0"); }); }); diff --git a/bastion/src/bastion/tests/syslog-listener.test.ts b/bastion/src/bastion/tests/syslog-listener.test.ts new file mode 100644 index 0000000..2ece0d5 --- /dev/null +++ b/bastion/src/bastion/tests/syslog-listener.test.ts @@ -0,0 +1,121 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { createSocket } from "node:dgram"; +import { mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { SyslogListener } from "../src/services/syslog-listener.js"; +import { InstallLogBuffer } from "../src/services/install-log.js"; +import { StateManager } from "../src/services/state.js"; + +function sendUdpSyslog(port: number, message: string): Promise { + return new Promise((resolve, reject) => { + const client = createSocket("udp4"); + const buf = Buffer.from(message); + client.send(buf, 0, buf.length, port, "127.0.0.1", (err) => { + client.close(); + if (err) reject(err); + else resolve(); + }); + }); +} + +describe("SyslogListener", () => { + let tmpDir: string; + let state: StateManager; + let installLog: InstallLogBuffer; + let syslog: SyslogListener; + const PORT = 15514; // use non-privileged port for testing + + beforeEach(() => { + tmpDir = mkdtempSync(join(tmpdir(), "syslog-test-")); + state = new StateManager(join(tmpDir, "state.json")); + state.init(); + installLog = new InstallLogBuffer(tmpDir); + syslog = new SyslogListener(PORT, installLog, state); + syslog.start(); + }); + + afterEach(() => { + syslog.stop(); + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("receives and stores syslog messages for registered IP", async () => { + const mac = "aa:bb:cc:dd:ee:ff"; + // Queue a machine so hostname can be resolved + state.update((s) => { + s.install_queue[mac] = { + hostname: "testnode", + disk: "/dev/sda", + role: "worker", + os: "fedora-43", + queued_at: new Date().toISOString(), + }; + }); + + // Register IP → MAC mapping + syslog.registerIp("127.0.0.1", mac); + + // Send a syslog message (RFC 3164 format) + await sendUdpSyslog(PORT, "<13>Mar 30 01:30:00 localhost anaconda[1234]: Installing package vim-enhanced"); + + // Wait for UDP delivery + await new Promise((r) => setTimeout(r, 200)); + + const lines = installLog.getLines(mac); + expect(lines.length).toBeGreaterThan(0); + expect(lines[0]!.line).toContain("anaconda"); + expect(lines[0]!.line).toContain("Installing package vim-enhanced"); + }); + + it("ignores messages from unknown IPs", async () => { + // Don't register any IP mapping + await sendUdpSyslog(PORT, "<13>Mar 30 01:30:00 localhost anaconda[1234]: test message"); + await new Promise((r) => setTimeout(r, 200)); + + // No MAC to check, but the listener should not crash + // and no logs should be stored for any MAC + expect(installLog.lineCount("unknown")).toBe(0); + }); + + it("resolves IP from installed machines state", async () => { + const mac = "11:22:33:44:55:66"; + state.update((s) => { + s.installed[mac] = { + hostname: "installed-node", + role: "worker", + ip: "127.0.0.1", + installed_at: new Date().toISOString(), + }; + }); + + await sendUdpSyslog(PORT, "<14>Mar 30 02:00:00 installed-node sshd[5678]: Accepted publickey for root"); + await new Promise((r) => setTimeout(r, 200)); + + const lines = installLog.getLines(mac); + expect(lines.length).toBeGreaterThan(0); + expect(lines[0]!.line).toContain("sshd"); + }); + + it("parses various syslog formats", async () => { + const mac = "aa:bb:cc:dd:ee:ff"; + syslog.registerIp("127.0.0.1", mac); + state.update((s) => { + s.install_queue[mac] = { + hostname: "testnode", + disk: "/dev/sda", + role: "worker", + os: "fedora-43", + queued_at: new Date().toISOString(), + }; + }); + + // Message without PID + await sendUdpSyslog(PORT, "<13>Mar 30 01:30:00 localhost kernel: NVMe device ready"); + await new Promise((r) => setTimeout(r, 200)); + + const lines = installLog.getLines(mac); + expect(lines.length).toBeGreaterThan(0); + expect(lines[0]!.line).toContain("kernel"); + }); +}); diff --git a/bastion/src/cli/src/api/client.ts b/bastion/src/cli/src/api/client.ts index 75380a4..241c848 100644 --- a/bastion/src/cli/src/api/client.ts +++ b/bastion/src/cli/src/api/client.ts @@ -94,8 +94,8 @@ export class LabdClient { return this.request("POST", "/api/machines/install", { body: opts }); } - async debugMachine(mac: string, opts?: { sshd?: boolean; pxeBoot?: boolean }): Promise<{ status: string; data?: { mac: string; hostname: string }; error?: string }> { - return this.request("POST", "/api/machines/debug", { body: { mac, sshd: opts?.sshd, pxeBoot: opts?.pxeBoot } }); + async debugMachine(mac: string, opts?: { pxeBoot?: boolean }): Promise<{ status: string; data?: { mac: string; hostname: string }; error?: string }> { + return this.request("POST", "/api/machines/debug", { body: { mac, pxeBoot: opts?.pxeBoot } }); } async forgetMachine(mac: string): Promise<{ status: string }> { diff --git a/bastion/src/cli/src/commands/debug.ts b/bastion/src/cli/src/commands/debug.ts index 45a13a7..2bb4f24 100644 --- a/bastion/src/cli/src/commands/debug.ts +++ b/bastion/src/cli/src/commands/debug.ts @@ -48,10 +48,9 @@ export function registerDebugCommand(parent: Command): void { parent .command("debug ") .description("PXE boot into Fedora rescue mode for debugging (target: hostname, MAC, or IP)") - .option("--sshd", "Start SSH + nc listener automatically, report IP to bastion") .option("--pxe-boot", "Boot installed system via PXE (kernel+initrd from network, root from NVMe)") .showHelpAfterError(true) - .action(async (target: string, opts: { sshd?: boolean; pxeBoot?: boolean }) => { + .action(async (target: string, opts: { pxeBoot?: boolean }) => { const client = getLabdClient(); // Resolve target from labd aggregated state @@ -75,7 +74,7 @@ export function registerDebugCommand(parent: Command): void { console.log(`Queuing debug mode for ${hostname} (${mac})...`); try { - const result = await client.debugMachine(mac, { sshd: opts.sshd === true, pxeBoot: opts.pxeBoot === true }); + const result = await client.debugMachine(mac, { pxeBoot: opts.pxeBoot === true }); if (result.error) { console.error(`Failed: ${result.error}`); process.exit(1); @@ -118,15 +117,21 @@ export function registerDebugCommand(parent: Command): void { } } - const sshdNote = opts.sshd - ? `\nSSH + nc listener will start automatically. Watch bastion logs for the IP callback. - Password: debug | nc 2323 for raw shell\n` - : ""; + // Determine bastion URL from labd config for the setup script URL + const bastionUrl = process.env["LABD_URL"] + ? process.env["LABD_URL"].replace(/\/ws\/bastion$/, "").replace(/^wss?:/, "http:") + : "http://:8080"; console.log(` Debug mode queued for ${hostname} (${mac}). Reboot the machine to enter Fedora rescue mode. -${sshdNote} + +SSH access (started by Anaconda): + ssh root@ (password: debug) + +For nc remote shell, run from rescue shell: + curl ${bastionUrl}/debug-setup.sh | bash + Once in rescue shell: # Activate LVM and mount installed system diff --git a/bastion/src/cli/src/commands/logs.ts b/bastion/src/cli/src/commands/logs.ts index 85a59c1..48630a6 100644 --- a/bastion/src/cli/src/commands/logs.ts +++ b/bastion/src/cli/src/commands/logs.ts @@ -39,19 +39,25 @@ export function registerLogsCommand(parent: Command): void { parent .command("logs ") .description("Show provisioning logs for a machine (hostname, MAC, or IP)") - .action(async (target: string) => { + .option("-f, --follow", "Follow log output in real-time") + .action(async (target: string, opts: { follow?: boolean }) => { const mac = await resolveToMac(target); + const BOLD = "\x1b[1m"; + const GREEN = "\x1b[32m"; + const YELLOW = "\x1b[33m"; + const RED = "\x1b[31m"; + const DIM = "\x1b[2m"; + const RESET = "\x1b[0m"; + + if (opts.follow) { + await followLogs(mac, { BOLD, GREEN, YELLOW, RED, DIM, RESET }); + return; + } + try { const data = await getLabdClient().getMachineLogs(mac); - const BOLD = "\x1b[1m"; - const GREEN = "\x1b[32m"; - const YELLOW = "\x1b[33m"; - const RED = "\x1b[31m"; - const DIM = "\x1b[2m"; - const RESET = "\x1b[0m"; - console.log(`${BOLD}${data["hostname"]}${RESET} (${mac})`); console.log(` Status: ${data["status"] === "installed" ? GREEN : YELLOW}${data["status"]}${RESET}`); console.log(` Role: ${data["role"]}`); @@ -83,3 +89,58 @@ export function registerLogsCommand(parent: Command): void { } }); } + +/** Follow logs by polling labd. */ +async function followLogs( + mac: string, + colors: { BOLD: string; GREEN: string; YELLOW: string; RED: string; DIM: string; RESET: string }, +): Promise { + const { BOLD, GREEN, YELLOW, RED, DIM, RESET } = colors; + const client = getLabdClient(); + + console.log(`${DIM}Following logs for ${mac} (Ctrl+C to stop)${RESET}`); + console.log(""); + + let lastStageCount = 0; + let lastStatus = ""; + + while (true) { + try { + const data = await client.getMachineLogs(mac); + const status = String(data["status"] ?? ""); + const log = data["log"] as Array<{ stage: string; detail: string; timestamp: string }> | undefined; + + // Print header once or on status change + if (status !== lastStatus) { + const hostname = String(data["hostname"] ?? mac); + const statusColor = status === "installed" ? GREEN : YELLOW; + console.log(` ${BOLD}${hostname}${RESET} ${statusColor}${status}${RESET}`); + lastStatus = status; + } + + // Print new stages + if (log && log.length > lastStageCount) { + for (let i = lastStageCount; i < log.length; i++) { + const entry = log[i]!; + const time = entry.timestamp.slice(11, 19); + const color = entry.stage === "complete" ? GREEN : entry.stage === "error" ? RED : YELLOW; + const detail = entry.detail ? ` ${DIM}-- ${entry.detail}${RESET}` : ""; + console.log(` ${DIM}${time}${RESET} ${color}${entry.stage}${RESET}${detail}`); + } + lastStageCount = log.length; + } + + // Done + if (status === "installed") { + const ip = data["ip"] ?? ""; + console.log(""); + console.log(` ${GREEN}${BOLD}Install complete!${RESET}${ip ? ` ${DIM}ssh lab@${ip}${RESET}` : ""}`); + process.exit(0); + } + } catch { + // Machine may not be in logs yet (still queued) + } + + await new Promise((r) => setTimeout(r, 5000)); + } +} diff --git a/bastion/src/labd/src/routes/bastions.ts b/bastion/src/labd/src/routes/bastions.ts index 218805e..9c8e181 100644 --- a/bastion/src/labd/src/routes/bastions.ts +++ b/bastion/src/labd/src/routes/bastions.ts @@ -151,7 +151,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void try { const result = await sendCommand(all[0]!.bastionId, { type: "command-install", - mac, hostname, disk: disk ?? "/dev/sda", role: role ?? "infra", os: os ?? "fedora-43", + mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43", }); return reply.code(result.status === "ok" ? 200 : 500).send(result); } catch (err) { @@ -164,7 +164,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void try { const result = await sendCommand(bastion.bastionId, { type: "command-install", - mac, hostname, disk: disk ?? "/dev/sda", role: role ?? "infra", os: os ?? "fedora-43", + mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43", }); return reply.code(result.status === "ok" ? 200 : 500).send(result); } catch (err) { @@ -174,10 +174,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void // Queue debug/rescue mode — route to correct bastion by MAC app.post<{ - Body: { mac?: string; sshd?: boolean; pxeBoot?: boolean }; + Body: { mac?: string; pxeBoot?: boolean }; }>("/api/machines/debug", async (request, reply) => { const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":"); - const sshd = request.body?.sshd ?? false; const pxeBoot = request.body?.pxeBoot ?? false; if (!mac) { return reply.code(400).send({ error: "mac is required" }); @@ -191,7 +190,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void } if (all.length === 1) { try { - const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac, sshd, pxeBoot }); + const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac, pxeBoot }); return reply.code(result.status === "ok" ? 200 : 500).send(result); } catch (err) { return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) }); @@ -201,7 +200,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void } try { - const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac, sshd, pxeBoot }); + const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac, pxeBoot }); return reply.code(result.status === "ok" ? 200 : 500).send(result); } catch (err) { return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) }); diff --git a/bastion/src/shared/src/protocol/index.ts b/bastion/src/shared/src/protocol/index.ts index 3b5054e..e2bdd1c 100644 --- a/bastion/src/shared/src/protocol/index.ts +++ b/bastion/src/shared/src/protocol/index.ts @@ -111,7 +111,7 @@ export type LabdBastionMessage = | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string } | { type: "command-forget"; requestId: string; mac: string } | { type: "command-role-update"; requestId: string; mac: string; role: string } - | { type: "command-debug"; requestId: string; mac: string; sshd?: boolean; pxeBoot?: boolean } + | { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean } | { type: "server-shutdown"; reconnectAfter: number }; export type BastionMessageType = BastionMessage["type"]; diff --git a/bastion/src/shared/src/types/state.ts b/bastion/src/shared/src/types/state.ts index eaadf1f..689f09a 100644 --- a/bastion/src/shared/src/types/state.ts +++ b/bastion/src/shared/src/types/state.ts @@ -101,7 +101,6 @@ export interface InstalledInfo { export interface DebugConfig { hostname: string; queued_at: string; - sshd?: boolean; pxeBoot?: boolean; } diff --git a/bastion/tests/integration/pxe-provision.test.ts b/bastion/tests/integration/pxe-provision.test.ts index 2d1f20b..9e5deb7 100644 --- a/bastion/tests/integration/pxe-provision.test.ts +++ b/bastion/tests/integration/pxe-provision.test.ts @@ -224,11 +224,12 @@ describe("PXE boot provisioning", () => { // Generate dnsmasq config generateDnsmasqConf(config); - // Start HTTP server - const { app, state } = createApp(config); + // Start HTTP server + syslog listener + const { app, state, syslog } = createApp(config); bastionApp = app; await app.listen({ port: config.httpPort, host: "0.0.0.0" }); - log(`Bastion HTTP server listening on :${HTTP_PORT}`); + syslog.start(); + log(`Bastion HTTP server listening on :${HTTP_PORT}, syslog on UDP :${config.syslogPort}`); // Start dnsmasq (fire-and-forget — it runs until killed) // May fail without root (DHCP socket needs CAP_NET_BIND_SERVICE); libvirt network provides DHCP fallback @@ -387,8 +388,8 @@ describe("PXE boot provisioning", () => { expect(data.progress).toBe("complete"); }); - it.skip("log lines were captured", async () => { - // Requires log streamer in %post — skipped until re-added + it("syslog install logs were captured", async () => { + // Anaconda forwards logs via syslog (logging --host directive in kickstart) const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`); const data = (await res.json()) as { log_total?: number; log_lines?: Array<{ line: string }> }; expect(data.log_total).toBeGreaterThan(0);