feat: PXE debug boot mode for rescue/diagnostics #4
@@ -62,13 +62,13 @@ _labctl() {
|
||||
COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision debug")
|
||||
COMPREPLY=($(compgen -W "--sshd -h --help" -- "$cur"))
|
||||
COMPREPLY=($(compgen -W "--pxe-boot -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision forget")
|
||||
COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision logs")
|
||||
COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
|
||||
COMPREPLY=($(compgen -W "-f --follow -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision makeiso")
|
||||
COMPREPLY=($(compgen -W "--arch --local --out -h --help" -- "$cur"))
|
||||
|
||||
@@ -138,7 +138,10 @@ complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operatin
|
||||
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l disk -d 'Target disk device (auto-detect if omitted)' -x
|
||||
|
||||
# provision debug options
|
||||
complete -c labctl -n "__labctl_in_cmd provision debug" -l sshd -d 'Start SSH + nc listener automatically, report IP to bastion'
|
||||
complete -c labctl -n "__labctl_in_cmd provision debug" -l pxe-boot -d 'Boot installed system via PXE (kernel+initrd from network, root from NVMe)'
|
||||
|
||||
# provision logs options
|
||||
complete -c labctl -n "__labctl_in_cmd provision logs" -s f -l follow -d 'Follow log output in real-time'
|
||||
|
||||
# provision makeiso options
|
||||
complete -c labctl -n "__labctl_in_cmd provision makeiso" -l arch -d 'Target architecture(s)' -xa 'x86_64 aarch64'
|
||||
|
||||
@@ -257,7 +257,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
|
||||
state.update((s) => {
|
||||
s.install_queue[msg.mac] = {
|
||||
hostname: msg.hostname,
|
||||
disk: msg.disk ?? "/dev/sda",
|
||||
disk: msg.disk ?? "",
|
||||
role: msg.role as import("@lab/shared").Role,
|
||||
os: msg.os as import("@lab/shared").OsId,
|
||||
queued_at: new Date().toISOString(),
|
||||
@@ -269,7 +269,6 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
|
||||
labdConn.onCommand("command-debug", async (msg) => {
|
||||
if (msg.type !== "command-debug") throw new Error("unexpected");
|
||||
const mac = msg.mac.toLowerCase();
|
||||
const sshd = msg.sshd ?? false;
|
||||
const pxeBoot = msg.pxeBoot ?? false;
|
||||
const currentState = state.load();
|
||||
const hostname =
|
||||
@@ -278,7 +277,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
|
||||
currentState.discovered[mac]?.product ??
|
||||
mac;
|
||||
state.update((s) => {
|
||||
s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd, pxeBoot };
|
||||
s.debug[mac] = { hostname, queued_at: new Date().toISOString(), pxeBoot };
|
||||
});
|
||||
return { status: "ok", data: { mac, hostname } };
|
||||
});
|
||||
|
||||
@@ -13,11 +13,13 @@ import { triggerPostProvisionK3s } from "../services/post-provision.js";
|
||||
import { progressBus } from "../services/progress-events.js";
|
||||
import type { ProgressEvent } from "../services/progress-events.js";
|
||||
import type { InstallLogBuffer } from "../services/install-log.js";
|
||||
import type { SyslogListener } from "../services/syslog-listener.js";
|
||||
|
||||
export function registerApiRoutes(
|
||||
app: FastifyInstance,
|
||||
state: StateManager,
|
||||
installLog: InstallLogBuffer,
|
||||
syslog: SyslogListener,
|
||||
): void {
|
||||
// List all machines
|
||||
app.get("/api/machines", async (_request, reply) => {
|
||||
@@ -84,6 +86,11 @@ export function registerApiRoutes(
|
||||
const { mac: rawMac, stage, detail } = request.body ?? {};
|
||||
const mac = (rawMac ?? "unknown").toLowerCase();
|
||||
const stageName = stage ?? "unknown";
|
||||
|
||||
// Register IP → MAC for syslog routing
|
||||
if (mac !== "unknown") {
|
||||
syslog.registerIp(request.ip, mac);
|
||||
}
|
||||
const detailStr = detail ?? "";
|
||||
|
||||
const GREEN = "\x1b[0;32m";
|
||||
@@ -191,10 +198,9 @@ export function registerApiRoutes(
|
||||
|
||||
// Queue debug/rescue mode for a machine
|
||||
app.post<{
|
||||
Body: { mac?: string; sshd?: boolean; pxeBoot?: boolean };
|
||||
Body: { mac?: string; pxeBoot?: boolean };
|
||||
}>("/api/debug", async (request, reply) => {
|
||||
const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":");
|
||||
const sshd = request.body?.sshd ?? false;
|
||||
const pxeBoot = request.body?.pxeBoot ?? false;
|
||||
if (mac === "") {
|
||||
return reply.status(400).send({ error: "mac is required" });
|
||||
@@ -209,7 +215,7 @@ export function registerApiRoutes(
|
||||
mac;
|
||||
|
||||
state.update((s) => {
|
||||
s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd, pxeBoot };
|
||||
s.debug[mac] = { hostname, queued_at: new Date().toISOString(), pxeBoot };
|
||||
});
|
||||
|
||||
logger.info(`DEBUG QUEUED: ${mac} -> ${hostname}`);
|
||||
|
||||
@@ -23,21 +23,44 @@ export function registerDispatchRoutes(
|
||||
config: BastionConfig,
|
||||
state: StateManager,
|
||||
): void {
|
||||
// Serve debug/rescue kickstart (minimal: SSH keys + network)
|
||||
app.get<{ Querystring: { mac?: string; sshd?: string } }>("/debug.ks", async (request, reply) => {
|
||||
const mac = (request.query.mac ?? "").toLowerCase().replace(/-/g, ":");
|
||||
const currentState = state.load();
|
||||
const wantSshd = request.query.sshd === "1" || currentState.debug[mac]?.sshd === true;
|
||||
|
||||
// Serve debug/rescue kickstart (minimal: SSH keys + network for inst.sshd)
|
||||
app.get<{ Querystring: { mac?: string } }>("/debug.ks", async (_request, reply) => {
|
||||
const ks = renderDebugKickstart({
|
||||
sshKeys: config.sshKeys ?? [],
|
||||
sshd: wantSshd,
|
||||
serverIp: config.serverIp,
|
||||
httpPort: config.httpPort,
|
||||
});
|
||||
return reply.type("text/plain").send(ks);
|
||||
});
|
||||
|
||||
// Shell script for manual debug setup (nc listener + IP reporting)
|
||||
// Usage from rescue shell: curl http://bastion:port/debug-setup.sh | bash
|
||||
app.get("/debug-setup.sh", async (_request, reply) => {
|
||||
const script = `#!/bin/bash
|
||||
# Lab Bastion debug setup — run from rescue shell
|
||||
set -x
|
||||
|
||||
IP_ADDR=$(ip -4 addr show | awk '/inet / && !/127.0.0/ {split($2,a,"/"); print a[1]; exit}')
|
||||
MAC_ADDR=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')
|
||||
|
||||
# Start persistent nc listener for remote shell
|
||||
(while true; do nc -l -p 2323 -e /bin/bash 2>/dev/null; done) &
|
||||
echo "nc shell listener on port 2323"
|
||||
|
||||
# Report IP to bastion
|
||||
curl -sf -X POST "http://${config.serverIp}:${config.httpPort}/api/progress" \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d "{\\"mac\\":\\"$MAC_ADDR\\",\\"stage\\":\\"debug-ready\\",\\"detail\\":\\"nc $IP_ADDR 2323\\"}" 2>/dev/null || true
|
||||
|
||||
echo ""
|
||||
echo "=== Debug environment ready ==="
|
||||
echo " nc $IP_ADDR 2323 (remote shell)"
|
||||
echo " ssh root@$IP_ADDR (password: debug)"
|
||||
echo "==============================="
|
||||
`;
|
||||
return reply.type("text/plain").send(script);
|
||||
});
|
||||
|
||||
app.get<{ Querystring: { mac?: string } }>("/dispatch", async (request, reply) => {
|
||||
const mac = (request.query.mac ?? "").toLowerCase().replace(/-/g, ":");
|
||||
const currentState = state.load();
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import type { BastionConfig } from "@lab/shared";
|
||||
import type { StateManager } from "../services/state.js";
|
||||
import type { SyslogListener } from "../services/syslog-listener.js";
|
||||
import { generateInstallKickstart, generateDiscoverKickstart } from "../services/kickstart-generator.js";
|
||||
import { renderUbuntuAutoinstall, renderUbuntuMetaData, type UbuntuAutoinstallParams } from "../templates/ubuntu-autoinstall.js";
|
||||
|
||||
@@ -12,6 +13,7 @@ export function registerKickstartRoutes(
|
||||
app: FastifyInstance,
|
||||
config: BastionConfig,
|
||||
state: StateManager,
|
||||
syslog: SyslogListener,
|
||||
): void {
|
||||
// Per-MAC install kickstart
|
||||
app.get<{ Querystring: { mac?: string } }>("/ks", async (request, reply) => {
|
||||
@@ -19,6 +21,11 @@ export function registerKickstartRoutes(
|
||||
const currentState = state.load();
|
||||
const queueEntry = currentState.install_queue[mac];
|
||||
|
||||
// Register IP → MAC so syslog listener can route Anaconda logs
|
||||
if (mac) {
|
||||
syslog.registerIp(request.ip, mac);
|
||||
}
|
||||
|
||||
const ks = generateInstallKickstart(config, {
|
||||
hostname: queueEntry?.hostname ?? "lab-node",
|
||||
disk: queueEntry?.disk ?? "",
|
||||
|
||||
@@ -43,8 +43,8 @@ export function createApp(config: BastionConfig): { app: ReturnType<typeof Fasti
|
||||
|
||||
// Register route handlers
|
||||
registerDispatchRoutes(app, config, state);
|
||||
registerKickstartRoutes(app, config, state);
|
||||
registerApiRoutes(app, state, installLog);
|
||||
registerKickstartRoutes(app, config, state, syslog);
|
||||
registerApiRoutes(app, state, installLog, syslog);
|
||||
// boot.iso is generated at startup and served as a static file from httpDir
|
||||
// (static serving supports HTTP Range requests, required by JetKVM streaming)
|
||||
|
||||
|
||||
@@ -30,6 +30,8 @@ export class SyslogListener {
|
||||
private port: number;
|
||||
private installLog: InstallLogBuffer;
|
||||
private state: StateManager;
|
||||
/** Explicit IP → MAC mapping registered from kickstart/progress requests. */
|
||||
private ipToMac = new Map<string, string>();
|
||||
|
||||
constructor(port: number, installLog: InstallLogBuffer, state: StateManager) {
|
||||
this.port = port;
|
||||
@@ -37,14 +39,21 @@ export class SyslogListener {
|
||||
this.state = state;
|
||||
}
|
||||
|
||||
/** Resolve a source IP to a MAC address using the install queue. */
|
||||
/** Register an IP → MAC mapping (called when we learn a machine's IP). */
|
||||
registerIp(ip: string, mac: string): void {
|
||||
this.ipToMac.set(ip, mac.toLowerCase());
|
||||
}
|
||||
|
||||
/** Resolve a source IP to a MAC address. */
|
||||
private resolveIpToMac(ip: string): string | null {
|
||||
// Check explicit mapping first (most reliable)
|
||||
const explicit = this.ipToMac.get(ip);
|
||||
if (explicit) return explicit;
|
||||
|
||||
const currentState = this.state.load();
|
||||
|
||||
// Check install queue — machines being installed have an IP from DHCP
|
||||
for (const [mac, entry] of Object.entries(currentState.install_queue)) {
|
||||
// The progress callback sends IP in "complete" detail, but during install
|
||||
// we need to match by what we know. Check if any progress mentions this IP.
|
||||
if (entry.progress_detail?.includes(ip)) return mac;
|
||||
}
|
||||
|
||||
|
||||
@@ -124,7 +124,7 @@ echo Kernel+initrd from PXE, root from NVMe
|
||||
echo =============================================
|
||||
echo
|
||||
|
||||
kernel http://${params.serverIp}:${params.httpPort}/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0 console=ttyS0,115200n8 modprobe.blacklist=amdgpu
|
||||
kernel http://${params.serverIp}:${params.httpPort}/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0
|
||||
initrd http://${params.serverIp}:${params.httpPort}/initrd.img
|
||||
boot
|
||||
`;
|
||||
|
||||
@@ -1,81 +1,33 @@
|
||||
// Debug/rescue kickstart template.
|
||||
// Minimal kickstart for Anaconda rescue mode.
|
||||
// When sshd=true: generates host keys, starts sshd, reports IP to bastion.
|
||||
// No dependency on mounted filesystems — fully self-contained.
|
||||
//
|
||||
// SSH access: Anaconda's inst.sshd starts sshd automatically.
|
||||
// The sshpw directive sets the password, sshkey adds authorized keys.
|
||||
// %pre/%post do NOT run in rescue mode — don't put setup code there.
|
||||
|
||||
export interface DebugKickstartParams {
|
||||
sshKeys: string[];
|
||||
sshd?: boolean;
|
||||
serverIp?: string;
|
||||
httpPort?: number;
|
||||
}
|
||||
|
||||
export function renderDebugKickstart(params: DebugKickstartParams): string {
|
||||
const sshpw = "sshpw --username=root --plaintext lab-root-pw";
|
||||
const sshkeyLine = params.sshKeys.length > 0
|
||||
? `sshkey --username=root "${params.sshKeys[0]}"`
|
||||
: "";
|
||||
|
||||
const sshdSetup = params.sshd ? `
|
||||
%pre --log=/tmp/debug-sshd.log
|
||||
#!/bin/bash
|
||||
set -x
|
||||
|
||||
# Wait for network to come up
|
||||
for i in $(seq 1 30); do
|
||||
IP_ADDR=$(ip -4 addr show | awk '/inet / && !/127.0.0/ {split($2,a,"/"); print a[1]; exit}')
|
||||
[ -n "$IP_ADDR" ] && break
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Generate host keys (self-contained, no mounted FS needed)
|
||||
ssh-keygen -t ed25519 -f /tmp/ssh_host_ed25519_key -N "" -q
|
||||
ssh-keygen -t rsa -f /tmp/ssh_host_rsa_key -N "" -q
|
||||
|
||||
# Write minimal sshd config
|
||||
cat > /tmp/sshd_config << 'SSHCFG'
|
||||
HostKey /tmp/ssh_host_ed25519_key
|
||||
HostKey /tmp/ssh_host_rsa_key
|
||||
PermitRootLogin yes
|
||||
PasswordAuthentication yes
|
||||
PubkeyAuthentication yes
|
||||
AuthorizedKeysFile /root/.ssh/authorized_keys
|
||||
SSHCFG
|
||||
|
||||
# Set root password for SSH access
|
||||
echo "root:debug" | chpasswd
|
||||
|
||||
# Set up SSH authorized keys
|
||||
mkdir -p /root/.ssh && chmod 700 /root/.ssh
|
||||
${params.sshKeys.map(k => `echo '${k}' >> /root/.ssh/authorized_keys`).join("\n")}
|
||||
chmod 600 /root/.ssh/authorized_keys 2>/dev/null || true
|
||||
|
||||
# Start sshd
|
||||
/usr/sbin/sshd -f /tmp/sshd_config -p 22
|
||||
echo "sshd started on port 22"
|
||||
|
||||
# Start persistent nc listener for remote shell
|
||||
(while true; do nc -l -p 2323 -e /bin/bash 2>/dev/null; done) &
|
||||
echo "nc shell listener on port 2323"
|
||||
|
||||
# Report IP to bastion
|
||||
MAC_ADDR=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')
|
||||
curl -sf -X POST "http://${params.serverIp}:${params.httpPort}/api/progress" \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d "{\\"mac\\":\\"$MAC_ADDR\\",\\"stage\\":\\"debug-ready\\",\\"detail\\":\\"ssh root@$IP_ADDR (pw: debug) | nc $IP_ADDR 2323\\"}" 2>/dev/null || true
|
||||
|
||||
echo "Debug environment ready: ssh root@$IP_ADDR or nc $IP_ADDR 2323"
|
||||
%end
|
||||
` : "";
|
||||
|
||||
return `# Lab Bastion -- Debug/Rescue Kickstart
|
||||
# Minimal: SSH + network for Anaconda rescue mode
|
||||
#
|
||||
# SSH is started by Anaconda (inst.sshd kernel param).
|
||||
# Password: debug | SSH keys from bastion config.
|
||||
# %pre/%post do NOT run in rescue mode.
|
||||
|
||||
lang en_US.UTF-8
|
||||
keyboard uk
|
||||
network --bootproto=dhcp --activate
|
||||
|
||||
${sshpw}
|
||||
sshpw --username=root --plaintext debug
|
||||
${sshkeyLine}
|
||||
${sshdSetup}`;
|
||||
`;
|
||||
}
|
||||
|
||||
@@ -134,10 +134,9 @@ network --bootproto=dhcp --activate --hostname=${fqdn}
|
||||
${auth}
|
||||
${userDirective}
|
||||
|
||||
bootloader --append="console=tty0 console=ttyS0,115200n8"
|
||||
bootloader --append="console=tty0"
|
||||
|
||||
# logging --host=${serverIp} --port=${syslogPort}
|
||||
# Disabled: syslog UDP port needs to be exposed in k3s service/hostPort first
|
||||
logging --host=${serverIp} --port=${syslogPort}
|
||||
|
||||
url --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-$releasever&arch=$basearch
|
||||
|
||||
@@ -342,17 +341,7 @@ echo "tmpfs /tmp tmpfs defaults,noatime,nosuid,nodev,size=4G 0 0" >> /etc/fstab
|
||||
|
||||
${isVanilla ? `# -- vanilla role: skip k3s kernel/sysctl/firewall setup --
|
||||
# -- Enable chronyd for time sync --
|
||||
systemctl enable chronyd || true
|
||||
|
||||
# -- Serial console (for debugging — auto-login as root on ttyS0) --
|
||||
# AWS EC2 compatible: ttyS0 @ 115200n8
|
||||
systemctl enable serial-getty@ttyS0.service || true
|
||||
|
||||
# -- Forward all system logs to serial console --
|
||||
cat > /etc/rsyslog.d/serial-console.conf << 'RSYSLOG'
|
||||
*.* /dev/ttyS0
|
||||
RSYSLOG
|
||||
systemctl enable rsyslog || true` : `# -- Kernel modules for k3s --
|
||||
systemctl enable chronyd || true` : `# -- Kernel modules for k3s --
|
||||
cat > /etc/modules-load.d/k3s.conf << 'MODULES'
|
||||
br_netfilter
|
||||
overlay
|
||||
@@ -396,6 +385,9 @@ fi
|
||||
|
||||
bastion_progress "post-install" "3-bootorder done"
|
||||
|
||||
# -- Enable SysRq magic keys (for emergency reboot via Alt+SysRq+REISUB) --
|
||||
echo "kernel.sysrq=1" > /etc/sysctl.d/90-sysrq.conf
|
||||
|
||||
# -- Provisioning metadata --
|
||||
cat > /etc/lab-provisioned << PROVEOF
|
||||
hostname: ${fqdn}
|
||||
|
||||
@@ -28,6 +28,7 @@ function createTestConfig(testDir: string): BastionConfig {
|
||||
gateway: "10.0.0.1",
|
||||
sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"],
|
||||
adminUser: "testadmin",
|
||||
syslogPort: 15514,
|
||||
skipDnsmasq: true,
|
||||
skipArtifacts: true,
|
||||
fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
|
||||
|
||||
@@ -206,10 +206,8 @@ describe("renderInstallKickstart", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("forwards system logs to serial console", () => {
|
||||
it("does not include serial console (causes 30s boot timeout on hardware without UART)", () => {
|
||||
const ks = renderInstallKickstart(baseParams({ role: "vanilla" }));
|
||||
expect(ks).toContain("serial-console.conf");
|
||||
expect(ks).toContain("/dev/ttyS0");
|
||||
expect(ks).toContain("rsyslog");
|
||||
expect(ks).not.toContain("ttyS0");
|
||||
});
|
||||
});
|
||||
|
||||
121
bastion/src/bastion/tests/syslog-listener.test.ts
Normal file
121
bastion/src/bastion/tests/syslog-listener.test.ts
Normal file
@@ -0,0 +1,121 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { createSocket } from "node:dgram";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { SyslogListener } from "../src/services/syslog-listener.js";
|
||||
import { InstallLogBuffer } from "../src/services/install-log.js";
|
||||
import { StateManager } from "../src/services/state.js";
|
||||
|
||||
function sendUdpSyslog(port: number, message: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const client = createSocket("udp4");
|
||||
const buf = Buffer.from(message);
|
||||
client.send(buf, 0, buf.length, port, "127.0.0.1", (err) => {
|
||||
client.close();
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
describe("SyslogListener", () => {
|
||||
let tmpDir: string;
|
||||
let state: StateManager;
|
||||
let installLog: InstallLogBuffer;
|
||||
let syslog: SyslogListener;
|
||||
const PORT = 15514; // use non-privileged port for testing
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), "syslog-test-"));
|
||||
state = new StateManager(join(tmpDir, "state.json"));
|
||||
state.init();
|
||||
installLog = new InstallLogBuffer(tmpDir);
|
||||
syslog = new SyslogListener(PORT, installLog, state);
|
||||
syslog.start();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
syslog.stop();
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("receives and stores syslog messages for registered IP", async () => {
|
||||
const mac = "aa:bb:cc:dd:ee:ff";
|
||||
// Queue a machine so hostname can be resolved
|
||||
state.update((s) => {
|
||||
s.install_queue[mac] = {
|
||||
hostname: "testnode",
|
||||
disk: "/dev/sda",
|
||||
role: "worker",
|
||||
os: "fedora-43",
|
||||
queued_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
// Register IP → MAC mapping
|
||||
syslog.registerIp("127.0.0.1", mac);
|
||||
|
||||
// Send a syslog message (RFC 3164 format)
|
||||
await sendUdpSyslog(PORT, "<13>Mar 30 01:30:00 localhost anaconda[1234]: Installing package vim-enhanced");
|
||||
|
||||
// Wait for UDP delivery
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
|
||||
const lines = installLog.getLines(mac);
|
||||
expect(lines.length).toBeGreaterThan(0);
|
||||
expect(lines[0]!.line).toContain("anaconda");
|
||||
expect(lines[0]!.line).toContain("Installing package vim-enhanced");
|
||||
});
|
||||
|
||||
it("ignores messages from unknown IPs", async () => {
|
||||
// Don't register any IP mapping
|
||||
await sendUdpSyslog(PORT, "<13>Mar 30 01:30:00 localhost anaconda[1234]: test message");
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
|
||||
// No MAC to check, but the listener should not crash
|
||||
// and no logs should be stored for any MAC
|
||||
expect(installLog.lineCount("unknown")).toBe(0);
|
||||
});
|
||||
|
||||
it("resolves IP from installed machines state", async () => {
|
||||
const mac = "11:22:33:44:55:66";
|
||||
state.update((s) => {
|
||||
s.installed[mac] = {
|
||||
hostname: "installed-node",
|
||||
role: "worker",
|
||||
ip: "127.0.0.1",
|
||||
installed_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
await sendUdpSyslog(PORT, "<14>Mar 30 02:00:00 installed-node sshd[5678]: Accepted publickey for root");
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
|
||||
const lines = installLog.getLines(mac);
|
||||
expect(lines.length).toBeGreaterThan(0);
|
||||
expect(lines[0]!.line).toContain("sshd");
|
||||
});
|
||||
|
||||
it("parses various syslog formats", async () => {
|
||||
const mac = "aa:bb:cc:dd:ee:ff";
|
||||
syslog.registerIp("127.0.0.1", mac);
|
||||
state.update((s) => {
|
||||
s.install_queue[mac] = {
|
||||
hostname: "testnode",
|
||||
disk: "/dev/sda",
|
||||
role: "worker",
|
||||
os: "fedora-43",
|
||||
queued_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
// Message without PID
|
||||
await sendUdpSyslog(PORT, "<13>Mar 30 01:30:00 localhost kernel: NVMe device ready");
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
|
||||
const lines = installLog.getLines(mac);
|
||||
expect(lines.length).toBeGreaterThan(0);
|
||||
expect(lines[0]!.line).toContain("kernel");
|
||||
});
|
||||
});
|
||||
@@ -94,8 +94,8 @@ export class LabdClient {
|
||||
return this.request("POST", "/api/machines/install", { body: opts });
|
||||
}
|
||||
|
||||
async debugMachine(mac: string, opts?: { sshd?: boolean; pxeBoot?: boolean }): Promise<{ status: string; data?: { mac: string; hostname: string }; error?: string }> {
|
||||
return this.request("POST", "/api/machines/debug", { body: { mac, sshd: opts?.sshd, pxeBoot: opts?.pxeBoot } });
|
||||
async debugMachine(mac: string, opts?: { pxeBoot?: boolean }): Promise<{ status: string; data?: { mac: string; hostname: string }; error?: string }> {
|
||||
return this.request("POST", "/api/machines/debug", { body: { mac, pxeBoot: opts?.pxeBoot } });
|
||||
}
|
||||
|
||||
async forgetMachine(mac: string): Promise<{ status: string }> {
|
||||
|
||||
@@ -48,10 +48,9 @@ export function registerDebugCommand(parent: Command): void {
|
||||
parent
|
||||
.command("debug <target>")
|
||||
.description("PXE boot into Fedora rescue mode for debugging (target: hostname, MAC, or IP)")
|
||||
.option("--sshd", "Start SSH + nc listener automatically, report IP to bastion")
|
||||
.option("--pxe-boot", "Boot installed system via PXE (kernel+initrd from network, root from NVMe)")
|
||||
.showHelpAfterError(true)
|
||||
.action(async (target: string, opts: { sshd?: boolean; pxeBoot?: boolean }) => {
|
||||
.action(async (target: string, opts: { pxeBoot?: boolean }) => {
|
||||
const client = getLabdClient();
|
||||
|
||||
// Resolve target from labd aggregated state
|
||||
@@ -75,7 +74,7 @@ export function registerDebugCommand(parent: Command): void {
|
||||
console.log(`Queuing debug mode for ${hostname} (${mac})...`);
|
||||
|
||||
try {
|
||||
const result = await client.debugMachine(mac, { sshd: opts.sshd === true, pxeBoot: opts.pxeBoot === true });
|
||||
const result = await client.debugMachine(mac, { pxeBoot: opts.pxeBoot === true });
|
||||
if (result.error) {
|
||||
console.error(`Failed: ${result.error}`);
|
||||
process.exit(1);
|
||||
@@ -118,15 +117,21 @@ export function registerDebugCommand(parent: Command): void {
|
||||
}
|
||||
}
|
||||
|
||||
const sshdNote = opts.sshd
|
||||
? `\nSSH + nc listener will start automatically. Watch bastion logs for the IP callback.
|
||||
Password: debug | nc <ip> 2323 for raw shell\n`
|
||||
: "";
|
||||
// Determine bastion URL from labd config for the setup script URL
|
||||
const bastionUrl = process.env["LABD_URL"]
|
||||
? process.env["LABD_URL"].replace(/\/ws\/bastion$/, "").replace(/^wss?:/, "http:")
|
||||
: "http://<bastion-ip>:8080";
|
||||
|
||||
console.log(`
|
||||
Debug mode queued for ${hostname} (${mac}).
|
||||
Reboot the machine to enter Fedora rescue mode.
|
||||
${sshdNote}
|
||||
|
||||
SSH access (started by Anaconda):
|
||||
ssh root@<ip> (password: debug)
|
||||
|
||||
For nc remote shell, run from rescue shell:
|
||||
curl ${bastionUrl}/debug-setup.sh | bash
|
||||
|
||||
Once in rescue shell:
|
||||
|
||||
# Activate LVM and mount installed system
|
||||
|
||||
@@ -39,12 +39,10 @@ export function registerLogsCommand(parent: Command): void {
|
||||
parent
|
||||
.command("logs <target>")
|
||||
.description("Show provisioning logs for a machine (hostname, MAC, or IP)")
|
||||
.action(async (target: string) => {
|
||||
.option("-f, --follow", "Follow log output in real-time")
|
||||
.action(async (target: string, opts: { follow?: boolean }) => {
|
||||
const mac = await resolveToMac(target);
|
||||
|
||||
try {
|
||||
const data = await getLabdClient().getMachineLogs(mac);
|
||||
|
||||
const BOLD = "\x1b[1m";
|
||||
const GREEN = "\x1b[32m";
|
||||
const YELLOW = "\x1b[33m";
|
||||
@@ -52,6 +50,14 @@ export function registerLogsCommand(parent: Command): void {
|
||||
const DIM = "\x1b[2m";
|
||||
const RESET = "\x1b[0m";
|
||||
|
||||
if (opts.follow) {
|
||||
await followLogs(mac, { BOLD, GREEN, YELLOW, RED, DIM, RESET });
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const data = await getLabdClient().getMachineLogs(mac);
|
||||
|
||||
console.log(`${BOLD}${data["hostname"]}${RESET} (${mac})`);
|
||||
console.log(` Status: ${data["status"] === "installed" ? GREEN : YELLOW}${data["status"]}${RESET}`);
|
||||
console.log(` Role: ${data["role"]}`);
|
||||
@@ -83,3 +89,58 @@ export function registerLogsCommand(parent: Command): void {
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Follow logs by polling labd. */
|
||||
async function followLogs(
|
||||
mac: string,
|
||||
colors: { BOLD: string; GREEN: string; YELLOW: string; RED: string; DIM: string; RESET: string },
|
||||
): Promise<void> {
|
||||
const { BOLD, GREEN, YELLOW, RED, DIM, RESET } = colors;
|
||||
const client = getLabdClient();
|
||||
|
||||
console.log(`${DIM}Following logs for ${mac} (Ctrl+C to stop)${RESET}`);
|
||||
console.log("");
|
||||
|
||||
let lastStageCount = 0;
|
||||
let lastStatus = "";
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
const data = await client.getMachineLogs(mac);
|
||||
const status = String(data["status"] ?? "");
|
||||
const log = data["log"] as Array<{ stage: string; detail: string; timestamp: string }> | undefined;
|
||||
|
||||
// Print header once or on status change
|
||||
if (status !== lastStatus) {
|
||||
const hostname = String(data["hostname"] ?? mac);
|
||||
const statusColor = status === "installed" ? GREEN : YELLOW;
|
||||
console.log(` ${BOLD}${hostname}${RESET} ${statusColor}${status}${RESET}`);
|
||||
lastStatus = status;
|
||||
}
|
||||
|
||||
// Print new stages
|
||||
if (log && log.length > lastStageCount) {
|
||||
for (let i = lastStageCount; i < log.length; i++) {
|
||||
const entry = log[i]!;
|
||||
const time = entry.timestamp.slice(11, 19);
|
||||
const color = entry.stage === "complete" ? GREEN : entry.stage === "error" ? RED : YELLOW;
|
||||
const detail = entry.detail ? ` ${DIM}-- ${entry.detail}${RESET}` : "";
|
||||
console.log(` ${DIM}${time}${RESET} ${color}${entry.stage}${RESET}${detail}`);
|
||||
}
|
||||
lastStageCount = log.length;
|
||||
}
|
||||
|
||||
// Done
|
||||
if (status === "installed") {
|
||||
const ip = data["ip"] ?? "";
|
||||
console.log("");
|
||||
console.log(` ${GREEN}${BOLD}Install complete!${RESET}${ip ? ` ${DIM}ssh lab@${ip}${RESET}` : ""}`);
|
||||
process.exit(0);
|
||||
}
|
||||
} catch {
|
||||
// Machine may not be in logs yet (still queued)
|
||||
}
|
||||
|
||||
await new Promise((r) => setTimeout(r, 5000));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,7 +151,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
try {
|
||||
const result = await sendCommand(all[0]!.bastionId, {
|
||||
type: "command-install",
|
||||
mac, hostname, disk: disk ?? "/dev/sda", role: role ?? "infra", os: os ?? "fedora-43",
|
||||
mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
|
||||
});
|
||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||
} catch (err) {
|
||||
@@ -164,7 +164,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
try {
|
||||
const result = await sendCommand(bastion.bastionId, {
|
||||
type: "command-install",
|
||||
mac, hostname, disk: disk ?? "/dev/sda", role: role ?? "infra", os: os ?? "fedora-43",
|
||||
mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
|
||||
});
|
||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||
} catch (err) {
|
||||
@@ -174,10 +174,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
|
||||
// Queue debug/rescue mode — route to correct bastion by MAC
|
||||
app.post<{
|
||||
Body: { mac?: string; sshd?: boolean; pxeBoot?: boolean };
|
||||
Body: { mac?: string; pxeBoot?: boolean };
|
||||
}>("/api/machines/debug", async (request, reply) => {
|
||||
const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":");
|
||||
const sshd = request.body?.sshd ?? false;
|
||||
const pxeBoot = request.body?.pxeBoot ?? false;
|
||||
if (!mac) {
|
||||
return reply.code(400).send({ error: "mac is required" });
|
||||
@@ -191,7 +190,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
}
|
||||
if (all.length === 1) {
|
||||
try {
|
||||
const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac, sshd, pxeBoot });
|
||||
const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac, pxeBoot });
|
||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||
} catch (err) {
|
||||
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
|
||||
@@ -201,7 +200,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac, sshd, pxeBoot });
|
||||
const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac, pxeBoot });
|
||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||
} catch (err) {
|
||||
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
|
||||
|
||||
@@ -111,7 +111,7 @@ export type LabdBastionMessage =
|
||||
| { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string }
|
||||
| { type: "command-forget"; requestId: string; mac: string }
|
||||
| { type: "command-role-update"; requestId: string; mac: string; role: string }
|
||||
| { type: "command-debug"; requestId: string; mac: string; sshd?: boolean; pxeBoot?: boolean }
|
||||
| { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean }
|
||||
| { type: "server-shutdown"; reconnectAfter: number };
|
||||
|
||||
export type BastionMessageType = BastionMessage["type"];
|
||||
|
||||
@@ -101,7 +101,6 @@ export interface InstalledInfo {
|
||||
export interface DebugConfig {
|
||||
hostname: string;
|
||||
queued_at: string;
|
||||
sshd?: boolean;
|
||||
pxeBoot?: boolean;
|
||||
}
|
||||
|
||||
|
||||
@@ -224,11 +224,12 @@ describe("PXE boot provisioning", () => {
|
||||
// Generate dnsmasq config
|
||||
generateDnsmasqConf(config);
|
||||
|
||||
// Start HTTP server
|
||||
const { app, state } = createApp(config);
|
||||
// Start HTTP server + syslog listener
|
||||
const { app, state, syslog } = createApp(config);
|
||||
bastionApp = app;
|
||||
await app.listen({ port: config.httpPort, host: "0.0.0.0" });
|
||||
log(`Bastion HTTP server listening on :${HTTP_PORT}`);
|
||||
syslog.start();
|
||||
log(`Bastion HTTP server listening on :${HTTP_PORT}, syslog on UDP :${config.syslogPort}`);
|
||||
|
||||
// Start dnsmasq (fire-and-forget — it runs until killed)
|
||||
// May fail without root (DHCP socket needs CAP_NET_BIND_SERVICE); libvirt network provides DHCP fallback
|
||||
@@ -387,8 +388,8 @@ describe("PXE boot provisioning", () => {
|
||||
expect(data.progress).toBe("complete");
|
||||
});
|
||||
|
||||
it.skip("log lines were captured", async () => {
|
||||
// Requires log streamer in %post — skipped until re-added
|
||||
it("syslog install logs were captured", async () => {
|
||||
// Anaconda forwards logs via syslog (logging --host directive in kickstart)
|
||||
const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`);
|
||||
const data = (await res.json()) as { log_total?: number; log_lines?: Array<{ line: string }> };
|
||||
expect(data.log_total).toBeGreaterThan(0);
|
||||
|
||||
Reference in New Issue
Block a user