diff --git a/bastion/src/bastion/src/routes/api.ts b/bastion/src/bastion/src/routes/api.ts index 51434fa..3294390 100644 --- a/bastion/src/bastion/src/routes/api.ts +++ b/bastion/src/bastion/src/routes/api.ts @@ -313,27 +313,31 @@ export function registerApiRoutes( // classification already on the record wins (see classifyOnboard). const onboarding = classifyOnboard({ mac, - manufacturer: data.manufacturer ?? "unknown", - product: data.product ?? "unknown", - board: data.board ?? "unknown", + manufacturer: data.manufacturer ?? existing?.manufacturer ?? "unknown", + product: data.product ?? existing?.product ?? "unknown", + board: data.board ?? existing?.board ?? "unknown", ...(existing?.onboard !== undefined ? { onboard: existing.onboard } : {}), ...(existing?.vendor_os !== undefined ? { vendor_os: existing.vendor_os } : {}), }); const rootDevice = data.root_device ?? existing?.root_device; const rootArgs = data.root_args ?? existing?.root_args; + // Absent fields keep whatever we already knew. Reporters are not all the full + // discovery kickstart: the rescue-shell probe posts only a root device, and + // blanking a machine's hardware inventory as a side effect of that would be + // silent data loss. const hwInfo: HardwareInfo = { mac, - product: data.product ?? "unknown", - board: data.board ?? "unknown", - serial: data.serial ?? "unknown", - manufacturer: data.manufacturer ?? "unknown", - cpu_model: data.cpu_model ?? "unknown", - cpu_cores: data.cpu_cores ?? 0, - memory_gb: data.memory_gb ?? 0, - arch: data.arch ?? "unknown", - disks: data.disks ?? [], - nics: data.nics ?? [], + product: data.product ?? existing?.product ?? "unknown", + board: data.board ?? existing?.board ?? "unknown", + serial: data.serial ?? existing?.serial ?? "unknown", + manufacturer: data.manufacturer ?? existing?.manufacturer ?? "unknown", + cpu_model: data.cpu_model ?? existing?.cpu_model ?? "unknown", + cpu_cores: data.cpu_cores ?? existing?.cpu_cores ?? 0, + memory_gb: data.memory_gb ?? existing?.memory_gb ?? 0, + arch: data.arch ?? existing?.arch ?? "unknown", + disks: data.disks ?? existing?.disks ?? [], + nics: data.nics ?? existing?.nics ?? [], first_seen: existing?.first_seen ?? now, last_seen: now, onboard: onboarding.onboard, diff --git a/bastion/src/bastion/src/routes/dispatch.ts b/bastion/src/bastion/src/routes/dispatch.ts index c790fa5..2a03921 100644 --- a/bastion/src/bastion/src/routes/dispatch.ts +++ b/bastion/src/bastion/src/routes/dispatch.ts @@ -92,10 +92,58 @@ curl -sf -X POST "http://${config.serverIp}:${config.httpPort}/api/progress" \\ -H "Content-Type: application/json" \\ -d "{\\"mac\\":\\"$MAC_ADDR\\",\\"stage\\":\\"debug-ready\\",\\"detail\\":\\"nc $IP_ADDR 2323\\"}" 2>/dev/null || true +# --- Find the installed root filesystem and report it --- +# This is what 'labctl provision debug --pxe-boot' needs. The rescue image cannot +# report it by itself: %pre/%post do not run in rescue mode, so it happens here. +vgchange -ay >/dev/null 2>&1 || true + +ROOT_DEVICE="" +ROOT_ARGS="" +PROBE_MNT=/tmp/lab-rootprobe +mkdir -p "$PROBE_MNT" + +# Candidates: every LVM logical volume plus every non-LVM partition with a filesystem. +for CAND in $(lvs --noheadings -o lv_path 2>/dev/null) \\ + $(blkid -o device 2>/dev/null | grep -v '^/dev/mapper/'); do + [ -b "$CAND" ] || continue + mount -o ro "$CAND" "$PROBE_MNT" >/dev/null 2>&1 || continue + # A root filesystem has both of these; /boot and /home do not. + if [ -f "$PROBE_MNT/etc/fstab" ] && [ -d "$PROBE_MNT/usr" ]; then + ROOT_DEVICE="$CAND" + PRETTY=$(. "$PROBE_MNT/etc/os-release" 2>/dev/null && echo "$PRETTY_NAME") + echo " found root: $CAND \${PRETTY:+($PRETTY)}" + if [ "$(lsblk -no TYPE "$CAND" 2>/dev/null | head -1)" = "lvm" ]; then + VGLV=$(lvs --noheadings -o vg_name,lv_name "$CAND" 2>/dev/null | awk '{print $1"/"$2}') + [ -n "$VGLV" ] && ROOT_ARGS="rd.lvm.lv=$VGLV" + # Swap comes from fstab here — /proc/swaps is the rescue image's, not the host's. + SWLV=$(awk '$3=="swap" && $1 ~ /^\\/dev\\// {print $1; exit}' "$PROBE_MNT/etc/fstab" 2>/dev/null) + if [ -n "$SWLV" ]; then + SWVGLV=$(lvs --noheadings -o vg_name,lv_name "$SWLV" 2>/dev/null | awk '{print $1"/"$2}') + [ -n "$SWVGLV" ] && [ "$SWVGLV" != "$VGLV" ] && ROOT_ARGS="$ROOT_ARGS rd.lvm.lv=$SWVGLV" + fi + fi + umount "$PROBE_MNT" >/dev/null 2>&1 || true + break + fi + umount "$PROBE_MNT" >/dev/null 2>&1 || true +done + +if [ -n "$ROOT_DEVICE" ]; then + curl -sf -X POST "http://${config.serverIp}:${config.httpPort}/api/discover" \\ + -H "Content-Type: application/json" \\ + -d "{\\"mac\\":\\"$MAC_ADDR\\",\\"root_device\\":\\"$ROOT_DEVICE\\",\\"root_args\\":\\"$ROOT_ARGS\\"}" 2>/dev/null \\ + && echo " reported to bastion — 'labctl provision debug --pxe-boot' will work now" +else + echo " no root filesystem found — --pxe-boot cannot be used on this machine" +fi + echo "" echo "=== Debug environment ready ===" echo " nc $IP_ADDR 2323 (remote shell)" echo " ssh root@$IP_ADDR (password: debug)" +if [ -n "$ROOT_DEVICE" ]; then + echo " root: $ROOT_DEVICE $ROOT_ARGS" +fi echo "===============================" `; return reply.type("text/plain").send(script); diff --git a/bastion/src/cli/src/api/client.ts b/bastion/src/cli/src/api/client.ts index 52add1e..3069dd7 100644 --- a/bastion/src/cli/src/api/client.ts +++ b/bastion/src/cli/src/api/client.ts @@ -110,6 +110,7 @@ export class LabdClient { memory_gb?: number; arch?: string; disks?: Array<{ name: string; size_gb: number; model: string }>; nics?: Array<{ name: string; mac: string; state: string }>; + root_device?: string; root_args?: string; }): Promise<{ status: string; error?: string }> { return this.request("POST", "/api/machines/discover", { body: data }); } diff --git a/bastion/src/cli/src/commands/debug.ts b/bastion/src/cli/src/commands/debug.ts index 903ca8a..4c54dde 100644 --- a/bastion/src/cli/src/commands/debug.ts +++ b/bastion/src/cli/src/commands/debug.ts @@ -8,6 +8,7 @@ import { join } from "node:path"; import { Command } from "commander"; import type { BastionState } from "@lab/shared"; import { getLabdClient } from "../api/config.js"; +import { ROOT_DEVICE_PROBE, parseRootProbe } from "../utils/hardware-probe.js"; /** Resolve a target (hostname, MAC, or IP) to {mac, hostname, ip} from state. */ function resolveTarget( @@ -44,6 +45,54 @@ function resolveTarget( return null; } +/** The local admin account to SSH as (root is not usable — it has no key here). */ +function sshUser(): string { + const adminUser = process.env["SUDO_USER"] ?? process.env["USER"] ?? ""; + return adminUser === "root" ? "" : adminUser; +} + +/** Common ssh arguments, ending with user@host. Null when there is no usable user. */ +function sshBaseArgs(ip: string): string[] | null { + const user = sshUser(); + if (user === "") return null; + + const sudoUser = process.env["SUDO_USER"]; + const realHome = sudoUser !== undefined ? join("/home", sudoUser) : homedir(); + const sshKey = ["id_ed25519", "id_rsa", "id_ecdsa"] + .map((name) => join(realHome, ".ssh", name)) + .find((k) => existsSync(k)); + + return [ + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "ConnectTimeout=10", + ...(sshKey !== undefined ? ["-i", sshKey] : []), + `${user}@${ip}`, + ]; +} + +/** + * Run a shell script on the target as root and return its stdout, or null. + * + * The script goes over stdin rather than the command line so it can contain quotes + * without a second round of shell escaping. `sudo -n` fails fast instead of hanging on + * a password prompt that would then eat the script. + */ +function sshCapture(ip: string, script: string): string | null { + const base = sshBaseArgs(ip); + if (base === null) return null; + try { + return execFileSync("ssh", [...base, "sudo", "-n", "sh", "-s"], { + input: script, + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + timeout: 30_000, + }); + } catch { + return null; + } +} + export function registerDebugCommand(parent: Command): void { parent .command("debug ") @@ -71,6 +120,31 @@ export function registerDebugCommand(parent: Command): void { } const { mac, hostname, ip } = resolved; + + // --pxe-boot needs a root= for the installed system. If the machine is still + // reachable, observe it now rather than assuming a disk layout: a wrong root= + // leaves the machine unbootable. If it isn't reachable, dispatch falls back to + // rescue and the operator reports the real one from there. + if (opts.pxeBoot === true && ip !== "") { + const known = state.installed[mac]?.root_device ?? state.discovered[mac]?.root_device; + if (known === undefined || known === "") { + console.log(`No root device recorded for ${hostname}. Probing over SSH...`); + const probe = sshCapture(ip, ROOT_DEVICE_PROBE); + const root = probe === null ? {} : parseRootProbe(probe); + if (root.root_device !== undefined) { + console.log(` root=${root.root_device}${root.root_args ? ` ${root.root_args}` : ""}`); + try { + await client.discoverMachine({ mac, ...root }); + } catch (err) { + console.error(` Could not record it: ${err instanceof Error ? err.message : String(err)}`); + } + } else { + console.log(" Probe failed. Booting rescue instead; report the root device with:"); + console.log(" curl http://:8080/debug-setup.sh | bash"); + } + } + } + console.log(`Queuing debug mode for ${hostname} (${mac})...`); try { @@ -86,32 +160,15 @@ export function registerDebugCommand(parent: Command): void { // Try SSH reboot into PXE if (ip !== "") { - const adminUser = process.env["SUDO_USER"] ?? process.env["USER"] ?? ""; - const effectiveUser = adminUser === "root" ? "" : adminUser; - - if (effectiveUser !== "") { - console.log(`\nAttempting SSH reboot into PXE (${effectiveUser}@${ip})...`); - - const sudoUser = process.env["SUDO_USER"]; - const realHome = sudoUser !== undefined ? join("/home", sudoUser) : homedir(); - const keyPaths = [ - join(realHome, ".ssh", "id_ed25519"), - join(realHome, ".ssh", "id_rsa"), - join(realHome, ".ssh", "id_ecdsa"), - ]; - const sshKey = keyPaths.find(k => existsSync(k)); - - const sshArgs = [ - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-o", "ConnectTimeout=10", - ...(sshKey !== undefined ? ["-i", sshKey] : []), - `${effectiveUser}@${ip}`, - 'PXE_ENTRY=$(sudo efibootmgr | grep -iE "pxe|network|ipv4" | head -1 | grep -oP "Boot\\K[0-9A-F]+"); if [ -n "$PXE_ENTRY" ]; then sudo efibootmgr --bootnext "$PXE_ENTRY" && echo "PXE set as next boot" && sudo reboot; else echo "No PXE boot entry found, rebooting anyway..." && sudo reboot; fi', - ]; + const base = sshBaseArgs(ip); + if (base !== null) { + console.log(`\nAttempting SSH reboot into PXE (${sshUser()}@${ip})...`); try { - execFileSync("ssh", sshArgs, { stdio: "inherit" }); + execFileSync("ssh", [ + ...base, + 'PXE_ENTRY=$(sudo efibootmgr | grep -iE "pxe|network|ipv4" | head -1 | grep -oP "Boot\\K[0-9A-F]+"); if [ -n "$PXE_ENTRY" ]; then sudo efibootmgr --bootnext "$PXE_ENTRY" && echo "PXE set as next boot" && sudo reboot; else echo "No PXE boot entry found, rebooting anyway..." && sudo reboot; fi', + ], { stdio: "inherit" }); } catch { // SSH connection closing during reboot is expected } diff --git a/bastion/src/cli/src/commands/recheck.ts b/bastion/src/cli/src/commands/recheck.ts index 2bfaba3..5695382 100644 --- a/bastion/src/cli/src/commands/recheck.ts +++ b/bastion/src/cli/src/commands/recheck.ts @@ -4,6 +4,7 @@ import type { Command } from "commander"; import { sshExec } from "@lab/modules"; import { getLabdClient } from "../api/config.js"; +import { ROOT_DEVICE_PROBE } from "../utils/hardware-probe.js"; const BOLD = "\x1b[1m"; const GREEN = "\x1b[0;32m"; @@ -24,7 +25,9 @@ const HW_COLLECT_SCRIPT = [ 'N=$(grep -c "^processor" /proc/cpuinfo 2>/dev/null || echo 0)', 'R=$(awk "/MemTotal/ {printf \\"%d\\", \\$2/1024/1024}" /proc/meminfo 2>/dev/null || echo 0)', 'A=$(uname -m)', - 'printf \'{"product":"%s","board":"%s","serial":"%s","manufacturer":"%s","cpu_model":"%s","cpu_cores":%s,"memory_gb":%s,"arch":"%s"}\\n\' "$P" "$B" "$S" "$M" "$C" "$N" "$R" "$A"', + // Root filesystem, so --pxe-boot has a root= to use instead of assuming our layout. + ROOT_DEVICE_PROBE, + 'printf \'{"product":"%s","board":"%s","serial":"%s","manufacturer":"%s","cpu_model":"%s","cpu_cores":%s,"memory_gb":%s,"arch":"%s","root_device":"%s","root_args":"%s"}\\n\' "$P" "$B" "$S" "$M" "$C" "$N" "$R" "$A" "$RD" "$RA"', ].join("; "); export function registerRecheckCommand(parent: Command): void { @@ -81,7 +84,8 @@ export function registerRecheckCommand(parent: Command): void { const cpu = hwData.cpu_model || "?"; const cores = hwData.cpu_cores || "?"; const mem = hwData.memory_gb || "?"; - console.log(`${GREEN}OK${RESET} ${DIM}${cpu}, ${cores} cores, ${mem}GB${RESET}`); + const root = hwData.root_device ? `, root=${hwData.root_device}` : ""; + console.log(`${GREEN}OK${RESET} ${DIM}${cpu}, ${cores} cores, ${mem}GB${root}${RESET}`); updated++; } catch (err) { console.log(`${RED}FAIL${RESET} ${DIM}${err instanceof Error ? err.message : String(err)}${RESET}`); diff --git a/bastion/src/cli/src/utils/hardware-probe.ts b/bastion/src/cli/src/utils/hardware-probe.ts new file mode 100644 index 0000000..26e6164 --- /dev/null +++ b/bastion/src/cli/src/utils/hardware-probe.ts @@ -0,0 +1,59 @@ +// Shell snippets for observing a machine's hardware over SSH. +// +// Pure shell + awk, no Python: these run on whatever the target happens to be, +// including a minimal rescue environment. + +/** + * Report the root filesystem and any dracut arguments needed to assemble it. + * + * Emits two lines: + * ROOT_DEVICE= + * ROOT_ARGS= + * + * Used by `--pxe-boot`, which boots the installed system with a kernel and initrd from + * the network. Getting root= wrong there leaves the machine unbootable, so this observes + * the machine rather than assuming our Fedora LVM layout. + * + * Device form is chosen for stability across reboots: LVM logical volumes keep their + * /dev/mapper path, anything else is reported by UUID, which survives device renumbering. + */ +export const ROOT_DEVICE_PROBE = [ + 'RD=$(findmnt -no SOURCE / 2>/dev/null | head -1)', + 'RA=""', + 'RT=$(lsblk -no TYPE "$RD" 2>/dev/null | head -1)', + 'if [ "$RT" = "lvm" ]; then', + ' VGLV=$(lvs --noheadings -o vg_name,lv_name "$RD" 2>/dev/null | awk \'{print $1"/"$2}\')', + ' [ -n "$VGLV" ] && RA="rd.lvm.lv=$VGLV"', + // Swap must be assembled too or resume= stalls the boot waiting for it. + ' SW=$(awk \'NR>1 {print $1; exit}\' /proc/swaps 2>/dev/null)', + ' if [ -n "$SW" ] && [ "$(lsblk -no TYPE "$SW" 2>/dev/null | head -1)" = "lvm" ]; then', + ' SWVGLV=$(lvs --noheadings -o vg_name,lv_name "$SW" 2>/dev/null | awk \'{print $1"/"$2}\')', + ' [ -n "$SWVGLV" ] && [ "$SWVGLV" != "$VGLV" ] && RA="$RA rd.lvm.lv=$SWVGLV"', + ' fi', + 'elif [ -n "$RD" ]; then', + ' U=$(findmnt -no UUID / 2>/dev/null | head -1)', + ' [ -n "$U" ] && RD="UUID=$U"', + 'fi', + 'printf \'ROOT_DEVICE=%s\\nROOT_ARGS=%s\\n\' "$RD" "$RA"', +].join("; "); + +export interface RootInfo { + root_device?: string; + root_args?: string; +} + +/** Parse the ROOT_DEVICE/ROOT_ARGS lines emitted by ROOT_DEVICE_PROBE. */ +export function parseRootProbe(stdout: string): RootInfo { + const out: RootInfo = {}; + for (const line of stdout.split("\n")) { + const trimmed = line.trim(); + if (trimmed.startsWith("ROOT_DEVICE=")) { + const v = trimmed.slice("ROOT_DEVICE=".length).trim(); + if (v !== "") out.root_device = v; + } else if (trimmed.startsWith("ROOT_ARGS=")) { + const v = trimmed.slice("ROOT_ARGS=".length).trim(); + if (v !== "") out.root_args = v; + } + } + return out; +} diff --git a/bastion/src/labd/src/routes/bastions.ts b/bastion/src/labd/src/routes/bastions.ts index d77a417..26347d4 100644 --- a/bastion/src/labd/src/routes/bastions.ts +++ b/bastion/src/labd/src/routes/bastions.ts @@ -299,6 +299,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void memory_gb?: number; arch?: string; disks?: Array<{ name: string; size_gb: number; model: string }>; nics?: Array<{ name: string; mac: string; state: string }>; + root_device?: string; root_args?: string; }; }>("/api/machines/discover", async (request, reply) => { const data = request.body ?? {};