diff --git a/bastion/src/bastion/src/main.ts b/bastion/src/bastion/src/main.ts index fe3a3ac..289551e 100644 --- a/bastion/src/bastion/src/main.ts +++ b/bastion/src/bastion/src/main.ts @@ -270,6 +270,7 @@ export async function startBastion(overrides: Partial = {}): Prom if (msg.type !== "command-debug") throw new Error("unexpected"); const mac = msg.mac.toLowerCase(); const sshd = msg.sshd ?? false; + const pxeBoot = msg.pxeBoot ?? false; const currentState = state.load(); const hostname = currentState.installed[mac]?.hostname ?? @@ -277,7 +278,7 @@ export async function startBastion(overrides: Partial = {}): Prom currentState.discovered[mac]?.product ?? mac; state.update((s) => { - s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd }; + s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd, pxeBoot }; }); return { status: "ok", data: { mac, hostname } }; }); diff --git a/bastion/src/bastion/src/routes/api.ts b/bastion/src/bastion/src/routes/api.ts index 5718357..5b9fe83 100644 --- a/bastion/src/bastion/src/routes/api.ts +++ b/bastion/src/bastion/src/routes/api.ts @@ -191,10 +191,11 @@ export function registerApiRoutes( // Queue debug/rescue mode for a machine app.post<{ - Body: { mac?: string; sshd?: boolean }; + Body: { mac?: string; sshd?: boolean; pxeBoot?: boolean }; }>("/api/debug", async (request, reply) => { const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":"); const sshd = request.body?.sshd ?? false; + const pxeBoot = request.body?.pxeBoot ?? false; if (mac === "") { return reply.status(400).send({ error: "mac is required" }); } @@ -208,7 +209,7 @@ export function registerApiRoutes( mac; state.update((s) => { - s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd }; + s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd, pxeBoot }; }); logger.info(`DEBUG QUEUED: ${mac} -> ${hostname}`); diff --git a/bastion/src/bastion/src/routes/dispatch.ts b/bastion/src/bastion/src/routes/dispatch.ts index 5361d0f..1954d4d 100644 --- a/bastion/src/bastion/src/routes/dispatch.ts +++ b/bastion/src/bastion/src/routes/dispatch.ts @@ -11,6 +11,7 @@ import { renderDiscoverIpxe, renderInstallIpxe, renderDebugIpxe, + renderPxeBootDebugIpxe, renderLocalBootIpxe, } from "../templates/boot.ipxe.js"; import { renderUbuntuInstallIpxe } from "../templates/ubuntu-boot.ipxe.js"; @@ -45,17 +46,27 @@ export function registerDispatchRoutes( const debugEntry = currentState.debug[mac]; if (debugEntry) { const hostname = debugEntry.hostname ?? "debug"; - logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (rescue mode)`); - state.update((s) => { delete s.debug[mac]; }); - const script = renderDebugIpxe({ - mac, - hostname, - serverIp: config.serverIp, - httpPort: config.httpPort, - fedoraMirror: config.fedoraMirror, - }); + let script: string; + if (debugEntry.pxeBoot) { + logger.info(`PXE BOOT DEBUG: ${mac} -> ${hostname} (kernel+initrd from PXE, root from NVMe)`); + script = renderPxeBootDebugIpxe({ + mac, + hostname, + serverIp: config.serverIp, + httpPort: config.httpPort, + }); + } else { + logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (rescue mode)`); + script = renderDebugIpxe({ + mac, + hostname, + serverIp: config.serverIp, + httpPort: config.httpPort, + fedoraMirror: config.fedoraMirror, + }); + } return reply.type("text/plain").send(script); } diff --git a/bastion/src/bastion/src/templates/boot.ipxe.ts b/bastion/src/bastion/src/templates/boot.ipxe.ts index 826633f..f56e815 100644 --- a/bastion/src/bastion/src/templates/boot.ipxe.ts +++ b/bastion/src/bastion/src/templates/boot.ipxe.ts @@ -102,6 +102,34 @@ boot `; } +/** + * iPXE script for PXE-boot debug mode -- boots the installed system's root + * filesystem using the bastion's PXE kernel+initrd instead of local GRUB. + * Workaround for UEFI firmware bugs that make local disk boot slow. + */ +export function renderPxeBootDebugIpxe(params: { + mac: string; + hostname: string; + serverIp: string; + httpPort: number; +}): string { + return `#!ipxe + +echo +echo ============================================= +echo Lab PXE Bastion - PXE BOOT (debug) +echo Target: ${params.hostname} +echo MAC: ${params.mac} +echo Kernel+initrd from PXE, root from NVMe +echo ============================================= +echo + +kernel http://${params.serverIp}:${params.httpPort}/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0 console=ttyS0,115200n8 modprobe.blacklist=amdgpu +initrd http://${params.serverIp}:${params.httpPort}/initrd.img +boot +`; +} + /** * iPXE script for already-installed machines -- exits to boot from local disk. */ diff --git a/bastion/src/cli/src/api/client.ts b/bastion/src/cli/src/api/client.ts index c7bfaa0..75380a4 100644 --- a/bastion/src/cli/src/api/client.ts +++ b/bastion/src/cli/src/api/client.ts @@ -94,8 +94,8 @@ export class LabdClient { return this.request("POST", "/api/machines/install", { body: opts }); } - async debugMachine(mac: string, opts?: { sshd?: boolean }): Promise<{ status: string; data?: { mac: string; hostname: string }; error?: string }> { - return this.request("POST", "/api/machines/debug", { body: { mac, sshd: opts?.sshd } }); + async debugMachine(mac: string, opts?: { sshd?: boolean; pxeBoot?: boolean }): Promise<{ status: string; data?: { mac: string; hostname: string }; error?: string }> { + return this.request("POST", "/api/machines/debug", { body: { mac, sshd: opts?.sshd, pxeBoot: opts?.pxeBoot } }); } async forgetMachine(mac: string): Promise<{ status: string }> { diff --git a/bastion/src/cli/src/commands/debug.ts b/bastion/src/cli/src/commands/debug.ts index aa3ccd7..45a13a7 100644 --- a/bastion/src/cli/src/commands/debug.ts +++ b/bastion/src/cli/src/commands/debug.ts @@ -49,8 +49,9 @@ export function registerDebugCommand(parent: Command): void { .command("debug ") .description("PXE boot into Fedora rescue mode for debugging (target: hostname, MAC, or IP)") .option("--sshd", "Start SSH + nc listener automatically, report IP to bastion") + .option("--pxe-boot", "Boot installed system via PXE (kernel+initrd from network, root from NVMe)") .showHelpAfterError(true) - .action(async (target: string, opts: { sshd?: boolean }) => { + .action(async (target: string, opts: { sshd?: boolean; pxeBoot?: boolean }) => { const client = getLabdClient(); // Resolve target from labd aggregated state @@ -74,7 +75,7 @@ export function registerDebugCommand(parent: Command): void { console.log(`Queuing debug mode for ${hostname} (${mac})...`); try { - const result = await client.debugMachine(mac, { sshd: opts.sshd === true }); + const result = await client.debugMachine(mac, { sshd: opts.sshd === true, pxeBoot: opts.pxeBoot === true }); if (result.error) { console.error(`Failed: ${result.error}`); process.exit(1); diff --git a/bastion/src/labd/src/routes/bastions.ts b/bastion/src/labd/src/routes/bastions.ts index ea694cc..218805e 100644 --- a/bastion/src/labd/src/routes/bastions.ts +++ b/bastion/src/labd/src/routes/bastions.ts @@ -174,10 +174,11 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void // Queue debug/rescue mode — route to correct bastion by MAC app.post<{ - Body: { mac?: string; sshd?: boolean }; + Body: { mac?: string; sshd?: boolean; pxeBoot?: boolean }; }>("/api/machines/debug", async (request, reply) => { const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":"); const sshd = request.body?.sshd ?? false; + const pxeBoot = request.body?.pxeBoot ?? false; if (!mac) { return reply.code(400).send({ error: "mac is required" }); } @@ -190,7 +191,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void } if (all.length === 1) { try { - const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac, sshd }); + const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac, sshd, pxeBoot }); return reply.code(result.status === "ok" ? 200 : 500).send(result); } catch (err) { return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) }); @@ -200,7 +201,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void } try { - const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac, sshd }); + const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac, sshd, pxeBoot }); return reply.code(result.status === "ok" ? 200 : 500).send(result); } catch (err) { return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) }); diff --git a/bastion/src/shared/src/protocol/index.ts b/bastion/src/shared/src/protocol/index.ts index 88dffbd..3b5054e 100644 --- a/bastion/src/shared/src/protocol/index.ts +++ b/bastion/src/shared/src/protocol/index.ts @@ -111,7 +111,7 @@ export type LabdBastionMessage = | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string } | { type: "command-forget"; requestId: string; mac: string } | { type: "command-role-update"; requestId: string; mac: string; role: string } - | { type: "command-debug"; requestId: string; mac: string; sshd?: boolean } + | { type: "command-debug"; requestId: string; mac: string; sshd?: boolean; pxeBoot?: boolean } | { type: "server-shutdown"; reconnectAfter: number }; export type BastionMessageType = BastionMessage["type"]; diff --git a/bastion/src/shared/src/types/state.ts b/bastion/src/shared/src/types/state.ts index a569cfa..eaadf1f 100644 --- a/bastion/src/shared/src/types/state.ts +++ b/bastion/src/shared/src/types/state.ts @@ -102,6 +102,7 @@ export interface DebugConfig { hostname: string; queued_at: string; sshd?: boolean; + pxeBoot?: boolean; } export interface BastionState {