feat: PXE debug boot mode for rescue/diagnostics #4
@@ -270,6 +270,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
|
|||||||
if (msg.type !== "command-debug") throw new Error("unexpected");
|
if (msg.type !== "command-debug") throw new Error("unexpected");
|
||||||
const mac = msg.mac.toLowerCase();
|
const mac = msg.mac.toLowerCase();
|
||||||
const sshd = msg.sshd ?? false;
|
const sshd = msg.sshd ?? false;
|
||||||
|
const pxeBoot = msg.pxeBoot ?? false;
|
||||||
const currentState = state.load();
|
const currentState = state.load();
|
||||||
const hostname =
|
const hostname =
|
||||||
currentState.installed[mac]?.hostname ??
|
currentState.installed[mac]?.hostname ??
|
||||||
@@ -277,7 +278,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
|
|||||||
currentState.discovered[mac]?.product ??
|
currentState.discovered[mac]?.product ??
|
||||||
mac;
|
mac;
|
||||||
state.update((s) => {
|
state.update((s) => {
|
||||||
s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd };
|
s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd, pxeBoot };
|
||||||
});
|
});
|
||||||
return { status: "ok", data: { mac, hostname } };
|
return { status: "ok", data: { mac, hostname } };
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -191,10 +191,11 @@ export function registerApiRoutes(
|
|||||||
|
|
||||||
// Queue debug/rescue mode for a machine
|
// Queue debug/rescue mode for a machine
|
||||||
app.post<{
|
app.post<{
|
||||||
Body: { mac?: string; sshd?: boolean };
|
Body: { mac?: string; sshd?: boolean; pxeBoot?: boolean };
|
||||||
}>("/api/debug", async (request, reply) => {
|
}>("/api/debug", async (request, reply) => {
|
||||||
const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":");
|
const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":");
|
||||||
const sshd = request.body?.sshd ?? false;
|
const sshd = request.body?.sshd ?? false;
|
||||||
|
const pxeBoot = request.body?.pxeBoot ?? false;
|
||||||
if (mac === "") {
|
if (mac === "") {
|
||||||
return reply.status(400).send({ error: "mac is required" });
|
return reply.status(400).send({ error: "mac is required" });
|
||||||
}
|
}
|
||||||
@@ -208,7 +209,7 @@ export function registerApiRoutes(
|
|||||||
mac;
|
mac;
|
||||||
|
|
||||||
state.update((s) => {
|
state.update((s) => {
|
||||||
s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd };
|
s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd, pxeBoot };
|
||||||
});
|
});
|
||||||
|
|
||||||
logger.info(`DEBUG QUEUED: ${mac} -> ${hostname}`);
|
logger.info(`DEBUG QUEUED: ${mac} -> ${hostname}`);
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import {
|
|||||||
renderDiscoverIpxe,
|
renderDiscoverIpxe,
|
||||||
renderInstallIpxe,
|
renderInstallIpxe,
|
||||||
renderDebugIpxe,
|
renderDebugIpxe,
|
||||||
|
renderPxeBootDebugIpxe,
|
||||||
renderLocalBootIpxe,
|
renderLocalBootIpxe,
|
||||||
} from "../templates/boot.ipxe.js";
|
} from "../templates/boot.ipxe.js";
|
||||||
import { renderUbuntuInstallIpxe } from "../templates/ubuntu-boot.ipxe.js";
|
import { renderUbuntuInstallIpxe } from "../templates/ubuntu-boot.ipxe.js";
|
||||||
@@ -45,17 +46,27 @@ export function registerDispatchRoutes(
|
|||||||
const debugEntry = currentState.debug[mac];
|
const debugEntry = currentState.debug[mac];
|
||||||
if (debugEntry) {
|
if (debugEntry) {
|
||||||
const hostname = debugEntry.hostname ?? "debug";
|
const hostname = debugEntry.hostname ?? "debug";
|
||||||
logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (rescue mode)`);
|
|
||||||
|
|
||||||
state.update((s) => { delete s.debug[mac]; });
|
state.update((s) => { delete s.debug[mac]; });
|
||||||
|
|
||||||
const script = renderDebugIpxe({
|
let script: string;
|
||||||
|
if (debugEntry.pxeBoot) {
|
||||||
|
logger.info(`PXE BOOT DEBUG: ${mac} -> ${hostname} (kernel+initrd from PXE, root from NVMe)`);
|
||||||
|
script = renderPxeBootDebugIpxe({
|
||||||
|
mac,
|
||||||
|
hostname,
|
||||||
|
serverIp: config.serverIp,
|
||||||
|
httpPort: config.httpPort,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (rescue mode)`);
|
||||||
|
script = renderDebugIpxe({
|
||||||
mac,
|
mac,
|
||||||
hostname,
|
hostname,
|
||||||
serverIp: config.serverIp,
|
serverIp: config.serverIp,
|
||||||
httpPort: config.httpPort,
|
httpPort: config.httpPort,
|
||||||
fedoraMirror: config.fedoraMirror,
|
fedoraMirror: config.fedoraMirror,
|
||||||
});
|
});
|
||||||
|
}
|
||||||
return reply.type("text/plain").send(script);
|
return reply.type("text/plain").send(script);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -102,6 +102,34 @@ boot
|
|||||||
`;
|
`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* iPXE script for PXE-boot debug mode -- boots the installed system's root
|
||||||
|
* filesystem using the bastion's PXE kernel+initrd instead of local GRUB.
|
||||||
|
* Workaround for UEFI firmware bugs that make local disk boot slow.
|
||||||
|
*/
|
||||||
|
export function renderPxeBootDebugIpxe(params: {
|
||||||
|
mac: string;
|
||||||
|
hostname: string;
|
||||||
|
serverIp: string;
|
||||||
|
httpPort: number;
|
||||||
|
}): string {
|
||||||
|
return `#!ipxe
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo =============================================
|
||||||
|
echo Lab PXE Bastion - PXE BOOT (debug)
|
||||||
|
echo Target: ${params.hostname}
|
||||||
|
echo MAC: ${params.mac}
|
||||||
|
echo Kernel+initrd from PXE, root from NVMe
|
||||||
|
echo =============================================
|
||||||
|
echo
|
||||||
|
|
||||||
|
kernel http://${params.serverIp}:${params.httpPort}/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0 console=ttyS0,115200n8 modprobe.blacklist=amdgpu
|
||||||
|
initrd http://${params.serverIp}:${params.httpPort}/initrd.img
|
||||||
|
boot
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* iPXE script for already-installed machines -- exits to boot from local disk.
|
* iPXE script for already-installed machines -- exits to boot from local disk.
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -94,8 +94,8 @@ export class LabdClient {
|
|||||||
return this.request("POST", "/api/machines/install", { body: opts });
|
return this.request("POST", "/api/machines/install", { body: opts });
|
||||||
}
|
}
|
||||||
|
|
||||||
async debugMachine(mac: string, opts?: { sshd?: boolean }): Promise<{ status: string; data?: { mac: string; hostname: string }; error?: string }> {
|
async debugMachine(mac: string, opts?: { sshd?: boolean; pxeBoot?: boolean }): Promise<{ status: string; data?: { mac: string; hostname: string }; error?: string }> {
|
||||||
return this.request("POST", "/api/machines/debug", { body: { mac, sshd: opts?.sshd } });
|
return this.request("POST", "/api/machines/debug", { body: { mac, sshd: opts?.sshd, pxeBoot: opts?.pxeBoot } });
|
||||||
}
|
}
|
||||||
|
|
||||||
async forgetMachine(mac: string): Promise<{ status: string }> {
|
async forgetMachine(mac: string): Promise<{ status: string }> {
|
||||||
|
|||||||
@@ -49,8 +49,9 @@ export function registerDebugCommand(parent: Command): void {
|
|||||||
.command("debug <target>")
|
.command("debug <target>")
|
||||||
.description("PXE boot into Fedora rescue mode for debugging (target: hostname, MAC, or IP)")
|
.description("PXE boot into Fedora rescue mode for debugging (target: hostname, MAC, or IP)")
|
||||||
.option("--sshd", "Start SSH + nc listener automatically, report IP to bastion")
|
.option("--sshd", "Start SSH + nc listener automatically, report IP to bastion")
|
||||||
|
.option("--pxe-boot", "Boot installed system via PXE (kernel+initrd from network, root from NVMe)")
|
||||||
.showHelpAfterError(true)
|
.showHelpAfterError(true)
|
||||||
.action(async (target: string, opts: { sshd?: boolean }) => {
|
.action(async (target: string, opts: { sshd?: boolean; pxeBoot?: boolean }) => {
|
||||||
const client = getLabdClient();
|
const client = getLabdClient();
|
||||||
|
|
||||||
// Resolve target from labd aggregated state
|
// Resolve target from labd aggregated state
|
||||||
@@ -74,7 +75,7 @@ export function registerDebugCommand(parent: Command): void {
|
|||||||
console.log(`Queuing debug mode for ${hostname} (${mac})...`);
|
console.log(`Queuing debug mode for ${hostname} (${mac})...`);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await client.debugMachine(mac, { sshd: opts.sshd === true });
|
const result = await client.debugMachine(mac, { sshd: opts.sshd === true, pxeBoot: opts.pxeBoot === true });
|
||||||
if (result.error) {
|
if (result.error) {
|
||||||
console.error(`Failed: ${result.error}`);
|
console.error(`Failed: ${result.error}`);
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
|
|||||||
@@ -174,10 +174,11 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
|||||||
|
|
||||||
// Queue debug/rescue mode — route to correct bastion by MAC
|
// Queue debug/rescue mode — route to correct bastion by MAC
|
||||||
app.post<{
|
app.post<{
|
||||||
Body: { mac?: string; sshd?: boolean };
|
Body: { mac?: string; sshd?: boolean; pxeBoot?: boolean };
|
||||||
}>("/api/machines/debug", async (request, reply) => {
|
}>("/api/machines/debug", async (request, reply) => {
|
||||||
const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":");
|
const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":");
|
||||||
const sshd = request.body?.sshd ?? false;
|
const sshd = request.body?.sshd ?? false;
|
||||||
|
const pxeBoot = request.body?.pxeBoot ?? false;
|
||||||
if (!mac) {
|
if (!mac) {
|
||||||
return reply.code(400).send({ error: "mac is required" });
|
return reply.code(400).send({ error: "mac is required" });
|
||||||
}
|
}
|
||||||
@@ -190,7 +191,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
|||||||
}
|
}
|
||||||
if (all.length === 1) {
|
if (all.length === 1) {
|
||||||
try {
|
try {
|
||||||
const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac, sshd });
|
const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac, sshd, pxeBoot });
|
||||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
|
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
|
||||||
@@ -200,7 +201,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac, sshd });
|
const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac, sshd, pxeBoot });
|
||||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
|
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
|
||||||
|
|||||||
@@ -111,7 +111,7 @@ export type LabdBastionMessage =
|
|||||||
| { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string }
|
| { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string }
|
||||||
| { type: "command-forget"; requestId: string; mac: string }
|
| { type: "command-forget"; requestId: string; mac: string }
|
||||||
| { type: "command-role-update"; requestId: string; mac: string; role: string }
|
| { type: "command-role-update"; requestId: string; mac: string; role: string }
|
||||||
| { type: "command-debug"; requestId: string; mac: string; sshd?: boolean }
|
| { type: "command-debug"; requestId: string; mac: string; sshd?: boolean; pxeBoot?: boolean }
|
||||||
| { type: "server-shutdown"; reconnectAfter: number };
|
| { type: "server-shutdown"; reconnectAfter: number };
|
||||||
|
|
||||||
export type BastionMessageType = BastionMessage["type"];
|
export type BastionMessageType = BastionMessage["type"];
|
||||||
|
|||||||
@@ -102,6 +102,7 @@ export interface DebugConfig {
|
|||||||
hostname: string;
|
hostname: string;
|
||||||
queued_at: string;
|
queued_at: string;
|
||||||
sshd?: boolean;
|
sshd?: boolean;
|
||||||
|
pxeBoot?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface BastionState {
|
export interface BastionState {
|
||||||
|
|||||||
Reference in New Issue
Block a user