feat: debug --sshd flag, auto SSH + nc listener + IP callback

When using `labctl provision debug <target> --sshd`, the rescue
kickstart generates host keys, starts sshd (pw: debug) and nc
listener (port 2323), and reports the IP back to bastion via
/api/progress callback. Fully self-contained, no mounted FS needed.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

bastion/src/bastion/src/main.ts

@@ -269,6 +269,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
     labdConn.onCommand("command-debug", async (msg) => {
       if (msg.type !== "command-debug") throw new Error("unexpected");
       const mac = msg.mac.toLowerCase();
+      const sshd = msg.sshd ?? false;
       const currentState = state.load();
       const hostname =
         currentState.installed[mac]?.hostname ??
@@ -276,7 +277,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
         currentState.discovered[mac]?.product ??
         mac;
       state.update((s) => {
-        s.debug[mac] = { hostname, queued_at: new Date().toISOString() };
+        s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd };
       });
       return { status: "ok", data: { mac, hostname } };
     });

bastion/src/bastion/src/routes/api.ts

@@ -191,9 +191,10 @@ export function registerApiRoutes(
 
   // Queue debug/rescue mode for a machine
   app.post<{
-    Body: { mac?: string };
+    Body: { mac?: string; sshd?: boolean };
   }>("/api/debug", async (request, reply) => {
     const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":");
+    const sshd = request.body?.sshd ?? false;
     if (mac === "") {
       return reply.status(400).send({ error: "mac is required" });
     }
@@ -207,7 +208,7 @@ export function registerApiRoutes(
       mac;
 
     state.update((s) => {
-      s.debug[mac] = { hostname, queued_at: new Date().toISOString() };
+      s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd };
     });
 
     logger.info(`DEBUG QUEUED: ${mac} -> ${hostname}`);

bastion/src/bastion/src/routes/dispatch.ts

@@ -23,8 +23,17 @@ export function registerDispatchRoutes(
   state: StateManager,
 ): void {
   // Serve debug/rescue kickstart (minimal: SSH keys + network)
-  app.get<{ Querystring: { mac?: string } }>("/debug.ks", async (_request, reply) => {
-    const ks = renderDebugKickstart({ sshKeys: config.sshKeys ?? [] });
+  app.get<{ Querystring: { mac?: string; sshd?: string } }>("/debug.ks", async (request, reply) => {
+    const mac = (request.query.mac ?? "").toLowerCase().replace(/-/g, ":");
+    const currentState = state.load();
+    const wantSshd = request.query.sshd === "1" || currentState.debug[mac]?.sshd === true;
+
+    const ks = renderDebugKickstart({
+      sshKeys: config.sshKeys ?? [],
+      sshd: wantSshd,
+      serverIp: config.serverIp,
+      httpPort: config.httpPort,
+    });
     return reply.type("text/plain").send(ks);
   });
 

bastion/src/bastion/src/templates/debug.ks.ts

@@ -1,9 +1,13 @@
 // Debug/rescue kickstart template.
-// Minimal: sets SSH access and network for Anaconda rescue mode.
-// No disk operations, no packages, no %post.
+// Minimal kickstart for Anaconda rescue mode.
+// When sshd=true: generates host keys, starts sshd, reports IP to bastion.
+// No dependency on mounted filesystems — fully self-contained.
 
 export interface DebugKickstartParams {
   sshKeys: string[];
+  sshd?: boolean;
+  serverIp?: string;
+  httpPort?: number;
 }
 
 export function renderDebugKickstart(params: DebugKickstartParams): string {
@@ -12,8 +16,55 @@ export function renderDebugKickstart(params: DebugKickstartParams): string {
     ? `sshkey --username=root "${params.sshKeys[0]}"`
     : "";
 
+  const sshdSetup = params.sshd ? `
+%post --nochroot --log=/tmp/debug-sshd.log
+#!/bin/bash
+set -x
+
+# Generate host keys (self-contained, no mounted FS needed)
+ssh-keygen -t ed25519 -f /tmp/ssh_host_ed25519_key -N "" -q
+ssh-keygen -t rsa -f /tmp/ssh_host_rsa_key -N "" -q
+
+# Write minimal sshd config
+cat > /tmp/sshd_config << 'SSHCFG'
+HostKey /tmp/ssh_host_ed25519_key
+HostKey /tmp/ssh_host_rsa_key
+PermitRootLogin yes
+PasswordAuthentication yes
+PubkeyAuthentication yes
+AuthorizedKeysFile /root/.ssh/authorized_keys
+SSHCFG
+
+# Set root password for SSH access
+echo "root:debug" | chpasswd
+
+# Set up SSH authorized keys
+mkdir -p /root/.ssh && chmod 700 /root/.ssh
+${params.sshKeys.map(k => `echo '${k}' >> /root/.ssh/authorized_keys`).join("\n")}
+chmod 600 /root/.ssh/authorized_keys 2>/dev/null || true
+
+# Start sshd
+/usr/sbin/sshd -f /tmp/sshd_config -p 22
+echo "sshd started on port 22"
+
+# Start persistent nc listener for remote shell
+(while true; do nc -l -p 2323 -e /bin/bash 2>/dev/null; done) &
+echo "nc shell listener on port 2323"
+
+# Report IP to bastion
+sleep 2
+IP_ADDR=$(ip -4 addr show | awk '/inet / && !/127.0.0/ {split($2,a,"/"); print a[1]; exit}')
+MAC_ADDR=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')
+curl -sf -X POST "http://${params.serverIp}:${params.httpPort}/api/progress" \\
+    -H "Content-Type: application/json" \\
+    -d "{\\"mac\\":\\"$MAC_ADDR\\",\\"stage\\":\\"debug-ready\\",\\"detail\\":\\"ssh root@$IP_ADDR (pw: debug)  |  nc $IP_ADDR 2323\\"}" 2>/dev/null || true
+
+echo "Debug environment ready: ssh root@$IP_ADDR or nc $IP_ADDR 2323"
+%end
+` : "";
+
   return `# Lab Bastion -- Debug/Rescue Kickstart
-# Minimal: only SSH + network for Anaconda rescue mode
+# Minimal: SSH + network for Anaconda rescue mode
 
 lang en_US.UTF-8
 keyboard uk
@@ -21,5 +72,5 @@ network --bootproto=dhcp --activate
 
 ${sshpw}
 ${sshkeyLine}
-`;
+${sshdSetup}`;
 }

bastion/src/cli/src/api/client.ts

@@ -94,8 +94,8 @@ export class LabdClient {
     return this.request("POST", "/api/machines/install", { body: opts });
   }
 
-  async debugMachine(mac: string): Promise<{ status: string; data?: { mac: string; hostname: string }; error?: string }> {
-    return this.request("POST", "/api/machines/debug", { body: { mac } });
+  async debugMachine(mac: string, opts?: { sshd?: boolean }): Promise<{ status: string; data?: { mac: string; hostname: string }; error?: string }> {
+    return this.request("POST", "/api/machines/debug", { body: { mac, sshd: opts?.sshd } });
   }
 
   async forgetMachine(mac: string): Promise<{ status: string }> {

bastion/src/cli/src/commands/debug.ts

@@ -48,8 +48,9 @@ export function registerDebugCommand(parent: Command): void {
   parent
     .command("debug <target>")
     .description("PXE boot into Fedora rescue mode for debugging (target: hostname, MAC, or IP)")
+    .option("--sshd", "Start SSH + nc listener automatically, report IP to bastion")
     .showHelpAfterError(true)
-    .action(async (target: string) => {
+    .action(async (target: string, opts: { sshd?: boolean }) => {
       const client = getLabdClient();
 
       // Resolve target from labd aggregated state
@@ -73,7 +74,7 @@ export function registerDebugCommand(parent: Command): void {
       console.log(`Queuing debug mode for ${hostname} (${mac})...`);
 
       try {
-        const result = await client.debugMachine(mac);
+        const result = await client.debugMachine(mac, { sshd: opts.sshd });
         if (result.error) {
           console.error(`Failed: ${result.error}`);
           process.exit(1);

bastion/src/labd/src/routes/bastions.ts

@@ -174,9 +174,10 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
 
   // Queue debug/rescue mode — route to correct bastion by MAC
   app.post<{
-    Body: { mac?: string };
+    Body: { mac?: string; sshd?: boolean };
   }>("/api/machines/debug", async (request, reply) => {
     const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":");
+    const sshd = request.body?.sshd ?? false;
     if (!mac) {
       return reply.code(400).send({ error: "mac is required" });
     }
@@ -189,7 +190,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
       }
       if (all.length === 1) {
         try {
-          const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac });
+          const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac, sshd });
           return reply.code(result.status === "ok" ? 200 : 500).send(result);
         } catch (err) {
           return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
@@ -199,7 +200,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
     }
 
     try {
-      const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac });
+      const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac, sshd });
       return reply.code(result.status === "ok" ? 200 : 500).send(result);
     } catch (err) {
       return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });

bastion/src/shared/src/protocol/index.ts

@@ -111,7 +111,7 @@ export type LabdBastionMessage =
   | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string }
   | { type: "command-forget"; requestId: string; mac: string }
   | { type: "command-role-update"; requestId: string; mac: string; role: string }
-  | { type: "command-debug"; requestId: string; mac: string }
+  | { type: "command-debug"; requestId: string; mac: string; sshd?: boolean }
   | { type: "server-shutdown"; reconnectAfter: number };
 
 export type BastionMessageType = BastionMessage["type"];

bastion/src/shared/src/types/state.ts

@@ -101,6 +101,7 @@ export interface InstalledInfo {
 export interface DebugConfig {
   hostname: string;
   queued_at: string;
+  sshd?: boolean;
 }
 
 export interface BastionState {