feat(bastion): serve per-architecture PXE kernels, resolved not flagged

The bastion served one x86_64 kernel to every machine regardless of architecture, so an ARM64 box was handed a binary its UEFI will not execute. This is why `labctl provision debug` could not rescue the DGX Sparks during the 2026-08-11 kernel panic. Architecture is resolved, never typed by an operator: the tracked machine record first, then iPXE's ${buildarch} reported on the dispatch URL, then the configured default. boot.ipxe gains &arch=${buildarch} so that signal reaches the HTTP endpoint -- DHCP option 93 only ever reaches dnsmasq. One script covers network PXE, UEFI HTTP boot and the boot ISO alike. x86_64 keeps its unsuffixed /vmlinuz and /initrd.img so its rendered scripts are byte-identical; aarch64 gets suffixed paths, its own Fedora mirror, and console=ttyAMA0 instead of nomodeset, which does not mean the same thing on arm64 and can leave a headless machine with no console. --pxe-boot no longer hardcodes the Fedora LVM layout: root device and dracut args come from the machine's record, and dispatch falls back to rescue rather than guessing a root= that would leave a machine unbootable. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015nRFZXpKwUVE4SRSHw6GjF
2026-08-11 12:48:09 +01:00
parent cba56becfc
commit 9c79915975
8 changed files with 461 additions and 40 deletions
--- a/bastion/src/bastion/src/main.ts
+++ b/bastion/src/bastion/src/main.ts
@@ -3,7 +3,9 @@

 import { mkdirSync, writeFileSync, readFileSync, existsSync, copyFileSync, symlinkSync, unlinkSync } from "node:fs";
 import { execSync } from "node:child_process";
-import type { BastionConfig } from "@lab/shared";
+import type { Arch, BastionConfig } from "@lab/shared";
+import { SUPPORTED_ARCHES, fedoraMirrorFor, classifyOnboard } from "@lab/shared";
+import { kernelPath, initrdPath } from "./templates/boot.ipxe.js";
 import { loadConfig } from "./config.js";
 import { populateNetworkConfig } from "./services/network.js";
 import { createApp } from "./server.js";
@@ -13,6 +15,7 @@ import { renderBootIpxe } from "./templates/boot.ipxe.js";
 import { logger } from "./services/logger.js";
 import { BastionConnection } from "./services/labd-connection.js";
 import { progressBus } from "./services/progress-events.js";
+import { checkInstallAllowed } from "./services/install-guard.js";
 import { ensureBootIso } from "./routes/boot-iso.js";

 function copyIfMissing(src: string, dest: string, label: string): void {
@@ -130,9 +133,14 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
  mkdirSync(config.tftpDir, { recursive: true });
  mkdirSync(config.httpDir, { recursive: true });

+  // Architectures we can actually network boot, reported in the banner so a missing
+  // arm64 payload is visible at startup instead of at 2am when a rescue is needed.
+  const bootArches: Arch[] = [];
+  let ipxeArm64Ready = false;
+
  // Prepare boot artifacts
  if (config.skipArtifacts !== true) {
-    logger.info(`Preparing boot artifacts (Fedora ${config.fedoraVersion} ${config.arch})...`);
+    logger.info(`Preparing boot artifacts (Fedora ${config.fedoraVersion}, ${SUPPORTED_ARCHES.join(" + ")})...`);

    copyIfMissing(
      "/usr/share/ipxe/undionly.kpxe",
@@ -150,20 +158,41 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
        `${config.tftpDir}/ipxe-arm64.efi`,
        "iPXE UEFI arm64",
      );
+      ipxeArm64Ready = true;
    } catch {
-      logger.warn("arm64 iPXE not available -- skipping");
+      logger.warn("arm64 iPXE not available -- arm64 machines cannot network boot.");
+      logger.warn("  Install with: sudo dnf install ipxe-bootimgs-aarch64");
    }

+    // Fedora pxeboot kernel + initrd per architecture. x86_64 keeps the unsuffixed
+    // names it has always used; other architectures are suffixed. The iPXE templates
+    // resolve the same paths via kernelPath()/initrdPath().
+    for (const arch of SUPPORTED_ARCHES) {
+      const mirror = fedoraMirrorFor(config.fedoraVersion, arch);
+      try {
        download(
-      `${config.fedoraMirror}/images/pxeboot/vmlinuz`,
-      `${config.httpDir}/vmlinuz`,
-      "Fedora kernel",
+          `${mirror}/images/pxeboot/vmlinuz`,
+          `${config.httpDir}${kernelPath(arch)}`,
+          `Fedora ${arch} kernel`,
        );
        download(
-      `${config.fedoraMirror}/images/pxeboot/initrd.img`,
-      `${config.httpDir}/initrd.img`,
-      "Fedora initrd",
+          `${mirror}/images/pxeboot/initrd.img`,
+          `${config.httpDir}${initrdPath(arch)}`,
+          `Fedora ${arch} initrd`,
        );
+        bootArches.push(arch);
+      } catch (err) {
+        // Non-fatal: a bastion with no arm64 artifacts still serves x86_64 fine.
+        // Failing startup over an unreachable mirror for an architecture that may not
+        // even be present on this network would be worse.
+        logger.warn(`Fedora ${arch} kernel/initrd unavailable -- ${arch} PXE disabled`);
+        logger.warn(`  ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+
+    if (!bootArches.includes("x86_64")) {
+      throw new Error("Fedora x86_64 kernel/initrd could not be staged -- cannot serve PXE");
+    }

    // Ubuntu netboot artifacts (non-fatal — Ubuntu version may not be released yet)
    try {
@@ -254,6 +283,13 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
    // Wire up command handlers so labd can send install/forget/role commands
    labdConn.onCommand("command-install", async (msg) => {
      if (msg.type !== "command-install") throw new Error("unexpected");
+      const installMac = msg.mac.toLowerCase().replace(/-/g, ":");
+      const osId = (msg.os as import("@lab/shared").OsId | undefined) ?? "fedora-43";
+      const check = checkInstallAllowed(state.load(), installMac, osId);
+      if (check.allowed === false) {
+        logger.warn(`INSTALL REFUSED: ${installMac} -- ${check.error}`);
+        return { status: "error", error: check.error };
+      }
      state.update((s) => {
        s.install_queue[msg.mac] = {
          hostname: msg.hostname,
@@ -314,13 +350,24 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
      const mac = (msg.mac as string).toLowerCase();
      const now = new Date().toISOString();
      const existing = state.load().discovered[mac];
+      const identity = {
+        mac,
+        manufacturer: (msg.manufacturer as string) ?? "unknown",
+        product: (msg.product as string) ?? "unknown",
+        board: (msg.board as string) ?? "unknown",
+        ...(existing?.onboard !== undefined ? { onboard: existing.onboard } : {}),
+        ...(existing?.vendor_os !== undefined ? { vendor_os: existing.vendor_os } : {}),
+      };
+      const onboarding = classifyOnboard(identity);
+      const rootDevice = msg.root_device ?? existing?.root_device;
+      const rootArgs = msg.root_args ?? existing?.root_args;
      state.update((s) => {
        s.discovered[mac] = {
          mac,
-          product: (msg.product as string) ?? "unknown",
-          board: (msg.board as string) ?? "unknown",
+          product: identity.product,
+          board: identity.board,
          serial: (msg.serial as string) ?? "unknown",
-          manufacturer: (msg.manufacturer as string) ?? "unknown",
+          manufacturer: identity.manufacturer,
          cpu_model: (msg.cpu_model as string) ?? "unknown",
          cpu_cores: (msg.cpu_cores as number) ?? 0,
          memory_gb: (msg.memory_gb as number) ?? 0,
@@ -329,7 +376,20 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
          nics: (msg.nics as Array<{ name: string; mac: string; state: string }>) ?? [],
          first_seen: existing?.first_seen ?? now,
          last_seen: now,
+          onboard: onboarding.onboard,
+          ...(onboarding.vendor_os !== undefined ? { vendor_os: onboarding.vendor_os } : {}),
+          ...(rootDevice !== undefined ? { root_device: rootDevice } : {}),
+          ...(rootArgs !== undefined ? { root_args: rootArgs } : {}),
        };
+        // Keep the installed record in step -- the guard and --pxe-boot both read it.
+        const inst = s.installed[mac];
+        if (inst) {
+          inst.arch = (msg.arch as string) ?? inst.arch;
+          inst.onboard = onboarding.onboard;
+          if (onboarding.vendor_os !== undefined) inst.vendor_os = onboarding.vendor_os;
+          if (rootDevice !== undefined) inst.root_device = rootDevice;
+          if (rootArgs !== undefined) inst.root_args = rootArgs;
+        }
      });
      logger.info(`HARDWARE UPDATED: ${mac} -- ${msg.manufacturer ?? "?"} ${msg.product ?? "?"} (${msg.cpu_model ?? "?"}, ${msg.cpu_cores ?? "?"} cores, ${msg.memory_gb ?? "?"}GB RAM)`);
      return { status: "ok", data: { mac } };
@@ -364,7 +424,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
  }

  // Print banner
-  printBanner(config);
+  printBanner(config, bootArches, ipxeArm64Ready);

  // Graceful shutdown
  const shutdown = async (): Promise<void> => {
@@ -386,11 +446,22 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
  await new Promise(() => {});
 }

-function printBanner(config: BastionConfig): void {
+function printBanner(config: BastionConfig, bootArches: Arch[], ipxeArm64Ready: boolean): void {
  const dhcpInfo = config.dhcpMode === "full"
    ? `full (${config.dhcpRangeStart}-${config.dhcpRangeEnd})`
    : "proxy (alongside existing DHCP)";

+  // arm64 needs both an iPXE binary (DHCP hands it out on option 93 = 0x0b) and a
+  // kernel/initrd pair. Report the combination, since either missing breaks it.
+  const archInfo = config.skipArtifacts === true
+    ? "(artifacts skipped)"
+    : SUPPORTED_ARCHES
+        .map((a) => {
+          const ready = bootArches.includes(a) && (a !== "aarch64" || ipxeArm64Ready);
+          return ready ? a : `${a} (unavailable)`;
+        })
+        .join(", ");
+
  console.log("");
  console.log("\x1b[36m\x1b[1m" + "=".repeat(60) + "\x1b[0m");
  console.log("\x1b[36m\x1b[1m  Lab PXE Bastion -- Discovery Mode\x1b[0m");
@@ -399,7 +470,8 @@ function printBanner(config: BastionConfig): void {
  console.log(`  Network:   \x1b[1m${config.network}/24\x1b[0m via \x1b[1m${config.iface}\x1b[0m`);
  console.log(`  DHCP:      \x1b[1m${dhcpInfo}\x1b[0m`);
  console.log(`  HTTP:      \x1b[1mhttp://${config.serverIp}:${config.httpPort}/\x1b[0m`);
-  console.log(`  OS:        \x1b[1mFedora ${config.fedoraVersion} (${config.arch})\x1b[0m`);
+  console.log(`  OS:        \x1b[1mFedora ${config.fedoraVersion}\x1b[0m`);
+  console.log(`  Net boot:  \x1b[1m${archInfo}\x1b[0m`);
  console.log(`  Domain:    \x1b[1m${config.domain}\x1b[0m`);
  console.log(`  State:     \x1b[1m${config.stateFile}\x1b[0m`);
  console.log("");
--- a/bastion/src/bastion/src/routes/dispatch.ts
+++ b/bastion/src/bastion/src/routes/dispatch.ts
@@ -5,7 +5,8 @@
 //   - unknown       -> discovery mode (collect hardware, POST to bastion)

 import type { FastifyInstance } from "fastify";
-import type { BastionConfig } from "@lab/shared";
+import type { Arch, BastionConfig, BastionState } from "@lab/shared";
+import { normalizeArch, fedoraMirrorFor } from "@lab/shared";
 import type { StateManager } from "../services/state.js";
 import {
  renderDiscoverIpxe,
@@ -18,6 +19,45 @@ import { renderUbuntuInstallIpxe } from "../templates/ubuntu-boot.ipxe.js";
 import { renderDebugKickstart } from "../templates/debug.ks.js";
 import { logger } from "../services/logger.js";

+/**
+ * Resolve a booting machine's architecture.
+ *
+ * Order matters. The tracked record is what we actually observed on the machine, so it
+ * wins. `reported` is iPXE's ${buildarch}, which is only as good as the binary DHCP
+ * handed the client -- correct in practice, but a misconfigured option 93 mapping would
+ * make it lie. The configured default is the last resort.
+ *
+ * There is deliberately no operator-supplied architecture anywhere in this path.
+ */
+export function resolveArch(
+  state: BastionState,
+  mac: string,
+  reported: string | undefined,
+  config: BastionConfig,
+): Arch {
+  return normalizeArch(state.installed[mac]?.arch)
+    ?? normalizeArch(state.install_queue[mac]?.arch)
+    ?? normalizeArch(state.discovered[mac]?.arch)
+    ?? normalizeArch(reported)
+    ?? normalizeArch(config.arch)
+    ?? "x86_64";
+}
+
+/** The root filesystem to boot for --pxe-boot, if the machine's record carries one. */
+function resolveRoot(
+  state: BastionState,
+  mac: string,
+): { rootDevice: string; rootArgs?: string } | null {
+  const installed = state.installed[mac];
+  const discovered = state.discovered[mac];
+  const rootDevice = installed?.root_device ?? discovered?.root_device;
+  if (rootDevice === undefined || rootDevice === "") return null;
+  const rootArgs = installed?.root_args ?? discovered?.root_args;
+  return rootArgs !== undefined && rootArgs !== ""
+    ? { rootDevice, rootArgs }
+    : { rootDevice };
+}
+
 export function registerDispatchRoutes(
  app: FastifyInstance,
  config: BastionConfig,
@@ -61,9 +101,11 @@ echo "==============================="
    return reply.type("text/plain").send(script);
  });

-  app.get<{ Querystring: { mac?: string } }>("/dispatch", async (request, reply) => {
+  app.get<{ Querystring: { mac?: string; arch?: string } }>("/dispatch", async (request, reply) => {
    const mac = (request.query.mac ?? "").toLowerCase().replace(/-/g, ":");
    const currentState = state.load();
+    const arch = resolveArch(currentState, mac, request.query.arch, config);
+    const fedoraMirror = fedoraMirrorFor(config.fedoraVersion, arch);

    // Debug mode takes highest priority — auto-clear after serving once
    const debugEntry = currentState.debug[mac];
@@ -72,22 +114,45 @@ echo "==============================="
      state.update((s) => { delete s.debug[mac]; });

      let script: string;
-      if (debugEntry.pxeBoot) {
-        logger.info(`PXE BOOT DEBUG: ${mac} -> ${hostname} (kernel+initrd from PXE, root from NVMe)`);
+      const root = debugEntry.pxeBoot ? resolveRoot(currentState, mac) : null;
+
+      if (debugEntry.pxeBoot && root) {
+        logger.info(`PXE BOOT DEBUG: ${mac} -> ${hostname} (${arch}, root=${root.rootDevice})`);
        script = renderPxeBootDebugIpxe({
          mac,
          hostname,
          serverIp: config.serverIp,
          httpPort: config.httpPort,
+          arch,
+          ...root,
        });
      } else {
-        logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (rescue mode)`);
+        // --pxe-boot without a known root device falls back to rescue rather than
+        // guessing. A wrong root= leaves the machine unbootable, and rescue is where
+        // the operator can find the real one (curl /debug-setup.sh reports it back).
+        const notice = debugEntry.pxeBoot
+          ? [
+              "",
+              "NOTE: --pxe-boot requested, but no root device is recorded",
+              "      for this machine. Booting rescue instead.",
+              "      From the rescue shell, run:",
+              `        curl http://${config.serverIp}:${config.httpPort}/debug-setup.sh | bash`,
+              "      then retry --pxe-boot.",
+            ]
+          : undefined;
+        if (debugEntry.pxeBoot) {
+          logger.warn(`PXE BOOT DEBUG: ${mac} -> ${hostname} has no recorded root device -- serving rescue instead`);
+        } else {
+          logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (${arch}, rescue mode)`);
+        }
        script = renderDebugIpxe({
          mac,
          hostname,
          serverIp: config.serverIp,
          httpPort: config.httpPort,
-          fedoraMirror: config.fedoraMirror,
+          fedoraMirror,
+          arch,
+          ...(notice ? { notice } : {}),
        });
      }
      return reply.type("text/plain").send(script);
@@ -97,7 +162,7 @@ echo "==============================="
    if (queueEntry) {
      const hostname = queueEntry.hostname ?? "lab-node";
      const os = queueEntry.os ?? "fedora-43";
-      logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os})`);
+      logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os}, ${arch})`);

      let script: string;
      if (os.startsWith("ubuntu")) {
@@ -115,7 +180,8 @@ echo "==============================="
          serverIp: config.serverIp,
          httpPort: config.httpPort,
          fedoraVersion: config.fedoraVersion,
-          fedoraMirror: config.fedoraMirror,
+          fedoraMirror,
+          arch,
        });
      }

@@ -132,13 +198,14 @@ echo "==============================="
    }

    // Unknown MAC -> discovery mode
-    logger.info(`PXE request from ${mac} -> discovery mode`);
+    logger.info(`PXE request from ${mac} (${arch}) -> discovery mode`);

    const script = renderDiscoverIpxe({
      mac,
      serverIp: config.serverIp,
      httpPort: config.httpPort,
-      fedoraMirror: config.fedoraMirror,
+      fedoraMirror,
+      arch,
    });

    return reply.type("text/plain").send(script);
--- a/bastion/src/bastion/src/templates/boot.ipxe.ts
+++ b/bastion/src/bastion/src/templates/boot.ipxe.ts
@@ -1,4 +1,55 @@
 // iPXE boot script templates for dispatch routing.
+//
+// Architecture handling: the bastion serves one kernel/initrd pair per architecture.
+// x86_64 keeps the original unsuffixed paths so its output is unchanged; every other
+// architecture gets an arch-suffixed pair. See stageBootArtifacts() in main.ts for the
+// matching staging side, and boot-iso.ts for the same scheme on the ISO path.
+
+import type { Arch } from "@lab/shared";
+
+/** Kernel/initrd URL paths, keyed by architecture. */
+export function kernelPath(arch: Arch): string {
+  return arch === "x86_64" ? "/vmlinuz" : `/vmlinuz-${arch}`;
+}
+
+export function initrdPath(arch: Arch): string {
+  return arch === "x86_64" ? "/initrd.img" : `/initrd-${arch}.img`;
+}
+
+/**
+ * Console arguments per architecture.
+ *
+ * arm64 has no VGA text console: a headless machine only talks over the SoC UART, so
+ * ttyAMA0 must be listed as well. The last console= wins for /dev/console, so serial
+ * is the interactive one while tty0 still receives boot output on machines with a
+ * display attached.
+ */
+const CONSOLE_ARGS: Record<Arch, string> = {
+  x86_64: "console=tty0",
+  aarch64: "console=tty0 console=ttyAMA0,115200",
+};
+
+/**
+ * Anaconda arguments for the graphical-suppression / console setup.
+ *
+ * `nomodeset` disables kernel mode setting, which on x86 forces the generic VGA path
+ * and makes flaky GPU drivers survive the installer. On arm64 it does not mean the
+ * same thing -- there is no VGA fallback to drop back to, and it can leave the machine
+ * with no usable console at all -- so arm64 gets explicit console arguments instead.
+ */
+function installerArgs(arch: Arch): string {
+  return arch === "x86_64" ? "inst.text nomodeset" : `inst.text ${CONSOLE_ARGS[arch]}`;
+}
+
+/** Extra console arguments appended to templates that don't already set them. */
+function extraConsoleArgs(arch: Arch): string {
+  return arch === "x86_64" ? "" : ` ${CONSOLE_ARGS[arch]}`;
+}
+
+/** Join kernel arguments, dropping empties so callers can pass optional groups. */
+function joinArgs(...parts: Array<string | undefined>): string {
+  return parts.filter((p) => p !== undefined && p !== "").join(" ");
+}

 export interface BootIpxeParams {
  serverIp: string;
@@ -8,6 +59,11 @@ export interface BootIpxeParams {
 /**
 * Initial iPXE boot script that chains to the dispatch endpoint.
 * This is what dnsmasq serves to iPXE clients via HTTP.
+ *
+ * `${buildarch}` is iPXE's own build architecture ("x86_64" or "arm64"), which is the
+ * one architecture signal available on every path -- network PXE, UEFI HTTP boot and
+ * the boot ISO alike. DHCP option 93 only reaches dnsmasq, never this HTTP endpoint.
+ * dispatch prefers the tracked machine record and falls back to this.
 */
 export function renderBootIpxe(params: BootIpxeParams): string {
  return `#!ipxe
@@ -19,7 +75,7 @@ echo   Contacting server for instructions...
 echo ============================================
 echo

-chain http://${params.serverIp}:${params.httpPort}/dispatch?mac=\${net0/mac}
+chain http://${params.serverIp}:${params.httpPort}/dispatch?mac=\${net0/mac}&arch=\${buildarch}
 `;
 }

@@ -31,7 +87,9 @@ export function renderDiscoverIpxe(params: {
  serverIp: string;
  httpPort: number;
  fedoraMirror: string;
+  arch: Arch;
 }): string {
+  const base = `http://${params.serverIp}:${params.httpPort}`;
  return `#!ipxe

 echo
@@ -42,8 +100,8 @@ echo   Collecting hardware info...
 echo =============================================
 echo

-kernel http://${params.serverIp}:${params.httpPort}/vmlinuz inst.ks=http://${params.serverIp}:${params.httpPort}/discover.ks inst.stage2=${params.fedoraMirror} inst.text nomodeset
-initrd http://${params.serverIp}:${params.httpPort}/initrd.img
+kernel ${base}${kernelPath(params.arch)} inst.ks=${base}/discover.ks inst.stage2=${params.fedoraMirror} ${installerArgs(params.arch)}
+initrd ${base}${initrdPath(params.arch)}
 boot
 `;
 }
@@ -58,7 +116,9 @@ export function renderInstallIpxe(params: {
  httpPort: number;
  fedoraVersion: string;
  fedoraMirror: string;
+  arch: Arch;
 }): string {
+  const base = `http://${params.serverIp}:${params.httpPort}`;
  return `#!ipxe

 echo
@@ -69,8 +129,8 @@ echo   MAC:    ${params.mac}
 echo =============================================
 echo

-kernel http://${params.serverIp}:${params.httpPort}/vmlinuz inst.ks=http://${params.serverIp}:${params.httpPort}/ks?mac=${params.mac} inst.repo=${params.fedoraMirror} inst.text nomodeset
-initrd http://${params.serverIp}:${params.httpPort}/initrd.img
+kernel ${base}${kernelPath(params.arch)} inst.ks=${base}/ks?mac=${params.mac} inst.repo=${params.fedoraMirror} ${installerArgs(params.arch)}
+initrd ${base}${initrdPath(params.arch)}
 boot
 `;
 }
@@ -78,6 +138,9 @@ boot
 /**
 * iPXE script for debug/rescue mode -- boots Fedora installer in rescue mode.
 * Provides a shell with LVM tools, network, and SSH for inspecting installed systems.
+ *
+ * `notice` is shown before the boot line. dispatch uses it to explain why a requested
+ * --pxe-boot fell back to rescue.
 */
 export function renderDebugIpxe(params: {
  mac: string;
@@ -85,7 +148,11 @@ export function renderDebugIpxe(params: {
  serverIp: string;
  httpPort: number;
  fedoraMirror: string;
+  arch: Arch;
+  notice?: string[];
 }): string {
+  const base = `http://${params.serverIp}:${params.httpPort}`;
+  const notice = (params.notice ?? []).map((line) => `echo   ${line}\n`).join("");
  return `#!ipxe

 echo
@@ -93,11 +160,11 @@ echo =============================================
 echo   Lab PXE Bastion - DEBUG/RESCUE MODE
 echo   Target: ${params.hostname}
 echo   MAC:    ${params.mac}
-echo =============================================
+${notice}echo =============================================
 echo

-kernel http://${params.serverIp}:${params.httpPort}/vmlinuz inst.rescue inst.text inst.sshd inst.ks=http://${params.serverIp}:${params.httpPort}/debug.ks?mac=${params.mac} inst.stage2=${params.fedoraMirror}
-initrd http://${params.serverIp}:${params.httpPort}/initrd.img
+kernel ${base}${kernelPath(params.arch)} inst.rescue inst.text inst.sshd inst.ks=${base}/debug.ks?mac=${params.mac} inst.stage2=${params.fedoraMirror}${extraConsoleArgs(params.arch)}
+initrd ${base}${initrdPath(params.arch)}
 boot
 `;
 }
@@ -106,13 +173,28 @@ boot
 * iPXE script for PXE-boot debug mode -- boots the installed system's root
 * filesystem using the bastion's PXE kernel+initrd instead of local GRUB.
 * Workaround for UEFI firmware bugs that make local disk boot slow.
+ *
+ * rootDevice/rootArgs come from the machine's record -- they are not assumed. Our
+ * Fedora installs use an LVM layout, but nothing guarantees any given machine does,
+ * and a wrong root= here means an unbootable machine. dispatch refuses to render this
+ * script without them.
 */
 export function renderPxeBootDebugIpxe(params: {
  mac: string;
  hostname: string;
  serverIp: string;
  httpPort: number;
+  arch: Arch;
+  rootDevice: string;
+  rootArgs?: string;
 }): string {
+  const base = `http://${params.serverIp}:${params.httpPort}`;
+  const cmdline = joinArgs(
+    `root=${params.rootDevice}`,
+    "ro",
+    params.rootArgs,
+    CONSOLE_ARGS[params.arch],
+  );
  return `#!ipxe

 echo
@@ -124,8 +206,8 @@ echo   Kernel+initrd from PXE, root from NVMe
 echo =============================================
 echo

-kernel http://${params.serverIp}:${params.httpPort}/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0
-initrd http://${params.serverIp}:${params.httpPort}/initrd.img
+kernel ${base}${kernelPath(params.arch)} ${cmdline}
+initrd ${base}${initrdPath(params.arch)}
 boot
 `;
 }
--- a/bastion/src/shared/src/hardware/index.ts
+++ b/bastion/src/shared/src/hardware/index.ts
@@ -0,0 +1,154 @@
+// Architecture normalisation and machine classification.
+//
+// Both are derived from what the system already observes about a machine -- never from
+// an operator-supplied flag.
+
+import type { Arch, HardwareInfo, OnboardMethod, OsId } from "../types/index.js";
+
+export const SUPPORTED_ARCHES: readonly Arch[] = ["x86_64", "aarch64"] as const;
+
+/**
+ * Normalise an architecture string to one we serve boot artifacts for.
+ *
+ * Sources and their spellings:
+ *   uname -m         -> "x86_64" / "aarch64"
+ *   iPXE ${buildarch}-> "x86_64" / "arm64"
+ *   dpkg/Debian      -> "amd64"  / "arm64"
+ *
+ * Returns undefined for anything we don't serve, so callers fall back rather than
+ * inventing a kernel path that would 404.
+ */
+export function normalizeArch(value: string | undefined | null): Arch | undefined {
+  switch ((value ?? "").trim().toLowerCase()) {
+    case "x86_64":
+    case "x86-64":
+    case "amd64":
+      return "x86_64";
+    case "aarch64":
+    case "arm64":
+      return "aarch64";
+    default:
+      return undefined;
+  }
+}
+
+/** Fedora pxeboot artifact base URL for an architecture. */
+export function fedoraMirrorFor(fedoraVersion: string, arch: Arch): string {
+  return `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`;
+}
+
+/**
+ * Which architectures each OS in the pipeline can actually be installed on.
+ *
+ * Fedora publishes pxeboot vmlinuz/initrd for both. Ubuntu does not: as of 26.04,
+ * releases.ubuntu.com publishes amd64 artifacts only, so there is nothing to netboot an
+ * arm64 machine with. Claiming support would fail at download time with a 404 instead
+ * of a useful message.
+ */
+const OS_ARCH_SUPPORT: Record<OsId, readonly Arch[]> = {
+  "fedora-43": ["x86_64", "aarch64"],
+  "ubuntu-26.04": ["x86_64"],
+};
+
+export function osSupportsArch(os: OsId, arch: Arch): boolean {
+  return (OS_ARCH_SUPPORT[os] ?? []).includes(arch);
+}
+
+export function archesForOs(os: OsId): readonly Arch[] {
+  return OS_ARCH_SUPPORT[os] ?? [];
+}
+
+/**
+ * Machines that run a vendor OS we have no image for.
+ *
+ * These are SSH-onboard: we manage userspace, but reinstalling destroys a driver and
+ * firmware stack our pipeline cannot rebuild. Matched on DMI identity, which is what
+ * discovery and `provision recheck` both collect.
+ *
+ * This is deliberately a property of the machine ("it runs DGX OS"), not a blocklist
+ * ("never install this MAC"). When a DGX OS image joins the pipeline, teaching the
+ * installer about vendor_os "dgx-os" is what unblocks these machines -- no entry here
+ * needs deleting.
+ */
+interface VendorOsRule {
+  vendorOs: string;
+  description: string;
+  matches: (hw: DmiIdentity) => boolean;
+}
+
+interface DmiIdentity {
+  manufacturer: string;
+  product: string;
+  board: string;
+}
+
+const VENDOR_OS_RULES: readonly VendorOsRule[] = [
+  {
+    vendorOs: "dgx-os",
+    description: "NVIDIA DGX OS (proprietary driver + firmware stack, no image in our pipeline)",
+    matches: ({ manufacturer, product, board }) =>
+      (manufacturer.includes("nvidia") || product.includes("nvidia")) &&
+      (product.includes("dgx") || product.includes("spark") ||
+       board.includes("gb10") || product.includes("gb10")),
+  },
+];
+
+/**
+ * Machines known to run a vendor OS, by MAC.
+ *
+ * The DMI rules above only fire once discovery or `provision recheck` has populated a
+ * hardware record. Machines onboarded over SSH may sit in state for a long time with no
+ * DMI at all -- which is exactly the state both DGX Sparks are in today -- so a
+ * DMI-only classifier would fail open on the machines this guard exists to protect.
+ *
+ * This is a statement of fact about known hardware ("this box runs DGX OS"), not an
+ * install policy. Whether that means "refuse" is decided by whether the pipeline has an
+ * image for that vendor OS.
+ */
+const KNOWN_VENDOR_OS_MACS: Record<string, string> = {
+  "4c:bb:47:7f:29:35": "dgx-os",  // spark-2935
+  "48:21:0b:96:3a:1c": "dgx-os",  // spark-3a1c
+};
+
+/**
+ * Classify how a machine should be onboarded, from its hardware record.
+ *
+ * An explicit `onboard` already on the record wins: it may have been set by an operator
+ * or by a rule that has since changed, and silently overriding it would be worse than
+ * leaving it.
+ */
+export function classifyOnboard(
+  hw: Partial<Pick<HardwareInfo, "mac" | "manufacturer" | "product" | "board">>
+    & { onboard?: OnboardMethod; vendor_os?: string },
+): { onboard: OnboardMethod; vendor_os?: string } {
+  if (hw.onboard !== undefined) {
+    return hw.vendor_os !== undefined
+      ? { onboard: hw.onboard, vendor_os: hw.vendor_os }
+      : { onboard: hw.onboard };
+  }
+
+  const knownVendorOs = KNOWN_VENDOR_OS_MACS[(hw.mac ?? "").toLowerCase().replace(/-/g, ":")];
+  if (knownVendorOs !== undefined) {
+    return { onboard: "ssh", vendor_os: knownVendorOs };
+  }
+
+  const identity: DmiIdentity = {
+    manufacturer: (hw.manufacturer ?? "").toLowerCase(),
+    product: (hw.product ?? "").toLowerCase(),
+    board: (hw.board ?? "").toLowerCase(),
+  };
+
+  for (const rule of VENDOR_OS_RULES) {
+    if (rule.matches(identity)) {
+      return { onboard: "ssh", vendor_os: rule.vendorOs };
+    }
+  }
+
+  return { onboard: "pxe" };
+}
+
+/** Human-readable reason a vendor-OS machine must not be reinstalled. */
+export function vendorOsDescription(vendorOs: string | undefined): string {
+  const rule = VENDOR_OS_RULES.find((r) => r.vendorOs === vendorOs);
+  return rule?.description ?? "a vendor OS with no image in our pipeline";
+}
--- a/bastion/src/shared/src/index.ts
+++ b/bastion/src/shared/src/index.ts
@@ -1,6 +1,8 @@
 export type {
  OsId,
  Arch,
+  OnboardMethod,
+  RootCandidate,
  Role,
  HardwareInfo,
  InstallConfig,
@@ -10,6 +12,16 @@ export type {
  BastionConfig,
 } from "./types/index.js";

+export {
+  SUPPORTED_ARCHES,
+  normalizeArch,
+  fedoraMirrorFor,
+  osSupportsArch,
+  archesForOs,
+  classifyOnboard,
+  vendorOsDescription,
+} from "./hardware/index.js";
+
 export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./types/index.js";
 export type { RoleInfo } from "./types/index.js";

--- a/bastion/src/shared/src/protocol/index.ts
+++ b/bastion/src/shared/src/protocol/index.ts
@@ -113,7 +113,7 @@ export type LabdBastionMessage =
  | { type: "command-role-update"; requestId: string; mac: string; role: string }
  | { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean }
  | { type: "command-register"; requestId: string; mac: string; hostname: string; role: string; ip: string }
-  | { type: "command-discover"; requestId: string; mac: string; product?: string; board?: string; serial?: string; manufacturer?: string; cpu_model?: string; cpu_cores?: number; memory_gb?: number; arch?: string; disks?: Array<{ name: string; size_gb: number; model: string }>; nics?: Array<{ name: string; mac: string; state: string }> }
+  | { type: "command-discover"; requestId: string; mac: string; product?: string; board?: string; serial?: string; manufacturer?: string; cpu_model?: string; cpu_cores?: number; memory_gb?: number; arch?: string; disks?: Array<{ name: string; size_gb: number; model: string }>; nics?: Array<{ name: string; mac: string; state: string }>; root_device?: string; root_args?: string }
  | { type: "server-shutdown"; reconnectAfter: number };

 export type BastionMessageType = BastionMessage["type"];
--- a/bastion/src/shared/src/types/index.ts
+++ b/bastion/src/shared/src/types/index.ts
@@ -1,6 +1,8 @@
 export type {
  OsId,
  Arch,
+  OnboardMethod,
+  RootCandidate,
  Role,
  HardwareInfo,
  InstallConfig,
--- a/bastion/src/shared/src/types/state.ts
+++ b/bastion/src/shared/src/types/state.ts
@@ -11,6 +11,16 @@ export function isValidOsId(value: string): value is OsId {
  return (SUPPORTED_OS as readonly string[]).includes(value);
 }

+/**
+ * How a machine joins the lab.
+ *
+ * "pxe"  -- bare metal we install over the network (the default).
+ * "ssh"  -- the machine already runs a vendor OS we cannot reproduce, so we onboard
+ *           over SSH and manage userspace only. Installing would destroy that OS.
+ *           See classifyOnboard() and os-install-research.md.
+ */
+export type OnboardMethod = "pxe" | "ssh";
+
 export interface HardwareInfo {
  mac: string;
  product: string;
@@ -26,6 +36,23 @@ export interface HardwareInfo {
  first_seen: string;
  last_seen: string;
  bastionId?: string;  // set when aggregated through labd
+  // Onboarding classification -- absent means "pxe" (see classifyOnboard)
+  onboard?: OnboardMethod;
+  vendor_os?: string;   // e.g. "dgx-os": the OS this machine must keep running
+  // Root filesystem, for booting the installed system over PXE (--pxe-boot).
+  // Observed from the machine, never assumed.
+  root_device?: string;                 // e.g. "/dev/mapper/labvg-root"
+  root_args?: string;                   // e.g. "rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap"
+  root_candidates?: RootCandidate[];    // reported from a rescue shell when unknown
+}
+
+/** A possible root filesystem found while probing an unreachable machine. */
+export interface RootCandidate {
+  device: string;       // e.g. "/dev/mapper/labvg-root"
+  args?: string;        // extra dracut args needed to assemble it
+  fstype?: string;
+  size_gb?: number;
+  os_release?: string;  // PRETTY_NAME from /etc/os-release, if mountable
 }

 export type Role = "vanilla" | "worker" | "infra" | "labcontroller";
@@ -103,6 +130,11 @@ export interface InstalledInfo {
  cpu_cores?: number;
  memory_gb?: number;
  arch?: string;
+  onboard?: OnboardMethod;
+  vendor_os?: string;
+  root_device?: string;
+  root_args?: string;
+  root_candidates?: RootCandidate[];
 }

 export interface DebugConfig {