diff --git a/bastion/src/bastion/src/main.ts b/bastion/src/bastion/src/main.ts index 7468f1f..b0ab842 100644 --- a/bastion/src/bastion/src/main.ts +++ b/bastion/src/bastion/src/main.ts @@ -3,7 +3,9 @@ import { mkdirSync, writeFileSync, readFileSync, existsSync, copyFileSync, symlinkSync, unlinkSync } from "node:fs"; import { execSync } from "node:child_process"; -import type { BastionConfig } from "@lab/shared"; +import type { Arch, BastionConfig } from "@lab/shared"; +import { SUPPORTED_ARCHES, fedoraMirrorFor, classifyOnboard } from "@lab/shared"; +import { kernelPath, initrdPath } from "./templates/boot.ipxe.js"; import { loadConfig } from "./config.js"; import { populateNetworkConfig } from "./services/network.js"; import { createApp } from "./server.js"; @@ -13,6 +15,7 @@ import { renderBootIpxe } from "./templates/boot.ipxe.js"; import { logger } from "./services/logger.js"; import { BastionConnection } from "./services/labd-connection.js"; import { progressBus } from "./services/progress-events.js"; +import { checkInstallAllowed } from "./services/install-guard.js"; import { ensureBootIso } from "./routes/boot-iso.js"; function copyIfMissing(src: string, dest: string, label: string): void { @@ -130,9 +133,14 @@ export async function startBastion(overrides: Partial = {}): Prom mkdirSync(config.tftpDir, { recursive: true }); mkdirSync(config.httpDir, { recursive: true }); + // Architectures we can actually network boot, reported in the banner so a missing + // arm64 payload is visible at startup instead of at 2am when a rescue is needed. + const bootArches: Arch[] = []; + let ipxeArm64Ready = false; + // Prepare boot artifacts if (config.skipArtifacts !== true) { - logger.info(`Preparing boot artifacts (Fedora ${config.fedoraVersion} ${config.arch})...`); + logger.info(`Preparing boot artifacts (Fedora ${config.fedoraVersion}, ${SUPPORTED_ARCHES.join(" + ")})...`); copyIfMissing( "/usr/share/ipxe/undionly.kpxe", @@ -150,20 +158,41 @@ export async function startBastion(overrides: Partial = {}): Prom `${config.tftpDir}/ipxe-arm64.efi`, "iPXE UEFI arm64", ); + ipxeArm64Ready = true; } catch { - logger.warn("arm64 iPXE not available -- skipping"); + logger.warn("arm64 iPXE not available -- arm64 machines cannot network boot."); + logger.warn(" Install with: sudo dnf install ipxe-bootimgs-aarch64"); } - download( - `${config.fedoraMirror}/images/pxeboot/vmlinuz`, - `${config.httpDir}/vmlinuz`, - "Fedora kernel", - ); - download( - `${config.fedoraMirror}/images/pxeboot/initrd.img`, - `${config.httpDir}/initrd.img`, - "Fedora initrd", - ); + // Fedora pxeboot kernel + initrd per architecture. x86_64 keeps the unsuffixed + // names it has always used; other architectures are suffixed. The iPXE templates + // resolve the same paths via kernelPath()/initrdPath(). + for (const arch of SUPPORTED_ARCHES) { + const mirror = fedoraMirrorFor(config.fedoraVersion, arch); + try { + download( + `${mirror}/images/pxeboot/vmlinuz`, + `${config.httpDir}${kernelPath(arch)}`, + `Fedora ${arch} kernel`, + ); + download( + `${mirror}/images/pxeboot/initrd.img`, + `${config.httpDir}${initrdPath(arch)}`, + `Fedora ${arch} initrd`, + ); + bootArches.push(arch); + } catch (err) { + // Non-fatal: a bastion with no arm64 artifacts still serves x86_64 fine. + // Failing startup over an unreachable mirror for an architecture that may not + // even be present on this network would be worse. + logger.warn(`Fedora ${arch} kernel/initrd unavailable -- ${arch} PXE disabled`); + logger.warn(` ${err instanceof Error ? err.message : String(err)}`); + } + } + + if (!bootArches.includes("x86_64")) { + throw new Error("Fedora x86_64 kernel/initrd could not be staged -- cannot serve PXE"); + } // Ubuntu netboot artifacts (non-fatal — Ubuntu version may not be released yet) try { @@ -254,6 +283,13 @@ export async function startBastion(overrides: Partial = {}): Prom // Wire up command handlers so labd can send install/forget/role commands labdConn.onCommand("command-install", async (msg) => { if (msg.type !== "command-install") throw new Error("unexpected"); + const installMac = msg.mac.toLowerCase().replace(/-/g, ":"); + const osId = (msg.os as import("@lab/shared").OsId | undefined) ?? "fedora-43"; + const check = checkInstallAllowed(state.load(), installMac, osId); + if (check.allowed === false) { + logger.warn(`INSTALL REFUSED: ${installMac} -- ${check.error}`); + return { status: "error", error: check.error }; + } state.update((s) => { s.install_queue[msg.mac] = { hostname: msg.hostname, @@ -314,13 +350,24 @@ export async function startBastion(overrides: Partial = {}): Prom const mac = (msg.mac as string).toLowerCase(); const now = new Date().toISOString(); const existing = state.load().discovered[mac]; + const identity = { + mac, + manufacturer: (msg.manufacturer as string) ?? "unknown", + product: (msg.product as string) ?? "unknown", + board: (msg.board as string) ?? "unknown", + ...(existing?.onboard !== undefined ? { onboard: existing.onboard } : {}), + ...(existing?.vendor_os !== undefined ? { vendor_os: existing.vendor_os } : {}), + }; + const onboarding = classifyOnboard(identity); + const rootDevice = msg.root_device ?? existing?.root_device; + const rootArgs = msg.root_args ?? existing?.root_args; state.update((s) => { s.discovered[mac] = { mac, - product: (msg.product as string) ?? "unknown", - board: (msg.board as string) ?? "unknown", + product: identity.product, + board: identity.board, serial: (msg.serial as string) ?? "unknown", - manufacturer: (msg.manufacturer as string) ?? "unknown", + manufacturer: identity.manufacturer, cpu_model: (msg.cpu_model as string) ?? "unknown", cpu_cores: (msg.cpu_cores as number) ?? 0, memory_gb: (msg.memory_gb as number) ?? 0, @@ -329,7 +376,20 @@ export async function startBastion(overrides: Partial = {}): Prom nics: (msg.nics as Array<{ name: string; mac: string; state: string }>) ?? [], first_seen: existing?.first_seen ?? now, last_seen: now, + onboard: onboarding.onboard, + ...(onboarding.vendor_os !== undefined ? { vendor_os: onboarding.vendor_os } : {}), + ...(rootDevice !== undefined ? { root_device: rootDevice } : {}), + ...(rootArgs !== undefined ? { root_args: rootArgs } : {}), }; + // Keep the installed record in step -- the guard and --pxe-boot both read it. + const inst = s.installed[mac]; + if (inst) { + inst.arch = (msg.arch as string) ?? inst.arch; + inst.onboard = onboarding.onboard; + if (onboarding.vendor_os !== undefined) inst.vendor_os = onboarding.vendor_os; + if (rootDevice !== undefined) inst.root_device = rootDevice; + if (rootArgs !== undefined) inst.root_args = rootArgs; + } }); logger.info(`HARDWARE UPDATED: ${mac} -- ${msg.manufacturer ?? "?"} ${msg.product ?? "?"} (${msg.cpu_model ?? "?"}, ${msg.cpu_cores ?? "?"} cores, ${msg.memory_gb ?? "?"}GB RAM)`); return { status: "ok", data: { mac } }; @@ -364,7 +424,7 @@ export async function startBastion(overrides: Partial = {}): Prom } // Print banner - printBanner(config); + printBanner(config, bootArches, ipxeArm64Ready); // Graceful shutdown const shutdown = async (): Promise => { @@ -386,11 +446,22 @@ export async function startBastion(overrides: Partial = {}): Prom await new Promise(() => {}); } -function printBanner(config: BastionConfig): void { +function printBanner(config: BastionConfig, bootArches: Arch[], ipxeArm64Ready: boolean): void { const dhcpInfo = config.dhcpMode === "full" ? `full (${config.dhcpRangeStart}-${config.dhcpRangeEnd})` : "proxy (alongside existing DHCP)"; + // arm64 needs both an iPXE binary (DHCP hands it out on option 93 = 0x0b) and a + // kernel/initrd pair. Report the combination, since either missing breaks it. + const archInfo = config.skipArtifacts === true + ? "(artifacts skipped)" + : SUPPORTED_ARCHES + .map((a) => { + const ready = bootArches.includes(a) && (a !== "aarch64" || ipxeArm64Ready); + return ready ? a : `${a} (unavailable)`; + }) + .join(", "); + console.log(""); console.log("\x1b[36m\x1b[1m" + "=".repeat(60) + "\x1b[0m"); console.log("\x1b[36m\x1b[1m Lab PXE Bastion -- Discovery Mode\x1b[0m"); @@ -399,7 +470,8 @@ function printBanner(config: BastionConfig): void { console.log(` Network: \x1b[1m${config.network}/24\x1b[0m via \x1b[1m${config.iface}\x1b[0m`); console.log(` DHCP: \x1b[1m${dhcpInfo}\x1b[0m`); console.log(` HTTP: \x1b[1mhttp://${config.serverIp}:${config.httpPort}/\x1b[0m`); - console.log(` OS: \x1b[1mFedora ${config.fedoraVersion} (${config.arch})\x1b[0m`); + console.log(` OS: \x1b[1mFedora ${config.fedoraVersion}\x1b[0m`); + console.log(` Net boot: \x1b[1m${archInfo}\x1b[0m`); console.log(` Domain: \x1b[1m${config.domain}\x1b[0m`); console.log(` State: \x1b[1m${config.stateFile}\x1b[0m`); console.log(""); diff --git a/bastion/src/bastion/src/routes/dispatch.ts b/bastion/src/bastion/src/routes/dispatch.ts index 0ecc1c4..c790fa5 100644 --- a/bastion/src/bastion/src/routes/dispatch.ts +++ b/bastion/src/bastion/src/routes/dispatch.ts @@ -5,7 +5,8 @@ // - unknown -> discovery mode (collect hardware, POST to bastion) import type { FastifyInstance } from "fastify"; -import type { BastionConfig } from "@lab/shared"; +import type { Arch, BastionConfig, BastionState } from "@lab/shared"; +import { normalizeArch, fedoraMirrorFor } from "@lab/shared"; import type { StateManager } from "../services/state.js"; import { renderDiscoverIpxe, @@ -18,6 +19,45 @@ import { renderUbuntuInstallIpxe } from "../templates/ubuntu-boot.ipxe.js"; import { renderDebugKickstart } from "../templates/debug.ks.js"; import { logger } from "../services/logger.js"; +/** + * Resolve a booting machine's architecture. + * + * Order matters. The tracked record is what we actually observed on the machine, so it + * wins. `reported` is iPXE's ${buildarch}, which is only as good as the binary DHCP + * handed the client -- correct in practice, but a misconfigured option 93 mapping would + * make it lie. The configured default is the last resort. + * + * There is deliberately no operator-supplied architecture anywhere in this path. + */ +export function resolveArch( + state: BastionState, + mac: string, + reported: string | undefined, + config: BastionConfig, +): Arch { + return normalizeArch(state.installed[mac]?.arch) + ?? normalizeArch(state.install_queue[mac]?.arch) + ?? normalizeArch(state.discovered[mac]?.arch) + ?? normalizeArch(reported) + ?? normalizeArch(config.arch) + ?? "x86_64"; +} + +/** The root filesystem to boot for --pxe-boot, if the machine's record carries one. */ +function resolveRoot( + state: BastionState, + mac: string, +): { rootDevice: string; rootArgs?: string } | null { + const installed = state.installed[mac]; + const discovered = state.discovered[mac]; + const rootDevice = installed?.root_device ?? discovered?.root_device; + if (rootDevice === undefined || rootDevice === "") return null; + const rootArgs = installed?.root_args ?? discovered?.root_args; + return rootArgs !== undefined && rootArgs !== "" + ? { rootDevice, rootArgs } + : { rootDevice }; +} + export function registerDispatchRoutes( app: FastifyInstance, config: BastionConfig, @@ -61,9 +101,11 @@ echo "===============================" return reply.type("text/plain").send(script); }); - app.get<{ Querystring: { mac?: string } }>("/dispatch", async (request, reply) => { + app.get<{ Querystring: { mac?: string; arch?: string } }>("/dispatch", async (request, reply) => { const mac = (request.query.mac ?? "").toLowerCase().replace(/-/g, ":"); const currentState = state.load(); + const arch = resolveArch(currentState, mac, request.query.arch, config); + const fedoraMirror = fedoraMirrorFor(config.fedoraVersion, arch); // Debug mode takes highest priority — auto-clear after serving once const debugEntry = currentState.debug[mac]; @@ -72,22 +114,45 @@ echo "===============================" state.update((s) => { delete s.debug[mac]; }); let script: string; - if (debugEntry.pxeBoot) { - logger.info(`PXE BOOT DEBUG: ${mac} -> ${hostname} (kernel+initrd from PXE, root from NVMe)`); + const root = debugEntry.pxeBoot ? resolveRoot(currentState, mac) : null; + + if (debugEntry.pxeBoot && root) { + logger.info(`PXE BOOT DEBUG: ${mac} -> ${hostname} (${arch}, root=${root.rootDevice})`); script = renderPxeBootDebugIpxe({ mac, hostname, serverIp: config.serverIp, httpPort: config.httpPort, + arch, + ...root, }); } else { - logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (rescue mode)`); + // --pxe-boot without a known root device falls back to rescue rather than + // guessing. A wrong root= leaves the machine unbootable, and rescue is where + // the operator can find the real one (curl /debug-setup.sh reports it back). + const notice = debugEntry.pxeBoot + ? [ + "", + "NOTE: --pxe-boot requested, but no root device is recorded", + " for this machine. Booting rescue instead.", + " From the rescue shell, run:", + ` curl http://${config.serverIp}:${config.httpPort}/debug-setup.sh | bash`, + " then retry --pxe-boot.", + ] + : undefined; + if (debugEntry.pxeBoot) { + logger.warn(`PXE BOOT DEBUG: ${mac} -> ${hostname} has no recorded root device -- serving rescue instead`); + } else { + logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (${arch}, rescue mode)`); + } script = renderDebugIpxe({ mac, hostname, serverIp: config.serverIp, httpPort: config.httpPort, - fedoraMirror: config.fedoraMirror, + fedoraMirror, + arch, + ...(notice ? { notice } : {}), }); } return reply.type("text/plain").send(script); @@ -97,7 +162,7 @@ echo "===============================" if (queueEntry) { const hostname = queueEntry.hostname ?? "lab-node"; const os = queueEntry.os ?? "fedora-43"; - logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os})`); + logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os}, ${arch})`); let script: string; if (os.startsWith("ubuntu")) { @@ -115,7 +180,8 @@ echo "===============================" serverIp: config.serverIp, httpPort: config.httpPort, fedoraVersion: config.fedoraVersion, - fedoraMirror: config.fedoraMirror, + fedoraMirror, + arch, }); } @@ -132,13 +198,14 @@ echo "===============================" } // Unknown MAC -> discovery mode - logger.info(`PXE request from ${mac} -> discovery mode`); + logger.info(`PXE request from ${mac} (${arch}) -> discovery mode`); const script = renderDiscoverIpxe({ mac, serverIp: config.serverIp, httpPort: config.httpPort, - fedoraMirror: config.fedoraMirror, + fedoraMirror, + arch, }); return reply.type("text/plain").send(script); diff --git a/bastion/src/bastion/src/templates/boot.ipxe.ts b/bastion/src/bastion/src/templates/boot.ipxe.ts index 95f36d2..c6d9ffb 100644 --- a/bastion/src/bastion/src/templates/boot.ipxe.ts +++ b/bastion/src/bastion/src/templates/boot.ipxe.ts @@ -1,4 +1,55 @@ // iPXE boot script templates for dispatch routing. +// +// Architecture handling: the bastion serves one kernel/initrd pair per architecture. +// x86_64 keeps the original unsuffixed paths so its output is unchanged; every other +// architecture gets an arch-suffixed pair. See stageBootArtifacts() in main.ts for the +// matching staging side, and boot-iso.ts for the same scheme on the ISO path. + +import type { Arch } from "@lab/shared"; + +/** Kernel/initrd URL paths, keyed by architecture. */ +export function kernelPath(arch: Arch): string { + return arch === "x86_64" ? "/vmlinuz" : `/vmlinuz-${arch}`; +} + +export function initrdPath(arch: Arch): string { + return arch === "x86_64" ? "/initrd.img" : `/initrd-${arch}.img`; +} + +/** + * Console arguments per architecture. + * + * arm64 has no VGA text console: a headless machine only talks over the SoC UART, so + * ttyAMA0 must be listed as well. The last console= wins for /dev/console, so serial + * is the interactive one while tty0 still receives boot output on machines with a + * display attached. + */ +const CONSOLE_ARGS: Record = { + x86_64: "console=tty0", + aarch64: "console=tty0 console=ttyAMA0,115200", +}; + +/** + * Anaconda arguments for the graphical-suppression / console setup. + * + * `nomodeset` disables kernel mode setting, which on x86 forces the generic VGA path + * and makes flaky GPU drivers survive the installer. On arm64 it does not mean the + * same thing -- there is no VGA fallback to drop back to, and it can leave the machine + * with no usable console at all -- so arm64 gets explicit console arguments instead. + */ +function installerArgs(arch: Arch): string { + return arch === "x86_64" ? "inst.text nomodeset" : `inst.text ${CONSOLE_ARGS[arch]}`; +} + +/** Extra console arguments appended to templates that don't already set them. */ +function extraConsoleArgs(arch: Arch): string { + return arch === "x86_64" ? "" : ` ${CONSOLE_ARGS[arch]}`; +} + +/** Join kernel arguments, dropping empties so callers can pass optional groups. */ +function joinArgs(...parts: Array): string { + return parts.filter((p) => p !== undefined && p !== "").join(" "); +} export interface BootIpxeParams { serverIp: string; @@ -8,6 +59,11 @@ export interface BootIpxeParams { /** * Initial iPXE boot script that chains to the dispatch endpoint. * This is what dnsmasq serves to iPXE clients via HTTP. + * + * `${buildarch}` is iPXE's own build architecture ("x86_64" or "arm64"), which is the + * one architecture signal available on every path -- network PXE, UEFI HTTP boot and + * the boot ISO alike. DHCP option 93 only reaches dnsmasq, never this HTTP endpoint. + * dispatch prefers the tracked machine record and falls back to this. */ export function renderBootIpxe(params: BootIpxeParams): string { return `#!ipxe @@ -19,7 +75,7 @@ echo Contacting server for instructions... echo ============================================ echo -chain http://${params.serverIp}:${params.httpPort}/dispatch?mac=\${net0/mac} +chain http://${params.serverIp}:${params.httpPort}/dispatch?mac=\${net0/mac}&arch=\${buildarch} `; } @@ -31,7 +87,9 @@ export function renderDiscoverIpxe(params: { serverIp: string; httpPort: number; fedoraMirror: string; + arch: Arch; }): string { + const base = `http://${params.serverIp}:${params.httpPort}`; return `#!ipxe echo @@ -42,8 +100,8 @@ echo Collecting hardware info... echo ============================================= echo -kernel http://${params.serverIp}:${params.httpPort}/vmlinuz inst.ks=http://${params.serverIp}:${params.httpPort}/discover.ks inst.stage2=${params.fedoraMirror} inst.text nomodeset -initrd http://${params.serverIp}:${params.httpPort}/initrd.img +kernel ${base}${kernelPath(params.arch)} inst.ks=${base}/discover.ks inst.stage2=${params.fedoraMirror} ${installerArgs(params.arch)} +initrd ${base}${initrdPath(params.arch)} boot `; } @@ -58,7 +116,9 @@ export function renderInstallIpxe(params: { httpPort: number; fedoraVersion: string; fedoraMirror: string; + arch: Arch; }): string { + const base = `http://${params.serverIp}:${params.httpPort}`; return `#!ipxe echo @@ -69,8 +129,8 @@ echo MAC: ${params.mac} echo ============================================= echo -kernel http://${params.serverIp}:${params.httpPort}/vmlinuz inst.ks=http://${params.serverIp}:${params.httpPort}/ks?mac=${params.mac} inst.repo=${params.fedoraMirror} inst.text nomodeset -initrd http://${params.serverIp}:${params.httpPort}/initrd.img +kernel ${base}${kernelPath(params.arch)} inst.ks=${base}/ks?mac=${params.mac} inst.repo=${params.fedoraMirror} ${installerArgs(params.arch)} +initrd ${base}${initrdPath(params.arch)} boot `; } @@ -78,6 +138,9 @@ boot /** * iPXE script for debug/rescue mode -- boots Fedora installer in rescue mode. * Provides a shell with LVM tools, network, and SSH for inspecting installed systems. + * + * `notice` is shown before the boot line. dispatch uses it to explain why a requested + * --pxe-boot fell back to rescue. */ export function renderDebugIpxe(params: { mac: string; @@ -85,7 +148,11 @@ export function renderDebugIpxe(params: { serverIp: string; httpPort: number; fedoraMirror: string; + arch: Arch; + notice?: string[]; }): string { + const base = `http://${params.serverIp}:${params.httpPort}`; + const notice = (params.notice ?? []).map((line) => `echo ${line}\n`).join(""); return `#!ipxe echo @@ -93,11 +160,11 @@ echo ============================================= echo Lab PXE Bastion - DEBUG/RESCUE MODE echo Target: ${params.hostname} echo MAC: ${params.mac} -echo ============================================= +${notice}echo ============================================= echo -kernel http://${params.serverIp}:${params.httpPort}/vmlinuz inst.rescue inst.text inst.sshd inst.ks=http://${params.serverIp}:${params.httpPort}/debug.ks?mac=${params.mac} inst.stage2=${params.fedoraMirror} -initrd http://${params.serverIp}:${params.httpPort}/initrd.img +kernel ${base}${kernelPath(params.arch)} inst.rescue inst.text inst.sshd inst.ks=${base}/debug.ks?mac=${params.mac} inst.stage2=${params.fedoraMirror}${extraConsoleArgs(params.arch)} +initrd ${base}${initrdPath(params.arch)} boot `; } @@ -106,13 +173,28 @@ boot * iPXE script for PXE-boot debug mode -- boots the installed system's root * filesystem using the bastion's PXE kernel+initrd instead of local GRUB. * Workaround for UEFI firmware bugs that make local disk boot slow. + * + * rootDevice/rootArgs come from the machine's record -- they are not assumed. Our + * Fedora installs use an LVM layout, but nothing guarantees any given machine does, + * and a wrong root= here means an unbootable machine. dispatch refuses to render this + * script without them. */ export function renderPxeBootDebugIpxe(params: { mac: string; hostname: string; serverIp: string; httpPort: number; + arch: Arch; + rootDevice: string; + rootArgs?: string; }): string { + const base = `http://${params.serverIp}:${params.httpPort}`; + const cmdline = joinArgs( + `root=${params.rootDevice}`, + "ro", + params.rootArgs, + CONSOLE_ARGS[params.arch], + ); return `#!ipxe echo @@ -124,8 +206,8 @@ echo Kernel+initrd from PXE, root from NVMe echo ============================================= echo -kernel http://${params.serverIp}:${params.httpPort}/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0 -initrd http://${params.serverIp}:${params.httpPort}/initrd.img +kernel ${base}${kernelPath(params.arch)} ${cmdline} +initrd ${base}${initrdPath(params.arch)} boot `; } diff --git a/bastion/src/shared/src/hardware/index.ts b/bastion/src/shared/src/hardware/index.ts new file mode 100644 index 0000000..2b58f98 --- /dev/null +++ b/bastion/src/shared/src/hardware/index.ts @@ -0,0 +1,154 @@ +// Architecture normalisation and machine classification. +// +// Both are derived from what the system already observes about a machine -- never from +// an operator-supplied flag. + +import type { Arch, HardwareInfo, OnboardMethod, OsId } from "../types/index.js"; + +export const SUPPORTED_ARCHES: readonly Arch[] = ["x86_64", "aarch64"] as const; + +/** + * Normalise an architecture string to one we serve boot artifacts for. + * + * Sources and their spellings: + * uname -m -> "x86_64" / "aarch64" + * iPXE ${buildarch}-> "x86_64" / "arm64" + * dpkg/Debian -> "amd64" / "arm64" + * + * Returns undefined for anything we don't serve, so callers fall back rather than + * inventing a kernel path that would 404. + */ +export function normalizeArch(value: string | undefined | null): Arch | undefined { + switch ((value ?? "").trim().toLowerCase()) { + case "x86_64": + case "x86-64": + case "amd64": + return "x86_64"; + case "aarch64": + case "arm64": + return "aarch64"; + default: + return undefined; + } +} + +/** Fedora pxeboot artifact base URL for an architecture. */ +export function fedoraMirrorFor(fedoraVersion: string, arch: Arch): string { + return `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`; +} + +/** + * Which architectures each OS in the pipeline can actually be installed on. + * + * Fedora publishes pxeboot vmlinuz/initrd for both. Ubuntu does not: as of 26.04, + * releases.ubuntu.com publishes amd64 artifacts only, so there is nothing to netboot an + * arm64 machine with. Claiming support would fail at download time with a 404 instead + * of a useful message. + */ +const OS_ARCH_SUPPORT: Record = { + "fedora-43": ["x86_64", "aarch64"], + "ubuntu-26.04": ["x86_64"], +}; + +export function osSupportsArch(os: OsId, arch: Arch): boolean { + return (OS_ARCH_SUPPORT[os] ?? []).includes(arch); +} + +export function archesForOs(os: OsId): readonly Arch[] { + return OS_ARCH_SUPPORT[os] ?? []; +} + +/** + * Machines that run a vendor OS we have no image for. + * + * These are SSH-onboard: we manage userspace, but reinstalling destroys a driver and + * firmware stack our pipeline cannot rebuild. Matched on DMI identity, which is what + * discovery and `provision recheck` both collect. + * + * This is deliberately a property of the machine ("it runs DGX OS"), not a blocklist + * ("never install this MAC"). When a DGX OS image joins the pipeline, teaching the + * installer about vendor_os "dgx-os" is what unblocks these machines -- no entry here + * needs deleting. + */ +interface VendorOsRule { + vendorOs: string; + description: string; + matches: (hw: DmiIdentity) => boolean; +} + +interface DmiIdentity { + manufacturer: string; + product: string; + board: string; +} + +const VENDOR_OS_RULES: readonly VendorOsRule[] = [ + { + vendorOs: "dgx-os", + description: "NVIDIA DGX OS (proprietary driver + firmware stack, no image in our pipeline)", + matches: ({ manufacturer, product, board }) => + (manufacturer.includes("nvidia") || product.includes("nvidia")) && + (product.includes("dgx") || product.includes("spark") || + board.includes("gb10") || product.includes("gb10")), + }, +]; + +/** + * Machines known to run a vendor OS, by MAC. + * + * The DMI rules above only fire once discovery or `provision recheck` has populated a + * hardware record. Machines onboarded over SSH may sit in state for a long time with no + * DMI at all -- which is exactly the state both DGX Sparks are in today -- so a + * DMI-only classifier would fail open on the machines this guard exists to protect. + * + * This is a statement of fact about known hardware ("this box runs DGX OS"), not an + * install policy. Whether that means "refuse" is decided by whether the pipeline has an + * image for that vendor OS. + */ +const KNOWN_VENDOR_OS_MACS: Record = { + "4c:bb:47:7f:29:35": "dgx-os", // spark-2935 + "48:21:0b:96:3a:1c": "dgx-os", // spark-3a1c +}; + +/** + * Classify how a machine should be onboarded, from its hardware record. + * + * An explicit `onboard` already on the record wins: it may have been set by an operator + * or by a rule that has since changed, and silently overriding it would be worse than + * leaving it. + */ +export function classifyOnboard( + hw: Partial> + & { onboard?: OnboardMethod; vendor_os?: string }, +): { onboard: OnboardMethod; vendor_os?: string } { + if (hw.onboard !== undefined) { + return hw.vendor_os !== undefined + ? { onboard: hw.onboard, vendor_os: hw.vendor_os } + : { onboard: hw.onboard }; + } + + const knownVendorOs = KNOWN_VENDOR_OS_MACS[(hw.mac ?? "").toLowerCase().replace(/-/g, ":")]; + if (knownVendorOs !== undefined) { + return { onboard: "ssh", vendor_os: knownVendorOs }; + } + + const identity: DmiIdentity = { + manufacturer: (hw.manufacturer ?? "").toLowerCase(), + product: (hw.product ?? "").toLowerCase(), + board: (hw.board ?? "").toLowerCase(), + }; + + for (const rule of VENDOR_OS_RULES) { + if (rule.matches(identity)) { + return { onboard: "ssh", vendor_os: rule.vendorOs }; + } + } + + return { onboard: "pxe" }; +} + +/** Human-readable reason a vendor-OS machine must not be reinstalled. */ +export function vendorOsDescription(vendorOs: string | undefined): string { + const rule = VENDOR_OS_RULES.find((r) => r.vendorOs === vendorOs); + return rule?.description ?? "a vendor OS with no image in our pipeline"; +} diff --git a/bastion/src/shared/src/index.ts b/bastion/src/shared/src/index.ts index 443edbc..7100826 100644 --- a/bastion/src/shared/src/index.ts +++ b/bastion/src/shared/src/index.ts @@ -1,6 +1,8 @@ export type { OsId, Arch, + OnboardMethod, + RootCandidate, Role, HardwareInfo, InstallConfig, @@ -10,6 +12,16 @@ export type { BastionConfig, } from "./types/index.js"; +export { + SUPPORTED_ARCHES, + normalizeArch, + fedoraMirrorFor, + osSupportsArch, + archesForOs, + classifyOnboard, + vendorOsDescription, +} from "./hardware/index.js"; + export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./types/index.js"; export type { RoleInfo } from "./types/index.js"; diff --git a/bastion/src/shared/src/protocol/index.ts b/bastion/src/shared/src/protocol/index.ts index d7e3018..d3fa3f6 100644 --- a/bastion/src/shared/src/protocol/index.ts +++ b/bastion/src/shared/src/protocol/index.ts @@ -113,7 +113,7 @@ export type LabdBastionMessage = | { type: "command-role-update"; requestId: string; mac: string; role: string } | { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean } | { type: "command-register"; requestId: string; mac: string; hostname: string; role: string; ip: string } - | { type: "command-discover"; requestId: string; mac: string; product?: string; board?: string; serial?: string; manufacturer?: string; cpu_model?: string; cpu_cores?: number; memory_gb?: number; arch?: string; disks?: Array<{ name: string; size_gb: number; model: string }>; nics?: Array<{ name: string; mac: string; state: string }> } + | { type: "command-discover"; requestId: string; mac: string; product?: string; board?: string; serial?: string; manufacturer?: string; cpu_model?: string; cpu_cores?: number; memory_gb?: number; arch?: string; disks?: Array<{ name: string; size_gb: number; model: string }>; nics?: Array<{ name: string; mac: string; state: string }>; root_device?: string; root_args?: string } | { type: "server-shutdown"; reconnectAfter: number }; export type BastionMessageType = BastionMessage["type"]; diff --git a/bastion/src/shared/src/types/index.ts b/bastion/src/shared/src/types/index.ts index 510ee31..35a19a9 100644 --- a/bastion/src/shared/src/types/index.ts +++ b/bastion/src/shared/src/types/index.ts @@ -1,6 +1,8 @@ export type { OsId, Arch, + OnboardMethod, + RootCandidate, Role, HardwareInfo, InstallConfig, diff --git a/bastion/src/shared/src/types/state.ts b/bastion/src/shared/src/types/state.ts index b8599fd..6e98a87 100644 --- a/bastion/src/shared/src/types/state.ts +++ b/bastion/src/shared/src/types/state.ts @@ -11,6 +11,16 @@ export function isValidOsId(value: string): value is OsId { return (SUPPORTED_OS as readonly string[]).includes(value); } +/** + * How a machine joins the lab. + * + * "pxe" -- bare metal we install over the network (the default). + * "ssh" -- the machine already runs a vendor OS we cannot reproduce, so we onboard + * over SSH and manage userspace only. Installing would destroy that OS. + * See classifyOnboard() and os-install-research.md. + */ +export type OnboardMethod = "pxe" | "ssh"; + export interface HardwareInfo { mac: string; product: string; @@ -26,6 +36,23 @@ export interface HardwareInfo { first_seen: string; last_seen: string; bastionId?: string; // set when aggregated through labd + // Onboarding classification -- absent means "pxe" (see classifyOnboard) + onboard?: OnboardMethod; + vendor_os?: string; // e.g. "dgx-os": the OS this machine must keep running + // Root filesystem, for booting the installed system over PXE (--pxe-boot). + // Observed from the machine, never assumed. + root_device?: string; // e.g. "/dev/mapper/labvg-root" + root_args?: string; // e.g. "rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap" + root_candidates?: RootCandidate[]; // reported from a rescue shell when unknown +} + +/** A possible root filesystem found while probing an unreachable machine. */ +export interface RootCandidate { + device: string; // e.g. "/dev/mapper/labvg-root" + args?: string; // extra dracut args needed to assemble it + fstype?: string; + size_gb?: number; + os_release?: string; // PRETTY_NAME from /etc/os-release, if mountable } export type Role = "vanilla" | "worker" | "infra" | "labcontroller"; @@ -103,6 +130,11 @@ export interface InstalledInfo { cpu_cores?: number; memory_gb?: number; arch?: string; + onboard?: OnboardMethod; + vendor_os?: string; + root_device?: string; + root_args?: string; + root_candidates?: RootCandidate[]; } export interface DebugConfig {