From d25c0ce64da4a23ae5a6be09511ad9ac39b72bf1 Mon Sep 17 00:00:00 2001 From: Michal Date: Tue, 11 Aug 2026 12:48:24 +0100 Subject: [PATCH] feat(bastion): refuse installs on machines running a vendor OS The DGX Sparks run DGX OS with a proprietary NVIDIA driver and firmware stack. No image in our pipeline restores it, so an install destroys the machine's software permanently -- and `labctl provision install` would happily do it. Machines carry an `onboard` classification and the `vendor_os` they must keep running. Both install entry points (the HTTP route and the labd command handler) refuse, naming the machine, what it runs, and pointing at `provision debug` instead. Rescue is deliberately never guarded: being unable to reinstall is exactly when you need a rescue shell. Classification is a fact about the machine, not a blocklist. When a DGX OS image joins the pipeline, teaching the installer about that vendor_os is what unblocks these boxes. It is keyed on DMI identity, with the two known Sparks also matched by MAC -- neither has DMI in bastion state today, so a DMI-only rule would fail open on exactly the machines this protects. Also refuses an OS/architecture combination with no netboot artifacts, which is Ubuntu on aarch64. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015nRFZXpKwUVE4SRSHw6GjF --- bastion/src/bastion/src/routes/api.ts | 40 ++++++++- .../src/bastion/src/services/install-guard.ts | 88 +++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 bastion/src/bastion/src/services/install-guard.ts diff --git a/bastion/src/bastion/src/routes/api.ts b/bastion/src/bastion/src/routes/api.ts index 8a75c65..51434fa 100644 --- a/bastion/src/bastion/src/routes/api.ts +++ b/bastion/src/bastion/src/routes/api.ts @@ -6,10 +6,11 @@ import type { FastifyInstance } from "fastify"; import type { HardwareInfo, InstalledInfo, Role } from "@lab/shared"; -import { isValidOsId, SUPPORTED_ROLES } from "@lab/shared"; +import { isValidOsId, SUPPORTED_ROLES, classifyOnboard } from "@lab/shared"; import type { StateManager } from "../services/state.js"; import { logger } from "../services/logger.js"; import { triggerPostProvisionK3s } from "../services/post-provision.js"; +import { checkInstallAllowed } from "../services/install-guard.js"; import { progressBus } from "../services/progress-events.js"; import type { ProgressEvent } from "../services/progress-events.js"; import type { InstallLogBuffer } from "../services/install-log.js"; @@ -53,6 +54,12 @@ export function registerApiRoutes( return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: fedora-43, ubuntu-26.04` }); } + const check = checkInstallAllowed(state.load(), mac, osId); + if (check.allowed === false) { + logger.warn(`INSTALL REFUSED: ${mac} -- ${check.error}`); + return reply.status(409).send({ error: check.error }); + } + state.update((s) => { s.install_queue[mac] = { hostname: hostname ?? "lab-node", @@ -284,6 +291,10 @@ export function registerApiRoutes( arch?: string; disks?: Array<{ name: string; size_gb: number; model: string }>; nics?: Array<{ name: string; mac: string; state: string }>; + // Root filesystem, when the reporter could observe it (recheck over SSH, or the + // probe script run from a rescue shell). Used by --pxe-boot. + root_device?: string; + root_args?: string; }; }>("/api/discover", async (request, reply) => { const data = request.body; @@ -298,6 +309,19 @@ export function registerApiRoutes( state.update((s) => { const existing = s.discovered[mac]; + // Classify onboarding from the DMI identity we just received. An explicit + // classification already on the record wins (see classifyOnboard). + const onboarding = classifyOnboard({ + mac, + manufacturer: data.manufacturer ?? "unknown", + product: data.product ?? "unknown", + board: data.board ?? "unknown", + ...(existing?.onboard !== undefined ? { onboard: existing.onboard } : {}), + ...(existing?.vendor_os !== undefined ? { vendor_os: existing.vendor_os } : {}), + }); + const rootDevice = data.root_device ?? existing?.root_device; + const rootArgs = data.root_args ?? existing?.root_args; + const hwInfo: HardwareInfo = { mac, product: data.product ?? "unknown", @@ -312,8 +336,22 @@ export function registerApiRoutes( nics: data.nics ?? [], first_seen: existing?.first_seen ?? now, last_seen: now, + onboard: onboarding.onboard, + ...(onboarding.vendor_os !== undefined ? { vendor_os: onboarding.vendor_os } : {}), + ...(rootDevice !== undefined ? { root_device: rootDevice } : {}), + ...(rootArgs !== undefined ? { root_args: rootArgs } : {}), }; s.discovered[mac] = hwInfo; + + // Keep the installed record in step -- the install guard and --pxe-boot read it. + const inst = s.installed[mac]; + if (inst) { + if (data.arch !== undefined) inst.arch = data.arch; + inst.onboard = onboarding.onboard; + if (onboarding.vendor_os !== undefined) inst.vendor_os = onboarding.vendor_os; + if (rootDevice !== undefined) inst.root_device = rootDevice; + if (rootArgs !== undefined) inst.root_args = rootArgs; + } }); const label = isNew ? "NEW MACHINE DISCOVERED" : "MACHINE RE-DISCOVERED"; diff --git a/bastion/src/bastion/src/services/install-guard.ts b/bastion/src/bastion/src/services/install-guard.ts new file mode 100644 index 0000000..670e866 --- /dev/null +++ b/bastion/src/bastion/src/services/install-guard.ts @@ -0,0 +1,88 @@ +// Pre-flight checks for queuing an OS install. +// +// Both entry points -- the HTTP /api/install route and the labd command-install handler +// -- run this, so `labctl provision install` and `provision reprovision` are covered +// whichever way the request arrives. +// +// Rescue/debug is deliberately NOT guarded. Being unable to reinstall a machine is +// exactly when you most need to boot it into a rescue shell. + +import type { BastionState, OsId } from "@lab/shared"; +import { classifyOnboard, normalizeArch, osSupportsArch, vendorOsDescription, archesForOs } from "@lab/shared"; + +export type InstallCheck = + | { allowed: true } + | { allowed: false; error: string }; + +/** Best-known identity for a MAC, merged across the three state maps. */ +function identify(state: BastionState, mac: string) { + const discovered = state.discovered[mac]; + const installed = state.installed[mac]; + const queued = state.install_queue[mac]; + + const manufacturer = discovered?.manufacturer ?? installed?.manufacturer; + const product = discovered?.product ?? installed?.product; + const board = discovered?.board; + const onboard = installed?.onboard ?? discovered?.onboard; + const vendorOs = installed?.vendor_os ?? discovered?.vendor_os; + + return { + hostname: installed?.hostname ?? queued?.hostname ?? discovered?.product ?? mac, + arch: normalizeArch(installed?.arch ?? queued?.arch ?? discovered?.arch), + identity: { + mac, + ...(manufacturer !== undefined ? { manufacturer } : {}), + ...(product !== undefined ? { product } : {}), + ...(board !== undefined ? { board } : {}), + ...(onboard !== undefined ? { onboard } : {}), + ...(vendorOs !== undefined ? { vendor_os: vendorOs } : {}), + }, + }; +} + +/** + * Decide whether `mac` may be queued for an install of `os`. + * + * Refusals name the machine and the reason, and point at the action that is available + * instead. An operator hitting this at 2am should not have to read the source to work + * out what happened. + */ +export function checkInstallAllowed( + state: BastionState, + mac: string, + os: OsId, +): InstallCheck { + const machine = identify(state, mac); + const { onboard, vendor_os } = classifyOnboard(machine.identity); + + // 1. Machines running a vendor OS we cannot rebuild. + if (onboard === "ssh") { + const what = vendorOsDescription(vendor_os); + return { + allowed: false, + error: + `Refusing to install ${machine.hostname} (${mac}): it runs ${what}. ` + + `No image in our pipeline can restore it, so installing ${os} would destroy that ` + + `driver and firmware stack permanently. This machine is SSH-onboard: we manage its ` + + `userspace, not its OS. ` + + `To boot it into a rescue shell instead, run: labctl provision debug ${machine.hostname}`, + // TODO: when a DGX OS / SparkOS image joins the pipeline, an install targeting a + // machine whose vendor_os matches that image should be allowed through here. + }; + } + + // 2. Architecture the OS has no netboot artifacts for. + if (machine.arch !== undefined && !osSupportsArch(os, machine.arch)) { + const supported = archesForOs(os); + return { + allowed: false, + error: + `Refusing to install ${os} on ${machine.hostname} (${mac}): ` + + `${os} has no ${machine.arch} netboot artifacts` + + (supported.length > 0 ? ` (only ${supported.join(", ")})` : "") + + `. Use an OS that supports ${machine.arch}.`, + }; + } + + return { allowed: true }; +}