diff --git a/bastion/src/bastion/src/routes/api.ts b/bastion/src/bastion/src/routes/api.ts index 17f7bda..7398046 100644 --- a/bastion/src/bastion/src/routes/api.ts +++ b/bastion/src/bastion/src/routes/api.ts @@ -15,6 +15,13 @@ import type { ProgressEvent } from "../services/progress-events.js"; import type { InstallLogBuffer } from "../services/install-log.js"; import type { SyslogListener } from "../services/syslog-listener.js"; +/** + * Seconds after dispatch with zero progress before a machine is called stalled. + * Generous: the slowest legitimate gap is fetching a ~600MB VyOS squashfs over + * HTTP before the hook can report anything. + */ +const STALL_THRESHOLD_S = 8 * 60; + export function registerApiRoutes( app: FastifyInstance, state: StateManager, @@ -442,6 +449,15 @@ export function registerApiRoutes( const installedEntry = currentState.installed[mac]; if (queueEntry) { + // A machine that was handed an install script but has reported nothing + // since is wedged BEFORE the installer environment came up — a bad + // kernel/initrd, no network in the initramfs, or the wrong NIC picked. + // Surfacing it here is what makes that diagnosable without a console. + const since = queueEntry.progress_at ?? queueEntry.dispatched_at; + const stalledForS = since !== undefined && queueEntry.progress === undefined + ? Math.floor((Date.now() - new Date(since).getTime()) / 1000) + : 0; + return reply.send({ mac, hostname: queueEntry.hostname, @@ -449,6 +465,9 @@ export function registerApiRoutes( progress: queueEntry.progress ?? "queued", progress_detail: queueEntry.progress_detail ?? "", progress_at: queueEntry.progress_at ?? queueEntry.queued_at, + dispatched_at: queueEntry.dispatched_at, + stalled_for_s: stalledForS, + stalled: stalledForS > STALL_THRESHOLD_S, role: queueEntry.role, os: queueEntry.os, stages: queueEntry.log ?? [], diff --git a/bastion/src/bastion/src/routes/dispatch.ts b/bastion/src/bastion/src/routes/dispatch.ts index f4637eb..9ff1fdf 100644 --- a/bastion/src/bastion/src/routes/dispatch.ts +++ b/bastion/src/bastion/src/routes/dispatch.ts @@ -100,6 +100,13 @@ echo "===============================" const os = queueEntry.os ?? "fedora-43"; logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os})`); + // Stamp the handoff so a machine that boots the installer but never + // reports can be spotted without a console. + state.update((s) => { + const entry = s.install_queue[mac]; + if (entry) entry.dispatched_at = new Date().toISOString(); + }); + let script: string; if (os.startsWith("vyos")) { script = renderVyosInstallIpxe({ diff --git a/bastion/src/bastion/src/templates/vyos-boot.ipxe.ts b/bastion/src/bastion/src/templates/vyos-boot.ipxe.ts index 8dacf2b..c917544 100644 --- a/bastion/src/bastion/src/templates/vyos-boot.ipxe.ts +++ b/bastion/src/bastion/src/templates/vyos-boot.ipxe.ts @@ -18,6 +18,20 @@ export function renderVyosInstallIpxe(params: { }): string { const base = `http://${params.serverIp}:${params.httpPort}`; + // Pin the boot NIC by MAC. live-boot otherwise scans for the first + // *connected* interface, and on a multi-NIC box that race is lost by + // whichever port negotiates slowest: on the Protectli VP2440 the SFP+ + // pair links first, so live-boot picked the fiber ports (which have no + // DHCP), burned 15s per port, and gave up with "Unable to find a live + // file system on the network" -- while the copper port that actually PXE + // booted came up at 4.6s and was never tried. + // + // live-boot's Device_from_bootif() strips the "01-" and matches the MAC + // against /sys/class/net/*. params.mac is the dispatch key, i.e. exactly + // the NIC that PXE booted -- more reliable than iPXE's ${net0} on a box + // where the booting NIC may not be net0. + const bootif = `01-${params.mac.toLowerCase().replace(/:/g, "-")}`; + // Deliberately NOT passing `nonetworking` (present in VyOS's own PXE docs): // live-config's hook component needs networking up to fetch the hook over // HTTP. Also no `console=ttyS0` — on hardware without a physical UART that @@ -32,7 +46,7 @@ echo MAC: ${params.mac} echo ============================================= echo -kernel ${base}/vyos-vmlinuz boot=live nopersistence noautologin fetch=${base}/vyos-filesystem.squashfs live-config.hooks=${base}/vyos/autoinstall.sh?mac=${params.mac} +kernel ${base}/vyos-vmlinuz boot=live nopersistence noautologin BOOTIF=${bootif} fetch=${base}/vyos-filesystem.squashfs live-config.hooks=${base}/vyos/autoinstall.sh?mac=${params.mac} initrd ${base}/vyos-initrd boot `; diff --git a/bastion/src/bastion/tests/vyos.test.ts b/bastion/src/bastion/tests/vyos.test.ts index 02914e5..e0535a7 100644 --- a/bastion/src/bastion/tests/vyos.test.ts +++ b/bastion/src/bastion/tests/vyos.test.ts @@ -514,3 +514,35 @@ print(json.dumps(out)) } }); }); + +describe("vyos boot NIC pinning", () => { + it("pins the boot interface by MAC via BOOTIF", async () => { + // Without this, live-boot picks the first *connected* NIC. On the VP2440 + // the SFP+ pair links before the copper PXE port, so live-boot tried the + // fiber ports (no DHCP), timed out 15s each, and failed with "Unable to + // find a live file system on the network". + const testDir = join(tmpdir(), `bastion-vyos-bootif-${Date.now()}`); + mkdirSync(join(testDir, "http"), { recursive: true }); + mkdirSync(join(testDir, "tftp"), { recursive: true }); + const { app: a, state: st } = createApp(createTestConfig(testDir)); + try { + const m = "64:62:66:25:96:47"; + st.update((s) => { + s.install_queue[m] = { + hostname: "vyos001", disk: "/dev/mmcblk0", role: "vanilla", + os: "vyos-rolling", queued_at: new Date().toISOString(), + }; + }); + const res = await a.inject({ method: "GET", url: `/dispatch?mac=${m}` }); + // live-boot's Device_from_bootif() expects 01- + expect(res.body).toContain("BOOTIF=01-64-62-66-25-96-47"); + // and it must be on the kernel line, before fetch= is attempted + const kernelLine = res.body.split("\n").find((l) => l.startsWith("kernel ")); + expect(kernelLine).toContain("BOOTIF=01-64-62-66-25-96-47"); + expect(kernelLine).toContain("fetch="); + } finally { + await a.close(); + rmSync(testDir, { recursive: true, force: true }); + } + }); +}); diff --git a/bastion/src/shared/src/types/state.ts b/bastion/src/shared/src/types/state.ts index 0d96238..72f5ee0 100644 --- a/bastion/src/shared/src/types/state.ts +++ b/bastion/src/shared/src/types/state.ts @@ -151,6 +151,13 @@ export interface InstallConfig { vyos?: VyosInstallSpec; // only consulted when os is "vyos-rolling" arch?: Arch; // detected from HardwareInfo or overridden queued_at: string; + /** + * When dispatch last served this machine an install boot script. Progress + * callbacks only start once the installer environment is up, so a machine + * dispatched long ago with no progress is wedged before that point (bad + * kernel/initrd, no network in the initramfs, wrong NIC picked...). + */ + dispatched_at?: string; progress?: string; progress_at?: string; progress_detail?: string; diff --git a/bastion/tests/integration/helpers/pxe-vm.ts b/bastion/tests/integration/helpers/pxe-vm.ts index 58ae4e2..f5374e2 100644 --- a/bastion/tests/integration/helpers/pxe-vm.ts +++ b/bastion/tests/integration/helpers/pxe-vm.ts @@ -29,6 +29,17 @@ export interface PxeVmConfig { diskSize: number; // GB network: string; // libvirt network name arch?: "x86_64" | "aarch64"; + /** + * Extra NICs enumerated BEFORE the PXE NIC, on a network with no route to + * the bastion (defaults to libvirt's "default"). + * + * Real multi-NIC boxes expose a class of bug a single-NIC VM cannot: an + * initramfs that picks "the first connected interface" grabs one of these + * instead of the NIC that PXE booted, and then cannot reach the bastion. + * Defaults to 0 (single NIC). + */ + decoyNics?: number; + decoyNetwork?: string; } /** Create a blank UEFI VM that PXE boots from the network. */ @@ -61,6 +72,10 @@ export function createPxeVm(config: PxeVmConfig): void { `--memory=${config.memory}`, `--vcpus=${config.vcpus}`, `--disk=path=${diskPath},format=qcow2,bus=virtio`, + // Decoys first so they enumerate ahead of the PXE NIC. They are up and + // carry a lease, but have no route to the bastion. + ...Array.from({ length: config.decoyNics ?? 0 }, () => + `--network=network=${config.decoyNetwork ?? "default"},model=virtio`), `--network=network=${config.network},model=virtio`, // UEFI firmware — required for PXE boot in modern mode `--boot=uefi,network,hd`, @@ -95,12 +110,21 @@ export function destroyPxeVm(name: string): void { } /** Get the MAC address of a VM's first NIC. */ -export function getVmMac(name: string): string | null { +export function getVmMac(name: string, network?: string): string | null { const result = virsh("domiflist", name); if (result.status !== 0) return null; // Output format: Interface Type Source Model MAC - const match = result.stdout.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i); - return match ? match[1].toLowerCase() : null; + // With decoy NICs present, match the line for the PXE network so we return + // the NIC that actually boots rather than whichever is listed first. + const lines = result.stdout.split("\n"); + const candidates = network === undefined + ? lines + : lines.filter((l) => l.split(/\s+/).includes(network)); + for (const line of candidates.length > 0 ? candidates : lines) { + const m = line.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i); + if (m) return m[1].toLowerCase(); + } + return null; } /** Reboot a VM (force off + start). */ diff --git a/bastion/tests/integration/vyos-provision.test.ts b/bastion/tests/integration/vyos-provision.test.ts index a5700d2..c3aa5b7 100644 --- a/bastion/tests/integration/vyos-provision.test.ts +++ b/bastion/tests/integration/vyos-provision.test.ts @@ -255,14 +255,20 @@ describe("VyOS provisioning", () => { await sleep(1000); log("Creating PXE VM..."); + // Two decoy NICs ahead of the PXE NIC, on a network with no route to the + // bastion. This reproduces the real VP2440 topology: live-boot scans for + // "the first connected interface", and without BOOTIF it picks a decoy, + // times out on DHCP/fetch, and dies with "Unable to find a live file + // system on the network". A single-NIC VM cannot catch that. createPxeVm({ name: VM_NAME, memory: VM_MEMORY, vcpus: VM_VCPUS, diskSize: VM_DISK_GB, network: PXE_NETWORK_NAME, + decoyNics: 2, }); - const mac = getVmMac(VM_NAME); + const mac = getVmMac(VM_NAME, PXE_NETWORK_NAME); if (!mac) throw new Error("Could not determine VM MAC address"); vmMac = mac; log(`VM MAC: ${vmMac}`);