From df2dfc5d71d8271f2ffba25b698be214f391ea05 Mon Sep 17 00:00:00 2001 From: Michal Date: Wed, 12 Aug 2026 12:35:09 +0100 Subject: [PATCH] fix(bastion): pin the VyOS boot NIC by MAC, and detect pre-installer stalls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both Protectli VP2440s failed to install on real hardware: they fetched kernel+initrd and then went silent. The console showed why — Looking for a connected Ethernet interface ... e2 ? e3 ? e4 ? e5 ? Connected e4 found Connected e5 found [4.595647] igc 0000:02:00.0 e2: NIC Link is Up IP-Config: e4 ... no response after 15 secs - giving up Unable to find a live file system on the network live-boot picks the first *connected* interface. The i40e SFP+ pair links before the igc copper port (up at 4.6s), so it chose the fiber ports, which have no DHCP, and never tried the NIC that actually PXE booted. Fix: pass BOOTIF=01- on the kernel cmdline. live-boot's Device_from_bootif() (verified present in this image) matches it against /sys/class/net and sets DEVICE directly. The MAC comes from the dispatch key — i.e. exactly the NIC that PXE booted — which is more reliable than iPXE's ${net0} on a box where the booting NIC may not be net0. Why the integration test missed it: the VM had ONE NIC, so "first connected interface" was trivially correct, and virtio links instantly so there was no negotiation race. createPxeVm now takes decoyNics, attaching extra NICs ahead of the PXE NIC on a network with no route to the bastion; the VyOS test uses 2. Without BOOTIF that reproduces the hardware failure. getVmMac is network-aware so it still returns the booting NIC. Also: the bastion had every clue and said nothing — it logged INSTALL STARTED, served kernel+initrd, then nothing for 7 minutes. dispatch now stamps dispatched_at, and /api/logs/:mac returns stalled_for_s / stalled (8 min threshold, sized for the ~600MB squashfs fetch), so a machine wedged before the installer environment comes up is diagnosable without a console. Verified on hardware: both firewalls installed, bond0 802.3ad + VLANs 2/3/9/10/200 + VRRP (priority 200/100, VIP .254 per VLAN) applied, and /config/lab-provisioned written. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DMVzWZgiKW2wquf5z8S1yH --- bastion/src/bastion/src/routes/api.ts | 19 +++++++++++ bastion/src/bastion/src/routes/dispatch.ts | 7 ++++ .../bastion/src/templates/vyos-boot.ipxe.ts | 16 +++++++++- bastion/src/bastion/tests/vyos.test.ts | 32 +++++++++++++++++++ bastion/src/shared/src/types/state.ts | 7 ++++ bastion/tests/integration/helpers/pxe-vm.ts | 30 +++++++++++++++-- .../tests/integration/vyos-provision.test.ts | 8 ++++- 7 files changed, 114 insertions(+), 5 deletions(-) diff --git a/bastion/src/bastion/src/routes/api.ts b/bastion/src/bastion/src/routes/api.ts index 17f7bda..7398046 100644 --- a/bastion/src/bastion/src/routes/api.ts +++ b/bastion/src/bastion/src/routes/api.ts @@ -15,6 +15,13 @@ import type { ProgressEvent } from "../services/progress-events.js"; import type { InstallLogBuffer } from "../services/install-log.js"; import type { SyslogListener } from "../services/syslog-listener.js"; +/** + * Seconds after dispatch with zero progress before a machine is called stalled. + * Generous: the slowest legitimate gap is fetching a ~600MB VyOS squashfs over + * HTTP before the hook can report anything. + */ +const STALL_THRESHOLD_S = 8 * 60; + export function registerApiRoutes( app: FastifyInstance, state: StateManager, @@ -442,6 +449,15 @@ export function registerApiRoutes( const installedEntry = currentState.installed[mac]; if (queueEntry) { + // A machine that was handed an install script but has reported nothing + // since is wedged BEFORE the installer environment came up — a bad + // kernel/initrd, no network in the initramfs, or the wrong NIC picked. + // Surfacing it here is what makes that diagnosable without a console. + const since = queueEntry.progress_at ?? queueEntry.dispatched_at; + const stalledForS = since !== undefined && queueEntry.progress === undefined + ? Math.floor((Date.now() - new Date(since).getTime()) / 1000) + : 0; + return reply.send({ mac, hostname: queueEntry.hostname, @@ -449,6 +465,9 @@ export function registerApiRoutes( progress: queueEntry.progress ?? "queued", progress_detail: queueEntry.progress_detail ?? "", progress_at: queueEntry.progress_at ?? queueEntry.queued_at, + dispatched_at: queueEntry.dispatched_at, + stalled_for_s: stalledForS, + stalled: stalledForS > STALL_THRESHOLD_S, role: queueEntry.role, os: queueEntry.os, stages: queueEntry.log ?? [], diff --git a/bastion/src/bastion/src/routes/dispatch.ts b/bastion/src/bastion/src/routes/dispatch.ts index f4637eb..9ff1fdf 100644 --- a/bastion/src/bastion/src/routes/dispatch.ts +++ b/bastion/src/bastion/src/routes/dispatch.ts @@ -100,6 +100,13 @@ echo "===============================" const os = queueEntry.os ?? "fedora-43"; logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os})`); + // Stamp the handoff so a machine that boots the installer but never + // reports can be spotted without a console. + state.update((s) => { + const entry = s.install_queue[mac]; + if (entry) entry.dispatched_at = new Date().toISOString(); + }); + let script: string; if (os.startsWith("vyos")) { script = renderVyosInstallIpxe({ diff --git a/bastion/src/bastion/src/templates/vyos-boot.ipxe.ts b/bastion/src/bastion/src/templates/vyos-boot.ipxe.ts index 8dacf2b..c917544 100644 --- a/bastion/src/bastion/src/templates/vyos-boot.ipxe.ts +++ b/bastion/src/bastion/src/templates/vyos-boot.ipxe.ts @@ -18,6 +18,20 @@ export function renderVyosInstallIpxe(params: { }): string { const base = `http://${params.serverIp}:${params.httpPort}`; + // Pin the boot NIC by MAC. live-boot otherwise scans for the first + // *connected* interface, and on a multi-NIC box that race is lost by + // whichever port negotiates slowest: on the Protectli VP2440 the SFP+ + // pair links first, so live-boot picked the fiber ports (which have no + // DHCP), burned 15s per port, and gave up with "Unable to find a live + // file system on the network" -- while the copper port that actually PXE + // booted came up at 4.6s and was never tried. + // + // live-boot's Device_from_bootif() strips the "01-" and matches the MAC + // against /sys/class/net/*. params.mac is the dispatch key, i.e. exactly + // the NIC that PXE booted -- more reliable than iPXE's ${net0} on a box + // where the booting NIC may not be net0. + const bootif = `01-${params.mac.toLowerCase().replace(/:/g, "-")}`; + // Deliberately NOT passing `nonetworking` (present in VyOS's own PXE docs): // live-config's hook component needs networking up to fetch the hook over // HTTP. Also no `console=ttyS0` — on hardware without a physical UART that @@ -32,7 +46,7 @@ echo MAC: ${params.mac} echo ============================================= echo -kernel ${base}/vyos-vmlinuz boot=live nopersistence noautologin fetch=${base}/vyos-filesystem.squashfs live-config.hooks=${base}/vyos/autoinstall.sh?mac=${params.mac} +kernel ${base}/vyos-vmlinuz boot=live nopersistence noautologin BOOTIF=${bootif} fetch=${base}/vyos-filesystem.squashfs live-config.hooks=${base}/vyos/autoinstall.sh?mac=${params.mac} initrd ${base}/vyos-initrd boot `; diff --git a/bastion/src/bastion/tests/vyos.test.ts b/bastion/src/bastion/tests/vyos.test.ts index 02914e5..e0535a7 100644 --- a/bastion/src/bastion/tests/vyos.test.ts +++ b/bastion/src/bastion/tests/vyos.test.ts @@ -514,3 +514,35 @@ print(json.dumps(out)) } }); }); + +describe("vyos boot NIC pinning", () => { + it("pins the boot interface by MAC via BOOTIF", async () => { + // Without this, live-boot picks the first *connected* NIC. On the VP2440 + // the SFP+ pair links before the copper PXE port, so live-boot tried the + // fiber ports (no DHCP), timed out 15s each, and failed with "Unable to + // find a live file system on the network". + const testDir = join(tmpdir(), `bastion-vyos-bootif-${Date.now()}`); + mkdirSync(join(testDir, "http"), { recursive: true }); + mkdirSync(join(testDir, "tftp"), { recursive: true }); + const { app: a, state: st } = createApp(createTestConfig(testDir)); + try { + const m = "64:62:66:25:96:47"; + st.update((s) => { + s.install_queue[m] = { + hostname: "vyos001", disk: "/dev/mmcblk0", role: "vanilla", + os: "vyos-rolling", queued_at: new Date().toISOString(), + }; + }); + const res = await a.inject({ method: "GET", url: `/dispatch?mac=${m}` }); + // live-boot's Device_from_bootif() expects 01- + expect(res.body).toContain("BOOTIF=01-64-62-66-25-96-47"); + // and it must be on the kernel line, before fetch= is attempted + const kernelLine = res.body.split("\n").find((l) => l.startsWith("kernel ")); + expect(kernelLine).toContain("BOOTIF=01-64-62-66-25-96-47"); + expect(kernelLine).toContain("fetch="); + } finally { + await a.close(); + rmSync(testDir, { recursive: true, force: true }); + } + }); +}); diff --git a/bastion/src/shared/src/types/state.ts b/bastion/src/shared/src/types/state.ts index 0d96238..72f5ee0 100644 --- a/bastion/src/shared/src/types/state.ts +++ b/bastion/src/shared/src/types/state.ts @@ -151,6 +151,13 @@ export interface InstallConfig { vyos?: VyosInstallSpec; // only consulted when os is "vyos-rolling" arch?: Arch; // detected from HardwareInfo or overridden queued_at: string; + /** + * When dispatch last served this machine an install boot script. Progress + * callbacks only start once the installer environment is up, so a machine + * dispatched long ago with no progress is wedged before that point (bad + * kernel/initrd, no network in the initramfs, wrong NIC picked...). + */ + dispatched_at?: string; progress?: string; progress_at?: string; progress_detail?: string; diff --git a/bastion/tests/integration/helpers/pxe-vm.ts b/bastion/tests/integration/helpers/pxe-vm.ts index 58ae4e2..f5374e2 100644 --- a/bastion/tests/integration/helpers/pxe-vm.ts +++ b/bastion/tests/integration/helpers/pxe-vm.ts @@ -29,6 +29,17 @@ export interface PxeVmConfig { diskSize: number; // GB network: string; // libvirt network name arch?: "x86_64" | "aarch64"; + /** + * Extra NICs enumerated BEFORE the PXE NIC, on a network with no route to + * the bastion (defaults to libvirt's "default"). + * + * Real multi-NIC boxes expose a class of bug a single-NIC VM cannot: an + * initramfs that picks "the first connected interface" grabs one of these + * instead of the NIC that PXE booted, and then cannot reach the bastion. + * Defaults to 0 (single NIC). + */ + decoyNics?: number; + decoyNetwork?: string; } /** Create a blank UEFI VM that PXE boots from the network. */ @@ -61,6 +72,10 @@ export function createPxeVm(config: PxeVmConfig): void { `--memory=${config.memory}`, `--vcpus=${config.vcpus}`, `--disk=path=${diskPath},format=qcow2,bus=virtio`, + // Decoys first so they enumerate ahead of the PXE NIC. They are up and + // carry a lease, but have no route to the bastion. + ...Array.from({ length: config.decoyNics ?? 0 }, () => + `--network=network=${config.decoyNetwork ?? "default"},model=virtio`), `--network=network=${config.network},model=virtio`, // UEFI firmware — required for PXE boot in modern mode `--boot=uefi,network,hd`, @@ -95,12 +110,21 @@ export function destroyPxeVm(name: string): void { } /** Get the MAC address of a VM's first NIC. */ -export function getVmMac(name: string): string | null { +export function getVmMac(name: string, network?: string): string | null { const result = virsh("domiflist", name); if (result.status !== 0) return null; // Output format: Interface Type Source Model MAC - const match = result.stdout.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i); - return match ? match[1].toLowerCase() : null; + // With decoy NICs present, match the line for the PXE network so we return + // the NIC that actually boots rather than whichever is listed first. + const lines = result.stdout.split("\n"); + const candidates = network === undefined + ? lines + : lines.filter((l) => l.split(/\s+/).includes(network)); + for (const line of candidates.length > 0 ? candidates : lines) { + const m = line.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i); + if (m) return m[1].toLowerCase(); + } + return null; } /** Reboot a VM (force off + start). */ diff --git a/bastion/tests/integration/vyos-provision.test.ts b/bastion/tests/integration/vyos-provision.test.ts index a5700d2..c3aa5b7 100644 --- a/bastion/tests/integration/vyos-provision.test.ts +++ b/bastion/tests/integration/vyos-provision.test.ts @@ -255,14 +255,20 @@ describe("VyOS provisioning", () => { await sleep(1000); log("Creating PXE VM..."); + // Two decoy NICs ahead of the PXE NIC, on a network with no route to the + // bastion. This reproduces the real VP2440 topology: live-boot scans for + // "the first connected interface", and without BOOTIF it picks a decoy, + // times out on DHCP/fetch, and dies with "Unable to find a live file + // system on the network". A single-NIC VM cannot catch that. createPxeVm({ name: VM_NAME, memory: VM_MEMORY, vcpus: VM_VCPUS, diskSize: VM_DISK_GB, network: PXE_NETWORK_NAME, + decoyNics: 2, }); - const mac = getVmMac(VM_NAME); + const mac = getVmMac(VM_NAME, PXE_NETWORK_NAME); if (!mac) throw new Error("Could not determine VM MAC address"); vmMac = mac; log(`VM MAC: ${vmMac}`);