bastion: progress callbacks report the queue mac, not the first NIC

The kickstart/autoinstall progress helpers detected their own MAC at
runtime (first 'ether' line of ip link). On a multi-NIC board the first
NIC need not be the PXE NIC the bastion keyed the install under, so the
'complete' callback arrived under an unknown mac and was a silent no-op:
the install_queue entry survived, every reboot re-dispatched the
installer, and the machine reinstalled in an endless loop. Observed on
thebeast (2 onboard NICs) 2026-10-20; the single-NIC VM drill could not
have caught it.

The per-MAC /ks and /autoinstall routes already know the queue mac --
bake it into the generated script and keep runtime detection only as the
fallback when no mac is known.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DceLLgGmyS7zTGLaLNjQBd

.gitea/workflows/ci.yml

@@ -160,14 +160,14 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: rpm-packages
-          path: bastion/dist/lab-*.rpm
+          path: bastion/dist/labctl-*.rpm
           retention-days: 7
 
       - name: Upload DEB artifacts
         uses: actions/upload-artifact@v3
         with:
           name: deb-packages
-          path: bastion/dist/lab*.deb
+          path: bastion/dist/labctl*.deb
           retention-days: 7
 
   # -- Release pipeline (main branch push only) --------------
@@ -198,7 +198,7 @@ jobs:
           GITEA_OWNER: ${{ env.GITEA_OWNER }}
           GITEA_REPO: lab
         run: |
-          for RPM_FILE in dist/lab-*.rpm; do
+          for RPM_FILE in dist/labctl-*.rpm; do
             [ -f "$RPM_FILE" ] || continue
             RPM_VERSION=$(rpm -qp --queryformat '%{VERSION}-%{RELEASE}' "$RPM_FILE")
             RPM_ARCH=$(rpm -qp --queryformat '%{ARCH}' "$RPM_FILE")
@@ -255,7 +255,7 @@ jobs:
           # Publish to each supported distribution
           DISTRIBUTIONS="trixie forky noble plucky"
 
-          for DEB_FILE in dist/lab*.deb; do
+          for DEB_FILE in dist/labctl*.deb; do
             [ -f "$DEB_FILE" ] || continue
             DEB_VERSION=$(dpkg-deb --field "$DEB_FILE" Version)
             DEB_ARCH=$(dpkg-deb --field "$DEB_FILE" Architecture)

bastion/Dockerfile.bastion

@@ -29,11 +29,13 @@ COPY src/modules/modules/ src/modules/modules/
 RUN pnpm build
 
 # ── Stage 1b: Build iPXE snp.efi (uses UEFI SNP protocol for ISO boot) ──
-FROM fedora:43 AS ipxe-builder
+# Pinned to the build platform: this stage cross-compiles BOTH arch outputs
+# itself, and the iPXE toolchain does not survive qemu-user emulation.
+FROM --platform=$BUILDPLATFORM fedora:43 AS ipxe-builder
 
 RUN dnf install -y git gcc make perl-interpreter xz-devel gcc-aarch64-linux-gnu && dnf clean all
 RUN git clone --depth=1 https://github.com/ipxe/ipxe.git /tmp/ipxe
-RUN cd /tmp/ipxe/src && make bin-x86_64-efi/snp.efi && \
+RUN cd /tmp/ipxe/src && make bin-x86_64-efi/snp.efi bin-x86_64-efi/ipxe.efi && \
     make CROSS_COMPILE=aarch64-linux-gnu- bin-arm64-efi/snp.efi
 
 # ── Stage 2: Production runtime (Fedora -- needs dnsmasq) ───────
@@ -56,6 +58,15 @@ RUN dnf install -y \
 # boot from CD-ROM/USB -- it requires PXE chainloading)
 COPY --from=ipxe-builder /tmp/ipxe/src/bin-x86_64-efi/snp.efi /usr/share/ipxe/ipxe-snp-x86_64.efi
 COPY --from=ipxe-builder /tmp/ipxe/src/bin-arm64-efi/snp.efi /usr/share/ipxe/arm64-efi/ipxe-snp.efi
+# Full iPXE build (native NIC drivers + snp + nii fallback), served as the
+# default UEFI x86_64 PXE binary. snponly rides the firmware's NII/UNDI stack,
+# and on some NICs that stack cannot RECEIVE UNICAST frames: DHCP (broadcast)
+# works, then every ARP reply is lost and HTTP times out. Observed on
+# thebeast's onboard NIC 2026-10-19 (tcpdump: labmaster answered every ARP
+# request within microseconds, iPXE kept re-asking). A native driver bypasses
+# the firmware stack entirely; NICs without one still fall back to snp/nii,
+# so machines that worked before are unaffected.
+COPY --from=ipxe-builder /tmp/ipxe/src/bin-x86_64-efi/ipxe.efi /usr/share/ipxe/ipxe-full-x86_64.efi
 
 # Install pnpm
 RUN npm install -g pnpm@9

bastion/src/bastion/src/main.ts

@@ -16,16 +16,19 @@ import { BastionConnection } from "./services/labd-connection.js";
 import { progressBus } from "./services/progress-events.js";
 import { ensureBootIso } from "./routes/boot-iso.js";
 
-function copyIfMissing(src: string, dest: string, label: string): void {
-  if (existsSync(dest)) {
-    logger.info(`  ${label} -- cached`);
-    return;
-  }
+// For binaries that ship WITH the image: the data dir is a persistent volume,
+// so a copy-if-missing would keep serving whatever build landed there first,
+// forever, across image upgrades. Compare content and replace on change.
+function copyIfChanged(src: string, dest: string, label: string): void {
   if (!existsSync(src)) {
     throw new Error(`${label}: source not found at ${src}`);
   }
+  if (existsSync(dest) && readFileSync(dest).equals(readFileSync(src))) {
+    logger.info(`  ${label} -- up to date`);
+    return;
+  }
   copyFileSync(src, dest);
-  logger.info(`  ${label} -- copied from ${src}`);
+  logger.info(`  ${label} -- installed from ${src}`);
 }
 
 function download(url: string, dest: string, label: string): void {
@@ -254,18 +257,21 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
   if (config.skipArtifacts !== true) {
     logger.info(`Preparing boot artifacts (Fedora ${config.fedoraVersion} ${config.arch})...`);
 
-    copyIfMissing(
+    copyIfChanged(
       "/usr/share/ipxe/undionly.kpxe",
       `${config.tftpDir}/undionly.kpxe`,
       "iPXE BIOS",
     );
-    copyIfMissing(
-      "/usr/share/ipxe/ipxe-snponly-x86_64.efi",
+    // Full build (native drivers + snp/nii fallback), NOT snponly: the
+    // firmware NII stack snponly rides cannot receive unicast on some NICs
+    // (thebeast 2026-10-19: DHCP fine, every ARP reply lost, HTTP timeout).
+    copyIfChanged(
+      "/usr/share/ipxe/ipxe-full-x86_64.efi",
       `${config.tftpDir}/ipxe.efi`,
       "iPXE UEFI x86_64",
     );
     try {
-      copyIfMissing(
+      copyIfChanged(
         "/usr/share/ipxe/arm64-efi/snponly.efi",
         `${config.tftpDir}/ipxe-arm64.efi`,
         "iPXE UEFI arm64",

bastion/src/bastion/src/routes/kickstart.ts

@@ -30,6 +30,9 @@ export function registerKickstartRoutes(
       hostname: queueEntry?.hostname ?? "lab-node",
       disk: queueEntry?.disk ?? "",
       role: queueEntry?.role ?? "worker",
+      // Progress callbacks must report the mac this install is QUEUED under,
+      // not whatever NIC enumerates first on the installed system.
+      ...(mac !== "" ? { mac } : {}),
       ...(queueEntry?.schemaVersion ? { schemaVersion: queueEntry.schemaVersion } : {}),
       ...(queueEntry?.dataDisks ? { dataDisks: queueEntry.dataDisks } : {}),
     });
@@ -197,6 +200,7 @@ esac
       hostname: queueEntry?.hostname ?? "lab-node",
       disk: queueEntry?.disk ?? "",
       role: queueEntry?.role ?? "worker",
+      ...(mac !== "" ? { mac } : {}),
       domain: config.domain,
       ubuntuVersion: config.ubuntuVersion,
       timezone: config.timezone,

bastion/src/bastion/src/services/kickstart-generator.ts

@@ -24,6 +24,8 @@ export function generateInstallKickstart(
     hostname: string;
     disk: string;
     role: Role;
+    /** PXE MAC the install is keyed under; baked into progress callbacks. */
+    mac?: string;
     schemaVersion?: 1 | 2;
     dataDisks?: import("@lab/shared").DataDisks;
   },
@@ -32,6 +34,7 @@ export function generateInstallKickstart(
     hostname: params.hostname,
     disk: params.disk,
     role: params.role,
+    ...(params.mac !== undefined && params.mac !== "" ? { mac: params.mac } : {}),
     ...(params.schemaVersion ? { schemaVersion: params.schemaVersion } : {}),
     ...(params.dataDisks ? { dataDisks: params.dataDisks } : {}),
     domain: config.domain,

bastion/src/bastion/src/templates/install.ks.ts

@@ -12,6 +12,15 @@ export interface InstallKickstartParams {
   hostname: string;
   disk: string;
   role: Role;
+  /**
+   * The PXE MAC the bastion keys this install under (install_queue /
+   * installed). When set, progress callbacks report THIS mac verbatim.
+   * Runtime detection (`ip link ... exit` = first NIC) is only a fallback:
+   * on a multi-NIC board the first NIC need not be the PXE NIC, and a
+   * completion reported under the wrong mac is a silent no-op — the queue
+   * entry survives and the machine reinstalls forever (thebeast 2026-10-20).
+   */
+  mac?: string;
   domain: string;
   fedoraVersion: string;
   timezone: string;
@@ -44,6 +53,7 @@ export function renderInstallKickstart(params: InstallKickstartParams): string {
     hostname,
     disk,
     role,
+    mac,
     domain,
     fedoraVersion,
     timezone,
@@ -55,6 +65,11 @@ export function renderInstallKickstart(params: InstallKickstartParams): string {
     adminUser,
   } = params;
 
+  // Shell line defining `mac` inside the bastion_progress helpers.
+  const macLine = mac !== undefined && mac !== ""
+    ? `local mac="${mac}"`
+    : `local mac=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')`;
+
   const fqdn = domain ? `${hostname}.${domain}` : hostname;
   const vg = "labvg";
   const now = new Date().toISOString();
@@ -335,7 +350,7 @@ set -x
 # Progress callback helper
 bastion_progress() {
     local stage="$1" detail="\${2:-}"
-    local mac=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')
+    ${macLine}
     curl -sf -X POST "http://${serverIp}:${httpPort}/api/progress" \\
         -H "Content-Type: application/json" \\
         -d "{\\"mac\\":\\"$mac\\",\\"stage\\":\\"$stage\\",\\"detail\\":\\"$detail\\"}" 2>/dev/null || true
@@ -553,7 +568,7 @@ set -x
 # Progress callback helper
 bastion_progress() {
     local stage="$1" detail="\${2:-}"
-    local mac=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')
+    ${macLine}
     curl -sf -X POST "http://${serverIp}:${httpPort}/api/progress" \\
         -H "Content-Type: application/json" \\
         -d "{\\"mac\\":\\"$mac\\",\\"stage\\":\\"$stage\\",\\"detail\\":\\"$detail\\"}" 2>/dev/null || true

bastion/src/bastion/src/templates/ubuntu-autoinstall.ts

@@ -9,6 +9,8 @@ export interface UbuntuAutoinstallParams {
   hostname: string;
   disk: string;
   role: Role;
+  /** PXE MAC the install is keyed under; baked into the progress callback. */
+  mac?: string;
   /** Fresh-install layout. Defaults to 1 so existing callers are unchanged. */
   schemaVersion?: 1 | 2;
   domain: string;
@@ -26,6 +28,7 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
     hostname,
     disk,
     role,
+    mac,
     domain,
     timezone,
     serverIp,
@@ -34,6 +37,12 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
     adminUser,
   } = params;
 
+  // MAC reported by the completion callback: the queue key when known;
+  // first-NIC runtime detection only as fallback (wrong on multi-NIC boards).
+  const reportMac = mac !== undefined && mac !== ""
+    ? mac
+    : `$(ip link show | awk "/ether/ && !/00:00:00:00/ {print \\$2; exit}")`;
+
   const fqdn = domain ? `${hostname}.${domain}` : hostname;
   const vg = "labvg";
   // ---------------------------------------------------------------------------
@@ -175,7 +184,7 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
 
   // Progress callback (complete)
   lateCommands.push(
-    `curtin in-target -- bash -c 'IP_ADDR=$(ip -4 addr show | awk "/inet / && !/127.0.0/ {split(\\$2,a,\\"/\\"); print a[1]; exit}"); curl -sf -X POST "http://${serverIp}:${httpPort}/api/progress" -H "Content-Type: application/json" -d "{\\"mac\\":\\"$(ip link show | awk "/ether/ && !/00:00:00:00/ {print \\$2; exit}")\\",\\"stage\\":\\"complete\\",\\"detail\\":\\"ready at $IP_ADDR\\"}" || true'`,
+    `curtin in-target -- bash -c 'IP_ADDR=$(ip -4 addr show | awk "/inet / && !/127.0.0/ {split(\\$2,a,\\"/\\"); print a[1]; exit}"); curl -sf -X POST "http://${serverIp}:${httpPort}/api/progress" -H "Content-Type: application/json" -d "{\\"mac\\":\\"${reportMac}\\",\\"stage\\":\\"complete\\",\\"detail\\":\\"ready at $IP_ADDR\\"}" || true'`,
   );
 
   // JSON.stringify, not `"${c}"`. JSON is a subset of YAML, so this produces a

bastion/src/bastion/tests/kickstart.test.ts

@@ -215,3 +215,21 @@ describe("renderInstallKickstart", () => {
     expect(ks).not.toContain("ttyS0");
   });
 });
+
+describe("progress callback mac", () => {
+  it("bakes the queue mac into every bastion_progress helper when given", () => {
+    const ks = renderInstallKickstart(baseParams({ mac: "34:97:f6:32:76:e9" }));
+    const baked = ks.match(/local mac="34:97:f6:32:76:e9"/g) ?? [];
+    expect(baked.length).toBeGreaterThanOrEqual(2); // %pre and %post helpers
+    // No runtime first-NIC detection left: on multi-NIC boards the first NIC
+    // need not be the PXE NIC, and a completion under the wrong mac is a
+    // silent no-op that reinstalls the machine forever.
+    expect(ks).not.toContain("local mac=$(ip link show");
+  });
+
+  it("falls back to runtime detection when no mac is known", () => {
+    const ks = renderInstallKickstart(baseParams());
+    expect(ks).toContain("local mac=$(ip link show");
+    expect(ks).not.toContain('local mac=""');
+  });
+});

bastion/src/bastion/tests/ubuntu-autoinstall.test.ts

@@ -182,3 +182,16 @@ describe("Ubuntu reinstall is destructive (known gap)", () => {
     expect(cfg.find((c) => c["type"] === "disk")?.["wipe"]).toBe("superblock-recursive");
   });
 });
+
+describe("progress callback mac", () => {
+  it("bakes the queue mac into the completion callback when given", () => {
+    const doc = renderUbuntuAutoinstall({ ...base, role: "worker", mac: "aa:bb:cc:dd:ee:ff" });
+    expect(doc).toContain("aa:bb:cc:dd:ee:ff");
+    expect(doc).not.toContain("ip link show");
+  });
+
+  it("falls back to runtime detection when no mac is known", () => {
+    const doc = renderUbuntuAutoinstall({ ...base, role: "worker" });
+    expect(doc).toContain("ip link show");
+  });
+});