feat(bastion): bring VyOS provisioning to Fedora-grade quality

Ports the Fedora provisioning features that matter for a router onto the VyOS path, and adds the libvirt integration test that proves them. - Live install logs: the driver streams the installer pty (ANSI-stripped, batched, best-effort) to POST /api/log, so `labctl provision logs -f` works during a VyOS install the way Anaconda's syslog does for Fedora. - installed.ip: report "ready at <ip>" -- the exact detail format routes/api.ts parses -- using the static mgmt address when known, else the live DHCP address. Without it VyOS machines landed with an empty IP, breaking provision list, logs-by-IP, recheck and reprovision. api.ts also guards the complete handler: VyOS boxes get the "vyos" SSH hint and never trigger the k3s post-provision. - EFI network-first boot order: port of the Fedora %post efibootmgr step, run from the live env after install (NVRAM, not disk). Best-effort. - Reinstall semantics: VyOS's installer already carries the previous config and SSH host keys forward -- the analog of Fedora's LV preservation -- so that stays the default. New --vyos-fresh-config overwrites the installed config.boot with the generated one instead, via a post-install target mount that also writes /config/lab-provisioned (mirrors Fedora's /etc/lab-provisioned, survives image upgrades). - reprovision/recheck default to the "vyos" SSH user for VyOS machines. Two hangs found by the VM test and fixed: - On reinstall the installer asks "Would you like to copy data to the new image?" (search_previous_installation). Unanswered, the driver blocked on stdin until its stall timeout -- a silent 15-minute hang. - The RAID regex missed "Would you like to choose two disks for RAID-1 mirroring?", which would wedge any multi-disk box. Both prompts default to yes, so a miss also risks an unwanted mirror. Both are now covered by a unit test asserting all 17 installer prompts match exactly one rule -- verified to fail against the unfixed code, so this class of bug is caught in a second instead of a 45-minute VM run. tests/integration/vyos-provision.test.ts: fresh install, reinstall preserves config + /config data, and freshConfig override. All 8 pass against the real nightly ISO (EXIT=0). 273 unit tests pass; no new lint errors in touched files. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DMVzWZgiKW2wquf5z8S1yH
2026-08-11 11:16:25 +01:00
parent cb9d99dd69
commit 5d00c42f5a
11 changed files with 806 additions and 20 deletions
--- a/bastion/tests/integration/vyos-provision.test.ts
+++ b/bastion/tests/integration/vyos-provision.test.ts
@@ -0,0 +1,381 @@
+// Integration test: full VyOS unattended provisioning flow.
+//
+// Validates the VyOS install path end-to-end, at the same depth as the Fedora
+// pxe-provision test:
+//   1. Bastion (HTTP + dnsmasq) on the isolated libvirt PXE network
+//   2. Blank UEFI VM PXE boots -> Fedora-based discovery (OS-neutral)
+//   3. Queue os=vyos-rolling -> live boot + live-config hook + pty driver
+//   4. Fresh-install asserts: installed.ip, streamed logs, applied config,
+//      /config/lab-provisioned, boot-order handling
+//   5. REINSTALL round: previous config + /config data carried forward
+//      ("reinstall without losing data", VyOS-flavored)
+//   6. freshConfig round: bastion-generated config wins, /config data kept
+//
+// Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo, internet
+// (first run downloads the ~600MB VyOS nightly ISO; artifacts are cached).
+// Run: sudo pnpm run test:integration:vyos
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, symlinkSync, writeFileSync } from "node:fs";
+import { execSync } from "node:child_process";
+import { join } from "node:path";
+import { homedir, tmpdir } from "node:os";
+import { log, waitForSsh } from "./helpers/libvirt.js";
+import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js";
+import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm } from "./helpers/pxe-vm.js";
+import { sshExec } from "./helpers/ssh.js";
+
+const VM_NAME = "lab-vyos-test";
+const VM_MEMORY = 4096;
+const VM_VCPUS = 4;
+const VM_DISK_GB = 10;       // VyOS image install needs ~2GB minimum
+const HTTP_PORT = 8099;
+const SSH_USER = "vyos";     // the only VyOS login user
+const BASTION_IP = PXE_GATEWAY;
+const DHCP_RANGE_START = `${PXE_SUBNET}.100`;
+const DHCP_RANGE_END = `${PXE_SUBNET}.200`;
+
+const DISCOVERY_TIMEOUT_MS = 5 * 60_000;
+const INSTALL_TIMEOUT_MS = 15 * 60_000;  // squashfs fetch + copy; much faster than Anaconda
+const SSH_TIMEOUT_MS = 8 * 60_000;
+
+const HOSTNAME_R1 = "vyos-r1";
+const HOSTNAME_R2 = "vyos-r2";
+const HOSTNAME_R3 = "vyos-r3";
+
+function findSshKey(): { pubKey: string; keyPath: string } {
+  const homes = [homedir()];
+  const sudoUser = process.env["SUDO_USER"];
+  if (sudoUser) homes.push(join("/home", sudoUser));
+  if (process.env["SSH_KEY_PATH"]) {
+    const keyPath = process.env["SSH_KEY_PATH"];
+    const pubPath = `${keyPath}.pub`;
+    if (existsSync(keyPath) && existsSync(pubPath)) {
+      return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath };
+    }
+  }
+  for (const home of homes) {
+    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
+      const keyPath = join(home, ".ssh", name);
+      const pubPath = `${keyPath}.pub`;
+      if (existsSync(keyPath) && existsSync(pubPath)) {
+        return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath };
+      }
+    }
+  }
+  throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/");
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+async function pollApi<T>(
+  url: string,
+  check: (data: T) => boolean,
+  timeoutMs: number,
+  intervalMs = 5000,
+): Promise<T> {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    try {
+      const res = await fetch(url);
+      if (res.ok) {
+        const data = (await res.json()) as T;
+        if (check(data)) return data;
+      }
+    } catch { /* not ready yet */ }
+    await sleep(intervalMs);
+  }
+  throw new Error(`Timeout after ${timeoutMs}ms polling ${url}`);
+}
+
+type LogsResponse = {
+  status: string;
+  progress: string;
+  progress_detail?: string;
+  ip?: string;
+  log_total?: number;
+  log_lines?: Array<{ line: string }>;
+};
+
+/** Queue a VyOS install, reboot the VM into PXE, wait for completion + SSH. */
+async function installRound(opts: {
+  mac: string;
+  hostname: string;
+  freshConfig?: boolean;
+}): Promise<string> {
+  const body = {
+    mac: opts.mac,
+    hostname: opts.hostname,
+    disk: "/dev/vda",
+    role: "vanilla",
+    os: "vyos-rolling",
+    vyos: {
+      mgmtInterface: "eth0",
+      mgmtAddress: "dhcp",
+      hwIds: { eth0: opts.mac },
+      ...(opts.freshConfig ? { freshConfig: true } : {}),
+    },
+  };
+  const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/install`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+  });
+  log(`Install queued (${opts.hostname}): ${JSON.stringify(await res.json())}`);
+
+  await sleep(5_000);
+  rebootPxeVm(VM_NAME);
+  await sleep(3_000);
+  deleteNftablesRejectRules();
+
+  const finalState = await pollApi<LogsResponse>(
+    `http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(opts.mac)}`,
+    (data) => data.status === "installed" || data.progress === "error",
+    INSTALL_TIMEOUT_MS,
+    10_000,
+  );
+  if (finalState.progress === "error") {
+    log(`INSTALL FAILED: ${JSON.stringify(finalState.progress_detail ?? finalState, null, 2)}`);
+    throw new Error(`VyOS install failed for ${opts.hostname}`);
+  }
+  const ip = finalState.ip ?? "";
+  log(`Install complete (${opts.hostname}). IP: ${ip}`);
+
+  // The driver force-reboots; the VM PXE boots, dispatch says installed ->
+  // localboot exit -> GRUB -> VyOS. nftables reject rules do not reappear
+  // (guest reboot, not a libvirt restart), but clearing is harmless.
+  deleteNftablesRejectRules();
+  await waitForSsh(ip, SSH_USER, SSH_TIMEOUT_MS, sshKeyPathGlobal);
+  return ip;
+}
+
+let sshKeyPathGlobal = "";
+
+describe("VyOS provisioning", () => {
+  let bastionApp: { close: () => Promise<void> };
+  let testDir: string;
+  let vmMac: string;
+  let vmIp: string;
+
+  beforeAll(async () => {
+    const { pubKey, keyPath } = findSshKey();
+    sshKeyPathGlobal = keyPath;
+
+    log("Setting up PXE test network...");
+    ensurePxeNetwork();
+
+    testDir = join(tmpdir(), `lab-vyos-test-${Date.now()}`);
+    mkdirSync(join(testDir, "tftp"), { recursive: true });
+    mkdirSync(join(testDir, "http"), { recursive: true });
+    mkdirSync(join(testDir, "logs"), { recursive: true });
+
+    log("Starting bastion...");
+    const { createApp } = await import("../../src/bastion/src/server.js");
+    const { loadConfig } = await import("../../src/bastion/src/config.js");
+    const { generateDnsmasqConf, startDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
+    const { generateDiscoverKickstart } = await import("../../src/bastion/src/services/kickstart-generator.js");
+    const { renderBootIpxe } = await import("../../src/bastion/src/templates/boot.ipxe.js");
+    const { prepareVyosArtifacts } = await import("../../src/bastion/src/main.js");
+
+    const config = loadConfig({
+      bastionDir: testDir,
+      httpPort: HTTP_PORT,
+      iface: "virbr-pxe",
+      serverIp: BASTION_IP,
+      network: `${PXE_SUBNET}.0`,
+      gateway: BASTION_IP,
+      dhcpMode: "full",
+      dhcpRangeStart: DHCP_RANGE_START,
+      dhcpRangeEnd: DHCP_RANGE_END,
+      domain: "pxe-test.local",
+      sshKeys: [pubKey],
+      adminUser: "lab",
+    });
+
+    // iPXE binary
+    const ipxeSrc = "/usr/share/ipxe/ipxe-snponly-x86_64.efi";
+    if (!existsSync(ipxeSrc)) {
+      throw new Error(`iPXE not found: ${ipxeSrc}. Install: sudo dnf install ipxe-bootimgs-x86`);
+    }
+    copyFileSync(ipxeSrc, join(config.tftpDir, "ipxe.efi"));
+    try { symlinkSync(join(config.tftpDir, "ipxe.efi"), join(config.httpDir, "ipxe.efi")); } catch { /* exists */ }
+
+    const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache";
+    execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" });
+
+    // Fedora kernel+initrd for DISCOVERY (OS-neutral, same as pxe test)
+    const kernel = join(cacheDir, `vmlinuz-${config.fedoraVersion}`);
+    const initrd = join(cacheDir, `initrd-${config.fedoraVersion}.img`);
+    if (!existsSync(kernel)) {
+      log(`Downloading Fedora ${config.fedoraVersion} kernel (discovery)...`);
+      execSync(`curl -# -L -f -o "${kernel}" "${config.fedoraMirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 300_000 });
+    }
+    if (!existsSync(initrd)) {
+      log(`Downloading Fedora ${config.fedoraVersion} initrd (discovery)...`);
+      execSync(`curl -# -L -f -o "${initrd}" "${config.fedoraMirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 300_000 });
+    }
+    copyFileSync(kernel, join(config.httpDir, "vmlinuz"));
+    copyFileSync(initrd, join(config.httpDir, "initrd.img"));
+
+    // VyOS netboot artifacts — cache the three extracted files across runs
+    const vyosCache = {
+      kernel: join(cacheDir, "vyos-vmlinuz"),
+      initrd: join(cacheDir, "vyos-initrd"),
+      squashfs: join(cacheDir, "vyos-filesystem.squashfs"),
+    };
+    if (Object.values(vyosCache).every((p) => existsSync(p))) {
+      log("VyOS netboot artifacts cached");
+      copyFileSync(vyosCache.kernel, join(config.httpDir, "vyos-vmlinuz"));
+      copyFileSync(vyosCache.initrd, join(config.httpDir, "vyos-initrd"));
+      copyFileSync(vyosCache.squashfs, join(config.httpDir, "vyos-filesystem.squashfs"));
+    } else {
+      log("Extracting VyOS artifacts from ISO (downloads ~600MB on first run)...");
+      prepareVyosArtifacts(config);
+      copyFileSync(join(config.httpDir, "vyos-vmlinuz"), vyosCache.kernel);
+      copyFileSync(join(config.httpDir, "vyos-initrd"), vyosCache.initrd);
+      copyFileSync(join(config.httpDir, "vyos-filesystem.squashfs"), vyosCache.squashfs);
+    }
+
+    writeFileSync(join(config.httpDir, "discover.ks"), generateDiscoverKickstart(config));
+    writeFileSync(join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }));
+    generateDnsmasqConf(config);
+
+    const { app, syslog } = createApp(config);
+    bastionApp = app;
+    await app.listen({ port: config.httpPort, host: "0.0.0.0" });
+    syslog.start();
+    log(`Bastion listening on :${HTTP_PORT}`);
+
+    log("Starting dnsmasq...");
+    startDnsmasq(config).catch((err) => {
+      log(`dnsmasq failed (expected without root): ${err instanceof Error ? err.message : String(err)}`);
+    });
+    await sleep(1000);
+
+    log("Creating PXE VM...");
+    createPxeVm({
+      name: VM_NAME,
+      memory: VM_MEMORY,
+      vcpus: VM_VCPUS,
+      diskSize: VM_DISK_GB,
+      network: PXE_NETWORK_NAME,
+    });
+    const mac = getVmMac(VM_NAME);
+    if (!mac) throw new Error("Could not determine VM MAC address");
+    vmMac = mac;
+    log(`VM MAC: ${vmMac}`);
+
+    log("Waiting for discovery...");
+    type MachinesResponse = { discovered: Record<string, unknown> };
+    await pollApi<MachinesResponse>(
+      `http://${BASTION_IP}:${HTTP_PORT}/api/machines`,
+      (data) => vmMac in data.discovered,
+      DISCOVERY_TIMEOUT_MS,
+    );
+    log("VM discovered. Running fresh VyOS install (round 1)...");
+
+    await sleep(15_000); // discovery reboot cycle
+    vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R1 });
+    log("Round 1 (fresh install) complete.");
+  }, DISCOVERY_TIMEOUT_MS + INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000);
+
+  afterAll(async () => {
+    log("Cleaning up...");
+    if (bastionApp) await bastionApp.close().catch(() => {});
+    const { stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
+    stopDnsmasq();
+    destroyPxeVm(VM_NAME);
+    destroyPxeNetwork();
+    if (testDir) rmSync(testDir, { recursive: true, force: true });
+  });
+
+  it("machine is installed with a real IP (WI-1: ready-at parsing)", async () => {
+    const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`);
+    const data = (await res.json()) as { installed: Record<string, { ip: string; os?: string }> };
+    const machine = data.installed[vmMac];
+    expect(machine).toBeDefined();
+    expect(machine.ip).toMatch(/^\d+\.\d+\.\d+\.\d+$/);
+    expect(machine.os).toBe("vyos-rolling");
+  });
+
+  it("install logs were streamed live (WI-2)", async () => {
+    const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`);
+    const data = (await res.json()) as LogsResponse;
+    expect(data.log_total).toBeGreaterThan(0);
+    const lines = (data.log_lines ?? []).map((l) => l.line).join("\n");
+    // Installer transcript lines and driver messages both flow through /api/log
+    expect(lines).toMatch(/Welcome to VyOS installation|>>> answered|base config:/);
+  });
+
+  it("SSH works as the vyos user with the injected key", () => {
+    const result = sshExec(vmIp, SSH_USER, "whoami", { keyPath: sshKeyPathGlobal });
+    expect(result.exitCode).toBe(0);
+    expect(result.stdout.trim()).toBe("vyos");
+  });
+
+  it("generated config was adopted (hostname + ssh key)", () => {
+    const result = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal });
+    expect(result.exitCode).toBe(0);
+    expect(result.stdout).toContain(`host-name "${HOSTNAME_R1}"`);
+    expect(result.stdout).toContain("public-keys");
+  });
+
+  it("boot-order step ran and reported (WI-3)", async () => {
+    const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`);
+    const data = (await res.json()) as LogsResponse;
+    const lines = (data.log_lines ?? []).map((l) => l.line).join("\n");
+    expect(lines).toContain("boot order:");
+  });
+
+  it("provisioning metadata persisted to /config (WI-4)", () => {
+    const result = sshExec(vmIp, SSH_USER, "cat /config/lab-provisioned 2>/dev/null || cat /opt/vyatta/etc/config/lab-provisioned", { keyPath: sshKeyPathGlobal });
+    expect(result.exitCode).toBe(0);
+    expect(result.stdout).toContain(`hostname=${HOSTNAME_R1}`);
+    expect(result.stdout).toContain("role=vanilla");
+    expect(result.stdout).toContain(`bastion=http://${BASTION_IP}:${HTTP_PORT}`);
+  });
+
+  it("reinstall preserves config and /config data (round 2)", async () => {
+    // Drop a marker in /config — the installer's previous-installation copy
+    // must carry it (and the whole old config) into the new image.
+    // `sync` is REQUIRED: rebootPxeVm uses `virsh destroy` (a hard power-cut),
+    // so an unsynced write never reaches the disk and the marker vanishes for
+    // reasons that have nothing to do with the installer.
+    const marker = sshExec(vmIp, SSH_USER, "echo LAB-MARKER-R2 > /config/lab-marker && sync && cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
+    expect(marker.exitCode).toBe(0);
+    expect(marker.stdout).toContain("LAB-MARKER-R2");
+
+    // Queue with a DIFFERENT hostname: with preserve semantics the previous
+    // config must win, so the hostname must NOT change.
+    vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R2 });
+
+    // Assert the config carry-forward first — it is the primary preservation
+    // signal and does not depend on the marker mechanism above.
+    const cfg = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal });
+    expect(cfg.stdout).toContain(`host-name "${HOSTNAME_R1}"`);   // old config carried
+    expect(cfg.stdout).not.toContain(`host-name "${HOSTNAME_R2}"`);
+
+    const markerAfter = sshExec(vmIp, SSH_USER, "cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
+    expect(markerAfter.exitCode).toBe(0);
+    expect(markerAfter.stdout).toContain("LAB-MARKER-R2");
+  }, INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 60_000);
+
+  it("freshConfig makes the generated config win, data still kept (round 3)", async () => {
+    // Re-assert the marker is on disk and synced before the next power-cut.
+    const pre = sshExec(vmIp, SSH_USER, "sync && cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
+    expect(pre.stdout).toContain("LAB-MARKER-R2");
+
+    vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R3, freshConfig: true });
+
+    const cfg = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal });
+    expect(cfg.stdout).toContain(`host-name "${HOSTNAME_R3}"`);   // generated config won
+
+    // The marker file (non-config data under /config) still survives —
+    // freshConfig replaces only config.boot, not the carried data.
+    const markerAfter = sshExec(vmIp, SSH_USER, "cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
+    expect(markerAfter.exitCode).toBe(0);
+    expect(markerAfter.stdout).toContain("LAB-MARKER-R2");
+  }, INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 60_000);
+});