lab/bastion/tests/integration/vyos-provision.test.ts

// Integration test: full VyOS unattended provisioning flow.
//
// Validates the VyOS install path end-to-end, at the same depth as the Fedora
// pxe-provision test:
//   1. Bastion (HTTP + dnsmasq) on the isolated libvirt PXE network
//   2. Blank UEFI VM PXE boots -> Fedora-based discovery (OS-neutral)
//   3. Queue os=vyos-rolling -> live boot + live-config hook + pty driver
//   4. Fresh-install asserts: installed.ip, streamed logs, applied config,
//      /config/lab-provisioned, boot-order handling
//   5. REINSTALL round: previous config + /config data carried forward
//      ("reinstall without losing data", VyOS-flavored)
//   6. freshConfig round: bastion-generated config wins, /config data kept
//
// Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo, internet
// (first run downloads the ~600MB VyOS nightly ISO; artifacts are cached).
// Run: sudo pnpm run test:integration:vyos

import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, symlinkSync, writeFileSync } from "node:fs";
import { execSync } from "node:child_process";
import { join } from "node:path";
import { homedir, tmpdir } from "node:os";
import { log, waitForSsh } from "./helpers/libvirt.js";
import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js";
import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm } from "./helpers/pxe-vm.js";
import { sshExec } from "./helpers/ssh.js";

const VM_NAME = "lab-vyos-test";
const VM_MEMORY = 4096;
const VM_VCPUS = 4;
const VM_DISK_GB = 10;       // VyOS image install needs ~2GB minimum
const HTTP_PORT = 8099;
const SSH_USER = "vyos";     // the only VyOS login user
const BASTION_IP = PXE_GATEWAY;
const DHCP_RANGE_START = `${PXE_SUBNET}.100`;
const DHCP_RANGE_END = `${PXE_SUBNET}.200`;

const DISCOVERY_TIMEOUT_MS = 5 * 60_000;
const INSTALL_TIMEOUT_MS = 15 * 60_000;  // squashfs fetch + copy; much faster than Anaconda
const SSH_TIMEOUT_MS = 8 * 60_000;

const HOSTNAME_R1 = "vyos-r1";
const HOSTNAME_R2 = "vyos-r2";
const HOSTNAME_R3 = "vyos-r3";

function findSshKey(): { pubKey: string; keyPath: string } {
  const homes = [homedir()];
  const sudoUser = process.env["SUDO_USER"];
  if (sudoUser) homes.push(join("/home", sudoUser));
  if (process.env["SSH_KEY_PATH"]) {
    const keyPath = process.env["SSH_KEY_PATH"];
    const pubPath = `${keyPath}.pub`;
    if (existsSync(keyPath) && existsSync(pubPath)) {
      return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath };
    }
  }
  for (const home of homes) {
    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
      const keyPath = join(home, ".ssh", name);
      const pubPath = `${keyPath}.pub`;
      if (existsSync(keyPath) && existsSync(pubPath)) {
        return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath };
      }
    }
  }
  throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/");
}

function sleep(ms: number): Promise<void> {
  return new Promise((r) => setTimeout(r, ms));
}

async function pollApi<T>(
  url: string,
  check: (data: T) => boolean,
  timeoutMs: number,
  intervalMs = 5000,
): Promise<T> {
  const start = Date.now();
  while (Date.now() - start < timeoutMs) {
    try {
      const res = await fetch(url);
      if (res.ok) {
        const data = (await res.json()) as T;
        if (check(data)) return data;
      }
    } catch { /* not ready yet */ }
    await sleep(intervalMs);
  }
  throw new Error(`Timeout after ${timeoutMs}ms polling ${url}`);
}

type LogsResponse = {
  status: string;
  progress: string;
  progress_detail?: string;
  ip?: string;
  log_total?: number;
  log_lines?: Array<{ line: string }>;
};

/** Queue a VyOS install, reboot the VM into PXE, wait for completion + SSH. */
async function installRound(opts: {
  mac: string;
  hostname: string;
  freshConfig?: boolean;
}): Promise<string> {
  const body = {
    mac: opts.mac,
    hostname: opts.hostname,
    disk: "/dev/vda",
    role: "vanilla",
    os: "vyos-rolling",
    vyos: {
      mgmtInterface: "eth0",
      mgmtAddress: "dhcp",
      hwIds: { eth0: opts.mac },
      ...(opts.freshConfig ? { freshConfig: true } : {}),
    },
  };
  const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/install`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify(body),
  });
  log(`Install queued (${opts.hostname}): ${JSON.stringify(await res.json())}`);

  await sleep(5_000);
  rebootPxeVm(VM_NAME);
  await sleep(3_000);
  deleteNftablesRejectRules();

  const finalState = await pollApi<LogsResponse>(
    `http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(opts.mac)}`,
    (data) => data.status === "installed" || data.progress === "error",
    INSTALL_TIMEOUT_MS,
    10_000,
  );
  if (finalState.progress === "error") {
    log(`INSTALL FAILED: ${JSON.stringify(finalState.progress_detail ?? finalState, null, 2)}`);
    throw new Error(`VyOS install failed for ${opts.hostname}`);
  }
  const ip = finalState.ip ?? "";
  log(`Install complete (${opts.hostname}). IP: ${ip}`);

  // The driver force-reboots; the VM PXE boots, dispatch says installed ->
  // localboot exit -> GRUB -> VyOS. nftables reject rules do not reappear
  // (guest reboot, not a libvirt restart), but clearing is harmless.
  deleteNftablesRejectRules();
  await waitForSsh(ip, SSH_USER, SSH_TIMEOUT_MS, sshKeyPathGlobal);
  return ip;
}

let sshKeyPathGlobal = "";

describe("VyOS provisioning", () => {
  let bastionApp: { close: () => Promise<void> };
  let testDir: string;
  let vmMac: string;
  let vmIp: string;

  beforeAll(async () => {
    const { pubKey, keyPath } = findSshKey();
    sshKeyPathGlobal = keyPath;

    log("Setting up PXE test network...");
    ensurePxeNetwork();

    testDir = join(tmpdir(), `lab-vyos-test-${Date.now()}`);
    mkdirSync(join(testDir, "tftp"), { recursive: true });
    mkdirSync(join(testDir, "http"), { recursive: true });
    mkdirSync(join(testDir, "logs"), { recursive: true });

    log("Starting bastion...");
    const { createApp } = await import("../../src/bastion/src/server.js");
    const { loadConfig } = await import("../../src/bastion/src/config.js");
    const { generateDnsmasqConf, startDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
    const { generateDiscoverKickstart } = await import("../../src/bastion/src/services/kickstart-generator.js");
    const { renderBootIpxe } = await import("../../src/bastion/src/templates/boot.ipxe.js");
    const { prepareVyosArtifacts } = await import("../../src/bastion/src/main.js");

    const config = loadConfig({
      bastionDir: testDir,
      httpPort: HTTP_PORT,
      iface: "virbr-pxe",
      serverIp: BASTION_IP,
      network: `${PXE_SUBNET}.0`,
      gateway: BASTION_IP,
      dhcpMode: "full",
      dhcpRangeStart: DHCP_RANGE_START,
      dhcpRangeEnd: DHCP_RANGE_END,
      domain: "pxe-test.local",
      sshKeys: [pubKey],
      adminUser: "lab",
    });

    // iPXE binary
    const ipxeSrc = "/usr/share/ipxe/ipxe-snponly-x86_64.efi";
    if (!existsSync(ipxeSrc)) {
      throw new Error(`iPXE not found: ${ipxeSrc}. Install: sudo dnf install ipxe-bootimgs-x86`);
    }
    copyFileSync(ipxeSrc, join(config.tftpDir, "ipxe.efi"));
    try { symlinkSync(join(config.tftpDir, "ipxe.efi"), join(config.httpDir, "ipxe.efi")); } catch { /* exists */ }

    const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache";
    execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" });

    // Fedora kernel+initrd for DISCOVERY (OS-neutral, same as pxe test)
    const kernel = join(cacheDir, `vmlinuz-${config.fedoraVersion}`);
    const initrd = join(cacheDir, `initrd-${config.fedoraVersion}.img`);
    if (!existsSync(kernel)) {
      log(`Downloading Fedora ${config.fedoraVersion} kernel (discovery)...`);
      execSync(`curl -# -L -f -o "${kernel}" "${config.fedoraMirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 300_000 });
    }
    if (!existsSync(initrd)) {
      log(`Downloading Fedora ${config.fedoraVersion} initrd (discovery)...`);
      execSync(`curl -# -L -f -o "${initrd}" "${config.fedoraMirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 300_000 });
    }
    copyFileSync(kernel, join(config.httpDir, "vmlinuz"));
    copyFileSync(initrd, join(config.httpDir, "initrd.img"));

    // VyOS netboot artifacts — cache the three extracted files across runs
    const vyosCache = {
      kernel: join(cacheDir, "vyos-vmlinuz"),
      initrd: join(cacheDir, "vyos-initrd"),
      squashfs: join(cacheDir, "vyos-filesystem.squashfs"),
    };
    if (Object.values(vyosCache).every((p) => existsSync(p))) {
      log("VyOS netboot artifacts cached");
      copyFileSync(vyosCache.kernel, join(config.httpDir, "vyos-vmlinuz"));
      copyFileSync(vyosCache.initrd, join(config.httpDir, "vyos-initrd"));
      copyFileSync(vyosCache.squashfs, join(config.httpDir, "vyos-filesystem.squashfs"));
    } else {
      log("Extracting VyOS artifacts from ISO (downloads ~600MB on first run)...");
      prepareVyosArtifacts(config);
      copyFileSync(join(config.httpDir, "vyos-vmlinuz"), vyosCache.kernel);
      copyFileSync(join(config.httpDir, "vyos-initrd"), vyosCache.initrd);
      copyFileSync(join(config.httpDir, "vyos-filesystem.squashfs"), vyosCache.squashfs);
    }

    writeFileSync(join(config.httpDir, "discover.ks"), generateDiscoverKickstart(config));
    writeFileSync(join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }));
    generateDnsmasqConf(config);

    const { app, syslog } = createApp(config);
    bastionApp = app;
    await app.listen({ port: config.httpPort, host: "0.0.0.0" });
    syslog.start();
    log(`Bastion listening on :${HTTP_PORT}`);

    log("Starting dnsmasq...");
    startDnsmasq(config).catch((err) => {
      log(`dnsmasq failed (expected without root): ${err instanceof Error ? err.message : String(err)}`);
    });
    await sleep(1000);

    log("Creating PXE VM...");
    // Two decoy NICs ahead of the PXE NIC, on a network with no route to the
    // bastion. This reproduces the real VP2440 topology: live-boot scans for
    // "the first connected interface", and without BOOTIF it picks a decoy,
    // times out on DHCP/fetch, and dies with "Unable to find a live file
    // system on the network". A single-NIC VM cannot catch that.
    createPxeVm({
      name: VM_NAME,
      memory: VM_MEMORY,
      vcpus: VM_VCPUS,
      diskSize: VM_DISK_GB,
      network: PXE_NETWORK_NAME,
      decoyNics: 2,
    });
    const mac = getVmMac(VM_NAME, PXE_NETWORK_NAME);
    if (!mac) throw new Error("Could not determine VM MAC address");
    vmMac = mac;
    log(`VM MAC: ${vmMac}`);

    log("Waiting for discovery...");
    type MachinesResponse = { discovered: Record<string, unknown> };
    await pollApi<MachinesResponse>(
      `http://${BASTION_IP}:${HTTP_PORT}/api/machines`,
      (data) => vmMac in data.discovered,
      DISCOVERY_TIMEOUT_MS,
    );
    log("VM discovered. Running fresh VyOS install (round 1)...");

    await sleep(15_000); // discovery reboot cycle
    vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R1 });
    log("Round 1 (fresh install) complete.");
  }, DISCOVERY_TIMEOUT_MS + INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000);

  afterAll(async () => {
    log("Cleaning up...");
    if (bastionApp) await bastionApp.close().catch(() => {});
    const { stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
    stopDnsmasq();
    destroyPxeVm(VM_NAME);
    destroyPxeNetwork();
    if (testDir) rmSync(testDir, { recursive: true, force: true });
  });

  it("machine is installed with a real IP (WI-1: ready-at parsing)", async () => {
    const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`);
    const data = (await res.json()) as { installed: Record<string, { ip: string; os?: string }> };
    const machine = data.installed[vmMac];
    expect(machine).toBeDefined();
    expect(machine.ip).toMatch(/^\d+\.\d+\.\d+\.\d+$/);
    expect(machine.os).toBe("vyos-rolling");
  });

  it("install logs were streamed live (WI-2)", async () => {
    const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`);
    const data = (await res.json()) as LogsResponse;
    expect(data.log_total).toBeGreaterThan(0);
    const lines = (data.log_lines ?? []).map((l) => l.line).join("\n");
    // Installer transcript lines and driver messages both flow through /api/log
    expect(lines).toMatch(/Welcome to VyOS installation|>>> answered|base config:/);
  });

  it("SSH works as the vyos user with the injected key", () => {
    const result = sshExec(vmIp, SSH_USER, "whoami", { keyPath: sshKeyPathGlobal });
    expect(result.exitCode).toBe(0);
    expect(result.stdout.trim()).toBe("vyos");
  });

  it("generated config was adopted (hostname + ssh key)", () => {
    const result = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal });
    expect(result.exitCode).toBe(0);
    expect(result.stdout).toContain(`host-name "${HOSTNAME_R1}"`);
    expect(result.stdout).toContain("public-keys");
  });

  it("boot-order step ran and reported (WI-3)", async () => {
    const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`);
    const data = (await res.json()) as LogsResponse;
    const lines = (data.log_lines ?? []).map((l) => l.line).join("\n");
    expect(lines).toContain("boot order:");
  });

  it("provisioning metadata persisted to /config (WI-4)", () => {
    const result = sshExec(vmIp, SSH_USER, "cat /config/lab-provisioned 2>/dev/null || cat /opt/vyatta/etc/config/lab-provisioned", { keyPath: sshKeyPathGlobal });
    expect(result.exitCode).toBe(0);
    expect(result.stdout).toContain(`hostname=${HOSTNAME_R1}`);
    expect(result.stdout).toContain("role=vanilla");
    expect(result.stdout).toContain(`bastion=http://${BASTION_IP}:${HTTP_PORT}`);
  });

  it("reinstall preserves config and /config data (round 2)", async () => {
    // Drop a marker in /config — the installer's previous-installation copy
    // must carry it (and the whole old config) into the new image.
    // `sync` is REQUIRED: rebootPxeVm uses `virsh destroy` (a hard power-cut),
    // so an unsynced write never reaches the disk and the marker vanishes for
    // reasons that have nothing to do with the installer.
    const marker = sshExec(vmIp, SSH_USER, "echo LAB-MARKER-R2 > /config/lab-marker && sync && cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
    expect(marker.exitCode).toBe(0);
    expect(marker.stdout).toContain("LAB-MARKER-R2");

    // Queue with a DIFFERENT hostname: with preserve semantics the previous
    // config must win, so the hostname must NOT change.
    vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R2 });

    // Assert the config carry-forward first — it is the primary preservation
    // signal and does not depend on the marker mechanism above.
    const cfg = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal });
    expect(cfg.stdout).toContain(`host-name "${HOSTNAME_R1}"`);   // old config carried
    expect(cfg.stdout).not.toContain(`host-name "${HOSTNAME_R2}"`);

    const markerAfter = sshExec(vmIp, SSH_USER, "cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
    expect(markerAfter.exitCode).toBe(0);
    expect(markerAfter.stdout).toContain("LAB-MARKER-R2");
  }, INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 60_000);

  it("freshConfig makes the generated config win, data still kept (round 3)", async () => {
    // Re-assert the marker is on disk and synced before the next power-cut.
    const pre = sshExec(vmIp, SSH_USER, "sync && cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
    expect(pre.stdout).toContain("LAB-MARKER-R2");

    vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R3, freshConfig: true });

    const cfg = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal });
    expect(cfg.stdout).toContain(`host-name "${HOSTNAME_R3}"`);   // generated config won

    // The marker file (non-config data under /config) still survives —
    // freshConfig replaces only config.boot, not the carried data.
    const markerAfter = sshExec(vmIp, SSH_USER, "cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
    expect(markerAfter.exitCode).toBe(0);
    expect(markerAfter.stdout).toContain("LAB-MARKER-R2");
  }, INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 60_000);
});