lab/bastion/tests/integration/asahi-firstboot.test.ts

// Integration test: Asahi first-boot LVM setup.
//
// Tests the first-boot script that creates the standard lab LVM layout
// on a separate data disk — simulating the Asahi provisioning flow where
// the root partition is pre-installed and a data partition is left for LVM.
//
// Uses a Fedora cloud VM with two disks:
//   disk0: 20GB root (Fedora cloud image)
//   disk1: 200GB empty (simulates the Asahi "Data" partition)
//
// The firstboot script should detect disk1, create labvg + LVs, mount them.
// Then we test reprovision: wipe marker, re-run, verify existing VG reused.
//
// Prerequisites: libvirt, virsh, virt-install, qemu, sudo access, lvm2
// Run: sudo pnpm run test:integration:asahi

import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { readFileSync, existsSync } from "node:fs";
import { execSync } from "node:child_process";
import { join } from "node:path";
import { homedir } from "node:os";
import { destroyVm, waitForVmIp, waitForSsh, log, ensureCloudImage, createCloudInitIso } from "./helpers/libvirt.js";
import { ensureTestNetwork, TEST_NETWORK_NAME } from "./helpers/network.js";
import { sshExec, sshRun } from "./helpers/ssh.js";
import { renderFirstbootScript } from "../../src/bastion/src/templates/asahi-firstboot.sh.js";

const VM_NAME = "lab-asahi-firstboot-test";
const VM_MEMORY = 4096;
const VM_VCPUS = 2;
const VM_ROOT_DISK_GB = 20;
const VM_DATA_DISK_GB = 200;    // Simulates the Asahi "Data" partition
const SSH_USER = "fedora";
const IMAGE_DIR = "/var/lib/libvirt/images";
const IS_ROOT = process.getuid?.() === 0;

const FEDORA_CLOUD_IMAGE = "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-43-1.6.x86_64.qcow2";

function run(cmd: string, opts?: { timeout?: number }): string {
  const full = IS_ROOT ? cmd : `sudo ${cmd}`;
  return execSync(full, { encoding: "utf-8", stdio: "pipe", timeout: opts?.timeout ?? 60_000 });
}

function findSshKey(): { pubKey: string; keyPath: string } {
  const homes = [homedir()];
  const sudoUser = process.env["SUDO_USER"];
  if (sudoUser) homes.push(join("/home", sudoUser));
  if (process.env["SSH_KEY_PATH"]) {
    const keyPath = process.env["SSH_KEY_PATH"];
    const pubPath = `${keyPath}.pub`;
    if (existsSync(keyPath) && existsSync(pubPath)) {
      return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath };
    }
  }
  for (const home of homes) {
    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
      const keyPath = join(home, ".ssh", name);
      const pubPath = `${keyPath}.pub`;
      if (existsSync(keyPath) && existsSync(pubPath)) {
        return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath };
      }
    }
  }
  throw new Error("No SSH key found");
}

/** Create a VM with two disks: root (cloud image) + empty data disk. */
function createTwoDiskVm(config: {
  name: string;
  memory: number;
  vcpus: number;
  rootDiskGb: number;
  dataDiskGb: number;
  network: string;
  cloudImageUrl: string;
  sshPubKey: string;
}): void {
  destroyVm(config.name);

  log(`Creating two-disk VM: ${config.name} (root=${config.rootDiskGb}GB, data=${config.dataDiskGb}GB)`);

  const baseImage = ensureCloudImage(config.cloudImageUrl, `${config.name}-base`);
  const rootDiskPath = join(IMAGE_DIR, `${config.name}.qcow2`);
  const dataDiskPath = join(IMAGE_DIR, `${config.name}-data.qcow2`);

  // Root disk from cloud image
  run(`cp "${baseImage}" "${rootDiskPath}"`);
  run(`qemu-img resize "${rootDiskPath}" ${config.rootDiskGb}G`);

  // Empty data disk
  run(`qemu-img create -f qcow2 "${dataDiskPath}" ${config.dataDiskGb}G`);

  // Cloud-init with LVM tools
  const cloudInitIso = createCloudInitIso(config.name, {
    name: config.name,
    memory: config.memory,
    vcpus: config.vcpus,
    diskSize: config.rootDiskGb,
    network: config.network,
    cloudImageUrl: config.cloudImageUrl,
    sshPubKey: config.sshPubKey,
    userData: `#cloud-config
hostname: ${config.name}
manage_etc_hosts: true
users:
  - default
  - name: fedora
    sudo: ALL=(ALL) NOPASSWD:ALL
    shell: /bin/bash
    ssh_authorized_keys:
      - ${config.sshPubKey}
ssh_pwauth: false
package_update: false
packages:
  - lvm2
  - xfsprogs
`,
  });

  const virtInstallArgs = [
    "virt-install",
    `--name=${config.name}`,
    `--memory=${config.memory}`,
    `--vcpus=${config.vcpus}`,
    `--disk=path=${rootDiskPath},format=qcow2`,
    `--disk=path=${dataDiskPath},format=qcow2`,  // Second disk for LVM
    `--disk=path=${cloudInitIso},device=cdrom`,
    `--network=network=${config.network},model=virtio`,
    "--os-variant=generic",
    "--import",
    "--noautoconsole",
    "--wait=0",
  ];

  run(virtInstallArgs.join(" "));
  log(`Two-disk VM ${config.name} created`);
}

describe("asahi firstboot LVM integration", () => {
  let vmIp: string;
  let sshKeyPath: string;
  let sshPubKey: string;

  beforeAll(async () => {
    const keys = findSshKey();
    sshKeyPath = keys.keyPath;
    sshPubKey = keys.pubKey;

    log("Setting up test network...");
    ensureTestNetwork();

    log("Creating two-disk VM...");
    createTwoDiskVm({
      name: VM_NAME,
      memory: VM_MEMORY,
      vcpus: VM_VCPUS,
      rootDiskGb: VM_ROOT_DISK_GB,
      dataDiskGb: VM_DATA_DISK_GB,
      network: TEST_NETWORK_NAME,
      cloudImageUrl: FEDORA_CLOUD_IMAGE,
      sshPubKey,
    });

    log("Waiting for VM IP...");
    vmIp = await waitForVmIp(VM_NAME, 120_000);

    log("Waiting for SSH...");
    await waitForSsh(vmIp, SSH_USER, 180_000, sshKeyPath);

    log("Waiting for cloud-init to finish...");
    await sshRun(vmIp, SSH_USER, "sudo cloud-init status --wait 2>/dev/null || sleep 30", "cloud-init", { keyPath: sshKeyPath });

    // Verify second disk exists
    const disks = sshExec(vmIp, SSH_USER, "lsblk -d -n -o NAME,SIZE", { keyPath: sshKeyPath });
    log(`Disks:\n${disks.stdout}`);
  }, 300_000);

  afterAll(async () => {
    log("Cleaning up VM...");
    destroyVm(VM_NAME);
    // Also remove data disk
    try { run(`rm -f "${join(IMAGE_DIR, `${VM_NAME}-data.qcow2`)}"`); } catch { /* ignore */ }
  });

  it("second disk is visible and unformatted", () => {
    const result = sshExec(vmIp, SSH_USER, "lsblk -d -n -o NAME,SIZE,TYPE | grep disk", { keyPath: sshKeyPath });
    const disks = result.stdout.trim().split("\n");
    expect(disks.length).toBeGreaterThanOrEqual(2);

    // Second disk (vdb) should exist
    const vdb = sshExec(vmIp, SSH_USER, "sudo blkid /dev/vdb 2>/dev/null; echo exit=$?", { keyPath: sshKeyPath });
    // Should have no filesystem (blkid returns nothing or non-zero)
    expect(vdb.stdout).toContain("exit=2");
  });

  it("firstboot script creates LVM on data disk", async () => {
    // Generate the firstboot script
    const script = renderFirstbootScript({
      hostname: "asahi-test",
      role: "infra",
      serverIp: "10.0.0.1",
      httpPort: 8080,
      sshKeys: [sshPubKey],
      adminUser: "testadmin",
      mac: "52:54:00:aa:bb:cc",
    });

    // Upload and run
    log("Uploading firstboot script...");
    await sshRun(vmIp, SSH_USER,
      `cat > /tmp/firstboot.sh << 'SCRIPT_EOF'\n${script}\nSCRIPT_EOF\nchmod +x /tmp/firstboot.sh`,
      "upload script", { keyPath: sshKeyPath });

    log("Running firstboot script...");
    const result = await sshRun(vmIp, SSH_USER,
      "sudo /tmp/firstboot.sh 2>&1",
      "firstboot", { keyPath: sshKeyPath, timeout: 120_000 });

    expect(result).toBe(0);
  }, 180_000);

  it("SSH still works after firstboot script", () => {
    const result = sshExec(vmIp, SSH_USER, "echo hello", { keyPath: sshKeyPath });
    if (result.stdout.trim() !== "hello") {
      log(`SSH debug: exitCode=${result.exitCode} stdout='${result.stdout}' stderr='${result.stderr}'`);
    }
    expect(result.stdout.trim()).toBe("hello");
  });

  it("volume group labvg exists", () => {
    const result = sshExec(vmIp, SSH_USER, "sudo vgs labvg --noheadings -o vg_name", { keyPath: sshKeyPath });
    expect(result.stdout.trim()).toBe("labvg");
  });

  it("all expected logical volumes exist", () => {
    const result = sshExec(vmIp, SSH_USER,
      "sudo lvs labvg --noheadings -o lv_name --sort lv_name",
      { keyPath: sshKeyPath });
    const lvs = result.stdout.trim().split("\n").map(l => l.trim()).sort();
    expect(lvs).toContain("home");
    expect(lvs).toContain("longhorn");
    expect(lvs).toContain("rancher");   // infra role
    expect(lvs).toContain("srv");
    expect(lvs).toContain("swap");
    expect(lvs).toContain("var");
    expect(lvs).toContain("varlog");
  });

  it("LV sizes match kickstart layout", () => {
    const result = sshExec(vmIp, SSH_USER,
      "sudo lvs labvg --noheadings -o lv_name,lv_size --units m --nosuffix",
      { keyPath: sshKeyPath });
    const lvMap = new Map<string, number>();
    for (const line of result.stdout.trim().split("\n")) {
      const [name, size] = line.trim().split(/\s+/);
      if (name && size) lvMap.set(name, Math.round(parseFloat(size)));
    }

    expect(lvMap.get("swap")).toBe(27648);
    expect(lvMap.get("var")).toBe(102400);
    expect(lvMap.get("varlog")).toBe(10240);
    expect(lvMap.get("home")).toBe(10240);
    expect(lvMap.get("srv")).toBe(20480);
    expect(lvMap.get("rancher")).toBe(20480);
    // longhorn gets remaining — should be at least 5GB (200GB disk - ~191GB used)
    expect(lvMap.get("longhorn")).toBeGreaterThan(5000);
  });

  it("non-var volumes are mounted with XFS", () => {
    const mounts = sshExec(vmIp, SSH_USER, "mount | grep labvg", { keyPath: sshKeyPath });
    // /var and /var/log deferred to next reboot (can't migrate live)
    expect(mounts.stdout).toContain("/home ");
    expect(mounts.stdout).toContain("/srv ");
    expect(mounts.stdout).toContain("/var/lib/rancher ");
    expect(mounts.stdout).toContain("/var/lib/longhorn ");
    expect(mounts.stdout).toContain("xfs");
  });

  it("swap is active", () => {
    const result = sshExec(vmIp, SSH_USER, "swapon --show --noheadings", { keyPath: sshKeyPath });
    // swapon may show /dev/dm-X or /dev/labvg/swap
    expect(result.stdout.length).toBeGreaterThan(0);
  });

  it("fstab has LVM entries", () => {
    const result = sshExec(vmIp, SSH_USER, "grep labvg /etc/fstab", { keyPath: sshKeyPath });
    const lines = result.stdout.trim().split("\n");
    expect(lines.length).toBeGreaterThanOrEqual(7); // swap + var + varlog + home + srv + rancher + longhorn
  });

  it("hostname was set", () => {
    const result = sshExec(vmIp, SSH_USER, "hostname", { keyPath: sshKeyPath });
    expect(result.stdout.trim()).toBe("asahi-test");
  });

  it("admin user was created with sudo", () => {
    const result = sshExec(vmIp, SSH_USER, "sudo id testadmin", { keyPath: sshKeyPath });
    expect(result.stdout).toContain("testadmin");
    expect(result.stdout).toContain("wheel");
  });

  it("provisioning metadata file exists", () => {
    const result = sshExec(vmIp, SSH_USER, "cat /etc/lab-provisioned", { keyPath: sshKeyPath });
    expect(result.stdout).toContain("hostname=asahi-test");
    expect(result.stdout).toContain("role=infra");
    expect(result.stdout).toContain("method=asahi-firstboot");
  });

  it("marker file prevents re-run", () => {
    const result = sshExec(vmIp, SSH_USER, "test -f /etc/lab-lvm-setup-done && echo yes", { keyPath: sshKeyPath });
    expect(result.stdout.trim()).toBe("yes");
  });

  // ── Reprovision test ──────────────────────────────────────────────

  it("reprovision: detects existing labvg and re-mounts", async () => {
    // Write a test file to a preserved LV
    await sshRun(vmIp, SSH_USER,
      "echo 'precious-data' | sudo tee /var/lib/rancher/test-preserve.txt",
      "write test data", { keyPath: sshKeyPath });

    // Remove marker to simulate fresh boot after reinstall
    await sshRun(vmIp, SSH_USER, "sudo rm /etc/lab-lvm-setup-done", "remove marker", { keyPath: sshKeyPath });

    // Unmount everything (simulate reinstall wiping root)
    await sshRun(vmIp, SSH_USER, `
      sudo umount /var/lib/longhorn 2>/dev/null || true
      sudo umount /var/lib/rancher 2>/dev/null || true
      sudo umount /srv 2>/dev/null || true
      sudo umount /home 2>/dev/null || true
      sudo umount /var/log 2>/dev/null || true
      # Don't unmount /var — it's in use
      sudo swapoff /dev/labvg/swap 2>/dev/null || true
      sudo sed -i '/labvg/d' /etc/fstab
    `, "unmount LVs", { keyPath: sshKeyPath });

    // Re-run firstboot script — should detect existing VG
    log("Re-running firstboot (reprovision)...");
    const result = await sshRun(vmIp, SSH_USER,
      "sudo /tmp/firstboot.sh 2>&1",
      "firstboot reprovision", { keyPath: sshKeyPath });
    expect(result).toBe(0);

    // Verify data was preserved
    const data = sshExec(vmIp, SSH_USER, "cat /var/lib/rancher/test-preserve.txt", { keyPath: sshKeyPath });
    expect(data.stdout.trim()).toBe("precious-data");

    // Verify marker was re-created
    const marker = sshExec(vmIp, SSH_USER, "test -f /etc/lab-lvm-setup-done && echo yes", { keyPath: sshKeyPath });
    expect(marker.stdout.trim()).toBe("yes");

    // Verify fstab was re-populated
    const fstab = sshExec(vmIp, SSH_USER, "grep labvg /etc/fstab", { keyPath: sshKeyPath });
    expect(fstab.stdout).toContain("/var/lib/rancher");
  }, 60_000);
});