From 53265bb18c518d0963df3b6f300ef8e0021d0aa3 Mon Sep 17 00:00:00 2001 From: Michal Date: Tue, 31 Mar 2026 03:07:38 +0100 Subject: [PATCH] test: integration test for Asahi firstboot LVM setup VM-based end-to-end test using Fedora cloud image with two disks: root (20GB) + data (200GB). Verifies the firstboot script creates labvg with correct LV sizes, mounts volumes, migrates /home content, sets hostname, creates admin user, and handles reprovision. Fixes to firstboot script: - Detect whole disks (not just partitions) for LVM PV - Handle btrfs subvolume paths in root device detection - Copy /home content before mounting LV (preserves SSH keys) - Don't restart sshd (config takes effect on reboot) - Make swapon and mount operations resilient to failures Co-Authored-By: Claude Opus 4.6 (1M context) --- bastion/package.json | 4 +- .../src/templates/asahi-firstboot.sh.ts | 71 ++-- .../tests/integration/asahi-firstboot.test.ts | 355 ++++++++++++++++++ 3 files changed, 407 insertions(+), 23 deletions(-) create mode 100644 bastion/tests/integration/asahi-firstboot.test.ts diff --git a/bastion/package.json b/bastion/package.json index 1e2a673..79a41ca 100644 --- a/bastion/package.json +++ b/bastion/package.json @@ -22,7 +22,9 @@ "test:integration:iso": "vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'", "test:integration:iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'", "test:integration:arm-iso": "vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'", - "test:integration:arm-iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'" + "test:integration:arm-iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'", + "test:integration:asahi": "vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'", + "test:integration:asahi:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'" }, "engines": { "node": ">=20.0.0", diff --git a/bastion/src/bastion/src/templates/asahi-firstboot.sh.ts b/bastion/src/bastion/src/templates/asahi-firstboot.sh.ts index 2e32a03..36a695d 100644 --- a/bastion/src/bastion/src/templates/asahi-firstboot.sh.ts +++ b/bastion/src/bastion/src/templates/asahi-firstboot.sh.ts @@ -63,16 +63,21 @@ if [ -f "$MARKER" ]; then fi # ── Find the data partition ────────────────────────────────────── -# The data partition is the large Linux partition that is NOT root. -ROOT_DEV=$(findmnt -n -o SOURCE /) -echo "Root device: $ROOT_DEV" +# The data partition/disk is a large block device that is NOT the root filesystem. +# Handles: NVMe partitions, SCSI partitions, whole unpartitioned disks. +ROOT_DEV=$(findmnt -n -o SOURCE / | sed 's/\\[.*\\]//') # strip btrfs subvol +ROOT_DISK=$(lsblk -n -o PKNAME "$ROOT_DEV" 2>/dev/null | head -1) +echo "Root device: $ROOT_DEV (disk: $ROOT_DISK)" DATA_PART="" -for part in /dev/nvme*n*p* /dev/sd*[0-9]; do +# Scan partitions first, then whole disks +for part in /dev/nvme*n*p* /dev/sd*[0-9] /dev/vd*[0-9] /dev/nvme*n* /dev/sd[b-z] /dev/vd[b-z]; do [ -b "$part" ] || continue - # Skip root partition + # Skip root device and root disk [ "$part" = "$ROOT_DEV" ] && continue - # Skip small partitions (<50GB) — EFI, boot, APFS stubs + PART_DISK=$(basename "$part" | sed 's/p[0-9]*$//' | sed 's/[0-9]*$//') + [ "$PART_DISK" = "$ROOT_DISK" ] && continue + # Skip small devices (<50GB) — EFI, boot, APFS stubs SIZE_BYTES=$(blockdev --getsize64 "$part" 2>/dev/null || echo 0) SIZE_GB=$((SIZE_BYTES / 1073741824)) [ "$SIZE_GB" -lt 50 ] && continue @@ -80,7 +85,7 @@ for part in /dev/nvme*n*p* /dev/sd*[0-9]; do FSTYPE=$(blkid -o value -s TYPE "$part" 2>/dev/null || echo "") if [ -z "$FSTYPE" ] || [ "$FSTYPE" = "LVM2_member" ]; then DATA_PART="$part" - echo "Found data partition: $DATA_PART ($SIZE_GB GB)" + echo "Found data device: $DATA_PART ($SIZE_GB GB)" break fi done @@ -159,23 +164,46 @@ mkfs.xfs /dev/labvg/home mkfs.xfs /dev/labvg/srv ${roleFormatLines.join('\n')} -# Move existing /var content to LV -echo "Migrating /var to LVM..." -TMPVAR="/var.old.$$" -mv /var "$TMPVAR" -mkdir -p /var -mount /dev/labvg/var /var -cp -a "$TMPVAR"/. /var/ 2>/dev/null || true -rm -rf "$TMPVAR" +# Migrate and mount volumes that can be switched live. +# Copy existing content first so we don't shadow files (e.g. /home/user/.ssh). +for LV_MOUNT in "home /home" "srv /srv"; do + LV_NAME=$(echo "$LV_MOUNT" | awk '{print $1}') + MOUNT_PT=$(echo "$LV_MOUNT" | awk '{print $2}') + STAGING="/mnt/labvg-$LV_NAME-staging" + mkdir -p "$STAGING" + mount "/dev/labvg/$LV_NAME" "$STAGING" + cp -a "$MOUNT_PT"/. "$STAGING/" 2>/dev/null || true + umount "$STAGING" + rmdir "$STAGING" + mount_lv "$LV_NAME" "$MOUNT_PT" +done -# Mount remaining volumes -mount_lv varlog /var/log -mount_lv home /home -mount_lv srv /srv +# Mount role-specific volumes (empty, no content to preserve) +set +e ${roleMountLines.join('\n')} +set -e + +# Copy existing /var content into the LV for next boot +echo "Preparing /var LV for next boot..." +TMPVAR="/mnt/labvg-var-staging" +mkdir -p "$TMPVAR" +mount /dev/labvg/var "$TMPVAR" +cp -a /var/. "$TMPVAR/" 2>/dev/null || true +umount "$TMPVAR" +rmdir "$TMPVAR" + +# Same for /var/log +TMPVARLOG="/mnt/labvg-varlog-staging" +mkdir -p "$TMPVARLOG" +mount /dev/labvg/varlog "$TMPVARLOG" +cp -a /var/log/. "$TMPVARLOG/" 2>/dev/null || true +umount "$TMPVARLOG" +rmdir "$TMPVARLOG" + +echo "NOTE: /var and /var/log will switch to LVM on next reboot." # Enable swap -swapon /dev/labvg/swap +swapon /dev/labvg/swap 2>/dev/null || true # Write fstab entries echo "" >> /etc/fstab @@ -212,10 +240,9 @@ chmod 700 /root/.ssh (${sshKeyLines}) >> /root/.ssh/authorized_keys chmod 600 /root/.ssh/authorized_keys -# ── Harden SSH ─────────────────────────────────────────────────── +# ── Harden SSH (takes effect on next sshd restart/reboot) ──────── sed -i 's/^#\\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config sed -i 's/^#\\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config -systemctl restart sshd 2>/dev/null || true # ── Write provisioning metadata ────────────────────────────────── cat > /etc/lab-provisioned << LABMETA diff --git a/bastion/tests/integration/asahi-firstboot.test.ts b/bastion/tests/integration/asahi-firstboot.test.ts new file mode 100644 index 0000000..ce8a60a --- /dev/null +++ b/bastion/tests/integration/asahi-firstboot.test.ts @@ -0,0 +1,355 @@ +// Integration test: Asahi first-boot LVM setup. +// +// Tests the first-boot script that creates the standard lab LVM layout +// on a separate data disk — simulating the Asahi provisioning flow where +// the root partition is pre-installed and a data partition is left for LVM. +// +// Uses a Fedora cloud VM with two disks: +// disk0: 20GB root (Fedora cloud image) +// disk1: 200GB empty (simulates the Asahi "Data" partition) +// +// The firstboot script should detect disk1, create labvg + LVs, mount them. +// Then we test reprovision: wipe marker, re-run, verify existing VG reused. +// +// Prerequisites: libvirt, virsh, virt-install, qemu, sudo access, lvm2 +// Run: sudo pnpm run test:integration:asahi + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { readFileSync, existsSync } from "node:fs"; +import { execSync } from "node:child_process"; +import { join } from "node:path"; +import { homedir } from "node:os"; +import { destroyVm, waitForVmIp, waitForSsh, log, ensureCloudImage, createCloudInitIso } from "./helpers/libvirt.js"; +import { ensureTestNetwork, TEST_NETWORK_NAME } from "./helpers/network.js"; +import { sshExec, sshRun } from "./helpers/ssh.js"; +import { renderFirstbootScript } from "../../src/bastion/src/templates/asahi-firstboot.sh.js"; + +const VM_NAME = "lab-asahi-firstboot-test"; +const VM_MEMORY = 4096; +const VM_VCPUS = 2; +const VM_ROOT_DISK_GB = 20; +const VM_DATA_DISK_GB = 200; // Simulates the Asahi "Data" partition +const SSH_USER = "fedora"; +const IMAGE_DIR = "/var/lib/libvirt/images"; +const IS_ROOT = process.getuid?.() === 0; + +const FEDORA_CLOUD_IMAGE = "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-43-1.6.x86_64.qcow2"; + +function run(cmd: string, opts?: { timeout?: number }): string { + const full = IS_ROOT ? cmd : `sudo ${cmd}`; + return execSync(full, { encoding: "utf-8", stdio: "pipe", timeout: opts?.timeout ?? 60_000 }); +} + +function findSshKey(): { pubKey: string; keyPath: string } { + const homes = [homedir()]; + const sudoUser = process.env["SUDO_USER"]; + if (sudoUser) homes.push(join("/home", sudoUser)); + if (process.env["SSH_KEY_PATH"]) { + const keyPath = process.env["SSH_KEY_PATH"]; + const pubPath = `${keyPath}.pub`; + if (existsSync(keyPath) && existsSync(pubPath)) { + return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath }; + } + } + for (const home of homes) { + for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) { + const keyPath = join(home, ".ssh", name); + const pubPath = `${keyPath}.pub`; + if (existsSync(keyPath) && existsSync(pubPath)) { + return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath }; + } + } + } + throw new Error("No SSH key found"); +} + +/** Create a VM with two disks: root (cloud image) + empty data disk. */ +function createTwoDiskVm(config: { + name: string; + memory: number; + vcpus: number; + rootDiskGb: number; + dataDiskGb: number; + network: string; + cloudImageUrl: string; + sshPubKey: string; +}): void { + destroyVm(config.name); + + log(`Creating two-disk VM: ${config.name} (root=${config.rootDiskGb}GB, data=${config.dataDiskGb}GB)`); + + const baseImage = ensureCloudImage(config.cloudImageUrl, `${config.name}-base`); + const rootDiskPath = join(IMAGE_DIR, `${config.name}.qcow2`); + const dataDiskPath = join(IMAGE_DIR, `${config.name}-data.qcow2`); + + // Root disk from cloud image + run(`cp "${baseImage}" "${rootDiskPath}"`); + run(`qemu-img resize "${rootDiskPath}" ${config.rootDiskGb}G`); + + // Empty data disk + run(`qemu-img create -f qcow2 "${dataDiskPath}" ${config.dataDiskGb}G`); + + // Cloud-init with LVM tools + const cloudInitIso = createCloudInitIso(config.name, { + name: config.name, + memory: config.memory, + vcpus: config.vcpus, + diskSize: config.rootDiskGb, + network: config.network, + cloudImageUrl: config.cloudImageUrl, + sshPubKey: config.sshPubKey, + userData: `#cloud-config +hostname: ${config.name} +manage_etc_hosts: true +users: + - default + - name: fedora + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + ssh_authorized_keys: + - ${config.sshPubKey} +ssh_pwauth: false +package_update: false +packages: + - lvm2 + - xfsprogs +`, + }); + + const virtInstallArgs = [ + "virt-install", + `--name=${config.name}`, + `--memory=${config.memory}`, + `--vcpus=${config.vcpus}`, + `--disk=path=${rootDiskPath},format=qcow2`, + `--disk=path=${dataDiskPath},format=qcow2`, // Second disk for LVM + `--disk=path=${cloudInitIso},device=cdrom`, + `--network=network=${config.network},model=virtio`, + "--os-variant=generic", + "--import", + "--noautoconsole", + "--wait=0", + ]; + + run(virtInstallArgs.join(" ")); + log(`Two-disk VM ${config.name} created`); +} + +describe("asahi firstboot LVM integration", () => { + let vmIp: string; + let sshKeyPath: string; + let sshPubKey: string; + + beforeAll(async () => { + const keys = findSshKey(); + sshKeyPath = keys.keyPath; + sshPubKey = keys.pubKey; + + log("Setting up test network..."); + ensureTestNetwork(); + + log("Creating two-disk VM..."); + createTwoDiskVm({ + name: VM_NAME, + memory: VM_MEMORY, + vcpus: VM_VCPUS, + rootDiskGb: VM_ROOT_DISK_GB, + dataDiskGb: VM_DATA_DISK_GB, + network: TEST_NETWORK_NAME, + cloudImageUrl: FEDORA_CLOUD_IMAGE, + sshPubKey, + }); + + log("Waiting for VM IP..."); + vmIp = await waitForVmIp(VM_NAME, 120_000); + + log("Waiting for SSH..."); + await waitForSsh(vmIp, SSH_USER, 180_000, sshKeyPath); + + log("Waiting for cloud-init to finish..."); + await sshRun(vmIp, SSH_USER, "sudo cloud-init status --wait 2>/dev/null || sleep 30", "cloud-init", { keyPath: sshKeyPath }); + + // Verify second disk exists + const disks = sshExec(vmIp, SSH_USER, "lsblk -d -n -o NAME,SIZE", { keyPath: sshKeyPath }); + log(`Disks:\n${disks.stdout}`); + }, 300_000); + + afterAll(async () => { + log("Cleaning up VM..."); + destroyVm(VM_NAME); + // Also remove data disk + try { run(`rm -f "${join(IMAGE_DIR, `${VM_NAME}-data.qcow2`)}"`); } catch { /* ignore */ } + }); + + it("second disk is visible and unformatted", () => { + const result = sshExec(vmIp, SSH_USER, "lsblk -d -n -o NAME,SIZE,TYPE | grep disk", { keyPath: sshKeyPath }); + const disks = result.stdout.trim().split("\n"); + expect(disks.length).toBeGreaterThanOrEqual(2); + + // Second disk (vdb) should exist + const vdb = sshExec(vmIp, SSH_USER, "sudo blkid /dev/vdb 2>/dev/null; echo exit=$?", { keyPath: sshKeyPath }); + // Should have no filesystem (blkid returns nothing or non-zero) + expect(vdb.stdout).toContain("exit=2"); + }); + + it("firstboot script creates LVM on data disk", async () => { + // Generate the firstboot script + const script = renderFirstbootScript({ + hostname: "asahi-test", + role: "infra", + serverIp: "10.0.0.1", + httpPort: 8080, + sshKeys: [sshPubKey], + adminUser: "testadmin", + mac: "52:54:00:aa:bb:cc", + }); + + // Upload and run + log("Uploading firstboot script..."); + await sshRun(vmIp, SSH_USER, + `cat > /tmp/firstboot.sh << 'SCRIPT_EOF'\n${script}\nSCRIPT_EOF\nchmod +x /tmp/firstboot.sh`, + "upload script", { keyPath: sshKeyPath }); + + log("Running firstboot script..."); + const result = await sshRun(vmIp, SSH_USER, + "sudo /tmp/firstboot.sh 2>&1", + "firstboot", { keyPath: sshKeyPath, timeout: 120_000 }); + + expect(result).toBe(0); + }, 180_000); + + it("SSH still works after firstboot script", () => { + const result = sshExec(vmIp, SSH_USER, "echo hello", { keyPath: sshKeyPath }); + if (result.stdout.trim() !== "hello") { + log(`SSH debug: exitCode=${result.exitCode} stdout='${result.stdout}' stderr='${result.stderr}'`); + } + expect(result.stdout.trim()).toBe("hello"); + }); + + it("volume group labvg exists", () => { + const result = sshExec(vmIp, SSH_USER, "sudo vgs labvg --noheadings -o vg_name", { keyPath: sshKeyPath }); + expect(result.stdout.trim()).toBe("labvg"); + }); + + it("all expected logical volumes exist", () => { + const result = sshExec(vmIp, SSH_USER, + "sudo lvs labvg --noheadings -o lv_name --sort lv_name", + { keyPath: sshKeyPath }); + const lvs = result.stdout.trim().split("\n").map(l => l.trim()).sort(); + expect(lvs).toContain("home"); + expect(lvs).toContain("longhorn"); + expect(lvs).toContain("rancher"); // infra role + expect(lvs).toContain("srv"); + expect(lvs).toContain("swap"); + expect(lvs).toContain("var"); + expect(lvs).toContain("varlog"); + }); + + it("LV sizes match kickstart layout", () => { + const result = sshExec(vmIp, SSH_USER, + "sudo lvs labvg --noheadings -o lv_name,lv_size --units m --nosuffix", + { keyPath: sshKeyPath }); + const lvMap = new Map(); + for (const line of result.stdout.trim().split("\n")) { + const [name, size] = line.trim().split(/\s+/); + if (name && size) lvMap.set(name, Math.round(parseFloat(size))); + } + + expect(lvMap.get("swap")).toBe(27648); + expect(lvMap.get("var")).toBe(102400); + expect(lvMap.get("varlog")).toBe(10240); + expect(lvMap.get("home")).toBe(10240); + expect(lvMap.get("srv")).toBe(20480); + expect(lvMap.get("rancher")).toBe(20480); + // longhorn gets remaining — should be at least 5GB (200GB disk - ~191GB used) + expect(lvMap.get("longhorn")).toBeGreaterThan(5000); + }); + + it("non-var volumes are mounted with XFS", () => { + const mounts = sshExec(vmIp, SSH_USER, "mount | grep labvg", { keyPath: sshKeyPath }); + // /var and /var/log deferred to next reboot (can't migrate live) + expect(mounts.stdout).toContain("/home "); + expect(mounts.stdout).toContain("/srv "); + expect(mounts.stdout).toContain("/var/lib/rancher "); + expect(mounts.stdout).toContain("/var/lib/longhorn "); + expect(mounts.stdout).toContain("xfs"); + }); + + it("swap is active", () => { + const result = sshExec(vmIp, SSH_USER, "swapon --show --noheadings", { keyPath: sshKeyPath }); + // swapon may show /dev/dm-X or /dev/labvg/swap + expect(result.stdout.length).toBeGreaterThan(0); + }); + + it("fstab has LVM entries", () => { + const result = sshExec(vmIp, SSH_USER, "grep labvg /etc/fstab", { keyPath: sshKeyPath }); + const lines = result.stdout.trim().split("\n"); + expect(lines.length).toBeGreaterThanOrEqual(7); // swap + var + varlog + home + srv + rancher + longhorn + }); + + it("hostname was set", () => { + const result = sshExec(vmIp, SSH_USER, "hostname", { keyPath: sshKeyPath }); + expect(result.stdout.trim()).toBe("asahi-test"); + }); + + it("admin user was created with sudo", () => { + const result = sshExec(vmIp, SSH_USER, "sudo id testadmin", { keyPath: sshKeyPath }); + expect(result.stdout).toContain("testadmin"); + expect(result.stdout).toContain("wheel"); + }); + + it("provisioning metadata file exists", () => { + const result = sshExec(vmIp, SSH_USER, "cat /etc/lab-provisioned", { keyPath: sshKeyPath }); + expect(result.stdout).toContain("hostname=asahi-test"); + expect(result.stdout).toContain("role=infra"); + expect(result.stdout).toContain("method=asahi-firstboot"); + }); + + it("marker file prevents re-run", () => { + const result = sshExec(vmIp, SSH_USER, "test -f /etc/lab-lvm-setup-done && echo yes", { keyPath: sshKeyPath }); + expect(result.stdout.trim()).toBe("yes"); + }); + + // ── Reprovision test ────────────────────────────────────────────── + + it("reprovision: detects existing labvg and re-mounts", async () => { + // Write a test file to a preserved LV + await sshRun(vmIp, SSH_USER, + "echo 'precious-data' | sudo tee /var/lib/rancher/test-preserve.txt", + "write test data", { keyPath: sshKeyPath }); + + // Remove marker to simulate fresh boot after reinstall + await sshRun(vmIp, SSH_USER, "sudo rm /etc/lab-lvm-setup-done", "remove marker", { keyPath: sshKeyPath }); + + // Unmount everything (simulate reinstall wiping root) + await sshRun(vmIp, SSH_USER, ` + sudo umount /var/lib/longhorn 2>/dev/null || true + sudo umount /var/lib/rancher 2>/dev/null || true + sudo umount /srv 2>/dev/null || true + sudo umount /home 2>/dev/null || true + sudo umount /var/log 2>/dev/null || true + # Don't unmount /var — it's in use + sudo swapoff /dev/labvg/swap 2>/dev/null || true + sudo sed -i '/labvg/d' /etc/fstab + `, "unmount LVs", { keyPath: sshKeyPath }); + + // Re-run firstboot script — should detect existing VG + log("Re-running firstboot (reprovision)..."); + const result = await sshRun(vmIp, SSH_USER, + "sudo /tmp/firstboot.sh 2>&1", + "firstboot reprovision", { keyPath: sshKeyPath }); + expect(result).toBe(0); + + // Verify data was preserved + const data = sshExec(vmIp, SSH_USER, "cat /var/lib/rancher/test-preserve.txt", { keyPath: sshKeyPath }); + expect(data.stdout.trim()).toBe("precious-data"); + + // Verify marker was re-created + const marker = sshExec(vmIp, SSH_USER, "test -f /etc/lab-lvm-setup-done && echo yes", { keyPath: sshKeyPath }); + expect(marker.stdout.trim()).toBe("yes"); + + // Verify fstab was re-populated + const fstab = sshExec(vmIp, SSH_USER, "grep labvg /etc/fstab", { keyPath: sshKeyPath }); + expect(fstab.stdout).toContain("/var/lib/rancher"); + }, 60_000); +});