feat: install logging, error trapping, PXE/ISO integration tests

Kickstart installs on real hardware failed silently — no error reporting, only 3 progress callbacks, zero log streaming. This overhaul makes every install fully observable. Kickstart improvements: - Error trapping in %pre and %post (trap ERR sends failure details to bastion) - 12+ granular progress stages (was 3): SSH, hostname, k3s prep, EFI boot, metadata - Background log streamer: tails %post output and batch-sends to /api/log - bastion_log() function for explicit log lines from kickstart scripts Bastion API: - POST /api/log — receives raw log lines from kickstart (single or batch) - InstallLogBuffer — per-MAC ring buffer (2000 lines) + file persistence - GET /api/logs/:mac — now returns log_lines + log_total alongside stages - SSE /api/logs/:mac/follow — uses named events (event: stage vs event: log) - Progress events forwarded to labd via bastion-progress WebSocket message - Post-provision k3s logs routed through progressBus (was console-only) dnsmasq fixes found during VM testing: - HTTP Boot filename: ipxe-real.efi → ipxe.efi (leftover from old 2-stage approach) - pxe-service directives: only in proxy mode (breaks OVMF PXE in full mode) - PXEClient vendor class echo for UEFI firmware compatibility Integration tests: - PXE boot test: blank UEFI VM → dnsmasq → HTTP Boot → iPXE → bastion → install - ISO boot test: blank VM boots from bastion-generated ISO → same flow - Shared helpers: pxe-network (no DHCP, nftables fix), pxe-vm (UEFI + ISO boot) - test-provision.sh: runs both PXE + ISO tests with prerequisite checks - 250GB sparse QCOW2 disk (LVM layout needs ~204GB) 201 unit tests passing (11 new). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-26 22:26:33 +00:00
parent ffc4a782d2
commit 46b017d77e
189 changed files with 16241 additions and 432 deletions
--- a/bastion/scripts/build-bastion.sh
+++ b/bastion/scripts/build-bastion.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Build bastion container image and push to Gitea container registry
+# Build bastion container image (multi-arch) and push to Gitea container registry
 set -e

 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
@@ -12,20 +12,28 @@ if [ -f .env ]; then
 fi

 # ── Argument parsing ───────────────────────────────────────────────
-TARGET_ARCH=""
+PUSH=false
+PLATFORMS="linux/amd64,linux/arm64"

 usage() {
  cat <<EOF
 Usage: $(basename "$0") [OPTIONS] [TAG]

-Build bastion container image and optionally push to registry.
+Build bastion container image (multi-arch) and optionally push to registry.

 Options:
-  --arch ARCH    Target platform: x86_64 or arm64 (default: host arch)
-  -h, --help     Show this help message
+  --push             Push to registry after building
+  --platforms LIST   Comma-separated platforms (default: linux/amd64,linux/arm64)
+  -h, --help         Show this help message

 Arguments:
-  TAG            Image tag (default: version from package.json)
+  TAG                Image tag (default: version from package.json)
+
+Examples:
+  $(basename "$0")                          # build multi-arch, no push
+  $(basename "$0") --push                   # build + push with version tag
+  $(basename "$0") --push latest            # build + push as :latest
+  $(basename "$0") --platforms linux/amd64   # build amd64 only
 EOF
  exit 0
 }
@@ -33,8 +41,12 @@ EOF
 POSITIONAL_ARGS=()
 while [[ $# -gt 0 ]]; do
  case "$1" in
-    --arch)
-      TARGET_ARCH="$2"
+    --push)
+      PUSH=true
+      shift
+      ;;
+    --platforms)
+      PLATFORMS="$2"
      shift 2
      ;;
    -h|--help)
@@ -47,56 +59,69 @@ while [[ $# -gt 0 ]]; do
  esac
 done

-# Registry defaults to internal address (external proxy has body size limit)
 REGISTRY="${GITEA_REGISTRY:-mysources.co.uk}"
-IMAGE="lab-bastion"
+REPO="michal/lab/bastion"
+FULL_IMAGE="$REGISTRY/$REPO"
 VERSION=$(node -p "require('./package.json').version")
 TAG="${POSITIONAL_ARGS[0]:-$VERSION}"

-# ── Resolve target platform ───────────────────────────────────────
-detect_host_arch() {
-  local machine
-  machine="$(uname -m)"
-  case "$machine" in
-    x86_64)  echo "x86_64" ;;
-    aarch64) echo "arm64" ;;
-    arm64)   echo "arm64" ;;
-    *)       echo "$machine" ;;
-  esac
-}
+echo "==> Building bastion image"
+echo "    Tag:       $TAG"
+echo "    Platforms: $PLATFORMS"
+echo "    Registry:  $FULL_IMAGE"

-docker_platform_for() {
-  case "$1" in
-    x86_64) echo "linux/amd64" ;;
-    arm64)  echo "linux/arm64" ;;
-  esac
-}
+# ── Build multi-arch manifest ────────────────────────────────────
+MANIFEST="lab-bastion:$TAG"

-ARCH="${TARGET_ARCH:-$(detect_host_arch)}"
-PLATFORM="$(docker_platform_for "$ARCH")"
+# Remove existing manifest/image with the same tag
+podman manifest rm "$MANIFEST" 2>/dev/null || true
+podman rmi "$MANIFEST" 2>/dev/null || true

-echo "==> Building bastion image (tag: $TAG, platform: $PLATFORM)..."
-podman build --platform "$PLATFORM" -t "$IMAGE:$TAG" -f stack/Dockerfile .
+echo "==> Building for platforms: $PLATFORMS..."
+podman build \
+  --platform "$PLATFORMS" \
+  --manifest "$MANIFEST" \
+  -f Dockerfile.bastion \
+  .

-echo "==> Tagging as $REGISTRY/michal/$IMAGE:$TAG..."
-podman tag "$IMAGE:$TAG" "$REGISTRY/michal/$IMAGE:$TAG"
+echo "==> Build complete. Manifest:"
+podman manifest inspect "$MANIFEST" | grep -E '"(architecture|os)"'
+
+# ── Push ─────────────────────────────────────────────────────────
+if [ "$PUSH" = true ]; then
+  if [ -z "$GITEA_TOKEN" ]; then
+    # Try reading from ~/.gitea-token
+    if [ -f "$HOME/.gitea-token" ]; then
+      GITEA_TOKEN="$(cat "$HOME/.gitea-token")"
+    else
+      echo "ERROR: GITEA_TOKEN not set and ~/.gitea-token not found"
+      exit 1
+    fi
+  fi

-if [ -n "$GITEA_TOKEN" ]; then
  echo "==> Logging in to $REGISTRY..."
-  podman login --tls-verify=false -u michal -p "$GITEA_TOKEN" "$REGISTRY"
+  podman login -u michal -p "$GITEA_TOKEN" "$REGISTRY"

-  echo "==> Pushing to $REGISTRY/michal/$IMAGE:$TAG..."
-  podman push --tls-verify=false "$REGISTRY/michal/$IMAGE:$TAG"
+  echo "==> Pushing $FULL_IMAGE:$TAG..."
+  podman manifest push --all "$MANIFEST" "docker://$FULL_IMAGE:$TAG"

-  # Ensure package is linked to the repository
+  # Also tag as :latest if not already
+  if [ "$TAG" != "latest" ]; then
+    echo "==> Also pushing as :latest..."
+    podman manifest push --all "$MANIFEST" "docker://$FULL_IMAGE:latest"
+  fi
+
+  # Link package to repository if script exists
  if [ -f "$SCRIPT_DIR/link-package.sh" ]; then
    source "$SCRIPT_DIR/link-package.sh"
-    link_package "container" "$IMAGE"
+    link_package "container" "bastion"
  fi
+
+  echo "==> Pushed successfully!"
 else
-  echo "==> GITEA_TOKEN not set, skipping push."
+  echo "==> Skipping push (use --push to push to registry)"
 fi

 echo "==> Done!"
-echo "    Image: $REGISTRY/michal/$IMAGE:$TAG"
-echo "    Platform: $PLATFORM"
+echo "    Image: $FULL_IMAGE:$TAG"
+echo "    Platforms: $PLATFORMS"
--- a/bastion/scripts/build-labd.sh
+++ b/bastion/scripts/build-labd.sh
@@ -0,0 +1,118 @@
+#!/bin/bash
+# Build labd container image (multi-arch) and push to Gitea container registry
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
+cd "$PROJECT_ROOT"
+
+# Load .env for GITEA_TOKEN
+if [ -f .env ]; then
+  set -a; source .env; set +a
+fi
+
+# ── Argument parsing ───────────────────────────────────────────────
+PUSH=false
+PLATFORMS="linux/amd64,linux/arm64"
+
+usage() {
+  cat <<EOF
+Usage: $(basename "$0") [OPTIONS] [TAG]
+
+Build labd container image (multi-arch) and optionally push to registry.
+
+Options:
+  --push             Push to registry after building
+  --platforms LIST   Comma-separated platforms (default: linux/amd64,linux/arm64)
+  -h, --help         Show this help message
+
+Arguments:
+  TAG                Image tag (default: version from package.json)
+EOF
+  exit 0
+}
+
+POSITIONAL_ARGS=()
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --push)
+      PUSH=true
+      shift
+      ;;
+    --platforms)
+      PLATFORMS="$2"
+      shift 2
+      ;;
+    -h|--help)
+      usage
+      ;;
+    *)
+      POSITIONAL_ARGS+=("$1")
+      shift
+      ;;
+  esac
+done
+
+REGISTRY="${GITEA_REGISTRY:-mysources.co.uk}"
+REPO="michal/lab/labd"
+FULL_IMAGE="$REGISTRY/$REPO"
+VERSION=$(node -p "require('./package.json').version")
+TAG="${POSITIONAL_ARGS[0]:-$VERSION}"
+
+echo "==> Building labd image"
+echo "    Tag:       $TAG"
+echo "    Platforms: $PLATFORMS"
+echo "    Registry:  $FULL_IMAGE"
+
+# ── Build multi-arch manifest ────────────────────────────────────
+MANIFEST="lab-labd:$TAG"
+
+# Remove existing manifest/image with the same tag
+podman manifest rm "$MANIFEST" 2>/dev/null || true
+podman rmi "$MANIFEST" 2>/dev/null || true
+
+echo "==> Building for platforms: $PLATFORMS..."
+podman build \
+  --platform "$PLATFORMS" \
+  --manifest "$MANIFEST" \
+  -f Dockerfile.labd \
+  .
+
+echo "==> Build complete. Manifest:"
+podman manifest inspect "$MANIFEST" | grep -E '"(architecture|os)"'
+
+# ── Push ─────────────────────────────────────────────────────────
+if [ "$PUSH" = true ]; then
+  if [ -z "$GITEA_TOKEN" ]; then
+    if [ -f "$HOME/.gitea-token" ]; then
+      GITEA_TOKEN="$(cat "$HOME/.gitea-token")"
+    else
+      echo "ERROR: GITEA_TOKEN not set and ~/.gitea-token not found"
+      exit 1
+    fi
+  fi
+
+  echo "==> Logging in to $REGISTRY..."
+  podman login -u michal -p "$GITEA_TOKEN" "$REGISTRY"
+
+  echo "==> Pushing $FULL_IMAGE:$TAG..."
+  podman manifest push --all "$MANIFEST" "docker://$FULL_IMAGE:$TAG"
+
+  if [ "$TAG" != "latest" ]; then
+    echo "==> Also pushing as :latest..."
+    podman manifest push --all "$MANIFEST" "docker://$FULL_IMAGE:latest"
+  fi
+
+  if [ -f "$SCRIPT_DIR/link-package.sh" ]; then
+    source "$SCRIPT_DIR/link-package.sh"
+    link_package "container" "labd"
+  fi
+
+  echo "==> Pushed successfully!"
+else
+  echo "==> Skipping push (use --push to push to registry)"
+fi
+
+echo "==> Done!"
+echo "    Image: $FULL_IMAGE:$TAG"
+echo "    Platforms: $PLATFORMS"
--- a/bastion/scripts/generate-completions.ts
+++ b/bastion/scripts/generate-completions.ts
@@ -154,8 +154,8 @@ function generateFish(root: CmdInfo): string {

  const allCmds = collectCommands(root);

-  // Helper function for fish: test if exactly the given subcommand chain is present
-  emit('# Helper: test if a subcommand chain is active');
+  // Helper: test if EXACTLY the given subcommand chain is present (for subcommand suggestions)
+  emit('# Helper: test if exactly a subcommand chain is active (no extra positional args)');
  emit(`function __${BIN}_using_cmd`);
  emit('    set -l tokens (commandline -opc)');
  emit('    set -l expected $argv');
@@ -181,6 +181,65 @@ function generateFish(root: CmdInfo): string {
  emit('end');
  emit('');

+  // Helper: test if command chain STARTS WITH the given prefix (for options that apply after args)
+  emit('# Helper: test if command starts with a subcommand chain (options still apply after args)');
+  emit(`function __${BIN}_in_cmd`);
+  emit('    set -l tokens (commandline -opc)');
+  emit('    set -l expected $argv');
+  emit('    set -l depth (count $expected)');
+  emit('    set -l found 0');
+  emit('    for tok in $tokens[2..]');
+  emit('        if string match -q -- "-*" $tok');
+  emit('            continue');
+  emit('        end');
+  emit('        set found (math $found + 1)');
+  emit('        if test $found -le $depth');
+  emit('            if test "$tok" != "$expected[$found]"');
+  emit('                return 1');
+  emit('            end');
+  emit('        end');
+  emit('    end');
+  emit('    test $found -ge $depth');
+  emit('end');
+  emit('');
+
+  // Dynamic completions: fetch machine data from bastion API
+  emit('# Dynamic: fetch machine hostnames from bastion (installed + queued)');
+  emit(`function __${BIN}_installed_hosts`);
+  emit('    curl -s http://localhost:8080/api/machines 2>/dev/null | ');
+  emit("        python3 -c 'import sys,json; d=json.load(sys.stdin); hosts=[v.get(\"hostname\",\"\") for v in {**d.get(\"install_queue\",{}), **d.get(\"installed\",{})}.values() if v.get(\"hostname\")]; [print(h) for h in set(hosts)]' 2>/dev/null");
+  emit('end');
+  emit('');
+
+  emit('# Dynamic: fetch all known MAC addresses (discovered + queue + installed)');
+  emit(`function __${BIN}_known_macs`);
+  emit('    curl -s http://localhost:8080/api/machines 2>/dev/null | ');
+  emit("        python3 -c 'import sys,json; d=json.load(sys.stdin); [print(k) for k in {**d.get(\"discovered\",{}), **d.get(\"install_queue\",{}), **d.get(\"installed\",{})}]' 2>/dev/null");
+  emit('end');
+  emit('');
+
+  emit('# Dynamic: fetch hostnames and MACs from all states');
+  emit(`function __${BIN}_hosts_and_macs`);
+  emit('    curl -s http://localhost:8080/api/machines 2>/dev/null | ');
+  emit("        python3 -c 'import sys,json; d=json.load(sys.stdin); a={**d.get(\"discovered\",{}), **d.get(\"install_queue\",{}), **d.get(\"installed\",{})}; macs=list(a.keys()); hosts=[v.get(\"hostname\",\"\") for v in {**d.get(\"install_queue\",{}), **d.get(\"installed\",{})}.values() if v.get(\"hostname\")]; [print(x) for x in set(macs+hosts)]' 2>/dev/null");
+  emit('end');
+  emit('');
+
+  // Target completions for commands that accept hostname/IP/MAC
+  emit('# Target argument completions');
+  // app k3s — takes hostname/IP
+  emit(`complete -c ${BIN} -n "__${BIN}_using_cmd app k3s install" -a "(__${BIN}_installed_hosts)" -d 'installed host'`);
+  emit(`complete -c ${BIN} -n "__${BIN}_using_cmd app k3s health" -a "(__${BIN}_installed_hosts)" -d 'installed host'`);
+  emit(`complete -c ${BIN} -n "__${BIN}_using_cmd app labcontroller deploy" -a "(__${BIN}_installed_hosts)" -d 'installed host'`);
+  emit(`complete -c ${BIN} -n "__${BIN}_using_cmd app labcontroller status" -a "(__${BIN}_installed_hosts)" -d 'installed host'`);
+  // provision install — takes MAC then hostname
+  emit(`complete -c ${BIN} -n "__${BIN}_using_cmd provision install" -a "(__${BIN}_known_macs)" -d 'MAC address'`);
+  // provision reprovision/forget/logs — takes MAC or hostname
+  emit(`complete -c ${BIN} -n "__${BIN}_using_cmd provision reprovision" -a "(__${BIN}_hosts_and_macs)" -d 'host or MAC'`);
+  emit(`complete -c ${BIN} -n "__${BIN}_using_cmd provision forget" -a "(__${BIN}_hosts_and_macs)" -d 'host or MAC'`);
+  emit(`complete -c ${BIN} -n "__${BIN}_using_cmd provision logs" -a "(__${BIN}_hosts_and_macs)" -d 'host or MAC'`);
+  emit('');
+
  // Top-level commands
  const topCmds = root.subcommands.filter((c) => !c.hidden);
  emit('# Top-level commands');
@@ -204,9 +263,9 @@ function generateFish(root: CmdInfo): string {
      emit('');
    }

-    // Options for this command
+    // Options for this command (use __in_cmd so options complete even after positional args)
    if (cmd.options.length > 0) {
-      const condition = `__${BIN}_using_cmd ${path.join(' ')}`;
+      const condition = `__${BIN}_in_cmd ${path.join(' ')}`;
      emit(`# ${path.join(' ')} options`);
      for (const opt of cmd.options) {
        const parts = [`complete -c ${BIN} -n "${condition}"`];
--- a/bastion/scripts/test-integration.sh
+++ b/bastion/scripts/test-integration.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+# Run integration tests inside a Node container with access to host libvirt.
+#
+# Usage: sudo ./scripts/test-integration.sh [vitest args...]
+# Example: sudo ./scripts/test-integration.sh -t k3s
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
+
+# Detect real user (even when running via sudo)
+REAL_USER="${SUDO_USER:-$(whoami)}"
+REAL_HOME="/home/${REAL_USER}"
+
+echo "==> Running integration tests in container"
+echo "    Project: ${PROJECT_ROOT}"
+echo "    User: ${REAL_USER}"
+echo "    SSH key: ${REAL_HOME}/.ssh/"
+echo ""
+
+# Check prerequisites
+if ! command -v podman &>/dev/null && ! command -v docker &>/dev/null; then
+  echo "ERROR: podman or docker required"
+  exit 1
+fi
+
+RUNTIME="podman"
+if ! command -v podman &>/dev/null; then
+  RUNTIME="docker"
+fi
+
+# Check libvirt socket
+if [ ! -S /var/run/libvirt/libvirt-sock ]; then
+  echo "ERROR: libvirt socket not found at /var/run/libvirt/libvirt-sock"
+  echo "       Is libvirtd running? Try: sudo systemctl start libvirtd"
+  exit 1
+fi
+
+# Create a temp dir for cloud-init artifacts (avoids SELinux /tmp relabel)
+WORK_TMP="/var/tmp/lab-integration-$$"
+mkdir -p "${WORK_TMP}"
+trap "rm -rf ${WORK_TMP}" EXIT
+
+exec $RUNTIME run --rm \
+  --name lab-integration-test \
+  --privileged \
+  --security-opt label=disable \
+  --network=host \
+  -v "${PROJECT_ROOT}:${PROJECT_ROOT}" \
+  -v "${REAL_HOME}/.ssh:${REAL_HOME}/.ssh:ro" \
+  -v "/var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock" \
+  -v "/var/lib/libvirt/images:/var/lib/libvirt/images" \
+  -v "${WORK_TMP}:/tmp/lab-integration-tests" \
+  -w "${PROJECT_ROOT}" \
+  -e "SSH_KEY_PATH=${REAL_HOME}/.ssh/id_rsa" \
+  -e "HOME=${REAL_HOME}" \
+  node:22-bookworm \
+  bash -c "
+    # Install system deps for libvirt client + cloud-init ISO creation
+    apt-get update -qq && apt-get install -y -qq libvirt-clients virtinst genisoimage openssh-client qemu-utils sudo >/dev/null 2>&1
+
+    # Install pnpm
+    corepack enable && corepack prepare pnpm@9 --activate >/dev/null 2>&1
+
+    echo '==> Installing project dependencies...'
+    pnpm install --frozen-lockfile 2>/dev/null
+
+    echo '==> Running integration tests...'
+    echo ''
+    pnpm run test:integration $*
+  "
--- a/bastion/scripts/test-provision.sh
+++ b/bastion/scripts/test-provision.sh
@@ -0,0 +1,131 @@
+#!/bin/bash
+# Run PXE and/or ISO boot integration tests.
+#
+# Usage:
+#   sudo ./scripts/test-provision.sh          # run both PXE + ISO tests
+#   sudo ./scripts/test-provision.sh pxe      # PXE only
+#   sudo ./scripts/test-provision.sh iso      # ISO only
+#
+# Prerequisites:
+#   libvirtd, OVMF (edk2-ovmf), iPXE (ipxe-bootimgs-x86),
+#   dnsmasq, xorriso, mtools, virt-install, qemu-img
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
+
+cd "$PROJECT_ROOT"
+
+# Detect real user for SSH keys
+REAL_USER="${SUDO_USER:-$(whoami)}"
+REAL_HOME=$(getent passwd "$REAL_USER" | cut -d: -f6)
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BOLD='\033[1m'
+RESET='\033[0m'
+
+echo ""
+echo -e "${BOLD}Lab Bastion -- Provision Integration Tests${RESET}"
+echo "==========================================="
+echo ""
+
+# --- Prerequisite checks ---
+MISSING=""
+for cmd in virsh virt-install qemu-img dnsmasq xorriso mformat mcopy curl; do
+    if ! command -v "$cmd" &>/dev/null; then
+        MISSING="$MISSING $cmd"
+    fi
+done
+
+if [ -n "$MISSING" ]; then
+    echo -e "${RED}Missing tools:${RESET}$MISSING"
+    echo "Install: sudo dnf install libvirt virt-install qemu-img dnsmasq xorriso mtools curl"
+    exit 1
+fi
+
+if ! systemctl is-active libvirtd &>/dev/null; then
+    echo -e "${RED}libvirtd not running.${RESET} Start with: sudo systemctl start libvirtd"
+    exit 1
+fi
+
+if [ ! -f /usr/share/edk2/ovmf/OVMF_CODE.fd ]; then
+    echo -e "${RED}OVMF firmware not found.${RESET} Install: sudo dnf install edk2-ovmf"
+    exit 1
+fi
+
+IPXE_EFI=""
+for f in /usr/share/ipxe/ipxe-snponly-x86_64.efi /usr/share/ipxe/ipxe-snp-x86_64.efi /usr/share/ipxe/ipxe-x86_64.efi; do
+    [ -f "$f" ] && IPXE_EFI="$f" && break
+done
+if [ -z "$IPXE_EFI" ]; then
+    echo -e "${RED}iPXE EFI binary not found.${RESET} Install: sudo dnf install ipxe-bootimgs-x86"
+    exit 1
+fi
+
+# Find SSH key
+SSH_KEY=""
+for name in id_ed25519 id_ecdsa id_rsa; do
+    if [ -f "$REAL_HOME/.ssh/$name" ] && [ -f "$REAL_HOME/.ssh/$name.pub" ]; then
+        SSH_KEY="$REAL_HOME/.ssh/$name"
+        break
+    fi
+done
+if [ -z "$SSH_KEY" ]; then
+    echo -e "${RED}No SSH key found in $REAL_HOME/.ssh/${RESET}"
+    exit 1
+fi
+
+echo -e "  User:    ${BOLD}$REAL_USER${RESET}"
+echo -e "  SSH key: ${BOLD}$SSH_KEY${RESET}"
+echo -e "  iPXE:    ${BOLD}$IPXE_EFI${RESET}"
+echo ""
+
+# --- Determine which tests to run ---
+MODE="${1:-both}"
+
+run_test() {
+    local name="$1" pattern="$2"
+    echo ""
+    echo -e "${YELLOW}━━━ Running $name test ━━━${RESET}"
+    echo ""
+
+    if SSH_KEY_PATH="$SSH_KEY" HOME="$REAL_HOME" \
+       npx vitest run -c tests/integration/vitest.config.ts -t "$pattern" 2>&1; then
+        echo ""
+        echo -e "${GREEN}✔ $name test passed${RESET}"
+        return 0
+    else
+        echo ""
+        echo -e "${RED}✘ $name test failed${RESET}"
+        return 1
+    fi
+}
+
+FAILED=0
+
+case "$MODE" in
+    pxe)
+        run_test "PXE boot" "PXE boot" || FAILED=1
+        ;;
+    iso)
+        run_test "ISO boot" "ISO boot" || FAILED=1
+        ;;
+    both|all)
+        run_test "PXE boot" "PXE boot" || FAILED=1
+        run_test "ISO boot" "ISO boot" || FAILED=1
+        ;;
+    *)
+        echo "Usage: $0 [pxe|iso|both]"
+        exit 1
+        ;;
+esac
+
+echo ""
+if [ "$FAILED" -eq 0 ]; then
+    echo -e "${GREEN}${BOLD}All provision tests passed.${RESET}"
+else
+    echo -e "${RED}${BOLD}Some tests failed.${RESET}"
+    exit 1
+fi