fix(k3s): route audit logs through journald, codify etcd member recovery

Two changes prompted by today's etcd raft panic on worker1-k8s0
(tocommit out of range, lost-write on follower) and the cascading
disk pressure that surfaced underneath it.

Audit logs to journald
- kube-apiserver now uses audit-log-path=- so audit events flow to
  k3s.service stdout and into journald instead of growing files in
  /var/log/kubernetes. The previous setup combined apiserver's
  internal rotation with a logrotate *.log glob that double-rotated
  the rotated files into permanent orphans (observed: 7+ GB).
- New journald-limits operation writes a SystemMaxUse=2G drop-in so
  audit volume cannot fill /var/log even under bursty load.
- log-rotation operation repurposed to decommission the obsolete
  logrotate rule and reap leftover audit files. Idempotent: no-op
  on fresh installs.

Etcd member recovery
- New recoverEtcdMember(broken, peer, hostname) codifies the
  documented k3s recovery: stop k3s, etcdctl member remove, wipe
  /var/lib/rancher/k3s/server/{db,tls,cred}, restart, poll for
  rejoin. Refuses to operate when cluster size < 3 to preserve
  quorum.

Tests
- 7 new unit tests covering both decommission paths and the
  recovery procedure (54 total, all green).
- install.test.ts asserts the file-based audit args are gone.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

CLAUDE.md

@@ -0,0 +1,19 @@
+## Skill routing
+
+When the user's request matches an available skill, ALWAYS invoke it using the Skill
+tool as your FIRST action. Do NOT answer directly, do NOT use other tools first.
+The skill has specialized workflows that produce better results than ad-hoc answers.
+
+Key routing rules:
+- Product ideas, "is this worth building", brainstorming → invoke gstack-office-hours
+- Bugs, errors, "why is this broken", 500 errors → invoke gstack-investigate
+- Ship, deploy, push, create PR → invoke gstack-ship
+- QA, test the site, find bugs → invoke gstack-qa
+- Code review, check my diff → invoke gstack-review
+- Update docs after shipping → invoke gstack-document-release
+- Weekly retro → invoke gstack-retro
+- Design system, brand → invoke gstack-design-consultation
+- Visual audit, design polish → invoke gstack-design-review
+- Architecture review → invoke gstack-plan-eng-review
+- Save progress, checkpoint, resume → invoke gstack-checkpoint
+- Code quality, health check → invoke gstack-health

TODOS.md

@@ -0,0 +1,47 @@
+# TODOS
+
+## P1 — Ship with Phase 1
+
+### v2.0 Architecture Document Update
+Update `bastion/docs/ARCHITECTURE.md` to cover v2.0: driver model, fleet system,
+Pulumi integration, Vault secrets, Deno evaluator, new CLI grammar. The existing
+doc covers v1.0 comprehensively (432 lines). v2.0 adds 5+ major subsystems.
+**Effort:** M (human: 1 week / CC: 1-2 days)
+**Depends on:** Phase 1 complete
+**Source:** CEO review 2026-04-01
+
+## P2 — Post-v2.0 Core
+
+### SSH Emergency Mode (scoped)
+SSH-based operations limited to: (1) earliest necessary box provisioning before agent
+is installed, and (2) emergency debugging/fixing operations that can't be done via agent.
+NOT a general-purpose DeploymentTarget alternative. The v1.0 `recheck` and `fix-ssh-root.sh`
+patterns are the model. Agent stays the primary management path.
+**Effort:** S (human: 1 week / CC: 1 day)
+**Depends on:** Phase 2 complete (DeploymentTarget interface exists)
+**Source:** CEO review 2026-04-01
+
+### Prometheus Metrics Endpoint
+Add `/metrics` endpoint to labd: resource counts by status, apply duration histograms,
+driver operation latency, fleet pipeline completion rates. Standard Prometheus scraping
+for Grafana dashboards and alerting.
+**Effort:** S (human: 2-3 days / CC: 2-3 hours)
+**Depends on:** Phase 1 (labd exists with resource store)
+**Source:** CEO review 2026-04-01 (observability gap)
+
+## P3 — Future Enhancements
+
+### Infrastructure Graph Visualization
+Visual representation of resource dependencies, environment topology, fleet status.
+Could be a web UI or terminal-based (like `kubectl tree`).
+**Source:** CEO review 2026-04-01
+
+### `labctl import` for Existing Cloud Resources
+Discover and import existing AWS/GCP resources into the state store.
+Pulumi's import functionality could be leveraged.
+**Source:** CEO review 2026-04-01
+
+### Built-in Secrets Rotation
+Automatic rotation of managed secrets (database passwords, API keys).
+Vault handles rotation but a labctl-native workflow could simplify.
+**Source:** CEO review 2026-04-01

bastion/bastion/.gitignore

@@ -0,0 +1,4 @@
+
+# Asahi build artifacts (large)
+.asahi-cache/
+asahi-repo/*.zip

bastion/completions/labctl.bash

@@ -82,6 +82,9 @@ _labctl() {
     "provision makeiso")
       COMPREPLY=($(compgen -W "--arch --local --out -h --help" -- "$cur"))
       return ;;
+    "provision recheck")
+      COMPREPLY=($(compgen -W "--user --target -h --help" -- "$cur"))
+      return ;;
     "config list")
       COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
       return ;;
@@ -107,7 +110,7 @@ _labctl() {
       COMPREPLY=($(compgen -W "bastion -h --help" -- "$cur"))
       return ;;
     "provision")
-      COMPREPLY=($(compgen -W "list install reprovision debug forget register asahi logs makeiso -h --help" -- "$cur"))
+      COMPREPLY=($(compgen -W "list install reprovision debug forget register asahi logs makeiso recheck -h --help" -- "$cur"))
       return ;;
     "config")
       COMPREPLY=($(compgen -W "list get set path -h --help" -- "$cur"))

bastion/completions/labctl.fish

@@ -128,6 +128,7 @@ complete -c labctl -n "__labctl_using_cmd provision" -a register -d 'Register an
 complete -c labctl -n "__labctl_using_cmd provision" -a asahi -d 'Show instructions to provision an Apple Silicon Mac with Asahi Linux'
 complete -c labctl -n "__labctl_using_cmd provision" -a logs -d 'Show provisioning logs for a machine (hostname, MAC, or IP)'
 complete -c labctl -n "__labctl_using_cmd provision" -a makeiso -d 'Generate a UEFI-bootable iPXE ISO for network provisioning'
+complete -c labctl -n "__labctl_using_cmd provision" -a recheck -d 'Refresh hardware info for all installed machines via SSH'
 
 # provision install options
 complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
@@ -154,6 +155,10 @@ complete -c labctl -n "__labctl_in_cmd provision makeiso" -l arch -d 'Target arc
 complete -c labctl -n "__labctl_in_cmd provision makeiso" -l local -d 'Build ISO locally instead of using bastion-hosted URL'
 complete -c labctl -n "__labctl_in_cmd provision makeiso" -l out -d 'Output path for local ISO build' -x
 
+# provision recheck options
+complete -c labctl -n "__labctl_in_cmd provision recheck" -l user -d 'SSH user' -x
+complete -c labctl -n "__labctl_in_cmd provision recheck" -l target -d 'Only recheck a specific machine (by hostname or MAC)' -x
+
 # config subcommands
 complete -c labctl -n "__labctl_using_cmd config" -a list -d 'Show all configuration values'
 complete -c labctl -n "__labctl_using_cmd config" -a get -d 'Get a configuration value'

bastion/scripts/build-bastion.sh

@@ -99,16 +99,22 @@ if [ "$PUSH" = true ]; then
     fi
   fi
 
+  # Use --tls-verify=false for plain HTTP registries (e.g. 10.0.0.194:3012)
+  TLS_FLAG=""
+  if [[ "$REGISTRY" =~ ^[0-9] ]] || [[ "$REGISTRY" =~ ^localhost ]]; then
+    TLS_FLAG="--tls-verify=false"
+  fi
+
   echo "==> Logging in to $REGISTRY..."
-  podman login -u michal -p "$GITEA_TOKEN" "$REGISTRY"
+  podman login $TLS_FLAG -u michal -p "$GITEA_TOKEN" "$REGISTRY"
 
   echo "==> Pushing $FULL_IMAGE:$TAG..."
-  podman manifest push --all "$MANIFEST" "docker://$FULL_IMAGE:$TAG"
+  podman manifest push --all $TLS_FLAG "$MANIFEST" "docker://$FULL_IMAGE:$TAG"
 
   # Also tag as :latest if not already
   if [ "$TAG" != "latest" ]; then
     echo "==> Also pushing as :latest..."
-    podman manifest push --all "$MANIFEST" "docker://$FULL_IMAGE:latest"
+    podman manifest push --all $TLS_FLAG "$MANIFEST" "docker://$FULL_IMAGE:latest"
   fi
 
   # Link package to repository if script exists

bastion/scripts/build-labd.sh

@@ -92,15 +92,21 @@ if [ "$PUSH" = true ]; then
     fi
   fi
 
+  # Use --tls-verify=false for plain HTTP registries (e.g. 10.0.0.194:3012)
+  TLS_FLAG=""
+  if [[ "$REGISTRY" =~ ^[0-9] ]] || [[ "$REGISTRY" =~ ^localhost ]]; then
+    TLS_FLAG="--tls-verify=false"
+  fi
+
   echo "==> Logging in to $REGISTRY..."
-  podman login -u michal -p "$GITEA_TOKEN" "$REGISTRY"
+  podman login $TLS_FLAG -u michal -p "$GITEA_TOKEN" "$REGISTRY"
 
   echo "==> Pushing $FULL_IMAGE:$TAG..."
-  podman manifest push --all "$MANIFEST" "docker://$FULL_IMAGE:$TAG"
+  podman manifest push --all $TLS_FLAG "$MANIFEST" "docker://$FULL_IMAGE:$TAG"
 
   if [ "$TAG" != "latest" ]; then
     echo "==> Also pushing as :latest..."
-    podman manifest push --all "$MANIFEST" "docker://$FULL_IMAGE:latest"
+    podman manifest push --all $TLS_FLAG "$MANIFEST" "docker://$FULL_IMAGE:latest"
   fi
 
   if [ -f "$SCRIPT_DIR/link-package.sh" ]; then

bastion/scripts/fix-ssh-root.sh

@@ -0,0 +1,131 @@
+#!/bin/bash
+# Fix root SSH access on all provisioned machines.
+# Tries root, lab, michal users to find one that works,
+# then ensures root has the SSH key and PermitRootLogin is enabled.
+set -euo pipefail
+
+SSH_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDMJ3FkUGbG174eoO5RjZd2eNV680FM5pgp0AgpW/QwlJExK3qxMk0DJSr4ICmzGUx4yujAXcrqU1otcOMPzzFzwc5heWpSmlNHU3TIW6NHEt0sF9ZTAbGLw2zSw3si5UouqFkCcENA40mePFJqY+Q9R8N1uvLgu4m/do+Zrn/mk5Ewc1V7OCRE5Acrnaec4T7LTB0BuVXcjPUfAmZ0q5fI+bKPR1q2Kc3+IeGhVkBuZ9OJVeXXhnpedm0uEbLeriK/jUYKYw/1QhsNDM8Tyty+UIGr9QVnWwzCMHB+wuQcDYC9mPGTqg0fYwX8Mp8xMi1PPxdsh1G7bj/cpWMAF43KswWORF2ul8ICGbaE1zEgIYXO790SuBjpBHhaC6Iegqi58hmCuP+a9893q/EU9HyrWTJHCZXC5E4kP1MsM57KrhEpszM6I3sW9f9zMTPd5QsCXFi4si4OMwX4kYNVu3fQGQPpseDPlTTSrT6uUdqj4Irm0c1m9cYTmK0vYgsM3ss= michal@fedora"
+
+SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5"
+USERS_TO_TRY=(root lab michal)
+
+# Machines: hostname ip
+MACHINES=(
+    "labmaster 192.168.8.11"
+    "worker0-k8s0 192.168.8.23"
+    "worker1-k8s0 192.168.8.13"
+    "worker2-k8s0 192.168.8.25"
+    "spark-2935 192.168.8.12"
+)
+
+BOLD="\033[1m"
+GREEN="\033[0;32m"
+RED="\033[0;31m"
+DIM="\033[2m"
+RESET="\033[0m"
+
+# Script to run on each machine (via sudo if needed)
+read -r -d '' FIX_SCRIPT << 'FIXEOF' || true
+#!/bin/bash
+set -e
+KEY="$1"
+
+# 1. Ensure root .ssh dir exists
+mkdir -p /root/.ssh
+chmod 700 /root/.ssh
+touch /root/.ssh/authorized_keys
+chmod 600 /root/.ssh/authorized_keys
+
+# 2. Add key if not present
+if ! grep -qF "$KEY" /root/.ssh/authorized_keys 2>/dev/null; then
+    echo "$KEY" >> /root/.ssh/authorized_keys
+    echo "KEY_ADDED"
+else
+    echo "KEY_EXISTS"
+fi
+
+# 3. Fix sshd_config for root login with keys
+SSHD_CONF="/etc/ssh/sshd_config"
+CHANGED=0
+
+# Ensure PermitRootLogin allows key auth
+CURRENT=$(grep -E "^PermitRootLogin" "$SSHD_CONF" 2>/dev/null | tail -1 || true)
+if [ "$CURRENT" = "PermitRootLogin prohibit-password" ] || [ "$CURRENT" = "PermitRootLogin without-password" ]; then
+    echo "SSHD_OK"
+elif [ "$CURRENT" = "PermitRootLogin yes" ]; then
+    echo "SSHD_OK"
+else
+    # Remove any existing PermitRootLogin lines
+    sed -i '/^#*PermitRootLogin/d' "$SSHD_CONF"
+    echo "PermitRootLogin prohibit-password" >> "$SSHD_CONF"
+    CHANGED=1
+    echo "SSHD_FIXED"
+fi
+
+# Ensure PubkeyAuthentication is enabled
+if grep -qE "^PubkeyAuthentication no" "$SSHD_CONF" 2>/dev/null; then
+    sed -i 's/^PubkeyAuthentication no/PubkeyAuthentication yes/' "$SSHD_CONF"
+    CHANGED=1
+    echo "PUBKEY_FIXED"
+else
+    echo "PUBKEY_OK"
+fi
+
+# Restart sshd if changed
+if [ "$CHANGED" -eq 1 ]; then
+    systemctl restart sshd 2>/dev/null || systemctl restart ssh 2>/dev/null || true
+    echo "SSHD_RESTARTED"
+fi
+
+# 4. Verify root can be reached
+echo "DONE"
+FIXEOF
+
+echo ""
+echo -e "${BOLD}Fixing root SSH access on all machines...${RESET}"
+echo ""
+
+for entry in "${MACHINES[@]}"; do
+    read -r hostname ip <<< "$entry"
+    printf "  %-24s ${DIM}(%s)${RESET}  " "$hostname" "$ip"
+
+    # Try each user until one works
+    WORKING_USER=""
+    for user in "${USERS_TO_TRY[@]}"; do
+        if ssh $SSH_OPTS "$user@$ip" "true" 2>/dev/null; then
+            WORKING_USER="$user"
+            break
+        fi
+    done
+
+    if [ -z "$WORKING_USER" ]; then
+        echo -e "${RED}UNREACHABLE${RESET} (tried: ${USERS_TO_TRY[*]})"
+        continue
+    fi
+
+    # Run fix script (with sudo if not root)
+    if [ "$WORKING_USER" = "root" ]; then
+        RESULT=$(ssh $SSH_OPTS "root@$ip" "bash -s -- '$SSH_KEY'" <<< "$FIX_SCRIPT" 2>&1)
+    else
+        RESULT=$(ssh $SSH_OPTS "$WORKING_USER@$ip" "sudo bash -s -- '$SSH_KEY'" <<< "$FIX_SCRIPT" 2>&1)
+    fi
+
+    # Parse result
+    DETAILS=""
+    if echo "$RESULT" | grep -q "KEY_ADDED"; then DETAILS="key added"; fi
+    if echo "$RESULT" | grep -q "KEY_EXISTS"; then DETAILS="key ok"; fi
+    if echo "$RESULT" | grep -q "SSHD_FIXED"; then DETAILS="$DETAILS, sshd fixed"; fi
+    if echo "$RESULT" | grep -q "SSHD_OK"; then DETAILS="$DETAILS, sshd ok"; fi
+    if echo "$RESULT" | grep -q "SSHD_RESTARTED"; then DETAILS="$DETAILS, restarted"; fi
+
+    # Verify root works now
+    if ssh $SSH_OPTS "root@$ip" "true" 2>/dev/null; then
+        echo -e "${GREEN}OK${RESET} ${DIM}(via $WORKING_USER: $DETAILS)${RESET}"
+    else
+        echo -e "${RED}PARTIAL${RESET} ${DIM}(via $WORKING_USER: $DETAILS -- root still blocked)${RESET}"
+    fi
+done
+
+echo ""
+echo -e "${BOLD}Done.${RESET} Verify: labctl provision recheck --user root"
+echo ""

bastion/src/bastion/src/main.ts

@@ -309,6 +309,32 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
       return { status: "ok", data: { mac, hostname: msg.hostname } };
     });
 
+    labdConn.onCommand("command-discover", async (msg) => {
+      if (msg.type !== "command-discover") throw new Error("unexpected");
+      const mac = (msg.mac as string).toLowerCase();
+      const now = new Date().toISOString();
+      const existing = state.load().discovered[mac];
+      state.update((s) => {
+        s.discovered[mac] = {
+          mac,
+          product: (msg.product as string) ?? "unknown",
+          board: (msg.board as string) ?? "unknown",
+          serial: (msg.serial as string) ?? "unknown",
+          manufacturer: (msg.manufacturer as string) ?? "unknown",
+          cpu_model: (msg.cpu_model as string) ?? "unknown",
+          cpu_cores: (msg.cpu_cores as number) ?? 0,
+          memory_gb: (msg.memory_gb as number) ?? 0,
+          arch: (msg.arch as string) ?? "unknown",
+          disks: (msg.disks as Array<{ name: string; size_gb: number; model: string }>) ?? [],
+          nics: (msg.nics as Array<{ name: string; mac: string; state: string }>) ?? [],
+          first_seen: existing?.first_seen ?? now,
+          last_seen: now,
+        };
+      });
+      logger.info(`HARDWARE UPDATED: ${mac} -- ${msg.manufacturer ?? "?"} ${msg.product ?? "?"} (${msg.cpu_model ?? "?"}, ${msg.cpu_cores ?? "?"} cores, ${msg.memory_gb ?? "?"}GB RAM)`);
+      return { status: "ok", data: { mac } };
+    });
+
     labdConn.onCommand("command-role-update", async (msg) => {
       if (msg.type !== "command-role-update") throw new Error("unexpected");
       const mac = msg.mac.toLowerCase();

bastion/src/bastion/src/routes/api.ts

@@ -139,12 +139,22 @@ export function registerApiRoutes(
             ? detailStr.replace("ready at ", "").trim()
             : "";
 
+          const hw = s.discovered[mac];
           const installedInfo: InstalledInfo = {
             hostname: cfg?.hostname ?? "?",
             role: cfg?.role ?? "?",
             ...(cfg?.os !== undefined ? { os: cfg.os } : {}),
             ip,
             installed_at: new Date().toISOString(),
+            // Preserve hardware info from discovery
+            ...(hw ? {
+              product: hw.product,
+              manufacturer: hw.manufacturer,
+              cpu_model: hw.cpu_model,
+              cpu_cores: hw.cpu_cores,
+              memory_gb: hw.memory_gb,
+              arch: hw.arch,
+            } : {}),
           };
           s.installed[mac] = installedInfo;
 
@@ -359,6 +369,23 @@ export function registerApiRoutes(
     });
   });
 
+  // Simple machine state query (used by ks-auto for ISO boot dispatch)
+  app.get<{
+    Params: { mac: string };
+  }>("/api/machine-state/:mac", async (request, reply) => {
+    const mac = request.params.mac.toLowerCase().replace(/-/g, ":");
+    const currentState = state.load();
+
+    if (currentState.debug[mac]) return reply.send("debug");
+    if (currentState.install_queue[mac]) {
+      const progress = currentState.install_queue[mac].progress;
+      return reply.send(progress ? "installing" : "queued");
+    }
+    if (currentState.installed[mac]) return reply.send("installed");
+    if (currentState.discovered[mac]) return reply.send("discovered");
+    return reply.send("unknown");
+  });
+
   // Update a machine's role (e.g. promote infra -> labcontroller)
   app.post<{
     Body: {

bastion/src/bastion/src/routes/asahi.ts

@@ -102,7 +102,8 @@ echo "    - Standard Asahi boot infrastructure (m1n1 + U-Boot)"
 echo "    - Fedora Asahi Remix root partition"
 echo "    - LVM data partition (remaining space)"
 echo ""
-echo "  On first boot, LVM volumes are created automatically."
+echo "  After first boot, SSH in and set up LVM:"
+echo "    ssh lab@<ip> 'curl -sf \${BASTION}/asahi/firstboot.sh | sudo bash'"
 echo ""
 
 # Run the installer
@@ -150,10 +151,10 @@ fi
   app.get<{
     Querystring: { hostname?: string; role?: string; mac?: string; user?: string };
   }>("/asahi/firstboot.sh", async (request, reply) => {
-    const hostname = request.query.hostname ?? "mac-studio";
+    const hostname = request.query.hostname ?? "unknown";
     const role = (request.query.role ?? "infra") as Role;
     const mac = request.query.mac ?? "unknown";
-    const user = request.query.user ?? config.adminUser;
+    const user = request.query.user ?? "lab";
 
     const script = renderFirstbootScript({
       hostname,

bastion/src/bastion/src/routes/boot-iso.ts

@@ -137,7 +137,7 @@ function generateIso(config: BastionConfig, outputPath: string): void {
     "# Map iPXE arch names to Fedora mirror paths (arm64 -> aarch64)",
     "set fedarch ${buildarch}",
     "iseq ${buildarch} arm64 && set fedarch aarch64 ||",
-    `kernel file:/vmlinuz-\${buildarch} inst.ks=${bastionUrl}/discover.ks inst.repo=${FEDORA_MIRROR_BASE}/${config.fedoraVersion}/Everything/\${fedarch}/os inst.text || goto no_kernel`,
+    `kernel file:/vmlinuz-\${buildarch} inst.ks=${bastionUrl}/ks-auto inst.repo=${FEDORA_MIRROR_BASE}/${config.fedoraVersion}/Everything/\${fedarch}/os inst.text || goto no_kernel`,
     `initrd file:/initrd-\${buildarch} || goto no_kernel`,
     "boot || shell",
     "",

bastion/src/bastion/src/routes/kickstart.ts

@@ -41,6 +41,150 @@ export function registerKickstartRoutes(
     return reply.type("text/plain").send(ks);
   });
 
+  // Auto-detecting kickstart for ISO boot (no-network machines like R1 ARM).
+  // %pre detects MAC, queries bastion state, writes dynamic kickstart to /tmp.
+  // Main body %include's it — so Anaconda gets either discover or install content.
+  app.get("/ks-auto", async (_request, reply) => {
+    const bastionUrl = `http://${config.serverIp}:${config.httpPort}`;
+
+    const ks = `# Lab Bastion -- Auto-detect kickstart (ISO boot)
+# %pre detects MAC, queries bastion state, writes /tmp/dynamic.ks.
+# Main body %include's it to get either discovery reboot or full install.
+
+%pre --erroronfail --log=/tmp/ks-auto.log
+#!/bin/bash
+set -x
+
+# -- Detect MAC address --
+MAC=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')
+echo "Detected MAC: $MAC"
+
+# -- Wait for network (Linux drivers may take a moment) --
+for i in $(seq 1 30); do
+    if curl -sf "${bastionUrl}/healthz" >/dev/null 2>&1; then
+        echo "Bastion reachable at ${bastionUrl}"
+        break
+    fi
+    echo "Waiting for network... ($i/30)"
+    sleep 2
+done
+
+# -- Query bastion for machine state --
+STATE=$(curl -sf "${bastionUrl}/api/machine-state/$MAC" 2>/dev/null || echo "unknown")
+echo "Machine state: $STATE"
+
+case "$STATE" in
+    queued|installing)
+        echo "=== Machine queued for install. Fetching install kickstart... ==="
+        curl -sf "${bastionUrl}/ks?mac=$MAC" > /tmp/dynamic.ks
+        if [ -s /tmp/dynamic.ks ]; then
+            echo "Install kickstart downloaded ($(wc -l < /tmp/dynamic.ks) lines)"
+        else
+            echo "ERROR: Failed to download install kickstart"
+            exit 1
+        fi
+
+        # Run any %pre scripts from the downloaded kickstart.
+        # Anaconda only runs %pre from the top-level file, not from %include'd files.
+        python3 -c "
+import re, subprocess
+content = open('/tmp/dynamic.ks').read()
+blocks = re.findall(r'%pre[^\\n]*\\n(.*?)%end', content, re.DOTALL)
+for i, script in enumerate(blocks):
+    path = f'/tmp/inner-pre-{i}.sh'
+    with open(path, 'w') as f:
+        f.write(script)
+    print(f'Running inner %pre script {i} ({len(script.splitlines())} lines)')
+    subprocess.run(['bash', path], check=False)
+"
+        ;;
+
+    debug)
+        echo "=== Debug mode ==="
+        curl -sf "${bastionUrl}/debug.ks?mac=$MAC" > /tmp/dynamic.ks 2>/dev/null
+        if [ ! -s /tmp/dynamic.ks ]; then
+            echo "rescue" > /tmp/dynamic.ks
+        fi
+        ;;
+
+    *)
+        echo "=== Running hardware discovery ==="
+        # Collect hardware info
+        PRODUCT=$(cat /sys/class/dmi/id/product_name 2>/dev/null || echo "unknown")
+        BOARD=$(cat /sys/class/dmi/id/board_name 2>/dev/null || echo "unknown")
+        SERIAL=$(cat /sys/class/dmi/id/product_serial 2>/dev/null || echo "unknown")
+        MANUFACTURER=$(cat /sys/class/dmi/id/sys_vendor 2>/dev/null || echo "unknown")
+        CPUMODEL=$(grep -m1 'model name' /proc/cpuinfo | cut -d: -f2 | sed 's/^ //')
+        CPUCORES=$(grep -c '^processor' /proc/cpuinfo)
+        MEMGB=$(awk '/MemTotal/ {printf "%d", $2/1024/1024}' /proc/meminfo)
+        ARCHTYPE=$(uname -m)
+
+        DISKS_JSON=$(lsblk -Jb -o NAME,SIZE,TYPE,MODEL 2>/dev/null | python3 -c "
+import sys, json
+data = json.load(sys.stdin)
+disks = [d for d in data.get('blockdevices', []) if d.get('type') == 'disk']
+result = []
+for d in disks:
+    size_gb = round(int(d.get('size', 0)) / 1073741824, 1)
+    result.append({'name': d.get('name', '?'), 'size_gb': size_gb, 'model': (d.get('model') or 'unknown').strip()})
+print(json.dumps(result))
+" 2>/dev/null || echo '[]')
+
+        NICS_JSON=$(ip -j link show 2>/dev/null | python3 -c "
+import sys, json
+nics = json.load(sys.stdin)
+result = []
+for n in nics:
+    if n.get('link_type') == 'loopback': continue
+    result.append({'name': n.get('ifname', '?'), 'mac': n.get('address', '?'), 'state': n.get('operstate', '?')})
+print(json.dumps(result))
+" 2>/dev/null || echo '[]')
+
+        PAYLOAD=$(python3 -c "
+import json
+print(json.dumps({
+    'mac': '$MAC', 'product': '$PRODUCT', 'board': '$BOARD', 'serial': '$SERIAL',
+    'manufacturer': '$MANUFACTURER', 'cpu_model': '$CPUMODEL',
+    'cpu_cores': int('$CPUCORES' or 0), 'memory_gb': int('$MEMGB' or 0),
+    'arch': '$ARCHTYPE', 'disks': $DISKS_JSON, 'nics': $NICS_JSON
+}))
+")
+
+        curl -sf -X POST "${bastionUrl}/api/discover" \\
+            -H "Content-Type: application/json" \\
+            -d "$PAYLOAD" || true
+
+        echo ""
+        echo "=== Discovery complete ==="
+        echo "Machine MAC: $MAC"
+        echo "Queue for install: labctl provision install $MAC <hostname> --role infra"
+        echo "Then reboot to start installation."
+        echo ""
+
+        # Write a minimal kickstart that just reboots
+        cat > /tmp/dynamic.ks << 'DISCOVER_KS'
+# Discovery mode -- reboot to allow install queue
+reboot
+DISCOVER_KS
+
+        # Force reboot now (don't wait for Anaconda)
+        sleep 3
+        echo 1 > /proc/sys/kernel/sysrq
+        echo b > /proc/sysrq-trigger
+        sleep 5
+        reboot -f
+        ;;
+esac
+
+%end
+
+# Include the dynamically chosen kickstart
+%include /tmp/dynamic.ks
+`;
+
+    return reply.type("text/plain").send(ks);
+  });
+
   // Ubuntu autoinstall user-data (cloud-init)
   app.get<{ Params: { mac: string } }>("/autoinstall/:mac/user-data", async (request, reply) => {
     const mac = request.params.mac.toLowerCase().replace(/-/g, ":");

bastion/src/bastion/src/services/labd-connection.ts

@@ -166,6 +166,7 @@ export class BastionConnection {
       case "command-role-update":
       case "command-debug":
       case "command-register":
+      case "command-discover":
         void this.handleCommand(msg);
         break;
     }

bastion/src/bastion/src/templates/asahi-firstboot.sh.ts

@@ -111,6 +111,29 @@ mount_lv() {
     fi
 }
 
+# ── Write fstab function (idempotent) ────────────────────────────
+write_lab_fstab() {
+    # Remove any previous lab LVM entries (clean slate)
+    sed -i '/# lab-lvm:/d' /etc/fstab
+    sed -i '/# Lab LVM volumes/d' /etc/fstab
+    grep -v "/dev/labvg/" /etc/fstab > /etc/fstab.tmp && mv /etc/fstab.tmp /etc/fstab
+    # Comment out non-LVM entries for mount points we manage
+    for mp in "/var " "/var/log " "/home " "/srv "; do
+        if grep -q "$mp" /etc/fstab; then
+            awk -v m="$mp" '{if($0 !~ /^#/ && index($0,m)) print "# lab-lvm: " $0; else print}' /etc/fstab > /etc/fstab.tmp
+            mv /etc/fstab.tmp /etc/fstab
+        fi
+    done
+    # Add fresh LVM entries
+    echo "# Lab LVM volumes" >> /etc/fstab
+    echo "/dev/labvg/swap    none         swap defaults 0 0" >> /etc/fstab
+    echo "/dev/labvg/var     /var         xfs  defaults 0 0" >> /etc/fstab
+    echo "/dev/labvg/varlog  /var/log     xfs  defaults 0 0" >> /etc/fstab
+    echo "/dev/labvg/home    /home        xfs  defaults 0 0" >> /etc/fstab
+    echo "/dev/labvg/srv     /srv         xfs  defaults 0 0" >> /etc/fstab
+    ${roleFstabLines.join('\n    ')}
+}
+
 # ── Check for existing VG ────────────────────────────────────────
 if vgs labvg &>/dev/null; then
     echo "Volume group 'labvg' already exists — reprovision detected."
@@ -129,22 +152,11 @@ ${roleMountLines.map(l => `    ${l}`).join('\n')}
         echo "  Enabled swap"
     fi
 
-    # Ensure fstab entries exist
+    # Ensure fstab entries exist — comment out conflicting btrfs subvol entries
-    grep -q "labvg" /etc/fstab || {
+    write_lab_fstab
-        echo "# Lab LVM volumes (re-added after reprovision)" >> /etc/fstab
-        echo "/dev/labvg/swap    none         swap defaults 0 0" >> /etc/fstab
-        echo "/dev/labvg/var     /var         xfs  defaults 0 0" >> /etc/fstab
-        echo "/dev/labvg/varlog  /var/log     xfs  defaults 0 0" >> /etc/fstab
-        echo "/dev/labvg/home    /home        xfs  defaults 0 0" >> /etc/fstab
-        echo "/dev/labvg/srv     /srv         xfs  defaults 0 0" >> /etc/fstab
-${roleFstabLines.map(l => `        ${l}`).join('\n')}
-    }
 
     echo "Existing LVM volumes re-mounted."
-    touch "$MARKER"
+else
-    exit 0
-fi
-
 # ── Fresh install: create LVM ────────────────────────────────────
 echo "Creating LVM on $DATA_PART..."
 
@@ -210,34 +222,39 @@ echo "NOTE: /var and /var/log will switch to LVM on next reboot."
 # Enable swap
 swapon /dev/labvg/swap 2>/dev/null || true
 
-# Write fstab entries
+write_lab_fstab
-echo "" >> /etc/fstab
-echo "# Lab LVM volumes" >> /etc/fstab
-echo "/dev/labvg/swap    none         swap defaults 0 0" >> /etc/fstab
-echo "/dev/labvg/var     /var         xfs  defaults 0 0" >> /etc/fstab
-echo "/dev/labvg/varlog  /var/log     xfs  defaults 0 0" >> /etc/fstab
-echo "/dev/labvg/home    /home        xfs  defaults 0 0" >> /etc/fstab
-echo "/dev/labvg/srv     /srv         xfs  defaults 0 0" >> /etc/fstab
-${roleFstabLines.join('\n')}
 
 echo "LVM setup complete."
 lvs labvg
 
-# ── Set hostname ─────────────────────────────────────────────────
+fi  # end if/else for reprovision vs fresh install
-hostnamectl set-hostname "${hostname}"
+
+# ── Set hostname (use configured value, or keep existing) ────────
+CONF_HOSTNAME="${hostname}"
+if [ "$CONF_HOSTNAME" != "unknown" ] && [ -n "$CONF_HOSTNAME" ]; then
+    hostnamectl set-hostname "$CONF_HOSTNAME"
+fi
+ACTUAL_HOSTNAME=$(hostname)
+
+# ── Detect MAC address ───────────────────────────────────────────
+CONF_MAC="${mac}"
+if [ "$CONF_MAC" = "unknown" ] || [ -z "$CONF_MAC" ]; then
+    CONF_MAC=$(ip -o link show | grep -v "lo:" | grep "state UP" | head -1 | grep -oP 'link/ether \\K[^ ]+' || echo "unknown")
+fi
 
 # ── Configure admin user ─────────────────────────────────────────
-if ! id "${adminUser}" &>/dev/null; then
+ADMIN="${adminUser}"
-    useradd -m -G wheel "${adminUser}"
+if ! id "$ADMIN" &>/dev/null; then
-    echo "${adminUser} ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/${adminUser}
+    useradd -m -G wheel "$ADMIN"
-    chmod 440 /etc/sudoers.d/${adminUser}
+    echo "$ADMIN ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$ADMIN
+    chmod 440 /etc/sudoers.d/$ADMIN
 fi
-ADMIN_SSH="/home/${adminUser}/.ssh"
+ADMIN_SSH="/home/$ADMIN/.ssh"
 mkdir -p "$ADMIN_SSH"
 chmod 700 "$ADMIN_SSH"
 ${sshKeyBlock}
 chmod 600 "$ADMIN_SSH/authorized_keys"
-chown -R ${adminUser}:${adminUser} "$ADMIN_SSH"
+chown -R $ADMIN:$ADMIN "$ADMIN_SSH"
 
 # Also authorize root
 mkdir -p /root/.ssh
@@ -246,14 +263,14 @@ ${rootSshKeyBlock}
 chmod 600 /root/.ssh/authorized_keys
 
 # ── Harden SSH (takes effect on next sshd restart/reboot) ────────
-sed -i 's/^#\\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config
+sed -i 's/^#*PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config
-sed -i 's/^#\\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config
+sed -i 's/^#*PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config
 
 # ── Write provisioning metadata ──────────────────────────────────
 cat > /etc/lab-provisioned << LABMETA
-hostname=${hostname}
+hostname=$ACTUAL_HOSTNAME
 role=${role}
-mac=${mac}
+mac=$CONF_MAC
 provisioned_at=$(date -Iseconds)
 method=asahi-firstboot
 LABMETA
@@ -263,9 +280,9 @@ IP=$(hostname -I | awk '{print $1}')
 echo "Registering with bastion at ${serverIp}:${httpPort}..."
 curl -sf -X POST "http://${serverIp}:${httpPort}/api/register" \\
     -H "Content-Type: application/json" \\
-    -d "{\\"mac\\":\\"${mac}\\",\\"hostname\\":\\"${hostname}\\",\\"role\\":\\"${role}\\",\\"ip\\":\\"$IP\\"}" \\
+    -d "{\\"mac\\":\\"$CONF_MAC\\",\\"hostname\\":\\"$ACTUAL_HOSTNAME\\",\\"role\\":\\"${role}\\",\\"ip\\":\\"$IP\\"}" \\
-    2>/dev/null && echo "  Registered as ${hostname} ($IP)" \\
+    2>/dev/null && echo "  Registered as $ACTUAL_HOSTNAME ($IP)" \\
-    || echo "  WARNING: Could not reach bastion — register manually with: labctl provision register ${mac} ${hostname} --role ${role} --ip $IP"
+    || echo "  WARNING: Could not reach bastion — register manually with: labctl provision register $CONF_MAC $ACTUAL_HOSTNAME --role ${role} --ip $IP"
 
 # ── Mark done ────────────────────────────────────────────────────
 touch "$MARKER"

bastion/src/bastion/tests/asahi.test.ts

@@ -184,7 +184,8 @@ describe("renderFirstbootScript", () => {
 
   it("sets hostname", () => {
     const script = renderFirstbootScript({ ...baseParams, role: "worker" });
-    expect(script).toContain('hostnamectl set-hostname "test-node"');
+    expect(script).toContain('CONF_HOSTNAME="test-node"');
+    expect(script).toContain("hostnamectl set-hostname");
   });
 
   it("includes bastion self-registration", () => {

bastion/src/cli/src/api/client.ts

@@ -104,6 +104,16 @@ export class LabdClient {
     return this.request("POST", "/api/machines/debug", { body: { mac, pxeBoot: opts?.pxeBoot } });
   }
 
+  async discoverMachine(data: {
+    mac: string; product?: string; board?: string; serial?: string;
+    manufacturer?: string; cpu_model?: string; cpu_cores?: number;
+    memory_gb?: number; arch?: string;
+    disks?: Array<{ name: string; size_gb: number; model: string }>;
+    nics?: Array<{ name: string; mac: string; state: string }>;
+  }): Promise<{ status: string; error?: string }> {
+    return this.request("POST", "/api/machines/discover", { body: data });
+  }
+
   async forgetMachine(mac: string): Promise<{ status: string }> {
     return this.request("DELETE", `/api/machines/${encodeURIComponent(mac)}`);
   }

bastion/src/cli/src/commands/app.ts

@@ -70,7 +70,7 @@ export function registerAppCommand(program: Command): void {
     .command("install <target>")
     .description("Install k3s on a target machine (hostname, IP, or MAC)")
     .option("--role <role>", "k3s role: infra (server) or worker (agent)", "infra")
-    .option("--user <user>", "SSH user", "lab")
+    .option("--user <user>", "SSH user", "root")
     .option("--k3s-server <url>", "k3s server URL (required for worker role)")
     .option("--k3s-token <token>", "k3s join token (required for worker role)")
     .action(async (target: string, opts: {
@@ -164,7 +164,7 @@ export function registerAppCommand(program: Command): void {
   k3sCmd
     .command("health [target]")
     .description("Check k3s health (all hosts if no target given)")
-    .option("--user <user>", "SSH user", "lab")
+    .option("--user <user>", "SSH user", "root")
     .action(async (target: string | undefined, opts: { user: string }) => {
       const sshKey = findSshKey();
 
@@ -304,7 +304,7 @@ export function registerAppCommand(program: Command): void {
   k3sCmd
     .command("list")
     .description("List installed machines and their k3s status")
-    .option("--user <user>", "SSH user", "lab")
+    .option("--user <user>", "SSH user", "root")
     .action(async (opts: { user: string }) => {
       let state: BastionState;
       try {

bastion/src/cli/src/commands/asahi.ts

@@ -59,9 +59,9 @@ export function registerAsahiCommand(parent: Command): void {
       console.log(`    labvg/longhorn (remaining space)${RESET}`);
       console.log("");
       console.log(`  After first boot, SSH in and run the firstboot script:`);
-      console.log(`    ${BOLD}ssh root@<ip> 'curl -sf ${bastionUrl}/asahi/firstboot.sh?hostname=<name>\\&role=infra | bash'${RESET}`);
+      console.log(`    ${BOLD}ssh root@<ip> 'curl -sf ${bastionUrl}/asahi/firstboot.sh | bash'${RESET}`);
       console.log("");
-      console.log(`  This sets up LVM and self-registers with the bastion.`);
+      console.log(`  This sets up LVM, detects hostname/MAC, and self-registers.`);
       console.log(`  Then install k3s:`);
       console.log(`    ${BOLD}labctl app k3s install <hostname> --role infra${RESET}`);
       console.log("");

bastion/src/cli/src/commands/labcontroller.ts

@@ -38,7 +38,7 @@ export function registerLabcontrollerCommands(appCmd: Command): void {
   lcCmd
     .command("deploy <target>")
     .description("Deploy labcontroller stack to a k3s node")
-    .option("--user <user>", "SSH user", "lab")
+    .option("--user <user>", "SSH user", "root")
     .option("--crdb-replicas <n>", "CockroachDB replicas", "1")
     .action(async (target: string, opts: {
       user: string;
@@ -193,7 +193,7 @@ export function registerLabcontrollerCommands(appCmd: Command): void {
   lcCmd
     .command("status [target]")
     .description("Check labcontroller deployment status (all hosts if no target)")
-    .option("--user <user>", "SSH user", "lab")
+    .option("--user <user>", "SSH user", "root")
     .action(async (target: string | undefined, opts: { user: string }) => {
       const sshKey = findSshKey();
       const sshOpts = sshKey ? { keyPath: sshKey } : {};

bastion/src/cli/src/commands/list.ts

@@ -69,10 +69,10 @@ export function registerListCommand(parent: Command): void {
         const hostname = inst?.hostname ?? queued?.hostname ?? "-";
         const role = inst?.role ?? queued?.role ?? "-";
         const ip = inst?.ip ?? "-";
-        const cpu = hw?.cpu_model ?? "-";
+        const cpu = hw?.cpu_model ?? inst?.cpu_model ?? "-";
-        const cores = hw?.cpu_cores != null ? String(hw.cpu_cores) : "-";
+        const cores = (hw?.cpu_cores ?? inst?.cpu_cores) != null ? String(hw?.cpu_cores ?? inst?.cpu_cores) : "-";
-        const ram = hw?.memory_gb != null ? `${hw.memory_gb}GB` : "-";
+        const ram = (hw?.memory_gb ?? inst?.memory_gb) != null ? `${hw?.memory_gb ?? inst?.memory_gb}GB` : "-";
-        const product = hw?.product ?? "-";
+        const product = hw?.product ?? inst?.product ?? "-";
 
         const color = statusColor(status);
 

bastion/src/cli/src/commands/recheck.ts

@@ -0,0 +1,94 @@
+// CLI command: provision recheck
+// SSH into all installed machines, collect hardware info, update bastion state.
+
+import type { Command } from "commander";
+import { sshExec } from "@lab/modules";
+import { getLabdClient } from "../api/config.js";
+
+const BOLD = "\x1b[1m";
+const GREEN = "\x1b[0;32m";
+const RED = "\x1b[0;31m";
+const DIM = "\x1b[2m";
+const RESET = "\x1b[0m";
+
+const SSH_OPTS = { timeoutMs: 30_000 };
+
+// Shell script that collects hardware info as JSON.
+// Kept simple — no Python, pure shell + awk.
+const HW_COLLECT_SCRIPT = [
+  'P=$(cat /sys/class/dmi/id/product_name 2>/dev/null || echo unknown)',
+  'B=$(cat /sys/class/dmi/id/board_name 2>/dev/null || echo unknown)',
+  'S=$(cat /sys/class/dmi/id/product_serial 2>/dev/null || echo unknown)',
+  'M=$(cat /sys/class/dmi/id/sys_vendor 2>/dev/null || echo unknown)',
+  'C=$(grep -m1 "model name" /proc/cpuinfo 2>/dev/null | cut -d: -f2 | sed "s/^ //" || grep -m1 Model /proc/cpuinfo 2>/dev/null | cut -d: -f2 | sed "s/^ //" || echo unknown)',
+  'N=$(grep -c "^processor" /proc/cpuinfo 2>/dev/null || echo 0)',
+  'R=$(awk "/MemTotal/ {printf \\"%d\\", \\$2/1024/1024}" /proc/meminfo 2>/dev/null || echo 0)',
+  'A=$(uname -m)',
+  'printf \'{"product":"%s","board":"%s","serial":"%s","manufacturer":"%s","cpu_model":"%s","cpu_cores":%s,"memory_gb":%s,"arch":"%s"}\\n\' "$P" "$B" "$S" "$M" "$C" "$N" "$R" "$A"',
+].join("; ");
+
+export function registerRecheckCommand(parent: Command): void {
+  parent
+    .command("recheck")
+    .description("Refresh hardware info for all installed machines via SSH")
+    .option("--user <user>", "SSH user", "root")
+    .option("--target <hostname>", "Only recheck a specific machine (by hostname or MAC)")
+    .action(async (opts: { user: string; target?: string }) => {
+      const client = getLabdClient();
+      let state;
+      try {
+        state = await client.getMachines();
+      } catch (err) {
+        console.error(`Cannot reach labd: ${err instanceof Error ? err.message : String(err)}`);
+        process.exit(1);
+      }
+
+      // Build list of machines to check
+      const targets: Array<{ mac: string; hostname: string; ip: string }> = [];
+      for (const [mac, info] of Object.entries(state.installed)) {
+        if (!info.ip) continue;
+        if (opts.target && info.hostname !== opts.target && mac !== opts.target) continue;
+        targets.push({ mac, hostname: info.hostname, ip: info.ip });
+      }
+
+      if (targets.length === 0) {
+        console.log("No installed machines with IPs to check.");
+        return;
+      }
+
+      console.log(`\n${BOLD}Rechecking ${targets.length} machine(s)...${RESET}\n`);
+
+      let updated = 0;
+      let failed = 0;
+
+      for (const { mac, hostname, ip } of targets) {
+        process.stdout.write(`  ${hostname.padEnd(24)} ${DIM}(${ip})${RESET}  `);
+
+        try {
+          const t0 = Date.now();
+          const result = await sshExec(ip, opts.user, HW_COLLECT_SCRIPT, SSH_OPTS);
+          const elapsed = Date.now() - t0;
+          if (result.exitCode !== 0) {
+            console.log(`${RED}SSH failed (exit ${result.exitCode}, ${elapsed}ms)${RESET}`);
+            if (result.stderr) console.log(`    ${DIM}${result.stderr.substring(0, 200)}${RESET}`);
+            console.log(`${RED}SSH failed (exit ${result.exitCode})${RESET}`);
+            failed++;
+            continue;
+          }
+
+          const hwData = JSON.parse(result.stdout.trim());
+          await client.discoverMachine({ mac, ...hwData });
+          const cpu = hwData.cpu_model || "?";
+          const cores = hwData.cpu_cores || "?";
+          const mem = hwData.memory_gb || "?";
+          console.log(`${GREEN}OK${RESET} ${DIM}${cpu}, ${cores} cores, ${mem}GB${RESET}`);
+          updated++;
+        } catch (err) {
+          console.log(`${RED}FAIL${RESET} ${DIM}${err instanceof Error ? err.message : String(err)}${RESET}`);
+          failed++;
+        }
+      }
+
+      console.log(`\n${BOLD}Done:${RESET} ${updated} updated, ${failed} failed\n`);
+    });
+}

bastion/src/cli/src/index.ts

@@ -20,6 +20,7 @@ import { registerRegisterCommand } from "./commands/register.js";
 import { registerAsahiCommand } from "./commands/asahi.js";
 import { registerLogsCommand } from "./commands/logs.js";
 import { registerMakeIsoCommand } from "./commands/makeiso.js";
+import { registerRecheckCommand } from "./commands/recheck.js";
 import { registerConfigCommand } from "./commands/config.js";
 import { registerLoginCommand } from "./commands/login.js";
 import { registerDoctorCommand } from "./commands/doctor.js";
@@ -104,6 +105,7 @@ export function createProgram(): Command {
   registerAsahiCommand(provisionCmd);
   registerLogsCommand(provisionCmd);
   registerMakeIsoCommand(provisionCmd);
+  registerRecheckCommand(provisionCmd);
 
   // config list/get/set/path
   registerConfigCommand(program);

bastion/src/labd/src/routes/bastions.ts

@@ -260,6 +260,37 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
     }
   });
 
+  // Update hardware info (discovery data) for a machine
+  app.post<{
+    Body: {
+      mac?: string; product?: string; board?: string; serial?: string;
+      manufacturer?: string; cpu_model?: string; cpu_cores?: number;
+      memory_gb?: number; arch?: string;
+      disks?: Array<{ name: string; size_gb: number; model: string }>;
+      nics?: Array<{ name: string; mac: string; state: string }>;
+    };
+  }>("/api/machines/discover", async (request, reply) => {
+    const data = request.body ?? {};
+    const mac = (data.mac ?? "").toLowerCase().replace(/-/g, ":");
+    if (!mac) {
+      return reply.code(400).send({ error: "mac is required" });
+    }
+
+    const bastion = bastionRegistry.findBastionByMac(mac);
+    const target = bastion ?? (bastionRegistry.getAll().length === 1 ? bastionRegistry.getAll()[0] : null);
+
+    if (!target) {
+      return reply.code(503).send({ error: "No bastion found for this MAC" });
+    }
+
+    try {
+      const result = await sendCommand(target.bastionId, { type: "command-discover", ...data, mac });
+      return reply.code(result.status === "ok" ? 200 : 500).send(result);
+    } catch (err) {
+      return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
+    }
+  });
+
   // Update role
   app.post<{
     Body: { mac?: string; role?: string };

bastion/src/modules/modules/k3s/src/groups/hardening.ts

@@ -1,19 +1,21 @@
-// Hardening: Pod Security Standards, certificate check, log rotation.
+// Hardening: Pod Security Standards, certificate check, journald cap, storage.
 
 import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
 import { runSequential } from "../utils.js";
 import { applyPodSecurityStandards } from "../operations/pod-security.js";
 import { checkCertExpiry } from "../operations/cert-check.js";
 import { configureLogRotation } from "../operations/log-rotation.js";
+import { configureJournaldLimits } from "../operations/journald-limits.js";
 import { configureLonghornDisk } from "../operations/longhorn-disk.js";
 
 export const hardeningGroup: OperationGroup = {
   name: "hardening",
-  description: "Pod security, certificate check, log rotation, storage",
+  description: "Pod security, certificate check, journald cap, storage",
   operations: [
     { name: "Apply Pod Security Standards", fn: applyPodSecurityStandards },
     { name: "Check certificate expiry", fn: checkCertExpiry },
-    { name: "Configure log rotation", fn: configureLogRotation },
+    { name: "Decommission file-based audit logs", fn: configureLogRotation },
+    { name: "Configure journald disk cap", fn: configureJournaldLimits },
     { name: "Configure Longhorn disk", fn: configureLonghornDisk },
   ],
 };

bastion/src/modules/modules/k3s/src/install.ts

@@ -76,7 +76,6 @@ sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config 2>/dev/nul
 # ── 5b. Create k3s config directory ──
 echo "[5/10] Writing k3s server configuration..."
 mkdir -p /etc/rancher/k3s
-mkdir -p /var/log/kubernetes
 
 cat > /etc/rancher/k3s/config.yaml << 'K3S_CONFIG'
 # k3s server configuration — CIS hardened
@@ -91,13 +90,10 @@ disable:
   - servicelb
   - traefik
 
-# API server hardening
+# API server hardening (audit-log-path=- routes audit to journald via stdout)
 kube-apiserver-arg:
   - "anonymous-auth=false"
-  - "audit-log-path=/var/log/kubernetes/audit.log"
+  - "audit-log-path=-"
-  - "audit-log-maxage=30"
-  - "audit-log-maxbackup=10"
-  - "audit-log-maxsize=100"
   - "audit-policy-file=/etc/rancher/k3s/audit-policy.yaml"
   - "enable-admission-plugins=NodeRestriction,PodSecurity"
   - "request-timeout=300s"

bastion/src/modules/modules/k3s/src/k3s-module.ts

@@ -78,9 +78,10 @@ export class K3sModule implements Module {
       return toModuleResult("install", [...prepResults, ...k3sResults], start);
     }
 
-    // Phase 3: Networking (server only — agents don't install Cilium)
+    // Phase 3: Networking (initial server only — joining servers get Cilium via daemonset)
     let netResults: OperationResult[] = [];
-    if (isServer) {
+    const isJoiningServer = isServer && !!opCtx.config.k3sServerUrl;
+    if (isServer && !isJoiningServer) {
       netResults = await runNetworking(opCtx);
     }
 

bastion/src/modules/modules/k3s/src/operations/etcd-recover.ts

@@ -0,0 +1,194 @@
+// Recover a broken etcd member by removing it from the cluster, wiping its
+// local state, and restarting k3s so it rejoins as a fresh member.
+//
+// Use case: a node panics on startup with
+//   "tocommit(N+1) is out of range [lastIndex(N)]. Was the raft log corrupted,
+//    truncated, or lost?"
+// This means the local raft WAL is missing the last entry the leader thinks
+// the follower acknowledged (lost write, unclean shutdown, etc). The fix is
+// always the same and well-documented; this codifies it so we don't fumble
+// the procedure under pressure.
+//
+// Preconditions:
+//   - At least one healthy peer is reachable so the cluster has quorum after
+//     we remove the broken member. (For a 3-node cluster: 2 healthy. For a
+//     5-node: 3 healthy.) If quorum would be lost, this function refuses.
+//   - SSH access to both the broken node and a healthy peer.
+//   - etcdctl available on the healthy peer (k3s does not bundle it; the
+//     procedure installs it on demand on Fedora).
+
+import type { SshClient } from "../types.js";
+
+const ETCD_TLS = {
+  ca: "/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt",
+  cert: "/var/lib/rancher/k3s/server/tls/etcd/server-client.crt",
+  key: "/var/lib/rancher/k3s/server/tls/etcd/server-client.key",
+} as const;
+
+const SSH_TIMEOUT = 60_000;
+
+export interface RecoverEtcdMemberOptions {
+  /** SSH client for the broken node (the one panicking). */
+  broken: SshClient;
+  /** SSH client for any healthy server peer in the same cluster. */
+  peer: SshClient;
+  /** Hostname (k8s node name) of the broken node. Used to find its etcd member id. */
+  brokenHostname: string;
+  /** Logger for progress output. */
+  log?: (msg: string) => void;
+}
+
+export interface RecoverEtcdMemberResult {
+  success: boolean;
+  changed: boolean;
+  message: string;
+  /** New etcd member id assigned after rejoin (when known). */
+  newMemberId?: string;
+  /** Old etcd member id that was removed. */
+  removedMemberId?: string;
+  error?: string;
+}
+
+function etcdctl(subcmd: string): string {
+  return [
+    "ETCDCTL_API=3 etcdctl",
+    `--cacert=${ETCD_TLS.ca}`,
+    `--cert=${ETCD_TLS.cert}`,
+    `--key=${ETCD_TLS.key}`,
+    "--endpoints=https://127.0.0.1:2379",
+    "--command-timeout=10s",
+    subcmd,
+  ].join(" ");
+}
+
+async function ensureEtcdctl(peer: SshClient): Promise<void> {
+  const probe = await peer.exec("command -v etcdctl 2>/dev/null", { timeoutMs: 5_000 });
+  if (probe.exitCode === 0 && probe.stdout.trim()) return;
+  // Best-effort install on Fedora. If the host isn't dnf-based, surface the
+  // error to the caller via the next etcdctl invocation.
+  await peer.exec("dnf install -y etcd 2>&1", { timeoutMs: 120_000 });
+}
+
+async function getMemberList(peer: SshClient): Promise<Array<{ id: string; name: string }>> {
+  const result = await peer.exec(etcdctl("member list"), { timeoutMs: SSH_TIMEOUT });
+  if (result.exitCode !== 0) {
+    throw new Error(`etcdctl member list failed: ${result.stderr || result.stdout}`);
+  }
+  // Format: <hex-id>, started, <name>, <peer-urls>, <client-urls>, <isLearner>
+  return result.stdout
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .map((line) => {
+      const [id, , name] = line.split(",").map((p) => p.trim());
+      return { id: id ?? "", name: name ?? "" };
+    })
+    .filter((m) => m.id);
+}
+
+export async function recoverEtcdMember(
+  opts: RecoverEtcdMemberOptions,
+): Promise<RecoverEtcdMemberResult> {
+  const log = opts.log ?? (() => {});
+
+  try {
+    log(`Looking up etcd member id for ${opts.brokenHostname} via peer...`);
+    await ensureEtcdctl(opts.peer);
+
+    const members = await getMemberList(opts.peer);
+    if (members.length < 3) {
+      return {
+        success: false,
+        changed: false,
+        message: "Refusing to remove a member from a cluster with <3 members (quorum would be lost)",
+        error: `member count = ${members.length}`,
+      };
+    }
+
+    // Member names are <hostname>-<random-suffix>; match by hostname prefix.
+    const broken = members.find((m) => m.name.startsWith(opts.brokenHostname));
+    if (!broken) {
+      return {
+        success: false,
+        changed: false,
+        message: `No etcd member found matching hostname ${opts.brokenHostname}`,
+        error: `members: ${members.map((m) => m.name).join(", ")}`,
+      };
+    }
+    log(`Broken member: ${broken.id} (${broken.name})`);
+
+    log("Step 1/4: stopping k3s on broken node");
+    await opts.broken.exec("systemctl stop k3s 2>&1", { timeoutMs: SSH_TIMEOUT });
+
+    log("Step 2/4: removing broken etcd member from cluster");
+    const remove = await opts.peer.exec(
+      etcdctl(`member remove ${broken.id}`),
+      { timeoutMs: SSH_TIMEOUT },
+    );
+    if (remove.exitCode !== 0) {
+      return {
+        success: false,
+        changed: false,
+        message: "etcdctl member remove failed",
+        error: remove.stderr || remove.stdout,
+        removedMemberId: broken.id,
+      };
+    }
+
+    log("Step 3/4: archiving corrupt etcd state and stale TLS/cred dirs on broken node");
+    const ts = Math.floor(Date.now() / 1000);
+    await opts.broken.exec(
+      [
+        `mv /var/lib/rancher/k3s/server/db /var/lib/rancher/k3s/server/db.corrupt-${ts} 2>/dev/null || true`,
+        "rm -rf /var/lib/rancher/k3s/server/tls /var/lib/rancher/k3s/server/cred",
+      ].join(" && "),
+      { timeoutMs: SSH_TIMEOUT },
+    );
+
+    log("Step 4/4: starting k3s on broken node — it will rejoin");
+    await opts.broken.exec("systemctl start k3s 2>&1", { timeoutMs: SSH_TIMEOUT });
+
+    // Poll for rejoin. The new member-id is what the cluster assigns on join.
+    let newMemberId: string | undefined;
+    for (let i = 0; i < 60; i++) {
+      await new Promise((r) => setTimeout(r, 5_000));
+      try {
+        const after = await getMemberList(opts.peer);
+        const rejoined = after.find(
+          (m) => m.name.startsWith(opts.brokenHostname) && m.id !== broken.id,
+        );
+        if (rejoined) {
+          newMemberId = rejoined.id;
+          break;
+        }
+      } catch {
+        // peer may briefly be unreachable mid-rejoin — keep polling
+      }
+    }
+
+    if (!newMemberId) {
+      return {
+        success: false,
+        changed: true,
+        message: "k3s started but new member did not appear in cluster within 5 minutes",
+        removedMemberId: broken.id,
+      };
+    }
+
+    log(`Rejoined as ${newMemberId}`);
+    return {
+      success: true,
+      changed: true,
+      message: `Recovered: removed ${broken.id}, rejoined as ${newMemberId}`,
+      removedMemberId: broken.id,
+      newMemberId,
+    };
+  } catch (err) {
+    return {
+      success: false,
+      changed: false,
+      message: "Recovery failed",
+      error: err instanceof Error ? err.message : String(err),
+    };
+  }
+}

bastion/src/modules/modules/k3s/src/operations/index.ts

@@ -11,7 +11,13 @@ export { installK3sBinary } from "./k3s-install.js";
 export { installCilium } from "./cilium.js";
 export { fixCoreDnsUpstream } from "./dns-fix.js";
 export { configureLogRotation } from "./log-rotation.js";
+export { configureJournaldLimits } from "./journald-limits.js";
 export { applyDefaultNetworkPolicies } from "./network-policy.js";
 export { applyPodSecurityStandards } from "./pod-security.js";
 export { checkCertExpiry } from "./cert-check.js";
 export { configureLonghornDisk } from "./longhorn-disk.js";
+export { recoverEtcdMember } from "./etcd-recover.js";
+export type {
+  RecoverEtcdMemberOptions,
+  RecoverEtcdMemberResult,
+} from "./etcd-recover.js";

bastion/src/modules/modules/k3s/src/operations/iscsi.ts

@@ -13,10 +13,11 @@ export const enableIscsi: Operation = async (ctx): Promise<OperationResult> => {
 
   // Install the package (detect distro)
   const osRelease = await ctx.ssh.exec("cat /etc/os-release", sshOpts(ctx));
-  const isFedora = osRelease.stdout.includes("fedora") || osRelease.stdout.includes("rhel") || osRelease.stdout.includes("centos");
+  const osLower = osRelease.stdout.toLowerCase();
+  const isFedora = osLower.includes("fedora") || osLower.includes("rhel") || osLower.includes("centos");
 
   const pkg = isFedora ? "iscsi-initiator-utils" : "open-iscsi";
-  const installCmd = isFedora ? `dnf install -y ${pkg}` : `apt-get install -y ${pkg}`;
+  const installCmd = isFedora ? `sudo dnf install -y ${pkg}` : `sudo apt-get install -y ${pkg}`;
 
   const install = await ctx.ssh.exec(installCmd, { timeoutMs: 120_000 });
   if (install.exitCode !== 0) {
@@ -24,7 +25,7 @@ export const enableIscsi: Operation = async (ctx): Promise<OperationResult> => {
   }
 
   // Enable and start
-  await ctx.ssh.exec("systemctl enable --now iscsid", sshOpts(ctx));
+  await ctx.ssh.exec("sudo systemctl enable --now iscsid", sshOpts(ctx));
 
   return { success: true, changed: true, message: `Installed ${pkg} and enabled iscsid` };
 };

bastion/src/modules/modules/k3s/src/operations/journald-limits.ts

@@ -0,0 +1,33 @@
+// Cap journald disk usage so audit logs (which now flow through journald via
+// kube-apiserver's stdout) cannot fill /var/log. Default journald uses up to
+// 10% of the filesystem, capped at 4 GB. In a /var/log of ~10 GB shared with
+// other services, that's still room for audit volume to evict useful logs.
+// 2 GB / 200 MB-per-file is a comfortable middle.
+
+import type { Operation, OperationResult } from "../types.js";
+import { sshOpts, writeRemoteFile } from "../utils.js";
+
+const DROPIN_CONTENT = `[Journal]
+SystemMaxUse=2G
+SystemKeepFree=1G
+SystemMaxFileSize=200M
+`;
+
+const DROPIN_PATH = "/etc/systemd/journald.conf.d/10-k3s-audit-cap.conf";
+
+export const configureJournaldLimits: Operation = async (ctx): Promise<OperationResult> => {
+  const changed = await writeRemoteFile(ctx, DROPIN_PATH, DROPIN_CONTENT);
+  if (changed) {
+    // Reload journald so the new limit applies without a reboot.
+    await ctx.ssh.exec(
+      "systemctl kill --signal=SIGUSR2 systemd-journald 2>/dev/null; " +
+      "systemctl restart systemd-journald 2>&1 || true",
+      sshOpts(ctx),
+    );
+  }
+  return {
+    success: true,
+    changed,
+    message: changed ? "journald limits configured (2 GB cap)" : "journald limits already configured",
+  };
+};

bastion/src/modules/modules/k3s/src/operations/k3s-config.ts

@@ -9,7 +9,18 @@ function isServerRole(role: string): boolean {
 
 function generateServerConfig(config: K3sConfig): string {
   const tlsSans = [config.hostname, config.ip, ...(config.tlsSans ?? [])];
-  return `# k3s server configuration — CIS hardened
+  const isJoining = !!config.k3sServerUrl;
+  const clusterLines = isJoining
+    ? `server: "${config.k3sServerUrl}"\ntoken: "${config.k3sToken}"`
+    : "cluster-init: true";
+  // audit-log-path=- routes audit events to k3s.service's stdout, which systemd
+  // forwards to journald. journald enforces its own size caps (see
+  // configureJournaldLimits) so audit volume cannot fill the disk. File-based
+  // audit logs led to /var/log/kubernetes growing to 7+ GB because apiserver's
+  // own rotation produced files that any logrotate glob would double-rotate
+  // and never expire.
+  return `# k3s server configuration — CIS hardened, etcd HA
+${clusterLines}
 protect-kernel-defaults: true
 secrets-encryption: true
 write-kubeconfig-mode: "0640"
@@ -25,10 +36,7 @@ node-label:
 
 kube-apiserver-arg:
   - "anonymous-auth=false"
-  - "audit-log-path=/var/log/kubernetes/audit.log"
+  - "audit-log-path=-"
-  - "audit-log-maxage=30"
-  - "audit-log-maxbackup=10"
-  - "audit-log-maxsize=100"
   - "audit-policy-file=/etc/rancher/k3s/audit-policy.yaml"
   - "enable-admission-plugins=NodeRestriction,PodSecurity"
   - "request-timeout=300s"
@@ -56,7 +64,7 @@ kubelet-arg:
 }
 
 export const writeK3sConfig: Operation = async (ctx): Promise<OperationResult> => {
-  await ctx.ssh.exec("mkdir -p /etc/rancher/k3s /var/log/kubernetes", sshOpts(ctx));
+  await ctx.ssh.exec("mkdir -p /etc/rancher/k3s", sshOpts(ctx));
 
   const content = isServerRole(ctx.config.role)
     ? generateServerConfig(ctx.config)

bastion/src/modules/modules/k3s/src/operations/k3s-install.ts

@@ -15,8 +15,21 @@ export const installK3sBinary: Operation = async (ctx): Promise<OperationResult>
   const alreadyInstalled = version.exitCode === 0;
 
   if (isServer) {
+    // Clean stale server state when joining an existing cluster
+    // (TLS certs from a previous run cause "newer than datastore" fatal error)
+    if (ctx.config.k3sServerUrl && ctx.config.k3sToken) {
+      await ctx.ssh.exec(
+        "rm -rf /var/lib/rancher/k3s/server/tls /var/lib/rancher/k3s/server/cred /var/lib/rancher/k3s/server/db",
+        sshOpts(ctx),
+      );
+    }
+
+    // If joining an existing cluster, pass K3S_URL and K3S_TOKEN
+    const joinEnv = ctx.config.k3sServerUrl && ctx.config.k3sToken
+      ? `K3S_URL="${ctx.config.k3sServerUrl}" K3S_TOKEN="${ctx.config.k3sToken}"`
+      : "";
     const result = await ctx.ssh.exec(
-      'curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server" INSTALL_K3S_SKIP_SELINUX_RPM=true sh -',
+      `curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server" INSTALL_K3S_SKIP_SELINUX_RPM=true ${joinEnv} sh -`,
       { timeoutMs: 300_000 },
     );
     if (result.exitCode !== 0) {

bastion/src/modules/modules/k3s/src/operations/log-rotation.ts

@@ -1,25 +1,44 @@
-// Configure log rotation for k3s.
+// Decommission file-based k8s audit logging in favor of journald.
+//
+// Earlier versions wrote audit events to /var/log/kubernetes/audit.log and
+// rotated them with a logrotate rule. Two failure modes followed: kube-apiserver
+// rotated internally (audit-{ts}.log), the *.log glob in logrotate
+// double-rotated those (-{date}), and the resulting filename matched no
+// retention policy, so the directory grew unbounded (we observed 7+ GB).
+//
+// k3s now sets audit-log-path=- so audit goes to stdout → journald, which
+// enforces SystemMaxUse caps. This operation removes the obsolete logrotate
+// rule and reaps any audit files left behind by the old setup. Idempotent: on
+// fresh installs everything is already absent and the operation is a no-op.
 
 import type { Operation, OperationResult } from "../types.js";
-import { writeRemoteFile } from "../utils.js";
+import { sshOpts } from "../utils.js";
 
-const LOGROTATE_CONFIG = `/var/log/kubernetes/*.log {
+const REMOVE_LOGROTATE = "rm -f /etc/logrotate.d/k3s";
-  daily
+
-  rotate 14
+// Bounded by a max-depth and explicit name pattern so we never reach outside
-  compress
+// the deprecated audit-log directory.
-  delaycompress
+const REAP_OLD_AUDIT_FILES =
-  missingok
+  "find /var/log/kubernetes -maxdepth 1 -type f " +
-  notifempty
+  "\\( -name 'audit*.log*' -o -name 'audit-*.log' \\) " +
-  copytruncate
+  "-delete 2>/dev/null; " +
-  maxsize 100M
+  "rmdir /var/log/kubernetes 2>/dev/null; true";
-}`;
 
 export const configureLogRotation: Operation = async (ctx): Promise<OperationResult> => {
-  const changed = await writeRemoteFile(ctx, "/etc/logrotate.d/k3s", LOGROTATE_CONFIG);
+  const before = await ctx.ssh.exec(
+    "test -e /etc/logrotate.d/k3s -o -d /var/log/kubernetes && echo present || echo absent",
+    sshOpts(ctx),
+  );
+  const wasPresent = before.stdout.trim() === "present";
+
+  await ctx.ssh.exec(REMOVE_LOGROTATE, sshOpts(ctx));
+  await ctx.ssh.exec(REAP_OLD_AUDIT_FILES, sshOpts(ctx));
 
   return {
     success: true,
-    changed,
+    changed: wasPresent,
-    message: changed ? "Log rotation configured" : "Log rotation already configured",
+    message: wasPresent
+      ? "Removed legacy file-based audit logging (now via journald)"
+      : "No legacy audit log artifacts present",
   };
 };

bastion/src/modules/modules/k3s/src/operations/longhorn-disk.ts

@@ -3,6 +3,7 @@
 
 import type { Operation, OperationResult } from "../types.js";
 import { sshOpts } from "../utils.js";
+import { sshExec as remoteSshExec } from "../../../../src/ssh.js";
 
 export const configureLonghornDisk: Operation = async (ctx): Promise<OperationResult> => {
   // Check if /var/lib/longhorn exists on this node
@@ -15,12 +16,11 @@ export const configureLonghornDisk: Operation = async (ctx): Promise<OperationRe
   const nodeNameResult = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
   const nodeName = nodeNameResult.stdout.trim();
 
-  // Apply the annotation via kubectl (works on server nodes, or via KUBECONFIG on agents)
-  const kubectlPrefix = "k3s kubectl";
   const annotation = JSON.stringify([{ path: "/var/lib/longhorn", allowScheduling: true }]);
 
+  // Try kubectl locally first (works on server nodes)
   const result = await ctx.ssh.exec(
-    `${kubectlPrefix} annotate node "${nodeName}" "node.longhorn.io/default-disks-config=${annotation}" --overwrite 2>&1 || true`,
+    `k3s kubectl annotate node "${nodeName}" "node.longhorn.io/default-disks-config=${annotation}" --overwrite 2>&1 || true`,
     sshOpts(ctx),
   );
 
@@ -28,7 +28,23 @@ export const configureLonghornDisk: Operation = async (ctx): Promise<OperationRe
     return { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName}` };
   }
 
-  // If kubectl isn't available (agent node without server access), that's OK —
+  // For worker/agent nodes without local kubectl: apply via the server
-  // the label is set, annotation can be applied from the server later
+  if (ctx.config.k3sServerUrl) {
+    // The CLI has SSH access to the server — use sshExec from there
+    const serverHost = new URL(ctx.config.k3sServerUrl).hostname;
+    try {
+      const remoteResult = await remoteSshExec(
+        serverHost, "root",
+        `k3s kubectl annotate node "${nodeName}" "node.longhorn.io/default-disks-config=${annotation}" --overwrite`,
+        { ...(ctx.ssh.keyPath ? { keyPath: ctx.ssh.keyPath } : {}), timeoutMs: 15_000 },
+      );
+      if (remoteResult.stdout.includes("annotated") || remoteResult.stdout.includes("unchanged")) {
+        return { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName} (via server)` };
+      }
+    } catch {
+      // Fall through to manual instruction
+    }
+  }
+
   return { success: true, changed: false, message: "Longhorn disk label set (annotation requires server kubectl)" };
 };

bastion/src/modules/modules/k3s/tests/install.test.ts

@@ -71,9 +71,14 @@ describe("k3s install script — server role", () => {
     expect(script).toContain("enable-admission-plugins=NodeRestriction,PodSecurity");
   });
 
-  it("configures audit logging", () => {
+  it("configures audit logging via journald (stdout)", () => {
-    expect(script).toContain("audit-log-path=/var/log/kubernetes/audit.log");
+    expect(script).toContain("audit-log-path=-");
-    expect(script).toContain("audit-log-maxage=30");
+    // file-based fields and the now-obsolete log directory must be gone
+    expect(script).not.toContain("/var/log/kubernetes/audit.log");
+    expect(script).not.toContain("audit-log-maxage");
+    expect(script).not.toContain("audit-log-maxbackup");
+    expect(script).not.toContain("audit-log-maxsize");
+    expect(script).not.toContain("mkdir -p /var/log/kubernetes");
   });
 
   it("cleans stale flannel vxlan before Cilium install", () => {

bastion/src/modules/modules/k3s/tests/operations.test.ts

@@ -348,3 +348,143 @@ describe("applyPodSecurityStandards", () => {
     expectCommand(ctx.ssh, "pod-security.kubernetes.io/audit=restricted");
   });
 });
+
+// --- Audit Logging Decommission (file-based → journald) ---
+
+import { configureLogRotation } from "../src/operations/log-rotation.js";
+import { configureJournaldLimits } from "../src/operations/journald-limits.js";
+
+describe("configureLogRotation (decommission file-based audit logs)", () => {
+  it("removes the legacy logrotate rule and reaps obsolete audit files", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(stdout("present")); // probe: legacy artifacts exist
+    ctx.ssh.exec.mockResolvedValue(OK);
+
+    const result = await configureLogRotation(ctx);
+
+    expect(result.success).toBe(true);
+    expect(result.changed).toBe(true);
+    expectCommand(ctx.ssh, "rm -f /etc/logrotate.d/k3s");
+    expectCommand(ctx.ssh, /find \/var\/log\/kubernetes.*audit.*-delete/);
+    expectCommand(ctx.ssh, "rmdir /var/log/kubernetes");
+  });
+
+  it("is a no-op when nothing legacy is present", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(stdout("absent"));
+    ctx.ssh.exec.mockResolvedValue(OK);
+
+    const result = await configureLogRotation(ctx);
+    expect(result.success).toBe(true);
+    expect(result.changed).toBe(false);
+  });
+});
+
+describe("configureJournaldLimits", () => {
+  it("writes a 2 GB SystemMaxUse drop-in and reloads journald when changed", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(stdout("__LABCTL_NOT_FOUND__")); // no existing drop-in
+    ctx.ssh.exec.mockResolvedValue(OK);
+
+    const result = await configureJournaldLimits(ctx);
+
+    expect(result.success).toBe(true);
+    expect(result.changed).toBe(true);
+    const writeCall = ctx.ssh.exec.mock.calls.find((c) => {
+      const cmd = c[0] as string;
+      return cmd.includes("10-k3s-audit-cap.conf") && cmd.includes("LABCTL_EOF");
+    });
+    expect(writeCall).toBeTruthy();
+    const written = writeCall?.[0] as string;
+    expect(written).toContain("SystemMaxUse=2G");
+    expect(written).toContain("SystemKeepFree=1G");
+    expectCommand(ctx.ssh, "systemctl restart systemd-journald");
+  });
+
+  it("does not restart journald when the drop-in is already correct", async () => {
+    const ctx = mockCtx();
+    const existing =
+      "[Journal]\nSystemMaxUse=2G\nSystemKeepFree=1G\nSystemMaxFileSize=200M\n";
+    ctx.ssh.exec.mockResolvedValueOnce(stdout(existing));
+    ctx.ssh.exec.mockResolvedValue(OK);
+
+    const result = await configureJournaldLimits(ctx);
+
+    expect(result.success).toBe(true);
+    expect(result.changed).toBe(false);
+    expectNoCommand(ctx.ssh, "systemctl restart systemd-journald");
+  });
+});
+
+// --- Etcd Recovery ---
+
+import { recoverEtcdMember } from "../src/operations/etcd-recover.js";
+import { mockSsh } from "./helpers.js";
+
+describe("recoverEtcdMember", () => {
+  it("refuses to operate when cluster is below 3 members (quorum risk)", async () => {
+    const broken = mockSsh();
+    const peer = mockSsh();
+    peer.exec.mockResolvedValueOnce(stdout("/usr/bin/etcdctl")); // etcdctl present
+    peer.exec.mockResolvedValueOnce(stdout(
+      "111, started, host-a-aaa, https://10.0.0.1:2380, https://10.0.0.1:2379, false\n" +
+      "222, started, host-b-bbb, https://10.0.0.2:2380, https://10.0.0.2:2379, false",
+    ));
+
+    const result = await recoverEtcdMember({ broken, peer, brokenHostname: "host-b" });
+
+    expect(result.success).toBe(false);
+    expect(result.message).toMatch(/quorum/i);
+    // Critically: must NOT have stopped k3s or removed anything
+    expect(broken.exec).not.toHaveBeenCalledWith(expect.stringContaining("systemctl stop k3s"), expect.anything());
+  });
+
+  it("performs full procedure when quorum is preserved", async () => {
+    const broken = mockSsh();
+    const peer = mockSsh();
+    // ensureEtcdctl: present
+    peer.exec.mockResolvedValueOnce(stdout("/usr/bin/etcdctl"));
+    // member list (3 members, target = host-b)
+    peer.exec.mockResolvedValueOnce(stdout(
+      "111, started, host-a-aaa, https://10.0.0.1:2380, https://10.0.0.1:2379, false\n" +
+      "222, started, host-b-bbb, https://10.0.0.2:2380, https://10.0.0.2:2379, false\n" +
+      "333, started, host-c-ccc, https://10.0.0.3:2380, https://10.0.0.3:2379, false",
+    ));
+    // member remove
+    peer.exec.mockResolvedValueOnce(stdout("Member 222 removed"));
+    // post-rejoin member list — new id 444 for host-b
+    peer.exec.mockResolvedValueOnce(stdout(
+      "111, started, host-a-aaa, https://10.0.0.1:2380, https://10.0.0.1:2379, false\n" +
+      "333, started, host-c-ccc, https://10.0.0.3:2380, https://10.0.0.3:2379, false\n" +
+      "444, started, host-b-zzz, https://10.0.0.2:2380, https://10.0.0.2:2379, false",
+    ));
+
+    const result = await recoverEtcdMember({ broken, peer, brokenHostname: "host-b" });
+
+    expect(result.success).toBe(true);
+    expect(result.removedMemberId).toBe("222");
+    expect(result.newMemberId).toBe("444");
+    expectCommand(broken,"systemctl stop k3s");
+    expectCommand(peer,"member remove 222");
+    expectCommand(broken,/db\.corrupt-/);
+    expectCommand(broken,/rm -rf .*\/server\/tls/);
+    expectCommand(broken,"systemctl start k3s");
+  });
+
+  it("fails clearly when no member matches the broken hostname", async () => {
+    const broken = mockSsh();
+    const peer = mockSsh();
+    peer.exec.mockResolvedValueOnce(stdout("/usr/bin/etcdctl"));
+    peer.exec.mockResolvedValueOnce(stdout(
+      "111, started, host-a-aaa, https://10.0.0.1:2380, https://10.0.0.1:2379, false\n" +
+      "222, started, host-b-bbb, https://10.0.0.2:2380, https://10.0.0.2:2379, false\n" +
+      "333, started, host-c-ccc, https://10.0.0.3:2380, https://10.0.0.3:2379, false",
+    ));
+
+    const result = await recoverEtcdMember({ broken, peer, brokenHostname: "host-d" });
+
+    expect(result.success).toBe(false);
+    expect(result.message).toMatch(/No etcd member found/);
+    expect(broken.exec).not.toHaveBeenCalledWith(expect.stringContaining("systemctl stop k3s"), expect.anything());
+  });
+});

bastion/src/shared/src/protocol/index.ts

@@ -113,6 +113,7 @@ export type LabdBastionMessage =
   | { type: "command-role-update"; requestId: string; mac: string; role: string }
   | { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean }
   | { type: "command-register"; requestId: string; mac: string; hostname: string; role: string; ip: string }
+  | { type: "command-discover"; requestId: string; mac: string; product?: string; board?: string; serial?: string; manufacturer?: string; cpu_model?: string; cpu_cores?: number; memory_gb?: number; arch?: string; disks?: Array<{ name: string; size_gb: number; model: string }>; nics?: Array<{ name: string; mac: string; state: string }> }
   | { type: "server-shutdown"; reconnectAfter: number };
 
 export type BastionMessageType = BastionMessage["type"];
@@ -127,7 +128,7 @@ const BASTION_MESSAGE_TYPES = new Set<string>([
 
 const LABD_BASTION_MESSAGE_TYPES = new Set<string>([
   "bastion-enrolled", "bastion-heartbeat-ack", "command-install",
-  "command-forget", "command-role-update", "command-debug", "command-register", "server-shutdown",
+  "command-forget", "command-role-update", "command-debug", "command-register", "command-discover", "server-shutdown",
 ]);
 
 export function isBastionMessage(msg: unknown): msg is BastionMessage {

bastion/src/shared/src/types/state.ts

@@ -96,6 +96,13 @@ export interface InstalledInfo {
   ip: string;
   installed_at: string;
   bastionId?: string;  // set when aggregated through labd
+  // Hardware info (copied from discovered on install completion)
+  product?: string;
+  manufacturer?: string;
+  cpu_model?: string;
+  cpu_cores?: number;
+  memory_gb?: number;
+  arch?: string;
 }
 
 export interface DebugConfig {