feat(k3s): enable swap and grow the rancher LV during host-prep

Replace the CIS-style disableSwap op with enableSwap: activate the
labvg-swap LV with an fstab entry (kubelet runs failSwapOn=false; zram
stays the fast tier, the LV is overflow before OOM kill). Add
growRancherLv: extend labvg/rancher to 120G when the VG has free space,
covering nodes installed before the kickstart sizing change and vanilla
nodes converted to k8s; skips with a clear message when the VG is full.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017f6jyeeDqP4ufyeL3UER9w

bastion/src/modules/modules/k3s/src/groups/host-prep.ts

@@ -1,21 +1,23 @@
-// Host preparation: kernel modules, sysctl, swap, firewall, SELinux.
+// Host preparation: kernel modules, sysctl, swap, storage, firewall, SELinux.
 
 import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
 import { runSequential } from "../utils.js";
 import { loadKernelModules } from "../operations/kernel-modules.js";
 import { applyCisHardening } from "../operations/sysctl.js";
-import { disableSwap } from "../operations/swap.js";
+import { enableSwap } from "../operations/swap.js";
+import { growRancherLv } from "../operations/rancher-storage.js";
 import { disableFirewall } from "../operations/firewall.js";
 import { setSelinuxPermissive } from "../operations/selinux.js";
 import { enableIscsi } from "../operations/iscsi.js";
 
 export const hostPrepGroup: OperationGroup = {
   name: "host-prep",
-  description: "Prepare host for k3s: kernel modules, sysctl, swap, firewall, SELinux, iSCSI",
+  description: "Prepare host for k3s: kernel modules, sysctl, swap, imageFs sizing, firewall, SELinux, iSCSI",
   operations: [
     { name: "Load kernel modules", fn: loadKernelModules },
     { name: "Apply CIS sysctl", fn: applyCisHardening },
-    { name: "Disable swap", fn: disableSwap },
+    { name: "Enable swap", fn: enableSwap },
+    { name: "Grow rancher LV", fn: growRancherLv },
     { name: "Disable firewall", fn: disableFirewall },
     { name: "Set SELinux permissive", fn: setSelinuxPermissive },
     { name: "Enable iSCSI", fn: enableIscsi },

bastion/src/modules/modules/k3s/src/operations/index.ts

@@ -1,6 +1,7 @@
 export { loadKernelModules } from "./kernel-modules.js";
 export { applyCisHardening } from "./sysctl.js";
-export { disableSwap } from "./swap.js";
+export { enableSwap } from "./swap.js";
+export { growRancherLv } from "./rancher-storage.js";
 export { enableIscsi } from "./iscsi.js";
 export { disableFirewall } from "./firewall.js";
 export { setSelinuxPermissive } from "./selinux.js";

bastion/src/modules/modules/k3s/src/operations/rancher-storage.ts

@@ -0,0 +1,48 @@
+// Grow the labvg/rancher LV (k3s image store / imageFs) to 120G.
+// 2026-08 incident: the original 20G LV sat at 85% used from steady-state
+// images alone, so one ~5G image pull tripped imagefs eviction and evicted
+// unrelated pods. Fresh installs are sized at 120G by the kickstart; this op
+// covers nodes installed before that change and vanilla nodes converted to
+// k8s later. Never removes or shrinks anything — if the VG lacks free space
+// (e.g. a longhorn --grow LV consumed it), it reports and moves on.
+
+import type { Operation, OperationResult } from "../types.js";
+import { sshOpts } from "../utils.js";
+
+const RANCHER_LV = "labvg/rancher";
+const TARGET_MIB = 122880; // 120G
+
+export const growRancherLv: Operation = async (ctx): Promise<OperationResult> => {
+  const lv = await ctx.ssh.exec(
+    `lvs --noheadings --units m --nosuffix -o lv_size ${RANCHER_LV} 2>/dev/null || true`,
+    sshOpts(ctx),
+  );
+  const sizeMib = Number.parseFloat(lv.stdout.trim());
+  if (Number.isNaN(sizeMib)) {
+    return { success: true, changed: false, message: "No labvg/rancher LV — imageFs shares /var, skipping" };
+  }
+  if (sizeMib >= TARGET_MIB) {
+    return { success: true, changed: false, message: `rancher LV already ${Math.round(sizeMib / 1024)}G` };
+  }
+
+  const vg = await ctx.ssh.exec(`vgs --noheadings --units m --nosuffix -o vg_free labvg`, sshOpts(ctx));
+  const freeMib = Number.parseFloat(vg.stdout.trim());
+  const neededMib = TARGET_MIB - sizeMib;
+  if (Number.isNaN(freeMib) || freeMib < neededMib) {
+    return {
+      success: true,
+      changed: false,
+      message: `VG labvg has ${Math.floor((Number.isNaN(freeMib) ? 0 : freeMib) / 1024)}G free — ` +
+        `need ${Math.ceil(neededMib / 1024)}G to grow rancher LV to 120G (manual LV rebuild required)`,
+    };
+  }
+
+  await ctx.ssh.exec(`lvextend -L ${TARGET_MIB}m /dev/${RANCHER_LV}`, sshOpts(ctx));
+  await ctx.ssh.exec(`xfs_growfs /var/lib/rancher`, sshOpts(ctx));
+
+  return {
+    success: true,
+    changed: true,
+    message: `rancher LV grown ${Math.round(sizeMib / 1024)}G → 120G`,
+  };
+};

bastion/src/modules/modules/k3s/src/operations/swap.ts

@@ -1,22 +1,40 @@
-// Disable swap (CIS requirement for k3s).
+// Enable swap so memory pressure spills to disk instead of OOM-killing.
+// kubelet runs with failSwapOn=false (k3s default); zram stays the fast tier,
+// the labvg-swap LV is the overflow tier. Replaces the old CIS-style
+// disableSwap op — a kernel OOM kill of a node daemon is worse than slow swap.
 
 import type { Operation, OperationResult } from "../types.js";
 import { sshOpts } from "../utils.js";
 
-export const disableSwap: Operation = async (ctx): Promise<OperationResult> => {
-  const check = await ctx.ssh.exec("swapon --show --noheadings", sshOpts(ctx));
-  const active = check.stdout.trim().length > 0;
+const SWAP_DEV = "/dev/mapper/labvg-swap";
 
-  if (active) {
-    await ctx.ssh.exec("swapoff -a", sshOpts(ctx));
+export const enableSwap: Operation = async (ctx): Promise<OperationResult> => {
+  const lv = await ctx.ssh.exec(`test -b ${SWAP_DEV} && echo yes || echo no`, sshOpts(ctx));
+  if (lv.stdout.trim() !== "yes") {
+    return { success: true, changed: false, message: "No labvg-swap LV — skipping swap enable" };
   }
 
-  // Remove swap entries from fstab permanently
-  await ctx.ssh.exec("sed -i '/\\sswap\\s/d' /etc/fstab", sshOpts(ctx));
+  const active = await ctx.ssh.exec(
+    `grep -q "^$(readlink -f ${SWAP_DEV}) " /proc/swaps && echo on || echo off`,
+    sshOpts(ctx),
+  );
+  const wasOff = active.stdout.trim() !== "on";
+
+  if (wasOff) {
+    // Format if the LV was never (or wrongly) initialised, then activate
+    await ctx.ssh.exec(`blkid ${SWAP_DEV} | grep -q 'TYPE="swap"' || mkswap ${SWAP_DEV}`, sshOpts(ctx));
+    await ctx.ssh.exec(`swapon ${SWAP_DEV}`, sshOpts(ctx));
+  }
+
+  // Persist across reboots (idempotent)
+  await ctx.ssh.exec(
+    `grep -q "labvg-swap" /etc/fstab || echo "${SWAP_DEV} none swap defaults 0 0" >> /etc/fstab`,
+    sshOpts(ctx),
+  );
 
   return {
     success: true,
-    changed: active,
-    message: active ? "Swap disabled" : "Swap already disabled",
+    changed: wasOff,
+    message: wasOff ? "LV swap enabled" : "LV swap already active",
   };
 };

bastion/src/modules/modules/k3s/tests/operations.test.ts

@@ -72,31 +72,97 @@ describe("applyCisHardening", () => {
 
 // --- Swap ---
 
-import { disableSwap } from "../src/operations/swap.js";
+import { enableSwap } from "../src/operations/swap.js";
 
-describe("disableSwap", () => {
-  it("disables active swap", async () => {
+describe("enableSwap", () => {
+  it("activates LV swap when present but off", async () => {
     const ctx = mockCtx();
     ctx.ssh.exec
-      .mockResolvedValueOnce(stdout("/dev/sda2 partition 2G"))  // swap active
-      .mockResolvedValueOnce(OK)   // swapoff
-      .mockResolvedValueOnce(OK);  // sed fstab
+      .mockResolvedValueOnce(stdout("yes"))  // LV exists
+      .mockResolvedValueOnce(stdout("off"))  // not in /proc/swaps
+      .mockResolvedValueOnce(OK)             // blkid || mkswap
+      .mockResolvedValueOnce(OK)             // swapon
+      .mockResolvedValueOnce(OK);            // fstab entry
 
-    const result = await disableSwap(ctx);
+    const result = await enableSwap(ctx);
     expect(result.success).toBe(true);
     expect(result.changed).toBe(true);
-    expectCommand(ctx.ssh, "swapoff -a");
+    expectCommand(ctx.ssh, "swapon /dev/mapper/labvg-swap");
   });
 
-  it("is idempotent when swap already off", async () => {
+  it("is idempotent when LV swap already active", async () => {
     const ctx = mockCtx();
     ctx.ssh.exec
-      .mockResolvedValueOnce(stdout(""))  // no swap
-      .mockResolvedValueOnce(OK);          // sed fstab (always runs)
+      .mockResolvedValueOnce(stdout("yes"))  // LV exists
+      .mockResolvedValueOnce(stdout("on"))   // already in /proc/swaps
+      .mockResolvedValueOnce(OK);            // fstab entry (always ensured)
 
-    const result = await disableSwap(ctx);
+    const result = await enableSwap(ctx);
     expect(result.changed).toBe(false);
-    expectNoCommand(ctx.ssh, "swapoff");
+    expectNoCommand(ctx.ssh, "swapon /dev/mapper/labvg-swap");
+  });
+
+  it("skips when no labvg-swap LV exists", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(stdout("no")); // LV missing
+
+    const result = await enableSwap(ctx);
+    expect(result.success).toBe(true);
+    expect(result.changed).toBe(false);
+    expectNoCommand(ctx.ssh, "swapon");
+  });
+});
+
+// --- Rancher LV (imageFs sizing) ---
+
+import { growRancherLv } from "../src/operations/rancher-storage.js";
+
+describe("growRancherLv", () => {
+  it("grows a 20G LV to 120G when the VG has space", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec
+      .mockResolvedValueOnce(stdout("  20480.00"))   // lv_size
+      .mockResolvedValueOnce(stdout("  747807.00"))  // vg_free
+      .mockResolvedValueOnce(OK)                     // lvextend
+      .mockResolvedValueOnce(OK);                    // xfs_growfs
+
+    const result = await growRancherLv(ctx);
+    expect(result.success).toBe(true);
+    expect(result.changed).toBe(true);
+    expectCommand(ctx.ssh, "lvextend -L 122880m /dev/labvg/rancher");
+    expectCommand(ctx.ssh, "xfs_growfs /var/lib/rancher");
+  });
+
+  it("is idempotent when the LV is already 120G", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(stdout("  122880.00")); // lv_size
+
+    const result = await growRancherLv(ctx);
+    expect(result.changed).toBe(false);
+    expectNoCommand(ctx.ssh, "lvextend");
+  });
+
+  it("reports without failing when the VG has no free space", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec
+      .mockResolvedValueOnce(stdout("  20480.00"))  // lv_size
+      .mockResolvedValueOnce(stdout("  0.00"));     // vg_free
+
+    const result = await growRancherLv(ctx);
+    expect(result.success).toBe(true);
+    expect(result.changed).toBe(false);
+    expect(result.message).toContain("free");
+    expectNoCommand(ctx.ssh, "lvextend");
+  });
+
+  it("skips when there is no rancher LV", async () => {
+    const ctx = mockCtx();
+    ctx.ssh.exec.mockResolvedValueOnce(stdout("")); // lvs empty
+
+    const result = await growRancherLv(ctx);
+    expect(result.success).toBe(true);
+    expect(result.changed).toBe(false);
+    expectNoCommand(ctx.ssh, "lvextend");
   });
 });
 

bastion/src/modules/modules/k3s/tests/smoke.test.ts

@@ -16,7 +16,7 @@ describe("smoke: full server install pipeline", () => {
     const pipeline: NamedOperation[] = [
       { name: "Kernel modules", fn: ops.loadKernelModules },
       { name: "Sysctl hardening", fn: ops.applyCisHardening },
-      { name: "Disable swap", fn: ops.disableSwap },
+      { name: "Enable swap", fn: ops.enableSwap },
       { name: "Disable firewall", fn: ops.disableFirewall },
       { name: "SELinux permissive", fn: ops.setSelinuxPermissive },
       { name: "Write k3s config", fn: ops.writeK3sConfig },
@@ -73,7 +73,7 @@ describe("smoke: pipeline stops on failure", () => {
     };
 
     const results = await runSequential(ctx, [
-      { name: "OK op", fn: ops.disableSwap },
+      { name: "OK op", fn: ops.enableSwap },
       { name: "Failing op", fn: failingOp },
       { name: "Never called", fn: neverCalled },
     ]);
@@ -98,11 +98,12 @@ describe("smoke: agent install rejects missing config", () => {
 });
 
 describe("smoke: all operations are exported", () => {
-  it("exports all 15 operations", () => {
+  it("exports all 16 operations", () => {
     const exported = [
       ops.loadKernelModules,
       ops.applyCisHardening,
-      ops.disableSwap,
+      ops.enableSwap,
+      ops.growRancherLv,
       ops.disableFirewall,
       ops.setSelinuxPermissive,
       ops.writeK3sConfig,
@@ -117,7 +118,7 @@ describe("smoke: all operations are exported", () => {
       ops.checkCertExpiry,
     ];
 
-    expect(exported).toHaveLength(15);
+    expect(exported).toHaveLength(16);
     for (const op of exported) {
       expect(typeof op).toBe("function");
     }