diff --git a/bastion/src/modules/modules/k3s/src/groups/host-prep.ts b/bastion/src/modules/modules/k3s/src/groups/host-prep.ts index ab1b4b8..d158415 100644 --- a/bastion/src/modules/modules/k3s/src/groups/host-prep.ts +++ b/bastion/src/modules/modules/k3s/src/groups/host-prep.ts @@ -1,21 +1,23 @@ -// Host preparation: kernel modules, sysctl, swap, firewall, SELinux. +// Host preparation: kernel modules, sysctl, swap, storage, firewall, SELinux. import type { OperationContext, OperationResult, OperationGroup } from "../types.js"; import { runSequential } from "../utils.js"; import { loadKernelModules } from "../operations/kernel-modules.js"; import { applyCisHardening } from "../operations/sysctl.js"; -import { disableSwap } from "../operations/swap.js"; +import { enableSwap } from "../operations/swap.js"; +import { growRancherLv } from "../operations/rancher-storage.js"; import { disableFirewall } from "../operations/firewall.js"; import { setSelinuxPermissive } from "../operations/selinux.js"; import { enableIscsi } from "../operations/iscsi.js"; export const hostPrepGroup: OperationGroup = { name: "host-prep", - description: "Prepare host for k3s: kernel modules, sysctl, swap, firewall, SELinux, iSCSI", + description: "Prepare host for k3s: kernel modules, sysctl, swap, imageFs sizing, firewall, SELinux, iSCSI", operations: [ { name: "Load kernel modules", fn: loadKernelModules }, { name: "Apply CIS sysctl", fn: applyCisHardening }, - { name: "Disable swap", fn: disableSwap }, + { name: "Enable swap", fn: enableSwap }, + { name: "Grow rancher LV", fn: growRancherLv }, { name: "Disable firewall", fn: disableFirewall }, { name: "Set SELinux permissive", fn: setSelinuxPermissive }, { name: "Enable iSCSI", fn: enableIscsi }, diff --git a/bastion/src/modules/modules/k3s/src/operations/index.ts b/bastion/src/modules/modules/k3s/src/operations/index.ts index f6434dc..d4783de 100644 --- a/bastion/src/modules/modules/k3s/src/operations/index.ts +++ b/bastion/src/modules/modules/k3s/src/operations/index.ts @@ -1,6 +1,7 @@ export { loadKernelModules } from "./kernel-modules.js"; export { applyCisHardening } from "./sysctl.js"; -export { disableSwap } from "./swap.js"; +export { enableSwap } from "./swap.js"; +export { growRancherLv } from "./rancher-storage.js"; export { enableIscsi } from "./iscsi.js"; export { disableFirewall } from "./firewall.js"; export { setSelinuxPermissive } from "./selinux.js"; diff --git a/bastion/src/modules/modules/k3s/src/operations/rancher-storage.ts b/bastion/src/modules/modules/k3s/src/operations/rancher-storage.ts new file mode 100644 index 0000000..a90fac7 --- /dev/null +++ b/bastion/src/modules/modules/k3s/src/operations/rancher-storage.ts @@ -0,0 +1,48 @@ +// Grow the labvg/rancher LV (k3s image store / imageFs) to 120G. +// 2026-08 incident: the original 20G LV sat at 85% used from steady-state +// images alone, so one ~5G image pull tripped imagefs eviction and evicted +// unrelated pods. Fresh installs are sized at 120G by the kickstart; this op +// covers nodes installed before that change and vanilla nodes converted to +// k8s later. Never removes or shrinks anything — if the VG lacks free space +// (e.g. a longhorn --grow LV consumed it), it reports and moves on. + +import type { Operation, OperationResult } from "../types.js"; +import { sshOpts } from "../utils.js"; + +const RANCHER_LV = "labvg/rancher"; +const TARGET_MIB = 122880; // 120G + +export const growRancherLv: Operation = async (ctx): Promise => { + const lv = await ctx.ssh.exec( + `lvs --noheadings --units m --nosuffix -o lv_size ${RANCHER_LV} 2>/dev/null || true`, + sshOpts(ctx), + ); + const sizeMib = Number.parseFloat(lv.stdout.trim()); + if (Number.isNaN(sizeMib)) { + return { success: true, changed: false, message: "No labvg/rancher LV — imageFs shares /var, skipping" }; + } + if (sizeMib >= TARGET_MIB) { + return { success: true, changed: false, message: `rancher LV already ${Math.round(sizeMib / 1024)}G` }; + } + + const vg = await ctx.ssh.exec(`vgs --noheadings --units m --nosuffix -o vg_free labvg`, sshOpts(ctx)); + const freeMib = Number.parseFloat(vg.stdout.trim()); + const neededMib = TARGET_MIB - sizeMib; + if (Number.isNaN(freeMib) || freeMib < neededMib) { + return { + success: true, + changed: false, + message: `VG labvg has ${Math.floor((Number.isNaN(freeMib) ? 0 : freeMib) / 1024)}G free — ` + + `need ${Math.ceil(neededMib / 1024)}G to grow rancher LV to 120G (manual LV rebuild required)`, + }; + } + + await ctx.ssh.exec(`lvextend -L ${TARGET_MIB}m /dev/${RANCHER_LV}`, sshOpts(ctx)); + await ctx.ssh.exec(`xfs_growfs /var/lib/rancher`, sshOpts(ctx)); + + return { + success: true, + changed: true, + message: `rancher LV grown ${Math.round(sizeMib / 1024)}G → 120G`, + }; +}; diff --git a/bastion/src/modules/modules/k3s/src/operations/swap.ts b/bastion/src/modules/modules/k3s/src/operations/swap.ts index cdc2b17..1a86211 100644 --- a/bastion/src/modules/modules/k3s/src/operations/swap.ts +++ b/bastion/src/modules/modules/k3s/src/operations/swap.ts @@ -1,22 +1,40 @@ -// Disable swap (CIS requirement for k3s). +// Enable swap so memory pressure spills to disk instead of OOM-killing. +// kubelet runs with failSwapOn=false (k3s default); zram stays the fast tier, +// the labvg-swap LV is the overflow tier. Replaces the old CIS-style +// disableSwap op — a kernel OOM kill of a node daemon is worse than slow swap. import type { Operation, OperationResult } from "../types.js"; import { sshOpts } from "../utils.js"; -export const disableSwap: Operation = async (ctx): Promise => { - const check = await ctx.ssh.exec("swapon --show --noheadings", sshOpts(ctx)); - const active = check.stdout.trim().length > 0; +const SWAP_DEV = "/dev/mapper/labvg-swap"; - if (active) { - await ctx.ssh.exec("swapoff -a", sshOpts(ctx)); +export const enableSwap: Operation = async (ctx): Promise => { + const lv = await ctx.ssh.exec(`test -b ${SWAP_DEV} && echo yes || echo no`, sshOpts(ctx)); + if (lv.stdout.trim() !== "yes") { + return { success: true, changed: false, message: "No labvg-swap LV — skipping swap enable" }; } - // Remove swap entries from fstab permanently - await ctx.ssh.exec("sed -i '/\\sswap\\s/d' /etc/fstab", sshOpts(ctx)); + const active = await ctx.ssh.exec( + `grep -q "^$(readlink -f ${SWAP_DEV}) " /proc/swaps && echo on || echo off`, + sshOpts(ctx), + ); + const wasOff = active.stdout.trim() !== "on"; + + if (wasOff) { + // Format if the LV was never (or wrongly) initialised, then activate + await ctx.ssh.exec(`blkid ${SWAP_DEV} | grep -q 'TYPE="swap"' || mkswap ${SWAP_DEV}`, sshOpts(ctx)); + await ctx.ssh.exec(`swapon ${SWAP_DEV}`, sshOpts(ctx)); + } + + // Persist across reboots (idempotent) + await ctx.ssh.exec( + `grep -q "labvg-swap" /etc/fstab || echo "${SWAP_DEV} none swap defaults 0 0" >> /etc/fstab`, + sshOpts(ctx), + ); return { success: true, - changed: active, - message: active ? "Swap disabled" : "Swap already disabled", + changed: wasOff, + message: wasOff ? "LV swap enabled" : "LV swap already active", }; }; diff --git a/bastion/src/modules/modules/k3s/tests/operations.test.ts b/bastion/src/modules/modules/k3s/tests/operations.test.ts index 5a61ab9..3502242 100644 --- a/bastion/src/modules/modules/k3s/tests/operations.test.ts +++ b/bastion/src/modules/modules/k3s/tests/operations.test.ts @@ -72,31 +72,97 @@ describe("applyCisHardening", () => { // --- Swap --- -import { disableSwap } from "../src/operations/swap.js"; +import { enableSwap } from "../src/operations/swap.js"; -describe("disableSwap", () => { - it("disables active swap", async () => { +describe("enableSwap", () => { + it("activates LV swap when present but off", async () => { const ctx = mockCtx(); ctx.ssh.exec - .mockResolvedValueOnce(stdout("/dev/sda2 partition 2G")) // swap active - .mockResolvedValueOnce(OK) // swapoff - .mockResolvedValueOnce(OK); // sed fstab + .mockResolvedValueOnce(stdout("yes")) // LV exists + .mockResolvedValueOnce(stdout("off")) // not in /proc/swaps + .mockResolvedValueOnce(OK) // blkid || mkswap + .mockResolvedValueOnce(OK) // swapon + .mockResolvedValueOnce(OK); // fstab entry - const result = await disableSwap(ctx); + const result = await enableSwap(ctx); expect(result.success).toBe(true); expect(result.changed).toBe(true); - expectCommand(ctx.ssh, "swapoff -a"); + expectCommand(ctx.ssh, "swapon /dev/mapper/labvg-swap"); }); - it("is idempotent when swap already off", async () => { + it("is idempotent when LV swap already active", async () => { const ctx = mockCtx(); ctx.ssh.exec - .mockResolvedValueOnce(stdout("")) // no swap - .mockResolvedValueOnce(OK); // sed fstab (always runs) + .mockResolvedValueOnce(stdout("yes")) // LV exists + .mockResolvedValueOnce(stdout("on")) // already in /proc/swaps + .mockResolvedValueOnce(OK); // fstab entry (always ensured) - const result = await disableSwap(ctx); + const result = await enableSwap(ctx); expect(result.changed).toBe(false); - expectNoCommand(ctx.ssh, "swapoff"); + expectNoCommand(ctx.ssh, "swapon /dev/mapper/labvg-swap"); + }); + + it("skips when no labvg-swap LV exists", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(stdout("no")); // LV missing + + const result = await enableSwap(ctx); + expect(result.success).toBe(true); + expect(result.changed).toBe(false); + expectNoCommand(ctx.ssh, "swapon"); + }); +}); + +// --- Rancher LV (imageFs sizing) --- + +import { growRancherLv } from "../src/operations/rancher-storage.js"; + +describe("growRancherLv", () => { + it("grows a 20G LV to 120G when the VG has space", async () => { + const ctx = mockCtx(); + ctx.ssh.exec + .mockResolvedValueOnce(stdout(" 20480.00")) // lv_size + .mockResolvedValueOnce(stdout(" 747807.00")) // vg_free + .mockResolvedValueOnce(OK) // lvextend + .mockResolvedValueOnce(OK); // xfs_growfs + + const result = await growRancherLv(ctx); + expect(result.success).toBe(true); + expect(result.changed).toBe(true); + expectCommand(ctx.ssh, "lvextend -L 122880m /dev/labvg/rancher"); + expectCommand(ctx.ssh, "xfs_growfs /var/lib/rancher"); + }); + + it("is idempotent when the LV is already 120G", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(stdout(" 122880.00")); // lv_size + + const result = await growRancherLv(ctx); + expect(result.changed).toBe(false); + expectNoCommand(ctx.ssh, "lvextend"); + }); + + it("reports without failing when the VG has no free space", async () => { + const ctx = mockCtx(); + ctx.ssh.exec + .mockResolvedValueOnce(stdout(" 20480.00")) // lv_size + .mockResolvedValueOnce(stdout(" 0.00")); // vg_free + + const result = await growRancherLv(ctx); + expect(result.success).toBe(true); + expect(result.changed).toBe(false); + expect(result.message).toContain("free"); + expectNoCommand(ctx.ssh, "lvextend"); + }); + + it("skips when there is no rancher LV", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(stdout("")); // lvs empty + + const result = await growRancherLv(ctx); + expect(result.success).toBe(true); + expect(result.changed).toBe(false); + expectNoCommand(ctx.ssh, "lvextend"); }); }); diff --git a/bastion/src/modules/modules/k3s/tests/smoke.test.ts b/bastion/src/modules/modules/k3s/tests/smoke.test.ts index e3234f8..8fbbb10 100644 --- a/bastion/src/modules/modules/k3s/tests/smoke.test.ts +++ b/bastion/src/modules/modules/k3s/tests/smoke.test.ts @@ -16,7 +16,7 @@ describe("smoke: full server install pipeline", () => { const pipeline: NamedOperation[] = [ { name: "Kernel modules", fn: ops.loadKernelModules }, { name: "Sysctl hardening", fn: ops.applyCisHardening }, - { name: "Disable swap", fn: ops.disableSwap }, + { name: "Enable swap", fn: ops.enableSwap }, { name: "Disable firewall", fn: ops.disableFirewall }, { name: "SELinux permissive", fn: ops.setSelinuxPermissive }, { name: "Write k3s config", fn: ops.writeK3sConfig }, @@ -73,7 +73,7 @@ describe("smoke: pipeline stops on failure", () => { }; const results = await runSequential(ctx, [ - { name: "OK op", fn: ops.disableSwap }, + { name: "OK op", fn: ops.enableSwap }, { name: "Failing op", fn: failingOp }, { name: "Never called", fn: neverCalled }, ]); @@ -98,11 +98,12 @@ describe("smoke: agent install rejects missing config", () => { }); describe("smoke: all operations are exported", () => { - it("exports all 15 operations", () => { + it("exports all 16 operations", () => { const exported = [ ops.loadKernelModules, ops.applyCisHardening, - ops.disableSwap, + ops.enableSwap, + ops.growRancherLv, ops.disableFirewall, ops.setSelinuxPermissive, ops.writeK3sConfig, @@ -117,7 +118,7 @@ describe("smoke: all operations are exported", () => { ops.checkCertExpiry, ]; - expect(exported).toHaveLength(15); + expect(exported).toHaveLength(16); for (const op of exported) { expect(typeof op).toBe("function"); }