lab/bastion/src/modules/modules/k3s/tests/operations.test.ts

// Unit tests for k3s operations.
// Each operation is tested for: correctness, idempotency, and error handling.

import { describe, it, expect, beforeEach } from "vitest";
import { mockCtx, OK, FAIL, stdout, expectCommand, expectNoCommand } from "./helpers.js";

// --- Kernel Modules ---

import { loadKernelModules } from "../src/operations/kernel-modules.js";

describe("loadKernelModules", () => {
  it("loads missing modules and writes config", async () => {
    const ctx = mockCtx();
    // lsmod checks: br_netfilter missing, overlay loaded, ip_conntrack missing
    ctx.ssh.exec
      .mockResolvedValueOnce(FAIL)   // br_netfilter not loaded
      .mockResolvedValueOnce(OK)     // modprobe br_netfilter
      .mockResolvedValueOnce(OK)     // overlay loaded
      .mockResolvedValueOnce(FAIL)   // ip_conntrack not loaded
      .mockResolvedValueOnce(OK)     // modprobe ip_conntrack
      .mockResolvedValueOnce(stdout("__LABCTL_NOT_FOUND__")) // cat config file
      .mockResolvedValueOnce(OK);    // write config

    const result = await loadKernelModules(ctx);
    expect(result.success).toBe(true);
    expect(result.changed).toBe(true);
    expect(result.details).toContain("Loaded: br_netfilter");
    expect(result.details).toContain("Already loaded: overlay");
  });

  it("is idempotent when all modules loaded and config exists", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec
      .mockResolvedValueOnce(OK) // br_netfilter loaded
      .mockResolvedValueOnce(OK) // overlay loaded
      .mockResolvedValueOnce(OK) // ip_conntrack loaded
      .mockResolvedValueOnce(stdout("br_netfilter\noverlay\nip_conntrack")); // config exists with correct content

    const result = await loadKernelModules(ctx);
    expect(result.success).toBe(true);
    expect(result.changed).toBe(false);
  });
});

// --- Sysctl ---

import { applyCisHardening } from "../src/operations/sysctl.js";

describe("applyCisHardening", () => {
  it("writes config and applies sysctl", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec
      .mockResolvedValueOnce(stdout("__LABCTL_NOT_FOUND__")) // file not found
      .mockResolvedValueOnce(OK)  // write file
      .mockResolvedValueOnce(OK); // sysctl --system

    const result = await applyCisHardening(ctx);
    expect(result.success).toBe(true);
    expect(result.changed).toBe(true);
    expectCommand(ctx.ssh, "sysctl --system");
  });

  it("skips sysctl when config unchanged", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec.mockResolvedValueOnce(stdout("# k3s CIS hardening\nnet.bridge.bridge-nf-call-iptables  = 1\nnet.bridge.bridge-nf-call-ip6tables = 1\nnet.ipv4.ip_forward                 = 1\nvm.panic_on_oom                     = 0\nvm.overcommit_memory                = 1\nkernel.panic                        = 10\nkernel.panic_on_oops                = 1\n# inotify limits for large clusters\nfs.inotify.max_user_instances       = 524288\nfs.inotify.max_user_watches         = 524288"));

    const result = await applyCisHardening(ctx);
    expect(result.changed).toBe(false);
    expectNoCommand(ctx.ssh, "sysctl --system");
  });
});

// --- Swap ---

import { disableSwap } from "../src/operations/swap.js";

describe("disableSwap", () => {
  it("disables active swap", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec
      .mockResolvedValueOnce(stdout("/dev/sda2 partition 2G"))  // swap active
      .mockResolvedValueOnce(OK)   // swapoff
      .mockResolvedValueOnce(OK);  // sed fstab

    const result = await disableSwap(ctx);
    expect(result.success).toBe(true);
    expect(result.changed).toBe(true);
    expectCommand(ctx.ssh, "swapoff -a");
  });

  it("is idempotent when swap already off", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec
      .mockResolvedValueOnce(stdout(""))  // no swap
      .mockResolvedValueOnce(OK);          // sed fstab (always runs)

    const result = await disableSwap(ctx);
    expect(result.changed).toBe(false);
    expectNoCommand(ctx.ssh, "swapoff");
  });
});

// --- Firewall ---

import { disableFirewall } from "../src/operations/firewall.js";

describe("disableFirewall", () => {
  it("disables active firewalld", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec
      .mockResolvedValueOnce(stdout("active"))  // firewalld active
      .mockResolvedValueOnce(OK)   // disable
      .mockResolvedValueOnce(OK)   // mask
      .mockResolvedValueOnce(FAIL);  // ufw not active

    const result = await disableFirewall(ctx);
    expect(result.success).toBe(true);
    expect(result.changed).toBe(true);
    expect(result.details).toContain("Disabled and masked: firewalld");
  });

  it("is idempotent when nothing active", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec
      .mockResolvedValueOnce(FAIL)  // firewalld not active
      .mockResolvedValueOnce(stdout("masked")) // already masked
      .mockResolvedValueOnce(FAIL)  // ufw not active
      .mockResolvedValueOnce(FAIL); // ufw not enabled

    const result = await disableFirewall(ctx);
    expect(result.changed).toBe(false);
  });
});

// --- SELinux ---

import { setSelinuxPermissive } from "../src/operations/selinux.js";

describe("setSelinuxPermissive", () => {
  it("sets enforcing to permissive", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec
      .mockResolvedValueOnce(stdout("Enforcing"))  // current mode
      .mockResolvedValueOnce(OK)   // setenforce 0
      .mockResolvedValueOnce(OK);  // sed config

    const result = await setSelinuxPermissive(ctx);
    expect(result.changed).toBe(true);
  });

  it("skips when already permissive", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec.mockResolvedValueOnce(stdout("Permissive"));

    const result = await setSelinuxPermissive(ctx);
    expect(result.changed).toBe(false);
  });

  it("skips when disabled", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec.mockResolvedValueOnce(stdout("Disabled"));

    const result = await setSelinuxPermissive(ctx);
    expect(result.changed).toBe(false);
  });
});

// --- K3s Config ---

import { writeK3sConfig } from "../src/operations/k3s-config.js";

describe("writeK3sConfig", () => {
  it("writes server config with TLS SANs", async () => {
    const ctx = mockCtx({ hostname: "node1.lab", ip: "10.0.1.1", role: "infra" });
    ctx.ssh.exec
      .mockResolvedValueOnce(OK)   // mkdir
      .mockResolvedValueOnce(stdout("__LABCTL_NOT_FOUND__")) // cat existing
      .mockResolvedValueOnce(OK);  // write

    const result = await writeK3sConfig(ctx);
    expect(result.changed).toBe(true);

    // Verify the written content includes TLS SANs
    const writeCall = ctx.ssh.exec.mock.calls[2]![0] as string;
    expect(writeCall).toContain("node1.lab");
    expect(writeCall).toContain("10.0.1.1");
    expect(writeCall).toContain("secrets-encryption: true");
    expect(writeCall).toContain("flannel-backend: none");
  });

  it("writes minimal agent config", async () => {
    const ctx = mockCtx({ role: "worker" });
    ctx.ssh.exec
      .mockResolvedValueOnce(OK)   // mkdir
      .mockResolvedValueOnce(stdout("__LABCTL_NOT_FOUND__"))
      .mockResolvedValueOnce(OK);

    const result = await writeK3sConfig(ctx);
    expect(result.changed).toBe(true);

    const writeCall = ctx.ssh.exec.mock.calls[2]![0] as string;
    expect(writeCall).toContain("protect-kernel-defaults: true");
    expect(writeCall).not.toContain("secrets-encryption");
  });
});

// --- CNI Cleanup ---

import { cleanupStaleCni } from "../src/operations/cni-cleanup.js";

describe("cleanupStaleCni", () => {
  it("stops k3s and removes stale interfaces", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec
      .mockResolvedValueOnce(stdout("active"))  // k3s is active
      .mockResolvedValueOnce(OK)    // systemctl stop k3s
      .mockResolvedValueOnce(OK)    // flannel.1 exists
      .mockResolvedValueOnce(OK)    // delete flannel.1
      .mockResolvedValueOnce(FAIL)  // cilium_vxlan not found
      .mockResolvedValueOnce(FAIL)  // cilium_host not found
      .mockResolvedValueOnce(FAIL)  // cilium_net not found
      .mockResolvedValueOnce(stdout(""))  // no vxlans
      .mockResolvedValueOnce(OK);   // rm -rf cni

    const result = await cleanupStaleCni(ctx);
    expect(result.success).toBe(true);
    expect(result.changed).toBe(true);
    expect(result.details).toContain("Stopped k3s service");
    expect(result.details).toContain("Removed interface: flannel.1");
  });

  it("is idempotent when nothing to clean", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec
      .mockResolvedValueOnce(FAIL)  // k3s not active
      .mockResolvedValueOnce(FAIL)  // flannel.1 not found
      .mockResolvedValueOnce(FAIL)  // cilium_vxlan
      .mockResolvedValueOnce(FAIL)  // cilium_host
      .mockResolvedValueOnce(FAIL)  // cilium_net
      .mockResolvedValueOnce(stdout(""))  // no vxlans
      .mockResolvedValueOnce(OK);   // rm -rf (always runs)

    const result = await cleanupStaleCni(ctx);
    expect(result.changed).toBe(false);
  });
});

// --- K3s Install ---

import { installK3sBinary } from "../src/operations/k3s-install.js";

describe("installK3sBinary", () => {
  it("installs k3s server", async () => {
    const ctx = mockCtx({ role: "infra" });
    ctx.ssh.exec
      .mockResolvedValueOnce(FAIL)  // k3s not installed
      .mockResolvedValueOnce(OK)    // curl install
      .mockResolvedValueOnce(OK)    // restart
      .mockResolvedValueOnce(OK);   // kubectl get nodes

    const result = await installK3sBinary(ctx);
    expect(result.success).toBe(true);
    expect(result.changed).toBe(true);
  });

  it("fails agent without server URL", async () => {
    const ctx = mockCtx({ role: "worker" });
    ctx.ssh.exec.mockResolvedValueOnce(FAIL);  // not installed

    const result = await installK3sBinary(ctx);
    expect(result.success).toBe(false);
    expect(result.error).toContain("Missing agent");
  });

  it("installs k3s agent with URL and token", async () => {
    const ctx = mockCtx({ role: "worker", k3sServerUrl: "https://10.0.0.1:6443", k3sToken: "secret" });
    ctx.ssh.exec
      .mockResolvedValueOnce(FAIL)  // not installed
      .mockResolvedValueOnce(OK)    // curl install
      .mockResolvedValueOnce(OK);   // restart

    const result = await installK3sBinary(ctx);
    expect(result.success).toBe(true);
    expectCommand(ctx.ssh, "K3S_URL=");
    expectCommand(ctx.ssh, "K3S_TOKEN=");
  });
});

// --- DNS Fix ---

import { fixCoreDnsUpstream } from "../src/operations/dns-fix.js";

describe("fixCoreDnsUpstream", () => {
  it("detects upstream DNS and writes resolv.conf", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec
      .mockResolvedValueOnce(stdout("192.168.8.1"))  // resolvectl
      .mockResolvedValueOnce(stdout("__LABCTL_NOT_FOUND__"))  // cat existing
      .mockResolvedValueOnce(OK)    // write resolv.conf
      .mockResolvedValueOnce(stdout("active"))  // k3s active
      .mockResolvedValueOnce(OK)    // restart
      .mockResolvedValueOnce(OK);   // kubectl get nodes

    const result = await fixCoreDnsUpstream(ctx);
    expect(result.success).toBe(true);
    expect(result.changed).toBe(true);
    expect(result.message).toContain("192.168.8.1");
  });

  it("falls back to /run/systemd/resolve/resolv.conf", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec
      .mockResolvedValueOnce(stdout(""))  // resolvectl empty
      .mockResolvedValueOnce(stdout("10.0.0.1"))  // fallback resolv.conf
      .mockResolvedValueOnce(stdout("__LABCTL_NOT_FOUND__"))
      .mockResolvedValueOnce(OK)
      .mockResolvedValueOnce(stdout("active"))
      .mockResolvedValueOnce(OK)
      .mockResolvedValueOnce(OK);

    const result = await fixCoreDnsUpstream(ctx);
    expect(result.changed).toBe(true);
  });

  it("skips when upstream cannot be detected", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec
      .mockResolvedValueOnce(stdout(""))       // resolvectl empty
      .mockResolvedValueOnce(stdout("127.0.0.53")); // fallback is still stub

    const result = await fixCoreDnsUpstream(ctx);
    expect(result.changed).toBe(false);
  });
});

// --- Pod Security ---

import { applyPodSecurityStandards } from "../src/operations/pod-security.js";

describe("applyPodSecurityStandards", () => {
  it("applies all three labels", async () => {
    const ctx = mockCtx();
    const result = await applyPodSecurityStandards(ctx);
    expect(result.success).toBe(true);
    expect(ctx.ssh.exec).toHaveBeenCalledTimes(3);
    expectCommand(ctx.ssh, "pod-security.kubernetes.io/enforce=restricted");
    expectCommand(ctx.ssh, "pod-security.kubernetes.io/warn=restricted");
    expectCommand(ctx.ssh, "pod-security.kubernetes.io/audit=restricted");
  });
});

// --- Audit Logging Decommission (file-based → journald) ---

import { configureLogRotation } from "../src/operations/log-rotation.js";
import { configureJournaldLimits } from "../src/operations/journald-limits.js";

describe("configureLogRotation (decommission file-based audit logs)", () => {
  it("removes the legacy logrotate rule and reaps obsolete audit files", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec.mockResolvedValueOnce(stdout("present")); // probe: legacy artifacts exist
    ctx.ssh.exec.mockResolvedValue(OK);

    const result = await configureLogRotation(ctx);

    expect(result.success).toBe(true);
    expect(result.changed).toBe(true);
    expectCommand(ctx.ssh, "rm -f /etc/logrotate.d/k3s");
    expectCommand(ctx.ssh, /find \/var\/log\/kubernetes.*audit.*-delete/);
    expectCommand(ctx.ssh, "rmdir /var/log/kubernetes");
  });

  it("is a no-op when nothing legacy is present", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec.mockResolvedValueOnce(stdout("absent"));
    ctx.ssh.exec.mockResolvedValue(OK);

    const result = await configureLogRotation(ctx);
    expect(result.success).toBe(true);
    expect(result.changed).toBe(false);
  });
});

describe("configureJournaldLimits", () => {
  it("writes a 2 GB SystemMaxUse drop-in and reloads journald when changed", async () => {
    const ctx = mockCtx();
    ctx.ssh.exec.mockResolvedValueOnce(stdout("__LABCTL_NOT_FOUND__")); // no existing drop-in
    ctx.ssh.exec.mockResolvedValue(OK);

    const result = await configureJournaldLimits(ctx);

    expect(result.success).toBe(true);
    expect(result.changed).toBe(true);
    const writeCall = ctx.ssh.exec.mock.calls.find((c) => {
      const cmd = c[0] as string;
      return cmd.includes("10-k3s-audit-cap.conf") && cmd.includes("LABCTL_EOF");
    });
    expect(writeCall).toBeTruthy();
    const written = writeCall?.[0] as string;
    expect(written).toContain("SystemMaxUse=2G");
    expect(written).toContain("SystemKeepFree=1G");
    expectCommand(ctx.ssh, "systemctl restart systemd-journald");
  });

  it("does not restart journald when the drop-in is already correct", async () => {
    const ctx = mockCtx();
    const existing =
      "[Journal]\nSystemMaxUse=2G\nSystemKeepFree=1G\nSystemMaxFileSize=200M\n";
    ctx.ssh.exec.mockResolvedValueOnce(stdout(existing));
    ctx.ssh.exec.mockResolvedValue(OK);

    const result = await configureJournaldLimits(ctx);

    expect(result.success).toBe(true);
    expect(result.changed).toBe(false);
    expectNoCommand(ctx.ssh, "systemctl restart systemd-journald");
  });
});

// --- Etcd Recovery ---

import { recoverEtcdMember } from "../src/operations/etcd-recover.js";
import { mockSsh } from "./helpers.js";

describe("recoverEtcdMember", () => {
  it("refuses to operate when cluster is below 3 members (quorum risk)", async () => {
    const broken = mockSsh();
    const peer = mockSsh();
    peer.exec.mockResolvedValueOnce(stdout("/usr/bin/etcdctl")); // etcdctl present
    peer.exec.mockResolvedValueOnce(stdout(
      "111, started, host-a-aaa, https://10.0.0.1:2380, https://10.0.0.1:2379, false\n" +
      "222, started, host-b-bbb, https://10.0.0.2:2380, https://10.0.0.2:2379, false",
    ));

    const result = await recoverEtcdMember({ broken, peer, brokenHostname: "host-b" });

    expect(result.success).toBe(false);
    expect(result.message).toMatch(/quorum/i);
    // Critically: must NOT have stopped k3s or removed anything
    expect(broken.exec).not.toHaveBeenCalledWith(expect.stringContaining("systemctl stop k3s"), expect.anything());
  });

  it("performs full procedure when quorum is preserved", async () => {
    const broken = mockSsh();
    const peer = mockSsh();
    // ensureEtcdctl: present
    peer.exec.mockResolvedValueOnce(stdout("/usr/bin/etcdctl"));
    // member list (3 members, target = host-b)
    peer.exec.mockResolvedValueOnce(stdout(
      "111, started, host-a-aaa, https://10.0.0.1:2380, https://10.0.0.1:2379, false\n" +
      "222, started, host-b-bbb, https://10.0.0.2:2380, https://10.0.0.2:2379, false\n" +
      "333, started, host-c-ccc, https://10.0.0.3:2380, https://10.0.0.3:2379, false",
    ));
    // member remove
    peer.exec.mockResolvedValueOnce(stdout("Member 222 removed"));
    // post-rejoin member list — new id 444 for host-b
    peer.exec.mockResolvedValueOnce(stdout(
      "111, started, host-a-aaa, https://10.0.0.1:2380, https://10.0.0.1:2379, false\n" +
      "333, started, host-c-ccc, https://10.0.0.3:2380, https://10.0.0.3:2379, false\n" +
      "444, started, host-b-zzz, https://10.0.0.2:2380, https://10.0.0.2:2379, false",
    ));

    const result = await recoverEtcdMember({ broken, peer, brokenHostname: "host-b" });

    expect(result.success).toBe(true);
    expect(result.removedMemberId).toBe("222");
    expect(result.newMemberId).toBe("444");
    expectCommand(broken,"systemctl stop k3s");
    expectCommand(peer,"member remove 222");
    expectCommand(broken,/db\.corrupt-/);
    expectCommand(broken,/rm -rf .*\/server\/tls/);
    expectCommand(broken,"systemctl start k3s");
  });

  it("fails clearly when no member matches the broken hostname", async () => {
    const broken = mockSsh();
    const peer = mockSsh();
    peer.exec.mockResolvedValueOnce(stdout("/usr/bin/etcdctl"));
    peer.exec.mockResolvedValueOnce(stdout(
      "111, started, host-a-aaa, https://10.0.0.1:2380, https://10.0.0.1:2379, false\n" +
      "222, started, host-b-bbb, https://10.0.0.2:2380, https://10.0.0.2:2379, false\n" +
      "333, started, host-c-ccc, https://10.0.0.3:2380, https://10.0.0.3:2379, false",
    ));

    const result = await recoverEtcdMember({ broken, peer, brokenHostname: "host-d" });

    expect(result.success).toBe(false);
    expect(result.message).toMatch(/No etcd member found/);
    expect(broken.exec).not.toHaveBeenCalledWith(expect.stringContaining("systemctl stop k3s"), expect.anything());
  });
});