lab/bastion/tests/integration/cluster-smoke.test.ts

// Integration test: form a real 3-server k3s cluster from PXE-installed VMs.
//
// WHY THIS EXISTS SEPARATELY FROM THE DRILL
// -----------------------------------------
// The rolling drill is a ~3 hour test. Almost everything that can go wrong in
// its first thirty minutes goes wrong here instead, in forty-five: the token
// handoff, the cluster-init/join discriminator, the etcd bootstrap race, root
// SSH for the module, and whether post-provision was actually suppressed.
// Debugging those inside a three-hour test is how you spend a day per attempt.
//
// It asserts on ETCD MEMBERSHIP, not just `kubectl get nodes`. A node object
// and an etcd member are different things, and the whole hazard in a rolling
// reinstall is that they can disagree.
//
// Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo. Set FEDORA_MIRROR to
// a local mirror or each install pulls the full package set over the internet.
//
//   sudo -E npx vitest run -c tests/integration/vitest.config.ts -t "cluster smoke"

import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { existsSync, readFileSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import {
  startTestBastion, provisionVms, destroyVms,
  type Harness, type ProvisionedVm,
} from "./helpers/provision-harness.js";
import {
  formCluster, clusterSnapshot, printSnapshot, etcdMembers, etcdHealthy,
  allServersReady, pickJoinTarget, kubectlOn, k8sNodeName,
} from "./helpers/k3s-cluster.js";
import { destroyPxeNetwork } from "./helpers/pxe-network.js";

const HTTP_PORT = 8097;
const SSH_USER = "lab";
const DISK_GB = 1000;
const NODES = ["smoke-a", "smoke-b", "smoke-c"];

function findSshKey(): { pubKey: string; keyPath: string } {
  const homes = [homedir()];
  const sudoUser = process.env["SUDO_USER"];
  if (sudoUser) homes.push(join("/home", sudoUser));
  for (const home of homes) {
    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
      const keyPath = join(home, ".ssh", name);
      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
      }
    }
  }
  throw new Error("no SSH keypair found");
}

describe("cluster smoke: three servers, one cluster", () => {
  let h: Harness;
  let vms: ProvisionedVm[] = [];

  /**
   * Kubernetes node names, NOT libvirt domain names.
   *
   * k3s registers a node by FQDN, so the VM `smoke-a` is the node
   * `smoke-a.pxe-test.local`. Asserting against the VM names would compare two
   * different namespaces and fail for a reason that has nothing to do with the
   * cluster.
   */
  const nodeNames = (): string[] => vms.map((v) => k8sNodeName(h, v)).sort();

  beforeAll(async () => {
    const { pubKey, keyPath } = findSshKey();
    destroyVms(...NODES);
    h = await startTestBastion({
      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey, sshKeyPath: keyPath,
      label: "cluster-smoke",
      // Mandatory. Left on, post-provision installs each node into its OWN
      // cluster and this test would be measuring three clusters of one.
      autoK3s: false,
    });
  }, 20 * 60_000);

  afterAll(async () => {
    // Dump state before anything is destroyed — the VMs are the only place
    // this information exists, and they are about to stop existing.
    try {
      if (vms.length > 0) printSnapshot("final", clusterSnapshot(h, vms[0]!));
    } catch { /* cluster may already be gone */ }
    destroyVms(...NODES);
    try { destroyPxeNetwork(); } catch { /* shared */ }
    await h?.close();
  }, 10 * 60_000);

  it("provisions three v2 combo nodes", async () => {
    vms = await provisionVms(h, NODES.map((name) => ({
      name, role: "k8s-combo" as const, schemaVersion: 2 as const,
      diskGb: DISK_GB, memory: 6144, vcpus: 4,
    })));
    expect(vms).toHaveLength(3);
    for (const vm of vms) expect(vm.ip).toMatch(/^\d+\.\d+\.\d+\.\d+$/);
  }, 60 * 60_000);

  it("forms ONE cluster, not three", async () => {
    await formCluster(h, vms, { role: "k8s-combo" });

    const snap = clusterSnapshot(h, vms[0]!);
    printSnapshot("after formation", snap);

    // The precise failure this guards: three separate single-node clusters
    // each report 1/1 Ready and look perfectly healthy in isolation.
    expect(snap.nodes).toHaveLength(3);
    expect(snap.readyCount).toBe(3);
    expect(snap.nodes.map((n) => n.name).sort()).toEqual(nodeNames());
  }, 90 * 60_000);

  it("has exactly three healthy etcd members", () => {
    const members = etcdMembers(h, vms[0]!);
    // .node, not .name: etcd names a member "<node>-<8 hex>", so comparing
    // raw names against node names never matches.
    expect(members.map((m) => m.node).sort()).toEqual(nodeNames());
    expect(members).toHaveLength(3);
    expect(members.every((m) => m.started)).toBe(true);
    expect(etcdHealthy(h, vms[0]!)).toBe(true);
  }, 5 * 60_000);

  it("agrees about membership from EVERY node, not just the first", () => {
    // Asked of each server independently: a split brain is three nodes that
    // each believe they are correct.
    for (const vm of vms) {
      const members = etcdMembers(h, vm).map((m) => m.node).sort();
      expect(members, `etcd membership as seen by ${vm.name}`).toEqual(nodeNames());
      const nodes = clusterSnapshot(h, vm).nodes.map((n) => n.name).sort();
      expect(nodes, `node list as seen by ${vm.name}`).toEqual(nodeNames());
    }
  }, 10 * 60_000);

  it("reports every server ready", () => {
    const ready = allServersReady(h, vms);
    expect(ready.ok, ready.detail).toBe(true);
  }, 5 * 60_000);

  it("makes all three control-plane and etcd", () => {
    const snap = clusterSnapshot(h, vms[0]!);
    for (const n of snap.nodes) {
      expect(n.roles, `${n.name} roles`).toContain("control-plane");
      expect(n.roles, `${n.name} roles`).toContain("etcd");
    }
  }, 5 * 60_000);

  it("can pick a join target other than a given node", () => {
    // The primitive the rolling drill depends on: after the first server is
    // wiped it must rejoin against a survivor.
    const target = pickJoinTarget(h, vms, vms[0]!);
    expect(target.name).not.toBe(vms[0]!.name);
  }, 5 * 60_000);

  it("carries the Longhorn disk label on every node", () => {
    // Set via k3s node-label. Without it Longhorn creates no disk on the node,
    // and the node then hosts zero replicas while looking entirely healthy.
    const out = kubectlOn(h, vms[0]!,
      `get nodes -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.labels.node\\.longhorn\\.io/create-default-disk}{"\\n"}{end}'`);
    for (const vm of vms) {
      expect(out, `${vm.name} create-default-disk label`)
        .toContain(`${k8sNodeName(h, vm)}=config`);
    }
  }, 5 * 60_000);
});