// THE MIGRATION DRILL: roll three storage+etcd nodes through // evacuate -> wipe -> reinstall -> rejoin, one at a time, with data intact. // // WHAT IT IS PROVING // ------------------ // The v1 -> v2 layout migration cannot preserve data through the wipe: %pre // only preserves when labvg already has the expected LV names, and a new layout // recreates the VG, taking the clearpart branch. So the data has to leave the // node first and come back afterwards. This asserts, on real machines, that it // does -- by sha256, either side of every step. // // WHY FOUR NODES FOR A THREE-NODE ROLL // ------------------------------------ // With strict anti-affinity (replica-soft-anti-affinity=false, what the fleet // runs) a volume puts at most one replica per node, so a 3-replica volume on a // 3-node cluster CANNOT be evicted: the two survivors already hold a copy each // and cannot take the third. Eviction would wait for its full timeout and fail. // The real fleet has exactly this shape, plus a 100 GiB disk on worker2 with // ~49 GiB free -- so capacity blocks it independently of anti-affinity. // // The fix is the one already planned for production: bring the NEW box up // first, then evacuate an old one onto it. The fourth node here is an AGENT, so // etcd stays a 3-member cluster and every roll runs against production's real // quorum arithmetic (2 of 3, no margin). // // ORDER OF THE DANGEROUS PART // --------------------------- // The irreversible pair -- etcd member removal and node deletion -- happens as // late as possible, immediately before the wipe. Anything that fails before it // has destroyed nothing. // // sudo -E npx vitest run -c tests/integration/vitest.config.ts -t "rolling drill" import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { existsSync, readFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { K3sModule } from "../../src/modules/modules/k3s/src/k3s-module.js"; import { startTestBastion, provisionVms, reprovisionVm, readLvs, destroyVms, type Harness, type ProvisionedVm, } from "./helpers/provision-harness.js"; import { PINNED_K3S_VERSION } from "./helpers/k3s-cluster.js"; import { formCluster, joinAgent, installK3sNode, serverUrlOf, moduleCtx, clusterSnapshot, printSnapshot, etcdMembers, etcdHealthy, allServersReady, pickJoinTarget, kubectlOn, kubectlJson, nodeExec, removeEtcdMember, waitForNodeReady, waitForApiPing, sleep, DRILL_TOKEN, k8sNodeName, forgetNodeName, } from "./helpers/k3s-cluster.js"; import { installLonghorn, waitForLonghornReady, createDataWorkload, waitForWorkload, writeBlob, verifyBlob, volumeState, waitForVolumeHealthy, LONGHORN_NS, backupTargetFromEnv, setBackupTarget, waitForBackupTarget, backupVolume, restoreFromBackup, readBlobDigest, } from "./helpers/longhorn.js"; import { destroyPxeNetwork } from "./helpers/pxe-network.js"; const HTTP_PORT = 8098; const SSH_USER = "lab"; const DISK_GB = 1000; const SERVERS = ["drill-s1", "drill-s2", "drill-s3"]; const SPARE = "drill-spare"; const ALL = [...SERVERS, SPARE]; /** The replica-3 volume: the one whose rebuild the roll depends on. */ const R3 = "drill-r3"; /** The replica-1 volume: single copy, the sharpest risk in the real fleet. */ const R1 = "drill-r1"; function findSshKey(): { pubKey: string; keyPath: string } { const homes = [homedir()]; const sudoUser = process.env["SUDO_USER"]; if (sudoUser) homes.push(join("/home", sudoUser)); for (const home of homes) { for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) { const keyPath = join(home, ".ssh", name); if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) { return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath }; } } } throw new Error("no SSH keypair found"); } describe("rolling drill: migrate three nodes without losing data", () => { let h: Harness; let vms: ProvisionedVm[] = []; let spare: ProvisionedVm; let servers: ProvisionedVm[] = []; let digestR3 = ""; let digestR1 = ""; /** * S3 backup target, from DRILL_S3_* in the environment. * * Undefined means the backup/restore phase SKIPS rather than passing * vacuously. A backup test that quietly does nothing is worse than none. */ const backupTarget = backupTargetFromEnv(); /** * Sticky abort. Once a step fails, the remaining steps SKIP rather than run * against a cluster that is already wrong -- a failed roll leaves a node * mid-migration, and rolling the next one on top of that turns one recoverable * problem into an unrecoverable one. */ let aborted: string | null = null; /** * `ctx` is vitest's test context. It is used to SKIP downstream steps rather * than let them return quietly — a silent return marks the test PASSED, so a * drill that died in provisioning reports "1 failed | 8 passed" and the eight * steps that never ran look like evidence. They are not evidence. */ const step = async ( ctx: { skip: (note?: string) => void }, name: string, fn: () => Promise, ): Promise => { if (aborted !== null) { console.log(`[drill] SKIPPED ${name} (aborted at: ${aborted})`); // eslint-disable-line no-console ctx.skip(`drill aborted at: ${aborted}`); return; } const t0 = Date.now(); console.log(`[drill] ==> ${name}`); // eslint-disable-line no-console try { await fn(); console.log(`[drill] <== ${name} (${Math.round((Date.now() - t0) / 1000)}s)`); // eslint-disable-line no-console } catch (e) { aborted = name; console.error(`[drill] !!! ${name} FAILED: ${e instanceof Error ? e.message : String(e)}`); // eslint-disable-line no-console throw e; } }; /** * The runbook's replica-1 pre-flight, rehearsed rather than described. * * A single-replica volume rebuilds from the only copy that exists, so * `evacuate` refuses it by default. The documented answer is to raise it to * 3, wait for healthy, evacuate, and put it back afterwards. Doing that here * means the drill exercises the procedure operators will actually follow * instead of quietly bypassing it with the override flag. */ const raiseReplicas = async (peer: ProvisionedVm, pvc: string, to: number): Promise => { const st = volumeState(h, peer, pvc); kubectlOn(h, peer, `-n ${LONGHORN_NS} patch volumes.longhorn.io ${st.pv} ` + `--type=merge -p '{"spec":{"numberOfReplicas":${to}}}'`); if (to <= 1) return true; try { // BOUNDED, and allowed to fail. // // Raising a volume's replica count only converges if every target node // will actually start a replica. A node that has just been wiped and // rejoined may not: observed here, a replica sat at // currentState=stopped / desireState=running / failedAt='' for forty // minutes with no rebuild and no error, while the volume stayed // degraded at 2-of-3 RW. Blocking the whole drill on that turns a // Longhorn quirk into a total failure and proves nothing. await waitForVolumeHealthy(h, peer, { pvc, expectReplicas: to, timeoutMs: 10 * 60_000 }); return true; } catch (e) { console.warn( // eslint-disable-line no-console `[drill] ${pvc} did not reach ${to} healthy replicas in 10 min: ` + `${e instanceof Error ? e.message : String(e)}`); return false; } }; /** Is this volume's only replica on the node we are about to evacuate? */ const replicaOn = (peer: ProvisionedVm, pvc: string, node: string): boolean => volumeState(h, peer, pvc).replicaNodes.includes(node); /** Read cluster state from a node that is NOT the one being worked on. */ const survivor = (exclude?: ProvisionedVm): ProvisionedVm => exclude ? pickJoinTarget(h, servers, exclude) : servers[0]!; beforeAll(async () => { const { pubKey, keyPath } = findSshKey(); destroyVms(...ALL); h = await startTestBastion({ httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey, sshKeyPath: keyPath, label: "rolling-drill", autoK3s: false, // we sequence cluster formation ourselves }); }, 20 * 60_000); afterAll(async () => { // Diagnostics BEFORE teardown: these VMs are the only record of what // happened and they are about to be deleted. try { const s = servers[0]; if (s) { printSnapshot("FINAL", clusterSnapshot(h, s)); for (const pvc of [R3, R1]) { try { const st = volumeState(h, s, pvc); console.log(`[drill] ${pvc}: ${st.state}/${st.robustness} on ${st.replicaNodes.join(",")}`); // eslint-disable-line no-console } catch { /* volume may be gone */ } } console.log(`[drill] digests: ${R3}=${digestR3.slice(0, 16)} ${R1}=${digestR1.slice(0, 16)}`); // eslint-disable-line no-console } } catch { /* cluster may be unreachable */ } // Best-effort: clear any eviction we may have left pending. Harmless on a // VM about to be destroyed, but this is the same call the runbook makes on // a real node, so exercising it here is free coverage. for (const vm of servers) { try { await new K3sModule().admit(moduleCtx(h, vm, "k8s-combo")); } catch { /* ignore */ } } if (process.env["LAB_DRILL_KEEP_VMS"] !== "1") { destroyVms(...ALL); try { destroyPxeNetwork(); } catch { /* shared */ } } else { console.log("[drill] LAB_DRILL_KEEP_VMS=1 — leaving VMs up for inspection"); // eslint-disable-line no-console } await h?.close(); }, 20 * 60_000); it("provisions four v2 nodes", async (ctx) => { await step(ctx, "provision", async () => { vms = await provisionVms(h, [ ...SERVERS.map((name) => ({ name, role: "k8s-combo" as const, schemaVersion: 2 as const, diskGb: DISK_GB, memory: 6144, vcpus: 4, })), { name: SPARE, role: "k8s-worker" as const, schemaVersion: 2 as const, diskGb: DISK_GB, memory: 4096, vcpus: 2 }, ]); servers = vms.filter((v) => SERVERS.includes(v.name)); spare = vms.find((v) => v.name === SPARE)!; expect(servers).toHaveLength(3); // Every node must actually have the longhorn LV, or nothing later works. for (const vm of vms) expect(Object.keys(readLvs(h, vm))).toContain("longhorn"); }); }, 90 * 60_000); it("forms one cluster with three etcd members plus a storage agent", async (ctx) => { await step(ctx, "form cluster", async () => { await formCluster(h, servers, { role: "k8s-combo" }); await joinAgent(h, spare, { server: servers[0]!, role: "k8s-worker" }); const snap = clusterSnapshot(h, servers[0]!); printSnapshot("formed", snap); expect(snap.nodes).toHaveLength(4); expect(snap.readyCount).toBe(4); // The spare is an agent on purpose: etcd stays at 3 so each roll runs // against production's real quorum arithmetic. expect(etcdMembers(h, servers[0]!)).toHaveLength(3); expect(etcdHealthy(h, servers[0]!)).toBe(true); // Every node must run the PINNED version, not whatever get.k3s.io calls // latest today. Setting k3sVersion is not the same as it taking effect: // toOpContext used to drop the field silently, so the pin was ignored and // nodes came up two minor versions ahead of the fleet. Assert the // outcome, on the machines. const versions = kubectlJson<{ items: { metadata: { name: string }; status: { nodeInfo: { kubeletVersion: string } }; }[] }>(h, servers[0]!, "get nodes"); expect(versions.items).toHaveLength(4); for (const n of versions.items) { expect(n.status.nodeInfo.kubeletVersion, `${n.metadata.name} must run the pinned ${PINNED_K3S_VERSION}`) .toBe(PINNED_K3S_VERSION); } console.log(`[drill] all 4 nodes on pinned ${PINNED_K3S_VERSION}`); // eslint-disable-line no-console }); }, 120 * 60_000); it("installs Longhorn with a schedulable disk on every node", async (ctx) => { await step(ctx, "install longhorn", async () => { await installLonghorn(h, servers[0]!, { defaultReplicas: 3, ...(backupTarget === undefined ? {} : { backup: backupTarget }), }); // Give Longhorn time to register the nodes, then run the PRODUCTION // repair path on each one. // // configureLonghornDisk runs during configure(), which happens BEFORE // Longhorn is installed -- there is no node record to repair yet, so it // only sets the annotation and returns. If Longhorn then misses that // annotation for any reason, nothing ever retries, and the node joins, // goes Ready, and silently hosts no replicas. Observed: one node in four // came up with no disk while its label and annotation were byte-identical // to its siblings'. // // admit() is exactly what the runbook tells an operator to run, and it // exercises the verify-and-repair branch that is otherwise only reachable // after a wipe-and-rejoin. await sleep(60_000); for (const vm of vms) { const role = vm.name === SPARE ? "k8s-worker" : "k8s-combo"; const res = await new K3sModule().admit(moduleCtx(h, vm, role)); for (const line of res.output) console.log(` [${vm.name}] ${line}`); // eslint-disable-line no-console } // All four VMs, including the agent: the spare having a schedulable disk // is the whole reason eviction can complete at all. await waitForLonghornReady(h, servers[0]!, vms); // Read it back rather than trusting the wait: every node must have a // disk, or replicas silently never land there. // // -o json and parse, NOT jsonpath. A jsonpath containing {"\n"} has to // survive a TS template literal AND a shell, and getting either escape // wrong yields `unterminated quoted string` at runtime — which is how // this assertion failed the first time it ran. const lhNodes = kubectlJson<{ items: { metadata: { name: string }; spec: { disks?: Record }; }[] }>(h, servers[0]!, `-n ${LONGHORN_NS} get nodes.longhorn.io`); for (const vm of vms) { const name = k8sNodeName(h, vm); const rec = lhNodes.items.find((n) => n.metadata.name === name); expect(rec, `${name} has no Longhorn node record`).toBeDefined(); const disks = Object.keys(rec?.spec.disks ?? {}); expect(disks.length, `${name} has no Longhorn disk — the default-disks-config ` + `annotation never parsed, so this node would silently host no replicas`) .toBeGreaterThan(0); } }); }, 40 * 60_000); it("writes known data and proves it survives a detach/attach", async (ctx) => { await step(ctx, "seed data", async () => { const s = servers[0]!; createDataWorkload(h, s, { name: R3, sizeGi: 2, replicas: 3 }); createDataWorkload(h, s, { name: R1, sizeGi: 1, replicas: 1 }); await waitForWorkload(h, s, R3); await waitForWorkload(h, s, R1); digestR3 = await writeBlob(h, s, { name: R3, mib: 512 }); digestR1 = await writeBlob(h, s, { name: R1, mib: 128 }); // Record the digest only AFTER it has survived one restart. Otherwise // the baseline could be a page-cache artefact and every later // comparison would be against a value that was never really on disk. await verifyBlob(h, s, { name: R3, expectDigest: digestR3 }); await verifyBlob(h, s, { name: R1, expectDigest: digestR1 }); const st = volumeState(h, s, R3); expect(st.robustness).toBe("healthy"); expect(new Set(st.replicaNodes).size).toBe(3); }); }, 60 * 60_000); it("backs up a volume to S3 and restores it byte-identical", async (ctx) => { await step(ctx, "backup and restore", async () => { const s = servers[0]!; if (backupTarget === undefined) { // Deliberately loud. A backup phase that silently passes because no // target was configured is exactly the kind of green tick that makes // people believe they have backups when they do not. throw new Error( "no backup target configured (DRILL_S3_TARGET / _ENDPOINT / _ACCESS_KEY / _SECRET_KEY). " + "The backup/restore phase cannot prove anything without one."); } // Set it on the CR. defaultSettings.backupTarget does nothing in 1.8 -- // see setBackupTarget for why. await setBackupTarget(h, s, backupTarget); // `available: true` is Longhorn saying it reached the endpoint AND // authenticated. Backing up before checking gives a failure that reads // like a volume problem. await waitForBackupTarget(h, s); const { backup } = await backupVolume(h, s, R3); // Restore into a SEPARATE volume. The original is untouched, so a failed // restore cannot damage the source -- which is the property that makes // this safe to rehearse at all. await restoreFromBackup(h, s, { backup, name: "drill-restored", sizeGi: 2 }); await waitForWorkload(h, s, "drill-restored"); // The whole point: the bytes that came back are the bytes that went in. // Compared against the digest recorded BEFORE any of this, which was // itself only trusted after surviving a detach/attach. const restored = readBlobDigest(h, s, "drill-restored"); expect(restored, "restored volume digest must equal the original").toBe(digestR3); console.log(`[drill] restored digest ${restored.slice(0, 16)}... matches original`); // eslint-disable-line no-console // And the restored volume is a real, independent volume. const st = volumeState(h, s, "drill-restored"); expect(st.pv).not.toBe(volumeState(h, s, R3).pv); expect(st.state).toBe("attached"); }); }, 45 * 60_000); it("REFUSES to evacuate when there is nowhere for replicas to go", async (ctx) => { await step(ctx, "negative pre-check", async () => { const s = servers[0]!; // Must be a node that actually HOLDS an r3 replica, and not the one we // read cluster state from. r3 has 3 replicas across 4 nodes, so one node // has none -- picking blindly makes this test pass or fail on placement // luck rather than on the behaviour it is meant to pin. // replicaNodes are KUBERNETES node names (smoke-a.pxe-test.local), not // libvirt domain names, so resolve before comparing. const r3Nodes = volumeState(h, s, R3).replicaNodes; const target = servers.find( (v) => v.name !== s.name && r3Nodes.includes(k8sNodeName(h, v))); expect(target, `no server other than ${s.name} holds an ${R3} replica (${r3Nodes.join(",")})`) .toBeDefined(); // Take the spare out of service: now 3 schedulable nodes for a 3-replica // volume, which is exactly the real fleet's shape. kubectlOn(h, s, `-n ${LONGHORN_NS} patch nodes.longhorn.io ${k8sNodeName(h, spare)} ` + `--type=merge -p '{"spec":{"allowScheduling":false}}'`); await sleep(10_000); const t0 = Date.now(); const res = await new K3sModule().evacuate(moduleCtx(h, target!, "k8s-combo")); const elapsed = (Date.now() - t0) / 1000; expect(res.success, "evacuation should have been refused").toBe(false); // Fast is the whole point: the old behaviour was a 60-minute wait. expect(elapsed, `refused in ${elapsed}s`).toBeLessThan(300); expect(res.errors.join(" ")).toMatch(/cannot complete|cannot be placed|wants 3 replicas/); // And it must not have touched anything on the way to saying no. const lhNode = kubectlOn(h, s, `-n ${LONGHORN_NS} get nodes.longhorn.io ${k8sNodeName(h, target!)} ` + `-o jsonpath='{.spec.evictionRequested}'`).trim(); expect(lhNode, "refusal must not leave an eviction pending").not.toBe("true"); // Put the spare back and un-cordon whatever the attempt cordoned. kubectlOn(h, s, `-n ${LONGHORN_NS} patch nodes.longhorn.io ${k8sNodeName(h, spare)} ` + `--type=merge -p '{"spec":{"allowScheduling":true}}'`); await new K3sModule().admit(moduleCtx(h, target!, "k8s-combo")); await sleep(15_000); }); }, 30 * 60_000); // The roll itself. Serial by construction -- each `it` depends on the last. for (const name of SERVERS) { it(`rolls ${name}: evacuate, wipe, reinstall, rejoin, verify`, async (ctx) => { await step(ctx, `roll ${name}`, async () => { const target = vms.find((v) => v.name === name)!; const peer = survivor(target); // Kubernetes and Longhorn both know this machine by its FQDN, not by // the libvirt domain name used everywhere else in this file. const nodeName = k8sNodeName(h, target); // --- GATE: never start a roll on a cluster that is already degraded. const pre = clusterSnapshot(h, peer); printSnapshot(`before rolling ${name}`, pre); expect(pre.etcdMembers, "etcd membership before roll").toBe(3); expect(pre.etcdHealthy, "etcd health before roll").toBe(true); expect(pre.readyCount, "all nodes Ready before roll").toBe(4); // --- Replica-1 pre-flight, exactly as the runbook prescribes. // // If drill-r1's single replica is on this node, evacuate MUST refuse: // moving it means rebuilding from the only copy in existence, and this // fleet has no backup target. Assert that refusal, then apply the // documented mitigation rather than overriding the guard. const r1Here = replicaOn(peer, R1, nodeName); let r1Raised = false; if (r1Here) { const refused = await new K3sModule().evacuate(moduleCtx(h, target, "k8s-combo")); expect(refused.success, "single-replica volume should have blocked evacuation").toBe(false); expect(refused.errors.join(" ")).toMatch(/single replica/); await new K3sModule().admit(moduleCtx(h, target, "k8s-combo")); r1Raised = await raiseReplicas(peer, R1, 3); } // --- Evacuate (cordon -> check -> evict+wait -> drain -> verify). // // If raising drill-r1 did not converge, fall back to the documented // override rather than stalling. The guard itself is already proven by // the refusal above and by the negative pre-check; what this roll is // here to prove is the migration. const evacConfig = r1Here && !r1Raised ? { evictionAllowSingleReplica: true } : {}; if (r1Here && !r1Raised) { console.warn( // eslint-disable-line no-console "[drill] proceeding with --allow-single-replica because the raise did not converge"); } const evac = await new K3sModule().evacuate(moduleCtx(h, target, "k8s-combo", evacConfig)); for (const line of evac.output) console.log(` [${name}] ${line}`); // eslint-disable-line no-console expect(evac.success, `evacuate ${name}: ${evac.errors.join("; ")}`).toBe(true); // Replicas really gone, and the r3 volume really rebuilt elsewhere. const afterEvac = await waitForVolumeHealthy(h, peer, { pvc: R3, expectReplicas: 3 }); expect(afterEvac.replicaNodes).not.toContain(nodeName); // --- Data intact after evacuation, BEFORE anything is destroyed. await verifyBlob(h, peer, { name: R3, expectDigest: digestR3 }); await verifyBlob(h, peer, { name: R1, expectDigest: digestR1 }); // --- The irreversible part starts here and not before. nodeExec(h, target, "systemctl stop k3s || true"); const removed = removeEtcdMember(h, peer, nodeName); expect(removed, `etcd member ${nodeName} should have existed`).toBe(true); kubectlOn(h, peer, `delete node ${nodeName} --wait=true --timeout=5m`, 6 * 60_000); // Quorum asserted POSITIVELY: 2 of 3 is survivable, silence is not proof. const mid = clusterSnapshot(h, peer); printSnapshot(`${name} removed`, mid); expect(mid.etcdMembers, "etcd must be down to exactly 2 members").toBe(2); expect(mid.etcdHealthy, "surviving etcd must still be healthy").toBe(true); const ready = allServersReady(h, servers.filter((v) => v.name !== name)); expect(ready.ok, `survivors /readyz: ${ready.detail}`).toBe(true); // --- Wipe and reinstall. The disk decides the schema, not the request. // // reprovisionVm returns a NEW record: the machine can come back on a // different DHCP address. Every later reference must use that record, // and the shared arrays have to be updated too or `survivor()` will // hand out a stale IP on the next roll. const rolled = await reprovisionVm(h, target, "k8s-combo"); vms = vms.map((v) => (v.name === name ? rolled : v)); servers = servers.map((v) => (v.name === name ? rolled : v)); // A reinstalled machine re-registers; do not trust the cached name. forgetNodeName(rolled); // v2 reinstall PRESERVES /var/lib/longhorn, so without this the next // step would measure replica re-adoption, not a real rebuild. The // actual v1->v2 migration recreates the VG and has no such leftovers. // // Clear the CONTENTS, never the directory itself. Longhorn's replica // process does a non-recursive `mkdir /host/var/lib/longhorn/replicas/ // `, which fails outright when the parent is missing: // // level=fatal msg="Error running start replica command" // error="mkdir /host/var/lib/longhorn/replicas/pvc-...: ..." // // Every replica scheduled onto the node then dies instantly and // permanently, while the node looks entirely healthy: longhorn-disk.cfg // survives the wipe, so the disk keeps its UUID and reports Ready and // Schedulable. The next node's eviction can never drain onto it and // stalls for its full hour. Two earlier drill runs died exactly here. nodeExec(h, rolled, "rm -rf /var/lib/longhorn/replicas/* /var/lib/longhorn/*.tmp 2>/dev/null; " + "mkdir -p /var/lib/longhorn/replicas && ls -d /var/lib/longhorn/replicas"); // --- Rejoin against a SURVIVOR, not against itself. await waitForApiPing(peer); await installK3sNode(h, rolled, { role: "k8s-combo", token: DRILL_TOKEN, serverUrl: serverUrlOf(peer), }); await waitForNodeReady(h, peer, rolled); // Exactly 3. Four would mean the old member survived the rejoin, which // silently raises quorum from 2-of-3 to 3-of-4 and removes all margin. const post = clusterSnapshot(h, peer); printSnapshot(`${name} rejoined`, post); expect(post.etcdMembers, "stale etcd member left behind").toBe(3); expect(post.etcdHealthy).toBe(true); expect(post.readyCount).toBe(4); // --- Back into service, then wait for the replica to rebuild onto it. const admit = await new K3sModule().admit(moduleCtx(h, rolled, "k8s-combo")); for (const line of admit.output) console.log(` [${name}] ${line}`); // eslint-disable-line no-console expect(admit.success, `admit ${name}: ${admit.errors.join("; ")}`).toBe(true); const healthy = await waitForVolumeHealthy(h, peer, { pvc: R3, expectReplicas: 3 }); expect(new Set(healthy.replicaNodes).size).toBe(3); // --- And the data is still the data. await verifyBlob(h, peer, { name: R3, expectDigest: digestR3 }); await verifyBlob(h, peer, { name: R1, expectDigest: digestR1 }); // Put drill-r1 back to a single replica, so the next roll faces the // same hazard rather than a progressively safer cluster. if (r1Here && r1Raised) await raiseReplicas(peer, R1, 1); // --- Does the rolled node actually take replicas BACK? // // Roll 1 originally "passed" with drill-r3 healthy on three nodes that // did not include the node just rolled -- the assertion counted // distinct nodes without ever checking that the reinstalled one was // among them. A node that rejoins, reports Ready and Schedulable, and // then hosts nothing is the exact failure this whole exercise exists to // catch, so record it explicitly. // RUNNING, not merely assigned. The first version of this check used // replicaNodes (which comes from spec.nodeID) and happily passed on a // node holding two replicas that were both state=stopped -- exactly the // condition that makes the NEXT eviction impossible, because Longhorn // keeps scheduling replacements onto a node that cannot start them. const running = kubectlJson<{ items: { spec: { nodeID?: string }; status?: { currentState?: string }; }[] }>(h, peer, `-n ${LONGHORN_NS} get replicas.longhorn.io`) .items.filter((r) => r.spec.nodeID === nodeName && r.status?.currentState === "running"); console.log( // eslint-disable-line no-console `[drill] after rolling ${name}: ${running.length} RUNNING replica(s) on the node`); expect(running.length, `${name} rejoined but hosts no RUNNING replica — it would look healthy and ` + `silently take no storage, and the next eviction could never drain onto it`) .toBeGreaterThan(0); }); }, 240 * 60_000); } it("ends with a whole cluster and the original bytes", async (ctx) => { await step(ctx, "final verification", async () => { const s = servers[0]!; const snap = clusterSnapshot(h, s); printSnapshot("end of drill", snap); expect(snap.nodes).toHaveLength(4); expect(snap.readyCount).toBe(4); expect(snap.etcdMembers).toBe(3); expect(snap.etcdHealthy).toBe(true); const r3 = volumeState(h, s, R3); expect(r3.robustness).toBe("healthy"); expect(new Set(r3.replicaNodes).size).toBe(3); // One last restart-and-check, so the final claim is about bytes read // back through a fresh attach rather than a cached mount. await verifyBlob(h, s, { name: R3, expectDigest: digestR3 }); await verifyBlob(h, s, { name: R1, expectDigest: digestR1 }); }); }, 60 * 60_000); });