lab/bastion/tests/integration/reinstall.test.ts

// Integration tests: reinstalling a machine must not destroy its data, and
// must not silently change its disk layout.
//
// This is the risk the whole v2 change carries. Every machine in the fleet was
// installed with v1, and `labctl provision reprovision` has to keep reproducing
// v1 on those boxes forever. If a reinstall quietly switched them to v2 it
// would recreate the volume group and take every Longhorn replica with it.
//
// Ported from test-reprovision.sh, which had the right idea (marker files in
// each preserved volume) but only ever ran by hand and was never in CI.
//
// Run: sudo -E npx vitest run -c tests/integration/vitest.config.ts -t "reinstall"

import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { existsSync, readFileSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import { v2Layout } from "../../src/shared/src/types/layout.js";
import {
  startTestBastion, provisionVm, reprovisionVm, readLvs, readSchemaTag,
  writeMarkers, readMarkers, destroyVms, type Harness,
} from "./helpers/provision-harness.js";
import { destroyPxeNetwork } from "./helpers/pxe-network.js";

const HTTP_PORT = 8095;
const SSH_USER = "lab";
const DISK_GB = 1000;

/** Mount points a reinstall is supposed to preserve on a v1 worker. */
const PRESERVED = ["/home", "/srv", "/var/lib/longhorn", "/var/lib/rancher"];

function findSshKey(): { pubKey: string; keyPath: string } {
  const homes = [homedir()];
  const sudoUser = process.env["SUDO_USER"];
  if (sudoUser) homes.push(join("/home", sudoUser));
  for (const home of homes) {
    for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
      const keyPath = join(home, ".ssh", name);
      if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) {
        return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath };
      }
    }
  }
  throw new Error("no SSH keypair found");
}

describe("reinstall preserves data and layout", () => {
  let h: Harness;
  const created: string[] = [];

  beforeAll(async () => {
    const { pubKey, keyPath } = findSshKey();
    h = await startTestBastion({
      httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey,
      sshKeyPath: keyPath, label: "reinstall",
    });
  }, 20 * 60_000);

  afterAll(async () => {
    destroyVms(...created);
    try { await h?.close(); } catch { /* closed */ }
    try { destroyPxeNetwork(); } catch { /* gone */ }
  }, 5 * 60_000);

  it("v1 worker: data survives and the layout stays v1", async () => {
    const name = "re-v1-worker";
    created.push(name);

    // Install at v1 -- i.e. exactly what every machine in the fleet is today.
    let vm = await provisionVm(h, { name, role: "worker", diskGb: DISK_GB });

    const before = readLvs(h, vm);
    expect(before["root"], "v1 root is 33G, not v2's 32G").toBeGreaterThan(33000);
    expect(before["rancher"], "v1 worker gets a 120G rancher LV").toBeGreaterThan(120000 - 8);
    expect(readSchemaTag(h, vm), "a v1 disk must NOT carry the v2 tag").not.toContain("lab-schema-v2");

    const token = writeMarkers(h, vm, PRESERVED);

    // Reinstall. No schemaVersion passed -- the disk decides.
    vm = await reprovisionVm(h, vm, "worker");

    const markers = readMarkers(h, vm, PRESERVED);
    for (const mount of PRESERVED) {
      expect(markers[mount], `${mount} must survive a reinstall`).toBe(token);
    }

    // And the layout must be untouched.
    const after = readLvs(h, vm);
    expect(Object.keys(after).sort()).toEqual(Object.keys(before).sort());
    expect(after["root"]).toBe(before["root"]);
    expect(after["rancher"]).toBe(before["rancher"]);
    expect(readSchemaTag(h, vm)).not.toContain("lab-schema-v2");
  }, 120 * 60_000);

  it("v2 worker: data survives and the layout stays v2", async () => {
    const name = "re-v2-worker";
    created.push(name);

    let vm = await provisionVm(h, { name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB });
    const before = readLvs(h, vm);
    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");

    const token = writeMarkers(h, vm, PRESERVED);
    vm = await reprovisionVm(h, vm, "k8s-worker");

    const markers = readMarkers(h, vm, PRESERVED);
    for (const mount of PRESERVED) {
      expect(markers[mount], `${mount} must survive a reinstall`).toBe(token);
    }

    const after = readLvs(h, vm);
    const expected = Object.fromEntries(v2Layout("k8s-worker").map((lv) => [lv.name, lv.sizeMib]));
    expect(Object.keys(after).sort()).toEqual(Object.keys(expected).sort());
    expect(after["root"]).toBe(before["root"]);
    // The tag must survive too, or the NEXT reinstall would fall back to v1.
    expect(readSchemaTag(h, vm)).toContain("lab-schema-v2");
  }, 120 * 60_000);

  it("a v1 machine is NOT upgraded to v2 just because v2 was requested", async () => {
    const name = "re-v1-sticky";
    created.push(name);

    let vm = await provisionVm(h, { name, role: "worker", diskGb: DISK_GB });
    const before = readLvs(h, vm);
    const token = writeMarkers(h, vm, ["/home", "/srv"]);

    // Ask for v2 on a disk that is v1. %pre must ignore it: silently
    // re-laying-out a live machine is how data gets destroyed.
    vm = await reprovisionVm(h, vm, "worker", 2);

    const after = readLvs(h, vm);
    expect(after["root"], "root must stay at the v1 size").toBe(before["root"]);
    expect(readSchemaTag(h, vm), "still not tagged v2").not.toContain("lab-schema-v2");
    const markers = readMarkers(h, vm, ["/home", "/srv"]);
    expect(markers["/home"]).toBe(token);
    expect(markers["/srv"]).toBe(token);
  }, 120 * 60_000);
});