Some checks failed
The fleet's disk layout was decided by literal role-name comparisons at 17 sites across 8 files. That is why `labcontroller` got no rancher LV (it matched neither "infra" nor "worker"), why labmaster carried an empty 728 GiB Longhorn LV, and why a 62 GiB container image tipped that node into DiskPressure and evicted bastion, labd, cockroach and openbao. Three generators each hardcoded their own sizes and disagreed: the kickstart said rancher=120G, ubuntu-autoinstall said 20G, asahi-firstboot said 20G under a comment asserting the sizes matched install.ks.ts. They did not. ROLE_REGISTRY already carried `k3s` and `parent` and nothing read them. This makes capabilities authoritative, moves every size into one shared module, and adds schema v2 (fixed 250G rancher / 300G longhorn, never --grow). Existing machines stay v1 forever: schema is detected from the on-disk VG tag, so a v1 box stays v1 even when v2 is requested. Proven on real VMs, not just unit tests: all four topologies installed and had their layouts read back off the machines; markers in /home, /srv, /var/lib/longhorn and /var/lib/rancher survived reinstall; and a 4-node cluster formed, took real data, and had all three servers rolled one at a time through evacuate / wipe / reinstall / rejoin with checksums verified before and after each roll. Drill: 10/10, evidence in migration/drill-evidence/. Production defects the drill found, none of which a unit test could reach: - sshOpts() capped every operation at 30s while drainNode asked kubectl for --timeout=15m, so the transport killed the drain after evicting most pods and returned a transcript that read like success. `labctl app k3s evacuate` could never have drained a real node. - generateAgentConfig put node-role.kubernetes.io/worker in the kubelet's node-label list, which NodeRestriction rejects, so no agent installed through K3sModule could start. systemd reported only "did not take the steps required by its unit configuration". - toOpContext built K3sConfig by enumerating three keys and silently dropped the rest, so the k3s version pin did nothing and --allow-single-replica was a no-op that claimed to override a data-loss guard. - The Longhorn disk annotation was mangled by shell quoting; with createDefaultDiskLabeledNodes the node then gets no disk at all, joins, goes Ready, and silently hosts no replica. - evacuate drained before evicting, which blocks on Longhorn's PDBs and would destroy a single-replica volume's only source. - etcd member ids were parsed from JSON, losing uint64 precision and emitting decimal where `member remove` needs hex -- at the moment a node is stopped and about to be wiped. - get.k3s.io installs latest, two minor versions ahead of the fleet, so a reinstalled node would rejoin on a newer k3s than its peers and roll an unplanned two-version control-plane upgrade. K3sConfig.k3sVersion now pins it. - `labctl app k3s install --role x` mapped anything that was not "worker" to "infra", so the runbook's own `--role k8s-combo` would have installed a different role with different disk expectations and no warning. Also: /var/lib/longhorn/replicas must never be deleted, only its contents, or every replica placed on the node dies on a non-recursive mkdir -- for ever, while the node reports Ready, Schedulable and healthy. etcd is now in the kickstart's package set for k3s server roles, because `k3s etcdctl` is not a command and a member that needs to recover its own membership has no internet. 542 unit tests / 39 files. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
187 lines
7.9 KiB
TypeScript
187 lines
7.9 KiB
TypeScript
import { describe, it, expect } from "vitest";
|
|
import { renderInstallKickstart } from "../src/templates/install.ks.js";
|
|
import type { InstallKickstartParams } from "../src/templates/install.ks.js";
|
|
import { SUPPORTED_ROLES, LEGACY_ROLES, v2Layout, type Role } from "@lab/shared";
|
|
|
|
/**
|
|
* Partition-layout tests.
|
|
*
|
|
* The existing ksvalidator test only sees the MAIN kickstart body. The actual
|
|
* layout is written by %pre into /tmp/part.ks inside a heredoc, so until now
|
|
* nothing validated it at all -- a broken layout rendered, shipped and was only
|
|
* discovered on the installed machine.
|
|
*
|
|
* These parse the heredocs. Following tests/ubuntu-autoinstall.test.ts:
|
|
* "A `toContain` assertion would not have caught that; only parsing does."
|
|
*/
|
|
|
|
function baseParams(over: Partial<InstallKickstartParams> = {}): InstallKickstartParams {
|
|
return {
|
|
hostname: "test-node", disk: "nvme0n1", role: "worker", domain: "ad.itaz.eu",
|
|
fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8",
|
|
serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514,
|
|
sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal",
|
|
...over,
|
|
};
|
|
}
|
|
|
|
/** Pull every `cat >[>] /tmp/part.ks << PARTEOF ... PARTEOF` body out of the script. */
|
|
function partKsBlocks(ks: string): string[] {
|
|
const out: string[] = [];
|
|
const re = /cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g;
|
|
let m: RegExpExecArray | null;
|
|
while ((m = re.exec(ks)) !== null) out.push(m[1]!);
|
|
return out;
|
|
}
|
|
|
|
interface Logvol { mount: string; name: string; fs: string; size?: number; useExisting: boolean }
|
|
|
|
function parseLogvols(block: string): Logvol[] {
|
|
return block.split("\n")
|
|
.filter((l) => l.trim().startsWith("logvol "))
|
|
.map((line) => {
|
|
const parts = line.trim().split(/\s+/);
|
|
const get = (flag: string) => {
|
|
const p = parts.find((x) => x.startsWith(`${flag}=`));
|
|
return p ? p.slice(flag.length + 1) : undefined;
|
|
};
|
|
return {
|
|
mount: parts[1]!,
|
|
name: get("--name") ?? "",
|
|
fs: get("--fstype") ?? "",
|
|
...(get("--size") ? { size: Number(get("--size")) } : {}),
|
|
useExisting: parts.includes("--useexisting"),
|
|
};
|
|
});
|
|
}
|
|
|
|
/** The fresh-install block is the one that also creates the PV. */
|
|
function freshBlock(ks: string): string {
|
|
const b = partKsBlocks(ks).find((x) => x.includes("part pv.01"));
|
|
if (!b) throw new Error("no fresh-install part.ks block found");
|
|
return b;
|
|
}
|
|
|
|
describe("v1 is frozen — existing machines must reinstall identically", () => {
|
|
it.each(LEGACY_ROLES)("%s v1 fresh layout is unchanged", (role) => {
|
|
const lvs = parseLogvols(freshBlock(renderInstallKickstart(baseParams({ role }))));
|
|
const byName = Object.fromEntries(lvs.map((l) => [l.name, l]));
|
|
|
|
// The exact v1 sizes, in MiB. If one of these changes, every existing box
|
|
// in the fleet would be re-laid-out on its next reinstall.
|
|
expect(byName["swap"]!.size).toBe(27648);
|
|
expect(byName["root"]!.size).toBe(33792);
|
|
expect(byName["var"]!.size).toBe(102400);
|
|
expect(byName["varlog"]!.size).toBe(10240);
|
|
expect(byName["home"]!.size).toBe(10240);
|
|
expect(byName["srv"]!.size).toBe(20480);
|
|
});
|
|
|
|
it("v1 gives rancher to infra and worker only — the historical bug, preserved", () => {
|
|
const has = (r: Role) => freshBlock(renderInstallKickstart(baseParams({ role: r }))).includes("--name=rancher");
|
|
expect(has("infra")).toBe(true);
|
|
expect(has("worker")).toBe(true);
|
|
expect(has("labcontroller")).toBe(false); // the bug, deliberately kept in v1
|
|
expect(has("vanilla")).toBe(false);
|
|
});
|
|
|
|
it("v1 is the default — no caller has to opt out", () => {
|
|
const ks = renderInstallKickstart(baseParams({ role: "worker" }));
|
|
expect(parseLogvols(freshBlock(ks)).find((l) => l.name === "root")!.size).toBe(33792);
|
|
});
|
|
});
|
|
|
|
describe("v2-only roles have no v1 layout", () => {
|
|
it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"] as const)(
|
|
"%s defaults to v2 when no schema is given", (role) => {
|
|
const lvs = parseLogvols(freshBlock(renderInstallKickstart(baseParams({ role }))));
|
|
expect(lvs.find((l) => l.name === "root")!.size).toBe(32768);
|
|
expect(lvs.some((l) => l.name === "rancher")).toBe(true);
|
|
});
|
|
|
|
it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"] as const)(
|
|
"%s throws on an EXPLICIT v1 rather than silently dropping its volumes", (role) => {
|
|
expect(() => renderInstallKickstart(baseParams({ role, schemaVersion: 1 }))).toThrow(/no v1 disk layout/);
|
|
});
|
|
});
|
|
|
|
describe("v2 renders the shared layout", () => {
|
|
it.each(SUPPORTED_ROLES)("%s v2 matches v2Layout() exactly", (role) => {
|
|
const ks = renderInstallKickstart(baseParams({ role, schemaVersion: 2 }));
|
|
const rendered = parseLogvols(freshBlock(ks));
|
|
const expected = v2Layout(role);
|
|
|
|
expect(rendered.map((l) => l.name).sort()).toEqual(expected.map((l) => l.name).sort());
|
|
for (const spec of expected) {
|
|
const got = rendered.find((l) => l.name === spec.name)!;
|
|
expect(got.size).toBe(spec.sizeMib);
|
|
expect(got.fs).toBe(spec.fs);
|
|
}
|
|
});
|
|
|
|
it("v2 gives labcontroller a rancher LV — the v1 bug, fixed", () => {
|
|
const ks = renderInstallKickstart(baseParams({ role: "labcontroller", schemaVersion: 2 }));
|
|
const rancher = parseLogvols(freshBlock(ks)).find((l) => l.name === "rancher");
|
|
expect(rancher).toBeDefined();
|
|
expect(rancher!.size).toBe(256000);
|
|
});
|
|
|
|
it("v2 never uses --grow", () => {
|
|
for (const role of SUPPORTED_ROLES) {
|
|
const block = freshBlock(renderInstallKickstart(baseParams({ role, schemaVersion: 2 })));
|
|
expect(block).not.toMatch(/logvol[^\n]*--grow/);
|
|
}
|
|
});
|
|
|
|
it("v2 uses the larger EFI and /boot partitions", () => {
|
|
const block = freshBlock(renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 2 })));
|
|
expect(block).toMatch(/part \/boot\/efi --fstype=efi --size=1024/);
|
|
expect(block).toMatch(/part \/boot --fstype=ext4 --size=4096/);
|
|
});
|
|
});
|
|
|
|
describe("reprovision protects existing data", () => {
|
|
it("reprovision reads the on-disk schema instead of trusting the caller", () => {
|
|
const ks = renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 2 }));
|
|
expect(ks).toContain("DISK_SCHEMA=v1");
|
|
expect(ks).toMatch(/vgs --noheadings -o vg_tags .*grep -q lab-schema-v2/);
|
|
});
|
|
|
|
it("the reprovision branch never wipes the disk", () => {
|
|
const ks = renderInstallKickstart(baseParams({ role: "worker" }));
|
|
const repro = partKsBlocks(ks).find((b) => b.includes("--useexisting"))!;
|
|
expect(repro).toContain("clearpart --none");
|
|
expect(repro).not.toContain("clearpart --all");
|
|
});
|
|
|
|
it("preserved data volumes are never reformatted", () => {
|
|
const ks = renderInstallKickstart(baseParams({ role: "worker" }));
|
|
for (const lv of ["longhorn", "rancher", "home", "srv"]) {
|
|
const re = new RegExp(`--name=${lv} --useexisting --noformat`);
|
|
expect(ks).toMatch(re);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("the %pre block survives the /ks-auto extractor", () => {
|
|
it("matches the same regex ks-auto uses to re-run %pre", () => {
|
|
// routes/kickstart.ts re-extracts %pre with python:
|
|
// re.findall(r'%pre[^\n]*\n(.*?)%end', text, re.DOTALL)
|
|
// A change that breaks this breaks ISO installs silently.
|
|
for (const role of SUPPORTED_ROLES) {
|
|
// v2-only roles throw at v1 by design, so only render the combinations
|
|
// that are actually legal.
|
|
// An explicit v1 is illegal for a v2-only role; everything else renders.
|
|
const versions = (LEGACY_ROLES as readonly string[]).includes(role)
|
|
? ([1, 2] as const)
|
|
: ([2] as const);
|
|
for (const schemaVersion of versions) {
|
|
const ks = renderInstallKickstart(baseParams({ role, schemaVersion }));
|
|
const blocks = [...ks.matchAll(/%pre[^\n]*\n([\s\S]*?)%end/g)];
|
|
expect(blocks.length).toBeGreaterThan(0);
|
|
expect(blocks[0]![1]).toContain("/tmp/part.ks");
|
|
}
|
|
}
|
|
});
|
|
});
|