Files
lab/bastion/src/bastion/tests/kickstart-layout.test.ts
Michal 84836cb8f2
Some checks failed
CI/CD / typecheck (push) Failing after 9s
CI/CD / test (push) Failing after 10s
CI/CD / lint (push) Failing after 24s
CI/CD / build (push) Has been skipped
CI/CD / publish-rpm (push) Has been skipped
CI/CD / publish-deb (push) Has been skipped
v2 partition schema: capability model, fixed sizing, and a proven rolling migration
The fleet's disk layout was decided by literal role-name comparisons at 17
sites across 8 files. That is why `labcontroller` got no rancher LV (it
matched neither "infra" nor "worker"), why labmaster carried an empty 728 GiB
Longhorn LV, and why a 62 GiB container image tipped that node into
DiskPressure and evicted bastion, labd, cockroach and openbao.

Three generators each hardcoded their own sizes and disagreed: the kickstart
said rancher=120G, ubuntu-autoinstall said 20G, asahi-firstboot said 20G under
a comment asserting the sizes matched install.ks.ts. They did not.

ROLE_REGISTRY already carried `k3s` and `parent` and nothing read them. This
makes capabilities authoritative, moves every size into one shared module, and
adds schema v2 (fixed 250G rancher / 300G longhorn, never --grow). Existing
machines stay v1 forever: schema is detected from the on-disk VG tag, so a v1
box stays v1 even when v2 is requested.

Proven on real VMs, not just unit tests: all four topologies installed and had
their layouts read back off the machines; markers in /home, /srv,
/var/lib/longhorn and /var/lib/rancher survived reinstall; and a 4-node cluster
formed, took real data, and had all three servers rolled one at a time through
evacuate / wipe / reinstall / rejoin with checksums verified before and after
each roll. Drill: 10/10, evidence in migration/drill-evidence/.

Production defects the drill found, none of which a unit test could reach:

- sshOpts() capped every operation at 30s while drainNode asked kubectl for
  --timeout=15m, so the transport killed the drain after evicting most pods and
  returned a transcript that read like success. `labctl app k3s evacuate` could
  never have drained a real node.
- generateAgentConfig put node-role.kubernetes.io/worker in the kubelet's
  node-label list, which NodeRestriction rejects, so no agent installed through
  K3sModule could start. systemd reported only "did not take the steps required
  by its unit configuration".
- toOpContext built K3sConfig by enumerating three keys and silently dropped
  the rest, so the k3s version pin did nothing and --allow-single-replica was a
  no-op that claimed to override a data-loss guard.
- The Longhorn disk annotation was mangled by shell quoting; with
  createDefaultDiskLabeledNodes the node then gets no disk at all, joins, goes
  Ready, and silently hosts no replica.
- evacuate drained before evicting, which blocks on Longhorn's PDBs and would
  destroy a single-replica volume's only source.
- etcd member ids were parsed from JSON, losing uint64 precision and emitting
  decimal where `member remove` needs hex -- at the moment a node is stopped
  and about to be wiped.
- get.k3s.io installs latest, two minor versions ahead of the fleet, so a
  reinstalled node would rejoin on a newer k3s than its peers and roll an
  unplanned two-version control-plane upgrade. K3sConfig.k3sVersion now pins it.
- `labctl app k3s install --role x` mapped anything that was not "worker" to
  "infra", so the runbook's own `--role k8s-combo` would have installed a
  different role with different disk expectations and no warning.

Also: /var/lib/longhorn/replicas must never be deleted, only its contents, or
every replica placed on the node dies on a non-recursive mkdir -- for ever,
while the node reports Ready, Schedulable and healthy.

etcd is now in the kickstart's package set for k3s server roles, because
`k3s etcdctl` is not a command and a member that needs to recover its own
membership has no internet.

542 unit tests / 39 files.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-10-09 22:33:59 +01:00

187 lines
7.9 KiB
TypeScript

import { describe, it, expect } from "vitest";
import { renderInstallKickstart } from "../src/templates/install.ks.js";
import type { InstallKickstartParams } from "../src/templates/install.ks.js";
import { SUPPORTED_ROLES, LEGACY_ROLES, v2Layout, type Role } from "@lab/shared";
/**
* Partition-layout tests.
*
* The existing ksvalidator test only sees the MAIN kickstart body. The actual
* layout is written by %pre into /tmp/part.ks inside a heredoc, so until now
* nothing validated it at all -- a broken layout rendered, shipped and was only
* discovered on the installed machine.
*
* These parse the heredocs. Following tests/ubuntu-autoinstall.test.ts:
* "A `toContain` assertion would not have caught that; only parsing does."
*/
function baseParams(over: Partial<InstallKickstartParams> = {}): InstallKickstartParams {
return {
hostname: "test-node", disk: "nvme0n1", role: "worker", domain: "ad.itaz.eu",
fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8",
serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514,
sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal",
...over,
};
}
/** Pull every `cat >[>] /tmp/part.ks << PARTEOF ... PARTEOF` body out of the script. */
function partKsBlocks(ks: string): string[] {
const out: string[] = [];
const re = /cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g;
let m: RegExpExecArray | null;
while ((m = re.exec(ks)) !== null) out.push(m[1]!);
return out;
}
interface Logvol { mount: string; name: string; fs: string; size?: number; useExisting: boolean }
function parseLogvols(block: string): Logvol[] {
return block.split("\n")
.filter((l) => l.trim().startsWith("logvol "))
.map((line) => {
const parts = line.trim().split(/\s+/);
const get = (flag: string) => {
const p = parts.find((x) => x.startsWith(`${flag}=`));
return p ? p.slice(flag.length + 1) : undefined;
};
return {
mount: parts[1]!,
name: get("--name") ?? "",
fs: get("--fstype") ?? "",
...(get("--size") ? { size: Number(get("--size")) } : {}),
useExisting: parts.includes("--useexisting"),
};
});
}
/** The fresh-install block is the one that also creates the PV. */
function freshBlock(ks: string): string {
const b = partKsBlocks(ks).find((x) => x.includes("part pv.01"));
if (!b) throw new Error("no fresh-install part.ks block found");
return b;
}
describe("v1 is frozen — existing machines must reinstall identically", () => {
it.each(LEGACY_ROLES)("%s v1 fresh layout is unchanged", (role) => {
const lvs = parseLogvols(freshBlock(renderInstallKickstart(baseParams({ role }))));
const byName = Object.fromEntries(lvs.map((l) => [l.name, l]));
// The exact v1 sizes, in MiB. If one of these changes, every existing box
// in the fleet would be re-laid-out on its next reinstall.
expect(byName["swap"]!.size).toBe(27648);
expect(byName["root"]!.size).toBe(33792);
expect(byName["var"]!.size).toBe(102400);
expect(byName["varlog"]!.size).toBe(10240);
expect(byName["home"]!.size).toBe(10240);
expect(byName["srv"]!.size).toBe(20480);
});
it("v1 gives rancher to infra and worker only — the historical bug, preserved", () => {
const has = (r: Role) => freshBlock(renderInstallKickstart(baseParams({ role: r }))).includes("--name=rancher");
expect(has("infra")).toBe(true);
expect(has("worker")).toBe(true);
expect(has("labcontroller")).toBe(false); // the bug, deliberately kept in v1
expect(has("vanilla")).toBe(false);
});
it("v1 is the default — no caller has to opt out", () => {
const ks = renderInstallKickstart(baseParams({ role: "worker" }));
expect(parseLogvols(freshBlock(ks)).find((l) => l.name === "root")!.size).toBe(33792);
});
});
describe("v2-only roles have no v1 layout", () => {
it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"] as const)(
"%s defaults to v2 when no schema is given", (role) => {
const lvs = parseLogvols(freshBlock(renderInstallKickstart(baseParams({ role }))));
expect(lvs.find((l) => l.name === "root")!.size).toBe(32768);
expect(lvs.some((l) => l.name === "rancher")).toBe(true);
});
it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"] as const)(
"%s throws on an EXPLICIT v1 rather than silently dropping its volumes", (role) => {
expect(() => renderInstallKickstart(baseParams({ role, schemaVersion: 1 }))).toThrow(/no v1 disk layout/);
});
});
describe("v2 renders the shared layout", () => {
it.each(SUPPORTED_ROLES)("%s v2 matches v2Layout() exactly", (role) => {
const ks = renderInstallKickstart(baseParams({ role, schemaVersion: 2 }));
const rendered = parseLogvols(freshBlock(ks));
const expected = v2Layout(role);
expect(rendered.map((l) => l.name).sort()).toEqual(expected.map((l) => l.name).sort());
for (const spec of expected) {
const got = rendered.find((l) => l.name === spec.name)!;
expect(got.size).toBe(spec.sizeMib);
expect(got.fs).toBe(spec.fs);
}
});
it("v2 gives labcontroller a rancher LV — the v1 bug, fixed", () => {
const ks = renderInstallKickstart(baseParams({ role: "labcontroller", schemaVersion: 2 }));
const rancher = parseLogvols(freshBlock(ks)).find((l) => l.name === "rancher");
expect(rancher).toBeDefined();
expect(rancher!.size).toBe(256000);
});
it("v2 never uses --grow", () => {
for (const role of SUPPORTED_ROLES) {
const block = freshBlock(renderInstallKickstart(baseParams({ role, schemaVersion: 2 })));
expect(block).not.toMatch(/logvol[^\n]*--grow/);
}
});
it("v2 uses the larger EFI and /boot partitions", () => {
const block = freshBlock(renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 2 })));
expect(block).toMatch(/part \/boot\/efi --fstype=efi --size=1024/);
expect(block).toMatch(/part \/boot --fstype=ext4 --size=4096/);
});
});
describe("reprovision protects existing data", () => {
it("reprovision reads the on-disk schema instead of trusting the caller", () => {
const ks = renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 2 }));
expect(ks).toContain("DISK_SCHEMA=v1");
expect(ks).toMatch(/vgs --noheadings -o vg_tags .*grep -q lab-schema-v2/);
});
it("the reprovision branch never wipes the disk", () => {
const ks = renderInstallKickstart(baseParams({ role: "worker" }));
const repro = partKsBlocks(ks).find((b) => b.includes("--useexisting"))!;
expect(repro).toContain("clearpart --none");
expect(repro).not.toContain("clearpart --all");
});
it("preserved data volumes are never reformatted", () => {
const ks = renderInstallKickstart(baseParams({ role: "worker" }));
for (const lv of ["longhorn", "rancher", "home", "srv"]) {
const re = new RegExp(`--name=${lv} --useexisting --noformat`);
expect(ks).toMatch(re);
}
});
});
describe("the %pre block survives the /ks-auto extractor", () => {
it("matches the same regex ks-auto uses to re-run %pre", () => {
// routes/kickstart.ts re-extracts %pre with python:
// re.findall(r'%pre[^\n]*\n(.*?)%end', text, re.DOTALL)
// A change that breaks this breaks ISO installs silently.
for (const role of SUPPORTED_ROLES) {
// v2-only roles throw at v1 by design, so only render the combinations
// that are actually legal.
// An explicit v1 is illegal for a v2-only role; everything else renders.
const versions = (LEGACY_ROLES as readonly string[]).includes(role)
? ([1, 2] as const)
: ([2] as const);
for (const schemaVersion of versions) {
const ks = renderInstallKickstart(baseParams({ role, schemaVersion }));
const blocks = [...ks.matchAll(/%pre[^\n]*\n([\s\S]*?)%end/g)];
expect(blocks.length).toBeGreaterThan(0);
expect(blocks[0]![1]).toContain("/tmp/part.ks");
}
}
});
});