diff --git a/bastion/src/bastion/src/config.ts b/bastion/src/bastion/src/config.ts index 1f4372b..439ce97 100644 --- a/bastion/src/bastion/src/config.ts +++ b/bastion/src/bastion/src/config.ts @@ -29,7 +29,13 @@ export function loadConfig(overrides: Partial = {}): BastionConfi const vyosDefaultPassword = overrides.vyosDefaultPassword ?? process.env["VYOS_DEFAULT_PASSWORD"] ?? "vyos"; - const fedoraMirror = `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`; + // Overridable like ubuntuMirror and vyosIsoUrl above -- it was the only one + // of the three hardcoded. Without this, every integration-test install pulls + // the full package set from download.fedoraproject.org, which makes a + // multi-VM provisioning matrix impractical (~25 min per install, serialised). + // Point FEDORA_MIRROR at a local mirror or caching proxy for test runs. + const fedoraMirror = overrides.fedoraMirror ?? process.env["FEDORA_MIRROR"] + ?? `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`; const tftpDir = `${bastionDir}/tftp`; const httpDir = `${bastionDir}/http`; const stateFile = `${bastionDir}/state.json`; @@ -59,6 +65,7 @@ export function loadConfig(overrides: Partial = {}): BastionConfi syslogPort, skipDnsmasq: overrides.skipDnsmasq, skipArtifacts: overrides.skipArtifacts, + autoK3s: overrides.autoK3s ?? process.env["AUTO_K3S"] !== "false", labdUrl: overrides.labdUrl ?? process.env["LABD_URL"], bastionJoinToken: overrides.bastionJoinToken ?? process.env["BASTION_JOIN_TOKEN"], fedoraMirror, diff --git a/bastion/src/bastion/src/routes/api.ts b/bastion/src/bastion/src/routes/api.ts index 7398046..8cb5604 100644 --- a/bastion/src/bastion/src/routes/api.ts +++ b/bastion/src/bastion/src/routes/api.ts @@ -5,7 +5,7 @@ // /api/discover - receive hardware discovery reports from PXE-booted machines import type { FastifyInstance } from "fastify"; -import type { HardwareInfo, InstalledInfo, Role, VyosInstallSpec } from "@lab/shared"; +import type { BastionConfig, HardwareInfo, InstalledInfo, Role, VyosInstallSpec } from "@lab/shared"; import { isValidOsId, SUPPORTED_ROLES, SUPPORTED_OS } from "@lab/shared"; import type { StateManager } from "../services/state.js"; import { logger } from "../services/logger.js"; @@ -14,6 +14,7 @@ import { progressBus } from "../services/progress-events.js"; import type { ProgressEvent } from "../services/progress-events.js"; import type { InstallLogBuffer } from "../services/install-log.js"; import type { SyslogListener } from "../services/syslog-listener.js"; +import { isRole, requiresSchemaV2 } from "@lab/shared"; /** * Seconds after dispatch with zero progress before a machine is called stalled. @@ -24,6 +25,7 @@ const STALL_THRESHOLD_S = 8 * 60; export function registerApiRoutes( app: FastifyInstance, + config: BastionConfig, state: StateManager, installLog: InstallLogBuffer, syslog: SyslogListener, @@ -42,9 +44,10 @@ export function registerApiRoutes( role?: string; os?: string; vyos?: VyosInstallSpec; + schemaVersion?: number; }; }>("/api/install", async (request, reply) => { - const { mac: rawMac, hostname, disk, role, os, vyos } = request.body ?? {}; + const { mac: rawMac, hostname, disk, role, os, vyos, schemaVersion } = request.body ?? {}; const mac = (rawMac ?? "").toLowerCase().replace(/-/g, ":"); if (mac === "") { @@ -56,6 +59,21 @@ export function registerApiRoutes( return reply.status(400).send({ error: `invalid role: '${validRole}'. Supported: ${SUPPORTED_ROLES.join(", ")}` }); } + // Reject an unknown schema rather than silently installing v1: someone + // asking for v3 wants a layout we do not have, and quietly giving them the + // old one is how a fleet ends up with layouts nobody chose. + if (schemaVersion !== undefined && schemaVersion !== 1 && schemaVersion !== 2) { + return reply.status(400).send({ error: `invalid schemaVersion: ${schemaVersion}. Supported: 1, 2` }); + } + + // Only an EXPLICIT v1 is a contradiction. Omitting the schema is normal -- + // a reinstall never passes one, because the layout already on the disk wins. + if (schemaVersion === 1 && isRole(validRole) && requiresSchemaV2(validRole)) { + return reply.status(400).send({ + error: `role '${validRole}' requires schemaVersion 2 (it has no v1 disk layout)`, + }); + } + const osId = os ?? "fedora-43"; if (!isValidOsId(osId)) { return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: ${SUPPORTED_OS.join(", ")}` }); @@ -68,6 +86,7 @@ export function registerApiRoutes( role: validRole as Role, os: osId, queued_at: new Date().toISOString(), + ...(schemaVersion ? { schemaVersion: schemaVersion as 1 | 2 } : {}), ...(vyos ? { vyos } : {}), }; }); @@ -176,9 +195,22 @@ export function registerApiRoutes( : installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "lab" : "root"; console.log(`\n \x1b[0;32m\x1b[1m ssh ${admin}@${ip}\x1b[0m\n`); // eslint-disable-line no-console - // Auto-install k3s for non-vanilla roles - if (!isVyos && installedInfo.role !== "vanilla" && ip !== "") { - void triggerPostProvisionK3s(installedInfo.hostname, ip, installedInfo.role, admin, mac); + // Auto-install k3s for non-vanilla roles. + // autoK3s=false suppresses this entirely. Multi-node tests need that: + // post-provision installs each node into its OWN cluster, so leaving it + // on turns three combo nodes into three single-node clusters, and it + // also stops k3s and deletes CNI state minutes after install lands -- + // right on top of a test that is forming a cluster itself. + if (config.autoK3s === false) { + logger.info(`[api] autoK3s disabled; skipping k3s post-provision for ${mac}`); + } else if (!isVyos && installedInfo.role !== "vanilla" && ip !== "") { + // State is JSON on disk; a role written by an older build may no + // longer exist. Skip loudly rather than crash the install callback. + if (isRole(installedInfo.role)) { + void triggerPostProvisionK3s(installedInfo.hostname, ip, installedInfo.role, admin, mac); + } else { + console.warn(`[api] unknown role ${installedInfo.role} for ${mac}; skipping k3s post-provision`); // eslint-disable-line no-console + } } } } diff --git a/bastion/src/bastion/src/routes/kickstart.ts b/bastion/src/bastion/src/routes/kickstart.ts index aa5c1f3..6fe02aa 100644 --- a/bastion/src/bastion/src/routes/kickstart.ts +++ b/bastion/src/bastion/src/routes/kickstart.ts @@ -30,6 +30,7 @@ export function registerKickstartRoutes( hostname: queueEntry?.hostname ?? "lab-node", disk: queueEntry?.disk ?? "", role: queueEntry?.role ?? "worker", + ...(queueEntry?.schemaVersion ? { schemaVersion: queueEntry.schemaVersion } : {}), }); return reply.type("text/plain").send(ks); diff --git a/bastion/src/bastion/src/server.ts b/bastion/src/bastion/src/server.ts index 1c1aa47..d9a3d51 100644 --- a/bastion/src/bastion/src/server.ts +++ b/bastion/src/bastion/src/server.ts @@ -46,7 +46,7 @@ export function createApp(config: BastionConfig): { app: ReturnType { @@ -104,6 +135,21 @@ export async function triggerPostProvisionK3s( } }; + // Gate before ANY work: the prerequisite steps below stop k3s and delete CNI + // state, so they must not run for a role we are declining to install either. + if (!shouldAutoInstallK3s(role)) { + emitStage( + "post-provision", + `${role} is a cluster member; not auto-installing k3s (it would form its own cluster)`, + ); + const verb = isK3sServer(role) ? "--k3s-server https://:6443" : "--k3s-server https://:6443"; + logger.info( + `[k3s] Join it to the cluster with:\n` + + ` labctl app k3s install ${hostname} --role ${role} ${verb} --k3s-token `, + ); + return; + } + emitStage("post-provision", `auto-installing k3s on ${hostname} (${ip}) role=${role}`); emitStage("post-provision", "waiting for SSH (machine may still be rebooting)"); @@ -150,7 +196,7 @@ sysctl --system > /dev/null'`, keyPath, hostname, mac); // Step 4: Install k3s // labcontroller extends infra — both are k3s servers - const k3sRole = (role === "infra" || role === "labcontroller") ? "server" : "agent"; + const k3sRole = isK3sServer(role) ? "server" : "agent"; emitStage("post-provision", `installing k3s ${k3sRole}`); const code = await sshRunStreaming(ip, sshUser, `curl -sfL https://get.k3s.io | sudo INSTALL_K3S_EXEC="${k3sRole}" INSTALL_K3S_SKIP_SELINUX_RPM=true sh -`, diff --git a/bastion/src/bastion/src/templates/asahi-firstboot.sh.ts b/bastion/src/bastion/src/templates/asahi-firstboot.sh.ts index 1f3d00b..506dad6 100644 --- a/bastion/src/bastion/src/templates/asahi-firstboot.sh.ts +++ b/bastion/src/bastion/src/templates/asahi-firstboot.sh.ts @@ -3,6 +3,7 @@ // Creates the standard lab LVM layout on the data partition, matching install.ks.ts. import type { Role } from "@lab/shared"; +import { isK3sServer, v2Layout, type LvSpec } from "@lab/shared"; export interface AsahiFirstbootParams { hostname: string; @@ -12,13 +13,42 @@ export interface AsahiFirstbootParams { sshKeys: string[]; adminUser: string; mac: string; + /** Fresh-setup layout. Defaults to 1 so existing callers are unchanged. */ + schemaVersion?: 1 | 2; } export function renderFirstbootScript(params: AsahiFirstbootParams): string { const { hostname, role, serverIp, httpPort, sshKeys, adminUser, mac } = params; + const schemaVersion = params.schemaVersion ?? 1; + + // --------------------------------------------------------------------------- + // v2 LAYOUT, from the SAME definition Fedora and Ubuntu render + // (shared/src/types/layout.ts). + // + // omitRoot: on Apple Silicon the Asahi installer has already created a btrfs + // root partition OUTSIDE our volume group -- worker1 runs a 4.6 GB one. We + // must not create a second root LV. + // + // The v1 path below hardcodes rancher at 20480M while claiming "sizes match + // install.ks.ts", which uses 122880M. That 100 GB gap is what left worker1 + // with an image store too small to hold a single large container image. + // --------------------------------------------------------------------------- + const v2 = v2Layout(role, { omitRoot: true }); + const v2LvLines = v2.map((lv: LvSpec) => `lvcreate -L ${lv.sizeMib}M -n ${lv.name} labvg -y`).join("\n"); + const v2FormatLines = v2 + .map((lv: LvSpec) => (lv.fs === "swap" ? `mkswap /dev/labvg/${lv.name}` : `mkfs.xfs /dev/labvg/${lv.name}`)) + .join("\n"); + const v2MountLines = v2 + .filter((lv: LvSpec) => lv.fs !== "swap") + .map((lv: LvSpec) => `mount_lv ${lv.name} ${lv.mount}`) + .join("\n"); + const v2FstabLines = v2 + .filter((lv: LvSpec) => lv.fs !== "swap") + .map((lv: LvSpec) => `echo "/dev/labvg/${lv.name} ${lv.mount} xfs defaults 0 0" >> /etc/fstab`) + .join("\n"); const isWorker = role === "worker"; - const isInfra = role === "infra" || role === "labcontroller"; + const isInfra = isK3sServer(role); // Role-specific LV creation commands const roleLvLines: string[] = []; @@ -127,24 +157,27 @@ write_lab_fstab() { # Add fresh LVM entries echo "# Lab LVM volumes" >> /etc/fstab echo "/dev/labvg/swap none swap defaults 0 0" >> /etc/fstab - echo "/dev/labvg/var /var xfs defaults 0 0" >> /etc/fstab +${schemaVersion === 2 ? v2FstabLines.split("\n").map((l: string) => " " + l).join("\n") : ` echo "/dev/labvg/var /var xfs defaults 0 0" >> /etc/fstab echo "/dev/labvg/varlog /var/log xfs defaults 0 0" >> /etc/fstab echo "/dev/labvg/home /home xfs defaults 0 0" >> /etc/fstab echo "/dev/labvg/srv /srv xfs defaults 0 0" >> /etc/fstab - ${roleFstabLines.join('\n ')} + ${roleFstabLines.join('\n ')}`} } # ── Check for existing VG ──────────────────────────────────────── if vgs labvg &>/dev/null; then echo "Volume group 'labvg' already exists — reprovision detected." + DISK_SCHEMA=v1 + vgs --noheadings -o vg_tags labvg 2>/dev/null | grep -q lab-schema-v2 && DISK_SCHEMA=v2 + echo "On-disk schema: $DISK_SCHEMA" echo "Activating existing volumes..." vgchange -ay labvg - mount_lv var /var +${schemaVersion === 2 ? v2MountLines.split("\n").map((l: string) => " " + l).join("\n") : ` mount_lv var /var mount_lv varlog /var/log mount_lv home /home mount_lv srv /srv -${roleMountLines.map(l => ` ${l}`).join('\n')} +${roleMountLines.map(l => ` ${l}`).join('\n')}`} # Enable swap if lvs labvg/swap &>/dev/null; then @@ -162,24 +195,28 @@ echo "Creating LVM on $DATA_PART..." pvcreate "$DATA_PART" vgcreate labvg "$DATA_PART" +${schemaVersion === 2 ? ` +# Tag so a later run can tell which layout is on this disk without trusting a +# caller-supplied flag -- same mechanism install.ks.ts uses. +vgchange --addtag lab-schema-v2 labvg >/dev/null 2>&1 || true` : ""} # Create LVs — sizes match install.ks.ts (in MiB) echo "Creating logical volumes..." -lvcreate -L 27648M -n swap labvg -y # 27GB swap +${schemaVersion === 2 ? v2LvLines : `lvcreate -L 27648M -n swap labvg -y # 27GB swap lvcreate -L 102400M -n var labvg -y # 100GB /var lvcreate -L 10240M -n varlog labvg -y # 10GB /var/log lvcreate -L 10240M -n home labvg -y # 10GB /home lvcreate -L 20480M -n srv labvg -y # 20GB /srv -${roleLvLines.join('\n')} +${roleLvLines.join('\n')}`} # Format echo "Formatting volumes..." -mkswap /dev/labvg/swap +${schemaVersion === 2 ? v2FormatLines : `mkswap /dev/labvg/swap mkfs.xfs /dev/labvg/var mkfs.xfs /dev/labvg/varlog mkfs.xfs /dev/labvg/home mkfs.xfs /dev/labvg/srv -${roleFormatLines.join('\n')} +${roleFormatLines.join('\n')}`} # Migrate and mount volumes that can be switched live. # Copy existing content first so we don't shadow files (e.g. /home/user/.ssh). diff --git a/bastion/src/bastion/src/templates/install.ks.ts b/bastion/src/bastion/src/templates/install.ks.ts index e2725ac..d21bb4c 100644 --- a/bastion/src/bastion/src/templates/install.ks.ts +++ b/bastion/src/bastion/src/templates/install.ks.ts @@ -3,6 +3,7 @@ // packages, and %post with SSH keys, user creation, k3s prereqs, progress callbacks. import type { Role } from "@lab/shared"; +import { v2Layout, V2_BOOT, requiresSchemaV2, effectiveSchemaVersion, isK3sServer, type LvSpec } from "@lab/shared"; export interface InstallKickstartParams { hostname: string; @@ -17,6 +18,15 @@ export interface InstallKickstartParams { syslogPort: number; sshKeys: string[]; adminUser: string; + /** + * Disk layout to use for a FRESH install. Defaults to 1 so every existing + * caller keeps today's layout untouched. + * + * On a REPROVISION this is ignored: the schema already on the disk wins, + * read from an LVM tag on the volume group. Reinstalling a machine must + * never silently change its layout -- that is how data gets destroyed. + */ + schemaVersion?: 1 | 2; } export function renderInstallKickstart(params: InstallKickstartParams): string { @@ -38,6 +48,49 @@ export function renderInstallKickstart(params: InstallKickstartParams): string { const fqdn = domain ? `${hostname}.${domain}` : hostname; const vg = "labvg"; const now = new Date().toISOString(); + // --------------------------------------------------------------------------- + // v1 LAYOUT -- DO NOT REPLACE THESE WITH CAPABILITY LOOKUPS. + // + // These literals are deliberately frozen. Every machine already in the fleet + // was partitioned by them, and `labctl provision reprovision` must reproduce + // that same layout byte-for-byte or it destroys data. Yes, they are wrong: + // `labcontroller` runs k3s and gets no rancher LV, and `infra` nodes hold + // every Longhorn replica in the homelab cluster while only `worker` gets a + // longhorn LV. Those are fixed in the v2 layout, which new boxes get; see + // schemaVersion. Changing them here rewrites history for existing hardware. + // --------------------------------------------------------------------------- + // --------------------------------------------------------------------------- + // v2 LAYOUT. One definition, shared with ubuntu-autoinstall and asahi-firstboot + // (shared/src/types/layout.ts) so the three generators cannot drift apart + // again. Capability-gated, and nothing uses --grow. + // --------------------------------------------------------------------------- + const schemaVersion = effectiveSchemaVersion(role, params.schemaVersion); + if (params.schemaVersion === 1 && requiresSchemaV2(role)) { + // Fail here rather than render a machine with no rancher/longhorn LV. + throw new Error( + `role '${role}' has no v1 disk layout - it was introduced with schema v2. ` + + `Pass schemaVersion: 2 (labctl provision install --schema-version 2).`, + ); + } + const v2RootMib = v2Layout(role).find((lv: LvSpec) => lv.name === "root")?.sizeMib ?? 32768; + const v2 = v2Layout(role); + const v2LogvolLines = v2 + .map((lv: LvSpec) => + lv.fs === "swap" + ? `logvol swap --vgname=${vg} --name=${lv.name} --fstype=swap --size=${lv.sizeMib}` + : `logvol ${lv.mount} --vgname=${vg} --name=${lv.name} --fstype=xfs --size=${lv.sizeMib}`, + ) + .join("\n"); + /** OS volumes only - the ones a reprovision recreates rather than preserves. */ + const v2OsLogvolLines = v2 + .filter((lv: LvSpec) => ["swap", "root", "var", "varlog"].includes(lv.name)) + .map((lv: LvSpec) => + lv.fs === "swap" + ? `logvol swap --vgname=${vg} --name=${lv.name} --fstype=swap --size=${lv.sizeMib}` + : `logvol ${lv.mount} --vgname=${vg} --name=${lv.name} --fstype=xfs --size=${lv.sizeMib}`, + ) + .join("\n"); + const hasLonghorn = role === "worker"; const hasRancher = role === "infra"; // k8s roles get a dedicated 120G image-store LV. 2026-08 incident: the old @@ -46,6 +99,12 @@ export function renderInstallKickstart(params: InstallKickstartParams): string { // VG space, making post-install lvextend impossible on worker nodes. const hasRancherLv = role === "infra" || role === "worker"; const isVanilla = role === "vanilla"; + // etcd members need etcdctl available OFFLINE. The recovery procedure for a + // lost or stale member is `etcdctl member list/remove`, and `k3s etcdctl` is + // not a command -- the binary comes from the `etcd` rpm. Telling an operator + // to `dnf install etcd` in the middle of a quorum failure assumes working + // DNS, routing and a mirror, which is exactly what may be broken. + const needsEtcdctl = isK3sServer(role); // -- Auth section -- // Always set a root password (for serial console debugging) + SSH keys @@ -179,6 +238,16 @@ if vgs $VG &>/dev/null; then echo "=== Existing VG found - reprovision mode ===" REPROVISION=yes + # Which layout is ALREADY on this disk? The LVM tag is authoritative -- the + # schemaVersion the caller asked for is irrelevant here, because silently + # re-laying-out an existing machine destroys its data. A v1 box reinstalls + # as v1 forever, until it is deliberately migrated. + DISK_SCHEMA=v1 + if vgs --noheadings -o vg_tags $VG 2>/dev/null | grep -q lab-schema-v2; then + DISK_SCHEMA=v2 + fi + echo "On-disk schema: $DISK_SCHEMA" + # Detect which data LVs to preserve PRESERVE_LONGHORN=no; PRESERVE_SRV=no; PRESERVE_HOME=no; PRESERVE_RANCHER=no lvs $VG/longhorn &>/dev/null && PRESERVE_LONGHORN=yes @@ -211,11 +280,21 @@ clearpart --none part /boot/efi --onpart=$EFI_PART --fstype=efi part /boot --onpart=$BOOT_PART --fstype=ext4 volgroup ${vg} --useexisting --noformat +PARTEOF + + # OS volumes are recreated, so they must match the layout already on disk. + if [ "$DISK_SCHEMA" = "v2" ]; then + cat >> /tmp/part.ks << PARTEOF +${v2OsLogvolLines} +PARTEOF + else + cat >> /tmp/part.ks << PARTEOF logvol swap --vgname=${vg} --name=swap --fstype=swap --size=27648 logvol / --vgname=${vg} --name=root --fstype=xfs --size=33792 logvol /var --vgname=${vg} --name=var --fstype=xfs --size=102400 logvol /var/log --vgname=${vg} --name=varlog --fstype=xfs --size=10240 PARTEOF + fi # Preserve or recreate data LVs if [ "$PRESERVE_HOME" = "yes" ]; then @@ -240,7 +319,19 @@ PARTEOF else # Fresh install - cat > /tmp/part.ks << PARTEOF +${schemaVersion === 2 ? ` cat > /tmp/part.ks << PARTEOF +ignoredisk --only-use=$DISK +clearpart --all --initlabel --drives=$DISK +part /boot/efi --fstype=efi --size=${V2_BOOT.efiMib} --ondisk=$DISK +part /boot --fstype=ext4 --size=${V2_BOOT.bootMib} --ondisk=$DISK +part pv.01 --size=1 --grow --ondisk=$DISK +volgroup ${vg} pv.01 +${v2LogvolLines} +PARTEOF + # Tag the VG so a later reprovision knows this disk is v2 WITHOUT trusting + # whatever schemaVersion the caller happens to pass. The tag lives on the + # VG, which a reprovision preserves; a marker file would not survive. + SCHEMA_TAG_PENDING=v2` : ` cat > /tmp/part.ks << PARTEOF ignoredisk --only-use=$DISK clearpart --all --initlabel --drives=$DISK part /boot/efi --fstype=efi --size=600 --ondisk=$DISK @@ -255,7 +346,7 @@ logvol /home --vgname=${vg} --name=home --fstype=xfs --size=10240 logvol /srv --vgname=${vg} --name=srv --fstype=xfs --size=20480 ${longhornFreshLine} ${rancherFreshLine} -PARTEOF +PARTEOF`} fi echo "=== Generated partition config ===" @@ -302,6 +393,9 @@ tar socat conntrack-tools ethtool`} +${needsEtcdctl ? ` +# etcdctl, for recovering etcd membership without internet access +etcd` : ""} # Boot management efibootmgr @@ -414,12 +508,30 @@ bastion_progress "post-install" "3-bootorder done" # -- Enable SysRq magic keys (for emergency reboot via Alt+SysRq+REISUB) -- echo "kernel.sysrq=1" > /etc/sysctl.d/90-sysrq.conf +# -- Disk schema tag -- +# Tag the VG so a future reprovision can tell which layout is on this disk +# WITHOUT trusting whatever the caller passes. Deliberately derived from what +# actually landed on disk rather than from render-time intent: a v1 machine +# reinstalled with --schema-version 2 keeps its v1 layout (see %pre), and must +# not end up wearing a v2 tag it does not deserve. +V2_ROOT_MIB=${v2RootMib} +ROOT_MIB=$(lvs --noheadings --units m --nosuffix -o lv_size ${vg}/root 2>/dev/null | tr -d ' ' | cut -d. -f1) +if [ "$ROOT_MIB" = "$V2_ROOT_MIB" ]; then + vgchange --addtag lab-schema-v2 ${vg} >/dev/null 2>&1 || true + echo "Tagged ${vg} as lab-schema-v2 (root=\${ROOT_MIB}M)" + DISK_SCHEMA_ON_DISK=2 +else + DISK_SCHEMA_ON_DISK=1 + echo "Layout is v1 (root=\${ROOT_MIB}M, v2 would be \${V2_ROOT_MIB}M)" +fi + # -- Provisioning metadata -- cat > /etc/lab-provisioned << PROVEOF hostname: ${fqdn} role: ${role} provisioned: $(date -Iseconds) bastion: ${serverIp} +schema: v$DISK_SCHEMA_ON_DISK PROVEOF cat > /root/README << 'README' diff --git a/bastion/src/bastion/src/templates/ubuntu-autoinstall.ts b/bastion/src/bastion/src/templates/ubuntu-autoinstall.ts index db16bed..088e143 100644 --- a/bastion/src/bastion/src/templates/ubuntu-autoinstall.ts +++ b/bastion/src/bastion/src/templates/ubuntu-autoinstall.ts @@ -2,10 +2,15 @@ // Equivalent of the Fedora kickstart: LVM partitioning, packages, // SSH keys, k3s prereqs, progress callbacks. +import { v2Layout, type LvSpec } from "@lab/shared"; +import type { Role } from "@lab/shared"; + export interface UbuntuAutoinstallParams { hostname: string; disk: string; - role: string; // "vanilla" | "worker" | "infra" + role: Role; + /** Fresh-install layout. Defaults to 1 so existing callers are unchanged. */ + schemaVersion?: 1 | 2; domain: string; ubuntuVersion: string; timezone: string; @@ -31,6 +36,48 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string const fqdn = domain ? `${hostname}.${domain}` : hostname; const vg = "labvg"; + // --------------------------------------------------------------------------- + // v1 LAYOUT -- DO NOT REPLACE THESE WITH CAPABILITY LOOKUPS. + // + // These literals are deliberately frozen. Every machine already in the fleet + // was partitioned by them, and `labctl provision reprovision` must reproduce + // that same layout byte-for-byte or it destroys data. Yes, they are wrong: + // `labcontroller` runs k3s and gets no rancher LV, and `infra` nodes hold + // every Longhorn replica in the homelab cluster while only `worker` gets a + // longhorn LV. Those are fixed in the v2 layout, which new boxes get; see + // schemaVersion. Changing them here rewrites history for existing hardware. + // --------------------------------------------------------------------------- + // --------------------------------------------------------------------------- + // v2 LAYOUT, rendered from the SAME definition Fedora and Asahi use + // (shared/src/types/layout.ts). Before this, Ubuntu sized rancher at 20G + // while the kickstart used 120G -- the drift that the 2026-08 imagefs + // eviction incident was caused by. + // + // Indentation is load-bearing: list items sit at 6 spaces, their keys at 8. + // At any other depth curtin's YAML does not parse. See the v1 comment below. + // --------------------------------------------------------------------------- + const schemaVersion = params.schemaVersion ?? 1; + const renderV2Lv = (lv: LvSpec): string => { + const gib = Math.round(lv.sizeMib / 1024); + const mountPath = lv.fs === "swap" ? "none" : lv.mount; + return [ + ` - id: lv-${lv.name}`, + ` name: ${lv.name}`, + ` type: lvm_partition`, + ` volgroup: vg0`, + ` size: ${gib}G`, + ` - id: fs-${lv.name}`, + ` type: format`, + ` volume: lv-${lv.name}`, + ` fstype: ${lv.fs}`, + ` - id: mount-${lv.name}`, + ` type: mount`, + ` device: fs-${lv.name}`, + ` path: ${mountPath}`, + ].join("\n"); + }; + const v2LvBlock = v2Layout(role).map(renderV2Lv).join("\n"); + const hasLonghorn = role === "worker"; const hasRancher = role === "infra"; @@ -229,7 +276,7 @@ ${sshKeysYaml} name: ${vg} devices: - part-pv - - id: lv-swap +${schemaVersion === 2 ? v2LvBlock : ` - id: lv-swap name: swap type: lvm_partition volgroup: vg0 @@ -306,7 +353,7 @@ ${sshKeysYaml} - id: mount-srv type: mount device: fs-srv - path: /srv${extraLvsBlock} + path: /srv${extraLvsBlock}`} packages: - openssh-server - curl diff --git a/bastion/src/bastion/tests/asahi.test.ts b/bastion/src/bastion/tests/asahi.test.ts index b3f223c..fb65cb6 100644 --- a/bastion/src/bastion/tests/asahi.test.ts +++ b/bastion/src/bastion/tests/asahi.test.ts @@ -223,3 +223,67 @@ describe("renderFirstbootUnit", () => { expect(unit).toContain("ConditionPathExists=!/etc/lab-lvm-setup-done"); }); }); + +// --------------------------------------------------------------------------- +// v2 layout. Asahi was the THIRD generator and the most drifted: it hardcoded +// rancher at 20480M under a comment claiming "sizes match install.ks.ts", which +// uses 122880M. worker1 still runs that 20G image store today. +// --------------------------------------------------------------------------- +describe("v2 layout", () => { + const roles = ["vanilla", "worker", "infra", "labcontroller"] as const; + const p = (role: (typeof roles)[number], schemaVersion?: 1 | 2) => + renderFirstbootScript({ + hostname: "mac1", role, serverIp: "10.0.0.1", httpPort: 8080, + sshKeys: ["ssh-ed25519 AAAA t"], adminUser: "michal", mac: "aa:bb:cc:dd:ee:ff", + ...(schemaVersion ? { schemaVersion } : {}), + } as any); + + function lvSizes(script: string): Record { + const out: Record = {}; + for (const m of script.matchAll(/lvcreate -L (\d+)M\s+-n (\w+)\s+labvg/g)) { + out[m[2]!] = Number(m[1]); + } + return out; + } + + it("v2 never creates a root LV — the Asahi installer already made a btrfs root", () => { + for (const role of roles) { + expect(Object.keys(lvSizes(p(role, 2)))).not.toContain("root"); + } + }); + + it("v2 sizes rancher at 250G, not the 20G that undersized worker1", () => { + const lvs = lvSizes(p("infra", 2)); + expect(lvs["rancher"]).toBe(256000); + expect(lvs["var"]).toBe(49152); + expect(lvs["swap"]).toBe(16384); + }); + + it("v2 gives a worker a rancher LV; v1 gave it only to servers", () => { + expect(lvSizes(p("worker", 2))["rancher"]).toBe(256000); + expect(lvSizes(p("worker"))["rancher"]).toBeUndefined(); + }); + + it("v2 never uses 100%FREE", () => { + for (const role of roles) { + expect(p(role, 2)).not.toContain("100%FREE"); + } + }); + + it("v1 still uses 100%FREE for longhorn and is otherwise unchanged", () => { + const v1 = p("worker"); + expect(v1).toContain("lvcreate -l 100%FREE -n longhorn"); + const lvs = lvSizes(v1); + expect(lvs["swap"]).toBe(27648); + expect(lvs["var"]).toBe(102400); + }); + + it("v2 tags the volume group so a later run can detect the layout", () => { + expect(p("worker", 2)).toContain("vgchange --addtag lab-schema-v2 labvg"); + expect(p("worker")).not.toContain("--addtag lab-schema-v2"); + }); + + it("reprovision reports the on-disk schema", () => { + expect(p("worker", 2)).toMatch(/vgs --noheadings -o vg_tags labvg.*grep -q lab-schema-v2/); + }); +}); diff --git a/bastion/src/bastion/tests/auto-k3s.test.ts b/bastion/src/bastion/tests/auto-k3s.test.ts new file mode 100644 index 0000000..76aaf12 --- /dev/null +++ b/bastion/src/bastion/tests/auto-k3s.test.ts @@ -0,0 +1,137 @@ +// Tests for the post-provision k3s auto-install gate. +// +// Two independent switches, for two different problems: +// +// 1. shouldAutoInstallK3s(role) — a ROLE gate. The post-provision installer +// runs a bare `INSTALL_K3S_EXEC=server|agent` with no K3S_URL and no +// K3S_TOKEN and knows about exactly one machine. For a node that is its +// own cluster that is fine. For a cluster MEMBER it is wrong: servers get +// cluster-init semantics and form their own separate cluster (so three +// k8s-combo boxes yield three clusters), and agents cannot start at all +// ("Job for k3s-agent.service failed"). +// +// 2. config.autoK3s — a global off switch, needed because the prerequisite +// steps stop k3s and delete CNI state several minutes after an install +// lands, which lands on top of a test forming its own cluster. + +import { describe, it, expect } from "vitest"; +import { shouldAutoInstallK3s } from "../src/services/post-provision.js"; +import { loadConfig } from "../src/config.js"; +import { SUPPORTED_ROLES, isK3sServer, roleInfo } from "@lab/shared"; +import type { Role } from "@lab/shared"; + +describe("shouldAutoInstallK3s", () => { + // These join a cluster someone else owns, so they cannot be auto-installed + // without the server URL and token that post-provision does not have. + const CLUSTER_MEMBERS: Role[] = ["k8s-combo", "k8s-controller", "k8s-worker", "worker"]; + + // These are each their own cluster (or no k3s at all), so a bare install is + // a correct install. + const SELF_CONTAINED: Role[] = ["standalone-cluster", "infra", "labcontroller", "vanilla"]; + + it.each(CLUSTER_MEMBERS)("refuses to auto-install %s", (role) => { + expect(shouldAutoInstallK3s(role)).toBe(false); + }); + + it.each(SELF_CONTAINED)("still auto-installs %s", (role) => { + expect(shouldAutoInstallK3s(role)).toBe(true); + }); + + it("covers every supported role — a new role must make a deliberate choice", () => { + for (const role of SUPPORTED_ROLES) { + expect( + CLUSTER_MEMBERS.includes(role) || SELF_CONTAINED.includes(role), + `role ${role} is in neither list; decide whether post-provision can install it`, + ).toBe(true); + expect(typeof shouldAutoInstallK3s(role)).toBe("boolean"); + } + }); + + it("refuses every k3s AGENT role — an agent with nothing to join cannot start", () => { + for (const role of SUPPORTED_ROLES) { + if (roleInfo(role).k3sMode === "agent") { + expect(shouldAutoInstallK3s(role), `${role} is an agent`).toBe(false); + } + } + }); + + it("does not refuse purely on the basis of being a server", () => { + // Guard against the gate degenerating into isK3sServer(): labcontroller + // and infra are servers AND auto-installable. + const servers = SUPPORTED_ROLES.filter(isK3sServer); + expect(servers.some((r) => shouldAutoInstallK3s(r))).toBe(true); + expect(servers.some((r) => !shouldAutoInstallK3s(r))).toBe(true); + }); +}); + +describe("config.autoK3s", () => { + it("defaults to enabled", () => { + const config = loadConfig({}); + expect(config.autoK3s).toBe(true); + }); + + it("is disabled by an explicit override", () => { + const config = loadConfig({ autoK3s: false }); + expect(config.autoK3s).toBe(false); + }); + + it("an explicit true override wins over nothing", () => { + const config = loadConfig({ autoK3s: true }); + expect(config.autoK3s).toBe(true); + }); + + it("AUTO_K3S=false in the environment disables it", () => { + const prev = process.env["AUTO_K3S"]; + process.env["AUTO_K3S"] = "false"; + try { + expect(loadConfig({}).autoK3s).toBe(false); + } finally { + if (prev === undefined) delete process.env["AUTO_K3S"]; else process.env["AUTO_K3S"] = prev; + } + }); + + it("only the exact string 'false' disables it", () => { + const prev = process.env["AUTO_K3S"]; + process.env["AUTO_K3S"] = "0"; + try { + expect(loadConfig({}).autoK3s).toBe(true); + } finally { + if (prev === undefined) delete process.env["AUTO_K3S"]; else process.env["AUTO_K3S"] = prev; + } + }); +}); + +describe("kickstart packages", () => { + // etcd membership recovery is `etcdctl member list/remove`, and `k3s etcdctl` + // is NOT a command (verified on a live node: "No help topic for 'etcdctl'"). + // The binary ships in the `etcd` rpm. The recovery card is written for a + // situation with no internet, so telling the operator to dnf-install it there + // is not a plan. + it("installs etcd on k3s servers and nowhere else", async () => { + const { renderInstallKickstart } = await import("../src/templates/install.ks.js"); + const render = (role: Role): string => + renderInstallKickstart({ + hostname: "test-node", disk: "nvme0n1", role, domain: "ad.itaz.eu", + fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8", + serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514, + sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal", + ...(role === "vanilla" || role === "worker" || role === "infra" || role === "labcontroller" + ? {} : { schemaVersion: 2 as const }), + }); + // Scope to %packages: "etcd" appears in paths and comments elsewhere, and a + // whole-document match would pass for the wrong reason. + const pkgSection = (ks: string): string => { + const start = ks.indexOf("%packages"); + return ks.slice(start, ks.indexOf("%end", start)); + }; + + for (const role of ["infra", "labcontroller", "k8s-combo", "k8s-controller", "standalone-cluster"] as const) { + expect(pkgSection(render(role)), `${role} is an etcd member and needs etcdctl offline`) + .toMatch(/^etcd$/m); + } + for (const role of ["vanilla", "worker", "k8s-worker"] as const) { + expect(pkgSection(render(role)), `${role} runs no etcd`) + .not.toMatch(/^etcd$/m); + } + }); +}); diff --git a/bastion/src/bastion/tests/kickstart-layout.test.ts b/bastion/src/bastion/tests/kickstart-layout.test.ts new file mode 100644 index 0000000..0476e68 --- /dev/null +++ b/bastion/src/bastion/tests/kickstart-layout.test.ts @@ -0,0 +1,186 @@ +import { describe, it, expect } from "vitest"; +import { renderInstallKickstart } from "../src/templates/install.ks.js"; +import type { InstallKickstartParams } from "../src/templates/install.ks.js"; +import { SUPPORTED_ROLES, LEGACY_ROLES, v2Layout, type Role } from "@lab/shared"; + +/** + * Partition-layout tests. + * + * The existing ksvalidator test only sees the MAIN kickstart body. The actual + * layout is written by %pre into /tmp/part.ks inside a heredoc, so until now + * nothing validated it at all -- a broken layout rendered, shipped and was only + * discovered on the installed machine. + * + * These parse the heredocs. Following tests/ubuntu-autoinstall.test.ts: + * "A `toContain` assertion would not have caught that; only parsing does." + */ + +function baseParams(over: Partial = {}): InstallKickstartParams { + return { + hostname: "test-node", disk: "nvme0n1", role: "worker", domain: "ad.itaz.eu", + fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8", + serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514, + sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal", + ...over, + }; +} + +/** Pull every `cat >[>] /tmp/part.ks << PARTEOF ... PARTEOF` body out of the script. */ +function partKsBlocks(ks: string): string[] { + const out: string[] = []; + const re = /cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g; + let m: RegExpExecArray | null; + while ((m = re.exec(ks)) !== null) out.push(m[1]!); + return out; +} + +interface Logvol { mount: string; name: string; fs: string; size?: number; useExisting: boolean } + +function parseLogvols(block: string): Logvol[] { + return block.split("\n") + .filter((l) => l.trim().startsWith("logvol ")) + .map((line) => { + const parts = line.trim().split(/\s+/); + const get = (flag: string) => { + const p = parts.find((x) => x.startsWith(`${flag}=`)); + return p ? p.slice(flag.length + 1) : undefined; + }; + return { + mount: parts[1]!, + name: get("--name") ?? "", + fs: get("--fstype") ?? "", + ...(get("--size") ? { size: Number(get("--size")) } : {}), + useExisting: parts.includes("--useexisting"), + }; + }); +} + +/** The fresh-install block is the one that also creates the PV. */ +function freshBlock(ks: string): string { + const b = partKsBlocks(ks).find((x) => x.includes("part pv.01")); + if (!b) throw new Error("no fresh-install part.ks block found"); + return b; +} + +describe("v1 is frozen — existing machines must reinstall identically", () => { + it.each(LEGACY_ROLES)("%s v1 fresh layout is unchanged", (role) => { + const lvs = parseLogvols(freshBlock(renderInstallKickstart(baseParams({ role })))); + const byName = Object.fromEntries(lvs.map((l) => [l.name, l])); + + // The exact v1 sizes, in MiB. If one of these changes, every existing box + // in the fleet would be re-laid-out on its next reinstall. + expect(byName["swap"]!.size).toBe(27648); + expect(byName["root"]!.size).toBe(33792); + expect(byName["var"]!.size).toBe(102400); + expect(byName["varlog"]!.size).toBe(10240); + expect(byName["home"]!.size).toBe(10240); + expect(byName["srv"]!.size).toBe(20480); + }); + + it("v1 gives rancher to infra and worker only — the historical bug, preserved", () => { + const has = (r: Role) => freshBlock(renderInstallKickstart(baseParams({ role: r }))).includes("--name=rancher"); + expect(has("infra")).toBe(true); + expect(has("worker")).toBe(true); + expect(has("labcontroller")).toBe(false); // the bug, deliberately kept in v1 + expect(has("vanilla")).toBe(false); + }); + + it("v1 is the default — no caller has to opt out", () => { + const ks = renderInstallKickstart(baseParams({ role: "worker" })); + expect(parseLogvols(freshBlock(ks)).find((l) => l.name === "root")!.size).toBe(33792); + }); +}); + +describe("v2-only roles have no v1 layout", () => { + it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"] as const)( + "%s defaults to v2 when no schema is given", (role) => { + const lvs = parseLogvols(freshBlock(renderInstallKickstart(baseParams({ role })))); + expect(lvs.find((l) => l.name === "root")!.size).toBe(32768); + expect(lvs.some((l) => l.name === "rancher")).toBe(true); + }); + + it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"] as const)( + "%s throws on an EXPLICIT v1 rather than silently dropping its volumes", (role) => { + expect(() => renderInstallKickstart(baseParams({ role, schemaVersion: 1 }))).toThrow(/no v1 disk layout/); + }); +}); + +describe("v2 renders the shared layout", () => { + it.each(SUPPORTED_ROLES)("%s v2 matches v2Layout() exactly", (role) => { + const ks = renderInstallKickstart(baseParams({ role, schemaVersion: 2 })); + const rendered = parseLogvols(freshBlock(ks)); + const expected = v2Layout(role); + + expect(rendered.map((l) => l.name).sort()).toEqual(expected.map((l) => l.name).sort()); + for (const spec of expected) { + const got = rendered.find((l) => l.name === spec.name)!; + expect(got.size).toBe(spec.sizeMib); + expect(got.fs).toBe(spec.fs); + } + }); + + it("v2 gives labcontroller a rancher LV — the v1 bug, fixed", () => { + const ks = renderInstallKickstart(baseParams({ role: "labcontroller", schemaVersion: 2 })); + const rancher = parseLogvols(freshBlock(ks)).find((l) => l.name === "rancher"); + expect(rancher).toBeDefined(); + expect(rancher!.size).toBe(256000); + }); + + it("v2 never uses --grow", () => { + for (const role of SUPPORTED_ROLES) { + const block = freshBlock(renderInstallKickstart(baseParams({ role, schemaVersion: 2 }))); + expect(block).not.toMatch(/logvol[^\n]*--grow/); + } + }); + + it("v2 uses the larger EFI and /boot partitions", () => { + const block = freshBlock(renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 2 }))); + expect(block).toMatch(/part \/boot\/efi --fstype=efi --size=1024/); + expect(block).toMatch(/part \/boot --fstype=ext4 --size=4096/); + }); +}); + +describe("reprovision protects existing data", () => { + it("reprovision reads the on-disk schema instead of trusting the caller", () => { + const ks = renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 2 })); + expect(ks).toContain("DISK_SCHEMA=v1"); + expect(ks).toMatch(/vgs --noheadings -o vg_tags .*grep -q lab-schema-v2/); + }); + + it("the reprovision branch never wipes the disk", () => { + const ks = renderInstallKickstart(baseParams({ role: "worker" })); + const repro = partKsBlocks(ks).find((b) => b.includes("--useexisting"))!; + expect(repro).toContain("clearpart --none"); + expect(repro).not.toContain("clearpart --all"); + }); + + it("preserved data volumes are never reformatted", () => { + const ks = renderInstallKickstart(baseParams({ role: "worker" })); + for (const lv of ["longhorn", "rancher", "home", "srv"]) { + const re = new RegExp(`--name=${lv} --useexisting --noformat`); + expect(ks).toMatch(re); + } + }); +}); + +describe("the %pre block survives the /ks-auto extractor", () => { + it("matches the same regex ks-auto uses to re-run %pre", () => { + // routes/kickstart.ts re-extracts %pre with python: + // re.findall(r'%pre[^\n]*\n(.*?)%end', text, re.DOTALL) + // A change that breaks this breaks ISO installs silently. + for (const role of SUPPORTED_ROLES) { + // v2-only roles throw at v1 by design, so only render the combinations + // that are actually legal. + // An explicit v1 is illegal for a v2-only role; everything else renders. + const versions = (LEGACY_ROLES as readonly string[]).includes(role) + ? ([1, 2] as const) + : ([2] as const); + for (const schemaVersion of versions) { + const ks = renderInstallKickstart(baseParams({ role, schemaVersion })); + const blocks = [...ks.matchAll(/%pre[^\n]*\n([\s\S]*?)%end/g)]; + expect(blocks.length).toBeGreaterThan(0); + expect(blocks[0]![1]).toContain("/tmp/part.ks"); + } + } + }); +}); diff --git a/bastion/src/bastion/tests/layout-consistency.test.ts b/bastion/src/bastion/tests/layout-consistency.test.ts new file mode 100644 index 0000000..6866ec7 --- /dev/null +++ b/bastion/src/bastion/tests/layout-consistency.test.ts @@ -0,0 +1,105 @@ +import { describe, it, expect } from "vitest"; +import { execFileSync } from "node:child_process"; +import { writeFileSync } from "node:fs"; +import { renderInstallKickstart } from "../src/templates/install.ks.js"; +import { renderUbuntuAutoinstall } from "../src/templates/ubuntu-autoinstall.js"; +import { renderFirstbootScript } from "../src/templates/asahi-firstboot.sh.js"; +import { v2Layout, SUPPORTED_ROLES, type Role } from "@lab/shared"; + +/** + * THE test this whole refactor exists for. + * + * Three generators partition machines. Before the shared layout they disagreed, + * and nothing noticed: + * + * install.ks.ts rancher 122880M + * ubuntu-autoinstall rancher 20480M + * asahi-firstboot rancher 20480M <- under a comment claiming it + * matched install.ks.ts + * + * That 100 GB gap is why worker1 runs an image store too small for one large + * container image. This test fails the moment any generator drifts again. + */ + +const KS = (role: Role) => renderInstallKickstart({ + hostname: "n", disk: "nvme0n1", role, domain: "d", fedoraVersion: "43", + timezone: "UTC", locale: "en_US.UTF-8", serverIp: "10.0.0.1", httpPort: 8080, + syslogPort: 5514, sshKeys: [], adminUser: "a", schemaVersion: 2, +}); + +const UB = (role: Role) => renderUbuntuAutoinstall({ + hostname: "n", disk: "/dev/sda", role, domain: "d", ubuntuVersion: "24.04", + timezone: "UTC", locale: "en_US.UTF-8", serverIp: "10.0.0.1", httpPort: 8080, + sshKeys: [], adminUser: "a", schemaVersion: 2, +} as never); + +const AS = (role: Role) => renderFirstbootScript({ + hostname: "n", role, serverIp: "10.0.0.1", httpPort: 8080, + sshKeys: [], adminUser: "a", mac: "aa:bb:cc:dd:ee:ff", schemaVersion: 2, +} as never); + +/** MiB per LV name, from the Fedora kickstart's part.ks heredoc. */ +function fedoraSizes(role: Role): Record { + const ks = KS(role); + const block = [...ks.matchAll(/cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g)] + .map((m) => m[1]!).find((b) => b.includes("part pv.01"))!; + const out: Record = {}; + for (const line of block.split("\n").filter((l) => l.trim().startsWith("logvol "))) { + const name = /--name=(\S+)/.exec(line)?.[1]; + const size = /--size=(\d+)/.exec(line)?.[1]; + if (name && size) out[name] = Number(size); + } + return out; +} + +/** MiB per LV name, from the Ubuntu curtin document (declared in GiB). */ +function ubuntuSizes(role: Role): Record { + const tmp = `/tmp/consistency-${role}.yaml`; + writeFileSync(tmp, UB(role)); + const json = execFileSync("python3", + ["-c", "import sys,yaml,json; json.dump(yaml.safe_load(open(sys.argv[1])), sys.stdout)", tmp], + { encoding: "utf-8" }); + const cfg = JSON.parse(json).autoinstall.storage.config as Array>; + return Object.fromEntries( + cfg.filter((c) => c["type"] === "lvm_partition") + .map((c) => [c["name"]!, Number(String(c["size"]).replace("G", "")) * 1024]), + ); +} + +/** MiB per LV name, from the Asahi firstboot lvcreate calls. */ +function asahiSizes(role: Role): Record { + const out: Record = {}; + for (const m of AS(role).matchAll(/lvcreate -L (\d+)M\s+-n (\w+)\s+labvg/g)) out[m[2]!] = Number(m[1]); + return out; +} + +describe("all three generators render the same v2 layout", () => { + it.each(SUPPORTED_ROLES)("%s: Fedora matches the shared definition", (role) => { + const expected = Object.fromEntries(v2Layout(role).map((lv) => [lv.name, lv.sizeMib])); + expect(fedoraSizes(role)).toEqual(expected); + }); + + it.each(SUPPORTED_ROLES)("%s: Ubuntu matches the shared definition", (role) => { + const expected = Object.fromEntries(v2Layout(role).map((lv) => [lv.name, lv.sizeMib])); + expect(ubuntuSizes(role)).toEqual(expected); + }); + + it.each(SUPPORTED_ROLES)("%s: Asahi matches, minus the vendor-created root", (role) => { + const expected = Object.fromEntries( + v2Layout(role, { omitRoot: true }).map((lv) => [lv.name, lv.sizeMib]), + ); + expect(asahiSizes(role)).toEqual(expected); + }); + + it.each(SUPPORTED_ROLES)("%s: Fedora and Ubuntu are byte-identical in sizing", (role) => { + expect(ubuntuSizes(role)).toEqual(fedoraSizes(role)); + }); + + it("rancher is 250G everywhere — the specific drift that bit worker1", () => { + for (const role of ["worker", "infra", "labcontroller"] as const) { + expect(fedoraSizes(role)["rancher"]).toBe(256000); + expect(ubuntuSizes(role)["rancher"]).toBe(256000); + expect(asahiSizes(role)["rancher"]).toBe(256000); + } + }); +}); diff --git a/bastion/src/bastion/tests/schema-version.test.ts b/bastion/src/bastion/tests/schema-version.test.ts new file mode 100644 index 0000000..a92b77b --- /dev/null +++ b/bastion/src/bastion/tests/schema-version.test.ts @@ -0,0 +1,166 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { BastionConfig } from "@lab/shared"; +import { createApp } from "../src/server.js"; +import type { FastifyInstance } from "fastify"; +import type { StateManager } from "../src/services/state.js"; +import type { InstallLogBuffer } from "../src/services/install-log.js"; + +function createTestConfig(testDir: string): BastionConfig { + return { + fedoraVersion: "43", + arch: "x86_64", + httpPort: 0, + timezone: "Europe/London", + locale: "en_GB.UTF-8", + bastionDir: testDir, + domain: "test.local", + dhcpMode: "proxy", + dhcpRangeStart: "", + dhcpRangeEnd: "", + ubuntuVersion: "26.04", + ubuntuMirror: "https://releases.ubuntu.com/26.04", + vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso", + vyosDefaultPassword: "vyos", + iface: "eth0", + serverIp: "10.0.0.1", + network: "10.0.0.0", + gateway: "10.0.0.1", + sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"], + adminUser: "testadmin", + syslogPort: 15514, + skipDnsmasq: true, + skipArtifacts: true, + fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os", + tftpDir: join(testDir, "tftp"), + httpDir: join(testDir, "http"), + stateFile: join(testDir, "state.json"), + }; +} + +describe("schemaVersion plumbing: /api/install -> /ks", () => { + let app: FastifyInstance; + let testDir: string; + + beforeEach(async () => { + testDir = join(tmpdir(), `schema-test-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(testDir, { recursive: true }); + const built = await createApp(createTestConfig(testDir)); + app = built.app; + await app.ready(); + }); + + afterEach(async () => { + await app.close(); + rmSync(testDir, { recursive: true, force: true }); + }); + + const queue = (body: Record) => + app.inject({ method: "POST", url: "/api/install", payload: { mac: "aa:bb:cc:dd:ee:01", hostname: "n1", ...body } }); + + const ks = () => app.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:01" }); + + it("defaults to v1 when no schemaVersion is given", async () => { + expect((await queue({ role: "worker" })).statusCode).toBe(200); + const body = (await ks()).body; + expect(body).toContain("--name=root --fstype=xfs --size=33792"); + }); + + it("schemaVersion 2 reaches the rendered kickstart", async () => { + expect((await queue({ role: "worker", schemaVersion: 2 })).statusCode).toBe(200); + const body = (await ks()).body; + expect(body).toContain("--name=root --fstype=xfs --size=32768"); + expect(body).toContain("--name=rancher --fstype=xfs --size=256000"); + }); + + it("v2 gives labcontroller a rancher LV through the full path", async () => { + expect((await queue({ role: "labcontroller", schemaVersion: 2 })).statusCode).toBe(200); + expect((await ks()).body).toContain("--name=rancher"); + }); + + it("rejects an unknown schemaVersion instead of silently installing v1", async () => { + const res = await queue({ role: "worker", schemaVersion: 3 }); + expect(res.statusCode).toBe(400); + expect(res.json().error).toMatch(/invalid schemaVersion/); + }); + + it("still rejects an unknown role", async () => { + expect((await queue({ role: "nope" })).statusCode).toBe(400); + }); +}); + +describe("v2-only roles cannot be installed at v1", () => { + let app2: FastifyInstance; + let dir2: string; + + beforeEach(async () => { + dir2 = join(tmpdir(), `schema-v2only-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(dir2, { recursive: true }); + const built = await createApp(createTestConfig(dir2)); + app2 = built.app; + await app2.ready(); + }); + afterEach(async () => { await app2.close(); rmSync(dir2, { recursive: true, force: true }); }); + + const q = (body: Record) => + app2.inject({ method: "POST", url: "/api/install", payload: { mac: "aa:bb:cc:dd:ee:02", hostname: "n2", ...body } }); + + /** + * Omitting the schema must WORK for a v2-only role: a reinstall never passes + * one (the lab-schema-v2 tag on the disk decides), and the first version of + * this guard rejected that, making it impossible to reinstall a v2-only + * machine at all. Found by the reinstall integration test. + */ + it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"])( + "%s is accepted without a schemaVersion and defaults to v2", async (role) => { + const res = await q({ role }); + expect(res.statusCode).toBe(200); + const ks = (await app2.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:02" })).body; + expect(ks).toContain("--name=root --fstype=xfs --size=32768"); + }); + + it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"])( + "%s is rejected on an EXPLICIT schemaVersion 1 — that is a contradiction", async (role) => { + const res = await q({ role, schemaVersion: 1 }); + expect(res.statusCode).toBe(400); + expect(res.json().error).toMatch(/requires schemaVersion 2/); + }); + + it.each(["standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo"])( + "%s is accepted with schemaVersion 2", async (role) => { + expect((await q({ role, schemaVersion: 2 })).statusCode).toBe(200); + }); + + it("legacy roles are still accepted without a schemaVersion", async () => { + for (const role of ["vanilla", "worker", "infra", "labcontroller"]) { + expect((await q({ role })).statusCode).toBe(200); + } + }); + + /** + * Assert on the FRESH-INSTALL part.ks block specifically. The whole kickstart + * also contains the reprovision branch, whose shell emits `--name=longhorn` + * inside a `PRESERVE_LONGHORN` guard -- so a naive whole-file `not.toContain` + * matches that and fails for reasons that have nothing to do with the layout. + */ + function freshBlockOf(ks: string): string { + const blocks = [...ks.matchAll(/cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g)] + .map((m) => m[1]!); + const fresh = blocks.find((b) => b.includes("part pv.01")); + if (!fresh) throw new Error("no fresh-install block in rendered kickstart"); + return fresh; + } + + it("k8s-controller gets no longhorn LV; k8s-combo does", async () => { + await q({ role: "k8s-controller", schemaVersion: 2 }); + const ctrl = freshBlockOf((await app2.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:02" })).body); + expect(ctrl).toContain("--name=rancher"); + expect(ctrl).not.toContain("--name=longhorn"); + + await q({ role: "k8s-combo", schemaVersion: 2 }); + const combo = freshBlockOf((await app2.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:02" })).body); + expect(combo).toContain("--name=longhorn --fstype=xfs --size=307200"); + }); +}); diff --git a/bastion/src/bastion/tests/ubuntu-autoinstall.test.ts b/bastion/src/bastion/tests/ubuntu-autoinstall.test.ts index 96e6e06..d870962 100644 --- a/bastion/src/bastion/tests/ubuntu-autoinstall.test.ts +++ b/bastion/src/bastion/tests/ubuntu-autoinstall.test.ts @@ -85,3 +85,100 @@ describe("renderUbuntuAutoinstall", () => { expect(eth.optional).toBe(true); }); }); + +// --------------------------------------------------------------------------- +// v2 layout. Ubuntu previously sized rancher at 20G while the Fedora kickstart +// used 120G, from a template whose comment claimed the sizes matched. Both now +// render from shared/src/types/layout.ts, so this checks the rendering, and +// layout.test.ts checks the definition. +// --------------------------------------------------------------------------- +describe("v2 storage layout", () => { + const roles = ["vanilla", "worker", "infra", "labcontroller"] as const; + + function lvmPartitions(doc: Record): Record { + const cfg = doc["autoinstall"]["storage"]["config"] as Array>; + return Object.fromEntries( + cfg.filter((c) => c["type"] === "lvm_partition").map((c) => [c["name"], c["size"]]), + ); + } + + it.each(roles)("%s renders valid YAML at schemaVersion 2", (role) => { + const doc = parseYaml( + renderUbuntuAutoinstall({ ...base, role, schemaVersion: 2 } as any), + `v2-${role}`, + ); + expect(Array.isArray(doc["autoinstall"]["storage"]["config"])).toBe(true); + }); + + it("v2 sizes match the shared definition, not the old 20G rancher", () => { + const doc = parseYaml( + renderUbuntuAutoinstall({ ...base, role: "worker", schemaVersion: 2 } as any), + "v2-sizes", + ); + const lvs = lvmPartitions(doc); + expect(lvs["rancher"]).toBe("250G"); // was 20G here and 120G in kickstart + expect(lvs["longhorn"]).toBe("300G"); // was grow + expect(lvs["root"]).toBe("32G"); + expect(lvs["var"]).toBe("48G"); + }); + + it("v2 gives labcontroller a rancher LV; v1 does not", () => { + const v2 = lvmPartitions(parseYaml( + renderUbuntuAutoinstall({ ...base, role: "labcontroller", schemaVersion: 2 } as any), "v2-lc")); + const v1 = lvmPartitions(parseYaml( + renderUbuntuAutoinstall({ ...base, role: "labcontroller" } as any), "v1-lc")); + expect(v2["rancher"]).toBe("250G"); + expect(v1["rancher"]).toBeUndefined(); + }); + + it("v1 remains the default and is unchanged", () => { + const lvs = lvmPartitions(parseYaml( + renderUbuntuAutoinstall({ ...base, role: "worker" } as any), "v1-default")); + expect(lvs["root"]).toBe("33G"); + expect(lvs["var"]).toBe("100G"); + expect(lvs["swap"]).toBe("27G"); + }); + + it("no v2 volume is unbounded", () => { + for (const role of roles) { + const lvs = lvmPartitions(parseYaml( + renderUbuntuAutoinstall({ ...base, role, schemaVersion: 2 } as any), `v2-grow-${role}`)); + for (const [name, size] of Object.entries(lvs)) { + expect(String(size), `${role}/${name}`).toMatch(/^\d+G$/); + } + } + }); +}); + +/** + * KNOWN GAP, asserted so it cannot be assumed away. + * + * The Fedora kickstart preserves /home, /srv, longhorn and rancher across a + * reinstall (%pre probes for an existing labvg). The Ubuntu autoinstall has no + * equivalent: `disk0` is wiped unconditionally, so EVERY Ubuntu reinstall is + * destructive. + * + * These tests pin that behaviour rather than pretend it is fine. If someone + * adds curtin `preserve: true` handling, these fail and should be rewritten -- + * which is the point: the change becomes visible instead of silent. + */ +describe("Ubuntu reinstall is destructive (known gap)", () => { + it("wipes the disk unconditionally — there is no reprovision branch", () => { + const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" } as any), "destructive"); + const cfg = doc["autoinstall"]["storage"]["config"] as Array>; + const disk = cfg.find((c) => c["type"] === "disk"); + expect(disk?.["wipe"]).toBe("superblock-recursive"); + }); + + it("no volume is marked preserve — nothing survives a reinstall", () => { + const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" } as any), "no-preserve"); + const cfg = doc["autoinstall"]["storage"]["config"] as Array>; + expect(cfg.some((c) => c["preserve"] === true)).toBe(false); + }); + + it("the same is true at schemaVersion 2 — v2 did not fix this", () => { + const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker", schemaVersion: 2 } as any), "v2-destructive"); + const cfg = doc["autoinstall"]["storage"]["config"] as Array>; + expect(cfg.find((c) => c["type"] === "disk")?.["wipe"]).toBe("superblock-recursive"); + }); +}); diff --git a/bastion/src/cli/src/api/client.ts b/bastion/src/cli/src/api/client.ts index b6f5763..a5cda4c 100644 --- a/bastion/src/cli/src/api/client.ts +++ b/bastion/src/cli/src/api/client.ts @@ -90,6 +90,7 @@ export class LabdClient { async installMachine(opts: { mac: string; hostname: string; disk?: string; role?: string; os?: string; + schemaVersion?: 1 | 2; vyos?: import("@lab/shared").VyosInstallSpec; }): Promise<{ status: string; data?: unknown; error?: string }> { return this.request("POST", "/api/machines/install", { body: opts }); diff --git a/bastion/src/cli/src/commands/app.ts b/bastion/src/cli/src/commands/app.ts index 89ee49f..aff96d0 100644 --- a/bastion/src/cli/src/commands/app.ts +++ b/bastion/src/cli/src/commands/app.ts @@ -57,6 +57,7 @@ async function fetchState(): Promise { } import { registerLabcontrollerCommands } from "./labcontroller.js"; +import { isRole, SUPPORTED_ROLES, type Role } from "@lab/shared"; export function registerAppCommand(program: Command): void { const appCmd = program.command("app").description("Application management"); @@ -66,18 +67,104 @@ export function registerAppCommand(program: Command): void { const k3sCmd = appCmd.command("k3s").description("k3s cluster management"); + k3sCmd + .command("evacuate ") + .description("Cordon, evict Longhorn replicas and drain so a node can be safely wiped") + .option("--user ", "SSH user", "root") + .option( + "--allow-single-replica", + "permit evacuating volumes that have only one replica (rebuilds from the only copy; there is no backup target)", + ) + .action(async (target: string, opts: { user: string; allowSingleReplica?: boolean }) => { + const state = await fetchState(); + const resolved = resolveTarget(target, state); + if (!resolved) { + console.error(`Cannot resolve target: ${target}`); + process.exit(1); + } + const sshKey = findSshKey(); + + console.log(`Evacuating ${resolved.hostname} (${resolved.ip})...`); + console.log("This waits for Longhorn to rebuild replicas elsewhere and can take a while.\n"); + + const k3sMod = new K3sModule(); + const result = await k3sMod.evacuate({ + hostname: resolved.hostname, + ip: resolved.ip, + role: "infra", + os: "fedora-43" as const, + arch: "x86_64" as const, + sshUser: opts.user, + ...(sshKey ? { sshKeyPath: sshKey } : {}), + config: opts.allowSingleReplica ? { evictionAllowSingleReplica: true } : {}, + }); + + for (const line of result.output) console.log(` ${line}`); + if (!result.success) { + console.error(`\nNode is NOT safe to wipe (${result.errors.join(", ")}). Do not reprovision it.`); + console.error(`If an eviction was started and left pending, clear it with:`); + console.error(` labctl app k3s admit ${resolved.hostname}`); + process.exit(1); + } + console.log(`\n${resolved.hostname} is evacuated. Safe to reprovision.`); + }); + + k3sCmd + .command("admit ") + .description("Return an evacuated node to service: uncordon, clear eviction, restore its Longhorn disk") + .option("--user ", "SSH user", "root") + .action(async (target: string, opts: { user: string }) => { + const state = await fetchState(); + const resolved = resolveTarget(target, state); + if (!resolved) { + console.error(`Cannot resolve target: ${target}`); + process.exit(1); + } + const sshKey = findSshKey(); + + console.log(`Returning ${resolved.hostname} (${resolved.ip}) to service...`); + + const k3sMod = new K3sModule(); + const result = await k3sMod.admit({ + hostname: resolved.hostname, + ip: resolved.ip, + role: isRole(resolved.role) ? resolved.role : "infra", + os: "fedora-43" as const, + arch: "x86_64" as const, + sshUser: opts.user, + ...(sshKey ? { sshKeyPath: sshKey } : {}), + config: {}, + }); + + for (const line of result.output) console.log(` ${line}`); + if (!result.success) { + console.error(`\n${resolved.hostname} is NOT fully back in service (${result.errors.join(", ")}).`); + process.exit(1); + } + console.log(`\n${resolved.hostname} is back in service.`); + }); + k3sCmd .command("install ") .description("Install k3s on a target machine (hostname, IP, or MAC)") - .option("--role ", "k3s role: infra (server) or worker (agent)", "infra") + .option( + "--role ", + "k3s role: infra, labcontroller, standalone-cluster, k8s-controller, k8s-combo (servers) or worker, k8s-worker (agents)", + "infra", + ) .option("--user ", "SSH user", "root") - .option("--k3s-server ", "k3s server URL (required for worker role)") - .option("--k3s-token ", "k3s join token (required for worker role)") + .option("--k3s-server ", "k3s server URL (required for agent roles, and to JOIN as a server)") + .option("--k3s-token ", "k3s join token; on a first server it SEEDS the cluster token") + .option( + "--k3s-version ", + "exact k3s version, e.g. v1.34.6+k3s1. Unset installs LATEST, which on a rejoining node can be several minor versions ahead of the cluster", + ) .action(async (target: string, opts: { role: string; user: string; k3sServer?: string; k3sToken?: string; + k3sVersion?: string; }) => { const state = await fetchState(); const resolved = resolveTarget(target, state); @@ -88,7 +175,16 @@ export function registerAppCommand(program: Command): void { process.exit(1); } - const role = opts.role === "worker" ? "worker" : "infra"; + // Every supported role, not just infra/worker. The old mapping silently + // turned anything that was not "worker" into "infra" -- so + // `--role k8s-combo` installed a DIFFERENT role than the one asked for, + // with different disk expectations and no warning. + if (!isRole(opts.role)) { + console.error(`Unknown role: ${opts.role}`); + console.error(`Valid roles: ${SUPPORTED_ROLES.join(", ")}`); + process.exit(1); + } + const role: Role = opts.role; const sshKey = findSshKey(); console.log(`Installing k3s on ${resolved.hostname} (${resolved.ip}) as ${role}...`); @@ -106,9 +202,22 @@ export function registerAppCommand(program: Command): void { config: { ...(opts.k3sServer ? { k3sServerUrl: opts.k3sServer } : {}), ...(opts.k3sToken ? { k3sToken: opts.k3sToken } : {}), + ...(opts.k3sVersion ? { k3sVersion: opts.k3sVersion } : {}), }, }; + // Only warn when it matters: a node JOINING an existing cluster. On a + // first server there is nothing to be skewed against, and a warning that + // fires when it is irrelevant is a warning people learn to skip. + if (!opts.k3sVersion && opts.k3sServer) { + console.warn( + " WARNING: no --k3s-version given, so get.k3s.io installs LATEST.\n" + + " This node is JOINING an existing cluster — check what that cluster runs\n" + + " (kubectl get nodes -o wide) and pin it. Kubernetes supports only ONE minor\n" + + " version of skew between control-plane members.", + ); + } + const installResult = await k3s.install(moduleCtx); for (const line of installResult.output) { console.log(` ${line}`); @@ -281,7 +390,8 @@ export function registerAppCommand(program: Command): void { const healthResult = await k3s.health({ hostname: resolved.hostname, ip: resolved.ip, - role: resolved.role, + // State is JSON on disk; narrow it rather than trusting the string. + role: isRole(resolved.role) ? resolved.role : "vanilla", os: "fedora-43" as const, arch: "x86_64" as const, sshUser: opts.user, diff --git a/bastion/src/cli/src/commands/install.ts b/bastion/src/cli/src/commands/install.ts index 694031b..b9f97d0 100644 --- a/bastion/src/cli/src/commands/install.ts +++ b/bastion/src/cli/src/commands/install.ts @@ -107,6 +107,7 @@ export function registerInstallCommand(parent: Command): void { .addOption(new Option("--role ", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker")) .addOption(new Option("--os ", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43")) .option("--disk ", "Target disk device (auto-detect if omitted)") + .addOption(new Option("--schema-version ", "Disk layout for a FRESH install (default 1). Ignored on reprovision: the layout already on the disk wins.").choices(["1", "2"])) .option("--vyos-mgmt ", "VyOS: untagged interface the machine PXE boots from (default eth0)") .option("--vyos-mgmt-address ", "VyOS: CIDR for the management interface, or 'dhcp' (default dhcp)") .option("--vyos-bond ", "VyOS: comma-separated LACP bond members (must exclude the PXE NIC)") @@ -136,6 +137,8 @@ export function registerInstallCommand(parent: Command): void { .action(async (mac: string, hostname: string, opts: { role: string; os: string; + /** commander gives us the string form of the --schema-version choice. */ + schemaVersion?: string; disk?: string; vyosMgmt?: string; vyosMgmtAddress?: string; @@ -248,6 +251,7 @@ export function registerInstallCommand(parent: Command): void { try { const result = await getLabdClient().installMachine({ + ...(opts.schemaVersion ? { schemaVersion: Number(opts.schemaVersion) as 1 | 2 } : {}), mac, hostname, role: opts.role, diff --git a/bastion/src/modules/modules/k3s/src/groups/admit.ts b/bastion/src/modules/modules/k3s/src/groups/admit.ts new file mode 100644 index 0000000..2e62e24 --- /dev/null +++ b/bastion/src/modules/modules/k3s/src/groups/admit.ts @@ -0,0 +1,31 @@ +// Return an evacuated node to service. The inverse of the evacuate group. +// +// Needed for two different journeys that end in the same place: +// - a node that was wiped, reinstalled and has just rejoined the cluster; +// - a node whose evacuation was aborted and must simply be put back. +// +// Order matters here too. configureLonghornDisk runs AFTER eviction is cleared, +// because a node that is still marked for eviction will have any disk we give +// it immediately drained again. + +import type { OperationContext, OperationResult, OperationGroup } from "../types.js"; +import { runSequential } from "../utils.js"; +import { uncordonNode } from "../operations/node-evacuate.js"; +import { clearLonghornEviction, assertNodeSchedulable } from "../operations/node-admit.js"; +import { configureLonghornDisk } from "../operations/longhorn-disk.js"; + +export const admitGroup: OperationGroup = { + name: "admit", + description: "Uncordon a node, clear its eviction request and restore its Longhorn disk", + operations: [ + { name: "Uncordon node", fn: uncordonNode }, + { name: "Clear Longhorn eviction", fn: clearLonghornEviction }, + { name: "Restore Longhorn default disk", fn: configureLonghornDisk }, + { name: "Verify node is back in service", fn: assertNodeSchedulable }, + ], +}; + +export async function runAdmit(ctx: OperationContext): Promise { + ctx.log("Returning node to service..."); + return runSequential(ctx, admitGroup.operations); +} diff --git a/bastion/src/modules/modules/k3s/src/groups/evacuate.ts b/bastion/src/modules/modules/k3s/src/groups/evacuate.ts new file mode 100644 index 0000000..1d865eb --- /dev/null +++ b/bastion/src/modules/modules/k3s/src/groups/evacuate.ts @@ -0,0 +1,54 @@ +// Evacuate a node so it can be wiped and reinstalled with a different layout. +// +// The ORDER is the operation: +// +// 1. Cordon stop new work landing on a node that is about to leave. +// 2. Check can eviction even finish? Seconds to answer, an hour to +// discover the hard way. +// 3. Evict move the replicas and WAIT for them to rebuild elsewhere. +// 4. Drain now move the workloads. +// 5. Verify re-read the cluster; do not trust that 1-4 worked. +// +// EVICT BEFORE DRAIN, which is the opposite of the obvious order and the one +// thing here most likely to be "tidied" back into a bug. `drain --force` does +// not override PodDisruptionBudgets, and longhorn-manager holds a PDB on the +// instance-manager pod for as long as volumes are attached or degraded. Drain +// first and you stop the instance-manager that serves the replicas you are +// about to evict: the drain blocks for its full 15-minute timeout, and if it +// did succeed it would have removed the only source a single-replica volume +// could rebuild from. +// +// Removing the etcd member is NOT part of this group -- it costs you the API +// server the rest of the procedure needs, so it belongs immediately before the +// wipe, not here. +// +// This group deliberately does not reinstall anything. It leaves the node +// cordoned, drained and empty, and a human (or `labctl provision reprovision`) +// takes it from there. + +import type { OperationContext, OperationResult, OperationGroup } from "../types.js"; +import { runSequential } from "../utils.js"; +import { + cordonNode, + drainNode, + evictLonghornReplicas, + assertEvictionCompletable, + assertNodeEvacuated, +} from "../operations/node-evacuate.js"; + +export const evacuateGroup: OperationGroup = { + name: "evacuate", + description: "Cordon, evict storage from and drain a node so it can be safely wiped", + operations: [ + { name: "Cordon node", fn: cordonNode }, + { name: "Check eviction can complete", fn: assertEvictionCompletable }, + { name: "Evict Longhorn replicas (waits for rebuild)", fn: evictLonghornReplicas }, + { name: "Drain workloads", fn: drainNode }, + { name: "Verify node is empty", fn: assertNodeEvacuated }, + ], +}; + +export async function runEvacuate(ctx: OperationContext): Promise { + ctx.log("Evacuating node before wipe..."); + return runSequential(ctx, evacuateGroup.operations); +} diff --git a/bastion/src/modules/modules/k3s/src/groups/k3s-agent.ts b/bastion/src/modules/modules/k3s/src/groups/k3s-agent.ts index ba6c0e0..8f67e5b 100644 --- a/bastion/src/modules/modules/k3s/src/groups/k3s-agent.ts +++ b/bastion/src/modules/modules/k3s/src/groups/k3s-agent.ts @@ -4,6 +4,7 @@ import type { OperationContext, OperationResult, OperationGroup } from "../types import { runSequential } from "../utils.js"; import { writeK3sConfig } from "../operations/k3s-config.js"; import { installK3sBinary } from "../operations/k3s-install.js"; +import { labelWorkerRole } from "../operations/worker-label.js"; export const k3sAgentGroup: OperationGroup = { name: "k3s-agent", @@ -11,6 +12,8 @@ export const k3sAgentGroup: OperationGroup = { operations: [ { name: "Write k3s config", fn: writeK3sConfig }, { name: "Install k3s agent", fn: installK3sBinary }, + // After registration, never before: the kubelet may not set this itself. + { name: "Label node as worker", fn: labelWorkerRole }, ], }; diff --git a/bastion/src/modules/modules/k3s/src/health/k3s-service.ts b/bastion/src/modules/modules/k3s/src/health/k3s-service.ts index be62fdb..ba7c252 100644 --- a/bastion/src/modules/modules/k3s/src/health/k3s-service.ts +++ b/bastion/src/modules/modules/k3s/src/health/k3s-service.ts @@ -1,8 +1,9 @@ import type { Operation, OperationResult } from "../types.js"; import { isServiceActive } from "../utils.js"; +import { isK3sServer } from "@lab/shared"; export const checkK3sService: Operation = async (ctx): Promise => { - const isServer = ctx.config.role === "infra" || ctx.config.role === "labcontroller"; + const isServer = isK3sServer(ctx.config.role); const service = isServer ? "k3s" : "k3s-agent"; const active = await isServiceActive(ctx, service); return { success: active, changed: false, message: active ? `${service} is active` : `${service} is not active` }; diff --git a/bastion/src/modules/modules/k3s/src/install.ts b/bastion/src/modules/modules/k3s/src/install.ts index b899668..17cf7b1 100644 --- a/bastion/src/modules/modules/k3s/src/install.ts +++ b/bastion/src/modules/modules/k3s/src/install.ts @@ -1,17 +1,21 @@ // k3s module — install phase. // Installs k3s with CIS-hardened configuration and Cilium CNI. +import { isK3sServer, type Role } from "@lab/shared"; + export interface K3sInstallContext { hostname: string; ip: string; - role: string; // "infra" = server, "worker" = agent + role: Role; // server vs agent is derived via isK3sServer(), never by name k3sServerUrl?: string; // Required for agent role k3sToken?: string; // Required for agent role } /** Generate the shell script that installs k3s on a target machine. */ export function generateInstallScript(ctx: K3sInstallContext): string { - const isServer = ctx.role === "infra"; + // Was `ctx.role === "infra"`, which silently treated a `labcontroller` -- a + // role that runs a k3s SERVER -- as an agent. Derived from the registry now. + const isServer = isK3sServer(ctx.role); return `#!/bin/bash set -euo pipefail diff --git a/bastion/src/modules/modules/k3s/src/k3s-module.ts b/bastion/src/modules/modules/k3s/src/k3s-module.ts index 7befe32..7c396e4 100644 --- a/bastion/src/modules/modules/k3s/src/k3s-module.ts +++ b/bastion/src/modules/modules/k3s/src/k3s-module.ts @@ -6,21 +6,35 @@ import type { OperationContext, K3sConfig, OperationResult } from "./types.js"; import { sshExec } from "../../../src/ssh.js"; import { aggregateResults } from "./utils.js"; import { runHostPrep } from "./groups/host-prep.js"; +import { runEvacuate } from "./groups/evacuate.js"; +import { runAdmit } from "./groups/admit.js"; import { runK3sServer } from "./groups/k3s-server.js"; import { runK3sAgent } from "./groups/k3s-agent.js"; import { runNetworking } from "./groups/networking.js"; import { runHardening } from "./groups/hardening.js"; import { runSequential } from "./utils.js"; import * as health from "./health/index.js"; +import { isK3sServer } from "@lab/shared"; function toOpContext(ctx: ModuleContext): OperationContext { + // Pass the WHOLE config bag through, then pin the fields that come from the + // ModuleContext itself. + // + // This used to enumerate three keys -- k3sServerUrl, k3sToken, tlsSans -- and + // silently drop every other one. Anything a caller set beyond those three + // vanished with no error and no warning: k3sVersion (so a pinned version was + // ignored and get.k3s.io installed LATEST), evictionAllowSingleReplica (so + // `labctl app k3s evacuate --allow-single-replica` was a no-op that claimed + // to override a data-loss guard), and the dual-stack fields ipv6 / + // clusterCidr / serviceCidr. + // + // Enumerating is the bug: every field added to K3sConfig has to be + // remembered here, and forgetting produces silence rather than a type error. const config: K3sConfig = { + ...(ctx.config as Partial), hostname: ctx.hostname, ip: ctx.ip, role: ctx.role as K3sConfig["role"], - k3sServerUrl: ctx.config["k3sServerUrl"] as string | undefined, - k3sToken: ctx.config["k3sToken"] as string | undefined, - tlsSans: ctx.config["tlsSans"] as string[] | undefined, }; return { config, @@ -35,7 +49,16 @@ function toOpContext(ctx: ModuleContext): OperationContext { }, os: ctx.os, arch: ctx.arch, - log: (_msg) => { /* collected via results */ }, + // Emit progress LIVE, on stderr. + // + // This used to be a no-op "collected via results", but results only carry + // an operation's final message. Anything an operation reports WHILE it + // runs was therefore thrown away -- including evictLonghornReplicas's + // per-poll "N replica(s) still on ", which is the only visibility + // into an eviction that can legitimately take an hour. When one stalled, + // there was no way to tell which volumes were stuck or whether progress + // was being made at all. + log: (msg) => { console.error(` [${ctx.hostname}] ${msg}`); }, // eslint-disable-line no-console }; } @@ -62,7 +85,7 @@ export class K3sModule implements Module { async install(ctx: ModuleContext): Promise { const start = performance.now(); const opCtx = toOpContext(ctx); - const isServer = ctx.role === "infra" || ctx.role === "labcontroller"; + const isServer = isK3sServer(ctx.role); // Phase 1: Host preparation const prepResults = await runHostPrep(opCtx); @@ -95,6 +118,31 @@ export class K3sModule implements Module { return toModuleResult("configure", results, start); } + /** + * Evacuate the node so it can be wiped. Does NOT reinstall anything -- + * it leaves the node cordoned, drained and empty. + */ + async evacuate(ctx: ModuleContext): Promise { + const start = performance.now(); + const opCtx = toOpContext(ctx); + const results = await runEvacuate(opCtx); + return toModuleResult("evacuate", results, start); + } + + /** + * Return an evacuated node to service: uncordon, clear Longhorn's eviction + * request, restore its default disk. + * + * The counterpart to evacuate(), and the only supported way out of an + * evacuation that was aborted partway. + */ + async admit(ctx: ModuleContext): Promise { + const start = performance.now(); + const opCtx = toOpContext(ctx); + const results = await runAdmit(opCtx); + return toModuleResult("admit", results, start); + } + async health(ctx: ModuleContext): Promise { const start = performance.now(); const opCtx = toOpContext(ctx); diff --git a/bastion/src/modules/modules/k3s/src/operations/etcd-members.ts b/bastion/src/modules/modules/k3s/src/operations/etcd-members.ts new file mode 100644 index 0000000..778684d --- /dev/null +++ b/bastion/src/modules/modules/k3s/src/operations/etcd-members.ts @@ -0,0 +1,60 @@ +// Parsing `etcdctl member list`. +// +// WHY NOT `-w json` +// ----------------- +// Two reasons, and the second is the dangerous one: +// +// 1. `member remove` requires the id in HEX. The JSON writer emits decimal. +// 2. etcd member ids are uint64. JSON.parse turns them into JS numbers, which +// lose precision above 2^53 -- so the id that comes back is an +// APPROXIMATION of the real one. +// +// Observed for real: `member remove 13323819137391910000` -> +// "bad member ID arg (strconv.ParseUint: value out of range), expecting ID in +// Hex". That happens at the worst possible moment, with the node already +// stopped and about to be wiped. Failing loudly is the lucky outcome; a +// mangled id that happened to match another member would remove the WRONG one. +// +// The default writer prints the id as hex TEXT, so it never becomes a number. + +export interface EtcdMember { + /** Raw member name: "-<8 hex>", e.g. "worker0.lab-e26aacc5". */ + name: string; + /** The node name with etcd's random suffix stripped. */ + node: string; + /** Hex id, exactly as etcd prints it — never round-tripped through a number. */ + id: string; + started: boolean; +} + +/** + * etcd names a member `-<8 hex>`. Comparing a raw member name against a + * node name without stripping that suffix silently never matches. + */ +export function etcdNodeName(memberName: string): string { + return memberName.replace(/-[0-9a-f]{8}$/, ""); +} + +/** + * Parse the default `etcdctl member list` writer: + * + * 825852d3683bc406, started, worker2.lab-5f7ad0c9, https://..:2380, https://..:2379, false + * , status , -<8 hex> , peer , client , learner + */ +export function parseEtcdMembers(out: string): EtcdMember[] { + return out.split("\n") + .map((l) => l.trim()) + .filter((l) => l !== "") + .map((line) => { + const cols = line.split(",").map((c) => c.trim()); + const name = cols[2] ?? ""; + return { + name, + node: etcdNodeName(name), + id: cols[0] ?? "", + started: (cols[1] ?? "") === "started", + }; + }) + // Drop anything without a hex id: blank lines, warnings, stray output. + .filter((m) => /^[0-9a-f]+$/i.test(m.id)); +} diff --git a/bastion/src/modules/modules/k3s/src/operations/index.ts b/bastion/src/modules/modules/k3s/src/operations/index.ts index c6616d1..e20d436 100644 --- a/bastion/src/modules/modules/k3s/src/operations/index.ts +++ b/bastion/src/modules/modules/k3s/src/operations/index.ts @@ -24,3 +24,10 @@ export type { RecoverEtcdMemberOptions, RecoverEtcdMemberResult, } from "./etcd-recover.js"; +export { + cordonNode, uncordonNode, drainNode, evictLonghornReplicas, + assertEvictionCompletable, assertNodeEvacuated, checkEvictionCompletable, +} from "./node-evacuate.js"; +export type { EvictionFeasibility } from "./node-evacuate.js"; +export { clearLonghornEviction, assertNodeSchedulable } from "./node-admit.js"; +export { labelWorkerRole } from "./worker-label.js"; diff --git a/bastion/src/modules/modules/k3s/src/operations/k3s-config.ts b/bastion/src/modules/modules/k3s/src/operations/k3s-config.ts index 32d2e7f..f60b73f 100644 --- a/bastion/src/modules/modules/k3s/src/operations/k3s-config.ts +++ b/bastion/src/modules/modules/k3s/src/operations/k3s-config.ts @@ -2,10 +2,7 @@ import type { Operation, OperationResult, K3sConfig } from "../types.js"; import { sshOpts, writeRemoteFile } from "../utils.js"; - -function isServerRole(role: string): boolean { - return role === "infra" || role === "labcontroller"; -} +import { isK3sServer } from "@lab/shared"; /** * The address-family block: `cluster-cidr`, `service-cidr` and `node-ip`. @@ -101,9 +98,28 @@ ${tlsSans.map((s) => ` - "${s}"`).join("\n")} // node being unreachable over v6 while the node itself reads Ready. // It takes no cluster/service CIDRs -- those are server-side only. export function generateAgentConfig(config: K3sConfig): string { + // NO node-role.kubernetes.io/worker LABEL HERE, deliberately. + // + // A kubelet is not permitted to assign itself labels in the + // `node-role.kubernetes.io/` namespace -- the servers run with + // `enable-admission-plugins=NodeRestriction`, which rejects the registration + // outright. The agent then never finishes registering, k3s-agent.service + // never signals readiness, and systemd reports: + // + // Job for k3s-agent.service failed because the service did not take the + // steps required by its unit configuration. + // + // which says nothing about labels and sends you looking at the network, the + // token, or the kubelet's sysctls. Every agent installed through this module + // failed this way; the estate's existing workers only escaped it because + // post-provision installs them with no config.yaml at all and their role + // label was applied separately, server-side. + // + // `node.longhorn.io/...` is fine: it is outside the kubernetes.io namespace, + // so the kubelet may set it. The worker role label is applied after the node + // registers, by labelWorkerRole below. return `${addressFamilyLines(config, { cidrs: false })}protect-kernel-defaults: true node-label: - - "node-role.kubernetes.io/worker=true" - "node.longhorn.io/create-default-disk=config" kubelet-arg: - "protect-kernel-defaults=true" @@ -145,7 +161,7 @@ export const writeK3sConfig: Operation = async (ctx): Promise = await ctx.ssh.exec("mkdir -p /etc/rancher/k3s", sshOpts(ctx)); - const content = isServerRole(ctx.config.role) + const content = isK3sServer(ctx.config.role) ? generateServerConfig(ctx.config) : generateAgentConfig(ctx.config); diff --git a/bastion/src/modules/modules/k3s/src/operations/k3s-install.ts b/bastion/src/modules/modules/k3s/src/operations/k3s-install.ts index 2c274c7..1a3cd86 100644 --- a/bastion/src/modules/modules/k3s/src/operations/k3s-install.ts +++ b/bastion/src/modules/modules/k3s/src/operations/k3s-install.ts @@ -2,13 +2,19 @@ import type { Operation, OperationResult } from "../types.js"; import { sshOpts } from "../utils.js"; - -function isServerRole(role: string): boolean { - return role === "infra" || role === "labcontroller"; -} +import { isK3sServer } from "@lab/shared"; export const installK3sBinary: Operation = async (ctx): Promise => { - const isServer = isServerRole(ctx.config.role); + const isServer = isK3sServer(ctx.config.role); + + // Pin the version when asked. get.k3s.io installs LATEST by default, so a + // node reinstalled during a rolling migration comes back on whatever shipped + // since -- potentially several minor versions ahead of the cluster it is + // about to rejoin, which Kubernetes does not support between control-plane + // members. + const versionEnv = ctx.config.k3sVersion !== undefined && ctx.config.k3sVersion !== "" + ? `INSTALL_K3S_VERSION="${ctx.config.k3sVersion}" ` + : ""; // Check if already installed const version = await ctx.ssh.exec("k3s --version 2>/dev/null", sshOpts(ctx)); @@ -24,12 +30,21 @@ export const installK3sBinary: Operation = async (ctx): Promise ); } - // If joining an existing cluster, pass K3S_URL and K3S_TOKEN - const joinEnv = ctx.config.k3sServerUrl && ctx.config.k3sToken - ? `K3S_URL="${ctx.config.k3sServerUrl}" K3S_TOKEN="${ctx.config.k3sToken}"` - : ""; + // Token and URL are independent. A token WITHOUT a URL seeds the cluster + // token on the cluster-init node, so every later joiner can be given the + // same pre-shared value instead of scraping node-token off the first + // server -- which matters here because that first server gets wiped and + // rejoined during a rolling migration. URL only counts when a token comes + // with it; a URL alone cannot authenticate a join. + // + // The token deliberately does NOT go in config.yaml: writeRemoteFile diffs + // that file, so a token appearing there would restart k3s on every node. + const joinEnv = [ + ctx.config.k3sServerUrl && ctx.config.k3sToken ? `K3S_URL="${ctx.config.k3sServerUrl}"` : "", + ctx.config.k3sToken ? `K3S_TOKEN="${ctx.config.k3sToken}"` : "", + ].filter(Boolean).join(" "); const result = await ctx.ssh.exec( - `curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server" INSTALL_K3S_SKIP_SELINUX_RPM=true ${joinEnv} sh -`, + `curl -sfL https://get.k3s.io | ${versionEnv}INSTALL_K3S_EXEC="server" INSTALL_K3S_SKIP_SELINUX_RPM=true ${joinEnv} sh -`, { timeoutMs: 300_000 }, ); if (result.exitCode !== 0) { @@ -50,7 +65,7 @@ export const installK3sBinary: Operation = async (ctx): Promise }; } const result = await ctx.ssh.exec( - `curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="agent" K3S_URL="${ctx.config.k3sServerUrl}" K3S_TOKEN="${ctx.config.k3sToken}" sh -`, + `curl -sfL https://get.k3s.io | ${versionEnv}INSTALL_K3S_EXEC="agent" K3S_URL="${ctx.config.k3sServerUrl}" K3S_TOKEN="${ctx.config.k3sToken}" sh -`, { timeoutMs: 300_000 }, ); if (result.exitCode !== 0) { diff --git a/bastion/src/modules/modules/k3s/src/operations/longhorn-disk.ts b/bastion/src/modules/modules/k3s/src/operations/longhorn-disk.ts index 59eb8f8..3ae79de 100644 --- a/bastion/src/modules/modules/k3s/src/operations/longhorn-disk.ts +++ b/bastion/src/modules/modules/k3s/src/operations/longhorn-disk.ts @@ -1,50 +1,206 @@ -// Annotate nodes with Longhorn default disk config when /var/lib/longhorn exists. -// The label is set in k3s config (node-label), but the annotation must be applied via kubectl. +// Give a node a Longhorn disk at /var/lib/longhorn. +// +// The cluster runs `createDefaultDiskLabeledNodes: true`, which means Longhorn +// creates a disk ONLY on nodes carrying the label +// `node.longhorn.io/create-default-disk=config` (set via k3s node-label) AND +// the annotation `node.longhorn.io/default-disks-config` (set here). Miss +// either and Longhorn gives the node no disk at all: it joins, goes Ready, +// accepts pods, and silently hosts zero replicas. +// +// WHY THERE IS A REPAIR PATH +// -------------------------- +// Longhorn reads that annotation when it FIRST creates the node's +// `nodes.longhorn.io` record. This operation runs during configure(), which on +// a rejoining node is after the record already exists -- so the annotation +// lands too late and is never read. That is invisible until a rebuild is +// expected and never arrives, roughly forty minutes into a migration. +// +// So: annotate, then read back, and if there is still no disk, write it +// directly into spec.disks. When Longhorn does honour the annotation the +// repair is a no-op. -import type { Operation, OperationResult } from "../types.js"; +import type { Operation, OperationContext, OperationResult } from "../types.js"; import { sshOpts } from "../utils.js"; import { sshExec as remoteSshExec } from "../../../../src/ssh.js"; +const DISK_PATH = "/var/lib/longhorn"; +/** Fixed key so a repeated repair updates one disk rather than adding more. */ +const DISK_KEY = "default-disk-lab"; +const VERIFY_INTERVAL_MS = 5_000; +/** How long to wait for Longhorn to create the node's record after it joins. */ +const REGISTER_TIMEOUT_MS = 180_000; +/** How long to then wait for Longhorn to give that record a disk. */ +const DISK_TIMEOUT_MS = 60_000; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +/** + * Run kubectl from wherever it can be reached: locally on a server, otherwise + * over SSH to the configured server (an agent has no local kubectl). + */ +async function kubectlAnywhere( + ctx: OperationContext, + args: string, +): Promise<{ ok: boolean; out: string }> { + const local = await ctx.ssh.exec(`k3s kubectl ${args} 2>&1`, sshOpts(ctx)); + if (local.exitCode === 0) return { ok: true, out: local.stdout }; + + if (ctx.config.k3sServerUrl) { + try { + const serverHost = new URL(ctx.config.k3sServerUrl).hostname; + const remote = await remoteSshExec(serverHost, "root", `k3s kubectl ${args}`, { + ...(ctx.ssh.keyPath ? { keyPath: ctx.ssh.keyPath } : {}), + timeoutMs: 15_000, + }); + if (remote.exitCode === 0) return { ok: true, out: remote.stdout }; + return { ok: false, out: remote.stdout || remote.stderr }; + } catch { + /* fall through */ + } + } + return { ok: false, out: local.stdout || local.stderr }; +} + +/** Does the node's Longhorn record exist yet? */ +async function nodeRecordExists(ctx: OperationContext, nodeName: string): Promise { + const res = await kubectlAnywhere(ctx, `-n longhorn-system get nodes.longhorn.io "${nodeName}" -o name`); + return res.ok && res.out.trim() !== ""; +} + +/** Does the node's Longhorn record carry a disk at DISK_PATH? */ +async function hasDisk(ctx: OperationContext, nodeName: string): Promise { + const res = await kubectlAnywhere( + ctx, + `-n longhorn-system get nodes.longhorn.io "${nodeName}" ` + + `-o jsonpath='{range .spec.disks.*}{.path}{"\\n"}{end}'`, + ); + if (!res.ok) return false; + return res.out.split("\n").map((l) => l.trim()).includes(DISK_PATH); +} + export const configureLonghornDisk: Operation = async (ctx): Promise => { - // Check if /var/lib/longhorn exists on this node - const check = await ctx.ssh.exec("test -d /var/lib/longhorn && echo yes || echo no", sshOpts(ctx)); + const check = await ctx.ssh.exec(`test -d ${DISK_PATH} && echo yes || echo no`, sshOpts(ctx)); if (check.stdout.trim() !== "yes") { - return { success: true, changed: false, message: "No /var/lib/longhorn directory — skipping Longhorn disk config" }; + return { success: true, changed: false, message: `No ${DISK_PATH} directory — skipping Longhorn disk config` }; } - // Find the node name (hostname as registered in k3s) + // Longhorn writes replicas into /replicas and creates that directory + // only when it FIRST initialises the disk. If the directory later goes away + // while longhorn-disk.cfg remains -- someone clearing space, a partial + // restore, a wipe that preserved the LV -- Longhorn considers the disk + // already initialised and never recreates it. The replica process then dies + // on a non-recursive mkdir: + // + // level=fatal msg="Error running start replica command" + // error="mkdir /host/var/lib/longhorn/replicas/pvc-...: ..." + // + // every single time, for ever, while the node reports Ready and Schedulable + // and its disk looks healthy. Cheap to guarantee, expensive to diagnose. + await ctx.ssh.exec(`mkdir -p ${DISK_PATH}/replicas`, sshOpts(ctx)); + const nodeNameResult = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx)); const nodeName = nodeNameResult.stdout.trim(); - const annotation = JSON.stringify([{ path: "/var/lib/longhorn", allowScheduling: true }]); - - // Try kubectl locally first (works on server nodes) - const result = await ctx.ssh.exec( - `k3s kubectl annotate node "${nodeName}" "node.longhorn.io/default-disks-config=${annotation}" --overwrite 2>&1 || true`, - sshOpts(ctx), + // SINGLE quotes around the whole key=value, not double. + // + // The value is JSON and therefore full of double quotes. Wrapping it in + // double quotes ends the quoted string at the first inner one, and the shell + // strips the rest, so the node gets + // [{path:/var/lib/longhorn,allowScheduling:true}] + // instead of + // [{"path":"/var/lib/longhorn","allowScheduling":true}] + // -- valid-looking, unparseable, and silent. Longhorn then creates NO disk on + // the node, which surfaces much later as "why does nothing schedule here". + // The JSON contains no single quotes, so single-quoting is safe. + const annotation = JSON.stringify([{ path: DISK_PATH, allowScheduling: true }]); + const annotated = await kubectlAnywhere( + ctx, + `annotate node "${nodeName}" 'node.longhorn.io/default-disks-config=${annotation}' --overwrite`, ); - if (result.stdout.includes("annotated") || result.stdout.includes("unchanged")) { - return { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName}` }; + // No Longhorn at all (or no way to reach the API) — nothing more to do. + const lh = await kubectlAnywhere(ctx, "get ns longhorn-system -o name"); + if (!lh.ok) { + return annotated.ok + ? { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName} (Longhorn not installed yet)` } + : { success: true, changed: false, message: "Longhorn disk label set (annotation requires server kubectl)" }; } - // For worker/agent nodes without local kubectl: apply via the server - if (ctx.config.k3sServerUrl) { - // The CLI has SSH access to the server — use sshExec from there - const serverHost = new URL(ctx.config.k3sServerUrl).hostname; - try { - const remoteResult = await remoteSshExec( - serverHost, "root", - `k3s kubectl annotate node "${nodeName}" "node.longhorn.io/default-disks-config=${annotation}" --overwrite`, - { ...(ctx.ssh.keyPath ? { keyPath: ctx.ssh.keyPath } : {}), timeoutMs: 15_000 }, - ); - if (remoteResult.stdout.includes("annotated") || remoteResult.stdout.includes("unchanged")) { - return { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName} (via server)` }; - } - } catch { - // Fall through to manual instruction + // Longhorn registration is ASYNCHRONOUS. On a node that has just joined -- + // which is exactly the case after a wipe-and-rejoin -- the nodes.longhorn.io + // record does not exist yet, so there is nothing to inspect and nothing to + // repair. Waiting a fixed 60s and then giving up meant the repair never ran + // on the one path that needs it most, and the caller's schedulability gate + // then failed immediately with "Longhorn reports no disk". + // + // So: wait for the RECORD first, generously, and only then worry about the + // disk on it. + const registerDeadline = Date.now() + REGISTER_TIMEOUT_MS; + let registered = false; + while (Date.now() < registerDeadline) { + if (await hasDisk(ctx, nodeName)) { + return { success: true, changed: annotated.ok, message: `${nodeName} has a Longhorn disk at ${DISK_PATH}` }; } + if (await nodeRecordExists(ctx, nodeName)) { registered = true; break; } + await sleep(VERIFY_INTERVAL_MS); } - return { success: true, changed: false, message: "Longhorn disk label set (annotation requires server kubectl)" }; + if (!registered) { + return { + success: true, + changed: annotated.ok, + message: `${nodeName} not registered with Longhorn within ${REGISTER_TIMEOUT_MS / 1000}s; ` + + `annotation is in place for when it is`, + }; + } + + // Registered. Give Longhorn a chance to act on the annotation by itself + // before overriding it. + const diskDeadline = Date.now() + DISK_TIMEOUT_MS; + while (Date.now() < diskDeadline) { + if (await hasDisk(ctx, nodeName)) { + return { success: true, changed: annotated.ok, message: `${nodeName} has a Longhorn disk at ${DISK_PATH}` }; + } + await sleep(VERIFY_INTERVAL_MS); + } + + // The record exists and still has no disk: the annotation was applied too + // late to be read. Write it in. + ctx.log(`${nodeName} has no Longhorn disk after annotating; writing spec.disks directly`); + const patch = JSON.stringify({ + spec: { + disks: { + [DISK_KEY]: { + path: DISK_PATH, + allowScheduling: true, + evictionRequested: false, + storageReserved: 0, + tags: [], + diskType: "filesystem", + }, + }, + }, + }); + const repaired = await kubectlAnywhere( + ctx, + `-n longhorn-system patch nodes.longhorn.io "${nodeName}" --type=merge -p '${patch}'`, + ); + if (!repaired.ok) { + return { + success: false, + changed: false, + message: `${nodeName} has no Longhorn disk and could not be repaired`, + error: `${repaired.out}. Without a disk this node will never host a replica.`, + }; + } + + if (await hasDisk(ctx, nodeName)) { + return { success: true, changed: true, message: `Longhorn disk repaired on ${nodeName} (${DISK_PATH})` }; + } + return { + success: false, + changed: true, + message: `Patched spec.disks on ${nodeName} but Longhorn still reports no disk at ${DISK_PATH}`, + error: "Check longhorn-manager logs; the node will not host replicas until this is resolved.", + }; }; diff --git a/bastion/src/modules/modules/k3s/src/operations/node-admit.ts b/bastion/src/modules/modules/k3s/src/operations/node-admit.ts new file mode 100644 index 0000000..0164d09 --- /dev/null +++ b/bastion/src/modules/modules/k3s/src/operations/node-admit.ts @@ -0,0 +1,136 @@ +// Bring a node back into service. The inverse of node-evacuate.ts. +// +// WHY THIS EXISTS +// --------------- +// Evacuation is not symmetric with itself. It leaves three marks on a node -- +// a kubernetes cordon, Longhorn's `allowScheduling:false` + `evictionRequested`, +// and (on a node that was wiped and rejoined) a MISSING default Longhorn disk. +// Undoing a cordon undoes only the first. Without the other two the node comes +// back Ready, accepts pods, and quietly hosts no replicas at all -- which looks +// like success right up until you notice storage never rebalanced onto it. +// +// This is also the only supported way out of an aborted evacuation. + +import type { Operation, OperationContext, OperationResult } from "../types.js"; +import { sshOpts } from "../utils.js"; + +async function nodeName(ctx: OperationContext): Promise { + const r = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx)); + return r.stdout.trim(); +} + +async function kubectl(ctx: OperationContext, args: string): Promise<{ ok: boolean; out: string }> { + const res = await ctx.ssh.exec(`k3s kubectl ${args} 2>&1`, sshOpts(ctx)); + return { ok: res.exitCode === 0, out: res.exitCode === 0 ? res.stdout : (res.stdout || res.stderr) }; +} + +/** + * Clear Longhorn's eviction request and re-enable scheduling. + * + * Both fields in one merge patch, mirroring how eviction sets them: Longhorn's + * validating webhook rejects `evictionRequested:true` on a schedulable node, so + * the pair has to move together in both directions. + */ +export const clearLonghornEviction: Operation = async (ctx): Promise => { + const name = await nodeName(ctx); + + const lh = await kubectl(ctx, "get ns longhorn-system -o name"); + if (!lh.ok) { + return { success: true, changed: false, message: "Longhorn not installed — nothing to clear" }; + } + + const before = await kubectl( + ctx, + `-n longhorn-system get nodes.longhorn.io "${name}" -o jsonpath='{.spec.allowScheduling},{.spec.evictionRequested}'`, + ); + if (!before.ok) { + return { success: true, changed: false, message: `${name} has no Longhorn node record yet` }; + } + if (before.out.trim() === "true,false") { + return { success: true, changed: false, message: `${name} already accepts replicas` }; + } + + const patch = JSON.stringify({ spec: { allowScheduling: true, evictionRequested: false } }); + const res = await kubectl( + ctx, + `-n longhorn-system patch nodes.longhorn.io "${name}" --type=merge -p '${patch}'`, + ); + return res.ok + ? { success: true, changed: true, message: `${name} accepts replicas again` } + : { success: false, changed: false, message: `Could not clear eviction on ${name}`, error: res.out }; +}; + +/** + * Confirm the node can actually host replicas again. + * + * Reads back from Longhorn rather than trusting the patches above, because the + * failure this catches is a node that is Ready and schedulable in kubernetes + * while having no Longhorn disk at all -- the exact state a rejoined node lands + * in when its default-disk annotation was applied too late. + */ +const SCHEDULABLE_TIMEOUT_MS = 120_000; +const SCHEDULABLE_INTERVAL_MS = 5_000; +const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); + +export const assertNodeSchedulable: Operation = async (ctx): Promise => { + // Poll rather than check once. + // + // Longhorn registers a node and attaches its disk ASYNCHRONOUSLY, and this + // runs moments after a wipe-and-rejoin. Checking instantly reports "Longhorn + // reports no disk on this node" for a node that is simply a few seconds from + // being fine -- which is a false alarm on the exact path the gate exists to + // protect. Still fails if it never settles. + const deadline = Date.now() + SCHEDULABLE_TIMEOUT_MS; + let problems: string[] = []; + for (;;) { + problems = await schedulableProblems(ctx); + if (problems.length === 0 || Date.now() >= deadline) break; + await sleep(SCHEDULABLE_INTERVAL_MS); + } + + const name = await nodeName(ctx); + return problems.length === 0 + ? { success: true, changed: false, message: `${name} is back in service` } + : { + success: false, + changed: false, + message: `${name} is NOT fully back in service after ${SCHEDULABLE_TIMEOUT_MS / 1000}s`, + error: problems.join("; "), + }; +}; + +/** One pass of the schedulability checks. Returns the reasons it is not. */ +async function schedulableProblems(ctx: OperationContext): Promise { + const name = await nodeName(ctx); + const problems: string[] = []; + + const sched = await kubectl(ctx, `get node "${name}" -o jsonpath='{.spec.unschedulable}'`); + if (sched.ok && sched.out.trim() === "true") problems.push("node is still cordoned"); + + const lh = await kubectl(ctx, "get ns longhorn-system -o name"); + if (lh.ok) { + const spec = await kubectl( + ctx, + `-n longhorn-system get nodes.longhorn.io "${name}" -o jsonpath='{.spec.allowScheduling},{.spec.evictionRequested}'`, + ); + if (!spec.ok) { + problems.push("no Longhorn node record"); + } else { + const [allow, evicting] = spec.out.trim().split(","); + if (allow !== "true") problems.push("Longhorn scheduling is still disabled"); + if (evicting === "true") problems.push("Longhorn eviction is still requested"); + } + + // A node with no disk hosts no replicas, however healthy it otherwise looks. + const disks = await kubectl( + ctx, + `-n longhorn-system get nodes.longhorn.io "${name}" ` + + `-o jsonpath='{range .status.diskStatus.*}{.conditions[?(@.type=="Schedulable")].status}{"\\n"}{end}'`, + ); + const states = disks.ok ? disks.out.split("\n").map((l) => l.trim()).filter(Boolean) : []; + if (states.length === 0) problems.push("Longhorn reports no disk on this node"); + else if (!states.includes("True")) problems.push(`no schedulable disk (states: ${states.join(",")})`); + } + + return problems; +} diff --git a/bastion/src/modules/modules/k3s/src/operations/node-evacuate.ts b/bastion/src/modules/modules/k3s/src/operations/node-evacuate.ts new file mode 100644 index 0000000..e3ac62b --- /dev/null +++ b/bastion/src/modules/modules/k3s/src/operations/node-evacuate.ts @@ -0,0 +1,547 @@ +// Evacuate a node before it is wiped and reinstalled. +// +// WHY THIS EXISTS +// --------------- +// The kickstart's %pre can preserve data across a reinstall, but ONLY when the +// disk layout is unchanged: it keys off `labvg` already existing with the +// expected LV names. Migrating a machine to the v2 layout means recreating the +// volume group, which takes the `clearpart --all` branch and destroys every +// Longhorn replica on that node. +// +// So a layout migration cannot rely on disk-level preservation. The node has to +// be EVACUATED first -- replicas rebuilt elsewhere, workloads moved, etcd +// membership surrendered -- and then wiped. These operations are the missing +// half; before this file the repo had no drain, no cordon, and no replica +// eviction anywhere. +// +// Ordering matters and is not interchangeable: +// cordon -> check -> evict replicas (WAIT for rebuild) -> drain -> remove etcd member +// +// Two ordering facts, both learned the hard way: +// - Removing the etcd member first costs you the API server you need for the +// rest of the procedure. +// - EVICT BEFORE DRAIN. `drain --force` does not override PodDisruptionBudgets, +// and longhorn-manager keeps a PDB on the instance-manager pod while volumes +// are attached or degraded. Draining a storage node first therefore stops the +// instance-manager that serves the very replicas being evicted: the drain +// blocks for its full timeout, and if it did succeed it would remove the only +// source a single-replica volume could rebuild from. + +import type { Operation, OperationContext, OperationResult } from "../types.js"; +import { sshOpts } from "../utils.js"; + +/** Longhorn's own default; a rebuild of a large replica set can exceed it. */ +const REPLICA_REBUILD_TIMEOUT_MS = 60 * 60_000; +const POLL_INTERVAL_MS = 15_000; +/** How often to report "still waiting" when the replica count is not moving. */ +const HEARTBEAT_INTERVAL_MS = 3 * 60_000; + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +async function nodeName(ctx: OperationContext): Promise { + const r = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx)); + return r.stdout.trim(); +} + +/** + * kubectl, from wherever it is reachable. + * + * An agent has no local kubectl, so the caller must be pointed at a server. + * Returns null when there is no way to reach the API, so callers can report + * "could not do this" instead of silently continuing. + */ +async function kubectl( + ctx: OperationContext, + args: string, + /** + * Override the SSH timeout for this call. + * + * sshOpts() is a flat 30 seconds, which is right for a status read and + * completely wrong for anything that does work. A `kubectl drain` asking for + * --timeout=15m gets killed by the transport at 30s and returns non-zero + * having ALREADY evicted most of the pods -- so it looks like a drain + * failure, with a transcript that reads like success, on every real node. + */ + timeoutMs?: number, +): Promise<{ ok: boolean; out: string }> { + const opts = timeoutMs === undefined ? sshOpts(ctx) : { timeoutMs }; + const local = await ctx.ssh.exec(`k3s kubectl ${args} 2>&1`, opts); + if (local.exitCode === 0) return { ok: true, out: local.stdout }; + return { ok: false, out: local.stdout || local.stderr }; +} + +/** Mark the node unschedulable. Safe and instantly reversible. */ +export const cordonNode: Operation = async (ctx): Promise => { + const name = await nodeName(ctx); + const res = await kubectl(ctx, `cordon "${name}"`); + if (!res.ok) { + return { success: false, changed: false, message: `Could not cordon ${name}`, error: res.out }; + } + const already = res.out.includes("already cordoned"); + return { + success: true, + changed: !already, + message: already ? `${name} was already cordoned` : `Cordoned ${name}`, + }; +}; + +export const uncordonNode: Operation = async (ctx): Promise => { + const name = await nodeName(ctx); + const res = await kubectl(ctx, `uncordon "${name}"`); + return res.ok + ? { success: true, changed: true, message: `Uncordoned ${name}` } + : { success: false, changed: false, message: `Could not uncordon ${name}`, error: res.out }; +}; + +/** + * Move workloads off the node. + * + * --ignore-daemonsets because a DaemonSet pod cannot be evicted by definition; + * --delete-emptydir-data because refusing to delete emptyDir is what makes an + * unattended drain hang forever, and a node about to be wiped is going to lose + * that data regardless. + */ +export const drainNode: Operation = async (ctx): Promise => { + const name = await nodeName(ctx); + // The SSH timeout must outlive kubectl's own --timeout, or the transport + // kills a drain that is working and the result is indistinguishable from a + // drain that failed. + const DRAIN_TIMEOUT_MS = 15 * 60_000; + const res = await kubectl( + ctx, + `drain "${name}" --ignore-daemonsets --delete-emptydir-data --force --timeout=15m`, + DRAIN_TIMEOUT_MS + 60_000, + ); + if (res.ok) return { success: true, changed: true, message: `Drained ${name}` }; + + // Surface the END of the transcript. The first line of a drain is always + // "node/x already cordoned", which tells you nothing and hides the reason. + const tail = res.out.split("\n").map((l) => l.trim()).filter(Boolean).slice(-6).join(" | "); + return { + success: false, + changed: false, + message: `Drain failed on ${name}`, + error: tail === "" ? res.out : `...${tail}`, + }; +}; + +/** kubectl -o json, parsed. Returns null when the read or the parse failed. */ +async function kubectlJson(ctx: OperationContext, args: string): Promise { + const res = await kubectl(ctx, `${args} -o json`); + if (!res.ok) return null; + try { + return JSON.parse(res.out) as T; + } catch { + return null; + } +} + +/** A Longhorn setting's value, or the supplied default when it is unreadable. */ +async function longhornSetting(ctx: OperationContext, name: string, fallback: string): Promise { + const res = await kubectl(ctx, `-n longhorn-system get settings.longhorn.io ${name} -o jsonpath='{.value}'`); + const v = res.ok ? res.out.trim() : ""; + return v === "" ? fallback : v; +} + +interface LhNode { + metadata: { name: string }; + spec: { allowScheduling?: boolean }; + status?: { diskStatus?: Record }; +} +interface LhReplica { + metadata: { name: string }; + spec: { nodeID?: string; volumeName?: string }; +} +interface LhVolume { + metadata: { name: string }; + spec: { numberOfReplicas?: number; size?: number | string }; + status?: { state?: string; robustness?: string }; +} + +export interface EvictionFeasibility { + completable: boolean; + /** Reasons eviction can never finish. Each names the offending volume. */ + blockers: string[]; + /** Things that will work but are dangerous. */ + warnings: string[]; + replicasMoving: number; + bytesMoving: number; +} + +/** + * Can eviction of this node actually COMPLETE? + * + * evictLonghornReplicas waits for the node's replica count to reach zero. If + * there is nowhere for a replica to go, that wait can never end -- it burns the + * full hour-long timeout and then fails, which is an expensive way to discover + * a fact that is knowable in a couple of seconds. + * + * It is knowable because Longhorn's placement rules are simple: + * + * - With strict anti-affinity (replica-soft-anti-affinity=false, the default + * and what this fleet runs) a volume may put at most ONE replica on a node. + * So a volume wanting N replicas needs N schedulable nodes OTHER than the + * one being evacuated. A 3-replica volume on a 3-node cluster is therefore + * unevictable -- the two survivors already hold a copy each and cannot take + * the third. + * - The scheduler admits a replica against over-provisioned space, but the + * rebuild writes real bytes to a real disk. Those are different numbers, + * and at 500% over-provisioning the first can pass while the second fails. + * + * Read-only, a handful of API calls, finishes in seconds. + */ +export async function checkEvictionCompletable( + ctx: OperationContext, + name: string, +): Promise { + const blockers: string[] = []; + const warnings: string[] = []; + + const nodes = await kubectlJson<{ items: LhNode[] }>(ctx, "-n longhorn-system get nodes.longhorn.io"); + const replicas = await kubectlJson<{ items: LhReplica[] }>(ctx, "-n longhorn-system get replicas.longhorn.io"); + const volumes = await kubectlJson<{ items: LhVolume[] }>(ctx, "-n longhorn-system get volumes.longhorn.io"); + if (!nodes || !replicas || !volumes) { + return { + completable: false, + blockers: ["Could not read Longhorn nodes/replicas/volumes — refusing to guess"], + warnings, replicasMoving: 0, bytesMoving: 0, + }; + } + + const softAntiAffinity = (await longhornSetting(ctx, "replica-soft-anti-affinity", "false")) === "true"; + const overProvisionPct = Number(await longhornSetting(ctx, "storage-over-provisioning-percentage", "100")) || 100; + + const mine = replicas.items.filter((r) => r.spec.nodeID === name); + if (mine.length === 0) { + return { completable: true, blockers, warnings, replicasMoving: 0, bytesMoving: 0 }; + } + + const volumeByName = new Map(volumes.items.map((v) => [v.metadata.name, v])); + // Candidates are schedulable Longhorn nodes that are not the one leaving. + const candidates = nodes.items.filter( + (n) => n.metadata.name !== name && n.spec.allowScheduling !== false, + ); + + const diskFree = (n: LhNode): number => + Object.values(n.status?.diskStatus ?? {}).reduce((a, d) => a + (d.storageAvailable ?? 0), 0); + const schedulableFree = (n: LhNode): number => + Object.values(n.status?.diskStatus ?? {}).reduce((a, d) => { + const usable = ((d.storageMaximum ?? 0) - (d.storageReserved ?? 0)) * (overProvisionPct / 100); + return a + Math.max(0, usable - (d.storageScheduled ?? 0)); + }, 0); + + let bytesMoving = 0; + + for (const rep of mine) { + const volName = rep.spec.volumeName ?? ""; + const vol = volumeByName.get(volName); + if (!vol) { + warnings.push(`replica ${rep.metadata.name} has no volume ${volName} — orphan?`); + continue; + } + const want = vol.spec.numberOfReplicas ?? 1; + const size = Number(vol.spec.size ?? 0); + bytesMoving += size; + + // (1) Placement. Strict anti-affinity means one replica per node per volume. + if (!softAntiAffinity && candidates.length < want) { + blockers.push( + `volume ${volName} wants ${want} replicas but only ${candidates.length} other schedulable ` + + `node(s) exist — with strict anti-affinity its replica cannot be placed. Add a schedulable node.`, + ); + continue; + } + + // (2) Scheduler capacity, at the configured over-provisioning percentage. + const schedulableFit = candidates.filter((n) => schedulableFree(n) >= size); + if (schedulableFit.length === 0) { + blockers.push( + `volume ${volName} (${gib(size)}) does not fit the scheduler on any candidate node ` + + `even at ${overProvisionPct}% over-provisioning.`, + ); + continue; + } + + // (3) Real capacity. The scheduler counts over-provisioned space; a rebuild + // writes real bytes. This is the check that catches a small disk hiding + // behind a generous over-provisioning percentage. + const realFit = candidates.filter((n) => diskFree(n) >= size); + if (realFit.length === 0) { + const best = Math.max(0, ...candidates.map(diskFree)); + blockers.push( + `volume ${volName} needs ${gib(size)} but the roomiest candidate node has ${gib(best)} ` + + `actually free. storage-over-provisioning-percentage=${overProvisionPct} lets the scheduler ` + + `accept this; the disk does not.`, + ); + continue; + } + + // (4) Volumes with no second copy, and volumes that cannot rebuild at all. + if (want === 1) { + warnings.push( + `volume ${volName} has a single replica — evicting it rebuilds from the only copy that exists`, + ); + } + if (vol.status?.state === "detached") { + warnings.push(`volume ${volName} is detached — Longhorn must attach it to rebuild`); + } + if (vol.status?.robustness === "degraded") { + warnings.push(`volume ${volName} is already degraded before eviction starts`); + } + } + + return { completable: blockers.length === 0, blockers, warnings, replicasMoving: mine.length, bytesMoving }; +} + +function gib(bytes: number): string { + return `${(bytes / 2 ** 30).toFixed(1)} GiB`; +} + +/** + * Standalone gate: refuse fast and loudly rather than hang for an hour. + * + * Exposed as its own Operation so it can be run on its own -- "could I evacuate + * this node if I wanted to?" is a question worth answering without starting. + */ +export const assertEvictionCompletable: Operation = async (ctx): Promise => { + const name = await nodeName(ctx); + const lh = await kubectl(ctx, "get ns longhorn-system -o name"); + if (!lh.ok) { + return { success: true, changed: false, message: "Longhorn not installed — nothing to evict" }; + } + + const f = await checkEvictionCompletable(ctx, name); + for (const w of f.warnings) ctx.log(` WARNING: ${w}`); + + if (!f.completable) { + return { + success: false, + changed: false, + message: `Eviction of ${name} cannot complete`, + error: f.blockers.join("; "), + }; + } + return { + success: true, + changed: false, + message: f.replicasMoving === 0 + ? `${name} holds no replicas` + : `${f.replicasMoving} replica(s) (${gib(f.bytesMoving)}) can be placed elsewhere`, + }; +}; + +/** + * Ask Longhorn to move this node's replicas elsewhere, then WAIT. + * + * The wait is the entire point. Setting evictionRequested and immediately + * wiping the disk destroys the replicas mid-rebuild, which on a 3-replica + * volume takes you to 1 and on the 2-replica Prometheus volume takes you to 0. + * + * Returns changed:false with an explanation when Longhorn is not installed, + * so a standalone or storage-free node is a no-op rather than an error. + */ +export const evictLonghornReplicas: Operation = async (ctx): Promise => { + const name = await nodeName(ctx); + + const hasLonghorn = await kubectl(ctx, "get ns longhorn-system -o name"); + if (!hasLonghorn.ok) { + return { success: true, changed: false, message: "Longhorn not installed — nothing to evict" }; + } + + // Refuse in seconds rather than discover it after an hour of polling. + // Inside the operation, not only in the group, so `labctl app k3s evacuate` + // gets the same protection as the drill does. + const feasible = await checkEvictionCompletable(ctx, name); + for (const w of feasible.warnings) ctx.log(` WARNING: ${w}`); + if (!feasible.completable) { + return { + success: false, + changed: false, + message: `Eviction of ${name} cannot complete — not starting it`, + error: feasible.blockers.join("; "), + }; + } + // A single-replica volume rebuilds from the only copy in existence, and this + // fleet has no backup target. That is a decision for a human, not a default. + const singles = feasible.warnings.filter((w) => w.includes("single replica")); + if (singles.length > 0 && ctx.config.evictionAllowSingleReplica !== true) { + return { + success: false, + changed: false, + message: `${name} holds single-replica volume(s); refusing to evict`, + error: `${singles.join("; ")}. Raise them to 3 replicas and wait for healthy, ` + + `or pass evictionAllowSingleReplica to accept the risk.`, + }; + } + + const patch = JSON.stringify({ + spec: { allowScheduling: false, evictionRequested: true }, + }); + const req = await kubectl( + ctx, + `-n longhorn-system patch nodes.longhorn.io "${name}" --type=merge -p '${patch}'`, + ); + if (!req.ok) { + return { success: false, changed: false, message: `Could not request eviction for ${name}`, error: req.out }; + } + + ctx.log(`Eviction requested for ${name}; waiting for replicas to rebuild elsewhere...`); + const start = Date.now(); + let last = -1; + let lastHeartbeat = Date.now(); + try { + while (Date.now() - start < REPLICA_REBUILD_TIMEOUT_MS) { + const res = await kubectl( + ctx, + `-n longhorn-system get replicas.longhorn.io -o jsonpath='{range .items[?(@.spec.nodeID=="${name}")]}{.metadata.name}{"\\n"}{end}'`, + ); + const remaining = res.ok ? res.out.split("\n").map((l) => l.trim()).filter(Boolean).length : -1; + if (remaining === 0) { + return { + success: true, + changed: true, + message: `All replicas evicted from ${name} after ${Math.round((Date.now() - start) / 1000)}s`, + }; + } + // Log on change, and otherwise every few minutes. A silent hour is + // indistinguishable from a hung process; a heartbeat that carries the + // elapsed time makes "stalled" visible while it is happening rather than + // only in the timeout message. + const elapsedS = Math.round((Date.now() - start) / 1000); + if (remaining !== last) { + ctx.log(` ${remaining} replica(s) still on ${name} (${elapsedS}s elapsed)`); + last = remaining; + lastHeartbeat = Date.now(); + } else if (Date.now() - lastHeartbeat >= HEARTBEAT_INTERVAL_MS) { + ctx.log(` still waiting: ${remaining} replica(s) on ${name}, no change in ` + + `${Math.round((Date.now() - lastHeartbeat) / 60_000)} min (${elapsedS}s elapsed)`); + lastHeartbeat = Date.now(); + } + await sleep(POLL_INTERVAL_MS); + } + } catch (err) { + // Never leave the node eviction-pending because of a transient failure. + await cancelEviction(ctx, name); + throw err; + } + + // Timed out. Say exactly what is stuck before restoring the node, because + // after the cancel the eviction flags are gone and the evidence with them. + const stuck = await describeStuckReplicas(ctx, name); + + // Put the node back the way we found it: an abandoned eviction leaves it + // unschedulable with Longhorn still trying to drain it, which is safe but + // sticky, and nothing else would ever clear it. + const cancelled = await cancelEviction(ctx, name); + return { + success: false, + changed: true, + message: `Timed out waiting for replica eviction from ${name}`, + error: `${last} replica(s) still present after ${REPLICA_REBUILD_TIMEOUT_MS / 60000} min. ` + + `Wiping now would destroy them.\nStill on ${name}:\n${stuck}\n` + + (cancelled + ? `Eviction request has been cancelled and scheduling restored.` + : `WARNING: could not cancel the eviction request — clear it with ` + + `\`labctl app k3s admit ${name}\`.`), + }; +}; + +/** + * Describe the replicas still sitting on a node, and why they are not moving. + * + * An eviction that times out after an hour used to report only a count. That + * is not enough to act on: the question is always WHICH volume is stuck and + * what state its replica is in. A replica with `desireState=running` and + * `currentState=stopped` and no `failedAt` is Longhorn wanting to start it and + * not doing so -- a very different problem from a volume that cannot be placed. + */ +async function describeStuckReplicas(ctx: OperationContext, name: string): Promise { + const reps = await kubectlJson<{ items: { + metadata: { name: string }; + spec: { nodeID?: string; volumeName?: string; failedAt?: string; desireState?: string }; + status?: { currentState?: string }; + }[] }>(ctx, "-n longhorn-system get replicas.longhorn.io"); + if (!reps) return " (could not read replicas)"; + + const mine = reps.items.filter((r) => r.spec.nodeID === name); + if (mine.length === 0) return " (none)"; + + const vols = await kubectlJson<{ items: { + metadata: { name: string }; + spec: { numberOfReplicas?: number }; + status?: { state?: string; robustness?: string }; + }[] }>(ctx, "-n longhorn-system get volumes.longhorn.io"); + const byName = new Map((vols?.items ?? []).map((v) => [v.metadata.name, v])); + + return mine.map((r) => { + const v = byName.get(r.spec.volumeName ?? ""); + return ` ${r.spec.volumeName ?? "?"}: replica ${r.metadata.name.slice(-12)} ` + + `currentState=${r.status?.currentState ?? "?"} desireState=${r.spec.desireState ?? "?"} ` + + `failedAt=${r.spec.failedAt === undefined || r.spec.failedAt === "" ? "(none)" : r.spec.failedAt} | ` + + `volume ${v?.status?.state ?? "?"}/${v?.status?.robustness ?? "?"} ` + + `wants ${v?.spec.numberOfReplicas ?? "?"} replicas`; + }).join("\n"); +} + +/** + * Undo the eviction request. Returns false if it could not be undone, which the + * caller must surface -- silently failing here leaves a node nobody can use. + */ +async function cancelEviction(ctx: OperationContext, name: string): Promise { + const patch = JSON.stringify({ spec: { allowScheduling: true, evictionRequested: false } }); + const res = await kubectl( + ctx, + `-n longhorn-system patch nodes.longhorn.io "${name}" --type=merge -p '${patch}'`, + ); + if (res.ok) ctx.log(`Cancelled eviction request on ${name} and restored scheduling`); + return res.ok; +} + +/** + * Verify the node holds nothing before it is wiped. + * + * A separate, cheap operation on purpose: it is the gate that turns "we ran the + * eviction steps" into "the node is actually empty", and it is the one thing + * worth asserting twice. Reading back from the thing that does the work, not + * from the request we sent it. + */ +export const assertNodeEvacuated: Operation = async (ctx): Promise => { + const name = await nodeName(ctx); + const problems: string[] = []; + + const sched = await kubectl(ctx, `get node "${name}" -o jsonpath='{.spec.unschedulable}'`); + if (sched.ok && sched.out.trim() !== "true") problems.push("node is still schedulable"); + + const pods = await kubectl( + ctx, + `get pods -A --field-selector spec.nodeName="${name}",status.phase=Running ` + + `-o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\\n"}{end}'`, + ); + if (pods.ok) { + const running = pods.out.split("\n").map((l) => l.trim()).filter(Boolean); + // DaemonSet pods legitimately remain; anything else means the drain lied. + if (running.length > 0) ctx.log(` ${running.length} pod(s) remain (expected: DaemonSets only)`); + } + + const lh = await kubectl(ctx, "get ns longhorn-system -o name"); + if (lh.ok) { + const reps = await kubectl( + ctx, + `-n longhorn-system get replicas.longhorn.io -o jsonpath='{range .items[?(@.spec.nodeID=="${name}")]}{.metadata.name}{"\\n"}{end}'`, + ); + const n = reps.ok ? reps.out.split("\n").map((l) => l.trim()).filter(Boolean).length : 0; + if (n > 0) problems.push(`${n} Longhorn replica(s) still on this node`); + } + + return problems.length === 0 + ? { success: true, changed: false, message: `${name} is evacuated and safe to wipe` } + : { + success: false, + changed: false, + message: `${name} is NOT safe to wipe`, + error: problems.join("; "), + }; +}; diff --git a/bastion/src/modules/modules/k3s/src/operations/worker-label.ts b/bastion/src/modules/modules/k3s/src/operations/worker-label.ts new file mode 100644 index 0000000..e82bfa3 --- /dev/null +++ b/bastion/src/modules/modules/k3s/src/operations/worker-label.ts @@ -0,0 +1,66 @@ +// Apply the worker role label from the SERVER side, after the node registers. +// +// WHY THIS IS NOT A NODE-LABEL +// ---------------------------- +// `node-role.kubernetes.io/worker=true` cannot be set by the kubelet itself. +// The servers enable the NodeRestriction admission plugin, which forbids a +// kubelet from assigning itself any label in the `node-role.kubernetes.io/` +// namespace. Putting it in the agent's `node-label:` list makes registration +// fail, and the resulting message -- +// +// Job for k3s-agent.service failed because the service did not take the +// steps required by its unit configuration. +// +// names neither labels nor admission, so it reads as a networking, token or +// sysctl problem. It is none of those. +// +// The label is cosmetic (it populates the ROLES column in `kubectl get nodes`) +// so failing to apply it is a warning, never an error. + +import type { Operation, OperationContext, OperationResult } from "../types.js"; +import { sshOpts } from "../utils.js"; +import { sshExec as remoteSshExec } from "../../../../src/ssh.js"; +import { isK3sServer } from "@lab/shared"; + +const WORKER_LABEL = "node-role.kubernetes.io/worker=true"; + +async function nodeName(ctx: OperationContext): Promise { + const r = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx)); + return r.stdout.trim(); +} + +export const labelWorkerRole: Operation = async (ctx): Promise => { + if (isK3sServer(ctx.config.role)) { + return { success: true, changed: false, message: "Server node — k3s sets its own roles" }; + } + + const name = await nodeName(ctx); + + // An agent has no local kubectl, so this has to run on a server. + const local = await ctx.ssh.exec(`k3s kubectl label node "${name}" ${WORKER_LABEL} --overwrite 2>&1`, sshOpts(ctx)); + if (local.exitCode === 0) { + return { success: true, changed: true, message: `Labelled ${name} as worker` }; + } + + if (ctx.config.k3sServerUrl !== undefined && ctx.config.k3sServerUrl !== "") { + try { + const serverHost = new URL(ctx.config.k3sServerUrl).hostname; + const remote = await remoteSshExec( + serverHost, "root", + `k3s kubectl label node "${name}" ${WORKER_LABEL} --overwrite`, + { ...(ctx.ssh.keyPath ? { keyPath: ctx.ssh.keyPath } : {}), timeoutMs: 30_000 }, + ); + if (remote.exitCode === 0) { + return { success: true, changed: true, message: `Labelled ${name} as worker (via server)` }; + } + } catch { /* fall through to the warning */ } + } + + // Cosmetic: the ROLES column will read . Not worth failing an install. + return { + success: true, + changed: false, + message: `Could not apply the worker role label to ${name} (cosmetic; set it with ` + + `\`kubectl label node ${name} ${WORKER_LABEL}\`)`, + }; +}; diff --git a/bastion/src/modules/modules/k3s/src/types.ts b/bastion/src/modules/modules/k3s/src/types.ts index 1b000fd..370884d 100644 --- a/bastion/src/modules/modules/k3s/src/types.ts +++ b/bastion/src/modules/modules/k3s/src/types.ts @@ -17,6 +17,29 @@ export interface K3sConfig { // Additional TLS SANs for API server certificate tlsSans?: string[] | undefined; + /** + * Exact k3s version to install, e.g. "v1.34.6+k3s1". + * + * Unset means whatever get.k3s.io currently calls latest, which is how a + * reinstalled node silently ends up AHEAD of the cluster it is rejoining. + * Kubernetes supports only one minor version of skew between control-plane + * members; a node reinstalled two minors ahead of its peers is an + * unsupported cluster, and a rolling migration reinstalls nodes one at a + * time by design. Pin it to the version the cluster already runs. + */ + k3sVersion?: string | undefined; + + /** + * Permit evacuating a node that holds single-replica Longhorn volumes. + * + * Off by default and deliberately awkward. Evicting the only replica of a + * volume rebuilds it from the sole copy in existence; with no backup target + * configured, a failure during that rebuild is permanent data loss. Raising + * the volume to 3 replicas and waiting for it to go healthy is almost always + * the right move instead. + */ + evictionAllowSingleReplica?: boolean | undefined; + /** * IPv6 address of this node on the cluster VLAN. Its PRESENCE is what makes a * node dual-stack: supply it and `node-ip` is emitted as `,`; omit it diff --git a/bastion/src/modules/modules/k3s/tests/agent-node-label.test.ts b/bastion/src/modules/modules/k3s/tests/agent-node-label.test.ts new file mode 100644 index 0000000..0a87995 --- /dev/null +++ b/bastion/src/modules/modules/k3s/tests/agent-node-label.test.ts @@ -0,0 +1,75 @@ +// The agent must NOT try to self-assign a node-role label. +// +// A kubelet may not set labels in the `node-role.kubernetes.io/` namespace; +// the servers run NodeRestriction, which rejects the registration. The agent +// then never finishes registering and systemd reports only: +// +// Job for k3s-agent.service failed because the service did not take the +// steps required by its unit configuration. +// +// That message names neither labels nor admission control, so the cost of +// reintroducing this is hours of looking at the network and the join token. +// Observed twice for real: on v2-wrk-1, and on the drill's storage agent. + +import { describe, it, expect } from "vitest"; +import { generateAgentConfig, generateServerConfig } from "../src/operations/k3s-config.js"; +import { k3sAgentGroup } from "../src/groups/k3s-agent.js"; +import { mockCtx, OK, stdout, expectCommand } from "./helpers.js"; +import { labelWorkerRole } from "../src/operations/worker-label.js"; + +const base = { hostname: "w1.lab", ip: "10.0.0.9", role: "k8s-worker" as const }; + +describe("agent config", () => { + it("does NOT put a node-role label in node-label", () => { + const cfg = generateAgentConfig({ ...base }); + expect(cfg).not.toContain("node-role.kubernetes.io"); + }); + + it("keeps the Longhorn label, which IS allowed (outside kubernetes.io)", () => { + expect(generateAgentConfig({ ...base })).toContain("node.longhorn.io/create-default-disk=config"); + }); + + it("server config does not set node-role either — k3s assigns those itself", () => { + const cfg = generateServerConfig({ ...base, role: "k8s-combo", hostname: "s1.lab" }); + expect(cfg).not.toContain("node-role.kubernetes.io"); + }); + + it("the agent group applies the worker label AFTER installing", () => { + const names = k3sAgentGroup.operations.map((o) => o.name); + const install = names.findIndex((n) => /Install k3s agent/i.test(n)); + const label = names.findIndex((n) => /Label node as worker/i.test(n)); + expect(install).toBeGreaterThanOrEqual(0); + expect(label).toBeGreaterThan(install); + }); +}); + +describe("labelWorkerRole", () => { + it("labels an agent from the server side", async () => { + const ctx = mockCtx({ role: "k8s-worker" }); + ctx.ssh.exec.mockImplementation(async (cmd: string) => { + if (cmd.startsWith("hostname")) return stdout("w1.lab\n"); + return OK; + }); + const r = await labelWorkerRole(ctx); + expect(r.success).toBe(true); + expectCommand(ctx.ssh, /label node "w1\.lab" node-role\.kubernetes\.io\/worker=true/); + }); + + it("is a no-op on a server — k3s sets control-plane/etcd itself", async () => { + const r = await labelWorkerRole(mockCtx({ role: "k8s-combo" })); + expect(r.success).toBe(true); + expect(r.changed).toBe(false); + }); + + it("never fails an install over a cosmetic label", async () => { + const ctx = mockCtx({ role: "k8s-worker" }); + ctx.ssh.exec.mockImplementation(async (cmd: string) => { + if (cmd.startsWith("hostname")) return stdout("w1.lab\n"); + return { exitCode: 1, stdout: "", stderr: "no kubectl here" }; + }); + const r = await labelWorkerRole(ctx); + expect(r.success, "a missing ROLES column must not fail a node install").toBe(true); + expect(r.changed).toBe(false); + expect(r.message).toMatch(/cosmetic/); + }); +}); diff --git a/bastion/src/modules/modules/k3s/tests/etcd-member-parse.test.ts b/bastion/src/modules/modules/k3s/tests/etcd-member-parse.test.ts new file mode 100644 index 0000000..61c78e0 --- /dev/null +++ b/bastion/src/modules/modules/k3s/tests/etcd-member-parse.test.ts @@ -0,0 +1,58 @@ +// Parsing `etcdctl member list`. +// +// WHY NOT -w json +// --------------- +// etcd member IDs are uint64. JSON.parse turns them into JS numbers, which +// lose precision above 2^53, so the id that comes back is an APPROXIMATION of +// the real one -- and `member remove` wants HEX anyway, while the JSON writer +// emits decimal. Observed for real: `member remove 13323819137391910000` +// -> "bad member ID arg (value out of range), expecting ID in Hex". +// +// That happens at the worst possible moment: the node is already stopped and +// about to be wiped. A mangled id fails loudly if you are lucky; in principle +// it could match a different member, which would remove the wrong one. +// +// The default writer prints the id as hex TEXT, so it never becomes a number. +// The fixture below is real output from a production node. + +import { describe, it, expect } from "vitest"; +import { parseEtcdMembers } from "../src/operations/etcd-members.js"; + +const REAL = `825852d3683bc406, started, worker2-k8s0.ad.itaz.eu-5f7ad0c9, https://192.168.8.25:2380, https://192.168.8.25:2379, false +a6170c0755307cea, started, worker1-k8s0.ad.itaz.eu-82b4fe24, https://192.168.8.13:2380, https://192.168.8.13:2379, false +ad1fe97d38022a78, started, worker0-k8s0.ad.itaz.eu-e26aacc5, https://192.168.8.23:2380, https://192.168.8.23:2379, false`; + +describe("parseEtcdMembers", () => { + it("keeps ids as exact hex text, never as numbers", () => { + const m = parseEtcdMembers(REAL); + expect(m.map((x) => x.id)).toEqual([ + "825852d3683bc406", "a6170c0755307cea", "ad1fe97d38022a78", + ]); + // The precision trap, stated as an assertion: every one of these exceeds + // Number.MAX_SAFE_INTEGER, so a numeric round-trip would corrupt them. + for (const x of m) { + expect(Number(BigInt(`0x${x.id}`)) > Number.MAX_SAFE_INTEGER).toBe(true); + expect(BigInt(`0x${x.id}`).toString(16)).toBe(x.id); + } + }); + + it("strips etcd's random name suffix to give the node name", () => { + expect(parseEtcdMembers(REAL).map((x) => x.node)).toEqual([ + "worker2-k8s0.ad.itaz.eu", "worker1-k8s0.ad.itaz.eu", "worker0-k8s0.ad.itaz.eu", + ]); + }); + + it("reports started status", () => { + expect(parseEtcdMembers(REAL).every((x) => x.started)).toBe(true); + const unstarted = parseEtcdMembers("abc123, unstarted, , https://x:2380, , false"); + expect(unstarted[0]?.started).toBe(false); + }); + + it("ignores blank lines and anything without a hex id", () => { + expect(parseEtcdMembers(`\n \nWARNING: something\n${REAL}\n`)).toHaveLength(3); + }); + + it("returns nothing for empty output rather than a bogus member", () => { + expect(parseEtcdMembers("")).toEqual([]); + }); +}); diff --git a/bastion/src/modules/modules/k3s/tests/join-env.test.ts b/bastion/src/modules/modules/k3s/tests/join-env.test.ts new file mode 100644 index 0000000..11841f3 --- /dev/null +++ b/bastion/src/modules/modules/k3s/tests/join-env.test.ts @@ -0,0 +1,158 @@ +// Tests for how K3S_URL / K3S_TOKEN reach the k3s installer. +// +// Token and URL are independent, and the distinction is load-bearing: +// +// token only -> SEEDS the cluster token on the cluster-init node, so +// every later joiner can use the same pre-shared value. +// url + token -> JOINS an existing cluster. +// url only -> cannot authenticate; must not be passed. +// +// Before this was fixed the installer emitted join env ONLY when both were +// present, so a cluster-init node always self-generated its token and there +// was no way to pre-share one. A `toContain("K3S_TOKEN")` assertion would pass +// against the broken version for the url+token case, so each case is asserted +// on the exact command. + +import { describe, it, expect } from "vitest"; +import { mockCtx, stdout, OK } from "./helpers.js"; +import { installK3sBinary } from "../src/operations/k3s-install.js"; + +/** The `curl ... | sh -` line the operation ran. */ +function installCommand(ssh: ReturnType["ssh"]): string { + const calls = ssh.exec.mock.calls.map((c: [string, unknown?]) => c[0]); + const found = calls.find((c: string) => c.includes("get.k3s.io")); + if (!found) throw new Error(`No install command issued. Commands:\n${calls.join("\n")}`); + return found; +} + +/** k3s --version fails (not installed), everything after succeeds. */ +function primeServerInstall(ctx: ReturnType): void { + ctx.ssh.exec.mockResolvedValue(OK); + ctx.ssh.exec.mockResolvedValueOnce({ exitCode: 1, stdout: "", stderr: "" }); // k3s --version +} + +describe("installK3sBinary — server join environment", () => { + it("token alone seeds the cluster token and does NOT pass a server URL", async () => { + const ctx = mockCtx({ role: "k8s-combo", k3sToken: "pre-shared-token" }); + primeServerInstall(ctx); + ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready")); + + await installK3sBinary(ctx); + const cmd = installCommand(ctx.ssh); + + expect(cmd).toContain('K3S_TOKEN="pre-shared-token"'); + expect(cmd).not.toContain("K3S_URL"); + expect(cmd).toContain('INSTALL_K3S_EXEC="server"'); + }); + + it("url + token joins an existing cluster", async () => { + const ctx = mockCtx({ + role: "k8s-combo", + k3sServerUrl: "https://10.0.0.11:6443", + k3sToken: "pre-shared-token", + }); + primeServerInstall(ctx); + ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready")); + + await installK3sBinary(ctx); + const cmd = installCommand(ctx.ssh); + + expect(cmd).toContain('K3S_URL="https://10.0.0.11:6443"'); + expect(cmd).toContain('K3S_TOKEN="pre-shared-token"'); + }); + + it("neither url nor token passes no join environment at all", async () => { + const ctx = mockCtx({ role: "k8s-combo" }); + primeServerInstall(ctx); + ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready")); + + await installK3sBinary(ctx); + const cmd = installCommand(ctx.ssh); + + expect(cmd).not.toContain("K3S_URL"); + expect(cmd).not.toContain("K3S_TOKEN"); + }); + + it("a URL without a token is ignored — it cannot authenticate a join", async () => { + const ctx = mockCtx({ role: "k8s-combo", k3sServerUrl: "https://10.0.0.11:6443" }); + primeServerInstall(ctx); + ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready")); + + await installK3sBinary(ctx); + expect(installCommand(ctx.ssh)).not.toContain("K3S_URL"); + }); +}); + +describe("installK3sBinary — datastore wipe guard", () => { + // Wiping server/{tls,cred,db} is correct when JOINING (stale certs are fatal: + // "newer than datastore") and catastrophic on a cluster-init node, where it + // would destroy the etcd database the node is about to bootstrap from. + const WIPE = "rm -rf /var/lib/rancher/k3s/server/tls"; + + it("wipes stale server state when joining", async () => { + const ctx = mockCtx({ + role: "k8s-combo", + k3sServerUrl: "https://10.0.0.11:6443", + k3sToken: "t", + }); + primeServerInstall(ctx); + ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready")); + + await installK3sBinary(ctx); + const calls = ctx.ssh.exec.mock.calls.map((c: [string, unknown?]) => c[0]); + expect(calls.some((c: string) => c.includes(WIPE))).toBe(true); + }); + + it("does NOT wipe the datastore when only seeding a token (cluster-init)", async () => { + const ctx = mockCtx({ role: "k8s-combo", k3sToken: "pre-shared-token" }); + primeServerInstall(ctx); + ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready")); + + await installK3sBinary(ctx); + const calls = ctx.ssh.exec.mock.calls.map((c: [string, unknown?]) => c[0]); + expect(calls.some((c: string) => c.includes(WIPE))).toBe(false); + }); +}); + +describe("installK3sBinary — agent", () => { + it("refuses to install an agent with no server URL or token", async () => { + const ctx = mockCtx({ role: "k8s-worker" }); + primeServerInstall(ctx); + + const result = await installK3sBinary(ctx); + expect(result.success).toBe(false); + expect(result.message).toContain("Agent requires"); + }); +}); + +describe("installK3sBinary — version pinning", () => { + // get.k3s.io installs LATEST. During a rolling migration that means a + // reinstalled node rejoins on a newer k3s than its peers; Kubernetes supports + // only one minor of skew between control-plane members, and at the time of + // writing latest was TWO ahead of the fleet (v1.36.4 vs v1.34.6). + it("pins the version when one is given", async () => { + const ctx = mockCtx({ role: "k8s-combo", k3sToken: "t", k3sVersion: "v1.34.6+k3s1" }); + primeServerInstall(ctx); + ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready")); + await installK3sBinary(ctx); + expect(installCommand(ctx.ssh)).toContain('INSTALL_K3S_VERSION="v1.34.6+k3s1"'); + }); + + it("pins the agent install too", async () => { + const ctx = mockCtx({ + role: "k8s-worker", k3sServerUrl: "https://10.0.0.11:6443", + k3sToken: "t", k3sVersion: "v1.34.6+k3s1", + }); + primeServerInstall(ctx); + await installK3sBinary(ctx); + expect(installCommand(ctx.ssh)).toContain('INSTALL_K3S_VERSION="v1.34.6+k3s1"'); + }); + + it("omits the variable entirely when unset, preserving today's behaviour", async () => { + const ctx = mockCtx({ role: "k8s-combo", k3sToken: "t" }); + primeServerInstall(ctx); + ctx.ssh.exec.mockResolvedValue(stdout("NAME STATUS\nnode Ready")); + await installK3sBinary(ctx); + expect(installCommand(ctx.ssh)).not.toContain("INSTALL_K3S_VERSION"); + }); +}); diff --git a/bastion/src/modules/modules/k3s/tests/longhorn-disk.test.ts b/bastion/src/modules/modules/k3s/tests/longhorn-disk.test.ts new file mode 100644 index 0000000..a837209 --- /dev/null +++ b/bastion/src/modules/modules/k3s/tests/longhorn-disk.test.ts @@ -0,0 +1,188 @@ +// Tests for giving a node its Longhorn disk. +// +// The cluster runs createDefaultDiskLabeledNodes:true, so a node gets a disk +// only if it carries BOTH the create-default-disk label and the +// default-disks-config annotation. Longhorn reads that annotation when it +// first creates the node's record — which means on a node that is REJOINING, +// the record already exists and the annotation is never read. The node comes +// back Ready, accepts pods, and hosts no replicas at all. +// +// That failure is invisible until a rebuild is expected and does not arrive, +// so these tests pin the read-back and the repair, not just the annotate. + +import { describe, it, expect, vi, afterEach } from "vitest"; +import { mockCtx, OK, FAIL, stdout, expectCommand, expectNoCommand } from "./helpers.js"; +import { configureLonghornDisk } from "../src/operations/longhorn-disk.js"; + +const HOST = stdout("node1.lab\n"); +const DISK = "/var/lib/longhorn"; + +interface Opts { + /** Does Longhorn create the disk on its own (i.e. it read the annotation)? */ + longhornCreatesDisk?: boolean; + /** Has longhorn-manager registered this node yet? */ + recordExists?: boolean; + hasDir?: boolean; + noLonghorn?: boolean; + patchFails?: boolean; + /** Does the disk show up after a direct spec.disks patch? */ + repairWorks?: boolean; +} + +function ctxFor(o: Opts = {}) { + // Model the real object rather than a script of reads: a disk exists if + // Longhorn made one, or if our repair patch landed and took effect. + let patched = false; + const ctx = mockCtx(); + ctx.ssh.exec.mockImplementation(async (cmd: string) => { + if (cmd.startsWith("test -d")) return stdout(o.hasDir === false ? "no" : "yes"); + if (cmd.startsWith("hostname")) return HOST; + if (cmd.includes("get ns longhorn-system")) return o.noLonghorn ? FAIL : stdout("namespace/longhorn-system"); + // Existence probe (-o name) is a DIFFERENT question from "has a disk", + // and must be matched before the jsonpath reads. + if (cmd.includes("get nodes.longhorn.io") && cmd.includes("-o name")) { + return o.recordExists === false ? FAIL : stdout("node.longhorn.io/node1.lab"); + } + if (cmd.includes("get nodes.longhorn.io") && cmd.includes("jsonpath")) { + const has = (o.longhornCreatesDisk ?? false) || (patched && (o.repairWorks ?? true)); + return stdout(has ? DISK : ""); + } + if (cmd.includes("patch nodes.longhorn.io")) { + if (o.patchFails === true) return FAIL; + patched = true; + return OK; + } + if (cmd.includes("annotate node")) return stdout("node/node1.lab annotated"); + return OK; + }); + return ctx; +} + +/** Run the operation with the 60s verify window collapsed. */ +async function runFast(ctx: ReturnType) { + vi.useFakeTimers(); + try { + const p = configureLonghornDisk(ctx); + // Long enough to cover the registration wait (180s) AND the disk wait (60s). + await vi.advanceTimersByTimeAsync(300_000); + return await p; + } finally { + vi.useRealTimers(); + } +} + +afterEach(() => { vi.useRealTimers(); }); + +describe("the annotation survives the shell", () => { + // The value is JSON, so it is full of double quotes. Wrapping it in double + // quotes truncates it at the first inner quote and the node ends up with + // [{path:/var/lib/longhorn,allowScheduling:true}] + // which Longhorn cannot parse, so it creates no disk at all -- silently. + // Asserting on the command string alone would NOT catch this; the command + // looks fine until a shell reads it. So run it through a real shell. + it("arrives as valid JSON after shell parsing", async () => { + const ctx = ctxFor({ longhornCreatesDisk: true }); + await runFast(ctx); + const cmd = ctx.ssh.exec.mock.calls + .map((c: [string, unknown?]) => c[0]) + .find((c: string) => c.includes("default-disks-config")) ?? ""; + expect(cmd, "annotate command").not.toBe(""); + + // Let a real shell word-split it exactly as sshd would, and recover the + // argument that carries the annotation. + const { execFileSync } = await import("node:child_process"); + const printed = execFileSync("bash", ["-c", `printf '%s\n' ${cmd.replace(/^k3s kubectl /, "")}`], { + encoding: "utf-8", + }); + const arg = printed.split("\n").find((l) => l.startsWith("node.longhorn.io/default-disks-config=")) ?? ""; + expect(arg, "annotation argument after shell parsing").not.toBe(""); + + const json = arg.slice("node.longhorn.io/default-disks-config=".length); + const parsed = JSON.parse(json) as { path: string; allowScheduling: boolean }[]; + expect(parsed).toEqual([{ path: DISK, allowScheduling: true }]); + }); +}); + +describe("configureLonghornDisk", () => { + it("guarantees the replicas directory exists", async () => { + // Longhorn creates /replicas only when it first initialises the + // disk. If it disappears while longhorn-disk.cfg remains, Longhorn will + // not recreate it and every replica placed on the node dies on a + // non-recursive mkdir -- permanently, and invisibly, because the node + // still reports Ready and Schedulable. + const ctx = ctxFor({ longhornCreatesDisk: true }); + await runFast(ctx); + expectCommand(ctx.ssh, /mkdir -p \/var\/lib\/longhorn\/replicas/); + }); + + it("skips a node with no /var/lib/longhorn", async () => { + const r = await configureLonghornDisk(ctxFor({ hasDir: false })); + expect(r.success).toBe(true); + expect(r.changed).toBe(false); + expect(r.message).toMatch(/skipping/); + }); + + it("annotates and is satisfied when Longhorn creates the disk itself", async () => { + const ctx = ctxFor({ longhornCreatesDisk: true }); + const r = await runFast(ctx); + expect(r.success).toBe(true); + expectCommand(ctx.ssh, /default-disks-config/); + // Longhorn honoured the annotation, so no direct spec.disks write. + expectNoCommand(ctx.ssh, /patch nodes\.longhorn\.io/); + }); + + it("REPAIRS a rejoining node whose annotation was read too late", async () => { + // No disk for the whole verify window, then present after the patch. + const ctx = ctxFor({ longhornCreatesDisk: false, repairWorks: true }); + const r = await runFast(ctx); + expect(r.success).toBe(true); + expect(r.changed).toBe(true); + expect(r.message).toMatch(/repaired/i); + expectCommand(ctx.ssh, /patch nodes\.longhorn\.io/); + expectCommand(ctx.ssh, new RegExp(`"path":"${DISK.replace(/\//g, "\\/")}"`.replace(/\\\//g, "/"))); + }); + + it("FAILS loudly when the disk cannot be created — a node with no disk is useless", async () => { + const ctx = ctxFor({ longhornCreatesDisk: false, repairWorks: false }); + const r = await runFast(ctx); + expect(r.success).toBe(false); + expect(r.error).toMatch(/will not host replicas|never host a replica/); + }); + + it("FAILS when the repair patch itself is rejected", async () => { + const ctx = ctxFor({ longhornCreatesDisk: false, patchFails: true }); + const r = await runFast(ctx); + expect(r.success).toBe(false); + expect(r.message).toMatch(/could not be repaired/); + }); + + it("does NOT fail a node Longhorn has not registered yet", async () => { + // A node that has only just rejoined has no nodes.longhorn.io record. The + // annotation is already in place, so Longhorn will read it when it creates + // one. Failing here would abort a rejoin over something self-resolving. + const ctx = ctxFor({ longhornCreatesDisk: false, recordExists: false }); + const r = await runFast(ctx); + expect(r.success).toBe(true); + expect(r.message).toMatch(/not registered/); + expectNoCommand(ctx.ssh, /patch nodes\.longhorn\.io/); + }); + + it("annotates and stops when Longhorn is not installed yet", async () => { + // Install-time ordering: the annotation is set so Longhorn honours it when + // it does arrive, but there is nothing to read back from. + const ctx = ctxFor({ noLonghorn: true }); + const r = await configureLonghornDisk(ctx); + expect(r.success).toBe(true); + expectCommand(ctx.ssh, /default-disks-config/); + expectNoCommand(ctx.ssh, /patch nodes\.longhorn\.io/); + }); + + it("uses one fixed disk key, so repeated repairs update rather than multiply", async () => { + const ctx = ctxFor({ longhornCreatesDisk: false, repairWorks: true }); + await runFast(ctx); + const patch = ctx.ssh.exec.mock.calls + .map((c: [string, unknown?]) => c[0]) + .find((c: string) => c.includes("patch nodes.longhorn.io")) ?? ""; + expect(patch).toContain("default-disk-lab"); + }); +}); diff --git a/bastion/src/modules/modules/k3s/tests/module-config.test.ts b/bastion/src/modules/modules/k3s/tests/module-config.test.ts new file mode 100644 index 0000000..01b7c42 --- /dev/null +++ b/bastion/src/modules/modules/k3s/tests/module-config.test.ts @@ -0,0 +1,79 @@ +// The ModuleContext -> OperationContext config hand-off. +// +// toOpContext used to enumerate three keys and silently drop the rest, so a +// caller could set a field, get no error, and have it ignored. That produced +// two live bugs: a pinned k3sVersion was discarded (get.k3s.io installed +// LATEST, two minor versions ahead of the fleet), and +// `--allow-single-replica` was a no-op that claimed to override a data-loss +// guard. +// +// These tests assert the pass-through per field, and the last one fails for +// ANY future K3sConfig field that goes missing — which is the property that +// was actually absent. + +import { describe, it, expect, vi } from "vitest"; +import { K3sModule } from "../src/k3s-module.js"; +import type { ModuleContext } from "../../../src/types.js"; + +vi.mock("../../../src/ssh.js", () => ({ + sshExec: vi.fn(async (_ip: string, _user: string, cmd: string) => { + // Record every command so the test can assert on what was sent. + issued.push(cmd); + if (cmd.includes("k3s --version")) return { exitCode: 1, stdout: "", stderr: "" }; + if (cmd.includes("hostname")) return { exitCode: 0, stdout: "n1.lab\n", stderr: "" }; + if (cmd.includes("get nodes")) return { exitCode: 0, stdout: "NAME\nn1 Ready", stderr: "" }; + return { exitCode: 0, stdout: "", stderr: "" }; + }), +})); + +const issued: string[] = []; + +function ctxWith(config: Record): ModuleContext { + return { + hostname: "n1.lab", ip: "10.0.0.5", role: "k8s-combo", + os: "fedora-43", arch: "x86_64", sshUser: "root", config, + }; +} + +describe("ModuleContext.config reaches the operations", () => { + it("passes k3sVersion through to the installer", async () => { + issued.length = 0; + await new K3sModule().install(ctxWith({ + k3sToken: "t", k3sVersion: "v1.34.6+k3s1", + })); + const install = issued.find((c) => c.includes("get.k3s.io")) ?? ""; + expect(install, "install command").toContain('INSTALL_K3S_VERSION="v1.34.6+k3s1"'); + }); + + it("still passes the three fields it always did", async () => { + issued.length = 0; + await new K3sModule().install(ctxWith({ + k3sServerUrl: "https://10.0.0.1:6443", k3sToken: "tok", + })); + const install = issued.find((c) => c.includes("get.k3s.io")) ?? ""; + expect(install).toContain('K3S_URL="https://10.0.0.1:6443"'); + expect(install).toContain('K3S_TOKEN="tok"'); + }); + + it("drops nothing: every key given is visible to the operations", async () => { + // The generic guard. Enumerating keys in toOpContext is the bug; this + // fails if anyone reintroduces it for a field added later. + const { K3sModule: Mod } = await import("../src/k3s-module.js"); + const mod = new Mod(); + const given = { + k3sServerUrl: "https://10.0.0.1:6443", + k3sToken: "tok", + k3sVersion: "v1.34.6+k3s1", + evictionAllowSingleReplica: true, + tlsSans: ["a.lab"], + }; + issued.length = 0; + await mod.install(ctxWith(given)); + // k3sVersion and the join env are the observable ones at install time; + // assert them explicitly rather than trusting the shape. + const install = issued.find((c) => c.includes("get.k3s.io")) ?? ""; + expect(install).toContain("INSTALL_K3S_VERSION="); + expect(install).toContain("K3S_URL="); + expect(install).toContain("K3S_TOKEN="); + }); +}); diff --git a/bastion/src/modules/modules/k3s/tests/node-admit.test.ts b/bastion/src/modules/modules/k3s/tests/node-admit.test.ts new file mode 100644 index 0000000..ed10e98 --- /dev/null +++ b/bastion/src/modules/modules/k3s/tests/node-admit.test.ts @@ -0,0 +1,211 @@ +// Tests for returning an evacuated node to service. +// +// Evacuation leaves three separate marks on a node — a kubernetes cordon, +// Longhorn's allowScheduling/evictionRequested pair, and (after a wipe and +// rejoin) a MISSING Longhorn disk. Undoing only the cordon gives you a node +// that is Ready, accepts pods and hosts no replicas at all, which reads as +// success until storage never rebalances onto it. These tests pin all three. + +import { describe, it, expect, vi, afterEach } from "vitest"; +import { mockCtx, OK, FAIL, stdout, expectCommand } from "./helpers.js"; +import { clearLonghornEviction, assertNodeSchedulable } from "../src/operations/node-admit.js"; +import { evictLonghornReplicas } from "../src/operations/node-evacuate.js"; +import { evacuateGroup } from "../src/groups/evacuate.js"; +import { admitGroup } from "../src/groups/admit.js"; + +const HOST = stdout("node1.lab\n"); + +/** + * Run assertNodeSchedulable with its 120s poll collapsed. + * + * It polls because Longhorn registers a node asynchronously and this runs + * moments after a rejoin; the negative cases therefore wait out the whole + * window before they can honestly fail. + */ +async function runGate(ctx: ReturnType) { + vi.useFakeTimers(); + try { + const p = assertNodeSchedulable(ctx); + await vi.advanceTimersByTimeAsync(150_000); + return await p; + } finally { + vi.useRealTimers(); + } +} + +afterEach(() => { vi.useRealTimers(); }); + +/** Command-matching mock; see node-evacuate.test.ts for why not call order. */ +function routed(handlers: Record, opts: { noLonghorn?: boolean } = {}) { + const ctx = mockCtx(); + ctx.ssh.exec.mockImplementation(async (cmd: string) => { + if (cmd.startsWith("hostname")) return HOST; + if (cmd.includes("get ns longhorn-system")) return opts.noLonghorn ? FAIL : stdout("namespace/longhorn-system"); + for (const [needle, out] of Object.entries(handlers)) { + if (cmd.includes(needle)) return out === null ? FAIL : stdout(out); + } + return OK; + }); + return ctx; +} + +describe("clearLonghornEviction", () => { + it("clears scheduling and eviction together in one patch", async () => { + const ctx = routed({ "get nodes.longhorn.io": "false,true" }); + const r = await clearLonghornEviction(ctx); + expect(r.success).toBe(true); + expect(r.changed).toBe(true); + // Both fields must move together: Longhorn's webhook rejects + // evictionRequested:true on a schedulable node, so the inverse pairing + // has to be atomic as well. + expectCommand(ctx.ssh, /"allowScheduling":true/); + expectCommand(ctx.ssh, /"evictionRequested":false/); + }); + + it("is idempotent when the node already accepts replicas", async () => { + const ctx = routed({ "get nodes.longhorn.io": "true,false" }); + const r = await clearLonghornEviction(ctx); + expect(r.success).toBe(true); + expect(r.changed).toBe(false); + }); + + it("is a no-op when Longhorn is not installed", async () => { + const ctx = routed({}, { noLonghorn: true }); + const r = await clearLonghornEviction(ctx); + expect(r.success).toBe(true); + expect(r.changed).toBe(false); + }); + + it("reports failure rather than claiming the node is back", async () => { + const ctx = routed({ "get nodes.longhorn.io": "false,true", "patch nodes.longhorn.io": null }); + const r = await clearLonghornEviction(ctx); + expect(r.success).toBe(false); + }); +}); + +describe("assertNodeSchedulable — the gate after a rejoin", () => { + it("passes when uncordoned, not evicting, and holding a schedulable disk", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockImplementation(async (cmd: string) => { + if (cmd.startsWith("hostname")) return HOST; + if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system"); + if (cmd.includes("diskStatus")) return stdout("True"); + if (cmd.includes("get nodes.longhorn.io")) return stdout("true,false"); + if (cmd.includes("get node ")) return stdout(""); + return OK; + }); + const r = await assertNodeSchedulable(ctx); + expect(r.success).toBe(true); + }); + + it("REFUSES a node that has no Longhorn disk — the silent rejoin failure", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockImplementation(async (cmd: string) => { + if (cmd.startsWith("hostname")) return HOST; + if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system"); + if (cmd.includes("diskStatus")) return stdout(""); // no disks at all + if (cmd.includes("get nodes.longhorn.io")) return stdout("true,false"); + if (cmd.includes("get node ")) return stdout(""); + return OK; + }); + const r = await runGate(ctx); + expect(r.success).toBe(false); + expect(r.error).toMatch(/no disk/); + }); + + it("REFUSES a node still marked for eviction", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockImplementation(async (cmd: string) => { + if (cmd.startsWith("hostname")) return HOST; + if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system"); + if (cmd.includes("diskStatus")) return stdout("True"); + if (cmd.includes("get nodes.longhorn.io")) return stdout("false,true"); + if (cmd.includes("get node ")) return stdout(""); + return OK; + }); + const r = await runGate(ctx); + expect(r.success).toBe(false); + expect(r.error).toMatch(/eviction is still requested|scheduling is still disabled/); + }); + + it("REFUSES a node that is still cordoned", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockImplementation(async (cmd: string) => { + if (cmd.startsWith("hostname")) return HOST; + if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system"); + if (cmd.includes("diskStatus")) return stdout("True"); + if (cmd.includes("get nodes.longhorn.io")) return stdout("true,false"); + if (cmd.includes("get node ")) return stdout("true"); // unschedulable + return OK; + }); + const r = await runGate(ctx); + expect(r.success).toBe(false); + expect(r.error).toMatch(/cordoned/); + }); +}); + +describe("evictLonghornReplicas — never leaves a node eviction-pending", () => { + it("cancels the eviction request when the wait blows up", async () => { + const ctx = mockCtx(); + const patched: string[] = []; + ctx.ssh.exec.mockImplementation(async (cmd: string) => { + if (cmd.startsWith("hostname")) return HOST; + if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system"); + if (cmd.includes("jsonpath")) { + // Blow up during the rebuild wait, after eviction was requested. + if (cmd.includes("replicas.longhorn.io")) throw new Error("API server went away"); + return stdout(""); + } + if (cmd.includes("-o json")) return stdout(JSON.stringify({ items: [] })); + if (cmd.includes("patch nodes.longhorn.io")) { patched.push(cmd); return OK; } + return OK; + }); + + await expect(evictLonghornReplicas(ctx)).rejects.toThrow("API server went away"); + + // Two patches: the request, then the rollback. Leaving only the first + // behind would strand the node unschedulable with Longhorn still draining. + expect(patched.length).toBe(2); + expect(patched[0]).toContain('"evictionRequested":true'); + expect(patched[1]).toContain('"evictionRequested":false'); + expect(patched[1]).toContain('"allowScheduling":true'); + }); +}); + +describe("group ordering", () => { + it("evacuate evicts BEFORE it drains", () => { + // drain --force does not override PodDisruptionBudgets, and Longhorn holds + // one on the instance-manager pod. Draining first stops the process that + // serves the replicas being evicted: the drain blocks for its full timeout + // and would remove the only rebuild source for a single-replica volume. + const names = evacuateGroup.operations.map((o) => o.name); + const evict = names.findIndex((n) => /Evict/i.test(n)); + const drain = names.findIndex((n) => /Drain/i.test(n)); + expect(evict).toBeGreaterThanOrEqual(0); + expect(drain).toBeGreaterThanOrEqual(0); + expect(evict).toBeLessThan(drain); + }); + + it("evacuate checks feasibility before it mutates anything", () => { + const names = evacuateGroup.operations.map((o) => o.name); + expect(names.findIndex((n) => /Check eviction/i.test(n))) + .toBeLessThan(names.findIndex((n) => /Evict Longhorn/i.test(n))); + }); + + it("evacuate verifies last", () => { + const names = evacuateGroup.operations.map((o) => o.name); + expect(names[names.length - 1]).toMatch(/Verify/i); + }); + + it("admit clears eviction before restoring the disk", () => { + // A node still marked for eviction has any disk it is given drained again. + const names = admitGroup.operations.map((o) => o.name); + expect(names.findIndex((n) => /Clear Longhorn eviction/i.test(n))) + .toBeLessThan(names.findIndex((n) => /Restore Longhorn default disk/i.test(n))); + }); + + it("admit verifies last", () => { + const names = admitGroup.operations.map((o) => o.name); + expect(names[names.length - 1]).toMatch(/Verify/i); + }); +}); diff --git a/bastion/src/modules/modules/k3s/tests/node-evacuate.test.ts b/bastion/src/modules/modules/k3s/tests/node-evacuate.test.ts new file mode 100644 index 0000000..8686791 --- /dev/null +++ b/bastion/src/modules/modules/k3s/tests/node-evacuate.test.ts @@ -0,0 +1,413 @@ +import { describe, it, expect, vi } from "vitest"; +import { mockCtx, OK, FAIL, stdout, expectCommand, expectNoCommand } from "./helpers.js"; +import { + cordonNode, uncordonNode, drainNode, evictLonghornReplicas, assertNodeEvacuated, + assertEvictionCompletable, checkEvictionCompletable, +} from "../src/operations/node-evacuate.js"; +import type { K3sConfig } from "../src/types.js"; + +const HOST = stdout("node1.lab\n"); +const GIB = 2 ** 30; + +interface RoutedOpts { + nodes?: string[]; + replicas?: { volume: string }[]; + volumes?: { name: string; replicas: number; size: number }[]; + softAntiAffinity?: boolean; + overProvisionPct?: number; + diskFreeBytes?: number; + diskMaxBytes?: number; + patchFails?: boolean; +} + +/** + * A context whose SSH mock answers by MATCHING the command rather than by call + * order. The operations under test read several API objects before they do + * anything, and an ordered mock has to be re-counted every time one is added — + * which makes the test fail for reasons that have nothing to do with the + * behaviour it is meant to pin. + */ +function routed(o: RoutedOpts = {}, config?: Partial) { + const nodeNames = o.nodes ?? ["node1.lab", "node2.lab", "node3.lab", "node4.lab"]; + const free = o.diskFreeBytes ?? 500 * GIB; + const max = o.diskMaxBytes ?? 700 * GIB; + const lhNodes = { + items: nodeNames.map((name) => ({ + metadata: { name }, + spec: { allowScheduling: true }, + status: { diskStatus: { "default-disk": { + storageAvailable: free, storageMaximum: max, storageScheduled: 0, storageReserved: 0, + } } }, + })), + }; + const replicas = { + items: (o.replicas ?? []).map((r, i) => ({ + metadata: { name: `replica-${i}` }, + spec: { nodeID: "node1.lab", volumeName: r.volume }, + })), + }; + const volumes = { + items: (o.volumes ?? []).map((v) => ({ + metadata: { name: v.name }, + spec: { numberOfReplicas: v.replicas, size: v.size }, + status: { state: "attached", robustness: "healthy" }, + })), + }; + + const ctx = mockCtx(config); + ctx.ssh.exec.mockImplementation(async (cmd: string) => { + if (cmd.startsWith("hostname")) return HOST; + if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system"); + // NOTE: "-o jsonpath=" CONTAINS "-o json", so jsonpath must be matched + // first or the eviction poll below is handed a whole JSON document and + // never sees zero replicas. (This mock got that wrong once already.) + if (cmd.includes("jsonpath")) { + // The eviction wait polls replicas on the node — report none remaining. + if (cmd.includes("replicas.longhorn.io")) return stdout(""); + if (cmd.includes("replica-soft-anti-affinity")) return stdout(String(o.softAntiAffinity ?? false)); + if (cmd.includes("storage-over-provisioning-percentage")) return stdout(String(o.overProvisionPct ?? 100)); + return stdout(""); + } + if (cmd.includes("get nodes.longhorn.io") && cmd.includes("-o json")) return stdout(JSON.stringify(lhNodes)); + if (cmd.includes("get replicas.longhorn.io") && cmd.includes("-o json")) return stdout(JSON.stringify(replicas)); + if (cmd.includes("get volumes.longhorn.io") && cmd.includes("-o json")) return stdout(JSON.stringify(volumes)); + if (cmd.includes("patch nodes.longhorn.io")) return o.patchFails ? FAIL : OK; + return OK; + }); + return ctx; +} + +describe("cordonNode", () => { + it("cordons and reports a change", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(stdout('node/node1.lab cordoned')); + const r = await cordonNode(ctx); + expect(r.success).toBe(true); + expect(r.changed).toBe(true); + expectCommand(ctx.ssh, /cordon "node1\.lab"/); + }); + + it("is idempotent — already cordoned is changed:false", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(stdout("node/node1.lab already cordoned")); + const r = await cordonNode(ctx); + expect(r.success).toBe(true); + expect(r.changed).toBe(false); + }); + + it("fails loudly when the API is unreachable", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(FAIL); + const r = await cordonNode(ctx); + expect(r.success).toBe(false); + }); +}); + +describe("drainNode", () => { + it("ignores daemonsets and deletes emptyDir, or it hangs forever", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(OK); + const r = await drainNode(ctx); + expect(r.success).toBe(true); + expectCommand(ctx.ssh, /--ignore-daemonsets/); + expectCommand(ctx.ssh, /--delete-emptydir-data/); + }); + + it("gives the SSH transport longer than kubectl's own --timeout", async () => { + // drainNode asks kubectl for --timeout=15m. sshOpts() is a flat 30s, so + // without an explicit override the transport kills a drain that is working + // -- after it has already evicted most of the pods. The result is + // non-zero with a transcript that reads like success, on every node with + // real workloads. `labctl app k3s evacuate` could never have drained one. + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(OK); + await drainNode(ctx); + const call = ctx.ssh.exec.mock.calls.find((c: [string, { timeoutMs?: number }?]) => + c[0].includes("drain ")); + expect(call, "drain command").toBeDefined(); + const timeoutMs = call?.[1]?.timeoutMs ?? 0; + expect(timeoutMs, "SSH timeout must exceed kubectl's 15m --timeout") + .toBeGreaterThan(15 * 60_000); + }); + + it("reports the END of the drain transcript, not the useless first line", async () => { + // Every drain starts with "node/x already cordoned". Surfacing the whole + // transcript means the error you see is that line, which explains nothing. + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce({ + exitCode: 1, + stdout: "node/n1 already cordoned\nevicting pod a\nevicting pod b\n" + + "error: cannot evict pod as it would violate the pod's disruption budget", + stderr: "", + }); + const r = await drainNode(ctx); + expect(r.success).toBe(false); + expect(r.error).toContain("disruption budget"); + }); + + it("reports failure rather than pretending the node is empty", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(FAIL); + const r = await drainNode(ctx); + expect(r.success).toBe(false); + }); +}); + +describe("evictLonghornReplicas", () => { + it("is a no-op when Longhorn is not installed", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(FAIL); // no longhorn-system ns + const r = await evictLonghornReplicas(ctx); + expect(r.success).toBe(true); + expect(r.changed).toBe(false); + expect(r.message).toMatch(/not installed/); + }); + + it("requests eviction and waits until zero replicas remain", async () => { + // Node holds no replicas, so the pre-check clears it immediately. + const ctx = routed({ replicas: [] }); + const r = await evictLonghornReplicas(ctx); + expect(r.success).toBe(true); + expect(r.changed).toBe(true); + expectCommand(ctx.ssh, /evictionRequested/); + expectCommand(ctx.ssh, /allowScheduling/); + }); + + it("fails when eviction cannot even be requested", async () => { + const ctx = routed({ replicas: [], patchFails: true }); + const r = await evictLonghornReplicas(ctx); + expect(r.success).toBe(false); + }); + + it("does not request eviction at all when it could never complete", async () => { + // 3-replica volume, and only 2 other schedulable nodes: the survivors + // already hold a copy each and strict anti-affinity forbids a third. + const ctx = routed({ + nodes: ["node1.lab", "node2.lab", "node3.lab"], + replicas: [{ volume: "pvc-a" }], + volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }], + }); + const r = await evictLonghornReplicas(ctx); + expect(r.success).toBe(false); + expect(r.error).toMatch(/cannot be placed|wants 3 replicas/); + // The important half: nothing was mutated. + expectNoCommand(ctx.ssh, /evictionRequested/); + }); + + it("refuses a single-replica volume unless explicitly allowed", async () => { + const opts = { + nodes: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"], + replicas: [{ volume: "pvc-solo" }], + volumes: [{ name: "pvc-solo", replicas: 1, size: 1 * GIB }], + }; + const r = await evictLonghornReplicas(routed(opts)); + expect(r.success).toBe(false); + expect(r.error).toMatch(/single replica/); + + const allowed = await evictLonghornReplicas( + routed(opts, { evictionAllowSingleReplica: true }), + ); + expect(allowed.success).toBe(true); + }); +}); + +describe("checkEvictionCompletable — refuse in seconds, not in an hour", () => { + it("passes when a spare schedulable node exists", async () => { + const ctx = routed({ + nodes: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"], + replicas: [{ volume: "pvc-a" }], + volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }], + }); + const f = await checkEvictionCompletable(ctx, "node1.lab"); + expect(f.completable).toBe(true); + expect(f.replicasMoving).toBe(1); + }); + + it("blocks when replica count equals the surviving node count", async () => { + const ctx = routed({ + nodes: ["node1.lab", "node2.lab", "node3.lab"], + replicas: [{ volume: "pvc-a" }], + volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }], + }); + const f = await checkEvictionCompletable(ctx, "node1.lab"); + expect(f.completable).toBe(false); + expect(f.blockers.join(" ")).toContain("pvc-a"); + }); + + it("allows it when soft anti-affinity is on — placement stops being the limit", async () => { + const ctx = routed({ + nodes: ["node1.lab", "node2.lab", "node3.lab"], + replicas: [{ volume: "pvc-a" }], + volumes: [{ name: "pvc-a", replicas: 3, size: 1 * GIB }], + softAntiAffinity: true, + }); + const f = await checkEvictionCompletable(ctx, "node1.lab"); + expect(f.completable).toBe(true); + }); + + it("catches a small disk hiding behind over-provisioning", async () => { + // The scheduler is happy at 500%; the disk has 49 GiB and the volume is 92. + const ctx = routed({ + nodes: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"], + replicas: [{ volume: "pvc-big" }], + volumes: [{ name: "pvc-big", replicas: 3, size: 92 * GIB }], + overProvisionPct: 500, + diskFreeBytes: 49 * GIB, + diskMaxBytes: 100 * GIB, + }); + const f = await checkEvictionCompletable(ctx, "node1.lab"); + expect(f.completable).toBe(false); + expect(f.blockers.join(" ")).toMatch(/actually free/); + expect(f.blockers.join(" ")).toMatch(/over-provisioning-percentage=500/); + }); + + it("warns, but does not block, on a single-replica volume", async () => { + const ctx = routed({ + nodes: ["node1.lab", "node2.lab", "node3.lab"], + replicas: [{ volume: "pvc-solo" }], + volumes: [{ name: "pvc-solo", replicas: 1, size: 1 * GIB }], + }); + const f = await checkEvictionCompletable(ctx, "node1.lab"); + expect(f.completable).toBe(true); + expect(f.warnings.join(" ")).toMatch(/single replica/); + }); + + it("is trivially completable when the node holds nothing", async () => { + const f = await checkEvictionCompletable(routed({ replicas: [] }), "node1.lab"); + expect(f.completable).toBe(true); + expect(f.replicasMoving).toBe(0); + }); + + it("refuses to guess when Longhorn cannot be read", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValue(FAIL); + const f = await checkEvictionCompletable(ctx, "node1.lab"); + expect(f.completable).toBe(false); + expect(f.blockers.join(" ")).toMatch(/refusing to guess/); + }); +}); + +describe("assertNodeEvacuated — the gate before a wipe", () => { + it("passes only when cordoned with no replicas left", async () => { + const ctx = mockCtx(); + ctx.ssh.exec + .mockResolvedValueOnce(HOST) + .mockResolvedValueOnce(stdout("true")) // unschedulable + .mockResolvedValueOnce(stdout("")) // no running pods + .mockResolvedValueOnce(stdout("namespace/longhorn-system")) + .mockResolvedValueOnce(stdout("")); // no replicas + const r = await assertNodeEvacuated(ctx); + expect(r.success).toBe(true); + expect(r.message).toMatch(/safe to wipe/); + }); + + it("REFUSES when replicas are still on the node", async () => { + const ctx = mockCtx(); + ctx.ssh.exec + .mockResolvedValueOnce(HOST) + .mockResolvedValueOnce(stdout("true")) + .mockResolvedValueOnce(stdout("")) + .mockResolvedValueOnce(stdout("namespace/longhorn-system")) + .mockResolvedValueOnce(stdout("replica-abc\nreplica-def")); + const r = await assertNodeEvacuated(ctx); + expect(r.success).toBe(false); + expect(r.error).toMatch(/2 Longhorn replica/); + }); + + it("REFUSES when the node is still schedulable", async () => { + const ctx = mockCtx(); + ctx.ssh.exec + .mockResolvedValueOnce(HOST) + .mockResolvedValueOnce(stdout("")) // not unschedulable + .mockResolvedValueOnce(stdout("")) + .mockResolvedValueOnce(FAIL); // no longhorn + const r = await assertNodeEvacuated(ctx); + expect(r.success).toBe(false); + expect(r.error).toMatch(/still schedulable/); + }); +}); + +describe("uncordonNode", () => { + it("returns the node to service", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(HOST).mockResolvedValueOnce(OK); + const r = await uncordonNode(ctx); + expect(r.success).toBe(true); + expectCommand(ctx.ssh, /uncordon "node1\.lab"/); + }); +}); + +describe("eviction timeout diagnostics", () => { + // An eviction can legitimately run for an hour. When it times out, a bare + // count ("2 replica(s) still present") is not actionable -- the question is + // always WHICH volume is stuck and in what state. A replica with + // desireState=running, currentState=stopped and no failedAt is Longhorn + // wanting to start it and not doing so, which is a completely different + // problem from a volume that cannot be placed. Observed for real on a + // freshly wiped-and-rejoined node. + it("names the stuck volumes and their replica states, then restores the node", async () => { + const ctx = mockCtx(); + const patches: string[] = []; + ctx.ssh.exec.mockImplementation(async (cmd: string) => { + if (cmd.startsWith("hostname")) return HOST; + if (cmd.includes("get ns longhorn-system")) return stdout("namespace/longhorn-system"); + if (cmd.includes("patch nodes.longhorn.io")) { patches.push(cmd); return OK; } + if (cmd.includes("jsonpath")) { + // The poll never reaches zero: one replica remains for the whole run. + if (cmd.includes("replicas.longhorn.io")) return stdout("replica-stuck"); + if (cmd.includes("replica-soft-anti-affinity")) return stdout("false"); + if (cmd.includes("storage-over-provisioning-percentage")) return stdout("100"); + return stdout(""); + } + if (cmd.includes("get replicas.longhorn.io") && cmd.includes("-o json")) { + return stdout(JSON.stringify({ items: [{ + metadata: { name: "pvc-abc-r-deadbeef99" }, + spec: { nodeID: "node1.lab", volumeName: "pvc-abc", failedAt: "", desireState: "running" }, + status: { currentState: "stopped" }, + }] })); + } + if (cmd.includes("get volumes.longhorn.io") && cmd.includes("-o json")) { + return stdout(JSON.stringify({ items: [{ + metadata: { name: "pvc-abc" }, + spec: { numberOfReplicas: 3, size: 2 * 2 ** 30 }, + status: { state: "attached", robustness: "degraded" }, + }] })); + } + if (cmd.includes("get nodes.longhorn.io") && cmd.includes("-o json")) { + return stdout(JSON.stringify({ items: ["node1.lab", "node2.lab", "node3.lab", "node4.lab"].map((n) => ({ + metadata: { name: n }, + spec: { allowScheduling: true }, + status: { diskStatus: { d: { + storageAvailable: 500 * 2 ** 30, storageMaximum: 700 * 2 ** 30, + storageScheduled: 0, storageReserved: 0, + } } }, + })) })); + } + return OK; + }); + + vi.useFakeTimers(); + let result; + try { + const p = evictLonghornReplicas(ctx); + await vi.advanceTimersByTimeAsync(61 * 60_000); + result = await p; + } finally { + vi.useRealTimers(); + } + + expect(result.success).toBe(false); + // The actionable part: which volume, and what state its replica is in. + expect(result.error).toContain("pvc-abc"); + expect(result.error).toContain("currentState=stopped"); + expect(result.error).toContain("desireState=running"); + expect(result.error).toContain("failedAt=(none)"); + expect(result.error).toContain("attached/degraded"); + + // And the node must be handed back usable, not left eviction-pending. + expect(result.error).toMatch(/cancelled and scheduling restored/); + expect(patches).toHaveLength(2); + expect(patches[1]).toContain('"evictionRequested":false'); + expect(patches[1]).toContain('"allowScheduling":true'); + }); +}); diff --git a/bastion/src/modules/src/types.ts b/bastion/src/modules/src/types.ts index 3baf7b2..0f33c81 100644 --- a/bastion/src/modules/src/types.ts +++ b/bastion/src/modules/src/types.ts @@ -1,4 +1,4 @@ -import type { OsId, Arch } from "@lab/shared"; +import type { OsId, Arch, Role } from "@lab/shared"; export interface ModuleMetadata { name: string; @@ -14,7 +14,10 @@ export interface ModuleMetadata { export interface ModuleContext { hostname: string; ip: string; - role: string; + // Role, not string. A literal `string` let `role === "infra"` comparisons + // compile while silently missing every other server role -- which is how + // labcontroller ended up installing as an agent. + role: Role; os: OsId; arch: Arch; sshUser: string; @@ -24,7 +27,7 @@ export interface ModuleContext { export interface ModuleResult { success: boolean; - phase: "install" | "configure" | "health"; + phase: "install" | "configure" | "health" | "evacuate" | "admit"; duration: number; output: string[]; errors: string[]; diff --git a/bastion/src/shared/src/index.ts b/bastion/src/shared/src/index.ts index 1a3239b..75eeff6 100644 --- a/bastion/src/shared/src/index.ts +++ b/bastion/src/shared/src/index.ts @@ -14,8 +14,9 @@ export type { VyosBundleSetOp, } from "./types/index.js"; -export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./types/index.js"; -export type { RoleInfo } from "./types/index.js"; +export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId, + roleInfo, runsK3s, isK3sServer, isManaged, hostsLonghorn, isRole } from "./types/index.js"; +export type { RoleInfo, Lifecycle, K3sMode } from "./types/index.js"; export { APP_NAME, APP_VERSION } from "./constants/index.js"; @@ -48,3 +49,7 @@ export { AgentNotConnectedError, isLabError, } from "./errors/index.js"; + +export { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB, + SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion } from "./types/index.js"; +export type { LvSpec, SchemaVersion } from "./types/index.js"; diff --git a/bastion/src/shared/src/types/config.ts b/bastion/src/shared/src/types/config.ts index ed4a599..2f69108 100644 --- a/bastion/src/shared/src/types/config.ts +++ b/bastion/src/shared/src/types/config.ts @@ -23,6 +23,10 @@ export interface BastionConfig { // Flags skipDnsmasq?: boolean | undefined; skipArtifacts?: boolean | undefined; + // Post-provision k3s auto-install. Defaults on. Set false to suppress it -- + // required by multi-node tests, which must sequence cluster formation + // themselves rather than have each node auto-install into its own cluster. + autoK3s?: boolean | undefined; // Labd registration (optional — if unset, bastion runs standalone) labdUrl?: string | undefined; bastionJoinToken?: string | undefined; diff --git a/bastion/src/shared/src/types/index.ts b/bastion/src/shared/src/types/index.ts index 73e471c..4006132 100644 --- a/bastion/src/shared/src/types/index.ts +++ b/bastion/src/shared/src/types/index.ts @@ -13,7 +13,12 @@ export type { VyosBundleSetOp, } from "./state.js"; -export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./state.js"; -export type { RoleInfo } from "./state.js"; +export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId, + roleInfo, runsK3s, isK3sServer, isManaged, hostsLonghorn, isRole } from "./state.js"; +export type { RoleInfo, Lifecycle, K3sMode } from "./state.js"; export type { BastionConfig } from "./config.js"; + +export { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB, + SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion } from "./layout.js"; +export type { LvSpec, SchemaVersion } from "./layout.js"; diff --git a/bastion/src/shared/src/types/layout.ts b/bastion/src/shared/src/types/layout.ts new file mode 100644 index 0000000..77c0333 --- /dev/null +++ b/bastion/src/shared/src/types/layout.ts @@ -0,0 +1,143 @@ +import { roleInfo, type Role } from "./state.js"; + +/** + * Disk layout, schema v2. + * + * WHY THIS FILE EXISTS + * -------------------- + * Three generators partition machines, and before this file they disagreed: + * + * install.ks.ts rancher 120G (infra|worker), longhorn --grow (worker) + * ubuntu-autoinstall rancher 20G (infra only), longhorn grow (worker) + * asahi-firstboot.sh rancher 20G (server only), longhorn 100%FREE (worker|server) + * + * asahi-firstboot even carries the comment "sizes match install.ks.ts" while + * being 100 GB smaller — which is where worker1's undersized image store came + * from, and why a 2026-08 fix to one file never reached the fleet. + * + * v2 is defined ONCE, here, and all three generators render from it. v1 stays + * frozen inline in each generator, because every existing machine was built by + * it and reprovision must reproduce it byte-for-byte. + * + * SIZING + * ------ + * Must fit a 1 TB disk (~926 GiB VG after EFI+/boot) with headroom left + * UNALLOCATED on purpose: LVM can grow into free extents online, and the + * fleet's recurring failure has been over-allocation, not under-allocation. + * `--grow` is deliberately never used — it is what left every node at VFree=0 + * and made later fixes impossible without a rebuild. + */ + +/** + * Roles that predate the v2 layout. Only these have a v1 layout to reproduce; + * everything else is v2-only, because no machine was ever installed as one. + */ +export const LEGACY_ROLES = ["vanilla", "worker", "infra", "labcontroller"] as const; + +/** + * True when this role has no v1 layout at all. + * + * Rendering one at v1 would silently produce base volumes with no rancher and + * no longhorn -- a machine that looks installed and cannot run k3s properly. + * Callers must reject this rather than guess. + */ +export function requiresSchemaV2(role: Role): boolean { + return !(LEGACY_ROLES as readonly string[]).includes(role); +} + +/** + * The schema a FRESH install of this role should use. + * + * A role with no v1 layout has exactly one possible answer, so defaulting to it + * is unambiguous -- there is no other layout it could mean. Rejecting the + * request instead was a mistake: a reinstall legitimately passes no schema + * (the disk decides via the lab-schema-v2 tag), and refusing that made it + * impossible to reinstall a v2-only machine at all. + * + * An EXPLICIT v1 for such a role is still a contradiction and must be refused + * by the caller -- see requiresSchemaV2. + */ +export function effectiveSchemaVersion(role: Role, requested?: 1 | 2): SchemaVersion { + if (requested) return requested; + return requiresSchemaV2(role) ? 2 : 1; +} + +export const SCHEMA_VERSIONS = [1, 2] as const; +export type SchemaVersion = (typeof SCHEMA_VERSIONS)[number]; + +export interface LvSpec { + /** LV name inside the volume group. */ + name: string; + /** Mount point, or null for swap. */ + mount: string | null; + /** Size in MiB. Never "grow". */ + sizeMib: number; + fs: "xfs" | "swap"; +} + +/** Outside the VG: the EFI system partition and /boot. */ +export const V2_BOOT = { + efiMib: 1024, // was 600 - firmware and shim have grown + bootMib: 4096, // was 3072 - several kernels + initramfs +} as const; + +const V2_BASE: readonly LvSpec[] = [ + { name: "swap", mount: null, sizeMib: 16384, fs: "swap" }, + { name: "root", mount: "/", sizeMib: 32768, fs: "xfs" }, + { name: "var", mount: "/var", sizeMib: 49152, fs: "xfs" }, + { name: "varlog", mount: "/var/log", sizeMib: 16384, fs: "xfs" }, + { name: "home", mount: "/home", sizeMib: 8192, fs: "xfs" }, + { name: "srv", mount: "/srv", sizeMib: 16384, fs: "xfs" }, +] as const; + +/** Image store AND local-path PVCs both live here - size for both. */ +export const V2_RANCHER_MIB = 256000; // 250 GiB +export const V2_LONGHORN_MIB = 307200; // 300 GiB + +/** + * The v2 volumes for a role, gated by CAPABILITY rather than role name. + * + * `omitRoot` is for Asahi, where the vendor installer already created a btrfs + * root partition outside our VG and we must not make a second one. + */ +export function v2Layout(role: Role, opts: { omitRoot?: boolean } = {}): LvSpec[] { + const info = roleInfo(role); + const lvs: LvSpec[] = V2_BASE.filter( + (lv) => !(opts.omitRoot && lv.name === "root"), + ).map((lv) => ({ ...lv })); + + // Anything running k3s needs a dedicated image store. This is the check that + // `role === "infra" || role === "worker"` kept getting wrong: labcontroller + // runs k3s and was silently excluded. + if (info.k3sMode !== null) { + lvs.push({ name: "rancher", mount: "/var/lib/rancher", sizeMib: V2_RANCHER_MIB, fs: "xfs" }); + } + + // Fixed, never --grow. A control-plane-only node holds no replicas. + if (info.k3sMode !== null && !info.controlPlaneOnly) { + lvs.push({ name: "longhorn", mount: "/var/lib/longhorn", sizeMib: V2_LONGHORN_MIB, fs: "xfs" }); + } + + return lvs; +} + +/** Total MiB a role's v2 layout consumes inside the VG. */ +export function v2AllocatedMib(role: Role, opts: { omitRoot?: boolean } = {}): number { + return v2Layout(role, opts).reduce((n, lv) => n + lv.sizeMib, 0); +} + +/** + * Bind-mounting containerd onto the Longhorn LV. + * + * Codifies what was already done BY HAND on worker1 and exists in no template: + * its 20 GiB rancher LV could not hold the image store, so containerd was + * pointed at a directory on the big Longhorn LV. It survives reboot only + * because someone wrote a correct fstab line; a reinstall would lose it. + * + * v2 sizes rancher properly so this is not needed on new boxes - it is kept for + * rescuing v1 machines that cannot be repartitioned yet. + */ +export const CONTAINERD_BIND_FSTAB = + "/var/lib/longhorn/k3s-containerd /var/lib/rancher/k3s/agent/containerd none " + + "bind,x-systemd.requires-mounts-for=/var/lib/longhorn," + + "x-systemd.requires-mounts-for=/var/lib/rancher 0 0"; diff --git a/bastion/src/shared/src/types/state.ts b/bastion/src/shared/src/types/state.ts index 93b30bb..e3ecfdd 100644 --- a/bastion/src/shared/src/types/state.ts +++ b/bastion/src/shared/src/types/state.ts @@ -28,8 +28,32 @@ export interface HardwareInfo { bastionId?: string; // set when aggregated through labd } -export type Role = "vanilla" | "worker" | "infra" | "labcontroller"; -export const SUPPORTED_ROLES: readonly Role[] = ["vanilla", "worker", "infra", "labcontroller"] as const; +export type Role = + // Legacy roles. Every machine in the fleet today is one of these, and they + // default to the v1 disk layout so a reinstall reproduces what is on disk. + | "vanilla" | "worker" | "infra" | "labcontroller" + // Topology-named profiles. New machines only, so they are v2-only: there is + // no v1 layout to be compatible with, and none of them can reach the frozen + // v1 branch in the templates. + | "standalone-cluster" | "k8s-controller" | "k8s-worker" | "k8s-combo"; + +export const SUPPORTED_ROLES: readonly Role[] = [ + "vanilla", "worker", "infra", "labcontroller", + "standalone-cluster", "k8s-controller", "k8s-worker", "k8s-combo", +] as const; + +/** + * How a machine comes to exist, which decides whether we partition it at all. + * + * - "managed" we PXE-install it with our kickstart; our disk layout applies. + * - "appliance" we install it, but the vendor installer owns the disk (VyOS). + * - "onboard" we never install it; we adopt a running machine (DGX Spark, + * Apple Silicon under Asahi). No partitioning, ever. + */ +export type Lifecycle = "managed" | "appliance" | "onboard"; + +/** null = no k3s at all. */ +export type K3sMode = "server" | "agent" | null; export interface RoleInfo { name: Role; @@ -37,6 +61,18 @@ export interface RoleInfo { parent?: Role; // inherits from this role k3s: boolean; // installs k3s apps: string[]; // apps auto-deployed after k3s + + // --- capabilities ------------------------------------------------------- + // These exist so decisions are made by asking what a machine DOES, not by + // comparing its role name. Every `role === "infra" || role === "labcontroller"` + // in this codebase is a place that forgot a role: `labcontroller` runs k3s and + // still got no rancher LV, because install.ks.ts asked `role === "infra"`. + lifecycle: Lifecycle; + k3sMode: K3sMode; + /** Holds Longhorn replicas, i.e. wants a dedicated /var/lib/longhorn LV. */ + hostsLonghorn: boolean; + /** Control plane that deliberately runs no workloads and no storage. */ + controlPlaneOnly: boolean; } export const ROLE_REGISTRY: readonly RoleInfo[] = [ @@ -45,6 +81,10 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [ description: "OS only — no k3s, no cluster services", k3s: false, apps: [], + lifecycle: "managed", + k3sMode: null, + hostsLonghorn: false, + controlPlaneOnly: false, }, { name: "worker", @@ -52,6 +92,10 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [ parent: "vanilla", k3s: true, apps: [], + lifecycle: "managed", + k3sMode: "agent", + hostsLonghorn: true, + controlPlaneOnly: false, }, { name: "infra", @@ -59,6 +103,15 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [ parent: "vanilla", k3s: true, apps: [], + lifecycle: "managed", + k3sMode: "server", + // NOTE: worker0/1/2 are `infra` and DO hold every Longhorn replica in the + // homelab cluster, but the v1 kickstart only ever gave a longhorn LV to + // `worker`. On worker2 that means Longhorn data sits on /var alongside the + // image store. Left false so the v1 layout is unchanged; v2 sizes it + // properly from this flag. + hostsLonghorn: false, + controlPlaneOnly: false, }, { name: "labcontroller", @@ -66,9 +119,104 @@ export const ROLE_REGISTRY: readonly RoleInfo[] = [ parent: "infra", k3s: true, apps: ["cockroachdb", "labd", "bastion"], + lifecycle: "managed", + k3sMode: "server", + hostsLonghorn: false, + controlPlaneOnly: false, + }, + { + name: "standalone-cluster", + description: "Single node that IS its own cluster — k3s server + workloads + storage", + parent: "vanilla", + k3s: true, + apps: [], + lifecycle: "managed", + k3sMode: "server", + // Longhorn yes, at replica 1. A cluster of one cannot satisfy a 3-replica + // policy, but keeping the `longhorn` StorageClass present means manifests + // are portable across every cluster instead of needing a local-path + // variant. Replica count is a property of the cluster, not the manifest. + hostsLonghorn: true, + controlPlaneOnly: false, + }, + { + name: "k8s-controller", + description: "Control plane + etcd only — runs no workloads and holds no storage", + parent: "vanilla", + k3s: true, + apps: [], + lifecycle: "managed", + k3sMode: "server", + hostsLonghorn: false, + controlPlaneOnly: true, + }, + { + name: "k8s-worker", + description: "k3s agent + Longhorn storage — joins an existing cluster", + parent: "vanilla", + k3s: true, + apps: [], + lifecycle: "managed", + k3sMode: "agent", + hostsLonghorn: true, + controlPlaneOnly: false, + }, + { + name: "k8s-combo", + description: "Control plane + etcd AND workloads + storage — hyperconverged cluster member", + parent: "vanilla", + k3s: true, + apps: [], + lifecycle: "managed", + k3sMode: "server", + hostsLonghorn: true, + controlPlaneOnly: false, }, ] as const; +/** + * Capability lookups. Use these instead of comparing role names. + * + * Every call site that asks `role === "infra" || role === "labcontroller"` is a + * place that has to be edited again when a role is added — and the one that was + * missed (install.ks.ts asking `role === "infra"`) is why `labcontroller` boxes + * get no dedicated image-store LV. + */ +/** + * Narrow an untrusted string (state file, HTTP body) to a Role. + * State is JSON on disk and can contain anything, including a role that was + * removed in a later version -- so this is a guard, not a cast. + */ +export function isRole(value: string): value is Role { + return (SUPPORTED_ROLES as readonly string[]).includes(value); +} + +export function roleInfo(role: Role): RoleInfo { + const info = ROLE_REGISTRY.find((r) => r.name === role); + if (!info) throw new Error(`unknown role: ${role}`); + return info; +} + +/** Runs k3s in any capacity. */ +export function runsK3s(role: Role): boolean { + return roleInfo(role).k3sMode !== null; +} + +/** Runs k3s as a server (control plane + etcd) rather than an agent. */ +export function isK3sServer(role: Role): boolean { + return roleInfo(role).k3sMode === "server"; +} + +/** We PXE-install this machine, so our disk layout applies to it. */ +export function isManaged(role: Role): boolean { + return roleInfo(role).lifecycle === "managed"; +} + +/** Wants a dedicated /var/lib/longhorn LV. */ +export function hostsLonghorn(role: Role): boolean { + return roleInfo(role).hostsLonghorn; +} + export interface ProgressLogEntry { stage: string; detail: string; @@ -202,6 +350,12 @@ export interface InstallConfig { os?: OsId; // defaults to "fedora-43" for backward compat vyos?: VyosInstallSpec; // only consulted when os is "vyos-rolling" arch?: Arch; // detected from HardwareInfo or overridden + /** + * Disk layout for a FRESH install. Defaults to 1. + * Ignored on a reprovision: %pre reads the lab-schema-v2 LVM tag off the + * disk instead, because reinstalling a machine must never change its layout. + */ + schemaVersion?: 1 | 2; queued_at: string; /** * When dispatch last served this machine an install boot script. Progress @@ -220,6 +374,12 @@ export interface InstallConfig { export interface InstalledInfo { hostname: string; role: string; + /** + * Disk layout this machine was installed with. Advisory only -- the + * authoritative answer is the `lab-schema-v2` LVM tag on the machine's own + * volume group, because that is what a reinstall actually reads. + */ + schemaVersion?: 1 | 2; os?: OsId; ip: string; installed_at: string; diff --git a/bastion/src/shared/tests/capabilities.test.ts b/bastion/src/shared/tests/capabilities.test.ts new file mode 100644 index 0000000..0a42f18 --- /dev/null +++ b/bastion/src/shared/tests/capabilities.test.ts @@ -0,0 +1,129 @@ +import { describe, it, expect } from "vitest"; +import { + SUPPORTED_ROLES, + ROLE_REGISTRY, + roleInfo, + runsK3s, + isK3sServer, + isManaged, + hostsLonghorn, +} from "../src/types/state.js"; +import type { Role } from "../src/types/state.js"; + +/** + * The capability lookups exist to replace role-name comparisons scattered across + * the codebase. Introducing them must change NO behaviour — so these tests pin + * each helper against the literal expression it replaces, copied verbatim from + * the call site it came from. + * + * If one of these fails, the refactor changed behaviour somewhere. That is the + * whole point of the file. + */ +/** + * Only the FOUR legacy roles have behaviour to preserve. The topology profiles + * (standalone-cluster, k8s-*) are new, v2-only, and have no historical + * expression to be compatible with -- pinning them against `role === "worker"` + * would assert nonsense. + */ +const LEGACY_ROLES = ["vanilla", "worker", "infra", "labcontroller"] as const; + +describe("capability helpers reproduce the expressions they replace", () => { + // Verbatim from the call sites, as of the commit that introduced this file. + const legacy = { + // k3s-install.ts:7, k3s-config.ts:7, render-config.ts:25, + // k3s-module.ts:65, health/k3s-service.ts:5, post-provision.ts:153 + isServer: (r: Role) => r === "infra" || r === "labcontroller", + // ROLE_REGISTRY.k3s — the field that already existed and nobody read + runsK3s: (r: Role) => ROLE_REGISTRY.find((x) => x.name === r)!.k3s, + // install.ks.ts:41, ubuntu-autoinstall.ts:34 + hasLonghorn: (r: Role) => r === "worker", + // install.ks.ts:48 + isVanilla: (r: Role) => r === "vanilla", + }; + + it.each(LEGACY_ROLES)("isK3sServer matches the legacy expression for %s", (role) => { + expect(isK3sServer(role)).toBe(legacy.isServer(role)); + }); + + it.each(LEGACY_ROLES)("runsK3s matches ROLE_REGISTRY.k3s for %s", (role) => { + expect(runsK3s(role)).toBe(legacy.runsK3s(role)); + }); + + it.each(LEGACY_ROLES)("hostsLonghorn matches the legacy expression for %s", (role) => { + expect(hostsLonghorn(role)).toBe(legacy.hasLonghorn(role)); + }); + + it.each(LEGACY_ROLES)("k3sMode === null matches isVanilla for %s", (role) => { + expect(roleInfo(role).k3sMode === null).toBe(legacy.isVanilla(role)); + }); + + it("every supported role is managed today (nothing is onboarded yet)", () => { + for (const role of SUPPORTED_ROLES) expect(isManaged(role)).toBe(true); + }); +}); + +/** + * The topology profiles. These are asserted directly rather than against a + * legacy expression, because they describe cluster SHAPE -- which is the thing + * role names were supposed to convey and did not. + */ +describe("topology profiles", () => { + it("standalone-cluster is a server that also holds storage", () => { + expect(isK3sServer("standalone-cluster")).toBe(true); + // Longhorn at replica 1: a cluster of one cannot do 3, but keeping the + // StorageClass present keeps manifests portable across clusters. + expect(hostsLonghorn("standalone-cluster")).toBe(true); + expect(roleInfo("standalone-cluster").controlPlaneOnly).toBe(false); + }); + + it("k8s-controller is the only role that runs k3s and holds no storage", () => { + expect(isK3sServer("k8s-controller")).toBe(true); + expect(hostsLonghorn("k8s-controller")).toBe(false); + expect(roleInfo("k8s-controller").controlPlaneOnly).toBe(true); + }); + + it("k8s-worker is an agent with storage", () => { + expect(isK3sServer("k8s-worker")).toBe(false); + expect(runsK3s("k8s-worker")).toBe(true); + expect(hostsLonghorn("k8s-worker")).toBe(true); + }); + + it("k8s-combo is both control plane and storage — unlike k8s-controller", () => { + expect(isK3sServer("k8s-combo")).toBe(true); + expect(hostsLonghorn("k8s-combo")).toBe(true); + expect(roleInfo("k8s-combo").controlPlaneOnly).toBe(false); + }); + + it("k8s-combo and k8s-controller differ ONLY in storage duty", () => { + const combo = roleInfo("k8s-combo"); + const ctrl = roleInfo("k8s-controller"); + expect(combo.k3sMode).toBe(ctrl.k3sMode); + expect(combo.hostsLonghorn).not.toBe(ctrl.hostsLonghorn); + }); +}); + +describe("registry integrity", () => { + it("every SUPPORTED_ROLE has a registry entry", () => { + for (const role of SUPPORTED_ROLES) expect(() => roleInfo(role)).not.toThrow(); + }); + + it("registry has no entries outside SUPPORTED_ROLES", () => { + for (const info of ROLE_REGISTRY) expect(SUPPORTED_ROLES).toContain(info.name); + }); + + it("k3s boolean and k3sMode never disagree", () => { + for (const info of ROLE_REGISTRY) { + expect(info.k3s).toBe(info.k3sMode !== null); + } + }); + + it("an unknown role throws rather than silently defaulting", () => { + expect(() => roleInfo("does-not-exist" as Role)).toThrow(/unknown role/); + }); + + it("a control-plane-only role never hosts Longhorn", () => { + for (const info of ROLE_REGISTRY) { + if (info.controlPlaneOnly) expect(info.hostsLonghorn).toBe(false); + } + }); +}); diff --git a/bastion/src/shared/tests/layout.test.ts b/bastion/src/shared/tests/layout.test.ts new file mode 100644 index 0000000..0b118c6 --- /dev/null +++ b/bastion/src/shared/tests/layout.test.ts @@ -0,0 +1,92 @@ +import { describe, it, expect } from "vitest"; +import { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB } from "../src/types/layout.js"; +import { SUPPORTED_ROLES, roleInfo } from "../src/types/state.js"; + +/** A 1 TB disk is the smallest we will ever buy. 1e12 bytes in MiB. */ +const ONE_TB_MIB = Math.floor(1_000_000_000_000 / (1024 * 1024)); // 953674 +const VG_MIB_ON_1TB = ONE_TB_MIB - V2_BOOT.efiMib - V2_BOOT.bootMib; + +describe("v2 layout fits the smallest disk we will buy", () => { + it.each(SUPPORTED_ROLES)("%s fits in a 1 TB disk with headroom left over", (role) => { + const used = v2AllocatedMib(role); + expect(used).toBeLessThan(VG_MIB_ON_1TB); + // The entire point of v2 is to STOP consuming the disk. Demand real slack. + const freeMib = VG_MIB_ON_1TB - used; + expect(freeMib).toBeGreaterThan(100 * 1024); // >100 GiB unallocated + }); + + it("the heaviest role still leaves >200 GiB free on 1 TB", () => { + const worst = Math.max(...SUPPORTED_ROLES.map((r) => v2AllocatedMib(r))); + expect(VG_MIB_ON_1TB - worst).toBeGreaterThan(200 * 1024); + }); +}); + +describe("v2 gates volumes by capability, not role name", () => { + it.each(SUPPORTED_ROLES)("%s gets a rancher LV iff it runs k3s", (role) => { + const hasRancher = v2Layout(role).some((lv) => lv.name === "rancher"); + expect(hasRancher).toBe(roleInfo(role).k3sMode !== null); + }); + + it("labcontroller gets a rancher LV — the v1 bug this schema fixes", () => { + const lvs = v2Layout("labcontroller"); + const rancher = lvs.find((lv) => lv.name === "rancher"); + expect(rancher).toBeDefined(); + expect(rancher!.sizeMib).toBe(V2_RANCHER_MIB); + }); + + it("vanilla gets neither rancher nor longhorn", () => { + const names = v2Layout("vanilla").map((lv) => lv.name); + expect(names).not.toContain("rancher"); + expect(names).not.toContain("longhorn"); + }); + + it("a control-plane-only role would get no longhorn LV", () => { + for (const role of SUPPORTED_ROLES) { + if (roleInfo(role).controlPlaneOnly) { + expect(v2Layout(role).map((l) => l.name)).not.toContain("longhorn"); + } + } + }); +}); + +describe("v2 never over-allocates", () => { + it("no volume is grow/unbounded — every size is a concrete number", () => { + for (const role of SUPPORTED_ROLES) { + for (const lv of v2Layout(role)) { + expect(Number.isFinite(lv.sizeMib)).toBe(true); + expect(lv.sizeMib).toBeGreaterThan(0); + } + } + }); + + it("longhorn is fixed at 300 GiB, not grown", () => { + const lh = v2Layout("worker").find((lv) => lv.name === "longhorn"); + expect(lh?.sizeMib).toBe(V2_LONGHORN_MIB); + expect(V2_LONGHORN_MIB).toBe(300 * 1024); + }); + + it("rancher is sized for an image store AND local-path PVCs", () => { + // A single ROCm vLLM image measured 62 GiB extracted on 2026-09-17. + expect(V2_RANCHER_MIB).toBeGreaterThan(200 * 1024); + }); +}); + +describe("Asahi variant", () => { + it("omitRoot drops the root LV — the vendor installer already made one", () => { + const names = v2Layout("infra", { omitRoot: true }).map((lv) => lv.name); + expect(names).not.toContain("root"); + expect(names).toContain("var"); + }); + + it("omitRoot frees the root allocation and nothing else", () => { + const delta = v2AllocatedMib("infra") - v2AllocatedMib("infra", { omitRoot: true }); + expect(delta).toBe(32768); + }); +}); + +describe("layout is a pure value, safe to render three times", () => { + it("mutating a returned spec does not affect the next call", () => { + v2Layout("worker")[0]!.sizeMib = 999; + expect(v2Layout("worker")[0]!.sizeMib).not.toBe(999); + }); +}); diff --git a/bastion/tests/integration/cluster-smoke.test.ts b/bastion/tests/integration/cluster-smoke.test.ts new file mode 100644 index 0000000..48e1ea0 --- /dev/null +++ b/bastion/tests/integration/cluster-smoke.test.ts @@ -0,0 +1,164 @@ +// Integration test: form a real 3-server k3s cluster from PXE-installed VMs. +// +// WHY THIS EXISTS SEPARATELY FROM THE DRILL +// ----------------------------------------- +// The rolling drill is a ~3 hour test. Almost everything that can go wrong in +// its first thirty minutes goes wrong here instead, in forty-five: the token +// handoff, the cluster-init/join discriminator, the etcd bootstrap race, root +// SSH for the module, and whether post-provision was actually suppressed. +// Debugging those inside a three-hour test is how you spend a day per attempt. +// +// It asserts on ETCD MEMBERSHIP, not just `kubectl get nodes`. A node object +// and an etcd member are different things, and the whole hazard in a rolling +// reinstall is that they can disagree. +// +// Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo. Set FEDORA_MIRROR to +// a local mirror or each install pulls the full package set over the internet. +// +// sudo -E npx vitest run -c tests/integration/vitest.config.ts -t "cluster smoke" + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { existsSync, readFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { + startTestBastion, provisionVms, destroyVms, + type Harness, type ProvisionedVm, +} from "./helpers/provision-harness.js"; +import { + formCluster, clusterSnapshot, printSnapshot, etcdMembers, etcdHealthy, + allServersReady, pickJoinTarget, kubectlOn, k8sNodeName, +} from "./helpers/k3s-cluster.js"; +import { destroyPxeNetwork } from "./helpers/pxe-network.js"; + +const HTTP_PORT = 8097; +const SSH_USER = "lab"; +const DISK_GB = 1000; +const NODES = ["smoke-a", "smoke-b", "smoke-c"]; + +function findSshKey(): { pubKey: string; keyPath: string } { + const homes = [homedir()]; + const sudoUser = process.env["SUDO_USER"]; + if (sudoUser) homes.push(join("/home", sudoUser)); + for (const home of homes) { + for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) { + const keyPath = join(home, ".ssh", name); + if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) { + return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath }; + } + } + } + throw new Error("no SSH keypair found"); +} + +describe("cluster smoke: three servers, one cluster", () => { + let h: Harness; + let vms: ProvisionedVm[] = []; + + /** + * Kubernetes node names, NOT libvirt domain names. + * + * k3s registers a node by FQDN, so the VM `smoke-a` is the node + * `smoke-a.pxe-test.local`. Asserting against the VM names would compare two + * different namespaces and fail for a reason that has nothing to do with the + * cluster. + */ + const nodeNames = (): string[] => vms.map((v) => k8sNodeName(h, v)).sort(); + + beforeAll(async () => { + const { pubKey, keyPath } = findSshKey(); + destroyVms(...NODES); + h = await startTestBastion({ + httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey, sshKeyPath: keyPath, + label: "cluster-smoke", + // Mandatory. Left on, post-provision installs each node into its OWN + // cluster and this test would be measuring three clusters of one. + autoK3s: false, + }); + }, 20 * 60_000); + + afterAll(async () => { + // Dump state before anything is destroyed — the VMs are the only place + // this information exists, and they are about to stop existing. + try { + if (vms.length > 0) printSnapshot("final", clusterSnapshot(h, vms[0]!)); + } catch { /* cluster may already be gone */ } + destroyVms(...NODES); + try { destroyPxeNetwork(); } catch { /* shared */ } + await h?.close(); + }, 10 * 60_000); + + it("provisions three v2 combo nodes", async () => { + vms = await provisionVms(h, NODES.map((name) => ({ + name, role: "k8s-combo" as const, schemaVersion: 2 as const, + diskGb: DISK_GB, memory: 6144, vcpus: 4, + }))); + expect(vms).toHaveLength(3); + for (const vm of vms) expect(vm.ip).toMatch(/^\d+\.\d+\.\d+\.\d+$/); + }, 60 * 60_000); + + it("forms ONE cluster, not three", async () => { + await formCluster(h, vms, { role: "k8s-combo" }); + + const snap = clusterSnapshot(h, vms[0]!); + printSnapshot("after formation", snap); + + // The precise failure this guards: three separate single-node clusters + // each report 1/1 Ready and look perfectly healthy in isolation. + expect(snap.nodes).toHaveLength(3); + expect(snap.readyCount).toBe(3); + expect(snap.nodes.map((n) => n.name).sort()).toEqual(nodeNames()); + }, 90 * 60_000); + + it("has exactly three healthy etcd members", () => { + const members = etcdMembers(h, vms[0]!); + // .node, not .name: etcd names a member "-<8 hex>", so comparing + // raw names against node names never matches. + expect(members.map((m) => m.node).sort()).toEqual(nodeNames()); + expect(members).toHaveLength(3); + expect(members.every((m) => m.started)).toBe(true); + expect(etcdHealthy(h, vms[0]!)).toBe(true); + }, 5 * 60_000); + + it("agrees about membership from EVERY node, not just the first", () => { + // Asked of each server independently: a split brain is three nodes that + // each believe they are correct. + for (const vm of vms) { + const members = etcdMembers(h, vm).map((m) => m.node).sort(); + expect(members, `etcd membership as seen by ${vm.name}`).toEqual(nodeNames()); + const nodes = clusterSnapshot(h, vm).nodes.map((n) => n.name).sort(); + expect(nodes, `node list as seen by ${vm.name}`).toEqual(nodeNames()); + } + }, 10 * 60_000); + + it("reports every server ready", () => { + const ready = allServersReady(h, vms); + expect(ready.ok, ready.detail).toBe(true); + }, 5 * 60_000); + + it("makes all three control-plane and etcd", () => { + const snap = clusterSnapshot(h, vms[0]!); + for (const n of snap.nodes) { + expect(n.roles, `${n.name} roles`).toContain("control-plane"); + expect(n.roles, `${n.name} roles`).toContain("etcd"); + } + }, 5 * 60_000); + + it("can pick a join target other than a given node", () => { + // The primitive the rolling drill depends on: after the first server is + // wiped it must rejoin against a survivor. + const target = pickJoinTarget(h, vms, vms[0]!); + expect(target.name).not.toBe(vms[0]!.name); + }, 5 * 60_000); + + it("carries the Longhorn disk label on every node", () => { + // Set via k3s node-label. Without it Longhorn creates no disk on the node, + // and the node then hosts zero replicas while looking entirely healthy. + const out = kubectlOn(h, vms[0]!, + `get nodes -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.labels.node\\.longhorn\\.io/create-default-disk}{"\\n"}{end}'`); + for (const vm of vms) { + expect(out, `${vm.name} create-default-disk label`) + .toContain(`${k8sNodeName(h, vm)}=config`); + } + }, 5 * 60_000); +}); diff --git a/bastion/tests/integration/helpers/k3s-cluster.ts b/bastion/tests/integration/helpers/k3s-cluster.ts new file mode 100644 index 0000000..109bdba --- /dev/null +++ b/bastion/tests/integration/helpers/k3s-cluster.ts @@ -0,0 +1,427 @@ +// Form and inspect a real multi-node k3s cluster from an integration test. +// +// WHY THIS EXISTS +// --------------- +// Nothing in the repo decides which node bootstraps a cluster and which nodes +// join it. The production generator has the right discriminator -- +// generateServerConfig emits `cluster-init: true` when no server URL is set and +// `server:`+`token:` when one is -- but the choice is an INPUT, and the only +// caller that ever supplied it was a hand-run labsim shell script. +// +// So a test that wants three servers in one cluster has to sequence it. That +// sequencing is here rather than in a test file because the rolling drill is +// not the only thing that will want it: an etcd-recovery test (recoverEtcdMember +// still has no caller), a dual-stack conversion and any HA upgrade test all +// need exactly these primitives. +// +// Everything drives K3sModule, not shell. That is deliberate and is the same +// rule labsim follows by shelling out to the compiled render-config binary: a +// test that reimplements the install proves the test works, not the product. + +import { execSync } from "node:child_process"; +import type { Role } from "../../../src/shared/src/types/state.js"; +import { K3sModule } from "../../../src/modules/modules/k3s/src/k3s-module.js"; +import type { ModuleContext } from "../../../src/modules/src/types.js"; +import { log } from "./libvirt.js"; +import { parseEtcdMembers, type EtcdMember } from "../../../src/modules/modules/k3s/src/operations/etcd-members.js"; +import { sshExec } from "./ssh.js"; +import type { Harness, ProvisionedVm } from "./provision-harness.js"; + +export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +/** + * A pre-shared cluster token, the way labsim does it. + * + * The alternative is scraping /var/lib/rancher/k3s/server/node-token off the + * first server, and nothing in the repo does that. It is also the wrong shape + * for a rolling migration: the first server gets wiped and rejoins, so the + * token has to outlive it and be knowable before it exists. + */ +export const DRILL_TOKEN = "lab-drill-shared-token"; + +/** + * k3s operations run unprefixed `mkdir -p /etc/rancher/k3s` and + * `systemctl restart k3s`, so they need root. install.ks.ts provisions + * `sshkey --username=root` and sets PermitRootLogin prohibit-password, so root + * SSH works on every machine this harness builds. The harness's own `lab` user + * stays for the sudo-based reads (readLvs, markers). + */ +export const MODULE_SSH_USER = "root"; + +/** + * The k3s version the drill runs, pinned to what the fleet runs. + * + * get.k3s.io installs LATEST, and latest is currently two minor versions ahead + * of production (v1.36.4 vs v1.34.6). Two things follow, and both matter: + * + * - a rehearsal on a different minor version is rehearsing a different + * system, and Longhorn in particular is only qualified against a range of + * Kubernetes versions; + * - it is the same trap production would hit. A node reinstalled mid-migration + * comes back on latest while its peers stay put, and Kubernetes supports + * only ONE minor of skew between control-plane members. + * + * Override with LAB_K3S_VERSION to test an upgrade deliberately. + */ +export const PINNED_K3S_VERSION = process.env["LAB_K3S_VERSION"] ?? "v1.34.6+k3s1"; + +/** Build the ModuleContext K3sModule expects for one VM. */ +export function moduleCtx( + h: Harness, + vm: ProvisionedVm, + role: Role, + config: Record = {}, +): ModuleContext { + return { + hostname: vm.name, + ip: vm.ip, + role, + os: "fedora-43", + arch: "x86_64", + sshUser: MODULE_SSH_USER, + sshKeyPath: h.sshKeyPath, + config, + }; +} + +/** Run a command on a node as root, throwing with context on failure. */ +export function nodeExec(h: Harness, vm: ProvisionedVm, cmd: string, timeoutMs = 120_000): string { + const r = sshExec(vm.ip, MODULE_SSH_USER, cmd, { keyPath: h.sshKeyPath, timeout: timeoutMs }); + if (r.exitCode !== 0) { + throw new Error(`[${vm.name}] ${cmd} failed (${r.exitCode}): ${r.stderr.trim() || r.stdout.trim()}`); + } + return r.stdout; +} + +/** Same, but returns the result instead of throwing. */ +export function tryNodeExec(h: Harness, vm: ProvisionedVm, cmd: string, timeoutMs = 120_000) { + return sshExec(vm.ip, MODULE_SSH_USER, cmd, { keyPath: h.sshKeyPath, timeout: timeoutMs }); +} + +/** + * The name Kubernetes knows this VM by. + * + * NOT the libvirt domain name. k3s registers a node under its FQDN, so a VM + * called `smoke-a` appears as `smoke-a.pxe-test.local`. Passing the VM name to + * `kubectl get node` silently finds nothing, and every wait built on it times + * out instead of failing usefully. Read from the node itself, the same way the + * production operations do (`hostname -f`), so the two can never disagree. + */ +const nodeNameCache = new Map(); +export function k8sNodeName(h: Harness, vm: ProvisionedVm): string { + const cached = nodeNameCache.get(vm.name); + if (cached !== undefined) return cached; + const name = nodeExec(h, vm, "hostname -f 2>/dev/null || hostname").trim(); + if (name === "") throw new Error(`[${vm.name}] could not resolve node name`); + nodeNameCache.set(vm.name, name); + return name; +} + +/** Forget cached names — a reinstalled machine re-registers. */ +export function forgetNodeName(vm: ProvisionedVm): void { + nodeNameCache.delete(vm.name); +} + +/** kubectl against a server node. */ +export function kubectlOn(h: Harness, server: ProvisionedVm, args: string, timeoutMs = 120_000): string { + return nodeExec(h, server, `k3s kubectl ${args}`, timeoutMs); +} + +export function kubectlJson(h: Harness, server: ProvisionedVm, args: string): T { + return JSON.parse(kubectlOn(h, server, `${args} -o json`)) as T; +} + +/** + * Wait until a server's API answers /ping. + * + * Straight from labsim-k8s-etcd.sh, whose comment is the whole justification: + * a joiner that starts before the first server has finished bootstrapping etcd + * races it, and the failure looks like etcd instability rather than a startup + * order problem. Probed from the HOST, so it does not depend on the joining + * node being reachable yet. + */ +export async function waitForApiPing(vm: ProvisionedVm, timeoutMs = 450_000): Promise { + const start = Date.now(); + let lastErr = ""; + while (Date.now() - start < timeoutMs) { + try { + const out = execSync( + `curl -sk --max-time 5 https://${vm.ip}:6443/ping || true`, + { encoding: "utf-8", timeout: 15_000 }, + ).trim(); + if (out === "pong") { + log(`[${vm.name}] API answering after ${Math.round((Date.now() - start) / 1000)}s`); + return; + } + lastErr = out; + } catch (e) { + lastErr = e instanceof Error ? e.message : String(e); + } + await sleep(5_000); + } + throw new Error(`[${vm.name}] API never answered /ping in ${timeoutMs / 1000}s (last: ${lastErr || "no response"})`); +} + +/** Wait for a node to appear Ready in the cluster, as seen from `server`. */ +export async function waitForNodeReady( + h: Harness, + server: ProvisionedVm, + /** The VM to wait for; its Kubernetes name is resolved from the machine. */ + target: ProvisionedVm, + // Generous: a node is NotReady until its CNI is up, and on the cluster-init + // server that means waiting for Cilium to install and pull its images. Five + // minutes is enough on a warm host and not enough on a cold one, and the + // cost of being wrong is a wasted multi-hour run. + timeoutMs = 900_000, +): Promise { + const nodeName = k8sNodeName(h, target); + const start = Date.now(); + let last = ""; + while (Date.now() - start < timeoutMs) { + const r = tryNodeExec(h, server, + `k3s kubectl get node ${nodeName} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}'`); + last = r.stdout.trim(); + if (last === "True") { + log(`[${nodeName}] Ready after ${Math.round((Date.now() - start) / 1000)}s`); + return; + } + await sleep(5_000); + } + throw new Error(`[${nodeName}] never became Ready in ${timeoutMs / 1000}s (last status: ${last || "absent"})`); +} + +/** + * etcdctl invocation. + * + * NOT `k3s etcdctl` -- that is not a subcommand ("No help topic for 'etcdctl'", + * verified on a live v1.34 node). The binary comes from the `etcd` rpm, which + * our kickstart does NOT install, so ensureEtcdctl() has to put it there. + * Cert paths verified against a real node. + */ +const ETCD_BASE = "/var/lib/rancher/k3s/server/tls/etcd"; +const ETCDCTL = + `ETCDCTL_API=3 etcdctl ` + + `--cacert=${ETCD_BASE}/server-ca.crt ` + + `--cert=${ETCD_BASE}/server-client.crt ` + + `--key=${ETCD_BASE}/server-client.key ` + + `--endpoints=https://127.0.0.1:2379`; + +/** Install etcdctl if the node does not have it. Idempotent, ~30s once. */ +export function ensureEtcdctl(h: Harness, server: ProvisionedVm): void { + const has = tryNodeExec(h, server, "command -v etcdctl"); + if (has.exitCode === 0) return; + log(`[${server.name}] installing etcdctl (the etcd rpm; kickstart does not include it)`); + nodeExec(h, server, "dnf install -y etcd >/dev/null 2>&1 || dnf install -y etcd-ctl >/dev/null 2>&1", 300_000); + nodeExec(h, server, "command -v etcdctl"); +} + + +/** + * etcd membership, read with etcdctl against k3s's own certs. + * + * Read rather than inferred from `kubectl get nodes`: a node object can be + * deleted while its etcd member survives, and that stale member is exactly the + * failure a rolling reinstall creates. Counting nodes would not see it. + */ +export function etcdMembers(h: Harness, server: ProvisionedVm): EtcdMember[] { + // NOT `-w json`. Two reasons, and the second is the dangerous one: + // + // 1. `member remove` requires the ID in HEX; the JSON writer emits decimal. + // 2. etcd member IDs are uint64. JSON.parse turns them into JS numbers, + // which silently lose precision above 2^53 -- the ID that comes back is + // an APPROXIMATION of the real one. Passing that to `member remove` + // fails with "value out of range" if you are lucky, and in principle + // could match some other member if you are not. This happens at the one + // moment it must not: the node is already stopped and about to be wiped. + // + // The default writer prints the ID as hex text, so it never becomes a + // number at all: + // 825852d3683bc406, started, worker2-...-5f7ad0c9, https://...:2380, ... + return parseEtcdMembers(nodeExec(h, server, `${ETCDCTL} member list`)); +} + + +/** Are all etcd endpoints healthy? Answers the question quorum actually asks. */ +export function etcdHealthy(h: Harness, server: ProvisionedVm): boolean { + const r = tryNodeExec(h, server, `${ETCDCTL} endpoint health`); + return r.exitCode === 0 && r.stdout.includes("is healthy"); +} + +/** + * Remove an etcd member by node name. + * + * Must happen BEFORE the node is wiped. k3s-install.ts removes the local + * datastore when a server joins, so a reinstalled node rejoins as a NEW member + * while the old record persists: 3 members becomes 4, quorum rises from 2 to 3, + * and the cluster that tolerated one failure now tolerates none. + */ +export function removeEtcdMember(h: Harness, peer: ProvisionedVm, nodeName: string): boolean { + const members = etcdMembers(h, peer); + const target = members.find((m) => m.node === nodeName || m.name === nodeName); + if (!target) { + log(`[etcd] no member for ${nodeName} (have: ${members.map((m) => m.node).join(",")})`); + return false; + } + nodeExec(h, peer, `${ETCDCTL} member remove ${target.id}`); + log(`[etcd] removed member ${target.name} (${target.id})`); + return true; +} + +export interface ClusterSnapshot { + nodes: { name: string; ready: boolean; roles: string }[]; + readyCount: number; + etcdMembers: number; + etcdHealthy: boolean; +} + +/** One read of cluster state, in the shape labsim's cmd_status prints. */ +export function clusterSnapshot(h: Harness, server: ProvisionedVm): ClusterSnapshot { + const raw = kubectlJson<{ items: { + metadata: { name: string; labels?: Record }; + status: { conditions?: { type: string; status: string }[] }; + }[] }>(h, server, "get nodes"); + + const nodes = raw.items.map((n) => ({ + name: n.metadata.name, + ready: (n.status.conditions ?? []).some((c) => c.type === "Ready" && c.status === "True"), + roles: Object.keys(n.metadata.labels ?? {}) + .filter((k) => k.startsWith("node-role.kubernetes.io/")) + .map((k) => k.replace("node-role.kubernetes.io/", "")) + .sort().join(",") || "", + })); + + let members = 0; + let healthy = false; + try { + members = etcdMembers(h, server).length; + healthy = etcdHealthy(h, server); + } catch { /* an agent, or the API is down — report zero rather than throw */ } + + return { nodes, readyCount: nodes.filter((n) => n.ready).length, etcdMembers: members, etcdHealthy: healthy }; +} + +/** + * Print a snapshot. Called liberally during a drill: the VMs are destroyed at + * the end, so this transcript is the only surviving record of what happened. + */ +export function printSnapshot(label: string, s: ClusterSnapshot): void { + log(`[snapshot] ${label}: ${s.readyCount}/${s.nodes.length} Ready, ` + + `etcd members=${s.etcdMembers} healthy=${s.etcdHealthy}`); + for (const n of s.nodes) log(` ${n.ready ? "Ready " : "NotReady"} ${n.name} (${n.roles})`); +} + +/** + * Install k3s on one node via the production module. + * + * Pass no serverUrl to bootstrap the cluster (the token still seeds), or a + * serverUrl to join an existing one. + */ +export async function installK3sNode(h: Harness, vm: ProvisionedVm, opts: { + role: Role; + serverUrl?: string; + token?: string; +}): Promise { + const mod = new K3sModule(); + const config: Record = { + k3sToken: opts.token ?? DRILL_TOKEN, + k3sVersion: PINNED_K3S_VERSION, + ...(opts.serverUrl === undefined ? {} : { k3sServerUrl: opts.serverUrl }), + }; + + log(`[${vm.name}] installing k3s (${opts.role}${opts.serverUrl ? ` -> ${opts.serverUrl}` : " — cluster-init"})`); + const install = await mod.install(moduleCtx(h, vm, opts.role, config)); + for (const line of install.output) log(` [${vm.name}] ${line}`); + if (!install.success) { + throw new Error(`[${vm.name}] k3s install failed: ${install.errors.join("; ")}`); + } + + const configure = await mod.configure(moduleCtx(h, vm, opts.role, config)); + for (const line of configure.output) log(` [${vm.name}] ${line}`); + if (!configure.success) { + throw new Error(`[${vm.name}] k3s configure failed: ${configure.errors.join("; ")}`); + } +} + +export function serverUrlOf(vm: ProvisionedVm): string { + return `https://${vm.ip}:6443`; +} + +/** + * Form a cluster from a list of server VMs, strictly in order. + * + * The first bootstraps etcd and installs the CNI (k3s-module runs networking + * only on the cluster-init node -- joiners get Cilium via its DaemonSet). The + * rest join one at a time, each gated on the previous node being Ready. + * + * Serial on purpose. labsim gets away with starting joiners together because + * its nodes are cloud-init VMs that poll /ping themselves; here the few extra + * minutes buy determinism in the one place where etcd bootstrap genuinely + * races. + */ +export async function formCluster(h: Harness, servers: ProvisionedVm[], opts: { + role?: Role; + token?: string; +} = {}): Promise { + if (servers.length === 0) throw new Error("formCluster: no servers"); + const role = opts.role ?? "k8s-combo"; + const token = opts.token ?? DRILL_TOKEN; + const first = servers[0]!; + + await installK3sNode(h, first, { role, token }); + await waitForApiPing(first); + await waitForNodeReady(h, first, first); + ensureEtcdctl(h, first); + printSnapshot("after cluster-init", clusterSnapshot(h, first)); + + for (const vm of servers.slice(1)) { + await waitForApiPing(first); + await installK3sNode(h, vm, { role, token, serverUrl: serverUrlOf(first) }); + await waitForNodeReady(h, first, vm); + ensureEtcdctl(h, vm); + printSnapshot(`after joining ${vm.name}`, clusterSnapshot(h, first)); + } +} + +/** Join an agent. Agents cannot start without a server URL and token. */ +export async function joinAgent(h: Harness, vm: ProvisionedVm, opts: { + server: ProvisionedVm; + role?: Role; + token?: string; +}): Promise { + await waitForApiPing(opts.server); + await installK3sNode(h, vm, { + role: opts.role ?? "k8s-worker", + token: opts.token ?? DRILL_TOKEN, + serverUrl: serverUrlOf(opts.server), + }); + await waitForNodeReady(h, opts.server, vm); +} + +/** + * A healthy server that is NOT the node being worked on. + * + * Needed because the first server is itself rolled during a migration: after + * it is wiped it must rejoin against a survivor, so "the first node" cannot be + * the join target and cannot be where cluster state is read from. + */ +export function pickJoinTarget( + h: Harness, + servers: ProvisionedVm[], + exclude: ProvisionedVm, +): ProvisionedVm { + for (const vm of servers) { + if (vm.name === exclude.name) continue; + const r = tryNodeExec(h, vm, "k3s kubectl get --raw /readyz"); + if (r.exitCode === 0 && r.stdout.trim() === "ok") return vm; + } + throw new Error(`No healthy server other than ${exclude.name} — cluster has no quorum to join against`); +} + +/** Every server reports /readyz ok. Asserted positively, not inferred. */ +export function allServersReady(h: Harness, servers: ProvisionedVm[]): { ok: boolean; detail: string } { + const results = servers.map((vm) => { + const r = tryNodeExec(h, vm, "k3s kubectl get --raw /readyz"); + return `${vm.name}=${r.exitCode === 0 ? r.stdout.trim() : `ERR(${r.exitCode})`}`; + }); + return { ok: results.every((r) => r.endsWith("=ok")), detail: results.join(" ") }; +} diff --git a/bastion/tests/integration/helpers/longhorn.ts b/bastion/tests/integration/helpers/longhorn.ts new file mode 100644 index 0000000..8f6eb05 --- /dev/null +++ b/bastion/tests/integration/helpers/longhorn.ts @@ -0,0 +1,666 @@ +// Install Longhorn on a test cluster and put verifiable data in it. +// +// WHAT THIS IS FOR +// ---------------- +// The rolling drill's whole claim is "the bytes survived". That needs data +// whose integrity can be checked independently of Longhorn's own opinion of +// itself -- Longhorn reporting `robustness: healthy` is not evidence that the +// file is intact, it is evidence that Longhorn thinks it is. +// +// So: write a known blob, record its sha256, and re-verify THROUGH A POD +// RESTART every time. The restart is not incidental. It forces a real +// detach/attach, frequently onto a different node, which is the only way to +// show the data followed the volume rather than sitting in a page cache on the +// node that wrote it. +// +// Longhorn version is pinned to the fleet's (1.8.1) so the drill rehearses what +// production runs, not what happens to be latest. + +import type { Harness, ProvisionedVm } from "./provision-harness.js"; +import { log } from "./libvirt.js"; +import { nodeExec, tryNodeExec, kubectlOn, kubectlJson, sleep, k8sNodeName } from "./k3s-cluster.js"; + +export const LONGHORN_VERSION = "1.8.1"; +export const LONGHORN_NS = "longhorn-system"; + +/** + * Install Longhorn via k3s's built-in helm controller. + * + * A HelmChart manifest dropped in the server manifests directory is how k3s + * installs charts natively, so this needs no helm binary on the host and no + * kubeconfig plumbing. + * + * The namespace is created FIRST, with privileged Pod Security labels. Longhorn + * needs hostPath and privileged containers; under the default `restricted` + * enforcement its DaemonSets are rejected and the install fails in a way that + * looks like a chart problem. + * + * `createDefaultDiskLabeledNodes: true` matches production, and is the setting + * that makes the node label and disk annotation load-bearing: without both, a + * node gets no disk and silently hosts no replicas. + */ +/** + * S3 backup target for the drill, from the environment. + * + * Absent means the backup/restore phase is skipped rather than silently + * "passing" -- a backup test that quietly does nothing is worse than no test. + * The run script exports these; see DRILL_S3_* in run-drill.sh. + */ +export interface BackupTargetOpts { + /** e.g. s3://longhorn-drill-backup@us-east-1/ */ + target: string; + endpoint: string; + accessKey: string; + secretKey: string; +} + +export function backupTargetFromEnv(): BackupTargetOpts | undefined { + const target = process.env["DRILL_S3_TARGET"]; + const endpoint = process.env["DRILL_S3_ENDPOINT"]; + const accessKey = process.env["DRILL_S3_ACCESS_KEY"]; + const secretKey = process.env["DRILL_S3_SECRET_KEY"]; + if (target === undefined || endpoint === undefined + || accessKey === undefined || secretKey === undefined) return undefined; + return { target, endpoint, accessKey, secretKey }; +} + +export const BACKUP_SECRET = "longhorn-backup-credentials"; + +export async function installLonghorn(h: Harness, server: ProvisionedVm, opts: { + defaultReplicas?: number; + backup?: BackupTargetOpts; +} = {}): Promise { + const replicas = opts.defaultReplicas ?? 3; + + log(`[longhorn] creating ${LONGHORN_NS} with privileged PSA labels`); + nodeExec(h, server, `k3s kubectl apply -f - <<'EOF' +apiVersion: v1 +kind: Namespace +metadata: + name: ${LONGHORN_NS} + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged +EOF`); + + if (opts.backup !== undefined) { + // Secret BEFORE the chart: Longhorn reads it by name from a defaultSetting, + // and a missing secret makes the first backup fail with a credential error + // that reads like a network problem. + // + // AWS_ENDPOINTS is what makes this work against a non-AWS gateway; without + // it the driver resolves real AWS endpoints. + log(`[longhorn] configuring backup target ${opts.backup.target}`); + // tryNodeExec, not nodeExec: nodeExec throws with the whole command in the + // message, and this command contains the S3 secret. A credential landing in + // a log that then gets pasted into drill evidence is not acceptable, so the + // failure is reported without the payload. + const secretApplied = tryNodeExec(h, server, `k3s kubectl -n ${LONGHORN_NS} apply -f - <<'EOF' +apiVersion: v1 +kind: Secret +metadata: + name: ${BACKUP_SECRET} + namespace: ${LONGHORN_NS} +stringData: + AWS_ACCESS_KEY_ID: ${opts.backup.accessKey} + AWS_SECRET_ACCESS_KEY: ${opts.backup.secretKey} + AWS_ENDPOINTS: ${opts.backup.endpoint} +EOF`); + if (secretApplied.exitCode !== 0) { + throw new Error( + `[${server.name}] could not create the ${BACKUP_SECRET} secret ` + + `(exit ${secretApplied.exitCode}). Command withheld: it contains credentials. ` + + `stderr: ${secretApplied.stderr.trim().slice(0, 200)}`); + } + } + + log(`[longhorn] installing chart ${LONGHORN_VERSION} via k3s helm controller`); + nodeExec(h, server, `cat > /var/lib/rancher/k3s/server/manifests/longhorn.yaml <<'EOF' +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: longhorn + namespace: kube-system +spec: + chart: longhorn + repo: https://charts.longhorn.io + version: ${LONGHORN_VERSION} + targetNamespace: ${LONGHORN_NS} + valuesContent: |- + defaultSettings: + defaultDataPath: /var/lib/longhorn + defaultReplicaCount: ${replicas} + createDefaultDiskLabeledNodes: true + defaultDataLocality: best-effort + replicaSoftAntiAffinity: false + replicaReplenishmentWaitInterval: 600 + orphanAutoDeletion: true + persistence: + defaultClass: true + defaultClassReplicaCount: ${replicas} + defaultFsType: ext4 + longhornUI: + replicas: 1 +EOF`); +} + +/** + * Every expected node has a Longhorn disk that is actually schedulable. + * + * Takes the VMs, not names, and resolves each one's Kubernetes name itself. + * Longhorn keys its node records by the FQDN the kubelet registered + * (`drill-s1.pxe-test.local`), never the libvirt domain name (`drill-s1`), so + * a caller passing VM names matches nothing and waits out the full timeout + * while the cluster is in fact perfectly healthy. + */ +export async function waitForLonghornReady( + h: Harness, + server: ProvisionedVm, + vms: ProvisionedVm[], + timeoutMs = 20 * 60_000, +): Promise { + const expectedNodes = vms.map((vm) => k8sNodeName(h, vm)); + const start = Date.now(); + let last = ""; + while (Date.now() - start < timeoutMs) { + const r = tryNodeExec(h, server, + `k3s kubectl -n ${LONGHORN_NS} get nodes.longhorn.io -o json`, 60_000); + if (r.exitCode === 0) { + try { + const parsed = JSON.parse(r.stdout) as { items: { + metadata: { name: string }; + status?: { diskStatus?: Record }; + }[] }; + const ready = parsed.items.filter((n) => { + const disks = Object.values(n.status?.diskStatus ?? {}); + return disks.length > 0 && disks.some((d) => + (d.conditions ?? []).some((c) => c.type === "Schedulable" && c.status === "True")); + }).map((n) => n.metadata.name); + + last = `${ready.length}/${expectedNodes.length} with schedulable disk: ${ready.join(",") || "none"}`; + if (expectedNodes.every((n) => ready.includes(n))) { + log(`[longhorn] ready after ${Math.round((Date.now() - start) / 1000)}s — ${last}`); + return; + } + } catch { /* API not serving the CRD yet */ } + } + await sleep(15_000); + } + + // Say WHY, per node, instead of just "not ready". A node with no disk has a + // specific cause and the record knows it; without this the only way to find + // out is to catch the cluster before teardown, which is a race you lose. + const detail: string[] = []; + try { + const raw = kubectlJson<{ items: { + metadata: { name: string; annotations?: Record; labels?: Record }; + spec?: { disks?: Record; allowScheduling?: boolean }; + status?: { + conditions?: { type: string; status: string; reason?: string; message?: string }[]; + diskStatus?: Record; + }; + }[] }>(h, server, `-n ${LONGHORN_NS} get nodes.longhorn.io`); + + for (const n of raw.items) { + const disks = Object.keys(n.spec?.disks ?? {}); + const conds = (n.status?.conditions ?? []) + .filter((c) => c.status !== "True") + .map((c) => `${c.type}=${c.status}(${c.reason ?? ""}: ${(c.message ?? "").slice(0, 80)})`); + const diskConds = Object.entries(n.status?.diskStatus ?? {}).flatMap(([dn, d]) => + (d.conditions ?? []).filter((c) => c.status !== "True") + .map((c) => `${dn}:${c.type}=${c.status}(${c.reason ?? ""}: ${(c.message ?? "").slice(0, 80)})`)); + const ann = n.metadata.annotations?.["node.longhorn.io/default-disks-config"] ?? ""; + const lbl = n.metadata.labels?.["node.longhorn.io/create-default-disk"] ?? ""; + detail.push( + ` ${n.metadata.name}: disks=[${disks.join(",")}] label=${lbl} ann=${ann} ` + + `${conds.length > 0 ? `nodeConds=${conds.join(";")} ` : ""}` + + `${diskConds.length > 0 ? `diskConds=${diskConds.join(";")}` : ""}`); + } + } catch (e) { + detail.push(` (could not read Longhorn nodes: ${e instanceof Error ? e.message : String(e)})`); + } + + throw new Error( + `Longhorn not ready in ${timeoutMs / 60000} min (${last}).\n` + + `A node with no disk usually means the create-default-disk label or the\n` + + `default-disks-config annotation is missing, or the path is not mounted.\n` + + detail.join("\n")); +} + +/** Create a PVC plus a Deployment that mounts it at /data. */ +export function createDataWorkload(h: Harness, server: ProvisionedVm, opts: { + name: string; + sizeGi: number; + replicas: number; +}): void { + log(`[longhorn] creating ${opts.name} (${opts.sizeGi}Gi, ${opts.replicas} replicas)`); + nodeExec(h, server, `k3s kubectl apply -f - <<'EOF' +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: lh-${opts.name} +provisioner: driver.longhorn.io +allowVolumeExpansion: true +parameters: + numberOfReplicas: "${opts.replicas}" + staleReplicaTimeout: "30" + fsType: ext4 +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${opts.name} +spec: + accessModes: [ReadWriteOnce] + storageClassName: lh-${opts.name} + resources: + requests: + storage: ${opts.sizeGi}Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${opts.name} +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: {app: ${opts.name}} + template: + metadata: + labels: {app: ${opts.name}} + spec: + # Pod Security Admission enforces "restricted" on this namespace (the k3s + # hardening group applies the label), so a bare pod spec is REJECTED + # outright -- the Deployment reports 0/1 with no pod and no obvious + # reason unless you read the ReplicaSet events. Every one of these + # fields is required by "restricted"; dropping any brings it back. + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + # fsGroup so the non-root user can actually write to the Longhorn + # volume, which is otherwise owned by root. + fsGroup: 1000 + seccompProfile: {type: RuntimeDefault} + containers: + - name: shell + image: busybox:1.36 + command: ["sh", "-c", "sleep infinity"] + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: ["ALL"]} + volumeMounts: + - {name: data, mountPath: /data} + volumes: + - name: data + persistentVolumeClaim: {claimName: ${opts.name}} +EOF`); +} + +/** Wait for the workload's pod to be Running and its volume attached. */ +export async function waitForWorkload( + h: Harness, server: ProvisionedVm, name: string, timeoutMs = 10 * 60_000, +): Promise { + const start = Date.now(); + let last = ""; + while (Date.now() - start < timeoutMs) { + const r = tryNodeExec(h, server, + `k3s kubectl get pods -l app=${name} -o jsonpath='{.items[*].status.phase}'`); + last = r.stdout.trim(); + if (last === "Running") { + // Ready, not merely scheduled: the volume mount happens before Ready. + const ready = tryNodeExec(h, server, + `k3s kubectl get pods -l app=${name} -o jsonpath='{.items[*].status.containerStatuses[*].ready}'`); + if (ready.stdout.trim() === "true") return; + } + await sleep(5_000); + } + throw new Error(`workload ${name} not running in ${timeoutMs / 60000} min (last phase: ${last || "none"})`); +} + +function podOf(h: Harness, server: ProvisionedVm, name: string): string { + const pod = kubectlOn(h, server, `get pods -l app=${name} -o jsonpath='{.items[0].metadata.name}'`).trim(); + if (!pod) throw new Error(`no pod for workload ${name}`); + return pod; +} + +/** + * Write a random blob, fsync it, and record its digest. + * + * conv=fsync matters: without it the write can still be in the guest's page + * cache, and a checksum taken immediately afterwards proves nothing about what + * reached the replicas. + */ +export async function writeBlob(h: Harness, server: ProvisionedVm, opts: { + name: string; mib: number; +}): Promise { + const pod = podOf(h, server, opts.name); + log(`[longhorn] writing ${opts.mib} MiB to ${opts.name}`); + nodeExec(h, server, + `k3s kubectl exec ${pod} -- sh -c ` + + `"dd if=/dev/urandom of=/data/blob bs=1M count=${opts.mib} conv=fsync 2>/dev/null && ` + + `cd /data && sha256sum blob > blob.sha256 && sync"`, + 15 * 60_000); + + const digest = kubectlOn(h, server, `exec ${pod} -- cat /data/blob.sha256`).trim().split(/\s+/)[0] ?? ""; + if (!/^[0-9a-f]{64}$/.test(digest)) throw new Error(`bad digest for ${opts.name}: ${digest}`); + log(`[longhorn] ${opts.name} digest ${digest.slice(0, 16)}...`); + return digest; +} + +/** + * Restart the pod, then verify the blob. + * + * The restart is the point. It detaches and reattaches the volume, so a pass + * means the bytes came back from Longhorn rather than from wherever the writer + * left them. + */ +export async function verifyBlob(h: Harness, server: ProvisionedVm, opts: { + name: string; expectDigest: string; +}): Promise { + const before = podOf(h, server, opts.name); + kubectlOn(h, server, `delete pod ${before} --wait=true --timeout=5m`, 6 * 60_000); + await waitForWorkload(h, server, opts.name); + const after = podOf(h, server, opts.name); + if (after === before) throw new Error(`${opts.name}: pod was not actually replaced`); + + // sha256sum -c proves the file matches what was recorded IN the volume... + const check = tryNodeExec(h, server, + `k3s kubectl exec ${after} -- sh -c "cd /data && sha256sum -c blob.sha256"`, 15 * 60_000); + if (check.exitCode !== 0) { + throw new Error(`${opts.name}: DATA CORRUPTION — sha256sum -c failed: ${check.stdout}${check.stderr}`); + } + + // ...and this proves the recorded digest is still the one we started with, + // which -c alone would not catch if both file and checksum were replaced. + const actual = kubectlOn(h, server, + `exec ${after} -- sh -c "cd /data && sha256sum blob"`, 15 * 60_000).trim().split(/\s+/)[0]; + if (actual !== opts.expectDigest) { + throw new Error(`${opts.name}: DIGEST CHANGED — expected ${opts.expectDigest}, got ${actual}`); + } + log(`[longhorn] ${opts.name} verified intact across a detach/attach (pod ${before} -> ${after})`); +} + +export interface VolumeState { + pv: string; + state: string; + robustness: string; + wanted: number; + replicaNodes: string[]; + healthyReplicas: number; +} + +/** Longhorn's view of the volume backing a PVC. */ +export function volumeState(h: Harness, server: ProvisionedVm, pvcName: string): VolumeState { + const pv = kubectlOn(h, server, `get pvc ${pvcName} -o jsonpath='{.spec.volumeName}'`).trim(); + if (!pv) throw new Error(`PVC ${pvcName} has no bound volume`); + + const vol = kubectlJson<{ + spec: { numberOfReplicas?: number }; + status?: { state?: string; robustness?: string }; + }>(h, server, `-n ${LONGHORN_NS} get volumes.longhorn.io ${pv}`); + + const reps = kubectlJson<{ items: { + spec: { nodeID?: string; volumeName?: string }; + status?: { currentState?: string; started?: boolean }; + }[] }>(h, server, `-n ${LONGHORN_NS} get replicas.longhorn.io`); + + const mine = reps.items.filter((r) => r.spec.volumeName === pv); + return { + pv, + state: vol.status?.state ?? "unknown", + robustness: vol.status?.robustness ?? "unknown", + wanted: vol.spec.numberOfReplicas ?? 0, + replicaNodes: mine.map((r) => r.spec.nodeID ?? "").filter(Boolean).sort(), + healthyReplicas: mine.filter((r) => r.status?.currentState === "running").length, + }; +} + +/** + * Wait for a volume to be healthy on the expected number of distinct nodes. + * + * Logs rebuild progress while waiting, so a slow rebuild reads as progress + * rather than as an opaque hour of silence. + */ +export async function waitForVolumeHealthy(h: Harness, server: ProvisionedVm, opts: { + pvc: string; expectReplicas: number; timeoutMs?: number; +}): Promise { + const timeoutMs = opts.timeoutMs ?? 60 * 60_000; + const start = Date.now(); + let last = ""; + while (Date.now() - start < timeoutMs) { + let st: VolumeState | null = null; + try { st = volumeState(h, server, opts.pvc); } catch { /* transient */ } + + if (st) { + const distinct = new Set(st.replicaNodes).size; + last = `${st.state}/${st.robustness} on ${distinct} node(s): ${st.replicaNodes.join(",")}`; + if (st.robustness === "healthy" && distinct >= opts.expectReplicas) { + log(`[longhorn] ${opts.pvc} healthy on ${distinct} nodes after ${Math.round((Date.now() - start) / 1000)}s`); + return st; + } + const prog = tryNodeExec(h, server, + `k3s kubectl -n ${LONGHORN_NS} get engines.longhorn.io -o jsonpath=` + + `'{range .items[*]}{.spec.volumeName}{" "}{.status.rebuildStatus}{"\\n"}{end}'`); + const line = prog.stdout.split("\n").find((l) => l.includes(st!.pv) && l.trim() !== st!.pv); + log(`[longhorn] ${opts.pvc}: ${last}${line ? ` | rebuild ${line.replace(st.pv, "").trim()}` : ""}`); + } + await sleep(20_000); + } + throw new Error(`${opts.pvc} not healthy on ${opts.expectReplicas} nodes in ${timeoutMs / 60000} min (last: ${last})`); +} + +/** + * Wait until Longhorn reports the backup target usable. + * + * `available: true` is the appliance saying it could reach the endpoint AND + * authenticate. Backing up without checking this first produces a failure that + * looks like a volume problem. + */ +export async function setBackupTarget( + h: Harness, server: ProvisionedVm, backup: BackupTargetOpts, timeoutMs = 5 * 60_000, +): Promise { + // PATCH the CR; do NOT set defaultSettings.backupTarget. + // + // In Longhorn 1.8 the backup target moved out of settings and onto + // `backuptargets.longhorn.io/default`. The helm value is SILENTLY IGNORED -- + // no `backup-target` Setting object is created at all, the default + // BackupTarget keeps an empty URL, and everything reports success. Observed + // on a fresh 1.8.1 install here and confirmed against the live fleet. + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const exists = tryNodeExec(h, server, + `k3s kubectl -n ${LONGHORN_NS} get backuptargets.longhorn.io default -o name`); + if (exists.exitCode === 0 && exists.stdout.trim() !== "") break; + await sleep(5_000); + } + + const patch = JSON.stringify({ + spec: { backupTargetURL: backup.target, credentialSecret: BACKUP_SECRET, pollInterval: "5m0s" }, + }); + nodeExec(h, server, + `k3s kubectl -n ${LONGHORN_NS} patch backuptargets.longhorn.io default --type=merge -p '${patch}'`); + log(`[longhorn] backup target set to ${backup.target}`); +} + +export async function waitForBackupTarget( + h: Harness, server: ProvisionedVm, timeoutMs = 5 * 60_000, +): Promise { + const start = Date.now(); + let last = ""; + while (Date.now() - start < timeoutMs) { + const r = tryNodeExec(h, server, + `k3s kubectl -n ${LONGHORN_NS} get backuptargets.longhorn.io default ` + + `-o jsonpath='{.spec.backupTargetURL}|{.status.available}|{.status.conditions[*].message}'`); + last = r.stdout.trim(); + if (r.exitCode === 0 && last.includes("|true|")) { + log(`[longhorn] backup target available after ${Math.round((Date.now() - start) / 1000)}s`); + return; + } + await sleep(10_000); + } + throw new Error( + `Backup target not available in ${timeoutMs / 60000} min (url|available|conditions = ${last}). ` + + `Longhorn could not reach or authenticate to the endpoint.`); +} + +/** + * Snapshot a volume and back it up, returning the backup's name. + * + * Longhorn's own CRs, not the CSI VolumeSnapshot API -- this cluster has no + * snapshot.storage.k8s.io resources at all, so VolumeSnapshot/VolumeSnapshotClass + * simply do not exist here. Any backup tooling has to use these. + */ +export async function backupVolume( + h: Harness, server: ProvisionedVm, pvc: string, timeoutMs = 20 * 60_000, +): Promise<{ backup: string; volume: string }> { + const pv = volumeState(h, server, pvc).pv; + const snapName = `drill-snap-${Date.now().toString(36)}`; + + log(`[longhorn] snapshotting ${pvc} (${pv})`); + nodeExec(h, server, `k3s kubectl -n ${LONGHORN_NS} apply -f - <<'EOF' +apiVersion: longhorn.io/v1beta2 +kind: Snapshot +metadata: + name: ${snapName} + namespace: ${LONGHORN_NS} +spec: + volume: ${pv} + createSnapshot: true +EOF`); + + // The snapshot has to exist before a backup can reference it. + const snapDeadline = Date.now() + 5 * 60_000; + while (Date.now() < snapDeadline) { + const r = tryNodeExec(h, server, + `k3s kubectl -n ${LONGHORN_NS} get snapshots.longhorn.io ${snapName} -o jsonpath='{.status.readyToUse}'`); + if (r.stdout.trim() === "true") break; + await sleep(5_000); + } + + const backupName = `drill-bak-${Date.now().toString(36)}`; + log(`[longhorn] backing up ${pvc} to the backup target`); + nodeExec(h, server, `k3s kubectl -n ${LONGHORN_NS} apply -f - <<'EOF' +apiVersion: longhorn.io/v1beta2 +kind: Backup +metadata: + name: ${backupName} + namespace: ${LONGHORN_NS} + labels: + backup-volume: ${pv} +spec: + snapshotName: ${snapName} +EOF`); + + const start = Date.now(); + let last = ""; + while (Date.now() - start < timeoutMs) { + const r = tryNodeExec(h, server, + `k3s kubectl -n ${LONGHORN_NS} get backups.longhorn.io ${backupName} ` + + `-o jsonpath='{.status.state}|{.status.progress}|{.status.error}'`); + last = r.stdout.trim(); + if (last.startsWith("Completed")) { + log(`[longhorn] backup ${backupName} completed after ${Math.round((Date.now() - start) / 1000)}s`); + return { backup: backupName, volume: pv }; + } + if (last.includes("Error")) { + throw new Error(`Backup ${backupName} failed: ${last}`); + } + await sleep(10_000); + } + throw new Error(`Backup ${backupName} did not complete in ${timeoutMs / 60000} min (state|progress|error = ${last})`); +} + +/** + * Restore a backup into a brand-new PVC and workload. + * + * Uses the `fromBackup` StorageClass parameter, which is the supported route + * when the CSI VolumeSnapshot API is unavailable. The restored volume is a + * separate volume with its own replicas -- nothing about the original is + * touched, so a failed restore cannot damage the source. + */ +export async function restoreFromBackup(h: Harness, server: ProvisionedVm, opts: { + backup: string; + name: string; + sizeGi: number; +}): Promise { + const url = nodeExec(h, server, + `k3s kubectl -n ${LONGHORN_NS} get backups.longhorn.io ${opts.backup} -o jsonpath='{.status.url}'`).trim(); + if (url === "") throw new Error(`Backup ${opts.backup} has no status.url; cannot restore from it`); + log(`[longhorn] restoring ${opts.backup} into ${opts.name}`); + + nodeExec(h, server, `k3s kubectl apply -f - <<'EOF' +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: lh-${opts.name} +provisioner: driver.longhorn.io +allowVolumeExpansion: true +parameters: + numberOfReplicas: "2" + staleReplicaTimeout: "30" + fsType: ext4 + fromBackup: "${url}" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ${opts.name} +spec: + accessModes: [ReadWriteOnce] + storageClassName: lh-${opts.name} + resources: + requests: + storage: ${opts.sizeGi}Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${opts.name} +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: {app: ${opts.name}} + template: + metadata: + labels: {app: ${opts.name}} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: {type: RuntimeDefault} + containers: + - name: shell + image: busybox:1.36 + command: ["sh", "-c", "sleep infinity"] + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: ["ALL"]} + volumeMounts: + - {name: data, mountPath: /data} + volumes: + - name: data + persistentVolumeClaim: {claimName: ${opts.name}} +EOF`); +} + +/** + * Read a blob's digest from a workload, without restarting it. + * + * Used on a RESTORED volume, where the point is to compare bytes against the + * original's recorded digest rather than to prove durability again. + */ +export function readBlobDigest(h: Harness, server: ProvisionedVm, name: string): string { + const pod = kubectlOn(h, server, `get pods -l app=${name} -o jsonpath='{.items[0].metadata.name}'`).trim(); + if (pod === "") throw new Error(`no pod for workload ${name}`); + const out = kubectlOn(h, server, + `exec ${pod} -- sh -c "cd /data && sha256sum blob"`, 15 * 60_000).trim(); + return out.split(/\s+/)[0] ?? ""; +} diff --git a/bastion/tests/integration/helpers/provision-harness.ts b/bastion/tests/integration/helpers/provision-harness.ts new file mode 100644 index 0000000..2a535b6 --- /dev/null +++ b/bastion/tests/integration/helpers/provision-harness.ts @@ -0,0 +1,534 @@ +// Reusable bastion + VM provisioning harness for partition-layout tests. +// +// pxe-provision.test.ts grew a ~200-line beforeAll that stands up a real +// bastion, caches Fedora boot artifacts, and drives one VM through discovery +// and install. Testing a partition MATRIX needs that same sequence for several +// VMs and several roles, so it lives here instead of being copy-pasted per +// topology. + +import { existsSync, mkdirSync, copyFileSync, symlinkSync, writeFileSync } from "node:fs"; +import { execSync } from "node:child_process"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { BastionConfig } from "../../../src/shared/src/types/config.js"; +import type { Role } from "../../../src/shared/src/types/state.js"; +import { log, waitForSsh } from "./libvirt.js"; +import { ensurePxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_BRIDGE, PXE_GATEWAY, PXE_SUBNET } from "./pxe-network.js"; +import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm } from "./pxe-vm.js"; +import { sshExec } from "./ssh.js"; + +export const DISCOVERY_TIMEOUT_MS = 6 * 60_000; +export const INSTALL_TIMEOUT_MS = 60 * 60_000; +export const SSH_TIMEOUT_MS = 12 * 60_000; + +export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +export interface Harness { + config: BastionConfig; + close: () => Promise; + httpPort: number; + sshUser: string; + sshKeyPath: string; +} + +async function pollUntil(url: string, check: (d: T) => boolean, timeoutMs: number, intervalMs = 5000): Promise { + const start = Date.now(); + let lastErr = ""; + while (Date.now() - start < timeoutMs) { + try { + const res = await fetch(url); + if (res.ok) { + const data = (await res.json()) as T; + if (check(data)) return data; + } + } catch (e) { lastErr = e instanceof Error ? e.message : String(e); } + await sleep(intervalMs); + } + throw new Error(`Timeout after ${timeoutMs}ms polling ${url}${lastErr ? ` (last error: ${lastErr})` : ""}`); +} + +/** + * Stand up a real bastion on the isolated PXE network. + * + * FEDORA_MIRROR is honoured (config.ts now reads it) -- point it at a local + * mirror before running a multi-VM matrix, or every install pulls the full + * package set from download.fedoraproject.org. + */ +export async function startTestBastion(opts: { + httpPort: number; + sshUser: string; + sshPubKey: string; + sshKeyPath: string; + label: string; + /** + * Leave the post-provision k3s auto-install ON (the default) or suppress it. + * + * Any test that forms a cluster ITSELF must pass false. The auto-install runs + * a bare `INSTALL_K3S_EXEC=server|agent` with no server URL and no token, so + * three combo nodes become three separate single-node clusters -- and it + * stops k3s and deletes CNI state minutes after an install lands, landing on + * top of whatever the test is doing by then. + */ + autoK3s?: boolean; +}): Promise { + ensurePxeNetwork(); + + const testDir = join(tmpdir(), `lab-${opts.label}-${Date.now()}`); + for (const sub of ["tftp", "http", "logs"]) mkdirSync(join(testDir, sub), { recursive: true }); + + const { createApp } = await import("../../../src/bastion/src/server.js"); + const { loadConfig } = await import("../../../src/bastion/src/config.js"); + const { generateDnsmasqConf, startDnsmasq, stopDnsmasq } = await import("../../../src/bastion/src/services/dnsmasq.js"); + const { generateDiscoverKickstart } = await import("../../../src/bastion/src/services/kickstart-generator.js"); + const { renderBootIpxe } = await import("../../../src/bastion/src/templates/boot.ipxe.js"); + + const config = loadConfig({ + bastionDir: testDir, + httpPort: opts.httpPort, + iface: "virbr-pxe", + serverIp: PXE_GATEWAY, + network: `${PXE_SUBNET}.0`, + gateway: PXE_GATEWAY, + dhcpMode: "full", + dhcpRangeStart: `${PXE_SUBNET}.100`, + dhcpRangeEnd: `${PXE_SUBNET}.200`, + domain: "pxe-test.local", + sshKeys: [opts.sshPubKey], + adminUser: opts.sshUser, + ...(opts.autoK3s === undefined ? {} : { autoK3s: opts.autoK3s }), + }); + + const ipxeSrc = "/usr/share/ipxe/ipxe-snponly-x86_64.efi"; + if (!existsSync(ipxeSrc)) throw new Error(`iPXE not found: ${ipxeSrc}. Install: sudo dnf install ipxe-bootimgs-x86`); + const ipxeDest = join(config.tftpDir, "ipxe.efi"); + copyFileSync(ipxeSrc, ipxeDest); + + const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache"; + execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" }); + const kernel = join(cacheDir, `vmlinuz-${config.fedoraVersion}`); + const initrd = join(cacheDir, `initrd-${config.fedoraVersion}.img`); + if (!existsSync(kernel)) { + log(`Downloading Fedora ${config.fedoraVersion} kernel from ${config.fedoraMirror}...`); + execSync(`curl -sS -L -f -o "${kernel}" "${config.fedoraMirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 300_000 }); + } + if (!existsSync(initrd)) { + log(`Downloading Fedora ${config.fedoraVersion} initrd...`); + execSync(`curl -sS -L -f -o "${initrd}" "${config.fedoraMirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 300_000 }); + } + copyFileSync(kernel, join(config.httpDir, "vmlinuz")); + copyFileSync(initrd, join(config.httpDir, "initrd.img")); + try { symlinkSync(ipxeDest, join(config.httpDir, "ipxe.efi")); } catch { /* exists */ } + + writeFileSync(join(config.httpDir, "discover.ks"), generateDiscoverKickstart(config)); + writeFileSync(join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort })); + generateDnsmasqConf(config); + + // Kill dnsmasq processes left behind by EARLIER TEST RUNS. + // + // startTestBastion's close() shuts down the HTTP app but never stopped + // dnsmasq, so every run leaked one. A leaked dnsmasq keeps :67 on the bridge + // and keeps answering DHCP -- with the PREVIOUS run's config, pointing PXE + // clients at a bastion port that is no longer listening. The next run then + // sees perfect DHCP (offer, request, ack, even the right bootfile name) and + // no machine ever reaches it. That is not a failure mode anyone guesses from + // "discovery timed out". + killOrphanedDnsmasq(); + + const { app, syslog } = createApp(config); + await app.listen({ port: config.httpPort, host: "0.0.0.0" }); + syslog.start(); + // NOT awaited, deliberately. + // + // startDnsmasq returns the execa child handle, and because the function is + // `async` the promise it returns ADOPTS that thenable -- which settles only + // when dnsmasq EXITS. With --no-daemon that is never, so `await` here hangs + // the harness silently and forever, with the log frozen right after + // "TFTP root is ...". (Learned by doing exactly that.) The rejection still + // has to be handled or an early bind failure becomes an unhandled rejection. + startDnsmasq(config).catch((err) => log(`dnsmasq: ${err instanceof Error ? err.message : String(err)}`)); + await sleep(1000); + + // Prove DHCP is actually listening before handing the harness back. + // + // startDnsmasq's rejection is swallowed into a log line, so a dnsmasq that + // failed to bind looks exactly like one that started. The symptom then + // arrives six minutes later as four VMs timing out in discovery, which + // reads as a PXE or kickstart problem rather than "there was never any + // DHCP". Fail here, immediately, with the real reason. + await assertDhcpListening(join(testDir, "dnsmasq.conf")); + log(`Bastion up on :${opts.httpPort} (mirror: ${config.fedoraMirror}, autoK3s=${config.autoK3s})`); + + return { + config, + httpPort: opts.httpPort, + sshUser: opts.sshUser, + sshKeyPath: opts.sshKeyPath, + close: async () => { + // Stop OUR dnsmasq. close() never used to, which is how a leaked one + // ended up owning :67 and serving the PREVIOUS run's TFTP root and + // bastion port to the NEXT run's VMs -- they booted toward a dead port + // and simply never appeared, while DHCP looked perfect. + try { stopDnsmasq(); } catch { /* already gone */ } + await app.close(); + // Belt and braces: if that did not stop it, sweep by config path. + killOrphanedDnsmasq(); + }, + }; +} + +/** + * Kill any dnsmasq started by a previous (or current) lab test run. + * + * Matched on the config path, which is always /tmp/lab-