diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 7474bff..5d33f9d 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -28,6 +28,8 @@ jobs: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 + with: + package_json_file: bastion/package.json - uses: actions/setup-node@v4 with: @@ -47,6 +49,8 @@ jobs: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 + with: + package_json_file: bastion/package.json - uses: actions/setup-node@v4 with: @@ -54,6 +58,9 @@ jobs: - run: pnpm install --frozen-lockfile + - name: Generate Prisma client (labd typecheck needs it) + run: pnpm -C src/labd run db:generate + - name: Typecheck run: pnpm typecheck @@ -66,6 +73,8 @@ jobs: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 + with: + package_json_file: bastion/package.json - uses: actions/setup-node@v4 with: @@ -73,6 +82,15 @@ jobs: - run: pnpm install --frozen-lockfile + - name: Generate Prisma client (labd typecheck needs it) + run: pnpm -C src/labd run db:generate + + - name: Install test validators (ubuntu-autoinstall parses YAML via python, kickstart.test runs ksvalidator, the bastion smoke start runs `ip route`) + # pykickstart is not in the runner image's apt sources -- pip it is. + run: | + sudo apt-get update -qq && sudo apt-get install -y -qq python3-yaml python3-pip iproute2 + sudo pip3 install --quiet --break-system-packages pykickstart + - name: Build (needed by completions check) run: pnpm build @@ -91,6 +109,8 @@ jobs: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 + with: + package_json_file: bastion/package.json - uses: actions/setup-node@v4 with: @@ -99,6 +119,9 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Generate Prisma client (labd typecheck needs it) + run: pnpm -C src/labd run db:generate + - name: Build all packages run: pnpm build @@ -123,13 +146,13 @@ jobs: - name: Package x86_64 RPM + DEB run: | - sed -e 's|^arch:.*|arch: amd64|' -e 's|src: ./dist/lab$|src: ./dist/lab-x86_64|' nfpm.yaml > /tmp/nfpm-x86_64.yaml + sed -e 's|^arch:.*|arch: amd64|' -e 's|src: ./dist/labctl$|src: ./dist/lab-x86_64|' nfpm.yaml > /tmp/nfpm-x86_64.yaml nfpm pkg --config /tmp/nfpm-x86_64.yaml --packager rpm --target dist/ nfpm pkg --config /tmp/nfpm-x86_64.yaml --packager deb --target dist/ - name: Package arm64 RPM + DEB run: | - sed -e 's|^arch:.*|arch: arm64|' -e 's|src: ./dist/lab$|src: ./dist/lab-arm64|' nfpm.yaml > /tmp/nfpm-arm64.yaml + sed -e 's|^arch:.*|arch: arm64|' -e 's|src: ./dist/labctl$|src: ./dist/lab-arm64|' nfpm.yaml > /tmp/nfpm-arm64.yaml nfpm pkg --config /tmp/nfpm-arm64.yaml --packager rpm --target dist/ nfpm pkg --config /tmp/nfpm-arm64.yaml --packager deb --target dist/ diff --git a/bastion/completions/labctl.bash b/bastion/completions/labctl.bash index 689de91..aea7a63 100644 --- a/bastion/completions/labctl.bash +++ b/bastion/completions/labctl.bash @@ -40,8 +40,14 @@ _labctl() { "app labcontroller status") COMPREPLY=($(compgen -W "--user -h --help" -- "$cur")) return ;; + "app k3s evacuate") + COMPREPLY=($(compgen -W "--user --allow-single-replica -h --help" -- "$cur")) + return ;; + "app k3s admit") + COMPREPLY=($(compgen -W "--user -h --help" -- "$cur")) + return ;; "app k3s install") - COMPREPLY=($(compgen -W "--role --user --k3s-server --k3s-token -h --help" -- "$cur")) + COMPREPLY=($(compgen -W "--role --user --k3s-server --k3s-token --k3s-version -h --help" -- "$cur")) return ;; "app k3s health") COMPREPLY=($(compgen -W "--user -h --help" -- "$cur")) @@ -59,7 +65,7 @@ _labctl() { COMPREPLY=($(compgen -W "-h --help" -- "$cur")) return ;; "provision install") - COMPREPLY=($(compgen -W "--role --os --disk --vyos-mgmt --vyos-mgmt-address --vyos-bond --vyos-bond-address --vyos-bond-vrrp --vlan-vip --vyos-vrrp-priority --vyos-mgmt-vlan --vlan --vyos-password --vyos-hwid --vyos-fresh-config -h --help" -- "$cur")) + COMPREPLY=($(compgen -W "--role --os --disk --schema-version --longhorn-disk --rancher-disk --vyos-mgmt --vyos-mgmt-address --vyos-bond --vyos-bond-address --vyos-bond-vrrp --vlan-vip --vyos-vrrp-priority --vyos-mgmt-vlan --vlan --vyos-password --vyos-hwid --vyos-fresh-config --vyos-bundle --vyos-api-key --vyos-api-listen -h --help" -- "$cur")) return ;; "provision reprovision") COMPREPLY=($(compgen -W "--role --os --disk --user -h --help" -- "$cur")) @@ -101,7 +107,7 @@ _labctl() { COMPREPLY=($(compgen -W "deploy status -h --help" -- "$cur")) return ;; "app k3s") - COMPREPLY=($(compgen -W "install health list kubeconfig -h --help" -- "$cur")) + COMPREPLY=($(compgen -W "evacuate admit install health list kubeconfig -h --help" -- "$cur")) return ;; "version") COMPREPLY=($(compgen -W "-h --help" -- "$cur")) diff --git a/bastion/completions/labctl.fish b/bastion/completions/labctl.fish index bcf2ac1..e8739de 100644 --- a/bastion/completions/labctl.fish +++ b/bastion/completions/labctl.fish @@ -131,9 +131,12 @@ complete -c labctl -n "__labctl_using_cmd provision" -a makeiso -d 'Generate a U complete -c labctl -n "__labctl_using_cmd provision" -a recheck -d 'Refresh hardware info for all installed machines via SSH' # provision install options -complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller' +complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo' complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling' -complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target disk device (auto-detect if omitted)' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target OS disk: kernel name or stable /dev/disk/by-id/ path (auto-detect if omitted)' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l schema-version -d 'Disk layout for a FRESH install (default 1). Ignored on reprovision: the layout already on the disk wins.' -xa '1 2' +complete -c labctl -n "__labctl_in_cmd provision install" -l longhorn-disk -d 'Dedicated whole disk for /var/lib/longhorn, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l rancher-disk -d 'Dedicated whole disk for /var/lib/rancher, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)' -x complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt -d 'VyOS: untagged interface the machine PXE boots from (default eth0)' -x complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-address -d 'VyOS: CIDR for the management interface, or \'dhcp\' (default dhcp)' -x complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond -d 'VyOS: comma-separated LACP bond members (must exclude the PXE NIC)' -x @@ -146,9 +149,12 @@ complete -c labctl -n "__labctl_in_cmd provision install" -l vlan -d 'VyOS: tagg complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-password -d 'VyOS: password for the \'vyos\' user' -x complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-hwid -d 'VyOS: pin an interface name to a MAC via hw-id (repeatable)' -x complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-fresh-config -d 'VyOS: on reinstall, overwrite the preserved config with the generated one' +complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bundle -d 'VyOS: apply a Pulumi-rendered bundle verbatim (kubernetes-deployment/infra/vyos/vyos-bundle.json). Replaces the derived --vyos-bond/--vlan/... config; secret values are left unset for `pulumi up`.' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-api-key -d 'VyOS: enable the HTTP API with this key so Pulumi can manage the router from first boot' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-api-listen -d 'VyOS: address the HTTP API binds to (default: the static management address). Required when management is DHCP; the API is never bound to all interfaces.' -x # provision reprovision options -complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller' +complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo' complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling' complete -c labctl -n "__labctl_in_cmd provision reprovision" -l disk -d 'Target disk device (auto-detect if omitted)' -x complete -c labctl -n "__labctl_in_cmd provision reprovision" -l user -d 'SSH user for the reboot (default: vyos for VyOS machines, else current user)' -x @@ -157,7 +163,7 @@ complete -c labctl -n "__labctl_in_cmd provision reprovision" -l user -d 'SSH us complete -c labctl -n "__labctl_in_cmd provision debug" -l pxe-boot -d 'Boot installed system via PXE (kernel+initrd from network, root from NVMe)' # provision register options -complete -c labctl -n "__labctl_in_cmd provision register" -l role -d 'Machine role' -xa 'vanilla worker infra labcontroller' +complete -c labctl -n "__labctl_in_cmd provision register" -l role -d 'Machine role' -xa 'vanilla worker infra labcontroller standalone-cluster k8s-controller k8s-worker k8s-combo' complete -c labctl -n "__labctl_in_cmd provision register" -l ip -d 'Machine IP address' -x # provision logs options @@ -200,16 +206,26 @@ complete -c labctl -n "__labctl_in_cmd app labcontroller deploy" -l crdb-replica complete -c labctl -n "__labctl_in_cmd app labcontroller status" -l user -d 'SSH user' -x # app k3s subcommands +complete -c labctl -n "__labctl_using_cmd app k3s" -a evacuate -d 'Cordon, evict Longhorn replicas and drain so a node can be safely wiped' +complete -c labctl -n "__labctl_using_cmd app k3s" -a admit -d 'Return an evacuated node to service: uncordon, clear eviction, restore its Longhorn disk' complete -c labctl -n "__labctl_using_cmd app k3s" -a install -d 'Install k3s on a target machine (hostname, IP, or MAC)' complete -c labctl -n "__labctl_using_cmd app k3s" -a health -d 'Check k3s health (all hosts if no target given)' complete -c labctl -n "__labctl_using_cmd app k3s" -a list -d 'List installed machines and their k3s status' complete -c labctl -n "__labctl_using_cmd app k3s" -a kubeconfig -d 'Fetch kubeconfig from a target and merge into ~/.kube/config' +# app k3s evacuate options +complete -c labctl -n "__labctl_in_cmd app k3s evacuate" -l user -d 'SSH user' -x +complete -c labctl -n "__labctl_in_cmd app k3s evacuate" -l allow-single-replica -d 'permit evacuating volumes that have only one replica (rebuilds from the only copy; there is no backup target)' + +# app k3s admit options +complete -c labctl -n "__labctl_in_cmd app k3s admit" -l user -d 'SSH user' -x + # app k3s install options -complete -c labctl -n "__labctl_in_cmd app k3s install" -l role -d 'k3s role: infra (server) or worker (agent)' -x +complete -c labctl -n "__labctl_in_cmd app k3s install" -l role -d 'k3s role: infra, labcontroller, standalone-cluster, k8s-controller, k8s-combo (servers) or worker, k8s-worker (agents)' -x complete -c labctl -n "__labctl_in_cmd app k3s install" -l user -d 'SSH user' -x -complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-server -d 'k3s server URL (required for worker role)' -x -complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-token -d 'k3s join token (required for worker role)' -x +complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-server -d 'k3s server URL (required for agent roles, and to JOIN as a server)' -x +complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-token -d 'k3s join token; on a first server it SEEDS the cluster token' -x +complete -c labctl -n "__labctl_in_cmd app k3s install" -l k3s-version -d 'exact k3s version, e.g. v1.34.6+k3s1. Unset installs LATEST, which on a rejoining node can be several minor versions ahead of the cluster' -x # app k3s health options complete -c labctl -n "__labctl_in_cmd app k3s health" -l user -d 'SSH user' -x diff --git a/bastion/src/bastion/src/main.ts b/bastion/src/bastion/src/main.ts index 2905650..e6a7e00 100644 --- a/bastion/src/bastion/src/main.ts +++ b/bastion/src/bastion/src/main.ts @@ -4,6 +4,7 @@ import { mkdirSync, writeFileSync, readFileSync, existsSync, copyFileSync, symlinkSync, unlinkSync } from "node:fs"; import { execSync } from "node:child_process"; import type { BastionConfig } from "@lab/shared"; +import { isRole, validateDataDisks } from "@lab/shared"; import { loadConfig } from "./config.js"; import { populateNetworkConfig } from "./services/network.js"; import { createApp } from "./server.js"; @@ -384,6 +385,22 @@ export async function startBastion(overrides: Partial = {}): Prom // Wire up command handlers so labd can send install/forget/role commands labdConn.onCommand("command-install", async (msg) => { if (msg.type !== "command-install") throw new Error("unexpected"); + // Same guard as POST /api/install: disks named by an operator must + // never silently degrade into LVs because they arrived over the socket. + const hasDataDisks = msg.dataDisks !== undefined + && Object.values(msg.dataDisks).some((d) => d !== undefined); + if (hasDataDisks) { + if ((msg.os ?? "fedora-43") !== "fedora-43") { + return { status: "error", error: `dataDisks are only supported for fedora installs (got os '${msg.os}')` }; + } + if (!isRole(msg.role)) { + return { status: "error", error: `invalid role: '${msg.role}'` }; + } + const problem = validateDataDisks(msg.role, msg.dataDisks!); + if (problem !== null) { + return { status: "error", error: `invalid dataDisks: ${problem}` }; + } + } state.update((s) => { s.install_queue[msg.mac] = { hostname: msg.hostname, @@ -391,6 +408,8 @@ export async function startBastion(overrides: Partial = {}): Prom role: msg.role as import("@lab/shared").Role, os: msg.os as import("@lab/shared").OsId, queued_at: new Date().toISOString(), + ...(msg.schemaVersion ? { schemaVersion: msg.schemaVersion } : {}), + ...(hasDataDisks ? { dataDisks: msg.dataDisks } : {}), ...(msg.vyos ? { vyos: msg.vyos } : {}), }; }); diff --git a/bastion/src/bastion/src/routes/api.ts b/bastion/src/bastion/src/routes/api.ts index 8cb5604..9ae4e81 100644 --- a/bastion/src/bastion/src/routes/api.ts +++ b/bastion/src/bastion/src/routes/api.ts @@ -14,7 +14,7 @@ import { progressBus } from "../services/progress-events.js"; import type { ProgressEvent } from "../services/progress-events.js"; import type { InstallLogBuffer } from "../services/install-log.js"; import type { SyslogListener } from "../services/syslog-listener.js"; -import { isRole, requiresSchemaV2 } from "@lab/shared"; +import { isRole, requiresSchemaV2, validateDataDisks, type DataDisks } from "@lab/shared"; /** * Seconds after dispatch with zero progress before a machine is called stalled. @@ -45,9 +45,10 @@ export function registerApiRoutes( os?: string; vyos?: VyosInstallSpec; schemaVersion?: number; + dataDisks?: DataDisks; }; }>("/api/install", async (request, reply) => { - const { mac: rawMac, hostname, disk, role, os, vyos, schemaVersion } = request.body ?? {}; + const { mac: rawMac, hostname, disk, role, os, vyos, schemaVersion, dataDisks } = request.body ?? {}; const mac = (rawMac ?? "").toLowerCase().replace(/-/g, ":"); if (mac === "") { @@ -79,6 +80,36 @@ export function registerApiRoutes( return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: ${SUPPORTED_OS.join(", ")}` }); } + // Dedicated data disks are rendered by the kickstart generator only, and + // only in the v2 layout. Rejected here rather than silently dropped at + // render time -- an operator who named disks must not get LVs instead. + const hasDataDisks = dataDisks !== undefined + && Object.values(dataDisks).some((d) => d !== undefined); + if (hasDataDisks) { + if (osId !== "fedora-43") { + return reply.status(400).send({ + error: `dataDisks are only supported for fedora installs (got os '${osId}')`, + }); + } + if (schemaVersion === 1) { + return reply.status(400).send({ + error: "dataDisks require schemaVersion 2 -- the v1 layout is frozen and single-disk", + }); + } + if (!isRole(validRole)) { + return reply.status(400).send({ error: `invalid role: '${validRole}'` }); + } + if (!requiresSchemaV2(validRole) && schemaVersion === undefined) { + return reply.status(400).send({ + error: `role '${validRole}' defaults to the v1 layout -- pass schemaVersion 2 to use dataDisks`, + }); + } + const problem = validateDataDisks(validRole, dataDisks as DataDisks); + if (problem !== null) { + return reply.status(400).send({ error: `invalid dataDisks: ${problem}` }); + } + } + state.update((s) => { s.install_queue[mac] = { hostname: hostname ?? "lab-node", @@ -87,6 +118,7 @@ export function registerApiRoutes( os: osId, queued_at: new Date().toISOString(), ...(schemaVersion ? { schemaVersion: schemaVersion as 1 | 2 } : {}), + ...(hasDataDisks ? { dataDisks } : {}), ...(vyos ? { vyos } : {}), }; }); diff --git a/bastion/src/bastion/src/routes/kickstart.ts b/bastion/src/bastion/src/routes/kickstart.ts index 6fe02aa..a9ae2b0 100644 --- a/bastion/src/bastion/src/routes/kickstart.ts +++ b/bastion/src/bastion/src/routes/kickstart.ts @@ -31,6 +31,7 @@ export function registerKickstartRoutes( disk: queueEntry?.disk ?? "", role: queueEntry?.role ?? "worker", ...(queueEntry?.schemaVersion ? { schemaVersion: queueEntry.schemaVersion } : {}), + ...(queueEntry?.dataDisks ? { dataDisks: queueEntry.dataDisks } : {}), }); return reply.type("text/plain").send(ks); diff --git a/bastion/src/bastion/src/services/kickstart-generator.ts b/bastion/src/bastion/src/services/kickstart-generator.ts index 49d23c9..9ce58d1 100644 --- a/bastion/src/bastion/src/services/kickstart-generator.ts +++ b/bastion/src/bastion/src/services/kickstart-generator.ts @@ -25,6 +25,7 @@ export function generateInstallKickstart( disk: string; role: Role; schemaVersion?: 1 | 2; + dataDisks?: import("@lab/shared").DataDisks; }, ): string { const ksParams: InstallKickstartParams = { @@ -32,6 +33,7 @@ export function generateInstallKickstart( disk: params.disk, role: params.role, ...(params.schemaVersion ? { schemaVersion: params.schemaVersion } : {}), + ...(params.dataDisks ? { dataDisks: params.dataDisks } : {}), domain: config.domain, fedoraVersion: config.fedoraVersion, timezone: config.timezone, diff --git a/bastion/src/bastion/src/templates/install.ks.ts b/bastion/src/bastion/src/templates/install.ks.ts index d21bb4c..69b26e3 100644 --- a/bastion/src/bastion/src/templates/install.ks.ts +++ b/bastion/src/bastion/src/templates/install.ks.ts @@ -2,8 +2,11 @@ // Full Fedora server install with LVM partitioning, %pre for reprovision detection, // packages, and %post with SSH keys, user creation, k3s prereqs, progress callbacks. -import type { Role } from "@lab/shared"; -import { v2Layout, V2_BOOT, requiresSchemaV2, effectiveSchemaVersion, isK3sServer, type LvSpec } from "@lab/shared"; +import type { Role, DataDisks, DataDiskName } from "@lab/shared"; +import { + v2Layout, V2_BOOT, requiresSchemaV2, effectiveSchemaVersion, isK3sServer, + validateDataDisks, DATA_DISK_LABELS, type LvSpec, +} from "@lab/shared"; export interface InstallKickstartParams { hostname: string; @@ -27,6 +30,13 @@ export interface InstallKickstartParams { * never silently change its layout -- that is how data gets destroyed. */ schemaVersion?: 1 | 2; + /** + * Dedicated whole-disk data volumes, by stable /dev/disk/by-id path (v2 + * only). The matching LV is not created; the disk gets one labelled xfs + * partition instead. On a reprovision %pre rediscovers these by label, so a + * reinstall preserves them whether or not the queue entry names them. + */ + dataDisks?: DataDisks; } export function renderInstallKickstart(params: InstallKickstartParams): string { @@ -72,8 +82,24 @@ export function renderInstallKickstart(params: InstallKickstartParams): string { `Pass schemaVersion: 2 (labctl provision install --schema-version 2).`, ); } + + const dataDisks = params.dataDisks ?? {}; + const hasDataDisks = Object.values(dataDisks).some((d) => d !== undefined); + if (hasDataDisks) { + // The v1 branches are frozen and know nothing about dedicated disks, so a + // v1 render with them would silently create BOTH the LV and the disk. + if (schemaVersion !== 2) { + throw new Error( + `dedicated data disks require the v2 layout - pass schemaVersion: 2 ` + + `(labctl provision install --schema-version 2)`, + ); + } + const problem = validateDataDisks(role, dataDisks); + if (problem !== null) throw new Error(problem); + } + const v2RootMib = v2Layout(role).find((lv: LvSpec) => lv.name === "root")?.sizeMib ?? 32768; - const v2 = v2Layout(role); + const v2 = v2Layout(role, { dataDisks }); const v2LogvolLines = v2 .map((lv: LvSpec) => lv.fs === "swap" @@ -149,14 +175,29 @@ chmod 440 /etc/sudoers.d/${adminUser}`; } // -- Disk detection -- + // An explicit disk may be a stable /dev/disk/by-id path: kernel names are + // not stable across boots on multi-NVMe hosts, so a pinned install must be + // able to name the disk by serial. Anaconda directives want the kernel name, + // so %pre resolves the symlink at install time. const diskLine = disk - ? `DISK="${disk}"` + ? disk.startsWith("/dev/") + ? `# OS disk pinned by stable path -- resolve to the kernel name anaconda wants +DISK="" +for _wait in $(seq 1 10); do + _os_target=$(readlink -f "${disk}" 2>/dev/null || true) + if [ -n "$_os_target" ] && [ -b "$_os_target" ]; then DISK=$(basename "$_os_target"); break; fi + sleep 1 +done +[ -z "$DISK" ] && { bastion_progress "error" "OS disk not found: ${disk}"; echo "ERROR: OS disk not found: ${disk}"; exit 1; }` + : `DISK="${disk}"` : `DISK="" # Wait up to 10s for NVMe/SCSI disks to appear (they init async in initrd) for _wait in $(seq 1 10); do for d in /dev/nvme0n1 /dev/nvme1n1 /dev/sda /dev/sdb /dev/vda; do [ -b "$d" ] || continue _bname=$(basename "$d") + # Never auto-pick a dedicated data disk as the OS disk + case "$_bname" in "$LONGHORN_DISK"|"$RANCHER_DISK") continue ;; esac # Skip removable disks (USB, CD-ROM, JetKVM virtual media) [ -f "/sys/block/$_bname/removable" ] && [ "$(cat /sys/block/$_bname/removable)" = "1" ] && continue # Skip USB-attached disks (JetKVM virtual media shows as SCSI over USB) @@ -172,6 +213,80 @@ for _wait in $(seq 1 10); do done [ -z "$DISK" ] && { echo "ERROR: no disk found"; exit 1; }`; + // -- Dedicated data disks: %pre resolution -- + // Resolved BEFORE the OS disk so auto-detect can refuse to pick one of them. + const dataDiskResolveBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[]) + .map((name) => { + const device = dataDisks[name]; + const VAR = name.toUpperCase(); + if (device === undefined) return `${VAR}_DISK=""`; + return `# ${name} gets a whole disk of its own, pinned by stable path +${VAR}_DISK_ID="${device}" +${VAR}_DISK="" +for _wait in $(seq 1 10); do + _dd_target=$(readlink -f "$${VAR}_DISK_ID" 2>/dev/null || true) + if [ -n "$_dd_target" ] && [ -b "$_dd_target" ]; then ${VAR}_DISK=$(basename "$_dd_target"); break; fi + sleep 1 +done +[ -z "$${VAR}_DISK" ] && { bastion_progress "error" "${name} disk not found: $${VAR}_DISK_ID"; echo "ERROR: ${name} disk not found: $${VAR}_DISK_ID"; exit 1; }`; + }) + .join("\n"); + + // -- Dedicated data disks: discovery and preservation decisions -- + // A labelled data partition is on-disk truth, exactly like the lab-schema-v2 + // LVM tag: it is preserved whether or not this install names the disk, and it + // is never silently wiped or shadowed. blkid finds it wherever tonight's NVMe + // enumeration put it. + const dataDiskDecisionBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[]) + .map((name) => { + const VAR = name.toUpperCase(); + const label = DATA_DISK_LABELS[name]; + const mount = name === "longhorn" ? "/var/lib/longhorn" : "/var/lib/rancher"; + return `${VAR}_PART=$(blkid -t LABEL=${label} -o device 2>/dev/null | head -1) +${VAR}_REUSE=no +if [ -n "$${VAR}_PART" ]; then + _dd_parent=$(lsblk -ndo pkname "$${VAR}_PART" 2>/dev/null) + if [ -n "$${VAR}_DISK" ] && [ "$_dd_parent" != "$${VAR}_DISK" ]; then + bastion_progress "error" "${label} label is on $_dd_parent, not on the configured ${name} disk $${VAR}_DISK" + echo "ERROR: ${label} label found on $_dd_parent but the ${name} disk is configured as $${VAR}_DISK -- refusing to guess" + exit 1 + fi + if [ "$REPROVISION" = "no" ] && [ -z "$${VAR}_DISK" ]; then + bastion_progress "error" "found ${label} data on $_dd_parent but this install configures no ${name} disk" + echo "ERROR: ${label} labelled data exists on $_dd_parent. Refusing to wipe or shadow it -- queue the install with a ${name} data disk, or wipe the label first" + exit 1 + fi + ${VAR}_DISK="$_dd_parent" + ${VAR}_REUSE=yes +fi +if [ "$PRESERVE_${VAR}" = "yes" ] && [ -n "$${VAR}_DISK" ]; then + bastion_progress "error" "both a ${name} LV and a dedicated ${name} disk exist" + echo "ERROR: VG $VG has a ${name} LV AND a dedicated ${name} disk is present -- two things cannot mount ${mount}. Migrate the LV off first" + exit 1 +fi`; + }) + .join("\n\n"); + + // -- Dedicated data disks: partition directives -- + // Appended to whichever /tmp/part.ks the branches above wrote. A reused + // partition is mounted --noformat; a fresh disk gets ONE partition spanning + // it. --grow is fine HERE: the whole point of a dedicated disk is to consume + // it, unlike LVs competing inside a shared VG. + const dataDiskPartBlocks = (Object.keys(DATA_DISK_LABELS) as DataDiskName[]) + .map((name) => { + const VAR = name.toUpperCase(); + const label = DATA_DISK_LABELS[name]; + const mount = name === "longhorn" ? "/var/lib/longhorn" : "/var/lib/rancher"; + return `if [ -n "$${VAR}_DISK" ]; then + if [ "$${VAR}_REUSE" = "yes" ]; then + echo "part ${mount} --onpart=$${VAR}_PART --noformat" >> /tmp/part.ks + else + echo "part ${mount} --fstype=xfs --ondisk=$${VAR}_DISK --grow --size=1 --label=${label}" >> /tmp/part.ks + fi +fi`; + }) + .join("\n"); + // -- Longhorn LV for fresh install -- const longhornFreshLine = hasLonghorn ? `logvol /var/lib/longhorn --vgname=${vg} --name=longhorn --fstype=xfs --grow --size=1` @@ -229,6 +344,9 @@ bastion_progress() { bastion_progress "partitioning" "detecting disk" VG="${vg}" + +${dataDiskResolveBlocks} + ${diskLine} REPROVISION=no @@ -265,6 +383,19 @@ else bastion_progress "partitioning" "fresh install on $DISK" fi +${dataDiskDecisionBlocks} + +# Every disk anaconda may touch; everything else stays invisible to it. +KS_DISKS="$DISK" +[ -n "$LONGHORN_DISK" ] && KS_DISKS="$KS_DISKS,$LONGHORN_DISK" +[ -n "$RANCHER_DISK" ] && [ "$RANCHER_DISK" != "$LONGHORN_DISK" ] && KS_DISKS="$KS_DISKS,$RANCHER_DISK" + +# Fresh installs wipe the OS disk plus any data disk being formatted for the +# first time -- NEVER one carrying a lab-* label (those are reused --noformat). +CLEARPART_DRIVES="$DISK" +[ -n "$LONGHORN_DISK" ] && [ "$LONGHORN_REUSE" = "no" ] && CLEARPART_DRIVES="$CLEARPART_DRIVES,$LONGHORN_DISK" +[ -n "$RANCHER_DISK" ] && [ "$RANCHER_REUSE" = "no" ] && [ "$RANCHER_DISK" != "$LONGHORN_DISK" ] && CLEARPART_DRIVES="$CLEARPART_DRIVES,$RANCHER_DISK" + if [ "$REPROVISION" = "yes" ]; then # Find existing boot partitions by type EFI_PART=$(blkid -t TYPE=vfat -o device /dev/\${DISK}* 2>/dev/null | head -1) @@ -275,7 +406,7 @@ if [ "$REPROVISION" = "yes" ]; then # Build partition config reusing existing PV/VG cat > /tmp/part.ks << PARTEOF -ignoredisk --only-use=$DISK +ignoredisk --only-use=$KS_DISKS clearpart --none part /boot/efi --onpart=$EFI_PART --fstype=efi part /boot --onpart=$BOOT_PART --fstype=ext4 @@ -320,8 +451,8 @@ PARTEOF else # Fresh install ${schemaVersion === 2 ? ` cat > /tmp/part.ks << PARTEOF -ignoredisk --only-use=$DISK -clearpart --all --initlabel --drives=$DISK +ignoredisk --only-use=$KS_DISKS +clearpart --all --initlabel --drives=$CLEARPART_DRIVES part /boot/efi --fstype=efi --size=${V2_BOOT.efiMib} --ondisk=$DISK part /boot --fstype=ext4 --size=${V2_BOOT.bootMib} --ondisk=$DISK part pv.01 --size=1 --grow --ondisk=$DISK @@ -349,6 +480,8 @@ ${rancherFreshLine} PARTEOF`} fi +${dataDiskPartBlocks} + echo "=== Generated partition config ===" cat /tmp/part.ks echo "===================================" diff --git a/bastion/src/bastion/tests/data-disks.test.ts b/bastion/src/bastion/tests/data-disks.test.ts new file mode 100644 index 0000000..4a8c222 --- /dev/null +++ b/bastion/src/bastion/tests/data-disks.test.ts @@ -0,0 +1,274 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { FastifyInstance } from "fastify"; +import { renderInstallKickstart } from "../src/templates/install.ks.js"; +import type { InstallKickstartParams } from "../src/templates/install.ks.js"; +import { createApp } from "../src/server.js"; +import { validateDataDisks, v2Layout, DATA_DISK_LABELS, type BastionConfig, type DataDisks } from "@lab/shared"; + +/** + * Dedicated whole-disk data volumes (thebeast: 1 TB OS + 2 TB longhorn + + * 2 TB rancher). The layout contract these tests pin down: + * + * - naming a data disk DROPS the matching LV -- never both. + * - disks are named by /dev/disk/by-id only; kernel names are rejected, + * because nvme enumeration order is not stable across boots. + * - a lab-longhorn / lab-rancher labelled partition is on-disk truth, like + * the lab-schema-v2 VG tag: reused --noformat, never wiped, never + * silently shadowed. + */ + +const BY_ID = "/dev/disk/by-id"; +const LONGHORN_ID = `${BY_ID}/nvme-Sabrent_Rocket_4.0_2TB_7F600711179D89261778`; +const RANCHER_ID = `${BY_ID}/nvme-Sabrent_Rocket_4.0_2TB_7FD4070C1EDA92197301`; +const OS_ID = `${BY_ID}/nvme-Samsung_SSD_970_EVO_Plus_1TB_S4EWNJ0N138234M`; + +function baseParams(over: Partial = {}): InstallKickstartParams { + return { + hostname: "thebeast", disk: "", role: "k8s-worker", domain: "ad.itaz.eu", + fedoraVersion: "43", timezone: "Europe/Warsaw", locale: "en_US.UTF-8", + serverIp: "192.168.8.11", httpPort: 8080, syslogPort: 5514, + sshKeys: ["ssh-ed25519 AAAA test"], adminUser: "michal", + dataDisks: { longhorn: LONGHORN_ID, rancher: RANCHER_ID }, + ...over, + }; +} + +/** The fresh-install part.ks heredoc (the one that creates the PV). */ +function freshBlock(ks: string): string { + const blocks = [...ks.matchAll(/cat >>?\s*\/tmp\/part\.ks\s*<<\s*PARTEOF\n([\s\S]*?)\nPARTEOF/g)] + .map((m) => m[1]!); + const fresh = blocks.find((b) => b.includes("part pv.01")); + if (!fresh) throw new Error("no fresh-install part.ks block found"); + return fresh; +} + +describe("validateDataDisks", () => { + const ok: DataDisks = { longhorn: LONGHORN_ID, rancher: RANCHER_ID }; + + it("accepts a sane spec for a role that hosts longhorn", () => { + expect(validateDataDisks("k8s-worker", ok)).toBeNull(); + expect(validateDataDisks("k8s-combo", ok)).toBeNull(); + expect(validateDataDisks("worker", ok)).toBeNull(); + }); + + it("rejects kernel names -- they are not stable across boots", () => { + expect(validateDataDisks("k8s-worker", { longhorn: "nvme1n1" })).toMatch(/by-id/); + expect(validateDataDisks("k8s-worker", { rancher: "/dev/nvme2n1" })).toMatch(/by-id/); + }); + + it("rejects a longhorn disk for a role that holds no replicas", () => { + expect(validateDataDisks("k8s-controller", { longhorn: LONGHORN_ID })).toMatch(/no Longhorn replicas/); + expect(validateDataDisks("vanilla", { longhorn: LONGHORN_ID })).toMatch(/no Longhorn replicas/); + }); + + it("rejects a rancher disk for a role that runs no k3s", () => { + expect(validateDataDisks("vanilla", { rancher: RANCHER_ID })).toMatch(/no k3s/); + }); + + it("a control-plane-only node may still take a rancher disk", () => { + expect(validateDataDisks("k8s-controller", { rancher: RANCHER_ID })).toBeNull(); + }); + + it("rejects the same disk doing both jobs", () => { + expect(validateDataDisks("k8s-worker", { longhorn: LONGHORN_ID, rancher: LONGHORN_ID })) + .toMatch(/same disk/); + }); +}); + +describe("v2Layout with dedicated disks", () => { + it("drops exactly the LV a dedicated disk replaces", () => { + const names = (d: DataDisks) => v2Layout("k8s-worker", { dataDisks: d }).map((l) => l.name); + expect(names({})).toContain("longhorn"); + expect(names({ longhorn: LONGHORN_ID })).not.toContain("longhorn"); + expect(names({ longhorn: LONGHORN_ID })).toContain("rancher"); + expect(names({ longhorn: LONGHORN_ID, rancher: RANCHER_ID })).not.toContain("rancher"); + // base volumes untouched + expect(names({ longhorn: LONGHORN_ID, rancher: RANCHER_ID })) + .toEqual(expect.arrayContaining(["swap", "root", "var", "varlog", "home", "srv"])); + }); +}); + +describe("renderInstallKickstart with data disks", () => { + it("refuses the v1 layout -- it is frozen and single-disk", () => { + expect(() => renderInstallKickstart(baseParams({ role: "worker", schemaVersion: 1 }))) + .toThrow(/v2 layout/); + // a legacy role with no explicit schema defaults to v1 -> same refusal + expect(() => renderInstallKickstart(baseParams({ role: "worker" }))) + .toThrow(/v2 layout/); + }); + + it("refuses a kernel-name data disk", () => { + expect(() => renderInstallKickstart(baseParams({ dataDisks: { longhorn: "nvme1n1" } }))) + .toThrow(/by-id/); + }); + + it("creates no longhorn/rancher LV when both mounts come from dedicated disks", () => { + const fresh = freshBlock(renderInstallKickstart(baseParams())); + expect(fresh).not.toContain("--name=longhorn"); + expect(fresh).not.toContain("--name=rancher"); + // base LVs still there + expect(fresh).toContain("--name=root"); + }); + + it("anaconda only sees the named disks, and only wipes unlabelled ones", () => { + const ks = renderInstallKickstart(baseParams()); + const fresh = freshBlock(ks); + expect(fresh).toContain("ignoredisk --only-use=$KS_DISKS"); + expect(fresh).toContain("clearpart --all --initlabel --drives=$CLEARPART_DRIVES"); + // a disk whose partition carries our label is excluded from clearpart + expect(ks).toContain('[ "$LONGHORN_REUSE" = "no" ] && CLEARPART_DRIVES='); + }); + + it("%pre resolves each by-id path and fails loudly when a disk is missing", () => { + const ks = renderInstallKickstart(baseParams()); + expect(ks).toContain(`LONGHORN_DISK_ID="${LONGHORN_ID}"`); + expect(ks).toContain(`RANCHER_DISK_ID="${RANCHER_ID}"`); + expect(ks).toContain('echo "ERROR: longhorn disk not found: $LONGHORN_DISK_ID"; exit 1'); + }); + + it("a labelled partition is reused --noformat; a fresh disk gets one labelled partition", () => { + const ks = renderInstallKickstart(baseParams()); + expect(ks).toContain("part /var/lib/longhorn --onpart=$LONGHORN_PART --noformat"); + expect(ks).toContain( + `part /var/lib/longhorn --fstype=xfs --ondisk=$LONGHORN_DISK --grow --size=1 --label=${DATA_DISK_LABELS.longhorn}`, + ); + expect(ks).toContain("part /var/lib/rancher --onpart=$RANCHER_PART --noformat"); + expect(ks).toContain( + `part /var/lib/rancher --fstype=xfs --ondisk=$RANCHER_DISK --grow --size=1 --label=${DATA_DISK_LABELS.rancher}`, + ); + }); + + it("refuses to run when a data LV and a dedicated disk both exist", () => { + const ks = renderInstallKickstart(baseParams()); + expect(ks).toContain('[ "$PRESERVE_LONGHORN" = "yes" ] && [ -n "$LONGHORN_DISK" ]'); + expect(ks).toContain('[ "$PRESERVE_RANCHER" = "yes" ] && [ -n "$RANCHER_DISK" ]'); + }); + + it("a fresh install never wipes or shadows a labelled disk it was not told about", () => { + // Rendered even WITHOUT dataDisks: the label is on-disk truth and the + // check protects against a queue entry that forgot the disks. + const ks = renderInstallKickstart(baseParams({ role: "k8s-worker", dataDisks: {} })); + expect(ks).toContain(`blkid -t LABEL=${DATA_DISK_LABELS.longhorn}`); + expect(ks).toContain("Refusing to wipe or shadow it"); + }); + + it("an OS disk given as a by-id path is resolved in %pre", () => { + const ks = renderInstallKickstart(baseParams({ disk: OS_ID })); + expect(ks).toContain(`readlink -f "${OS_ID}"`); + expect(ks).not.toContain(`DISK="${OS_ID}"`); // never fed to anaconda unresolved + }); + + it("auto-detect never picks a data disk as the OS disk", () => { + const ks = renderInstallKickstart(baseParams({ disk: "" })); + expect(ks).toContain('case "$_bname" in "$LONGHORN_DISK"|"$RANCHER_DISK") continue ;; esac'); + }); + + it("a plain kernel-name OS disk still renders exactly as before", () => { + const ks = renderInstallKickstart(baseParams({ disk: "vda", dataDisks: {} })); + expect(ks).toContain('DISK="vda"'); + }); + + it("the %pre block still survives the /ks-auto extractor", () => { + const ks = renderInstallKickstart(baseParams()); + const blocks = [...ks.matchAll(/%pre[^\n]*\n([\s\S]*?)%end/g)]; + expect(blocks.length).toBeGreaterThan(0); + expect(blocks[0]![1]).toContain("/tmp/part.ks"); + }); +}); + +describe("dataDisks plumbing: /api/install -> /ks", () => { + let app: FastifyInstance; + let testDir: string; + + function testConfig(dir: string): BastionConfig { + return { + fedoraVersion: "43", arch: "x86_64", httpPort: 0, + timezone: "Europe/London", locale: "en_GB.UTF-8", + bastionDir: dir, domain: "test.local", dhcpMode: "proxy", + dhcpRangeStart: "", dhcpRangeEnd: "", + ubuntuVersion: "26.04", ubuntuMirror: "https://releases.ubuntu.com/26.04", + vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso", + vyosDefaultPassword: "vyos", iface: "eth0", + serverIp: "10.0.0.1", network: "10.0.0.0", gateway: "10.0.0.1", + sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"], + adminUser: "testadmin", syslogPort: 15514, + skipDnsmasq: true, skipArtifacts: true, + fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os", + tftpDir: join(dir, "tftp"), httpDir: join(dir, "http"), stateFile: join(dir, "state.json"), + }; + } + + beforeEach(async () => { + testDir = join(tmpdir(), `datadisks-test-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(testDir, { recursive: true }); + const built = await createApp(testConfig(testDir)); + app = built.app; + await app.ready(); + }); + + afterEach(async () => { + await app.close(); + rmSync(testDir, { recursive: true, force: true }); + }); + + const queue = (body: Record) => + app.inject({ method: "POST", url: "/api/install", payload: { mac: "aa:bb:cc:dd:ee:03", hostname: "thebeast", ...body } }); + + const ks = async () => (await app.inject({ method: "GET", url: "/ks?mac=aa:bb:cc:dd:ee:03" })).body; + + it("named disks reach the rendered kickstart, and their LVs are gone", async () => { + const res = await queue({ role: "k8s-worker", dataDisks: { longhorn: LONGHORN_ID, rancher: RANCHER_ID } }); + expect(res.statusCode).toBe(200); + const body = await ks(); + expect(body).toContain(LONGHORN_ID); + expect(body).toContain(RANCHER_ID); + expect(freshBlock(body)).not.toContain("--name=longhorn"); + expect(freshBlock(body)).not.toContain("--name=rancher"); + }); + + it("rejects dataDisks on a non-fedora install instead of silently dropping them", async () => { + const res = await queue({ role: "k8s-worker", os: "ubuntu-26.04", dataDisks: { longhorn: LONGHORN_ID } }); + expect(res.statusCode).toBe(400); + expect(res.json().error).toMatch(/fedora/); + }); + + it("rejects dataDisks with an explicit v1 schema", async () => { + const res = await queue({ role: "worker", schemaVersion: 1, dataDisks: { longhorn: LONGHORN_ID } }); + expect(res.statusCode).toBe(400); + expect(res.json().error).toMatch(/schemaVersion 2/); + }); + + it("rejects dataDisks for a legacy role that would default to v1", async () => { + const res = await queue({ role: "worker", dataDisks: { longhorn: LONGHORN_ID } }); + expect(res.statusCode).toBe(400); + expect(res.json().error).toMatch(/schemaVersion 2/); + }); + + it("legacy role + explicit v2 + dataDisks is accepted", async () => { + const res = await queue({ role: "worker", schemaVersion: 2, dataDisks: { longhorn: LONGHORN_ID } }); + expect(res.statusCode).toBe(200); + expect(await ks()).toContain(LONGHORN_ID); + }); + + it("rejects a kernel-name disk with the shared message", async () => { + const res = await queue({ role: "k8s-worker", dataDisks: { longhorn: "nvme1n1" } }); + expect(res.statusCode).toBe(400); + expect(res.json().error).toMatch(/by-id/); + }); + + it("rejects a capability mismatch (longhorn disk on a control-plane-only role)", async () => { + const res = await queue({ role: "k8s-controller", dataDisks: { longhorn: LONGHORN_ID } }); + expect(res.statusCode).toBe(400); + expect(res.json().error).toMatch(/no Longhorn replicas/); + }); + + it("an install without dataDisks is untouched by all of this", async () => { + expect((await queue({ role: "k8s-worker" })).statusCode).toBe(200); + const body = await ks(); + expect(freshBlock(body)).toContain("--name=longhorn --fstype=xfs --size=307200"); + expect(freshBlock(body)).toContain("--name=rancher --fstype=xfs --size=256000"); + }); +}); diff --git a/bastion/src/cli/src/api/client.ts b/bastion/src/cli/src/api/client.ts index a5cda4c..00d6021 100644 --- a/bastion/src/cli/src/api/client.ts +++ b/bastion/src/cli/src/api/client.ts @@ -91,6 +91,7 @@ export class LabdClient { async installMachine(opts: { mac: string; hostname: string; disk?: string; role?: string; os?: string; schemaVersion?: 1 | 2; + dataDisks?: import("@lab/shared").DataDisks; vyos?: import("@lab/shared").VyosInstallSpec; }): Promise<{ status: string; data?: unknown; error?: string }> { return this.request("POST", "/api/machines/install", { body: opts }); diff --git a/bastion/src/cli/src/commands/install.ts b/bastion/src/cli/src/commands/install.ts index b9f97d0..e49e4fb 100644 --- a/bastion/src/cli/src/commands/install.ts +++ b/bastion/src/cli/src/commands/install.ts @@ -3,8 +3,8 @@ import { readFileSync } from "node:fs"; import { Command, Option, InvalidArgumentError } from "commander"; -import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY } from "@lab/shared"; -import type { VyosBundle, VyosInstallSpec, VyosVlanSpec } from "@lab/shared"; +import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isRole, requiresSchemaV2, validateDataDisks } from "@lab/shared"; +import type { DataDisks, VyosBundle, VyosInstallSpec, VyosVlanSpec } from "@lab/shared"; import { getLabdClient } from "../api/config.js"; /** @@ -106,8 +106,16 @@ export function registerInstallCommand(parent: Command): void { .addHelpText("after", roleTable()) .addOption(new Option("--role ", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker")) .addOption(new Option("--os ", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43")) - .option("--disk ", "Target disk device (auto-detect if omitted)") + .option("--disk ", "Target OS disk: kernel name or stable /dev/disk/by-id/ path (auto-detect if omitted)") .addOption(new Option("--schema-version ", "Disk layout for a FRESH install (default 1). Ignored on reprovision: the layout already on the disk wins.").choices(["1", "2"])) + .option( + "--longhorn-disk ", + "Dedicated whole disk for /var/lib/longhorn, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)", + ) + .option( + "--rancher-disk ", + "Dedicated whole disk for /var/lib/rancher, as a stable /dev/disk/by-id/ path (fedora, schema v2 only)", + ) .option("--vyos-mgmt ", "VyOS: untagged interface the machine PXE boots from (default eth0)") .option("--vyos-mgmt-address ", "VyOS: CIDR for the management interface, or 'dhcp' (default dhcp)") .option("--vyos-bond ", "VyOS: comma-separated LACP bond members (must exclude the PXE NIC)") @@ -140,6 +148,8 @@ export function registerInstallCommand(parent: Command): void { /** commander gives us the string form of the --schema-version choice. */ schemaVersion?: string; disk?: string; + longhornDisk?: string; + rancherDisk?: string; vyosMgmt?: string; vyosMgmtAddress?: string; vyosBond?: string; @@ -166,6 +176,32 @@ export function registerInstallCommand(parent: Command): void { process.exit(1); } + // Dedicated data disks -- validated here for fast feedback, and again on + // the bastion, which is the enforcement point. + const dataDisks: DataDisks = { + ...(opts.longhornDisk !== undefined && opts.longhornDisk !== "" + ? { longhorn: opts.longhornDisk } : {}), + ...(opts.rancherDisk !== undefined && opts.rancherDisk !== "" + ? { rancher: opts.rancherDisk } : {}), + }; + const hasDataDisks = Object.keys(dataDisks).length > 0; + if (hasDataDisks) { + if (opts.os !== "fedora-43") { + console.error(`--longhorn-disk/--rancher-disk only work with --os fedora-43 (got --os ${opts.os})`); + process.exit(1); + } + if (opts.schemaVersion === "1" + || (opts.schemaVersion === undefined && isRole(opts.role) && !requiresSchemaV2(opts.role))) { + console.error("--longhorn-disk/--rancher-disk need the v2 layout -- add --schema-version 2"); + process.exit(1); + } + const problem = isRole(opts.role) ? validateDataDisks(opts.role, dataDisks) : null; + if (problem !== null) { + console.error(`Invalid data disks: ${problem}`); + process.exit(1); + } + } + const bondMembers = opts.vyosBond !== undefined && opts.vyosBond !== "" ? opts.vyosBond.split(",").map((s) => s.trim()).filter((s) => s.length > 0) : []; @@ -257,6 +293,7 @@ export function registerInstallCommand(parent: Command): void { role: opts.role, os: opts.os, ...(opts.disk ? { disk: opts.disk } : {}), + ...(hasDataDisks ? { dataDisks } : {}), ...(hasVyosOptions ? { vyos } : {}), }); diff --git a/bastion/src/cli/tests/smoke-bastion.test.ts b/bastion/src/cli/tests/smoke-bastion.test.ts index dd50230..2030e50 100644 --- a/bastion/src/cli/tests/smoke-bastion.test.ts +++ b/bastion/src/cli/tests/smoke-bastion.test.ts @@ -143,7 +143,10 @@ describe("bastion smoke tests", () => { await sleep(200); } - expect(stdout).toContain("Waiting for PXE boot requests"); + // On failure, show what the process actually said -- an empty stdout with + // a hidden stderr is undiagnosable from a CI log. + expect(stdout, `bastion never printed its banner; stderr was:\n${stderr}`) + .toContain("Waiting for PXE boot requests"); expect(stdout).toContain("HTTP server listening"); // Verify the process is still alive after startup diff --git a/bastion/src/labd/src/routes/bastions.ts b/bastion/src/labd/src/routes/bastions.ts index 8792c5c..b88dd54 100644 --- a/bastion/src/labd/src/routes/bastions.ts +++ b/bastion/src/labd/src/routes/bastions.ts @@ -164,9 +164,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void // Queue install — route to correct bastion by MAC app.post<{ - Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string; vyos?: VyosInstallSpec }; + Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string; schemaVersion?: 1 | 2; dataDisks?: import("@lab/shared").DataDisks; vyos?: VyosInstallSpec }; }>("/api/machines/install", async (request, reply) => { - const { mac, hostname, disk, role, os, vyos } = request.body ?? {}; + const { mac, hostname, disk, role, os, schemaVersion, dataDisks, vyos } = request.body ?? {}; if (!mac || !hostname) { return reply.code(400).send({ error: "mac and hostname are required" }); } @@ -184,6 +184,8 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void const result = await sendCommand(all[0]!.bastionId, { type: "command-install", mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43", + ...(schemaVersion ? { schemaVersion } : {}), + ...(dataDisks ? { dataDisks } : {}), ...(vyos ? { vyos } : {}), }); return reply.code(result.status === "ok" ? 200 : 500).send(result); @@ -198,6 +200,8 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void const result = await sendCommand(bastion.bastionId, { type: "command-install", mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43", + ...(schemaVersion ? { schemaVersion } : {}), + ...(dataDisks ? { dataDisks } : {}), ...(vyos ? { vyos } : {}), }); return reply.code(result.status === "ok" ? 200 : 500).send(result); diff --git a/bastion/src/shared/src/index.ts b/bastion/src/shared/src/index.ts index 75eeff6..2e22ac3 100644 --- a/bastion/src/shared/src/index.ts +++ b/bastion/src/shared/src/index.ts @@ -51,5 +51,6 @@ export { } from "./errors/index.js"; export { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB, - SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion } from "./types/index.js"; -export type { LvSpec, SchemaVersion } from "./types/index.js"; + SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion, + validateDataDisks, DATA_DISK_LABELS } from "./types/index.js"; +export type { LvSpec, SchemaVersion, DataDisks, DataDiskName } from "./types/index.js"; diff --git a/bastion/src/shared/src/protocol/index.ts b/bastion/src/shared/src/protocol/index.ts index d7f626a..ab75d3c 100644 --- a/bastion/src/shared/src/protocol/index.ts +++ b/bastion/src/shared/src/protocol/index.ts @@ -109,7 +109,7 @@ export type BastionMessage = export type LabdBastionMessage = | { type: "bastion-enrolled"; bastionId: string } | { type: "bastion-heartbeat-ack"; serverTime: string } - | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string; vyos?: VyosInstallSpec } + | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string; schemaVersion?: 1 | 2; dataDisks?: import("../types/layout.js").DataDisks; vyos?: VyosInstallSpec } | { type: "command-forget"; requestId: string; mac: string } | { type: "command-role-update"; requestId: string; mac: string; role: string } | { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean } diff --git a/bastion/src/shared/src/types/index.ts b/bastion/src/shared/src/types/index.ts index 4006132..a357f8a 100644 --- a/bastion/src/shared/src/types/index.ts +++ b/bastion/src/shared/src/types/index.ts @@ -20,5 +20,6 @@ export type { RoleInfo, Lifecycle, K3sMode } from "./state.js"; export type { BastionConfig } from "./config.js"; export { v2Layout, v2AllocatedMib, V2_BOOT, V2_RANCHER_MIB, V2_LONGHORN_MIB, - SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion } from "./layout.js"; -export type { LvSpec, SchemaVersion } from "./layout.js"; + SCHEMA_VERSIONS, CONTAINERD_BIND_FSTAB, LEGACY_ROLES, requiresSchemaV2, effectiveSchemaVersion, + validateDataDisks, DATA_DISK_LABELS } from "./layout.js"; +export type { LvSpec, SchemaVersion, DataDisks, DataDiskName } from "./layout.js"; diff --git a/bastion/src/shared/src/types/layout.ts b/bastion/src/shared/src/types/layout.ts index 77c0333..bde4b23 100644 --- a/bastion/src/shared/src/types/layout.ts +++ b/bastion/src/shared/src/types/layout.ts @@ -94,13 +94,80 @@ const V2_BASE: readonly LvSpec[] = [ export const V2_RANCHER_MIB = 256000; // 250 GiB export const V2_LONGHORN_MIB = 307200; // 300 GiB +/** + * Dedicated whole-disk data volumes (v2 only). + * + * The fixed 250G/300G LVs exist to FIT a 1 TB OS disk. A machine with spare + * NVMe bays (thebeast: 1 TB OS + 2 TB + 2 TB) wants the opposite trade: give + * each data mount a whole disk and keep the OS VG small. Each value is a + * stable /dev/disk/by-id/ path -- NEVER a kernel name, because nvme0n1/nvme1n1 + * enumeration order is not stable across boots on multi-NVMe hosts, and a + * swapped name here formats the wrong disk. + * + * The disk gets one partition spanning it, xfs, labelled (DATA_DISK_LABELS) + * so a reprovision can recognise and preserve it the same way the LVs are + * recognised by name inside the VG. The corresponding LV is simply not + * created -- v2Layout() drops it when a dedicated disk supplies that mount. + */ +export interface DataDisks { + /** Whole disk for /var/lib/longhorn (replica store). */ + longhorn?: string; + /** Whole disk for /var/lib/rancher (image store + local-path PVCs). */ + rancher?: string; +} + +/** + * Filesystem labels that mark a dedicated data disk's partition. They are the + * reprovision contract: %pre finds these by label and mounts them --noformat, + * exactly as it finds LVs by name. Changing a label orphans every disk + * already carrying the old one. + */ +export const DATA_DISK_LABELS = { + longhorn: "lab-longhorn", + rancher: "lab-rancher", +} as const; + +export type DataDiskName = keyof typeof DATA_DISK_LABELS; + +const BY_ID_PREFIX = "/dev/disk/by-id/"; + +/** + * Why a DataDisks spec is unusable for this role, or null when it is fine. + * Shared by the HTTP API, the websocket command handler and the CLI so all + * three reject with the same words instead of three drifting checks. + */ +export function validateDataDisks(role: Role, disks: DataDisks): string | null { + const info = roleInfo(role); + for (const name of Object.keys(DATA_DISK_LABELS) as DataDiskName[]) { + const device = disks[name]; + if (device === undefined) continue; + if (!device.startsWith(BY_ID_PREFIX)) { + return `${name} disk must be a stable ${BY_ID_PREFIX} path (got '${device}'). ` + + `Kernel names like nvme0n1 change across boots on multi-NVMe hosts.`; + } + } + if (disks.rancher !== undefined && info.k3sMode === null) { + return `role '${role}' runs no k3s -- a dedicated rancher disk makes no sense for it`; + } + if (disks.longhorn !== undefined && (info.k3sMode === null || info.controlPlaneOnly)) { + return `role '${role}' holds no Longhorn replicas -- a dedicated longhorn disk makes no sense for it`; + } + if (disks.longhorn !== undefined && disks.longhorn === disks.rancher) { + return `longhorn and rancher point at the same disk (${disks.longhorn}) -- they need one each`; + } + return null; +} + /** * The v2 volumes for a role, gated by CAPABILITY rather than role name. * * `omitRoot` is for Asahi, where the vendor installer already created a btrfs * root partition outside our VG and we must not make a second one. + * + * `dataDisks` drops the longhorn/rancher LV whose mount a dedicated disk + * supplies -- two things mounting /var/lib/longhorn is an invalid layout. */ -export function v2Layout(role: Role, opts: { omitRoot?: boolean } = {}): LvSpec[] { +export function v2Layout(role: Role, opts: { omitRoot?: boolean; dataDisks?: DataDisks } = {}): LvSpec[] { const info = roleInfo(role); const lvs: LvSpec[] = V2_BASE.filter( (lv) => !(opts.omitRoot && lv.name === "root"), @@ -109,12 +176,12 @@ export function v2Layout(role: Role, opts: { omitRoot?: boolean } = {}): LvSpec[ // Anything running k3s needs a dedicated image store. This is the check that // `role === "infra" || role === "worker"` kept getting wrong: labcontroller // runs k3s and was silently excluded. - if (info.k3sMode !== null) { + if (info.k3sMode !== null && opts.dataDisks?.rancher === undefined) { lvs.push({ name: "rancher", mount: "/var/lib/rancher", sizeMib: V2_RANCHER_MIB, fs: "xfs" }); } // Fixed, never --grow. A control-plane-only node holds no replicas. - if (info.k3sMode !== null && !info.controlPlaneOnly) { + if (info.k3sMode !== null && !info.controlPlaneOnly && opts.dataDisks?.longhorn === undefined) { lvs.push({ name: "longhorn", mount: "/var/lib/longhorn", sizeMib: V2_LONGHORN_MIB, fs: "xfs" }); } @@ -122,7 +189,7 @@ export function v2Layout(role: Role, opts: { omitRoot?: boolean } = {}): LvSpec[ } /** Total MiB a role's v2 layout consumes inside the VG. */ -export function v2AllocatedMib(role: Role, opts: { omitRoot?: boolean } = {}): number { +export function v2AllocatedMib(role: Role, opts: { omitRoot?: boolean; dataDisks?: DataDisks } = {}): number { return v2Layout(role, opts).reduce((n, lv) => n + lv.sizeMib, 0); } diff --git a/bastion/src/shared/src/types/state.ts b/bastion/src/shared/src/types/state.ts index e3ecfdd..7af2c20 100644 --- a/bastion/src/shared/src/types/state.ts +++ b/bastion/src/shared/src/types/state.ts @@ -356,6 +356,13 @@ export interface InstallConfig { * disk instead, because reinstalling a machine must never change its layout. */ schemaVersion?: 1 | 2; + /** + * Dedicated whole-disk data volumes (v2 + fedora only), by stable + * /dev/disk/by-id path. See DataDisks in layout.ts. On a reprovision these + * are optional: %pre rediscovers the disks by filesystem label and preserves + * them whether or not the queue entry names them. + */ + dataDisks?: import("./layout.js").DataDisks; queued_at: string; /** * When dispatch last served this machine an install boot script. Progress diff --git a/bastion/tests/integration/data-disks.test.ts b/bastion/tests/integration/data-disks.test.ts new file mode 100644 index 0000000..a018452 --- /dev/null +++ b/bastion/tests/integration/data-disks.test.ts @@ -0,0 +1,173 @@ +// Integration test: dedicated data disks, on a real VM. +// +// The unit tests prove the kickstart RENDERS the right %pre and part lines. +// This proves anaconda ACTUALLY PRODUCES the layout thebeast will get: OS on +// one disk, /var/lib/longhorn and /var/lib/rancher each a whole-disk labelled +// xfs partition on their own disks -- and that a reprovision with NO flags +// rediscovers both by label and remounts them without reformatting. +// +// The VM mirrors thebeast's topology (1 OS disk + 2 data disks addressed by +// /dev/disk/by-id/ serial); virtio serials stand in for NVMe serials, which is +// exactly the aliasing the feature is specified over (any by-id path). +// +// Run (on a libvirt host): +// sudo -E npx vitest run -c tests/integration/vitest.config.ts tests/integration/data-disks.test.ts +// +// Set FEDORA_MIRROR to a local mirror or each install pulls packages from the +// internet (~12-15 min per install either way; two installs in this file). + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { existsSync, readFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { v2Layout, DATA_DISK_LABELS } from "../../src/shared/src/types/layout.js"; +import { + startTestBastion, provisionVm, reprovisionVm, readLvs, readSchemaTag, + writeMarkers, readMarkers, destroyVms, virtioById, + type Harness, type ProvisionedVm, +} from "./helpers/provision-harness.js"; +import { destroyPxeNetwork } from "./helpers/pxe-network.js"; +import { sshExec } from "./helpers/ssh.js"; + +const HTTP_PORT = 8094; +const SSH_USER = "lab"; +const DISK_GB = 1000; // same sparse OS disk as every other layout test + +// Short virtio serials (virtio-blk truncates at 20 chars), echoing thebeast's +// real disk serial fragments so the drill log reads like the runbook. +const LH = { serial: "LHDRILL7F600711", sizeGb: 20 }; +const RN = { serial: "RNDRILL7FD4070C", sizeGb: 20 }; + +const PRESERVED = ["/home", "/srv", "/var/lib/longhorn", "/var/lib/rancher"]; + +function findSshKey(): { pubKey: string; keyPath: string } { + // A passphrase-protected default key fails SILENTLY here: BatchMode ssh + // never offers it without an agent, the server logs only "Connection closed + // [preauth]", and the test reads as "installed machine never came up" when + // the machine was fine. LAB_TEST_SSH_KEY names a dedicated passphrase-less + // keypair and sidesteps the whole class. + const envKey = process.env["LAB_TEST_SSH_KEY"]; + if (envKey) { + if (!existsSync(envKey) || !existsSync(`${envKey}.pub`)) { + throw new Error(`LAB_TEST_SSH_KEY=${envKey}: key or .pub missing`); + } + return { pubKey: readFileSync(`${envKey}.pub`, "utf-8").trim(), keyPath: envKey }; + } + const homes = [homedir()]; + const sudoUser = process.env["SUDO_USER"]; + if (sudoUser) homes.push(join("/home", sudoUser)); + for (const home of homes) { + for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) { + const keyPath = join(home, ".ssh", name); + if (existsSync(keyPath) && existsSync(`${keyPath}.pub`)) { + return { pubKey: readFileSync(`${keyPath}.pub`, "utf-8").trim(), keyPath }; + } + } + } + throw new Error("no SSH keypair found"); +} + +/** Run a command on the VM and insist it succeeds. */ +function ssh(h: Harness, vm: ProvisionedVm, cmd: string): string { + const r = sshExec(vm.ip, h.sshUser, cmd, { keyPath: h.sshKeyPath }); + if (r.exitCode !== 0) throw new Error(`[${vm.name}] \`${cmd}\` failed: ${r.stderr || r.stdout}`); + return r.stdout.trim(); +} + +/** + * What actually backs a mount point, and whether it is the whole-disk labelled + * partition the feature promises. + */ +function inspectMount(h: Harness, vm: ProvisionedVm, mount: string, serial: string) { + const source = ssh(h, vm, `findmnt -no SOURCE ${mount}`); + const fstype = ssh(h, vm, `findmnt -no FSTYPE ${mount}`); + const label = ssh(h, vm, `sudo blkid -s LABEL -o value ${source}`); + const uuid = ssh(h, vm, `sudo blkid -s UUID -o value ${source}`); + // The partition must live on the disk with OUR serial, not wherever the + // kernel happened to enumerate something. + const disk = ssh(h, vm, `readlink -f ${virtioById(serial)}`); + const parent = "/dev/" + ssh(h, vm, `lsblk -ndo pkname ${source}`); + return { source, fstype, label, uuid, disk, parent }; +} + +describe("dedicated data disks on real hardware", () => { + let h: Harness; + const created: string[] = []; + + beforeAll(async () => { + const { pubKey, keyPath } = findSshKey(); + h = await startTestBastion({ + httpPort: HTTP_PORT, sshUser: SSH_USER, sshPubKey: pubKey, + sshKeyPath: keyPath, label: "datadisks", + }); + }, 20 * 60_000); + + afterAll(async () => { + // KEEP_VMS=1 leaves the VM behind for a post-mortem (virsh console, + // screenshot, mounting the qcow2). Without it a failed boot is destroyed + // along with every clue about why it failed. + if (!process.env["KEEP_VMS"]) destroyVms(...created); + try { await h?.close(); } catch { /* closed */ } + try { destroyPxeNetwork(); } catch { /* gone */ } + }, 5 * 60_000); + + it("k8s-worker with longhorn+rancher disks: install, then flagless reprovision rediscovers both", async () => { + const name = "dd-worker"; + created.push(name); + const dataDisks = { longhorn: virtioById(LH.serial), rancher: virtioById(RN.serial) }; + + // --- Fresh install, thebeast-style ----------------------------------- + let vm = await provisionVm(h, { + name, role: "k8s-worker", schemaVersion: 2, diskGb: DISK_GB, + dataDisks: { longhorn: LH, rancher: RN }, + }); + + // The VG must hold exactly the v2 set MINUS the LVs the disks replace. + const lvs = readLvs(h, vm); + const expected = v2Layout("k8s-worker", { dataDisks }).map((lv) => lv.name).sort(); + expect(Object.keys(lvs).sort(), "volume set").toEqual(expected); + expect(lvs["longhorn"]).toBeUndefined(); + expect(lvs["rancher"]).toBeUndefined(); + expect(readSchemaTag(h, vm)).toContain("lab-schema-v2"); + + // Each mount is a whole-disk xfs partition, labelled, on the RIGHT disk. + const lhBefore = inspectMount(h, vm, "/var/lib/longhorn", LH.serial); + expect(lhBefore.fstype).toBe("xfs"); + expect(lhBefore.label).toBe(DATA_DISK_LABELS.longhorn); + expect(lhBefore.parent, "/var/lib/longhorn sits on the longhorn disk").toBe(lhBefore.disk); + + const rnBefore = inspectMount(h, vm, "/var/lib/rancher", RN.serial); + expect(rnBefore.fstype).toBe("xfs"); + expect(rnBefore.label).toBe(DATA_DISK_LABELS.rancher); + expect(rnBefore.parent, "/var/lib/rancher sits on the rancher disk").toBe(rnBefore.disk); + + const token = writeMarkers(h, vm, PRESERVED); + + // --- Reprovision with NO dataDisks flags ------------------------------ + // This is the claim that matters for day 2: the labels on disk are the + // truth, and a plain `labctl provision reprovision` must find and keep + // them. Passing the flags again here would hide a regression in exactly + // that rediscovery. + vm = await reprovisionVm(h, vm, "k8s-worker"); + + const markers = readMarkers(h, vm, PRESERVED); + for (const mount of PRESERVED) { + expect(markers[mount], `${mount} must survive a reinstall`).toBe(token); + } + + // Still LV-less for longhorn/rancher, still v2. + const lvsAfter = readLvs(h, vm); + expect(Object.keys(lvsAfter).sort()).toEqual(expected); + expect(readSchemaTag(h, vm)).toContain("lab-schema-v2"); + + // Same partitions, same filesystems. An unchanged UUID is the proof that + // --noformat really happened -- a remade xfs with the same label would + // pass every other check here. + const lhAfter = inspectMount(h, vm, "/var/lib/longhorn", LH.serial); + expect(lhAfter.uuid, "longhorn fs was NOT reformatted").toBe(lhBefore.uuid); + expect(lhAfter.parent).toBe(lhAfter.disk); + const rnAfter = inspectMount(h, vm, "/var/lib/rancher", RN.serial); + expect(rnAfter.uuid, "rancher fs was NOT reformatted").toBe(rnBefore.uuid); + expect(rnAfter.parent).toBe(rnAfter.disk); + }, 150 * 60_000); +}); diff --git a/bastion/tests/integration/helpers/provision-harness.ts b/bastion/tests/integration/helpers/provision-harness.ts index 2a535b6..e883a48 100644 --- a/bastion/tests/integration/helpers/provision-harness.ts +++ b/bastion/tests/integration/helpers/provision-harness.ts @@ -14,7 +14,7 @@ import type { BastionConfig } from "../../../src/shared/src/types/config.js"; import type { Role } from "../../../src/shared/src/types/state.js"; import { log, waitForSsh } from "./libvirt.js"; import { ensurePxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_BRIDGE, PXE_GATEWAY, PXE_SUBNET } from "./pxe-network.js"; -import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm } from "./pxe-vm.js"; +import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm, readSerialLog } from "./pxe-vm.js"; import { sshExec } from "./ssh.js"; export const DISCOVERY_TIMEOUT_MS = 6 * 60_000; @@ -295,7 +295,23 @@ async function waitForInstallStart( } } -export interface ProvisionedVm { name: string; mac: string; ip: string } +export interface ProvisionedVm { name: string; mac: string; ip: string; serialPort: number } + +/** + * "SSH never came up" on its own is undiagnosable once afterAll has destroyed + * the VM -- the machine may be sitting in emergency mode with the exact error + * on its serial console. Capture that console into the failure. + */ +async function sshFailurePostMortem(name: string, serialPort: number, cause: unknown): Promise { + let state = "unknown"; + try { state = execSync(`sudo virsh domstate ${name}`, { encoding: "utf-8" }).trim(); } catch { /* gone */ } + let serial = "(no serial output)"; + try { serial = await readSerialLog(serialPort, { lastLines: 60 }); } catch { /* no console */ } + throw new Error( + `[${name}] ${cause instanceof Error ? cause.message : String(cause)}\n` + + `domstate: ${state}\n--- serial console (last lines) ---\n${serial}`, + ); +} /** * Serial ports are a global resource on the host, so hand out a distinct one @@ -304,6 +320,15 @@ export interface ProvisionedVm { name: string; mac: string; ip: string } */ let nextSerialPort = 4600; +/** + * The stable path a virtio data disk gets inside the guest. This is what the + * install request must carry: the feature refuses anything outside + * /dev/disk/by-id/, exactly as it does for real NVMe serials. + */ +export function virtioById(serial: string): string { + return `/dev/disk/by-id/virtio-${serial}`; +} + /** Create a blank VM, let it discover, queue the install, wait for SSH. */ export async function provisionVm(h: Harness, spec: { name: string; @@ -312,16 +337,26 @@ export async function provisionVm(h: Harness, spec: { diskGb: number; memory?: number; vcpus?: number; + /** + * Dedicated data disks: attached to the VM with the given virtio serial AND + * passed to /api/install as /dev/disk/by-id/virtio-. Mirrors + * thebeast's layout (OS disk + separate longhorn + rancher disks). + */ + dataDisks?: { longhorn?: { serial: string; sizeGb: number }; rancher?: { serial: string; sizeGb: number } }; }): Promise { const base = `http://${PXE_GATEWAY}:${h.httpPort}`; - log(`[${spec.name}] creating VM (${spec.diskGb}GB disk, role=${spec.role}, schema=v${spec.schemaVersion ?? 1})`); + const vmDataDisks = [spec.dataDisks?.longhorn, spec.dataDisks?.rancher] + .filter((d): d is { serial: string; sizeGb: number } => d !== undefined); + log(`[${spec.name}] creating VM (${spec.diskGb}GB disk + ${vmDataDisks.length} data disk(s), role=${spec.role}, schema=v${spec.schemaVersion ?? 1})`); + const serialPort = nextSerialPort++; createPxeVm({ name: spec.name, memory: spec.memory ?? 4096, vcpus: spec.vcpus ?? 4, diskSize: spec.diskGb, network: PXE_NETWORK_NAME, - serialPort: nextSerialPort++, + serialPort, + ...(vmDataDisks.length > 0 ? { dataDisks: vmDataDisks } : {}), }); deleteNftablesRejectRules(); @@ -339,6 +374,12 @@ export async function provisionVm(h: Harness, spec: { body: JSON.stringify({ mac, hostname: spec.name, disk: "", role: spec.role, ...(spec.schemaVersion ? { schemaVersion: spec.schemaVersion } : {}), + ...(spec.dataDisks ? { + dataDisks: { + ...(spec.dataDisks.longhorn ? { longhorn: virtioById(spec.dataDisks.longhorn.serial) } : {}), + ...(spec.dataDisks.rancher ? { rancher: virtioById(spec.dataDisks.rancher.serial) } : {}), + }, + } : {}), }), }); if (!res.ok) throw new Error(`[${spec.name}] install queue rejected: ${res.status} ${await res.text()}`); @@ -376,9 +417,13 @@ export async function provisionVm(h: Harness, spec: { deleteNftablesRejectRules(); log(`[${spec.name}] waiting for SSH at ${ip}...`); - await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath); + try { + await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath); + } catch (e) { + await sshFailurePostMortem(spec.name, serialPort, e); + } log(`[${spec.name}] up at ${ip}`); - return { name: spec.name, mac, ip }; + return { name: spec.name, mac, ip, serialPort }; } /** @@ -498,7 +543,11 @@ export async function reprovisionVm( } catch { /* already running */ } deleteNftablesRejectRules(); - await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath); + try { + await waitForSsh(ip, h.sshUser, SSH_TIMEOUT_MS, h.sshKeyPath); + } catch (e) { + await sshFailurePostMortem(vm.name, vm.serialPort, e); + } log(`[${vm.name}] back up at ${ip} after reinstall`); return { ...vm, ip }; } diff --git a/bastion/tests/integration/helpers/pxe-vm.ts b/bastion/tests/integration/helpers/pxe-vm.ts index 2972200..13dea2d 100644 --- a/bastion/tests/integration/helpers/pxe-vm.ts +++ b/bastion/tests/integration/helpers/pxe-vm.ts @@ -47,6 +47,15 @@ export interface PxeVmConfig { */ decoyNics?: number; decoyNetwork?: string; + /** + * Extra blank data disks, attached after the OS disk. Each gets a fixed + * virtio serial, so inside the guest it has a stable + * /dev/disk/by-id/virtio- path -- the moral equivalent of the + * nvme-..._ links the dedicated-data-disk feature requires on real + * hardware. virtio-blk truncates serials at 20 characters, so keep them + * short or the by-id link quietly won't match. + */ + dataDisks?: { serial: string; sizeGb: number }[]; } /** Create a blank UEFI VM that PXE boots from the network. */ @@ -60,6 +69,14 @@ export function createPxeVm(config: PxeVmConfig): void { const diskPath = join(IMAGE_DIR, `${config.name}.qcow2`); run(`qemu-img create -f qcow2 "${diskPath}" ${config.diskSize}G`); + const dataDiskArgs: string[] = []; + for (const dd of config.dataDisks ?? []) { + if (dd.serial.length > 20) throw new Error(`virtio serial '${dd.serial}' exceeds 20 chars -- the guest by-id link would be truncated`); + const p = join(IMAGE_DIR, `${config.name}-${dd.serial}.qcow2`); + run(`qemu-img create -f qcow2 "${p}" ${dd.sizeGb}G`); + dataDiskArgs.push(`--disk=path=${p},format=qcow2,bus=virtio,serial=${dd.serial}`); + } + // UEFI firmware paths (Fedora) if (arch === "aarch64") { const aavmf = "/usr/share/edk2/aarch64/QEMU_EFI.fd"; @@ -79,6 +96,7 @@ export function createPxeVm(config: PxeVmConfig): void { `--memory=${config.memory}`, `--vcpus=${config.vcpus}`, `--disk=path=${diskPath},format=qcow2,bus=virtio`, + ...dataDiskArgs, // Decoys first so they enumerate ahead of the PXE NIC. They are up and // carry a lease, but have no route to the bastion. ...Array.from({ length: config.decoyNics ?? 0 }, () => diff --git a/migration/THEBEAST-K8S-WORKER.md b/migration/THEBEAST-K8S-WORKER.md new file mode 100644 index 0000000..3ea8d1d --- /dev/null +++ b/migration/THEBEAST-K8S-WORKER.md @@ -0,0 +1,113 @@ +# thebeast → prod k8s worker — conversion runbook + +Converting the old desktop **thebeast** (currently Fedora/ZFS desktop at +`10.0.1.22`) into a `k8s-worker` node of the **prod cluster (k8s0)** via the +PXE bastion, using the dedicated-data-disk layout (`--longhorn-disk` / +`--rancher-disk`, introduced with this runbook). + +Decisions already taken (2026-10-14): Windows disk is sacrificed (unused for +years, no backup wanted) · cluster = prod k8s0 · OS = Fedora `k8s-worker` +(thebeast is mainly an AMD-GPU machine: 2× AMD + 1× RTX 3080 Ti, no unified +memory) · the node will carry a dedicated taint and also serve CI/CD. + +**VM-drilled 2026-10-14** (on thebeast itself, evidence: +`drill-evidence/data-disks-2026-10-14.txt`): fresh install lands the exact +layout below, and a flagless reprovision rediscovers both data disks by label +and remounts them without reformatting (UUIDs unchanged, marker files intact). + +## Disk map — ALWAYS by-id, never nvme0n1 + +Kernel NVMe names are **not stable across boots on this machine** (observed). +Every command below uses `/dev/disk/by-id`. + +| target | disk | by-id | +|---|---|---| +| OS (wipes Windows) | Samsung 970 EVO Plus 1TB | `nvme-Samsung_SSD_970_EVO_Plus_1TB_S4EWNJ0N138234M` | +| /var/lib/longhorn | Sabrent Rocket 4.0 2TB | `nvme-Sabrent_Rocket_4.0_2TB_7F600711179D89261778` | +| /var/lib/rancher | Sabrent Rocket 4.0 2TB | `nvme-Sabrent_Rocket_4.0_2TB_7FD4070C1EDA92197301` | +| DISCONNECT after final shutdown | Samsung 970 EVO Plus 2TB `S4J4NF0NC00809J` | sits on its own motherboard add-on card, PCIe x4, model discontinued | + +NICs: `enp69s0` `34:97:f6:32:76:e9` (current uplink — this MAC is what the +bastion will see), `enp70s0` `34:97:f6:31:b3:99` (down). + +## 0. Before the final shutdown — data + +**rpool is a 4-wide raidz1 already DEGRADED** (member +`S4J4NF0R102776W` removed since 2026-09-14). There is **zero redundancy +left**: one more failure loses the pool, and a disk pulled out of it is NOT +importable alone. Finish the copies first, then never boot the desktop again. + +Already safe (verified 2026-10-14): `~/developer` (all 113 repos) + the +credential/dotfile set → bazzite (Oct 12 rsync, 1 file of drift). + +Copied to bazzite and rsync-verified 2026-10-14: `~/keyssecrets`, +`~/dicesecrets`, `~/Videos/canaries` (24G), `~/Pictures`, `~/Documents`. +`~/opt/vaultbackup` and `~/migration-backup` declared done/not-needed (user, +2026-10-14). + +Still open before shutdown: +- `~/Videos/{Webcam,subscribescreen,Screencasts,music}` (~370M) and + `~/backups/sogo-final-2026-09-13` — small, not yet copied +- Verify the Syncthing replicas on synology0-nas for `~/Logseq`, + `~/DICESharedBoostNote`, `~/Boostnote`, `~/MangaReader_backup` — thebeast's + syncthing shows 0 connected peers, so the NAS copy is **unverified** +- Final re-sync of `~/.claude` + atuin (activity postdates the Oct 12 copy) + +## 1. Physical + network + +1. Final desktop shutdown. +2. Disconnect the Samsung 2TB `S4J4NF0NC00809J` (the add-on card). +3. Move thebeast's switch port to the `192.168.8.0/24` VLAN (same as k8s0 and + labmaster). The bastion's proxy-DHCP (`dhcp-range=192.168.8.0,proxy`) + coexists with the real DHCP there — no DHCP changes needed. +4. Make sure network boot is enabled/first in firmware (UEFI PXE, IPv4). + +## 2. Discovery + +PXE boot. Unknown MAC → discovery image → the machine self-reports: + +``` +labctl provision list # thebeast appears under discovered (MAC 34:97:f6:32:76:e9) +``` + +## 3. Install + +``` +labctl provision install 34:97:f6:32:76:e9 thebeast \ + --role k8s-worker \ + --disk /dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_1TB_S4EWNJ0N138234M \ + --longhorn-disk /dev/disk/by-id/nvme-Sabrent_Rocket_4.0_2TB_7F600711179D89261778 \ + --rancher-disk /dev/disk/by-id/nvme-Sabrent_Rocket_4.0_2TB_7FD4070C1EDA92197301 +``` + +(`k8s-worker` is v2-only, so no `--schema-version` needed.) + +What the layout does: OS disk gets EFI + /boot + `labvg` with the v2 base LVs +only — **no** longhorn/rancher LVs. Each 2 TB disk gets ONE xfs partition +spanning it, labelled `lab-longhorn` / `lab-rancher`, mounted at +`/var/lib/longhorn` / `/var/lib/rancher`. The ZFS signatures on those disks +are wiped by the install — which is the point of section 0. + +Reinstall semantics from then on: the labels are on-disk truth. A reprovision +rediscovers both data disks by label (no flags needed) and mounts them +`--noformat`; a fresh install refuses to wipe or shadow a labelled disk it was +not told about. + +## 4. Join k8s0 + +As on every k8s-worker, k3s is installed with `INSTALL_K3S_SKIP_START`; the +join is deliberate (see `/root/README` on the node): + +``` +curl -sfL https://get.k3s.io | K3S_URL=https://:6443 K3S_TOKEN= sh - +``` + +Add the agreed CI/CD taint/labels at join time (k3s agent flags), e.g.: + +``` +--node-taint dedicated=cicd:NoSchedule --node-label itaz.eu/cicd=true +``` + +(exact taint TBD with the Pulumi model; GPU bits — ROCm for the 2 AMD cards, +nvidia for the 3080 Ti — are cluster-side follow-ups in +thelab-kubernetes-pulumi, not part of the install.) diff --git a/migration/drill-evidence/data-disks-2026-10-14.txt b/migration/drill-evidence/data-disks-2026-10-14.txt new file mode 100644 index 0000000..6f5711f --- /dev/null +++ b/migration/drill-evidence/data-disks-2026-10-14.txt @@ -0,0 +1,40 @@ +Dedicated data disks drill — fresh install + flagless reprovision, real VM +run finished: 2026-10-14T23:30+01:00 host: thebeast +test: bastion/tests/integration/data-disks.test.ts (1 passed, 1164s) +topology: 1 VM, role k8s-worker schema v2, 1000G OS disk + 2x 20G data disks +addressed as /dev/disk/by-id/virtio-LHDRILL7F600711 (longhorn) and +/dev/disk/by-id/virtio-RNDRILL7FD4070C (rancher) — the same by-id contract +thebeast's NVMe serials use. + +Proven: +- fresh install: labvg = home/root/srv/swap/var/varlog ONLY (no longhorn, + no rancher LV); /var/lib/longhorn and /var/lib/rancher are whole-disk xfs + partitions labelled lab-longhorn / lab-rancher, each verified to sit on + the disk with the matching serial; lab-schema-v2 tag present; fstab by UUID. +- reprovision with NO dataDisks flags: both disks rediscovered by label, + remounted --noformat (filesystem UUIDs identical before/after), marker + files intact in /home /srv /var/lib/longhorn /var/lib/rancher, LV set and + schema tag unchanged. + +=== checkpoints === +[22:10:40] [dd-worker] creating VM (1000GB disk + 2 data disk(s), role=k8s-worker, schema=v2) +[22:13:02] [dd-worker] queueing install... +[22:15:25] [dd-worker] installing (up to 60 min)... +[22:21:24] [dd-worker] up at 192.168.251.194 +[22:21:24] [dd-worker] on-disk LVs: home=8G root=32G srv=16G swap=16G var=48G varlog=16G +[22:21:27] [dd-worker] wrote marker preserve-1792016486955-iu1rih into /home, /srv, /var/lib/longhorn, /var/lib/rancher +[22:21:27] [dd-worker] reprovisioning (role=k8s-worker, schema: whatever is on disk)... +[22:29:58] [dd-worker] back up at 192.168.251.194 after reinstall +[22:29:59] [dd-worker] markers after reinstall: all four mounts carry preserve-1792016486955-iu1rih +[22:29:59] [dd-worker] on-disk LVs: home=8G root=32G srv=16G swap=16G var=48G varlog=16G +✓ k8s-worker with longhorn+rancher disks: install, then flagless reprovision rediscovers both (1161s) + +Run notes (cost of learning, so nobody pays it twice): +- the drill machine's bastion imports resolve @lab/shared to dist/ -- run + `pnpm build` after switching branches or the API answers with the OLD role + registry ("invalid role: 'k8s-worker'"). +- a passphrase-protected default ~/.ssh/id_rsa fails as "SSH not available" + with a perfectly healthy machine (BatchMode never offers it without an + agent). Use LAB_TEST_SSH_KEY with a dedicated passphrase-less key. +- run: cd bastion && LAB_TEST_SSH_KEY=~/.ssh/lab-integration_ed25519 \ + sudo -E npx vitest run -c tests/integration/vitest.config.ts tests/integration/data-disks.test.ts