diff --git a/bastion/completions/labctl.bash b/bastion/completions/labctl.bash index 628e688..689de91 100644 --- a/bastion/completions/labctl.bash +++ b/bastion/completions/labctl.bash @@ -59,10 +59,10 @@ _labctl() { COMPREPLY=($(compgen -W "-h --help" -- "$cur")) return ;; "provision install") - COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur")) + COMPREPLY=($(compgen -W "--role --os --disk --vyos-mgmt --vyos-mgmt-address --vyos-bond --vyos-bond-address --vyos-bond-vrrp --vlan-vip --vyos-vrrp-priority --vyos-mgmt-vlan --vlan --vyos-password --vyos-hwid --vyos-fresh-config -h --help" -- "$cur")) return ;; "provision reprovision") - COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur")) + COMPREPLY=($(compgen -W "--role --os --disk --user -h --help" -- "$cur")) return ;; "provision debug") COMPREPLY=($(compgen -W "--pxe-boot -h --help" -- "$cur")) diff --git a/bastion/completions/labctl.fish b/bastion/completions/labctl.fish index 26c1204..bcf2ac1 100644 --- a/bastion/completions/labctl.fish +++ b/bastion/completions/labctl.fish @@ -132,13 +132,26 @@ complete -c labctl -n "__labctl_using_cmd provision" -a recheck -d 'Refresh hard # provision install options complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller' -complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04' +complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling' complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target disk device (auto-detect if omitted)' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt -d 'VyOS: untagged interface the machine PXE boots from (default eth0)' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-address -d 'VyOS: CIDR for the management interface, or \'dhcp\' (default dhcp)' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond -d 'VyOS: comma-separated LACP bond members (must exclude the PXE NIC)' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond-address -d 'VyOS: address on the untagged bond (trunk native VLAN)' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond-vrrp -d 'VyOS: VRRP VIP floated on the untagged bond' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l vlan-vip -d 'VyOS: VRRP VIP for a --vlan entry (repeatable)' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-vrrp-priority -d 'VyOS: VRRP priority for all groups on this box (higher = master)' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-vlan -d 'VyOS: tagged management VLAN on the PXE port' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l vlan -d 'VyOS: tagged VLAN sub-interface on the bond (repeatable)' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-password -d 'VyOS: password for the \'vyos\' user' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-hwid -d 'VyOS: pin an interface name to a MAC via hw-id (repeatable)' -x +complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-fresh-config -d 'VyOS: on reinstall, overwrite the preserved config with the generated one' # provision reprovision options complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller' -complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04' +complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling' complete -c labctl -n "__labctl_in_cmd provision reprovision" -l disk -d 'Target disk device (auto-detect if omitted)' -x +complete -c labctl -n "__labctl_in_cmd provision reprovision" -l user -d 'SSH user for the reboot (default: vyos for VyOS machines, else current user)' -x # provision debug options complete -c labctl -n "__labctl_in_cmd provision debug" -l pxe-boot -d 'Boot installed system via PXE (kernel+initrd from network, root from NVMe)' diff --git a/bastion/package.json b/bastion/package.json index fc1b5a3..02b0f42 100644 --- a/bastion/package.json +++ b/bastion/package.json @@ -21,6 +21,8 @@ "test:integration:pxe:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'PXE boot'", "test:integration:iso": "vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'", "test:integration:iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'", + "test:integration:vyos": "vitest run -c tests/integration/vitest.config.ts -t 'VyOS provisioning'", + "test:integration:vyos:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'VyOS provisioning'", "test:integration:arm-iso": "vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'", "test:integration:arm-iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'", "test:integration:asahi": "vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'", diff --git a/bastion/src/bastion/src/config.ts b/bastion/src/bastion/src/config.ts index 4ebfe9b..1f4372b 100644 --- a/bastion/src/bastion/src/config.ts +++ b/bastion/src/bastion/src/config.ts @@ -20,6 +20,15 @@ export function loadConfig(overrides: Partial = {}): BastionConfi const ubuntuMirror = overrides.ubuntuMirror ?? process.env["UBUNTU_MIRROR"] ?? `https://releases.ubuntu.com/${ubuntuVersion}`; + // "latest" resolves the newest nightly ISO from the vyos-nightly-build GitHub + // releases at startup. downloads.vyos.io no longer serves direct rolling ISOs + // (it returns the vyos.io site, and nightly builds sit behind a signup form); + // GitHub releases are the remaining free, unauthenticated direct source. + // LTS ISOs are subscription-only. Set VYOS_ISO_URL to pin a specific build. + const vyosIsoUrl = overrides.vyosIsoUrl ?? process.env["VYOS_ISO_URL"] ?? "latest"; + const vyosDefaultPassword = overrides.vyosDefaultPassword + ?? process.env["VYOS_DEFAULT_PASSWORD"] ?? "vyos"; + const fedoraMirror = `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`; const tftpDir = `${bastionDir}/tftp`; const httpDir = `${bastionDir}/http`; @@ -38,6 +47,8 @@ export function loadConfig(overrides: Partial = {}): BastionConfi dhcpRangeEnd, ubuntuVersion, ubuntuMirror, + vyosIsoUrl, + vyosDefaultPassword, // These are populated at runtime by the network service iface: overrides.iface ?? "", serverIp: overrides.serverIp ?? "", diff --git a/bastion/src/bastion/src/main.ts b/bastion/src/bastion/src/main.ts index 7468f1f..2905650 100644 --- a/bastion/src/bastion/src/main.ts +++ b/bastion/src/bastion/src/main.ts @@ -40,6 +40,125 @@ function download(url: string, dest: string, label: string): void { } } +/** + * Pick the largest regular-file initrd from an `xorriso -lsl` listing. + * + * /live carries decoys: a 0-byte initrd.img placeholder on some images, or an + * initrd.img SYMLINK to the real version-suffixed file on others. Parsing is + * field-based (ls -l layout: perms links uid gid size month day time 'name') + * and considers only lines whose mode string marks a regular file — symlinks + * report their link size, not the target's, and must not win. + */ +export function pickLargestInitrd( + listing: string, +): { name: string; size: number } | undefined { + let best: { name: string; size: number } | undefined; + for (const line of listing.split("\n")) { + if (!line.startsWith("-")) continue; // regular files only + const quoted = /'([^']+)'/.exec(line); + const fields = line.trim().split(/\s+/); + const size = parseInt(fields[4] ?? "", 10); + const name = quoted?.[1] ?? ""; + if (!name.startsWith("initrd")) continue; + if (!Number.isFinite(size) || size <= 0) continue; + if (best === undefined || size > best.size) { + best = { name, size }; + } + } + return best; +} + +const VYOS_NIGHTLY_RELEASES = + "https://api.github.com/repos/vyos/vyos-nightly-build/releases/latest"; + +/** + * Resolve the configured VyOS ISO URL, expanding the "latest" sentinel. + * + * The nightly asset filename embeds a build date, so there is no stable + * "latest.iso" path to hardcode — the newest release has to be looked up. + * Any other value is used verbatim, which is how VYOS_ISO_URL pins a build + * or points at a locally mirrored copy. + */ +function resolveVyosIsoUrl(configured: string): string { + if (configured !== "latest") return configured; + + const body = execSync(`curl -sSfL "${VYOS_NIGHTLY_RELEASES}"`, { + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + }); + const release = JSON.parse(body) as { + tag_name?: string; + assets?: Array<{ name: string; browser_download_url: string }>; + }; + + const asset = (release.assets ?? []).find((a) => + /generic-amd64\.iso$/.test(a.name), + ); + if (!asset) { + throw new Error( + `No generic-amd64 ISO asset in VyOS nightly release ${release.tag_name ?? "?"}`, + ); + } + + logger.info(` VyOS ISO resolved to ${asset.name} (${release.tag_name ?? "?"})`); + return asset.browser_download_url; +} + +/** + * Extract VyOS netboot artifacts from the release ISO. + * + * VyOS publishes no netboot bundle, so kernel/initrd/squashfs have to come out + * of the ISO. xorriso is already in the bastion image (used for boot.iso) and + * extracts without root or a loop mount. + * + * The initrd needs care: /live contains an empty initrd.img placeholder + * alongside the real one, which carries a version-suffixed name. Booting the + * 0-byte file fails with no useful diagnostic, so pick the largest initrd*. + */ +export function prepareVyosArtifacts(config: BastionConfig): void { + const kernel = `${config.httpDir}/vyos-vmlinuz`; + const initrd = `${config.httpDir}/vyos-initrd`; + const squashfs = `${config.httpDir}/vyos-filesystem.squashfs`; + + if (existsSync(kernel) && existsSync(initrd) && existsSync(squashfs)) { + logger.info(" VyOS netboot artifacts -- cached"); + return; + } + + const iso = `${config.bastionDir}/vyos.iso`; + download(resolveVyosIsoUrl(config.vyosIsoUrl), iso, "VyOS ISO"); + + const extract = (isoPath: string, dest: string, label: string): void => { + execSync( + `xorriso -osirrox on -indev "${iso}" -extract "${isoPath}" "${dest}"`, + { stdio: "pipe" }, + ); + logger.info(` ${label} -- extracted from ${isoPath}`); + }; + + extract("/live/vmlinuz", kernel, "VyOS kernel"); + extract("/live/filesystem.squashfs", squashfs, "VyOS squashfs"); + + // Pick the real initrd by size from the ISO's own directory listing. + const listing = execSync(`xorriso -indev "${iso}" -lsl /live/ --`, { + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + }); + + const best = pickLargestInitrd(listing); + if (best === undefined) { + throw new Error("No non-empty initrd found in /live on the VyOS ISO"); + } + extract(`/live/${best.name}`, initrd, `VyOS initrd (${best.name}, ${best.size} bytes)`); + + // The ISO is only needed to produce the three artifacts above. + try { + unlinkSync(iso); + } catch { + // Non-fatal: leaving it costs disk but nothing else. + } +} + function symlinkSafe(target: string, linkPath: string): void { try { symlinkSync(target, linkPath); @@ -182,6 +301,17 @@ export async function startBastion(overrides: Partial = {}): Prom logger.warn(`Ubuntu ${config.ubuntuVersion} artifacts not available -- Ubuntu provisioning disabled`); } + // VyOS netboot artifacts (non-fatal — same policy as Ubuntu) + try { + logger.info("Preparing VyOS netboot artifacts..."); + prepareVyosArtifacts(config); + } catch (err) { + logger.warn( + `VyOS artifacts not available -- VyOS provisioning disabled ` + + `(${err instanceof Error ? err.message : String(err)})`, + ); + } + // Symlink iPXE binaries into HTTP dir for UEFI HTTP Boot for (const name of ["ipxe.efi", "ipxe-arm64.efi"]) { const src = `${config.tftpDir}/${name}`; @@ -261,6 +391,7 @@ export async function startBastion(overrides: Partial = {}): Prom role: msg.role as import("@lab/shared").Role, os: msg.os as import("@lab/shared").OsId, queued_at: new Date().toISOString(), + ...(msg.vyos ? { vyos: msg.vyos } : {}), }; }); return { status: "ok", data: { mac: msg.mac, hostname: msg.hostname } }; diff --git a/bastion/src/bastion/src/routes/api.ts b/bastion/src/bastion/src/routes/api.ts index 8a75c65..7398046 100644 --- a/bastion/src/bastion/src/routes/api.ts +++ b/bastion/src/bastion/src/routes/api.ts @@ -5,8 +5,8 @@ // /api/discover - receive hardware discovery reports from PXE-booted machines import type { FastifyInstance } from "fastify"; -import type { HardwareInfo, InstalledInfo, Role } from "@lab/shared"; -import { isValidOsId, SUPPORTED_ROLES } from "@lab/shared"; +import type { HardwareInfo, InstalledInfo, Role, VyosInstallSpec } from "@lab/shared"; +import { isValidOsId, SUPPORTED_ROLES, SUPPORTED_OS } from "@lab/shared"; import type { StateManager } from "../services/state.js"; import { logger } from "../services/logger.js"; import { triggerPostProvisionK3s } from "../services/post-provision.js"; @@ -15,6 +15,13 @@ import type { ProgressEvent } from "../services/progress-events.js"; import type { InstallLogBuffer } from "../services/install-log.js"; import type { SyslogListener } from "../services/syslog-listener.js"; +/** + * Seconds after dispatch with zero progress before a machine is called stalled. + * Generous: the slowest legitimate gap is fetching a ~600MB VyOS squashfs over + * HTTP before the hook can report anything. + */ +const STALL_THRESHOLD_S = 8 * 60; + export function registerApiRoutes( app: FastifyInstance, state: StateManager, @@ -34,9 +41,10 @@ export function registerApiRoutes( disk?: string; role?: string; os?: string; + vyos?: VyosInstallSpec; }; }>("/api/install", async (request, reply) => { - const { mac: rawMac, hostname, disk, role, os } = request.body ?? {}; + const { mac: rawMac, hostname, disk, role, os, vyos } = request.body ?? {}; const mac = (rawMac ?? "").toLowerCase().replace(/-/g, ":"); if (mac === "") { @@ -50,7 +58,7 @@ export function registerApiRoutes( const osId = os ?? "fedora-43"; if (!isValidOsId(osId)) { - return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: fedora-43, ubuntu-26.04` }); + return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: ${SUPPORTED_OS.join(", ")}` }); } state.update((s) => { @@ -60,6 +68,7 @@ export function registerApiRoutes( role: validRole as Role, os: osId, queued_at: new Date().toISOString(), + ...(vyos ? { vyos } : {}), }; }); @@ -158,11 +167,17 @@ export function registerApiRoutes( }; s.installed[mac] = installedInfo; - const admin = installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "lab" : "root"; + // VyOS: the only login user is "vyos", and a router never runs k3s — + // without this guard a non-vanilla role + recorded IP would trigger + // the k3s post-provision against a VyOS box. + const isVyos = (installedInfo.os ?? "").startsWith("vyos"); + const admin = isVyos + ? "vyos" + : installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "lab" : "root"; console.log(`\n \x1b[0;32m\x1b[1m ssh ${admin}@${ip}\x1b[0m\n`); // eslint-disable-line no-console // Auto-install k3s for non-vanilla roles - if (installedInfo.role !== "vanilla" && ip !== "") { + if (!isVyos && installedInfo.role !== "vanilla" && ip !== "") { void triggerPostProvisionK3s(installedInfo.hostname, ip, installedInfo.role, admin, mac); } } @@ -434,6 +449,15 @@ export function registerApiRoutes( const installedEntry = currentState.installed[mac]; if (queueEntry) { + // A machine that was handed an install script but has reported nothing + // since is wedged BEFORE the installer environment came up — a bad + // kernel/initrd, no network in the initramfs, or the wrong NIC picked. + // Surfacing it here is what makes that diagnosable without a console. + const since = queueEntry.progress_at ?? queueEntry.dispatched_at; + const stalledForS = since !== undefined && queueEntry.progress === undefined + ? Math.floor((Date.now() - new Date(since).getTime()) / 1000) + : 0; + return reply.send({ mac, hostname: queueEntry.hostname, @@ -441,6 +465,9 @@ export function registerApiRoutes( progress: queueEntry.progress ?? "queued", progress_detail: queueEntry.progress_detail ?? "", progress_at: queueEntry.progress_at ?? queueEntry.queued_at, + dispatched_at: queueEntry.dispatched_at, + stalled_for_s: stalledForS, + stalled: stalledForS > STALL_THRESHOLD_S, role: queueEntry.role, os: queueEntry.os, stages: queueEntry.log ?? [], diff --git a/bastion/src/bastion/src/routes/dispatch.ts b/bastion/src/bastion/src/routes/dispatch.ts index 0ecc1c4..9ff1fdf 100644 --- a/bastion/src/bastion/src/routes/dispatch.ts +++ b/bastion/src/bastion/src/routes/dispatch.ts @@ -15,6 +15,7 @@ import { renderLocalBootIpxe, } from "../templates/boot.ipxe.js"; import { renderUbuntuInstallIpxe } from "../templates/ubuntu-boot.ipxe.js"; +import { renderVyosInstallIpxe } from "../templates/vyos-boot.ipxe.js"; import { renderDebugKickstart } from "../templates/debug.ks.js"; import { logger } from "../services/logger.js"; @@ -99,8 +100,22 @@ echo "===============================" const os = queueEntry.os ?? "fedora-43"; logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os})`); + // Stamp the handoff so a machine that boots the installer but never + // reports can be spotted without a console. + state.update((s) => { + const entry = s.install_queue[mac]; + if (entry) entry.dispatched_at = new Date().toISOString(); + }); + let script: string; - if (os.startsWith("ubuntu")) { + if (os.startsWith("vyos")) { + script = renderVyosInstallIpxe({ + mac, + hostname, + serverIp: config.serverIp, + httpPort: config.httpPort, + }); + } else if (os.startsWith("ubuntu")) { script = renderUbuntuInstallIpxe({ mac, hostname, diff --git a/bastion/src/bastion/src/routes/vyos.ts b/bastion/src/bastion/src/routes/vyos.ts new file mode 100644 index 0000000..60cc9e7 --- /dev/null +++ b/bastion/src/bastion/src/routes/vyos.ts @@ -0,0 +1,71 @@ +// VyOS network install routes. +// +// VyOS has no unattended installer, so the automation is injected via +// live-config's `hooks` component: the iPXE script passes +// live-config.hooks=<.../vyos/autoinstall.sh>, live-config wgets it and runs it +// as root, and that script fetches and executes the generated install driver. + +import type { FastifyInstance } from "fastify"; +import type { BastionConfig } from "@lab/shared"; +import type { StateManager } from "../services/state.js"; +import { buildVyosConfigSpec } from "../templates/vyos-config-spec.js"; +import { renderVyosInstallPy } from "../templates/vyos-install.py.js"; +import { logger } from "../services/logger.js"; + +function normalizeMac(value: string | undefined): string { + return (value ?? "").toLowerCase().replace(/-/g, ":"); +} + +export function registerVyosRoutes( + app: FastifyInstance, + config: BastionConfig, + state: StateManager, +): void { + // live-config hook. Kept minimal: everything version-specific lives in the + // generated Python. wget is guaranteed present -- live-config used it to + // fetch this very script. + app.get<{ Querystring: { mac?: string } }>("/vyos/autoinstall.sh", async (request, reply) => { + const mac = normalizeMac(request.query.mac); + const base = `http://${config.serverIp}:${config.httpPort}`; + + logger.info(`VYOS AUTOINSTALL HOOK served to ${mac || "unknown MAC"}`); + + const script = `#!/bin/sh +# Lab PXE Bastion -- VyOS unattended install hook (run by live-config as root) +set -eu + +wget -q "${base}/vyos/install.py?mac=${mac}" -O /tmp/vyos-install.py +exec python3 /tmp/vyos-install.py +`; + return reply.type("text/plain").send(script); + }); + + // Per-MAC install driver, with the machine's config spec baked in. + app.get<{ Querystring: { mac?: string } }>("/vyos/install.py", async (request, reply) => { + const mac = normalizeMac(request.query.mac); + const queueEntry = state.load().install_queue[mac]; + + const spec = buildVyosConfigSpec({ + hostname: queueEntry?.hostname ?? "vyos", + spec: queueEntry?.vyos, + defaultPassword: config.vyosDefaultPassword, + sshKeys: config.sshKeys, + disk: queueEntry?.disk, + }); + + logger.info( + `VYOS INSTALL DRIVER served to ${mac} (${spec.hostname}, ` + + `${spec.sets.length} config ops, disk="${spec.disk || "auto"}")`, + ); + + const script = renderVyosInstallPy({ + spec, + mac, + serverIp: config.serverIp, + httpPort: config.httpPort, + role: queueEntry?.role ?? "vanilla", + }); + + return reply.type("text/plain").send(script); + }); +} diff --git a/bastion/src/bastion/src/server.ts b/bastion/src/bastion/src/server.ts index d3078d8..1c1aa47 100644 --- a/bastion/src/bastion/src/server.ts +++ b/bastion/src/bastion/src/server.ts @@ -12,6 +12,7 @@ import { registerDispatchRoutes } from "./routes/dispatch.js"; import { registerKickstartRoutes } from "./routes/kickstart.js"; import { registerApiRoutes } from "./routes/api.js"; import { registerAsahiRoutes } from "./routes/asahi.js"; +import { registerVyosRoutes } from "./routes/vyos.js"; export function createApp(config: BastionConfig): { app: ReturnType; state: StateManager; installLog: InstallLogBuffer; syslog: SyslogListener } { @@ -47,6 +48,7 @@ export function createApp(config: BastionConfig): { app: ReturnType" — + * the exact format routes/api.ts parses installed.ip from). The mgmt + * address when static; "" means detect the live DHCP address at runtime. + */ + reportAddress: string; + /** Whether to accept RAID-1 when the installer finds more than one disk. */ + raid: boolean; + /** Overwrite the installed config.boot with the generated one on reinstall. */ + freshConfig: boolean; + sets: VyosSetOp[]; + /** Paths that are VyOS tag nodes — must be marked as such in the ConfigTree. */ + tags: string[][]; +} + +/** + * Normalise a target disk to the form the installer expects. + * + * find_disks() enumerates via `lsblk -Jbp` (-p = full paths), so its valid + * responses are "/dev/mmcblk0"-style. A bare "mmcblk0" is rejected by + * ask_input()'s valid_responses check and re-prompts forever. + */ +function normalizeDiskPath(value: string | undefined): string { + const raw = (value ?? "").trim(); + if (raw === "") return ""; + return raw.startsWith("/dev/") ? raw : `/dev/${raw}`; +} + +export function buildVyosConfigSpec(params: { + hostname: string; + spec?: VyosInstallSpec | undefined; + defaultPassword: string; + sshKeys?: string[] | undefined; + disk?: string | undefined; +}): VyosConfigSpec { + const spec = params.spec ?? {}; + const mgmt = spec.mgmtInterface ?? "eth0"; + const mgmtAddress = spec.mgmtAddress ?? "dhcp"; + const bondMembers = spec.bondMembers ?? []; + const vlans = spec.vlans ?? []; + + const sets: VyosSetOp[] = []; + const tags: string[][] = [ + ["interfaces", "ethernet"], + ["system", "login", "user"], + ]; + + const hwIds = spec.hwIds ?? {}; + const pinHwId = (iface: string): void => { + const mac = hwIds[iface]; + if (mac !== undefined && mac !== "") { + sets.push({ path: ["interfaces", "ethernet", iface, "hw-id"], value: mac }); + } + }; + + sets.push({ path: ["system", "host-name"], value: params.hostname }); + + // Management interface — the NIC that PXE booted, left untagged and unbonded. + sets.push({ path: ["interfaces", "ethernet", mgmt, "address"], value: mgmtAddress }); + pinHwId(mgmt); + + // Tagged management VLAN on the PXE port. Emitted regardless of bonding, so + // the box stays reachable on the management VLAN while still booting untagged + // on whichever VLAN the bastion's proxy DHCP serves. + const mgmtVlan = spec.mgmtVlan; + if (mgmtVlan !== undefined) { + tags.push(["interfaces", "ethernet", mgmt, "vif"]); + const vif = ["interfaces", "ethernet", mgmt, "vif", String(mgmtVlan.id)]; + sets.push({ path: [...vif, "address"], value: mgmtVlan.address }); + if (mgmtVlan.description !== undefined && mgmtVlan.description !== "") { + sets.push({ path: [...vif, "description"], value: mgmtVlan.description }); + } + } + + // LACP bond. Members must exclude the PXE NIC; firmware PXE cannot run over LACP. + const bonded = bondMembers.length > 0; + if (bonded) { + tags.push(["interfaces", "bonding"]); + sets.push({ path: ["interfaces", "bonding", "bond0", "mode"], value: "802.3ad" }); + sets.push({ path: ["interfaces", "bonding", "bond0", "hash-policy"], value: "layer2+3" }); + for (const member of bondMembers) { + sets.push({ + path: ["interfaces", "bonding", "bond0", "member", "interface"], + value: member, + replace: false, + }); + pinHwId(member); + } + // Address on the trunk's native/untagged VLAN. + if (spec.bondAddress !== undefined && spec.bondAddress !== "") { + sets.push({ path: ["interfaces", "bonding", "bond0", "address"], value: spec.bondAddress }); + } + } + + // VRRP groups accumulate here; emitted (plus a sync group) after the VLANs. + // interface accepts dotted vifs (constraint regex `[0-9]+(.\d+)?`), address + // is a tag node (the VIP is the tag value itself), vrid range is 1-255. + const vrrpGroups: Array<{ name: string; iface: string; vrid: number; vip: string }> = []; + if (bonded && spec.bondVrrp !== undefined && spec.bondVrrp !== "") { + // vrid 1 for the untagged group: the native VLAN is never a vif, so this + // cannot collide with a vlan-id-derived vrid. + vrrpGroups.push({ name: "native", iface: "bond0", vrid: 1, vip: spec.bondVrrp }); + } + + // Tagged VLAN sub-interfaces hang off the bond when there is one, else off mgmt. + const parent = bonded + ? ["interfaces", "bonding", "bond0"] + : ["interfaces", "ethernet", mgmt]; + if (vlans.length > 0) { + tags.push([...parent, "vif"]); + const parentName = bonded ? "bond0" : mgmt; + for (const vlan of vlans) { + const vif = [...parent, "vif", String(vlan.id)]; + sets.push({ path: [...vif, "address"], value: vlan.address }); + if (vlan.description !== undefined && vlan.description !== "") { + sets.push({ path: [...vif, "description"], value: vlan.description }); + } + if (vlan.vrrp !== undefined && vlan.vrrp !== "") { + vrrpGroups.push({ + name: `vlan${vlan.id}`, + iface: `${parentName}.${vlan.id}`, + vrid: vlan.id, + vip: vlan.vrrp, + }); + } + } + } + + // Emit VRRP groups plus one sync group so all VLANs fail over together — + // without it a single-link event could split mastership across the pair. + if (vrrpGroups.length > 0) { + tags.push(["high-availability", "vrrp", "group"]); + tags.push(["high-availability", "vrrp", "sync-group"]); + const priority = String(spec.vrrpPriority ?? 100); + for (const g of vrrpGroups) { + const base = ["high-availability", "vrrp", "group", g.name]; + sets.push({ path: [...base, "interface"], value: g.iface }); + sets.push({ path: [...base, "vrid"], value: String(g.vrid) }); + sets.push({ path: [...base, "priority"], value: priority }); + // address is a tag node: the VIP is the path's final segment, no value. + sets.push({ path: [...base, "address", g.vip] }); + tags.push([...base, "address"]); + sets.push({ + path: ["high-availability", "vrrp", "sync-group", "MAIN", "member"], + value: g.name, + replace: false, + }); + } + } + + sets.push({ path: ["service", "ssh", "port"], value: "22" }); + + const sshKeys = params.sshKeys ?? []; + if (sshKeys.length > 0) { + tags.push(["system", "login", "user", "vyos", "authentication", "public-keys"]); + sshKeys.forEach((entry, index) => { + const parts = entry.trim().split(/\s+/); + const type = parts[0] ?? ""; + const key = parts[1] ?? ""; + if (!type.startsWith("ssh-") && !type.startsWith("ecdsa-")) return; + if (!key) return; + const name = parts[2] ?? `lab-key-${index}`; + const base = ["system", "login", "user", "vyos", "authentication", "public-keys", name]; + sets.push({ path: [...base, "type"], value: type }); + sets.push({ path: [...base, "key"], value: key }); + }); + } + + return { + hostname: params.hostname, + imageName: "", + password: spec.password ?? params.defaultPassword, + console: "K", + disk: normalizeDiskPath(params.disk), + // Static mgmt address wins; under DHCP the driver detects the live IP. + reportAddress: mgmtAddress.includes("/") ? (mgmtAddress.split("/")[0] ?? "") : "", + raid: false, + freshConfig: spec.freshConfig ?? false, + sets, + tags, + }; +} diff --git a/bastion/src/bastion/src/templates/vyos-install.py.ts b/bastion/src/bastion/src/templates/vyos-install.py.ts new file mode 100644 index 0000000..bb5b066 --- /dev/null +++ b/bastion/src/bastion/src/templates/vyos-install.py.ts @@ -0,0 +1,514 @@ +// Renders the Python program that performs the unattended VyOS install. +// +// It runs as root inside the live system, fetched and executed by live-config's +// `hooks` component (see vyos-boot.ipxe.ts). It does three things: +// 1. builds config.boot from the image's own default via vyos.configtree +// 2. drives the interactive `install image` through a pty +// 3. reports progress back to the bastion, then reboots +// +// A pty is used rather than piping stdin because the installer reads the +// password through getpass(), which opens /dev/tty directly and would ignore a +// pipe. Prompts are matched by text rather than replayed positionally: the +// installer skips the boot-config question when it finds a previous +// installation, so a fixed answer sequence desyncs on reinstall. + +import type { VyosConfigSpec } from "./vyos-config-spec.js"; + +export function renderVyosInstallPy(params: { + spec: VyosConfigSpec; + mac: string; + serverIp: string; + httpPort: number; + role: string; +}): string { + // Base64 so arbitrary values (passwords, descriptions, SSH keys) can never + // terminate the Python string literal that carries them. + const specB64 = Buffer.from(JSON.stringify(params.spec), "utf-8").toString("base64"); + + return `#!/usr/bin/env python3 +"""Unattended VyOS install driver -- generated by the lab PXE bastion.""" + +import base64 +import json +import os +import pty +import re +import select +import subprocess +import sys +import time +import urllib.request + +SPEC = json.loads(base64.b64decode("${specB64}").decode("utf-8")) +BASTION = "http://${params.serverIp}:${params.httpPort}" +MAC = "${params.mac}" +ROLE = ${JSON.stringify(params.role ?? "vanilla")} + +INSTALLER = "/usr/libexec/vyos/op_mode/image_installer.py" +CONFIG_DIR = "/opt/vyatta/etc/config" +# The installer copies the rootfs from the boot MEDIUM path -- which only a +# CD/USB boot provides. With fetch= (HTTP netboot) nothing is mounted there +# (verified in VM: Errno 2), so the squashfs must be linked or re-fetched into +# place before 'install image' runs. +ROOTFS_EXPECTED = "/usr/lib/live/mount/medium/live/filesystem.squashfs" +SQUASHFS_URL = "http://${params.serverIp}:${params.httpPort}/vyos-filesystem.squashfs" +# The live-config hook runs BEFORE vyos-router creates the /opt/vyatta compat +# path, so the squashfs's own location must be tried too (verified in VM: only +# /usr/share/vyos/config.boot.default exists at hook time). +DEFAULT_CONFIG_CANDIDATES = [ + "/opt/vyatta/etc/config.boot.default", + "/usr/share/vyos/config.boot.default", +] +STALL_TIMEOUT = 900 # seconds without installer output before giving up + + +def detect_ip(): + """Best-effort local IP as seen on the route toward the bastion. + + Matches Fedora's semantics (IP captured during install): under DHCP the + installed system will renew on the same NIC/subnet the live env used. + """ + import socket + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("${params.serverIp}", ${params.httpPort})) + ip = s.getsockname()[0] + s.close() + return ip + except Exception: + return "" + + +class LogStreamer: + """Stream install output to the bastion's /api/log so 'labctl provision + logs -f' works live for VyOS, like Anaconda's syslog does for Fedora. + + Strictly best-effort: a failed POST drops the batch and must never stall + the pty read loop or fail the install. + """ + + ANSI = re.compile(rb"\\x1b\\[[0-9;?]*[a-zA-Z]|\\x1b[=>]|\\r") + + def __init__(self): + self.partial = b"" + self.pending = [] + self.last_flush = time.time() + + def feed(self, chunk): + """Raw pty bytes: split into lines, strip ANSI noise, queue.""" + self.partial += chunk + while b"\\n" in self.partial: + raw, self.partial = self.partial.split(b"\\n", 1) + text = self.ANSI.sub(b"", raw).decode("utf-8", "replace").rstrip() + if text: + self.pending.append(text) + self.maybe_flush() + + def line(self, text): + """A driver-originated message (already a clean string).""" + self.pending.append(text) + self.maybe_flush() + + def maybe_flush(self): + if len(self.pending) >= 20 or (self.pending and time.time() - self.last_flush >= 2): + self.flush() + + def flush(self): + if not self.pending: + return + batch, self.pending = self.pending[:200], self.pending[200:] + self.last_flush = time.time() + try: + body = json.dumps({"mac": MAC, "lines": batch}).encode() + req = urllib.request.Request( + BASTION + "/api/log", + data=body, + headers={"Content-Type": "application/json"}, + ) + urllib.request.urlopen(req, timeout=5).read() + except Exception: + pass + + +STREAM = LogStreamer() + + +def say(msg): + """Print locally and stream to the bastion log buffer.""" + print(msg) + STREAM.line(str(msg)) + + +def report(stage, detail=""): + """Best-effort progress callback; never fatal.""" + STREAM.flush() + try: + body = json.dumps({"mac": MAC, "stage": stage, "detail": detail}).encode() + req = urllib.request.Request( + BASTION + "/api/progress", + data=body, + headers={"Content-Type": "application/json"}, + ) + urllib.request.urlopen(req, timeout=5).read() + except Exception: + pass + + +def build_config(): + """Apply our set operations onto the image's own default config. + + Using config.boot.default as the base keeps the vyos-config-version trailer + consistent with the running image, so first boot does not run migrations. + """ + from vyos.configtree import ConfigTree + + default_config = next( + (p for p in DEFAULT_CONFIG_CANDIDATES if os.path.exists(p)), None) + if default_config is None: + raise FileNotFoundError( + "no config.boot.default found (tried %s)" % ", ".join(DEFAULT_CONFIG_CANDIDATES)) + say("base config: %s" % default_config) + + with open(default_config) as handle: + config = ConfigTree(handle.read()) + + for op in SPEC["sets"]: + replace = op.get("replace", True) + if "value" in op and op["value"] is not None: + config.set(op["path"], value=op["value"], replace=replace) + else: + config.set(op["path"]) + + # Tag nodes must be marked after the nodes exist, as the installer itself does. + for tag in SPEC["tags"]: + try: + config.set_tag(tag) + except Exception as err: + say("warning: set_tag %s failed: %s" % (tag, err)) + + os.makedirs(CONFIG_DIR, exist_ok=True) + target = os.path.join(CONFIG_DIR, "config.boot") + + # Re-attach the vyos-config-version footer: ConfigTree.to_string() emits + # only the config body, and a config without the footer is treated as + # ancient -- the boot migrator then runs every migration over it and (as + # observed in the VM test) crashes in system/31-to-32. Building the footer + # from the running system pins it to the exact image being installed. + body = config.to_string() + try: + from vyos.component_version import version_info_from_system + info = version_info_from_system() + info.update_config_body(body) + info.write(target) + say("wrote %s (footer: %s)" % (target, info.release)) + except Exception as err: + say("warning: version footer failed (%s); writing bare config" % err) + with open(target, "w") as handle: + handle.write(body) + return target + + +def find_live_squashfs(): + """Locate the squashfs live-boot fetched, without walking into the mounted + rootfs or overlay (each would mean traversing the entire OS tree).""" + explicit = [ + "/run/live/medium/live/filesystem.squashfs", + "/lib/live/mount/medium/live/filesystem.squashfs", + ] + for path in explicit: + if os.path.isfile(path) and os.path.getsize(path) > 0: + return path + for root in ("/run/live", "/lib/live/mount", "/usr/lib/live/mount"): + for dirpath, dirs, files in os.walk(root): + depth = dirpath.count(os.sep) - root.count(os.sep) + dirs[:] = [d for d in dirs + if d not in ("rootfs", "overlay") + and not d.endswith(".squashfs") + and depth < 3] + if "filesystem.squashfs" in files: + path = os.path.join(dirpath, "filesystem.squashfs") + if os.path.isfile(path) and os.path.getsize(path) > 0: + return path + return None + + +def ensure_rootfs(): + """Make FILE_ROOTFS_SRC exist so the installer can copy the system image.""" + if os.path.isfile(ROOTFS_EXPECTED) and os.path.getsize(ROOTFS_EXPECTED) > 0: + return + src = find_live_squashfs() + if src is None: + say("squashfs not in live mounts; re-fetching %s" % SQUASHFS_URL) + src = "/tmp/filesystem.squashfs" + urllib.request.urlretrieve(SQUASHFS_URL, src) + os.makedirs(os.path.dirname(ROOTFS_EXPECTED), exist_ok=True) + if os.path.lexists(ROOTFS_EXPECTED): + os.remove(ROOTFS_EXPECTED) + os.symlink(src, ROOTFS_EXPECTED) + say("rootfs source: %s -> %s" % (ROOTFS_EXPECTED, src)) + + +def build_rules(): + """Prompt -> response table for the interactive installer.""" + password = SPEC["password"].encode() + b"\\n" + image_name = SPEC["imageName"].encode() + b"\\n" + disk = SPEC["disk"].encode() + b"\\n" + console = SPEC["console"].encode() + b"\\n" + raid = (b"yes\\n" if SPEC["raid"] else b"no\\n") + + return [ + (re.compile(rb"Would you like to continue\\?"), b"yes\\n"), + (re.compile(rb"What would you like to name this image\\?"), image_name), + (re.compile(rb"Please confirm password for the .vyos. user:"), password), + (re.compile(rb"Please enter a password for the .vyos. user:"), password), + (re.compile(rb"What console should be used by default"), console), + # Three RAID variants: "configure RAID-1 mirroring?", "...on them?", + # and "choose two disks for RAID-1 mirroring?" -- all default to YES, + # so a missed one both hangs the install and risks an unwanted mirror. + (re.compile(rb"Would you like to [^?]*RAID-1 mirroring"), raid), + (re.compile(rb"Installation will delete all data on (?:the drive|both drives)\\. Continue\\?"), b"yes\\n"), + (re.compile(rb"Which one should be used for installation\\?"), disk), + (re.compile(rb"Would you like to use all the free space on the drive\\?"), b"yes\\n"), + (re.compile(rb"Which file would you like as boot config\\?"), b"1\\n"), + # Reinstall path only (search_previous_installation): carrying the old + # /config and SSH host keys forward is VyOS's "reinstall without losing + # data". Always yes -- freshConfig replaces config.boot afterwards, so + # answering no here would also discard non-config data under /config. + (re.compile(rb"Would you like to copy data to the new image\\?"), b"yes\\n"), + (re.compile(rb"Would you like to copy the encrypted config to the new image\\?"), b"yes\\n"), + # More than one previous image found -- take the first offered. + (re.compile(rb"From which image would you like to save config information\\?"), b"1\\n"), + (re.compile(rb"From which image would you like to copy the encrypted config\\?"), b"1\\n"), + ] + + +def run_installer(): + """Drive image_installer.py over a pty, answering prompts as they appear.""" + rules = build_rules() + master, slave = pty.openpty() + + proc = subprocess.Popen( + [INSTALLER, "--action", "install"], + stdin=slave, + stdout=slave, + stderr=slave, + close_fds=True, + preexec_fn=os.setsid, + ) + os.close(slave) + + buf = b"" + transcript = b"" # rolling tail of everything the installer printed + last_output = time.time() + + while True: + ready, _, _ = select.select([master], [], [], 1.0) + + if ready: + try: + chunk = os.read(master, 4096) + except OSError: + break + if not chunk: + break + + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + buf += chunk + transcript = (transcript + chunk)[-8000:] + STREAM.feed(chunk) + last_output = time.time() + + # Answer every prompt currently in the buffer, earliest first, so + # ordering is preserved even when the installer skips questions -- + # and so a single chunk carrying two prompts gets both answers. + while True: + best = None + for pattern, response in rules: + found = pattern.search(buf) + if found and (best is None or found.start() < best[0].start()): + best = (found, response) + if best is None: + break + found, response = best + os.write(master, response) + transcript = (transcript + b"\\n>>> answered: " + response)[-8000:] + STREAM.line(">>> answered: " + response.decode("utf-8", "replace").strip()) + buf = buf[found.end():] + + # Bound memory if the installer emits a lot without prompting. + if len(buf) > 65536: + buf = buf[-8192:] + + elif proc.poll() is not None: + break + + STREAM.maybe_flush() + + if time.time() - last_output > STALL_TIMEOUT: + proc.kill() + raise SystemExit("installer produced no output for %ds" % STALL_TIMEOUT) + + os.close(master) + return proc.wait(), transcript.decode("utf-8", "replace") + + +def ensure_network_boot_first(): + """Keep network boot first so the bastion intercepts every reboot. + + Port of the Fedora kickstart's %post efibootmgr step (install.ks.ts) -- + what makes reprovision-by-reboot work. Best-effort: skipped on BIOS boots + or when efibootmgr is absent. Runs from the live env after the installer; + efibootmgr edits NVRAM, not the disk, so installer cleanup is irrelevant. + """ + import shutil + if not os.path.isdir("/sys/firmware/efi") or shutil.which("efibootmgr") is None: + say("boot order: skipped (BIOS boot or efibootmgr missing)") + return + try: + out = subprocess.run(["efibootmgr"], capture_output=True, text=True, timeout=30).stdout + order = [] + network_entry = None + for line in out.splitlines(): + m = re.match(r"^BootOrder:\\s*(.*)$", line) + if m: + order = [x.strip() for x in m.group(1).split(",") if x.strip()] + continue + m = re.match(r"^Boot([0-9A-Fa-f]{4})\\*?\\s+(.*)$", line) + if m and network_entry is None: + if re.search(r"network|pxe|ipv4|ipv6|http", m.group(2), re.IGNORECASE): + network_entry = m.group(1).upper() + if network_entry is None or not order: + say("boot order: no network boot entry found; leaving as is") + return + new_order = [network_entry] + [x for x in order if x.upper() != network_entry] + if [x.upper() for x in order] == [x.upper() for x in new_order]: + say("boot order: network entry Boot%s already first" % network_entry) + return + subprocess.run(["efibootmgr", "-o", ",".join(new_order)], + capture_output=True, timeout=30) + say("boot order: moved network entry Boot%s first" % network_entry) + except Exception as err: + say("warning: boot order adjustment failed: %s" % err) + + +def with_target_mounted(fn): + """Mount the installed root partition, call fn(rw_dir), always unmount. + + The installer has unmounted and cleaned the target by the time this runs, + so the block device is free. The partition holding boot//rw is the + VyOS root; the glob also yields the installed image's rw dir directly. + """ + import glob + disk = SPEC["disk"] + if not disk: + # No pinned disk (installer picked the default) -- enumerate all disks. + candidates = ["/dev/" + b for b in os.listdir("/sys/block") + if not b.startswith(("loop", "ram", "zram", "sr"))] + else: + candidates = [disk] + + mnt = "/mnt/lab-target" + os.makedirs(mnt, exist_ok=True) + for dev in candidates: + name = os.path.basename(dev) + parts = sorted(p for p in os.listdir("/sys/block/%s" % name) + if p.startswith(name)) if os.path.isdir("/sys/block/%s" % name) else [] + for part in parts: + pdev = "/dev/" + part + if subprocess.run(["mount", pdev, mnt], capture_output=True).returncode != 0: + continue + try: + rw_dirs = glob.glob(os.path.join(mnt, "boot", "*", "rw")) + if rw_dirs: + fn(rw_dirs[0]) + return True + finally: + subprocess.run(["umount", mnt], capture_output=True) + return False + + +def post_install_target_steps(): + """Metadata + optional fresh-config overwrite inside the installed image.""" + def apply(rw_dir): + config_dir = os.path.join(rw_dir, "opt/vyatta/etc/config") + os.makedirs(config_dir, exist_ok=True) + + # /config/lab-provisioned -- survives VyOS image upgrades. Mirrors the + # Fedora kickstart's /etc/lab-provisioned. + try: + with open(os.path.join(config_dir, "lab-provisioned"), "w") as handle: + handle.write("hostname=%s\\n" % SPEC["hostname"]) + handle.write("role=%s\\n" % ROLE) + handle.write("provisioned=%s\\n" % time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())) + handle.write("bastion=%s\\n" % BASTION) + say("wrote /config/lab-provisioned") + except Exception as err: + say("warning: lab-provisioned metadata failed: %s" % err) + + # freshConfig: make the bastion-generated config win over the previous + # installation's carried-forward config. Explicit intent -- failure is + # fatal (raised out of with_target_mounted). + if SPEC.get("freshConfig"): + import shutil + shutil.copyfile(os.path.join(CONFIG_DIR, "config.boot"), + os.path.join(config_dir, "config.boot")) + say("freshConfig: replaced installed config.boot with generated config") + + mounted = with_target_mounted(apply) + if not mounted: + if SPEC.get("freshConfig"): + raise RuntimeError("freshConfig requested but installed root partition not found") + say("warning: installed root partition not found; skipping metadata") + + +def main(): + report("vyos-install", "building config.boot") + try: + build_config() + except Exception as err: + report("error", "config generation failed: %s" % err) + raise + + report("vyos-install", "staging rootfs for installer") + try: + ensure_rootfs() + except Exception as err: + report("error", "rootfs staging failed: %s" % err) + raise + + report("vyos-install", "running install image") + code, transcript = run_installer() + + if code != 0: + # Surface the installer's last words in bastion progress -- the console + # they were printed on is usually invisible during unattended installs. + report("error", "install image exited %d | tail: %s" % (code, transcript[-4000:])) + raise SystemExit(code) + + report("post-install", "boot order + metadata") + ensure_network_boot_first() + try: + post_install_target_steps() + except Exception as err: + report("error", "post-install target steps failed: %s" % err) + raise + + # "complete" is the stage the bastion uses to move a machine out of the + # install queue into installed state, and "ready at " is the exact + # detail format it parses installed.ip from -- see routes/api.ts. + ip = SPEC.get("reportAddress") or detect_ip() + report("complete", "ready at %s" % ip if ip else "VyOS installed, rebooting") + os.system("sync") + # --force: this driver is a child of live-config.service, whose start job is + # still running -- a normal reboot deadlocks waiting for it (verified in VM: + # shutdown blocked >1min on "start job is running for live-config"). The + # installer has already unmounted and cleaned the target, so an immediate + # reboot is safe. + os.system("systemctl reboot --force") + + +if __name__ == "__main__": + main() +`; +} diff --git a/bastion/src/bastion/tests/dispatch.test.ts b/bastion/src/bastion/tests/dispatch.test.ts index 3d07ac4..edaea4d 100644 --- a/bastion/src/bastion/tests/dispatch.test.ts +++ b/bastion/src/bastion/tests/dispatch.test.ts @@ -22,6 +22,8 @@ function createTestConfig(testDir: string): BastionConfig { dhcpRangeEnd: "", ubuntuVersion: "26.04", ubuntuMirror: "https://releases.ubuntu.com/26.04", + vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso", + vyosDefaultPassword: "vyos", iface: "eth0", serverIp: "10.0.0.1", network: "10.0.0.0", diff --git a/bastion/src/bastion/tests/kickstart.test.ts b/bastion/src/bastion/tests/kickstart.test.ts index 771f5d1..5a5dada 100644 --- a/bastion/src/bastion/tests/kickstart.test.ts +++ b/bastion/src/bastion/tests/kickstart.test.ts @@ -96,9 +96,9 @@ describe("renderInstallKickstart", () => { expect(ks).toContain("/api/progress"); }); - it("infra role has /var/lib/rancher partition", () => { + it("infra role has 120G /var/lib/rancher partition", () => { const ks = renderInstallKickstart(baseParams({ role: "infra" })); - expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=20480"); + expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=122880"); }); it("infra role has k3s install", () => { @@ -106,10 +106,14 @@ describe("renderInstallKickstart", () => { expect(ks).toContain("curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true sh -"); }); - it("worker role does NOT have /var/lib/rancher partition in fresh install", () => { + it("worker role has 120G /var/lib/rancher partition (imageFs must be sized before longhorn --grow)", () => { const ks = renderInstallKickstart(baseParams({ role: "worker" })); - // Worker should not have the fresh-install rancher partition line - expect(ks).not.toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=20480"); + expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=122880"); + }); + + it("vanilla role does NOT have /var/lib/rancher partition in fresh install", () => { + const ks = renderInstallKickstart(baseParams({ role: "vanilla" })); + expect(ks).not.toContain("--name=rancher --fstype=xfs"); }); it("worker role does NOT have k3s install", () => { diff --git a/bastion/src/bastion/tests/vyos.test.ts b/bastion/src/bastion/tests/vyos.test.ts new file mode 100644 index 0000000..e0535a7 --- /dev/null +++ b/bastion/src/bastion/tests/vyos.test.ts @@ -0,0 +1,548 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import type { BastionConfig } from "@lab/shared"; +import type { FastifyInstance } from "fastify"; +import { createApp } from "../src/server.js"; +import type { StateManager } from "../src/services/state.js"; +import { buildVyosConfigSpec } from "../src/templates/vyos-config-spec.js"; +import { renderVyosInstallPy } from "../src/templates/vyos-install.py.js"; + +function createTestConfig(testDir: string): BastionConfig { + return { + fedoraVersion: "43", + arch: "x86_64", + httpPort: 0, + timezone: "Europe/London", + locale: "en_GB.UTF-8", + bastionDir: testDir, + domain: "test.local", + dhcpMode: "proxy", + dhcpRangeStart: "", + dhcpRangeEnd: "", + ubuntuVersion: "26.04", + ubuntuMirror: "https://releases.ubuntu.com/26.04", + vyosIsoUrl: "https://example.invalid/vyos.iso", + vyosDefaultPassword: "test-pw", + iface: "eth0", + serverIp: "10.0.0.1", + network: "10.0.0.0", + gateway: "10.0.0.1", + sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST lab@test"], + adminUser: "testadmin", + syslogPort: 15515, + skipDnsmasq: true, + skipArtifacts: true, + fedoraMirror: "https://example.invalid/fedora", + tftpDir: join(testDir, "tftp"), + httpDir: join(testDir, "http"), + stateFile: join(testDir, "state.json"), + }; +} + +/** Pull the base64 spec back out of the generated Python driver. */ +function decodeSpecFrom(python: string): Record { + const match = /base64\.b64decode\("([^"]+)"\)/.exec(python); + if (!match?.[1]) throw new Error("no base64 spec found in generated driver"); + return JSON.parse(Buffer.from(match[1], "base64").toString("utf-8")); +} + +describe("vyos config spec", () => { + it("puts VLANs on the bond when members are given", () => { + const spec = buildVyosConfigSpec({ + hostname: "fw1", + defaultPassword: "pw", + spec: { + mgmtInterface: "eth0", + mgmtAddress: "10.0.8.2/24", + bondMembers: ["eth2", "eth3"], + vlans: [{ id: 10, address: "10.0.10.1/24", description: "k8s" }], + }, + }); + + const paths = spec.sets.map((s) => s.path.join(" ")); + expect(paths).toContain("interfaces bonding bond0 mode"); + expect(paths).toContain("interfaces bonding bond0 vif 10 address"); + // VLANs must hang off the bond, not the management NIC. + expect(paths).not.toContain("interfaces ethernet eth0 vif 10 address"); + + // Bond members are a multi-value node — appending, not replacing, is what + // keeps the second member from overwriting the first. + const members = spec.sets.filter( + (s) => s.path.join(" ") === "interfaces bonding bond0 member interface", + ); + expect(members.map((m) => m.value)).toEqual(["eth2", "eth3"]); + expect(members.every((m) => m.replace === false)).toBe(true); + }); + + it("falls back to VLANs on the management NIC when unbonded", () => { + const spec = buildVyosConfigSpec({ + hostname: "fw2", + defaultPassword: "pw", + spec: { mgmtInterface: "eth1", vlans: [{ id: 20, address: "10.0.20.1/24" }] }, + }); + + const paths = spec.sets.map((s) => s.path.join(" ")); + expect(paths).toContain("interfaces ethernet eth1 vif 20 address"); + }); + + it("normalises the target disk to a full /dev path", () => { + // find_disks() enumerates with `lsblk -Jbp`, so valid responses are full + // paths; a bare name fails valid_responses and re-prompts forever. + expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw", disk: "/dev/mmcblk0" }).disk) + .toBe("/dev/mmcblk0"); + expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw", disk: "mmcblk0" }).disk) + .toBe("/dev/mmcblk0"); + expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw" }).disk).toBe(""); + }); + + it("defaults to dhcp on eth0 and never opts into RAID", () => { + const spec = buildVyosConfigSpec({ hostname: "fw4", defaultPassword: "pw" }); + const address = spec.sets.find( + (s) => s.path.join(" ") === "interfaces ethernet eth0 address", + ); + expect(address?.value).toBe("dhcp"); + // The installer's RAID prompt defaults to yes; a second disk must not + // silently produce a mirror. + expect(spec.raid).toBe(false); + }); +}); + +describe("vyos routes", () => { + let testDir: string; + let app: FastifyInstance; + let state: StateManager; + const mac = "aa:bb:cc:11:22:33"; + + beforeEach(() => { + testDir = join(tmpdir(), `bastion-vyos-test-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(join(testDir, "http"), { recursive: true }); + mkdirSync(join(testDir, "tftp"), { recursive: true }); + + const result = createApp(createTestConfig(testDir)); + app = result.app; + state = result.state; + }); + + afterEach(async () => { + await app.close(); + rmSync(testDir, { recursive: true, force: true }); + }); + + it("dispatches a queued vyos machine to the live-boot script", async () => { + state.update((s) => { + s.install_queue[mac] = { + hostname: "fw1", + disk: "/dev/nvme0n1", + role: "worker", + os: "vyos-rolling", + queued_at: new Date().toISOString(), + }; + }); + + const response = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}` }); + expect(response.statusCode).toBe(200); + + expect(response.body).toContain("/vyos-vmlinuz"); + expect(response.body).toContain("fetch=http://10.0.0.1:0/vyos-filesystem.squashfs"); + expect(response.body).toContain(`live-config.hooks=http://10.0.0.1:0/vyos/autoinstall.sh?mac=${mac}`); + + // `nonetworking` appears in VyOS's own PXE docs but breaks the hook fetch, + // and console=ttyS0 costs 30s per systemd phase on boards with no UART. + expect(response.body).not.toContain("nonetworking"); + expect(response.body).not.toContain("console=ttyS0"); + }); + + it("serves a hook that fetches and executes the install driver", async () => { + const response = await app.inject({ method: "GET", url: `/vyos/autoinstall.sh?mac=${mac}` }); + expect(response.statusCode).toBe(200); + expect(response.body).toContain(`/vyos/install.py?mac=${mac}`); + expect(response.body).toContain("python3 /tmp/vyos-install.py"); + }); + + it("bakes the machine's config into the generated install driver", async () => { + state.update((s) => { + s.install_queue[mac] = { + hostname: "fw1", + disk: "/dev/nvme0n1", + role: "worker", + os: "vyos-rolling", + queued_at: new Date().toISOString(), + vyos: { + mgmtInterface: "eth0", + mgmtAddress: "10.0.8.2/24", + bondMembers: ["eth2", "eth3"], + vlans: [{ id: 10, address: "10.0.10.1/24" }], + password: "s3cret", + }, + }; + }); + + const response = await app.inject({ method: "GET", url: `/vyos/install.py?mac=${mac}` }); + expect(response.statusCode).toBe(200); + + // Builds config from the image's own default so the vyos-config-version + // trailer matches and first boot skips migrations. + expect(response.body).toContain("/opt/vyatta/etc/config.boot.default"); + expect(response.body).toContain("/usr/libexec/vyos/op_mode/image_installer.py"); + // "complete" is what moves the machine out of the install queue. + expect(response.body).toContain('report("complete"'); + + const spec = decodeSpecFrom(response.body); + expect(spec["hostname"]).toBe("fw1"); + expect(spec["password"]).toBe("s3cret"); + expect(spec["disk"]).toBe("/dev/nvme0n1"); + + const paths = (spec["sets"] as Array<{ path: string[] }>).map((s) => s.path.join(" ")); + expect(paths).toContain("interfaces bonding bond0 vif 10 address"); + expect(paths).toContain("system host-name"); + }); + + it("falls back to the bastion default password when none is set", async () => { + state.update((s) => { + s.install_queue[mac] = { + hostname: "fw9", + disk: "", + role: "worker", + os: "vyos-rolling", + queued_at: new Date().toISOString(), + }; + }); + + const response = await app.inject({ method: "GET", url: `/vyos/install.py?mac=${mac}` }); + const spec = decodeSpecFrom(response.body); + expect(spec["password"]).toBe("test-pw"); + // Empty disk means "accept the installer's first-disk default". + expect(spec["disk"]).toBe(""); + }); +}); + +describe("vyos hw-id pinning", () => { + it("emits hw-id for the mgmt interface and each bond member", () => { + // Discovery sees enp2s0/enp1s0f0np0 under Fedora, but VyOS enumerates its + // own eth. Pinning by MAC is what makes the mapping deterministic. + const spec = buildVyosConfigSpec({ + hostname: "fw1", + defaultPassword: "pw", + spec: { + mgmtInterface: "eth2", + bondMembers: ["eth0", "eth1"], + hwIds: { + eth2: "64:62:66:25:96:47", + eth0: "64:62:66:25:96:45", + eth1: "64:62:66:25:96:46", + }, + }, + }); + + const hw = spec.sets.filter((s) => s.path[s.path.length - 1] === "hw-id"); + expect(hw.map((s) => [s.path[2], s.value])).toEqual([ + ["eth2", "64:62:66:25:96:47"], + ["eth0", "64:62:66:25:96:45"], + ["eth1", "64:62:66:25:96:46"], + ]); + }); + + it("omits hw-id entirely when no mapping is given", () => { + const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" }); + expect(spec.sets.some((s) => s.path.includes("hw-id"))).toBe(false); + }); +}); + +describe("vyos management VLAN", () => { + it("puts the mgmt VLAN on the PXE port while the bond carries routed VLANs", () => { + // Trunked PXE port: boots untagged on the VLAN the bastion serves, stays + // reachable on the tagged management VLAN. + const spec = buildVyosConfigSpec({ + hostname: "vyos001", + defaultPassword: "pw", + spec: { + mgmtInterface: "eth2", + mgmtAddress: "dhcp", + mgmtVlan: { id: 3, address: "192.168.3.4/24", description: "kvm" }, + bondMembers: ["eth0", "eth1"], + vlans: [{ id: 2, address: "192.168.8.2/23" }], + }, + }); + + const paths = spec.sets.map((s) => s.path.join(" ")); + expect(paths).toContain("interfaces ethernet eth2 vif 3 address"); + expect(paths).toContain("interfaces bonding bond0 vif 2 address"); + // The mgmt VLAN must not land on the bond. + expect(paths).not.toContain("interfaces bonding bond0 vif 3 address"); + expect(spec.tags.map((t) => t.join(" "))).toContain("interfaces ethernet eth2 vif"); + }); +}); + +describe("vyos VRRP HA", () => { + const haSpec = { + mgmtInterface: "eth2", + mgmtAddress: "dhcp", + bondMembers: ["eth0", "eth1"], + bondAddress: "192.168.1.252/24", + bondVrrp: "192.168.1.254/24", + vrrpPriority: 200, + vlans: [ + { id: 3, address: "192.168.3.4/24", vrrp: "192.168.3.254/24" }, + { id: 200, address: "192.168.2.252/24" }, // no VIP on this one + ], + }; + + it("emits a vrrp group per VIP with vrid = VLAN id and dotted vif interface", () => { + const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw", spec: haSpec }); + const paths = spec.sets.map((s) => `${s.path.join(" ")}${s.value !== undefined ? "=" + s.value : ""}`); + + expect(paths).toContain("interfaces bonding bond0 address=192.168.1.252/24"); + // untagged bond group: vrid 1, interface bond0 itself + expect(paths).toContain("high-availability vrrp group native interface=bond0"); + expect(paths).toContain("high-availability vrrp group native vrid=1"); + // address is a tag node -- VIP is the final path segment, no value + expect(paths).toContain("high-availability vrrp group native address 192.168.1.254/24"); + // VLAN group: vrid = VLAN id, dotted vif + expect(paths).toContain("high-availability vrrp group vlan3 interface=bond0.3"); + expect(paths).toContain("high-availability vrrp group vlan3 vrid=3"); + expect(paths).toContain("high-availability vrrp group vlan3 address 192.168.3.254/24"); + // VLAN without a VIP gets no group + expect(paths.some((p) => p.includes("group vlan200"))).toBe(false); + }); + + it("applies the box-wide priority and one sync group over all groups", () => { + const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw", spec: haSpec }); + const prio = spec.sets.filter((s) => s.path[s.path.length - 1] === "priority" + && s.path[0] === "high-availability"); + expect(prio).toHaveLength(2); + expect(prio.every((s) => s.value === "200")).toBe(true); + + // sync group binds the pair: all groups fail over together + const members = spec.sets.filter( + (s) => s.path.join(" ") === "high-availability vrrp sync-group MAIN member", + ); + expect(members.map((m) => m.value)).toEqual(["native", "vlan3"]); + expect(members.every((m) => m.replace === false)).toBe(true); + }); + + it("emits no high-availability nodes when no VIPs are given", () => { + const spec = buildVyosConfigSpec({ + hostname: "fw1", + defaultPassword: "pw", + spec: { bondMembers: ["eth0", "eth1"], vlans: [{ id: 3, address: "192.168.3.4/24" }] }, + }); + expect(spec.sets.some((s) => s.path[0] === "high-availability")).toBe(false); + }); +}); + +describe("pickLargestInitrd", async () => { + const { pickLargestInitrd } = await import("../src/main.js"); + + // Verbatim from `xorriso -lsl /live/` on vyos-2026.08.05-0033-rolling. + const realListing = `total 8 +-r--r--r-- 1 0 0 22255 Aug 5 01:33 'filesystem.packages' +-r--r--r-- 1 0 0 6 Aug 5 01:33 'filesystem.packages-remove' +-r--r--r-- 1 0 0 541192192 Aug 5 01:33 'filesystem.squashfs' +-r--r--r-- 1 0 0 50352547 Aug 5 01:33 'initrd.img' +-r--r--r-- 1 0 0 50352547 Aug 5 01:33 'initrd.img-6.18.41-vyos' +-r--r--r-- 1 0 0 20 Aug 5 01:33 'packages.txt' +-r--r--r-- 1 0 0 9135104 Aug 2 19:54 'vmlinuz' +-r--r--r-- 1 0 0 9135104 Aug 2 19:54 'vmlinuz-6.18.41-vyos' +`; + + it("picks a full-size initrd from a real nightly listing", () => { + expect(pickLargestInitrd(realListing)).toEqual({ name: "initrd.img", size: 50352547 }); + }); + + it("ignores 0-byte decoys and symlinks (which report link size, not target size)", () => { + const listing = `total 8 +-r--r--r-- 1 0 0 0 Aug 5 01:33 'initrd.img' +lrwxrwxrwx 1 0 0 24 Aug 5 01:33 'initrd.img-link' -> 'initrd.img-6.18.41-vyos' +-r--r--r-- 1 0 0 50352547 Aug 5 01:33 'initrd.img-6.18.41-vyos' +`; + expect(pickLargestInitrd(listing)).toEqual({ name: "initrd.img-6.18.41-vyos", size: 50352547 }); + }); + + it("returns undefined when only decoys exist", () => { + expect(pickLargestInitrd("-r--r--r-- 1 0 0 0 Aug 5 01:33 'initrd.img'\n")).toBeUndefined(); + }); +}); + +describe("vyos fedora-parity features", () => { + it("computes reportAddress from a static mgmt address, empty for dhcp", () => { + const staticSpec = buildVyosConfigSpec({ + hostname: "fw1", defaultPassword: "pw", + spec: { mgmtAddress: "192.168.8.2/23" }, + }); + expect(staticSpec.reportAddress).toBe("192.168.8.2"); + + const dhcpSpec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" }); + expect(dhcpSpec.reportAddress).toBe(""); + }); + + it("defaults freshConfig off (reinstall preserves the on-disk config)", () => { + expect(buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" }).freshConfig).toBe(false); + expect(buildVyosConfigSpec({ + hostname: "fw1", defaultPassword: "pw", spec: { freshConfig: true }, + }).freshConfig).toBe(true); + }); + + it("driver streams logs to /api/log and reports 'ready at' on completion", async () => { + const testDir = join(tmpdir(), `bastion-vyos-parity-${Date.now()}`); + mkdirSync(join(testDir, "http"), { recursive: true }); + mkdirSync(join(testDir, "tftp"), { recursive: true }); + const { app: parityApp, state: parityState } = createApp(createTestConfig(testDir)); + try { + parityState.update((s) => { + s.install_queue["aa:bb:cc:44:55:66"] = { + hostname: "fw9", disk: "/dev/vda", role: "vanilla", + os: "vyos-rolling", queued_at: new Date().toISOString(), + }; + }); + const response = await parityApp.inject({ + method: "GET", url: "/vyos/install.py?mac=aa:bb:cc:44:55:66", + }); + expect(response.body).toContain("/api/log"); + expect(response.body).toContain('"lines": batch'); + expect(response.body).toContain('report("complete", "ready at %s"'); + expect(response.body).toContain("ensure_network_boot_first"); + expect(response.body).toContain("lab-provisioned"); + expect(response.body).toContain('ROLE = "vanilla"'); + } finally { + await parityApp.close(); + rmSync(testDir, { recursive: true, force: true }); + } + }); + + it("complete with 'ready at' records installed.ip for a vyos machine", async () => { + const testDir = join(tmpdir(), `bastion-vyos-complete-${Date.now()}`); + mkdirSync(join(testDir, "http"), { recursive: true }); + mkdirSync(join(testDir, "tftp"), { recursive: true }); + const { app: cApp, state: cState } = createApp(createTestConfig(testDir)); + try { + const mac2 = "aa:bb:cc:77:88:99"; + cState.update((s) => { + s.install_queue[mac2] = { + hostname: "fw1", disk: "/dev/vda", role: "vanilla", + os: "vyos-rolling", queued_at: new Date().toISOString(), + }; + }); + const response = await cApp.inject({ + method: "POST", url: "/api/progress", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ mac: mac2, stage: "complete", detail: "ready at 192.168.8.2" }), + }); + expect(response.statusCode).toBe(200); + const installed = cState.load().installed[mac2]; + expect(installed?.ip).toBe("192.168.8.2"); + expect(installed?.os).toBe("vyos-rolling"); + } finally { + await cApp.close(); + rmSync(testDir, { recursive: true, force: true }); + } + }); +}); + +describe("vyos installer prompt coverage", () => { + // Every interactive prompt image_installer.py can emit, copied verbatim from + // the MSG_* constants (including the reinstall-only search_previous_installation + // ones). An unanswered prompt does not fail loudly -- the installer simply + // blocks on stdin until the driver's stall timeout, which is how the reinstall + // path silently hung for 15 minutes in the VM test. + const PROMPTS: Record = { + continue: "Would you like to continue? [y/N] ", + imageName: "What would you like to name this image? (Default: 1.5-rolling) ", + password: 'Please enter a password for the "vyos" user: ', + passwordConfirm: 'Please confirm password for the "vyos" user: ', + console: "What console should be used by default? (K: KVM, S: Serial)? (Default: K) ", + raidConfigure: "Would you like to configure RAID-1 mirroring? [Y/n] ", + raidFoundDisks: "Would you like to configure RAID-1 mirroring on them? [Y/n] ", + raidChooseDisks: "Would you like to choose two disks for RAID-1 mirroring? [Y/n] ", + diskSelect: "Which one should be used for installation? (Default: /dev/vda) ", + diskConfirm: "Installation will delete all data on the drive. Continue? [y/N] ", + raidConfirm: "Installation will delete all data on both drives. Continue? [y/N] ", + rootSizeAll: "Would you like to use all the free space on the drive? [Y/n] ", + bootConfig: "Which file would you like as boot config? ", + copyData: "Would you like to copy data to the new image? [Y/n] ", + chooseCopyData: "From which image would you like to save config information? ", + copyEncData: "Would you like to copy the encrypted config to the new image? [Y/n] ", + chooseCopyEncData: "From which image would you like to copy the encrypted config? ", + }; + + it("answers every installer prompt exactly once", () => { + const { execFileSync } = require("node:child_process") as typeof import("node:child_process"); + const { writeFileSync, unlinkSync, mkdtempSync } = require("node:fs") as typeof import("node:fs"); + + // Skip cleanly where python3 is unavailable (same spirit as the + // ksvalidator-backed kickstart test). + try { + execFileSync("python3", ["--version"], { stdio: "pipe" }); + } catch { + return; + } + + const spec = buildVyosConfigSpec({ + hostname: "fw1", defaultPassword: "pw", disk: "/dev/vda", + }); + const driver = renderVyosInstallPy({ + spec, mac: "aa:bb:cc:11:22:33", serverIp: "10.0.0.1", httpPort: 8080, role: "vanilla", + }); + + const dir = mkdtempSync(join(tmpdir(), "vyos-rules-")); + const driverPath = join(dir, "driver.py"); + const checkPath = join(dir, "check.py"); + writeFileSync(driverPath, driver); + writeFileSync(checkPath, ` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("drv", ${JSON.stringify(driverPath)}) +drv = importlib.util.module_from_spec(spec); spec.loader.exec_module(drv) +rules = drv.build_rules() +prompts = json.loads(sys.argv[1]) +out = {} +for label, text in prompts.items(): + out[label] = len([r for p, r in rules if p.search(text.encode())]) +print(json.dumps(out)) +`); + + try { + const stdout = execFileSync("python3", [checkPath, JSON.stringify(PROMPTS)], { + encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"], + }); + const counts = JSON.parse(stdout) as Record; + const unanswered = Object.entries(counts).filter(([, n]) => n !== 1); + expect(unanswered).toEqual([]); + } finally { + try { unlinkSync(driverPath); unlinkSync(checkPath); } catch { /* best effort */ } + try { rmSync(dir, { recursive: true, force: true }); } catch { /* best effort */ } + } + }); +}); + +describe("vyos boot NIC pinning", () => { + it("pins the boot interface by MAC via BOOTIF", async () => { + // Without this, live-boot picks the first *connected* NIC. On the VP2440 + // the SFP+ pair links before the copper PXE port, so live-boot tried the + // fiber ports (no DHCP), timed out 15s each, and failed with "Unable to + // find a live file system on the network". + const testDir = join(tmpdir(), `bastion-vyos-bootif-${Date.now()}`); + mkdirSync(join(testDir, "http"), { recursive: true }); + mkdirSync(join(testDir, "tftp"), { recursive: true }); + const { app: a, state: st } = createApp(createTestConfig(testDir)); + try { + const m = "64:62:66:25:96:47"; + st.update((s) => { + s.install_queue[m] = { + hostname: "vyos001", disk: "/dev/mmcblk0", role: "vanilla", + os: "vyos-rolling", queued_at: new Date().toISOString(), + }; + }); + const res = await a.inject({ method: "GET", url: `/dispatch?mac=${m}` }); + // live-boot's Device_from_bootif() expects 01- + expect(res.body).toContain("BOOTIF=01-64-62-66-25-96-47"); + // and it must be on the kernel line, before fetch= is attempted + const kernelLine = res.body.split("\n").find((l) => l.startsWith("kernel ")); + expect(kernelLine).toContain("BOOTIF=01-64-62-66-25-96-47"); + expect(kernelLine).toContain("fetch="); + } finally { + await a.close(); + rmSync(testDir, { recursive: true, force: true }); + } + }); +}); diff --git a/bastion/src/cli/src/api/client.ts b/bastion/src/cli/src/api/client.ts index 52add1e..b6f5763 100644 --- a/bastion/src/cli/src/api/client.ts +++ b/bastion/src/cli/src/api/client.ts @@ -90,6 +90,7 @@ export class LabdClient { async installMachine(opts: { mac: string; hostname: string; disk?: string; role?: string; os?: string; + vyos?: import("@lab/shared").VyosInstallSpec; }): Promise<{ status: string; data?: unknown; error?: string }> { return this.request("POST", "/api/machines/install", { body: opts }); } diff --git a/bastion/src/cli/src/commands/install.ts b/bastion/src/cli/src/commands/install.ts index 6da3f11..9ea6786 100644 --- a/bastion/src/cli/src/commands/install.ts +++ b/bastion/src/cli/src/commands/install.ts @@ -1,10 +1,30 @@ // CLI command: provision install // Queue a discovered machine for OS installation via labd. -import { Command, Option } from "commander"; +import { Command, Option, InvalidArgumentError } from "commander"; import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY } from "@lab/shared"; +import type { VyosInstallSpec, VyosVlanSpec } from "@lab/shared"; import { getLabdClient } from "../api/config.js"; +/** Parse a repeated --vlan flag: ":[:]". */ +export function parseVlan(value: string, previous: VyosVlanSpec[] = []): VyosVlanSpec[] { + const parts = value.split(":"); + const id = Number(parts[0]); + const address = parts[1] ?? ""; + // InvalidArgumentError makes commander print a clean message instead of + // dumping a stack trace at the operator. + if (!Number.isInteger(id) || id < 1 || id > 4094) { + throw new InvalidArgumentError(`Invalid VLAN id in "${value}" (expected 1-4094)`); + } + if (!address.includes("/")) { + throw new InvalidArgumentError( + `Invalid VLAN address in "${value}" (expected CIDR, e.g. 10.0.10.1/24)`, + ); + } + const description = parts.slice(2).join(":"); + return [...previous, { id, address, ...(description ? { description } : {}) }]; +} + function roleTable(): string { const lines: string[] = ["", "Available roles:"]; for (const r of ROLE_REGISTRY) { @@ -15,6 +35,38 @@ function roleTable(): string { return lines.join("\n"); } +/** Parse a repeated --vlan-vip flag: ":" — VRRP VIP for a --vlan entry. */ +export function parseVlanVip( + value: string, + previous: Record = {}, +): Record { + const index = value.indexOf(":"); + const id = Number(index === -1 ? Number.NaN : value.slice(0, index)); + const cidr = index === -1 ? "" : value.slice(index + 1).trim(); + if (!Number.isInteger(id) || id < 1 || id > 4094 || !cidr.includes("/")) { + throw new InvalidArgumentError( + `Invalid VLAN VIP "${value}" (expected :, e.g. 3:192.168.3.254/24)`, + ); + } + return { ...previous, [id]: cidr }; +} + +/** Parse a repeated --vyos-hwid flag: "=". */ +export function parseHwId( + value: string, + previous: Record = {}, +): Record { + const index = value.indexOf("="); + const iface = index === -1 ? "" : value.slice(0, index).trim(); + const mac = index === -1 ? "" : value.slice(index + 1).trim().toLowerCase(); + if (iface === "" || !/^([0-9a-f]{2}:){5}[0-9a-f]{2}$/.test(mac)) { + throw new InvalidArgumentError( + `Invalid hw-id "${value}" (expected =, e.g. eth2=64:62:66:25:96:47)`, + ); + } + return { ...previous, [iface]: mac }; +} + export function registerInstallCommand(parent: Command): void { parent .command("install ") @@ -24,10 +76,34 @@ export function registerInstallCommand(parent: Command): void { .addOption(new Option("--role ", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker")) .addOption(new Option("--os ", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43")) .option("--disk ", "Target disk device (auto-detect if omitted)") + .option("--vyos-mgmt ", "VyOS: untagged interface the machine PXE boots from (default eth0)") + .option("--vyos-mgmt-address ", "VyOS: CIDR for the management interface, or 'dhcp' (default dhcp)") + .option("--vyos-bond ", "VyOS: comma-separated LACP bond members (must exclude the PXE NIC)") + .option("--vyos-bond-address ", "VyOS: address on the untagged bond (trunk native VLAN)") + .option("--vyos-bond-vrrp ", "VyOS: VRRP VIP floated on the untagged bond") + .option("--vlan-vip ", "VyOS: VRRP VIP for a --vlan entry (repeatable)", parseVlanVip) + .option("--vyos-vrrp-priority ", "VyOS: VRRP priority for all groups on this box (higher = master)") + .option("--vyos-mgmt-vlan ", "VyOS: tagged management VLAN on the PXE port") + .option("--vlan ", "VyOS: tagged VLAN sub-interface on the bond (repeatable)", parseVlan) + .option("--vyos-password ", "VyOS: password for the 'vyos' user") + .option("--vyos-hwid ", "VyOS: pin an interface name to a MAC via hw-id (repeatable)", parseHwId) + .option("--vyos-fresh-config", "VyOS: on reinstall, overwrite the preserved config with the generated one") .action(async (mac: string, hostname: string, opts: { role: string; os: string; disk?: string; + vyosMgmt?: string; + vyosMgmtAddress?: string; + vyosBond?: string; + vyosBondAddress?: string; + vyosBondVrrp?: string; + vlan?: VyosVlanSpec[]; + vlanVip?: Record; + vyosVrrpPriority?: string; + vyosMgmtVlan?: string; + vyosPassword?: string; + vyosHwid?: Record; + vyosFreshConfig?: boolean; }) => { if (!isValidOsId(opts.os)) { console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`); @@ -39,6 +115,67 @@ export function registerInstallCommand(parent: Command): void { process.exit(1); } + const bondMembers = opts.vyosBond !== undefined && opts.vyosBond !== "" + ? opts.vyosBond.split(",").map((s) => s.trim()).filter((s) => s.length > 0) + : []; + + // Attach --vlan-vip entries to their --vlan definitions. A VIP for a VLAN + // that was never defined is a typo that would otherwise vanish silently. + const vips = opts.vlanVip ?? {}; + const vlans = (opts.vlan ?? []).map((v) => + vips[v.id] !== undefined ? { ...v, vrrp: vips[v.id] as string } : v, + ); + for (const id of Object.keys(vips)) { + if (!vlans.some((v) => String(v.id) === id)) { + console.error(`--vlan-vip ${id}:... has no matching --vlan ${id}:... entry`); + process.exit(1); + } + } + + const vrrpPriority = opts.vyosVrrpPriority !== undefined && opts.vyosVrrpPriority !== "" + ? Number(opts.vyosVrrpPriority) + : undefined; + if (vrrpPriority !== undefined + && (!Number.isInteger(vrrpPriority) || vrrpPriority < 1 || vrrpPriority > 255)) { + console.error(`--vyos-vrrp-priority must be an integer 1-255 (got ${opts.vyosVrrpPriority})`); + process.exit(1); + } + const vyos: VyosInstallSpec = { + ...(opts.vyosMgmt !== undefined && opts.vyosMgmt !== "" + ? { mgmtInterface: opts.vyosMgmt } : {}), + ...(opts.vyosMgmtAddress !== undefined && opts.vyosMgmtAddress !== "" + ? { mgmtAddress: opts.vyosMgmtAddress } : {}), + ...(bondMembers.length > 0 ? { bondMembers } : {}), + ...(opts.vyosBondAddress !== undefined && opts.vyosBondAddress !== "" + ? { bondAddress: opts.vyosBondAddress } : {}), + ...(opts.vyosBondVrrp !== undefined && opts.vyosBondVrrp !== "" + ? { bondVrrp: opts.vyosBondVrrp } : {}), + ...(vrrpPriority !== undefined ? { vrrpPriority } : {}), + ...(vlans.length > 0 ? { vlans } : {}), + ...(opts.vyosPassword !== undefined && opts.vyosPassword !== "" + ? { password: opts.vyosPassword } : {}), + ...(opts.vyosHwid !== undefined && Object.keys(opts.vyosHwid).length > 0 + ? { hwIds: opts.vyosHwid } : {}), + ...(opts.vyosMgmtVlan !== undefined && opts.vyosMgmtVlan !== "" + ? { mgmtVlan: parseVlan(opts.vyosMgmtVlan)[0] as VyosVlanSpec } : {}), + ...(opts.vyosFreshConfig === true ? { freshConfig: true } : {}), + }; + const hasVyosOptions = Object.keys(vyos).length > 0; + + if (hasVyosOptions && !opts.os.startsWith("vyos")) { + console.error(`VyOS options require --os vyos-rolling (got --os ${opts.os})`); + process.exit(1); + } + + // Firmware PXE cannot run over LACP, so the NIC that boots the installer + // must stay out of the bond — otherwise the next reinstall has no path in. + const mgmt = vyos.mgmtInterface ?? "eth0"; + if (bondMembers.includes(mgmt)) { + console.error(`--vyos-bond must not include the PXE/management interface "${mgmt}"`); + console.error("PXE cannot boot over an LACP bond; keep that NIC unbonded."); + process.exit(1); + } + try { const result = await getLabdClient().installMachine({ mac, @@ -46,11 +183,14 @@ export function registerInstallCommand(parent: Command): void { role: opts.role, os: opts.os, ...(opts.disk ? { disk: opts.disk } : {}), + ...(hasVyosOptions ? { vyos } : {}), }); console.log(JSON.stringify(result, null, 2)); console.log(""); - const osLabel = opts.os.startsWith("ubuntu") ? "Ubuntu" : "Fedora"; + const osLabel = opts.os.startsWith("ubuntu") + ? "Ubuntu" + : opts.os.startsWith("vyos") ? "VyOS" : "Fedora"; console.log(`Power on the machine to start ${osLabel} installation.`); const roleInfo = ROLE_REGISTRY.find(r => r.name === opts.role); diff --git a/bastion/src/cli/src/commands/recheck.ts b/bastion/src/cli/src/commands/recheck.ts index 2bfaba3..cce23e4 100644 --- a/bastion/src/cli/src/commands/recheck.ts +++ b/bastion/src/cli/src/commands/recheck.ts @@ -44,11 +44,14 @@ export function registerRecheckCommand(parent: Command): void { } // Build list of machines to check - const targets: Array<{ mac: string; hostname: string; ip: string }> = []; + const targets: Array<{ mac: string; hostname: string; ip: string; sshUser: string }> = []; + const userIsDefault = opts.user === "root"; for (const [mac, info] of Object.entries(state.installed)) { if (!info.ip) continue; if (opts.target && info.hostname !== opts.target && mac !== opts.target) continue; - targets.push({ mac, hostname: info.hostname, ip: info.ip }); + // VyOS boxes only have the "vyos" login; honor an explicit --user. + const sshUser = userIsDefault && (info.os ?? "").startsWith("vyos") ? "vyos" : opts.user; + targets.push({ mac, hostname: info.hostname, ip: info.ip, sshUser }); } if (targets.length === 0) { @@ -61,12 +64,12 @@ export function registerRecheckCommand(parent: Command): void { let updated = 0; let failed = 0; - for (const { mac, hostname, ip } of targets) { + for (const { mac, hostname, ip, sshUser } of targets) { process.stdout.write(` ${hostname.padEnd(24)} ${DIM}(${ip})${RESET} `); try { const t0 = Date.now(); - const result = await sshExec(ip, opts.user, HW_COLLECT_SCRIPT, SSH_OPTS); + const result = await sshExec(ip, sshUser, HW_COLLECT_SCRIPT, SSH_OPTS); const elapsed = Date.now() - t0; if (result.exitCode !== 0) { console.log(`${RED}SSH failed (exit ${result.exitCode}, ${elapsed}ms)${RESET}`); diff --git a/bastion/src/cli/src/commands/reprovision.ts b/bastion/src/cli/src/commands/reprovision.ts index 9f213f6..4821d40 100644 --- a/bastion/src/cli/src/commands/reprovision.ts +++ b/bastion/src/cli/src/commands/reprovision.ts @@ -24,12 +24,12 @@ function roleTable(): string { function resolveTarget( target: string, state: BastionState, -): { mac: string; hostname: string; ip: string } | null { +): { mac: string; hostname: string; ip: string; os?: string } | null { const normalized = target.toLowerCase().replace(/-/g, ":"); if (state.installed[normalized]) { const info = state.installed[normalized]; - return { mac: normalized, hostname: info.hostname, ip: info.ip }; + return { mac: normalized, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) }; } if (state.discovered[normalized]) { @@ -38,13 +38,13 @@ function resolveTarget( for (const [mac, info] of Object.entries(state.installed)) { if (info.hostname === target || info.hostname.startsWith(target + ".")) { - return { mac, hostname: info.hostname, ip: info.ip }; + return { mac, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) }; } } for (const [mac, info] of Object.entries(state.installed)) { if (info.ip === target) { - return { mac, hostname: info.hostname, ip: info.ip }; + return { mac, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) }; } } @@ -60,10 +60,12 @@ export function registerReprovisionCommand(parent: Command): void { .addOption(new Option("--role ", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker")) .addOption(new Option("--os ", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43")) .option("--disk ", "Target disk device (auto-detect if omitted)") + .option("--user ", "SSH user for the reboot (default: vyos for VyOS machines, else current user)") .action(async (target: string, hostnameOverride: string | undefined, opts: { role: string; os: string; disk?: string; + user?: string; }) => { if (!isValidOsId(opts.os)) { console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`); @@ -123,7 +125,11 @@ export function registerReprovisionCommand(parent: Command): void { return; } - const adminUser = process.env["SUDO_USER"] ?? process.env["USER"] ?? ""; + // SSH user: explicit flag > the machine's current OS (VyOS boxes only + // have the "vyos" login) > the invoking user. + const currentOsIsVyos = (resolved.os ?? "").startsWith("vyos"); + const adminUser = opts.user + ?? (currentOsIsVyos ? "vyos" : (process.env["SUDO_USER"] ?? process.env["USER"] ?? "")); const effectiveUser = adminUser === "root" ? "" : adminUser; if (effectiveUser === "") { diff --git a/bastion/src/cli/tests/install-vyos.test.ts b/bastion/src/cli/tests/install-vyos.test.ts new file mode 100644 index 0000000..2f1a62e --- /dev/null +++ b/bastion/src/cli/tests/install-vyos.test.ts @@ -0,0 +1,35 @@ +// Tests for VyOS install option parsing. + +import { describe, it, expect } from "vitest"; +import { parseVlan } from "../src/commands/install.js"; + +describe("parseVlan", () => { + it("parses id and CIDR", () => { + expect(parseVlan("10:10.0.10.1/24")).toEqual([{ id: 10, address: "10.0.10.1/24" }]); + }); + + it("accumulates across repeated flags", () => { + const first = parseVlan("10:10.0.10.1/24"); + const both = parseVlan("20:10.0.20.1/24", first); + expect(both).toHaveLength(2); + expect(both[1]).toEqual({ id: 20, address: "10.0.20.1/24" }); + }); + + it("keeps a description, including one containing colons", () => { + expect(parseVlan("30:10.0.30.1/24:mgmt:secondary")).toEqual([ + { id: 30, address: "10.0.30.1/24", description: "mgmt:secondary" }, + ]); + }); + + it("rejects an address that is not CIDR", () => { + // A bare address would produce a VyOS config that fails to commit on first + // boot, long after the operator has stopped watching. + expect(() => parseVlan("10:10.0.10.1")).toThrow(/CIDR/); + }); + + it("rejects out-of-range and non-numeric VLAN ids", () => { + expect(() => parseVlan("0:10.0.10.1/24")).toThrow(/1-4094/); + expect(() => parseVlan("4095:10.0.10.1/24")).toThrow(/1-4094/); + expect(() => parseVlan("abc:10.0.10.1/24")).toThrow(/1-4094/); + }); +}); diff --git a/bastion/src/labd/src/routes/bastions.ts b/bastion/src/labd/src/routes/bastions.ts index d77a417..8792c5c 100644 --- a/bastion/src/labd/src/routes/bastions.ts +++ b/bastion/src/labd/src/routes/bastions.ts @@ -10,6 +10,7 @@ import type { FastifyInstance } from "fastify"; import type { DbClient } from "../server.js"; import { bastionRegistry } from "../services/bastion-registry.js"; import { generateRequestId } from "@lab/shared"; +import type { VyosInstallSpec } from "@lab/shared"; const COMMAND_TIMEOUT_MS = 15_000; @@ -163,9 +164,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void // Queue install — route to correct bastion by MAC app.post<{ - Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string }; + Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string; vyos?: VyosInstallSpec }; }>("/api/machines/install", async (request, reply) => { - const { mac, hostname, disk, role, os } = request.body ?? {}; + const { mac, hostname, disk, role, os, vyos } = request.body ?? {}; if (!mac || !hostname) { return reply.code(400).send({ error: "mac and hostname are required" }); } @@ -183,6 +184,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void const result = await sendCommand(all[0]!.bastionId, { type: "command-install", mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43", + ...(vyos ? { vyos } : {}), }); return reply.code(result.status === "ok" ? 200 : 500).send(result); } catch (err) { @@ -196,6 +198,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void const result = await sendCommand(bastion.bastionId, { type: "command-install", mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43", + ...(vyos ? { vyos } : {}), }); return reply.code(result.status === "ok" ? 200 : 500).send(result); } catch (err) { diff --git a/bastion/src/modules/modules/k3s/src/groups/host-prep.ts b/bastion/src/modules/modules/k3s/src/groups/host-prep.ts index ab1b4b8..d158415 100644 --- a/bastion/src/modules/modules/k3s/src/groups/host-prep.ts +++ b/bastion/src/modules/modules/k3s/src/groups/host-prep.ts @@ -1,21 +1,23 @@ -// Host preparation: kernel modules, sysctl, swap, firewall, SELinux. +// Host preparation: kernel modules, sysctl, swap, storage, firewall, SELinux. import type { OperationContext, OperationResult, OperationGroup } from "../types.js"; import { runSequential } from "../utils.js"; import { loadKernelModules } from "../operations/kernel-modules.js"; import { applyCisHardening } from "../operations/sysctl.js"; -import { disableSwap } from "../operations/swap.js"; +import { enableSwap } from "../operations/swap.js"; +import { growRancherLv } from "../operations/rancher-storage.js"; import { disableFirewall } from "../operations/firewall.js"; import { setSelinuxPermissive } from "../operations/selinux.js"; import { enableIscsi } from "../operations/iscsi.js"; export const hostPrepGroup: OperationGroup = { name: "host-prep", - description: "Prepare host for k3s: kernel modules, sysctl, swap, firewall, SELinux, iSCSI", + description: "Prepare host for k3s: kernel modules, sysctl, swap, imageFs sizing, firewall, SELinux, iSCSI", operations: [ { name: "Load kernel modules", fn: loadKernelModules }, { name: "Apply CIS sysctl", fn: applyCisHardening }, - { name: "Disable swap", fn: disableSwap }, + { name: "Enable swap", fn: enableSwap }, + { name: "Grow rancher LV", fn: growRancherLv }, { name: "Disable firewall", fn: disableFirewall }, { name: "Set SELinux permissive", fn: setSelinuxPermissive }, { name: "Enable iSCSI", fn: enableIscsi }, diff --git a/bastion/src/modules/modules/k3s/src/operations/index.ts b/bastion/src/modules/modules/k3s/src/operations/index.ts index f6434dc..d4783de 100644 --- a/bastion/src/modules/modules/k3s/src/operations/index.ts +++ b/bastion/src/modules/modules/k3s/src/operations/index.ts @@ -1,6 +1,7 @@ export { loadKernelModules } from "./kernel-modules.js"; export { applyCisHardening } from "./sysctl.js"; -export { disableSwap } from "./swap.js"; +export { enableSwap } from "./swap.js"; +export { growRancherLv } from "./rancher-storage.js"; export { enableIscsi } from "./iscsi.js"; export { disableFirewall } from "./firewall.js"; export { setSelinuxPermissive } from "./selinux.js"; diff --git a/bastion/src/modules/modules/k3s/src/operations/rancher-storage.ts b/bastion/src/modules/modules/k3s/src/operations/rancher-storage.ts new file mode 100644 index 0000000..a90fac7 --- /dev/null +++ b/bastion/src/modules/modules/k3s/src/operations/rancher-storage.ts @@ -0,0 +1,48 @@ +// Grow the labvg/rancher LV (k3s image store / imageFs) to 120G. +// 2026-08 incident: the original 20G LV sat at 85% used from steady-state +// images alone, so one ~5G image pull tripped imagefs eviction and evicted +// unrelated pods. Fresh installs are sized at 120G by the kickstart; this op +// covers nodes installed before that change and vanilla nodes converted to +// k8s later. Never removes or shrinks anything — if the VG lacks free space +// (e.g. a longhorn --grow LV consumed it), it reports and moves on. + +import type { Operation, OperationResult } from "../types.js"; +import { sshOpts } from "../utils.js"; + +const RANCHER_LV = "labvg/rancher"; +const TARGET_MIB = 122880; // 120G + +export const growRancherLv: Operation = async (ctx): Promise => { + const lv = await ctx.ssh.exec( + `lvs --noheadings --units m --nosuffix -o lv_size ${RANCHER_LV} 2>/dev/null || true`, + sshOpts(ctx), + ); + const sizeMib = Number.parseFloat(lv.stdout.trim()); + if (Number.isNaN(sizeMib)) { + return { success: true, changed: false, message: "No labvg/rancher LV — imageFs shares /var, skipping" }; + } + if (sizeMib >= TARGET_MIB) { + return { success: true, changed: false, message: `rancher LV already ${Math.round(sizeMib / 1024)}G` }; + } + + const vg = await ctx.ssh.exec(`vgs --noheadings --units m --nosuffix -o vg_free labvg`, sshOpts(ctx)); + const freeMib = Number.parseFloat(vg.stdout.trim()); + const neededMib = TARGET_MIB - sizeMib; + if (Number.isNaN(freeMib) || freeMib < neededMib) { + return { + success: true, + changed: false, + message: `VG labvg has ${Math.floor((Number.isNaN(freeMib) ? 0 : freeMib) / 1024)}G free — ` + + `need ${Math.ceil(neededMib / 1024)}G to grow rancher LV to 120G (manual LV rebuild required)`, + }; + } + + await ctx.ssh.exec(`lvextend -L ${TARGET_MIB}m /dev/${RANCHER_LV}`, sshOpts(ctx)); + await ctx.ssh.exec(`xfs_growfs /var/lib/rancher`, sshOpts(ctx)); + + return { + success: true, + changed: true, + message: `rancher LV grown ${Math.round(sizeMib / 1024)}G → 120G`, + }; +}; diff --git a/bastion/src/modules/modules/k3s/src/operations/swap.ts b/bastion/src/modules/modules/k3s/src/operations/swap.ts index cdc2b17..1a86211 100644 --- a/bastion/src/modules/modules/k3s/src/operations/swap.ts +++ b/bastion/src/modules/modules/k3s/src/operations/swap.ts @@ -1,22 +1,40 @@ -// Disable swap (CIS requirement for k3s). +// Enable swap so memory pressure spills to disk instead of OOM-killing. +// kubelet runs with failSwapOn=false (k3s default); zram stays the fast tier, +// the labvg-swap LV is the overflow tier. Replaces the old CIS-style +// disableSwap op — a kernel OOM kill of a node daemon is worse than slow swap. import type { Operation, OperationResult } from "../types.js"; import { sshOpts } from "../utils.js"; -export const disableSwap: Operation = async (ctx): Promise => { - const check = await ctx.ssh.exec("swapon --show --noheadings", sshOpts(ctx)); - const active = check.stdout.trim().length > 0; +const SWAP_DEV = "/dev/mapper/labvg-swap"; - if (active) { - await ctx.ssh.exec("swapoff -a", sshOpts(ctx)); +export const enableSwap: Operation = async (ctx): Promise => { + const lv = await ctx.ssh.exec(`test -b ${SWAP_DEV} && echo yes || echo no`, sshOpts(ctx)); + if (lv.stdout.trim() !== "yes") { + return { success: true, changed: false, message: "No labvg-swap LV — skipping swap enable" }; } - // Remove swap entries from fstab permanently - await ctx.ssh.exec("sed -i '/\\sswap\\s/d' /etc/fstab", sshOpts(ctx)); + const active = await ctx.ssh.exec( + `grep -q "^$(readlink -f ${SWAP_DEV}) " /proc/swaps && echo on || echo off`, + sshOpts(ctx), + ); + const wasOff = active.stdout.trim() !== "on"; + + if (wasOff) { + // Format if the LV was never (or wrongly) initialised, then activate + await ctx.ssh.exec(`blkid ${SWAP_DEV} | grep -q 'TYPE="swap"' || mkswap ${SWAP_DEV}`, sshOpts(ctx)); + await ctx.ssh.exec(`swapon ${SWAP_DEV}`, sshOpts(ctx)); + } + + // Persist across reboots (idempotent) + await ctx.ssh.exec( + `grep -q "labvg-swap" /etc/fstab || echo "${SWAP_DEV} none swap defaults 0 0" >> /etc/fstab`, + sshOpts(ctx), + ); return { success: true, - changed: active, - message: active ? "Swap disabled" : "Swap already disabled", + changed: wasOff, + message: wasOff ? "LV swap enabled" : "LV swap already active", }; }; diff --git a/bastion/src/modules/modules/k3s/tests/operations.test.ts b/bastion/src/modules/modules/k3s/tests/operations.test.ts index 5a61ab9..3502242 100644 --- a/bastion/src/modules/modules/k3s/tests/operations.test.ts +++ b/bastion/src/modules/modules/k3s/tests/operations.test.ts @@ -72,31 +72,97 @@ describe("applyCisHardening", () => { // --- Swap --- -import { disableSwap } from "../src/operations/swap.js"; +import { enableSwap } from "../src/operations/swap.js"; -describe("disableSwap", () => { - it("disables active swap", async () => { +describe("enableSwap", () => { + it("activates LV swap when present but off", async () => { const ctx = mockCtx(); ctx.ssh.exec - .mockResolvedValueOnce(stdout("/dev/sda2 partition 2G")) // swap active - .mockResolvedValueOnce(OK) // swapoff - .mockResolvedValueOnce(OK); // sed fstab + .mockResolvedValueOnce(stdout("yes")) // LV exists + .mockResolvedValueOnce(stdout("off")) // not in /proc/swaps + .mockResolvedValueOnce(OK) // blkid || mkswap + .mockResolvedValueOnce(OK) // swapon + .mockResolvedValueOnce(OK); // fstab entry - const result = await disableSwap(ctx); + const result = await enableSwap(ctx); expect(result.success).toBe(true); expect(result.changed).toBe(true); - expectCommand(ctx.ssh, "swapoff -a"); + expectCommand(ctx.ssh, "swapon /dev/mapper/labvg-swap"); }); - it("is idempotent when swap already off", async () => { + it("is idempotent when LV swap already active", async () => { const ctx = mockCtx(); ctx.ssh.exec - .mockResolvedValueOnce(stdout("")) // no swap - .mockResolvedValueOnce(OK); // sed fstab (always runs) + .mockResolvedValueOnce(stdout("yes")) // LV exists + .mockResolvedValueOnce(stdout("on")) // already in /proc/swaps + .mockResolvedValueOnce(OK); // fstab entry (always ensured) - const result = await disableSwap(ctx); + const result = await enableSwap(ctx); expect(result.changed).toBe(false); - expectNoCommand(ctx.ssh, "swapoff"); + expectNoCommand(ctx.ssh, "swapon /dev/mapper/labvg-swap"); + }); + + it("skips when no labvg-swap LV exists", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(stdout("no")); // LV missing + + const result = await enableSwap(ctx); + expect(result.success).toBe(true); + expect(result.changed).toBe(false); + expectNoCommand(ctx.ssh, "swapon"); + }); +}); + +// --- Rancher LV (imageFs sizing) --- + +import { growRancherLv } from "../src/operations/rancher-storage.js"; + +describe("growRancherLv", () => { + it("grows a 20G LV to 120G when the VG has space", async () => { + const ctx = mockCtx(); + ctx.ssh.exec + .mockResolvedValueOnce(stdout(" 20480.00")) // lv_size + .mockResolvedValueOnce(stdout(" 747807.00")) // vg_free + .mockResolvedValueOnce(OK) // lvextend + .mockResolvedValueOnce(OK); // xfs_growfs + + const result = await growRancherLv(ctx); + expect(result.success).toBe(true); + expect(result.changed).toBe(true); + expectCommand(ctx.ssh, "lvextend -L 122880m /dev/labvg/rancher"); + expectCommand(ctx.ssh, "xfs_growfs /var/lib/rancher"); + }); + + it("is idempotent when the LV is already 120G", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(stdout(" 122880.00")); // lv_size + + const result = await growRancherLv(ctx); + expect(result.changed).toBe(false); + expectNoCommand(ctx.ssh, "lvextend"); + }); + + it("reports without failing when the VG has no free space", async () => { + const ctx = mockCtx(); + ctx.ssh.exec + .mockResolvedValueOnce(stdout(" 20480.00")) // lv_size + .mockResolvedValueOnce(stdout(" 0.00")); // vg_free + + const result = await growRancherLv(ctx); + expect(result.success).toBe(true); + expect(result.changed).toBe(false); + expect(result.message).toContain("free"); + expectNoCommand(ctx.ssh, "lvextend"); + }); + + it("skips when there is no rancher LV", async () => { + const ctx = mockCtx(); + ctx.ssh.exec.mockResolvedValueOnce(stdout("")); // lvs empty + + const result = await growRancherLv(ctx); + expect(result.success).toBe(true); + expect(result.changed).toBe(false); + expectNoCommand(ctx.ssh, "lvextend"); }); }); diff --git a/bastion/src/modules/modules/k3s/tests/smoke.test.ts b/bastion/src/modules/modules/k3s/tests/smoke.test.ts index e3234f8..8fbbb10 100644 --- a/bastion/src/modules/modules/k3s/tests/smoke.test.ts +++ b/bastion/src/modules/modules/k3s/tests/smoke.test.ts @@ -16,7 +16,7 @@ describe("smoke: full server install pipeline", () => { const pipeline: NamedOperation[] = [ { name: "Kernel modules", fn: ops.loadKernelModules }, { name: "Sysctl hardening", fn: ops.applyCisHardening }, - { name: "Disable swap", fn: ops.disableSwap }, + { name: "Enable swap", fn: ops.enableSwap }, { name: "Disable firewall", fn: ops.disableFirewall }, { name: "SELinux permissive", fn: ops.setSelinuxPermissive }, { name: "Write k3s config", fn: ops.writeK3sConfig }, @@ -73,7 +73,7 @@ describe("smoke: pipeline stops on failure", () => { }; const results = await runSequential(ctx, [ - { name: "OK op", fn: ops.disableSwap }, + { name: "OK op", fn: ops.enableSwap }, { name: "Failing op", fn: failingOp }, { name: "Never called", fn: neverCalled }, ]); @@ -98,11 +98,12 @@ describe("smoke: agent install rejects missing config", () => { }); describe("smoke: all operations are exported", () => { - it("exports all 15 operations", () => { + it("exports all 16 operations", () => { const exported = [ ops.loadKernelModules, ops.applyCisHardening, - ops.disableSwap, + ops.enableSwap, + ops.growRancherLv, ops.disableFirewall, ops.setSelinuxPermissive, ops.writeK3sConfig, @@ -117,7 +118,7 @@ describe("smoke: all operations are exported", () => { ops.checkCertExpiry, ]; - expect(exported).toHaveLength(15); + expect(exported).toHaveLength(16); for (const op of exported) { expect(typeof op).toBe("function"); } diff --git a/bastion/src/shared/src/index.ts b/bastion/src/shared/src/index.ts index 443edbc..24accf0 100644 --- a/bastion/src/shared/src/index.ts +++ b/bastion/src/shared/src/index.ts @@ -8,6 +8,8 @@ export type { DebugConfig, BastionState, BastionConfig, + VyosVlanSpec, + VyosInstallSpec, } from "./types/index.js"; export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./types/index.js"; diff --git a/bastion/src/shared/src/protocol/index.ts b/bastion/src/shared/src/protocol/index.ts index d7e3018..d7f626a 100644 --- a/bastion/src/shared/src/protocol/index.ts +++ b/bastion/src/shared/src/protocol/index.ts @@ -1,6 +1,7 @@ // Protocol types for agent-labd WebSocket communication. import { randomUUID } from "node:crypto"; +import type { VyosInstallSpec } from "../types/state.js"; // --- Agent -> labd messages --- @@ -108,7 +109,7 @@ export type BastionMessage = export type LabdBastionMessage = | { type: "bastion-enrolled"; bastionId: string } | { type: "bastion-heartbeat-ack"; serverTime: string } - | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string } + | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string; vyos?: VyosInstallSpec } | { type: "command-forget"; requestId: string; mac: string } | { type: "command-role-update"; requestId: string; mac: string; role: string } | { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean } diff --git a/bastion/src/shared/src/types/config.ts b/bastion/src/shared/src/types/config.ts index 7c1d72f..ed4a599 100644 --- a/bastion/src/shared/src/types/config.ts +++ b/bastion/src/shared/src/types/config.ts @@ -14,6 +14,10 @@ export interface BastionConfig { // Ubuntu support ubuntuVersion: string; ubuntuMirror: string; + // VyOS support — netboot artifacts are extracted from the ISO at startup. + // LTS ISOs are subscription-only, so this defaults to a rolling release. + vyosIsoUrl: string; + vyosDefaultPassword: string; // Syslog listener for install logs (Anaconda logging --host) syslogPort: number; // Flags diff --git a/bastion/src/shared/src/types/index.ts b/bastion/src/shared/src/types/index.ts index 510ee31..fd2d53c 100644 --- a/bastion/src/shared/src/types/index.ts +++ b/bastion/src/shared/src/types/index.ts @@ -7,6 +7,8 @@ export type { InstalledInfo, DebugConfig, BastionState, + VyosVlanSpec, + VyosInstallSpec, } from "./state.js"; export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./state.js"; diff --git a/bastion/src/shared/src/types/state.ts b/bastion/src/shared/src/types/state.ts index b8599fd..72f5ee0 100644 --- a/bastion/src/shared/src/types/state.ts +++ b/bastion/src/shared/src/types/state.ts @@ -2,10 +2,10 @@ export type ProvisionStackType = "dhcpproxy" | "iso" | "cloud-init"; -export type OsId = "fedora-43" | "ubuntu-26.04"; +export type OsId = "fedora-43" | "ubuntu-26.04" | "vyos-rolling"; export type Arch = "x86_64" | "aarch64"; -export const SUPPORTED_OS: readonly OsId[] = ["fedora-43", "ubuntu-26.04"] as const; +export const SUPPORTED_OS: readonly OsId[] = ["fedora-43", "ubuntu-26.04", "vyos-rolling"] as const; export function isValidOsId(value: string): value is OsId { return (SUPPORTED_OS as readonly string[]).includes(value); @@ -75,13 +75,89 @@ export interface ProgressLogEntry { timestamp: string; } +/** A tagged VLAN sub-interface on the bond (or on the mgmt NIC when unbonded). */ +export interface VyosVlanSpec { + id: number; + address: string; // CIDR, e.g. "10.0.10.1/24" + description?: string; + /** + * VRRP virtual address (CIDR) floated on this VLAN. Emitted as a + * high-availability vrrp group with vrid = VLAN id, so the same spec on both + * HA peers (with different priorities) produces a matching group pair. + */ + vrrp?: string; +} + +/** + * VyOS-specific install parameters. Rendered into the config.boot that the + * installer adopts, so the router comes up already configured. + * + * NOTE: bondMembers must NOT include the interface PXE booted from. Firmware + * PXE cannot run over LACP, so the install-time NIC has to stay unbonded. + */ +export interface VyosInstallSpec { + /** Interfaces aggregated into bond0 with LACP (802.3ad). Omit for no bond. */ + bondMembers?: string[]; + /** CIDR address on bond0 itself — the switch trunk's native/untagged VLAN. */ + bondAddress?: string; + /** VRRP virtual address (CIDR) floated on the untagged bond (vrid 1). */ + bondVrrp?: string; + /** + * VRRP priority for every group on this box. Higher wins mastership. + * The HA pair differs ONLY here (e.g. 200 on the primary, 100 on the + * standby) — addresses differ per box, VIPs and vrids match. + */ + vrrpPriority?: number; + /** Tagged VLAN sub-interfaces, created on bond0 when bonded, else on mgmtInterface. */ + vlans?: VyosVlanSpec[]; + /** Untagged interface the machine PXE booted from. Defaults to "eth0". */ + mgmtInterface?: string; + /** CIDR address for mgmtInterface, or "dhcp". Defaults to "dhcp". */ + mgmtAddress?: string; + /** + * Tagged management VLAN on mgmtInterface, separate from the routed VLANs + * carried by the bond. + * + * Needed when the PXE port is a trunk: it boots untagged on the VLAN the + * bastion's proxy DHCP serves, and carries the management VLAN tagged so the + * router stays reachable there without giving up reinstallability. + */ + mgmtVlan?: VyosVlanSpec; + /** Password for the "vyos" user. Falls back to the bastion default. */ + password?: string; + /** + * On reinstall the VyOS installer carries the previous on-disk config (and + * SSH host keys) forward -- the "reinstall without losing data" default. + * Set true to make the bastion-generated config win instead: after install + * the driver overwrites the installed image's config.boot. + */ + freshConfig?: boolean; + /** + * VyOS interface name -> MAC, emitted as `hw-id` so names bind deterministically. + * + * Discovery runs under Fedora and reports predictable names (enp2s0, + * enp1s0f0np0), but VyOS enumerates its own eth names, so a name observed + * during discovery cannot be used directly. Pinning by MAC removes the guess + * about which physical port a given eth is. + */ + hwIds?: Record; +} + export interface InstallConfig { hostname: string; disk: string; role: Role; os?: OsId; // defaults to "fedora-43" for backward compat + vyos?: VyosInstallSpec; // only consulted when os is "vyos-rolling" arch?: Arch; // detected from HardwareInfo or overridden queued_at: string; + /** + * When dispatch last served this machine an install boot script. Progress + * callbacks only start once the installer environment is up, so a machine + * dispatched long ago with no progress is wedged before that point (bad + * kernel/initrd, no network in the initramfs, wrong NIC picked...). + */ + dispatched_at?: string; progress?: string; progress_at?: string; progress_detail?: string; diff --git a/bastion/tests/integration/helpers/pxe-vm.ts b/bastion/tests/integration/helpers/pxe-vm.ts index 58ae4e2..f5374e2 100644 --- a/bastion/tests/integration/helpers/pxe-vm.ts +++ b/bastion/tests/integration/helpers/pxe-vm.ts @@ -29,6 +29,17 @@ export interface PxeVmConfig { diskSize: number; // GB network: string; // libvirt network name arch?: "x86_64" | "aarch64"; + /** + * Extra NICs enumerated BEFORE the PXE NIC, on a network with no route to + * the bastion (defaults to libvirt's "default"). + * + * Real multi-NIC boxes expose a class of bug a single-NIC VM cannot: an + * initramfs that picks "the first connected interface" grabs one of these + * instead of the NIC that PXE booted, and then cannot reach the bastion. + * Defaults to 0 (single NIC). + */ + decoyNics?: number; + decoyNetwork?: string; } /** Create a blank UEFI VM that PXE boots from the network. */ @@ -61,6 +72,10 @@ export function createPxeVm(config: PxeVmConfig): void { `--memory=${config.memory}`, `--vcpus=${config.vcpus}`, `--disk=path=${diskPath},format=qcow2,bus=virtio`, + // Decoys first so they enumerate ahead of the PXE NIC. They are up and + // carry a lease, but have no route to the bastion. + ...Array.from({ length: config.decoyNics ?? 0 }, () => + `--network=network=${config.decoyNetwork ?? "default"},model=virtio`), `--network=network=${config.network},model=virtio`, // UEFI firmware — required for PXE boot in modern mode `--boot=uefi,network,hd`, @@ -95,12 +110,21 @@ export function destroyPxeVm(name: string): void { } /** Get the MAC address of a VM's first NIC. */ -export function getVmMac(name: string): string | null { +export function getVmMac(name: string, network?: string): string | null { const result = virsh("domiflist", name); if (result.status !== 0) return null; // Output format: Interface Type Source Model MAC - const match = result.stdout.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i); - return match ? match[1].toLowerCase() : null; + // With decoy NICs present, match the line for the PXE network so we return + // the NIC that actually boots rather than whichever is listed first. + const lines = result.stdout.split("\n"); + const candidates = network === undefined + ? lines + : lines.filter((l) => l.split(/\s+/).includes(network)); + for (const line of candidates.length > 0 ? candidates : lines) { + const m = line.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i); + if (m) return m[1].toLowerCase(); + } + return null; } /** Reboot a VM (force off + start). */ diff --git a/bastion/tests/integration/vyos-provision.test.ts b/bastion/tests/integration/vyos-provision.test.ts new file mode 100644 index 0000000..c3aa5b7 --- /dev/null +++ b/bastion/tests/integration/vyos-provision.test.ts @@ -0,0 +1,387 @@ +// Integration test: full VyOS unattended provisioning flow. +// +// Validates the VyOS install path end-to-end, at the same depth as the Fedora +// pxe-provision test: +// 1. Bastion (HTTP + dnsmasq) on the isolated libvirt PXE network +// 2. Blank UEFI VM PXE boots -> Fedora-based discovery (OS-neutral) +// 3. Queue os=vyos-rolling -> live boot + live-config hook + pty driver +// 4. Fresh-install asserts: installed.ip, streamed logs, applied config, +// /config/lab-provisioned, boot-order handling +// 5. REINSTALL round: previous config + /config data carried forward +// ("reinstall without losing data", VyOS-flavored) +// 6. freshConfig round: bastion-generated config wins, /config data kept +// +// Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo, internet +// (first run downloads the ~600MB VyOS nightly ISO; artifacts are cached). +// Run: sudo pnpm run test:integration:vyos + +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, symlinkSync, writeFileSync } from "node:fs"; +import { execSync } from "node:child_process"; +import { join } from "node:path"; +import { homedir, tmpdir } from "node:os"; +import { log, waitForSsh } from "./helpers/libvirt.js"; +import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js"; +import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm } from "./helpers/pxe-vm.js"; +import { sshExec } from "./helpers/ssh.js"; + +const VM_NAME = "lab-vyos-test"; +const VM_MEMORY = 4096; +const VM_VCPUS = 4; +const VM_DISK_GB = 10; // VyOS image install needs ~2GB minimum +const HTTP_PORT = 8099; +const SSH_USER = "vyos"; // the only VyOS login user +const BASTION_IP = PXE_GATEWAY; +const DHCP_RANGE_START = `${PXE_SUBNET}.100`; +const DHCP_RANGE_END = `${PXE_SUBNET}.200`; + +const DISCOVERY_TIMEOUT_MS = 5 * 60_000; +const INSTALL_TIMEOUT_MS = 15 * 60_000; // squashfs fetch + copy; much faster than Anaconda +const SSH_TIMEOUT_MS = 8 * 60_000; + +const HOSTNAME_R1 = "vyos-r1"; +const HOSTNAME_R2 = "vyos-r2"; +const HOSTNAME_R3 = "vyos-r3"; + +function findSshKey(): { pubKey: string; keyPath: string } { + const homes = [homedir()]; + const sudoUser = process.env["SUDO_USER"]; + if (sudoUser) homes.push(join("/home", sudoUser)); + if (process.env["SSH_KEY_PATH"]) { + const keyPath = process.env["SSH_KEY_PATH"]; + const pubPath = `${keyPath}.pub`; + if (existsSync(keyPath) && existsSync(pubPath)) { + return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath }; + } + } + for (const home of homes) { + for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) { + const keyPath = join(home, ".ssh", name); + const pubPath = `${keyPath}.pub`; + if (existsSync(keyPath) && existsSync(pubPath)) { + return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath }; + } + } + } + throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/"); +} + +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +async function pollApi( + url: string, + check: (data: T) => boolean, + timeoutMs: number, + intervalMs = 5000, +): Promise { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + try { + const res = await fetch(url); + if (res.ok) { + const data = (await res.json()) as T; + if (check(data)) return data; + } + } catch { /* not ready yet */ } + await sleep(intervalMs); + } + throw new Error(`Timeout after ${timeoutMs}ms polling ${url}`); +} + +type LogsResponse = { + status: string; + progress: string; + progress_detail?: string; + ip?: string; + log_total?: number; + log_lines?: Array<{ line: string }>; +}; + +/** Queue a VyOS install, reboot the VM into PXE, wait for completion + SSH. */ +async function installRound(opts: { + mac: string; + hostname: string; + freshConfig?: boolean; +}): Promise { + const body = { + mac: opts.mac, + hostname: opts.hostname, + disk: "/dev/vda", + role: "vanilla", + os: "vyos-rolling", + vyos: { + mgmtInterface: "eth0", + mgmtAddress: "dhcp", + hwIds: { eth0: opts.mac }, + ...(opts.freshConfig ? { freshConfig: true } : {}), + }, + }; + const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/install`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + log(`Install queued (${opts.hostname}): ${JSON.stringify(await res.json())}`); + + await sleep(5_000); + rebootPxeVm(VM_NAME); + await sleep(3_000); + deleteNftablesRejectRules(); + + const finalState = await pollApi( + `http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(opts.mac)}`, + (data) => data.status === "installed" || data.progress === "error", + INSTALL_TIMEOUT_MS, + 10_000, + ); + if (finalState.progress === "error") { + log(`INSTALL FAILED: ${JSON.stringify(finalState.progress_detail ?? finalState, null, 2)}`); + throw new Error(`VyOS install failed for ${opts.hostname}`); + } + const ip = finalState.ip ?? ""; + log(`Install complete (${opts.hostname}). IP: ${ip}`); + + // The driver force-reboots; the VM PXE boots, dispatch says installed -> + // localboot exit -> GRUB -> VyOS. nftables reject rules do not reappear + // (guest reboot, not a libvirt restart), but clearing is harmless. + deleteNftablesRejectRules(); + await waitForSsh(ip, SSH_USER, SSH_TIMEOUT_MS, sshKeyPathGlobal); + return ip; +} + +let sshKeyPathGlobal = ""; + +describe("VyOS provisioning", () => { + let bastionApp: { close: () => Promise }; + let testDir: string; + let vmMac: string; + let vmIp: string; + + beforeAll(async () => { + const { pubKey, keyPath } = findSshKey(); + sshKeyPathGlobal = keyPath; + + log("Setting up PXE test network..."); + ensurePxeNetwork(); + + testDir = join(tmpdir(), `lab-vyos-test-${Date.now()}`); + mkdirSync(join(testDir, "tftp"), { recursive: true }); + mkdirSync(join(testDir, "http"), { recursive: true }); + mkdirSync(join(testDir, "logs"), { recursive: true }); + + log("Starting bastion..."); + const { createApp } = await import("../../src/bastion/src/server.js"); + const { loadConfig } = await import("../../src/bastion/src/config.js"); + const { generateDnsmasqConf, startDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js"); + const { generateDiscoverKickstart } = await import("../../src/bastion/src/services/kickstart-generator.js"); + const { renderBootIpxe } = await import("../../src/bastion/src/templates/boot.ipxe.js"); + const { prepareVyosArtifacts } = await import("../../src/bastion/src/main.js"); + + const config = loadConfig({ + bastionDir: testDir, + httpPort: HTTP_PORT, + iface: "virbr-pxe", + serverIp: BASTION_IP, + network: `${PXE_SUBNET}.0`, + gateway: BASTION_IP, + dhcpMode: "full", + dhcpRangeStart: DHCP_RANGE_START, + dhcpRangeEnd: DHCP_RANGE_END, + domain: "pxe-test.local", + sshKeys: [pubKey], + adminUser: "lab", + }); + + // iPXE binary + const ipxeSrc = "/usr/share/ipxe/ipxe-snponly-x86_64.efi"; + if (!existsSync(ipxeSrc)) { + throw new Error(`iPXE not found: ${ipxeSrc}. Install: sudo dnf install ipxe-bootimgs-x86`); + } + copyFileSync(ipxeSrc, join(config.tftpDir, "ipxe.efi")); + try { symlinkSync(join(config.tftpDir, "ipxe.efi"), join(config.httpDir, "ipxe.efi")); } catch { /* exists */ } + + const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache"; + execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" }); + + // Fedora kernel+initrd for DISCOVERY (OS-neutral, same as pxe test) + const kernel = join(cacheDir, `vmlinuz-${config.fedoraVersion}`); + const initrd = join(cacheDir, `initrd-${config.fedoraVersion}.img`); + if (!existsSync(kernel)) { + log(`Downloading Fedora ${config.fedoraVersion} kernel (discovery)...`); + execSync(`curl -# -L -f -o "${kernel}" "${config.fedoraMirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 300_000 }); + } + if (!existsSync(initrd)) { + log(`Downloading Fedora ${config.fedoraVersion} initrd (discovery)...`); + execSync(`curl -# -L -f -o "${initrd}" "${config.fedoraMirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 300_000 }); + } + copyFileSync(kernel, join(config.httpDir, "vmlinuz")); + copyFileSync(initrd, join(config.httpDir, "initrd.img")); + + // VyOS netboot artifacts — cache the three extracted files across runs + const vyosCache = { + kernel: join(cacheDir, "vyos-vmlinuz"), + initrd: join(cacheDir, "vyos-initrd"), + squashfs: join(cacheDir, "vyos-filesystem.squashfs"), + }; + if (Object.values(vyosCache).every((p) => existsSync(p))) { + log("VyOS netboot artifacts cached"); + copyFileSync(vyosCache.kernel, join(config.httpDir, "vyos-vmlinuz")); + copyFileSync(vyosCache.initrd, join(config.httpDir, "vyos-initrd")); + copyFileSync(vyosCache.squashfs, join(config.httpDir, "vyos-filesystem.squashfs")); + } else { + log("Extracting VyOS artifacts from ISO (downloads ~600MB on first run)..."); + prepareVyosArtifacts(config); + copyFileSync(join(config.httpDir, "vyos-vmlinuz"), vyosCache.kernel); + copyFileSync(join(config.httpDir, "vyos-initrd"), vyosCache.initrd); + copyFileSync(join(config.httpDir, "vyos-filesystem.squashfs"), vyosCache.squashfs); + } + + writeFileSync(join(config.httpDir, "discover.ks"), generateDiscoverKickstart(config)); + writeFileSync(join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort })); + generateDnsmasqConf(config); + + const { app, syslog } = createApp(config); + bastionApp = app; + await app.listen({ port: config.httpPort, host: "0.0.0.0" }); + syslog.start(); + log(`Bastion listening on :${HTTP_PORT}`); + + log("Starting dnsmasq..."); + startDnsmasq(config).catch((err) => { + log(`dnsmasq failed (expected without root): ${err instanceof Error ? err.message : String(err)}`); + }); + await sleep(1000); + + log("Creating PXE VM..."); + // Two decoy NICs ahead of the PXE NIC, on a network with no route to the + // bastion. This reproduces the real VP2440 topology: live-boot scans for + // "the first connected interface", and without BOOTIF it picks a decoy, + // times out on DHCP/fetch, and dies with "Unable to find a live file + // system on the network". A single-NIC VM cannot catch that. + createPxeVm({ + name: VM_NAME, + memory: VM_MEMORY, + vcpus: VM_VCPUS, + diskSize: VM_DISK_GB, + network: PXE_NETWORK_NAME, + decoyNics: 2, + }); + const mac = getVmMac(VM_NAME, PXE_NETWORK_NAME); + if (!mac) throw new Error("Could not determine VM MAC address"); + vmMac = mac; + log(`VM MAC: ${vmMac}`); + + log("Waiting for discovery..."); + type MachinesResponse = { discovered: Record }; + await pollApi( + `http://${BASTION_IP}:${HTTP_PORT}/api/machines`, + (data) => vmMac in data.discovered, + DISCOVERY_TIMEOUT_MS, + ); + log("VM discovered. Running fresh VyOS install (round 1)..."); + + await sleep(15_000); // discovery reboot cycle + vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R1 }); + log("Round 1 (fresh install) complete."); + }, DISCOVERY_TIMEOUT_MS + INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000); + + afterAll(async () => { + log("Cleaning up..."); + if (bastionApp) await bastionApp.close().catch(() => {}); + const { stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js"); + stopDnsmasq(); + destroyPxeVm(VM_NAME); + destroyPxeNetwork(); + if (testDir) rmSync(testDir, { recursive: true, force: true }); + }); + + it("machine is installed with a real IP (WI-1: ready-at parsing)", async () => { + const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`); + const data = (await res.json()) as { installed: Record }; + const machine = data.installed[vmMac]; + expect(machine).toBeDefined(); + expect(machine.ip).toMatch(/^\d+\.\d+\.\d+\.\d+$/); + expect(machine.os).toBe("vyos-rolling"); + }); + + it("install logs were streamed live (WI-2)", async () => { + const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`); + const data = (await res.json()) as LogsResponse; + expect(data.log_total).toBeGreaterThan(0); + const lines = (data.log_lines ?? []).map((l) => l.line).join("\n"); + // Installer transcript lines and driver messages both flow through /api/log + expect(lines).toMatch(/Welcome to VyOS installation|>>> answered|base config:/); + }); + + it("SSH works as the vyos user with the injected key", () => { + const result = sshExec(vmIp, SSH_USER, "whoami", { keyPath: sshKeyPathGlobal }); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe("vyos"); + }); + + it("generated config was adopted (hostname + ssh key)", () => { + const result = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal }); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain(`host-name "${HOSTNAME_R1}"`); + expect(result.stdout).toContain("public-keys"); + }); + + it("boot-order step ran and reported (WI-3)", async () => { + const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`); + const data = (await res.json()) as LogsResponse; + const lines = (data.log_lines ?? []).map((l) => l.line).join("\n"); + expect(lines).toContain("boot order:"); + }); + + it("provisioning metadata persisted to /config (WI-4)", () => { + const result = sshExec(vmIp, SSH_USER, "cat /config/lab-provisioned 2>/dev/null || cat /opt/vyatta/etc/config/lab-provisioned", { keyPath: sshKeyPathGlobal }); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain(`hostname=${HOSTNAME_R1}`); + expect(result.stdout).toContain("role=vanilla"); + expect(result.stdout).toContain(`bastion=http://${BASTION_IP}:${HTTP_PORT}`); + }); + + it("reinstall preserves config and /config data (round 2)", async () => { + // Drop a marker in /config — the installer's previous-installation copy + // must carry it (and the whole old config) into the new image. + // `sync` is REQUIRED: rebootPxeVm uses `virsh destroy` (a hard power-cut), + // so an unsynced write never reaches the disk and the marker vanishes for + // reasons that have nothing to do with the installer. + const marker = sshExec(vmIp, SSH_USER, "echo LAB-MARKER-R2 > /config/lab-marker && sync && cat /config/lab-marker", { keyPath: sshKeyPathGlobal }); + expect(marker.exitCode).toBe(0); + expect(marker.stdout).toContain("LAB-MARKER-R2"); + + // Queue with a DIFFERENT hostname: with preserve semantics the previous + // config must win, so the hostname must NOT change. + vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R2 }); + + // Assert the config carry-forward first — it is the primary preservation + // signal and does not depend on the marker mechanism above. + const cfg = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal }); + expect(cfg.stdout).toContain(`host-name "${HOSTNAME_R1}"`); // old config carried + expect(cfg.stdout).not.toContain(`host-name "${HOSTNAME_R2}"`); + + const markerAfter = sshExec(vmIp, SSH_USER, "cat /config/lab-marker", { keyPath: sshKeyPathGlobal }); + expect(markerAfter.exitCode).toBe(0); + expect(markerAfter.stdout).toContain("LAB-MARKER-R2"); + }, INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 60_000); + + it("freshConfig makes the generated config win, data still kept (round 3)", async () => { + // Re-assert the marker is on disk and synced before the next power-cut. + const pre = sshExec(vmIp, SSH_USER, "sync && cat /config/lab-marker", { keyPath: sshKeyPathGlobal }); + expect(pre.stdout).toContain("LAB-MARKER-R2"); + + vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R3, freshConfig: true }); + + const cfg = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal }); + expect(cfg.stdout).toContain(`host-name "${HOSTNAME_R3}"`); // generated config won + + // The marker file (non-config data under /config) still survives — + // freshConfig replaces only config.boot, not the carried data. + const markerAfter = sshExec(vmIp, SSH_USER, "cat /config/lab-marker", { keyPath: sshKeyPathGlobal }); + expect(markerAfter.exitCode).toBe(0); + expect(markerAfter.stdout).toContain("LAB-MARKER-R2"); + }, INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 60_000); +}); diff --git a/labsim/.gitignore b/labsim/.gitignore new file mode 100644 index 0000000..73e9e69 --- /dev/null +++ b/labsim/.gitignore @@ -0,0 +1,4 @@ +# runtime artifacts, not source +*.log +labsim_matrix_lib.py +__pycache__/ diff --git a/labsim/README.md b/labsim/README.md new file mode 100644 index 0000000..615c93d --- /dev/null +++ b/labsim/README.md @@ -0,0 +1,104 @@ +# labsim — libvirt replica of the lab network + +A throwaway copy of the production VLAN topology for testing routing, firewall +rules and failover **without touching the real network**. Same VLAN IDs and +roles as UniFi, deliberately different IP ranges so nothing can be confused for +production. + +## Topology + +Each VLAN is its own isolated libvirt network with one tiny Alpine VM on it. + +| VLAN | Name | Sim subnet | VM address | Mirrors production | +|-----:|------|------------|-----------|--------------------| +| 1 | management | 172.31.1.0/24 | 172.31.1.10 | 192.168.1.0/24 | +| 2 | k8s | 172.31.2.0/24 | 172.31.2.10 | 192.168.8.0/23 | +| 3 | kvm | 172.31.3.0/24 | 172.31.3.10 | 192.168.3.0/24 | +| 9 | private | 172.31.9.0/24 | 172.31.9.10 | 10.0.9.0/23 | +| 10 | lot | 172.31.10.0/24 | 172.31.10.10 | 10.0.0.0/23 | +| 200 | roomates | 172.31.200.0/24 | 172.31.200.10 | 192.168.2.0/24 | + +The sim subnet always encodes the VLAN id: `172.31..0/24`. + +Address plan, identical on every VLAN: + +| Address | Role | +|---------|------| +| `.1` | gateway under test — a router VM you add (not created by default) | +| `.2` | host bridge — how you reach the VMs from this workstation | +| `.10` | the VLAN's micro VM | +| `.254` | reserved for a VRRP VIP, mirroring production | + +The host sits at `.2` purely so you can SSH in. It is deliberately **not** the +VMs' default route — that is `.1` — so inter-VLAN tests fail loudly when no +router is present instead of being silently served by the host's own routing +table. libvirt also installs reject rules that stop these networks forwarding +to each other, so traffic between VLANs only works once a router VM bridges +them. + +## Usage + +```bash +./labsim-up.sh # bring up every VLAN (idempotent) +./labsim-up.sh 2 3 # only VLANs 2 and 3 +./labsim-down.sh # destroy VMs + networks, keep the base image +./labsim-down.sh --purge # also delete the downloaded Alpine image +``` + +Each VM: 256 MB, 1 vCPU, a copy-on-write overlay on one shared 176 MB Alpine +image (so six VMs cost a few MB of disk, not 1 GB). + +## Access + +```bash +ssh alpine@172.31.2.10 # normal user (password: labsim) +ssh root@172.31.2.10 # privileged — this image has no sudo +curl http://172.31.2.10/ # hello-world page naming the VLAN +``` + +Console, when the network is the thing that is broken: + +```bash +sudo virsh console labsim-2-k8s # root / labsim +``` + +## Watching it + +```bash +./labsim-matrix.py --watch 2 # terminal grid, changed cells highlighted +./monitoring-up.sh # topology page + Prometheus + Grafana +``` + +- **http://localhost:9101/** — live mesh: a node per VLAN, the router in the + middle, one line per pair coloured green/red with the ICMP RTT on it. Hover a + line for per-direction detail. Refreshes every 5s. This is the one to watch + while changing firewall rules. +- **http://localhost:3000/d/labsim-matrix** — Grafana (anonymous, no login) for + *history*: when did a path flip, and how has latency moved. +- **http://localhost:9101/metrics** — `labsim_reachable{src,dst,proto}` and + `labsim_rtt_ms{src,dst}`. + +## Notes for whoever extends this + +Things that cost time the first time round, all verified on this image: + +- **No `sudo`.** Alpine ships `doas`; cloud-init's `sudo:` directive is inert + here. Use `root@` for privileged work. +- **cloud-init leaves users locked** (`!*` in `/etc/shadow`) unless + `lock_passwd: false`, and sshd then refuses key auth for that user. +- **One failing `runcmd` aborts every command after it.** Each entry is + `|| true` for that reason. +- **busybox here has no `httpd` applet**, and the VMs have no internet to + `apk add` one — so the hello-world server is `python3 -m http.server` + (python3 is already present because cloud-init depends on it). +- **`start-stop-daemon --exec /usr/bin/python3` matches cloud-init's own + python3** at boot and refuses to start anything. +- **busybox `pgrep -f PATTERN` matches its own argv**, so a "skip if already + running" guard always fires. Verified: `guard_exit=0` with nothing listening. + +## Not modelled (yet) + +VLANs are separate L2 segments rather than one 802.1Q trunk, so this exercises +inter-VLAN routing but not a `bond0.` trunk config specifically. A router +VM would attach one NIC per VLAN. Adding a tagged-trunk variant is the obvious +next step if the bond/vif config itself needs testing. diff --git a/labsim/labsim-down.sh b/labsim/labsim-down.sh new file mode 100755 index 0000000..bb4f016 --- /dev/null +++ b/labsim/labsim-down.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Tear down the lab network simulation. +# +# By default this destroys VMs and networks but KEEPS the downloaded base +# image, so the next bring-up is fast. Pass --purge to remove that too. +# +# Usage: ./labsim-down.sh [--purge] [vlan-id ...] +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/lib.sh" +source "$SCRIPT_DIR/ovs.sh" + +PURGE=false +ARGS=() +for a in "$@"; do + case "$a" in + --purge) PURGE=true ;; + *) ARGS+=("$a") ;; + esac +done + +selected_vlans "${ARGS[@]+"${ARGS[@]}"}" + +for entry in "${SELECTED[@]}"; do + IFS=: read -r vid name prefix _r <<<"$entry" + vm="$(vm_name "$vid" "$name")" + + if virsh_q dominfo "$vm" >/dev/null 2>&1; then + log "destroying VM $vm" + virsh_q destroy "$vm" >/dev/null 2>&1 || true + virsh_q undefine "$vm" --nvram >/dev/null 2>&1 || virsh_q undefine "$vm" >/dev/null 2>&1 || true + fi + sudo rm -f "$IMG_DIR/${vm}.qcow2" "$IMG_DIR/${vm}-seed.iso" + +done + +# Legacy per-VLAN Linux-bridge networks from before the OVS migration. If +# these survive they keep a duplicate .2/24 on a dead bridge, and the +# kernel may prefer that route over the OVS host leg — which looks exactly +# like "the VM is unreachable" while ping -I hostvN works fine. +for entry in "${SELECTED[@]}"; do + IFS=: read -r vid _n _p _r <<<"$entry" + legacy="labsim-vlan${vid}" + if virsh_q net-info "$legacy" >/dev/null 2>&1; then + log "removing legacy network $legacy" + virsh_q net-destroy "$legacy" >/dev/null 2>&1 || true + virsh_q net-undefine "$legacy" >/dev/null 2>&1 || true + fi +done + +log "removing OVS fabric" +ovs_down + +if [ "$PURGE" = true ]; then + log "purging base image $BASE_IMAGE" + sudo rm -f "$BASE_IMAGE" + sudo rmdir "$IMG_DIR" 2>/dev/null || true +fi + +log "environment is DOWN" diff --git a/labsim/labsim-exporter.py b/labsim/labsim-exporter.py new file mode 100755 index 0000000..f82ab29 --- /dev/null +++ b/labsim/labsim-exporter.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Prometheus exporter for the labsim connectivity matrix. + +Runs the same sweep as labsim-matrix.py on an interval and exposes it as +metrics, so Grafana can show the mesh as a heatmap and — more usefully — a +history of exactly when a cell flipped after a firewall change. + + labsim_reachable{src,dst,proto} 1 = reachable, 0 = blocked + labsim_sweep_seconds how long the last sweep took + labsim_sweep_total sweeps completed since start + labsim_up 1 while the exporter is alive + +Deliberately stdlib-only (http.server + threads): this runs on the workstation +next to libvirt, and adding a dependency to watch a lab network is silly. + + ./labsim-exporter.py --port 9101 --interval 15 +""" +from __future__ import annotations + +import argparse +import http.server +import json +import os +import threading +import time + +import labsim_matrix_lib as m # thin import shim, see below + + +class Collector: + def __init__(self, interval: int, timeout: int) -> None: + self.interval = interval + self.timeout = timeout + self.vlans = m.load_vlans() + self.lock = threading.Lock() + self.results: dict = {} + self.duration = 0.0 + self.sweeps = 0 + + def loop(self) -> None: + while True: + started = time.time() + try: + results = m.sweep(self.vlans, self.timeout) + with self.lock: + self.results = results + self.duration = time.time() - started + self.sweeps += 1 + except Exception: # noqa: BLE001 - never let the loop die + pass + time.sleep(max(1.0, self.interval - (time.time() - started))) + + def snapshot(self) -> dict: + """Everything the topology page needs, in one JSON payload.""" + with self.lock: + results, duration = dict(self.results), self.duration + reach = total = 0 + for data in results.values(): + if "__error__" in data: + continue + for protos in data.values(): + for proto, ok in protos.items(): + if proto == "rtt_ms": + continue + total += 1 + if ok: + reach += 1 + return {"vlans": self.vlans, "results": results, "reachable": reach, + "total": total, "sweep_seconds": duration} + + def render(self) -> str: + with self.lock: + results, duration, sweeps = dict(self.results), self.duration, self.sweeps + + out = [ + "# HELP labsim_reachable 1 if dst is reachable from src over proto", + "# TYPE labsim_reachable gauge", + ] + rtts = [] + for src, data in results.items(): + if "__error__" in data: + continue + for dst, protos in data.items(): + for proto, ok in protos.items(): + if proto == "rtt_ms": + if isinstance(ok, (int, float)): + rtts.append((src, dst, ok)) + continue + out.append( + f'labsim_reachable{{src="{src}",dst="{dst}",proto="{proto}"}} {1 if ok else 0}') + out += ["# HELP labsim_rtt_ms ICMP round-trip time", + "# TYPE labsim_rtt_ms gauge"] + for src, dst, val in rtts: + out.append(f'labsim_rtt_ms{{src="{src}",dst="{dst}"}} {val}') + out += [ + "# HELP labsim_sweep_seconds duration of the last sweep", + "# TYPE labsim_sweep_seconds gauge", + f"labsim_sweep_seconds {duration:.3f}", + "# HELP labsim_sweep_total sweeps completed", + "# TYPE labsim_sweep_total counter", + f"labsim_sweep_total {sweeps}", + "# HELP labsim_up exporter liveness", + "# TYPE labsim_up gauge", + "labsim_up 1", + ] + return "\n".join(out) + "\n" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=9101) + ap.add_argument("--interval", type=int, default=15) + ap.add_argument("--timeout", type=int, default=30) + args = ap.parse_args() + + collector = Collector(args.interval, args.timeout) + threading.Thread(target=collector.loop, daemon=True).start() + + here = os.path.dirname(os.path.abspath(__file__)) + + class Handler(http.server.BaseHTTPRequestHandler): + def _send(self, body: bytes, ctype: str) -> None: + self.send_response(200) + self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 - stdlib API + path = self.path.split("?")[0].rstrip("/") + if path in ("", "/topology"): + # Live topology view — the thing you actually watch. + try: + with open(os.path.join(here, "topology.html"), "rb") as fh: + self._send(fh.read(), "text/html; charset=utf-8") + except OSError: + self.send_error(500, "topology.html missing") + elif path == "/api/matrix": + self._send(json.dumps(collector.snapshot()).encode(), "application/json") + elif path == "/metrics": + self._send(collector.render().encode(), "text/plain; version=0.0.4") + else: + self.send_error(404) + + def log_message(self, *_args) -> None: # keep the console quiet + return + + srv = http.server.ThreadingHTTPServer(("0.0.0.0", args.port), Handler) + print(f"labsim topology http://localhost:{args.port}/") + print(f"labsim metrics http://localhost:{args.port}/metrics (sweep every {args.interval}s)") + srv.serve_forever() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/labsim/labsim-matrix.py b/labsim/labsim-matrix.py new file mode 100755 index 0000000..425624f --- /dev/null +++ b/labsim/labsim-matrix.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Full-mesh connectivity matrix for the labsim VLANs. + +Probes every VLAN VM from every other VLAN VM (ICMP + TCP/22 + TCP/80) and +prints a grid. Use --watch to keep it live: cells that changed since the last +sweep are highlighted, so adding or removing a VyOS firewall rule shows up +within one refresh. + +Deliberately dependency-free on the guests: the probe runs with python3, which +is already installed there (cloud-init needs it), so nothing has to be +installed on VMs that have no internet. + + ./labsim-matrix.py # one sweep + ./labsim-matrix.py --watch # live, refresh every 5s + ./labsim-matrix.py --watch 2 # live, every 2s + ./labsim-matrix.py --proto icmp # single protocol + ./labsim-matrix.py --json # machine-readable +""" +from __future__ import annotations + +import argparse +import concurrent.futures +import json +import os +import subprocess +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +CONF = os.path.join(HERE, "vlans.conf") + +GREEN, RED, GREY, YELLOW, BOLD, RESET = ( + "\033[0;32m", "\033[0;31m", "\033[0;90m", "\033[1;33m", "\033[1m", "\033[0m") + +PROTOS = ("icmp", "tcp22", "tcp80") + +# Runs ON the guest. Keep it stdlib-only and quick — a hung probe delays the +# whole sweep, so every check is hard-bounded by a timeout. +PROBE = r''' +import json, re, socket, subprocess, sys +targets = json.load(sys.stdin) +out = {} +for name, ip in targets.items(): + res = {} + try: + p = subprocess.run(["ping", "-c", "1", "-W", "1", ip], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=4) + res["icmp"] = p.returncode == 0 + # RTT as well as pass/fail: a path that is up but slow is a different + # problem from one that is down, and the grid alone cannot show it. + res["rtt_ms"] = None + if res["icmp"]: + m = re.search(r"time[=<]\s*([0-9.]+)\s*ms", p.stdout.decode("utf-8", "replace")) + if m: + res["rtt_ms"] = float(m.group(1)) + except Exception: + res["icmp"] = False + res["rtt_ms"] = None + for port in (22, 80): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(1.5) + try: + s.connect((ip, port)); res["tcp%d" % port] = True + except Exception: + res["tcp%d" % port] = False + finally: + try: s.close() + except Exception: pass + out[name] = res +print(json.dumps(out)) +''' + + +def load_vlans() -> list[dict]: + vlans = [] + with open(CONF) as fh: + for line in fh: + line = line.strip() + if not line or line.startswith("#"): + continue + vid, name, prefix, real = line.split(":", 3) + vlans.append({"vid": vid, "name": name, "ip": f"{prefix}.10", + "label": f"{vid}:{name}", "real": real}) + return vlans + + +def probe_from(src: dict, targets: list[dict], timeout: int) -> tuple[str, dict]: + """SSH once into src and probe every target from there.""" + payload = json.dumps({t["label"]: t["ip"] for t in targets if t["label"] != src["label"]}) + cmd = [ + "ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", + "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", "-o", "LogLevel=ERROR", + f"alpine@{src['ip']}", "python3", "-", + ] + try: + # The probe script goes on stdin, the target list follows it — the guest + # reads the script from argv-less stdin, so send both in one stream. + proc = subprocess.run( + cmd, input=PROBE.replace("json.load(sys.stdin)", f"json.loads({payload!r})"), + capture_output=True, text=True, timeout=timeout) + if proc.returncode != 0: + return src["label"], {"__error__": (proc.stderr or "ssh failed").strip()[:60]} + return src["label"], json.loads(proc.stdout) + except subprocess.TimeoutExpired: + return src["label"], {"__error__": "probe timed out"} + except Exception as exc: # noqa: BLE001 - report, never crash the sweep + return src["label"], {"__error__": f"{type(exc).__name__}: {exc}"[:60]} + + +def sweep(vlans: list[dict], timeout: int) -> dict: + results: dict = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=len(vlans)) as pool: + futures = [pool.submit(probe_from, v, vlans, timeout) for v in vlans] + for fut in concurrent.futures.as_completed(futures): + label, data = fut.result() + results[label] = data + return results + + +def cell(ok: bool | None, changed: bool) -> str: + if ok is None: + return f"{GREY} · {RESET}" + mark = "ok " if ok else "-- " + colour = GREEN if ok else RED + if changed: + return f"{YELLOW}{BOLD}{'OK*' if ok else 'XX*':<4}{RESET}" + return f"{colour}{mark}{RESET}" + + +def render(vlans: list[dict], results: dict, prev: dict | None, protos: tuple[str, ...]) -> None: + labels = [v["label"] for v in vlans] + width = max(len(x) for x in labels) + 2 + + for proto in protos: + print(f"\n{BOLD}{proto.upper()}{RESET} (rows = source, columns = destination)") + header = " " * width + "".join(f"{lbl:<{width}}" for lbl in labels) + print(f"{GREY}{header}{RESET}") + + for src in vlans: + row = f"{src['label']:<{width}}" + data = results.get(src["label"], {}) + if "__error__" in data: + print(row + f"{RED}{data['__error__']}{RESET}") + continue + for dst in vlans: + if dst["label"] == src["label"]: + row += f"{GREY}{'·':<{width}}{RESET}" + continue + ok = data.get(dst["label"], {}).get(proto) + was = (prev or {}).get(src["label"], {}).get(dst["label"], {}).get(proto) + changed = prev is not None and was is not None and was != ok + txt = cell(ok, changed) + row += txt + " " * (width - 4) + print(row) + + reach = sum(1 for s in results.values() if "__error__" not in s + for d in s.values() for p in protos if d.get(p) is True) + total = sum(1 for s in results.values() if "__error__" not in s + for _d in s.values() for _p in protos) + print(f"\n reachable: {reach}/{total} " + f"{GREEN}ok{RESET}=allowed {RED}--{RESET}=blocked/no route " + f"{YELLOW}*{RESET}=changed since last sweep") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--watch", nargs="?", const=5, type=int, metavar="SECONDS", + help="refresh continuously (default every 5s)") + ap.add_argument("--proto", choices=PROTOS, help="only this protocol") + ap.add_argument("--json", action="store_true", help="emit raw JSON and exit") + ap.add_argument("--timeout", type=int, default=30, help="per-host probe timeout") + args = ap.parse_args() + + vlans = load_vlans() + protos = (args.proto,) if args.proto else PROTOS + + if args.json: + print(json.dumps(sweep(vlans, args.timeout), indent=2)) + return 0 + + prev = None + while True: + started = time.time() + results = sweep(vlans, args.timeout) + if args.watch: + os.system("clear") + print(f"{BOLD}labsim connectivity matrix{RESET} " + f"{time.strftime('%H:%M:%S')} (refresh {args.watch}s, Ctrl-C to stop)") + render(vlans, results, prev, protos) + if not args.watch: + return 0 + prev = results + time.sleep(max(0.0, args.watch - (time.time() - started))) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except KeyboardInterrupt: + print() + sys.exit(130) diff --git a/labsim/labsim-up.sh b/labsim/labsim-up.sh new file mode 100755 index 0000000..eb3133d --- /dev/null +++ b/labsim/labsim-up.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Bring up the lab network simulation: one isolated libvirt network per VLAN, +# each with a single tiny Alpine VM offering SSH + a hello-world HTTP page. +# +# Idempotent: re-running only creates what is missing. Safe to run repeatedly. +# +# Usage: ./labsim-up.sh [vlan-id ...] (default: every VLAN in vlans.conf) +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/lib.sh" +source "$SCRIPT_DIR/ovs.sh" + +require_tools +[ -f "$BASE_IMAGE" ] || die "base image missing: $BASE_IMAGE (see README)" + +SSH_PUB="$(find_ssh_pubkey)" +log "Using SSH key: ${SSH_PUB%% *} ...${SSH_PUB##* }" + +selected_vlans "$@" + +# --- switch fabric ------------------------------------------------------ +log "bringing up OVS fabric ($OVS_BR) with host legs per VLAN" +ovs_up + +# --- VMs ------------------------------------------------------------------ +for entry in "${SELECTED[@]}"; do + IFS=: read -r vid name prefix real <<<"$entry" + vm="$(vm_name "$vid" "$name")" + ip="${prefix}.10" + + if virsh_q dominfo "$vm" >/dev/null 2>&1; then + state="$(virsh_q domstate "$vm" 2>/dev/null | head -1 | tr -d '\n')" + if [ "$state" = "running" ]; then + log "VM $vm already running ($ip)" + continue + fi + log "VM $vm exists but is $state — starting" + virsh_q start "$vm" >/dev/null + continue + fi + + log "creating VM $vm ($ip on vlan $vid/$name)" + + disk="$IMG_DIR/${vm}.qcow2" + seed="$IMG_DIR/${vm}-seed.iso" + + # Copy-on-write overlay: each VM costs a few MB, not 176. + sudo qemu-img create -q -f qcow2 -F qcow2 -b "$BASE_IMAGE" "$disk" "$VM_DISK" >/dev/null + + build_seed "$seed" "$vm" "$vid" "$name" "$prefix" "$ip" "$real" "$SSH_PUB" + + sudo virt-install \ + --connect "$LIBVIRT_URI" \ + --name "$vm" \ + --memory "$VM_MEM" --vcpus "$VM_CPUS" \ + --disk "path=$disk,format=qcow2,bus=virtio" \ + --disk "path=$seed,device=cdrom,readonly=on" \ + --network "network=$OVS_NET,portgroup=vlan${vid},model=virtio" \ + --os-variant alpinelinux3.18 \ + --graphics none --noautoconsole --import >/dev/null +done + +echo +log "waiting for VMs to answer on SSH + HTTP..." +wait_ready +echo +status_table +echo +log "environment is UP. Tear down with: $SCRIPT_DIR/labsim-down.sh" diff --git a/labsim/lib.sh b/labsim/lib.sh new file mode 100644 index 0000000..9f54c94 --- /dev/null +++ b/labsim/lib.sh @@ -0,0 +1,216 @@ +#!/bin/bash +# Shared helpers for the lab network simulation. +# shellcheck disable=SC2034 + +LIBVIRT_URI="${LIBVIRT_URI:-qemu:///system}" +IMG_DIR="${IMG_DIR:-/var/lib/libvirt/images/labsim}" +BASE_IMAGE="${BASE_IMAGE:-$IMG_DIR/alpine-base.qcow2}" +ALPINE_URL="${ALPINE_URL:-https://dl-cdn.alpinelinux.org/alpine/latest-stable/releases/cloud/generic_alpine-3.24.1-x86_64-bios-cloudinit-r0.qcow2}" + +VM_MEM="${VM_MEM:-256}" # MB — Alpine is happy here +VM_CPUS="${VM_CPUS:-1}" +VM_DISK="${VM_DISK:-1G}" +PREFIX="labsim" + +CONF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/vlans.conf" + +log() { printf '\033[0;36m[labsim]\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33m[labsim]\033[0m %s\n' "$*" >&2; } +die() { printf '\033[0;31m[labsim]\033[0m %s\n' "$*" >&2; exit 1; } + +virsh_q() { sudo virsh --connect "$LIBVIRT_URI" "$@"; } + +net_name() { echo "${PREFIX}-vlan$1"; } +vm_name() { echo "${PREFIX}-$1-$2"; } # labsim-2-k8s +# Linux bridge names are capped at 15 chars — keep it short and unique. +br_name() { echo "vbr-ls$1"; } + +require_tools() { + for t in virsh virt-install qemu-img genisoimage; do + command -v "$t" >/dev/null 2>&1 || die "missing required tool: $t" + done + sudo -n true 2>/dev/null || warn "sudo may prompt for a password" +} + +find_ssh_pubkey() { + local home="${SUDO_USER:+/home/$SUDO_USER}" + home="${home:-$HOME}" + for n in id_ed25519 id_ecdsa id_rsa; do + [ -f "$home/.ssh/$n.pub" ] && { cat "$home/.ssh/$n.pub"; return; } + done + die "no SSH public key found in $home/.ssh" +} + +# Populate SELECTED[] from argv (VLAN ids) or the whole config. +selected_vlans() { + SELECTED=() + local want=("$@") + while IFS= read -r line; do + [[ "$line" =~ ^[[:space:]]*# ]] && continue + [[ -z "${line// }" ]] && continue + local vid="${line%%:*}" + if [ ${#want[@]} -eq 0 ]; then + SELECTED+=("$line") + else + for w in "${want[@]}"; do [ "$w" = "$vid" ] && SELECTED+=("$line"); done + fi + done < "$CONF" + [ ${#SELECTED[@]} -gt 0 ] || die "no VLANs selected (checked $CONF)" +} + +# cloud-init NoCloud seed: static addressing + SSH key + hello-world HTTP. +build_seed() { + local iso="$1" vm="$2" vid="$3" name="$4" prefix="$5" ip="$6" real="$7" pubkey="$8" + local tmp; tmp="$(mktemp -d)" + + cat > "$tmp/meta-data" < "$tmp/network-config" < "$tmp/user-data" < +

labsim vlan $vid — $name

+

host: $vm

+

address: $ip/24

+

gateway under test: ${prefix}.1

+

mirrors production: $real

+ + - path: /etc/local.d/labsim-http.start + permissions: '0755' + content: | + #!/bin/sh + # This image's busybox has no httpd applet ("applet not found"), and the + # VMs are isolated so apk cannot fetch one. python3 is already present + # (cloud-init depends on it), so serve with http.server — no packages, + # no internet. + # + # Two traps already hit here, both silent: + # - start-stop-daemon --exec /usr/bin/python3 matches cloud-init's OWN + # python3 at boot, says "already running", starts nothing. + # - busybox pgrep -f PATTERN matches its own argv, so a + # "skip if running" guard always fires (verified: guard_exit=0 with + # nothing listening). + # So: no guard, no start-stop-daemon. Binding twice is harmless — the + # second just fails to bind. + nohup /usr/bin/python3 -m http.server 80 --directory /var/www \\ + >/var/log/labsim-http.log 2>&1 & +runcmd: + # cloud-init's network-config (v1, above) already applies the address, so do + # NOT restart networking here — it fails, and one failing runcmd aborts every + # command after it, which is what silently left httpd unstarted. Each command + # is || true for the same reason. + - [ sh, -c, "rc-update add sshd default || true" ] + - [ sh, -c, "rc-update add local default || true" ] + - [ sh, -c, "/etc/local.d/labsim-http.start || true" ] +EOF + + # Validate before building the ISO. The heredoc above is intentionally + # unquoted (it interpolates $ip/$prefix), which means backticks or $( ) in + # ANY line — including comments — get executed by the host shell and their + # output silently corrupts the YAML. Cheap check, expensive bug. + python3 -c "import yaml,sys; yaml.safe_load(open(sys.argv[1]))" "$tmp/user-data" \ + || die "generated user-data is not valid YAML (backticks or \$( ) in build_seed?): $tmp/user-data" + + sudo genisoimage -quiet -output "$iso" -volid cidata -joliet -rock \ + "$tmp/user-data" "$tmp/meta-data" "$tmp/network-config" + rm -rf "$tmp" +} + +ssh_to() { + local ip="$1"; shift + timeout 12 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=5 -o BatchMode=yes -o LogLevel=ERROR \ + "alpine@$ip" "$@" 2>/dev/null +} + +wait_ready() { + local deadline=$((SECONDS + 240)) pending=1 + while [ $SECONDS -lt $deadline ]; do + pending=0 + for entry in "${SELECTED[@]}"; do + IFS=: read -r vid _n prefix _r <<<"$entry" + # Wait for BOTH: sshd is up well before cloud-init's runcmd starts the + # web server, so checking SSH alone reports "ready" then shows HTTP FAIL. + ssh_to "${prefix}.10" true >/dev/null 2>&1 \ + && curl -sS -o /dev/null --max-time 4 "http://${prefix}.10/" 2>/dev/null \ + || pending=$((pending + 1)) + done + [ $pending -eq 0 ] && { log "all ${#SELECTED[@]} VMs reachable"; return 0; } + sleep 5 + done + warn "$pending VM(s) still not answering SSH after 240s — see status below" + return 0 +} + +status_table() { + printf ' %-18s %-6s %-16s %-9s %-7s %s\n' VM VLAN ADDRESS STATE SSH HTTP + printf ' %-18s %-6s %-16s %-9s %-7s %s\n' ------------------ ------ ---------------- --------- ------- ---- + for entry in "${SELECTED[@]}"; do + IFS=: read -r vid name prefix _r <<<"$entry" + local vm ip state ssh http + vm="$(vm_name "$vid" "$name")"; ip="${prefix}.10" + state="$(virsh_q domstate "$vm" 2>/dev/null | head -1 | tr -d '\n')" + [ -z "$state" ] && state="absent" + ssh_to "$ip" true >/dev/null 2>&1 && ssh=ok || ssh=FAIL + if curl -sS -o /dev/null --max-time 5 "http://$ip/" 2>/dev/null; then http=ok; else http=FAIL; fi + printf ' %-18s %-6s %-16s %-9s %-7s %s\n' "$vm" "$vid" "$ip" "$state" "$ssh" "$http" + done +} diff --git a/labsim/monitoring-up.sh b/labsim/monitoring-up.sh new file mode 100755 index 0000000..273cda2 --- /dev/null +++ b/labsim/monitoring-up.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Prometheus + Grafana for the labsim connectivity matrix. +# +# Grafana runs with anonymous auth as Admin — NO LOGIN. That is deliberate for +# a throwaway lab on localhost; do not copy this into anything reachable. +# +# ./monitoring-up.sh start exporter + prometheus + grafana +# ./monitoring-up.sh --down stop and remove them +# +# Grafana: http://localhost:3000 (dashboard "labsim — VLAN connectivity matrix") +# Prometheus: http://localhost:9090 +# Exporter: http://localhost:9101/metrics +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/lib.sh" + +GRAFANA_PORT="${GRAFANA_PORT:-3000}" +PROM_PORT="${PROM_PORT:-9090}" +EXPORTER_PORT="${EXPORTER_PORT:-9101}" +NET="labsim-mon" + +if [ "${1:-}" = "--down" ]; then + pkill -f "labsim-exporter.py" 2>/dev/null || true + podman rm -f labsim-grafana labsim-prometheus >/dev/null 2>&1 || true + podman network rm -f "$NET" >/dev/null 2>&1 || true + log "monitoring stopped" + exit 0 +fi + +command -v podman >/dev/null 2>&1 || die "podman not installed" + +# --- exporter (on the host: it needs SSH access to the VMs) ---------------- +if pgrep -f "labsim-exporter.py" >/dev/null 2>&1; then + log "exporter already running on :$EXPORTER_PORT" +else + log "starting exporter on :$EXPORTER_PORT" + nohup "$SCRIPT_DIR/labsim-exporter.py" --port "$EXPORTER_PORT" --interval 15 \ + > /tmp/labsim-exporter.log 2>&1 & + sleep 3 +fi +curl -sS --max-time 5 "http://127.0.0.1:${EXPORTER_PORT}/metrics" >/dev/null \ + || die "exporter not answering on :$EXPORTER_PORT (see /tmp/labsim-exporter.log)" + +podman network exists "$NET" 2>/dev/null || podman network create "$NET" >/dev/null + +# --- prometheus ----------------------------------------------------------- +podman rm -f labsim-prometheus >/dev/null 2>&1 || true +log "starting prometheus on :$PROM_PORT" +podman run -d --name labsim-prometheus --network "$NET" \ + -p "${PROM_PORT}:9090" \ + -v "$SCRIPT_DIR/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z" \ + --add-host "host.containers.internal:host-gateway" \ + docker.io/prom/prometheus:latest >/dev/null + +# --- grafana (anonymous, no login) ---------------------------------------- +podman rm -f labsim-grafana >/dev/null 2>&1 || true +log "starting grafana on :$GRAFANA_PORT (anonymous auth — no password)" +podman run -d --name labsim-grafana --network "$NET" \ + -p "${GRAFANA_PORT}:3000" \ + -e GF_AUTH_ANONYMOUS_ENABLED=true \ + -e GF_AUTH_ANONYMOUS_ORG_ROLE=Admin \ + -e GF_AUTH_DISABLE_LOGIN_FORM=true \ + -e GF_AUTH_BASIC_ENABLED=false \ + -e GF_SECURITY_ALLOW_EMBEDDING=true \ + -e GF_USERS_DEFAULT_THEME=dark \ + -v "$SCRIPT_DIR/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro,Z" \ + docker.io/grafana/grafana:latest >/dev/null + +log "waiting for grafana..." +for _ in $(seq 1 40); do + if curl -sS --max-time 3 "http://127.0.0.1:${GRAFANA_PORT}/api/health" >/dev/null 2>&1; then + break + fi + sleep 3 +done + +echo +log "Topology: http://localhost:${EXPORTER_PORT}/ <- live mesh, red/green + RTT" +log "Grafana: http://localhost:${GRAFANA_PORT}/d/labsim-matrix (no login, history)" +log "Prometheus: http://localhost:${PROM_PORT}" +log "Exporter: http://localhost:${EXPORTER_PORT}/metrics" diff --git a/labsim/monitoring/grafana/provisioning/dashboards/dashboards.yml b/labsim/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..cafb391 --- /dev/null +++ b/labsim/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,9 @@ +apiVersion: 1 +providers: + - name: labsim + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + options: + path: /etc/grafana/provisioning/dashboards diff --git a/labsim/monitoring/grafana/provisioning/dashboards/labsim.json b/labsim/monitoring/grafana/provisioning/dashboards/labsim.json new file mode 100644 index 0000000..44e7a84 --- /dev/null +++ b/labsim/monitoring/grafana/provisioning/dashboards/labsim.json @@ -0,0 +1,59 @@ +{ + "uid": "labsim-matrix", + "title": "labsim — VLAN connectivity matrix", + "tags": ["labsim"], + "timezone": "browser", + "refresh": "10s", + "time": { "from": "now-30m", "to": "now" }, + "panels": [ + { + "type": "stat", + "title": "Reachable paths", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "targets": [ { "expr": "sum(labsim_reachable)", "refId": "A" } ], + "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", + "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 90 } ] } } } + }, + { + "type": "stat", + "title": "Blocked paths", + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }, + "targets": [ { "expr": "count(labsim_reachable == 0) or vector(0)", "refId": "A" } ], + "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", + "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 } ] } } } + }, + { + "type": "stat", + "title": "Sweep duration (s)", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, + "targets": [ { "expr": "labsim_sweep_seconds", "refId": "A" } ] + }, + { + "type": "stat", + "title": "Sweeps", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, + "targets": [ { "expr": "labsim_sweep_total", "refId": "A" } ] + }, + { + "type": "heatmap", + "title": "ICMP matrix (src → dst) — green = reachable", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 }, + "targets": [ { "expr": "labsim_reachable{proto=\"icmp\"}", + "legendFormat": "{{src}} → {{dst}}", "refId": "A" } ] + }, + { + "type": "state-timeline", + "title": "Every path over time — a firewall change shows up here immediately", + "gridPos": { "h": 12, "w": 24, "x": 0, "y": 14 }, + "targets": [ { "expr": "labsim_reachable", + "legendFormat": "{{proto}} {{src}} → {{dst}}", "refId": "A" } ], + "fieldConfig": { "defaults": { + "mappings": [ { "type": "value", "options": { + "0": { "text": "blocked", "color": "red", "index": 0 }, + "1": { "text": "ok", "color": "green", "index": 1 } } } ] } }, + "options": { "mergeValues": true, "showValue": "never" } + } + ], + "schemaVersion": 39, + "version": 1 +} diff --git a/labsim/monitoring/grafana/provisioning/datasources/prometheus.yml b/labsim/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..ca7795c --- /dev/null +++ b/labsim/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,7 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://labsim-prometheus:9090 + isDefault: true diff --git a/labsim/monitoring/prometheus.yml b/labsim/monitoring/prometheus.yml new file mode 100644 index 0000000..e602ca3 --- /dev/null +++ b/labsim/monitoring/prometheus.yml @@ -0,0 +1,9 @@ +# Scrapes the labsim connectivity exporter running on the host. +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: labsim + static_configs: + - targets: ['host.containers.internal:9101'] diff --git a/labsim/ovs.sh b/labsim/ovs.sh new file mode 100644 index 0000000..9d1c718 --- /dev/null +++ b/labsim/ovs.sh @@ -0,0 +1,161 @@ +#!/bin/bash +# Open vSwitch fabric for labsim — the "switch" the whole sim hangs off. +# +# Why OVS and not a Linux bridge: a Linux bridge cannot do LACP at all, and its +# VLAN support is awkward to drive from libvirt. OVS gives real 802.1Q access +# and trunk ports plus real LACP bonds, so a router VM can run the SAME bond0 + +# vif config as the production VP2440s instead of an approximation. +# +# Layout: +# ovs-labsim the switch +# ├─ vm ports access ports, tag= (micro VM per VLAN) +# ├─ hostv internal ports, tag= (host leg, for SSH) +# └─ lag-vyos LACP bond, trunk of all VLANs (router under test) +# shellcheck disable=SC2034 + +OVS_BR="${OVS_BR:-ovs-labsim}" +OVS_NET="${OVS_NET:-labsim-ovs}" # libvirt network wrapping the bridge +LAG_NAME="${LAG_NAME:-lag-vyos}" + +ovs() { sudo ovs-vsctl "$@"; } + +ovs_require() { + command -v ovs-vsctl >/dev/null 2>&1 || die "openvswitch not installed (dnf install openvswitch)" + systemctl is-active --quiet openvswitch || sudo systemctl start openvswitch \ + || die "could not start openvswitch" +} + +# All VLAN ids from the config, comma separated — used for trunk ports. +vlan_id_list() { + local ids=() + for entry in "${SELECTED[@]}"; do ids+=("${entry%%:*}"); done + (IFS=,; echo "${ids[*]}") +} + +ovs_up() { + ovs_require + ovs --may-exist add-br "$OVS_BR" + + # Host leg per VLAN: an OVS internal port carrying that VLAN's tag, given the + # .2 address. This is how you SSH to the VMs. It is deliberately NOT their + # default route (.1 is), so inter-VLAN tests exercise the router, not the + # host's routing table. + for entry in "${SELECTED[@]}"; do + IFS=: read -r vid _name prefix _real <<<"$entry" + local port="hostv${vid}" + ovs --may-exist add-port "$OVS_BR" "$port" tag="$vid" \ + -- set interface "$port" type=internal + sudo ip link set "$port" up 2>/dev/null || true + sudo ip addr replace "${prefix}.2/24" dev "$port" + done + + ovs_define_libvirt_net +} + +# A libvirt network that hands out OVS ports: one portgroup per VLAN (access) +# plus a trunk portgroup for the router. +ovs_define_libvirt_net() { + local pg="" ids + for entry in "${SELECTED[@]}"; do + IFS=: read -r vid name _p _r <<<"$entry" + pg+=" + + +" + done + + # Trunk: VLAN 1 native/untagged, everything else tagged — the production + # shape. libvirt expresses this declaratively via nativeMode='untagged' + # (see libvirt formatnetwork.html), so it does not need fixing up by hand. + # It also matters functionally: LACPDUs are untagged, and a trunk with no + # native VLAN has nowhere to put them. + local trunk=" + +" + for entry in "${SELECTED[@]}"; do + IFS=: read -r vid _n _p _r <<<"$entry" + if [ "$vid" = "1" ]; then + trunk+=" +" + else + trunk+=" +" + fi + done + trunk+=" + +" + + local xml=" + ${OVS_NET} + + + +${pg}${trunk}" + + if virsh_q net-info "$OVS_NET" >/dev/null 2>&1; then + virsh_q net-destroy "$OVS_NET" >/dev/null 2>&1 || true + virsh_q net-undefine "$OVS_NET" >/dev/null 2>&1 || true + fi + echo "$xml" | virsh_q net-define /dev/stdin >/dev/null + virsh_q net-start "$OVS_NET" >/dev/null + log "libvirt network $OVS_NET bound to $OVS_BR (access portgroups + trunk)" +} + +# Replace the router VM's two individual OVS ports with a single LACP bond. +# libvirt attaches each NIC separately; only ovs-vsctl can bond them, and the +# taps only exist once the VM is running — so this runs post-start. +ovs_bond_router() { + local vm="$1" + local taps + # NB: domiflist indents its rows, so anchor on the FIELD not the line — + # /^vnet/ silently matches nothing and the bond never gets built. + taps="$(virsh_q domiflist "$vm" 2>/dev/null | awk '$1 ~ /^vnet/ {print $1}')" + local count; count="$(echo "$taps" | grep -c .)" + [ "$count" -eq 2 ] || { warn "router $vm has $count tap(s), expected 2 — skipping bond"; return 1; } + + # Already bonded? (idempotent re-runs) + if ovs list-ports "$OVS_BR" 2>/dev/null | grep -qx "$LAG_NAME"; then + log "LACP bond $LAG_NAME already present" + return 0 + fi + + local t1 t2; t1="$(echo "$taps" | sed -n 1p)"; t2="$(echo "$taps" | sed -n 2p)" + log "bonding $t1 + $t2 into $LAG_NAME (LACP active, balance-tcp)" + ovs del-port "$OVS_BR" "$t1" 2>/dev/null || true + ovs del-port "$OVS_BR" "$t2" 2>/dev/null || true + + # bond_mode=balance-tcp is REQUIRED: OVS defaults a bond to active-backup, + # which does not speak LACP at all (confirmed on ovs-discuss). It is also the + # equivalent of VyOS's 802.3ad + layer2+3 hashing. + # + # lacp-fallback-ab breaks a genuine deadlock: OVS keeps members disabled + # until LACP negotiates, while the partner needs carrier before it will send + # LACPDUs. Falling back to active-backup brings the links up so negotiation + # can start. + # + # native-untagged + tag=1 carries the untagged LACPDUs and the management + # VLAN, matching production. libvirt's portgroup VLAN config does NOT apply + # here — the bond is a port libvirt never created — so set it inline. + local tagged; tagged="$(vlan_id_list | tr ',' '\n' | grep -vx 1 | paste -sd, -)" + ovs add-bond "$OVS_BR" "$LAG_NAME" "$t1" "$t2" \ + lacp=active bond_mode=balance-tcp \ + vlan_mode=native-untagged tag=1 trunks="$tagged" \ + -- set port "$LAG_NAME" other_config:lacp-time=fast \ + -- set port "$LAG_NAME" other_config:lacp-fallback-ab=true +} + +ovs_bond_status() { + echo "--- ovs bond ---" + sudo ovs-appctl bond/show "$LAG_NAME" 2>/dev/null | grep -E "bond_mode|lacp_status|^member|may_enable" || echo "(no bond)" + echo "--- lacp ---" + sudo ovs-appctl lacp/show "$LAG_NAME" 2>/dev/null | grep -E "status|aggregation key|^member|attached" || true +} + +ovs_down() { + virsh_q net-destroy "$OVS_NET" >/dev/null 2>&1 || true + virsh_q net-undefine "$OVS_NET" >/dev/null 2>&1 || true + if command -v ovs-vsctl >/dev/null 2>&1; then + ovs --if-exists del-br "$OVS_BR" 2>/dev/null || true + fi +} diff --git a/labsim/router-install.py b/labsim/router-install.py new file mode 100755 index 0000000..26352a4 --- /dev/null +++ b/labsim/router-install.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Drive the labsim VyOS router over its serial console. + +Three phases: + --phase live wait for the live system and log in + --phase install run `install image` unattended + --phase configure apply bond0 (LACP) + per-VLAN gateway addresses + +The installer prompt list is the same one the bastion's install driver answers +(src/bastion/src/templates/vyos-install.py.ts). Two of them are easy to miss and +both hang forever rather than failing: the reinstall-only "copy data to the new +image?", and "choose two disks for RAID-1 mirroring?" — every RAID prompt +defaults to YES. +""" +from __future__ import annotations + +import argparse +import sys +import time + +import pexpect + +PASSWORD = "vyos" +PROMPT = r"[\$#] $" + + +def console(vm: str, timeout: int = 60) -> pexpect.spawn: + c = pexpect.spawn(f"sudo virsh console {vm} --force", encoding="utf-8", timeout=timeout) + c.expect("Connected to domain", timeout=30) + return c + + +def login(c: pexpect.spawn, timeout: int = 300) -> None: + """Get to a shell prompt, whether we land at a login or an open session.""" + deadline = time.time() + timeout + while time.time() < deadline: + c.sendline("") + i = c.expect(["login:", PROMPT, pexpect.TIMEOUT], timeout=20) + if i == 0: + c.sendline("vyos") + c.expect("assword:", timeout=20) + c.sendline(PASSWORD) + j = c.expect([PROMPT, "incorrect", pexpect.TIMEOUT], timeout=30) + if j == 0: + return + elif i == 1: + return + raise SystemExit("timed out waiting for a VyOS shell") + + +def run(c: pexpect.spawn, cmd: str, timeout: int = 60) -> str: + c.sendline(cmd) + c.expect(PROMPT, timeout=timeout) + return c.before or "" + + +def phase_install(c: pexpect.spawn) -> None: + """Answer `install image` end to end.""" + rules: list[tuple[str, str]] = [ + (r"Would you like to continue\?", "yes"), + (r"What would you like to name this image\?", ""), + (r"Please confirm password for the .vyos. user:", PASSWORD), + (r"Please enter a password for the .vyos. user:", PASSWORD), + (r"What console should be used by default", "K"), + # every RAID variant defaults to YES — decline them all + (r"Would you like to [^?]*RAID-1 mirroring", "no"), + (r"Installation will delete all data on (?:the drive|both drives)\. Continue\?", "yes"), + (r"Which one should be used for installation\?", "/dev/vda"), + (r"Would you like to use all the free space on the drive\?", "yes"), + (r"Which file would you like as boot config\?", "1"), + # reinstall-only; unanswered it blocks on stdin until the world ends + (r"Would you like to copy data to the new image\?", "yes"), + (r"From which image would you like to save config information\?", "1"), + ] + patterns = [r for r, _ in rules] + [r"The image installed successfully", + r"Unable to install VyOS", pexpect.TIMEOUT] + + c.sendline("install image") + for _ in range(60): + i = c.expect(patterns, timeout=180) + if i < len(rules): + c.sendline(rules[i][1]) + continue + if i == len(rules): + print(" installer: success") + return + if i == len(rules) + 1: + raise SystemExit("installer reported failure") + raise SystemExit("installer went quiet (unanswered prompt?)") + raise SystemExit("installer exceeded expected prompt count") + + +def phase_configure(c: pexpect.spawn, vlans: list[tuple[str, str, str]]) -> None: + """bond0 over eth0+eth1 with LACP, then a gateway address per VLAN.""" + # Production shape: VLAN 1 (management) is the NATIVE/untagged VLAN on the + # bond, everything else is a tagged vif. This matters beyond fidelity — + # LACPDUs are untagged, so a trunk with no native VLAN has nowhere to put + # them and the bond never negotiates. + native = [v for v in vlans if v[0] == "1"] + tagged = [v for v in vlans if v[0] != "1"] + + cmds = [ + "configure", + "set interfaces bonding bond0 mode '802.3ad'", + "set interfaces bonding bond0 hash-policy 'layer2+3'", + "set interfaces bonding bond0 lacp-rate 'fast'", + "set interfaces bonding bond0 member interface 'eth0'", + "set interfaces bonding bond0 member interface 'eth1'", + "set service ssh port '22'", + "set system login user vyos authentication plaintext-password 'vyos'", + ] + for vid, name, prefix in native: + cmds.append(f"set interfaces bonding bond0 address '{prefix}.1/24'") + cmds.append(f"set interfaces bonding bond0 description '{name} (native)'") + for vid, name, prefix in tagged: + cmds.append(f"set interfaces bonding bond0 vif {vid} address '{prefix}.1/24'") + cmds.append(f"set interfaces bonding bond0 vif {vid} description '{name}'") + cmds += ["commit", "save", "exit"] + + for cmd in cmds: + out = run(c, cmd, timeout=180) + low = out.lower() + if "invalid" in low or "syntax error" in low or "commit failed" in low: + print(f" !! {cmd}\n{out.strip()[-300:]}") + raise SystemExit(f"config command rejected: {cmd}") + print(" config committed and saved") + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--vm", required=True) + ap.add_argument("--phase", required=True, choices=["live", "install", "configure"]) + ap.add_argument("--vlans", default="", help="space separated vid:name:prefix:real entries") + args = ap.parse_args() + + c = console(args.vm) + try: + login(c) + if args.phase == "live": + print(" live system reachable") + elif args.phase == "install": + phase_install(c) + else: + vlans = [] + for entry in args.vlans.split(): + parts = entry.split(":") + if len(parts) >= 3: + vlans.append((parts[0], parts[1], parts[2])) + if not vlans: + raise SystemExit("no VLANs passed to configure") + phase_configure(c, vlans) + return 0 + finally: + try: + c.sendline("") + c.close(force=True) + except Exception: + pass + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/labsim/router-up.sh b/labsim/router-up.sh new file mode 100755 index 0000000..fd88124 --- /dev/null +++ b/labsim/router-up.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# Add the VyOS router under test to labsim. +# +# Mirrors the production VP2440 pair: TWO NICs bonded with LACP carrying a +# trunk of every VLAN, then bond0. sub-interfaces holding the .1 gateway +# address on each. That is the same config shape the real firewalls run, so a +# rule tested here means something. +# +# NIC model is e1000e, NOT virtio, and that is load-bearing: with virtio the +# guest's bonding driver reports its slaves "MII Status: down" despite +# carrier=1 and never emits a single LACPDU, so the bond sits in +# AD_STATE_DEFAULTED forever. Known issue — see the netdev thread "bonding +# (IEEE 802.3ad) not working with qemu/virtio"; e1000e fixes it with no other +# change. 802.3ad also requires the MII link monitor, which virtio cannot back. +# +# host OVS "switch" VyOS VM +# hostv (.2) ──────── ovs-labsim ──── lag-vyos ═════ eth0 + eth1 +# (tagged) (LACP, trunk) └─ bond0. = .1 +# +# Usage: ./router-up.sh build + install + configure +# ./router-up.sh --status show bond/LACP + interface state +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/lib.sh" +source "$SCRIPT_DIR/ovs.sh" + +ROUTER_VM="${ROUTER_VM:-labsim-vyos}" +ROUTER_MEM="${ROUTER_MEM:-2048}" +ROUTER_CPUS="${ROUTER_CPUS:-2}" +ROUTER_DISK_GB="${ROUTER_DISK_GB:-8}" +VYOS_ISO="${VYOS_ISO:-$IMG_DIR/vyos.iso}" +VYOS_CACHE="/var/lib/libvirt/images/lab-pxe-cache" + +selected_vlans + +if [ "${1:-}" = "--status" ]; then + ovs_bond_status + echo "--- vyos gateway addresses (probed from each host leg) ---" + for entry in "${SELECTED[@]}"; do + IFS=: read -r vid _n prefix _r <<<"$entry" + printf ' vlan %-5s %-16s ' "$vid" "${prefix}.1" + ping -c1 -W2 "${prefix}.1" >/dev/null 2>&1 && echo up || echo down + done + exit 0 +fi + +ovs_require + +# --- ISO ------------------------------------------------------------------ +if [ ! -f "$VYOS_ISO" ]; then + # Reuse the bastion's cached nightly if it is already on this box. + if [ -f "$VYOS_CACHE/vyos.iso" ]; then + log "reusing cached VyOS ISO" + sudo cp "$VYOS_CACHE/vyos.iso" "$VYOS_ISO" + else + log "resolving latest VyOS nightly ISO..." + url="$(curl -sSL https://api.github.com/repos/vyos/vyos-nightly-build/releases/latest \ + | python3 -c "import json,sys;print(next(a['browser_download_url'] for a in json.load(sys.stdin)['assets'] if a['name'].endswith('generic-amd64.iso')))")" + log "downloading $url" + sudo curl -sSL --max-time 1800 -o "$VYOS_ISO" "$url" + fi +fi +[ -f "$VYOS_ISO" ] || die "no VyOS ISO at $VYOS_ISO" + +# --- VM ------------------------------------------------------------------- +if virsh_q dominfo "$ROUTER_VM" >/dev/null 2>&1; then + log "router VM $ROUTER_VM exists" + virsh_q start "$ROUTER_VM" >/dev/null 2>&1 || true +else + log "creating router VM $ROUTER_VM (2 NICs on the trunk, for LACP)" + sudo qemu-img create -q -f qcow2 "$IMG_DIR/${ROUTER_VM}.qcow2" "${ROUTER_DISK_GB}G" >/dev/null + + # Two trunk NICs — OVS bonds them after boot (libvirt cannot create bonds). + sudo virt-install \ + --connect "$LIBVIRT_URI" \ + --name "$ROUTER_VM" \ + --memory "$ROUTER_MEM" --vcpus "$ROUTER_CPUS" \ + --disk "path=$IMG_DIR/${ROUTER_VM}.qcow2,format=qcow2,bus=virtio" \ + --disk "path=$VYOS_ISO,device=cdrom,readonly=on" \ + --network "network=$OVS_NET,portgroup=trunk,model=e1000e,trustGuestRxFilters=yes" \ + --network "network=$OVS_NET,portgroup=trunk,model=e1000e,trustGuestRxFilters=yes" \ + --boot cdrom,hd \ + --os-variant debian12 \ + --graphics none --noautoconsole --import >/dev/null +fi + +log "waiting for the live system to boot (VyOS live login)..." +python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase live || die "live boot failed" + +log "installing VyOS to disk (unattended over the console)..." +python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase install || die "install failed" + +# Boot the INSTALLED system from here on. Without this the VM was created with +# --boot cdrom,hd and every restart re-runs the ISO, so the live system comes +# back with no config and every `commit; save` silently evaporates. +log "switching boot to disk and ejecting the install media..." +virsh_q destroy "$ROUTER_VM" >/dev/null 2>&1 || true +sleep 2 +sudo virt-xml "$ROUTER_VM" --edit --boot hd >/dev/null +sudo virt-xml "$ROUTER_VM" --remove-device --disk device=cdrom >/dev/null 2>&1 || true +virsh_q start "$ROUTER_VM" >/dev/null +sleep 10 + +# Bond the taps only now: they are recreated by the restart above, so bonding +# before this would bond stale interfaces. +ovs_bond_router "$ROUTER_VM" + +log "applying router config (bond0 LACP + VLAN gateways)..." +python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase configure \ + --vlans "$(printf '%s\n' "${SELECTED[@]}" | tr '\n' ' ')" || die "configure failed" + +log "waiting for LACP to negotiate..." +for _ in $(seq 1 30); do + if sudo ovs-appctl lacp/show "$LAG_NAME" 2>/dev/null | grep -q "current attached"; then + log "LACP negotiated"; break + fi + sleep 5 +done + +echo +ovs_bond_status +echo +log "router is up. Check reachability with: $SCRIPT_DIR/labsim-matrix.py --watch 2" diff --git a/labsim/topology.html b/labsim/topology.html new file mode 100644 index 0000000..f4d910f --- /dev/null +++ b/labsim/topology.html @@ -0,0 +1,181 @@ + + + + +labsim — live VLAN topology + + + +
+

labsim — live VLAN topology

+ … + every path is probed from a VM to every other VM, through the VyOS router + +
+ +
+
+

Mesh — line colour is reachability, label is ICMP RTT

+ +
+ reachable + blocked + hover a line for detail · node ring turns red if anything to/from it is blocked +
+
+ + +
+ + + + diff --git a/labsim/vlans.conf b/labsim/vlans.conf new file mode 100644 index 0000000..2b6712b --- /dev/null +++ b/labsim/vlans.conf @@ -0,0 +1,21 @@ +# Lab network simulation — VLAN map. +# +# Mirrors the real UniFi topology (same VLAN IDs, same roles) but with +# deliberately DIFFERENT IP ranges so nothing here can collide with, or be +# confused for, production. The sim subnet always encodes the VLAN id: +# +# 172.31..0/24 +# +# Per-subnet address plan (same shape on every VLAN): +# .1 gateway under test (VyOS/router VM — not created by default) +# .2 host bridge (how you SSH in from this workstation) +# .10 the micro VM for this VLAN +# .254 VRRP VIP (reserved, mirrors production) +# +# Format: vlan_id:name:sim_subnet_prefix:real_subnet(for reference) +1:management:172.31.1:192.168.1.0/24 +2:k8s:172.31.2:192.168.8.0/23 +3:kvm:172.31.3:192.168.3.0/24 +9:private:172.31.9:10.0.9.0/23 +10:lot:172.31.10:10.0.0.0/23 +200:roomates:172.31.200:192.168.2.0/24