Compare commits
19 Commits
fix/k3s-au
...
feat/vyos-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
72c54edce2 | ||
|
|
33be713d0c | ||
|
|
a5b36678ed | ||
|
|
c91e44f796 | ||
|
|
df2dfc5d71 | ||
|
|
e36a7a193c | ||
|
|
5d00c42f5a | ||
|
|
cb9d99dd69 | ||
|
|
4f9a6f64e4 | ||
|
|
12fa954a05 | ||
| 7181a61cec | |||
|
|
cdf3b5c045 | ||
| f3c50f71ef | |||
|
|
98b0ccc6c9 | ||
|
|
37a3b51e57 | ||
|
|
d6e1f3c74d | ||
|
|
52e831b8c1 | ||
| f5af24699a | |||
|
|
04faa079e2 |
@@ -11,6 +11,7 @@ WORKDIR /app
|
||||
# Copy workspace config and package manifests first (layer cache)
|
||||
COPY pnpm-workspace.yaml pnpm-lock.yaml package.json tsconfig.base.json tsconfig.json ./
|
||||
COPY src/shared/package.json src/shared/tsconfig.json src/shared/
|
||||
COPY src/core/package.json src/core/tsconfig.json src/core/
|
||||
COPY src/labd/package.json src/labd/tsconfig.json src/labd/
|
||||
|
||||
# Install all dependencies (dev included -- needed for build)
|
||||
@@ -22,10 +23,13 @@ RUN pnpm --filter @lab/labd exec prisma generate
|
||||
|
||||
# Copy source code
|
||||
COPY src/shared/src/ src/shared/src/
|
||||
COPY src/core/src/ src/core/src/
|
||||
COPY src/labd/src/ src/labd/src/
|
||||
|
||||
# Build TypeScript (shared first via project references)
|
||||
RUN pnpm --filter @lab/shared build && pnpm --filter @lab/labd build
|
||||
# Build TypeScript (shared + core before labd via project references)
|
||||
RUN pnpm --filter @lab/shared build \
|
||||
&& pnpm --filter @lab/core build \
|
||||
&& pnpm --filter @lab/labd build
|
||||
|
||||
# Hoist the generated Prisma client so stage 2 can COPY it from a stable path
|
||||
RUN mkdir -p /app/_prisma && \
|
||||
@@ -41,6 +45,7 @@ WORKDIR /app
|
||||
# Copy workspace config and package manifests
|
||||
COPY pnpm-workspace.yaml pnpm-lock.yaml package.json ./
|
||||
COPY src/shared/package.json src/shared/
|
||||
COPY src/core/package.json src/core/
|
||||
COPY src/labd/package.json src/labd/
|
||||
|
||||
# Install production dependencies only
|
||||
@@ -48,6 +53,7 @@ RUN pnpm install --frozen-lockfile --prod 2>/dev/null || pnpm install --prod
|
||||
|
||||
# Copy built output from builder
|
||||
COPY --from=builder /app/src/shared/dist/ src/shared/dist/
|
||||
COPY --from=builder /app/src/core/dist/ src/core/dist/
|
||||
COPY --from=builder /app/src/labd/dist/ src/labd/dist/
|
||||
|
||||
# Copy Prisma schema + generated client into pnpm store location
|
||||
|
||||
@@ -59,10 +59,10 @@ _labctl() {
|
||||
COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision install")
|
||||
COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur"))
|
||||
COMPREPLY=($(compgen -W "--role --os --disk --vyos-mgmt --vyos-mgmt-address --vyos-bond --vyos-bond-address --vyos-bond-vrrp --vlan-vip --vyos-vrrp-priority --vyos-mgmt-vlan --vlan --vyos-password --vyos-hwid --vyos-fresh-config -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision reprovision")
|
||||
COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur"))
|
||||
COMPREPLY=($(compgen -W "--role --os --disk --user -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision debug")
|
||||
COMPREPLY=($(compgen -W "--pxe-boot -h --help" -- "$cur"))
|
||||
|
||||
@@ -132,13 +132,26 @@ complete -c labctl -n "__labctl_using_cmd provision" -a recheck -d 'Refresh hard
|
||||
|
||||
# provision install options
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04'
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling'
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target disk device (auto-detect if omitted)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt -d 'VyOS: untagged interface the machine PXE boots from (default eth0)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-address -d 'VyOS: CIDR for the management interface, or \'dhcp\' (default dhcp)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond -d 'VyOS: comma-separated LACP bond members (must exclude the PXE NIC)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond-address -d 'VyOS: address on the untagged bond (trunk native VLAN)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond-vrrp -d 'VyOS: VRRP VIP floated on the untagged bond' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vlan-vip -d 'VyOS: VRRP VIP for a --vlan entry (repeatable)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-vrrp-priority -d 'VyOS: VRRP priority for all groups on this box (higher = master)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-vlan -d 'VyOS: tagged management VLAN on the PXE port' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vlan -d 'VyOS: tagged VLAN sub-interface on the bond (repeatable)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-password -d 'VyOS: password for the \'vyos\' user' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-hwid -d 'VyOS: pin an interface name to a MAC via hw-id (repeatable)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-fresh-config -d 'VyOS: on reinstall, overwrite the preserved config with the generated one'
|
||||
|
||||
# provision reprovision options
|
||||
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
|
||||
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04'
|
||||
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling'
|
||||
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l disk -d 'Target disk device (auto-detect if omitted)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l user -d 'SSH user for the reboot (default: vyos for VyOS machines, else current user)' -x
|
||||
|
||||
# provision debug options
|
||||
complete -c labctl -n "__labctl_in_cmd provision debug" -l pxe-boot -d 'Boot installed system via PXE (kernel+initrd from network, root from NVMe)'
|
||||
|
||||
@@ -21,6 +21,8 @@
|
||||
"test:integration:pxe:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'PXE boot'",
|
||||
"test:integration:iso": "vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'",
|
||||
"test:integration:iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'",
|
||||
"test:integration:vyos": "vitest run -c tests/integration/vitest.config.ts -t 'VyOS provisioning'",
|
||||
"test:integration:vyos:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'VyOS provisioning'",
|
||||
"test:integration:arm-iso": "vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'",
|
||||
"test:integration:arm-iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'",
|
||||
"test:integration:asahi": "vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'",
|
||||
|
||||
1847
bastion/pnpm-lock.yaml
generated
1847
bastion/pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -20,6 +20,15 @@ export function loadConfig(overrides: Partial<BastionConfig> = {}): BastionConfi
|
||||
const ubuntuMirror = overrides.ubuntuMirror ?? process.env["UBUNTU_MIRROR"]
|
||||
?? `https://releases.ubuntu.com/${ubuntuVersion}`;
|
||||
|
||||
// "latest" resolves the newest nightly ISO from the vyos-nightly-build GitHub
|
||||
// releases at startup. downloads.vyos.io no longer serves direct rolling ISOs
|
||||
// (it returns the vyos.io site, and nightly builds sit behind a signup form);
|
||||
// GitHub releases are the remaining free, unauthenticated direct source.
|
||||
// LTS ISOs are subscription-only. Set VYOS_ISO_URL to pin a specific build.
|
||||
const vyosIsoUrl = overrides.vyosIsoUrl ?? process.env["VYOS_ISO_URL"] ?? "latest";
|
||||
const vyosDefaultPassword = overrides.vyosDefaultPassword
|
||||
?? process.env["VYOS_DEFAULT_PASSWORD"] ?? "vyos";
|
||||
|
||||
const fedoraMirror = `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`;
|
||||
const tftpDir = `${bastionDir}/tftp`;
|
||||
const httpDir = `${bastionDir}/http`;
|
||||
@@ -38,6 +47,8 @@ export function loadConfig(overrides: Partial<BastionConfig> = {}): BastionConfi
|
||||
dhcpRangeEnd,
|
||||
ubuntuVersion,
|
||||
ubuntuMirror,
|
||||
vyosIsoUrl,
|
||||
vyosDefaultPassword,
|
||||
// These are populated at runtime by the network service
|
||||
iface: overrides.iface ?? "",
|
||||
serverIp: overrides.serverIp ?? "",
|
||||
|
||||
@@ -40,6 +40,125 @@ function download(url: string, dest: string, label: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the largest regular-file initrd from an `xorriso -lsl` listing.
|
||||
*
|
||||
* /live carries decoys: a 0-byte initrd.img placeholder on some images, or an
|
||||
* initrd.img SYMLINK to the real version-suffixed file on others. Parsing is
|
||||
* field-based (ls -l layout: perms links uid gid size month day time 'name')
|
||||
* and considers only lines whose mode string marks a regular file — symlinks
|
||||
* report their link size, not the target's, and must not win.
|
||||
*/
|
||||
export function pickLargestInitrd(
|
||||
listing: string,
|
||||
): { name: string; size: number } | undefined {
|
||||
let best: { name: string; size: number } | undefined;
|
||||
for (const line of listing.split("\n")) {
|
||||
if (!line.startsWith("-")) continue; // regular files only
|
||||
const quoted = /'([^']+)'/.exec(line);
|
||||
const fields = line.trim().split(/\s+/);
|
||||
const size = parseInt(fields[4] ?? "", 10);
|
||||
const name = quoted?.[1] ?? "";
|
||||
if (!name.startsWith("initrd")) continue;
|
||||
if (!Number.isFinite(size) || size <= 0) continue;
|
||||
if (best === undefined || size > best.size) {
|
||||
best = { name, size };
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
const VYOS_NIGHTLY_RELEASES =
|
||||
"https://api.github.com/repos/vyos/vyos-nightly-build/releases/latest";
|
||||
|
||||
/**
|
||||
* Resolve the configured VyOS ISO URL, expanding the "latest" sentinel.
|
||||
*
|
||||
* The nightly asset filename embeds a build date, so there is no stable
|
||||
* "latest.iso" path to hardcode — the newest release has to be looked up.
|
||||
* Any other value is used verbatim, which is how VYOS_ISO_URL pins a build
|
||||
* or points at a locally mirrored copy.
|
||||
*/
|
||||
function resolveVyosIsoUrl(configured: string): string {
|
||||
if (configured !== "latest") return configured;
|
||||
|
||||
const body = execSync(`curl -sSfL "${VYOS_NIGHTLY_RELEASES}"`, {
|
||||
encoding: "utf-8",
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
const release = JSON.parse(body) as {
|
||||
tag_name?: string;
|
||||
assets?: Array<{ name: string; browser_download_url: string }>;
|
||||
};
|
||||
|
||||
const asset = (release.assets ?? []).find((a) =>
|
||||
/generic-amd64\.iso$/.test(a.name),
|
||||
);
|
||||
if (!asset) {
|
||||
throw new Error(
|
||||
`No generic-amd64 ISO asset in VyOS nightly release ${release.tag_name ?? "?"}`,
|
||||
);
|
||||
}
|
||||
|
||||
logger.info(` VyOS ISO resolved to ${asset.name} (${release.tag_name ?? "?"})`);
|
||||
return asset.browser_download_url;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract VyOS netboot artifacts from the release ISO.
|
||||
*
|
||||
* VyOS publishes no netboot bundle, so kernel/initrd/squashfs have to come out
|
||||
* of the ISO. xorriso is already in the bastion image (used for boot.iso) and
|
||||
* extracts without root or a loop mount.
|
||||
*
|
||||
* The initrd needs care: /live contains an empty initrd.img placeholder
|
||||
* alongside the real one, which carries a version-suffixed name. Booting the
|
||||
* 0-byte file fails with no useful diagnostic, so pick the largest initrd*.
|
||||
*/
|
||||
export function prepareVyosArtifacts(config: BastionConfig): void {
|
||||
const kernel = `${config.httpDir}/vyos-vmlinuz`;
|
||||
const initrd = `${config.httpDir}/vyos-initrd`;
|
||||
const squashfs = `${config.httpDir}/vyos-filesystem.squashfs`;
|
||||
|
||||
if (existsSync(kernel) && existsSync(initrd) && existsSync(squashfs)) {
|
||||
logger.info(" VyOS netboot artifacts -- cached");
|
||||
return;
|
||||
}
|
||||
|
||||
const iso = `${config.bastionDir}/vyos.iso`;
|
||||
download(resolveVyosIsoUrl(config.vyosIsoUrl), iso, "VyOS ISO");
|
||||
|
||||
const extract = (isoPath: string, dest: string, label: string): void => {
|
||||
execSync(
|
||||
`xorriso -osirrox on -indev "${iso}" -extract "${isoPath}" "${dest}"`,
|
||||
{ stdio: "pipe" },
|
||||
);
|
||||
logger.info(` ${label} -- extracted from ${isoPath}`);
|
||||
};
|
||||
|
||||
extract("/live/vmlinuz", kernel, "VyOS kernel");
|
||||
extract("/live/filesystem.squashfs", squashfs, "VyOS squashfs");
|
||||
|
||||
// Pick the real initrd by size from the ISO's own directory listing.
|
||||
const listing = execSync(`xorriso -indev "${iso}" -lsl /live/ --`, {
|
||||
encoding: "utf-8",
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
const best = pickLargestInitrd(listing);
|
||||
if (best === undefined) {
|
||||
throw new Error("No non-empty initrd found in /live on the VyOS ISO");
|
||||
}
|
||||
extract(`/live/${best.name}`, initrd, `VyOS initrd (${best.name}, ${best.size} bytes)`);
|
||||
|
||||
// The ISO is only needed to produce the three artifacts above.
|
||||
try {
|
||||
unlinkSync(iso);
|
||||
} catch {
|
||||
// Non-fatal: leaving it costs disk but nothing else.
|
||||
}
|
||||
}
|
||||
|
||||
function symlinkSafe(target: string, linkPath: string): void {
|
||||
try {
|
||||
symlinkSync(target, linkPath);
|
||||
@@ -182,6 +301,17 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
|
||||
logger.warn(`Ubuntu ${config.ubuntuVersion} artifacts not available -- Ubuntu provisioning disabled`);
|
||||
}
|
||||
|
||||
// VyOS netboot artifacts (non-fatal — same policy as Ubuntu)
|
||||
try {
|
||||
logger.info("Preparing VyOS netboot artifacts...");
|
||||
prepareVyosArtifacts(config);
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
`VyOS artifacts not available -- VyOS provisioning disabled ` +
|
||||
`(${err instanceof Error ? err.message : String(err)})`,
|
||||
);
|
||||
}
|
||||
|
||||
// Symlink iPXE binaries into HTTP dir for UEFI HTTP Boot
|
||||
for (const name of ["ipxe.efi", "ipxe-arm64.efi"]) {
|
||||
const src = `${config.tftpDir}/${name}`;
|
||||
@@ -261,6 +391,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
|
||||
role: msg.role as import("@lab/shared").Role,
|
||||
os: msg.os as import("@lab/shared").OsId,
|
||||
queued_at: new Date().toISOString(),
|
||||
...(msg.vyos ? { vyos: msg.vyos } : {}),
|
||||
};
|
||||
});
|
||||
return { status: "ok", data: { mac: msg.mac, hostname: msg.hostname } };
|
||||
|
||||
@@ -5,8 +5,8 @@
|
||||
// /api/discover - receive hardware discovery reports from PXE-booted machines
|
||||
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import type { HardwareInfo, InstalledInfo, Role } from "@lab/shared";
|
||||
import { isValidOsId, SUPPORTED_ROLES } from "@lab/shared";
|
||||
import type { HardwareInfo, InstalledInfo, Role, VyosInstallSpec } from "@lab/shared";
|
||||
import { isValidOsId, SUPPORTED_ROLES, SUPPORTED_OS } from "@lab/shared";
|
||||
import type { StateManager } from "../services/state.js";
|
||||
import { logger } from "../services/logger.js";
|
||||
import { triggerPostProvisionK3s } from "../services/post-provision.js";
|
||||
@@ -15,6 +15,13 @@ import type { ProgressEvent } from "../services/progress-events.js";
|
||||
import type { InstallLogBuffer } from "../services/install-log.js";
|
||||
import type { SyslogListener } from "../services/syslog-listener.js";
|
||||
|
||||
/**
|
||||
* Seconds after dispatch with zero progress before a machine is called stalled.
|
||||
* Generous: the slowest legitimate gap is fetching a ~600MB VyOS squashfs over
|
||||
* HTTP before the hook can report anything.
|
||||
*/
|
||||
const STALL_THRESHOLD_S = 8 * 60;
|
||||
|
||||
export function registerApiRoutes(
|
||||
app: FastifyInstance,
|
||||
state: StateManager,
|
||||
@@ -34,9 +41,10 @@ export function registerApiRoutes(
|
||||
disk?: string;
|
||||
role?: string;
|
||||
os?: string;
|
||||
vyos?: VyosInstallSpec;
|
||||
};
|
||||
}>("/api/install", async (request, reply) => {
|
||||
const { mac: rawMac, hostname, disk, role, os } = request.body ?? {};
|
||||
const { mac: rawMac, hostname, disk, role, os, vyos } = request.body ?? {};
|
||||
const mac = (rawMac ?? "").toLowerCase().replace(/-/g, ":");
|
||||
|
||||
if (mac === "") {
|
||||
@@ -50,7 +58,7 @@ export function registerApiRoutes(
|
||||
|
||||
const osId = os ?? "fedora-43";
|
||||
if (!isValidOsId(osId)) {
|
||||
return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: fedora-43, ubuntu-26.04` });
|
||||
return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: ${SUPPORTED_OS.join(", ")}` });
|
||||
}
|
||||
|
||||
state.update((s) => {
|
||||
@@ -60,6 +68,7 @@ export function registerApiRoutes(
|
||||
role: validRole as Role,
|
||||
os: osId,
|
||||
queued_at: new Date().toISOString(),
|
||||
...(vyos ? { vyos } : {}),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -158,11 +167,17 @@ export function registerApiRoutes(
|
||||
};
|
||||
s.installed[mac] = installedInfo;
|
||||
|
||||
const admin = installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "lab" : "root";
|
||||
// VyOS: the only login user is "vyos", and a router never runs k3s —
|
||||
// without this guard a non-vanilla role + recorded IP would trigger
|
||||
// the k3s post-provision against a VyOS box.
|
||||
const isVyos = (installedInfo.os ?? "").startsWith("vyos");
|
||||
const admin = isVyos
|
||||
? "vyos"
|
||||
: installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "lab" : "root";
|
||||
console.log(`\n \x1b[0;32m\x1b[1m ssh ${admin}@${ip}\x1b[0m\n`); // eslint-disable-line no-console
|
||||
|
||||
// Auto-install k3s for non-vanilla roles
|
||||
if (installedInfo.role !== "vanilla" && ip !== "") {
|
||||
if (!isVyos && installedInfo.role !== "vanilla" && ip !== "") {
|
||||
void triggerPostProvisionK3s(installedInfo.hostname, ip, installedInfo.role, admin, mac);
|
||||
}
|
||||
}
|
||||
@@ -434,6 +449,15 @@ export function registerApiRoutes(
|
||||
const installedEntry = currentState.installed[mac];
|
||||
|
||||
if (queueEntry) {
|
||||
// A machine that was handed an install script but has reported nothing
|
||||
// since is wedged BEFORE the installer environment came up — a bad
|
||||
// kernel/initrd, no network in the initramfs, or the wrong NIC picked.
|
||||
// Surfacing it here is what makes that diagnosable without a console.
|
||||
const since = queueEntry.progress_at ?? queueEntry.dispatched_at;
|
||||
const stalledForS = since !== undefined && queueEntry.progress === undefined
|
||||
? Math.floor((Date.now() - new Date(since).getTime()) / 1000)
|
||||
: 0;
|
||||
|
||||
return reply.send({
|
||||
mac,
|
||||
hostname: queueEntry.hostname,
|
||||
@@ -441,6 +465,9 @@ export function registerApiRoutes(
|
||||
progress: queueEntry.progress ?? "queued",
|
||||
progress_detail: queueEntry.progress_detail ?? "",
|
||||
progress_at: queueEntry.progress_at ?? queueEntry.queued_at,
|
||||
dispatched_at: queueEntry.dispatched_at,
|
||||
stalled_for_s: stalledForS,
|
||||
stalled: stalledForS > STALL_THRESHOLD_S,
|
||||
role: queueEntry.role,
|
||||
os: queueEntry.os,
|
||||
stages: queueEntry.log ?? [],
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
renderLocalBootIpxe,
|
||||
} from "../templates/boot.ipxe.js";
|
||||
import { renderUbuntuInstallIpxe } from "../templates/ubuntu-boot.ipxe.js";
|
||||
import { renderVyosInstallIpxe } from "../templates/vyos-boot.ipxe.js";
|
||||
import { renderDebugKickstart } from "../templates/debug.ks.js";
|
||||
import { logger } from "../services/logger.js";
|
||||
|
||||
@@ -99,8 +100,22 @@ echo "==============================="
|
||||
const os = queueEntry.os ?? "fedora-43";
|
||||
logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os})`);
|
||||
|
||||
// Stamp the handoff so a machine that boots the installer but never
|
||||
// reports can be spotted without a console.
|
||||
state.update((s) => {
|
||||
const entry = s.install_queue[mac];
|
||||
if (entry) entry.dispatched_at = new Date().toISOString();
|
||||
});
|
||||
|
||||
let script: string;
|
||||
if (os.startsWith("ubuntu")) {
|
||||
if (os.startsWith("vyos")) {
|
||||
script = renderVyosInstallIpxe({
|
||||
mac,
|
||||
hostname,
|
||||
serverIp: config.serverIp,
|
||||
httpPort: config.httpPort,
|
||||
});
|
||||
} else if (os.startsWith("ubuntu")) {
|
||||
script = renderUbuntuInstallIpxe({
|
||||
mac,
|
||||
hostname,
|
||||
|
||||
71
bastion/src/bastion/src/routes/vyos.ts
Normal file
71
bastion/src/bastion/src/routes/vyos.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
// VyOS network install routes.
|
||||
//
|
||||
// VyOS has no unattended installer, so the automation is injected via
|
||||
// live-config's `hooks` component: the iPXE script passes
|
||||
// live-config.hooks=<.../vyos/autoinstall.sh>, live-config wgets it and runs it
|
||||
// as root, and that script fetches and executes the generated install driver.
|
||||
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import type { BastionConfig } from "@lab/shared";
|
||||
import type { StateManager } from "../services/state.js";
|
||||
import { buildVyosConfigSpec } from "../templates/vyos-config-spec.js";
|
||||
import { renderVyosInstallPy } from "../templates/vyos-install.py.js";
|
||||
import { logger } from "../services/logger.js";
|
||||
|
||||
function normalizeMac(value: string | undefined): string {
|
||||
return (value ?? "").toLowerCase().replace(/-/g, ":");
|
||||
}
|
||||
|
||||
export function registerVyosRoutes(
|
||||
app: FastifyInstance,
|
||||
config: BastionConfig,
|
||||
state: StateManager,
|
||||
): void {
|
||||
// live-config hook. Kept minimal: everything version-specific lives in the
|
||||
// generated Python. wget is guaranteed present -- live-config used it to
|
||||
// fetch this very script.
|
||||
app.get<{ Querystring: { mac?: string } }>("/vyos/autoinstall.sh", async (request, reply) => {
|
||||
const mac = normalizeMac(request.query.mac);
|
||||
const base = `http://${config.serverIp}:${config.httpPort}`;
|
||||
|
||||
logger.info(`VYOS AUTOINSTALL HOOK served to ${mac || "unknown MAC"}`);
|
||||
|
||||
const script = `#!/bin/sh
|
||||
# Lab PXE Bastion -- VyOS unattended install hook (run by live-config as root)
|
||||
set -eu
|
||||
|
||||
wget -q "${base}/vyos/install.py?mac=${mac}" -O /tmp/vyos-install.py
|
||||
exec python3 /tmp/vyos-install.py
|
||||
`;
|
||||
return reply.type("text/plain").send(script);
|
||||
});
|
||||
|
||||
// Per-MAC install driver, with the machine's config spec baked in.
|
||||
app.get<{ Querystring: { mac?: string } }>("/vyos/install.py", async (request, reply) => {
|
||||
const mac = normalizeMac(request.query.mac);
|
||||
const queueEntry = state.load().install_queue[mac];
|
||||
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: queueEntry?.hostname ?? "vyos",
|
||||
spec: queueEntry?.vyos,
|
||||
defaultPassword: config.vyosDefaultPassword,
|
||||
sshKeys: config.sshKeys,
|
||||
disk: queueEntry?.disk,
|
||||
});
|
||||
|
||||
logger.info(
|
||||
`VYOS INSTALL DRIVER served to ${mac} (${spec.hostname}, ` +
|
||||
`${spec.sets.length} config ops, disk="${spec.disk || "auto"}")`,
|
||||
);
|
||||
|
||||
const script = renderVyosInstallPy({
|
||||
spec,
|
||||
mac,
|
||||
serverIp: config.serverIp,
|
||||
httpPort: config.httpPort,
|
||||
role: queueEntry?.role ?? "vanilla",
|
||||
});
|
||||
|
||||
return reply.type("text/plain").send(script);
|
||||
});
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import { registerDispatchRoutes } from "./routes/dispatch.js";
|
||||
import { registerKickstartRoutes } from "./routes/kickstart.js";
|
||||
import { registerApiRoutes } from "./routes/api.js";
|
||||
import { registerAsahiRoutes } from "./routes/asahi.js";
|
||||
import { registerVyosRoutes } from "./routes/vyos.js";
|
||||
|
||||
|
||||
export function createApp(config: BastionConfig): { app: ReturnType<typeof Fastify>; state: StateManager; installLog: InstallLogBuffer; syslog: SyslogListener } {
|
||||
@@ -47,6 +48,7 @@ export function createApp(config: BastionConfig): { app: ReturnType<typeof Fasti
|
||||
registerKickstartRoutes(app, config, state, syslog);
|
||||
registerApiRoutes(app, state, installLog, syslog);
|
||||
registerAsahiRoutes(app, config);
|
||||
registerVyosRoutes(app, config, state);
|
||||
// boot.iso is generated at startup and served as a static file from httpDir
|
||||
// (static serving supports HTTP Range requests, required by JetKVM streaming)
|
||||
|
||||
|
||||
@@ -40,6 +40,11 @@ export function renderInstallKickstart(params: InstallKickstartParams): string {
|
||||
const now = new Date().toISOString();
|
||||
const hasLonghorn = role === "worker";
|
||||
const hasRancher = role === "infra";
|
||||
// k8s roles get a dedicated 120G image-store LV. 2026-08 incident: the old
|
||||
// 20G LV idled at 85% used, so a single ~5G image pull tripped imagefs
|
||||
// eviction. Must be sized here — longhorn's --grow consumes all remaining
|
||||
// VG space, making post-install lvextend impossible on worker nodes.
|
||||
const hasRancherLv = role === "infra" || role === "worker";
|
||||
const isVanilla = role === "vanilla";
|
||||
|
||||
// -- Auth section --
|
||||
@@ -113,9 +118,9 @@ done
|
||||
? `logvol /var/lib/longhorn --vgname=${vg} --name=longhorn --fstype=xfs --grow --size=1`
|
||||
: "";
|
||||
|
||||
// -- Rancher LV for fresh install (infra role) --
|
||||
const rancherFreshLine = hasRancher
|
||||
? `logvol /var/lib/rancher --vgname=${vg} --name=rancher --fstype=xfs --size=20480`
|
||||
// -- Rancher LV for fresh install (k8s roles: worker + infra) --
|
||||
const rancherFreshLine = hasRancherLv
|
||||
? `logvol /var/lib/rancher --vgname=${vg} --name=rancher --fstype=xfs --size=122880`
|
||||
: "";
|
||||
|
||||
return `# Lab Bastion -- Fedora ${fedoraVersion} server install
|
||||
|
||||
53
bastion/src/bastion/src/templates/vyos-boot.ipxe.ts
Normal file
53
bastion/src/bastion/src/templates/vyos-boot.ipxe.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
// iPXE boot script template for VyOS network install.
|
||||
//
|
||||
// VyOS ships no unattended installer: `install image` is unconditionally
|
||||
// interactive (image_installer.py's install action takes no arguments, and
|
||||
// --no-prompt is wired only to `add`). So PXE boots the *live* system and the
|
||||
// automation is injected through live-config's `hooks` component, which fetches
|
||||
// a script over HTTP and runs it as root late in live boot.
|
||||
//
|
||||
// Unlike the Fedora/Ubuntu paths this boots a live image rather than an
|
||||
// installer, so there is no kickstart/autoinstall equivalent — see
|
||||
// routes/vyos.ts for the hook that actually drives the install.
|
||||
|
||||
export function renderVyosInstallIpxe(params: {
|
||||
mac: string;
|
||||
hostname: string;
|
||||
serverIp: string;
|
||||
httpPort: number;
|
||||
}): string {
|
||||
const base = `http://${params.serverIp}:${params.httpPort}`;
|
||||
|
||||
// Pin the boot NIC by MAC. live-boot otherwise scans for the first
|
||||
// *connected* interface, and on a multi-NIC box that race is lost by
|
||||
// whichever port negotiates slowest: on the Protectli VP2440 the SFP+
|
||||
// pair links first, so live-boot picked the fiber ports (which have no
|
||||
// DHCP), burned 15s per port, and gave up with "Unable to find a live
|
||||
// file system on the network" -- while the copper port that actually PXE
|
||||
// booted came up at 4.6s and was never tried.
|
||||
//
|
||||
// live-boot's Device_from_bootif() strips the "01-" and matches the MAC
|
||||
// against /sys/class/net/*. params.mac is the dispatch key, i.e. exactly
|
||||
// the NIC that PXE booted -- more reliable than iPXE's ${net0} on a box
|
||||
// where the booting NIC may not be net0.
|
||||
const bootif = `01-${params.mac.toLowerCase().replace(/:/g, "-")}`;
|
||||
|
||||
// Deliberately NOT passing `nonetworking` (present in VyOS's own PXE docs):
|
||||
// live-config's hook component needs networking up to fetch the hook over
|
||||
// HTTP. Also no `console=ttyS0` — on hardware without a physical UART that
|
||||
// costs 30s at every systemd boot phase.
|
||||
return `#!ipxe
|
||||
|
||||
echo
|
||||
echo =============================================
|
||||
echo Lab PXE Bastion - INSTALLING VyOS
|
||||
echo Target: ${params.hostname}
|
||||
echo MAC: ${params.mac}
|
||||
echo =============================================
|
||||
echo
|
||||
|
||||
kernel ${base}/vyos-vmlinuz boot=live nopersistence noautologin BOOTIF=${bootif} fetch=${base}/vyos-filesystem.squashfs live-config.hooks=${base}/vyos/autoinstall.sh?mac=${params.mac}
|
||||
initrd ${base}/vyos-initrd
|
||||
boot
|
||||
`;
|
||||
}
|
||||
208
bastion/src/bastion/src/templates/vyos-config-spec.ts
Normal file
208
bastion/src/bastion/src/templates/vyos-config-spec.ts
Normal file
@@ -0,0 +1,208 @@
|
||||
// Builds the VyOS configuration spec applied by the autoinstall hook.
|
||||
//
|
||||
// We deliberately do NOT emit a config.boot file as text. A config.boot carries a
|
||||
// `vyos-config-version` trailer; without a trailer matching the running image,
|
||||
// VyOS runs its migration scripts from version 0 on first boot. Instead the hook
|
||||
// loads the image's own /opt/vyatta/etc/config.boot.default through vyos.configtree
|
||||
// and applies these set operations on top, so syntax and version trailer always
|
||||
// match the exact image being installed.
|
||||
|
||||
import type { VyosInstallSpec } from "@lab/shared";
|
||||
|
||||
export interface VyosSetOp {
|
||||
path: string[];
|
||||
value?: string;
|
||||
/** false appends to a multi-value node (e.g. bond members) instead of replacing. */
|
||||
replace?: boolean;
|
||||
}
|
||||
|
||||
export interface VyosConfigSpec {
|
||||
hostname: string;
|
||||
/** "" means accept the installer default (the running image's version string). */
|
||||
imageName: string;
|
||||
password: string;
|
||||
console: "K" | "S";
|
||||
/** Target disk name (e.g. "nvme0n1"); "" accepts the installer's first-disk default. */
|
||||
disk: string;
|
||||
/**
|
||||
* IP the driver should report in the "complete" callback ("ready at <ip>" —
|
||||
* the exact format routes/api.ts parses installed.ip from). The mgmt
|
||||
* address when static; "" means detect the live DHCP address at runtime.
|
||||
*/
|
||||
reportAddress: string;
|
||||
/** Whether to accept RAID-1 when the installer finds more than one disk. */
|
||||
raid: boolean;
|
||||
/** Overwrite the installed config.boot with the generated one on reinstall. */
|
||||
freshConfig: boolean;
|
||||
sets: VyosSetOp[];
|
||||
/** Paths that are VyOS tag nodes — must be marked as such in the ConfigTree. */
|
||||
tags: string[][];
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalise a target disk to the form the installer expects.
|
||||
*
|
||||
* find_disks() enumerates via `lsblk -Jbp` (-p = full paths), so its valid
|
||||
* responses are "/dev/mmcblk0"-style. A bare "mmcblk0" is rejected by
|
||||
* ask_input()'s valid_responses check and re-prompts forever.
|
||||
*/
|
||||
function normalizeDiskPath(value: string | undefined): string {
|
||||
const raw = (value ?? "").trim();
|
||||
if (raw === "") return "";
|
||||
return raw.startsWith("/dev/") ? raw : `/dev/${raw}`;
|
||||
}
|
||||
|
||||
export function buildVyosConfigSpec(params: {
|
||||
hostname: string;
|
||||
spec?: VyosInstallSpec | undefined;
|
||||
defaultPassword: string;
|
||||
sshKeys?: string[] | undefined;
|
||||
disk?: string | undefined;
|
||||
}): VyosConfigSpec {
|
||||
const spec = params.spec ?? {};
|
||||
const mgmt = spec.mgmtInterface ?? "eth0";
|
||||
const mgmtAddress = spec.mgmtAddress ?? "dhcp";
|
||||
const bondMembers = spec.bondMembers ?? [];
|
||||
const vlans = spec.vlans ?? [];
|
||||
|
||||
const sets: VyosSetOp[] = [];
|
||||
const tags: string[][] = [
|
||||
["interfaces", "ethernet"],
|
||||
["system", "login", "user"],
|
||||
];
|
||||
|
||||
const hwIds = spec.hwIds ?? {};
|
||||
const pinHwId = (iface: string): void => {
|
||||
const mac = hwIds[iface];
|
||||
if (mac !== undefined && mac !== "") {
|
||||
sets.push({ path: ["interfaces", "ethernet", iface, "hw-id"], value: mac });
|
||||
}
|
||||
};
|
||||
|
||||
sets.push({ path: ["system", "host-name"], value: params.hostname });
|
||||
|
||||
// Management interface — the NIC that PXE booted, left untagged and unbonded.
|
||||
sets.push({ path: ["interfaces", "ethernet", mgmt, "address"], value: mgmtAddress });
|
||||
pinHwId(mgmt);
|
||||
|
||||
// Tagged management VLAN on the PXE port. Emitted regardless of bonding, so
|
||||
// the box stays reachable on the management VLAN while still booting untagged
|
||||
// on whichever VLAN the bastion's proxy DHCP serves.
|
||||
const mgmtVlan = spec.mgmtVlan;
|
||||
if (mgmtVlan !== undefined) {
|
||||
tags.push(["interfaces", "ethernet", mgmt, "vif"]);
|
||||
const vif = ["interfaces", "ethernet", mgmt, "vif", String(mgmtVlan.id)];
|
||||
sets.push({ path: [...vif, "address"], value: mgmtVlan.address });
|
||||
if (mgmtVlan.description !== undefined && mgmtVlan.description !== "") {
|
||||
sets.push({ path: [...vif, "description"], value: mgmtVlan.description });
|
||||
}
|
||||
}
|
||||
|
||||
// LACP bond. Members must exclude the PXE NIC; firmware PXE cannot run over LACP.
|
||||
const bonded = bondMembers.length > 0;
|
||||
if (bonded) {
|
||||
tags.push(["interfaces", "bonding"]);
|
||||
sets.push({ path: ["interfaces", "bonding", "bond0", "mode"], value: "802.3ad" });
|
||||
sets.push({ path: ["interfaces", "bonding", "bond0", "hash-policy"], value: "layer2+3" });
|
||||
for (const member of bondMembers) {
|
||||
sets.push({
|
||||
path: ["interfaces", "bonding", "bond0", "member", "interface"],
|
||||
value: member,
|
||||
replace: false,
|
||||
});
|
||||
pinHwId(member);
|
||||
}
|
||||
// Address on the trunk's native/untagged VLAN.
|
||||
if (spec.bondAddress !== undefined && spec.bondAddress !== "") {
|
||||
sets.push({ path: ["interfaces", "bonding", "bond0", "address"], value: spec.bondAddress });
|
||||
}
|
||||
}
|
||||
|
||||
// VRRP groups accumulate here; emitted (plus a sync group) after the VLANs.
|
||||
// interface accepts dotted vifs (constraint regex `[0-9]+(.\d+)?`), address
|
||||
// is a tag node (the VIP is the tag value itself), vrid range is 1-255.
|
||||
const vrrpGroups: Array<{ name: string; iface: string; vrid: number; vip: string }> = [];
|
||||
if (bonded && spec.bondVrrp !== undefined && spec.bondVrrp !== "") {
|
||||
// vrid 1 for the untagged group: the native VLAN is never a vif, so this
|
||||
// cannot collide with a vlan-id-derived vrid.
|
||||
vrrpGroups.push({ name: "native", iface: "bond0", vrid: 1, vip: spec.bondVrrp });
|
||||
}
|
||||
|
||||
// Tagged VLAN sub-interfaces hang off the bond when there is one, else off mgmt.
|
||||
const parent = bonded
|
||||
? ["interfaces", "bonding", "bond0"]
|
||||
: ["interfaces", "ethernet", mgmt];
|
||||
if (vlans.length > 0) {
|
||||
tags.push([...parent, "vif"]);
|
||||
const parentName = bonded ? "bond0" : mgmt;
|
||||
for (const vlan of vlans) {
|
||||
const vif = [...parent, "vif", String(vlan.id)];
|
||||
sets.push({ path: [...vif, "address"], value: vlan.address });
|
||||
if (vlan.description !== undefined && vlan.description !== "") {
|
||||
sets.push({ path: [...vif, "description"], value: vlan.description });
|
||||
}
|
||||
if (vlan.vrrp !== undefined && vlan.vrrp !== "") {
|
||||
vrrpGroups.push({
|
||||
name: `vlan${vlan.id}`,
|
||||
iface: `${parentName}.${vlan.id}`,
|
||||
vrid: vlan.id,
|
||||
vip: vlan.vrrp,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Emit VRRP groups plus one sync group so all VLANs fail over together —
|
||||
// without it a single-link event could split mastership across the pair.
|
||||
if (vrrpGroups.length > 0) {
|
||||
tags.push(["high-availability", "vrrp", "group"]);
|
||||
tags.push(["high-availability", "vrrp", "sync-group"]);
|
||||
const priority = String(spec.vrrpPriority ?? 100);
|
||||
for (const g of vrrpGroups) {
|
||||
const base = ["high-availability", "vrrp", "group", g.name];
|
||||
sets.push({ path: [...base, "interface"], value: g.iface });
|
||||
sets.push({ path: [...base, "vrid"], value: String(g.vrid) });
|
||||
sets.push({ path: [...base, "priority"], value: priority });
|
||||
// address is a tag node: the VIP is the path's final segment, no value.
|
||||
sets.push({ path: [...base, "address", g.vip] });
|
||||
tags.push([...base, "address"]);
|
||||
sets.push({
|
||||
path: ["high-availability", "vrrp", "sync-group", "MAIN", "member"],
|
||||
value: g.name,
|
||||
replace: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
sets.push({ path: ["service", "ssh", "port"], value: "22" });
|
||||
|
||||
const sshKeys = params.sshKeys ?? [];
|
||||
if (sshKeys.length > 0) {
|
||||
tags.push(["system", "login", "user", "vyos", "authentication", "public-keys"]);
|
||||
sshKeys.forEach((entry, index) => {
|
||||
const parts = entry.trim().split(/\s+/);
|
||||
const type = parts[0] ?? "";
|
||||
const key = parts[1] ?? "";
|
||||
if (!type.startsWith("ssh-") && !type.startsWith("ecdsa-")) return;
|
||||
if (!key) return;
|
||||
const name = parts[2] ?? `lab-key-${index}`;
|
||||
const base = ["system", "login", "user", "vyos", "authentication", "public-keys", name];
|
||||
sets.push({ path: [...base, "type"], value: type });
|
||||
sets.push({ path: [...base, "key"], value: key });
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
hostname: params.hostname,
|
||||
imageName: "",
|
||||
password: spec.password ?? params.defaultPassword,
|
||||
console: "K",
|
||||
disk: normalizeDiskPath(params.disk),
|
||||
// Static mgmt address wins; under DHCP the driver detects the live IP.
|
||||
reportAddress: mgmtAddress.includes("/") ? (mgmtAddress.split("/")[0] ?? "") : "",
|
||||
raid: false,
|
||||
freshConfig: spec.freshConfig ?? false,
|
||||
sets,
|
||||
tags,
|
||||
};
|
||||
}
|
||||
514
bastion/src/bastion/src/templates/vyos-install.py.ts
Normal file
514
bastion/src/bastion/src/templates/vyos-install.py.ts
Normal file
@@ -0,0 +1,514 @@
|
||||
// Renders the Python program that performs the unattended VyOS install.
|
||||
//
|
||||
// It runs as root inside the live system, fetched and executed by live-config's
|
||||
// `hooks` component (see vyos-boot.ipxe.ts). It does three things:
|
||||
// 1. builds config.boot from the image's own default via vyos.configtree
|
||||
// 2. drives the interactive `install image` through a pty
|
||||
// 3. reports progress back to the bastion, then reboots
|
||||
//
|
||||
// A pty is used rather than piping stdin because the installer reads the
|
||||
// password through getpass(), which opens /dev/tty directly and would ignore a
|
||||
// pipe. Prompts are matched by text rather than replayed positionally: the
|
||||
// installer skips the boot-config question when it finds a previous
|
||||
// installation, so a fixed answer sequence desyncs on reinstall.
|
||||
|
||||
import type { VyosConfigSpec } from "./vyos-config-spec.js";
|
||||
|
||||
export function renderVyosInstallPy(params: {
|
||||
spec: VyosConfigSpec;
|
||||
mac: string;
|
||||
serverIp: string;
|
||||
httpPort: number;
|
||||
role: string;
|
||||
}): string {
|
||||
// Base64 so arbitrary values (passwords, descriptions, SSH keys) can never
|
||||
// terminate the Python string literal that carries them.
|
||||
const specB64 = Buffer.from(JSON.stringify(params.spec), "utf-8").toString("base64");
|
||||
|
||||
return `#!/usr/bin/env python3
|
||||
"""Unattended VyOS install driver -- generated by the lab PXE bastion."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import pty
|
||||
import re
|
||||
import select
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
SPEC = json.loads(base64.b64decode("${specB64}").decode("utf-8"))
|
||||
BASTION = "http://${params.serverIp}:${params.httpPort}"
|
||||
MAC = "${params.mac}"
|
||||
ROLE = ${JSON.stringify(params.role ?? "vanilla")}
|
||||
|
||||
INSTALLER = "/usr/libexec/vyos/op_mode/image_installer.py"
|
||||
CONFIG_DIR = "/opt/vyatta/etc/config"
|
||||
# The installer copies the rootfs from the boot MEDIUM path -- which only a
|
||||
# CD/USB boot provides. With fetch= (HTTP netboot) nothing is mounted there
|
||||
# (verified in VM: Errno 2), so the squashfs must be linked or re-fetched into
|
||||
# place before 'install image' runs.
|
||||
ROOTFS_EXPECTED = "/usr/lib/live/mount/medium/live/filesystem.squashfs"
|
||||
SQUASHFS_URL = "http://${params.serverIp}:${params.httpPort}/vyos-filesystem.squashfs"
|
||||
# The live-config hook runs BEFORE vyos-router creates the /opt/vyatta compat
|
||||
# path, so the squashfs's own location must be tried too (verified in VM: only
|
||||
# /usr/share/vyos/config.boot.default exists at hook time).
|
||||
DEFAULT_CONFIG_CANDIDATES = [
|
||||
"/opt/vyatta/etc/config.boot.default",
|
||||
"/usr/share/vyos/config.boot.default",
|
||||
]
|
||||
STALL_TIMEOUT = 900 # seconds without installer output before giving up
|
||||
|
||||
|
||||
def detect_ip():
|
||||
"""Best-effort local IP as seen on the route toward the bastion.
|
||||
|
||||
Matches Fedora's semantics (IP captured during install): under DHCP the
|
||||
installed system will renew on the same NIC/subnet the live env used.
|
||||
"""
|
||||
import socket
|
||||
try:
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
s.connect(("${params.serverIp}", ${params.httpPort}))
|
||||
ip = s.getsockname()[0]
|
||||
s.close()
|
||||
return ip
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
class LogStreamer:
|
||||
"""Stream install output to the bastion's /api/log so 'labctl provision
|
||||
logs -f' works live for VyOS, like Anaconda's syslog does for Fedora.
|
||||
|
||||
Strictly best-effort: a failed POST drops the batch and must never stall
|
||||
the pty read loop or fail the install.
|
||||
"""
|
||||
|
||||
ANSI = re.compile(rb"\\x1b\\[[0-9;?]*[a-zA-Z]|\\x1b[=>]|\\r")
|
||||
|
||||
def __init__(self):
|
||||
self.partial = b""
|
||||
self.pending = []
|
||||
self.last_flush = time.time()
|
||||
|
||||
def feed(self, chunk):
|
||||
"""Raw pty bytes: split into lines, strip ANSI noise, queue."""
|
||||
self.partial += chunk
|
||||
while b"\\n" in self.partial:
|
||||
raw, self.partial = self.partial.split(b"\\n", 1)
|
||||
text = self.ANSI.sub(b"", raw).decode("utf-8", "replace").rstrip()
|
||||
if text:
|
||||
self.pending.append(text)
|
||||
self.maybe_flush()
|
||||
|
||||
def line(self, text):
|
||||
"""A driver-originated message (already a clean string)."""
|
||||
self.pending.append(text)
|
||||
self.maybe_flush()
|
||||
|
||||
def maybe_flush(self):
|
||||
if len(self.pending) >= 20 or (self.pending and time.time() - self.last_flush >= 2):
|
||||
self.flush()
|
||||
|
||||
def flush(self):
|
||||
if not self.pending:
|
||||
return
|
||||
batch, self.pending = self.pending[:200], self.pending[200:]
|
||||
self.last_flush = time.time()
|
||||
try:
|
||||
body = json.dumps({"mac": MAC, "lines": batch}).encode()
|
||||
req = urllib.request.Request(
|
||||
BASTION + "/api/log",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
urllib.request.urlopen(req, timeout=5).read()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
STREAM = LogStreamer()
|
||||
|
||||
|
||||
def say(msg):
|
||||
"""Print locally and stream to the bastion log buffer."""
|
||||
print(msg)
|
||||
STREAM.line(str(msg))
|
||||
|
||||
|
||||
def report(stage, detail=""):
|
||||
"""Best-effort progress callback; never fatal."""
|
||||
STREAM.flush()
|
||||
try:
|
||||
body = json.dumps({"mac": MAC, "stage": stage, "detail": detail}).encode()
|
||||
req = urllib.request.Request(
|
||||
BASTION + "/api/progress",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
urllib.request.urlopen(req, timeout=5).read()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def build_config():
|
||||
"""Apply our set operations onto the image's own default config.
|
||||
|
||||
Using config.boot.default as the base keeps the vyos-config-version trailer
|
||||
consistent with the running image, so first boot does not run migrations.
|
||||
"""
|
||||
from vyos.configtree import ConfigTree
|
||||
|
||||
default_config = next(
|
||||
(p for p in DEFAULT_CONFIG_CANDIDATES if os.path.exists(p)), None)
|
||||
if default_config is None:
|
||||
raise FileNotFoundError(
|
||||
"no config.boot.default found (tried %s)" % ", ".join(DEFAULT_CONFIG_CANDIDATES))
|
||||
say("base config: %s" % default_config)
|
||||
|
||||
with open(default_config) as handle:
|
||||
config = ConfigTree(handle.read())
|
||||
|
||||
for op in SPEC["sets"]:
|
||||
replace = op.get("replace", True)
|
||||
if "value" in op and op["value"] is not None:
|
||||
config.set(op["path"], value=op["value"], replace=replace)
|
||||
else:
|
||||
config.set(op["path"])
|
||||
|
||||
# Tag nodes must be marked after the nodes exist, as the installer itself does.
|
||||
for tag in SPEC["tags"]:
|
||||
try:
|
||||
config.set_tag(tag)
|
||||
except Exception as err:
|
||||
say("warning: set_tag %s failed: %s" % (tag, err))
|
||||
|
||||
os.makedirs(CONFIG_DIR, exist_ok=True)
|
||||
target = os.path.join(CONFIG_DIR, "config.boot")
|
||||
|
||||
# Re-attach the vyos-config-version footer: ConfigTree.to_string() emits
|
||||
# only the config body, and a config without the footer is treated as
|
||||
# ancient -- the boot migrator then runs every migration over it and (as
|
||||
# observed in the VM test) crashes in system/31-to-32. Building the footer
|
||||
# from the running system pins it to the exact image being installed.
|
||||
body = config.to_string()
|
||||
try:
|
||||
from vyos.component_version import version_info_from_system
|
||||
info = version_info_from_system()
|
||||
info.update_config_body(body)
|
||||
info.write(target)
|
||||
say("wrote %s (footer: %s)" % (target, info.release))
|
||||
except Exception as err:
|
||||
say("warning: version footer failed (%s); writing bare config" % err)
|
||||
with open(target, "w") as handle:
|
||||
handle.write(body)
|
||||
return target
|
||||
|
||||
|
||||
def find_live_squashfs():
|
||||
"""Locate the squashfs live-boot fetched, without walking into the mounted
|
||||
rootfs or overlay (each would mean traversing the entire OS tree)."""
|
||||
explicit = [
|
||||
"/run/live/medium/live/filesystem.squashfs",
|
||||
"/lib/live/mount/medium/live/filesystem.squashfs",
|
||||
]
|
||||
for path in explicit:
|
||||
if os.path.isfile(path) and os.path.getsize(path) > 0:
|
||||
return path
|
||||
for root in ("/run/live", "/lib/live/mount", "/usr/lib/live/mount"):
|
||||
for dirpath, dirs, files in os.walk(root):
|
||||
depth = dirpath.count(os.sep) - root.count(os.sep)
|
||||
dirs[:] = [d for d in dirs
|
||||
if d not in ("rootfs", "overlay")
|
||||
and not d.endswith(".squashfs")
|
||||
and depth < 3]
|
||||
if "filesystem.squashfs" in files:
|
||||
path = os.path.join(dirpath, "filesystem.squashfs")
|
||||
if os.path.isfile(path) and os.path.getsize(path) > 0:
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def ensure_rootfs():
|
||||
"""Make FILE_ROOTFS_SRC exist so the installer can copy the system image."""
|
||||
if os.path.isfile(ROOTFS_EXPECTED) and os.path.getsize(ROOTFS_EXPECTED) > 0:
|
||||
return
|
||||
src = find_live_squashfs()
|
||||
if src is None:
|
||||
say("squashfs not in live mounts; re-fetching %s" % SQUASHFS_URL)
|
||||
src = "/tmp/filesystem.squashfs"
|
||||
urllib.request.urlretrieve(SQUASHFS_URL, src)
|
||||
os.makedirs(os.path.dirname(ROOTFS_EXPECTED), exist_ok=True)
|
||||
if os.path.lexists(ROOTFS_EXPECTED):
|
||||
os.remove(ROOTFS_EXPECTED)
|
||||
os.symlink(src, ROOTFS_EXPECTED)
|
||||
say("rootfs source: %s -> %s" % (ROOTFS_EXPECTED, src))
|
||||
|
||||
|
||||
def build_rules():
|
||||
"""Prompt -> response table for the interactive installer."""
|
||||
password = SPEC["password"].encode() + b"\\n"
|
||||
image_name = SPEC["imageName"].encode() + b"\\n"
|
||||
disk = SPEC["disk"].encode() + b"\\n"
|
||||
console = SPEC["console"].encode() + b"\\n"
|
||||
raid = (b"yes\\n" if SPEC["raid"] else b"no\\n")
|
||||
|
||||
return [
|
||||
(re.compile(rb"Would you like to continue\\?"), b"yes\\n"),
|
||||
(re.compile(rb"What would you like to name this image\\?"), image_name),
|
||||
(re.compile(rb"Please confirm password for the .vyos. user:"), password),
|
||||
(re.compile(rb"Please enter a password for the .vyos. user:"), password),
|
||||
(re.compile(rb"What console should be used by default"), console),
|
||||
# Three RAID variants: "configure RAID-1 mirroring?", "...on them?",
|
||||
# and "choose two disks for RAID-1 mirroring?" -- all default to YES,
|
||||
# so a missed one both hangs the install and risks an unwanted mirror.
|
||||
(re.compile(rb"Would you like to [^?]*RAID-1 mirroring"), raid),
|
||||
(re.compile(rb"Installation will delete all data on (?:the drive|both drives)\\. Continue\\?"), b"yes\\n"),
|
||||
(re.compile(rb"Which one should be used for installation\\?"), disk),
|
||||
(re.compile(rb"Would you like to use all the free space on the drive\\?"), b"yes\\n"),
|
||||
(re.compile(rb"Which file would you like as boot config\\?"), b"1\\n"),
|
||||
# Reinstall path only (search_previous_installation): carrying the old
|
||||
# /config and SSH host keys forward is VyOS's "reinstall without losing
|
||||
# data". Always yes -- freshConfig replaces config.boot afterwards, so
|
||||
# answering no here would also discard non-config data under /config.
|
||||
(re.compile(rb"Would you like to copy data to the new image\\?"), b"yes\\n"),
|
||||
(re.compile(rb"Would you like to copy the encrypted config to the new image\\?"), b"yes\\n"),
|
||||
# More than one previous image found -- take the first offered.
|
||||
(re.compile(rb"From which image would you like to save config information\\?"), b"1\\n"),
|
||||
(re.compile(rb"From which image would you like to copy the encrypted config\\?"), b"1\\n"),
|
||||
]
|
||||
|
||||
|
||||
def run_installer():
|
||||
"""Drive image_installer.py over a pty, answering prompts as they appear."""
|
||||
rules = build_rules()
|
||||
master, slave = pty.openpty()
|
||||
|
||||
proc = subprocess.Popen(
|
||||
[INSTALLER, "--action", "install"],
|
||||
stdin=slave,
|
||||
stdout=slave,
|
||||
stderr=slave,
|
||||
close_fds=True,
|
||||
preexec_fn=os.setsid,
|
||||
)
|
||||
os.close(slave)
|
||||
|
||||
buf = b""
|
||||
transcript = b"" # rolling tail of everything the installer printed
|
||||
last_output = time.time()
|
||||
|
||||
while True:
|
||||
ready, _, _ = select.select([master], [], [], 1.0)
|
||||
|
||||
if ready:
|
||||
try:
|
||||
chunk = os.read(master, 4096)
|
||||
except OSError:
|
||||
break
|
||||
if not chunk:
|
||||
break
|
||||
|
||||
sys.stdout.buffer.write(chunk)
|
||||
sys.stdout.buffer.flush()
|
||||
buf += chunk
|
||||
transcript = (transcript + chunk)[-8000:]
|
||||
STREAM.feed(chunk)
|
||||
last_output = time.time()
|
||||
|
||||
# Answer every prompt currently in the buffer, earliest first, so
|
||||
# ordering is preserved even when the installer skips questions --
|
||||
# and so a single chunk carrying two prompts gets both answers.
|
||||
while True:
|
||||
best = None
|
||||
for pattern, response in rules:
|
||||
found = pattern.search(buf)
|
||||
if found and (best is None or found.start() < best[0].start()):
|
||||
best = (found, response)
|
||||
if best is None:
|
||||
break
|
||||
found, response = best
|
||||
os.write(master, response)
|
||||
transcript = (transcript + b"\\n>>> answered: " + response)[-8000:]
|
||||
STREAM.line(">>> answered: " + response.decode("utf-8", "replace").strip())
|
||||
buf = buf[found.end():]
|
||||
|
||||
# Bound memory if the installer emits a lot without prompting.
|
||||
if len(buf) > 65536:
|
||||
buf = buf[-8192:]
|
||||
|
||||
elif proc.poll() is not None:
|
||||
break
|
||||
|
||||
STREAM.maybe_flush()
|
||||
|
||||
if time.time() - last_output > STALL_TIMEOUT:
|
||||
proc.kill()
|
||||
raise SystemExit("installer produced no output for %ds" % STALL_TIMEOUT)
|
||||
|
||||
os.close(master)
|
||||
return proc.wait(), transcript.decode("utf-8", "replace")
|
||||
|
||||
|
||||
def ensure_network_boot_first():
|
||||
"""Keep network boot first so the bastion intercepts every reboot.
|
||||
|
||||
Port of the Fedora kickstart's %post efibootmgr step (install.ks.ts) --
|
||||
what makes reprovision-by-reboot work. Best-effort: skipped on BIOS boots
|
||||
or when efibootmgr is absent. Runs from the live env after the installer;
|
||||
efibootmgr edits NVRAM, not the disk, so installer cleanup is irrelevant.
|
||||
"""
|
||||
import shutil
|
||||
if not os.path.isdir("/sys/firmware/efi") or shutil.which("efibootmgr") is None:
|
||||
say("boot order: skipped (BIOS boot or efibootmgr missing)")
|
||||
return
|
||||
try:
|
||||
out = subprocess.run(["efibootmgr"], capture_output=True, text=True, timeout=30).stdout
|
||||
order = []
|
||||
network_entry = None
|
||||
for line in out.splitlines():
|
||||
m = re.match(r"^BootOrder:\\s*(.*)$", line)
|
||||
if m:
|
||||
order = [x.strip() for x in m.group(1).split(",") if x.strip()]
|
||||
continue
|
||||
m = re.match(r"^Boot([0-9A-Fa-f]{4})\\*?\\s+(.*)$", line)
|
||||
if m and network_entry is None:
|
||||
if re.search(r"network|pxe|ipv4|ipv6|http", m.group(2), re.IGNORECASE):
|
||||
network_entry = m.group(1).upper()
|
||||
if network_entry is None or not order:
|
||||
say("boot order: no network boot entry found; leaving as is")
|
||||
return
|
||||
new_order = [network_entry] + [x for x in order if x.upper() != network_entry]
|
||||
if [x.upper() for x in order] == [x.upper() for x in new_order]:
|
||||
say("boot order: network entry Boot%s already first" % network_entry)
|
||||
return
|
||||
subprocess.run(["efibootmgr", "-o", ",".join(new_order)],
|
||||
capture_output=True, timeout=30)
|
||||
say("boot order: moved network entry Boot%s first" % network_entry)
|
||||
except Exception as err:
|
||||
say("warning: boot order adjustment failed: %s" % err)
|
||||
|
||||
|
||||
def with_target_mounted(fn):
|
||||
"""Mount the installed root partition, call fn(rw_dir), always unmount.
|
||||
|
||||
The installer has unmounted and cleaned the target by the time this runs,
|
||||
so the block device is free. The partition holding boot/<image>/rw is the
|
||||
VyOS root; the glob also yields the installed image's rw dir directly.
|
||||
"""
|
||||
import glob
|
||||
disk = SPEC["disk"]
|
||||
if not disk:
|
||||
# No pinned disk (installer picked the default) -- enumerate all disks.
|
||||
candidates = ["/dev/" + b for b in os.listdir("/sys/block")
|
||||
if not b.startswith(("loop", "ram", "zram", "sr"))]
|
||||
else:
|
||||
candidates = [disk]
|
||||
|
||||
mnt = "/mnt/lab-target"
|
||||
os.makedirs(mnt, exist_ok=True)
|
||||
for dev in candidates:
|
||||
name = os.path.basename(dev)
|
||||
parts = sorted(p for p in os.listdir("/sys/block/%s" % name)
|
||||
if p.startswith(name)) if os.path.isdir("/sys/block/%s" % name) else []
|
||||
for part in parts:
|
||||
pdev = "/dev/" + part
|
||||
if subprocess.run(["mount", pdev, mnt], capture_output=True).returncode != 0:
|
||||
continue
|
||||
try:
|
||||
rw_dirs = glob.glob(os.path.join(mnt, "boot", "*", "rw"))
|
||||
if rw_dirs:
|
||||
fn(rw_dirs[0])
|
||||
return True
|
||||
finally:
|
||||
subprocess.run(["umount", mnt], capture_output=True)
|
||||
return False
|
||||
|
||||
|
||||
def post_install_target_steps():
|
||||
"""Metadata + optional fresh-config overwrite inside the installed image."""
|
||||
def apply(rw_dir):
|
||||
config_dir = os.path.join(rw_dir, "opt/vyatta/etc/config")
|
||||
os.makedirs(config_dir, exist_ok=True)
|
||||
|
||||
# /config/lab-provisioned -- survives VyOS image upgrades. Mirrors the
|
||||
# Fedora kickstart's /etc/lab-provisioned.
|
||||
try:
|
||||
with open(os.path.join(config_dir, "lab-provisioned"), "w") as handle:
|
||||
handle.write("hostname=%s\\n" % SPEC["hostname"])
|
||||
handle.write("role=%s\\n" % ROLE)
|
||||
handle.write("provisioned=%s\\n" % time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
|
||||
handle.write("bastion=%s\\n" % BASTION)
|
||||
say("wrote /config/lab-provisioned")
|
||||
except Exception as err:
|
||||
say("warning: lab-provisioned metadata failed: %s" % err)
|
||||
|
||||
# freshConfig: make the bastion-generated config win over the previous
|
||||
# installation's carried-forward config. Explicit intent -- failure is
|
||||
# fatal (raised out of with_target_mounted).
|
||||
if SPEC.get("freshConfig"):
|
||||
import shutil
|
||||
shutil.copyfile(os.path.join(CONFIG_DIR, "config.boot"),
|
||||
os.path.join(config_dir, "config.boot"))
|
||||
say("freshConfig: replaced installed config.boot with generated config")
|
||||
|
||||
mounted = with_target_mounted(apply)
|
||||
if not mounted:
|
||||
if SPEC.get("freshConfig"):
|
||||
raise RuntimeError("freshConfig requested but installed root partition not found")
|
||||
say("warning: installed root partition not found; skipping metadata")
|
||||
|
||||
|
||||
def main():
|
||||
report("vyos-install", "building config.boot")
|
||||
try:
|
||||
build_config()
|
||||
except Exception as err:
|
||||
report("error", "config generation failed: %s" % err)
|
||||
raise
|
||||
|
||||
report("vyos-install", "staging rootfs for installer")
|
||||
try:
|
||||
ensure_rootfs()
|
||||
except Exception as err:
|
||||
report("error", "rootfs staging failed: %s" % err)
|
||||
raise
|
||||
|
||||
report("vyos-install", "running install image")
|
||||
code, transcript = run_installer()
|
||||
|
||||
if code != 0:
|
||||
# Surface the installer's last words in bastion progress -- the console
|
||||
# they were printed on is usually invisible during unattended installs.
|
||||
report("error", "install image exited %d | tail: %s" % (code, transcript[-4000:]))
|
||||
raise SystemExit(code)
|
||||
|
||||
report("post-install", "boot order + metadata")
|
||||
ensure_network_boot_first()
|
||||
try:
|
||||
post_install_target_steps()
|
||||
except Exception as err:
|
||||
report("error", "post-install target steps failed: %s" % err)
|
||||
raise
|
||||
|
||||
# "complete" is the stage the bastion uses to move a machine out of the
|
||||
# install queue into installed state, and "ready at <ip>" is the exact
|
||||
# detail format it parses installed.ip from -- see routes/api.ts.
|
||||
ip = SPEC.get("reportAddress") or detect_ip()
|
||||
report("complete", "ready at %s" % ip if ip else "VyOS installed, rebooting")
|
||||
os.system("sync")
|
||||
# --force: this driver is a child of live-config.service, whose start job is
|
||||
# still running -- a normal reboot deadlocks waiting for it (verified in VM:
|
||||
# shutdown blocked >1min on "start job is running for live-config"). The
|
||||
# installer has already unmounted and cleaned the target, so an immediate
|
||||
# reboot is safe.
|
||||
os.system("systemctl reboot --force")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
`;
|
||||
}
|
||||
@@ -22,6 +22,8 @@ function createTestConfig(testDir: string): BastionConfig {
|
||||
dhcpRangeEnd: "",
|
||||
ubuntuVersion: "26.04",
|
||||
ubuntuMirror: "https://releases.ubuntu.com/26.04",
|
||||
vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso",
|
||||
vyosDefaultPassword: "vyos",
|
||||
iface: "eth0",
|
||||
serverIp: "10.0.0.1",
|
||||
network: "10.0.0.0",
|
||||
|
||||
@@ -96,9 +96,9 @@ describe("renderInstallKickstart", () => {
|
||||
expect(ks).toContain("/api/progress");
|
||||
});
|
||||
|
||||
it("infra role has /var/lib/rancher partition", () => {
|
||||
it("infra role has 120G /var/lib/rancher partition", () => {
|
||||
const ks = renderInstallKickstart(baseParams({ role: "infra" }));
|
||||
expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=20480");
|
||||
expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=122880");
|
||||
});
|
||||
|
||||
it("infra role has k3s install", () => {
|
||||
@@ -106,10 +106,14 @@ describe("renderInstallKickstart", () => {
|
||||
expect(ks).toContain("curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true sh -");
|
||||
});
|
||||
|
||||
it("worker role does NOT have /var/lib/rancher partition in fresh install", () => {
|
||||
it("worker role has 120G /var/lib/rancher partition (imageFs must be sized before longhorn --grow)", () => {
|
||||
const ks = renderInstallKickstart(baseParams({ role: "worker" }));
|
||||
// Worker should not have the fresh-install rancher partition line
|
||||
expect(ks).not.toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=20480");
|
||||
expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=122880");
|
||||
});
|
||||
|
||||
it("vanilla role does NOT have /var/lib/rancher partition in fresh install", () => {
|
||||
const ks = renderInstallKickstart(baseParams({ role: "vanilla" }));
|
||||
expect(ks).not.toContain("--name=rancher --fstype=xfs");
|
||||
});
|
||||
|
||||
it("worker role does NOT have k3s install", () => {
|
||||
|
||||
548
bastion/src/bastion/tests/vyos.test.ts
Normal file
548
bastion/src/bastion/tests/vyos.test.ts
Normal file
@@ -0,0 +1,548 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { mkdirSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import type { BastionConfig } from "@lab/shared";
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import { createApp } from "../src/server.js";
|
||||
import type { StateManager } from "../src/services/state.js";
|
||||
import { buildVyosConfigSpec } from "../src/templates/vyos-config-spec.js";
|
||||
import { renderVyosInstallPy } from "../src/templates/vyos-install.py.js";
|
||||
|
||||
function createTestConfig(testDir: string): BastionConfig {
|
||||
return {
|
||||
fedoraVersion: "43",
|
||||
arch: "x86_64",
|
||||
httpPort: 0,
|
||||
timezone: "Europe/London",
|
||||
locale: "en_GB.UTF-8",
|
||||
bastionDir: testDir,
|
||||
domain: "test.local",
|
||||
dhcpMode: "proxy",
|
||||
dhcpRangeStart: "",
|
||||
dhcpRangeEnd: "",
|
||||
ubuntuVersion: "26.04",
|
||||
ubuntuMirror: "https://releases.ubuntu.com/26.04",
|
||||
vyosIsoUrl: "https://example.invalid/vyos.iso",
|
||||
vyosDefaultPassword: "test-pw",
|
||||
iface: "eth0",
|
||||
serverIp: "10.0.0.1",
|
||||
network: "10.0.0.0",
|
||||
gateway: "10.0.0.1",
|
||||
sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST lab@test"],
|
||||
adminUser: "testadmin",
|
||||
syslogPort: 15515,
|
||||
skipDnsmasq: true,
|
||||
skipArtifacts: true,
|
||||
fedoraMirror: "https://example.invalid/fedora",
|
||||
tftpDir: join(testDir, "tftp"),
|
||||
httpDir: join(testDir, "http"),
|
||||
stateFile: join(testDir, "state.json"),
|
||||
};
|
||||
}
|
||||
|
||||
/** Pull the base64 spec back out of the generated Python driver. */
|
||||
function decodeSpecFrom(python: string): Record<string, unknown> {
|
||||
const match = /base64\.b64decode\("([^"]+)"\)/.exec(python);
|
||||
if (!match?.[1]) throw new Error("no base64 spec found in generated driver");
|
||||
return JSON.parse(Buffer.from(match[1], "base64").toString("utf-8"));
|
||||
}
|
||||
|
||||
describe("vyos config spec", () => {
|
||||
it("puts VLANs on the bond when members are given", () => {
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "fw1",
|
||||
defaultPassword: "pw",
|
||||
spec: {
|
||||
mgmtInterface: "eth0",
|
||||
mgmtAddress: "10.0.8.2/24",
|
||||
bondMembers: ["eth2", "eth3"],
|
||||
vlans: [{ id: 10, address: "10.0.10.1/24", description: "k8s" }],
|
||||
},
|
||||
});
|
||||
|
||||
const paths = spec.sets.map((s) => s.path.join(" "));
|
||||
expect(paths).toContain("interfaces bonding bond0 mode");
|
||||
expect(paths).toContain("interfaces bonding bond0 vif 10 address");
|
||||
// VLANs must hang off the bond, not the management NIC.
|
||||
expect(paths).not.toContain("interfaces ethernet eth0 vif 10 address");
|
||||
|
||||
// Bond members are a multi-value node — appending, not replacing, is what
|
||||
// keeps the second member from overwriting the first.
|
||||
const members = spec.sets.filter(
|
||||
(s) => s.path.join(" ") === "interfaces bonding bond0 member interface",
|
||||
);
|
||||
expect(members.map((m) => m.value)).toEqual(["eth2", "eth3"]);
|
||||
expect(members.every((m) => m.replace === false)).toBe(true);
|
||||
});
|
||||
|
||||
it("falls back to VLANs on the management NIC when unbonded", () => {
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "fw2",
|
||||
defaultPassword: "pw",
|
||||
spec: { mgmtInterface: "eth1", vlans: [{ id: 20, address: "10.0.20.1/24" }] },
|
||||
});
|
||||
|
||||
const paths = spec.sets.map((s) => s.path.join(" "));
|
||||
expect(paths).toContain("interfaces ethernet eth1 vif 20 address");
|
||||
});
|
||||
|
||||
it("normalises the target disk to a full /dev path", () => {
|
||||
// find_disks() enumerates with `lsblk -Jbp`, so valid responses are full
|
||||
// paths; a bare name fails valid_responses and re-prompts forever.
|
||||
expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw", disk: "/dev/mmcblk0" }).disk)
|
||||
.toBe("/dev/mmcblk0");
|
||||
expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw", disk: "mmcblk0" }).disk)
|
||||
.toBe("/dev/mmcblk0");
|
||||
expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw" }).disk).toBe("");
|
||||
});
|
||||
|
||||
it("defaults to dhcp on eth0 and never opts into RAID", () => {
|
||||
const spec = buildVyosConfigSpec({ hostname: "fw4", defaultPassword: "pw" });
|
||||
const address = spec.sets.find(
|
||||
(s) => s.path.join(" ") === "interfaces ethernet eth0 address",
|
||||
);
|
||||
expect(address?.value).toBe("dhcp");
|
||||
// The installer's RAID prompt defaults to yes; a second disk must not
|
||||
// silently produce a mirror.
|
||||
expect(spec.raid).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("vyos routes", () => {
|
||||
let testDir: string;
|
||||
let app: FastifyInstance;
|
||||
let state: StateManager;
|
||||
const mac = "aa:bb:cc:11:22:33";
|
||||
|
||||
beforeEach(() => {
|
||||
testDir = join(tmpdir(), `bastion-vyos-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
||||
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||
|
||||
const result = createApp(createTestConfig(testDir));
|
||||
app = result.app;
|
||||
state = result.state;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await app.close();
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("dispatches a queued vyos machine to the live-boot script", async () => {
|
||||
state.update((s) => {
|
||||
s.install_queue[mac] = {
|
||||
hostname: "fw1",
|
||||
disk: "/dev/nvme0n1",
|
||||
role: "worker",
|
||||
os: "vyos-rolling",
|
||||
queued_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}` });
|
||||
expect(response.statusCode).toBe(200);
|
||||
|
||||
expect(response.body).toContain("/vyos-vmlinuz");
|
||||
expect(response.body).toContain("fetch=http://10.0.0.1:0/vyos-filesystem.squashfs");
|
||||
expect(response.body).toContain(`live-config.hooks=http://10.0.0.1:0/vyos/autoinstall.sh?mac=${mac}`);
|
||||
|
||||
// `nonetworking` appears in VyOS's own PXE docs but breaks the hook fetch,
|
||||
// and console=ttyS0 costs 30s per systemd phase on boards with no UART.
|
||||
expect(response.body).not.toContain("nonetworking");
|
||||
expect(response.body).not.toContain("console=ttyS0");
|
||||
});
|
||||
|
||||
it("serves a hook that fetches and executes the install driver", async () => {
|
||||
const response = await app.inject({ method: "GET", url: `/vyos/autoinstall.sh?mac=${mac}` });
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.body).toContain(`/vyos/install.py?mac=${mac}`);
|
||||
expect(response.body).toContain("python3 /tmp/vyos-install.py");
|
||||
});
|
||||
|
||||
it("bakes the machine's config into the generated install driver", async () => {
|
||||
state.update((s) => {
|
||||
s.install_queue[mac] = {
|
||||
hostname: "fw1",
|
||||
disk: "/dev/nvme0n1",
|
||||
role: "worker",
|
||||
os: "vyos-rolling",
|
||||
queued_at: new Date().toISOString(),
|
||||
vyos: {
|
||||
mgmtInterface: "eth0",
|
||||
mgmtAddress: "10.0.8.2/24",
|
||||
bondMembers: ["eth2", "eth3"],
|
||||
vlans: [{ id: 10, address: "10.0.10.1/24" }],
|
||||
password: "s3cret",
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "GET", url: `/vyos/install.py?mac=${mac}` });
|
||||
expect(response.statusCode).toBe(200);
|
||||
|
||||
// Builds config from the image's own default so the vyos-config-version
|
||||
// trailer matches and first boot skips migrations.
|
||||
expect(response.body).toContain("/opt/vyatta/etc/config.boot.default");
|
||||
expect(response.body).toContain("/usr/libexec/vyos/op_mode/image_installer.py");
|
||||
// "complete" is what moves the machine out of the install queue.
|
||||
expect(response.body).toContain('report("complete"');
|
||||
|
||||
const spec = decodeSpecFrom(response.body);
|
||||
expect(spec["hostname"]).toBe("fw1");
|
||||
expect(spec["password"]).toBe("s3cret");
|
||||
expect(spec["disk"]).toBe("/dev/nvme0n1");
|
||||
|
||||
const paths = (spec["sets"] as Array<{ path: string[] }>).map((s) => s.path.join(" "));
|
||||
expect(paths).toContain("interfaces bonding bond0 vif 10 address");
|
||||
expect(paths).toContain("system host-name");
|
||||
});
|
||||
|
||||
it("falls back to the bastion default password when none is set", async () => {
|
||||
state.update((s) => {
|
||||
s.install_queue[mac] = {
|
||||
hostname: "fw9",
|
||||
disk: "",
|
||||
role: "worker",
|
||||
os: "vyos-rolling",
|
||||
queued_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "GET", url: `/vyos/install.py?mac=${mac}` });
|
||||
const spec = decodeSpecFrom(response.body);
|
||||
expect(spec["password"]).toBe("test-pw");
|
||||
// Empty disk means "accept the installer's first-disk default".
|
||||
expect(spec["disk"]).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("vyos hw-id pinning", () => {
|
||||
it("emits hw-id for the mgmt interface and each bond member", () => {
|
||||
// Discovery sees enp2s0/enp1s0f0np0 under Fedora, but VyOS enumerates its
|
||||
// own eth<N>. Pinning by MAC is what makes the mapping deterministic.
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "fw1",
|
||||
defaultPassword: "pw",
|
||||
spec: {
|
||||
mgmtInterface: "eth2",
|
||||
bondMembers: ["eth0", "eth1"],
|
||||
hwIds: {
|
||||
eth2: "64:62:66:25:96:47",
|
||||
eth0: "64:62:66:25:96:45",
|
||||
eth1: "64:62:66:25:96:46",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const hw = spec.sets.filter((s) => s.path[s.path.length - 1] === "hw-id");
|
||||
expect(hw.map((s) => [s.path[2], s.value])).toEqual([
|
||||
["eth2", "64:62:66:25:96:47"],
|
||||
["eth0", "64:62:66:25:96:45"],
|
||||
["eth1", "64:62:66:25:96:46"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("omits hw-id entirely when no mapping is given", () => {
|
||||
const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" });
|
||||
expect(spec.sets.some((s) => s.path.includes("hw-id"))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("vyos management VLAN", () => {
|
||||
it("puts the mgmt VLAN on the PXE port while the bond carries routed VLANs", () => {
|
||||
// Trunked PXE port: boots untagged on the VLAN the bastion serves, stays
|
||||
// reachable on the tagged management VLAN.
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "vyos001",
|
||||
defaultPassword: "pw",
|
||||
spec: {
|
||||
mgmtInterface: "eth2",
|
||||
mgmtAddress: "dhcp",
|
||||
mgmtVlan: { id: 3, address: "192.168.3.4/24", description: "kvm" },
|
||||
bondMembers: ["eth0", "eth1"],
|
||||
vlans: [{ id: 2, address: "192.168.8.2/23" }],
|
||||
},
|
||||
});
|
||||
|
||||
const paths = spec.sets.map((s) => s.path.join(" "));
|
||||
expect(paths).toContain("interfaces ethernet eth2 vif 3 address");
|
||||
expect(paths).toContain("interfaces bonding bond0 vif 2 address");
|
||||
// The mgmt VLAN must not land on the bond.
|
||||
expect(paths).not.toContain("interfaces bonding bond0 vif 3 address");
|
||||
expect(spec.tags.map((t) => t.join(" "))).toContain("interfaces ethernet eth2 vif");
|
||||
});
|
||||
});
|
||||
|
||||
describe("vyos VRRP HA", () => {
|
||||
const haSpec = {
|
||||
mgmtInterface: "eth2",
|
||||
mgmtAddress: "dhcp",
|
||||
bondMembers: ["eth0", "eth1"],
|
||||
bondAddress: "192.168.1.252/24",
|
||||
bondVrrp: "192.168.1.254/24",
|
||||
vrrpPriority: 200,
|
||||
vlans: [
|
||||
{ id: 3, address: "192.168.3.4/24", vrrp: "192.168.3.254/24" },
|
||||
{ id: 200, address: "192.168.2.252/24" }, // no VIP on this one
|
||||
],
|
||||
};
|
||||
|
||||
it("emits a vrrp group per VIP with vrid = VLAN id and dotted vif interface", () => {
|
||||
const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw", spec: haSpec });
|
||||
const paths = spec.sets.map((s) => `${s.path.join(" ")}${s.value !== undefined ? "=" + s.value : ""}`);
|
||||
|
||||
expect(paths).toContain("interfaces bonding bond0 address=192.168.1.252/24");
|
||||
// untagged bond group: vrid 1, interface bond0 itself
|
||||
expect(paths).toContain("high-availability vrrp group native interface=bond0");
|
||||
expect(paths).toContain("high-availability vrrp group native vrid=1");
|
||||
// address is a tag node -- VIP is the final path segment, no value
|
||||
expect(paths).toContain("high-availability vrrp group native address 192.168.1.254/24");
|
||||
// VLAN group: vrid = VLAN id, dotted vif
|
||||
expect(paths).toContain("high-availability vrrp group vlan3 interface=bond0.3");
|
||||
expect(paths).toContain("high-availability vrrp group vlan3 vrid=3");
|
||||
expect(paths).toContain("high-availability vrrp group vlan3 address 192.168.3.254/24");
|
||||
// VLAN without a VIP gets no group
|
||||
expect(paths.some((p) => p.includes("group vlan200"))).toBe(false);
|
||||
});
|
||||
|
||||
it("applies the box-wide priority and one sync group over all groups", () => {
|
||||
const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw", spec: haSpec });
|
||||
const prio = spec.sets.filter((s) => s.path[s.path.length - 1] === "priority"
|
||||
&& s.path[0] === "high-availability");
|
||||
expect(prio).toHaveLength(2);
|
||||
expect(prio.every((s) => s.value === "200")).toBe(true);
|
||||
|
||||
// sync group binds the pair: all groups fail over together
|
||||
const members = spec.sets.filter(
|
||||
(s) => s.path.join(" ") === "high-availability vrrp sync-group MAIN member",
|
||||
);
|
||||
expect(members.map((m) => m.value)).toEqual(["native", "vlan3"]);
|
||||
expect(members.every((m) => m.replace === false)).toBe(true);
|
||||
});
|
||||
|
||||
it("emits no high-availability nodes when no VIPs are given", () => {
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "fw1",
|
||||
defaultPassword: "pw",
|
||||
spec: { bondMembers: ["eth0", "eth1"], vlans: [{ id: 3, address: "192.168.3.4/24" }] },
|
||||
});
|
||||
expect(spec.sets.some((s) => s.path[0] === "high-availability")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("pickLargestInitrd", async () => {
|
||||
const { pickLargestInitrd } = await import("../src/main.js");
|
||||
|
||||
// Verbatim from `xorriso -lsl /live/` on vyos-2026.08.05-0033-rolling.
|
||||
const realListing = `total 8
|
||||
-r--r--r-- 1 0 0 22255 Aug 5 01:33 'filesystem.packages'
|
||||
-r--r--r-- 1 0 0 6 Aug 5 01:33 'filesystem.packages-remove'
|
||||
-r--r--r-- 1 0 0 541192192 Aug 5 01:33 'filesystem.squashfs'
|
||||
-r--r--r-- 1 0 0 50352547 Aug 5 01:33 'initrd.img'
|
||||
-r--r--r-- 1 0 0 50352547 Aug 5 01:33 'initrd.img-6.18.41-vyos'
|
||||
-r--r--r-- 1 0 0 20 Aug 5 01:33 'packages.txt'
|
||||
-r--r--r-- 1 0 0 9135104 Aug 2 19:54 'vmlinuz'
|
||||
-r--r--r-- 1 0 0 9135104 Aug 2 19:54 'vmlinuz-6.18.41-vyos'
|
||||
`;
|
||||
|
||||
it("picks a full-size initrd from a real nightly listing", () => {
|
||||
expect(pickLargestInitrd(realListing)).toEqual({ name: "initrd.img", size: 50352547 });
|
||||
});
|
||||
|
||||
it("ignores 0-byte decoys and symlinks (which report link size, not target size)", () => {
|
||||
const listing = `total 8
|
||||
-r--r--r-- 1 0 0 0 Aug 5 01:33 'initrd.img'
|
||||
lrwxrwxrwx 1 0 0 24 Aug 5 01:33 'initrd.img-link' -> 'initrd.img-6.18.41-vyos'
|
||||
-r--r--r-- 1 0 0 50352547 Aug 5 01:33 'initrd.img-6.18.41-vyos'
|
||||
`;
|
||||
expect(pickLargestInitrd(listing)).toEqual({ name: "initrd.img-6.18.41-vyos", size: 50352547 });
|
||||
});
|
||||
|
||||
it("returns undefined when only decoys exist", () => {
|
||||
expect(pickLargestInitrd("-r--r--r-- 1 0 0 0 Aug 5 01:33 'initrd.img'\n")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("vyos fedora-parity features", () => {
|
||||
it("computes reportAddress from a static mgmt address, empty for dhcp", () => {
|
||||
const staticSpec = buildVyosConfigSpec({
|
||||
hostname: "fw1", defaultPassword: "pw",
|
||||
spec: { mgmtAddress: "192.168.8.2/23" },
|
||||
});
|
||||
expect(staticSpec.reportAddress).toBe("192.168.8.2");
|
||||
|
||||
const dhcpSpec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" });
|
||||
expect(dhcpSpec.reportAddress).toBe("");
|
||||
});
|
||||
|
||||
it("defaults freshConfig off (reinstall preserves the on-disk config)", () => {
|
||||
expect(buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" }).freshConfig).toBe(false);
|
||||
expect(buildVyosConfigSpec({
|
||||
hostname: "fw1", defaultPassword: "pw", spec: { freshConfig: true },
|
||||
}).freshConfig).toBe(true);
|
||||
});
|
||||
|
||||
it("driver streams logs to /api/log and reports 'ready at' on completion", async () => {
|
||||
const testDir = join(tmpdir(), `bastion-vyos-parity-${Date.now()}`);
|
||||
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||
const { app: parityApp, state: parityState } = createApp(createTestConfig(testDir));
|
||||
try {
|
||||
parityState.update((s) => {
|
||||
s.install_queue["aa:bb:cc:44:55:66"] = {
|
||||
hostname: "fw9", disk: "/dev/vda", role: "vanilla",
|
||||
os: "vyos-rolling", queued_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
const response = await parityApp.inject({
|
||||
method: "GET", url: "/vyos/install.py?mac=aa:bb:cc:44:55:66",
|
||||
});
|
||||
expect(response.body).toContain("/api/log");
|
||||
expect(response.body).toContain('"lines": batch');
|
||||
expect(response.body).toContain('report("complete", "ready at %s"');
|
||||
expect(response.body).toContain("ensure_network_boot_first");
|
||||
expect(response.body).toContain("lab-provisioned");
|
||||
expect(response.body).toContain('ROLE = "vanilla"');
|
||||
} finally {
|
||||
await parityApp.close();
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("complete with 'ready at' records installed.ip for a vyos machine", async () => {
|
||||
const testDir = join(tmpdir(), `bastion-vyos-complete-${Date.now()}`);
|
||||
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||
const { app: cApp, state: cState } = createApp(createTestConfig(testDir));
|
||||
try {
|
||||
const mac2 = "aa:bb:cc:77:88:99";
|
||||
cState.update((s) => {
|
||||
s.install_queue[mac2] = {
|
||||
hostname: "fw1", disk: "/dev/vda", role: "vanilla",
|
||||
os: "vyos-rolling", queued_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
const response = await cApp.inject({
|
||||
method: "POST", url: "/api/progress",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ mac: mac2, stage: "complete", detail: "ready at 192.168.8.2" }),
|
||||
});
|
||||
expect(response.statusCode).toBe(200);
|
||||
const installed = cState.load().installed[mac2];
|
||||
expect(installed?.ip).toBe("192.168.8.2");
|
||||
expect(installed?.os).toBe("vyos-rolling");
|
||||
} finally {
|
||||
await cApp.close();
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("vyos installer prompt coverage", () => {
|
||||
// Every interactive prompt image_installer.py can emit, copied verbatim from
|
||||
// the MSG_* constants (including the reinstall-only search_previous_installation
|
||||
// ones). An unanswered prompt does not fail loudly -- the installer simply
|
||||
// blocks on stdin until the driver's stall timeout, which is how the reinstall
|
||||
// path silently hung for 15 minutes in the VM test.
|
||||
const PROMPTS: Record<string, string> = {
|
||||
continue: "Would you like to continue? [y/N] ",
|
||||
imageName: "What would you like to name this image? (Default: 1.5-rolling) ",
|
||||
password: 'Please enter a password for the "vyos" user: ',
|
||||
passwordConfirm: 'Please confirm password for the "vyos" user: ',
|
||||
console: "What console should be used by default? (K: KVM, S: Serial)? (Default: K) ",
|
||||
raidConfigure: "Would you like to configure RAID-1 mirroring? [Y/n] ",
|
||||
raidFoundDisks: "Would you like to configure RAID-1 mirroring on them? [Y/n] ",
|
||||
raidChooseDisks: "Would you like to choose two disks for RAID-1 mirroring? [Y/n] ",
|
||||
diskSelect: "Which one should be used for installation? (Default: /dev/vda) ",
|
||||
diskConfirm: "Installation will delete all data on the drive. Continue? [y/N] ",
|
||||
raidConfirm: "Installation will delete all data on both drives. Continue? [y/N] ",
|
||||
rootSizeAll: "Would you like to use all the free space on the drive? [Y/n] ",
|
||||
bootConfig: "Which file would you like as boot config? ",
|
||||
copyData: "Would you like to copy data to the new image? [Y/n] ",
|
||||
chooseCopyData: "From which image would you like to save config information? ",
|
||||
copyEncData: "Would you like to copy the encrypted config to the new image? [Y/n] ",
|
||||
chooseCopyEncData: "From which image would you like to copy the encrypted config? ",
|
||||
};
|
||||
|
||||
it("answers every installer prompt exactly once", () => {
|
||||
const { execFileSync } = require("node:child_process") as typeof import("node:child_process");
|
||||
const { writeFileSync, unlinkSync, mkdtempSync } = require("node:fs") as typeof import("node:fs");
|
||||
|
||||
// Skip cleanly where python3 is unavailable (same spirit as the
|
||||
// ksvalidator-backed kickstart test).
|
||||
try {
|
||||
execFileSync("python3", ["--version"], { stdio: "pipe" });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "fw1", defaultPassword: "pw", disk: "/dev/vda",
|
||||
});
|
||||
const driver = renderVyosInstallPy({
|
||||
spec, mac: "aa:bb:cc:11:22:33", serverIp: "10.0.0.1", httpPort: 8080, role: "vanilla",
|
||||
});
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "vyos-rules-"));
|
||||
const driverPath = join(dir, "driver.py");
|
||||
const checkPath = join(dir, "check.py");
|
||||
writeFileSync(driverPath, driver);
|
||||
writeFileSync(checkPath, `
|
||||
import importlib.util, json, sys
|
||||
spec = importlib.util.spec_from_file_location("drv", ${JSON.stringify(driverPath)})
|
||||
drv = importlib.util.module_from_spec(spec); spec.loader.exec_module(drv)
|
||||
rules = drv.build_rules()
|
||||
prompts = json.loads(sys.argv[1])
|
||||
out = {}
|
||||
for label, text in prompts.items():
|
||||
out[label] = len([r for p, r in rules if p.search(text.encode())])
|
||||
print(json.dumps(out))
|
||||
`);
|
||||
|
||||
try {
|
||||
const stdout = execFileSync("python3", [checkPath, JSON.stringify(PROMPTS)], {
|
||||
encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
const counts = JSON.parse(stdout) as Record<string, number>;
|
||||
const unanswered = Object.entries(counts).filter(([, n]) => n !== 1);
|
||||
expect(unanswered).toEqual([]);
|
||||
} finally {
|
||||
try { unlinkSync(driverPath); unlinkSync(checkPath); } catch { /* best effort */ }
|
||||
try { rmSync(dir, { recursive: true, force: true }); } catch { /* best effort */ }
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("vyos boot NIC pinning", () => {
|
||||
it("pins the boot interface by MAC via BOOTIF", async () => {
|
||||
// Without this, live-boot picks the first *connected* NIC. On the VP2440
|
||||
// the SFP+ pair links before the copper PXE port, so live-boot tried the
|
||||
// fiber ports (no DHCP), timed out 15s each, and failed with "Unable to
|
||||
// find a live file system on the network".
|
||||
const testDir = join(tmpdir(), `bastion-vyos-bootif-${Date.now()}`);
|
||||
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||
const { app: a, state: st } = createApp(createTestConfig(testDir));
|
||||
try {
|
||||
const m = "64:62:66:25:96:47";
|
||||
st.update((s) => {
|
||||
s.install_queue[m] = {
|
||||
hostname: "vyos001", disk: "/dev/mmcblk0", role: "vanilla",
|
||||
os: "vyos-rolling", queued_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
const res = await a.inject({ method: "GET", url: `/dispatch?mac=${m}` });
|
||||
// live-boot's Device_from_bootif() expects 01-<mac with dashes>
|
||||
expect(res.body).toContain("BOOTIF=01-64-62-66-25-96-47");
|
||||
// and it must be on the kernel line, before fetch= is attempted
|
||||
const kernelLine = res.body.split("\n").find((l) => l.startsWith("kernel "));
|
||||
expect(kernelLine).toContain("BOOTIF=01-64-62-66-25-96-47");
|
||||
expect(kernelLine).toContain("fetch=");
|
||||
} finally {
|
||||
await a.close();
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -90,6 +90,7 @@ export class LabdClient {
|
||||
|
||||
async installMachine(opts: {
|
||||
mac: string; hostname: string; disk?: string; role?: string; os?: string;
|
||||
vyos?: import("@lab/shared").VyosInstallSpec;
|
||||
}): Promise<{ status: string; data?: unknown; error?: string }> {
|
||||
return this.request("POST", "/api/machines/install", { body: opts });
|
||||
}
|
||||
|
||||
@@ -1,10 +1,30 @@
|
||||
// CLI command: provision install
|
||||
// Queue a discovered machine for OS installation via labd.
|
||||
|
||||
import { Command, Option } from "commander";
|
||||
import { Command, Option, InvalidArgumentError } from "commander";
|
||||
import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY } from "@lab/shared";
|
||||
import type { VyosInstallSpec, VyosVlanSpec } from "@lab/shared";
|
||||
import { getLabdClient } from "../api/config.js";
|
||||
|
||||
/** Parse a repeated --vlan flag: "<id>:<cidr>[:<description>]". */
|
||||
export function parseVlan(value: string, previous: VyosVlanSpec[] = []): VyosVlanSpec[] {
|
||||
const parts = value.split(":");
|
||||
const id = Number(parts[0]);
|
||||
const address = parts[1] ?? "";
|
||||
// InvalidArgumentError makes commander print a clean message instead of
|
||||
// dumping a stack trace at the operator.
|
||||
if (!Number.isInteger(id) || id < 1 || id > 4094) {
|
||||
throw new InvalidArgumentError(`Invalid VLAN id in "${value}" (expected 1-4094)`);
|
||||
}
|
||||
if (!address.includes("/")) {
|
||||
throw new InvalidArgumentError(
|
||||
`Invalid VLAN address in "${value}" (expected CIDR, e.g. 10.0.10.1/24)`,
|
||||
);
|
||||
}
|
||||
const description = parts.slice(2).join(":");
|
||||
return [...previous, { id, address, ...(description ? { description } : {}) }];
|
||||
}
|
||||
|
||||
function roleTable(): string {
|
||||
const lines: string[] = ["", "Available roles:"];
|
||||
for (const r of ROLE_REGISTRY) {
|
||||
@@ -15,6 +35,38 @@ function roleTable(): string {
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
/** Parse a repeated --vlan-vip flag: "<id>:<cidr>" — VRRP VIP for a --vlan entry. */
|
||||
export function parseVlanVip(
|
||||
value: string,
|
||||
previous: Record<number, string> = {},
|
||||
): Record<number, string> {
|
||||
const index = value.indexOf(":");
|
||||
const id = Number(index === -1 ? Number.NaN : value.slice(0, index));
|
||||
const cidr = index === -1 ? "" : value.slice(index + 1).trim();
|
||||
if (!Number.isInteger(id) || id < 1 || id > 4094 || !cidr.includes("/")) {
|
||||
throw new InvalidArgumentError(
|
||||
`Invalid VLAN VIP "${value}" (expected <id>:<cidr>, e.g. 3:192.168.3.254/24)`,
|
||||
);
|
||||
}
|
||||
return { ...previous, [id]: cidr };
|
||||
}
|
||||
|
||||
/** Parse a repeated --vyos-hwid flag: "<iface>=<mac>". */
|
||||
export function parseHwId(
|
||||
value: string,
|
||||
previous: Record<string, string> = {},
|
||||
): Record<string, string> {
|
||||
const index = value.indexOf("=");
|
||||
const iface = index === -1 ? "" : value.slice(0, index).trim();
|
||||
const mac = index === -1 ? "" : value.slice(index + 1).trim().toLowerCase();
|
||||
if (iface === "" || !/^([0-9a-f]{2}:){5}[0-9a-f]{2}$/.test(mac)) {
|
||||
throw new InvalidArgumentError(
|
||||
`Invalid hw-id "${value}" (expected <iface>=<mac>, e.g. eth2=64:62:66:25:96:47)`,
|
||||
);
|
||||
}
|
||||
return { ...previous, [iface]: mac };
|
||||
}
|
||||
|
||||
export function registerInstallCommand(parent: Command): void {
|
||||
parent
|
||||
.command("install <mac> <hostname>")
|
||||
@@ -24,10 +76,34 @@ export function registerInstallCommand(parent: Command): void {
|
||||
.addOption(new Option("--role <role>", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker"))
|
||||
.addOption(new Option("--os <os>", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43"))
|
||||
.option("--disk <device>", "Target disk device (auto-detect if omitted)")
|
||||
.option("--vyos-mgmt <iface>", "VyOS: untagged interface the machine PXE boots from (default eth0)")
|
||||
.option("--vyos-mgmt-address <addr>", "VyOS: CIDR for the management interface, or 'dhcp' (default dhcp)")
|
||||
.option("--vyos-bond <ifaces>", "VyOS: comma-separated LACP bond members (must exclude the PXE NIC)")
|
||||
.option("--vyos-bond-address <cidr>", "VyOS: address on the untagged bond (trunk native VLAN)")
|
||||
.option("--vyos-bond-vrrp <cidr>", "VyOS: VRRP VIP floated on the untagged bond")
|
||||
.option("--vlan-vip <id:cidr>", "VyOS: VRRP VIP for a --vlan entry (repeatable)", parseVlanVip)
|
||||
.option("--vyos-vrrp-priority <n>", "VyOS: VRRP priority for all groups on this box (higher = master)")
|
||||
.option("--vyos-mgmt-vlan <id:cidr[:desc]>", "VyOS: tagged management VLAN on the PXE port")
|
||||
.option("--vlan <id:cidr[:desc]>", "VyOS: tagged VLAN sub-interface on the bond (repeatable)", parseVlan)
|
||||
.option("--vyos-password <password>", "VyOS: password for the 'vyos' user")
|
||||
.option("--vyos-hwid <iface=mac>", "VyOS: pin an interface name to a MAC via hw-id (repeatable)", parseHwId)
|
||||
.option("--vyos-fresh-config", "VyOS: on reinstall, overwrite the preserved config with the generated one")
|
||||
.action(async (mac: string, hostname: string, opts: {
|
||||
role: string;
|
||||
os: string;
|
||||
disk?: string;
|
||||
vyosMgmt?: string;
|
||||
vyosMgmtAddress?: string;
|
||||
vyosBond?: string;
|
||||
vyosBondAddress?: string;
|
||||
vyosBondVrrp?: string;
|
||||
vlan?: VyosVlanSpec[];
|
||||
vlanVip?: Record<number, string>;
|
||||
vyosVrrpPriority?: string;
|
||||
vyosMgmtVlan?: string;
|
||||
vyosPassword?: string;
|
||||
vyosHwid?: Record<string, string>;
|
||||
vyosFreshConfig?: boolean;
|
||||
}) => {
|
||||
if (!isValidOsId(opts.os)) {
|
||||
console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`);
|
||||
@@ -39,6 +115,67 @@ export function registerInstallCommand(parent: Command): void {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const bondMembers = opts.vyosBond !== undefined && opts.vyosBond !== ""
|
||||
? opts.vyosBond.split(",").map((s) => s.trim()).filter((s) => s.length > 0)
|
||||
: [];
|
||||
|
||||
// Attach --vlan-vip entries to their --vlan definitions. A VIP for a VLAN
|
||||
// that was never defined is a typo that would otherwise vanish silently.
|
||||
const vips = opts.vlanVip ?? {};
|
||||
const vlans = (opts.vlan ?? []).map((v) =>
|
||||
vips[v.id] !== undefined ? { ...v, vrrp: vips[v.id] as string } : v,
|
||||
);
|
||||
for (const id of Object.keys(vips)) {
|
||||
if (!vlans.some((v) => String(v.id) === id)) {
|
||||
console.error(`--vlan-vip ${id}:... has no matching --vlan ${id}:... entry`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
const vrrpPriority = opts.vyosVrrpPriority !== undefined && opts.vyosVrrpPriority !== ""
|
||||
? Number(opts.vyosVrrpPriority)
|
||||
: undefined;
|
||||
if (vrrpPriority !== undefined
|
||||
&& (!Number.isInteger(vrrpPriority) || vrrpPriority < 1 || vrrpPriority > 255)) {
|
||||
console.error(`--vyos-vrrp-priority must be an integer 1-255 (got ${opts.vyosVrrpPriority})`);
|
||||
process.exit(1);
|
||||
}
|
||||
const vyos: VyosInstallSpec = {
|
||||
...(opts.vyosMgmt !== undefined && opts.vyosMgmt !== ""
|
||||
? { mgmtInterface: opts.vyosMgmt } : {}),
|
||||
...(opts.vyosMgmtAddress !== undefined && opts.vyosMgmtAddress !== ""
|
||||
? { mgmtAddress: opts.vyosMgmtAddress } : {}),
|
||||
...(bondMembers.length > 0 ? { bondMembers } : {}),
|
||||
...(opts.vyosBondAddress !== undefined && opts.vyosBondAddress !== ""
|
||||
? { bondAddress: opts.vyosBondAddress } : {}),
|
||||
...(opts.vyosBondVrrp !== undefined && opts.vyosBondVrrp !== ""
|
||||
? { bondVrrp: opts.vyosBondVrrp } : {}),
|
||||
...(vrrpPriority !== undefined ? { vrrpPriority } : {}),
|
||||
...(vlans.length > 0 ? { vlans } : {}),
|
||||
...(opts.vyosPassword !== undefined && opts.vyosPassword !== ""
|
||||
? { password: opts.vyosPassword } : {}),
|
||||
...(opts.vyosHwid !== undefined && Object.keys(opts.vyosHwid).length > 0
|
||||
? { hwIds: opts.vyosHwid } : {}),
|
||||
...(opts.vyosMgmtVlan !== undefined && opts.vyosMgmtVlan !== ""
|
||||
? { mgmtVlan: parseVlan(opts.vyosMgmtVlan)[0] as VyosVlanSpec } : {}),
|
||||
...(opts.vyosFreshConfig === true ? { freshConfig: true } : {}),
|
||||
};
|
||||
const hasVyosOptions = Object.keys(vyos).length > 0;
|
||||
|
||||
if (hasVyosOptions && !opts.os.startsWith("vyos")) {
|
||||
console.error(`VyOS options require --os vyos-rolling (got --os ${opts.os})`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Firmware PXE cannot run over LACP, so the NIC that boots the installer
|
||||
// must stay out of the bond — otherwise the next reinstall has no path in.
|
||||
const mgmt = vyos.mgmtInterface ?? "eth0";
|
||||
if (bondMembers.includes(mgmt)) {
|
||||
console.error(`--vyos-bond must not include the PXE/management interface "${mgmt}"`);
|
||||
console.error("PXE cannot boot over an LACP bond; keep that NIC unbonded.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await getLabdClient().installMachine({
|
||||
mac,
|
||||
@@ -46,11 +183,14 @@ export function registerInstallCommand(parent: Command): void {
|
||||
role: opts.role,
|
||||
os: opts.os,
|
||||
...(opts.disk ? { disk: opts.disk } : {}),
|
||||
...(hasVyosOptions ? { vyos } : {}),
|
||||
});
|
||||
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
console.log("");
|
||||
const osLabel = opts.os.startsWith("ubuntu") ? "Ubuntu" : "Fedora";
|
||||
const osLabel = opts.os.startsWith("ubuntu")
|
||||
? "Ubuntu"
|
||||
: opts.os.startsWith("vyos") ? "VyOS" : "Fedora";
|
||||
console.log(`Power on the machine to start ${osLabel} installation.`);
|
||||
|
||||
const roleInfo = ROLE_REGISTRY.find(r => r.name === opts.role);
|
||||
|
||||
@@ -44,11 +44,14 @@ export function registerRecheckCommand(parent: Command): void {
|
||||
}
|
||||
|
||||
// Build list of machines to check
|
||||
const targets: Array<{ mac: string; hostname: string; ip: string }> = [];
|
||||
const targets: Array<{ mac: string; hostname: string; ip: string; sshUser: string }> = [];
|
||||
const userIsDefault = opts.user === "root";
|
||||
for (const [mac, info] of Object.entries(state.installed)) {
|
||||
if (!info.ip) continue;
|
||||
if (opts.target && info.hostname !== opts.target && mac !== opts.target) continue;
|
||||
targets.push({ mac, hostname: info.hostname, ip: info.ip });
|
||||
// VyOS boxes only have the "vyos" login; honor an explicit --user.
|
||||
const sshUser = userIsDefault && (info.os ?? "").startsWith("vyos") ? "vyos" : opts.user;
|
||||
targets.push({ mac, hostname: info.hostname, ip: info.ip, sshUser });
|
||||
}
|
||||
|
||||
if (targets.length === 0) {
|
||||
@@ -61,12 +64,12 @@ export function registerRecheckCommand(parent: Command): void {
|
||||
let updated = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const { mac, hostname, ip } of targets) {
|
||||
for (const { mac, hostname, ip, sshUser } of targets) {
|
||||
process.stdout.write(` ${hostname.padEnd(24)} ${DIM}(${ip})${RESET} `);
|
||||
|
||||
try {
|
||||
const t0 = Date.now();
|
||||
const result = await sshExec(ip, opts.user, HW_COLLECT_SCRIPT, SSH_OPTS);
|
||||
const result = await sshExec(ip, sshUser, HW_COLLECT_SCRIPT, SSH_OPTS);
|
||||
const elapsed = Date.now() - t0;
|
||||
if (result.exitCode !== 0) {
|
||||
console.log(`${RED}SSH failed (exit ${result.exitCode}, ${elapsed}ms)${RESET}`);
|
||||
|
||||
@@ -24,12 +24,12 @@ function roleTable(): string {
|
||||
function resolveTarget(
|
||||
target: string,
|
||||
state: BastionState,
|
||||
): { mac: string; hostname: string; ip: string } | null {
|
||||
): { mac: string; hostname: string; ip: string; os?: string } | null {
|
||||
const normalized = target.toLowerCase().replace(/-/g, ":");
|
||||
|
||||
if (state.installed[normalized]) {
|
||||
const info = state.installed[normalized];
|
||||
return { mac: normalized, hostname: info.hostname, ip: info.ip };
|
||||
return { mac: normalized, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
|
||||
}
|
||||
|
||||
if (state.discovered[normalized]) {
|
||||
@@ -38,13 +38,13 @@ function resolveTarget(
|
||||
|
||||
for (const [mac, info] of Object.entries(state.installed)) {
|
||||
if (info.hostname === target || info.hostname.startsWith(target + ".")) {
|
||||
return { mac, hostname: info.hostname, ip: info.ip };
|
||||
return { mac, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
|
||||
}
|
||||
}
|
||||
|
||||
for (const [mac, info] of Object.entries(state.installed)) {
|
||||
if (info.ip === target) {
|
||||
return { mac, hostname: info.hostname, ip: info.ip };
|
||||
return { mac, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,10 +60,12 @@ export function registerReprovisionCommand(parent: Command): void {
|
||||
.addOption(new Option("--role <role>", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker"))
|
||||
.addOption(new Option("--os <os>", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43"))
|
||||
.option("--disk <device>", "Target disk device (auto-detect if omitted)")
|
||||
.option("--user <user>", "SSH user for the reboot (default: vyos for VyOS machines, else current user)")
|
||||
.action(async (target: string, hostnameOverride: string | undefined, opts: {
|
||||
role: string;
|
||||
os: string;
|
||||
disk?: string;
|
||||
user?: string;
|
||||
}) => {
|
||||
if (!isValidOsId(opts.os)) {
|
||||
console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`);
|
||||
@@ -123,7 +125,11 @@ export function registerReprovisionCommand(parent: Command): void {
|
||||
return;
|
||||
}
|
||||
|
||||
const adminUser = process.env["SUDO_USER"] ?? process.env["USER"] ?? "";
|
||||
// SSH user: explicit flag > the machine's current OS (VyOS boxes only
|
||||
// have the "vyos" login) > the invoking user.
|
||||
const currentOsIsVyos = (resolved.os ?? "").startsWith("vyos");
|
||||
const adminUser = opts.user
|
||||
?? (currentOsIsVyos ? "vyos" : (process.env["SUDO_USER"] ?? process.env["USER"] ?? ""));
|
||||
const effectiveUser = adminUser === "root" ? "" : adminUser;
|
||||
|
||||
if (effectiveUser === "") {
|
||||
|
||||
@@ -11,7 +11,7 @@ export function registerStartCommand(parent: Command): void {
|
||||
.command("start")
|
||||
.description("Start the bastion server (HTTP + dnsmasq PXE)")
|
||||
.option("--port <port>", "HTTP port", "8080")
|
||||
.option("--dir <dir>", "Bastion data directory", "/tmp/lab-bastion")
|
||||
.option("--dir <dir>", "Bastion data directory", process.env["BASTION_DIR"] ?? "/tmp/lab-bastion")
|
||||
.option("--domain <domain>", "Internal domain for hostnames", "ad.itaz.eu")
|
||||
.option("--dhcp-mode <mode>", "DHCP mode: proxy or full", "proxy")
|
||||
.option("--fedora <version>", "Fedora version", "43")
|
||||
|
||||
@@ -8,7 +8,7 @@ export function registerStopCommand(parent: Command): void {
|
||||
parent
|
||||
.command("stop")
|
||||
.description("Stop a running bastion server")
|
||||
.option("--dir <dir>", "Bastion data directory", "/tmp/lab-bastion")
|
||||
.option("--dir <dir>", "Bastion data directory", process.env["BASTION_DIR"] ?? "/tmp/lab-bastion")
|
||||
.action((opts: { dir: string }) => {
|
||||
const pidFile = `${opts.dir}/bastion.pid`;
|
||||
|
||||
|
||||
35
bastion/src/cli/tests/install-vyos.test.ts
Normal file
35
bastion/src/cli/tests/install-vyos.test.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
// Tests for VyOS install option parsing.
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { parseVlan } from "../src/commands/install.js";
|
||||
|
||||
describe("parseVlan", () => {
|
||||
it("parses id and CIDR", () => {
|
||||
expect(parseVlan("10:10.0.10.1/24")).toEqual([{ id: 10, address: "10.0.10.1/24" }]);
|
||||
});
|
||||
|
||||
it("accumulates across repeated flags", () => {
|
||||
const first = parseVlan("10:10.0.10.1/24");
|
||||
const both = parseVlan("20:10.0.20.1/24", first);
|
||||
expect(both).toHaveLength(2);
|
||||
expect(both[1]).toEqual({ id: 20, address: "10.0.20.1/24" });
|
||||
});
|
||||
|
||||
it("keeps a description, including one containing colons", () => {
|
||||
expect(parseVlan("30:10.0.30.1/24:mgmt:secondary")).toEqual([
|
||||
{ id: 30, address: "10.0.30.1/24", description: "mgmt:secondary" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects an address that is not CIDR", () => {
|
||||
// A bare address would produce a VyOS config that fails to commit on first
|
||||
// boot, long after the operator has stopped watching.
|
||||
expect(() => parseVlan("10:10.0.10.1")).toThrow(/CIDR/);
|
||||
});
|
||||
|
||||
it("rejects out-of-range and non-numeric VLAN ids", () => {
|
||||
expect(() => parseVlan("0:10.0.10.1/24")).toThrow(/1-4094/);
|
||||
expect(() => parseVlan("4095:10.0.10.1/24")).toThrow(/1-4094/);
|
||||
expect(() => parseVlan("abc:10.0.10.1/24")).toThrow(/1-4094/);
|
||||
});
|
||||
});
|
||||
23
bastion/src/core/package.json
Normal file
23
bastion/src/core/package.json
Normal file
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "@lab/core",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"import": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc --build",
|
||||
"clean": "rimraf dist",
|
||||
"test": "vitest",
|
||||
"test:run": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@pulumi/pulumi": "^3.0.0"
|
||||
}
|
||||
}
|
||||
75
bastion/src/core/src/audit.ts
Normal file
75
bastion/src/core/src/audit.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
// Audit event types for the labctl platform.
|
||||
// Every mutation is tracked with correlation IDs for causal chains.
|
||||
|
||||
export type AuditEventKind =
|
||||
| "resource_created"
|
||||
| "resource_updated"
|
||||
| "resource_deleted"
|
||||
| "resource_state_change"
|
||||
| "plan_generated"
|
||||
| "apply_started"
|
||||
| "apply_step"
|
||||
| "apply_completed"
|
||||
| "driver_translate"
|
||||
| "driver_execute"
|
||||
| "driver_error"
|
||||
| "fleet_discovery"
|
||||
| "fleet_classification"
|
||||
| "fleet_approval"
|
||||
| "fleet_auto_approve"
|
||||
| "pipeline_started"
|
||||
| "pipeline_step_started"
|
||||
| "pipeline_step_completed"
|
||||
| "pipeline_completed"
|
||||
| "deploy_started"
|
||||
| "deploy_completed"
|
||||
| "deploy_failed"
|
||||
| "drift_detected"
|
||||
| "drift_corrected"
|
||||
| "sync_triggered"
|
||||
| "sync_completed"
|
||||
| "auth_login"
|
||||
| "auth_logout"
|
||||
| "auth_bootstrap"
|
||||
| "rbac_decision"
|
||||
| "impersonation"
|
||||
| "server_started"
|
||||
| "controller_started"
|
||||
| "agent_connected"
|
||||
| "agent_disconnected"
|
||||
| "bastion_registered";
|
||||
|
||||
export type AuditSource =
|
||||
| "cli"
|
||||
| "labd"
|
||||
| "agent"
|
||||
| "driver"
|
||||
| "fleet-controller"
|
||||
| "sync-controller";
|
||||
|
||||
export type AuditResult = "success" | "failure" | "denied" | "skipped";
|
||||
|
||||
export interface AuditEvent {
|
||||
id: string;
|
||||
timestamp: Date;
|
||||
eventKind: AuditEventKind;
|
||||
source: AuditSource;
|
||||
verified: boolean;
|
||||
|
||||
userId?: string;
|
||||
userName?: string;
|
||||
sessionId?: string;
|
||||
environmentName?: string;
|
||||
accountName?: string;
|
||||
|
||||
resourceKind?: string;
|
||||
resourceName?: string;
|
||||
|
||||
correlationId: string;
|
||||
parentEventId?: string;
|
||||
|
||||
details: Record<string, unknown>;
|
||||
result: AuditResult;
|
||||
error?: string;
|
||||
durationMs?: number;
|
||||
}
|
||||
50
bastion/src/core/src/auth.ts
Normal file
50
bastion/src/core/src/auth.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
// Auth types for the labctl platform.
|
||||
// Bearer token auth for CLI/SDK. mTLS stays for agent/bastion.
|
||||
|
||||
export type UserRole = "USER" | "ADMIN";
|
||||
|
||||
export interface User {
|
||||
id: string;
|
||||
email: string;
|
||||
name?: string;
|
||||
role: UserRole;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export interface Session {
|
||||
id: string;
|
||||
userId: string;
|
||||
token: string;
|
||||
expiresAt: Date;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export interface Group {
|
||||
id: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
export type SubjectKind = "User" | "Group" | "ServiceAccount";
|
||||
|
||||
export interface RoleBinding {
|
||||
role: "view" | "edit" | "create" | "delete" | "run" | "admin";
|
||||
resource: string;
|
||||
name?: string;
|
||||
environment?: string;
|
||||
action?: string;
|
||||
}
|
||||
|
||||
export interface RbacSubject {
|
||||
kind: SubjectKind;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface RbacDefinition {
|
||||
id: string;
|
||||
name: string;
|
||||
subjects: RbacSubject[];
|
||||
roleBindings: RoleBinding[];
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
24
bastion/src/core/src/environment.ts
Normal file
24
bastion/src/core/src/environment.ts
Normal file
@@ -0,0 +1,24 @@
|
||||
// Environment and Account types.
|
||||
// An Environment is a logical boundary (production, staging, dev).
|
||||
// An Account is a configured driver instance with credentials.
|
||||
|
||||
export interface Environment {
|
||||
id: string;
|
||||
name: string;
|
||||
status: "active" | "archived";
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export interface Account {
|
||||
id: string;
|
||||
name: string;
|
||||
driver: string;
|
||||
config: Record<string, unknown>;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export interface Binding {
|
||||
id: string;
|
||||
environmentId: string;
|
||||
accountId: string;
|
||||
}
|
||||
9
bastion/src/core/src/index.ts
Normal file
9
bastion/src/core/src/index.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
// @lab/core — foundation types for the labctl platform.
|
||||
// Phase 1 stub: resource types, auth types, audit types, Output<T>.
|
||||
// Phase 5 adds: CompositeResource, evaluator integration, full SDK.
|
||||
|
||||
export * from "./resource.js";
|
||||
export * from "./environment.js";
|
||||
export * from "./audit.js";
|
||||
export * from "./auth.js";
|
||||
export { Output, output, all, interpolate, secret } from "./output.js";
|
||||
5
bastion/src/core/src/output.ts
Normal file
5
bastion/src/core/src/output.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
// Re-export Pulumi's Output<T> type for use across the platform.
|
||||
// Cloud drivers use this for future values (endpoints, IPs, kubeconfigs).
|
||||
// Phase 1: type re-export only. Phase 5 adds full evaluator integration.
|
||||
|
||||
export { Output, output, all, interpolate, secret } from "@pulumi/pulumi";
|
||||
83
bastion/src/core/src/resource.ts
Normal file
83
bastion/src/core/src/resource.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
// Core resource types for the labctl platform.
|
||||
// Every managed thing (Server, Database, App, Cluster) is a Resource.
|
||||
|
||||
export type ResourceOrigin = "file" | "cli" | "fleet" | "imported";
|
||||
export type ResourceManagedBy = "gitops" | "manual" | "auto";
|
||||
|
||||
export type ResourceStatus =
|
||||
| "pending"
|
||||
| "creating"
|
||||
| "ready"
|
||||
| "updating"
|
||||
| "deleting"
|
||||
| "error"
|
||||
| "unknown";
|
||||
|
||||
export interface ResourceMetadata {
|
||||
kind: string;
|
||||
name: string;
|
||||
environmentId: string;
|
||||
accountId: string;
|
||||
origin: ResourceOrigin;
|
||||
managedBy: ResourceManagedBy;
|
||||
sourceRef?: string;
|
||||
}
|
||||
|
||||
export interface ResourceState {
|
||||
status: ResourceStatus;
|
||||
message?: string;
|
||||
lastReconciled?: Date;
|
||||
platformRef?: string;
|
||||
}
|
||||
|
||||
export interface Resource<TSpec = Record<string, unknown>> {
|
||||
id: string;
|
||||
metadata: ResourceMetadata;
|
||||
desiredSpec: TSpec;
|
||||
actualSpec?: TSpec;
|
||||
state: ResourceState;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
// Well-known resource kinds. Drivers register additional kinds.
|
||||
export const RESOURCE_KINDS = {
|
||||
SERVER: "server",
|
||||
DATABASE: "database",
|
||||
CACHE: "cache",
|
||||
CLUSTER: "cluster",
|
||||
APP: "app",
|
||||
SERVICE: "service",
|
||||
CRONJOB: "cronjob",
|
||||
NETWORK: "network",
|
||||
LOADBALANCER: "loadbalancer",
|
||||
DNSZONE: "dnszone",
|
||||
CERTIFICATE: "certificate",
|
||||
OBJECTSTORE: "objectstore",
|
||||
QUEUE: "queue",
|
||||
SECRET: "secret",
|
||||
FLEET: "fleet",
|
||||
} as const;
|
||||
|
||||
export type ResourceKind = (typeof RESOURCE_KINDS)[keyof typeof RESOURCE_KINDS];
|
||||
|
||||
// Resource aliases for CLI (kubectl-style shortnames)
|
||||
export const RESOURCE_ALIASES: Record<string, string> = {
|
||||
srv: "server",
|
||||
db: "database",
|
||||
cl: "cluster",
|
||||
svc: "service",
|
||||
cj: "cronjob",
|
||||
lb: "loadbalancer",
|
||||
dns: "dnszone",
|
||||
cert: "certificate",
|
||||
os: "objectstore",
|
||||
mq: "queue",
|
||||
sec: "secret",
|
||||
fl: "fleet",
|
||||
};
|
||||
|
||||
export function resolveResourceKind(input: string): string {
|
||||
const lower = input.toLowerCase();
|
||||
return RESOURCE_ALIASES[lower] ?? lower;
|
||||
}
|
||||
8
bastion/src/core/tsconfig.json
Normal file
8
bastion/src/core/tsconfig.json
Normal file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "src",
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
@@ -26,8 +26,10 @@
|
||||
"dependencies": {
|
||||
"@fastify/rate-limit": "^10.3.0",
|
||||
"@fastify/websocket": "^11.0.2",
|
||||
"@lab/core": "workspace:^",
|
||||
"@lab/shared": "workspace:*",
|
||||
"@prisma/client": "^6.9.0",
|
||||
"bcryptjs": "^3.0.3",
|
||||
"fastify": "^5.3.3",
|
||||
"winston": "^3.17.0",
|
||||
"ws": "^8.19.0",
|
||||
@@ -37,6 +39,7 @@
|
||||
"seed": "tsx prisma/seed.ts"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bcryptjs": "^3.0.0",
|
||||
"@types/node": "^22.14.1",
|
||||
"@types/ws": "^8.18.1",
|
||||
"prisma": "^6.9.0",
|
||||
|
||||
@@ -7,23 +7,241 @@ datasource db {
|
||||
url = env("DATABASE_URL")
|
||||
}
|
||||
|
||||
// ── Auth (mcpctl pattern: email/password + bearer token sessions) ──
|
||||
|
||||
model User {
|
||||
id String @id @default(cuid())
|
||||
email String @unique
|
||||
password String // bcrypt
|
||||
name String?
|
||||
role UserRole @default(USER)
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
sessions Session[]
|
||||
auditLogs AuditEvent[]
|
||||
groups GroupMember[]
|
||||
}
|
||||
|
||||
enum UserRole {
|
||||
USER
|
||||
ADMIN
|
||||
}
|
||||
|
||||
model Session {
|
||||
id String @id @default(cuid())
|
||||
userId String
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
token String @unique
|
||||
expiresAt DateTime
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
@@index([userId])
|
||||
@@index([token])
|
||||
}
|
||||
|
||||
model Group {
|
||||
id String @id @default(cuid())
|
||||
name String @unique
|
||||
description String?
|
||||
createdAt DateTime @default(now())
|
||||
members GroupMember[]
|
||||
}
|
||||
|
||||
model GroupMember {
|
||||
id String @id @default(cuid())
|
||||
groupId String
|
||||
group Group @relation(fields: [groupId], references: [id], onDelete: Cascade)
|
||||
userId String
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@unique([groupId, userId])
|
||||
}
|
||||
|
||||
model ServiceAccount {
|
||||
id String @id @default(cuid())
|
||||
name String @unique
|
||||
token String @unique
|
||||
createdAt DateTime @default(now())
|
||||
}
|
||||
|
||||
// ── RBAC (mcpctl pattern: named definitions with JSON subjects/bindings) ──
|
||||
|
||||
model RbacDefinition {
|
||||
id String @id @default(cuid())
|
||||
name String @unique
|
||||
subjects Json // [{kind: "User"|"Group"|"ServiceAccount", name: string}]
|
||||
roleBindings Json // [{role, resource, name?, environment?, action?}]
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
}
|
||||
|
||||
// ── Audit (mcpctl pattern: fire-and-forget with correlation IDs) ──
|
||||
|
||||
model AuditEvent {
|
||||
id String @id @default(cuid())
|
||||
timestamp DateTime @default(now())
|
||||
eventKind String
|
||||
source String // cli | labd | agent | driver | fleet-controller | sync-controller
|
||||
verified Boolean @default(false)
|
||||
|
||||
userId String?
|
||||
user User? @relation(fields: [userId], references: [id])
|
||||
userName String?
|
||||
sessionId String?
|
||||
environmentName String?
|
||||
accountName String?
|
||||
|
||||
resourceKind String?
|
||||
resourceName String?
|
||||
|
||||
correlationId String
|
||||
parentEventId String?
|
||||
|
||||
details Json @default("{}")
|
||||
result String // success | failure | denied | skipped
|
||||
error String?
|
||||
durationMs Int?
|
||||
|
||||
@@index([correlationId])
|
||||
@@index([eventKind, timestamp])
|
||||
@@index([environmentName, timestamp])
|
||||
@@index([resourceKind, resourceName])
|
||||
@@index([userId, timestamp])
|
||||
}
|
||||
|
||||
// ── Core infrastructure ──
|
||||
|
||||
model Environment {
|
||||
id String @id @default(cuid())
|
||||
name String @unique
|
||||
status String @default("active") // active | archived
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
bindings Binding[]
|
||||
resources Resource[]
|
||||
}
|
||||
|
||||
model Account {
|
||||
id String @id @default(cuid())
|
||||
name String @unique
|
||||
driver String // baremetal-pxe | aws | gcp | kubernetes | ovh
|
||||
config Json @default("{}")
|
||||
// Credentials stored in Infisical, referenced by secretPath
|
||||
secretPath String?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
bindings Binding[]
|
||||
resources Resource[]
|
||||
}
|
||||
|
||||
model Binding {
|
||||
id String @id @default(cuid())
|
||||
environmentId String
|
||||
environment Environment @relation(fields: [environmentId], references: [id], onDelete: Cascade)
|
||||
accountId String
|
||||
account Account @relation(fields: [accountId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@unique([environmentId, accountId])
|
||||
}
|
||||
|
||||
model Resource {
|
||||
id String @id @default(cuid())
|
||||
kind String
|
||||
name String
|
||||
environmentId String
|
||||
environment Environment @relation(fields: [environmentId], references: [id])
|
||||
accountId String
|
||||
account Account @relation(fields: [accountId], references: [id])
|
||||
origin String @default("cli") // file | cli | fleet | imported
|
||||
managedBy String @default("manual") // gitops | manual | auto
|
||||
sourceRef String?
|
||||
desiredSpec Json @default("{}")
|
||||
actualSpec Json?
|
||||
platformRef String?
|
||||
status String @default("pending") // pending | creating | ready | updating | deleting | error
|
||||
statusMessage String?
|
||||
lastReconciled DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
@@unique([kind, name, environmentId])
|
||||
@@index([environmentId])
|
||||
@@index([accountId])
|
||||
@@index([kind, status])
|
||||
}
|
||||
|
||||
model Secret {
|
||||
id String @id @default(cuid())
|
||||
name String @unique
|
||||
// Encrypted data — application-layer encryption as fallback if Infisical unavailable
|
||||
data Json @default("{}")
|
||||
version Int @default(1)
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
}
|
||||
|
||||
// ── Fleet ──
|
||||
|
||||
model Fleet {
|
||||
id String @id @default(cuid())
|
||||
name String
|
||||
environmentId String
|
||||
accountId String
|
||||
selector Json // fact-matching rules
|
||||
onboardPipeline Json // step definitions
|
||||
offboardPipeline Json?
|
||||
approvalConfig Json?
|
||||
status String @default("active")
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
members FleetMember[]
|
||||
}
|
||||
|
||||
model FleetMember {
|
||||
id String @id @default(cuid())
|
||||
fleetId String
|
||||
fleet Fleet @relation(fields: [fleetId], references: [id], onDelete: Cascade)
|
||||
serverId String
|
||||
status String // discovered | pending | onboarding | active | offboarding | removed
|
||||
joinedAt DateTime @default(now())
|
||||
|
||||
@@index([fleetId])
|
||||
}
|
||||
|
||||
// ── Git sources (for sync controller) ──
|
||||
|
||||
model GitSource {
|
||||
id String @id @default(cuid())
|
||||
name String @unique
|
||||
repo String
|
||||
branch String @default("main")
|
||||
path String @default("environments/")
|
||||
lastSync DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
}
|
||||
|
||||
// ── Existing v1.0 models (kept for bastion/agent compatibility) ──
|
||||
|
||||
model Server {
|
||||
id String @id @default(uuid())
|
||||
hostname String @unique
|
||||
mac String? @unique
|
||||
cloud String @default("baremetal")
|
||||
environment String @default("default")
|
||||
role String @default("worker")
|
||||
labels Json @default("{}")
|
||||
id String @id @default(uuid())
|
||||
hostname String @unique
|
||||
mac String? @unique
|
||||
cloud String @default("baremetal")
|
||||
environment String @default("default")
|
||||
role String @default("worker")
|
||||
labels Json @default("{}")
|
||||
ip String?
|
||||
agentVersion String?
|
||||
status String @default("unknown") // unknown, online, offline, provisioning
|
||||
status String @default("unknown")
|
||||
lastHeartbeat DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
agent Agent?
|
||||
auditLogs AuditLog[]
|
||||
agent Agent?
|
||||
}
|
||||
|
||||
model Agent {
|
||||
@@ -33,112 +251,29 @@ model Agent {
|
||||
certificatePem String?
|
||||
enrolledAt DateTime @default(now())
|
||||
lastSeen DateTime?
|
||||
facts Json? // hardware facts reported by agent
|
||||
|
||||
@@index([serverId])
|
||||
}
|
||||
|
||||
model User {
|
||||
id String @id @default(uuid())
|
||||
username String @unique
|
||||
displayName String?
|
||||
certFingerprint String? @unique
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
roleBindings UserRole[]
|
||||
auditLogs AuditLog[]
|
||||
}
|
||||
|
||||
model Role {
|
||||
id String @id @default(uuid())
|
||||
name String @unique
|
||||
description String?
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
permissions Permission[]
|
||||
userBindings UserRole[]
|
||||
}
|
||||
|
||||
model Permission {
|
||||
id String @id @default(uuid())
|
||||
roleId String
|
||||
role Role @relation(fields: [roleId], references: [id], onDelete: Cascade)
|
||||
type String @default("allow") // allow or deny
|
||||
action String // read, exec, apply, destroy, manage, admin, kubectl, *
|
||||
cloud String @default("*")
|
||||
environment String @default("*")
|
||||
server String @default("*")
|
||||
|
||||
@@index([roleId])
|
||||
}
|
||||
|
||||
model UserRole {
|
||||
id String @id @default(uuid())
|
||||
userId String
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
roleId String
|
||||
role Role @relation(fields: [roleId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@unique([userId, roleId])
|
||||
@@index([userId])
|
||||
@@index([roleId])
|
||||
}
|
||||
|
||||
model JoinToken {
|
||||
id String @id @default(uuid())
|
||||
token String @unique
|
||||
type String @default("one-time") // one-time or reusable
|
||||
type String @default("one-time")
|
||||
label String?
|
||||
usedBy String? // server hostname that used it
|
||||
usedBy String?
|
||||
usedAt DateTime?
|
||||
revokedAt DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
expiresAt DateTime?
|
||||
}
|
||||
|
||||
model AuditLog {
|
||||
id String @id @default(uuid())
|
||||
userId String?
|
||||
user User? @relation(fields: [userId], references: [id])
|
||||
serverId String?
|
||||
server Server? @relation(fields: [serverId], references: [id])
|
||||
sessionId String?
|
||||
action String // exec, kubectl, apply, login, rbac-denied, etc.
|
||||
resourceType String? // server, cluster, role, app, etc.
|
||||
resourceName String?
|
||||
args String? // sanitized command args
|
||||
result String @default("success") // success, denied, error
|
||||
durationMs Int?
|
||||
sourceIp String?
|
||||
timestamp DateTime @default(now())
|
||||
|
||||
@@index([userId])
|
||||
@@index([serverId])
|
||||
@@index([sessionId])
|
||||
@@index([timestamp])
|
||||
@@index([action])
|
||||
}
|
||||
|
||||
model PulumiRun {
|
||||
id String @id @default(uuid())
|
||||
userId String
|
||||
stackName String
|
||||
action String // up, preview, destroy
|
||||
status String @default("pending") // pending, running, succeeded, failed
|
||||
output String?
|
||||
startedAt DateTime @default(now())
|
||||
completedAt DateTime?
|
||||
|
||||
@@index([userId])
|
||||
@@index([stackName])
|
||||
}
|
||||
|
||||
model Bastion {
|
||||
id String @id @default(uuid())
|
||||
hostname String @unique
|
||||
network String
|
||||
serverIp String
|
||||
status String @default("offline") // online, offline
|
||||
status String @default("offline")
|
||||
lastHeartbeat DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
@@ -149,7 +284,7 @@ model Cluster {
|
||||
name String @unique
|
||||
cloud String @default("baremetal")
|
||||
environment String @default("default")
|
||||
kubeconfigEnc String? // encrypted kubeconfig
|
||||
kubeconfigEnc String?
|
||||
labels Json @default("{}")
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
65
bastion/src/labd/src/middleware/bearer-auth.ts
Normal file
65
bastion/src/labd/src/middleware/bearer-auth.ts
Normal file
@@ -0,0 +1,65 @@
|
||||
// Bearer token auth middleware for Fastify.
|
||||
// Validates Authorization header, resolves user identity, attaches to request.
|
||||
|
||||
import type { FastifyRequest, FastifyReply } from "fastify";
|
||||
import type { AuthService } from "../services/auth.js";
|
||||
|
||||
declare module "fastify" {
|
||||
interface FastifyRequest {
|
||||
userId?: string;
|
||||
userEmail?: string;
|
||||
userRole?: string;
|
||||
}
|
||||
}
|
||||
|
||||
// Paths that don't require authentication
|
||||
const PUBLIC_PATHS = new Set([
|
||||
"/health",
|
||||
"/api/auth/login",
|
||||
"/ws/bastion",
|
||||
"/ws/agent",
|
||||
"/api/auth/enroll",
|
||||
]);
|
||||
|
||||
export function createBearerAuthMiddleware(authService: AuthService) {
|
||||
return async function bearerAuth(
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
): Promise<void> {
|
||||
// Skip auth for public paths
|
||||
if (PUBLIC_PATHS.has(request.url.split("?")[0] ?? "")) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip auth for WebSocket upgrade requests (handled by their own auth)
|
||||
if (request.headers.upgrade === "websocket") {
|
||||
return;
|
||||
}
|
||||
|
||||
const authHeader = request.headers.authorization;
|
||||
if (!authHeader) {
|
||||
void reply.code(401).send({ error: "Authorization header required" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!authHeader.startsWith("Bearer ")) {
|
||||
void reply.code(401).send({ error: "Invalid authorization format, expected: Bearer <token>" });
|
||||
return;
|
||||
}
|
||||
|
||||
const token = authHeader.slice(7);
|
||||
if (token.length === 0) {
|
||||
void reply.code(401).send({ error: "Empty bearer token" });
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const identity = await authService.validateToken(token);
|
||||
request.userId = identity.userId;
|
||||
request.userEmail = identity.email;
|
||||
request.userRole = identity.role;
|
||||
} catch {
|
||||
void reply.code(401).send({ error: "Invalid or expired token. Run: labctl login" });
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import type { FastifyInstance } from "fastify";
|
||||
import type { DbClient } from "../server.js";
|
||||
import { bastionRegistry } from "../services/bastion-registry.js";
|
||||
import { generateRequestId } from "@lab/shared";
|
||||
import type { VyosInstallSpec } from "@lab/shared";
|
||||
|
||||
const COMMAND_TIMEOUT_MS = 15_000;
|
||||
|
||||
@@ -84,7 +85,6 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
app.get("/api/machines", async () => {
|
||||
const live = bastionRegistry.getAggregatedState();
|
||||
|
||||
// Merge DB records for machines not currently in any bastion's live state
|
||||
try {
|
||||
const dbServers = (await db.server.findMany({})) as Array<{
|
||||
mac: string | null; hostname: string; role: string; ip: string | null;
|
||||
@@ -93,9 +93,49 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
for (const s of dbServers) {
|
||||
if (!s.mac) continue;
|
||||
const mac = s.mac.toLowerCase();
|
||||
// Only add from DB if not already in live state
|
||||
|
||||
// DB knows this machine has been installed at some point if it has a real
|
||||
// hostname+role (not just product-name-as-hostname and role="unknown").
|
||||
// Status alone is unreliable: a rediscovery can re-set it without erasing the
|
||||
// install identity. If the bastion restarted and lost its installed map, the
|
||||
// machine will only show up in live.discovered — promote it here so the CLI
|
||||
// still sees hostname/role/IP.
|
||||
const dbKnowsInstalled =
|
||||
s.role !== "unknown" && s.role !== "" &&
|
||||
s.hostname !== "" && s.hostname !== s.mac;
|
||||
|
||||
if (dbKnowsInstalled && !(mac in live.installed) && !(mac in live.install_queue)) {
|
||||
const hw = live.discovered[mac];
|
||||
live.installed[mac] = {
|
||||
hostname: s.hostname,
|
||||
role: s.role,
|
||||
ip: s.ip ?? "",
|
||||
installed_at: "",
|
||||
bastionId: hw?.bastionId ?? "db",
|
||||
...(hw ? {
|
||||
product: hw.product,
|
||||
manufacturer: hw.manufacturer,
|
||||
cpu_model: hw.cpu_model,
|
||||
cpu_cores: hw.cpu_cores,
|
||||
memory_gb: hw.memory_gb,
|
||||
arch: hw.arch,
|
||||
} : {}),
|
||||
};
|
||||
delete live.discovered[mac];
|
||||
continue;
|
||||
}
|
||||
|
||||
// Unknown-to-live MAC: fall back to whatever the DB says.
|
||||
if (!(mac in live.discovered) && !(mac in live.install_queue) && !(mac in live.installed)) {
|
||||
if (s.status === "discovered") {
|
||||
if (s.status === "online" || s.status === "offline") {
|
||||
live.installed[mac] = {
|
||||
hostname: s.hostname,
|
||||
role: s.role,
|
||||
ip: s.ip ?? "",
|
||||
installed_at: "",
|
||||
bastionId: "db",
|
||||
};
|
||||
} else {
|
||||
live.discovered[mac] = {
|
||||
mac,
|
||||
product: String(s.labels?.product ?? "unknown"),
|
||||
@@ -112,14 +152,6 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
last_seen: "",
|
||||
bastionId: "db",
|
||||
};
|
||||
} else if (s.status === "online" || s.status === "offline") {
|
||||
live.installed[mac] = {
|
||||
hostname: s.hostname,
|
||||
role: s.role,
|
||||
ip: s.ip ?? "",
|
||||
installed_at: "",
|
||||
bastionId: "db",
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -132,9 +164,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
|
||||
// Queue install — route to correct bastion by MAC
|
||||
app.post<{
|
||||
Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string };
|
||||
Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string; vyos?: VyosInstallSpec };
|
||||
}>("/api/machines/install", async (request, reply) => {
|
||||
const { mac, hostname, disk, role, os } = request.body ?? {};
|
||||
const { mac, hostname, disk, role, os, vyos } = request.body ?? {};
|
||||
if (!mac || !hostname) {
|
||||
return reply.code(400).send({ error: "mac and hostname are required" });
|
||||
}
|
||||
@@ -152,6 +184,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
const result = await sendCommand(all[0]!.bastionId, {
|
||||
type: "command-install",
|
||||
mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
|
||||
...(vyos ? { vyos } : {}),
|
||||
});
|
||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||
} catch (err) {
|
||||
@@ -165,6 +198,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
const result = await sendCommand(bastion.bastionId, {
|
||||
type: "command-install",
|
||||
mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
|
||||
...(vyos ? { vyos } : {}),
|
||||
});
|
||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||
} catch (err) {
|
||||
|
||||
191
bastion/src/labd/src/routes/environments.ts
Normal file
191
bastion/src/labd/src/routes/environments.ts
Normal file
@@ -0,0 +1,191 @@
|
||||
// Environment and Account management routes.
|
||||
// GET/POST /api/environments — list/create environments
|
||||
// GET/POST /api/accounts — list/create accounts
|
||||
// POST /api/accounts/bind — bind account to environment
|
||||
// GET /api/bindings — list bindings
|
||||
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import type { PrismaClient, Prisma } from "@prisma/client";
|
||||
import type { RbacService } from "../services/rbac.js";
|
||||
import type { AuditService } from "../services/audit.js";
|
||||
|
||||
export function registerEnvironmentRoutes(
|
||||
app: FastifyInstance,
|
||||
db: PrismaClient,
|
||||
rbacService: RbacService,
|
||||
auditService: AuditService,
|
||||
): void {
|
||||
// List environments
|
||||
app.get("/api/environments", async (_request, reply) => {
|
||||
const envs = await db.environment.findMany({ orderBy: { name: "asc" } });
|
||||
return reply.send(envs);
|
||||
});
|
||||
|
||||
// Create environment
|
||||
app.post<{
|
||||
Body: { name?: string };
|
||||
}>("/api/environments", async (request, reply) => {
|
||||
const { name } = request.body ?? {};
|
||||
if (!name) {
|
||||
return reply.code(400).send({ error: "name is required" });
|
||||
}
|
||||
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "admin",
|
||||
resource: "environments",
|
||||
});
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
try {
|
||||
const env = await db.environment.create({ data: { name } });
|
||||
auditService.emit({
|
||||
eventKind: "resource_created",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userId: request.userId ?? null,
|
||||
resourceKind: "environment",
|
||||
resourceName: name,
|
||||
result: "success",
|
||||
});
|
||||
return reply.code(201).send(env);
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message.includes("Unique constraint")) {
|
||||
return reply.code(409).send({ error: `Environment '${name}' already exists` });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
|
||||
// List accounts
|
||||
app.get("/api/accounts", async (_request, reply) => {
|
||||
const accounts = await db.account.findMany({
|
||||
orderBy: { name: "asc" },
|
||||
select: { id: true, name: true, driver: true, config: true, createdAt: true, updatedAt: true },
|
||||
});
|
||||
return reply.send(accounts);
|
||||
});
|
||||
|
||||
// Create account
|
||||
app.post<{
|
||||
Body: { name?: string; driver?: string; config?: Record<string, unknown> };
|
||||
}>("/api/accounts", async (request, reply) => {
|
||||
const { name, driver, config } = request.body ?? {};
|
||||
if (!name || !driver) {
|
||||
return reply.code(400).send({ error: "name and driver are required" });
|
||||
}
|
||||
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "admin",
|
||||
resource: "accounts",
|
||||
});
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
try {
|
||||
const account = await db.account.create({
|
||||
data: { name, driver, config: (config ?? {}) as Prisma.InputJsonValue },
|
||||
});
|
||||
auditService.emit({
|
||||
eventKind: "resource_created",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userId: request.userId ?? null,
|
||||
resourceKind: "account",
|
||||
resourceName: name,
|
||||
result: "success",
|
||||
details: { driver },
|
||||
});
|
||||
return reply.code(201).send(account);
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message.includes("Unique constraint")) {
|
||||
return reply.code(409).send({ error: `Account '${name}' already exists` });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
|
||||
// Bind account to environment
|
||||
app.post<{
|
||||
Body: { environmentId?: string; accountId?: string };
|
||||
}>("/api/accounts/bind", async (request, reply) => {
|
||||
const { environmentId, accountId } = request.body ?? {};
|
||||
if (!environmentId || !accountId) {
|
||||
return reply.code(400).send({ error: "environmentId and accountId are required" });
|
||||
}
|
||||
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "admin",
|
||||
resource: "accounts",
|
||||
});
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
try {
|
||||
const binding = await db.binding.create({
|
||||
data: { environmentId, accountId },
|
||||
});
|
||||
return reply.code(201).send(binding);
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message.includes("Unique constraint")) {
|
||||
return reply.code(409).send({ error: "This account is already bound to this environment" });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
|
||||
// List bindings
|
||||
app.get("/api/bindings", async (_request, reply) => {
|
||||
const bindings = await db.binding.findMany({
|
||||
include: { environment: true, account: true },
|
||||
});
|
||||
return reply.send(bindings);
|
||||
});
|
||||
|
||||
// Audit event query
|
||||
app.get<{
|
||||
Querystring: {
|
||||
last?: string;
|
||||
kind?: string;
|
||||
env?: string;
|
||||
correlation?: string;
|
||||
limit?: string;
|
||||
};
|
||||
}>("/api/events", async (request, reply) => {
|
||||
const { last, kind, env, correlation, limit } = request.query as { last?: string; kind?: string; env?: string; correlation?: string; limit?: string };
|
||||
|
||||
const where: Record<string, unknown> = {};
|
||||
|
||||
if (last) {
|
||||
const match = last.match(/^(\d+)(h|d|m)$/);
|
||||
if (match) {
|
||||
const [, num, unit] = match;
|
||||
const ms = { h: 3_600_000, d: 86_400_000, m: 60_000 }[unit!]!;
|
||||
where.timestamp = { gte: new Date(Date.now() - parseInt(num!) * ms) };
|
||||
}
|
||||
}
|
||||
if (kind) where.eventKind = kind;
|
||||
if (env) where.environmentName = env;
|
||||
if (correlation) where.correlationId = correlation;
|
||||
|
||||
const events = await db.auditEvent.findMany({
|
||||
where,
|
||||
orderBy: { timestamp: "desc" },
|
||||
take: Math.min(parseInt(limit ?? "100"), 500),
|
||||
});
|
||||
|
||||
return reply.send(events);
|
||||
});
|
||||
}
|
||||
196
bastion/src/labd/src/routes/resources.ts
Normal file
196
bastion/src/labd/src/routes/resources.ts
Normal file
@@ -0,0 +1,196 @@
|
||||
// Resource CRUD routes with RBAC enforcement.
|
||||
// GET /api/resources — list (filtered by RBAC scope)
|
||||
// GET /api/resources/:id — get
|
||||
// POST /api/resources — create
|
||||
// PUT /api/resources/:id — update
|
||||
// DELETE /api/resources/:id — delete (marks as deleting)
|
||||
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import type { ResourceStore, CreateResourceInput } from "../services/resource-store.js";
|
||||
import type { RbacService } from "../services/rbac.js";
|
||||
import type { AuditService } from "../services/audit.js";
|
||||
import { resolveResourceKind } from "@lab/core";
|
||||
|
||||
export function registerResourceRoutes(
|
||||
app: FastifyInstance,
|
||||
resourceStore: ResourceStore,
|
||||
rbacService: RbacService,
|
||||
auditService: AuditService,
|
||||
): void {
|
||||
// List resources (filtered by kind, environment, status)
|
||||
app.get<{
|
||||
Querystring: { kind?: string; environment?: string; status?: string };
|
||||
}>("/api/resources", async (request, reply) => {
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "view",
|
||||
resource: request.query.kind ? resolveResourceKind(request.query.kind) : undefined,
|
||||
});
|
||||
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
const resources = await resourceStore.list({
|
||||
kind: request.query.kind ? resolveResourceKind(request.query.kind) : undefined,
|
||||
environmentId: request.query.environment,
|
||||
status: request.query.status,
|
||||
});
|
||||
|
||||
return reply.send(resources);
|
||||
});
|
||||
|
||||
// Get single resource
|
||||
app.get<{
|
||||
Params: { id: string };
|
||||
}>("/api/resources/:id", async (request, reply) => {
|
||||
const resource = await resourceStore.get(request.params.id);
|
||||
if (!resource) {
|
||||
return reply.code(404).send({ error: "Resource not found" });
|
||||
}
|
||||
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "view",
|
||||
resource: resource.kind,
|
||||
name: resource.name,
|
||||
});
|
||||
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
return reply.send(resource);
|
||||
});
|
||||
|
||||
// Create resource
|
||||
app.post<{
|
||||
Body: CreateResourceInput;
|
||||
}>("/api/resources", async (request, reply) => {
|
||||
const input = request.body;
|
||||
if (!input?.kind || !input?.name || !input?.environmentId || !input?.accountId) {
|
||||
return reply.code(400).send({ error: "kind, name, environmentId, and accountId are required" });
|
||||
}
|
||||
|
||||
const kind = resolveResourceKind(input.kind);
|
||||
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "create",
|
||||
resource: kind,
|
||||
});
|
||||
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
const correlationId = auditService.createCorrelation();
|
||||
|
||||
try {
|
||||
const resource = await resourceStore.create({ ...input, kind });
|
||||
|
||||
auditService.emit({
|
||||
eventKind: "resource_created",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userId: request.userId ?? null,
|
||||
userName: request.userEmail ?? null,
|
||||
resourceKind: kind,
|
||||
resourceName: input.name,
|
||||
correlationId,
|
||||
result: "success",
|
||||
});
|
||||
|
||||
return reply.code(201).send(resource);
|
||||
} catch (err) {
|
||||
// Prisma unique constraint violation
|
||||
if (err instanceof Error && err.message.includes("Unique constraint")) {
|
||||
return reply.code(409).send({ error: `Resource ${kind}/${input.name} already exists in this environment` });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
|
||||
// Update resource
|
||||
app.put<{
|
||||
Params: { id: string };
|
||||
Body: { desiredSpec?: Record<string, unknown>; status?: string };
|
||||
}>("/api/resources/:id", async (request, reply) => {
|
||||
const resource = await resourceStore.get(request.params.id);
|
||||
if (!resource) {
|
||||
return reply.code(404).send({ error: "Resource not found" });
|
||||
}
|
||||
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "edit",
|
||||
resource: resource.kind,
|
||||
name: resource.name,
|
||||
});
|
||||
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
const updated = await resourceStore.update(request.params.id, request.body);
|
||||
|
||||
auditService.emit({
|
||||
eventKind: "resource_updated",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userId: request.userId ?? null,
|
||||
userName: request.userEmail ?? null,
|
||||
resourceKind: resource.kind,
|
||||
resourceName: resource.name,
|
||||
result: "success",
|
||||
});
|
||||
|
||||
return reply.send(updated);
|
||||
});
|
||||
|
||||
// Delete resource (marks as deleting)
|
||||
app.delete<{
|
||||
Params: { id: string };
|
||||
}>("/api/resources/:id", async (request, reply) => {
|
||||
const resource = await resourceStore.get(request.params.id);
|
||||
if (!resource) {
|
||||
return reply.code(404).send({ error: "Resource not found" });
|
||||
}
|
||||
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "delete",
|
||||
resource: resource.kind,
|
||||
name: resource.name,
|
||||
});
|
||||
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
await resourceStore.delete(request.params.id);
|
||||
|
||||
auditService.emit({
|
||||
eventKind: "resource_deleted",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userId: request.userId ?? null,
|
||||
userName: request.userEmail ?? null,
|
||||
resourceKind: resource.kind,
|
||||
resourceName: resource.name,
|
||||
result: "success",
|
||||
});
|
||||
|
||||
return reply.send({ status: "deleting", id: request.params.id });
|
||||
});
|
||||
}
|
||||
81
bastion/src/labd/src/routes/v2-auth.ts
Normal file
81
bastion/src/labd/src/routes/v2-auth.ts
Normal file
@@ -0,0 +1,81 @@
|
||||
// v2 Auth routes: bearer token login/logout.
|
||||
// POST /api/auth/login — email + password → session token
|
||||
// POST /api/auth/logout — revoke session
|
||||
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import type { AuthService } from "../services/auth.js";
|
||||
import type { AuditService } from "../services/audit.js";
|
||||
import { AuthError } from "../services/auth.js";
|
||||
|
||||
export function registerV2AuthRoutes(
|
||||
app: FastifyInstance,
|
||||
authService: AuthService,
|
||||
auditService: AuditService,
|
||||
): void {
|
||||
app.post<{
|
||||
Body: { email?: string; password?: string };
|
||||
}>("/api/auth/login", async (request, reply) => {
|
||||
const { email, password } = request.body ?? {};
|
||||
|
||||
if (!email || !password) {
|
||||
return reply.code(400).send({ error: "email and password are required" });
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await authService.login(email, password);
|
||||
|
||||
auditService.emit({
|
||||
eventKind: result.isBootstrap ? "auth_bootstrap" : "auth_login",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userId: result.userId,
|
||||
userName: email,
|
||||
result: "success",
|
||||
details: { isBootstrap: result.isBootstrap },
|
||||
});
|
||||
|
||||
return reply.send({
|
||||
token: result.token,
|
||||
expiresAt: result.expiresAt.toISOString(),
|
||||
isBootstrap: result.isBootstrap,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof AuthError) {
|
||||
auditService.emit({
|
||||
eventKind: "auth_login",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userName: email,
|
||||
result: "failure",
|
||||
error: err.message,
|
||||
});
|
||||
return reply.code(401).send({ error: err.message });
|
||||
}
|
||||
return reply.code(500).send({ error: "Login failed" });
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/api/auth/logout", async (request, reply) => {
|
||||
const token = request.headers.authorization?.slice(7);
|
||||
if (!token) {
|
||||
return reply.code(400).send({ error: "Authorization header required" });
|
||||
}
|
||||
|
||||
try {
|
||||
await authService.logout(token);
|
||||
auditService.emit({
|
||||
eventKind: "auth_logout",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userId: request.userId ?? null,
|
||||
result: "success",
|
||||
});
|
||||
return reply.send({ status: "logged_out" });
|
||||
} catch (err) {
|
||||
if (err instanceof AuthError) {
|
||||
return reply.code(400).send({ error: err.message });
|
||||
}
|
||||
return reply.code(500).send({ error: "Logout failed" });
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import Fastify from "fastify";
|
||||
import websocket from "@fastify/websocket";
|
||||
import type { PrismaClient } from "@prisma/client";
|
||||
import type { LabdConfig } from "./config.js";
|
||||
import { logger } from "./services/logger.js";
|
||||
import { registerHealthRoutes } from "./routes/health.js";
|
||||
@@ -9,8 +10,16 @@ import { registerServerRoutes } from "./routes/servers.js";
|
||||
import { registerAuthRoutes } from "./routes/auth.js";
|
||||
import { registerAgentRoutes } from "./routes/agents.js";
|
||||
import { registerBastionRoutes } from "./routes/bastions.js";
|
||||
import { registerV2AuthRoutes } from "./routes/v2-auth.js";
|
||||
import { registerEnvironmentRoutes } from "./routes/environments.js";
|
||||
import { registerResourceRoutes } from "./routes/resources.js";
|
||||
import { setupRateLimiting } from "./middleware/rate-limit.js";
|
||||
import { createBearerAuthMiddleware } from "./middleware/bearer-auth.js";
|
||||
import { bastionRegistry } from "./services/bastion-registry.js";
|
||||
import { AuthService } from "./services/auth.js";
|
||||
import { RbacService } from "./services/rbac.js";
|
||||
import { ResourceStore } from "./services/resource-store.js";
|
||||
import { AuditService } from "./services/audit.js";
|
||||
import { isBastionMessage } from "@lab/shared";
|
||||
|
||||
export interface DbClient {
|
||||
@@ -37,6 +46,7 @@ export interface DbClient {
|
||||
|
||||
export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
|
||||
app: ReturnType<typeof Fastify>;
|
||||
auditService: AuditService;
|
||||
}> {
|
||||
const app = Fastify({
|
||||
logger: false, // We use winston instead
|
||||
@@ -48,13 +58,39 @@ export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
|
||||
// Register WebSocket support
|
||||
void app.register(websocket);
|
||||
|
||||
// Register route handlers
|
||||
// v2 services. The structural DbClient is a subset of the real PrismaClient;
|
||||
// at runtime db IS the PrismaClient instance, so the cast is safe. Tests that
|
||||
// exercise v2 routes provide a PrismaClient-shaped mock (see auth-bootstrap,
|
||||
// rbac-deny, audit-correlation tests).
|
||||
const prisma = db as unknown as PrismaClient;
|
||||
const authService = new AuthService(prisma);
|
||||
const rbacService = new RbacService(prisma);
|
||||
const resourceStore = new ResourceStore(prisma);
|
||||
const auditService = new AuditService(prisma);
|
||||
auditService.start();
|
||||
|
||||
// Register v1 (legacy) route handlers
|
||||
registerHealthRoutes(app, db);
|
||||
registerServerRoutes(app, db);
|
||||
registerAuthRoutes(app, db);
|
||||
registerAgentRoutes(app);
|
||||
registerBastionRoutes(app, db);
|
||||
|
||||
// v2 routes live in a scope with bearer-auth as preHandler. Public paths
|
||||
// (login, /health, websockets) are skipped inside the middleware itself.
|
||||
// v1 routes above are unaffected — they're registered on the root scope.
|
||||
await app.register(async (scope) => {
|
||||
scope.addHook("preHandler", createBearerAuthMiddleware(authService));
|
||||
registerV2AuthRoutes(scope, authService, auditService);
|
||||
registerEnvironmentRoutes(scope, prisma, rbacService, auditService);
|
||||
registerResourceRoutes(scope, resourceStore, rbacService, auditService);
|
||||
});
|
||||
|
||||
// Flush pending audit events on shutdown so we never lose the last batch.
|
||||
app.addHook("onClose", async () => {
|
||||
auditService.stop();
|
||||
});
|
||||
|
||||
// WebSocket handler for agent connections
|
||||
app.register(async (fastify) => {
|
||||
fastify.get("/ws/agent", { websocket: true }, (socket, _request) => {
|
||||
@@ -192,7 +228,9 @@ export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
|
||||
labels: { cpu: hw.cpu_model, cores: hw.cpu_cores, memory_gb: hw.memory_gb, arch: hw.arch, product: hw.product, manufacturer: hw.manufacturer },
|
||||
},
|
||||
update: {
|
||||
status: "discovered",
|
||||
// Leave status alone — a previously "online"/"offline" record
|
||||
// must not be downgraded to "discovered" just because the bastion
|
||||
// restarted and re-discovered the MAC via DHCP/PXE.
|
||||
lastHeartbeat: new Date(),
|
||||
labels: { cpu: hw.cpu_model, cores: hw.cpu_cores, memory_gb: hw.memory_gb, arch: hw.arch, product: hw.product, manufacturer: hw.manufacturer },
|
||||
},
|
||||
@@ -265,5 +303,5 @@ export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
|
||||
logger.info(`HTTP: ${request.ip} ${request.method} ${request.url}`);
|
||||
});
|
||||
|
||||
return { app };
|
||||
return { app, auditService };
|
||||
}
|
||||
|
||||
106
bastion/src/labd/src/services/audit.ts
Normal file
106
bastion/src/labd/src/services/audit.ts
Normal file
@@ -0,0 +1,106 @@
|
||||
// Audit service: fire-and-forget event collection with batching.
|
||||
// Batches 50 events or flushes every 5 seconds, whichever comes first.
|
||||
// Failures never block the operation being audited.
|
||||
|
||||
import { randomBytes } from "node:crypto";
|
||||
import type { PrismaClient, Prisma } from "@prisma/client";
|
||||
import { logger } from "./logger.js";
|
||||
|
||||
const BATCH_SIZE = 50;
|
||||
const FLUSH_INTERVAL_MS = 5_000;
|
||||
|
||||
export interface AuditEventInput {
|
||||
eventKind: string;
|
||||
source: string;
|
||||
verified?: boolean;
|
||||
userId?: string | null;
|
||||
userName?: string | null;
|
||||
sessionId?: string | null;
|
||||
environmentName?: string | null;
|
||||
accountName?: string | null;
|
||||
resourceKind?: string | null;
|
||||
resourceName?: string | null;
|
||||
correlationId?: string | null;
|
||||
parentEventId?: string | null;
|
||||
details?: Record<string, unknown>;
|
||||
result: string;
|
||||
error?: string | null;
|
||||
durationMs?: number | null;
|
||||
}
|
||||
|
||||
export class AuditService {
|
||||
private batch: AuditEventInput[] = [];
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
constructor(private readonly db: PrismaClient) {}
|
||||
|
||||
start(): void {
|
||||
this.timer = setInterval(() => {
|
||||
void this.flush();
|
||||
}, FLUSH_INTERVAL_MS);
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (this.timer) {
|
||||
clearInterval(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
void this.flush();
|
||||
}
|
||||
|
||||
emit(event: AuditEventInput): void {
|
||||
// Generate correlation ID if not provided
|
||||
if (!event.correlationId) {
|
||||
event.correlationId = `corr_${randomBytes(8).toString("hex")}`;
|
||||
}
|
||||
|
||||
this.batch.push(event);
|
||||
|
||||
if (this.batch.length >= BATCH_SIZE) {
|
||||
void this.flush();
|
||||
}
|
||||
}
|
||||
|
||||
/** Create a correlation context for a chain of related events. */
|
||||
createCorrelation(): string {
|
||||
return `corr_${randomBytes(8).toString("hex")}`;
|
||||
}
|
||||
|
||||
/** Flush all pending events synchronously. Tests await this; production
|
||||
* relies on the interval timer or stop() during shutdown. */
|
||||
async flushPending(): Promise<void> {
|
||||
await this.flush();
|
||||
}
|
||||
|
||||
private async flush(): Promise<void> {
|
||||
if (this.batch.length === 0) return;
|
||||
|
||||
const events = this.batch.splice(0);
|
||||
try {
|
||||
await this.db.auditEvent.createMany({
|
||||
data: events.map((e) => ({
|
||||
eventKind: e.eventKind,
|
||||
source: e.source,
|
||||
verified: e.verified ?? false,
|
||||
userId: e.userId ?? null,
|
||||
userName: e.userName ?? null,
|
||||
sessionId: e.sessionId ?? null,
|
||||
environmentName: e.environmentName ?? null,
|
||||
accountName: e.accountName ?? null,
|
||||
resourceKind: e.resourceKind ?? null,
|
||||
resourceName: e.resourceName ?? null,
|
||||
correlationId: e.correlationId ?? `corr_${randomBytes(8).toString("hex")}`,
|
||||
parentEventId: e.parentEventId ?? null,
|
||||
details: (e.details ?? {}) as Prisma.InputJsonValue,
|
||||
result: e.result,
|
||||
error: e.error ?? null,
|
||||
durationMs: e.durationMs ?? null,
|
||||
})),
|
||||
});
|
||||
logger.info(`AUDIT: flushed ${events.length} events`);
|
||||
} catch (err) {
|
||||
// Fire-and-forget: audit failures never block operations
|
||||
logger.warn(`AUDIT: failed to flush ${events.length} events: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
119
bastion/src/labd/src/services/auth.ts
Normal file
119
bastion/src/labd/src/services/auth.ts
Normal file
@@ -0,0 +1,119 @@
|
||||
// Auth service: bearer token authentication with bootstrap flow.
|
||||
// First login creates the admin user. Subsequent logins return session tokens.
|
||||
|
||||
import { randomBytes } from "node:crypto";
|
||||
import bcrypt from "bcryptjs";
|
||||
import type { PrismaClient } from "@prisma/client";
|
||||
import { logger } from "./logger.js";
|
||||
|
||||
const SESSION_EXPIRY_DAYS = 30;
|
||||
const BCRYPT_ROUNDS = 12;
|
||||
|
||||
export interface LoginResult {
|
||||
token: string;
|
||||
expiresAt: Date;
|
||||
userId: string;
|
||||
isBootstrap: boolean;
|
||||
}
|
||||
|
||||
export class AuthService {
|
||||
constructor(private readonly db: PrismaClient) {}
|
||||
|
||||
async login(email: string, password: string): Promise<LoginResult> {
|
||||
const userCount = await this.db.user.count();
|
||||
|
||||
// Bootstrap: first login creates admin user
|
||||
if (userCount === 0) {
|
||||
return this.bootstrap(email, password);
|
||||
}
|
||||
|
||||
const user = await this.db.user.findUnique({ where: { email } });
|
||||
if (!user) {
|
||||
// Same error for unknown user and wrong password (no enumeration)
|
||||
throw new AuthError("Invalid email or password");
|
||||
}
|
||||
|
||||
const valid = await bcrypt.compare(password, user.password);
|
||||
if (!valid) {
|
||||
throw new AuthError("Invalid email or password");
|
||||
}
|
||||
|
||||
const session = await this.createSession(user.id);
|
||||
logger.info(`AUTH LOGIN: ${email} (${user.id.slice(0, 8)}...)`);
|
||||
|
||||
return {
|
||||
token: session.token,
|
||||
expiresAt: session.expiresAt,
|
||||
userId: user.id,
|
||||
isBootstrap: false,
|
||||
};
|
||||
}
|
||||
|
||||
async logout(token: string): Promise<void> {
|
||||
const session = await this.db.session.findUnique({ where: { token } });
|
||||
if (!session) {
|
||||
throw new AuthError("Invalid session");
|
||||
}
|
||||
await this.db.session.delete({ where: { id: session.id } });
|
||||
logger.info(`AUTH LOGOUT: session ${session.id.slice(0, 8)}...`);
|
||||
}
|
||||
|
||||
async validateToken(token: string): Promise<{ userId: string; email: string; role: string }> {
|
||||
const session = await this.db.session.findUnique({
|
||||
where: { token },
|
||||
include: { user: true },
|
||||
});
|
||||
|
||||
if (!session) {
|
||||
throw new AuthError("Invalid token");
|
||||
}
|
||||
if (session.expiresAt < new Date()) {
|
||||
await this.db.session.delete({ where: { id: session.id } });
|
||||
throw new AuthError("Token expired");
|
||||
}
|
||||
|
||||
return {
|
||||
userId: session.user.id,
|
||||
email: session.user.email,
|
||||
role: session.user.role,
|
||||
};
|
||||
}
|
||||
|
||||
private async bootstrap(email: string, password: string): Promise<LoginResult> {
|
||||
const hashed = await bcrypt.hash(password, BCRYPT_ROUNDS);
|
||||
const user = await this.db.user.create({
|
||||
data: {
|
||||
email,
|
||||
password: hashed,
|
||||
role: "ADMIN",
|
||||
name: email.split("@")[0] ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
const session = await this.createSession(user.id);
|
||||
logger.info(`AUTH BOOTSTRAP: created admin user ${email} (${user.id.slice(0, 8)}...)`);
|
||||
|
||||
return {
|
||||
token: session.token,
|
||||
expiresAt: session.expiresAt,
|
||||
userId: user.id,
|
||||
isBootstrap: true,
|
||||
};
|
||||
}
|
||||
|
||||
private async createSession(userId: string) {
|
||||
const token = randomBytes(32).toString("hex");
|
||||
const expiresAt = new Date(Date.now() + SESSION_EXPIRY_DAYS * 24 * 60 * 60 * 1000);
|
||||
|
||||
return this.db.session.create({
|
||||
data: { userId, token, expiresAt },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export class AuthError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "AuthError";
|
||||
}
|
||||
}
|
||||
123
bastion/src/labd/src/services/rbac.ts
Normal file
123
bastion/src/labd/src/services/rbac.ts
Normal file
@@ -0,0 +1,123 @@
|
||||
// RBAC service: environment-scoped permission checks.
|
||||
// Uses named RbacDefinition records with JSON subjects and roleBindings.
|
||||
//
|
||||
// Resolution flow:
|
||||
// 1. Find all RbacDefinitions where subjects match the current user/groups
|
||||
// 2. Collect all roleBindings from matching definitions
|
||||
// 3. Check if any binding grants the requested action on the requested resource
|
||||
|
||||
import type { PrismaClient } from "@prisma/client";
|
||||
import { logger } from "./logger.js";
|
||||
|
||||
export interface RbacCheck {
|
||||
userId: string;
|
||||
userEmail: string;
|
||||
userRole: string;
|
||||
action: string; // "view" | "edit" | "create" | "delete" | "run" | "admin"
|
||||
resource?: string | undefined; // "servers" | "databases" | "clusters" | "*"
|
||||
name?: string | undefined; // specific resource name
|
||||
environment?: string | undefined; // specific environment name
|
||||
}
|
||||
|
||||
export interface RbacResult {
|
||||
allowed: boolean;
|
||||
reason: string;
|
||||
matchedDefinition?: string;
|
||||
}
|
||||
|
||||
interface StoredSubject {
|
||||
kind: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
interface StoredBinding {
|
||||
role: string;
|
||||
resource?: string;
|
||||
name?: string;
|
||||
environment?: string;
|
||||
action?: string;
|
||||
}
|
||||
|
||||
export class RbacService {
|
||||
constructor(private readonly db: PrismaClient) {}
|
||||
|
||||
async check(req: RbacCheck): Promise<RbacResult> {
|
||||
// Admin users bypass RBAC
|
||||
if (req.userRole === "ADMIN") {
|
||||
return { allowed: true, reason: "admin role" };
|
||||
}
|
||||
|
||||
// Collect user's group memberships
|
||||
const memberships = await this.db.groupMember.findMany({
|
||||
where: { userId: req.userId },
|
||||
include: { group: true },
|
||||
});
|
||||
const groupNames = memberships.map((m) => m.group.name);
|
||||
|
||||
// Find all RBAC definitions
|
||||
const definitions = await this.db.rbacDefinition.findMany();
|
||||
|
||||
for (const def of definitions) {
|
||||
const subjects = def.subjects as unknown as StoredSubject[];
|
||||
const bindings = def.roleBindings as unknown as StoredBinding[];
|
||||
|
||||
// Check if this definition's subjects match the user
|
||||
const subjectMatch = subjects.some((s) => {
|
||||
if (s.kind === "User" && s.name === req.userEmail) return true;
|
||||
if (s.kind === "Group" && groupNames.includes(s.name)) return true;
|
||||
return false;
|
||||
});
|
||||
|
||||
if (!subjectMatch) continue;
|
||||
|
||||
// Check if any binding grants the requested permission
|
||||
for (const binding of bindings) {
|
||||
if (this.bindingMatches(binding, req)) {
|
||||
logger.info(`RBAC ALLOW: ${req.userEmail} ${req.action} ${req.resource ?? "*"}${req.name ? `/${req.name}` : ""} via ${def.name}`);
|
||||
return {
|
||||
allowed: true,
|
||||
reason: `granted by ${def.name}`,
|
||||
matchedDefinition: def.name,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`RBAC DENY: ${req.userEmail} ${req.action} ${req.resource ?? "*"}${req.name ? `/${req.name}` : ""}`);
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `no matching role binding for ${req.action} on ${req.resource ?? "*"}`,
|
||||
};
|
||||
}
|
||||
|
||||
private bindingMatches(binding: StoredBinding, req: RbacCheck): boolean {
|
||||
// Check role grants the action
|
||||
if (!this.roleGrantsAction(binding.role, req.action)) return false;
|
||||
|
||||
// Check resource scope
|
||||
if (binding.resource && binding.resource !== "*" && binding.resource !== req.resource) return false;
|
||||
|
||||
// Check name scope
|
||||
if (binding.name && binding.name !== req.name) return false;
|
||||
|
||||
// Check environment scope
|
||||
if (binding.environment && binding.environment !== req.environment) return false;
|
||||
|
||||
// Check operation scope (for "run" role with specific actions)
|
||||
if (binding.action && binding.action !== "*" && binding.action !== req.action) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private roleGrantsAction(role: string, action: string): boolean {
|
||||
const grants: Record<string, string[]> = {
|
||||
admin: ["view", "edit", "create", "delete", "run", "admin"],
|
||||
edit: ["view", "edit", "create", "delete"],
|
||||
create: ["create"],
|
||||
delete: ["delete"],
|
||||
view: ["view"],
|
||||
run: ["run"],
|
||||
};
|
||||
return grants[role]?.includes(action) ?? false;
|
||||
}
|
||||
}
|
||||
108
bastion/src/labd/src/services/resource-store.ts
Normal file
108
bastion/src/labd/src/services/resource-store.ts
Normal file
@@ -0,0 +1,108 @@
|
||||
// Resource store: CRUD for generic resources with origin/managedBy tracking.
|
||||
// All mutations go through this service so RBAC and audit are applied consistently.
|
||||
|
||||
import type { PrismaClient, Resource as PrismaResource, Prisma } from "@prisma/client";
|
||||
import { logger } from "./logger.js";
|
||||
|
||||
export interface CreateResourceInput {
|
||||
kind: string;
|
||||
name: string;
|
||||
environmentId: string;
|
||||
accountId: string;
|
||||
origin?: string;
|
||||
managedBy?: string;
|
||||
sourceRef?: string;
|
||||
desiredSpec: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface UpdateResourceInput {
|
||||
desiredSpec?: Record<string, unknown>;
|
||||
status?: string;
|
||||
statusMessage?: string;
|
||||
actualSpec?: Record<string, unknown>;
|
||||
platformRef?: string;
|
||||
}
|
||||
|
||||
export interface ListResourcesFilter {
|
||||
kind?: string | undefined;
|
||||
environmentId?: string | undefined;
|
||||
accountId?: string | undefined;
|
||||
status?: string | undefined;
|
||||
}
|
||||
|
||||
export class ResourceStore {
|
||||
constructor(private readonly db: PrismaClient) {}
|
||||
|
||||
async create(input: CreateResourceInput): Promise<PrismaResource> {
|
||||
const resource = await this.db.resource.create({
|
||||
data: {
|
||||
kind: input.kind,
|
||||
name: input.name,
|
||||
environmentId: input.environmentId,
|
||||
accountId: input.accountId,
|
||||
origin: input.origin ?? "cli",
|
||||
managedBy: input.managedBy ?? "manual",
|
||||
sourceRef: input.sourceRef ?? null,
|
||||
desiredSpec: input.desiredSpec as Prisma.InputJsonValue,
|
||||
status: "pending",
|
||||
},
|
||||
});
|
||||
|
||||
logger.info(`RESOURCE CREATED: ${input.kind}/${input.name} in env ${input.environmentId.slice(0, 8)}...`);
|
||||
return resource;
|
||||
}
|
||||
|
||||
async get(id: string): Promise<PrismaResource | null> {
|
||||
return this.db.resource.findUnique({ where: { id } });
|
||||
}
|
||||
|
||||
async getByKindNameEnv(kind: string, name: string, environmentId: string): Promise<PrismaResource | null> {
|
||||
return this.db.resource.findUnique({
|
||||
where: { kind_name_environmentId: { kind, name, environmentId } },
|
||||
});
|
||||
}
|
||||
|
||||
async list(filter: ListResourcesFilter = {}): Promise<PrismaResource[]> {
|
||||
return this.db.resource.findMany({
|
||||
where: {
|
||||
...(filter.kind ? { kind: filter.kind } : {}),
|
||||
...(filter.environmentId ? { environmentId: filter.environmentId } : {}),
|
||||
...(filter.accountId ? { accountId: filter.accountId } : {}),
|
||||
...(filter.status ? { status: filter.status } : {}),
|
||||
},
|
||||
orderBy: { createdAt: "desc" },
|
||||
});
|
||||
}
|
||||
|
||||
async update(id: string, input: UpdateResourceInput): Promise<PrismaResource> {
|
||||
const data: Prisma.ResourceUpdateInput = {};
|
||||
if (input.desiredSpec !== undefined) data.desiredSpec = input.desiredSpec as Prisma.InputJsonValue;
|
||||
if (input.status !== undefined) data.status = input.status;
|
||||
if (input.statusMessage !== undefined) data.statusMessage = input.statusMessage;
|
||||
if (input.actualSpec !== undefined) data.actualSpec = input.actualSpec as Prisma.InputJsonValue;
|
||||
if (input.platformRef !== undefined) data.platformRef = input.platformRef;
|
||||
if (input.status === "ready") data.lastReconciled = new Date();
|
||||
|
||||
const resource = await this.db.resource.update({ where: { id }, data });
|
||||
|
||||
logger.info(`RESOURCE UPDATED: ${resource.kind}/${resource.name} -> ${input.status ?? "spec change"}`);
|
||||
return resource;
|
||||
}
|
||||
|
||||
async delete(id: string): Promise<void> {
|
||||
const resource = await this.db.resource.findUnique({ where: { id } });
|
||||
if (!resource) return;
|
||||
|
||||
// Mark as deleting first (driver handles actual deletion)
|
||||
await this.db.resource.update({
|
||||
where: { id },
|
||||
data: { status: "deleting" },
|
||||
});
|
||||
|
||||
logger.info(`RESOURCE DELETING: ${resource.kind}/${resource.name}`);
|
||||
}
|
||||
|
||||
async hardDelete(id: string): Promise<void> {
|
||||
await this.db.resource.delete({ where: { id } });
|
||||
}
|
||||
}
|
||||
144
bastion/src/labd/tests/bastions-machines.test.ts
Normal file
144
bastion/src/labd/tests/bastions-machines.test.ts
Normal file
@@ -0,0 +1,144 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import Fastify from "fastify";
|
||||
import { registerBastionRoutes } from "../src/routes/bastions.js";
|
||||
import { bastionRegistry } from "../src/services/bastion-registry.js";
|
||||
import type { DbClient } from "../src/server.js";
|
||||
import type { BastionState } from "@lab/shared";
|
||||
|
||||
function createMockDb(servers: unknown[] = []): DbClient {
|
||||
return {
|
||||
$queryRaw: vi.fn().mockResolvedValue([{ "?column?": 1 }]),
|
||||
server: {
|
||||
findMany: vi.fn().mockResolvedValue(servers),
|
||||
findUnique: vi.fn().mockResolvedValue(null),
|
||||
upsert: vi.fn().mockResolvedValue({}),
|
||||
},
|
||||
joinToken: {
|
||||
findUnique: vi.fn().mockResolvedValue(null),
|
||||
findMany: vi.fn().mockResolvedValue([]),
|
||||
create: vi.fn().mockResolvedValue({ id: "t" }),
|
||||
update: vi.fn().mockResolvedValue({}),
|
||||
},
|
||||
bastion: {
|
||||
upsert: vi.fn().mockResolvedValue({}),
|
||||
findMany: vi.fn().mockResolvedValue([]),
|
||||
findUnique: vi.fn().mockResolvedValue(null),
|
||||
update: vi.fn().mockResolvedValue({}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function registerFakeBastion(bastionId: string, state: BastionState): void {
|
||||
bastionRegistry.register({
|
||||
bastionId,
|
||||
hostname: "fake",
|
||||
network: "192.168.8.0/24",
|
||||
serverIp: "192.168.8.11",
|
||||
// socket is referenced only on commands, not during aggregation
|
||||
socket: { on: () => undefined, off: () => undefined, send: () => undefined, close: () => undefined } as never,
|
||||
connectedAt: new Date(),
|
||||
lastHeartbeat: new Date(),
|
||||
state,
|
||||
});
|
||||
}
|
||||
|
||||
describe("GET /api/machines aggregation", () => {
|
||||
beforeEach(() => {
|
||||
for (const b of bastionRegistry.getAll()) bastionRegistry.unregister(b.bastionId);
|
||||
});
|
||||
|
||||
it("promotes a live-discovered MAC to installed when the DB has a real hostname+role for it", async () => {
|
||||
// Simulates the worker0-k8s0 bug: bastion restarted, lost its installed map,
|
||||
// rediscovered the machine via DHCP/PXE. DB still has hostname=worker0-k8s0,
|
||||
// role=infra, ip=192.168.8.23. Without the fix, the CLI sees a "discovered"
|
||||
// row with no hostname/role/IP. With the fix, the row is promoted to
|
||||
// "installed" with full identity preserved.
|
||||
const mac = "78:55:36:08:28:fb";
|
||||
registerFakeBastion("b1", {
|
||||
discovered: {
|
||||
[mac]: {
|
||||
mac, product: "SER", board: "SER", serial: "x", manufacturer: "AZW",
|
||||
cpu_model: "AMD Ryzen 7 255", cpu_cores: 16, memory_gb: 58, arch: "x86_64",
|
||||
disks: [], nics: [], first_seen: "", last_seen: "",
|
||||
},
|
||||
},
|
||||
install_queue: {},
|
||||
installed: {},
|
||||
debug: {},
|
||||
});
|
||||
|
||||
const app = Fastify({ logger: false });
|
||||
const db = createMockDb([
|
||||
{ mac, hostname: "worker0-k8s0", role: "infra", ip: "192.168.8.23", status: "discovered", labels: {} },
|
||||
]);
|
||||
registerBastionRoutes(app, db);
|
||||
|
||||
const res = await app.inject({ method: "GET", url: "/api/machines" });
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
|
||||
expect(body.discovered[mac]).toBeUndefined();
|
||||
expect(body.installed[mac]).toMatchObject({
|
||||
hostname: "worker0-k8s0",
|
||||
role: "infra",
|
||||
ip: "192.168.8.23",
|
||||
cpu_model: "AMD Ryzen 7 255",
|
||||
cpu_cores: 16,
|
||||
memory_gb: 58,
|
||||
});
|
||||
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it("leaves a fresh-discovery MAC in discovered when DB only has a discovery-shaped record", async () => {
|
||||
const mac = "aa:bb:cc:dd:ee:ff";
|
||||
registerFakeBastion("b1", {
|
||||
discovered: {
|
||||
[mac]: {
|
||||
mac, product: "SER", board: "SER", serial: "x", manufacturer: "AZW",
|
||||
cpu_model: "AMD Ryzen 7", cpu_cores: 8, memory_gb: 32, arch: "x86_64",
|
||||
disks: [], nics: [], first_seen: "", last_seen: "",
|
||||
},
|
||||
},
|
||||
install_queue: {},
|
||||
installed: {},
|
||||
debug: {},
|
||||
});
|
||||
|
||||
const app = Fastify({ logger: false });
|
||||
// Matches what labd writes on first discovery: hostname=product, role="unknown"
|
||||
const db = createMockDb([
|
||||
{ mac, hostname: "SER", role: "unknown", ip: null, status: "discovered", labels: {} },
|
||||
]);
|
||||
registerBastionRoutes(app, db);
|
||||
|
||||
const res = await app.inject({ method: "GET", url: "/api/machines" });
|
||||
const body = JSON.parse(res.body);
|
||||
|
||||
expect(body.discovered[mac]).toBeDefined();
|
||||
expect(body.installed[mac]).toBeUndefined();
|
||||
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it("falls back to DB for MACs not in any live bucket", async () => {
|
||||
const mac = "11:22:33:44:55:66";
|
||||
// No bastions connected
|
||||
const app = Fastify({ logger: false });
|
||||
const db = createMockDb([
|
||||
{ mac, hostname: "worker1-k8s0", role: "infra", ip: "192.168.8.13", status: "online", labels: {} },
|
||||
]);
|
||||
registerBastionRoutes(app, db);
|
||||
|
||||
const res = await app.inject({ method: "GET", url: "/api/machines" });
|
||||
const body = JSON.parse(res.body);
|
||||
|
||||
expect(body.installed[mac]).toMatchObject({
|
||||
hostname: "worker1-k8s0",
|
||||
role: "infra",
|
||||
ip: "192.168.8.13",
|
||||
});
|
||||
|
||||
await app.close();
|
||||
});
|
||||
});
|
||||
425
bastion/src/labd/tests/v2-smoke.test.ts
Normal file
425
bastion/src/labd/tests/v2-smoke.test.ts
Normal file
@@ -0,0 +1,425 @@
|
||||
// End-to-end smoke tests for the v2.0 Phase 1 surface (auth bootstrap, RBAC,
|
||||
// audit correlation). These exercise the wiring in createApp(): the bearer
|
||||
// auth middleware, the v2 routes scope, and the AuditService lifecycle.
|
||||
//
|
||||
// We don't spin up CockroachDB. Instead we provide a PrismaClient-shaped
|
||||
// in-memory mock that matches the surface the v2 services actually touch.
|
||||
// Tests follow the project convention of using mock DBs + Fastify.inject().
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
||||
import bcrypt from "bcryptjs";
|
||||
import { createApp } from "../src/server.js";
|
||||
import type { DbClient } from "../src/server.js";
|
||||
import type { AuditService } from "../src/services/audit.js";
|
||||
|
||||
const TEST_CONFIG = { port: 0, host: "127.0.0.1", databaseUrl: "", caDir: "/tmp", logLevel: "silent" };
|
||||
|
||||
interface UserRow { id: string; email: string; password: string; role: string; name: string | null; }
|
||||
interface SessionRow { id: string; userId: string; token: string; expiresAt: Date; user?: UserRow; }
|
||||
interface RbacDefRow { id: string; name: string; subjects: unknown; roleBindings: unknown; }
|
||||
interface AuditEventRow {
|
||||
id: string;
|
||||
eventKind: string;
|
||||
source: string;
|
||||
verified: boolean;
|
||||
userId: string | null;
|
||||
userName: string | null;
|
||||
environmentName: string | null;
|
||||
resourceKind: string | null;
|
||||
correlationId: string | null;
|
||||
parentEventId: string | null;
|
||||
details: unknown;
|
||||
result: string;
|
||||
error: string | null;
|
||||
durationMs: number | null;
|
||||
timestamp: Date;
|
||||
}
|
||||
|
||||
interface Stores {
|
||||
users: Map<string, UserRow>;
|
||||
sessions: Map<string, SessionRow>;
|
||||
groupMembers: Array<{ userId: string; group: { name: string } }>;
|
||||
rbacDefs: RbacDefRow[];
|
||||
auditEvents: AuditEventRow[];
|
||||
resources: Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
function makeStores(): Stores {
|
||||
return {
|
||||
users: new Map(),
|
||||
sessions: new Map(),
|
||||
groupMembers: [],
|
||||
rbacDefs: [],
|
||||
auditEvents: [],
|
||||
resources: [],
|
||||
};
|
||||
}
|
||||
|
||||
function makeMockDb(s: Stores): DbClient {
|
||||
let idCounter = 0;
|
||||
const newId = (prefix: string): string => `${prefix}-${++idCounter}`;
|
||||
|
||||
return {
|
||||
$queryRaw: vi.fn(async () => [{ "?column?": 1 }]),
|
||||
server: { findMany: vi.fn(async () => []), findUnique: vi.fn(), upsert: vi.fn() },
|
||||
joinToken: { findUnique: vi.fn(), findMany: vi.fn(), create: vi.fn(), update: vi.fn() },
|
||||
bastion: { upsert: vi.fn(), findMany: vi.fn(), findUnique: vi.fn(), update: vi.fn() },
|
||||
|
||||
user: {
|
||||
count: vi.fn(async () => s.users.size),
|
||||
findUnique: vi.fn(async (args: { where: { email?: string; id?: string } }) => {
|
||||
if (args.where.email) {
|
||||
for (const u of s.users.values()) if (u.email === args.where.email) return u;
|
||||
}
|
||||
if (args.where.id) return s.users.get(args.where.id) ?? null;
|
||||
return null;
|
||||
}),
|
||||
create: vi.fn(async (args: { data: Omit<UserRow, "id"> }) => {
|
||||
const id = newId("user");
|
||||
const row: UserRow = { id, ...args.data };
|
||||
s.users.set(id, row);
|
||||
return row;
|
||||
}),
|
||||
},
|
||||
session: {
|
||||
findUnique: vi.fn(async (args: { where: { token?: string; id?: string }; include?: { user?: boolean } }) => {
|
||||
let session: SessionRow | undefined;
|
||||
if (args.where.token) {
|
||||
for (const sess of s.sessions.values()) if (sess.token === args.where.token) { session = sess; break; }
|
||||
} else if (args.where.id) {
|
||||
session = s.sessions.get(args.where.id);
|
||||
}
|
||||
if (!session) return null;
|
||||
if (args.include?.user) {
|
||||
return { ...session, user: s.users.get(session.userId)! };
|
||||
}
|
||||
return session;
|
||||
}),
|
||||
create: vi.fn(async (args: { data: { userId: string; token: string; expiresAt: Date } }) => {
|
||||
const id = newId("sess");
|
||||
const row: SessionRow = { id, ...args.data };
|
||||
s.sessions.set(id, row);
|
||||
return row;
|
||||
}),
|
||||
delete: vi.fn(async (args: { where: { id: string } }) => {
|
||||
s.sessions.delete(args.where.id);
|
||||
return null;
|
||||
}),
|
||||
},
|
||||
groupMember: {
|
||||
findMany: vi.fn(async (args: { where: { userId: string } }) =>
|
||||
s.groupMembers.filter((m) => m.userId === args.where.userId),
|
||||
),
|
||||
},
|
||||
rbacDefinition: {
|
||||
findMany: vi.fn(async () => s.rbacDefs),
|
||||
},
|
||||
auditEvent: {
|
||||
createMany: vi.fn(async (args: { data: Array<Omit<AuditEventRow, "id" | "timestamp">> }) => {
|
||||
const ts = new Date();
|
||||
for (const e of args.data) {
|
||||
s.auditEvents.push({ id: newId("evt"), timestamp: ts, ...e });
|
||||
}
|
||||
return { count: args.data.length };
|
||||
}),
|
||||
findMany: vi.fn(async (args: { where?: Record<string, unknown>; orderBy?: unknown; take?: number }) => {
|
||||
const where = args.where ?? {};
|
||||
const filtered = s.auditEvents.filter((e) => {
|
||||
if (where["eventKind"] && e.eventKind !== where["eventKind"]) return false;
|
||||
if (where["correlationId"] && e.correlationId !== where["correlationId"]) return false;
|
||||
if (where["environmentName"] && e.environmentName !== where["environmentName"]) return false;
|
||||
return true;
|
||||
});
|
||||
return filtered.slice(0, args.take ?? 100);
|
||||
}),
|
||||
},
|
||||
resource: {
|
||||
findMany: vi.fn(async () => s.resources),
|
||||
findUnique: vi.fn(),
|
||||
create: vi.fn(),
|
||||
update: vi.fn(),
|
||||
delete: vi.fn(),
|
||||
},
|
||||
environment: { findMany: vi.fn(async () => []), findUnique: vi.fn(), create: vi.fn() },
|
||||
account: { findMany: vi.fn(async () => []), findUnique: vi.fn(), create: vi.fn() },
|
||||
binding: { findMany: vi.fn(async () => []), create: vi.fn() },
|
||||
} as unknown as DbClient;
|
||||
}
|
||||
|
||||
async function buildApp(s: Stores) {
|
||||
const db = makeMockDb(s);
|
||||
const result = await createApp(TEST_CONFIG, db);
|
||||
await result.app.ready();
|
||||
return result;
|
||||
}
|
||||
|
||||
describe("v2 auth: bootstrap flow", () => {
|
||||
let stores: Stores;
|
||||
let app: Awaited<ReturnType<typeof buildApp>>["app"];
|
||||
let auditService: AuditService;
|
||||
|
||||
beforeEach(async () => {
|
||||
stores = makeStores();
|
||||
const built = await buildApp(stores);
|
||||
app = built.app;
|
||||
auditService = built.auditService;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await app.close(); // triggers auditService.stop()
|
||||
});
|
||||
|
||||
it("first login with no users seeds the admin and returns a session token", async () => {
|
||||
expect(stores.users.size).toBe(0);
|
||||
|
||||
const resp = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/auth/login",
|
||||
payload: { email: "admin@itaz.eu", password: "s3cret-pw" },
|
||||
});
|
||||
|
||||
expect(resp.statusCode).toBe(200);
|
||||
const body = resp.json();
|
||||
expect(body.isBootstrap).toBe(true);
|
||||
expect(body.token).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect(typeof body.expiresAt).toBe("string");
|
||||
|
||||
expect(stores.users.size).toBe(1);
|
||||
const created = [...stores.users.values()][0]!;
|
||||
expect(created.email).toBe("admin@itaz.eu");
|
||||
expect(created.role).toBe("ADMIN");
|
||||
// Password is hashed, not stored plaintext.
|
||||
expect(created.password).not.toBe("s3cret-pw");
|
||||
expect(await bcrypt.compare("s3cret-pw", created.password)).toBe(true);
|
||||
|
||||
// Bootstrap emits an audit event.
|
||||
await auditService.flushPending();
|
||||
const bootstrapEvents = stores.auditEvents.filter((e) => e.eventKind === "auth_bootstrap");
|
||||
expect(bootstrapEvents).toHaveLength(1);
|
||||
expect(bootstrapEvents[0]!.result).toBe("success");
|
||||
expect(bootstrapEvents[0]!.userName).toBe("admin@itaz.eu");
|
||||
});
|
||||
|
||||
it("returns 400 for missing credentials", async () => {
|
||||
const resp = await app.inject({ method: "POST", url: "/api/auth/login", payload: {} });
|
||||
expect(resp.statusCode).toBe(400);
|
||||
});
|
||||
|
||||
it("second login uses normal flow (no isBootstrap)", async () => {
|
||||
// Bootstrap once
|
||||
await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/auth/login",
|
||||
payload: { email: "admin@itaz.eu", password: "s3cret-pw" },
|
||||
});
|
||||
expect(stores.users.size).toBe(1);
|
||||
|
||||
// Login again
|
||||
const resp = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/auth/login",
|
||||
payload: { email: "admin@itaz.eu", password: "s3cret-pw" },
|
||||
});
|
||||
|
||||
expect(resp.statusCode).toBe(200);
|
||||
expect(resp.json().isBootstrap).toBe(false);
|
||||
expect(stores.users.size).toBe(1); // no new user
|
||||
});
|
||||
|
||||
it("rejects wrong password with 401", async () => {
|
||||
// Seed admin
|
||||
await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/auth/login",
|
||||
payload: { email: "admin@itaz.eu", password: "s3cret-pw" },
|
||||
});
|
||||
|
||||
const resp = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/auth/login",
|
||||
payload: { email: "admin@itaz.eu", password: "wrong" },
|
||||
});
|
||||
expect(resp.statusCode).toBe(401);
|
||||
|
||||
// Failed login is also audited.
|
||||
await auditService.flushPending();
|
||||
const fails = stores.auditEvents.filter((e) => e.eventKind === "auth_login" && e.result === "failure");
|
||||
expect(fails).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("v2 RBAC: env-scoped denial", () => {
|
||||
let stores: Stores;
|
||||
let app: Awaited<ReturnType<typeof buildApp>>["app"];
|
||||
|
||||
async function seedSession(role: string): Promise<string> {
|
||||
stores.users.set("u-1", {
|
||||
id: "u-1",
|
||||
email: `${role.toLowerCase()}@itaz.eu`,
|
||||
password: "x",
|
||||
role,
|
||||
name: null,
|
||||
});
|
||||
const token = "test-token-" + role;
|
||||
stores.sessions.set("s-1", {
|
||||
id: "s-1",
|
||||
userId: "u-1",
|
||||
token,
|
||||
expiresAt: new Date(Date.now() + 86_400_000),
|
||||
});
|
||||
return token;
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
stores = makeStores();
|
||||
app = (await buildApp(stores)).app;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it("non-admin user with no role bindings gets 403 on /api/resources", async () => {
|
||||
const token = await seedSession("EDITOR"); // not admin, no bindings
|
||||
|
||||
const resp = await app.inject({
|
||||
method: "GET",
|
||||
url: "/api/resources",
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
});
|
||||
|
||||
expect(resp.statusCode).toBe(403);
|
||||
expect(resp.json().error).toMatch(/no matching role binding/);
|
||||
});
|
||||
|
||||
it("missing/empty bearer token gets 401 (auth, not RBAC)", async () => {
|
||||
const r1 = await app.inject({ method: "GET", url: "/api/resources" });
|
||||
expect(r1.statusCode).toBe(401);
|
||||
|
||||
const r2 = await app.inject({
|
||||
method: "GET",
|
||||
url: "/api/resources",
|
||||
headers: { authorization: "Bearer " },
|
||||
});
|
||||
expect(r2.statusCode).toBe(401);
|
||||
});
|
||||
|
||||
it("invalid bearer token gets 401", async () => {
|
||||
const resp = await app.inject({
|
||||
method: "GET",
|
||||
url: "/api/resources",
|
||||
headers: { authorization: "Bearer not-a-real-token" },
|
||||
});
|
||||
expect(resp.statusCode).toBe(401);
|
||||
});
|
||||
|
||||
it("admin role bypasses RBAC", async () => {
|
||||
const token = await seedSession("ADMIN");
|
||||
|
||||
const resp = await app.inject({
|
||||
method: "GET",
|
||||
url: "/api/resources",
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
});
|
||||
|
||||
expect(resp.statusCode).toBe(200);
|
||||
expect(resp.json()).toEqual([]);
|
||||
});
|
||||
|
||||
it("user with binding for env A is denied for resources in env B", async () => {
|
||||
const token = await seedSession("EDITOR");
|
||||
stores.groupMembers.push({ userId: "u-1", group: { name: "team-a" } });
|
||||
stores.rbacDefs.push({
|
||||
id: "rbac-1",
|
||||
name: "team-a-edit-on-env-a",
|
||||
subjects: [{ kind: "Group", name: "team-a" }],
|
||||
roleBindings: [{ role: "edit", environment: "env-a" }],
|
||||
});
|
||||
|
||||
// List in env-a → should pass RBAC (no env query so it's global view, but
|
||||
// the binding scope is environment-specific → for global list the binding
|
||||
// doesn't apply when an environment scope is set on the binding).
|
||||
// Smoke test the targeted denial: trying to create in env-b is rejected.
|
||||
const respB = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/resources",
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
payload: { kind: "database", name: "x", environmentId: "env-b", accountId: "acc-1" },
|
||||
});
|
||||
|
||||
expect(respB.statusCode).toBe(403);
|
||||
expect(respB.json().error).toMatch(/no matching role binding/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("v2 audit: correlation chain visible via /api/events", () => {
|
||||
let stores: Stores;
|
||||
let app: Awaited<ReturnType<typeof buildApp>>["app"];
|
||||
let auditService: AuditService;
|
||||
|
||||
beforeEach(async () => {
|
||||
stores = makeStores();
|
||||
const built = await buildApp(stores);
|
||||
app = built.app;
|
||||
auditService = built.auditService;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it("emitted audit events are queryable by correlation id", async () => {
|
||||
// Seed admin so /api/events is accessible (it sits behind bearer auth)
|
||||
const loginResp = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/auth/login",
|
||||
payload: { email: "admin@itaz.eu", password: "pw" },
|
||||
});
|
||||
const token = loginResp.json().token;
|
||||
|
||||
// Force flush so the bootstrap event is in the DB
|
||||
await auditService.flushPending();
|
||||
|
||||
expect(stores.auditEvents.length).toBeGreaterThan(0);
|
||||
const bootstrap = stores.auditEvents.find((e) => e.eventKind === "auth_bootstrap")!;
|
||||
expect(bootstrap.correlationId).toMatch(/^corr_[a-f0-9]{16}$/);
|
||||
|
||||
// Query /api/events filtered by correlation id
|
||||
const queryResp = await app.inject({
|
||||
method: "GET",
|
||||
url: `/api/events?correlation=${bootstrap.correlationId}`,
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
});
|
||||
|
||||
expect(queryResp.statusCode).toBe(200);
|
||||
const events = queryResp.json() as Array<{ correlationId: string; eventKind: string }>;
|
||||
expect(events.length).toBe(1);
|
||||
expect(events[0]!.eventKind).toBe("auth_bootstrap");
|
||||
expect(events[0]!.correlationId).toBe(bootstrap.correlationId);
|
||||
});
|
||||
|
||||
it("explicit parent/child correlation chain is preserved across emits", async () => {
|
||||
const correlationId = auditService.createCorrelation();
|
||||
|
||||
auditService.emit({
|
||||
eventKind: "test_parent",
|
||||
source: "test",
|
||||
result: "success",
|
||||
correlationId,
|
||||
});
|
||||
auditService.emit({
|
||||
eventKind: "test_child",
|
||||
source: "test",
|
||||
result: "success",
|
||||
correlationId,
|
||||
parentEventId: "evt-1",
|
||||
});
|
||||
|
||||
await auditService.flushPending();
|
||||
|
||||
const chain = stores.auditEvents.filter((e) => e.correlationId === correlationId);
|
||||
expect(chain).toHaveLength(2);
|
||||
expect(chain.map((e) => e.eventKind).sort()).toEqual(["test_child", "test_parent"]);
|
||||
expect(chain.find((e) => e.eventKind === "test_child")!.parentEventId).toBe("evt-1");
|
||||
});
|
||||
});
|
||||
@@ -1,21 +1,23 @@
|
||||
// Host preparation: kernel modules, sysctl, swap, firewall, SELinux.
|
||||
// Host preparation: kernel modules, sysctl, swap, storage, firewall, SELinux.
|
||||
|
||||
import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
|
||||
import { runSequential } from "../utils.js";
|
||||
import { loadKernelModules } from "../operations/kernel-modules.js";
|
||||
import { applyCisHardening } from "../operations/sysctl.js";
|
||||
import { disableSwap } from "../operations/swap.js";
|
||||
import { enableSwap } from "../operations/swap.js";
|
||||
import { growRancherLv } from "../operations/rancher-storage.js";
|
||||
import { disableFirewall } from "../operations/firewall.js";
|
||||
import { setSelinuxPermissive } from "../operations/selinux.js";
|
||||
import { enableIscsi } from "../operations/iscsi.js";
|
||||
|
||||
export const hostPrepGroup: OperationGroup = {
|
||||
name: "host-prep",
|
||||
description: "Prepare host for k3s: kernel modules, sysctl, swap, firewall, SELinux, iSCSI",
|
||||
description: "Prepare host for k3s: kernel modules, sysctl, swap, imageFs sizing, firewall, SELinux, iSCSI",
|
||||
operations: [
|
||||
{ name: "Load kernel modules", fn: loadKernelModules },
|
||||
{ name: "Apply CIS sysctl", fn: applyCisHardening },
|
||||
{ name: "Disable swap", fn: disableSwap },
|
||||
{ name: "Enable swap", fn: enableSwap },
|
||||
{ name: "Grow rancher LV", fn: growRancherLv },
|
||||
{ name: "Disable firewall", fn: disableFirewall },
|
||||
{ name: "Set SELinux permissive", fn: setSelinuxPermissive },
|
||||
{ name: "Enable iSCSI", fn: enableIscsi },
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
|
||||
import { runSequential } from "../utils.js";
|
||||
import { installCilium } from "../operations/cilium.js";
|
||||
import { installMultus } from "../operations/multus.js";
|
||||
import { installVlanSetup } from "../operations/vlan-setup.js";
|
||||
import { fixCoreDnsUpstream } from "../operations/dns-fix.js";
|
||||
import { applyDefaultNetworkPolicies } from "../operations/network-policy.js";
|
||||
|
||||
@@ -11,6 +13,11 @@ export const networkingGroup: OperationGroup = {
|
||||
description: "Install Cilium CNI, fix DNS, apply network policies",
|
||||
operations: [
|
||||
{ name: "Install Cilium CNI", fn: installCilium },
|
||||
// Multus + vlan-setup: give pods a second interface on VLAN 10 (macvlan)
|
||||
// for LAN device discovery (Matter/HomeKit mDNS). Must follow Cilium
|
||||
// (needs cni.exclusive=false + bpf.vlanBypass={10} from installCilium).
|
||||
{ name: "Install Multus CNI", fn: installMultus },
|
||||
{ name: "Install vlan-setup (lan10 + CNI plugins)", fn: installVlanSetup },
|
||||
{ name: "Fix CoreDNS upstream", fn: fixCoreDnsUpstream },
|
||||
{ name: "Apply network policies", fn: applyDefaultNetworkPolicies },
|
||||
],
|
||||
|
||||
@@ -38,12 +38,20 @@ export const installCilium: Operation = async (ctx): Promise<OperationResult> =>
|
||||
// Install Cilium
|
||||
// - No hardcoded devices: Cilium auto-detects per node (heterogeneous NICs like eno1 vs enP7s7)
|
||||
// - k8sServiceHost/Port: k3s agents proxy the API on 127.0.0.1:6444 (not 6443)
|
||||
// - cni.exclusive=false: required so Multus can install its CNI config alongside
|
||||
// Cilium (Cilium otherwise deletes any non-Cilium CNI conf).
|
||||
// - bpf.vlanBypass={10}: allow VLAN 10 (LoT) tagged traffic through the eBPF
|
||||
// host VLAN filter, so pods on a macvlan/VLAN-10 interface receive multicast
|
||||
// (Matter/mDNS ff02::fb + 224.0.0.251). Without this Cilium drops it
|
||||
// ("VLAN traffic disallowed by VLAN filter", bpf_host.c).
|
||||
const installResult = await ctx.ssh.exec(
|
||||
`KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install \
|
||||
--set kubeProxyReplacement=true \
|
||||
--set ipam.mode=kubernetes \
|
||||
--set k8sServiceHost=127.0.0.1 \
|
||||
--set k8sServicePort=6444`,
|
||||
--set k8sServicePort=6444 \
|
||||
--set cni.exclusive=false \
|
||||
--set bpf.vlanBypass="{10}"`,
|
||||
{ timeoutMs: 300_000 },
|
||||
);
|
||||
if (installResult.exitCode !== 0) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
export { loadKernelModules } from "./kernel-modules.js";
|
||||
export { applyCisHardening } from "./sysctl.js";
|
||||
export { disableSwap } from "./swap.js";
|
||||
export { enableSwap } from "./swap.js";
|
||||
export { growRancherLv } from "./rancher-storage.js";
|
||||
export { enableIscsi } from "./iscsi.js";
|
||||
export { disableFirewall } from "./firewall.js";
|
||||
export { setSelinuxPermissive } from "./selinux.js";
|
||||
@@ -9,6 +10,8 @@ export { writeAuditPolicy } from "./audit-policy.js";
|
||||
export { cleanupStaleCni } from "./cni-cleanup.js";
|
||||
export { installK3sBinary } from "./k3s-install.js";
|
||||
export { installCilium } from "./cilium.js";
|
||||
export { installMultus } from "./multus.js";
|
||||
export { installVlanSetup } from "./vlan-setup.js";
|
||||
export { fixCoreDnsUpstream } from "./dns-fix.js";
|
||||
export { configureLogRotation } from "./log-rotation.js";
|
||||
export { configureJournaldLimits } from "./journald-limits.js";
|
||||
|
||||
34
bastion/src/modules/modules/k3s/src/operations/multus.ts
Normal file
34
bastion/src/modules/modules/k3s/src/operations/multus.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
// Install Multus CNI (thick plugin) — the meta-CNI that lets pods attach an
|
||||
// extra interface (macvlan on VLAN 10) alongside Cilium, via a
|
||||
// NetworkAttachmentDefinition. Required for Home Assistant's LAN presence
|
||||
// (Matter/HomeKit mDNS discovery). Cilium must be installed with
|
||||
// cni.exclusive=false first (see cilium.ts) or it deletes Multus's CNI conf.
|
||||
|
||||
import type { Operation, OperationResult } from "../types.js";
|
||||
import { sshOpts } from "../utils.js";
|
||||
|
||||
const MULTUS_VERSION = "v4.1.4";
|
||||
const MULTUS_MANIFEST = `https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/${MULTUS_VERSION}/deployments/multus-daemonset-thick.yml`;
|
||||
|
||||
export const installMultus: Operation = async (ctx): Promise<OperationResult> => {
|
||||
const K = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml";
|
||||
|
||||
// Idempotent: skip if the Multus DaemonSet is already present.
|
||||
const check = await ctx.ssh.exec(
|
||||
`${K} kubectl -n kube-system get ds kube-multus-ds -o name 2>/dev/null`,
|
||||
sshOpts(ctx),
|
||||
);
|
||||
if (check.exitCode === 0 && check.stdout.includes("kube-multus-ds")) {
|
||||
return { success: true, changed: false, message: `Multus already installed (${MULTUS_VERSION})` };
|
||||
}
|
||||
|
||||
const apply = await ctx.ssh.exec(
|
||||
`${K} kubectl apply -f ${MULTUS_MANIFEST}`,
|
||||
{ ...sshOpts(ctx), timeoutMs: 120_000 },
|
||||
);
|
||||
if (apply.exitCode !== 0) {
|
||||
return { success: false, changed: false, message: "Failed to apply Multus manifest", error: apply.stderr };
|
||||
}
|
||||
|
||||
return { success: true, changed: true, message: `Installed Multus ${MULTUS_VERSION} (thick)` };
|
||||
};
|
||||
@@ -0,0 +1,48 @@
|
||||
// Grow the labvg/rancher LV (k3s image store / imageFs) to 120G.
|
||||
// 2026-08 incident: the original 20G LV sat at 85% used from steady-state
|
||||
// images alone, so one ~5G image pull tripped imagefs eviction and evicted
|
||||
// unrelated pods. Fresh installs are sized at 120G by the kickstart; this op
|
||||
// covers nodes installed before that change and vanilla nodes converted to
|
||||
// k8s later. Never removes or shrinks anything — if the VG lacks free space
|
||||
// (e.g. a longhorn --grow LV consumed it), it reports and moves on.
|
||||
|
||||
import type { Operation, OperationResult } from "../types.js";
|
||||
import { sshOpts } from "../utils.js";
|
||||
|
||||
const RANCHER_LV = "labvg/rancher";
|
||||
const TARGET_MIB = 122880; // 120G
|
||||
|
||||
export const growRancherLv: Operation = async (ctx): Promise<OperationResult> => {
|
||||
const lv = await ctx.ssh.exec(
|
||||
`lvs --noheadings --units m --nosuffix -o lv_size ${RANCHER_LV} 2>/dev/null || true`,
|
||||
sshOpts(ctx),
|
||||
);
|
||||
const sizeMib = Number.parseFloat(lv.stdout.trim());
|
||||
if (Number.isNaN(sizeMib)) {
|
||||
return { success: true, changed: false, message: "No labvg/rancher LV — imageFs shares /var, skipping" };
|
||||
}
|
||||
if (sizeMib >= TARGET_MIB) {
|
||||
return { success: true, changed: false, message: `rancher LV already ${Math.round(sizeMib / 1024)}G` };
|
||||
}
|
||||
|
||||
const vg = await ctx.ssh.exec(`vgs --noheadings --units m --nosuffix -o vg_free labvg`, sshOpts(ctx));
|
||||
const freeMib = Number.parseFloat(vg.stdout.trim());
|
||||
const neededMib = TARGET_MIB - sizeMib;
|
||||
if (Number.isNaN(freeMib) || freeMib < neededMib) {
|
||||
return {
|
||||
success: true,
|
||||
changed: false,
|
||||
message: `VG labvg has ${Math.floor((Number.isNaN(freeMib) ? 0 : freeMib) / 1024)}G free — ` +
|
||||
`need ${Math.ceil(neededMib / 1024)}G to grow rancher LV to 120G (manual LV rebuild required)`,
|
||||
};
|
||||
}
|
||||
|
||||
await ctx.ssh.exec(`lvextend -L ${TARGET_MIB}m /dev/${RANCHER_LV}`, sshOpts(ctx));
|
||||
await ctx.ssh.exec(`xfs_growfs /var/lib/rancher`, sshOpts(ctx));
|
||||
|
||||
return {
|
||||
success: true,
|
||||
changed: true,
|
||||
message: `rancher LV grown ${Math.round(sizeMib / 1024)}G → 120G`,
|
||||
};
|
||||
};
|
||||
@@ -1,22 +1,40 @@
|
||||
// Disable swap (CIS requirement for k3s).
|
||||
// Enable swap so memory pressure spills to disk instead of OOM-killing.
|
||||
// kubelet runs with failSwapOn=false (k3s default); zram stays the fast tier,
|
||||
// the labvg-swap LV is the overflow tier. Replaces the old CIS-style
|
||||
// disableSwap op — a kernel OOM kill of a node daemon is worse than slow swap.
|
||||
|
||||
import type { Operation, OperationResult } from "../types.js";
|
||||
import { sshOpts } from "../utils.js";
|
||||
|
||||
export const disableSwap: Operation = async (ctx): Promise<OperationResult> => {
|
||||
const check = await ctx.ssh.exec("swapon --show --noheadings", sshOpts(ctx));
|
||||
const active = check.stdout.trim().length > 0;
|
||||
const SWAP_DEV = "/dev/mapper/labvg-swap";
|
||||
|
||||
if (active) {
|
||||
await ctx.ssh.exec("swapoff -a", sshOpts(ctx));
|
||||
export const enableSwap: Operation = async (ctx): Promise<OperationResult> => {
|
||||
const lv = await ctx.ssh.exec(`test -b ${SWAP_DEV} && echo yes || echo no`, sshOpts(ctx));
|
||||
if (lv.stdout.trim() !== "yes") {
|
||||
return { success: true, changed: false, message: "No labvg-swap LV — skipping swap enable" };
|
||||
}
|
||||
|
||||
// Remove swap entries from fstab permanently
|
||||
await ctx.ssh.exec("sed -i '/\\sswap\\s/d' /etc/fstab", sshOpts(ctx));
|
||||
const active = await ctx.ssh.exec(
|
||||
`grep -q "^$(readlink -f ${SWAP_DEV}) " /proc/swaps && echo on || echo off`,
|
||||
sshOpts(ctx),
|
||||
);
|
||||
const wasOff = active.stdout.trim() !== "on";
|
||||
|
||||
if (wasOff) {
|
||||
// Format if the LV was never (or wrongly) initialised, then activate
|
||||
await ctx.ssh.exec(`blkid ${SWAP_DEV} | grep -q 'TYPE="swap"' || mkswap ${SWAP_DEV}`, sshOpts(ctx));
|
||||
await ctx.ssh.exec(`swapon ${SWAP_DEV}`, sshOpts(ctx));
|
||||
}
|
||||
|
||||
// Persist across reboots (idempotent)
|
||||
await ctx.ssh.exec(
|
||||
`grep -q "labvg-swap" /etc/fstab || echo "${SWAP_DEV} none swap defaults 0 0" >> /etc/fstab`,
|
||||
sshOpts(ctx),
|
||||
);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
changed: active,
|
||||
message: active ? "Swap disabled" : "Swap already disabled",
|
||||
changed: wasOff,
|
||||
message: wasOff ? "LV swap enabled" : "LV swap already active",
|
||||
};
|
||||
};
|
||||
|
||||
96
bastion/src/modules/modules/k3s/src/operations/vlan-setup.ts
Normal file
96
bastion/src/modules/modules/k3s/src/operations/vlan-setup.ts
Normal file
@@ -0,0 +1,96 @@
|
||||
// vlan-setup DaemonSet — the node-level half of the macvlan/VLAN-10 story.
|
||||
// On every node it (1) installs the reference CNI plugins (macvlan/ipvlan/
|
||||
// static/host-local/vlan/tuning) into /opt/cni/bin if missing, and (2) creates
|
||||
// a `lan10` VLAN-10 sub-interface on the primary NIC that macvlan
|
||||
// NetworkAttachmentDefinitions use as their master. Idempotent + self-healing
|
||||
// (re-creates lan10 if it disappears). Paired with Multus (multus.ts) + Cilium
|
||||
// bpf.vlanBypass={10} (cilium.ts).
|
||||
|
||||
import type { Operation, OperationResult } from "../types.js";
|
||||
import { sshOpts } from "../utils.js";
|
||||
|
||||
const MANIFEST = `apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: macvlan-sys
|
||||
labels:
|
||||
pod-security.kubernetes.io/enforce: privileged
|
||||
pod-security.kubernetes.io/audit: privileged
|
||||
pod-security.kubernetes.io/warn: privileged
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: vlan-setup
|
||||
namespace: macvlan-sys
|
||||
spec:
|
||||
selector:
|
||||
matchLabels: { app: vlan-setup }
|
||||
template:
|
||||
metadata:
|
||||
labels: { app: vlan-setup }
|
||||
spec:
|
||||
hostNetwork: true
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
containers:
|
||||
- name: vlan
|
||||
image: nicolaka/netshoot
|
||||
securityContext:
|
||||
privileged: true
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -x
|
||||
# install reference CNI plugins (macvlan/ipvlan/static/host-local) if missing
|
||||
if [ ! -f /host/opt/cni/bin/macvlan ] || [ ! -f /host/opt/cni/bin/ipvlan ]; then
|
||||
case "$(uname -m)" in x86_64) A=amd64;; aarch64) A=arm64;; *) A=amd64;; esac
|
||||
curl -sSL "https://github.com/containernetworking/plugins/releases/download/v1.5.1/cni-plugins-linux-$A-v1.5.1.tgz" -o /tmp/cni.tgz
|
||||
tar -xzf /tmp/cni.tgz -C /host/opt/cni/bin ./macvlan ./ipvlan ./static ./host-local ./vlan ./tuning
|
||||
fi
|
||||
# detect the primary NIC (default route dev, else the one holding 192.168.8.x)
|
||||
NIC="$(ip -o -4 route show default 2>/dev/null | awk '{print $5; exit}')"
|
||||
[ -z "$NIC" ] && NIC="$(ip -o -4 addr show 2>/dev/null | awk '/192\\.168\\.8\\./{print $2; exit}')"
|
||||
echo "primary NIC = $NIC"
|
||||
while true; do
|
||||
if [ -n "$NIC" ]; then
|
||||
ip link show lan10 >/dev/null 2>&1 || ip link add link "$NIC" name lan10 type vlan id 10
|
||||
ip link set lan10 up
|
||||
# NIC-driver workarounds for VLAN multicast RX
|
||||
ip link set "$NIC" allmulticast on 2>/dev/null
|
||||
ethtool -K "$NIC" rxvlan off rx-vlan-filter off 2>/dev/null
|
||||
fi
|
||||
sleep 30
|
||||
done
|
||||
volumeMounts:
|
||||
- name: cnibin
|
||||
mountPath: /host/opt/cni/bin
|
||||
volumes:
|
||||
- name: cnibin
|
||||
hostPath:
|
||||
path: /opt/cni/bin
|
||||
`;
|
||||
|
||||
export const installVlanSetup: Operation = async (ctx): Promise<OperationResult> => {
|
||||
const K = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml";
|
||||
|
||||
const check = await ctx.ssh.exec(
|
||||
`${K} kubectl -n macvlan-sys get ds vlan-setup -o name 2>/dev/null`,
|
||||
sshOpts(ctx),
|
||||
);
|
||||
if (check.exitCode === 0 && check.stdout.includes("vlan-setup")) {
|
||||
return { success: true, changed: false, message: "vlan-setup DaemonSet already installed" };
|
||||
}
|
||||
|
||||
const b64 = Buffer.from(MANIFEST).toString("base64");
|
||||
const apply = await ctx.ssh.exec(
|
||||
`echo ${b64} | base64 -d | ${K} kubectl apply -f -`,
|
||||
{ ...sshOpts(ctx), timeoutMs: 60_000 },
|
||||
);
|
||||
if (apply.exitCode !== 0) {
|
||||
return { success: false, changed: false, message: "Failed to apply vlan-setup DaemonSet", error: apply.stderr };
|
||||
}
|
||||
|
||||
return { success: true, changed: true, message: "Installed vlan-setup DaemonSet (lan10 + CNI plugins)" };
|
||||
};
|
||||
@@ -72,31 +72,97 @@ describe("applyCisHardening", () => {
|
||||
|
||||
// --- Swap ---
|
||||
|
||||
import { disableSwap } from "../src/operations/swap.js";
|
||||
import { enableSwap } from "../src/operations/swap.js";
|
||||
|
||||
describe("disableSwap", () => {
|
||||
it("disables active swap", async () => {
|
||||
describe("enableSwap", () => {
|
||||
it("activates LV swap when present but off", async () => {
|
||||
const ctx = mockCtx();
|
||||
ctx.ssh.exec
|
||||
.mockResolvedValueOnce(stdout("/dev/sda2 partition 2G")) // swap active
|
||||
.mockResolvedValueOnce(OK) // swapoff
|
||||
.mockResolvedValueOnce(OK); // sed fstab
|
||||
.mockResolvedValueOnce(stdout("yes")) // LV exists
|
||||
.mockResolvedValueOnce(stdout("off")) // not in /proc/swaps
|
||||
.mockResolvedValueOnce(OK) // blkid || mkswap
|
||||
.mockResolvedValueOnce(OK) // swapon
|
||||
.mockResolvedValueOnce(OK); // fstab entry
|
||||
|
||||
const result = await disableSwap(ctx);
|
||||
const result = await enableSwap(ctx);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.changed).toBe(true);
|
||||
expectCommand(ctx.ssh, "swapoff -a");
|
||||
expectCommand(ctx.ssh, "swapon /dev/mapper/labvg-swap");
|
||||
});
|
||||
|
||||
it("is idempotent when swap already off", async () => {
|
||||
it("is idempotent when LV swap already active", async () => {
|
||||
const ctx = mockCtx();
|
||||
ctx.ssh.exec
|
||||
.mockResolvedValueOnce(stdout("")) // no swap
|
||||
.mockResolvedValueOnce(OK); // sed fstab (always runs)
|
||||
.mockResolvedValueOnce(stdout("yes")) // LV exists
|
||||
.mockResolvedValueOnce(stdout("on")) // already in /proc/swaps
|
||||
.mockResolvedValueOnce(OK); // fstab entry (always ensured)
|
||||
|
||||
const result = await disableSwap(ctx);
|
||||
const result = await enableSwap(ctx);
|
||||
expect(result.changed).toBe(false);
|
||||
expectNoCommand(ctx.ssh, "swapoff");
|
||||
expectNoCommand(ctx.ssh, "swapon /dev/mapper/labvg-swap");
|
||||
});
|
||||
|
||||
it("skips when no labvg-swap LV exists", async () => {
|
||||
const ctx = mockCtx();
|
||||
ctx.ssh.exec.mockResolvedValueOnce(stdout("no")); // LV missing
|
||||
|
||||
const result = await enableSwap(ctx);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.changed).toBe(false);
|
||||
expectNoCommand(ctx.ssh, "swapon");
|
||||
});
|
||||
});
|
||||
|
||||
// --- Rancher LV (imageFs sizing) ---
|
||||
|
||||
import { growRancherLv } from "../src/operations/rancher-storage.js";
|
||||
|
||||
describe("growRancherLv", () => {
|
||||
it("grows a 20G LV to 120G when the VG has space", async () => {
|
||||
const ctx = mockCtx();
|
||||
ctx.ssh.exec
|
||||
.mockResolvedValueOnce(stdout(" 20480.00")) // lv_size
|
||||
.mockResolvedValueOnce(stdout(" 747807.00")) // vg_free
|
||||
.mockResolvedValueOnce(OK) // lvextend
|
||||
.mockResolvedValueOnce(OK); // xfs_growfs
|
||||
|
||||
const result = await growRancherLv(ctx);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.changed).toBe(true);
|
||||
expectCommand(ctx.ssh, "lvextend -L 122880m /dev/labvg/rancher");
|
||||
expectCommand(ctx.ssh, "xfs_growfs /var/lib/rancher");
|
||||
});
|
||||
|
||||
it("is idempotent when the LV is already 120G", async () => {
|
||||
const ctx = mockCtx();
|
||||
ctx.ssh.exec.mockResolvedValueOnce(stdout(" 122880.00")); // lv_size
|
||||
|
||||
const result = await growRancherLv(ctx);
|
||||
expect(result.changed).toBe(false);
|
||||
expectNoCommand(ctx.ssh, "lvextend");
|
||||
});
|
||||
|
||||
it("reports without failing when the VG has no free space", async () => {
|
||||
const ctx = mockCtx();
|
||||
ctx.ssh.exec
|
||||
.mockResolvedValueOnce(stdout(" 20480.00")) // lv_size
|
||||
.mockResolvedValueOnce(stdout(" 0.00")); // vg_free
|
||||
|
||||
const result = await growRancherLv(ctx);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.changed).toBe(false);
|
||||
expect(result.message).toContain("free");
|
||||
expectNoCommand(ctx.ssh, "lvextend");
|
||||
});
|
||||
|
||||
it("skips when there is no rancher LV", async () => {
|
||||
const ctx = mockCtx();
|
||||
ctx.ssh.exec.mockResolvedValueOnce(stdout("")); // lvs empty
|
||||
|
||||
const result = await growRancherLv(ctx);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.changed).toBe(false);
|
||||
expectNoCommand(ctx.ssh, "lvextend");
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ describe("smoke: full server install pipeline", () => {
|
||||
const pipeline: NamedOperation[] = [
|
||||
{ name: "Kernel modules", fn: ops.loadKernelModules },
|
||||
{ name: "Sysctl hardening", fn: ops.applyCisHardening },
|
||||
{ name: "Disable swap", fn: ops.disableSwap },
|
||||
{ name: "Enable swap", fn: ops.enableSwap },
|
||||
{ name: "Disable firewall", fn: ops.disableFirewall },
|
||||
{ name: "SELinux permissive", fn: ops.setSelinuxPermissive },
|
||||
{ name: "Write k3s config", fn: ops.writeK3sConfig },
|
||||
@@ -73,7 +73,7 @@ describe("smoke: pipeline stops on failure", () => {
|
||||
};
|
||||
|
||||
const results = await runSequential(ctx, [
|
||||
{ name: "OK op", fn: ops.disableSwap },
|
||||
{ name: "OK op", fn: ops.enableSwap },
|
||||
{ name: "Failing op", fn: failingOp },
|
||||
{ name: "Never called", fn: neverCalled },
|
||||
]);
|
||||
@@ -98,11 +98,12 @@ describe("smoke: agent install rejects missing config", () => {
|
||||
});
|
||||
|
||||
describe("smoke: all operations are exported", () => {
|
||||
it("exports all 15 operations", () => {
|
||||
it("exports all 16 operations", () => {
|
||||
const exported = [
|
||||
ops.loadKernelModules,
|
||||
ops.applyCisHardening,
|
||||
ops.disableSwap,
|
||||
ops.enableSwap,
|
||||
ops.growRancherLv,
|
||||
ops.disableFirewall,
|
||||
ops.setSelinuxPermissive,
|
||||
ops.writeK3sConfig,
|
||||
@@ -117,7 +118,7 @@ describe("smoke: all operations are exported", () => {
|
||||
ops.checkCertExpiry,
|
||||
];
|
||||
|
||||
expect(exported).toHaveLength(15);
|
||||
expect(exported).toHaveLength(16);
|
||||
for (const op of exported) {
|
||||
expect(typeof op).toBe("function");
|
||||
}
|
||||
|
||||
@@ -8,6 +8,8 @@ export type {
|
||||
DebugConfig,
|
||||
BastionState,
|
||||
BastionConfig,
|
||||
VyosVlanSpec,
|
||||
VyosInstallSpec,
|
||||
} from "./types/index.js";
|
||||
|
||||
export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./types/index.js";
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Protocol types for agent-labd WebSocket communication.
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { VyosInstallSpec } from "../types/state.js";
|
||||
|
||||
// --- Agent -> labd messages ---
|
||||
|
||||
@@ -108,7 +109,7 @@ export type BastionMessage =
|
||||
export type LabdBastionMessage =
|
||||
| { type: "bastion-enrolled"; bastionId: string }
|
||||
| { type: "bastion-heartbeat-ack"; serverTime: string }
|
||||
| { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string }
|
||||
| { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string; vyos?: VyosInstallSpec }
|
||||
| { type: "command-forget"; requestId: string; mac: string }
|
||||
| { type: "command-role-update"; requestId: string; mac: string; role: string }
|
||||
| { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean }
|
||||
|
||||
@@ -14,6 +14,10 @@ export interface BastionConfig {
|
||||
// Ubuntu support
|
||||
ubuntuVersion: string;
|
||||
ubuntuMirror: string;
|
||||
// VyOS support — netboot artifacts are extracted from the ISO at startup.
|
||||
// LTS ISOs are subscription-only, so this defaults to a rolling release.
|
||||
vyosIsoUrl: string;
|
||||
vyosDefaultPassword: string;
|
||||
// Syslog listener for install logs (Anaconda logging --host)
|
||||
syslogPort: number;
|
||||
// Flags
|
||||
|
||||
@@ -7,6 +7,8 @@ export type {
|
||||
InstalledInfo,
|
||||
DebugConfig,
|
||||
BastionState,
|
||||
VyosVlanSpec,
|
||||
VyosInstallSpec,
|
||||
} from "./state.js";
|
||||
|
||||
export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./state.js";
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
|
||||
export type ProvisionStackType = "dhcpproxy" | "iso" | "cloud-init";
|
||||
|
||||
export type OsId = "fedora-43" | "ubuntu-26.04";
|
||||
export type OsId = "fedora-43" | "ubuntu-26.04" | "vyos-rolling";
|
||||
export type Arch = "x86_64" | "aarch64";
|
||||
|
||||
export const SUPPORTED_OS: readonly OsId[] = ["fedora-43", "ubuntu-26.04"] as const;
|
||||
export const SUPPORTED_OS: readonly OsId[] = ["fedora-43", "ubuntu-26.04", "vyos-rolling"] as const;
|
||||
|
||||
export function isValidOsId(value: string): value is OsId {
|
||||
return (SUPPORTED_OS as readonly string[]).includes(value);
|
||||
@@ -75,13 +75,89 @@ export interface ProgressLogEntry {
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
/** A tagged VLAN sub-interface on the bond (or on the mgmt NIC when unbonded). */
|
||||
export interface VyosVlanSpec {
|
||||
id: number;
|
||||
address: string; // CIDR, e.g. "10.0.10.1/24"
|
||||
description?: string;
|
||||
/**
|
||||
* VRRP virtual address (CIDR) floated on this VLAN. Emitted as a
|
||||
* high-availability vrrp group with vrid = VLAN id, so the same spec on both
|
||||
* HA peers (with different priorities) produces a matching group pair.
|
||||
*/
|
||||
vrrp?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* VyOS-specific install parameters. Rendered into the config.boot that the
|
||||
* installer adopts, so the router comes up already configured.
|
||||
*
|
||||
* NOTE: bondMembers must NOT include the interface PXE booted from. Firmware
|
||||
* PXE cannot run over LACP, so the install-time NIC has to stay unbonded.
|
||||
*/
|
||||
export interface VyosInstallSpec {
|
||||
/** Interfaces aggregated into bond0 with LACP (802.3ad). Omit for no bond. */
|
||||
bondMembers?: string[];
|
||||
/** CIDR address on bond0 itself — the switch trunk's native/untagged VLAN. */
|
||||
bondAddress?: string;
|
||||
/** VRRP virtual address (CIDR) floated on the untagged bond (vrid 1). */
|
||||
bondVrrp?: string;
|
||||
/**
|
||||
* VRRP priority for every group on this box. Higher wins mastership.
|
||||
* The HA pair differs ONLY here (e.g. 200 on the primary, 100 on the
|
||||
* standby) — addresses differ per box, VIPs and vrids match.
|
||||
*/
|
||||
vrrpPriority?: number;
|
||||
/** Tagged VLAN sub-interfaces, created on bond0 when bonded, else on mgmtInterface. */
|
||||
vlans?: VyosVlanSpec[];
|
||||
/** Untagged interface the machine PXE booted from. Defaults to "eth0". */
|
||||
mgmtInterface?: string;
|
||||
/** CIDR address for mgmtInterface, or "dhcp". Defaults to "dhcp". */
|
||||
mgmtAddress?: string;
|
||||
/**
|
||||
* Tagged management VLAN on mgmtInterface, separate from the routed VLANs
|
||||
* carried by the bond.
|
||||
*
|
||||
* Needed when the PXE port is a trunk: it boots untagged on the VLAN the
|
||||
* bastion's proxy DHCP serves, and carries the management VLAN tagged so the
|
||||
* router stays reachable there without giving up reinstallability.
|
||||
*/
|
||||
mgmtVlan?: VyosVlanSpec;
|
||||
/** Password for the "vyos" user. Falls back to the bastion default. */
|
||||
password?: string;
|
||||
/**
|
||||
* On reinstall the VyOS installer carries the previous on-disk config (and
|
||||
* SSH host keys) forward -- the "reinstall without losing data" default.
|
||||
* Set true to make the bastion-generated config win instead: after install
|
||||
* the driver overwrites the installed image's config.boot.
|
||||
*/
|
||||
freshConfig?: boolean;
|
||||
/**
|
||||
* VyOS interface name -> MAC, emitted as `hw-id` so names bind deterministically.
|
||||
*
|
||||
* Discovery runs under Fedora and reports predictable names (enp2s0,
|
||||
* enp1s0f0np0), but VyOS enumerates its own eth<N> names, so a name observed
|
||||
* during discovery cannot be used directly. Pinning by MAC removes the guess
|
||||
* about which physical port a given eth<N> is.
|
||||
*/
|
||||
hwIds?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface InstallConfig {
|
||||
hostname: string;
|
||||
disk: string;
|
||||
role: Role;
|
||||
os?: OsId; // defaults to "fedora-43" for backward compat
|
||||
vyos?: VyosInstallSpec; // only consulted when os is "vyos-rolling"
|
||||
arch?: Arch; // detected from HardwareInfo or overridden
|
||||
queued_at: string;
|
||||
/**
|
||||
* When dispatch last served this machine an install boot script. Progress
|
||||
* callbacks only start once the installer environment is up, so a machine
|
||||
* dispatched long ago with no progress is wedged before that point (bad
|
||||
* kernel/initrd, no network in the initramfs, wrong NIC picked...).
|
||||
*/
|
||||
dispatched_at?: string;
|
||||
progress?: string;
|
||||
progress_at?: string;
|
||||
progress_detail?: string;
|
||||
|
||||
@@ -29,6 +29,17 @@ export interface PxeVmConfig {
|
||||
diskSize: number; // GB
|
||||
network: string; // libvirt network name
|
||||
arch?: "x86_64" | "aarch64";
|
||||
/**
|
||||
* Extra NICs enumerated BEFORE the PXE NIC, on a network with no route to
|
||||
* the bastion (defaults to libvirt's "default").
|
||||
*
|
||||
* Real multi-NIC boxes expose a class of bug a single-NIC VM cannot: an
|
||||
* initramfs that picks "the first connected interface" grabs one of these
|
||||
* instead of the NIC that PXE booted, and then cannot reach the bastion.
|
||||
* Defaults to 0 (single NIC).
|
||||
*/
|
||||
decoyNics?: number;
|
||||
decoyNetwork?: string;
|
||||
}
|
||||
|
||||
/** Create a blank UEFI VM that PXE boots from the network. */
|
||||
@@ -61,6 +72,10 @@ export function createPxeVm(config: PxeVmConfig): void {
|
||||
`--memory=${config.memory}`,
|
||||
`--vcpus=${config.vcpus}`,
|
||||
`--disk=path=${diskPath},format=qcow2,bus=virtio`,
|
||||
// Decoys first so they enumerate ahead of the PXE NIC. They are up and
|
||||
// carry a lease, but have no route to the bastion.
|
||||
...Array.from({ length: config.decoyNics ?? 0 }, () =>
|
||||
`--network=network=${config.decoyNetwork ?? "default"},model=virtio`),
|
||||
`--network=network=${config.network},model=virtio`,
|
||||
// UEFI firmware — required for PXE boot in modern mode
|
||||
`--boot=uefi,network,hd`,
|
||||
@@ -95,12 +110,21 @@ export function destroyPxeVm(name: string): void {
|
||||
}
|
||||
|
||||
/** Get the MAC address of a VM's first NIC. */
|
||||
export function getVmMac(name: string): string | null {
|
||||
export function getVmMac(name: string, network?: string): string | null {
|
||||
const result = virsh("domiflist", name);
|
||||
if (result.status !== 0) return null;
|
||||
// Output format: Interface Type Source Model MAC
|
||||
const match = result.stdout.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i);
|
||||
return match ? match[1].toLowerCase() : null;
|
||||
// With decoy NICs present, match the line for the PXE network so we return
|
||||
// the NIC that actually boots rather than whichever is listed first.
|
||||
const lines = result.stdout.split("\n");
|
||||
const candidates = network === undefined
|
||||
? lines
|
||||
: lines.filter((l) => l.split(/\s+/).includes(network));
|
||||
for (const line of candidates.length > 0 ? candidates : lines) {
|
||||
const m = line.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i);
|
||||
if (m) return m[1].toLowerCase();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Reboot a VM (force off + start). */
|
||||
|
||||
387
bastion/tests/integration/vyos-provision.test.ts
Normal file
387
bastion/tests/integration/vyos-provision.test.ts
Normal file
@@ -0,0 +1,387 @@
|
||||
// Integration test: full VyOS unattended provisioning flow.
|
||||
//
|
||||
// Validates the VyOS install path end-to-end, at the same depth as the Fedora
|
||||
// pxe-provision test:
|
||||
// 1. Bastion (HTTP + dnsmasq) on the isolated libvirt PXE network
|
||||
// 2. Blank UEFI VM PXE boots -> Fedora-based discovery (OS-neutral)
|
||||
// 3. Queue os=vyos-rolling -> live boot + live-config hook + pty driver
|
||||
// 4. Fresh-install asserts: installed.ip, streamed logs, applied config,
|
||||
// /config/lab-provisioned, boot-order handling
|
||||
// 5. REINSTALL round: previous config + /config data carried forward
|
||||
// ("reinstall without losing data", VyOS-flavored)
|
||||
// 6. freshConfig round: bastion-generated config wins, /config data kept
|
||||
//
|
||||
// Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo, internet
|
||||
// (first run downloads the ~600MB VyOS nightly ISO; artifacts are cached).
|
||||
// Run: sudo pnpm run test:integration:vyos
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll } from "vitest";
|
||||
import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { execSync } from "node:child_process";
|
||||
import { join } from "node:path";
|
||||
import { homedir, tmpdir } from "node:os";
|
||||
import { log, waitForSsh } from "./helpers/libvirt.js";
|
||||
import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js";
|
||||
import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm } from "./helpers/pxe-vm.js";
|
||||
import { sshExec } from "./helpers/ssh.js";
|
||||
|
||||
const VM_NAME = "lab-vyos-test";
|
||||
const VM_MEMORY = 4096;
|
||||
const VM_VCPUS = 4;
|
||||
const VM_DISK_GB = 10; // VyOS image install needs ~2GB minimum
|
||||
const HTTP_PORT = 8099;
|
||||
const SSH_USER = "vyos"; // the only VyOS login user
|
||||
const BASTION_IP = PXE_GATEWAY;
|
||||
const DHCP_RANGE_START = `${PXE_SUBNET}.100`;
|
||||
const DHCP_RANGE_END = `${PXE_SUBNET}.200`;
|
||||
|
||||
const DISCOVERY_TIMEOUT_MS = 5 * 60_000;
|
||||
const INSTALL_TIMEOUT_MS = 15 * 60_000; // squashfs fetch + copy; much faster than Anaconda
|
||||
const SSH_TIMEOUT_MS = 8 * 60_000;
|
||||
|
||||
const HOSTNAME_R1 = "vyos-r1";
|
||||
const HOSTNAME_R2 = "vyos-r2";
|
||||
const HOSTNAME_R3 = "vyos-r3";
|
||||
|
||||
function findSshKey(): { pubKey: string; keyPath: string } {
|
||||
const homes = [homedir()];
|
||||
const sudoUser = process.env["SUDO_USER"];
|
||||
if (sudoUser) homes.push(join("/home", sudoUser));
|
||||
if (process.env["SSH_KEY_PATH"]) {
|
||||
const keyPath = process.env["SSH_KEY_PATH"];
|
||||
const pubPath = `${keyPath}.pub`;
|
||||
if (existsSync(keyPath) && existsSync(pubPath)) {
|
||||
return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath };
|
||||
}
|
||||
}
|
||||
for (const home of homes) {
|
||||
for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
|
||||
const keyPath = join(home, ".ssh", name);
|
||||
const pubPath = `${keyPath}.pub`;
|
||||
if (existsSync(keyPath) && existsSync(pubPath)) {
|
||||
return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath };
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/");
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((r) => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
async function pollApi<T>(
|
||||
url: string,
|
||||
check: (data: T) => boolean,
|
||||
timeoutMs: number,
|
||||
intervalMs = 5000,
|
||||
): Promise<T> {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
if (res.ok) {
|
||||
const data = (await res.json()) as T;
|
||||
if (check(data)) return data;
|
||||
}
|
||||
} catch { /* not ready yet */ }
|
||||
await sleep(intervalMs);
|
||||
}
|
||||
throw new Error(`Timeout after ${timeoutMs}ms polling ${url}`);
|
||||
}
|
||||
|
||||
type LogsResponse = {
|
||||
status: string;
|
||||
progress: string;
|
||||
progress_detail?: string;
|
||||
ip?: string;
|
||||
log_total?: number;
|
||||
log_lines?: Array<{ line: string }>;
|
||||
};
|
||||
|
||||
/** Queue a VyOS install, reboot the VM into PXE, wait for completion + SSH. */
|
||||
async function installRound(opts: {
|
||||
mac: string;
|
||||
hostname: string;
|
||||
freshConfig?: boolean;
|
||||
}): Promise<string> {
|
||||
const body = {
|
||||
mac: opts.mac,
|
||||
hostname: opts.hostname,
|
||||
disk: "/dev/vda",
|
||||
role: "vanilla",
|
||||
os: "vyos-rolling",
|
||||
vyos: {
|
||||
mgmtInterface: "eth0",
|
||||
mgmtAddress: "dhcp",
|
||||
hwIds: { eth0: opts.mac },
|
||||
...(opts.freshConfig ? { freshConfig: true } : {}),
|
||||
},
|
||||
};
|
||||
const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/install`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
log(`Install queued (${opts.hostname}): ${JSON.stringify(await res.json())}`);
|
||||
|
||||
await sleep(5_000);
|
||||
rebootPxeVm(VM_NAME);
|
||||
await sleep(3_000);
|
||||
deleteNftablesRejectRules();
|
||||
|
||||
const finalState = await pollApi<LogsResponse>(
|
||||
`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(opts.mac)}`,
|
||||
(data) => data.status === "installed" || data.progress === "error",
|
||||
INSTALL_TIMEOUT_MS,
|
||||
10_000,
|
||||
);
|
||||
if (finalState.progress === "error") {
|
||||
log(`INSTALL FAILED: ${JSON.stringify(finalState.progress_detail ?? finalState, null, 2)}`);
|
||||
throw new Error(`VyOS install failed for ${opts.hostname}`);
|
||||
}
|
||||
const ip = finalState.ip ?? "";
|
||||
log(`Install complete (${opts.hostname}). IP: ${ip}`);
|
||||
|
||||
// The driver force-reboots; the VM PXE boots, dispatch says installed ->
|
||||
// localboot exit -> GRUB -> VyOS. nftables reject rules do not reappear
|
||||
// (guest reboot, not a libvirt restart), but clearing is harmless.
|
||||
deleteNftablesRejectRules();
|
||||
await waitForSsh(ip, SSH_USER, SSH_TIMEOUT_MS, sshKeyPathGlobal);
|
||||
return ip;
|
||||
}
|
||||
|
||||
let sshKeyPathGlobal = "";
|
||||
|
||||
describe("VyOS provisioning", () => {
|
||||
let bastionApp: { close: () => Promise<void> };
|
||||
let testDir: string;
|
||||
let vmMac: string;
|
||||
let vmIp: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { pubKey, keyPath } = findSshKey();
|
||||
sshKeyPathGlobal = keyPath;
|
||||
|
||||
log("Setting up PXE test network...");
|
||||
ensurePxeNetwork();
|
||||
|
||||
testDir = join(tmpdir(), `lab-vyos-test-${Date.now()}`);
|
||||
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||
mkdirSync(join(testDir, "logs"), { recursive: true });
|
||||
|
||||
log("Starting bastion...");
|
||||
const { createApp } = await import("../../src/bastion/src/server.js");
|
||||
const { loadConfig } = await import("../../src/bastion/src/config.js");
|
||||
const { generateDnsmasqConf, startDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
|
||||
const { generateDiscoverKickstart } = await import("../../src/bastion/src/services/kickstart-generator.js");
|
||||
const { renderBootIpxe } = await import("../../src/bastion/src/templates/boot.ipxe.js");
|
||||
const { prepareVyosArtifacts } = await import("../../src/bastion/src/main.js");
|
||||
|
||||
const config = loadConfig({
|
||||
bastionDir: testDir,
|
||||
httpPort: HTTP_PORT,
|
||||
iface: "virbr-pxe",
|
||||
serverIp: BASTION_IP,
|
||||
network: `${PXE_SUBNET}.0`,
|
||||
gateway: BASTION_IP,
|
||||
dhcpMode: "full",
|
||||
dhcpRangeStart: DHCP_RANGE_START,
|
||||
dhcpRangeEnd: DHCP_RANGE_END,
|
||||
domain: "pxe-test.local",
|
||||
sshKeys: [pubKey],
|
||||
adminUser: "lab",
|
||||
});
|
||||
|
||||
// iPXE binary
|
||||
const ipxeSrc = "/usr/share/ipxe/ipxe-snponly-x86_64.efi";
|
||||
if (!existsSync(ipxeSrc)) {
|
||||
throw new Error(`iPXE not found: ${ipxeSrc}. Install: sudo dnf install ipxe-bootimgs-x86`);
|
||||
}
|
||||
copyFileSync(ipxeSrc, join(config.tftpDir, "ipxe.efi"));
|
||||
try { symlinkSync(join(config.tftpDir, "ipxe.efi"), join(config.httpDir, "ipxe.efi")); } catch { /* exists */ }
|
||||
|
||||
const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache";
|
||||
execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" });
|
||||
|
||||
// Fedora kernel+initrd for DISCOVERY (OS-neutral, same as pxe test)
|
||||
const kernel = join(cacheDir, `vmlinuz-${config.fedoraVersion}`);
|
||||
const initrd = join(cacheDir, `initrd-${config.fedoraVersion}.img`);
|
||||
if (!existsSync(kernel)) {
|
||||
log(`Downloading Fedora ${config.fedoraVersion} kernel (discovery)...`);
|
||||
execSync(`curl -# -L -f -o "${kernel}" "${config.fedoraMirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 300_000 });
|
||||
}
|
||||
if (!existsSync(initrd)) {
|
||||
log(`Downloading Fedora ${config.fedoraVersion} initrd (discovery)...`);
|
||||
execSync(`curl -# -L -f -o "${initrd}" "${config.fedoraMirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 300_000 });
|
||||
}
|
||||
copyFileSync(kernel, join(config.httpDir, "vmlinuz"));
|
||||
copyFileSync(initrd, join(config.httpDir, "initrd.img"));
|
||||
|
||||
// VyOS netboot artifacts — cache the three extracted files across runs
|
||||
const vyosCache = {
|
||||
kernel: join(cacheDir, "vyos-vmlinuz"),
|
||||
initrd: join(cacheDir, "vyos-initrd"),
|
||||
squashfs: join(cacheDir, "vyos-filesystem.squashfs"),
|
||||
};
|
||||
if (Object.values(vyosCache).every((p) => existsSync(p))) {
|
||||
log("VyOS netboot artifacts cached");
|
||||
copyFileSync(vyosCache.kernel, join(config.httpDir, "vyos-vmlinuz"));
|
||||
copyFileSync(vyosCache.initrd, join(config.httpDir, "vyos-initrd"));
|
||||
copyFileSync(vyosCache.squashfs, join(config.httpDir, "vyos-filesystem.squashfs"));
|
||||
} else {
|
||||
log("Extracting VyOS artifacts from ISO (downloads ~600MB on first run)...");
|
||||
prepareVyosArtifacts(config);
|
||||
copyFileSync(join(config.httpDir, "vyos-vmlinuz"), vyosCache.kernel);
|
||||
copyFileSync(join(config.httpDir, "vyos-initrd"), vyosCache.initrd);
|
||||
copyFileSync(join(config.httpDir, "vyos-filesystem.squashfs"), vyosCache.squashfs);
|
||||
}
|
||||
|
||||
writeFileSync(join(config.httpDir, "discover.ks"), generateDiscoverKickstart(config));
|
||||
writeFileSync(join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }));
|
||||
generateDnsmasqConf(config);
|
||||
|
||||
const { app, syslog } = createApp(config);
|
||||
bastionApp = app;
|
||||
await app.listen({ port: config.httpPort, host: "0.0.0.0" });
|
||||
syslog.start();
|
||||
log(`Bastion listening on :${HTTP_PORT}`);
|
||||
|
||||
log("Starting dnsmasq...");
|
||||
startDnsmasq(config).catch((err) => {
|
||||
log(`dnsmasq failed (expected without root): ${err instanceof Error ? err.message : String(err)}`);
|
||||
});
|
||||
await sleep(1000);
|
||||
|
||||
log("Creating PXE VM...");
|
||||
// Two decoy NICs ahead of the PXE NIC, on a network with no route to the
|
||||
// bastion. This reproduces the real VP2440 topology: live-boot scans for
|
||||
// "the first connected interface", and without BOOTIF it picks a decoy,
|
||||
// times out on DHCP/fetch, and dies with "Unable to find a live file
|
||||
// system on the network". A single-NIC VM cannot catch that.
|
||||
createPxeVm({
|
||||
name: VM_NAME,
|
||||
memory: VM_MEMORY,
|
||||
vcpus: VM_VCPUS,
|
||||
diskSize: VM_DISK_GB,
|
||||
network: PXE_NETWORK_NAME,
|
||||
decoyNics: 2,
|
||||
});
|
||||
const mac = getVmMac(VM_NAME, PXE_NETWORK_NAME);
|
||||
if (!mac) throw new Error("Could not determine VM MAC address");
|
||||
vmMac = mac;
|
||||
log(`VM MAC: ${vmMac}`);
|
||||
|
||||
log("Waiting for discovery...");
|
||||
type MachinesResponse = { discovered: Record<string, unknown> };
|
||||
await pollApi<MachinesResponse>(
|
||||
`http://${BASTION_IP}:${HTTP_PORT}/api/machines`,
|
||||
(data) => vmMac in data.discovered,
|
||||
DISCOVERY_TIMEOUT_MS,
|
||||
);
|
||||
log("VM discovered. Running fresh VyOS install (round 1)...");
|
||||
|
||||
await sleep(15_000); // discovery reboot cycle
|
||||
vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R1 });
|
||||
log("Round 1 (fresh install) complete.");
|
||||
}, DISCOVERY_TIMEOUT_MS + INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000);
|
||||
|
||||
afterAll(async () => {
|
||||
log("Cleaning up...");
|
||||
if (bastionApp) await bastionApp.close().catch(() => {});
|
||||
const { stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
|
||||
stopDnsmasq();
|
||||
destroyPxeVm(VM_NAME);
|
||||
destroyPxeNetwork();
|
||||
if (testDir) rmSync(testDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("machine is installed with a real IP (WI-1: ready-at parsing)", async () => {
|
||||
const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`);
|
||||
const data = (await res.json()) as { installed: Record<string, { ip: string; os?: string }> };
|
||||
const machine = data.installed[vmMac];
|
||||
expect(machine).toBeDefined();
|
||||
expect(machine.ip).toMatch(/^\d+\.\d+\.\d+\.\d+$/);
|
||||
expect(machine.os).toBe("vyos-rolling");
|
||||
});
|
||||
|
||||
it("install logs were streamed live (WI-2)", async () => {
|
||||
const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`);
|
||||
const data = (await res.json()) as LogsResponse;
|
||||
expect(data.log_total).toBeGreaterThan(0);
|
||||
const lines = (data.log_lines ?? []).map((l) => l.line).join("\n");
|
||||
// Installer transcript lines and driver messages both flow through /api/log
|
||||
expect(lines).toMatch(/Welcome to VyOS installation|>>> answered|base config:/);
|
||||
});
|
||||
|
||||
it("SSH works as the vyos user with the injected key", () => {
|
||||
const result = sshExec(vmIp, SSH_USER, "whoami", { keyPath: sshKeyPathGlobal });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.stdout.trim()).toBe("vyos");
|
||||
});
|
||||
|
||||
it("generated config was adopted (hostname + ssh key)", () => {
|
||||
const result = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.stdout).toContain(`host-name "${HOSTNAME_R1}"`);
|
||||
expect(result.stdout).toContain("public-keys");
|
||||
});
|
||||
|
||||
it("boot-order step ran and reported (WI-3)", async () => {
|
||||
const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`);
|
||||
const data = (await res.json()) as LogsResponse;
|
||||
const lines = (data.log_lines ?? []).map((l) => l.line).join("\n");
|
||||
expect(lines).toContain("boot order:");
|
||||
});
|
||||
|
||||
it("provisioning metadata persisted to /config (WI-4)", () => {
|
||||
const result = sshExec(vmIp, SSH_USER, "cat /config/lab-provisioned 2>/dev/null || cat /opt/vyatta/etc/config/lab-provisioned", { keyPath: sshKeyPathGlobal });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.stdout).toContain(`hostname=${HOSTNAME_R1}`);
|
||||
expect(result.stdout).toContain("role=vanilla");
|
||||
expect(result.stdout).toContain(`bastion=http://${BASTION_IP}:${HTTP_PORT}`);
|
||||
});
|
||||
|
||||
it("reinstall preserves config and /config data (round 2)", async () => {
|
||||
// Drop a marker in /config — the installer's previous-installation copy
|
||||
// must carry it (and the whole old config) into the new image.
|
||||
// `sync` is REQUIRED: rebootPxeVm uses `virsh destroy` (a hard power-cut),
|
||||
// so an unsynced write never reaches the disk and the marker vanishes for
|
||||
// reasons that have nothing to do with the installer.
|
||||
const marker = sshExec(vmIp, SSH_USER, "echo LAB-MARKER-R2 > /config/lab-marker && sync && cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
|
||||
expect(marker.exitCode).toBe(0);
|
||||
expect(marker.stdout).toContain("LAB-MARKER-R2");
|
||||
|
||||
// Queue with a DIFFERENT hostname: with preserve semantics the previous
|
||||
// config must win, so the hostname must NOT change.
|
||||
vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R2 });
|
||||
|
||||
// Assert the config carry-forward first — it is the primary preservation
|
||||
// signal and does not depend on the marker mechanism above.
|
||||
const cfg = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal });
|
||||
expect(cfg.stdout).toContain(`host-name "${HOSTNAME_R1}"`); // old config carried
|
||||
expect(cfg.stdout).not.toContain(`host-name "${HOSTNAME_R2}"`);
|
||||
|
||||
const markerAfter = sshExec(vmIp, SSH_USER, "cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
|
||||
expect(markerAfter.exitCode).toBe(0);
|
||||
expect(markerAfter.stdout).toContain("LAB-MARKER-R2");
|
||||
}, INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 60_000);
|
||||
|
||||
it("freshConfig makes the generated config win, data still kept (round 3)", async () => {
|
||||
// Re-assert the marker is on disk and synced before the next power-cut.
|
||||
const pre = sshExec(vmIp, SSH_USER, "sync && cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
|
||||
expect(pre.stdout).toContain("LAB-MARKER-R2");
|
||||
|
||||
vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R3, freshConfig: true });
|
||||
|
||||
const cfg = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal });
|
||||
expect(cfg.stdout).toContain(`host-name "${HOSTNAME_R3}"`); // generated config won
|
||||
|
||||
// The marker file (non-config data under /config) still survives —
|
||||
// freshConfig replaces only config.boot, not the carried data.
|
||||
const markerAfter = sshExec(vmIp, SSH_USER, "cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
|
||||
expect(markerAfter.exitCode).toBe(0);
|
||||
expect(markerAfter.stdout).toContain("LAB-MARKER-R2");
|
||||
}, INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 60_000);
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
{
|
||||
"files": [],
|
||||
"references": [
|
||||
{ "path": "src/core" },
|
||||
{ "path": "src/shared" },
|
||||
{ "path": "src/bastion" },
|
||||
{ "path": "src/cli" },
|
||||
|
||||
4
labsim/.gitignore
vendored
Normal file
4
labsim/.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
# runtime artifacts, not source
|
||||
*.log
|
||||
labsim_matrix_lib.py
|
||||
__pycache__/
|
||||
104
labsim/README.md
Normal file
104
labsim/README.md
Normal file
@@ -0,0 +1,104 @@
|
||||
# labsim — libvirt replica of the lab network
|
||||
|
||||
A throwaway copy of the production VLAN topology for testing routing, firewall
|
||||
rules and failover **without touching the real network**. Same VLAN IDs and
|
||||
roles as UniFi, deliberately different IP ranges so nothing can be confused for
|
||||
production.
|
||||
|
||||
## Topology
|
||||
|
||||
Each VLAN is its own isolated libvirt network with one tiny Alpine VM on it.
|
||||
|
||||
| VLAN | Name | Sim subnet | VM address | Mirrors production |
|
||||
|-----:|------|------------|-----------|--------------------|
|
||||
| 1 | management | 172.31.1.0/24 | 172.31.1.10 | 192.168.1.0/24 |
|
||||
| 2 | k8s | 172.31.2.0/24 | 172.31.2.10 | 192.168.8.0/23 |
|
||||
| 3 | kvm | 172.31.3.0/24 | 172.31.3.10 | 192.168.3.0/24 |
|
||||
| 9 | private | 172.31.9.0/24 | 172.31.9.10 | 10.0.9.0/23 |
|
||||
| 10 | lot | 172.31.10.0/24 | 172.31.10.10 | 10.0.0.0/23 |
|
||||
| 200 | roomates | 172.31.200.0/24 | 172.31.200.10 | 192.168.2.0/24 |
|
||||
|
||||
The sim subnet always encodes the VLAN id: `172.31.<vlan>.0/24`.
|
||||
|
||||
Address plan, identical on every VLAN:
|
||||
|
||||
| Address | Role |
|
||||
|---------|------|
|
||||
| `.1` | gateway under test — a router VM you add (not created by default) |
|
||||
| `.2` | host bridge — how you reach the VMs from this workstation |
|
||||
| `.10` | the VLAN's micro VM |
|
||||
| `.254` | reserved for a VRRP VIP, mirroring production |
|
||||
|
||||
The host sits at `.2` purely so you can SSH in. It is deliberately **not** the
|
||||
VMs' default route — that is `.1` — so inter-VLAN tests fail loudly when no
|
||||
router is present instead of being silently served by the host's own routing
|
||||
table. libvirt also installs reject rules that stop these networks forwarding
|
||||
to each other, so traffic between VLANs only works once a router VM bridges
|
||||
them.
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
./labsim-up.sh # bring up every VLAN (idempotent)
|
||||
./labsim-up.sh 2 3 # only VLANs 2 and 3
|
||||
./labsim-down.sh # destroy VMs + networks, keep the base image
|
||||
./labsim-down.sh --purge # also delete the downloaded Alpine image
|
||||
```
|
||||
|
||||
Each VM: 256 MB, 1 vCPU, a copy-on-write overlay on one shared 176 MB Alpine
|
||||
image (so six VMs cost a few MB of disk, not 1 GB).
|
||||
|
||||
## Access
|
||||
|
||||
```bash
|
||||
ssh alpine@172.31.2.10 # normal user (password: labsim)
|
||||
ssh root@172.31.2.10 # privileged — this image has no sudo
|
||||
curl http://172.31.2.10/ # hello-world page naming the VLAN
|
||||
```
|
||||
|
||||
Console, when the network is the thing that is broken:
|
||||
|
||||
```bash
|
||||
sudo virsh console labsim-2-k8s # root / labsim
|
||||
```
|
||||
|
||||
## Watching it
|
||||
|
||||
```bash
|
||||
./labsim-matrix.py --watch 2 # terminal grid, changed cells highlighted
|
||||
./monitoring-up.sh # topology page + Prometheus + Grafana
|
||||
```
|
||||
|
||||
- **http://localhost:9101/** — live mesh: a node per VLAN, the router in the
|
||||
middle, one line per pair coloured green/red with the ICMP RTT on it. Hover a
|
||||
line for per-direction detail. Refreshes every 5s. This is the one to watch
|
||||
while changing firewall rules.
|
||||
- **http://localhost:3000/d/labsim-matrix** — Grafana (anonymous, no login) for
|
||||
*history*: when did a path flip, and how has latency moved.
|
||||
- **http://localhost:9101/metrics** — `labsim_reachable{src,dst,proto}` and
|
||||
`labsim_rtt_ms{src,dst}`.
|
||||
|
||||
## Notes for whoever extends this
|
||||
|
||||
Things that cost time the first time round, all verified on this image:
|
||||
|
||||
- **No `sudo`.** Alpine ships `doas`; cloud-init's `sudo:` directive is inert
|
||||
here. Use `root@` for privileged work.
|
||||
- **cloud-init leaves users locked** (`!*` in `/etc/shadow`) unless
|
||||
`lock_passwd: false`, and sshd then refuses key auth for that user.
|
||||
- **One failing `runcmd` aborts every command after it.** Each entry is
|
||||
`|| true` for that reason.
|
||||
- **busybox here has no `httpd` applet**, and the VMs have no internet to
|
||||
`apk add` one — so the hello-world server is `python3 -m http.server`
|
||||
(python3 is already present because cloud-init depends on it).
|
||||
- **`start-stop-daemon --exec /usr/bin/python3` matches cloud-init's own
|
||||
python3** at boot and refuses to start anything.
|
||||
- **busybox `pgrep -f PATTERN` matches its own argv**, so a "skip if already
|
||||
running" guard always fires. Verified: `guard_exit=0` with nothing listening.
|
||||
|
||||
## Not modelled (yet)
|
||||
|
||||
VLANs are separate L2 segments rather than one 802.1Q trunk, so this exercises
|
||||
inter-VLAN routing but not a `bond0.<vif>` trunk config specifically. A router
|
||||
VM would attach one NIC per VLAN. Adding a tagged-trunk variant is the obvious
|
||||
next step if the bond/vif config itself needs testing.
|
||||
61
labsim/labsim-down.sh
Executable file
61
labsim/labsim-down.sh
Executable file
@@ -0,0 +1,61 @@
|
||||
#!/bin/bash
|
||||
# Tear down the lab network simulation.
|
||||
#
|
||||
# By default this destroys VMs and networks but KEEPS the downloaded base
|
||||
# image, so the next bring-up is fast. Pass --purge to remove that too.
|
||||
#
|
||||
# Usage: ./labsim-down.sh [--purge] [vlan-id ...]
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "$SCRIPT_DIR/lib.sh"
|
||||
source "$SCRIPT_DIR/ovs.sh"
|
||||
|
||||
PURGE=false
|
||||
ARGS=()
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--purge) PURGE=true ;;
|
||||
*) ARGS+=("$a") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
selected_vlans "${ARGS[@]+"${ARGS[@]}"}"
|
||||
|
||||
for entry in "${SELECTED[@]}"; do
|
||||
IFS=: read -r vid name prefix _r <<<"$entry"
|
||||
vm="$(vm_name "$vid" "$name")"
|
||||
|
||||
if virsh_q dominfo "$vm" >/dev/null 2>&1; then
|
||||
log "destroying VM $vm"
|
||||
virsh_q destroy "$vm" >/dev/null 2>&1 || true
|
||||
virsh_q undefine "$vm" --nvram >/dev/null 2>&1 || virsh_q undefine "$vm" >/dev/null 2>&1 || true
|
||||
fi
|
||||
sudo rm -f "$IMG_DIR/${vm}.qcow2" "$IMG_DIR/${vm}-seed.iso"
|
||||
|
||||
done
|
||||
|
||||
# Legacy per-VLAN Linux-bridge networks from before the OVS migration. If
|
||||
# these survive they keep a duplicate <prefix>.2/24 on a dead bridge, and the
|
||||
# kernel may prefer that route over the OVS host leg — which looks exactly
|
||||
# like "the VM is unreachable" while ping -I hostvN works fine.
|
||||
for entry in "${SELECTED[@]}"; do
|
||||
IFS=: read -r vid _n _p _r <<<"$entry"
|
||||
legacy="labsim-vlan${vid}"
|
||||
if virsh_q net-info "$legacy" >/dev/null 2>&1; then
|
||||
log "removing legacy network $legacy"
|
||||
virsh_q net-destroy "$legacy" >/dev/null 2>&1 || true
|
||||
virsh_q net-undefine "$legacy" >/dev/null 2>&1 || true
|
||||
fi
|
||||
done
|
||||
|
||||
log "removing OVS fabric"
|
||||
ovs_down
|
||||
|
||||
if [ "$PURGE" = true ]; then
|
||||
log "purging base image $BASE_IMAGE"
|
||||
sudo rm -f "$BASE_IMAGE"
|
||||
sudo rmdir "$IMG_DIR" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
log "environment is DOWN"
|
||||
157
labsim/labsim-exporter.py
Executable file
157
labsim/labsim-exporter.py
Executable file
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prometheus exporter for the labsim connectivity matrix.
|
||||
|
||||
Runs the same sweep as labsim-matrix.py on an interval and exposes it as
|
||||
metrics, so Grafana can show the mesh as a heatmap and — more usefully — a
|
||||
history of exactly when a cell flipped after a firewall change.
|
||||
|
||||
labsim_reachable{src,dst,proto} 1 = reachable, 0 = blocked
|
||||
labsim_sweep_seconds how long the last sweep took
|
||||
labsim_sweep_total sweeps completed since start
|
||||
labsim_up 1 while the exporter is alive
|
||||
|
||||
Deliberately stdlib-only (http.server + threads): this runs on the workstation
|
||||
next to libvirt, and adding a dependency to watch a lab network is silly.
|
||||
|
||||
./labsim-exporter.py --port 9101 --interval 15
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
|
||||
import labsim_matrix_lib as m # thin import shim, see below
|
||||
|
||||
|
||||
class Collector:
|
||||
def __init__(self, interval: int, timeout: int) -> None:
|
||||
self.interval = interval
|
||||
self.timeout = timeout
|
||||
self.vlans = m.load_vlans()
|
||||
self.lock = threading.Lock()
|
||||
self.results: dict = {}
|
||||
self.duration = 0.0
|
||||
self.sweeps = 0
|
||||
|
||||
def loop(self) -> None:
|
||||
while True:
|
||||
started = time.time()
|
||||
try:
|
||||
results = m.sweep(self.vlans, self.timeout)
|
||||
with self.lock:
|
||||
self.results = results
|
||||
self.duration = time.time() - started
|
||||
self.sweeps += 1
|
||||
except Exception: # noqa: BLE001 - never let the loop die
|
||||
pass
|
||||
time.sleep(max(1.0, self.interval - (time.time() - started)))
|
||||
|
||||
def snapshot(self) -> dict:
|
||||
"""Everything the topology page needs, in one JSON payload."""
|
||||
with self.lock:
|
||||
results, duration = dict(self.results), self.duration
|
||||
reach = total = 0
|
||||
for data in results.values():
|
||||
if "__error__" in data:
|
||||
continue
|
||||
for protos in data.values():
|
||||
for proto, ok in protos.items():
|
||||
if proto == "rtt_ms":
|
||||
continue
|
||||
total += 1
|
||||
if ok:
|
||||
reach += 1
|
||||
return {"vlans": self.vlans, "results": results, "reachable": reach,
|
||||
"total": total, "sweep_seconds": duration}
|
||||
|
||||
def render(self) -> str:
|
||||
with self.lock:
|
||||
results, duration, sweeps = dict(self.results), self.duration, self.sweeps
|
||||
|
||||
out = [
|
||||
"# HELP labsim_reachable 1 if dst is reachable from src over proto",
|
||||
"# TYPE labsim_reachable gauge",
|
||||
]
|
||||
rtts = []
|
||||
for src, data in results.items():
|
||||
if "__error__" in data:
|
||||
continue
|
||||
for dst, protos in data.items():
|
||||
for proto, ok in protos.items():
|
||||
if proto == "rtt_ms":
|
||||
if isinstance(ok, (int, float)):
|
||||
rtts.append((src, dst, ok))
|
||||
continue
|
||||
out.append(
|
||||
f'labsim_reachable{{src="{src}",dst="{dst}",proto="{proto}"}} {1 if ok else 0}')
|
||||
out += ["# HELP labsim_rtt_ms ICMP round-trip time",
|
||||
"# TYPE labsim_rtt_ms gauge"]
|
||||
for src, dst, val in rtts:
|
||||
out.append(f'labsim_rtt_ms{{src="{src}",dst="{dst}"}} {val}')
|
||||
out += [
|
||||
"# HELP labsim_sweep_seconds duration of the last sweep",
|
||||
"# TYPE labsim_sweep_seconds gauge",
|
||||
f"labsim_sweep_seconds {duration:.3f}",
|
||||
"# HELP labsim_sweep_total sweeps completed",
|
||||
"# TYPE labsim_sweep_total counter",
|
||||
f"labsim_sweep_total {sweeps}",
|
||||
"# HELP labsim_up exporter liveness",
|
||||
"# TYPE labsim_up gauge",
|
||||
"labsim_up 1",
|
||||
]
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--port", type=int, default=9101)
|
||||
ap.add_argument("--interval", type=int, default=15)
|
||||
ap.add_argument("--timeout", type=int, default=30)
|
||||
args = ap.parse_args()
|
||||
|
||||
collector = Collector(args.interval, args.timeout)
|
||||
threading.Thread(target=collector.loop, daemon=True).start()
|
||||
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def _send(self, body: bytes, ctype: str) -> None:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", ctype)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802 - stdlib API
|
||||
path = self.path.split("?")[0].rstrip("/")
|
||||
if path in ("", "/topology"):
|
||||
# Live topology view — the thing you actually watch.
|
||||
try:
|
||||
with open(os.path.join(here, "topology.html"), "rb") as fh:
|
||||
self._send(fh.read(), "text/html; charset=utf-8")
|
||||
except OSError:
|
||||
self.send_error(500, "topology.html missing")
|
||||
elif path == "/api/matrix":
|
||||
self._send(json.dumps(collector.snapshot()).encode(), "application/json")
|
||||
elif path == "/metrics":
|
||||
self._send(collector.render().encode(), "text/plain; version=0.0.4")
|
||||
else:
|
||||
self.send_error(404)
|
||||
|
||||
def log_message(self, *_args) -> None: # keep the console quiet
|
||||
return
|
||||
|
||||
srv = http.server.ThreadingHTTPServer(("0.0.0.0", args.port), Handler)
|
||||
print(f"labsim topology http://localhost:{args.port}/")
|
||||
print(f"labsim metrics http://localhost:{args.port}/metrics (sweep every {args.interval}s)")
|
||||
srv.serve_forever()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
202
labsim/labsim-matrix.py
Executable file
202
labsim/labsim-matrix.py
Executable file
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Full-mesh connectivity matrix for the labsim VLANs.
|
||||
|
||||
Probes every VLAN VM from every other VLAN VM (ICMP + TCP/22 + TCP/80) and
|
||||
prints a grid. Use --watch to keep it live: cells that changed since the last
|
||||
sweep are highlighted, so adding or removing a VyOS firewall rule shows up
|
||||
within one refresh.
|
||||
|
||||
Deliberately dependency-free on the guests: the probe runs with python3, which
|
||||
is already installed there (cloud-init needs it), so nothing has to be
|
||||
installed on VMs that have no internet.
|
||||
|
||||
./labsim-matrix.py # one sweep
|
||||
./labsim-matrix.py --watch # live, refresh every 5s
|
||||
./labsim-matrix.py --watch 2 # live, every 2s
|
||||
./labsim-matrix.py --proto icmp # single protocol
|
||||
./labsim-matrix.py --json # machine-readable
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
CONF = os.path.join(HERE, "vlans.conf")
|
||||
|
||||
GREEN, RED, GREY, YELLOW, BOLD, RESET = (
|
||||
"\033[0;32m", "\033[0;31m", "\033[0;90m", "\033[1;33m", "\033[1m", "\033[0m")
|
||||
|
||||
PROTOS = ("icmp", "tcp22", "tcp80")
|
||||
|
||||
# Runs ON the guest. Keep it stdlib-only and quick — a hung probe delays the
|
||||
# whole sweep, so every check is hard-bounded by a timeout.
|
||||
PROBE = r'''
|
||||
import json, re, socket, subprocess, sys
|
||||
targets = json.load(sys.stdin)
|
||||
out = {}
|
||||
for name, ip in targets.items():
|
||||
res = {}
|
||||
try:
|
||||
p = subprocess.run(["ping", "-c", "1", "-W", "1", ip],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=4)
|
||||
res["icmp"] = p.returncode == 0
|
||||
# RTT as well as pass/fail: a path that is up but slow is a different
|
||||
# problem from one that is down, and the grid alone cannot show it.
|
||||
res["rtt_ms"] = None
|
||||
if res["icmp"]:
|
||||
m = re.search(r"time[=<]\s*([0-9.]+)\s*ms", p.stdout.decode("utf-8", "replace"))
|
||||
if m:
|
||||
res["rtt_ms"] = float(m.group(1))
|
||||
except Exception:
|
||||
res["icmp"] = False
|
||||
res["rtt_ms"] = None
|
||||
for port in (22, 80):
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
s.settimeout(1.5)
|
||||
try:
|
||||
s.connect((ip, port)); res["tcp%d" % port] = True
|
||||
except Exception:
|
||||
res["tcp%d" % port] = False
|
||||
finally:
|
||||
try: s.close()
|
||||
except Exception: pass
|
||||
out[name] = res
|
||||
print(json.dumps(out))
|
||||
'''
|
||||
|
||||
|
||||
def load_vlans() -> list[dict]:
|
||||
vlans = []
|
||||
with open(CONF) as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
vid, name, prefix, real = line.split(":", 3)
|
||||
vlans.append({"vid": vid, "name": name, "ip": f"{prefix}.10",
|
||||
"label": f"{vid}:{name}", "real": real})
|
||||
return vlans
|
||||
|
||||
|
||||
def probe_from(src: dict, targets: list[dict], timeout: int) -> tuple[str, dict]:
|
||||
"""SSH once into src and probe every target from there."""
|
||||
payload = json.dumps({t["label"]: t["ip"] for t in targets if t["label"] != src["label"]})
|
||||
cmd = [
|
||||
"ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "BatchMode=yes", "-o", "ConnectTimeout=5", "-o", "LogLevel=ERROR",
|
||||
f"alpine@{src['ip']}", "python3", "-",
|
||||
]
|
||||
try:
|
||||
# The probe script goes on stdin, the target list follows it — the guest
|
||||
# reads the script from argv-less stdin, so send both in one stream.
|
||||
proc = subprocess.run(
|
||||
cmd, input=PROBE.replace("json.load(sys.stdin)", f"json.loads({payload!r})"),
|
||||
capture_output=True, text=True, timeout=timeout)
|
||||
if proc.returncode != 0:
|
||||
return src["label"], {"__error__": (proc.stderr or "ssh failed").strip()[:60]}
|
||||
return src["label"], json.loads(proc.stdout)
|
||||
except subprocess.TimeoutExpired:
|
||||
return src["label"], {"__error__": "probe timed out"}
|
||||
except Exception as exc: # noqa: BLE001 - report, never crash the sweep
|
||||
return src["label"], {"__error__": f"{type(exc).__name__}: {exc}"[:60]}
|
||||
|
||||
|
||||
def sweep(vlans: list[dict], timeout: int) -> dict:
|
||||
results: dict = {}
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=len(vlans)) as pool:
|
||||
futures = [pool.submit(probe_from, v, vlans, timeout) for v in vlans]
|
||||
for fut in concurrent.futures.as_completed(futures):
|
||||
label, data = fut.result()
|
||||
results[label] = data
|
||||
return results
|
||||
|
||||
|
||||
def cell(ok: bool | None, changed: bool) -> str:
|
||||
if ok is None:
|
||||
return f"{GREY} · {RESET}"
|
||||
mark = "ok " if ok else "-- "
|
||||
colour = GREEN if ok else RED
|
||||
if changed:
|
||||
return f"{YELLOW}{BOLD}{'OK*' if ok else 'XX*':<4}{RESET}"
|
||||
return f"{colour}{mark}{RESET}"
|
||||
|
||||
|
||||
def render(vlans: list[dict], results: dict, prev: dict | None, protos: tuple[str, ...]) -> None:
|
||||
labels = [v["label"] for v in vlans]
|
||||
width = max(len(x) for x in labels) + 2
|
||||
|
||||
for proto in protos:
|
||||
print(f"\n{BOLD}{proto.upper()}{RESET} (rows = source, columns = destination)")
|
||||
header = " " * width + "".join(f"{lbl:<{width}}" for lbl in labels)
|
||||
print(f"{GREY}{header}{RESET}")
|
||||
|
||||
for src in vlans:
|
||||
row = f"{src['label']:<{width}}"
|
||||
data = results.get(src["label"], {})
|
||||
if "__error__" in data:
|
||||
print(row + f"{RED}{data['__error__']}{RESET}")
|
||||
continue
|
||||
for dst in vlans:
|
||||
if dst["label"] == src["label"]:
|
||||
row += f"{GREY}{'·':<{width}}{RESET}"
|
||||
continue
|
||||
ok = data.get(dst["label"], {}).get(proto)
|
||||
was = (prev or {}).get(src["label"], {}).get(dst["label"], {}).get(proto)
|
||||
changed = prev is not None and was is not None and was != ok
|
||||
txt = cell(ok, changed)
|
||||
row += txt + " " * (width - 4)
|
||||
print(row)
|
||||
|
||||
reach = sum(1 for s in results.values() if "__error__" not in s
|
||||
for d in s.values() for p in protos if d.get(p) is True)
|
||||
total = sum(1 for s in results.values() if "__error__" not in s
|
||||
for _d in s.values() for _p in protos)
|
||||
print(f"\n reachable: {reach}/{total} "
|
||||
f"{GREEN}ok{RESET}=allowed {RED}--{RESET}=blocked/no route "
|
||||
f"{YELLOW}*{RESET}=changed since last sweep")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--watch", nargs="?", const=5, type=int, metavar="SECONDS",
|
||||
help="refresh continuously (default every 5s)")
|
||||
ap.add_argument("--proto", choices=PROTOS, help="only this protocol")
|
||||
ap.add_argument("--json", action="store_true", help="emit raw JSON and exit")
|
||||
ap.add_argument("--timeout", type=int, default=30, help="per-host probe timeout")
|
||||
args = ap.parse_args()
|
||||
|
||||
vlans = load_vlans()
|
||||
protos = (args.proto,) if args.proto else PROTOS
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(sweep(vlans, args.timeout), indent=2))
|
||||
return 0
|
||||
|
||||
prev = None
|
||||
while True:
|
||||
started = time.time()
|
||||
results = sweep(vlans, args.timeout)
|
||||
if args.watch:
|
||||
os.system("clear")
|
||||
print(f"{BOLD}labsim connectivity matrix{RESET} "
|
||||
f"{time.strftime('%H:%M:%S')} (refresh {args.watch}s, Ctrl-C to stop)")
|
||||
render(vlans, results, prev, protos)
|
||||
if not args.watch:
|
||||
return 0
|
||||
prev = results
|
||||
time.sleep(max(0.0, args.watch - (time.time() - started)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except KeyboardInterrupt:
|
||||
print()
|
||||
sys.exit(130)
|
||||
70
labsim/labsim-up.sh
Executable file
70
labsim/labsim-up.sh
Executable file
@@ -0,0 +1,70 @@
|
||||
#!/bin/bash
|
||||
# Bring up the lab network simulation: one isolated libvirt network per VLAN,
|
||||
# each with a single tiny Alpine VM offering SSH + a hello-world HTTP page.
|
||||
#
|
||||
# Idempotent: re-running only creates what is missing. Safe to run repeatedly.
|
||||
#
|
||||
# Usage: ./labsim-up.sh [vlan-id ...] (default: every VLAN in vlans.conf)
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "$SCRIPT_DIR/lib.sh"
|
||||
source "$SCRIPT_DIR/ovs.sh"
|
||||
|
||||
require_tools
|
||||
[ -f "$BASE_IMAGE" ] || die "base image missing: $BASE_IMAGE (see README)"
|
||||
|
||||
SSH_PUB="$(find_ssh_pubkey)"
|
||||
log "Using SSH key: ${SSH_PUB%% *} ...${SSH_PUB##* }"
|
||||
|
||||
selected_vlans "$@"
|
||||
|
||||
# --- switch fabric ------------------------------------------------------
|
||||
log "bringing up OVS fabric ($OVS_BR) with host legs per VLAN"
|
||||
ovs_up
|
||||
|
||||
# --- VMs ------------------------------------------------------------------
|
||||
for entry in "${SELECTED[@]}"; do
|
||||
IFS=: read -r vid name prefix real <<<"$entry"
|
||||
vm="$(vm_name "$vid" "$name")"
|
||||
ip="${prefix}.10"
|
||||
|
||||
if virsh_q dominfo "$vm" >/dev/null 2>&1; then
|
||||
state="$(virsh_q domstate "$vm" 2>/dev/null | head -1 | tr -d '\n')"
|
||||
if [ "$state" = "running" ]; then
|
||||
log "VM $vm already running ($ip)"
|
||||
continue
|
||||
fi
|
||||
log "VM $vm exists but is $state — starting"
|
||||
virsh_q start "$vm" >/dev/null
|
||||
continue
|
||||
fi
|
||||
|
||||
log "creating VM $vm ($ip on vlan $vid/$name)"
|
||||
|
||||
disk="$IMG_DIR/${vm}.qcow2"
|
||||
seed="$IMG_DIR/${vm}-seed.iso"
|
||||
|
||||
# Copy-on-write overlay: each VM costs a few MB, not 176.
|
||||
sudo qemu-img create -q -f qcow2 -F qcow2 -b "$BASE_IMAGE" "$disk" "$VM_DISK" >/dev/null
|
||||
|
||||
build_seed "$seed" "$vm" "$vid" "$name" "$prefix" "$ip" "$real" "$SSH_PUB"
|
||||
|
||||
sudo virt-install \
|
||||
--connect "$LIBVIRT_URI" \
|
||||
--name "$vm" \
|
||||
--memory "$VM_MEM" --vcpus "$VM_CPUS" \
|
||||
--disk "path=$disk,format=qcow2,bus=virtio" \
|
||||
--disk "path=$seed,device=cdrom,readonly=on" \
|
||||
--network "network=$OVS_NET,portgroup=vlan${vid},model=virtio" \
|
||||
--os-variant alpinelinux3.18 \
|
||||
--graphics none --noautoconsole --import >/dev/null
|
||||
done
|
||||
|
||||
echo
|
||||
log "waiting for VMs to answer on SSH + HTTP..."
|
||||
wait_ready
|
||||
echo
|
||||
status_table
|
||||
echo
|
||||
log "environment is UP. Tear down with: $SCRIPT_DIR/labsim-down.sh"
|
||||
216
labsim/lib.sh
Normal file
216
labsim/lib.sh
Normal file
@@ -0,0 +1,216 @@
|
||||
#!/bin/bash
|
||||
# Shared helpers for the lab network simulation.
|
||||
# shellcheck disable=SC2034
|
||||
|
||||
LIBVIRT_URI="${LIBVIRT_URI:-qemu:///system}"
|
||||
IMG_DIR="${IMG_DIR:-/var/lib/libvirt/images/labsim}"
|
||||
BASE_IMAGE="${BASE_IMAGE:-$IMG_DIR/alpine-base.qcow2}"
|
||||
ALPINE_URL="${ALPINE_URL:-https://dl-cdn.alpinelinux.org/alpine/latest-stable/releases/cloud/generic_alpine-3.24.1-x86_64-bios-cloudinit-r0.qcow2}"
|
||||
|
||||
VM_MEM="${VM_MEM:-256}" # MB — Alpine is happy here
|
||||
VM_CPUS="${VM_CPUS:-1}"
|
||||
VM_DISK="${VM_DISK:-1G}"
|
||||
PREFIX="labsim"
|
||||
|
||||
CONF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/vlans.conf"
|
||||
|
||||
log() { printf '\033[0;36m[labsim]\033[0m %s\n' "$*"; }
|
||||
warn() { printf '\033[1;33m[labsim]\033[0m %s\n' "$*" >&2; }
|
||||
die() { printf '\033[0;31m[labsim]\033[0m %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
virsh_q() { sudo virsh --connect "$LIBVIRT_URI" "$@"; }
|
||||
|
||||
net_name() { echo "${PREFIX}-vlan$1"; }
|
||||
vm_name() { echo "${PREFIX}-$1-$2"; } # labsim-2-k8s
|
||||
# Linux bridge names are capped at 15 chars — keep it short and unique.
|
||||
br_name() { echo "vbr-ls$1"; }
|
||||
|
||||
require_tools() {
|
||||
for t in virsh virt-install qemu-img genisoimage; do
|
||||
command -v "$t" >/dev/null 2>&1 || die "missing required tool: $t"
|
||||
done
|
||||
sudo -n true 2>/dev/null || warn "sudo may prompt for a password"
|
||||
}
|
||||
|
||||
find_ssh_pubkey() {
|
||||
local home="${SUDO_USER:+/home/$SUDO_USER}"
|
||||
home="${home:-$HOME}"
|
||||
for n in id_ed25519 id_ecdsa id_rsa; do
|
||||
[ -f "$home/.ssh/$n.pub" ] && { cat "$home/.ssh/$n.pub"; return; }
|
||||
done
|
||||
die "no SSH public key found in $home/.ssh"
|
||||
}
|
||||
|
||||
# Populate SELECTED[] from argv (VLAN ids) or the whole config.
|
||||
selected_vlans() {
|
||||
SELECTED=()
|
||||
local want=("$@")
|
||||
while IFS= read -r line; do
|
||||
[[ "$line" =~ ^[[:space:]]*# ]] && continue
|
||||
[[ -z "${line// }" ]] && continue
|
||||
local vid="${line%%:*}"
|
||||
if [ ${#want[@]} -eq 0 ]; then
|
||||
SELECTED+=("$line")
|
||||
else
|
||||
for w in "${want[@]}"; do [ "$w" = "$vid" ] && SELECTED+=("$line"); done
|
||||
fi
|
||||
done < "$CONF"
|
||||
[ ${#SELECTED[@]} -gt 0 ] || die "no VLANs selected (checked $CONF)"
|
||||
}
|
||||
|
||||
# cloud-init NoCloud seed: static addressing + SSH key + hello-world HTTP.
|
||||
build_seed() {
|
||||
local iso="$1" vm="$2" vid="$3" name="$4" prefix="$5" ip="$6" real="$7" pubkey="$8"
|
||||
local tmp; tmp="$(mktemp -d)"
|
||||
|
||||
cat > "$tmp/meta-data" <<EOF
|
||||
instance-id: $vm
|
||||
local-hostname: $vm
|
||||
EOF
|
||||
|
||||
# Alpine's cloud-init does not reliably apply netplan-style network-config,
|
||||
# and these networks have no DHCP server on purpose — so configure the
|
||||
# interface the Alpine-native way instead (verified: hostname applied but no
|
||||
# address, i.e. the seed was read and network-config was ignored).
|
||||
#
|
||||
# The default route deliberately points at the router under test (.1), not
|
||||
# the host (.2), so a broken/absent router shows up as a failed test rather
|
||||
# than being silently papered over by host routing. post-up ... || true keeps
|
||||
# the interface up even while no router exists yet.
|
||||
cat > "$tmp/network-config" <<EOF
|
||||
version: 1
|
||||
config:
|
||||
- type: physical
|
||||
name: eth0
|
||||
subnets:
|
||||
- type: static
|
||||
address: $ip
|
||||
netmask: 255.255.255.0
|
||||
# Default route via the router under test. Without this the VMs can
|
||||
# reach their own /24 and their gateway, but nothing beyond it — which
|
||||
# looks exactly like "the router is broken" in the matrix.
|
||||
gateway: ${prefix}.1
|
||||
EOF
|
||||
|
||||
cat > "$tmp/user-data" <<EOF
|
||||
#cloud-config
|
||||
hostname: $vm
|
||||
users:
|
||||
- name: alpine
|
||||
# NOTE: this Alpine image ships no sudo (and cloud-init's sudo: directive
|
||||
# is therefore inert). For privileged work in these VMs, ssh as root —
|
||||
# the key is installed there too.
|
||||
shell: /bin/ash
|
||||
# Without this cloud-init leaves the account locked ("!*" in /etc/shadow)
|
||||
# and sshd refuses key auth for it — verified on the first build.
|
||||
lock_passwd: false
|
||||
plain_text_passwd: labsim
|
||||
ssh_authorized_keys:
|
||||
- $pubkey
|
||||
ssh_authorized_keys:
|
||||
- $pubkey
|
||||
disable_root: false
|
||||
chpasswd:
|
||||
list: |
|
||||
root:labsim
|
||||
expire: false
|
||||
write_files:
|
||||
- path: /etc/network/interfaces
|
||||
content: |
|
||||
auto lo
|
||||
iface lo inet loopback
|
||||
auto eth0
|
||||
iface eth0 inet static
|
||||
address $ip
|
||||
netmask 255.255.255.0
|
||||
post-up ip route add default via ${prefix}.1 || true
|
||||
- path: /var/www/index.html
|
||||
content: |
|
||||
<html><body>
|
||||
<h1>labsim vlan $vid — $name</h1>
|
||||
<p>host: $vm</p>
|
||||
<p>address: $ip/24</p>
|
||||
<p>gateway under test: ${prefix}.1</p>
|
||||
<p>mirrors production: $real</p>
|
||||
</body></html>
|
||||
- path: /etc/local.d/labsim-http.start
|
||||
permissions: '0755'
|
||||
content: |
|
||||
#!/bin/sh
|
||||
# This image's busybox has no httpd applet ("applet not found"), and the
|
||||
# VMs are isolated so apk cannot fetch one. python3 is already present
|
||||
# (cloud-init depends on it), so serve with http.server — no packages,
|
||||
# no internet.
|
||||
#
|
||||
# Two traps already hit here, both silent:
|
||||
# - start-stop-daemon --exec /usr/bin/python3 matches cloud-init's OWN
|
||||
# python3 at boot, says "already running", starts nothing.
|
||||
# - busybox pgrep -f PATTERN matches its own argv, so a
|
||||
# "skip if running" guard always fires (verified: guard_exit=0 with
|
||||
# nothing listening).
|
||||
# So: no guard, no start-stop-daemon. Binding twice is harmless — the
|
||||
# second just fails to bind.
|
||||
nohup /usr/bin/python3 -m http.server 80 --directory /var/www \\
|
||||
>/var/log/labsim-http.log 2>&1 &
|
||||
runcmd:
|
||||
# cloud-init's network-config (v1, above) already applies the address, so do
|
||||
# NOT restart networking here — it fails, and one failing runcmd aborts every
|
||||
# command after it, which is what silently left httpd unstarted. Each command
|
||||
# is || true for the same reason.
|
||||
- [ sh, -c, "rc-update add sshd default || true" ]
|
||||
- [ sh, -c, "rc-update add local default || true" ]
|
||||
- [ sh, -c, "/etc/local.d/labsim-http.start || true" ]
|
||||
EOF
|
||||
|
||||
# Validate before building the ISO. The heredoc above is intentionally
|
||||
# unquoted (it interpolates $ip/$prefix), which means backticks or $( ) in
|
||||
# ANY line — including comments — get executed by the host shell and their
|
||||
# output silently corrupts the YAML. Cheap check, expensive bug.
|
||||
python3 -c "import yaml,sys; yaml.safe_load(open(sys.argv[1]))" "$tmp/user-data" \
|
||||
|| die "generated user-data is not valid YAML (backticks or \$( ) in build_seed?): $tmp/user-data"
|
||||
|
||||
sudo genisoimage -quiet -output "$iso" -volid cidata -joliet -rock \
|
||||
"$tmp/user-data" "$tmp/meta-data" "$tmp/network-config"
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
ssh_to() {
|
||||
local ip="$1"; shift
|
||||
timeout 12 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
-o ConnectTimeout=5 -o BatchMode=yes -o LogLevel=ERROR \
|
||||
"alpine@$ip" "$@" 2>/dev/null
|
||||
}
|
||||
|
||||
wait_ready() {
|
||||
local deadline=$((SECONDS + 240)) pending=1
|
||||
while [ $SECONDS -lt $deadline ]; do
|
||||
pending=0
|
||||
for entry in "${SELECTED[@]}"; do
|
||||
IFS=: read -r vid _n prefix _r <<<"$entry"
|
||||
# Wait for BOTH: sshd is up well before cloud-init's runcmd starts the
|
||||
# web server, so checking SSH alone reports "ready" then shows HTTP FAIL.
|
||||
ssh_to "${prefix}.10" true >/dev/null 2>&1 \
|
||||
&& curl -sS -o /dev/null --max-time 4 "http://${prefix}.10/" 2>/dev/null \
|
||||
|| pending=$((pending + 1))
|
||||
done
|
||||
[ $pending -eq 0 ] && { log "all ${#SELECTED[@]} VMs reachable"; return 0; }
|
||||
sleep 5
|
||||
done
|
||||
warn "$pending VM(s) still not answering SSH after 240s — see status below"
|
||||
return 0
|
||||
}
|
||||
|
||||
status_table() {
|
||||
printf ' %-18s %-6s %-16s %-9s %-7s %s\n' VM VLAN ADDRESS STATE SSH HTTP
|
||||
printf ' %-18s %-6s %-16s %-9s %-7s %s\n' ------------------ ------ ---------------- --------- ------- ----
|
||||
for entry in "${SELECTED[@]}"; do
|
||||
IFS=: read -r vid name prefix _r <<<"$entry"
|
||||
local vm ip state ssh http
|
||||
vm="$(vm_name "$vid" "$name")"; ip="${prefix}.10"
|
||||
state="$(virsh_q domstate "$vm" 2>/dev/null | head -1 | tr -d '\n')"
|
||||
[ -z "$state" ] && state="absent"
|
||||
ssh_to "$ip" true >/dev/null 2>&1 && ssh=ok || ssh=FAIL
|
||||
if curl -sS -o /dev/null --max-time 5 "http://$ip/" 2>/dev/null; then http=ok; else http=FAIL; fi
|
||||
printf ' %-18s %-6s %-16s %-9s %-7s %s\n' "$vm" "$vid" "$ip" "$state" "$ssh" "$http"
|
||||
done
|
||||
}
|
||||
82
labsim/monitoring-up.sh
Executable file
82
labsim/monitoring-up.sh
Executable file
@@ -0,0 +1,82 @@
|
||||
#!/bin/bash
|
||||
# Prometheus + Grafana for the labsim connectivity matrix.
|
||||
#
|
||||
# Grafana runs with anonymous auth as Admin — NO LOGIN. That is deliberate for
|
||||
# a throwaway lab on localhost; do not copy this into anything reachable.
|
||||
#
|
||||
# ./monitoring-up.sh start exporter + prometheus + grafana
|
||||
# ./monitoring-up.sh --down stop and remove them
|
||||
#
|
||||
# Grafana: http://localhost:3000 (dashboard "labsim — VLAN connectivity matrix")
|
||||
# Prometheus: http://localhost:9090
|
||||
# Exporter: http://localhost:9101/metrics
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "$SCRIPT_DIR/lib.sh"
|
||||
|
||||
GRAFANA_PORT="${GRAFANA_PORT:-3000}"
|
||||
PROM_PORT="${PROM_PORT:-9090}"
|
||||
EXPORTER_PORT="${EXPORTER_PORT:-9101}"
|
||||
NET="labsim-mon"
|
||||
|
||||
if [ "${1:-}" = "--down" ]; then
|
||||
pkill -f "labsim-exporter.py" 2>/dev/null || true
|
||||
podman rm -f labsim-grafana labsim-prometheus >/dev/null 2>&1 || true
|
||||
podman network rm -f "$NET" >/dev/null 2>&1 || true
|
||||
log "monitoring stopped"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
command -v podman >/dev/null 2>&1 || die "podman not installed"
|
||||
|
||||
# --- exporter (on the host: it needs SSH access to the VMs) ----------------
|
||||
if pgrep -f "labsim-exporter.py" >/dev/null 2>&1; then
|
||||
log "exporter already running on :$EXPORTER_PORT"
|
||||
else
|
||||
log "starting exporter on :$EXPORTER_PORT"
|
||||
nohup "$SCRIPT_DIR/labsim-exporter.py" --port "$EXPORTER_PORT" --interval 15 \
|
||||
> /tmp/labsim-exporter.log 2>&1 &
|
||||
sleep 3
|
||||
fi
|
||||
curl -sS --max-time 5 "http://127.0.0.1:${EXPORTER_PORT}/metrics" >/dev/null \
|
||||
|| die "exporter not answering on :$EXPORTER_PORT (see /tmp/labsim-exporter.log)"
|
||||
|
||||
podman network exists "$NET" 2>/dev/null || podman network create "$NET" >/dev/null
|
||||
|
||||
# --- prometheus -----------------------------------------------------------
|
||||
podman rm -f labsim-prometheus >/dev/null 2>&1 || true
|
||||
log "starting prometheus on :$PROM_PORT"
|
||||
podman run -d --name labsim-prometheus --network "$NET" \
|
||||
-p "${PROM_PORT}:9090" \
|
||||
-v "$SCRIPT_DIR/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z" \
|
||||
--add-host "host.containers.internal:host-gateway" \
|
||||
docker.io/prom/prometheus:latest >/dev/null
|
||||
|
||||
# --- grafana (anonymous, no login) ----------------------------------------
|
||||
podman rm -f labsim-grafana >/dev/null 2>&1 || true
|
||||
log "starting grafana on :$GRAFANA_PORT (anonymous auth — no password)"
|
||||
podman run -d --name labsim-grafana --network "$NET" \
|
||||
-p "${GRAFANA_PORT}:3000" \
|
||||
-e GF_AUTH_ANONYMOUS_ENABLED=true \
|
||||
-e GF_AUTH_ANONYMOUS_ORG_ROLE=Admin \
|
||||
-e GF_AUTH_DISABLE_LOGIN_FORM=true \
|
||||
-e GF_AUTH_BASIC_ENABLED=false \
|
||||
-e GF_SECURITY_ALLOW_EMBEDDING=true \
|
||||
-e GF_USERS_DEFAULT_THEME=dark \
|
||||
-v "$SCRIPT_DIR/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro,Z" \
|
||||
docker.io/grafana/grafana:latest >/dev/null
|
||||
|
||||
log "waiting for grafana..."
|
||||
for _ in $(seq 1 40); do
|
||||
if curl -sS --max-time 3 "http://127.0.0.1:${GRAFANA_PORT}/api/health" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
|
||||
echo
|
||||
log "Topology: http://localhost:${EXPORTER_PORT}/ <- live mesh, red/green + RTT"
|
||||
log "Grafana: http://localhost:${GRAFANA_PORT}/d/labsim-matrix (no login, history)"
|
||||
log "Prometheus: http://localhost:${PROM_PORT}"
|
||||
log "Exporter: http://localhost:${EXPORTER_PORT}/metrics"
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: labsim
|
||||
folder: ''
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 10
|
||||
options:
|
||||
path: /etc/grafana/provisioning/dashboards
|
||||
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"uid": "labsim-matrix",
|
||||
"title": "labsim — VLAN connectivity matrix",
|
||||
"tags": ["labsim"],
|
||||
"timezone": "browser",
|
||||
"refresh": "10s",
|
||||
"time": { "from": "now-30m", "to": "now" },
|
||||
"panels": [
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Reachable paths",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
|
||||
"targets": [ { "expr": "sum(labsim_reachable)", "refId": "A" } ],
|
||||
"fieldConfig": { "defaults": { "thresholds": { "mode": "absolute",
|
||||
"steps": [ { "color": "red", "value": null }, { "color": "green", "value": 90 } ] } } }
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Blocked paths",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
|
||||
"targets": [ { "expr": "count(labsim_reachable == 0) or vector(0)", "refId": "A" } ],
|
||||
"fieldConfig": { "defaults": { "thresholds": { "mode": "absolute",
|
||||
"steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 } ] } } }
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Sweep duration (s)",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
|
||||
"targets": [ { "expr": "labsim_sweep_seconds", "refId": "A" } ]
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Sweeps",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
|
||||
"targets": [ { "expr": "labsim_sweep_total", "refId": "A" } ]
|
||||
},
|
||||
{
|
||||
"type": "heatmap",
|
||||
"title": "ICMP matrix (src → dst) — green = reachable",
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
|
||||
"targets": [ { "expr": "labsim_reachable{proto=\"icmp\"}",
|
||||
"legendFormat": "{{src}} → {{dst}}", "refId": "A" } ]
|
||||
},
|
||||
{
|
||||
"type": "state-timeline",
|
||||
"title": "Every path over time — a firewall change shows up here immediately",
|
||||
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 14 },
|
||||
"targets": [ { "expr": "labsim_reachable",
|
||||
"legendFormat": "{{proto}} {{src}} → {{dst}}", "refId": "A" } ],
|
||||
"fieldConfig": { "defaults": {
|
||||
"mappings": [ { "type": "value", "options": {
|
||||
"0": { "text": "blocked", "color": "red", "index": 0 },
|
||||
"1": { "text": "ok", "color": "green", "index": 1 } } } ] } },
|
||||
"options": { "mergeValues": true, "showValue": "never" }
|
||||
}
|
||||
],
|
||||
"schemaVersion": 39,
|
||||
"version": 1
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://labsim-prometheus:9090
|
||||
isDefault: true
|
||||
9
labsim/monitoring/prometheus.yml
Normal file
9
labsim/monitoring/prometheus.yml
Normal file
@@ -0,0 +1,9 @@
|
||||
# Scrapes the labsim connectivity exporter running on the host.
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: labsim
|
||||
static_configs:
|
||||
- targets: ['host.containers.internal:9101']
|
||||
161
labsim/ovs.sh
Normal file
161
labsim/ovs.sh
Normal file
@@ -0,0 +1,161 @@
|
||||
#!/bin/bash
|
||||
# Open vSwitch fabric for labsim — the "switch" the whole sim hangs off.
|
||||
#
|
||||
# Why OVS and not a Linux bridge: a Linux bridge cannot do LACP at all, and its
|
||||
# VLAN support is awkward to drive from libvirt. OVS gives real 802.1Q access
|
||||
# and trunk ports plus real LACP bonds, so a router VM can run the SAME bond0 +
|
||||
# vif config as the production VP2440s instead of an approximation.
|
||||
#
|
||||
# Layout:
|
||||
# ovs-labsim the switch
|
||||
# ├─ vm ports access ports, tag=<vlan> (micro VM per VLAN)
|
||||
# ├─ hostv<vlan> internal ports, tag=<vlan> (host leg, for SSH)
|
||||
# └─ lag-vyos LACP bond, trunk of all VLANs (router under test)
|
||||
# shellcheck disable=SC2034
|
||||
|
||||
OVS_BR="${OVS_BR:-ovs-labsim}"
|
||||
OVS_NET="${OVS_NET:-labsim-ovs}" # libvirt network wrapping the bridge
|
||||
LAG_NAME="${LAG_NAME:-lag-vyos}"
|
||||
|
||||
ovs() { sudo ovs-vsctl "$@"; }
|
||||
|
||||
ovs_require() {
|
||||
command -v ovs-vsctl >/dev/null 2>&1 || die "openvswitch not installed (dnf install openvswitch)"
|
||||
systemctl is-active --quiet openvswitch || sudo systemctl start openvswitch \
|
||||
|| die "could not start openvswitch"
|
||||
}
|
||||
|
||||
# All VLAN ids from the config, comma separated — used for trunk ports.
|
||||
vlan_id_list() {
|
||||
local ids=()
|
||||
for entry in "${SELECTED[@]}"; do ids+=("${entry%%:*}"); done
|
||||
(IFS=,; echo "${ids[*]}")
|
||||
}
|
||||
|
||||
ovs_up() {
|
||||
ovs_require
|
||||
ovs --may-exist add-br "$OVS_BR"
|
||||
|
||||
# Host leg per VLAN: an OVS internal port carrying that VLAN's tag, given the
|
||||
# .2 address. This is how you SSH to the VMs. It is deliberately NOT their
|
||||
# default route (.1 is), so inter-VLAN tests exercise the router, not the
|
||||
# host's routing table.
|
||||
for entry in "${SELECTED[@]}"; do
|
||||
IFS=: read -r vid _name prefix _real <<<"$entry"
|
||||
local port="hostv${vid}"
|
||||
ovs --may-exist add-port "$OVS_BR" "$port" tag="$vid" \
|
||||
-- set interface "$port" type=internal
|
||||
sudo ip link set "$port" up 2>/dev/null || true
|
||||
sudo ip addr replace "${prefix}.2/24" dev "$port"
|
||||
done
|
||||
|
||||
ovs_define_libvirt_net
|
||||
}
|
||||
|
||||
# A libvirt network that hands out OVS ports: one portgroup per VLAN (access)
|
||||
# plus a trunk portgroup for the router.
|
||||
ovs_define_libvirt_net() {
|
||||
local pg="" ids
|
||||
for entry in "${SELECTED[@]}"; do
|
||||
IFS=: read -r vid name _p _r <<<"$entry"
|
||||
pg+=" <portgroup name='vlan${vid}'>
|
||||
<vlan><tag id='${vid}'/></vlan>
|
||||
</portgroup>
|
||||
"
|
||||
done
|
||||
|
||||
# Trunk: VLAN 1 native/untagged, everything else tagged — the production
|
||||
# shape. libvirt expresses this declaratively via nativeMode='untagged'
|
||||
# (see libvirt formatnetwork.html), so it does not need fixing up by hand.
|
||||
# It also matters functionally: LACPDUs are untagged, and a trunk with no
|
||||
# native VLAN has nowhere to put them.
|
||||
local trunk=" <portgroup name='trunk'>
|
||||
<vlan trunk='yes'>
|
||||
"
|
||||
for entry in "${SELECTED[@]}"; do
|
||||
IFS=: read -r vid _n _p _r <<<"$entry"
|
||||
if [ "$vid" = "1" ]; then
|
||||
trunk+=" <tag id='1' nativeMode='untagged'/>
|
||||
"
|
||||
else
|
||||
trunk+=" <tag id='${vid}'/>
|
||||
"
|
||||
fi
|
||||
done
|
||||
trunk+=" </vlan>
|
||||
</portgroup>
|
||||
"
|
||||
|
||||
local xml="<network>
|
||||
<name>${OVS_NET}</name>
|
||||
<forward mode='bridge'/>
|
||||
<bridge name='${OVS_BR}'/>
|
||||
<virtualport type='openvswitch'/>
|
||||
${pg}${trunk}</network>"
|
||||
|
||||
if virsh_q net-info "$OVS_NET" >/dev/null 2>&1; then
|
||||
virsh_q net-destroy "$OVS_NET" >/dev/null 2>&1 || true
|
||||
virsh_q net-undefine "$OVS_NET" >/dev/null 2>&1 || true
|
||||
fi
|
||||
echo "$xml" | virsh_q net-define /dev/stdin >/dev/null
|
||||
virsh_q net-start "$OVS_NET" >/dev/null
|
||||
log "libvirt network $OVS_NET bound to $OVS_BR (access portgroups + trunk)"
|
||||
}
|
||||
|
||||
# Replace the router VM's two individual OVS ports with a single LACP bond.
|
||||
# libvirt attaches each NIC separately; only ovs-vsctl can bond them, and the
|
||||
# taps only exist once the VM is running — so this runs post-start.
|
||||
ovs_bond_router() {
|
||||
local vm="$1"
|
||||
local taps
|
||||
# NB: domiflist indents its rows, so anchor on the FIELD not the line —
|
||||
# /^vnet/ silently matches nothing and the bond never gets built.
|
||||
taps="$(virsh_q domiflist "$vm" 2>/dev/null | awk '$1 ~ /^vnet/ {print $1}')"
|
||||
local count; count="$(echo "$taps" | grep -c .)"
|
||||
[ "$count" -eq 2 ] || { warn "router $vm has $count tap(s), expected 2 — skipping bond"; return 1; }
|
||||
|
||||
# Already bonded? (idempotent re-runs)
|
||||
if ovs list-ports "$OVS_BR" 2>/dev/null | grep -qx "$LAG_NAME"; then
|
||||
log "LACP bond $LAG_NAME already present"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local t1 t2; t1="$(echo "$taps" | sed -n 1p)"; t2="$(echo "$taps" | sed -n 2p)"
|
||||
log "bonding $t1 + $t2 into $LAG_NAME (LACP active, balance-tcp)"
|
||||
ovs del-port "$OVS_BR" "$t1" 2>/dev/null || true
|
||||
ovs del-port "$OVS_BR" "$t2" 2>/dev/null || true
|
||||
|
||||
# bond_mode=balance-tcp is REQUIRED: OVS defaults a bond to active-backup,
|
||||
# which does not speak LACP at all (confirmed on ovs-discuss). It is also the
|
||||
# equivalent of VyOS's 802.3ad + layer2+3 hashing.
|
||||
#
|
||||
# lacp-fallback-ab breaks a genuine deadlock: OVS keeps members disabled
|
||||
# until LACP negotiates, while the partner needs carrier before it will send
|
||||
# LACPDUs. Falling back to active-backup brings the links up so negotiation
|
||||
# can start.
|
||||
#
|
||||
# native-untagged + tag=1 carries the untagged LACPDUs and the management
|
||||
# VLAN, matching production. libvirt's portgroup VLAN config does NOT apply
|
||||
# here — the bond is a port libvirt never created — so set it inline.
|
||||
local tagged; tagged="$(vlan_id_list | tr ',' '\n' | grep -vx 1 | paste -sd, -)"
|
||||
ovs add-bond "$OVS_BR" "$LAG_NAME" "$t1" "$t2" \
|
||||
lacp=active bond_mode=balance-tcp \
|
||||
vlan_mode=native-untagged tag=1 trunks="$tagged" \
|
||||
-- set port "$LAG_NAME" other_config:lacp-time=fast \
|
||||
-- set port "$LAG_NAME" other_config:lacp-fallback-ab=true
|
||||
}
|
||||
|
||||
ovs_bond_status() {
|
||||
echo "--- ovs bond ---"
|
||||
sudo ovs-appctl bond/show "$LAG_NAME" 2>/dev/null | grep -E "bond_mode|lacp_status|^member|may_enable" || echo "(no bond)"
|
||||
echo "--- lacp ---"
|
||||
sudo ovs-appctl lacp/show "$LAG_NAME" 2>/dev/null | grep -E "status|aggregation key|^member|attached" || true
|
||||
}
|
||||
|
||||
ovs_down() {
|
||||
virsh_q net-destroy "$OVS_NET" >/dev/null 2>&1 || true
|
||||
virsh_q net-undefine "$OVS_NET" >/dev/null 2>&1 || true
|
||||
if command -v ovs-vsctl >/dev/null 2>&1; then
|
||||
ovs --if-exists del-br "$OVS_BR" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
162
labsim/router-install.py
Executable file
162
labsim/router-install.py
Executable file
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Drive the labsim VyOS router over its serial console.
|
||||
|
||||
Three phases:
|
||||
--phase live wait for the live system and log in
|
||||
--phase install run `install image` unattended
|
||||
--phase configure apply bond0 (LACP) + per-VLAN gateway addresses
|
||||
|
||||
The installer prompt list is the same one the bastion's install driver answers
|
||||
(src/bastion/src/templates/vyos-install.py.ts). Two of them are easy to miss and
|
||||
both hang forever rather than failing: the reinstall-only "copy data to the new
|
||||
image?", and "choose two disks for RAID-1 mirroring?" — every RAID prompt
|
||||
defaults to YES.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import time
|
||||
|
||||
import pexpect
|
||||
|
||||
PASSWORD = "vyos"
|
||||
PROMPT = r"[\$#] $"
|
||||
|
||||
|
||||
def console(vm: str, timeout: int = 60) -> pexpect.spawn:
|
||||
c = pexpect.spawn(f"sudo virsh console {vm} --force", encoding="utf-8", timeout=timeout)
|
||||
c.expect("Connected to domain", timeout=30)
|
||||
return c
|
||||
|
||||
|
||||
def login(c: pexpect.spawn, timeout: int = 300) -> None:
|
||||
"""Get to a shell prompt, whether we land at a login or an open session."""
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
c.sendline("")
|
||||
i = c.expect(["login:", PROMPT, pexpect.TIMEOUT], timeout=20)
|
||||
if i == 0:
|
||||
c.sendline("vyos")
|
||||
c.expect("assword:", timeout=20)
|
||||
c.sendline(PASSWORD)
|
||||
j = c.expect([PROMPT, "incorrect", pexpect.TIMEOUT], timeout=30)
|
||||
if j == 0:
|
||||
return
|
||||
elif i == 1:
|
||||
return
|
||||
raise SystemExit("timed out waiting for a VyOS shell")
|
||||
|
||||
|
||||
def run(c: pexpect.spawn, cmd: str, timeout: int = 60) -> str:
|
||||
c.sendline(cmd)
|
||||
c.expect(PROMPT, timeout=timeout)
|
||||
return c.before or ""
|
||||
|
||||
|
||||
def phase_install(c: pexpect.spawn) -> None:
|
||||
"""Answer `install image` end to end."""
|
||||
rules: list[tuple[str, str]] = [
|
||||
(r"Would you like to continue\?", "yes"),
|
||||
(r"What would you like to name this image\?", ""),
|
||||
(r"Please confirm password for the .vyos. user:", PASSWORD),
|
||||
(r"Please enter a password for the .vyos. user:", PASSWORD),
|
||||
(r"What console should be used by default", "K"),
|
||||
# every RAID variant defaults to YES — decline them all
|
||||
(r"Would you like to [^?]*RAID-1 mirroring", "no"),
|
||||
(r"Installation will delete all data on (?:the drive|both drives)\. Continue\?", "yes"),
|
||||
(r"Which one should be used for installation\?", "/dev/vda"),
|
||||
(r"Would you like to use all the free space on the drive\?", "yes"),
|
||||
(r"Which file would you like as boot config\?", "1"),
|
||||
# reinstall-only; unanswered it blocks on stdin until the world ends
|
||||
(r"Would you like to copy data to the new image\?", "yes"),
|
||||
(r"From which image would you like to save config information\?", "1"),
|
||||
]
|
||||
patterns = [r for r, _ in rules] + [r"The image installed successfully",
|
||||
r"Unable to install VyOS", pexpect.TIMEOUT]
|
||||
|
||||
c.sendline("install image")
|
||||
for _ in range(60):
|
||||
i = c.expect(patterns, timeout=180)
|
||||
if i < len(rules):
|
||||
c.sendline(rules[i][1])
|
||||
continue
|
||||
if i == len(rules):
|
||||
print(" installer: success")
|
||||
return
|
||||
if i == len(rules) + 1:
|
||||
raise SystemExit("installer reported failure")
|
||||
raise SystemExit("installer went quiet (unanswered prompt?)")
|
||||
raise SystemExit("installer exceeded expected prompt count")
|
||||
|
||||
|
||||
def phase_configure(c: pexpect.spawn, vlans: list[tuple[str, str, str]]) -> None:
|
||||
"""bond0 over eth0+eth1 with LACP, then a gateway address per VLAN."""
|
||||
# Production shape: VLAN 1 (management) is the NATIVE/untagged VLAN on the
|
||||
# bond, everything else is a tagged vif. This matters beyond fidelity —
|
||||
# LACPDUs are untagged, so a trunk with no native VLAN has nowhere to put
|
||||
# them and the bond never negotiates.
|
||||
native = [v for v in vlans if v[0] == "1"]
|
||||
tagged = [v for v in vlans if v[0] != "1"]
|
||||
|
||||
cmds = [
|
||||
"configure",
|
||||
"set interfaces bonding bond0 mode '802.3ad'",
|
||||
"set interfaces bonding bond0 hash-policy 'layer2+3'",
|
||||
"set interfaces bonding bond0 lacp-rate 'fast'",
|
||||
"set interfaces bonding bond0 member interface 'eth0'",
|
||||
"set interfaces bonding bond0 member interface 'eth1'",
|
||||
"set service ssh port '22'",
|
||||
"set system login user vyos authentication plaintext-password 'vyos'",
|
||||
]
|
||||
for vid, name, prefix in native:
|
||||
cmds.append(f"set interfaces bonding bond0 address '{prefix}.1/24'")
|
||||
cmds.append(f"set interfaces bonding bond0 description '{name} (native)'")
|
||||
for vid, name, prefix in tagged:
|
||||
cmds.append(f"set interfaces bonding bond0 vif {vid} address '{prefix}.1/24'")
|
||||
cmds.append(f"set interfaces bonding bond0 vif {vid} description '{name}'")
|
||||
cmds += ["commit", "save", "exit"]
|
||||
|
||||
for cmd in cmds:
|
||||
out = run(c, cmd, timeout=180)
|
||||
low = out.lower()
|
||||
if "invalid" in low or "syntax error" in low or "commit failed" in low:
|
||||
print(f" !! {cmd}\n{out.strip()[-300:]}")
|
||||
raise SystemExit(f"config command rejected: {cmd}")
|
||||
print(" config committed and saved")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--vm", required=True)
|
||||
ap.add_argument("--phase", required=True, choices=["live", "install", "configure"])
|
||||
ap.add_argument("--vlans", default="", help="space separated vid:name:prefix:real entries")
|
||||
args = ap.parse_args()
|
||||
|
||||
c = console(args.vm)
|
||||
try:
|
||||
login(c)
|
||||
if args.phase == "live":
|
||||
print(" live system reachable")
|
||||
elif args.phase == "install":
|
||||
phase_install(c)
|
||||
else:
|
||||
vlans = []
|
||||
for entry in args.vlans.split():
|
||||
parts = entry.split(":")
|
||||
if len(parts) >= 3:
|
||||
vlans.append((parts[0], parts[1], parts[2]))
|
||||
if not vlans:
|
||||
raise SystemExit("no VLANs passed to configure")
|
||||
phase_configure(c, vlans)
|
||||
return 0
|
||||
finally:
|
||||
try:
|
||||
c.sendline("")
|
||||
c.close(force=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
124
labsim/router-up.sh
Executable file
124
labsim/router-up.sh
Executable file
@@ -0,0 +1,124 @@
|
||||
#!/bin/bash
|
||||
# Add the VyOS router under test to labsim.
|
||||
#
|
||||
# Mirrors the production VP2440 pair: TWO NICs bonded with LACP carrying a
|
||||
# trunk of every VLAN, then bond0.<vlan> sub-interfaces holding the .1 gateway
|
||||
# address on each. That is the same config shape the real firewalls run, so a
|
||||
# rule tested here means something.
|
||||
#
|
||||
# NIC model is e1000e, NOT virtio, and that is load-bearing: with virtio the
|
||||
# guest's bonding driver reports its slaves "MII Status: down" despite
|
||||
# carrier=1 and never emits a single LACPDU, so the bond sits in
|
||||
# AD_STATE_DEFAULTED forever. Known issue — see the netdev thread "bonding
|
||||
# (IEEE 802.3ad) not working with qemu/virtio"; e1000e fixes it with no other
|
||||
# change. 802.3ad also requires the MII link monitor, which virtio cannot back.
|
||||
#
|
||||
# host OVS "switch" VyOS VM
|
||||
# hostv<vlan> (.2) ──────── ovs-labsim ──── lag-vyos ═════ eth0 + eth1
|
||||
# (tagged) (LACP, trunk) └─ bond0.<vlan> = .1
|
||||
#
|
||||
# Usage: ./router-up.sh build + install + configure
|
||||
# ./router-up.sh --status show bond/LACP + interface state
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "$SCRIPT_DIR/lib.sh"
|
||||
source "$SCRIPT_DIR/ovs.sh"
|
||||
|
||||
ROUTER_VM="${ROUTER_VM:-labsim-vyos}"
|
||||
ROUTER_MEM="${ROUTER_MEM:-2048}"
|
||||
ROUTER_CPUS="${ROUTER_CPUS:-2}"
|
||||
ROUTER_DISK_GB="${ROUTER_DISK_GB:-8}"
|
||||
VYOS_ISO="${VYOS_ISO:-$IMG_DIR/vyos.iso}"
|
||||
VYOS_CACHE="/var/lib/libvirt/images/lab-pxe-cache"
|
||||
|
||||
selected_vlans
|
||||
|
||||
if [ "${1:-}" = "--status" ]; then
|
||||
ovs_bond_status
|
||||
echo "--- vyos gateway addresses (probed from each host leg) ---"
|
||||
for entry in "${SELECTED[@]}"; do
|
||||
IFS=: read -r vid _n prefix _r <<<"$entry"
|
||||
printf ' vlan %-5s %-16s ' "$vid" "${prefix}.1"
|
||||
ping -c1 -W2 "${prefix}.1" >/dev/null 2>&1 && echo up || echo down
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ovs_require
|
||||
|
||||
# --- ISO ------------------------------------------------------------------
|
||||
if [ ! -f "$VYOS_ISO" ]; then
|
||||
# Reuse the bastion's cached nightly if it is already on this box.
|
||||
if [ -f "$VYOS_CACHE/vyos.iso" ]; then
|
||||
log "reusing cached VyOS ISO"
|
||||
sudo cp "$VYOS_CACHE/vyos.iso" "$VYOS_ISO"
|
||||
else
|
||||
log "resolving latest VyOS nightly ISO..."
|
||||
url="$(curl -sSL https://api.github.com/repos/vyos/vyos-nightly-build/releases/latest \
|
||||
| python3 -c "import json,sys;print(next(a['browser_download_url'] for a in json.load(sys.stdin)['assets'] if a['name'].endswith('generic-amd64.iso')))")"
|
||||
log "downloading $url"
|
||||
sudo curl -sSL --max-time 1800 -o "$VYOS_ISO" "$url"
|
||||
fi
|
||||
fi
|
||||
[ -f "$VYOS_ISO" ] || die "no VyOS ISO at $VYOS_ISO"
|
||||
|
||||
# --- VM -------------------------------------------------------------------
|
||||
if virsh_q dominfo "$ROUTER_VM" >/dev/null 2>&1; then
|
||||
log "router VM $ROUTER_VM exists"
|
||||
virsh_q start "$ROUTER_VM" >/dev/null 2>&1 || true
|
||||
else
|
||||
log "creating router VM $ROUTER_VM (2 NICs on the trunk, for LACP)"
|
||||
sudo qemu-img create -q -f qcow2 "$IMG_DIR/${ROUTER_VM}.qcow2" "${ROUTER_DISK_GB}G" >/dev/null
|
||||
|
||||
# Two trunk NICs — OVS bonds them after boot (libvirt cannot create bonds).
|
||||
sudo virt-install \
|
||||
--connect "$LIBVIRT_URI" \
|
||||
--name "$ROUTER_VM" \
|
||||
--memory "$ROUTER_MEM" --vcpus "$ROUTER_CPUS" \
|
||||
--disk "path=$IMG_DIR/${ROUTER_VM}.qcow2,format=qcow2,bus=virtio" \
|
||||
--disk "path=$VYOS_ISO,device=cdrom,readonly=on" \
|
||||
--network "network=$OVS_NET,portgroup=trunk,model=e1000e,trustGuestRxFilters=yes" \
|
||||
--network "network=$OVS_NET,portgroup=trunk,model=e1000e,trustGuestRxFilters=yes" \
|
||||
--boot cdrom,hd \
|
||||
--os-variant debian12 \
|
||||
--graphics none --noautoconsole --import >/dev/null
|
||||
fi
|
||||
|
||||
log "waiting for the live system to boot (VyOS live login)..."
|
||||
python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase live || die "live boot failed"
|
||||
|
||||
log "installing VyOS to disk (unattended over the console)..."
|
||||
python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase install || die "install failed"
|
||||
|
||||
# Boot the INSTALLED system from here on. Without this the VM was created with
|
||||
# --boot cdrom,hd and every restart re-runs the ISO, so the live system comes
|
||||
# back with no config and every `commit; save` silently evaporates.
|
||||
log "switching boot to disk and ejecting the install media..."
|
||||
virsh_q destroy "$ROUTER_VM" >/dev/null 2>&1 || true
|
||||
sleep 2
|
||||
sudo virt-xml "$ROUTER_VM" --edit --boot hd >/dev/null
|
||||
sudo virt-xml "$ROUTER_VM" --remove-device --disk device=cdrom >/dev/null 2>&1 || true
|
||||
virsh_q start "$ROUTER_VM" >/dev/null
|
||||
sleep 10
|
||||
|
||||
# Bond the taps only now: they are recreated by the restart above, so bonding
|
||||
# before this would bond stale interfaces.
|
||||
ovs_bond_router "$ROUTER_VM"
|
||||
|
||||
log "applying router config (bond0 LACP + VLAN gateways)..."
|
||||
python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase configure \
|
||||
--vlans "$(printf '%s\n' "${SELECTED[@]}" | tr '\n' ' ')" || die "configure failed"
|
||||
|
||||
log "waiting for LACP to negotiate..."
|
||||
for _ in $(seq 1 30); do
|
||||
if sudo ovs-appctl lacp/show "$LAG_NAME" 2>/dev/null | grep -q "current attached"; then
|
||||
log "LACP negotiated"; break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
|
||||
echo
|
||||
ovs_bond_status
|
||||
echo
|
||||
log "router is up. Check reachability with: $SCRIPT_DIR/labsim-matrix.py --watch 2"
|
||||
181
labsim/topology.html
Normal file
181
labsim/topology.html
Normal file
@@ -0,0 +1,181 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>labsim — live VLAN topology</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg:#0e1116; --panel:#161b22; --line:#30363d; --text:#e6edf3; --dim:#8b949e;
|
||||
--ok:#3fb950; --bad:#f85149; --warn:#d29922; --router:#58a6ff;
|
||||
}
|
||||
* { box-sizing:border-box; }
|
||||
body { margin:0; background:var(--bg); color:var(--text);
|
||||
font:14px/1.5 ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif; }
|
||||
header { display:flex; align-items:baseline; gap:16px; flex-wrap:wrap;
|
||||
padding:14px 20px; border-bottom:1px solid var(--line); }
|
||||
h1 { font-size:16px; margin:0; font-weight:650; letter-spacing:.2px; }
|
||||
.meta { color:var(--dim); font-size:12px; }
|
||||
.pill { padding:2px 8px; border-radius:999px; font-size:12px; font-weight:600; }
|
||||
.pill.ok { background:rgba(63,185,80,.15); color:var(--ok); }
|
||||
.pill.bad { background:rgba(248,81,73,.15); color:var(--bad); }
|
||||
main { display:grid; grid-template-columns:minmax(0,1.35fr) minmax(320px,.65fr);
|
||||
gap:16px; padding:16px 20px; align-items:start; }
|
||||
@media (max-width:1000px){ main { grid-template-columns:1fr; } }
|
||||
.card { background:var(--panel); border:1px solid var(--line); border-radius:10px; padding:14px; }
|
||||
.card h2 { margin:0 0 10px; font-size:13px; font-weight:600; color:var(--dim);
|
||||
text-transform:uppercase; letter-spacing:.6px; }
|
||||
svg { width:100%; height:auto; display:block; }
|
||||
.edge { stroke-width:2.5; transition:stroke .25s, opacity .25s; }
|
||||
.edge.ok { stroke:var(--ok); opacity:.55; }
|
||||
.edge.bad { stroke:var(--bad); opacity:.95; stroke-dasharray:7 5; }
|
||||
.edge:hover { opacity:1; stroke-width:4; }
|
||||
.node circle { fill:#0d1117; stroke-width:2.5; }
|
||||
.node text { text-anchor:middle; font-size:11px; font-weight:600; fill:var(--text); }
|
||||
.node .sub { font-size:9.5px; font-weight:400; fill:var(--dim); }
|
||||
.rtt { font-size:9px; fill:var(--dim); text-anchor:middle; }
|
||||
table { width:100%; border-collapse:collapse; font-size:12.5px; }
|
||||
th,td { text-align:left; padding:5px 8px; border-bottom:1px solid var(--line); }
|
||||
th { color:var(--dim); font-weight:600; font-size:11px; text-transform:uppercase; }
|
||||
td.n { text-align:right; font-variant-numeric:tabular-nums; }
|
||||
.b-ok { color:var(--ok); } .b-bad { color:var(--bad); }
|
||||
.empty { color:var(--dim); padding:10px 4px; }
|
||||
.legend { display:flex; gap:14px; align-items:center; color:var(--dim);
|
||||
font-size:11.5px; margin-top:10px; flex-wrap:wrap; }
|
||||
.swatch { display:inline-block; width:22px; height:0; border-top:2.5px solid; margin-right:5px;
|
||||
vertical-align:middle; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>labsim — live VLAN topology</h1>
|
||||
<span id="summary" class="pill ok">…</span>
|
||||
<span class="meta">every path is probed <em>from</em> a VM <em>to</em> every other VM, through the VyOS router</span>
|
||||
<span class="meta" id="clock" style="margin-left:auto"></span>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<section class="card">
|
||||
<h2>Mesh — line colour is reachability, label is ICMP RTT</h2>
|
||||
<svg id="topo" viewBox="0 0 720 560" role="img" aria-label="VLAN topology"></svg>
|
||||
<div class="legend">
|
||||
<span><i class="swatch" style="border-color:var(--ok)"></i>reachable</span>
|
||||
<span><i class="swatch" style="border-color:var(--bad); border-top-style:dashed"></i>blocked</span>
|
||||
<span>hover a line for detail · node ring turns red if anything to/from it is blocked</span>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<aside style="display:grid; gap:16px">
|
||||
<section class="card">
|
||||
<h2>Blocked paths</h2>
|
||||
<div id="blocked"></div>
|
||||
</section>
|
||||
<section class="card">
|
||||
<h2>Latency (ICMP, ms)</h2>
|
||||
<table><thead><tr><th>path</th><th class="n">rtt</th></tr></thead>
|
||||
<tbody id="lat"></tbody></table>
|
||||
</section>
|
||||
</aside>
|
||||
</main>
|
||||
|
||||
<script>
|
||||
const REFRESH_MS = 5000;
|
||||
const CX = 360, CY = 250, R = 185;
|
||||
|
||||
function polar(i, n) {
|
||||
const a = (i / n) * Math.PI * 2 - Math.PI / 2;
|
||||
return { x: CX + R * Math.cos(a), y: CY + R * Math.sin(a) };
|
||||
}
|
||||
|
||||
function render(data) {
|
||||
const vlans = data.vlans, res = data.results;
|
||||
const svg = document.getElementById('topo');
|
||||
const n = vlans.length;
|
||||
const pos = vlans.map((_, i) => polar(i, n));
|
||||
let out = '';
|
||||
|
||||
// Router in the middle — every inter-VLAN packet really does traverse it.
|
||||
out += `<circle cx="${CX}" cy="${CY}" r="40" fill="#0d1117" stroke="var(--router)" stroke-width="2.5"/>`;
|
||||
out += `<text x="${CX}" y="${CY-6}" text-anchor="middle" font-size="12" font-weight="700" fill="var(--router)">VyOS</text>`;
|
||||
out += `<text x="${CX}" y="${CY+9}" text-anchor="middle" font-size="8.5" fill="var(--dim)">bond0</text>`;
|
||||
out += `<text x="${CX}" y="${CY+20}" text-anchor="middle" font-size="8.5" fill="var(--dim)">LACP</text>`;
|
||||
|
||||
const bad = new Set();
|
||||
// One line per unordered pair; a pair is bad if EITHER direction fails.
|
||||
for (let i = 0; i < n; i++) {
|
||||
for (let j = i + 1; j < n; j++) {
|
||||
const a = vlans[i].label, b = vlans[j].label;
|
||||
const ab = (res[a] || {})[b] || {}, ba = (res[b] || {})[a] || {};
|
||||
const okAB = ab.icmp === true, okBA = ba.icmp === true;
|
||||
const ok = okAB && okBA;
|
||||
if (!ok) { bad.add(a); bad.add(b); }
|
||||
const rtts = [ab.rtt_ms, ba.rtt_ms].filter(v => typeof v === 'number');
|
||||
const rtt = rtts.length ? (rtts.reduce((s,v)=>s+v,0)/rtts.length) : null;
|
||||
// Place the label ~32% along the edge, not at the midpoint: diagonals of
|
||||
// a 6-node mesh all cross the centre, so midpoint labels stack on top of
|
||||
// the router node. Plus a small perpendicular nudge off the line itself.
|
||||
const dx = pos[j].x - pos[i].x, dy = pos[j].y - pos[i].y;
|
||||
const len = Math.hypot(dx, dy) || 1;
|
||||
const t = 0.32;
|
||||
const mx = pos[i].x + dx * t + (-dy / len) * 8;
|
||||
const my = pos[i].y + dy * t + ( dx / len) * 8;
|
||||
const tip = `${a} ↔ ${b}\n→ ${okAB ? 'ok' : 'BLOCKED'} ← ${okBA ? 'ok' : 'BLOCKED'}` +
|
||||
(rtt !== null ? `\nrtt ${rtt.toFixed(2)} ms` : '');
|
||||
out += `<line class="edge ${ok?'ok':'bad'}" x1="${pos[i].x}" y1="${pos[i].y}" x2="${pos[j].x}" y2="${pos[j].y}"><title>${tip}</title></line>`;
|
||||
if (ok && rtt !== null)
|
||||
out += `<text class="rtt" x="${mx}" y="${my}">${rtt.toFixed(2)}</text>`;
|
||||
}
|
||||
}
|
||||
|
||||
vlans.forEach((v, i) => {
|
||||
const p = pos[i], isBad = bad.has(v.label);
|
||||
out += `<g class="node"><circle cx="${p.x}" cy="${p.y}" r="30" stroke="${isBad?'var(--bad)':'var(--ok)'}"/>` +
|
||||
`<text x="${p.x}" y="${p.y-2}">${v.name}</text>` +
|
||||
`<text class="sub" x="${p.x}" y="${p.y+11}">vlan ${v.vid}</text>` +
|
||||
`<text class="sub" x="${p.x}" y="${p.y+47}">${v.ip}</text></g>`;
|
||||
});
|
||||
svg.innerHTML = out;
|
||||
|
||||
// Blocked list — the thing you actually act on.
|
||||
const rows = [];
|
||||
for (const src of vlans) for (const dst of vlans) {
|
||||
if (src.label === dst.label) continue;
|
||||
const d = (res[src.label] || {})[dst.label] || {};
|
||||
for (const proto of ['icmp','tcp22','tcp80'])
|
||||
if (d[proto] === false) rows.push(`${src.label} → ${dst.label} <span style="color:var(--dim)">(${proto})</span>`);
|
||||
}
|
||||
document.getElementById('blocked').innerHTML = rows.length
|
||||
? `<table><tbody>${rows.map(r=>`<tr><td class="b-bad">${r}</td></tr>`).join('')}</tbody></table>`
|
||||
: `<div class="empty">none — all ${vlans.length*(vlans.length-1)*3} paths open</div>`;
|
||||
|
||||
// Latency table, slowest first.
|
||||
const lat = [];
|
||||
for (const src of vlans) for (const dst of vlans) {
|
||||
if (src.label === dst.label) continue;
|
||||
const d = (res[src.label] || {})[dst.label] || {};
|
||||
if (typeof d.rtt_ms === 'number') lat.push([`${src.label} → ${dst.label}`, d.rtt_ms]);
|
||||
}
|
||||
lat.sort((a,b) => b[1]-a[1]);
|
||||
document.getElementById('lat').innerHTML = lat.slice(0,12)
|
||||
.map(([k,v]) => `<tr><td>${k}</td><td class="n">${v.toFixed(2)}</td></tr>`).join('')
|
||||
|| `<tr><td class="empty" colspan="2">no RTT data</td></tr>`;
|
||||
|
||||
const total = data.total, reach = data.reachable;
|
||||
const pill = document.getElementById('summary');
|
||||
pill.textContent = `${reach}/${total} paths open`;
|
||||
pill.className = 'pill ' + (reach === total ? 'ok' : 'bad');
|
||||
document.getElementById('clock').textContent =
|
||||
`updated ${new Date().toLocaleTimeString()} · sweep ${data.sweep_seconds.toFixed(2)}s · refresh ${REFRESH_MS/1000}s`;
|
||||
}
|
||||
|
||||
async function tick() {
|
||||
try {
|
||||
const r = await fetch('/api/matrix', {cache:'no-store'});
|
||||
render(await r.json());
|
||||
} catch (e) {
|
||||
document.getElementById('clock').textContent = 'exporter unreachable — ' + e;
|
||||
}
|
||||
}
|
||||
tick(); setInterval(tick, REFRESH_MS);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
21
labsim/vlans.conf
Normal file
21
labsim/vlans.conf
Normal file
@@ -0,0 +1,21 @@
|
||||
# Lab network simulation — VLAN map.
|
||||
#
|
||||
# Mirrors the real UniFi topology (same VLAN IDs, same roles) but with
|
||||
# deliberately DIFFERENT IP ranges so nothing here can collide with, or be
|
||||
# confused for, production. The sim subnet always encodes the VLAN id:
|
||||
#
|
||||
# 172.31.<vlan-id>.0/24
|
||||
#
|
||||
# Per-subnet address plan (same shape on every VLAN):
|
||||
# .1 gateway under test (VyOS/router VM — not created by default)
|
||||
# .2 host bridge (how you SSH in from this workstation)
|
||||
# .10 the micro VM for this VLAN
|
||||
# .254 VRRP VIP (reserved, mirrors production)
|
||||
#
|
||||
# Format: vlan_id:name:sim_subnet_prefix:real_subnet(for reference)
|
||||
1:management:172.31.1:192.168.1.0/24
|
||||
2:k8s:172.31.2:192.168.8.0/23
|
||||
3:kvm:172.31.3:192.168.3.0/24
|
||||
9:private:172.31.9:10.0.9.0/23
|
||||
10:lot:172.31.10:10.0.0.0/23
|
||||
200:roomates:172.31.200:192.168.2.0/24
|
||||
Reference in New Issue
Block a user