Compare commits
19 Commits
fix/k3s-au
...
feat/vyos-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
72c54edce2 | ||
|
|
33be713d0c | ||
|
|
a5b36678ed | ||
|
|
c91e44f796 | ||
|
|
df2dfc5d71 | ||
|
|
e36a7a193c | ||
|
|
5d00c42f5a | ||
|
|
cb9d99dd69 | ||
|
|
4f9a6f64e4 | ||
|
|
12fa954a05 | ||
| 7181a61cec | |||
|
|
cdf3b5c045 | ||
| f3c50f71ef | |||
|
|
98b0ccc6c9 | ||
|
|
37a3b51e57 | ||
|
|
d6e1f3c74d | ||
|
|
52e831b8c1 | ||
| f5af24699a | |||
|
|
04faa079e2 |
@@ -11,6 +11,7 @@ WORKDIR /app
|
|||||||
# Copy workspace config and package manifests first (layer cache)
|
# Copy workspace config and package manifests first (layer cache)
|
||||||
COPY pnpm-workspace.yaml pnpm-lock.yaml package.json tsconfig.base.json tsconfig.json ./
|
COPY pnpm-workspace.yaml pnpm-lock.yaml package.json tsconfig.base.json tsconfig.json ./
|
||||||
COPY src/shared/package.json src/shared/tsconfig.json src/shared/
|
COPY src/shared/package.json src/shared/tsconfig.json src/shared/
|
||||||
|
COPY src/core/package.json src/core/tsconfig.json src/core/
|
||||||
COPY src/labd/package.json src/labd/tsconfig.json src/labd/
|
COPY src/labd/package.json src/labd/tsconfig.json src/labd/
|
||||||
|
|
||||||
# Install all dependencies (dev included -- needed for build)
|
# Install all dependencies (dev included -- needed for build)
|
||||||
@@ -22,10 +23,13 @@ RUN pnpm --filter @lab/labd exec prisma generate
|
|||||||
|
|
||||||
# Copy source code
|
# Copy source code
|
||||||
COPY src/shared/src/ src/shared/src/
|
COPY src/shared/src/ src/shared/src/
|
||||||
|
COPY src/core/src/ src/core/src/
|
||||||
COPY src/labd/src/ src/labd/src/
|
COPY src/labd/src/ src/labd/src/
|
||||||
|
|
||||||
# Build TypeScript (shared first via project references)
|
# Build TypeScript (shared + core before labd via project references)
|
||||||
RUN pnpm --filter @lab/shared build && pnpm --filter @lab/labd build
|
RUN pnpm --filter @lab/shared build \
|
||||||
|
&& pnpm --filter @lab/core build \
|
||||||
|
&& pnpm --filter @lab/labd build
|
||||||
|
|
||||||
# Hoist the generated Prisma client so stage 2 can COPY it from a stable path
|
# Hoist the generated Prisma client so stage 2 can COPY it from a stable path
|
||||||
RUN mkdir -p /app/_prisma && \
|
RUN mkdir -p /app/_prisma && \
|
||||||
@@ -41,6 +45,7 @@ WORKDIR /app
|
|||||||
# Copy workspace config and package manifests
|
# Copy workspace config and package manifests
|
||||||
COPY pnpm-workspace.yaml pnpm-lock.yaml package.json ./
|
COPY pnpm-workspace.yaml pnpm-lock.yaml package.json ./
|
||||||
COPY src/shared/package.json src/shared/
|
COPY src/shared/package.json src/shared/
|
||||||
|
COPY src/core/package.json src/core/
|
||||||
COPY src/labd/package.json src/labd/
|
COPY src/labd/package.json src/labd/
|
||||||
|
|
||||||
# Install production dependencies only
|
# Install production dependencies only
|
||||||
@@ -48,6 +53,7 @@ RUN pnpm install --frozen-lockfile --prod 2>/dev/null || pnpm install --prod
|
|||||||
|
|
||||||
# Copy built output from builder
|
# Copy built output from builder
|
||||||
COPY --from=builder /app/src/shared/dist/ src/shared/dist/
|
COPY --from=builder /app/src/shared/dist/ src/shared/dist/
|
||||||
|
COPY --from=builder /app/src/core/dist/ src/core/dist/
|
||||||
COPY --from=builder /app/src/labd/dist/ src/labd/dist/
|
COPY --from=builder /app/src/labd/dist/ src/labd/dist/
|
||||||
|
|
||||||
# Copy Prisma schema + generated client into pnpm store location
|
# Copy Prisma schema + generated client into pnpm store location
|
||||||
|
|||||||
@@ -59,10 +59,10 @@ _labctl() {
|
|||||||
COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
|
COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
|
||||||
return ;;
|
return ;;
|
||||||
"provision install")
|
"provision install")
|
||||||
COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur"))
|
COMPREPLY=($(compgen -W "--role --os --disk --vyos-mgmt --vyos-mgmt-address --vyos-bond --vyos-bond-address --vyos-bond-vrrp --vlan-vip --vyos-vrrp-priority --vyos-mgmt-vlan --vlan --vyos-password --vyos-hwid --vyos-fresh-config -h --help" -- "$cur"))
|
||||||
return ;;
|
return ;;
|
||||||
"provision reprovision")
|
"provision reprovision")
|
||||||
COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur"))
|
COMPREPLY=($(compgen -W "--role --os --disk --user -h --help" -- "$cur"))
|
||||||
return ;;
|
return ;;
|
||||||
"provision debug")
|
"provision debug")
|
||||||
COMPREPLY=($(compgen -W "--pxe-boot -h --help" -- "$cur"))
|
COMPREPLY=($(compgen -W "--pxe-boot -h --help" -- "$cur"))
|
||||||
|
|||||||
@@ -132,13 +132,26 @@ complete -c labctl -n "__labctl_using_cmd provision" -a recheck -d 'Refresh hard
|
|||||||
|
|
||||||
# provision install options
|
# provision install options
|
||||||
complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
|
complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
|
||||||
complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04'
|
complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling'
|
||||||
complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target disk device (auto-detect if omitted)' -x
|
complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target disk device (auto-detect if omitted)' -x
|
||||||
|
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt -d 'VyOS: untagged interface the machine PXE boots from (default eth0)' -x
|
||||||
|
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-address -d 'VyOS: CIDR for the management interface, or \'dhcp\' (default dhcp)' -x
|
||||||
|
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond -d 'VyOS: comma-separated LACP bond members (must exclude the PXE NIC)' -x
|
||||||
|
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond-address -d 'VyOS: address on the untagged bond (trunk native VLAN)' -x
|
||||||
|
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond-vrrp -d 'VyOS: VRRP VIP floated on the untagged bond' -x
|
||||||
|
complete -c labctl -n "__labctl_in_cmd provision install" -l vlan-vip -d 'VyOS: VRRP VIP for a --vlan entry (repeatable)' -x
|
||||||
|
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-vrrp-priority -d 'VyOS: VRRP priority for all groups on this box (higher = master)' -x
|
||||||
|
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-vlan -d 'VyOS: tagged management VLAN on the PXE port' -x
|
||||||
|
complete -c labctl -n "__labctl_in_cmd provision install" -l vlan -d 'VyOS: tagged VLAN sub-interface on the bond (repeatable)' -x
|
||||||
|
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-password -d 'VyOS: password for the \'vyos\' user' -x
|
||||||
|
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-hwid -d 'VyOS: pin an interface name to a MAC via hw-id (repeatable)' -x
|
||||||
|
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-fresh-config -d 'VyOS: on reinstall, overwrite the preserved config with the generated one'
|
||||||
|
|
||||||
# provision reprovision options
|
# provision reprovision options
|
||||||
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
|
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
|
||||||
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04'
|
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling'
|
||||||
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l disk -d 'Target disk device (auto-detect if omitted)' -x
|
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l disk -d 'Target disk device (auto-detect if omitted)' -x
|
||||||
|
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l user -d 'SSH user for the reboot (default: vyos for VyOS machines, else current user)' -x
|
||||||
|
|
||||||
# provision debug options
|
# provision debug options
|
||||||
complete -c labctl -n "__labctl_in_cmd provision debug" -l pxe-boot -d 'Boot installed system via PXE (kernel+initrd from network, root from NVMe)'
|
complete -c labctl -n "__labctl_in_cmd provision debug" -l pxe-boot -d 'Boot installed system via PXE (kernel+initrd from network, root from NVMe)'
|
||||||
|
|||||||
@@ -21,6 +21,8 @@
|
|||||||
"test:integration:pxe:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'PXE boot'",
|
"test:integration:pxe:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'PXE boot'",
|
||||||
"test:integration:iso": "vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'",
|
"test:integration:iso": "vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'",
|
||||||
"test:integration:iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'",
|
"test:integration:iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'",
|
||||||
|
"test:integration:vyos": "vitest run -c tests/integration/vitest.config.ts -t 'VyOS provisioning'",
|
||||||
|
"test:integration:vyos:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'VyOS provisioning'",
|
||||||
"test:integration:arm-iso": "vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'",
|
"test:integration:arm-iso": "vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'",
|
||||||
"test:integration:arm-iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'",
|
"test:integration:arm-iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'",
|
||||||
"test:integration:asahi": "vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'",
|
"test:integration:asahi": "vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'",
|
||||||
|
|||||||
1847
bastion/pnpm-lock.yaml
generated
1847
bastion/pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -20,6 +20,15 @@ export function loadConfig(overrides: Partial<BastionConfig> = {}): BastionConfi
|
|||||||
const ubuntuMirror = overrides.ubuntuMirror ?? process.env["UBUNTU_MIRROR"]
|
const ubuntuMirror = overrides.ubuntuMirror ?? process.env["UBUNTU_MIRROR"]
|
||||||
?? `https://releases.ubuntu.com/${ubuntuVersion}`;
|
?? `https://releases.ubuntu.com/${ubuntuVersion}`;
|
||||||
|
|
||||||
|
// "latest" resolves the newest nightly ISO from the vyos-nightly-build GitHub
|
||||||
|
// releases at startup. downloads.vyos.io no longer serves direct rolling ISOs
|
||||||
|
// (it returns the vyos.io site, and nightly builds sit behind a signup form);
|
||||||
|
// GitHub releases are the remaining free, unauthenticated direct source.
|
||||||
|
// LTS ISOs are subscription-only. Set VYOS_ISO_URL to pin a specific build.
|
||||||
|
const vyosIsoUrl = overrides.vyosIsoUrl ?? process.env["VYOS_ISO_URL"] ?? "latest";
|
||||||
|
const vyosDefaultPassword = overrides.vyosDefaultPassword
|
||||||
|
?? process.env["VYOS_DEFAULT_PASSWORD"] ?? "vyos";
|
||||||
|
|
||||||
const fedoraMirror = `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`;
|
const fedoraMirror = `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`;
|
||||||
const tftpDir = `${bastionDir}/tftp`;
|
const tftpDir = `${bastionDir}/tftp`;
|
||||||
const httpDir = `${bastionDir}/http`;
|
const httpDir = `${bastionDir}/http`;
|
||||||
@@ -38,6 +47,8 @@ export function loadConfig(overrides: Partial<BastionConfig> = {}): BastionConfi
|
|||||||
dhcpRangeEnd,
|
dhcpRangeEnd,
|
||||||
ubuntuVersion,
|
ubuntuVersion,
|
||||||
ubuntuMirror,
|
ubuntuMirror,
|
||||||
|
vyosIsoUrl,
|
||||||
|
vyosDefaultPassword,
|
||||||
// These are populated at runtime by the network service
|
// These are populated at runtime by the network service
|
||||||
iface: overrides.iface ?? "",
|
iface: overrides.iface ?? "",
|
||||||
serverIp: overrides.serverIp ?? "",
|
serverIp: overrides.serverIp ?? "",
|
||||||
|
|||||||
@@ -40,6 +40,125 @@ function download(url: string, dest: string, label: string): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pick the largest regular-file initrd from an `xorriso -lsl` listing.
|
||||||
|
*
|
||||||
|
* /live carries decoys: a 0-byte initrd.img placeholder on some images, or an
|
||||||
|
* initrd.img SYMLINK to the real version-suffixed file on others. Parsing is
|
||||||
|
* field-based (ls -l layout: perms links uid gid size month day time 'name')
|
||||||
|
* and considers only lines whose mode string marks a regular file — symlinks
|
||||||
|
* report their link size, not the target's, and must not win.
|
||||||
|
*/
|
||||||
|
export function pickLargestInitrd(
|
||||||
|
listing: string,
|
||||||
|
): { name: string; size: number } | undefined {
|
||||||
|
let best: { name: string; size: number } | undefined;
|
||||||
|
for (const line of listing.split("\n")) {
|
||||||
|
if (!line.startsWith("-")) continue; // regular files only
|
||||||
|
const quoted = /'([^']+)'/.exec(line);
|
||||||
|
const fields = line.trim().split(/\s+/);
|
||||||
|
const size = parseInt(fields[4] ?? "", 10);
|
||||||
|
const name = quoted?.[1] ?? "";
|
||||||
|
if (!name.startsWith("initrd")) continue;
|
||||||
|
if (!Number.isFinite(size) || size <= 0) continue;
|
||||||
|
if (best === undefined || size > best.size) {
|
||||||
|
best = { name, size };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return best;
|
||||||
|
}
|
||||||
|
|
||||||
|
const VYOS_NIGHTLY_RELEASES =
|
||||||
|
"https://api.github.com/repos/vyos/vyos-nightly-build/releases/latest";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve the configured VyOS ISO URL, expanding the "latest" sentinel.
|
||||||
|
*
|
||||||
|
* The nightly asset filename embeds a build date, so there is no stable
|
||||||
|
* "latest.iso" path to hardcode — the newest release has to be looked up.
|
||||||
|
* Any other value is used verbatim, which is how VYOS_ISO_URL pins a build
|
||||||
|
* or points at a locally mirrored copy.
|
||||||
|
*/
|
||||||
|
function resolveVyosIsoUrl(configured: string): string {
|
||||||
|
if (configured !== "latest") return configured;
|
||||||
|
|
||||||
|
const body = execSync(`curl -sSfL "${VYOS_NIGHTLY_RELEASES}"`, {
|
||||||
|
encoding: "utf-8",
|
||||||
|
stdio: ["pipe", "pipe", "pipe"],
|
||||||
|
});
|
||||||
|
const release = JSON.parse(body) as {
|
||||||
|
tag_name?: string;
|
||||||
|
assets?: Array<{ name: string; browser_download_url: string }>;
|
||||||
|
};
|
||||||
|
|
||||||
|
const asset = (release.assets ?? []).find((a) =>
|
||||||
|
/generic-amd64\.iso$/.test(a.name),
|
||||||
|
);
|
||||||
|
if (!asset) {
|
||||||
|
throw new Error(
|
||||||
|
`No generic-amd64 ISO asset in VyOS nightly release ${release.tag_name ?? "?"}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(` VyOS ISO resolved to ${asset.name} (${release.tag_name ?? "?"})`);
|
||||||
|
return asset.browser_download_url;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract VyOS netboot artifacts from the release ISO.
|
||||||
|
*
|
||||||
|
* VyOS publishes no netboot bundle, so kernel/initrd/squashfs have to come out
|
||||||
|
* of the ISO. xorriso is already in the bastion image (used for boot.iso) and
|
||||||
|
* extracts without root or a loop mount.
|
||||||
|
*
|
||||||
|
* The initrd needs care: /live contains an empty initrd.img placeholder
|
||||||
|
* alongside the real one, which carries a version-suffixed name. Booting the
|
||||||
|
* 0-byte file fails with no useful diagnostic, so pick the largest initrd*.
|
||||||
|
*/
|
||||||
|
export function prepareVyosArtifacts(config: BastionConfig): void {
|
||||||
|
const kernel = `${config.httpDir}/vyos-vmlinuz`;
|
||||||
|
const initrd = `${config.httpDir}/vyos-initrd`;
|
||||||
|
const squashfs = `${config.httpDir}/vyos-filesystem.squashfs`;
|
||||||
|
|
||||||
|
if (existsSync(kernel) && existsSync(initrd) && existsSync(squashfs)) {
|
||||||
|
logger.info(" VyOS netboot artifacts -- cached");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const iso = `${config.bastionDir}/vyos.iso`;
|
||||||
|
download(resolveVyosIsoUrl(config.vyosIsoUrl), iso, "VyOS ISO");
|
||||||
|
|
||||||
|
const extract = (isoPath: string, dest: string, label: string): void => {
|
||||||
|
execSync(
|
||||||
|
`xorriso -osirrox on -indev "${iso}" -extract "${isoPath}" "${dest}"`,
|
||||||
|
{ stdio: "pipe" },
|
||||||
|
);
|
||||||
|
logger.info(` ${label} -- extracted from ${isoPath}`);
|
||||||
|
};
|
||||||
|
|
||||||
|
extract("/live/vmlinuz", kernel, "VyOS kernel");
|
||||||
|
extract("/live/filesystem.squashfs", squashfs, "VyOS squashfs");
|
||||||
|
|
||||||
|
// Pick the real initrd by size from the ISO's own directory listing.
|
||||||
|
const listing = execSync(`xorriso -indev "${iso}" -lsl /live/ --`, {
|
||||||
|
encoding: "utf-8",
|
||||||
|
stdio: ["pipe", "pipe", "pipe"],
|
||||||
|
});
|
||||||
|
|
||||||
|
const best = pickLargestInitrd(listing);
|
||||||
|
if (best === undefined) {
|
||||||
|
throw new Error("No non-empty initrd found in /live on the VyOS ISO");
|
||||||
|
}
|
||||||
|
extract(`/live/${best.name}`, initrd, `VyOS initrd (${best.name}, ${best.size} bytes)`);
|
||||||
|
|
||||||
|
// The ISO is only needed to produce the three artifacts above.
|
||||||
|
try {
|
||||||
|
unlinkSync(iso);
|
||||||
|
} catch {
|
||||||
|
// Non-fatal: leaving it costs disk but nothing else.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function symlinkSafe(target: string, linkPath: string): void {
|
function symlinkSafe(target: string, linkPath: string): void {
|
||||||
try {
|
try {
|
||||||
symlinkSync(target, linkPath);
|
symlinkSync(target, linkPath);
|
||||||
@@ -182,6 +301,17 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
|
|||||||
logger.warn(`Ubuntu ${config.ubuntuVersion} artifacts not available -- Ubuntu provisioning disabled`);
|
logger.warn(`Ubuntu ${config.ubuntuVersion} artifacts not available -- Ubuntu provisioning disabled`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// VyOS netboot artifacts (non-fatal — same policy as Ubuntu)
|
||||||
|
try {
|
||||||
|
logger.info("Preparing VyOS netboot artifacts...");
|
||||||
|
prepareVyosArtifacts(config);
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(
|
||||||
|
`VyOS artifacts not available -- VyOS provisioning disabled ` +
|
||||||
|
`(${err instanceof Error ? err.message : String(err)})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// Symlink iPXE binaries into HTTP dir for UEFI HTTP Boot
|
// Symlink iPXE binaries into HTTP dir for UEFI HTTP Boot
|
||||||
for (const name of ["ipxe.efi", "ipxe-arm64.efi"]) {
|
for (const name of ["ipxe.efi", "ipxe-arm64.efi"]) {
|
||||||
const src = `${config.tftpDir}/${name}`;
|
const src = `${config.tftpDir}/${name}`;
|
||||||
@@ -261,6 +391,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
|
|||||||
role: msg.role as import("@lab/shared").Role,
|
role: msg.role as import("@lab/shared").Role,
|
||||||
os: msg.os as import("@lab/shared").OsId,
|
os: msg.os as import("@lab/shared").OsId,
|
||||||
queued_at: new Date().toISOString(),
|
queued_at: new Date().toISOString(),
|
||||||
|
...(msg.vyos ? { vyos: msg.vyos } : {}),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
return { status: "ok", data: { mac: msg.mac, hostname: msg.hostname } };
|
return { status: "ok", data: { mac: msg.mac, hostname: msg.hostname } };
|
||||||
|
|||||||
@@ -5,8 +5,8 @@
|
|||||||
// /api/discover - receive hardware discovery reports from PXE-booted machines
|
// /api/discover - receive hardware discovery reports from PXE-booted machines
|
||||||
|
|
||||||
import type { FastifyInstance } from "fastify";
|
import type { FastifyInstance } from "fastify";
|
||||||
import type { HardwareInfo, InstalledInfo, Role } from "@lab/shared";
|
import type { HardwareInfo, InstalledInfo, Role, VyosInstallSpec } from "@lab/shared";
|
||||||
import { isValidOsId, SUPPORTED_ROLES } from "@lab/shared";
|
import { isValidOsId, SUPPORTED_ROLES, SUPPORTED_OS } from "@lab/shared";
|
||||||
import type { StateManager } from "../services/state.js";
|
import type { StateManager } from "../services/state.js";
|
||||||
import { logger } from "../services/logger.js";
|
import { logger } from "../services/logger.js";
|
||||||
import { triggerPostProvisionK3s } from "../services/post-provision.js";
|
import { triggerPostProvisionK3s } from "../services/post-provision.js";
|
||||||
@@ -15,6 +15,13 @@ import type { ProgressEvent } from "../services/progress-events.js";
|
|||||||
import type { InstallLogBuffer } from "../services/install-log.js";
|
import type { InstallLogBuffer } from "../services/install-log.js";
|
||||||
import type { SyslogListener } from "../services/syslog-listener.js";
|
import type { SyslogListener } from "../services/syslog-listener.js";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Seconds after dispatch with zero progress before a machine is called stalled.
|
||||||
|
* Generous: the slowest legitimate gap is fetching a ~600MB VyOS squashfs over
|
||||||
|
* HTTP before the hook can report anything.
|
||||||
|
*/
|
||||||
|
const STALL_THRESHOLD_S = 8 * 60;
|
||||||
|
|
||||||
export function registerApiRoutes(
|
export function registerApiRoutes(
|
||||||
app: FastifyInstance,
|
app: FastifyInstance,
|
||||||
state: StateManager,
|
state: StateManager,
|
||||||
@@ -34,9 +41,10 @@ export function registerApiRoutes(
|
|||||||
disk?: string;
|
disk?: string;
|
||||||
role?: string;
|
role?: string;
|
||||||
os?: string;
|
os?: string;
|
||||||
|
vyos?: VyosInstallSpec;
|
||||||
};
|
};
|
||||||
}>("/api/install", async (request, reply) => {
|
}>("/api/install", async (request, reply) => {
|
||||||
const { mac: rawMac, hostname, disk, role, os } = request.body ?? {};
|
const { mac: rawMac, hostname, disk, role, os, vyos } = request.body ?? {};
|
||||||
const mac = (rawMac ?? "").toLowerCase().replace(/-/g, ":");
|
const mac = (rawMac ?? "").toLowerCase().replace(/-/g, ":");
|
||||||
|
|
||||||
if (mac === "") {
|
if (mac === "") {
|
||||||
@@ -50,7 +58,7 @@ export function registerApiRoutes(
|
|||||||
|
|
||||||
const osId = os ?? "fedora-43";
|
const osId = os ?? "fedora-43";
|
||||||
if (!isValidOsId(osId)) {
|
if (!isValidOsId(osId)) {
|
||||||
return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: fedora-43, ubuntu-26.04` });
|
return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: ${SUPPORTED_OS.join(", ")}` });
|
||||||
}
|
}
|
||||||
|
|
||||||
state.update((s) => {
|
state.update((s) => {
|
||||||
@@ -60,6 +68,7 @@ export function registerApiRoutes(
|
|||||||
role: validRole as Role,
|
role: validRole as Role,
|
||||||
os: osId,
|
os: osId,
|
||||||
queued_at: new Date().toISOString(),
|
queued_at: new Date().toISOString(),
|
||||||
|
...(vyos ? { vyos } : {}),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -158,11 +167,17 @@ export function registerApiRoutes(
|
|||||||
};
|
};
|
||||||
s.installed[mac] = installedInfo;
|
s.installed[mac] = installedInfo;
|
||||||
|
|
||||||
const admin = installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "lab" : "root";
|
// VyOS: the only login user is "vyos", and a router never runs k3s —
|
||||||
|
// without this guard a non-vanilla role + recorded IP would trigger
|
||||||
|
// the k3s post-provision against a VyOS box.
|
||||||
|
const isVyos = (installedInfo.os ?? "").startsWith("vyos");
|
||||||
|
const admin = isVyos
|
||||||
|
? "vyos"
|
||||||
|
: installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "lab" : "root";
|
||||||
console.log(`\n \x1b[0;32m\x1b[1m ssh ${admin}@${ip}\x1b[0m\n`); // eslint-disable-line no-console
|
console.log(`\n \x1b[0;32m\x1b[1m ssh ${admin}@${ip}\x1b[0m\n`); // eslint-disable-line no-console
|
||||||
|
|
||||||
// Auto-install k3s for non-vanilla roles
|
// Auto-install k3s for non-vanilla roles
|
||||||
if (installedInfo.role !== "vanilla" && ip !== "") {
|
if (!isVyos && installedInfo.role !== "vanilla" && ip !== "") {
|
||||||
void triggerPostProvisionK3s(installedInfo.hostname, ip, installedInfo.role, admin, mac);
|
void triggerPostProvisionK3s(installedInfo.hostname, ip, installedInfo.role, admin, mac);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -434,6 +449,15 @@ export function registerApiRoutes(
|
|||||||
const installedEntry = currentState.installed[mac];
|
const installedEntry = currentState.installed[mac];
|
||||||
|
|
||||||
if (queueEntry) {
|
if (queueEntry) {
|
||||||
|
// A machine that was handed an install script but has reported nothing
|
||||||
|
// since is wedged BEFORE the installer environment came up — a bad
|
||||||
|
// kernel/initrd, no network in the initramfs, or the wrong NIC picked.
|
||||||
|
// Surfacing it here is what makes that diagnosable without a console.
|
||||||
|
const since = queueEntry.progress_at ?? queueEntry.dispatched_at;
|
||||||
|
const stalledForS = since !== undefined && queueEntry.progress === undefined
|
||||||
|
? Math.floor((Date.now() - new Date(since).getTime()) / 1000)
|
||||||
|
: 0;
|
||||||
|
|
||||||
return reply.send({
|
return reply.send({
|
||||||
mac,
|
mac,
|
||||||
hostname: queueEntry.hostname,
|
hostname: queueEntry.hostname,
|
||||||
@@ -441,6 +465,9 @@ export function registerApiRoutes(
|
|||||||
progress: queueEntry.progress ?? "queued",
|
progress: queueEntry.progress ?? "queued",
|
||||||
progress_detail: queueEntry.progress_detail ?? "",
|
progress_detail: queueEntry.progress_detail ?? "",
|
||||||
progress_at: queueEntry.progress_at ?? queueEntry.queued_at,
|
progress_at: queueEntry.progress_at ?? queueEntry.queued_at,
|
||||||
|
dispatched_at: queueEntry.dispatched_at,
|
||||||
|
stalled_for_s: stalledForS,
|
||||||
|
stalled: stalledForS > STALL_THRESHOLD_S,
|
||||||
role: queueEntry.role,
|
role: queueEntry.role,
|
||||||
os: queueEntry.os,
|
os: queueEntry.os,
|
||||||
stages: queueEntry.log ?? [],
|
stages: queueEntry.log ?? [],
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import {
|
|||||||
renderLocalBootIpxe,
|
renderLocalBootIpxe,
|
||||||
} from "../templates/boot.ipxe.js";
|
} from "../templates/boot.ipxe.js";
|
||||||
import { renderUbuntuInstallIpxe } from "../templates/ubuntu-boot.ipxe.js";
|
import { renderUbuntuInstallIpxe } from "../templates/ubuntu-boot.ipxe.js";
|
||||||
|
import { renderVyosInstallIpxe } from "../templates/vyos-boot.ipxe.js";
|
||||||
import { renderDebugKickstart } from "../templates/debug.ks.js";
|
import { renderDebugKickstart } from "../templates/debug.ks.js";
|
||||||
import { logger } from "../services/logger.js";
|
import { logger } from "../services/logger.js";
|
||||||
|
|
||||||
@@ -99,8 +100,22 @@ echo "==============================="
|
|||||||
const os = queueEntry.os ?? "fedora-43";
|
const os = queueEntry.os ?? "fedora-43";
|
||||||
logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os})`);
|
logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os})`);
|
||||||
|
|
||||||
|
// Stamp the handoff so a machine that boots the installer but never
|
||||||
|
// reports can be spotted without a console.
|
||||||
|
state.update((s) => {
|
||||||
|
const entry = s.install_queue[mac];
|
||||||
|
if (entry) entry.dispatched_at = new Date().toISOString();
|
||||||
|
});
|
||||||
|
|
||||||
let script: string;
|
let script: string;
|
||||||
if (os.startsWith("ubuntu")) {
|
if (os.startsWith("vyos")) {
|
||||||
|
script = renderVyosInstallIpxe({
|
||||||
|
mac,
|
||||||
|
hostname,
|
||||||
|
serverIp: config.serverIp,
|
||||||
|
httpPort: config.httpPort,
|
||||||
|
});
|
||||||
|
} else if (os.startsWith("ubuntu")) {
|
||||||
script = renderUbuntuInstallIpxe({
|
script = renderUbuntuInstallIpxe({
|
||||||
mac,
|
mac,
|
||||||
hostname,
|
hostname,
|
||||||
|
|||||||
71
bastion/src/bastion/src/routes/vyos.ts
Normal file
71
bastion/src/bastion/src/routes/vyos.ts
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
// VyOS network install routes.
|
||||||
|
//
|
||||||
|
// VyOS has no unattended installer, so the automation is injected via
|
||||||
|
// live-config's `hooks` component: the iPXE script passes
|
||||||
|
// live-config.hooks=<.../vyos/autoinstall.sh>, live-config wgets it and runs it
|
||||||
|
// as root, and that script fetches and executes the generated install driver.
|
||||||
|
|
||||||
|
import type { FastifyInstance } from "fastify";
|
||||||
|
import type { BastionConfig } from "@lab/shared";
|
||||||
|
import type { StateManager } from "../services/state.js";
|
||||||
|
import { buildVyosConfigSpec } from "../templates/vyos-config-spec.js";
|
||||||
|
import { renderVyosInstallPy } from "../templates/vyos-install.py.js";
|
||||||
|
import { logger } from "../services/logger.js";
|
||||||
|
|
||||||
|
function normalizeMac(value: string | undefined): string {
|
||||||
|
return (value ?? "").toLowerCase().replace(/-/g, ":");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function registerVyosRoutes(
|
||||||
|
app: FastifyInstance,
|
||||||
|
config: BastionConfig,
|
||||||
|
state: StateManager,
|
||||||
|
): void {
|
||||||
|
// live-config hook. Kept minimal: everything version-specific lives in the
|
||||||
|
// generated Python. wget is guaranteed present -- live-config used it to
|
||||||
|
// fetch this very script.
|
||||||
|
app.get<{ Querystring: { mac?: string } }>("/vyos/autoinstall.sh", async (request, reply) => {
|
||||||
|
const mac = normalizeMac(request.query.mac);
|
||||||
|
const base = `http://${config.serverIp}:${config.httpPort}`;
|
||||||
|
|
||||||
|
logger.info(`VYOS AUTOINSTALL HOOK served to ${mac || "unknown MAC"}`);
|
||||||
|
|
||||||
|
const script = `#!/bin/sh
|
||||||
|
# Lab PXE Bastion -- VyOS unattended install hook (run by live-config as root)
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
wget -q "${base}/vyos/install.py?mac=${mac}" -O /tmp/vyos-install.py
|
||||||
|
exec python3 /tmp/vyos-install.py
|
||||||
|
`;
|
||||||
|
return reply.type("text/plain").send(script);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Per-MAC install driver, with the machine's config spec baked in.
|
||||||
|
app.get<{ Querystring: { mac?: string } }>("/vyos/install.py", async (request, reply) => {
|
||||||
|
const mac = normalizeMac(request.query.mac);
|
||||||
|
const queueEntry = state.load().install_queue[mac];
|
||||||
|
|
||||||
|
const spec = buildVyosConfigSpec({
|
||||||
|
hostname: queueEntry?.hostname ?? "vyos",
|
||||||
|
spec: queueEntry?.vyos,
|
||||||
|
defaultPassword: config.vyosDefaultPassword,
|
||||||
|
sshKeys: config.sshKeys,
|
||||||
|
disk: queueEntry?.disk,
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
`VYOS INSTALL DRIVER served to ${mac} (${spec.hostname}, ` +
|
||||||
|
`${spec.sets.length} config ops, disk="${spec.disk || "auto"}")`,
|
||||||
|
);
|
||||||
|
|
||||||
|
const script = renderVyosInstallPy({
|
||||||
|
spec,
|
||||||
|
mac,
|
||||||
|
serverIp: config.serverIp,
|
||||||
|
httpPort: config.httpPort,
|
||||||
|
role: queueEntry?.role ?? "vanilla",
|
||||||
|
});
|
||||||
|
|
||||||
|
return reply.type("text/plain").send(script);
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -12,6 +12,7 @@ import { registerDispatchRoutes } from "./routes/dispatch.js";
|
|||||||
import { registerKickstartRoutes } from "./routes/kickstart.js";
|
import { registerKickstartRoutes } from "./routes/kickstart.js";
|
||||||
import { registerApiRoutes } from "./routes/api.js";
|
import { registerApiRoutes } from "./routes/api.js";
|
||||||
import { registerAsahiRoutes } from "./routes/asahi.js";
|
import { registerAsahiRoutes } from "./routes/asahi.js";
|
||||||
|
import { registerVyosRoutes } from "./routes/vyos.js";
|
||||||
|
|
||||||
|
|
||||||
export function createApp(config: BastionConfig): { app: ReturnType<typeof Fastify>; state: StateManager; installLog: InstallLogBuffer; syslog: SyslogListener } {
|
export function createApp(config: BastionConfig): { app: ReturnType<typeof Fastify>; state: StateManager; installLog: InstallLogBuffer; syslog: SyslogListener } {
|
||||||
@@ -47,6 +48,7 @@ export function createApp(config: BastionConfig): { app: ReturnType<typeof Fasti
|
|||||||
registerKickstartRoutes(app, config, state, syslog);
|
registerKickstartRoutes(app, config, state, syslog);
|
||||||
registerApiRoutes(app, state, installLog, syslog);
|
registerApiRoutes(app, state, installLog, syslog);
|
||||||
registerAsahiRoutes(app, config);
|
registerAsahiRoutes(app, config);
|
||||||
|
registerVyosRoutes(app, config, state);
|
||||||
// boot.iso is generated at startup and served as a static file from httpDir
|
// boot.iso is generated at startup and served as a static file from httpDir
|
||||||
// (static serving supports HTTP Range requests, required by JetKVM streaming)
|
// (static serving supports HTTP Range requests, required by JetKVM streaming)
|
||||||
|
|
||||||
|
|||||||
@@ -40,6 +40,11 @@ export function renderInstallKickstart(params: InstallKickstartParams): string {
|
|||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
const hasLonghorn = role === "worker";
|
const hasLonghorn = role === "worker";
|
||||||
const hasRancher = role === "infra";
|
const hasRancher = role === "infra";
|
||||||
|
// k8s roles get a dedicated 120G image-store LV. 2026-08 incident: the old
|
||||||
|
// 20G LV idled at 85% used, so a single ~5G image pull tripped imagefs
|
||||||
|
// eviction. Must be sized here — longhorn's --grow consumes all remaining
|
||||||
|
// VG space, making post-install lvextend impossible on worker nodes.
|
||||||
|
const hasRancherLv = role === "infra" || role === "worker";
|
||||||
const isVanilla = role === "vanilla";
|
const isVanilla = role === "vanilla";
|
||||||
|
|
||||||
// -- Auth section --
|
// -- Auth section --
|
||||||
@@ -113,9 +118,9 @@ done
|
|||||||
? `logvol /var/lib/longhorn --vgname=${vg} --name=longhorn --fstype=xfs --grow --size=1`
|
? `logvol /var/lib/longhorn --vgname=${vg} --name=longhorn --fstype=xfs --grow --size=1`
|
||||||
: "";
|
: "";
|
||||||
|
|
||||||
// -- Rancher LV for fresh install (infra role) --
|
// -- Rancher LV for fresh install (k8s roles: worker + infra) --
|
||||||
const rancherFreshLine = hasRancher
|
const rancherFreshLine = hasRancherLv
|
||||||
? `logvol /var/lib/rancher --vgname=${vg} --name=rancher --fstype=xfs --size=20480`
|
? `logvol /var/lib/rancher --vgname=${vg} --name=rancher --fstype=xfs --size=122880`
|
||||||
: "";
|
: "";
|
||||||
|
|
||||||
return `# Lab Bastion -- Fedora ${fedoraVersion} server install
|
return `# Lab Bastion -- Fedora ${fedoraVersion} server install
|
||||||
|
|||||||
53
bastion/src/bastion/src/templates/vyos-boot.ipxe.ts
Normal file
53
bastion/src/bastion/src/templates/vyos-boot.ipxe.ts
Normal file
@@ -0,0 +1,53 @@
|
|||||||
|
// iPXE boot script template for VyOS network install.
|
||||||
|
//
|
||||||
|
// VyOS ships no unattended installer: `install image` is unconditionally
|
||||||
|
// interactive (image_installer.py's install action takes no arguments, and
|
||||||
|
// --no-prompt is wired only to `add`). So PXE boots the *live* system and the
|
||||||
|
// automation is injected through live-config's `hooks` component, which fetches
|
||||||
|
// a script over HTTP and runs it as root late in live boot.
|
||||||
|
//
|
||||||
|
// Unlike the Fedora/Ubuntu paths this boots a live image rather than an
|
||||||
|
// installer, so there is no kickstart/autoinstall equivalent — see
|
||||||
|
// routes/vyos.ts for the hook that actually drives the install.
|
||||||
|
|
||||||
|
export function renderVyosInstallIpxe(params: {
|
||||||
|
mac: string;
|
||||||
|
hostname: string;
|
||||||
|
serverIp: string;
|
||||||
|
httpPort: number;
|
||||||
|
}): string {
|
||||||
|
const base = `http://${params.serverIp}:${params.httpPort}`;
|
||||||
|
|
||||||
|
// Pin the boot NIC by MAC. live-boot otherwise scans for the first
|
||||||
|
// *connected* interface, and on a multi-NIC box that race is lost by
|
||||||
|
// whichever port negotiates slowest: on the Protectli VP2440 the SFP+
|
||||||
|
// pair links first, so live-boot picked the fiber ports (which have no
|
||||||
|
// DHCP), burned 15s per port, and gave up with "Unable to find a live
|
||||||
|
// file system on the network" -- while the copper port that actually PXE
|
||||||
|
// booted came up at 4.6s and was never tried.
|
||||||
|
//
|
||||||
|
// live-boot's Device_from_bootif() strips the "01-" and matches the MAC
|
||||||
|
// against /sys/class/net/*. params.mac is the dispatch key, i.e. exactly
|
||||||
|
// the NIC that PXE booted -- more reliable than iPXE's ${net0} on a box
|
||||||
|
// where the booting NIC may not be net0.
|
||||||
|
const bootif = `01-${params.mac.toLowerCase().replace(/:/g, "-")}`;
|
||||||
|
|
||||||
|
// Deliberately NOT passing `nonetworking` (present in VyOS's own PXE docs):
|
||||||
|
// live-config's hook component needs networking up to fetch the hook over
|
||||||
|
// HTTP. Also no `console=ttyS0` — on hardware without a physical UART that
|
||||||
|
// costs 30s at every systemd boot phase.
|
||||||
|
return `#!ipxe
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo =============================================
|
||||||
|
echo Lab PXE Bastion - INSTALLING VyOS
|
||||||
|
echo Target: ${params.hostname}
|
||||||
|
echo MAC: ${params.mac}
|
||||||
|
echo =============================================
|
||||||
|
echo
|
||||||
|
|
||||||
|
kernel ${base}/vyos-vmlinuz boot=live nopersistence noautologin BOOTIF=${bootif} fetch=${base}/vyos-filesystem.squashfs live-config.hooks=${base}/vyos/autoinstall.sh?mac=${params.mac}
|
||||||
|
initrd ${base}/vyos-initrd
|
||||||
|
boot
|
||||||
|
`;
|
||||||
|
}
|
||||||
208
bastion/src/bastion/src/templates/vyos-config-spec.ts
Normal file
208
bastion/src/bastion/src/templates/vyos-config-spec.ts
Normal file
@@ -0,0 +1,208 @@
|
|||||||
|
// Builds the VyOS configuration spec applied by the autoinstall hook.
|
||||||
|
//
|
||||||
|
// We deliberately do NOT emit a config.boot file as text. A config.boot carries a
|
||||||
|
// `vyos-config-version` trailer; without a trailer matching the running image,
|
||||||
|
// VyOS runs its migration scripts from version 0 on first boot. Instead the hook
|
||||||
|
// loads the image's own /opt/vyatta/etc/config.boot.default through vyos.configtree
|
||||||
|
// and applies these set operations on top, so syntax and version trailer always
|
||||||
|
// match the exact image being installed.
|
||||||
|
|
||||||
|
import type { VyosInstallSpec } from "@lab/shared";
|
||||||
|
|
||||||
|
export interface VyosSetOp {
|
||||||
|
path: string[];
|
||||||
|
value?: string;
|
||||||
|
/** false appends to a multi-value node (e.g. bond members) instead of replacing. */
|
||||||
|
replace?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VyosConfigSpec {
|
||||||
|
hostname: string;
|
||||||
|
/** "" means accept the installer default (the running image's version string). */
|
||||||
|
imageName: string;
|
||||||
|
password: string;
|
||||||
|
console: "K" | "S";
|
||||||
|
/** Target disk name (e.g. "nvme0n1"); "" accepts the installer's first-disk default. */
|
||||||
|
disk: string;
|
||||||
|
/**
|
||||||
|
* IP the driver should report in the "complete" callback ("ready at <ip>" —
|
||||||
|
* the exact format routes/api.ts parses installed.ip from). The mgmt
|
||||||
|
* address when static; "" means detect the live DHCP address at runtime.
|
||||||
|
*/
|
||||||
|
reportAddress: string;
|
||||||
|
/** Whether to accept RAID-1 when the installer finds more than one disk. */
|
||||||
|
raid: boolean;
|
||||||
|
/** Overwrite the installed config.boot with the generated one on reinstall. */
|
||||||
|
freshConfig: boolean;
|
||||||
|
sets: VyosSetOp[];
|
||||||
|
/** Paths that are VyOS tag nodes — must be marked as such in the ConfigTree. */
|
||||||
|
tags: string[][];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalise a target disk to the form the installer expects.
|
||||||
|
*
|
||||||
|
* find_disks() enumerates via `lsblk -Jbp` (-p = full paths), so its valid
|
||||||
|
* responses are "/dev/mmcblk0"-style. A bare "mmcblk0" is rejected by
|
||||||
|
* ask_input()'s valid_responses check and re-prompts forever.
|
||||||
|
*/
|
||||||
|
function normalizeDiskPath(value: string | undefined): string {
|
||||||
|
const raw = (value ?? "").trim();
|
||||||
|
if (raw === "") return "";
|
||||||
|
return raw.startsWith("/dev/") ? raw : `/dev/${raw}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildVyosConfigSpec(params: {
|
||||||
|
hostname: string;
|
||||||
|
spec?: VyosInstallSpec | undefined;
|
||||||
|
defaultPassword: string;
|
||||||
|
sshKeys?: string[] | undefined;
|
||||||
|
disk?: string | undefined;
|
||||||
|
}): VyosConfigSpec {
|
||||||
|
const spec = params.spec ?? {};
|
||||||
|
const mgmt = spec.mgmtInterface ?? "eth0";
|
||||||
|
const mgmtAddress = spec.mgmtAddress ?? "dhcp";
|
||||||
|
const bondMembers = spec.bondMembers ?? [];
|
||||||
|
const vlans = spec.vlans ?? [];
|
||||||
|
|
||||||
|
const sets: VyosSetOp[] = [];
|
||||||
|
const tags: string[][] = [
|
||||||
|
["interfaces", "ethernet"],
|
||||||
|
["system", "login", "user"],
|
||||||
|
];
|
||||||
|
|
||||||
|
const hwIds = spec.hwIds ?? {};
|
||||||
|
const pinHwId = (iface: string): void => {
|
||||||
|
const mac = hwIds[iface];
|
||||||
|
if (mac !== undefined && mac !== "") {
|
||||||
|
sets.push({ path: ["interfaces", "ethernet", iface, "hw-id"], value: mac });
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
sets.push({ path: ["system", "host-name"], value: params.hostname });
|
||||||
|
|
||||||
|
// Management interface — the NIC that PXE booted, left untagged and unbonded.
|
||||||
|
sets.push({ path: ["interfaces", "ethernet", mgmt, "address"], value: mgmtAddress });
|
||||||
|
pinHwId(mgmt);
|
||||||
|
|
||||||
|
// Tagged management VLAN on the PXE port. Emitted regardless of bonding, so
|
||||||
|
// the box stays reachable on the management VLAN while still booting untagged
|
||||||
|
// on whichever VLAN the bastion's proxy DHCP serves.
|
||||||
|
const mgmtVlan = spec.mgmtVlan;
|
||||||
|
if (mgmtVlan !== undefined) {
|
||||||
|
tags.push(["interfaces", "ethernet", mgmt, "vif"]);
|
||||||
|
const vif = ["interfaces", "ethernet", mgmt, "vif", String(mgmtVlan.id)];
|
||||||
|
sets.push({ path: [...vif, "address"], value: mgmtVlan.address });
|
||||||
|
if (mgmtVlan.description !== undefined && mgmtVlan.description !== "") {
|
||||||
|
sets.push({ path: [...vif, "description"], value: mgmtVlan.description });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// LACP bond. Members must exclude the PXE NIC; firmware PXE cannot run over LACP.
|
||||||
|
const bonded = bondMembers.length > 0;
|
||||||
|
if (bonded) {
|
||||||
|
tags.push(["interfaces", "bonding"]);
|
||||||
|
sets.push({ path: ["interfaces", "bonding", "bond0", "mode"], value: "802.3ad" });
|
||||||
|
sets.push({ path: ["interfaces", "bonding", "bond0", "hash-policy"], value: "layer2+3" });
|
||||||
|
for (const member of bondMembers) {
|
||||||
|
sets.push({
|
||||||
|
path: ["interfaces", "bonding", "bond0", "member", "interface"],
|
||||||
|
value: member,
|
||||||
|
replace: false,
|
||||||
|
});
|
||||||
|
pinHwId(member);
|
||||||
|
}
|
||||||
|
// Address on the trunk's native/untagged VLAN.
|
||||||
|
if (spec.bondAddress !== undefined && spec.bondAddress !== "") {
|
||||||
|
sets.push({ path: ["interfaces", "bonding", "bond0", "address"], value: spec.bondAddress });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// VRRP groups accumulate here; emitted (plus a sync group) after the VLANs.
|
||||||
|
// interface accepts dotted vifs (constraint regex `[0-9]+(.\d+)?`), address
|
||||||
|
// is a tag node (the VIP is the tag value itself), vrid range is 1-255.
|
||||||
|
const vrrpGroups: Array<{ name: string; iface: string; vrid: number; vip: string }> = [];
|
||||||
|
if (bonded && spec.bondVrrp !== undefined && spec.bondVrrp !== "") {
|
||||||
|
// vrid 1 for the untagged group: the native VLAN is never a vif, so this
|
||||||
|
// cannot collide with a vlan-id-derived vrid.
|
||||||
|
vrrpGroups.push({ name: "native", iface: "bond0", vrid: 1, vip: spec.bondVrrp });
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tagged VLAN sub-interfaces hang off the bond when there is one, else off mgmt.
|
||||||
|
const parent = bonded
|
||||||
|
? ["interfaces", "bonding", "bond0"]
|
||||||
|
: ["interfaces", "ethernet", mgmt];
|
||||||
|
if (vlans.length > 0) {
|
||||||
|
tags.push([...parent, "vif"]);
|
||||||
|
const parentName = bonded ? "bond0" : mgmt;
|
||||||
|
for (const vlan of vlans) {
|
||||||
|
const vif = [...parent, "vif", String(vlan.id)];
|
||||||
|
sets.push({ path: [...vif, "address"], value: vlan.address });
|
||||||
|
if (vlan.description !== undefined && vlan.description !== "") {
|
||||||
|
sets.push({ path: [...vif, "description"], value: vlan.description });
|
||||||
|
}
|
||||||
|
if (vlan.vrrp !== undefined && vlan.vrrp !== "") {
|
||||||
|
vrrpGroups.push({
|
||||||
|
name: `vlan${vlan.id}`,
|
||||||
|
iface: `${parentName}.${vlan.id}`,
|
||||||
|
vrid: vlan.id,
|
||||||
|
vip: vlan.vrrp,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Emit VRRP groups plus one sync group so all VLANs fail over together —
|
||||||
|
// without it a single-link event could split mastership across the pair.
|
||||||
|
if (vrrpGroups.length > 0) {
|
||||||
|
tags.push(["high-availability", "vrrp", "group"]);
|
||||||
|
tags.push(["high-availability", "vrrp", "sync-group"]);
|
||||||
|
const priority = String(spec.vrrpPriority ?? 100);
|
||||||
|
for (const g of vrrpGroups) {
|
||||||
|
const base = ["high-availability", "vrrp", "group", g.name];
|
||||||
|
sets.push({ path: [...base, "interface"], value: g.iface });
|
||||||
|
sets.push({ path: [...base, "vrid"], value: String(g.vrid) });
|
||||||
|
sets.push({ path: [...base, "priority"], value: priority });
|
||||||
|
// address is a tag node: the VIP is the path's final segment, no value.
|
||||||
|
sets.push({ path: [...base, "address", g.vip] });
|
||||||
|
tags.push([...base, "address"]);
|
||||||
|
sets.push({
|
||||||
|
path: ["high-availability", "vrrp", "sync-group", "MAIN", "member"],
|
||||||
|
value: g.name,
|
||||||
|
replace: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sets.push({ path: ["service", "ssh", "port"], value: "22" });
|
||||||
|
|
||||||
|
const sshKeys = params.sshKeys ?? [];
|
||||||
|
if (sshKeys.length > 0) {
|
||||||
|
tags.push(["system", "login", "user", "vyos", "authentication", "public-keys"]);
|
||||||
|
sshKeys.forEach((entry, index) => {
|
||||||
|
const parts = entry.trim().split(/\s+/);
|
||||||
|
const type = parts[0] ?? "";
|
||||||
|
const key = parts[1] ?? "";
|
||||||
|
if (!type.startsWith("ssh-") && !type.startsWith("ecdsa-")) return;
|
||||||
|
if (!key) return;
|
||||||
|
const name = parts[2] ?? `lab-key-${index}`;
|
||||||
|
const base = ["system", "login", "user", "vyos", "authentication", "public-keys", name];
|
||||||
|
sets.push({ path: [...base, "type"], value: type });
|
||||||
|
sets.push({ path: [...base, "key"], value: key });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
hostname: params.hostname,
|
||||||
|
imageName: "",
|
||||||
|
password: spec.password ?? params.defaultPassword,
|
||||||
|
console: "K",
|
||||||
|
disk: normalizeDiskPath(params.disk),
|
||||||
|
// Static mgmt address wins; under DHCP the driver detects the live IP.
|
||||||
|
reportAddress: mgmtAddress.includes("/") ? (mgmtAddress.split("/")[0] ?? "") : "",
|
||||||
|
raid: false,
|
||||||
|
freshConfig: spec.freshConfig ?? false,
|
||||||
|
sets,
|
||||||
|
tags,
|
||||||
|
};
|
||||||
|
}
|
||||||
514
bastion/src/bastion/src/templates/vyos-install.py.ts
Normal file
514
bastion/src/bastion/src/templates/vyos-install.py.ts
Normal file
@@ -0,0 +1,514 @@
|
|||||||
|
// Renders the Python program that performs the unattended VyOS install.
|
||||||
|
//
|
||||||
|
// It runs as root inside the live system, fetched and executed by live-config's
|
||||||
|
// `hooks` component (see vyos-boot.ipxe.ts). It does three things:
|
||||||
|
// 1. builds config.boot from the image's own default via vyos.configtree
|
||||||
|
// 2. drives the interactive `install image` through a pty
|
||||||
|
// 3. reports progress back to the bastion, then reboots
|
||||||
|
//
|
||||||
|
// A pty is used rather than piping stdin because the installer reads the
|
||||||
|
// password through getpass(), which opens /dev/tty directly and would ignore a
|
||||||
|
// pipe. Prompts are matched by text rather than replayed positionally: the
|
||||||
|
// installer skips the boot-config question when it finds a previous
|
||||||
|
// installation, so a fixed answer sequence desyncs on reinstall.
|
||||||
|
|
||||||
|
import type { VyosConfigSpec } from "./vyos-config-spec.js";
|
||||||
|
|
||||||
|
export function renderVyosInstallPy(params: {
|
||||||
|
spec: VyosConfigSpec;
|
||||||
|
mac: string;
|
||||||
|
serverIp: string;
|
||||||
|
httpPort: number;
|
||||||
|
role: string;
|
||||||
|
}): string {
|
||||||
|
// Base64 so arbitrary values (passwords, descriptions, SSH keys) can never
|
||||||
|
// terminate the Python string literal that carries them.
|
||||||
|
const specB64 = Buffer.from(JSON.stringify(params.spec), "utf-8").toString("base64");
|
||||||
|
|
||||||
|
return `#!/usr/bin/env python3
|
||||||
|
"""Unattended VyOS install driver -- generated by the lab PXE bastion."""
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import pty
|
||||||
|
import re
|
||||||
|
import select
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
SPEC = json.loads(base64.b64decode("${specB64}").decode("utf-8"))
|
||||||
|
BASTION = "http://${params.serverIp}:${params.httpPort}"
|
||||||
|
MAC = "${params.mac}"
|
||||||
|
ROLE = ${JSON.stringify(params.role ?? "vanilla")}
|
||||||
|
|
||||||
|
INSTALLER = "/usr/libexec/vyos/op_mode/image_installer.py"
|
||||||
|
CONFIG_DIR = "/opt/vyatta/etc/config"
|
||||||
|
# The installer copies the rootfs from the boot MEDIUM path -- which only a
|
||||||
|
# CD/USB boot provides. With fetch= (HTTP netboot) nothing is mounted there
|
||||||
|
# (verified in VM: Errno 2), so the squashfs must be linked or re-fetched into
|
||||||
|
# place before 'install image' runs.
|
||||||
|
ROOTFS_EXPECTED = "/usr/lib/live/mount/medium/live/filesystem.squashfs"
|
||||||
|
SQUASHFS_URL = "http://${params.serverIp}:${params.httpPort}/vyos-filesystem.squashfs"
|
||||||
|
# The live-config hook runs BEFORE vyos-router creates the /opt/vyatta compat
|
||||||
|
# path, so the squashfs's own location must be tried too (verified in VM: only
|
||||||
|
# /usr/share/vyos/config.boot.default exists at hook time).
|
||||||
|
DEFAULT_CONFIG_CANDIDATES = [
|
||||||
|
"/opt/vyatta/etc/config.boot.default",
|
||||||
|
"/usr/share/vyos/config.boot.default",
|
||||||
|
]
|
||||||
|
STALL_TIMEOUT = 900 # seconds without installer output before giving up
|
||||||
|
|
||||||
|
|
||||||
|
def detect_ip():
|
||||||
|
"""Best-effort local IP as seen on the route toward the bastion.
|
||||||
|
|
||||||
|
Matches Fedora's semantics (IP captured during install): under DHCP the
|
||||||
|
installed system will renew on the same NIC/subnet the live env used.
|
||||||
|
"""
|
||||||
|
import socket
|
||||||
|
try:
|
||||||
|
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||||
|
s.connect(("${params.serverIp}", ${params.httpPort}))
|
||||||
|
ip = s.getsockname()[0]
|
||||||
|
s.close()
|
||||||
|
return ip
|
||||||
|
except Exception:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
class LogStreamer:
|
||||||
|
"""Stream install output to the bastion's /api/log so 'labctl provision
|
||||||
|
logs -f' works live for VyOS, like Anaconda's syslog does for Fedora.
|
||||||
|
|
||||||
|
Strictly best-effort: a failed POST drops the batch and must never stall
|
||||||
|
the pty read loop or fail the install.
|
||||||
|
"""
|
||||||
|
|
||||||
|
ANSI = re.compile(rb"\\x1b\\[[0-9;?]*[a-zA-Z]|\\x1b[=>]|\\r")
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.partial = b""
|
||||||
|
self.pending = []
|
||||||
|
self.last_flush = time.time()
|
||||||
|
|
||||||
|
def feed(self, chunk):
|
||||||
|
"""Raw pty bytes: split into lines, strip ANSI noise, queue."""
|
||||||
|
self.partial += chunk
|
||||||
|
while b"\\n" in self.partial:
|
||||||
|
raw, self.partial = self.partial.split(b"\\n", 1)
|
||||||
|
text = self.ANSI.sub(b"", raw).decode("utf-8", "replace").rstrip()
|
||||||
|
if text:
|
||||||
|
self.pending.append(text)
|
||||||
|
self.maybe_flush()
|
||||||
|
|
||||||
|
def line(self, text):
|
||||||
|
"""A driver-originated message (already a clean string)."""
|
||||||
|
self.pending.append(text)
|
||||||
|
self.maybe_flush()
|
||||||
|
|
||||||
|
def maybe_flush(self):
|
||||||
|
if len(self.pending) >= 20 or (self.pending and time.time() - self.last_flush >= 2):
|
||||||
|
self.flush()
|
||||||
|
|
||||||
|
def flush(self):
|
||||||
|
if not self.pending:
|
||||||
|
return
|
||||||
|
batch, self.pending = self.pending[:200], self.pending[200:]
|
||||||
|
self.last_flush = time.time()
|
||||||
|
try:
|
||||||
|
body = json.dumps({"mac": MAC, "lines": batch}).encode()
|
||||||
|
req = urllib.request.Request(
|
||||||
|
BASTION + "/api/log",
|
||||||
|
data=body,
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
)
|
||||||
|
urllib.request.urlopen(req, timeout=5).read()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
STREAM = LogStreamer()
|
||||||
|
|
||||||
|
|
||||||
|
def say(msg):
|
||||||
|
"""Print locally and stream to the bastion log buffer."""
|
||||||
|
print(msg)
|
||||||
|
STREAM.line(str(msg))
|
||||||
|
|
||||||
|
|
||||||
|
def report(stage, detail=""):
|
||||||
|
"""Best-effort progress callback; never fatal."""
|
||||||
|
STREAM.flush()
|
||||||
|
try:
|
||||||
|
body = json.dumps({"mac": MAC, "stage": stage, "detail": detail}).encode()
|
||||||
|
req = urllib.request.Request(
|
||||||
|
BASTION + "/api/progress",
|
||||||
|
data=body,
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
)
|
||||||
|
urllib.request.urlopen(req, timeout=5).read()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def build_config():
|
||||||
|
"""Apply our set operations onto the image's own default config.
|
||||||
|
|
||||||
|
Using config.boot.default as the base keeps the vyos-config-version trailer
|
||||||
|
consistent with the running image, so first boot does not run migrations.
|
||||||
|
"""
|
||||||
|
from vyos.configtree import ConfigTree
|
||||||
|
|
||||||
|
default_config = next(
|
||||||
|
(p for p in DEFAULT_CONFIG_CANDIDATES if os.path.exists(p)), None)
|
||||||
|
if default_config is None:
|
||||||
|
raise FileNotFoundError(
|
||||||
|
"no config.boot.default found (tried %s)" % ", ".join(DEFAULT_CONFIG_CANDIDATES))
|
||||||
|
say("base config: %s" % default_config)
|
||||||
|
|
||||||
|
with open(default_config) as handle:
|
||||||
|
config = ConfigTree(handle.read())
|
||||||
|
|
||||||
|
for op in SPEC["sets"]:
|
||||||
|
replace = op.get("replace", True)
|
||||||
|
if "value" in op and op["value"] is not None:
|
||||||
|
config.set(op["path"], value=op["value"], replace=replace)
|
||||||
|
else:
|
||||||
|
config.set(op["path"])
|
||||||
|
|
||||||
|
# Tag nodes must be marked after the nodes exist, as the installer itself does.
|
||||||
|
for tag in SPEC["tags"]:
|
||||||
|
try:
|
||||||
|
config.set_tag(tag)
|
||||||
|
except Exception as err:
|
||||||
|
say("warning: set_tag %s failed: %s" % (tag, err))
|
||||||
|
|
||||||
|
os.makedirs(CONFIG_DIR, exist_ok=True)
|
||||||
|
target = os.path.join(CONFIG_DIR, "config.boot")
|
||||||
|
|
||||||
|
# Re-attach the vyos-config-version footer: ConfigTree.to_string() emits
|
||||||
|
# only the config body, and a config without the footer is treated as
|
||||||
|
# ancient -- the boot migrator then runs every migration over it and (as
|
||||||
|
# observed in the VM test) crashes in system/31-to-32. Building the footer
|
||||||
|
# from the running system pins it to the exact image being installed.
|
||||||
|
body = config.to_string()
|
||||||
|
try:
|
||||||
|
from vyos.component_version import version_info_from_system
|
||||||
|
info = version_info_from_system()
|
||||||
|
info.update_config_body(body)
|
||||||
|
info.write(target)
|
||||||
|
say("wrote %s (footer: %s)" % (target, info.release))
|
||||||
|
except Exception as err:
|
||||||
|
say("warning: version footer failed (%s); writing bare config" % err)
|
||||||
|
with open(target, "w") as handle:
|
||||||
|
handle.write(body)
|
||||||
|
return target
|
||||||
|
|
||||||
|
|
||||||
|
def find_live_squashfs():
|
||||||
|
"""Locate the squashfs live-boot fetched, without walking into the mounted
|
||||||
|
rootfs or overlay (each would mean traversing the entire OS tree)."""
|
||||||
|
explicit = [
|
||||||
|
"/run/live/medium/live/filesystem.squashfs",
|
||||||
|
"/lib/live/mount/medium/live/filesystem.squashfs",
|
||||||
|
]
|
||||||
|
for path in explicit:
|
||||||
|
if os.path.isfile(path) and os.path.getsize(path) > 0:
|
||||||
|
return path
|
||||||
|
for root in ("/run/live", "/lib/live/mount", "/usr/lib/live/mount"):
|
||||||
|
for dirpath, dirs, files in os.walk(root):
|
||||||
|
depth = dirpath.count(os.sep) - root.count(os.sep)
|
||||||
|
dirs[:] = [d for d in dirs
|
||||||
|
if d not in ("rootfs", "overlay")
|
||||||
|
and not d.endswith(".squashfs")
|
||||||
|
and depth < 3]
|
||||||
|
if "filesystem.squashfs" in files:
|
||||||
|
path = os.path.join(dirpath, "filesystem.squashfs")
|
||||||
|
if os.path.isfile(path) and os.path.getsize(path) > 0:
|
||||||
|
return path
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_rootfs():
|
||||||
|
"""Make FILE_ROOTFS_SRC exist so the installer can copy the system image."""
|
||||||
|
if os.path.isfile(ROOTFS_EXPECTED) and os.path.getsize(ROOTFS_EXPECTED) > 0:
|
||||||
|
return
|
||||||
|
src = find_live_squashfs()
|
||||||
|
if src is None:
|
||||||
|
say("squashfs not in live mounts; re-fetching %s" % SQUASHFS_URL)
|
||||||
|
src = "/tmp/filesystem.squashfs"
|
||||||
|
urllib.request.urlretrieve(SQUASHFS_URL, src)
|
||||||
|
os.makedirs(os.path.dirname(ROOTFS_EXPECTED), exist_ok=True)
|
||||||
|
if os.path.lexists(ROOTFS_EXPECTED):
|
||||||
|
os.remove(ROOTFS_EXPECTED)
|
||||||
|
os.symlink(src, ROOTFS_EXPECTED)
|
||||||
|
say("rootfs source: %s -> %s" % (ROOTFS_EXPECTED, src))
|
||||||
|
|
||||||
|
|
||||||
|
def build_rules():
|
||||||
|
"""Prompt -> response table for the interactive installer."""
|
||||||
|
password = SPEC["password"].encode() + b"\\n"
|
||||||
|
image_name = SPEC["imageName"].encode() + b"\\n"
|
||||||
|
disk = SPEC["disk"].encode() + b"\\n"
|
||||||
|
console = SPEC["console"].encode() + b"\\n"
|
||||||
|
raid = (b"yes\\n" if SPEC["raid"] else b"no\\n")
|
||||||
|
|
||||||
|
return [
|
||||||
|
(re.compile(rb"Would you like to continue\\?"), b"yes\\n"),
|
||||||
|
(re.compile(rb"What would you like to name this image\\?"), image_name),
|
||||||
|
(re.compile(rb"Please confirm password for the .vyos. user:"), password),
|
||||||
|
(re.compile(rb"Please enter a password for the .vyos. user:"), password),
|
||||||
|
(re.compile(rb"What console should be used by default"), console),
|
||||||
|
# Three RAID variants: "configure RAID-1 mirroring?", "...on them?",
|
||||||
|
# and "choose two disks for RAID-1 mirroring?" -- all default to YES,
|
||||||
|
# so a missed one both hangs the install and risks an unwanted mirror.
|
||||||
|
(re.compile(rb"Would you like to [^?]*RAID-1 mirroring"), raid),
|
||||||
|
(re.compile(rb"Installation will delete all data on (?:the drive|both drives)\\. Continue\\?"), b"yes\\n"),
|
||||||
|
(re.compile(rb"Which one should be used for installation\\?"), disk),
|
||||||
|
(re.compile(rb"Would you like to use all the free space on the drive\\?"), b"yes\\n"),
|
||||||
|
(re.compile(rb"Which file would you like as boot config\\?"), b"1\\n"),
|
||||||
|
# Reinstall path only (search_previous_installation): carrying the old
|
||||||
|
# /config and SSH host keys forward is VyOS's "reinstall without losing
|
||||||
|
# data". Always yes -- freshConfig replaces config.boot afterwards, so
|
||||||
|
# answering no here would also discard non-config data under /config.
|
||||||
|
(re.compile(rb"Would you like to copy data to the new image\\?"), b"yes\\n"),
|
||||||
|
(re.compile(rb"Would you like to copy the encrypted config to the new image\\?"), b"yes\\n"),
|
||||||
|
# More than one previous image found -- take the first offered.
|
||||||
|
(re.compile(rb"From which image would you like to save config information\\?"), b"1\\n"),
|
||||||
|
(re.compile(rb"From which image would you like to copy the encrypted config\\?"), b"1\\n"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def run_installer():
|
||||||
|
"""Drive image_installer.py over a pty, answering prompts as they appear."""
|
||||||
|
rules = build_rules()
|
||||||
|
master, slave = pty.openpty()
|
||||||
|
|
||||||
|
proc = subprocess.Popen(
|
||||||
|
[INSTALLER, "--action", "install"],
|
||||||
|
stdin=slave,
|
||||||
|
stdout=slave,
|
||||||
|
stderr=slave,
|
||||||
|
close_fds=True,
|
||||||
|
preexec_fn=os.setsid,
|
||||||
|
)
|
||||||
|
os.close(slave)
|
||||||
|
|
||||||
|
buf = b""
|
||||||
|
transcript = b"" # rolling tail of everything the installer printed
|
||||||
|
last_output = time.time()
|
||||||
|
|
||||||
|
while True:
|
||||||
|
ready, _, _ = select.select([master], [], [], 1.0)
|
||||||
|
|
||||||
|
if ready:
|
||||||
|
try:
|
||||||
|
chunk = os.read(master, 4096)
|
||||||
|
except OSError:
|
||||||
|
break
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
|
||||||
|
sys.stdout.buffer.write(chunk)
|
||||||
|
sys.stdout.buffer.flush()
|
||||||
|
buf += chunk
|
||||||
|
transcript = (transcript + chunk)[-8000:]
|
||||||
|
STREAM.feed(chunk)
|
||||||
|
last_output = time.time()
|
||||||
|
|
||||||
|
# Answer every prompt currently in the buffer, earliest first, so
|
||||||
|
# ordering is preserved even when the installer skips questions --
|
||||||
|
# and so a single chunk carrying two prompts gets both answers.
|
||||||
|
while True:
|
||||||
|
best = None
|
||||||
|
for pattern, response in rules:
|
||||||
|
found = pattern.search(buf)
|
||||||
|
if found and (best is None or found.start() < best[0].start()):
|
||||||
|
best = (found, response)
|
||||||
|
if best is None:
|
||||||
|
break
|
||||||
|
found, response = best
|
||||||
|
os.write(master, response)
|
||||||
|
transcript = (transcript + b"\\n>>> answered: " + response)[-8000:]
|
||||||
|
STREAM.line(">>> answered: " + response.decode("utf-8", "replace").strip())
|
||||||
|
buf = buf[found.end():]
|
||||||
|
|
||||||
|
# Bound memory if the installer emits a lot without prompting.
|
||||||
|
if len(buf) > 65536:
|
||||||
|
buf = buf[-8192:]
|
||||||
|
|
||||||
|
elif proc.poll() is not None:
|
||||||
|
break
|
||||||
|
|
||||||
|
STREAM.maybe_flush()
|
||||||
|
|
||||||
|
if time.time() - last_output > STALL_TIMEOUT:
|
||||||
|
proc.kill()
|
||||||
|
raise SystemExit("installer produced no output for %ds" % STALL_TIMEOUT)
|
||||||
|
|
||||||
|
os.close(master)
|
||||||
|
return proc.wait(), transcript.decode("utf-8", "replace")
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_network_boot_first():
|
||||||
|
"""Keep network boot first so the bastion intercepts every reboot.
|
||||||
|
|
||||||
|
Port of the Fedora kickstart's %post efibootmgr step (install.ks.ts) --
|
||||||
|
what makes reprovision-by-reboot work. Best-effort: skipped on BIOS boots
|
||||||
|
or when efibootmgr is absent. Runs from the live env after the installer;
|
||||||
|
efibootmgr edits NVRAM, not the disk, so installer cleanup is irrelevant.
|
||||||
|
"""
|
||||||
|
import shutil
|
||||||
|
if not os.path.isdir("/sys/firmware/efi") or shutil.which("efibootmgr") is None:
|
||||||
|
say("boot order: skipped (BIOS boot or efibootmgr missing)")
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
out = subprocess.run(["efibootmgr"], capture_output=True, text=True, timeout=30).stdout
|
||||||
|
order = []
|
||||||
|
network_entry = None
|
||||||
|
for line in out.splitlines():
|
||||||
|
m = re.match(r"^BootOrder:\\s*(.*)$", line)
|
||||||
|
if m:
|
||||||
|
order = [x.strip() for x in m.group(1).split(",") if x.strip()]
|
||||||
|
continue
|
||||||
|
m = re.match(r"^Boot([0-9A-Fa-f]{4})\\*?\\s+(.*)$", line)
|
||||||
|
if m and network_entry is None:
|
||||||
|
if re.search(r"network|pxe|ipv4|ipv6|http", m.group(2), re.IGNORECASE):
|
||||||
|
network_entry = m.group(1).upper()
|
||||||
|
if network_entry is None or not order:
|
||||||
|
say("boot order: no network boot entry found; leaving as is")
|
||||||
|
return
|
||||||
|
new_order = [network_entry] + [x for x in order if x.upper() != network_entry]
|
||||||
|
if [x.upper() for x in order] == [x.upper() for x in new_order]:
|
||||||
|
say("boot order: network entry Boot%s already first" % network_entry)
|
||||||
|
return
|
||||||
|
subprocess.run(["efibootmgr", "-o", ",".join(new_order)],
|
||||||
|
capture_output=True, timeout=30)
|
||||||
|
say("boot order: moved network entry Boot%s first" % network_entry)
|
||||||
|
except Exception as err:
|
||||||
|
say("warning: boot order adjustment failed: %s" % err)
|
||||||
|
|
||||||
|
|
||||||
|
def with_target_mounted(fn):
|
||||||
|
"""Mount the installed root partition, call fn(rw_dir), always unmount.
|
||||||
|
|
||||||
|
The installer has unmounted and cleaned the target by the time this runs,
|
||||||
|
so the block device is free. The partition holding boot/<image>/rw is the
|
||||||
|
VyOS root; the glob also yields the installed image's rw dir directly.
|
||||||
|
"""
|
||||||
|
import glob
|
||||||
|
disk = SPEC["disk"]
|
||||||
|
if not disk:
|
||||||
|
# No pinned disk (installer picked the default) -- enumerate all disks.
|
||||||
|
candidates = ["/dev/" + b for b in os.listdir("/sys/block")
|
||||||
|
if not b.startswith(("loop", "ram", "zram", "sr"))]
|
||||||
|
else:
|
||||||
|
candidates = [disk]
|
||||||
|
|
||||||
|
mnt = "/mnt/lab-target"
|
||||||
|
os.makedirs(mnt, exist_ok=True)
|
||||||
|
for dev in candidates:
|
||||||
|
name = os.path.basename(dev)
|
||||||
|
parts = sorted(p for p in os.listdir("/sys/block/%s" % name)
|
||||||
|
if p.startswith(name)) if os.path.isdir("/sys/block/%s" % name) else []
|
||||||
|
for part in parts:
|
||||||
|
pdev = "/dev/" + part
|
||||||
|
if subprocess.run(["mount", pdev, mnt], capture_output=True).returncode != 0:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
rw_dirs = glob.glob(os.path.join(mnt, "boot", "*", "rw"))
|
||||||
|
if rw_dirs:
|
||||||
|
fn(rw_dirs[0])
|
||||||
|
return True
|
||||||
|
finally:
|
||||||
|
subprocess.run(["umount", mnt], capture_output=True)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def post_install_target_steps():
|
||||||
|
"""Metadata + optional fresh-config overwrite inside the installed image."""
|
||||||
|
def apply(rw_dir):
|
||||||
|
config_dir = os.path.join(rw_dir, "opt/vyatta/etc/config")
|
||||||
|
os.makedirs(config_dir, exist_ok=True)
|
||||||
|
|
||||||
|
# /config/lab-provisioned -- survives VyOS image upgrades. Mirrors the
|
||||||
|
# Fedora kickstart's /etc/lab-provisioned.
|
||||||
|
try:
|
||||||
|
with open(os.path.join(config_dir, "lab-provisioned"), "w") as handle:
|
||||||
|
handle.write("hostname=%s\\n" % SPEC["hostname"])
|
||||||
|
handle.write("role=%s\\n" % ROLE)
|
||||||
|
handle.write("provisioned=%s\\n" % time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
|
||||||
|
handle.write("bastion=%s\\n" % BASTION)
|
||||||
|
say("wrote /config/lab-provisioned")
|
||||||
|
except Exception as err:
|
||||||
|
say("warning: lab-provisioned metadata failed: %s" % err)
|
||||||
|
|
||||||
|
# freshConfig: make the bastion-generated config win over the previous
|
||||||
|
# installation's carried-forward config. Explicit intent -- failure is
|
||||||
|
# fatal (raised out of with_target_mounted).
|
||||||
|
if SPEC.get("freshConfig"):
|
||||||
|
import shutil
|
||||||
|
shutil.copyfile(os.path.join(CONFIG_DIR, "config.boot"),
|
||||||
|
os.path.join(config_dir, "config.boot"))
|
||||||
|
say("freshConfig: replaced installed config.boot with generated config")
|
||||||
|
|
||||||
|
mounted = with_target_mounted(apply)
|
||||||
|
if not mounted:
|
||||||
|
if SPEC.get("freshConfig"):
|
||||||
|
raise RuntimeError("freshConfig requested but installed root partition not found")
|
||||||
|
say("warning: installed root partition not found; skipping metadata")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
report("vyos-install", "building config.boot")
|
||||||
|
try:
|
||||||
|
build_config()
|
||||||
|
except Exception as err:
|
||||||
|
report("error", "config generation failed: %s" % err)
|
||||||
|
raise
|
||||||
|
|
||||||
|
report("vyos-install", "staging rootfs for installer")
|
||||||
|
try:
|
||||||
|
ensure_rootfs()
|
||||||
|
except Exception as err:
|
||||||
|
report("error", "rootfs staging failed: %s" % err)
|
||||||
|
raise
|
||||||
|
|
||||||
|
report("vyos-install", "running install image")
|
||||||
|
code, transcript = run_installer()
|
||||||
|
|
||||||
|
if code != 0:
|
||||||
|
# Surface the installer's last words in bastion progress -- the console
|
||||||
|
# they were printed on is usually invisible during unattended installs.
|
||||||
|
report("error", "install image exited %d | tail: %s" % (code, transcript[-4000:]))
|
||||||
|
raise SystemExit(code)
|
||||||
|
|
||||||
|
report("post-install", "boot order + metadata")
|
||||||
|
ensure_network_boot_first()
|
||||||
|
try:
|
||||||
|
post_install_target_steps()
|
||||||
|
except Exception as err:
|
||||||
|
report("error", "post-install target steps failed: %s" % err)
|
||||||
|
raise
|
||||||
|
|
||||||
|
# "complete" is the stage the bastion uses to move a machine out of the
|
||||||
|
# install queue into installed state, and "ready at <ip>" is the exact
|
||||||
|
# detail format it parses installed.ip from -- see routes/api.ts.
|
||||||
|
ip = SPEC.get("reportAddress") or detect_ip()
|
||||||
|
report("complete", "ready at %s" % ip if ip else "VyOS installed, rebooting")
|
||||||
|
os.system("sync")
|
||||||
|
# --force: this driver is a child of live-config.service, whose start job is
|
||||||
|
# still running -- a normal reboot deadlocks waiting for it (verified in VM:
|
||||||
|
# shutdown blocked >1min on "start job is running for live-config"). The
|
||||||
|
# installer has already unmounted and cleaned the target, so an immediate
|
||||||
|
# reboot is safe.
|
||||||
|
os.system("systemctl reboot --force")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
`;
|
||||||
|
}
|
||||||
@@ -22,6 +22,8 @@ function createTestConfig(testDir: string): BastionConfig {
|
|||||||
dhcpRangeEnd: "",
|
dhcpRangeEnd: "",
|
||||||
ubuntuVersion: "26.04",
|
ubuntuVersion: "26.04",
|
||||||
ubuntuMirror: "https://releases.ubuntu.com/26.04",
|
ubuntuMirror: "https://releases.ubuntu.com/26.04",
|
||||||
|
vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso",
|
||||||
|
vyosDefaultPassword: "vyos",
|
||||||
iface: "eth0",
|
iface: "eth0",
|
||||||
serverIp: "10.0.0.1",
|
serverIp: "10.0.0.1",
|
||||||
network: "10.0.0.0",
|
network: "10.0.0.0",
|
||||||
|
|||||||
@@ -96,9 +96,9 @@ describe("renderInstallKickstart", () => {
|
|||||||
expect(ks).toContain("/api/progress");
|
expect(ks).toContain("/api/progress");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("infra role has /var/lib/rancher partition", () => {
|
it("infra role has 120G /var/lib/rancher partition", () => {
|
||||||
const ks = renderInstallKickstart(baseParams({ role: "infra" }));
|
const ks = renderInstallKickstart(baseParams({ role: "infra" }));
|
||||||
expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=20480");
|
expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=122880");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("infra role has k3s install", () => {
|
it("infra role has k3s install", () => {
|
||||||
@@ -106,10 +106,14 @@ describe("renderInstallKickstart", () => {
|
|||||||
expect(ks).toContain("curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true sh -");
|
expect(ks).toContain("curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true sh -");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("worker role does NOT have /var/lib/rancher partition in fresh install", () => {
|
it("worker role has 120G /var/lib/rancher partition (imageFs must be sized before longhorn --grow)", () => {
|
||||||
const ks = renderInstallKickstart(baseParams({ role: "worker" }));
|
const ks = renderInstallKickstart(baseParams({ role: "worker" }));
|
||||||
// Worker should not have the fresh-install rancher partition line
|
expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=122880");
|
||||||
expect(ks).not.toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=20480");
|
});
|
||||||
|
|
||||||
|
it("vanilla role does NOT have /var/lib/rancher partition in fresh install", () => {
|
||||||
|
const ks = renderInstallKickstart(baseParams({ role: "vanilla" }));
|
||||||
|
expect(ks).not.toContain("--name=rancher --fstype=xfs");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("worker role does NOT have k3s install", () => {
|
it("worker role does NOT have k3s install", () => {
|
||||||
|
|||||||
548
bastion/src/bastion/tests/vyos.test.ts
Normal file
548
bastion/src/bastion/tests/vyos.test.ts
Normal file
@@ -0,0 +1,548 @@
|
|||||||
|
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||||
|
import { mkdirSync, rmSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import type { BastionConfig } from "@lab/shared";
|
||||||
|
import type { FastifyInstance } from "fastify";
|
||||||
|
import { createApp } from "../src/server.js";
|
||||||
|
import type { StateManager } from "../src/services/state.js";
|
||||||
|
import { buildVyosConfigSpec } from "../src/templates/vyos-config-spec.js";
|
||||||
|
import { renderVyosInstallPy } from "../src/templates/vyos-install.py.js";
|
||||||
|
|
||||||
|
function createTestConfig(testDir: string): BastionConfig {
|
||||||
|
return {
|
||||||
|
fedoraVersion: "43",
|
||||||
|
arch: "x86_64",
|
||||||
|
httpPort: 0,
|
||||||
|
timezone: "Europe/London",
|
||||||
|
locale: "en_GB.UTF-8",
|
||||||
|
bastionDir: testDir,
|
||||||
|
domain: "test.local",
|
||||||
|
dhcpMode: "proxy",
|
||||||
|
dhcpRangeStart: "",
|
||||||
|
dhcpRangeEnd: "",
|
||||||
|
ubuntuVersion: "26.04",
|
||||||
|
ubuntuMirror: "https://releases.ubuntu.com/26.04",
|
||||||
|
vyosIsoUrl: "https://example.invalid/vyos.iso",
|
||||||
|
vyosDefaultPassword: "test-pw",
|
||||||
|
iface: "eth0",
|
||||||
|
serverIp: "10.0.0.1",
|
||||||
|
network: "10.0.0.0",
|
||||||
|
gateway: "10.0.0.1",
|
||||||
|
sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST lab@test"],
|
||||||
|
adminUser: "testadmin",
|
||||||
|
syslogPort: 15515,
|
||||||
|
skipDnsmasq: true,
|
||||||
|
skipArtifacts: true,
|
||||||
|
fedoraMirror: "https://example.invalid/fedora",
|
||||||
|
tftpDir: join(testDir, "tftp"),
|
||||||
|
httpDir: join(testDir, "http"),
|
||||||
|
stateFile: join(testDir, "state.json"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Pull the base64 spec back out of the generated Python driver. */
|
||||||
|
function decodeSpecFrom(python: string): Record<string, unknown> {
|
||||||
|
const match = /base64\.b64decode\("([^"]+)"\)/.exec(python);
|
||||||
|
if (!match?.[1]) throw new Error("no base64 spec found in generated driver");
|
||||||
|
return JSON.parse(Buffer.from(match[1], "base64").toString("utf-8"));
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("vyos config spec", () => {
|
||||||
|
it("puts VLANs on the bond when members are given", () => {
|
||||||
|
const spec = buildVyosConfigSpec({
|
||||||
|
hostname: "fw1",
|
||||||
|
defaultPassword: "pw",
|
||||||
|
spec: {
|
||||||
|
mgmtInterface: "eth0",
|
||||||
|
mgmtAddress: "10.0.8.2/24",
|
||||||
|
bondMembers: ["eth2", "eth3"],
|
||||||
|
vlans: [{ id: 10, address: "10.0.10.1/24", description: "k8s" }],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const paths = spec.sets.map((s) => s.path.join(" "));
|
||||||
|
expect(paths).toContain("interfaces bonding bond0 mode");
|
||||||
|
expect(paths).toContain("interfaces bonding bond0 vif 10 address");
|
||||||
|
// VLANs must hang off the bond, not the management NIC.
|
||||||
|
expect(paths).not.toContain("interfaces ethernet eth0 vif 10 address");
|
||||||
|
|
||||||
|
// Bond members are a multi-value node — appending, not replacing, is what
|
||||||
|
// keeps the second member from overwriting the first.
|
||||||
|
const members = spec.sets.filter(
|
||||||
|
(s) => s.path.join(" ") === "interfaces bonding bond0 member interface",
|
||||||
|
);
|
||||||
|
expect(members.map((m) => m.value)).toEqual(["eth2", "eth3"]);
|
||||||
|
expect(members.every((m) => m.replace === false)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back to VLANs on the management NIC when unbonded", () => {
|
||||||
|
const spec = buildVyosConfigSpec({
|
||||||
|
hostname: "fw2",
|
||||||
|
defaultPassword: "pw",
|
||||||
|
spec: { mgmtInterface: "eth1", vlans: [{ id: 20, address: "10.0.20.1/24" }] },
|
||||||
|
});
|
||||||
|
|
||||||
|
const paths = spec.sets.map((s) => s.path.join(" "));
|
||||||
|
expect(paths).toContain("interfaces ethernet eth1 vif 20 address");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("normalises the target disk to a full /dev path", () => {
|
||||||
|
// find_disks() enumerates with `lsblk -Jbp`, so valid responses are full
|
||||||
|
// paths; a bare name fails valid_responses and re-prompts forever.
|
||||||
|
expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw", disk: "/dev/mmcblk0" }).disk)
|
||||||
|
.toBe("/dev/mmcblk0");
|
||||||
|
expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw", disk: "mmcblk0" }).disk)
|
||||||
|
.toBe("/dev/mmcblk0");
|
||||||
|
expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw" }).disk).toBe("");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("defaults to dhcp on eth0 and never opts into RAID", () => {
|
||||||
|
const spec = buildVyosConfigSpec({ hostname: "fw4", defaultPassword: "pw" });
|
||||||
|
const address = spec.sets.find(
|
||||||
|
(s) => s.path.join(" ") === "interfaces ethernet eth0 address",
|
||||||
|
);
|
||||||
|
expect(address?.value).toBe("dhcp");
|
||||||
|
// The installer's RAID prompt defaults to yes; a second disk must not
|
||||||
|
// silently produce a mirror.
|
||||||
|
expect(spec.raid).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("vyos routes", () => {
|
||||||
|
let testDir: string;
|
||||||
|
let app: FastifyInstance;
|
||||||
|
let state: StateManager;
|
||||||
|
const mac = "aa:bb:cc:11:22:33";
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
testDir = join(tmpdir(), `bastion-vyos-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
||||||
|
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||||
|
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||||
|
|
||||||
|
const result = createApp(createTestConfig(testDir));
|
||||||
|
app = result.app;
|
||||||
|
state = result.state;
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await app.close();
|
||||||
|
rmSync(testDir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("dispatches a queued vyos machine to the live-boot script", async () => {
|
||||||
|
state.update((s) => {
|
||||||
|
s.install_queue[mac] = {
|
||||||
|
hostname: "fw1",
|
||||||
|
disk: "/dev/nvme0n1",
|
||||||
|
role: "worker",
|
||||||
|
os: "vyos-rolling",
|
||||||
|
queued_at: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const response = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}` });
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
expect(response.body).toContain("/vyos-vmlinuz");
|
||||||
|
expect(response.body).toContain("fetch=http://10.0.0.1:0/vyos-filesystem.squashfs");
|
||||||
|
expect(response.body).toContain(`live-config.hooks=http://10.0.0.1:0/vyos/autoinstall.sh?mac=${mac}`);
|
||||||
|
|
||||||
|
// `nonetworking` appears in VyOS's own PXE docs but breaks the hook fetch,
|
||||||
|
// and console=ttyS0 costs 30s per systemd phase on boards with no UART.
|
||||||
|
expect(response.body).not.toContain("nonetworking");
|
||||||
|
expect(response.body).not.toContain("console=ttyS0");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("serves a hook that fetches and executes the install driver", async () => {
|
||||||
|
const response = await app.inject({ method: "GET", url: `/vyos/autoinstall.sh?mac=${mac}` });
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
expect(response.body).toContain(`/vyos/install.py?mac=${mac}`);
|
||||||
|
expect(response.body).toContain("python3 /tmp/vyos-install.py");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("bakes the machine's config into the generated install driver", async () => {
|
||||||
|
state.update((s) => {
|
||||||
|
s.install_queue[mac] = {
|
||||||
|
hostname: "fw1",
|
||||||
|
disk: "/dev/nvme0n1",
|
||||||
|
role: "worker",
|
||||||
|
os: "vyos-rolling",
|
||||||
|
queued_at: new Date().toISOString(),
|
||||||
|
vyos: {
|
||||||
|
mgmtInterface: "eth0",
|
||||||
|
mgmtAddress: "10.0.8.2/24",
|
||||||
|
bondMembers: ["eth2", "eth3"],
|
||||||
|
vlans: [{ id: 10, address: "10.0.10.1/24" }],
|
||||||
|
password: "s3cret",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const response = await app.inject({ method: "GET", url: `/vyos/install.py?mac=${mac}` });
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
// Builds config from the image's own default so the vyos-config-version
|
||||||
|
// trailer matches and first boot skips migrations.
|
||||||
|
expect(response.body).toContain("/opt/vyatta/etc/config.boot.default");
|
||||||
|
expect(response.body).toContain("/usr/libexec/vyos/op_mode/image_installer.py");
|
||||||
|
// "complete" is what moves the machine out of the install queue.
|
||||||
|
expect(response.body).toContain('report("complete"');
|
||||||
|
|
||||||
|
const spec = decodeSpecFrom(response.body);
|
||||||
|
expect(spec["hostname"]).toBe("fw1");
|
||||||
|
expect(spec["password"]).toBe("s3cret");
|
||||||
|
expect(spec["disk"]).toBe("/dev/nvme0n1");
|
||||||
|
|
||||||
|
const paths = (spec["sets"] as Array<{ path: string[] }>).map((s) => s.path.join(" "));
|
||||||
|
expect(paths).toContain("interfaces bonding bond0 vif 10 address");
|
||||||
|
expect(paths).toContain("system host-name");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back to the bastion default password when none is set", async () => {
|
||||||
|
state.update((s) => {
|
||||||
|
s.install_queue[mac] = {
|
||||||
|
hostname: "fw9",
|
||||||
|
disk: "",
|
||||||
|
role: "worker",
|
||||||
|
os: "vyos-rolling",
|
||||||
|
queued_at: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const response = await app.inject({ method: "GET", url: `/vyos/install.py?mac=${mac}` });
|
||||||
|
const spec = decodeSpecFrom(response.body);
|
||||||
|
expect(spec["password"]).toBe("test-pw");
|
||||||
|
// Empty disk means "accept the installer's first-disk default".
|
||||||
|
expect(spec["disk"]).toBe("");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("vyos hw-id pinning", () => {
|
||||||
|
it("emits hw-id for the mgmt interface and each bond member", () => {
|
||||||
|
// Discovery sees enp2s0/enp1s0f0np0 under Fedora, but VyOS enumerates its
|
||||||
|
// own eth<N>. Pinning by MAC is what makes the mapping deterministic.
|
||||||
|
const spec = buildVyosConfigSpec({
|
||||||
|
hostname: "fw1",
|
||||||
|
defaultPassword: "pw",
|
||||||
|
spec: {
|
||||||
|
mgmtInterface: "eth2",
|
||||||
|
bondMembers: ["eth0", "eth1"],
|
||||||
|
hwIds: {
|
||||||
|
eth2: "64:62:66:25:96:47",
|
||||||
|
eth0: "64:62:66:25:96:45",
|
||||||
|
eth1: "64:62:66:25:96:46",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const hw = spec.sets.filter((s) => s.path[s.path.length - 1] === "hw-id");
|
||||||
|
expect(hw.map((s) => [s.path[2], s.value])).toEqual([
|
||||||
|
["eth2", "64:62:66:25:96:47"],
|
||||||
|
["eth0", "64:62:66:25:96:45"],
|
||||||
|
["eth1", "64:62:66:25:96:46"],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("omits hw-id entirely when no mapping is given", () => {
|
||||||
|
const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" });
|
||||||
|
expect(spec.sets.some((s) => s.path.includes("hw-id"))).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("vyos management VLAN", () => {
|
||||||
|
it("puts the mgmt VLAN on the PXE port while the bond carries routed VLANs", () => {
|
||||||
|
// Trunked PXE port: boots untagged on the VLAN the bastion serves, stays
|
||||||
|
// reachable on the tagged management VLAN.
|
||||||
|
const spec = buildVyosConfigSpec({
|
||||||
|
hostname: "vyos001",
|
||||||
|
defaultPassword: "pw",
|
||||||
|
spec: {
|
||||||
|
mgmtInterface: "eth2",
|
||||||
|
mgmtAddress: "dhcp",
|
||||||
|
mgmtVlan: { id: 3, address: "192.168.3.4/24", description: "kvm" },
|
||||||
|
bondMembers: ["eth0", "eth1"],
|
||||||
|
vlans: [{ id: 2, address: "192.168.8.2/23" }],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const paths = spec.sets.map((s) => s.path.join(" "));
|
||||||
|
expect(paths).toContain("interfaces ethernet eth2 vif 3 address");
|
||||||
|
expect(paths).toContain("interfaces bonding bond0 vif 2 address");
|
||||||
|
// The mgmt VLAN must not land on the bond.
|
||||||
|
expect(paths).not.toContain("interfaces bonding bond0 vif 3 address");
|
||||||
|
expect(spec.tags.map((t) => t.join(" "))).toContain("interfaces ethernet eth2 vif");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("vyos VRRP HA", () => {
|
||||||
|
const haSpec = {
|
||||||
|
mgmtInterface: "eth2",
|
||||||
|
mgmtAddress: "dhcp",
|
||||||
|
bondMembers: ["eth0", "eth1"],
|
||||||
|
bondAddress: "192.168.1.252/24",
|
||||||
|
bondVrrp: "192.168.1.254/24",
|
||||||
|
vrrpPriority: 200,
|
||||||
|
vlans: [
|
||||||
|
{ id: 3, address: "192.168.3.4/24", vrrp: "192.168.3.254/24" },
|
||||||
|
{ id: 200, address: "192.168.2.252/24" }, // no VIP on this one
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
it("emits a vrrp group per VIP with vrid = VLAN id and dotted vif interface", () => {
|
||||||
|
const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw", spec: haSpec });
|
||||||
|
const paths = spec.sets.map((s) => `${s.path.join(" ")}${s.value !== undefined ? "=" + s.value : ""}`);
|
||||||
|
|
||||||
|
expect(paths).toContain("interfaces bonding bond0 address=192.168.1.252/24");
|
||||||
|
// untagged bond group: vrid 1, interface bond0 itself
|
||||||
|
expect(paths).toContain("high-availability vrrp group native interface=bond0");
|
||||||
|
expect(paths).toContain("high-availability vrrp group native vrid=1");
|
||||||
|
// address is a tag node -- VIP is the final path segment, no value
|
||||||
|
expect(paths).toContain("high-availability vrrp group native address 192.168.1.254/24");
|
||||||
|
// VLAN group: vrid = VLAN id, dotted vif
|
||||||
|
expect(paths).toContain("high-availability vrrp group vlan3 interface=bond0.3");
|
||||||
|
expect(paths).toContain("high-availability vrrp group vlan3 vrid=3");
|
||||||
|
expect(paths).toContain("high-availability vrrp group vlan3 address 192.168.3.254/24");
|
||||||
|
// VLAN without a VIP gets no group
|
||||||
|
expect(paths.some((p) => p.includes("group vlan200"))).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("applies the box-wide priority and one sync group over all groups", () => {
|
||||||
|
const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw", spec: haSpec });
|
||||||
|
const prio = spec.sets.filter((s) => s.path[s.path.length - 1] === "priority"
|
||||||
|
&& s.path[0] === "high-availability");
|
||||||
|
expect(prio).toHaveLength(2);
|
||||||
|
expect(prio.every((s) => s.value === "200")).toBe(true);
|
||||||
|
|
||||||
|
// sync group binds the pair: all groups fail over together
|
||||||
|
const members = spec.sets.filter(
|
||||||
|
(s) => s.path.join(" ") === "high-availability vrrp sync-group MAIN member",
|
||||||
|
);
|
||||||
|
expect(members.map((m) => m.value)).toEqual(["native", "vlan3"]);
|
||||||
|
expect(members.every((m) => m.replace === false)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("emits no high-availability nodes when no VIPs are given", () => {
|
||||||
|
const spec = buildVyosConfigSpec({
|
||||||
|
hostname: "fw1",
|
||||||
|
defaultPassword: "pw",
|
||||||
|
spec: { bondMembers: ["eth0", "eth1"], vlans: [{ id: 3, address: "192.168.3.4/24" }] },
|
||||||
|
});
|
||||||
|
expect(spec.sets.some((s) => s.path[0] === "high-availability")).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("pickLargestInitrd", async () => {
|
||||||
|
const { pickLargestInitrd } = await import("../src/main.js");
|
||||||
|
|
||||||
|
// Verbatim from `xorriso -lsl /live/` on vyos-2026.08.05-0033-rolling.
|
||||||
|
const realListing = `total 8
|
||||||
|
-r--r--r-- 1 0 0 22255 Aug 5 01:33 'filesystem.packages'
|
||||||
|
-r--r--r-- 1 0 0 6 Aug 5 01:33 'filesystem.packages-remove'
|
||||||
|
-r--r--r-- 1 0 0 541192192 Aug 5 01:33 'filesystem.squashfs'
|
||||||
|
-r--r--r-- 1 0 0 50352547 Aug 5 01:33 'initrd.img'
|
||||||
|
-r--r--r-- 1 0 0 50352547 Aug 5 01:33 'initrd.img-6.18.41-vyos'
|
||||||
|
-r--r--r-- 1 0 0 20 Aug 5 01:33 'packages.txt'
|
||||||
|
-r--r--r-- 1 0 0 9135104 Aug 2 19:54 'vmlinuz'
|
||||||
|
-r--r--r-- 1 0 0 9135104 Aug 2 19:54 'vmlinuz-6.18.41-vyos'
|
||||||
|
`;
|
||||||
|
|
||||||
|
it("picks a full-size initrd from a real nightly listing", () => {
|
||||||
|
expect(pickLargestInitrd(realListing)).toEqual({ name: "initrd.img", size: 50352547 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ignores 0-byte decoys and symlinks (which report link size, not target size)", () => {
|
||||||
|
const listing = `total 8
|
||||||
|
-r--r--r-- 1 0 0 0 Aug 5 01:33 'initrd.img'
|
||||||
|
lrwxrwxrwx 1 0 0 24 Aug 5 01:33 'initrd.img-link' -> 'initrd.img-6.18.41-vyos'
|
||||||
|
-r--r--r-- 1 0 0 50352547 Aug 5 01:33 'initrd.img-6.18.41-vyos'
|
||||||
|
`;
|
||||||
|
expect(pickLargestInitrd(listing)).toEqual({ name: "initrd.img-6.18.41-vyos", size: 50352547 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns undefined when only decoys exist", () => {
|
||||||
|
expect(pickLargestInitrd("-r--r--r-- 1 0 0 0 Aug 5 01:33 'initrd.img'\n")).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("vyos fedora-parity features", () => {
|
||||||
|
it("computes reportAddress from a static mgmt address, empty for dhcp", () => {
|
||||||
|
const staticSpec = buildVyosConfigSpec({
|
||||||
|
hostname: "fw1", defaultPassword: "pw",
|
||||||
|
spec: { mgmtAddress: "192.168.8.2/23" },
|
||||||
|
});
|
||||||
|
expect(staticSpec.reportAddress).toBe("192.168.8.2");
|
||||||
|
|
||||||
|
const dhcpSpec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" });
|
||||||
|
expect(dhcpSpec.reportAddress).toBe("");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("defaults freshConfig off (reinstall preserves the on-disk config)", () => {
|
||||||
|
expect(buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" }).freshConfig).toBe(false);
|
||||||
|
expect(buildVyosConfigSpec({
|
||||||
|
hostname: "fw1", defaultPassword: "pw", spec: { freshConfig: true },
|
||||||
|
}).freshConfig).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("driver streams logs to /api/log and reports 'ready at' on completion", async () => {
|
||||||
|
const testDir = join(tmpdir(), `bastion-vyos-parity-${Date.now()}`);
|
||||||
|
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||||
|
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||||
|
const { app: parityApp, state: parityState } = createApp(createTestConfig(testDir));
|
||||||
|
try {
|
||||||
|
parityState.update((s) => {
|
||||||
|
s.install_queue["aa:bb:cc:44:55:66"] = {
|
||||||
|
hostname: "fw9", disk: "/dev/vda", role: "vanilla",
|
||||||
|
os: "vyos-rolling", queued_at: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
const response = await parityApp.inject({
|
||||||
|
method: "GET", url: "/vyos/install.py?mac=aa:bb:cc:44:55:66",
|
||||||
|
});
|
||||||
|
expect(response.body).toContain("/api/log");
|
||||||
|
expect(response.body).toContain('"lines": batch');
|
||||||
|
expect(response.body).toContain('report("complete", "ready at %s"');
|
||||||
|
expect(response.body).toContain("ensure_network_boot_first");
|
||||||
|
expect(response.body).toContain("lab-provisioned");
|
||||||
|
expect(response.body).toContain('ROLE = "vanilla"');
|
||||||
|
} finally {
|
||||||
|
await parityApp.close();
|
||||||
|
rmSync(testDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("complete with 'ready at' records installed.ip for a vyos machine", async () => {
|
||||||
|
const testDir = join(tmpdir(), `bastion-vyos-complete-${Date.now()}`);
|
||||||
|
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||||
|
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||||
|
const { app: cApp, state: cState } = createApp(createTestConfig(testDir));
|
||||||
|
try {
|
||||||
|
const mac2 = "aa:bb:cc:77:88:99";
|
||||||
|
cState.update((s) => {
|
||||||
|
s.install_queue[mac2] = {
|
||||||
|
hostname: "fw1", disk: "/dev/vda", role: "vanilla",
|
||||||
|
os: "vyos-rolling", queued_at: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
const response = await cApp.inject({
|
||||||
|
method: "POST", url: "/api/progress",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({ mac: mac2, stage: "complete", detail: "ready at 192.168.8.2" }),
|
||||||
|
});
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
const installed = cState.load().installed[mac2];
|
||||||
|
expect(installed?.ip).toBe("192.168.8.2");
|
||||||
|
expect(installed?.os).toBe("vyos-rolling");
|
||||||
|
} finally {
|
||||||
|
await cApp.close();
|
||||||
|
rmSync(testDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("vyos installer prompt coverage", () => {
|
||||||
|
// Every interactive prompt image_installer.py can emit, copied verbatim from
|
||||||
|
// the MSG_* constants (including the reinstall-only search_previous_installation
|
||||||
|
// ones). An unanswered prompt does not fail loudly -- the installer simply
|
||||||
|
// blocks on stdin until the driver's stall timeout, which is how the reinstall
|
||||||
|
// path silently hung for 15 minutes in the VM test.
|
||||||
|
const PROMPTS: Record<string, string> = {
|
||||||
|
continue: "Would you like to continue? [y/N] ",
|
||||||
|
imageName: "What would you like to name this image? (Default: 1.5-rolling) ",
|
||||||
|
password: 'Please enter a password for the "vyos" user: ',
|
||||||
|
passwordConfirm: 'Please confirm password for the "vyos" user: ',
|
||||||
|
console: "What console should be used by default? (K: KVM, S: Serial)? (Default: K) ",
|
||||||
|
raidConfigure: "Would you like to configure RAID-1 mirroring? [Y/n] ",
|
||||||
|
raidFoundDisks: "Would you like to configure RAID-1 mirroring on them? [Y/n] ",
|
||||||
|
raidChooseDisks: "Would you like to choose two disks for RAID-1 mirroring? [Y/n] ",
|
||||||
|
diskSelect: "Which one should be used for installation? (Default: /dev/vda) ",
|
||||||
|
diskConfirm: "Installation will delete all data on the drive. Continue? [y/N] ",
|
||||||
|
raidConfirm: "Installation will delete all data on both drives. Continue? [y/N] ",
|
||||||
|
rootSizeAll: "Would you like to use all the free space on the drive? [Y/n] ",
|
||||||
|
bootConfig: "Which file would you like as boot config? ",
|
||||||
|
copyData: "Would you like to copy data to the new image? [Y/n] ",
|
||||||
|
chooseCopyData: "From which image would you like to save config information? ",
|
||||||
|
copyEncData: "Would you like to copy the encrypted config to the new image? [Y/n] ",
|
||||||
|
chooseCopyEncData: "From which image would you like to copy the encrypted config? ",
|
||||||
|
};
|
||||||
|
|
||||||
|
it("answers every installer prompt exactly once", () => {
|
||||||
|
const { execFileSync } = require("node:child_process") as typeof import("node:child_process");
|
||||||
|
const { writeFileSync, unlinkSync, mkdtempSync } = require("node:fs") as typeof import("node:fs");
|
||||||
|
|
||||||
|
// Skip cleanly where python3 is unavailable (same spirit as the
|
||||||
|
// ksvalidator-backed kickstart test).
|
||||||
|
try {
|
||||||
|
execFileSync("python3", ["--version"], { stdio: "pipe" });
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const spec = buildVyosConfigSpec({
|
||||||
|
hostname: "fw1", defaultPassword: "pw", disk: "/dev/vda",
|
||||||
|
});
|
||||||
|
const driver = renderVyosInstallPy({
|
||||||
|
spec, mac: "aa:bb:cc:11:22:33", serverIp: "10.0.0.1", httpPort: 8080, role: "vanilla",
|
||||||
|
});
|
||||||
|
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), "vyos-rules-"));
|
||||||
|
const driverPath = join(dir, "driver.py");
|
||||||
|
const checkPath = join(dir, "check.py");
|
||||||
|
writeFileSync(driverPath, driver);
|
||||||
|
writeFileSync(checkPath, `
|
||||||
|
import importlib.util, json, sys
|
||||||
|
spec = importlib.util.spec_from_file_location("drv", ${JSON.stringify(driverPath)})
|
||||||
|
drv = importlib.util.module_from_spec(spec); spec.loader.exec_module(drv)
|
||||||
|
rules = drv.build_rules()
|
||||||
|
prompts = json.loads(sys.argv[1])
|
||||||
|
out = {}
|
||||||
|
for label, text in prompts.items():
|
||||||
|
out[label] = len([r for p, r in rules if p.search(text.encode())])
|
||||||
|
print(json.dumps(out))
|
||||||
|
`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const stdout = execFileSync("python3", [checkPath, JSON.stringify(PROMPTS)], {
|
||||||
|
encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"],
|
||||||
|
});
|
||||||
|
const counts = JSON.parse(stdout) as Record<string, number>;
|
||||||
|
const unanswered = Object.entries(counts).filter(([, n]) => n !== 1);
|
||||||
|
expect(unanswered).toEqual([]);
|
||||||
|
} finally {
|
||||||
|
try { unlinkSync(driverPath); unlinkSync(checkPath); } catch { /* best effort */ }
|
||||||
|
try { rmSync(dir, { recursive: true, force: true }); } catch { /* best effort */ }
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("vyos boot NIC pinning", () => {
|
||||||
|
it("pins the boot interface by MAC via BOOTIF", async () => {
|
||||||
|
// Without this, live-boot picks the first *connected* NIC. On the VP2440
|
||||||
|
// the SFP+ pair links before the copper PXE port, so live-boot tried the
|
||||||
|
// fiber ports (no DHCP), timed out 15s each, and failed with "Unable to
|
||||||
|
// find a live file system on the network".
|
||||||
|
const testDir = join(tmpdir(), `bastion-vyos-bootif-${Date.now()}`);
|
||||||
|
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||||
|
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||||
|
const { app: a, state: st } = createApp(createTestConfig(testDir));
|
||||||
|
try {
|
||||||
|
const m = "64:62:66:25:96:47";
|
||||||
|
st.update((s) => {
|
||||||
|
s.install_queue[m] = {
|
||||||
|
hostname: "vyos001", disk: "/dev/mmcblk0", role: "vanilla",
|
||||||
|
os: "vyos-rolling", queued_at: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
const res = await a.inject({ method: "GET", url: `/dispatch?mac=${m}` });
|
||||||
|
// live-boot's Device_from_bootif() expects 01-<mac with dashes>
|
||||||
|
expect(res.body).toContain("BOOTIF=01-64-62-66-25-96-47");
|
||||||
|
// and it must be on the kernel line, before fetch= is attempted
|
||||||
|
const kernelLine = res.body.split("\n").find((l) => l.startsWith("kernel "));
|
||||||
|
expect(kernelLine).toContain("BOOTIF=01-64-62-66-25-96-47");
|
||||||
|
expect(kernelLine).toContain("fetch=");
|
||||||
|
} finally {
|
||||||
|
await a.close();
|
||||||
|
rmSync(testDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -90,6 +90,7 @@ export class LabdClient {
|
|||||||
|
|
||||||
async installMachine(opts: {
|
async installMachine(opts: {
|
||||||
mac: string; hostname: string; disk?: string; role?: string; os?: string;
|
mac: string; hostname: string; disk?: string; role?: string; os?: string;
|
||||||
|
vyos?: import("@lab/shared").VyosInstallSpec;
|
||||||
}): Promise<{ status: string; data?: unknown; error?: string }> {
|
}): Promise<{ status: string; data?: unknown; error?: string }> {
|
||||||
return this.request("POST", "/api/machines/install", { body: opts });
|
return this.request("POST", "/api/machines/install", { body: opts });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,30 @@
|
|||||||
// CLI command: provision install
|
// CLI command: provision install
|
||||||
// Queue a discovered machine for OS installation via labd.
|
// Queue a discovered machine for OS installation via labd.
|
||||||
|
|
||||||
import { Command, Option } from "commander";
|
import { Command, Option, InvalidArgumentError } from "commander";
|
||||||
import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY } from "@lab/shared";
|
import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY } from "@lab/shared";
|
||||||
|
import type { VyosInstallSpec, VyosVlanSpec } from "@lab/shared";
|
||||||
import { getLabdClient } from "../api/config.js";
|
import { getLabdClient } from "../api/config.js";
|
||||||
|
|
||||||
|
/** Parse a repeated --vlan flag: "<id>:<cidr>[:<description>]". */
|
||||||
|
export function parseVlan(value: string, previous: VyosVlanSpec[] = []): VyosVlanSpec[] {
|
||||||
|
const parts = value.split(":");
|
||||||
|
const id = Number(parts[0]);
|
||||||
|
const address = parts[1] ?? "";
|
||||||
|
// InvalidArgumentError makes commander print a clean message instead of
|
||||||
|
// dumping a stack trace at the operator.
|
||||||
|
if (!Number.isInteger(id) || id < 1 || id > 4094) {
|
||||||
|
throw new InvalidArgumentError(`Invalid VLAN id in "${value}" (expected 1-4094)`);
|
||||||
|
}
|
||||||
|
if (!address.includes("/")) {
|
||||||
|
throw new InvalidArgumentError(
|
||||||
|
`Invalid VLAN address in "${value}" (expected CIDR, e.g. 10.0.10.1/24)`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const description = parts.slice(2).join(":");
|
||||||
|
return [...previous, { id, address, ...(description ? { description } : {}) }];
|
||||||
|
}
|
||||||
|
|
||||||
function roleTable(): string {
|
function roleTable(): string {
|
||||||
const lines: string[] = ["", "Available roles:"];
|
const lines: string[] = ["", "Available roles:"];
|
||||||
for (const r of ROLE_REGISTRY) {
|
for (const r of ROLE_REGISTRY) {
|
||||||
@@ -15,6 +35,38 @@ function roleTable(): string {
|
|||||||
return lines.join("\n");
|
return lines.join("\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Parse a repeated --vlan-vip flag: "<id>:<cidr>" — VRRP VIP for a --vlan entry. */
|
||||||
|
export function parseVlanVip(
|
||||||
|
value: string,
|
||||||
|
previous: Record<number, string> = {},
|
||||||
|
): Record<number, string> {
|
||||||
|
const index = value.indexOf(":");
|
||||||
|
const id = Number(index === -1 ? Number.NaN : value.slice(0, index));
|
||||||
|
const cidr = index === -1 ? "" : value.slice(index + 1).trim();
|
||||||
|
if (!Number.isInteger(id) || id < 1 || id > 4094 || !cidr.includes("/")) {
|
||||||
|
throw new InvalidArgumentError(
|
||||||
|
`Invalid VLAN VIP "${value}" (expected <id>:<cidr>, e.g. 3:192.168.3.254/24)`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return { ...previous, [id]: cidr };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Parse a repeated --vyos-hwid flag: "<iface>=<mac>". */
|
||||||
|
export function parseHwId(
|
||||||
|
value: string,
|
||||||
|
previous: Record<string, string> = {},
|
||||||
|
): Record<string, string> {
|
||||||
|
const index = value.indexOf("=");
|
||||||
|
const iface = index === -1 ? "" : value.slice(0, index).trim();
|
||||||
|
const mac = index === -1 ? "" : value.slice(index + 1).trim().toLowerCase();
|
||||||
|
if (iface === "" || !/^([0-9a-f]{2}:){5}[0-9a-f]{2}$/.test(mac)) {
|
||||||
|
throw new InvalidArgumentError(
|
||||||
|
`Invalid hw-id "${value}" (expected <iface>=<mac>, e.g. eth2=64:62:66:25:96:47)`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return { ...previous, [iface]: mac };
|
||||||
|
}
|
||||||
|
|
||||||
export function registerInstallCommand(parent: Command): void {
|
export function registerInstallCommand(parent: Command): void {
|
||||||
parent
|
parent
|
||||||
.command("install <mac> <hostname>")
|
.command("install <mac> <hostname>")
|
||||||
@@ -24,10 +76,34 @@ export function registerInstallCommand(parent: Command): void {
|
|||||||
.addOption(new Option("--role <role>", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker"))
|
.addOption(new Option("--role <role>", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker"))
|
||||||
.addOption(new Option("--os <os>", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43"))
|
.addOption(new Option("--os <os>", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43"))
|
||||||
.option("--disk <device>", "Target disk device (auto-detect if omitted)")
|
.option("--disk <device>", "Target disk device (auto-detect if omitted)")
|
||||||
|
.option("--vyos-mgmt <iface>", "VyOS: untagged interface the machine PXE boots from (default eth0)")
|
||||||
|
.option("--vyos-mgmt-address <addr>", "VyOS: CIDR for the management interface, or 'dhcp' (default dhcp)")
|
||||||
|
.option("--vyos-bond <ifaces>", "VyOS: comma-separated LACP bond members (must exclude the PXE NIC)")
|
||||||
|
.option("--vyos-bond-address <cidr>", "VyOS: address on the untagged bond (trunk native VLAN)")
|
||||||
|
.option("--vyos-bond-vrrp <cidr>", "VyOS: VRRP VIP floated on the untagged bond")
|
||||||
|
.option("--vlan-vip <id:cidr>", "VyOS: VRRP VIP for a --vlan entry (repeatable)", parseVlanVip)
|
||||||
|
.option("--vyos-vrrp-priority <n>", "VyOS: VRRP priority for all groups on this box (higher = master)")
|
||||||
|
.option("--vyos-mgmt-vlan <id:cidr[:desc]>", "VyOS: tagged management VLAN on the PXE port")
|
||||||
|
.option("--vlan <id:cidr[:desc]>", "VyOS: tagged VLAN sub-interface on the bond (repeatable)", parseVlan)
|
||||||
|
.option("--vyos-password <password>", "VyOS: password for the 'vyos' user")
|
||||||
|
.option("--vyos-hwid <iface=mac>", "VyOS: pin an interface name to a MAC via hw-id (repeatable)", parseHwId)
|
||||||
|
.option("--vyos-fresh-config", "VyOS: on reinstall, overwrite the preserved config with the generated one")
|
||||||
.action(async (mac: string, hostname: string, opts: {
|
.action(async (mac: string, hostname: string, opts: {
|
||||||
role: string;
|
role: string;
|
||||||
os: string;
|
os: string;
|
||||||
disk?: string;
|
disk?: string;
|
||||||
|
vyosMgmt?: string;
|
||||||
|
vyosMgmtAddress?: string;
|
||||||
|
vyosBond?: string;
|
||||||
|
vyosBondAddress?: string;
|
||||||
|
vyosBondVrrp?: string;
|
||||||
|
vlan?: VyosVlanSpec[];
|
||||||
|
vlanVip?: Record<number, string>;
|
||||||
|
vyosVrrpPriority?: string;
|
||||||
|
vyosMgmtVlan?: string;
|
||||||
|
vyosPassword?: string;
|
||||||
|
vyosHwid?: Record<string, string>;
|
||||||
|
vyosFreshConfig?: boolean;
|
||||||
}) => {
|
}) => {
|
||||||
if (!isValidOsId(opts.os)) {
|
if (!isValidOsId(opts.os)) {
|
||||||
console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`);
|
console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`);
|
||||||
@@ -39,6 +115,67 @@ export function registerInstallCommand(parent: Command): void {
|
|||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const bondMembers = opts.vyosBond !== undefined && opts.vyosBond !== ""
|
||||||
|
? opts.vyosBond.split(",").map((s) => s.trim()).filter((s) => s.length > 0)
|
||||||
|
: [];
|
||||||
|
|
||||||
|
// Attach --vlan-vip entries to their --vlan definitions. A VIP for a VLAN
|
||||||
|
// that was never defined is a typo that would otherwise vanish silently.
|
||||||
|
const vips = opts.vlanVip ?? {};
|
||||||
|
const vlans = (opts.vlan ?? []).map((v) =>
|
||||||
|
vips[v.id] !== undefined ? { ...v, vrrp: vips[v.id] as string } : v,
|
||||||
|
);
|
||||||
|
for (const id of Object.keys(vips)) {
|
||||||
|
if (!vlans.some((v) => String(v.id) === id)) {
|
||||||
|
console.error(`--vlan-vip ${id}:... has no matching --vlan ${id}:... entry`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const vrrpPriority = opts.vyosVrrpPriority !== undefined && opts.vyosVrrpPriority !== ""
|
||||||
|
? Number(opts.vyosVrrpPriority)
|
||||||
|
: undefined;
|
||||||
|
if (vrrpPriority !== undefined
|
||||||
|
&& (!Number.isInteger(vrrpPriority) || vrrpPriority < 1 || vrrpPriority > 255)) {
|
||||||
|
console.error(`--vyos-vrrp-priority must be an integer 1-255 (got ${opts.vyosVrrpPriority})`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
const vyos: VyosInstallSpec = {
|
||||||
|
...(opts.vyosMgmt !== undefined && opts.vyosMgmt !== ""
|
||||||
|
? { mgmtInterface: opts.vyosMgmt } : {}),
|
||||||
|
...(opts.vyosMgmtAddress !== undefined && opts.vyosMgmtAddress !== ""
|
||||||
|
? { mgmtAddress: opts.vyosMgmtAddress } : {}),
|
||||||
|
...(bondMembers.length > 0 ? { bondMembers } : {}),
|
||||||
|
...(opts.vyosBondAddress !== undefined && opts.vyosBondAddress !== ""
|
||||||
|
? { bondAddress: opts.vyosBondAddress } : {}),
|
||||||
|
...(opts.vyosBondVrrp !== undefined && opts.vyosBondVrrp !== ""
|
||||||
|
? { bondVrrp: opts.vyosBondVrrp } : {}),
|
||||||
|
...(vrrpPriority !== undefined ? { vrrpPriority } : {}),
|
||||||
|
...(vlans.length > 0 ? { vlans } : {}),
|
||||||
|
...(opts.vyosPassword !== undefined && opts.vyosPassword !== ""
|
||||||
|
? { password: opts.vyosPassword } : {}),
|
||||||
|
...(opts.vyosHwid !== undefined && Object.keys(opts.vyosHwid).length > 0
|
||||||
|
? { hwIds: opts.vyosHwid } : {}),
|
||||||
|
...(opts.vyosMgmtVlan !== undefined && opts.vyosMgmtVlan !== ""
|
||||||
|
? { mgmtVlan: parseVlan(opts.vyosMgmtVlan)[0] as VyosVlanSpec } : {}),
|
||||||
|
...(opts.vyosFreshConfig === true ? { freshConfig: true } : {}),
|
||||||
|
};
|
||||||
|
const hasVyosOptions = Object.keys(vyos).length > 0;
|
||||||
|
|
||||||
|
if (hasVyosOptions && !opts.os.startsWith("vyos")) {
|
||||||
|
console.error(`VyOS options require --os vyos-rolling (got --os ${opts.os})`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Firmware PXE cannot run over LACP, so the NIC that boots the installer
|
||||||
|
// must stay out of the bond — otherwise the next reinstall has no path in.
|
||||||
|
const mgmt = vyos.mgmtInterface ?? "eth0";
|
||||||
|
if (bondMembers.includes(mgmt)) {
|
||||||
|
console.error(`--vyos-bond must not include the PXE/management interface "${mgmt}"`);
|
||||||
|
console.error("PXE cannot boot over an LACP bond; keep that NIC unbonded.");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await getLabdClient().installMachine({
|
const result = await getLabdClient().installMachine({
|
||||||
mac,
|
mac,
|
||||||
@@ -46,11 +183,14 @@ export function registerInstallCommand(parent: Command): void {
|
|||||||
role: opts.role,
|
role: opts.role,
|
||||||
os: opts.os,
|
os: opts.os,
|
||||||
...(opts.disk ? { disk: opts.disk } : {}),
|
...(opts.disk ? { disk: opts.disk } : {}),
|
||||||
|
...(hasVyosOptions ? { vyos } : {}),
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log(JSON.stringify(result, null, 2));
|
console.log(JSON.stringify(result, null, 2));
|
||||||
console.log("");
|
console.log("");
|
||||||
const osLabel = opts.os.startsWith("ubuntu") ? "Ubuntu" : "Fedora";
|
const osLabel = opts.os.startsWith("ubuntu")
|
||||||
|
? "Ubuntu"
|
||||||
|
: opts.os.startsWith("vyos") ? "VyOS" : "Fedora";
|
||||||
console.log(`Power on the machine to start ${osLabel} installation.`);
|
console.log(`Power on the machine to start ${osLabel} installation.`);
|
||||||
|
|
||||||
const roleInfo = ROLE_REGISTRY.find(r => r.name === opts.role);
|
const roleInfo = ROLE_REGISTRY.find(r => r.name === opts.role);
|
||||||
|
|||||||
@@ -44,11 +44,14 @@ export function registerRecheckCommand(parent: Command): void {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Build list of machines to check
|
// Build list of machines to check
|
||||||
const targets: Array<{ mac: string; hostname: string; ip: string }> = [];
|
const targets: Array<{ mac: string; hostname: string; ip: string; sshUser: string }> = [];
|
||||||
|
const userIsDefault = opts.user === "root";
|
||||||
for (const [mac, info] of Object.entries(state.installed)) {
|
for (const [mac, info] of Object.entries(state.installed)) {
|
||||||
if (!info.ip) continue;
|
if (!info.ip) continue;
|
||||||
if (opts.target && info.hostname !== opts.target && mac !== opts.target) continue;
|
if (opts.target && info.hostname !== opts.target && mac !== opts.target) continue;
|
||||||
targets.push({ mac, hostname: info.hostname, ip: info.ip });
|
// VyOS boxes only have the "vyos" login; honor an explicit --user.
|
||||||
|
const sshUser = userIsDefault && (info.os ?? "").startsWith("vyos") ? "vyos" : opts.user;
|
||||||
|
targets.push({ mac, hostname: info.hostname, ip: info.ip, sshUser });
|
||||||
}
|
}
|
||||||
|
|
||||||
if (targets.length === 0) {
|
if (targets.length === 0) {
|
||||||
@@ -61,12 +64,12 @@ export function registerRecheckCommand(parent: Command): void {
|
|||||||
let updated = 0;
|
let updated = 0;
|
||||||
let failed = 0;
|
let failed = 0;
|
||||||
|
|
||||||
for (const { mac, hostname, ip } of targets) {
|
for (const { mac, hostname, ip, sshUser } of targets) {
|
||||||
process.stdout.write(` ${hostname.padEnd(24)} ${DIM}(${ip})${RESET} `);
|
process.stdout.write(` ${hostname.padEnd(24)} ${DIM}(${ip})${RESET} `);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const t0 = Date.now();
|
const t0 = Date.now();
|
||||||
const result = await sshExec(ip, opts.user, HW_COLLECT_SCRIPT, SSH_OPTS);
|
const result = await sshExec(ip, sshUser, HW_COLLECT_SCRIPT, SSH_OPTS);
|
||||||
const elapsed = Date.now() - t0;
|
const elapsed = Date.now() - t0;
|
||||||
if (result.exitCode !== 0) {
|
if (result.exitCode !== 0) {
|
||||||
console.log(`${RED}SSH failed (exit ${result.exitCode}, ${elapsed}ms)${RESET}`);
|
console.log(`${RED}SSH failed (exit ${result.exitCode}, ${elapsed}ms)${RESET}`);
|
||||||
|
|||||||
@@ -24,12 +24,12 @@ function roleTable(): string {
|
|||||||
function resolveTarget(
|
function resolveTarget(
|
||||||
target: string,
|
target: string,
|
||||||
state: BastionState,
|
state: BastionState,
|
||||||
): { mac: string; hostname: string; ip: string } | null {
|
): { mac: string; hostname: string; ip: string; os?: string } | null {
|
||||||
const normalized = target.toLowerCase().replace(/-/g, ":");
|
const normalized = target.toLowerCase().replace(/-/g, ":");
|
||||||
|
|
||||||
if (state.installed[normalized]) {
|
if (state.installed[normalized]) {
|
||||||
const info = state.installed[normalized];
|
const info = state.installed[normalized];
|
||||||
return { mac: normalized, hostname: info.hostname, ip: info.ip };
|
return { mac: normalized, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
|
||||||
}
|
}
|
||||||
|
|
||||||
if (state.discovered[normalized]) {
|
if (state.discovered[normalized]) {
|
||||||
@@ -38,13 +38,13 @@ function resolveTarget(
|
|||||||
|
|
||||||
for (const [mac, info] of Object.entries(state.installed)) {
|
for (const [mac, info] of Object.entries(state.installed)) {
|
||||||
if (info.hostname === target || info.hostname.startsWith(target + ".")) {
|
if (info.hostname === target || info.hostname.startsWith(target + ".")) {
|
||||||
return { mac, hostname: info.hostname, ip: info.ip };
|
return { mac, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const [mac, info] of Object.entries(state.installed)) {
|
for (const [mac, info] of Object.entries(state.installed)) {
|
||||||
if (info.ip === target) {
|
if (info.ip === target) {
|
||||||
return { mac, hostname: info.hostname, ip: info.ip };
|
return { mac, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -60,10 +60,12 @@ export function registerReprovisionCommand(parent: Command): void {
|
|||||||
.addOption(new Option("--role <role>", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker"))
|
.addOption(new Option("--role <role>", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker"))
|
||||||
.addOption(new Option("--os <os>", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43"))
|
.addOption(new Option("--os <os>", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43"))
|
||||||
.option("--disk <device>", "Target disk device (auto-detect if omitted)")
|
.option("--disk <device>", "Target disk device (auto-detect if omitted)")
|
||||||
|
.option("--user <user>", "SSH user for the reboot (default: vyos for VyOS machines, else current user)")
|
||||||
.action(async (target: string, hostnameOverride: string | undefined, opts: {
|
.action(async (target: string, hostnameOverride: string | undefined, opts: {
|
||||||
role: string;
|
role: string;
|
||||||
os: string;
|
os: string;
|
||||||
disk?: string;
|
disk?: string;
|
||||||
|
user?: string;
|
||||||
}) => {
|
}) => {
|
||||||
if (!isValidOsId(opts.os)) {
|
if (!isValidOsId(opts.os)) {
|
||||||
console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`);
|
console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`);
|
||||||
@@ -123,7 +125,11 @@ export function registerReprovisionCommand(parent: Command): void {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const adminUser = process.env["SUDO_USER"] ?? process.env["USER"] ?? "";
|
// SSH user: explicit flag > the machine's current OS (VyOS boxes only
|
||||||
|
// have the "vyos" login) > the invoking user.
|
||||||
|
const currentOsIsVyos = (resolved.os ?? "").startsWith("vyos");
|
||||||
|
const adminUser = opts.user
|
||||||
|
?? (currentOsIsVyos ? "vyos" : (process.env["SUDO_USER"] ?? process.env["USER"] ?? ""));
|
||||||
const effectiveUser = adminUser === "root" ? "" : adminUser;
|
const effectiveUser = adminUser === "root" ? "" : adminUser;
|
||||||
|
|
||||||
if (effectiveUser === "") {
|
if (effectiveUser === "") {
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ export function registerStartCommand(parent: Command): void {
|
|||||||
.command("start")
|
.command("start")
|
||||||
.description("Start the bastion server (HTTP + dnsmasq PXE)")
|
.description("Start the bastion server (HTTP + dnsmasq PXE)")
|
||||||
.option("--port <port>", "HTTP port", "8080")
|
.option("--port <port>", "HTTP port", "8080")
|
||||||
.option("--dir <dir>", "Bastion data directory", "/tmp/lab-bastion")
|
.option("--dir <dir>", "Bastion data directory", process.env["BASTION_DIR"] ?? "/tmp/lab-bastion")
|
||||||
.option("--domain <domain>", "Internal domain for hostnames", "ad.itaz.eu")
|
.option("--domain <domain>", "Internal domain for hostnames", "ad.itaz.eu")
|
||||||
.option("--dhcp-mode <mode>", "DHCP mode: proxy or full", "proxy")
|
.option("--dhcp-mode <mode>", "DHCP mode: proxy or full", "proxy")
|
||||||
.option("--fedora <version>", "Fedora version", "43")
|
.option("--fedora <version>", "Fedora version", "43")
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ export function registerStopCommand(parent: Command): void {
|
|||||||
parent
|
parent
|
||||||
.command("stop")
|
.command("stop")
|
||||||
.description("Stop a running bastion server")
|
.description("Stop a running bastion server")
|
||||||
.option("--dir <dir>", "Bastion data directory", "/tmp/lab-bastion")
|
.option("--dir <dir>", "Bastion data directory", process.env["BASTION_DIR"] ?? "/tmp/lab-bastion")
|
||||||
.action((opts: { dir: string }) => {
|
.action((opts: { dir: string }) => {
|
||||||
const pidFile = `${opts.dir}/bastion.pid`;
|
const pidFile = `${opts.dir}/bastion.pid`;
|
||||||
|
|
||||||
|
|||||||
35
bastion/src/cli/tests/install-vyos.test.ts
Normal file
35
bastion/src/cli/tests/install-vyos.test.ts
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
// Tests for VyOS install option parsing.
|
||||||
|
|
||||||
|
import { describe, it, expect } from "vitest";
|
||||||
|
import { parseVlan } from "../src/commands/install.js";
|
||||||
|
|
||||||
|
describe("parseVlan", () => {
|
||||||
|
it("parses id and CIDR", () => {
|
||||||
|
expect(parseVlan("10:10.0.10.1/24")).toEqual([{ id: 10, address: "10.0.10.1/24" }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("accumulates across repeated flags", () => {
|
||||||
|
const first = parseVlan("10:10.0.10.1/24");
|
||||||
|
const both = parseVlan("20:10.0.20.1/24", first);
|
||||||
|
expect(both).toHaveLength(2);
|
||||||
|
expect(both[1]).toEqual({ id: 20, address: "10.0.20.1/24" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps a description, including one containing colons", () => {
|
||||||
|
expect(parseVlan("30:10.0.30.1/24:mgmt:secondary")).toEqual([
|
||||||
|
{ id: 30, address: "10.0.30.1/24", description: "mgmt:secondary" },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects an address that is not CIDR", () => {
|
||||||
|
// A bare address would produce a VyOS config that fails to commit on first
|
||||||
|
// boot, long after the operator has stopped watching.
|
||||||
|
expect(() => parseVlan("10:10.0.10.1")).toThrow(/CIDR/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects out-of-range and non-numeric VLAN ids", () => {
|
||||||
|
expect(() => parseVlan("0:10.0.10.1/24")).toThrow(/1-4094/);
|
||||||
|
expect(() => parseVlan("4095:10.0.10.1/24")).toThrow(/1-4094/);
|
||||||
|
expect(() => parseVlan("abc:10.0.10.1/24")).toThrow(/1-4094/);
|
||||||
|
});
|
||||||
|
});
|
||||||
23
bastion/src/core/package.json
Normal file
23
bastion/src/core/package.json
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
{
|
||||||
|
"name": "@lab/core",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"private": true,
|
||||||
|
"type": "module",
|
||||||
|
"main": "./dist/index.js",
|
||||||
|
"types": "./dist/index.d.ts",
|
||||||
|
"exports": {
|
||||||
|
".": {
|
||||||
|
"import": "./dist/index.js",
|
||||||
|
"types": "./dist/index.d.ts"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsc --build",
|
||||||
|
"clean": "rimraf dist",
|
||||||
|
"test": "vitest",
|
||||||
|
"test:run": "vitest run"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@pulumi/pulumi": "^3.0.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
75
bastion/src/core/src/audit.ts
Normal file
75
bastion/src/core/src/audit.ts
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
// Audit event types for the labctl platform.
|
||||||
|
// Every mutation is tracked with correlation IDs for causal chains.
|
||||||
|
|
||||||
|
export type AuditEventKind =
|
||||||
|
| "resource_created"
|
||||||
|
| "resource_updated"
|
||||||
|
| "resource_deleted"
|
||||||
|
| "resource_state_change"
|
||||||
|
| "plan_generated"
|
||||||
|
| "apply_started"
|
||||||
|
| "apply_step"
|
||||||
|
| "apply_completed"
|
||||||
|
| "driver_translate"
|
||||||
|
| "driver_execute"
|
||||||
|
| "driver_error"
|
||||||
|
| "fleet_discovery"
|
||||||
|
| "fleet_classification"
|
||||||
|
| "fleet_approval"
|
||||||
|
| "fleet_auto_approve"
|
||||||
|
| "pipeline_started"
|
||||||
|
| "pipeline_step_started"
|
||||||
|
| "pipeline_step_completed"
|
||||||
|
| "pipeline_completed"
|
||||||
|
| "deploy_started"
|
||||||
|
| "deploy_completed"
|
||||||
|
| "deploy_failed"
|
||||||
|
| "drift_detected"
|
||||||
|
| "drift_corrected"
|
||||||
|
| "sync_triggered"
|
||||||
|
| "sync_completed"
|
||||||
|
| "auth_login"
|
||||||
|
| "auth_logout"
|
||||||
|
| "auth_bootstrap"
|
||||||
|
| "rbac_decision"
|
||||||
|
| "impersonation"
|
||||||
|
| "server_started"
|
||||||
|
| "controller_started"
|
||||||
|
| "agent_connected"
|
||||||
|
| "agent_disconnected"
|
||||||
|
| "bastion_registered";
|
||||||
|
|
||||||
|
export type AuditSource =
|
||||||
|
| "cli"
|
||||||
|
| "labd"
|
||||||
|
| "agent"
|
||||||
|
| "driver"
|
||||||
|
| "fleet-controller"
|
||||||
|
| "sync-controller";
|
||||||
|
|
||||||
|
export type AuditResult = "success" | "failure" | "denied" | "skipped";
|
||||||
|
|
||||||
|
export interface AuditEvent {
|
||||||
|
id: string;
|
||||||
|
timestamp: Date;
|
||||||
|
eventKind: AuditEventKind;
|
||||||
|
source: AuditSource;
|
||||||
|
verified: boolean;
|
||||||
|
|
||||||
|
userId?: string;
|
||||||
|
userName?: string;
|
||||||
|
sessionId?: string;
|
||||||
|
environmentName?: string;
|
||||||
|
accountName?: string;
|
||||||
|
|
||||||
|
resourceKind?: string;
|
||||||
|
resourceName?: string;
|
||||||
|
|
||||||
|
correlationId: string;
|
||||||
|
parentEventId?: string;
|
||||||
|
|
||||||
|
details: Record<string, unknown>;
|
||||||
|
result: AuditResult;
|
||||||
|
error?: string;
|
||||||
|
durationMs?: number;
|
||||||
|
}
|
||||||
50
bastion/src/core/src/auth.ts
Normal file
50
bastion/src/core/src/auth.ts
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
// Auth types for the labctl platform.
|
||||||
|
// Bearer token auth for CLI/SDK. mTLS stays for agent/bastion.
|
||||||
|
|
||||||
|
export type UserRole = "USER" | "ADMIN";
|
||||||
|
|
||||||
|
export interface User {
|
||||||
|
id: string;
|
||||||
|
email: string;
|
||||||
|
name?: string;
|
||||||
|
role: UserRole;
|
||||||
|
createdAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Session {
|
||||||
|
id: string;
|
||||||
|
userId: string;
|
||||||
|
token: string;
|
||||||
|
expiresAt: Date;
|
||||||
|
createdAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Group {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
description?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type SubjectKind = "User" | "Group" | "ServiceAccount";
|
||||||
|
|
||||||
|
export interface RoleBinding {
|
||||||
|
role: "view" | "edit" | "create" | "delete" | "run" | "admin";
|
||||||
|
resource: string;
|
||||||
|
name?: string;
|
||||||
|
environment?: string;
|
||||||
|
action?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RbacSubject {
|
||||||
|
kind: SubjectKind;
|
||||||
|
name: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RbacDefinition {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
subjects: RbacSubject[];
|
||||||
|
roleBindings: RoleBinding[];
|
||||||
|
createdAt: Date;
|
||||||
|
updatedAt: Date;
|
||||||
|
}
|
||||||
24
bastion/src/core/src/environment.ts
Normal file
24
bastion/src/core/src/environment.ts
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
// Environment and Account types.
|
||||||
|
// An Environment is a logical boundary (production, staging, dev).
|
||||||
|
// An Account is a configured driver instance with credentials.
|
||||||
|
|
||||||
|
export interface Environment {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
status: "active" | "archived";
|
||||||
|
createdAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Account {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
driver: string;
|
||||||
|
config: Record<string, unknown>;
|
||||||
|
createdAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Binding {
|
||||||
|
id: string;
|
||||||
|
environmentId: string;
|
||||||
|
accountId: string;
|
||||||
|
}
|
||||||
9
bastion/src/core/src/index.ts
Normal file
9
bastion/src/core/src/index.ts
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
// @lab/core — foundation types for the labctl platform.
|
||||||
|
// Phase 1 stub: resource types, auth types, audit types, Output<T>.
|
||||||
|
// Phase 5 adds: CompositeResource, evaluator integration, full SDK.
|
||||||
|
|
||||||
|
export * from "./resource.js";
|
||||||
|
export * from "./environment.js";
|
||||||
|
export * from "./audit.js";
|
||||||
|
export * from "./auth.js";
|
||||||
|
export { Output, output, all, interpolate, secret } from "./output.js";
|
||||||
5
bastion/src/core/src/output.ts
Normal file
5
bastion/src/core/src/output.ts
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
// Re-export Pulumi's Output<T> type for use across the platform.
|
||||||
|
// Cloud drivers use this for future values (endpoints, IPs, kubeconfigs).
|
||||||
|
// Phase 1: type re-export only. Phase 5 adds full evaluator integration.
|
||||||
|
|
||||||
|
export { Output, output, all, interpolate, secret } from "@pulumi/pulumi";
|
||||||
83
bastion/src/core/src/resource.ts
Normal file
83
bastion/src/core/src/resource.ts
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
// Core resource types for the labctl platform.
|
||||||
|
// Every managed thing (Server, Database, App, Cluster) is a Resource.
|
||||||
|
|
||||||
|
export type ResourceOrigin = "file" | "cli" | "fleet" | "imported";
|
||||||
|
export type ResourceManagedBy = "gitops" | "manual" | "auto";
|
||||||
|
|
||||||
|
export type ResourceStatus =
|
||||||
|
| "pending"
|
||||||
|
| "creating"
|
||||||
|
| "ready"
|
||||||
|
| "updating"
|
||||||
|
| "deleting"
|
||||||
|
| "error"
|
||||||
|
| "unknown";
|
||||||
|
|
||||||
|
export interface ResourceMetadata {
|
||||||
|
kind: string;
|
||||||
|
name: string;
|
||||||
|
environmentId: string;
|
||||||
|
accountId: string;
|
||||||
|
origin: ResourceOrigin;
|
||||||
|
managedBy: ResourceManagedBy;
|
||||||
|
sourceRef?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ResourceState {
|
||||||
|
status: ResourceStatus;
|
||||||
|
message?: string;
|
||||||
|
lastReconciled?: Date;
|
||||||
|
platformRef?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Resource<TSpec = Record<string, unknown>> {
|
||||||
|
id: string;
|
||||||
|
metadata: ResourceMetadata;
|
||||||
|
desiredSpec: TSpec;
|
||||||
|
actualSpec?: TSpec;
|
||||||
|
state: ResourceState;
|
||||||
|
createdAt: Date;
|
||||||
|
updatedAt: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Well-known resource kinds. Drivers register additional kinds.
|
||||||
|
export const RESOURCE_KINDS = {
|
||||||
|
SERVER: "server",
|
||||||
|
DATABASE: "database",
|
||||||
|
CACHE: "cache",
|
||||||
|
CLUSTER: "cluster",
|
||||||
|
APP: "app",
|
||||||
|
SERVICE: "service",
|
||||||
|
CRONJOB: "cronjob",
|
||||||
|
NETWORK: "network",
|
||||||
|
LOADBALANCER: "loadbalancer",
|
||||||
|
DNSZONE: "dnszone",
|
||||||
|
CERTIFICATE: "certificate",
|
||||||
|
OBJECTSTORE: "objectstore",
|
||||||
|
QUEUE: "queue",
|
||||||
|
SECRET: "secret",
|
||||||
|
FLEET: "fleet",
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
export type ResourceKind = (typeof RESOURCE_KINDS)[keyof typeof RESOURCE_KINDS];
|
||||||
|
|
||||||
|
// Resource aliases for CLI (kubectl-style shortnames)
|
||||||
|
export const RESOURCE_ALIASES: Record<string, string> = {
|
||||||
|
srv: "server",
|
||||||
|
db: "database",
|
||||||
|
cl: "cluster",
|
||||||
|
svc: "service",
|
||||||
|
cj: "cronjob",
|
||||||
|
lb: "loadbalancer",
|
||||||
|
dns: "dnszone",
|
||||||
|
cert: "certificate",
|
||||||
|
os: "objectstore",
|
||||||
|
mq: "queue",
|
||||||
|
sec: "secret",
|
||||||
|
fl: "fleet",
|
||||||
|
};
|
||||||
|
|
||||||
|
export function resolveResourceKind(input: string): string {
|
||||||
|
const lower = input.toLowerCase();
|
||||||
|
return RESOURCE_ALIASES[lower] ?? lower;
|
||||||
|
}
|
||||||
8
bastion/src/core/tsconfig.json
Normal file
8
bastion/src/core/tsconfig.json
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"extends": "../../tsconfig.base.json",
|
||||||
|
"compilerOptions": {
|
||||||
|
"rootDir": "src",
|
||||||
|
"outDir": "dist"
|
||||||
|
},
|
||||||
|
"include": ["src/**/*.ts"]
|
||||||
|
}
|
||||||
@@ -26,8 +26,10 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@fastify/rate-limit": "^10.3.0",
|
"@fastify/rate-limit": "^10.3.0",
|
||||||
"@fastify/websocket": "^11.0.2",
|
"@fastify/websocket": "^11.0.2",
|
||||||
|
"@lab/core": "workspace:^",
|
||||||
"@lab/shared": "workspace:*",
|
"@lab/shared": "workspace:*",
|
||||||
"@prisma/client": "^6.9.0",
|
"@prisma/client": "^6.9.0",
|
||||||
|
"bcryptjs": "^3.0.3",
|
||||||
"fastify": "^5.3.3",
|
"fastify": "^5.3.3",
|
||||||
"winston": "^3.17.0",
|
"winston": "^3.17.0",
|
||||||
"ws": "^8.19.0",
|
"ws": "^8.19.0",
|
||||||
@@ -37,6 +39,7 @@
|
|||||||
"seed": "tsx prisma/seed.ts"
|
"seed": "tsx prisma/seed.ts"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
"@types/bcryptjs": "^3.0.0",
|
||||||
"@types/node": "^22.14.1",
|
"@types/node": "^22.14.1",
|
||||||
"@types/ws": "^8.18.1",
|
"@types/ws": "^8.18.1",
|
||||||
"prisma": "^6.9.0",
|
"prisma": "^6.9.0",
|
||||||
|
|||||||
@@ -7,6 +7,225 @@ datasource db {
|
|||||||
url = env("DATABASE_URL")
|
url = env("DATABASE_URL")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Auth (mcpctl pattern: email/password + bearer token sessions) ──
|
||||||
|
|
||||||
|
model User {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
email String @unique
|
||||||
|
password String // bcrypt
|
||||||
|
name String?
|
||||||
|
role UserRole @default(USER)
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
sessions Session[]
|
||||||
|
auditLogs AuditEvent[]
|
||||||
|
groups GroupMember[]
|
||||||
|
}
|
||||||
|
|
||||||
|
enum UserRole {
|
||||||
|
USER
|
||||||
|
ADMIN
|
||||||
|
}
|
||||||
|
|
||||||
|
model Session {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
userId String
|
||||||
|
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||||
|
token String @unique
|
||||||
|
expiresAt DateTime
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
|
||||||
|
@@index([userId])
|
||||||
|
@@index([token])
|
||||||
|
}
|
||||||
|
|
||||||
|
model Group {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
name String @unique
|
||||||
|
description String?
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
members GroupMember[]
|
||||||
|
}
|
||||||
|
|
||||||
|
model GroupMember {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
groupId String
|
||||||
|
group Group @relation(fields: [groupId], references: [id], onDelete: Cascade)
|
||||||
|
userId String
|
||||||
|
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||||
|
|
||||||
|
@@unique([groupId, userId])
|
||||||
|
}
|
||||||
|
|
||||||
|
model ServiceAccount {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
name String @unique
|
||||||
|
token String @unique
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── RBAC (mcpctl pattern: named definitions with JSON subjects/bindings) ──
|
||||||
|
|
||||||
|
model RbacDefinition {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
name String @unique
|
||||||
|
subjects Json // [{kind: "User"|"Group"|"ServiceAccount", name: string}]
|
||||||
|
roleBindings Json // [{role, resource, name?, environment?, action?}]
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Audit (mcpctl pattern: fire-and-forget with correlation IDs) ──
|
||||||
|
|
||||||
|
model AuditEvent {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
timestamp DateTime @default(now())
|
||||||
|
eventKind String
|
||||||
|
source String // cli | labd | agent | driver | fleet-controller | sync-controller
|
||||||
|
verified Boolean @default(false)
|
||||||
|
|
||||||
|
userId String?
|
||||||
|
user User? @relation(fields: [userId], references: [id])
|
||||||
|
userName String?
|
||||||
|
sessionId String?
|
||||||
|
environmentName String?
|
||||||
|
accountName String?
|
||||||
|
|
||||||
|
resourceKind String?
|
||||||
|
resourceName String?
|
||||||
|
|
||||||
|
correlationId String
|
||||||
|
parentEventId String?
|
||||||
|
|
||||||
|
details Json @default("{}")
|
||||||
|
result String // success | failure | denied | skipped
|
||||||
|
error String?
|
||||||
|
durationMs Int?
|
||||||
|
|
||||||
|
@@index([correlationId])
|
||||||
|
@@index([eventKind, timestamp])
|
||||||
|
@@index([environmentName, timestamp])
|
||||||
|
@@index([resourceKind, resourceName])
|
||||||
|
@@index([userId, timestamp])
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Core infrastructure ──
|
||||||
|
|
||||||
|
model Environment {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
name String @unique
|
||||||
|
status String @default("active") // active | archived
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
bindings Binding[]
|
||||||
|
resources Resource[]
|
||||||
|
}
|
||||||
|
|
||||||
|
model Account {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
name String @unique
|
||||||
|
driver String // baremetal-pxe | aws | gcp | kubernetes | ovh
|
||||||
|
config Json @default("{}")
|
||||||
|
// Credentials stored in Infisical, referenced by secretPath
|
||||||
|
secretPath String?
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
bindings Binding[]
|
||||||
|
resources Resource[]
|
||||||
|
}
|
||||||
|
|
||||||
|
model Binding {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
environmentId String
|
||||||
|
environment Environment @relation(fields: [environmentId], references: [id], onDelete: Cascade)
|
||||||
|
accountId String
|
||||||
|
account Account @relation(fields: [accountId], references: [id], onDelete: Cascade)
|
||||||
|
|
||||||
|
@@unique([environmentId, accountId])
|
||||||
|
}
|
||||||
|
|
||||||
|
model Resource {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
kind String
|
||||||
|
name String
|
||||||
|
environmentId String
|
||||||
|
environment Environment @relation(fields: [environmentId], references: [id])
|
||||||
|
accountId String
|
||||||
|
account Account @relation(fields: [accountId], references: [id])
|
||||||
|
origin String @default("cli") // file | cli | fleet | imported
|
||||||
|
managedBy String @default("manual") // gitops | manual | auto
|
||||||
|
sourceRef String?
|
||||||
|
desiredSpec Json @default("{}")
|
||||||
|
actualSpec Json?
|
||||||
|
platformRef String?
|
||||||
|
status String @default("pending") // pending | creating | ready | updating | deleting | error
|
||||||
|
statusMessage String?
|
||||||
|
lastReconciled DateTime?
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
@@unique([kind, name, environmentId])
|
||||||
|
@@index([environmentId])
|
||||||
|
@@index([accountId])
|
||||||
|
@@index([kind, status])
|
||||||
|
}
|
||||||
|
|
||||||
|
model Secret {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
name String @unique
|
||||||
|
// Encrypted data — application-layer encryption as fallback if Infisical unavailable
|
||||||
|
data Json @default("{}")
|
||||||
|
version Int @default(1)
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Fleet ──
|
||||||
|
|
||||||
|
model Fleet {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
name String
|
||||||
|
environmentId String
|
||||||
|
accountId String
|
||||||
|
selector Json // fact-matching rules
|
||||||
|
onboardPipeline Json // step definitions
|
||||||
|
offboardPipeline Json?
|
||||||
|
approvalConfig Json?
|
||||||
|
status String @default("active")
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
|
members FleetMember[]
|
||||||
|
}
|
||||||
|
|
||||||
|
model FleetMember {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
fleetId String
|
||||||
|
fleet Fleet @relation(fields: [fleetId], references: [id], onDelete: Cascade)
|
||||||
|
serverId String
|
||||||
|
status String // discovered | pending | onboarding | active | offboarding | removed
|
||||||
|
joinedAt DateTime @default(now())
|
||||||
|
|
||||||
|
@@index([fleetId])
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Git sources (for sync controller) ──
|
||||||
|
|
||||||
|
model GitSource {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
name String @unique
|
||||||
|
repo String
|
||||||
|
branch String @default("main")
|
||||||
|
path String @default("environments/")
|
||||||
|
lastSync DateTime?
|
||||||
|
createdAt DateTime @default(now())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Existing v1.0 models (kept for bastion/agent compatibility) ──
|
||||||
|
|
||||||
model Server {
|
model Server {
|
||||||
id String @id @default(uuid())
|
id String @id @default(uuid())
|
||||||
hostname String @unique
|
hostname String @unique
|
||||||
@@ -17,13 +236,12 @@ model Server {
|
|||||||
labels Json @default("{}")
|
labels Json @default("{}")
|
||||||
ip String?
|
ip String?
|
||||||
agentVersion String?
|
agentVersion String?
|
||||||
status String @default("unknown") // unknown, online, offline, provisioning
|
status String @default("unknown")
|
||||||
lastHeartbeat DateTime?
|
lastHeartbeat DateTime?
|
||||||
createdAt DateTime @default(now())
|
createdAt DateTime @default(now())
|
||||||
updatedAt DateTime @updatedAt
|
updatedAt DateTime @updatedAt
|
||||||
|
|
||||||
agent Agent?
|
agent Agent?
|
||||||
auditLogs AuditLog[]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
model Agent {
|
model Agent {
|
||||||
@@ -33,112 +251,29 @@ model Agent {
|
|||||||
certificatePem String?
|
certificatePem String?
|
||||||
enrolledAt DateTime @default(now())
|
enrolledAt DateTime @default(now())
|
||||||
lastSeen DateTime?
|
lastSeen DateTime?
|
||||||
|
facts Json? // hardware facts reported by agent
|
||||||
|
|
||||||
@@index([serverId])
|
@@index([serverId])
|
||||||
}
|
}
|
||||||
|
|
||||||
model User {
|
|
||||||
id String @id @default(uuid())
|
|
||||||
username String @unique
|
|
||||||
displayName String?
|
|
||||||
certFingerprint String? @unique
|
|
||||||
createdAt DateTime @default(now())
|
|
||||||
updatedAt DateTime @updatedAt
|
|
||||||
|
|
||||||
roleBindings UserRole[]
|
|
||||||
auditLogs AuditLog[]
|
|
||||||
}
|
|
||||||
|
|
||||||
model Role {
|
|
||||||
id String @id @default(uuid())
|
|
||||||
name String @unique
|
|
||||||
description String?
|
|
||||||
createdAt DateTime @default(now())
|
|
||||||
|
|
||||||
permissions Permission[]
|
|
||||||
userBindings UserRole[]
|
|
||||||
}
|
|
||||||
|
|
||||||
model Permission {
|
|
||||||
id String @id @default(uuid())
|
|
||||||
roleId String
|
|
||||||
role Role @relation(fields: [roleId], references: [id], onDelete: Cascade)
|
|
||||||
type String @default("allow") // allow or deny
|
|
||||||
action String // read, exec, apply, destroy, manage, admin, kubectl, *
|
|
||||||
cloud String @default("*")
|
|
||||||
environment String @default("*")
|
|
||||||
server String @default("*")
|
|
||||||
|
|
||||||
@@index([roleId])
|
|
||||||
}
|
|
||||||
|
|
||||||
model UserRole {
|
|
||||||
id String @id @default(uuid())
|
|
||||||
userId String
|
|
||||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
|
||||||
roleId String
|
|
||||||
role Role @relation(fields: [roleId], references: [id], onDelete: Cascade)
|
|
||||||
|
|
||||||
@@unique([userId, roleId])
|
|
||||||
@@index([userId])
|
|
||||||
@@index([roleId])
|
|
||||||
}
|
|
||||||
|
|
||||||
model JoinToken {
|
model JoinToken {
|
||||||
id String @id @default(uuid())
|
id String @id @default(uuid())
|
||||||
token String @unique
|
token String @unique
|
||||||
type String @default("one-time") // one-time or reusable
|
type String @default("one-time")
|
||||||
label String?
|
label String?
|
||||||
usedBy String? // server hostname that used it
|
usedBy String?
|
||||||
usedAt DateTime?
|
usedAt DateTime?
|
||||||
revokedAt DateTime?
|
revokedAt DateTime?
|
||||||
createdAt DateTime @default(now())
|
createdAt DateTime @default(now())
|
||||||
expiresAt DateTime?
|
expiresAt DateTime?
|
||||||
}
|
}
|
||||||
|
|
||||||
model AuditLog {
|
|
||||||
id String @id @default(uuid())
|
|
||||||
userId String?
|
|
||||||
user User? @relation(fields: [userId], references: [id])
|
|
||||||
serverId String?
|
|
||||||
server Server? @relation(fields: [serverId], references: [id])
|
|
||||||
sessionId String?
|
|
||||||
action String // exec, kubectl, apply, login, rbac-denied, etc.
|
|
||||||
resourceType String? // server, cluster, role, app, etc.
|
|
||||||
resourceName String?
|
|
||||||
args String? // sanitized command args
|
|
||||||
result String @default("success") // success, denied, error
|
|
||||||
durationMs Int?
|
|
||||||
sourceIp String?
|
|
||||||
timestamp DateTime @default(now())
|
|
||||||
|
|
||||||
@@index([userId])
|
|
||||||
@@index([serverId])
|
|
||||||
@@index([sessionId])
|
|
||||||
@@index([timestamp])
|
|
||||||
@@index([action])
|
|
||||||
}
|
|
||||||
|
|
||||||
model PulumiRun {
|
|
||||||
id String @id @default(uuid())
|
|
||||||
userId String
|
|
||||||
stackName String
|
|
||||||
action String // up, preview, destroy
|
|
||||||
status String @default("pending") // pending, running, succeeded, failed
|
|
||||||
output String?
|
|
||||||
startedAt DateTime @default(now())
|
|
||||||
completedAt DateTime?
|
|
||||||
|
|
||||||
@@index([userId])
|
|
||||||
@@index([stackName])
|
|
||||||
}
|
|
||||||
|
|
||||||
model Bastion {
|
model Bastion {
|
||||||
id String @id @default(uuid())
|
id String @id @default(uuid())
|
||||||
hostname String @unique
|
hostname String @unique
|
||||||
network String
|
network String
|
||||||
serverIp String
|
serverIp String
|
||||||
status String @default("offline") // online, offline
|
status String @default("offline")
|
||||||
lastHeartbeat DateTime?
|
lastHeartbeat DateTime?
|
||||||
createdAt DateTime @default(now())
|
createdAt DateTime @default(now())
|
||||||
updatedAt DateTime @updatedAt
|
updatedAt DateTime @updatedAt
|
||||||
@@ -149,7 +284,7 @@ model Cluster {
|
|||||||
name String @unique
|
name String @unique
|
||||||
cloud String @default("baremetal")
|
cloud String @default("baremetal")
|
||||||
environment String @default("default")
|
environment String @default("default")
|
||||||
kubeconfigEnc String? // encrypted kubeconfig
|
kubeconfigEnc String?
|
||||||
labels Json @default("{}")
|
labels Json @default("{}")
|
||||||
createdAt DateTime @default(now())
|
createdAt DateTime @default(now())
|
||||||
updatedAt DateTime @updatedAt
|
updatedAt DateTime @updatedAt
|
||||||
|
|||||||
65
bastion/src/labd/src/middleware/bearer-auth.ts
Normal file
65
bastion/src/labd/src/middleware/bearer-auth.ts
Normal file
@@ -0,0 +1,65 @@
|
|||||||
|
// Bearer token auth middleware for Fastify.
|
||||||
|
// Validates Authorization header, resolves user identity, attaches to request.
|
||||||
|
|
||||||
|
import type { FastifyRequest, FastifyReply } from "fastify";
|
||||||
|
import type { AuthService } from "../services/auth.js";
|
||||||
|
|
||||||
|
declare module "fastify" {
|
||||||
|
interface FastifyRequest {
|
||||||
|
userId?: string;
|
||||||
|
userEmail?: string;
|
||||||
|
userRole?: string;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Paths that don't require authentication
|
||||||
|
const PUBLIC_PATHS = new Set([
|
||||||
|
"/health",
|
||||||
|
"/api/auth/login",
|
||||||
|
"/ws/bastion",
|
||||||
|
"/ws/agent",
|
||||||
|
"/api/auth/enroll",
|
||||||
|
]);
|
||||||
|
|
||||||
|
export function createBearerAuthMiddleware(authService: AuthService) {
|
||||||
|
return async function bearerAuth(
|
||||||
|
request: FastifyRequest,
|
||||||
|
reply: FastifyReply,
|
||||||
|
): Promise<void> {
|
||||||
|
// Skip auth for public paths
|
||||||
|
if (PUBLIC_PATHS.has(request.url.split("?")[0] ?? "")) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip auth for WebSocket upgrade requests (handled by their own auth)
|
||||||
|
if (request.headers.upgrade === "websocket") {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const authHeader = request.headers.authorization;
|
||||||
|
if (!authHeader) {
|
||||||
|
void reply.code(401).send({ error: "Authorization header required" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!authHeader.startsWith("Bearer ")) {
|
||||||
|
void reply.code(401).send({ error: "Invalid authorization format, expected: Bearer <token>" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const token = authHeader.slice(7);
|
||||||
|
if (token.length === 0) {
|
||||||
|
void reply.code(401).send({ error: "Empty bearer token" });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const identity = await authService.validateToken(token);
|
||||||
|
request.userId = identity.userId;
|
||||||
|
request.userEmail = identity.email;
|
||||||
|
request.userRole = identity.role;
|
||||||
|
} catch {
|
||||||
|
void reply.code(401).send({ error: "Invalid or expired token. Run: labctl login" });
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ import type { FastifyInstance } from "fastify";
|
|||||||
import type { DbClient } from "../server.js";
|
import type { DbClient } from "../server.js";
|
||||||
import { bastionRegistry } from "../services/bastion-registry.js";
|
import { bastionRegistry } from "../services/bastion-registry.js";
|
||||||
import { generateRequestId } from "@lab/shared";
|
import { generateRequestId } from "@lab/shared";
|
||||||
|
import type { VyosInstallSpec } from "@lab/shared";
|
||||||
|
|
||||||
const COMMAND_TIMEOUT_MS = 15_000;
|
const COMMAND_TIMEOUT_MS = 15_000;
|
||||||
|
|
||||||
@@ -84,7 +85,6 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
|||||||
app.get("/api/machines", async () => {
|
app.get("/api/machines", async () => {
|
||||||
const live = bastionRegistry.getAggregatedState();
|
const live = bastionRegistry.getAggregatedState();
|
||||||
|
|
||||||
// Merge DB records for machines not currently in any bastion's live state
|
|
||||||
try {
|
try {
|
||||||
const dbServers = (await db.server.findMany({})) as Array<{
|
const dbServers = (await db.server.findMany({})) as Array<{
|
||||||
mac: string | null; hostname: string; role: string; ip: string | null;
|
mac: string | null; hostname: string; role: string; ip: string | null;
|
||||||
@@ -93,9 +93,49 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
|||||||
for (const s of dbServers) {
|
for (const s of dbServers) {
|
||||||
if (!s.mac) continue;
|
if (!s.mac) continue;
|
||||||
const mac = s.mac.toLowerCase();
|
const mac = s.mac.toLowerCase();
|
||||||
// Only add from DB if not already in live state
|
|
||||||
|
// DB knows this machine has been installed at some point if it has a real
|
||||||
|
// hostname+role (not just product-name-as-hostname and role="unknown").
|
||||||
|
// Status alone is unreliable: a rediscovery can re-set it without erasing the
|
||||||
|
// install identity. If the bastion restarted and lost its installed map, the
|
||||||
|
// machine will only show up in live.discovered — promote it here so the CLI
|
||||||
|
// still sees hostname/role/IP.
|
||||||
|
const dbKnowsInstalled =
|
||||||
|
s.role !== "unknown" && s.role !== "" &&
|
||||||
|
s.hostname !== "" && s.hostname !== s.mac;
|
||||||
|
|
||||||
|
if (dbKnowsInstalled && !(mac in live.installed) && !(mac in live.install_queue)) {
|
||||||
|
const hw = live.discovered[mac];
|
||||||
|
live.installed[mac] = {
|
||||||
|
hostname: s.hostname,
|
||||||
|
role: s.role,
|
||||||
|
ip: s.ip ?? "",
|
||||||
|
installed_at: "",
|
||||||
|
bastionId: hw?.bastionId ?? "db",
|
||||||
|
...(hw ? {
|
||||||
|
product: hw.product,
|
||||||
|
manufacturer: hw.manufacturer,
|
||||||
|
cpu_model: hw.cpu_model,
|
||||||
|
cpu_cores: hw.cpu_cores,
|
||||||
|
memory_gb: hw.memory_gb,
|
||||||
|
arch: hw.arch,
|
||||||
|
} : {}),
|
||||||
|
};
|
||||||
|
delete live.discovered[mac];
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unknown-to-live MAC: fall back to whatever the DB says.
|
||||||
if (!(mac in live.discovered) && !(mac in live.install_queue) && !(mac in live.installed)) {
|
if (!(mac in live.discovered) && !(mac in live.install_queue) && !(mac in live.installed)) {
|
||||||
if (s.status === "discovered") {
|
if (s.status === "online" || s.status === "offline") {
|
||||||
|
live.installed[mac] = {
|
||||||
|
hostname: s.hostname,
|
||||||
|
role: s.role,
|
||||||
|
ip: s.ip ?? "",
|
||||||
|
installed_at: "",
|
||||||
|
bastionId: "db",
|
||||||
|
};
|
||||||
|
} else {
|
||||||
live.discovered[mac] = {
|
live.discovered[mac] = {
|
||||||
mac,
|
mac,
|
||||||
product: String(s.labels?.product ?? "unknown"),
|
product: String(s.labels?.product ?? "unknown"),
|
||||||
@@ -112,14 +152,6 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
|||||||
last_seen: "",
|
last_seen: "",
|
||||||
bastionId: "db",
|
bastionId: "db",
|
||||||
};
|
};
|
||||||
} else if (s.status === "online" || s.status === "offline") {
|
|
||||||
live.installed[mac] = {
|
|
||||||
hostname: s.hostname,
|
|
||||||
role: s.role,
|
|
||||||
ip: s.ip ?? "",
|
|
||||||
installed_at: "",
|
|
||||||
bastionId: "db",
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -132,9 +164,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
|||||||
|
|
||||||
// Queue install — route to correct bastion by MAC
|
// Queue install — route to correct bastion by MAC
|
||||||
app.post<{
|
app.post<{
|
||||||
Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string };
|
Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string; vyos?: VyosInstallSpec };
|
||||||
}>("/api/machines/install", async (request, reply) => {
|
}>("/api/machines/install", async (request, reply) => {
|
||||||
const { mac, hostname, disk, role, os } = request.body ?? {};
|
const { mac, hostname, disk, role, os, vyos } = request.body ?? {};
|
||||||
if (!mac || !hostname) {
|
if (!mac || !hostname) {
|
||||||
return reply.code(400).send({ error: "mac and hostname are required" });
|
return reply.code(400).send({ error: "mac and hostname are required" });
|
||||||
}
|
}
|
||||||
@@ -152,6 +184,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
|||||||
const result = await sendCommand(all[0]!.bastionId, {
|
const result = await sendCommand(all[0]!.bastionId, {
|
||||||
type: "command-install",
|
type: "command-install",
|
||||||
mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
|
mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
|
||||||
|
...(vyos ? { vyos } : {}),
|
||||||
});
|
});
|
||||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -165,6 +198,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
|||||||
const result = await sendCommand(bastion.bastionId, {
|
const result = await sendCommand(bastion.bastionId, {
|
||||||
type: "command-install",
|
type: "command-install",
|
||||||
mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
|
mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
|
||||||
|
...(vyos ? { vyos } : {}),
|
||||||
});
|
});
|
||||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|||||||
191
bastion/src/labd/src/routes/environments.ts
Normal file
191
bastion/src/labd/src/routes/environments.ts
Normal file
@@ -0,0 +1,191 @@
|
|||||||
|
// Environment and Account management routes.
|
||||||
|
// GET/POST /api/environments — list/create environments
|
||||||
|
// GET/POST /api/accounts — list/create accounts
|
||||||
|
// POST /api/accounts/bind — bind account to environment
|
||||||
|
// GET /api/bindings — list bindings
|
||||||
|
|
||||||
|
import type { FastifyInstance } from "fastify";
|
||||||
|
import type { PrismaClient, Prisma } from "@prisma/client";
|
||||||
|
import type { RbacService } from "../services/rbac.js";
|
||||||
|
import type { AuditService } from "../services/audit.js";
|
||||||
|
|
||||||
|
export function registerEnvironmentRoutes(
|
||||||
|
app: FastifyInstance,
|
||||||
|
db: PrismaClient,
|
||||||
|
rbacService: RbacService,
|
||||||
|
auditService: AuditService,
|
||||||
|
): void {
|
||||||
|
// List environments
|
||||||
|
app.get("/api/environments", async (_request, reply) => {
|
||||||
|
const envs = await db.environment.findMany({ orderBy: { name: "asc" } });
|
||||||
|
return reply.send(envs);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Create environment
|
||||||
|
app.post<{
|
||||||
|
Body: { name?: string };
|
||||||
|
}>("/api/environments", async (request, reply) => {
|
||||||
|
const { name } = request.body ?? {};
|
||||||
|
if (!name) {
|
||||||
|
return reply.code(400).send({ error: "name is required" });
|
||||||
|
}
|
||||||
|
|
||||||
|
const rbac = await rbacService.check({
|
||||||
|
userId: request.userId!,
|
||||||
|
userEmail: request.userEmail!,
|
||||||
|
userRole: request.userRole!,
|
||||||
|
action: "admin",
|
||||||
|
resource: "environments",
|
||||||
|
});
|
||||||
|
if (!rbac.allowed) {
|
||||||
|
return reply.code(403).send({ error: rbac.reason });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const env = await db.environment.create({ data: { name } });
|
||||||
|
auditService.emit({
|
||||||
|
eventKind: "resource_created",
|
||||||
|
source: "labd",
|
||||||
|
verified: true,
|
||||||
|
userId: request.userId ?? null,
|
||||||
|
resourceKind: "environment",
|
||||||
|
resourceName: name,
|
||||||
|
result: "success",
|
||||||
|
});
|
||||||
|
return reply.code(201).send(env);
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof Error && err.message.includes("Unique constraint")) {
|
||||||
|
return reply.code(409).send({ error: `Environment '${name}' already exists` });
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// List accounts
|
||||||
|
app.get("/api/accounts", async (_request, reply) => {
|
||||||
|
const accounts = await db.account.findMany({
|
||||||
|
orderBy: { name: "asc" },
|
||||||
|
select: { id: true, name: true, driver: true, config: true, createdAt: true, updatedAt: true },
|
||||||
|
});
|
||||||
|
return reply.send(accounts);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Create account
|
||||||
|
app.post<{
|
||||||
|
Body: { name?: string; driver?: string; config?: Record<string, unknown> };
|
||||||
|
}>("/api/accounts", async (request, reply) => {
|
||||||
|
const { name, driver, config } = request.body ?? {};
|
||||||
|
if (!name || !driver) {
|
||||||
|
return reply.code(400).send({ error: "name and driver are required" });
|
||||||
|
}
|
||||||
|
|
||||||
|
const rbac = await rbacService.check({
|
||||||
|
userId: request.userId!,
|
||||||
|
userEmail: request.userEmail!,
|
||||||
|
userRole: request.userRole!,
|
||||||
|
action: "admin",
|
||||||
|
resource: "accounts",
|
||||||
|
});
|
||||||
|
if (!rbac.allowed) {
|
||||||
|
return reply.code(403).send({ error: rbac.reason });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const account = await db.account.create({
|
||||||
|
data: { name, driver, config: (config ?? {}) as Prisma.InputJsonValue },
|
||||||
|
});
|
||||||
|
auditService.emit({
|
||||||
|
eventKind: "resource_created",
|
||||||
|
source: "labd",
|
||||||
|
verified: true,
|
||||||
|
userId: request.userId ?? null,
|
||||||
|
resourceKind: "account",
|
||||||
|
resourceName: name,
|
||||||
|
result: "success",
|
||||||
|
details: { driver },
|
||||||
|
});
|
||||||
|
return reply.code(201).send(account);
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof Error && err.message.includes("Unique constraint")) {
|
||||||
|
return reply.code(409).send({ error: `Account '${name}' already exists` });
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Bind account to environment
|
||||||
|
app.post<{
|
||||||
|
Body: { environmentId?: string; accountId?: string };
|
||||||
|
}>("/api/accounts/bind", async (request, reply) => {
|
||||||
|
const { environmentId, accountId } = request.body ?? {};
|
||||||
|
if (!environmentId || !accountId) {
|
||||||
|
return reply.code(400).send({ error: "environmentId and accountId are required" });
|
||||||
|
}
|
||||||
|
|
||||||
|
const rbac = await rbacService.check({
|
||||||
|
userId: request.userId!,
|
||||||
|
userEmail: request.userEmail!,
|
||||||
|
userRole: request.userRole!,
|
||||||
|
action: "admin",
|
||||||
|
resource: "accounts",
|
||||||
|
});
|
||||||
|
if (!rbac.allowed) {
|
||||||
|
return reply.code(403).send({ error: rbac.reason });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const binding = await db.binding.create({
|
||||||
|
data: { environmentId, accountId },
|
||||||
|
});
|
||||||
|
return reply.code(201).send(binding);
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof Error && err.message.includes("Unique constraint")) {
|
||||||
|
return reply.code(409).send({ error: "This account is already bound to this environment" });
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// List bindings
|
||||||
|
app.get("/api/bindings", async (_request, reply) => {
|
||||||
|
const bindings = await db.binding.findMany({
|
||||||
|
include: { environment: true, account: true },
|
||||||
|
});
|
||||||
|
return reply.send(bindings);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Audit event query
|
||||||
|
app.get<{
|
||||||
|
Querystring: {
|
||||||
|
last?: string;
|
||||||
|
kind?: string;
|
||||||
|
env?: string;
|
||||||
|
correlation?: string;
|
||||||
|
limit?: string;
|
||||||
|
};
|
||||||
|
}>("/api/events", async (request, reply) => {
|
||||||
|
const { last, kind, env, correlation, limit } = request.query as { last?: string; kind?: string; env?: string; correlation?: string; limit?: string };
|
||||||
|
|
||||||
|
const where: Record<string, unknown> = {};
|
||||||
|
|
||||||
|
if (last) {
|
||||||
|
const match = last.match(/^(\d+)(h|d|m)$/);
|
||||||
|
if (match) {
|
||||||
|
const [, num, unit] = match;
|
||||||
|
const ms = { h: 3_600_000, d: 86_400_000, m: 60_000 }[unit!]!;
|
||||||
|
where.timestamp = { gte: new Date(Date.now() - parseInt(num!) * ms) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (kind) where.eventKind = kind;
|
||||||
|
if (env) where.environmentName = env;
|
||||||
|
if (correlation) where.correlationId = correlation;
|
||||||
|
|
||||||
|
const events = await db.auditEvent.findMany({
|
||||||
|
where,
|
||||||
|
orderBy: { timestamp: "desc" },
|
||||||
|
take: Math.min(parseInt(limit ?? "100"), 500),
|
||||||
|
});
|
||||||
|
|
||||||
|
return reply.send(events);
|
||||||
|
});
|
||||||
|
}
|
||||||
196
bastion/src/labd/src/routes/resources.ts
Normal file
196
bastion/src/labd/src/routes/resources.ts
Normal file
@@ -0,0 +1,196 @@
|
|||||||
|
// Resource CRUD routes with RBAC enforcement.
|
||||||
|
// GET /api/resources — list (filtered by RBAC scope)
|
||||||
|
// GET /api/resources/:id — get
|
||||||
|
// POST /api/resources — create
|
||||||
|
// PUT /api/resources/:id — update
|
||||||
|
// DELETE /api/resources/:id — delete (marks as deleting)
|
||||||
|
|
||||||
|
import type { FastifyInstance } from "fastify";
|
||||||
|
import type { ResourceStore, CreateResourceInput } from "../services/resource-store.js";
|
||||||
|
import type { RbacService } from "../services/rbac.js";
|
||||||
|
import type { AuditService } from "../services/audit.js";
|
||||||
|
import { resolveResourceKind } from "@lab/core";
|
||||||
|
|
||||||
|
export function registerResourceRoutes(
|
||||||
|
app: FastifyInstance,
|
||||||
|
resourceStore: ResourceStore,
|
||||||
|
rbacService: RbacService,
|
||||||
|
auditService: AuditService,
|
||||||
|
): void {
|
||||||
|
// List resources (filtered by kind, environment, status)
|
||||||
|
app.get<{
|
||||||
|
Querystring: { kind?: string; environment?: string; status?: string };
|
||||||
|
}>("/api/resources", async (request, reply) => {
|
||||||
|
const rbac = await rbacService.check({
|
||||||
|
userId: request.userId!,
|
||||||
|
userEmail: request.userEmail!,
|
||||||
|
userRole: request.userRole!,
|
||||||
|
action: "view",
|
||||||
|
resource: request.query.kind ? resolveResourceKind(request.query.kind) : undefined,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!rbac.allowed) {
|
||||||
|
return reply.code(403).send({ error: rbac.reason });
|
||||||
|
}
|
||||||
|
|
||||||
|
const resources = await resourceStore.list({
|
||||||
|
kind: request.query.kind ? resolveResourceKind(request.query.kind) : undefined,
|
||||||
|
environmentId: request.query.environment,
|
||||||
|
status: request.query.status,
|
||||||
|
});
|
||||||
|
|
||||||
|
return reply.send(resources);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Get single resource
|
||||||
|
app.get<{
|
||||||
|
Params: { id: string };
|
||||||
|
}>("/api/resources/:id", async (request, reply) => {
|
||||||
|
const resource = await resourceStore.get(request.params.id);
|
||||||
|
if (!resource) {
|
||||||
|
return reply.code(404).send({ error: "Resource not found" });
|
||||||
|
}
|
||||||
|
|
||||||
|
const rbac = await rbacService.check({
|
||||||
|
userId: request.userId!,
|
||||||
|
userEmail: request.userEmail!,
|
||||||
|
userRole: request.userRole!,
|
||||||
|
action: "view",
|
||||||
|
resource: resource.kind,
|
||||||
|
name: resource.name,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!rbac.allowed) {
|
||||||
|
return reply.code(403).send({ error: rbac.reason });
|
||||||
|
}
|
||||||
|
|
||||||
|
return reply.send(resource);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Create resource
|
||||||
|
app.post<{
|
||||||
|
Body: CreateResourceInput;
|
||||||
|
}>("/api/resources", async (request, reply) => {
|
||||||
|
const input = request.body;
|
||||||
|
if (!input?.kind || !input?.name || !input?.environmentId || !input?.accountId) {
|
||||||
|
return reply.code(400).send({ error: "kind, name, environmentId, and accountId are required" });
|
||||||
|
}
|
||||||
|
|
||||||
|
const kind = resolveResourceKind(input.kind);
|
||||||
|
|
||||||
|
const rbac = await rbacService.check({
|
||||||
|
userId: request.userId!,
|
||||||
|
userEmail: request.userEmail!,
|
||||||
|
userRole: request.userRole!,
|
||||||
|
action: "create",
|
||||||
|
resource: kind,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!rbac.allowed) {
|
||||||
|
return reply.code(403).send({ error: rbac.reason });
|
||||||
|
}
|
||||||
|
|
||||||
|
const correlationId = auditService.createCorrelation();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const resource = await resourceStore.create({ ...input, kind });
|
||||||
|
|
||||||
|
auditService.emit({
|
||||||
|
eventKind: "resource_created",
|
||||||
|
source: "labd",
|
||||||
|
verified: true,
|
||||||
|
userId: request.userId ?? null,
|
||||||
|
userName: request.userEmail ?? null,
|
||||||
|
resourceKind: kind,
|
||||||
|
resourceName: input.name,
|
||||||
|
correlationId,
|
||||||
|
result: "success",
|
||||||
|
});
|
||||||
|
|
||||||
|
return reply.code(201).send(resource);
|
||||||
|
} catch (err) {
|
||||||
|
// Prisma unique constraint violation
|
||||||
|
if (err instanceof Error && err.message.includes("Unique constraint")) {
|
||||||
|
return reply.code(409).send({ error: `Resource ${kind}/${input.name} already exists in this environment` });
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Update resource
|
||||||
|
app.put<{
|
||||||
|
Params: { id: string };
|
||||||
|
Body: { desiredSpec?: Record<string, unknown>; status?: string };
|
||||||
|
}>("/api/resources/:id", async (request, reply) => {
|
||||||
|
const resource = await resourceStore.get(request.params.id);
|
||||||
|
if (!resource) {
|
||||||
|
return reply.code(404).send({ error: "Resource not found" });
|
||||||
|
}
|
||||||
|
|
||||||
|
const rbac = await rbacService.check({
|
||||||
|
userId: request.userId!,
|
||||||
|
userEmail: request.userEmail!,
|
||||||
|
userRole: request.userRole!,
|
||||||
|
action: "edit",
|
||||||
|
resource: resource.kind,
|
||||||
|
name: resource.name,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!rbac.allowed) {
|
||||||
|
return reply.code(403).send({ error: rbac.reason });
|
||||||
|
}
|
||||||
|
|
||||||
|
const updated = await resourceStore.update(request.params.id, request.body);
|
||||||
|
|
||||||
|
auditService.emit({
|
||||||
|
eventKind: "resource_updated",
|
||||||
|
source: "labd",
|
||||||
|
verified: true,
|
||||||
|
userId: request.userId ?? null,
|
||||||
|
userName: request.userEmail ?? null,
|
||||||
|
resourceKind: resource.kind,
|
||||||
|
resourceName: resource.name,
|
||||||
|
result: "success",
|
||||||
|
});
|
||||||
|
|
||||||
|
return reply.send(updated);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Delete resource (marks as deleting)
|
||||||
|
app.delete<{
|
||||||
|
Params: { id: string };
|
||||||
|
}>("/api/resources/:id", async (request, reply) => {
|
||||||
|
const resource = await resourceStore.get(request.params.id);
|
||||||
|
if (!resource) {
|
||||||
|
return reply.code(404).send({ error: "Resource not found" });
|
||||||
|
}
|
||||||
|
|
||||||
|
const rbac = await rbacService.check({
|
||||||
|
userId: request.userId!,
|
||||||
|
userEmail: request.userEmail!,
|
||||||
|
userRole: request.userRole!,
|
||||||
|
action: "delete",
|
||||||
|
resource: resource.kind,
|
||||||
|
name: resource.name,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!rbac.allowed) {
|
||||||
|
return reply.code(403).send({ error: rbac.reason });
|
||||||
|
}
|
||||||
|
|
||||||
|
await resourceStore.delete(request.params.id);
|
||||||
|
|
||||||
|
auditService.emit({
|
||||||
|
eventKind: "resource_deleted",
|
||||||
|
source: "labd",
|
||||||
|
verified: true,
|
||||||
|
userId: request.userId ?? null,
|
||||||
|
userName: request.userEmail ?? null,
|
||||||
|
resourceKind: resource.kind,
|
||||||
|
resourceName: resource.name,
|
||||||
|
result: "success",
|
||||||
|
});
|
||||||
|
|
||||||
|
return reply.send({ status: "deleting", id: request.params.id });
|
||||||
|
});
|
||||||
|
}
|
||||||
81
bastion/src/labd/src/routes/v2-auth.ts
Normal file
81
bastion/src/labd/src/routes/v2-auth.ts
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
// v2 Auth routes: bearer token login/logout.
|
||||||
|
// POST /api/auth/login — email + password → session token
|
||||||
|
// POST /api/auth/logout — revoke session
|
||||||
|
|
||||||
|
import type { FastifyInstance } from "fastify";
|
||||||
|
import type { AuthService } from "../services/auth.js";
|
||||||
|
import type { AuditService } from "../services/audit.js";
|
||||||
|
import { AuthError } from "../services/auth.js";
|
||||||
|
|
||||||
|
export function registerV2AuthRoutes(
|
||||||
|
app: FastifyInstance,
|
||||||
|
authService: AuthService,
|
||||||
|
auditService: AuditService,
|
||||||
|
): void {
|
||||||
|
app.post<{
|
||||||
|
Body: { email?: string; password?: string };
|
||||||
|
}>("/api/auth/login", async (request, reply) => {
|
||||||
|
const { email, password } = request.body ?? {};
|
||||||
|
|
||||||
|
if (!email || !password) {
|
||||||
|
return reply.code(400).send({ error: "email and password are required" });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await authService.login(email, password);
|
||||||
|
|
||||||
|
auditService.emit({
|
||||||
|
eventKind: result.isBootstrap ? "auth_bootstrap" : "auth_login",
|
||||||
|
source: "labd",
|
||||||
|
verified: true,
|
||||||
|
userId: result.userId,
|
||||||
|
userName: email,
|
||||||
|
result: "success",
|
||||||
|
details: { isBootstrap: result.isBootstrap },
|
||||||
|
});
|
||||||
|
|
||||||
|
return reply.send({
|
||||||
|
token: result.token,
|
||||||
|
expiresAt: result.expiresAt.toISOString(),
|
||||||
|
isBootstrap: result.isBootstrap,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof AuthError) {
|
||||||
|
auditService.emit({
|
||||||
|
eventKind: "auth_login",
|
||||||
|
source: "labd",
|
||||||
|
verified: true,
|
||||||
|
userName: email,
|
||||||
|
result: "failure",
|
||||||
|
error: err.message,
|
||||||
|
});
|
||||||
|
return reply.code(401).send({ error: err.message });
|
||||||
|
}
|
||||||
|
return reply.code(500).send({ error: "Login failed" });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
app.post("/api/auth/logout", async (request, reply) => {
|
||||||
|
const token = request.headers.authorization?.slice(7);
|
||||||
|
if (!token) {
|
||||||
|
return reply.code(400).send({ error: "Authorization header required" });
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await authService.logout(token);
|
||||||
|
auditService.emit({
|
||||||
|
eventKind: "auth_logout",
|
||||||
|
source: "labd",
|
||||||
|
verified: true,
|
||||||
|
userId: request.userId ?? null,
|
||||||
|
result: "success",
|
||||||
|
});
|
||||||
|
return reply.send({ status: "logged_out" });
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof AuthError) {
|
||||||
|
return reply.code(400).send({ error: err.message });
|
||||||
|
}
|
||||||
|
return reply.code(500).send({ error: "Logout failed" });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import Fastify from "fastify";
|
import Fastify from "fastify";
|
||||||
import websocket from "@fastify/websocket";
|
import websocket from "@fastify/websocket";
|
||||||
|
import type { PrismaClient } from "@prisma/client";
|
||||||
import type { LabdConfig } from "./config.js";
|
import type { LabdConfig } from "./config.js";
|
||||||
import { logger } from "./services/logger.js";
|
import { logger } from "./services/logger.js";
|
||||||
import { registerHealthRoutes } from "./routes/health.js";
|
import { registerHealthRoutes } from "./routes/health.js";
|
||||||
@@ -9,8 +10,16 @@ import { registerServerRoutes } from "./routes/servers.js";
|
|||||||
import { registerAuthRoutes } from "./routes/auth.js";
|
import { registerAuthRoutes } from "./routes/auth.js";
|
||||||
import { registerAgentRoutes } from "./routes/agents.js";
|
import { registerAgentRoutes } from "./routes/agents.js";
|
||||||
import { registerBastionRoutes } from "./routes/bastions.js";
|
import { registerBastionRoutes } from "./routes/bastions.js";
|
||||||
|
import { registerV2AuthRoutes } from "./routes/v2-auth.js";
|
||||||
|
import { registerEnvironmentRoutes } from "./routes/environments.js";
|
||||||
|
import { registerResourceRoutes } from "./routes/resources.js";
|
||||||
import { setupRateLimiting } from "./middleware/rate-limit.js";
|
import { setupRateLimiting } from "./middleware/rate-limit.js";
|
||||||
|
import { createBearerAuthMiddleware } from "./middleware/bearer-auth.js";
|
||||||
import { bastionRegistry } from "./services/bastion-registry.js";
|
import { bastionRegistry } from "./services/bastion-registry.js";
|
||||||
|
import { AuthService } from "./services/auth.js";
|
||||||
|
import { RbacService } from "./services/rbac.js";
|
||||||
|
import { ResourceStore } from "./services/resource-store.js";
|
||||||
|
import { AuditService } from "./services/audit.js";
|
||||||
import { isBastionMessage } from "@lab/shared";
|
import { isBastionMessage } from "@lab/shared";
|
||||||
|
|
||||||
export interface DbClient {
|
export interface DbClient {
|
||||||
@@ -37,6 +46,7 @@ export interface DbClient {
|
|||||||
|
|
||||||
export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
|
export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
|
||||||
app: ReturnType<typeof Fastify>;
|
app: ReturnType<typeof Fastify>;
|
||||||
|
auditService: AuditService;
|
||||||
}> {
|
}> {
|
||||||
const app = Fastify({
|
const app = Fastify({
|
||||||
logger: false, // We use winston instead
|
logger: false, // We use winston instead
|
||||||
@@ -48,13 +58,39 @@ export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
|
|||||||
// Register WebSocket support
|
// Register WebSocket support
|
||||||
void app.register(websocket);
|
void app.register(websocket);
|
||||||
|
|
||||||
// Register route handlers
|
// v2 services. The structural DbClient is a subset of the real PrismaClient;
|
||||||
|
// at runtime db IS the PrismaClient instance, so the cast is safe. Tests that
|
||||||
|
// exercise v2 routes provide a PrismaClient-shaped mock (see auth-bootstrap,
|
||||||
|
// rbac-deny, audit-correlation tests).
|
||||||
|
const prisma = db as unknown as PrismaClient;
|
||||||
|
const authService = new AuthService(prisma);
|
||||||
|
const rbacService = new RbacService(prisma);
|
||||||
|
const resourceStore = new ResourceStore(prisma);
|
||||||
|
const auditService = new AuditService(prisma);
|
||||||
|
auditService.start();
|
||||||
|
|
||||||
|
// Register v1 (legacy) route handlers
|
||||||
registerHealthRoutes(app, db);
|
registerHealthRoutes(app, db);
|
||||||
registerServerRoutes(app, db);
|
registerServerRoutes(app, db);
|
||||||
registerAuthRoutes(app, db);
|
registerAuthRoutes(app, db);
|
||||||
registerAgentRoutes(app);
|
registerAgentRoutes(app);
|
||||||
registerBastionRoutes(app, db);
|
registerBastionRoutes(app, db);
|
||||||
|
|
||||||
|
// v2 routes live in a scope with bearer-auth as preHandler. Public paths
|
||||||
|
// (login, /health, websockets) are skipped inside the middleware itself.
|
||||||
|
// v1 routes above are unaffected — they're registered on the root scope.
|
||||||
|
await app.register(async (scope) => {
|
||||||
|
scope.addHook("preHandler", createBearerAuthMiddleware(authService));
|
||||||
|
registerV2AuthRoutes(scope, authService, auditService);
|
||||||
|
registerEnvironmentRoutes(scope, prisma, rbacService, auditService);
|
||||||
|
registerResourceRoutes(scope, resourceStore, rbacService, auditService);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Flush pending audit events on shutdown so we never lose the last batch.
|
||||||
|
app.addHook("onClose", async () => {
|
||||||
|
auditService.stop();
|
||||||
|
});
|
||||||
|
|
||||||
// WebSocket handler for agent connections
|
// WebSocket handler for agent connections
|
||||||
app.register(async (fastify) => {
|
app.register(async (fastify) => {
|
||||||
fastify.get("/ws/agent", { websocket: true }, (socket, _request) => {
|
fastify.get("/ws/agent", { websocket: true }, (socket, _request) => {
|
||||||
@@ -192,7 +228,9 @@ export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
|
|||||||
labels: { cpu: hw.cpu_model, cores: hw.cpu_cores, memory_gb: hw.memory_gb, arch: hw.arch, product: hw.product, manufacturer: hw.manufacturer },
|
labels: { cpu: hw.cpu_model, cores: hw.cpu_cores, memory_gb: hw.memory_gb, arch: hw.arch, product: hw.product, manufacturer: hw.manufacturer },
|
||||||
},
|
},
|
||||||
update: {
|
update: {
|
||||||
status: "discovered",
|
// Leave status alone — a previously "online"/"offline" record
|
||||||
|
// must not be downgraded to "discovered" just because the bastion
|
||||||
|
// restarted and re-discovered the MAC via DHCP/PXE.
|
||||||
lastHeartbeat: new Date(),
|
lastHeartbeat: new Date(),
|
||||||
labels: { cpu: hw.cpu_model, cores: hw.cpu_cores, memory_gb: hw.memory_gb, arch: hw.arch, product: hw.product, manufacturer: hw.manufacturer },
|
labels: { cpu: hw.cpu_model, cores: hw.cpu_cores, memory_gb: hw.memory_gb, arch: hw.arch, product: hw.product, manufacturer: hw.manufacturer },
|
||||||
},
|
},
|
||||||
@@ -265,5 +303,5 @@ export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
|
|||||||
logger.info(`HTTP: ${request.ip} ${request.method} ${request.url}`);
|
logger.info(`HTTP: ${request.ip} ${request.method} ${request.url}`);
|
||||||
});
|
});
|
||||||
|
|
||||||
return { app };
|
return { app, auditService };
|
||||||
}
|
}
|
||||||
|
|||||||
106
bastion/src/labd/src/services/audit.ts
Normal file
106
bastion/src/labd/src/services/audit.ts
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
// Audit service: fire-and-forget event collection with batching.
|
||||||
|
// Batches 50 events or flushes every 5 seconds, whichever comes first.
|
||||||
|
// Failures never block the operation being audited.
|
||||||
|
|
||||||
|
import { randomBytes } from "node:crypto";
|
||||||
|
import type { PrismaClient, Prisma } from "@prisma/client";
|
||||||
|
import { logger } from "./logger.js";
|
||||||
|
|
||||||
|
const BATCH_SIZE = 50;
|
||||||
|
const FLUSH_INTERVAL_MS = 5_000;
|
||||||
|
|
||||||
|
export interface AuditEventInput {
|
||||||
|
eventKind: string;
|
||||||
|
source: string;
|
||||||
|
verified?: boolean;
|
||||||
|
userId?: string | null;
|
||||||
|
userName?: string | null;
|
||||||
|
sessionId?: string | null;
|
||||||
|
environmentName?: string | null;
|
||||||
|
accountName?: string | null;
|
||||||
|
resourceKind?: string | null;
|
||||||
|
resourceName?: string | null;
|
||||||
|
correlationId?: string | null;
|
||||||
|
parentEventId?: string | null;
|
||||||
|
details?: Record<string, unknown>;
|
||||||
|
result: string;
|
||||||
|
error?: string | null;
|
||||||
|
durationMs?: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class AuditService {
|
||||||
|
private batch: AuditEventInput[] = [];
|
||||||
|
private timer: ReturnType<typeof setInterval> | null = null;
|
||||||
|
|
||||||
|
constructor(private readonly db: PrismaClient) {}
|
||||||
|
|
||||||
|
start(): void {
|
||||||
|
this.timer = setInterval(() => {
|
||||||
|
void this.flush();
|
||||||
|
}, FLUSH_INTERVAL_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
stop(): void {
|
||||||
|
if (this.timer) {
|
||||||
|
clearInterval(this.timer);
|
||||||
|
this.timer = null;
|
||||||
|
}
|
||||||
|
void this.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
emit(event: AuditEventInput): void {
|
||||||
|
// Generate correlation ID if not provided
|
||||||
|
if (!event.correlationId) {
|
||||||
|
event.correlationId = `corr_${randomBytes(8).toString("hex")}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.batch.push(event);
|
||||||
|
|
||||||
|
if (this.batch.length >= BATCH_SIZE) {
|
||||||
|
void this.flush();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Create a correlation context for a chain of related events. */
|
||||||
|
createCorrelation(): string {
|
||||||
|
return `corr_${randomBytes(8).toString("hex")}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Flush all pending events synchronously. Tests await this; production
|
||||||
|
* relies on the interval timer or stop() during shutdown. */
|
||||||
|
async flushPending(): Promise<void> {
|
||||||
|
await this.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async flush(): Promise<void> {
|
||||||
|
if (this.batch.length === 0) return;
|
||||||
|
|
||||||
|
const events = this.batch.splice(0);
|
||||||
|
try {
|
||||||
|
await this.db.auditEvent.createMany({
|
||||||
|
data: events.map((e) => ({
|
||||||
|
eventKind: e.eventKind,
|
||||||
|
source: e.source,
|
||||||
|
verified: e.verified ?? false,
|
||||||
|
userId: e.userId ?? null,
|
||||||
|
userName: e.userName ?? null,
|
||||||
|
sessionId: e.sessionId ?? null,
|
||||||
|
environmentName: e.environmentName ?? null,
|
||||||
|
accountName: e.accountName ?? null,
|
||||||
|
resourceKind: e.resourceKind ?? null,
|
||||||
|
resourceName: e.resourceName ?? null,
|
||||||
|
correlationId: e.correlationId ?? `corr_${randomBytes(8).toString("hex")}`,
|
||||||
|
parentEventId: e.parentEventId ?? null,
|
||||||
|
details: (e.details ?? {}) as Prisma.InputJsonValue,
|
||||||
|
result: e.result,
|
||||||
|
error: e.error ?? null,
|
||||||
|
durationMs: e.durationMs ?? null,
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
logger.info(`AUDIT: flushed ${events.length} events`);
|
||||||
|
} catch (err) {
|
||||||
|
// Fire-and-forget: audit failures never block operations
|
||||||
|
logger.warn(`AUDIT: failed to flush ${events.length} events: ${err instanceof Error ? err.message : String(err)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
119
bastion/src/labd/src/services/auth.ts
Normal file
119
bastion/src/labd/src/services/auth.ts
Normal file
@@ -0,0 +1,119 @@
|
|||||||
|
// Auth service: bearer token authentication with bootstrap flow.
|
||||||
|
// First login creates the admin user. Subsequent logins return session tokens.
|
||||||
|
|
||||||
|
import { randomBytes } from "node:crypto";
|
||||||
|
import bcrypt from "bcryptjs";
|
||||||
|
import type { PrismaClient } from "@prisma/client";
|
||||||
|
import { logger } from "./logger.js";
|
||||||
|
|
||||||
|
const SESSION_EXPIRY_DAYS = 30;
|
||||||
|
const BCRYPT_ROUNDS = 12;
|
||||||
|
|
||||||
|
export interface LoginResult {
|
||||||
|
token: string;
|
||||||
|
expiresAt: Date;
|
||||||
|
userId: string;
|
||||||
|
isBootstrap: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class AuthService {
|
||||||
|
constructor(private readonly db: PrismaClient) {}
|
||||||
|
|
||||||
|
async login(email: string, password: string): Promise<LoginResult> {
|
||||||
|
const userCount = await this.db.user.count();
|
||||||
|
|
||||||
|
// Bootstrap: first login creates admin user
|
||||||
|
if (userCount === 0) {
|
||||||
|
return this.bootstrap(email, password);
|
||||||
|
}
|
||||||
|
|
||||||
|
const user = await this.db.user.findUnique({ where: { email } });
|
||||||
|
if (!user) {
|
||||||
|
// Same error for unknown user and wrong password (no enumeration)
|
||||||
|
throw new AuthError("Invalid email or password");
|
||||||
|
}
|
||||||
|
|
||||||
|
const valid = await bcrypt.compare(password, user.password);
|
||||||
|
if (!valid) {
|
||||||
|
throw new AuthError("Invalid email or password");
|
||||||
|
}
|
||||||
|
|
||||||
|
const session = await this.createSession(user.id);
|
||||||
|
logger.info(`AUTH LOGIN: ${email} (${user.id.slice(0, 8)}...)`);
|
||||||
|
|
||||||
|
return {
|
||||||
|
token: session.token,
|
||||||
|
expiresAt: session.expiresAt,
|
||||||
|
userId: user.id,
|
||||||
|
isBootstrap: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async logout(token: string): Promise<void> {
|
||||||
|
const session = await this.db.session.findUnique({ where: { token } });
|
||||||
|
if (!session) {
|
||||||
|
throw new AuthError("Invalid session");
|
||||||
|
}
|
||||||
|
await this.db.session.delete({ where: { id: session.id } });
|
||||||
|
logger.info(`AUTH LOGOUT: session ${session.id.slice(0, 8)}...`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async validateToken(token: string): Promise<{ userId: string; email: string; role: string }> {
|
||||||
|
const session = await this.db.session.findUnique({
|
||||||
|
where: { token },
|
||||||
|
include: { user: true },
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!session) {
|
||||||
|
throw new AuthError("Invalid token");
|
||||||
|
}
|
||||||
|
if (session.expiresAt < new Date()) {
|
||||||
|
await this.db.session.delete({ where: { id: session.id } });
|
||||||
|
throw new AuthError("Token expired");
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
userId: session.user.id,
|
||||||
|
email: session.user.email,
|
||||||
|
role: session.user.role,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async bootstrap(email: string, password: string): Promise<LoginResult> {
|
||||||
|
const hashed = await bcrypt.hash(password, BCRYPT_ROUNDS);
|
||||||
|
const user = await this.db.user.create({
|
||||||
|
data: {
|
||||||
|
email,
|
||||||
|
password: hashed,
|
||||||
|
role: "ADMIN",
|
||||||
|
name: email.split("@")[0] ?? null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const session = await this.createSession(user.id);
|
||||||
|
logger.info(`AUTH BOOTSTRAP: created admin user ${email} (${user.id.slice(0, 8)}...)`);
|
||||||
|
|
||||||
|
return {
|
||||||
|
token: session.token,
|
||||||
|
expiresAt: session.expiresAt,
|
||||||
|
userId: user.id,
|
||||||
|
isBootstrap: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async createSession(userId: string) {
|
||||||
|
const token = randomBytes(32).toString("hex");
|
||||||
|
const expiresAt = new Date(Date.now() + SESSION_EXPIRY_DAYS * 24 * 60 * 60 * 1000);
|
||||||
|
|
||||||
|
return this.db.session.create({
|
||||||
|
data: { userId, token, expiresAt },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class AuthError extends Error {
|
||||||
|
constructor(message: string) {
|
||||||
|
super(message);
|
||||||
|
this.name = "AuthError";
|
||||||
|
}
|
||||||
|
}
|
||||||
123
bastion/src/labd/src/services/rbac.ts
Normal file
123
bastion/src/labd/src/services/rbac.ts
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
// RBAC service: environment-scoped permission checks.
|
||||||
|
// Uses named RbacDefinition records with JSON subjects and roleBindings.
|
||||||
|
//
|
||||||
|
// Resolution flow:
|
||||||
|
// 1. Find all RbacDefinitions where subjects match the current user/groups
|
||||||
|
// 2. Collect all roleBindings from matching definitions
|
||||||
|
// 3. Check if any binding grants the requested action on the requested resource
|
||||||
|
|
||||||
|
import type { PrismaClient } from "@prisma/client";
|
||||||
|
import { logger } from "./logger.js";
|
||||||
|
|
||||||
|
export interface RbacCheck {
|
||||||
|
userId: string;
|
||||||
|
userEmail: string;
|
||||||
|
userRole: string;
|
||||||
|
action: string; // "view" | "edit" | "create" | "delete" | "run" | "admin"
|
||||||
|
resource?: string | undefined; // "servers" | "databases" | "clusters" | "*"
|
||||||
|
name?: string | undefined; // specific resource name
|
||||||
|
environment?: string | undefined; // specific environment name
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RbacResult {
|
||||||
|
allowed: boolean;
|
||||||
|
reason: string;
|
||||||
|
matchedDefinition?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface StoredSubject {
|
||||||
|
kind: string;
|
||||||
|
name: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface StoredBinding {
|
||||||
|
role: string;
|
||||||
|
resource?: string;
|
||||||
|
name?: string;
|
||||||
|
environment?: string;
|
||||||
|
action?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RbacService {
|
||||||
|
constructor(private readonly db: PrismaClient) {}
|
||||||
|
|
||||||
|
async check(req: RbacCheck): Promise<RbacResult> {
|
||||||
|
// Admin users bypass RBAC
|
||||||
|
if (req.userRole === "ADMIN") {
|
||||||
|
return { allowed: true, reason: "admin role" };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect user's group memberships
|
||||||
|
const memberships = await this.db.groupMember.findMany({
|
||||||
|
where: { userId: req.userId },
|
||||||
|
include: { group: true },
|
||||||
|
});
|
||||||
|
const groupNames = memberships.map((m) => m.group.name);
|
||||||
|
|
||||||
|
// Find all RBAC definitions
|
||||||
|
const definitions = await this.db.rbacDefinition.findMany();
|
||||||
|
|
||||||
|
for (const def of definitions) {
|
||||||
|
const subjects = def.subjects as unknown as StoredSubject[];
|
||||||
|
const bindings = def.roleBindings as unknown as StoredBinding[];
|
||||||
|
|
||||||
|
// Check if this definition's subjects match the user
|
||||||
|
const subjectMatch = subjects.some((s) => {
|
||||||
|
if (s.kind === "User" && s.name === req.userEmail) return true;
|
||||||
|
if (s.kind === "Group" && groupNames.includes(s.name)) return true;
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!subjectMatch) continue;
|
||||||
|
|
||||||
|
// Check if any binding grants the requested permission
|
||||||
|
for (const binding of bindings) {
|
||||||
|
if (this.bindingMatches(binding, req)) {
|
||||||
|
logger.info(`RBAC ALLOW: ${req.userEmail} ${req.action} ${req.resource ?? "*"}${req.name ? `/${req.name}` : ""} via ${def.name}`);
|
||||||
|
return {
|
||||||
|
allowed: true,
|
||||||
|
reason: `granted by ${def.name}`,
|
||||||
|
matchedDefinition: def.name,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`RBAC DENY: ${req.userEmail} ${req.action} ${req.resource ?? "*"}${req.name ? `/${req.name}` : ""}`);
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: `no matching role binding for ${req.action} on ${req.resource ?? "*"}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private bindingMatches(binding: StoredBinding, req: RbacCheck): boolean {
|
||||||
|
// Check role grants the action
|
||||||
|
if (!this.roleGrantsAction(binding.role, req.action)) return false;
|
||||||
|
|
||||||
|
// Check resource scope
|
||||||
|
if (binding.resource && binding.resource !== "*" && binding.resource !== req.resource) return false;
|
||||||
|
|
||||||
|
// Check name scope
|
||||||
|
if (binding.name && binding.name !== req.name) return false;
|
||||||
|
|
||||||
|
// Check environment scope
|
||||||
|
if (binding.environment && binding.environment !== req.environment) return false;
|
||||||
|
|
||||||
|
// Check operation scope (for "run" role with specific actions)
|
||||||
|
if (binding.action && binding.action !== "*" && binding.action !== req.action) return false;
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private roleGrantsAction(role: string, action: string): boolean {
|
||||||
|
const grants: Record<string, string[]> = {
|
||||||
|
admin: ["view", "edit", "create", "delete", "run", "admin"],
|
||||||
|
edit: ["view", "edit", "create", "delete"],
|
||||||
|
create: ["create"],
|
||||||
|
delete: ["delete"],
|
||||||
|
view: ["view"],
|
||||||
|
run: ["run"],
|
||||||
|
};
|
||||||
|
return grants[role]?.includes(action) ?? false;
|
||||||
|
}
|
||||||
|
}
|
||||||
108
bastion/src/labd/src/services/resource-store.ts
Normal file
108
bastion/src/labd/src/services/resource-store.ts
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
// Resource store: CRUD for generic resources with origin/managedBy tracking.
|
||||||
|
// All mutations go through this service so RBAC and audit are applied consistently.
|
||||||
|
|
||||||
|
import type { PrismaClient, Resource as PrismaResource, Prisma } from "@prisma/client";
|
||||||
|
import { logger } from "./logger.js";
|
||||||
|
|
||||||
|
export interface CreateResourceInput {
|
||||||
|
kind: string;
|
||||||
|
name: string;
|
||||||
|
environmentId: string;
|
||||||
|
accountId: string;
|
||||||
|
origin?: string;
|
||||||
|
managedBy?: string;
|
||||||
|
sourceRef?: string;
|
||||||
|
desiredSpec: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface UpdateResourceInput {
|
||||||
|
desiredSpec?: Record<string, unknown>;
|
||||||
|
status?: string;
|
||||||
|
statusMessage?: string;
|
||||||
|
actualSpec?: Record<string, unknown>;
|
||||||
|
platformRef?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ListResourcesFilter {
|
||||||
|
kind?: string | undefined;
|
||||||
|
environmentId?: string | undefined;
|
||||||
|
accountId?: string | undefined;
|
||||||
|
status?: string | undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class ResourceStore {
|
||||||
|
constructor(private readonly db: PrismaClient) {}
|
||||||
|
|
||||||
|
async create(input: CreateResourceInput): Promise<PrismaResource> {
|
||||||
|
const resource = await this.db.resource.create({
|
||||||
|
data: {
|
||||||
|
kind: input.kind,
|
||||||
|
name: input.name,
|
||||||
|
environmentId: input.environmentId,
|
||||||
|
accountId: input.accountId,
|
||||||
|
origin: input.origin ?? "cli",
|
||||||
|
managedBy: input.managedBy ?? "manual",
|
||||||
|
sourceRef: input.sourceRef ?? null,
|
||||||
|
desiredSpec: input.desiredSpec as Prisma.InputJsonValue,
|
||||||
|
status: "pending",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.info(`RESOURCE CREATED: ${input.kind}/${input.name} in env ${input.environmentId.slice(0, 8)}...`);
|
||||||
|
return resource;
|
||||||
|
}
|
||||||
|
|
||||||
|
async get(id: string): Promise<PrismaResource | null> {
|
||||||
|
return this.db.resource.findUnique({ where: { id } });
|
||||||
|
}
|
||||||
|
|
||||||
|
async getByKindNameEnv(kind: string, name: string, environmentId: string): Promise<PrismaResource | null> {
|
||||||
|
return this.db.resource.findUnique({
|
||||||
|
where: { kind_name_environmentId: { kind, name, environmentId } },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async list(filter: ListResourcesFilter = {}): Promise<PrismaResource[]> {
|
||||||
|
return this.db.resource.findMany({
|
||||||
|
where: {
|
||||||
|
...(filter.kind ? { kind: filter.kind } : {}),
|
||||||
|
...(filter.environmentId ? { environmentId: filter.environmentId } : {}),
|
||||||
|
...(filter.accountId ? { accountId: filter.accountId } : {}),
|
||||||
|
...(filter.status ? { status: filter.status } : {}),
|
||||||
|
},
|
||||||
|
orderBy: { createdAt: "desc" },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async update(id: string, input: UpdateResourceInput): Promise<PrismaResource> {
|
||||||
|
const data: Prisma.ResourceUpdateInput = {};
|
||||||
|
if (input.desiredSpec !== undefined) data.desiredSpec = input.desiredSpec as Prisma.InputJsonValue;
|
||||||
|
if (input.status !== undefined) data.status = input.status;
|
||||||
|
if (input.statusMessage !== undefined) data.statusMessage = input.statusMessage;
|
||||||
|
if (input.actualSpec !== undefined) data.actualSpec = input.actualSpec as Prisma.InputJsonValue;
|
||||||
|
if (input.platformRef !== undefined) data.platformRef = input.platformRef;
|
||||||
|
if (input.status === "ready") data.lastReconciled = new Date();
|
||||||
|
|
||||||
|
const resource = await this.db.resource.update({ where: { id }, data });
|
||||||
|
|
||||||
|
logger.info(`RESOURCE UPDATED: ${resource.kind}/${resource.name} -> ${input.status ?? "spec change"}`);
|
||||||
|
return resource;
|
||||||
|
}
|
||||||
|
|
||||||
|
async delete(id: string): Promise<void> {
|
||||||
|
const resource = await this.db.resource.findUnique({ where: { id } });
|
||||||
|
if (!resource) return;
|
||||||
|
|
||||||
|
// Mark as deleting first (driver handles actual deletion)
|
||||||
|
await this.db.resource.update({
|
||||||
|
where: { id },
|
||||||
|
data: { status: "deleting" },
|
||||||
|
});
|
||||||
|
|
||||||
|
logger.info(`RESOURCE DELETING: ${resource.kind}/${resource.name}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async hardDelete(id: string): Promise<void> {
|
||||||
|
await this.db.resource.delete({ where: { id } });
|
||||||
|
}
|
||||||
|
}
|
||||||
144
bastion/src/labd/tests/bastions-machines.test.ts
Normal file
144
bastion/src/labd/tests/bastions-machines.test.ts
Normal file
@@ -0,0 +1,144 @@
|
|||||||
|
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||||
|
import Fastify from "fastify";
|
||||||
|
import { registerBastionRoutes } from "../src/routes/bastions.js";
|
||||||
|
import { bastionRegistry } from "../src/services/bastion-registry.js";
|
||||||
|
import type { DbClient } from "../src/server.js";
|
||||||
|
import type { BastionState } from "@lab/shared";
|
||||||
|
|
||||||
|
function createMockDb(servers: unknown[] = []): DbClient {
|
||||||
|
return {
|
||||||
|
$queryRaw: vi.fn().mockResolvedValue([{ "?column?": 1 }]),
|
||||||
|
server: {
|
||||||
|
findMany: vi.fn().mockResolvedValue(servers),
|
||||||
|
findUnique: vi.fn().mockResolvedValue(null),
|
||||||
|
upsert: vi.fn().mockResolvedValue({}),
|
||||||
|
},
|
||||||
|
joinToken: {
|
||||||
|
findUnique: vi.fn().mockResolvedValue(null),
|
||||||
|
findMany: vi.fn().mockResolvedValue([]),
|
||||||
|
create: vi.fn().mockResolvedValue({ id: "t" }),
|
||||||
|
update: vi.fn().mockResolvedValue({}),
|
||||||
|
},
|
||||||
|
bastion: {
|
||||||
|
upsert: vi.fn().mockResolvedValue({}),
|
||||||
|
findMany: vi.fn().mockResolvedValue([]),
|
||||||
|
findUnique: vi.fn().mockResolvedValue(null),
|
||||||
|
update: vi.fn().mockResolvedValue({}),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function registerFakeBastion(bastionId: string, state: BastionState): void {
|
||||||
|
bastionRegistry.register({
|
||||||
|
bastionId,
|
||||||
|
hostname: "fake",
|
||||||
|
network: "192.168.8.0/24",
|
||||||
|
serverIp: "192.168.8.11",
|
||||||
|
// socket is referenced only on commands, not during aggregation
|
||||||
|
socket: { on: () => undefined, off: () => undefined, send: () => undefined, close: () => undefined } as never,
|
||||||
|
connectedAt: new Date(),
|
||||||
|
lastHeartbeat: new Date(),
|
||||||
|
state,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("GET /api/machines aggregation", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
for (const b of bastionRegistry.getAll()) bastionRegistry.unregister(b.bastionId);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("promotes a live-discovered MAC to installed when the DB has a real hostname+role for it", async () => {
|
||||||
|
// Simulates the worker0-k8s0 bug: bastion restarted, lost its installed map,
|
||||||
|
// rediscovered the machine via DHCP/PXE. DB still has hostname=worker0-k8s0,
|
||||||
|
// role=infra, ip=192.168.8.23. Without the fix, the CLI sees a "discovered"
|
||||||
|
// row with no hostname/role/IP. With the fix, the row is promoted to
|
||||||
|
// "installed" with full identity preserved.
|
||||||
|
const mac = "78:55:36:08:28:fb";
|
||||||
|
registerFakeBastion("b1", {
|
||||||
|
discovered: {
|
||||||
|
[mac]: {
|
||||||
|
mac, product: "SER", board: "SER", serial: "x", manufacturer: "AZW",
|
||||||
|
cpu_model: "AMD Ryzen 7 255", cpu_cores: 16, memory_gb: 58, arch: "x86_64",
|
||||||
|
disks: [], nics: [], first_seen: "", last_seen: "",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
install_queue: {},
|
||||||
|
installed: {},
|
||||||
|
debug: {},
|
||||||
|
});
|
||||||
|
|
||||||
|
const app = Fastify({ logger: false });
|
||||||
|
const db = createMockDb([
|
||||||
|
{ mac, hostname: "worker0-k8s0", role: "infra", ip: "192.168.8.23", status: "discovered", labels: {} },
|
||||||
|
]);
|
||||||
|
registerBastionRoutes(app, db);
|
||||||
|
|
||||||
|
const res = await app.inject({ method: "GET", url: "/api/machines" });
|
||||||
|
expect(res.statusCode).toBe(200);
|
||||||
|
const body = JSON.parse(res.body);
|
||||||
|
|
||||||
|
expect(body.discovered[mac]).toBeUndefined();
|
||||||
|
expect(body.installed[mac]).toMatchObject({
|
||||||
|
hostname: "worker0-k8s0",
|
||||||
|
role: "infra",
|
||||||
|
ip: "192.168.8.23",
|
||||||
|
cpu_model: "AMD Ryzen 7 255",
|
||||||
|
cpu_cores: 16,
|
||||||
|
memory_gb: 58,
|
||||||
|
});
|
||||||
|
|
||||||
|
await app.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves a fresh-discovery MAC in discovered when DB only has a discovery-shaped record", async () => {
|
||||||
|
const mac = "aa:bb:cc:dd:ee:ff";
|
||||||
|
registerFakeBastion("b1", {
|
||||||
|
discovered: {
|
||||||
|
[mac]: {
|
||||||
|
mac, product: "SER", board: "SER", serial: "x", manufacturer: "AZW",
|
||||||
|
cpu_model: "AMD Ryzen 7", cpu_cores: 8, memory_gb: 32, arch: "x86_64",
|
||||||
|
disks: [], nics: [], first_seen: "", last_seen: "",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
install_queue: {},
|
||||||
|
installed: {},
|
||||||
|
debug: {},
|
||||||
|
});
|
||||||
|
|
||||||
|
const app = Fastify({ logger: false });
|
||||||
|
// Matches what labd writes on first discovery: hostname=product, role="unknown"
|
||||||
|
const db = createMockDb([
|
||||||
|
{ mac, hostname: "SER", role: "unknown", ip: null, status: "discovered", labels: {} },
|
||||||
|
]);
|
||||||
|
registerBastionRoutes(app, db);
|
||||||
|
|
||||||
|
const res = await app.inject({ method: "GET", url: "/api/machines" });
|
||||||
|
const body = JSON.parse(res.body);
|
||||||
|
|
||||||
|
expect(body.discovered[mac]).toBeDefined();
|
||||||
|
expect(body.installed[mac]).toBeUndefined();
|
||||||
|
|
||||||
|
await app.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back to DB for MACs not in any live bucket", async () => {
|
||||||
|
const mac = "11:22:33:44:55:66";
|
||||||
|
// No bastions connected
|
||||||
|
const app = Fastify({ logger: false });
|
||||||
|
const db = createMockDb([
|
||||||
|
{ mac, hostname: "worker1-k8s0", role: "infra", ip: "192.168.8.13", status: "online", labels: {} },
|
||||||
|
]);
|
||||||
|
registerBastionRoutes(app, db);
|
||||||
|
|
||||||
|
const res = await app.inject({ method: "GET", url: "/api/machines" });
|
||||||
|
const body = JSON.parse(res.body);
|
||||||
|
|
||||||
|
expect(body.installed[mac]).toMatchObject({
|
||||||
|
hostname: "worker1-k8s0",
|
||||||
|
role: "infra",
|
||||||
|
ip: "192.168.8.13",
|
||||||
|
});
|
||||||
|
|
||||||
|
await app.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
425
bastion/src/labd/tests/v2-smoke.test.ts
Normal file
425
bastion/src/labd/tests/v2-smoke.test.ts
Normal file
@@ -0,0 +1,425 @@
|
|||||||
|
// End-to-end smoke tests for the v2.0 Phase 1 surface (auth bootstrap, RBAC,
|
||||||
|
// audit correlation). These exercise the wiring in createApp(): the bearer
|
||||||
|
// auth middleware, the v2 routes scope, and the AuditService lifecycle.
|
||||||
|
//
|
||||||
|
// We don't spin up CockroachDB. Instead we provide a PrismaClient-shaped
|
||||||
|
// in-memory mock that matches the surface the v2 services actually touch.
|
||||||
|
// Tests follow the project convention of using mock DBs + Fastify.inject().
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
||||||
|
import bcrypt from "bcryptjs";
|
||||||
|
import { createApp } from "../src/server.js";
|
||||||
|
import type { DbClient } from "../src/server.js";
|
||||||
|
import type { AuditService } from "../src/services/audit.js";
|
||||||
|
|
||||||
|
const TEST_CONFIG = { port: 0, host: "127.0.0.1", databaseUrl: "", caDir: "/tmp", logLevel: "silent" };
|
||||||
|
|
||||||
|
interface UserRow { id: string; email: string; password: string; role: string; name: string | null; }
|
||||||
|
interface SessionRow { id: string; userId: string; token: string; expiresAt: Date; user?: UserRow; }
|
||||||
|
interface RbacDefRow { id: string; name: string; subjects: unknown; roleBindings: unknown; }
|
||||||
|
interface AuditEventRow {
|
||||||
|
id: string;
|
||||||
|
eventKind: string;
|
||||||
|
source: string;
|
||||||
|
verified: boolean;
|
||||||
|
userId: string | null;
|
||||||
|
userName: string | null;
|
||||||
|
environmentName: string | null;
|
||||||
|
resourceKind: string | null;
|
||||||
|
correlationId: string | null;
|
||||||
|
parentEventId: string | null;
|
||||||
|
details: unknown;
|
||||||
|
result: string;
|
||||||
|
error: string | null;
|
||||||
|
durationMs: number | null;
|
||||||
|
timestamp: Date;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Stores {
|
||||||
|
users: Map<string, UserRow>;
|
||||||
|
sessions: Map<string, SessionRow>;
|
||||||
|
groupMembers: Array<{ userId: string; group: { name: string } }>;
|
||||||
|
rbacDefs: RbacDefRow[];
|
||||||
|
auditEvents: AuditEventRow[];
|
||||||
|
resources: Array<Record<string, unknown>>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeStores(): Stores {
|
||||||
|
return {
|
||||||
|
users: new Map(),
|
||||||
|
sessions: new Map(),
|
||||||
|
groupMembers: [],
|
||||||
|
rbacDefs: [],
|
||||||
|
auditEvents: [],
|
||||||
|
resources: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeMockDb(s: Stores): DbClient {
|
||||||
|
let idCounter = 0;
|
||||||
|
const newId = (prefix: string): string => `${prefix}-${++idCounter}`;
|
||||||
|
|
||||||
|
return {
|
||||||
|
$queryRaw: vi.fn(async () => [{ "?column?": 1 }]),
|
||||||
|
server: { findMany: vi.fn(async () => []), findUnique: vi.fn(), upsert: vi.fn() },
|
||||||
|
joinToken: { findUnique: vi.fn(), findMany: vi.fn(), create: vi.fn(), update: vi.fn() },
|
||||||
|
bastion: { upsert: vi.fn(), findMany: vi.fn(), findUnique: vi.fn(), update: vi.fn() },
|
||||||
|
|
||||||
|
user: {
|
||||||
|
count: vi.fn(async () => s.users.size),
|
||||||
|
findUnique: vi.fn(async (args: { where: { email?: string; id?: string } }) => {
|
||||||
|
if (args.where.email) {
|
||||||
|
for (const u of s.users.values()) if (u.email === args.where.email) return u;
|
||||||
|
}
|
||||||
|
if (args.where.id) return s.users.get(args.where.id) ?? null;
|
||||||
|
return null;
|
||||||
|
}),
|
||||||
|
create: vi.fn(async (args: { data: Omit<UserRow, "id"> }) => {
|
||||||
|
const id = newId("user");
|
||||||
|
const row: UserRow = { id, ...args.data };
|
||||||
|
s.users.set(id, row);
|
||||||
|
return row;
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
session: {
|
||||||
|
findUnique: vi.fn(async (args: { where: { token?: string; id?: string }; include?: { user?: boolean } }) => {
|
||||||
|
let session: SessionRow | undefined;
|
||||||
|
if (args.where.token) {
|
||||||
|
for (const sess of s.sessions.values()) if (sess.token === args.where.token) { session = sess; break; }
|
||||||
|
} else if (args.where.id) {
|
||||||
|
session = s.sessions.get(args.where.id);
|
||||||
|
}
|
||||||
|
if (!session) return null;
|
||||||
|
if (args.include?.user) {
|
||||||
|
return { ...session, user: s.users.get(session.userId)! };
|
||||||
|
}
|
||||||
|
return session;
|
||||||
|
}),
|
||||||
|
create: vi.fn(async (args: { data: { userId: string; token: string; expiresAt: Date } }) => {
|
||||||
|
const id = newId("sess");
|
||||||
|
const row: SessionRow = { id, ...args.data };
|
||||||
|
s.sessions.set(id, row);
|
||||||
|
return row;
|
||||||
|
}),
|
||||||
|
delete: vi.fn(async (args: { where: { id: string } }) => {
|
||||||
|
s.sessions.delete(args.where.id);
|
||||||
|
return null;
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
groupMember: {
|
||||||
|
findMany: vi.fn(async (args: { where: { userId: string } }) =>
|
||||||
|
s.groupMembers.filter((m) => m.userId === args.where.userId),
|
||||||
|
),
|
||||||
|
},
|
||||||
|
rbacDefinition: {
|
||||||
|
findMany: vi.fn(async () => s.rbacDefs),
|
||||||
|
},
|
||||||
|
auditEvent: {
|
||||||
|
createMany: vi.fn(async (args: { data: Array<Omit<AuditEventRow, "id" | "timestamp">> }) => {
|
||||||
|
const ts = new Date();
|
||||||
|
for (const e of args.data) {
|
||||||
|
s.auditEvents.push({ id: newId("evt"), timestamp: ts, ...e });
|
||||||
|
}
|
||||||
|
return { count: args.data.length };
|
||||||
|
}),
|
||||||
|
findMany: vi.fn(async (args: { where?: Record<string, unknown>; orderBy?: unknown; take?: number }) => {
|
||||||
|
const where = args.where ?? {};
|
||||||
|
const filtered = s.auditEvents.filter((e) => {
|
||||||
|
if (where["eventKind"] && e.eventKind !== where["eventKind"]) return false;
|
||||||
|
if (where["correlationId"] && e.correlationId !== where["correlationId"]) return false;
|
||||||
|
if (where["environmentName"] && e.environmentName !== where["environmentName"]) return false;
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
return filtered.slice(0, args.take ?? 100);
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
resource: {
|
||||||
|
findMany: vi.fn(async () => s.resources),
|
||||||
|
findUnique: vi.fn(),
|
||||||
|
create: vi.fn(),
|
||||||
|
update: vi.fn(),
|
||||||
|
delete: vi.fn(),
|
||||||
|
},
|
||||||
|
environment: { findMany: vi.fn(async () => []), findUnique: vi.fn(), create: vi.fn() },
|
||||||
|
account: { findMany: vi.fn(async () => []), findUnique: vi.fn(), create: vi.fn() },
|
||||||
|
binding: { findMany: vi.fn(async () => []), create: vi.fn() },
|
||||||
|
} as unknown as DbClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function buildApp(s: Stores) {
|
||||||
|
const db = makeMockDb(s);
|
||||||
|
const result = await createApp(TEST_CONFIG, db);
|
||||||
|
await result.app.ready();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("v2 auth: bootstrap flow", () => {
|
||||||
|
let stores: Stores;
|
||||||
|
let app: Awaited<ReturnType<typeof buildApp>>["app"];
|
||||||
|
let auditService: AuditService;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
stores = makeStores();
|
||||||
|
const built = await buildApp(stores);
|
||||||
|
app = built.app;
|
||||||
|
auditService = built.auditService;
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await app.close(); // triggers auditService.stop()
|
||||||
|
});
|
||||||
|
|
||||||
|
it("first login with no users seeds the admin and returns a session token", async () => {
|
||||||
|
expect(stores.users.size).toBe(0);
|
||||||
|
|
||||||
|
const resp = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: "/api/auth/login",
|
||||||
|
payload: { email: "admin@itaz.eu", password: "s3cret-pw" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(resp.statusCode).toBe(200);
|
||||||
|
const body = resp.json();
|
||||||
|
expect(body.isBootstrap).toBe(true);
|
||||||
|
expect(body.token).toMatch(/^[a-f0-9]{64}$/);
|
||||||
|
expect(typeof body.expiresAt).toBe("string");
|
||||||
|
|
||||||
|
expect(stores.users.size).toBe(1);
|
||||||
|
const created = [...stores.users.values()][0]!;
|
||||||
|
expect(created.email).toBe("admin@itaz.eu");
|
||||||
|
expect(created.role).toBe("ADMIN");
|
||||||
|
// Password is hashed, not stored plaintext.
|
||||||
|
expect(created.password).not.toBe("s3cret-pw");
|
||||||
|
expect(await bcrypt.compare("s3cret-pw", created.password)).toBe(true);
|
||||||
|
|
||||||
|
// Bootstrap emits an audit event.
|
||||||
|
await auditService.flushPending();
|
||||||
|
const bootstrapEvents = stores.auditEvents.filter((e) => e.eventKind === "auth_bootstrap");
|
||||||
|
expect(bootstrapEvents).toHaveLength(1);
|
||||||
|
expect(bootstrapEvents[0]!.result).toBe("success");
|
||||||
|
expect(bootstrapEvents[0]!.userName).toBe("admin@itaz.eu");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns 400 for missing credentials", async () => {
|
||||||
|
const resp = await app.inject({ method: "POST", url: "/api/auth/login", payload: {} });
|
||||||
|
expect(resp.statusCode).toBe(400);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("second login uses normal flow (no isBootstrap)", async () => {
|
||||||
|
// Bootstrap once
|
||||||
|
await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: "/api/auth/login",
|
||||||
|
payload: { email: "admin@itaz.eu", password: "s3cret-pw" },
|
||||||
|
});
|
||||||
|
expect(stores.users.size).toBe(1);
|
||||||
|
|
||||||
|
// Login again
|
||||||
|
const resp = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: "/api/auth/login",
|
||||||
|
payload: { email: "admin@itaz.eu", password: "s3cret-pw" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(resp.statusCode).toBe(200);
|
||||||
|
expect(resp.json().isBootstrap).toBe(false);
|
||||||
|
expect(stores.users.size).toBe(1); // no new user
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects wrong password with 401", async () => {
|
||||||
|
// Seed admin
|
||||||
|
await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: "/api/auth/login",
|
||||||
|
payload: { email: "admin@itaz.eu", password: "s3cret-pw" },
|
||||||
|
});
|
||||||
|
|
||||||
|
const resp = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: "/api/auth/login",
|
||||||
|
payload: { email: "admin@itaz.eu", password: "wrong" },
|
||||||
|
});
|
||||||
|
expect(resp.statusCode).toBe(401);
|
||||||
|
|
||||||
|
// Failed login is also audited.
|
||||||
|
await auditService.flushPending();
|
||||||
|
const fails = stores.auditEvents.filter((e) => e.eventKind === "auth_login" && e.result === "failure");
|
||||||
|
expect(fails).toHaveLength(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("v2 RBAC: env-scoped denial", () => {
|
||||||
|
let stores: Stores;
|
||||||
|
let app: Awaited<ReturnType<typeof buildApp>>["app"];
|
||||||
|
|
||||||
|
async function seedSession(role: string): Promise<string> {
|
||||||
|
stores.users.set("u-1", {
|
||||||
|
id: "u-1",
|
||||||
|
email: `${role.toLowerCase()}@itaz.eu`,
|
||||||
|
password: "x",
|
||||||
|
role,
|
||||||
|
name: null,
|
||||||
|
});
|
||||||
|
const token = "test-token-" + role;
|
||||||
|
stores.sessions.set("s-1", {
|
||||||
|
id: "s-1",
|
||||||
|
userId: "u-1",
|
||||||
|
token,
|
||||||
|
expiresAt: new Date(Date.now() + 86_400_000),
|
||||||
|
});
|
||||||
|
return token;
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
stores = makeStores();
|
||||||
|
app = (await buildApp(stores)).app;
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await app.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("non-admin user with no role bindings gets 403 on /api/resources", async () => {
|
||||||
|
const token = await seedSession("EDITOR"); // not admin, no bindings
|
||||||
|
|
||||||
|
const resp = await app.inject({
|
||||||
|
method: "GET",
|
||||||
|
url: "/api/resources",
|
||||||
|
headers: { authorization: `Bearer ${token}` },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(resp.statusCode).toBe(403);
|
||||||
|
expect(resp.json().error).toMatch(/no matching role binding/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("missing/empty bearer token gets 401 (auth, not RBAC)", async () => {
|
||||||
|
const r1 = await app.inject({ method: "GET", url: "/api/resources" });
|
||||||
|
expect(r1.statusCode).toBe(401);
|
||||||
|
|
||||||
|
const r2 = await app.inject({
|
||||||
|
method: "GET",
|
||||||
|
url: "/api/resources",
|
||||||
|
headers: { authorization: "Bearer " },
|
||||||
|
});
|
||||||
|
expect(r2.statusCode).toBe(401);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("invalid bearer token gets 401", async () => {
|
||||||
|
const resp = await app.inject({
|
||||||
|
method: "GET",
|
||||||
|
url: "/api/resources",
|
||||||
|
headers: { authorization: "Bearer not-a-real-token" },
|
||||||
|
});
|
||||||
|
expect(resp.statusCode).toBe(401);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("admin role bypasses RBAC", async () => {
|
||||||
|
const token = await seedSession("ADMIN");
|
||||||
|
|
||||||
|
const resp = await app.inject({
|
||||||
|
method: "GET",
|
||||||
|
url: "/api/resources",
|
||||||
|
headers: { authorization: `Bearer ${token}` },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(resp.statusCode).toBe(200);
|
||||||
|
expect(resp.json()).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("user with binding for env A is denied for resources in env B", async () => {
|
||||||
|
const token = await seedSession("EDITOR");
|
||||||
|
stores.groupMembers.push({ userId: "u-1", group: { name: "team-a" } });
|
||||||
|
stores.rbacDefs.push({
|
||||||
|
id: "rbac-1",
|
||||||
|
name: "team-a-edit-on-env-a",
|
||||||
|
subjects: [{ kind: "Group", name: "team-a" }],
|
||||||
|
roleBindings: [{ role: "edit", environment: "env-a" }],
|
||||||
|
});
|
||||||
|
|
||||||
|
// List in env-a → should pass RBAC (no env query so it's global view, but
|
||||||
|
// the binding scope is environment-specific → for global list the binding
|
||||||
|
// doesn't apply when an environment scope is set on the binding).
|
||||||
|
// Smoke test the targeted denial: trying to create in env-b is rejected.
|
||||||
|
const respB = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: "/api/resources",
|
||||||
|
headers: { authorization: `Bearer ${token}` },
|
||||||
|
payload: { kind: "database", name: "x", environmentId: "env-b", accountId: "acc-1" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(respB.statusCode).toBe(403);
|
||||||
|
expect(respB.json().error).toMatch(/no matching role binding/);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("v2 audit: correlation chain visible via /api/events", () => {
|
||||||
|
let stores: Stores;
|
||||||
|
let app: Awaited<ReturnType<typeof buildApp>>["app"];
|
||||||
|
let auditService: AuditService;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
stores = makeStores();
|
||||||
|
const built = await buildApp(stores);
|
||||||
|
app = built.app;
|
||||||
|
auditService = built.auditService;
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await app.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("emitted audit events are queryable by correlation id", async () => {
|
||||||
|
// Seed admin so /api/events is accessible (it sits behind bearer auth)
|
||||||
|
const loginResp = await app.inject({
|
||||||
|
method: "POST",
|
||||||
|
url: "/api/auth/login",
|
||||||
|
payload: { email: "admin@itaz.eu", password: "pw" },
|
||||||
|
});
|
||||||
|
const token = loginResp.json().token;
|
||||||
|
|
||||||
|
// Force flush so the bootstrap event is in the DB
|
||||||
|
await auditService.flushPending();
|
||||||
|
|
||||||
|
expect(stores.auditEvents.length).toBeGreaterThan(0);
|
||||||
|
const bootstrap = stores.auditEvents.find((e) => e.eventKind === "auth_bootstrap")!;
|
||||||
|
expect(bootstrap.correlationId).toMatch(/^corr_[a-f0-9]{16}$/);
|
||||||
|
|
||||||
|
// Query /api/events filtered by correlation id
|
||||||
|
const queryResp = await app.inject({
|
||||||
|
method: "GET",
|
||||||
|
url: `/api/events?correlation=${bootstrap.correlationId}`,
|
||||||
|
headers: { authorization: `Bearer ${token}` },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(queryResp.statusCode).toBe(200);
|
||||||
|
const events = queryResp.json() as Array<{ correlationId: string; eventKind: string }>;
|
||||||
|
expect(events.length).toBe(1);
|
||||||
|
expect(events[0]!.eventKind).toBe("auth_bootstrap");
|
||||||
|
expect(events[0]!.correlationId).toBe(bootstrap.correlationId);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("explicit parent/child correlation chain is preserved across emits", async () => {
|
||||||
|
const correlationId = auditService.createCorrelation();
|
||||||
|
|
||||||
|
auditService.emit({
|
||||||
|
eventKind: "test_parent",
|
||||||
|
source: "test",
|
||||||
|
result: "success",
|
||||||
|
correlationId,
|
||||||
|
});
|
||||||
|
auditService.emit({
|
||||||
|
eventKind: "test_child",
|
||||||
|
source: "test",
|
||||||
|
result: "success",
|
||||||
|
correlationId,
|
||||||
|
parentEventId: "evt-1",
|
||||||
|
});
|
||||||
|
|
||||||
|
await auditService.flushPending();
|
||||||
|
|
||||||
|
const chain = stores.auditEvents.filter((e) => e.correlationId === correlationId);
|
||||||
|
expect(chain).toHaveLength(2);
|
||||||
|
expect(chain.map((e) => e.eventKind).sort()).toEqual(["test_child", "test_parent"]);
|
||||||
|
expect(chain.find((e) => e.eventKind === "test_child")!.parentEventId).toBe("evt-1");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,21 +1,23 @@
|
|||||||
// Host preparation: kernel modules, sysctl, swap, firewall, SELinux.
|
// Host preparation: kernel modules, sysctl, swap, storage, firewall, SELinux.
|
||||||
|
|
||||||
import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
|
import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
|
||||||
import { runSequential } from "../utils.js";
|
import { runSequential } from "../utils.js";
|
||||||
import { loadKernelModules } from "../operations/kernel-modules.js";
|
import { loadKernelModules } from "../operations/kernel-modules.js";
|
||||||
import { applyCisHardening } from "../operations/sysctl.js";
|
import { applyCisHardening } from "../operations/sysctl.js";
|
||||||
import { disableSwap } from "../operations/swap.js";
|
import { enableSwap } from "../operations/swap.js";
|
||||||
|
import { growRancherLv } from "../operations/rancher-storage.js";
|
||||||
import { disableFirewall } from "../operations/firewall.js";
|
import { disableFirewall } from "../operations/firewall.js";
|
||||||
import { setSelinuxPermissive } from "../operations/selinux.js";
|
import { setSelinuxPermissive } from "../operations/selinux.js";
|
||||||
import { enableIscsi } from "../operations/iscsi.js";
|
import { enableIscsi } from "../operations/iscsi.js";
|
||||||
|
|
||||||
export const hostPrepGroup: OperationGroup = {
|
export const hostPrepGroup: OperationGroup = {
|
||||||
name: "host-prep",
|
name: "host-prep",
|
||||||
description: "Prepare host for k3s: kernel modules, sysctl, swap, firewall, SELinux, iSCSI",
|
description: "Prepare host for k3s: kernel modules, sysctl, swap, imageFs sizing, firewall, SELinux, iSCSI",
|
||||||
operations: [
|
operations: [
|
||||||
{ name: "Load kernel modules", fn: loadKernelModules },
|
{ name: "Load kernel modules", fn: loadKernelModules },
|
||||||
{ name: "Apply CIS sysctl", fn: applyCisHardening },
|
{ name: "Apply CIS sysctl", fn: applyCisHardening },
|
||||||
{ name: "Disable swap", fn: disableSwap },
|
{ name: "Enable swap", fn: enableSwap },
|
||||||
|
{ name: "Grow rancher LV", fn: growRancherLv },
|
||||||
{ name: "Disable firewall", fn: disableFirewall },
|
{ name: "Disable firewall", fn: disableFirewall },
|
||||||
{ name: "Set SELinux permissive", fn: setSelinuxPermissive },
|
{ name: "Set SELinux permissive", fn: setSelinuxPermissive },
|
||||||
{ name: "Enable iSCSI", fn: enableIscsi },
|
{ name: "Enable iSCSI", fn: enableIscsi },
|
||||||
|
|||||||
@@ -3,6 +3,8 @@
|
|||||||
import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
|
import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
|
||||||
import { runSequential } from "../utils.js";
|
import { runSequential } from "../utils.js";
|
||||||
import { installCilium } from "../operations/cilium.js";
|
import { installCilium } from "../operations/cilium.js";
|
||||||
|
import { installMultus } from "../operations/multus.js";
|
||||||
|
import { installVlanSetup } from "../operations/vlan-setup.js";
|
||||||
import { fixCoreDnsUpstream } from "../operations/dns-fix.js";
|
import { fixCoreDnsUpstream } from "../operations/dns-fix.js";
|
||||||
import { applyDefaultNetworkPolicies } from "../operations/network-policy.js";
|
import { applyDefaultNetworkPolicies } from "../operations/network-policy.js";
|
||||||
|
|
||||||
@@ -11,6 +13,11 @@ export const networkingGroup: OperationGroup = {
|
|||||||
description: "Install Cilium CNI, fix DNS, apply network policies",
|
description: "Install Cilium CNI, fix DNS, apply network policies",
|
||||||
operations: [
|
operations: [
|
||||||
{ name: "Install Cilium CNI", fn: installCilium },
|
{ name: "Install Cilium CNI", fn: installCilium },
|
||||||
|
// Multus + vlan-setup: give pods a second interface on VLAN 10 (macvlan)
|
||||||
|
// for LAN device discovery (Matter/HomeKit mDNS). Must follow Cilium
|
||||||
|
// (needs cni.exclusive=false + bpf.vlanBypass={10} from installCilium).
|
||||||
|
{ name: "Install Multus CNI", fn: installMultus },
|
||||||
|
{ name: "Install vlan-setup (lan10 + CNI plugins)", fn: installVlanSetup },
|
||||||
{ name: "Fix CoreDNS upstream", fn: fixCoreDnsUpstream },
|
{ name: "Fix CoreDNS upstream", fn: fixCoreDnsUpstream },
|
||||||
{ name: "Apply network policies", fn: applyDefaultNetworkPolicies },
|
{ name: "Apply network policies", fn: applyDefaultNetworkPolicies },
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -38,12 +38,20 @@ export const installCilium: Operation = async (ctx): Promise<OperationResult> =>
|
|||||||
// Install Cilium
|
// Install Cilium
|
||||||
// - No hardcoded devices: Cilium auto-detects per node (heterogeneous NICs like eno1 vs enP7s7)
|
// - No hardcoded devices: Cilium auto-detects per node (heterogeneous NICs like eno1 vs enP7s7)
|
||||||
// - k8sServiceHost/Port: k3s agents proxy the API on 127.0.0.1:6444 (not 6443)
|
// - k8sServiceHost/Port: k3s agents proxy the API on 127.0.0.1:6444 (not 6443)
|
||||||
|
// - cni.exclusive=false: required so Multus can install its CNI config alongside
|
||||||
|
// Cilium (Cilium otherwise deletes any non-Cilium CNI conf).
|
||||||
|
// - bpf.vlanBypass={10}: allow VLAN 10 (LoT) tagged traffic through the eBPF
|
||||||
|
// host VLAN filter, so pods on a macvlan/VLAN-10 interface receive multicast
|
||||||
|
// (Matter/mDNS ff02::fb + 224.0.0.251). Without this Cilium drops it
|
||||||
|
// ("VLAN traffic disallowed by VLAN filter", bpf_host.c).
|
||||||
const installResult = await ctx.ssh.exec(
|
const installResult = await ctx.ssh.exec(
|
||||||
`KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install \
|
`KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install \
|
||||||
--set kubeProxyReplacement=true \
|
--set kubeProxyReplacement=true \
|
||||||
--set ipam.mode=kubernetes \
|
--set ipam.mode=kubernetes \
|
||||||
--set k8sServiceHost=127.0.0.1 \
|
--set k8sServiceHost=127.0.0.1 \
|
||||||
--set k8sServicePort=6444`,
|
--set k8sServicePort=6444 \
|
||||||
|
--set cni.exclusive=false \
|
||||||
|
--set bpf.vlanBypass="{10}"`,
|
||||||
{ timeoutMs: 300_000 },
|
{ timeoutMs: 300_000 },
|
||||||
);
|
);
|
||||||
if (installResult.exitCode !== 0) {
|
if (installResult.exitCode !== 0) {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
export { loadKernelModules } from "./kernel-modules.js";
|
export { loadKernelModules } from "./kernel-modules.js";
|
||||||
export { applyCisHardening } from "./sysctl.js";
|
export { applyCisHardening } from "./sysctl.js";
|
||||||
export { disableSwap } from "./swap.js";
|
export { enableSwap } from "./swap.js";
|
||||||
|
export { growRancherLv } from "./rancher-storage.js";
|
||||||
export { enableIscsi } from "./iscsi.js";
|
export { enableIscsi } from "./iscsi.js";
|
||||||
export { disableFirewall } from "./firewall.js";
|
export { disableFirewall } from "./firewall.js";
|
||||||
export { setSelinuxPermissive } from "./selinux.js";
|
export { setSelinuxPermissive } from "./selinux.js";
|
||||||
@@ -9,6 +10,8 @@ export { writeAuditPolicy } from "./audit-policy.js";
|
|||||||
export { cleanupStaleCni } from "./cni-cleanup.js";
|
export { cleanupStaleCni } from "./cni-cleanup.js";
|
||||||
export { installK3sBinary } from "./k3s-install.js";
|
export { installK3sBinary } from "./k3s-install.js";
|
||||||
export { installCilium } from "./cilium.js";
|
export { installCilium } from "./cilium.js";
|
||||||
|
export { installMultus } from "./multus.js";
|
||||||
|
export { installVlanSetup } from "./vlan-setup.js";
|
||||||
export { fixCoreDnsUpstream } from "./dns-fix.js";
|
export { fixCoreDnsUpstream } from "./dns-fix.js";
|
||||||
export { configureLogRotation } from "./log-rotation.js";
|
export { configureLogRotation } from "./log-rotation.js";
|
||||||
export { configureJournaldLimits } from "./journald-limits.js";
|
export { configureJournaldLimits } from "./journald-limits.js";
|
||||||
|
|||||||
34
bastion/src/modules/modules/k3s/src/operations/multus.ts
Normal file
34
bastion/src/modules/modules/k3s/src/operations/multus.ts
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
// Install Multus CNI (thick plugin) — the meta-CNI that lets pods attach an
|
||||||
|
// extra interface (macvlan on VLAN 10) alongside Cilium, via a
|
||||||
|
// NetworkAttachmentDefinition. Required for Home Assistant's LAN presence
|
||||||
|
// (Matter/HomeKit mDNS discovery). Cilium must be installed with
|
||||||
|
// cni.exclusive=false first (see cilium.ts) or it deletes Multus's CNI conf.
|
||||||
|
|
||||||
|
import type { Operation, OperationResult } from "../types.js";
|
||||||
|
import { sshOpts } from "../utils.js";
|
||||||
|
|
||||||
|
const MULTUS_VERSION = "v4.1.4";
|
||||||
|
const MULTUS_MANIFEST = `https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/${MULTUS_VERSION}/deployments/multus-daemonset-thick.yml`;
|
||||||
|
|
||||||
|
export const installMultus: Operation = async (ctx): Promise<OperationResult> => {
|
||||||
|
const K = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml";
|
||||||
|
|
||||||
|
// Idempotent: skip if the Multus DaemonSet is already present.
|
||||||
|
const check = await ctx.ssh.exec(
|
||||||
|
`${K} kubectl -n kube-system get ds kube-multus-ds -o name 2>/dev/null`,
|
||||||
|
sshOpts(ctx),
|
||||||
|
);
|
||||||
|
if (check.exitCode === 0 && check.stdout.includes("kube-multus-ds")) {
|
||||||
|
return { success: true, changed: false, message: `Multus already installed (${MULTUS_VERSION})` };
|
||||||
|
}
|
||||||
|
|
||||||
|
const apply = await ctx.ssh.exec(
|
||||||
|
`${K} kubectl apply -f ${MULTUS_MANIFEST}`,
|
||||||
|
{ ...sshOpts(ctx), timeoutMs: 120_000 },
|
||||||
|
);
|
||||||
|
if (apply.exitCode !== 0) {
|
||||||
|
return { success: false, changed: false, message: "Failed to apply Multus manifest", error: apply.stderr };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { success: true, changed: true, message: `Installed Multus ${MULTUS_VERSION} (thick)` };
|
||||||
|
};
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
// Grow the labvg/rancher LV (k3s image store / imageFs) to 120G.
|
||||||
|
// 2026-08 incident: the original 20G LV sat at 85% used from steady-state
|
||||||
|
// images alone, so one ~5G image pull tripped imagefs eviction and evicted
|
||||||
|
// unrelated pods. Fresh installs are sized at 120G by the kickstart; this op
|
||||||
|
// covers nodes installed before that change and vanilla nodes converted to
|
||||||
|
// k8s later. Never removes or shrinks anything — if the VG lacks free space
|
||||||
|
// (e.g. a longhorn --grow LV consumed it), it reports and moves on.
|
||||||
|
|
||||||
|
import type { Operation, OperationResult } from "../types.js";
|
||||||
|
import { sshOpts } from "../utils.js";
|
||||||
|
|
||||||
|
const RANCHER_LV = "labvg/rancher";
|
||||||
|
const TARGET_MIB = 122880; // 120G
|
||||||
|
|
||||||
|
export const growRancherLv: Operation = async (ctx): Promise<OperationResult> => {
|
||||||
|
const lv = await ctx.ssh.exec(
|
||||||
|
`lvs --noheadings --units m --nosuffix -o lv_size ${RANCHER_LV} 2>/dev/null || true`,
|
||||||
|
sshOpts(ctx),
|
||||||
|
);
|
||||||
|
const sizeMib = Number.parseFloat(lv.stdout.trim());
|
||||||
|
if (Number.isNaN(sizeMib)) {
|
||||||
|
return { success: true, changed: false, message: "No labvg/rancher LV — imageFs shares /var, skipping" };
|
||||||
|
}
|
||||||
|
if (sizeMib >= TARGET_MIB) {
|
||||||
|
return { success: true, changed: false, message: `rancher LV already ${Math.round(sizeMib / 1024)}G` };
|
||||||
|
}
|
||||||
|
|
||||||
|
const vg = await ctx.ssh.exec(`vgs --noheadings --units m --nosuffix -o vg_free labvg`, sshOpts(ctx));
|
||||||
|
const freeMib = Number.parseFloat(vg.stdout.trim());
|
||||||
|
const neededMib = TARGET_MIB - sizeMib;
|
||||||
|
if (Number.isNaN(freeMib) || freeMib < neededMib) {
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
changed: false,
|
||||||
|
message: `VG labvg has ${Math.floor((Number.isNaN(freeMib) ? 0 : freeMib) / 1024)}G free — ` +
|
||||||
|
`need ${Math.ceil(neededMib / 1024)}G to grow rancher LV to 120G (manual LV rebuild required)`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
await ctx.ssh.exec(`lvextend -L ${TARGET_MIB}m /dev/${RANCHER_LV}`, sshOpts(ctx));
|
||||||
|
await ctx.ssh.exec(`xfs_growfs /var/lib/rancher`, sshOpts(ctx));
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
changed: true,
|
||||||
|
message: `rancher LV grown ${Math.round(sizeMib / 1024)}G → 120G`,
|
||||||
|
};
|
||||||
|
};
|
||||||
@@ -1,22 +1,40 @@
|
|||||||
// Disable swap (CIS requirement for k3s).
|
// Enable swap so memory pressure spills to disk instead of OOM-killing.
|
||||||
|
// kubelet runs with failSwapOn=false (k3s default); zram stays the fast tier,
|
||||||
|
// the labvg-swap LV is the overflow tier. Replaces the old CIS-style
|
||||||
|
// disableSwap op — a kernel OOM kill of a node daemon is worse than slow swap.
|
||||||
|
|
||||||
import type { Operation, OperationResult } from "../types.js";
|
import type { Operation, OperationResult } from "../types.js";
|
||||||
import { sshOpts } from "../utils.js";
|
import { sshOpts } from "../utils.js";
|
||||||
|
|
||||||
export const disableSwap: Operation = async (ctx): Promise<OperationResult> => {
|
const SWAP_DEV = "/dev/mapper/labvg-swap";
|
||||||
const check = await ctx.ssh.exec("swapon --show --noheadings", sshOpts(ctx));
|
|
||||||
const active = check.stdout.trim().length > 0;
|
|
||||||
|
|
||||||
if (active) {
|
export const enableSwap: Operation = async (ctx): Promise<OperationResult> => {
|
||||||
await ctx.ssh.exec("swapoff -a", sshOpts(ctx));
|
const lv = await ctx.ssh.exec(`test -b ${SWAP_DEV} && echo yes || echo no`, sshOpts(ctx));
|
||||||
|
if (lv.stdout.trim() !== "yes") {
|
||||||
|
return { success: true, changed: false, message: "No labvg-swap LV — skipping swap enable" };
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove swap entries from fstab permanently
|
const active = await ctx.ssh.exec(
|
||||||
await ctx.ssh.exec("sed -i '/\\sswap\\s/d' /etc/fstab", sshOpts(ctx));
|
`grep -q "^$(readlink -f ${SWAP_DEV}) " /proc/swaps && echo on || echo off`,
|
||||||
|
sshOpts(ctx),
|
||||||
|
);
|
||||||
|
const wasOff = active.stdout.trim() !== "on";
|
||||||
|
|
||||||
|
if (wasOff) {
|
||||||
|
// Format if the LV was never (or wrongly) initialised, then activate
|
||||||
|
await ctx.ssh.exec(`blkid ${SWAP_DEV} | grep -q 'TYPE="swap"' || mkswap ${SWAP_DEV}`, sshOpts(ctx));
|
||||||
|
await ctx.ssh.exec(`swapon ${SWAP_DEV}`, sshOpts(ctx));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Persist across reboots (idempotent)
|
||||||
|
await ctx.ssh.exec(
|
||||||
|
`grep -q "labvg-swap" /etc/fstab || echo "${SWAP_DEV} none swap defaults 0 0" >> /etc/fstab`,
|
||||||
|
sshOpts(ctx),
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
success: true,
|
success: true,
|
||||||
changed: active,
|
changed: wasOff,
|
||||||
message: active ? "Swap disabled" : "Swap already disabled",
|
message: wasOff ? "LV swap enabled" : "LV swap already active",
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|||||||
96
bastion/src/modules/modules/k3s/src/operations/vlan-setup.ts
Normal file
96
bastion/src/modules/modules/k3s/src/operations/vlan-setup.ts
Normal file
@@ -0,0 +1,96 @@
|
|||||||
|
// vlan-setup DaemonSet — the node-level half of the macvlan/VLAN-10 story.
|
||||||
|
// On every node it (1) installs the reference CNI plugins (macvlan/ipvlan/
|
||||||
|
// static/host-local/vlan/tuning) into /opt/cni/bin if missing, and (2) creates
|
||||||
|
// a `lan10` VLAN-10 sub-interface on the primary NIC that macvlan
|
||||||
|
// NetworkAttachmentDefinitions use as their master. Idempotent + self-healing
|
||||||
|
// (re-creates lan10 if it disappears). Paired with Multus (multus.ts) + Cilium
|
||||||
|
// bpf.vlanBypass={10} (cilium.ts).
|
||||||
|
|
||||||
|
import type { Operation, OperationResult } from "../types.js";
|
||||||
|
import { sshOpts } from "../utils.js";
|
||||||
|
|
||||||
|
const MANIFEST = `apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: macvlan-sys
|
||||||
|
labels:
|
||||||
|
pod-security.kubernetes.io/enforce: privileged
|
||||||
|
pod-security.kubernetes.io/audit: privileged
|
||||||
|
pod-security.kubernetes.io/warn: privileged
|
||||||
|
---
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: DaemonSet
|
||||||
|
metadata:
|
||||||
|
name: vlan-setup
|
||||||
|
namespace: macvlan-sys
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels: { app: vlan-setup }
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels: { app: vlan-setup }
|
||||||
|
spec:
|
||||||
|
hostNetwork: true
|
||||||
|
tolerations:
|
||||||
|
- operator: Exists
|
||||||
|
containers:
|
||||||
|
- name: vlan
|
||||||
|
image: nicolaka/netshoot
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
command:
|
||||||
|
- sh
|
||||||
|
- -c
|
||||||
|
- |
|
||||||
|
set -x
|
||||||
|
# install reference CNI plugins (macvlan/ipvlan/static/host-local) if missing
|
||||||
|
if [ ! -f /host/opt/cni/bin/macvlan ] || [ ! -f /host/opt/cni/bin/ipvlan ]; then
|
||||||
|
case "$(uname -m)" in x86_64) A=amd64;; aarch64) A=arm64;; *) A=amd64;; esac
|
||||||
|
curl -sSL "https://github.com/containernetworking/plugins/releases/download/v1.5.1/cni-plugins-linux-$A-v1.5.1.tgz" -o /tmp/cni.tgz
|
||||||
|
tar -xzf /tmp/cni.tgz -C /host/opt/cni/bin ./macvlan ./ipvlan ./static ./host-local ./vlan ./tuning
|
||||||
|
fi
|
||||||
|
# detect the primary NIC (default route dev, else the one holding 192.168.8.x)
|
||||||
|
NIC="$(ip -o -4 route show default 2>/dev/null | awk '{print $5; exit}')"
|
||||||
|
[ -z "$NIC" ] && NIC="$(ip -o -4 addr show 2>/dev/null | awk '/192\\.168\\.8\\./{print $2; exit}')"
|
||||||
|
echo "primary NIC = $NIC"
|
||||||
|
while true; do
|
||||||
|
if [ -n "$NIC" ]; then
|
||||||
|
ip link show lan10 >/dev/null 2>&1 || ip link add link "$NIC" name lan10 type vlan id 10
|
||||||
|
ip link set lan10 up
|
||||||
|
# NIC-driver workarounds for VLAN multicast RX
|
||||||
|
ip link set "$NIC" allmulticast on 2>/dev/null
|
||||||
|
ethtool -K "$NIC" rxvlan off rx-vlan-filter off 2>/dev/null
|
||||||
|
fi
|
||||||
|
sleep 30
|
||||||
|
done
|
||||||
|
volumeMounts:
|
||||||
|
- name: cnibin
|
||||||
|
mountPath: /host/opt/cni/bin
|
||||||
|
volumes:
|
||||||
|
- name: cnibin
|
||||||
|
hostPath:
|
||||||
|
path: /opt/cni/bin
|
||||||
|
`;
|
||||||
|
|
||||||
|
export const installVlanSetup: Operation = async (ctx): Promise<OperationResult> => {
|
||||||
|
const K = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml";
|
||||||
|
|
||||||
|
const check = await ctx.ssh.exec(
|
||||||
|
`${K} kubectl -n macvlan-sys get ds vlan-setup -o name 2>/dev/null`,
|
||||||
|
sshOpts(ctx),
|
||||||
|
);
|
||||||
|
if (check.exitCode === 0 && check.stdout.includes("vlan-setup")) {
|
||||||
|
return { success: true, changed: false, message: "vlan-setup DaemonSet already installed" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const b64 = Buffer.from(MANIFEST).toString("base64");
|
||||||
|
const apply = await ctx.ssh.exec(
|
||||||
|
`echo ${b64} | base64 -d | ${K} kubectl apply -f -`,
|
||||||
|
{ ...sshOpts(ctx), timeoutMs: 60_000 },
|
||||||
|
);
|
||||||
|
if (apply.exitCode !== 0) {
|
||||||
|
return { success: false, changed: false, message: "Failed to apply vlan-setup DaemonSet", error: apply.stderr };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { success: true, changed: true, message: "Installed vlan-setup DaemonSet (lan10 + CNI plugins)" };
|
||||||
|
};
|
||||||
@@ -72,31 +72,97 @@ describe("applyCisHardening", () => {
|
|||||||
|
|
||||||
// --- Swap ---
|
// --- Swap ---
|
||||||
|
|
||||||
import { disableSwap } from "../src/operations/swap.js";
|
import { enableSwap } from "../src/operations/swap.js";
|
||||||
|
|
||||||
describe("disableSwap", () => {
|
describe("enableSwap", () => {
|
||||||
it("disables active swap", async () => {
|
it("activates LV swap when present but off", async () => {
|
||||||
const ctx = mockCtx();
|
const ctx = mockCtx();
|
||||||
ctx.ssh.exec
|
ctx.ssh.exec
|
||||||
.mockResolvedValueOnce(stdout("/dev/sda2 partition 2G")) // swap active
|
.mockResolvedValueOnce(stdout("yes")) // LV exists
|
||||||
.mockResolvedValueOnce(OK) // swapoff
|
.mockResolvedValueOnce(stdout("off")) // not in /proc/swaps
|
||||||
.mockResolvedValueOnce(OK); // sed fstab
|
.mockResolvedValueOnce(OK) // blkid || mkswap
|
||||||
|
.mockResolvedValueOnce(OK) // swapon
|
||||||
|
.mockResolvedValueOnce(OK); // fstab entry
|
||||||
|
|
||||||
const result = await disableSwap(ctx);
|
const result = await enableSwap(ctx);
|
||||||
expect(result.success).toBe(true);
|
expect(result.success).toBe(true);
|
||||||
expect(result.changed).toBe(true);
|
expect(result.changed).toBe(true);
|
||||||
expectCommand(ctx.ssh, "swapoff -a");
|
expectCommand(ctx.ssh, "swapon /dev/mapper/labvg-swap");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("is idempotent when swap already off", async () => {
|
it("is idempotent when LV swap already active", async () => {
|
||||||
const ctx = mockCtx();
|
const ctx = mockCtx();
|
||||||
ctx.ssh.exec
|
ctx.ssh.exec
|
||||||
.mockResolvedValueOnce(stdout("")) // no swap
|
.mockResolvedValueOnce(stdout("yes")) // LV exists
|
||||||
.mockResolvedValueOnce(OK); // sed fstab (always runs)
|
.mockResolvedValueOnce(stdout("on")) // already in /proc/swaps
|
||||||
|
.mockResolvedValueOnce(OK); // fstab entry (always ensured)
|
||||||
|
|
||||||
const result = await disableSwap(ctx);
|
const result = await enableSwap(ctx);
|
||||||
expect(result.changed).toBe(false);
|
expect(result.changed).toBe(false);
|
||||||
expectNoCommand(ctx.ssh, "swapoff");
|
expectNoCommand(ctx.ssh, "swapon /dev/mapper/labvg-swap");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips when no labvg-swap LV exists", async () => {
|
||||||
|
const ctx = mockCtx();
|
||||||
|
ctx.ssh.exec.mockResolvedValueOnce(stdout("no")); // LV missing
|
||||||
|
|
||||||
|
const result = await enableSwap(ctx);
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(result.changed).toBe(false);
|
||||||
|
expectNoCommand(ctx.ssh, "swapon");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- Rancher LV (imageFs sizing) ---
|
||||||
|
|
||||||
|
import { growRancherLv } from "../src/operations/rancher-storage.js";
|
||||||
|
|
||||||
|
describe("growRancherLv", () => {
|
||||||
|
it("grows a 20G LV to 120G when the VG has space", async () => {
|
||||||
|
const ctx = mockCtx();
|
||||||
|
ctx.ssh.exec
|
||||||
|
.mockResolvedValueOnce(stdout(" 20480.00")) // lv_size
|
||||||
|
.mockResolvedValueOnce(stdout(" 747807.00")) // vg_free
|
||||||
|
.mockResolvedValueOnce(OK) // lvextend
|
||||||
|
.mockResolvedValueOnce(OK); // xfs_growfs
|
||||||
|
|
||||||
|
const result = await growRancherLv(ctx);
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(result.changed).toBe(true);
|
||||||
|
expectCommand(ctx.ssh, "lvextend -L 122880m /dev/labvg/rancher");
|
||||||
|
expectCommand(ctx.ssh, "xfs_growfs /var/lib/rancher");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is idempotent when the LV is already 120G", async () => {
|
||||||
|
const ctx = mockCtx();
|
||||||
|
ctx.ssh.exec.mockResolvedValueOnce(stdout(" 122880.00")); // lv_size
|
||||||
|
|
||||||
|
const result = await growRancherLv(ctx);
|
||||||
|
expect(result.changed).toBe(false);
|
||||||
|
expectNoCommand(ctx.ssh, "lvextend");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports without failing when the VG has no free space", async () => {
|
||||||
|
const ctx = mockCtx();
|
||||||
|
ctx.ssh.exec
|
||||||
|
.mockResolvedValueOnce(stdout(" 20480.00")) // lv_size
|
||||||
|
.mockResolvedValueOnce(stdout(" 0.00")); // vg_free
|
||||||
|
|
||||||
|
const result = await growRancherLv(ctx);
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(result.changed).toBe(false);
|
||||||
|
expect(result.message).toContain("free");
|
||||||
|
expectNoCommand(ctx.ssh, "lvextend");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips when there is no rancher LV", async () => {
|
||||||
|
const ctx = mockCtx();
|
||||||
|
ctx.ssh.exec.mockResolvedValueOnce(stdout("")); // lvs empty
|
||||||
|
|
||||||
|
const result = await growRancherLv(ctx);
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(result.changed).toBe(false);
|
||||||
|
expectNoCommand(ctx.ssh, "lvextend");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ describe("smoke: full server install pipeline", () => {
|
|||||||
const pipeline: NamedOperation[] = [
|
const pipeline: NamedOperation[] = [
|
||||||
{ name: "Kernel modules", fn: ops.loadKernelModules },
|
{ name: "Kernel modules", fn: ops.loadKernelModules },
|
||||||
{ name: "Sysctl hardening", fn: ops.applyCisHardening },
|
{ name: "Sysctl hardening", fn: ops.applyCisHardening },
|
||||||
{ name: "Disable swap", fn: ops.disableSwap },
|
{ name: "Enable swap", fn: ops.enableSwap },
|
||||||
{ name: "Disable firewall", fn: ops.disableFirewall },
|
{ name: "Disable firewall", fn: ops.disableFirewall },
|
||||||
{ name: "SELinux permissive", fn: ops.setSelinuxPermissive },
|
{ name: "SELinux permissive", fn: ops.setSelinuxPermissive },
|
||||||
{ name: "Write k3s config", fn: ops.writeK3sConfig },
|
{ name: "Write k3s config", fn: ops.writeK3sConfig },
|
||||||
@@ -73,7 +73,7 @@ describe("smoke: pipeline stops on failure", () => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const results = await runSequential(ctx, [
|
const results = await runSequential(ctx, [
|
||||||
{ name: "OK op", fn: ops.disableSwap },
|
{ name: "OK op", fn: ops.enableSwap },
|
||||||
{ name: "Failing op", fn: failingOp },
|
{ name: "Failing op", fn: failingOp },
|
||||||
{ name: "Never called", fn: neverCalled },
|
{ name: "Never called", fn: neverCalled },
|
||||||
]);
|
]);
|
||||||
@@ -98,11 +98,12 @@ describe("smoke: agent install rejects missing config", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
describe("smoke: all operations are exported", () => {
|
describe("smoke: all operations are exported", () => {
|
||||||
it("exports all 15 operations", () => {
|
it("exports all 16 operations", () => {
|
||||||
const exported = [
|
const exported = [
|
||||||
ops.loadKernelModules,
|
ops.loadKernelModules,
|
||||||
ops.applyCisHardening,
|
ops.applyCisHardening,
|
||||||
ops.disableSwap,
|
ops.enableSwap,
|
||||||
|
ops.growRancherLv,
|
||||||
ops.disableFirewall,
|
ops.disableFirewall,
|
||||||
ops.setSelinuxPermissive,
|
ops.setSelinuxPermissive,
|
||||||
ops.writeK3sConfig,
|
ops.writeK3sConfig,
|
||||||
@@ -117,7 +118,7 @@ describe("smoke: all operations are exported", () => {
|
|||||||
ops.checkCertExpiry,
|
ops.checkCertExpiry,
|
||||||
];
|
];
|
||||||
|
|
||||||
expect(exported).toHaveLength(15);
|
expect(exported).toHaveLength(16);
|
||||||
for (const op of exported) {
|
for (const op of exported) {
|
||||||
expect(typeof op).toBe("function");
|
expect(typeof op).toBe("function");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ export type {
|
|||||||
DebugConfig,
|
DebugConfig,
|
||||||
BastionState,
|
BastionState,
|
||||||
BastionConfig,
|
BastionConfig,
|
||||||
|
VyosVlanSpec,
|
||||||
|
VyosInstallSpec,
|
||||||
} from "./types/index.js";
|
} from "./types/index.js";
|
||||||
|
|
||||||
export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./types/index.js";
|
export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./types/index.js";
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
// Protocol types for agent-labd WebSocket communication.
|
// Protocol types for agent-labd WebSocket communication.
|
||||||
|
|
||||||
import { randomUUID } from "node:crypto";
|
import { randomUUID } from "node:crypto";
|
||||||
|
import type { VyosInstallSpec } from "../types/state.js";
|
||||||
|
|
||||||
// --- Agent -> labd messages ---
|
// --- Agent -> labd messages ---
|
||||||
|
|
||||||
@@ -108,7 +109,7 @@ export type BastionMessage =
|
|||||||
export type LabdBastionMessage =
|
export type LabdBastionMessage =
|
||||||
| { type: "bastion-enrolled"; bastionId: string }
|
| { type: "bastion-enrolled"; bastionId: string }
|
||||||
| { type: "bastion-heartbeat-ack"; serverTime: string }
|
| { type: "bastion-heartbeat-ack"; serverTime: string }
|
||||||
| { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string }
|
| { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string; vyos?: VyosInstallSpec }
|
||||||
| { type: "command-forget"; requestId: string; mac: string }
|
| { type: "command-forget"; requestId: string; mac: string }
|
||||||
| { type: "command-role-update"; requestId: string; mac: string; role: string }
|
| { type: "command-role-update"; requestId: string; mac: string; role: string }
|
||||||
| { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean }
|
| { type: "command-debug"; requestId: string; mac: string; pxeBoot?: boolean }
|
||||||
|
|||||||
@@ -14,6 +14,10 @@ export interface BastionConfig {
|
|||||||
// Ubuntu support
|
// Ubuntu support
|
||||||
ubuntuVersion: string;
|
ubuntuVersion: string;
|
||||||
ubuntuMirror: string;
|
ubuntuMirror: string;
|
||||||
|
// VyOS support — netboot artifacts are extracted from the ISO at startup.
|
||||||
|
// LTS ISOs are subscription-only, so this defaults to a rolling release.
|
||||||
|
vyosIsoUrl: string;
|
||||||
|
vyosDefaultPassword: string;
|
||||||
// Syslog listener for install logs (Anaconda logging --host)
|
// Syslog listener for install logs (Anaconda logging --host)
|
||||||
syslogPort: number;
|
syslogPort: number;
|
||||||
// Flags
|
// Flags
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ export type {
|
|||||||
InstalledInfo,
|
InstalledInfo,
|
||||||
DebugConfig,
|
DebugConfig,
|
||||||
BastionState,
|
BastionState,
|
||||||
|
VyosVlanSpec,
|
||||||
|
VyosInstallSpec,
|
||||||
} from "./state.js";
|
} from "./state.js";
|
||||||
|
|
||||||
export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./state.js";
|
export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./state.js";
|
||||||
|
|||||||
@@ -2,10 +2,10 @@
|
|||||||
|
|
||||||
export type ProvisionStackType = "dhcpproxy" | "iso" | "cloud-init";
|
export type ProvisionStackType = "dhcpproxy" | "iso" | "cloud-init";
|
||||||
|
|
||||||
export type OsId = "fedora-43" | "ubuntu-26.04";
|
export type OsId = "fedora-43" | "ubuntu-26.04" | "vyos-rolling";
|
||||||
export type Arch = "x86_64" | "aarch64";
|
export type Arch = "x86_64" | "aarch64";
|
||||||
|
|
||||||
export const SUPPORTED_OS: readonly OsId[] = ["fedora-43", "ubuntu-26.04"] as const;
|
export const SUPPORTED_OS: readonly OsId[] = ["fedora-43", "ubuntu-26.04", "vyos-rolling"] as const;
|
||||||
|
|
||||||
export function isValidOsId(value: string): value is OsId {
|
export function isValidOsId(value: string): value is OsId {
|
||||||
return (SUPPORTED_OS as readonly string[]).includes(value);
|
return (SUPPORTED_OS as readonly string[]).includes(value);
|
||||||
@@ -75,13 +75,89 @@ export interface ProgressLogEntry {
|
|||||||
timestamp: string;
|
timestamp: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** A tagged VLAN sub-interface on the bond (or on the mgmt NIC when unbonded). */
|
||||||
|
export interface VyosVlanSpec {
|
||||||
|
id: number;
|
||||||
|
address: string; // CIDR, e.g. "10.0.10.1/24"
|
||||||
|
description?: string;
|
||||||
|
/**
|
||||||
|
* VRRP virtual address (CIDR) floated on this VLAN. Emitted as a
|
||||||
|
* high-availability vrrp group with vrid = VLAN id, so the same spec on both
|
||||||
|
* HA peers (with different priorities) produces a matching group pair.
|
||||||
|
*/
|
||||||
|
vrrp?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* VyOS-specific install parameters. Rendered into the config.boot that the
|
||||||
|
* installer adopts, so the router comes up already configured.
|
||||||
|
*
|
||||||
|
* NOTE: bondMembers must NOT include the interface PXE booted from. Firmware
|
||||||
|
* PXE cannot run over LACP, so the install-time NIC has to stay unbonded.
|
||||||
|
*/
|
||||||
|
export interface VyosInstallSpec {
|
||||||
|
/** Interfaces aggregated into bond0 with LACP (802.3ad). Omit for no bond. */
|
||||||
|
bondMembers?: string[];
|
||||||
|
/** CIDR address on bond0 itself — the switch trunk's native/untagged VLAN. */
|
||||||
|
bondAddress?: string;
|
||||||
|
/** VRRP virtual address (CIDR) floated on the untagged bond (vrid 1). */
|
||||||
|
bondVrrp?: string;
|
||||||
|
/**
|
||||||
|
* VRRP priority for every group on this box. Higher wins mastership.
|
||||||
|
* The HA pair differs ONLY here (e.g. 200 on the primary, 100 on the
|
||||||
|
* standby) — addresses differ per box, VIPs and vrids match.
|
||||||
|
*/
|
||||||
|
vrrpPriority?: number;
|
||||||
|
/** Tagged VLAN sub-interfaces, created on bond0 when bonded, else on mgmtInterface. */
|
||||||
|
vlans?: VyosVlanSpec[];
|
||||||
|
/** Untagged interface the machine PXE booted from. Defaults to "eth0". */
|
||||||
|
mgmtInterface?: string;
|
||||||
|
/** CIDR address for mgmtInterface, or "dhcp". Defaults to "dhcp". */
|
||||||
|
mgmtAddress?: string;
|
||||||
|
/**
|
||||||
|
* Tagged management VLAN on mgmtInterface, separate from the routed VLANs
|
||||||
|
* carried by the bond.
|
||||||
|
*
|
||||||
|
* Needed when the PXE port is a trunk: it boots untagged on the VLAN the
|
||||||
|
* bastion's proxy DHCP serves, and carries the management VLAN tagged so the
|
||||||
|
* router stays reachable there without giving up reinstallability.
|
||||||
|
*/
|
||||||
|
mgmtVlan?: VyosVlanSpec;
|
||||||
|
/** Password for the "vyos" user. Falls back to the bastion default. */
|
||||||
|
password?: string;
|
||||||
|
/**
|
||||||
|
* On reinstall the VyOS installer carries the previous on-disk config (and
|
||||||
|
* SSH host keys) forward -- the "reinstall without losing data" default.
|
||||||
|
* Set true to make the bastion-generated config win instead: after install
|
||||||
|
* the driver overwrites the installed image's config.boot.
|
||||||
|
*/
|
||||||
|
freshConfig?: boolean;
|
||||||
|
/**
|
||||||
|
* VyOS interface name -> MAC, emitted as `hw-id` so names bind deterministically.
|
||||||
|
*
|
||||||
|
* Discovery runs under Fedora and reports predictable names (enp2s0,
|
||||||
|
* enp1s0f0np0), but VyOS enumerates its own eth<N> names, so a name observed
|
||||||
|
* during discovery cannot be used directly. Pinning by MAC removes the guess
|
||||||
|
* about which physical port a given eth<N> is.
|
||||||
|
*/
|
||||||
|
hwIds?: Record<string, string>;
|
||||||
|
}
|
||||||
|
|
||||||
export interface InstallConfig {
|
export interface InstallConfig {
|
||||||
hostname: string;
|
hostname: string;
|
||||||
disk: string;
|
disk: string;
|
||||||
role: Role;
|
role: Role;
|
||||||
os?: OsId; // defaults to "fedora-43" for backward compat
|
os?: OsId; // defaults to "fedora-43" for backward compat
|
||||||
|
vyos?: VyosInstallSpec; // only consulted when os is "vyos-rolling"
|
||||||
arch?: Arch; // detected from HardwareInfo or overridden
|
arch?: Arch; // detected from HardwareInfo or overridden
|
||||||
queued_at: string;
|
queued_at: string;
|
||||||
|
/**
|
||||||
|
* When dispatch last served this machine an install boot script. Progress
|
||||||
|
* callbacks only start once the installer environment is up, so a machine
|
||||||
|
* dispatched long ago with no progress is wedged before that point (bad
|
||||||
|
* kernel/initrd, no network in the initramfs, wrong NIC picked...).
|
||||||
|
*/
|
||||||
|
dispatched_at?: string;
|
||||||
progress?: string;
|
progress?: string;
|
||||||
progress_at?: string;
|
progress_at?: string;
|
||||||
progress_detail?: string;
|
progress_detail?: string;
|
||||||
|
|||||||
@@ -29,6 +29,17 @@ export interface PxeVmConfig {
|
|||||||
diskSize: number; // GB
|
diskSize: number; // GB
|
||||||
network: string; // libvirt network name
|
network: string; // libvirt network name
|
||||||
arch?: "x86_64" | "aarch64";
|
arch?: "x86_64" | "aarch64";
|
||||||
|
/**
|
||||||
|
* Extra NICs enumerated BEFORE the PXE NIC, on a network with no route to
|
||||||
|
* the bastion (defaults to libvirt's "default").
|
||||||
|
*
|
||||||
|
* Real multi-NIC boxes expose a class of bug a single-NIC VM cannot: an
|
||||||
|
* initramfs that picks "the first connected interface" grabs one of these
|
||||||
|
* instead of the NIC that PXE booted, and then cannot reach the bastion.
|
||||||
|
* Defaults to 0 (single NIC).
|
||||||
|
*/
|
||||||
|
decoyNics?: number;
|
||||||
|
decoyNetwork?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Create a blank UEFI VM that PXE boots from the network. */
|
/** Create a blank UEFI VM that PXE boots from the network. */
|
||||||
@@ -61,6 +72,10 @@ export function createPxeVm(config: PxeVmConfig): void {
|
|||||||
`--memory=${config.memory}`,
|
`--memory=${config.memory}`,
|
||||||
`--vcpus=${config.vcpus}`,
|
`--vcpus=${config.vcpus}`,
|
||||||
`--disk=path=${diskPath},format=qcow2,bus=virtio`,
|
`--disk=path=${diskPath},format=qcow2,bus=virtio`,
|
||||||
|
// Decoys first so they enumerate ahead of the PXE NIC. They are up and
|
||||||
|
// carry a lease, but have no route to the bastion.
|
||||||
|
...Array.from({ length: config.decoyNics ?? 0 }, () =>
|
||||||
|
`--network=network=${config.decoyNetwork ?? "default"},model=virtio`),
|
||||||
`--network=network=${config.network},model=virtio`,
|
`--network=network=${config.network},model=virtio`,
|
||||||
// UEFI firmware — required for PXE boot in modern mode
|
// UEFI firmware — required for PXE boot in modern mode
|
||||||
`--boot=uefi,network,hd`,
|
`--boot=uefi,network,hd`,
|
||||||
@@ -95,12 +110,21 @@ export function destroyPxeVm(name: string): void {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Get the MAC address of a VM's first NIC. */
|
/** Get the MAC address of a VM's first NIC. */
|
||||||
export function getVmMac(name: string): string | null {
|
export function getVmMac(name: string, network?: string): string | null {
|
||||||
const result = virsh("domiflist", name);
|
const result = virsh("domiflist", name);
|
||||||
if (result.status !== 0) return null;
|
if (result.status !== 0) return null;
|
||||||
// Output format: Interface Type Source Model MAC
|
// Output format: Interface Type Source Model MAC
|
||||||
const match = result.stdout.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i);
|
// With decoy NICs present, match the line for the PXE network so we return
|
||||||
return match ? match[1].toLowerCase() : null;
|
// the NIC that actually boots rather than whichever is listed first.
|
||||||
|
const lines = result.stdout.split("\n");
|
||||||
|
const candidates = network === undefined
|
||||||
|
? lines
|
||||||
|
: lines.filter((l) => l.split(/\s+/).includes(network));
|
||||||
|
for (const line of candidates.length > 0 ? candidates : lines) {
|
||||||
|
const m = line.match(/([0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2}:[0-9a-f]{2})/i);
|
||||||
|
if (m) return m[1].toLowerCase();
|
||||||
|
}
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Reboot a VM (force off + start). */
|
/** Reboot a VM (force off + start). */
|
||||||
|
|||||||
387
bastion/tests/integration/vyos-provision.test.ts
Normal file
387
bastion/tests/integration/vyos-provision.test.ts
Normal file
@@ -0,0 +1,387 @@
|
|||||||
|
// Integration test: full VyOS unattended provisioning flow.
|
||||||
|
//
|
||||||
|
// Validates the VyOS install path end-to-end, at the same depth as the Fedora
|
||||||
|
// pxe-provision test:
|
||||||
|
// 1. Bastion (HTTP + dnsmasq) on the isolated libvirt PXE network
|
||||||
|
// 2. Blank UEFI VM PXE boots -> Fedora-based discovery (OS-neutral)
|
||||||
|
// 3. Queue os=vyos-rolling -> live boot + live-config hook + pty driver
|
||||||
|
// 4. Fresh-install asserts: installed.ip, streamed logs, applied config,
|
||||||
|
// /config/lab-provisioned, boot-order handling
|
||||||
|
// 5. REINSTALL round: previous config + /config data carried forward
|
||||||
|
// ("reinstall without losing data", VyOS-flavored)
|
||||||
|
// 6. freshConfig round: bastion-generated config wins, /config data kept
|
||||||
|
//
|
||||||
|
// Prerequisites: libvirtd, OVMF, ipxe-bootimgs-x86, sudo, internet
|
||||||
|
// (first run downloads the ~600MB VyOS nightly ISO; artifacts are cached).
|
||||||
|
// Run: sudo pnpm run test:integration:vyos
|
||||||
|
|
||||||
|
import { describe, it, expect, beforeAll, afterAll } from "vitest";
|
||||||
|
import { readFileSync, existsSync, mkdirSync, rmSync, copyFileSync, symlinkSync, writeFileSync } from "node:fs";
|
||||||
|
import { execSync } from "node:child_process";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { homedir, tmpdir } from "node:os";
|
||||||
|
import { log, waitForSsh } from "./helpers/libvirt.js";
|
||||||
|
import { ensurePxeNetwork, destroyPxeNetwork, deleteNftablesRejectRules, PXE_NETWORK_NAME, PXE_GATEWAY, PXE_SUBNET } from "./helpers/pxe-network.js";
|
||||||
|
import { createPxeVm, destroyPxeVm, getVmMac, rebootPxeVm } from "./helpers/pxe-vm.js";
|
||||||
|
import { sshExec } from "./helpers/ssh.js";
|
||||||
|
|
||||||
|
const VM_NAME = "lab-vyos-test";
|
||||||
|
const VM_MEMORY = 4096;
|
||||||
|
const VM_VCPUS = 4;
|
||||||
|
const VM_DISK_GB = 10; // VyOS image install needs ~2GB minimum
|
||||||
|
const HTTP_PORT = 8099;
|
||||||
|
const SSH_USER = "vyos"; // the only VyOS login user
|
||||||
|
const BASTION_IP = PXE_GATEWAY;
|
||||||
|
const DHCP_RANGE_START = `${PXE_SUBNET}.100`;
|
||||||
|
const DHCP_RANGE_END = `${PXE_SUBNET}.200`;
|
||||||
|
|
||||||
|
const DISCOVERY_TIMEOUT_MS = 5 * 60_000;
|
||||||
|
const INSTALL_TIMEOUT_MS = 15 * 60_000; // squashfs fetch + copy; much faster than Anaconda
|
||||||
|
const SSH_TIMEOUT_MS = 8 * 60_000;
|
||||||
|
|
||||||
|
const HOSTNAME_R1 = "vyos-r1";
|
||||||
|
const HOSTNAME_R2 = "vyos-r2";
|
||||||
|
const HOSTNAME_R3 = "vyos-r3";
|
||||||
|
|
||||||
|
function findSshKey(): { pubKey: string; keyPath: string } {
|
||||||
|
const homes = [homedir()];
|
||||||
|
const sudoUser = process.env["SUDO_USER"];
|
||||||
|
if (sudoUser) homes.push(join("/home", sudoUser));
|
||||||
|
if (process.env["SSH_KEY_PATH"]) {
|
||||||
|
const keyPath = process.env["SSH_KEY_PATH"];
|
||||||
|
const pubPath = `${keyPath}.pub`;
|
||||||
|
if (existsSync(keyPath) && existsSync(pubPath)) {
|
||||||
|
return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const home of homes) {
|
||||||
|
for (const name of ["id_ed25519", "id_ecdsa", "id_rsa"]) {
|
||||||
|
const keyPath = join(home, ".ssh", name);
|
||||||
|
const pubPath = `${keyPath}.pub`;
|
||||||
|
if (existsSync(keyPath) && existsSync(pubPath)) {
|
||||||
|
return { pubKey: readFileSync(pubPath, "utf-8").trim(), keyPath };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new Error("No SSH key found — set SSH_KEY_PATH or ensure keys exist in ~/.ssh/");
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((r) => setTimeout(r, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function pollApi<T>(
|
||||||
|
url: string,
|
||||||
|
check: (data: T) => boolean,
|
||||||
|
timeoutMs: number,
|
||||||
|
intervalMs = 5000,
|
||||||
|
): Promise<T> {
|
||||||
|
const start = Date.now();
|
||||||
|
while (Date.now() - start < timeoutMs) {
|
||||||
|
try {
|
||||||
|
const res = await fetch(url);
|
||||||
|
if (res.ok) {
|
||||||
|
const data = (await res.json()) as T;
|
||||||
|
if (check(data)) return data;
|
||||||
|
}
|
||||||
|
} catch { /* not ready yet */ }
|
||||||
|
await sleep(intervalMs);
|
||||||
|
}
|
||||||
|
throw new Error(`Timeout after ${timeoutMs}ms polling ${url}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
type LogsResponse = {
|
||||||
|
status: string;
|
||||||
|
progress: string;
|
||||||
|
progress_detail?: string;
|
||||||
|
ip?: string;
|
||||||
|
log_total?: number;
|
||||||
|
log_lines?: Array<{ line: string }>;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Queue a VyOS install, reboot the VM into PXE, wait for completion + SSH. */
|
||||||
|
async function installRound(opts: {
|
||||||
|
mac: string;
|
||||||
|
hostname: string;
|
||||||
|
freshConfig?: boolean;
|
||||||
|
}): Promise<string> {
|
||||||
|
const body = {
|
||||||
|
mac: opts.mac,
|
||||||
|
hostname: opts.hostname,
|
||||||
|
disk: "/dev/vda",
|
||||||
|
role: "vanilla",
|
||||||
|
os: "vyos-rolling",
|
||||||
|
vyos: {
|
||||||
|
mgmtInterface: "eth0",
|
||||||
|
mgmtAddress: "dhcp",
|
||||||
|
hwIds: { eth0: opts.mac },
|
||||||
|
...(opts.freshConfig ? { freshConfig: true } : {}),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/install`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
});
|
||||||
|
log(`Install queued (${opts.hostname}): ${JSON.stringify(await res.json())}`);
|
||||||
|
|
||||||
|
await sleep(5_000);
|
||||||
|
rebootPxeVm(VM_NAME);
|
||||||
|
await sleep(3_000);
|
||||||
|
deleteNftablesRejectRules();
|
||||||
|
|
||||||
|
const finalState = await pollApi<LogsResponse>(
|
||||||
|
`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(opts.mac)}`,
|
||||||
|
(data) => data.status === "installed" || data.progress === "error",
|
||||||
|
INSTALL_TIMEOUT_MS,
|
||||||
|
10_000,
|
||||||
|
);
|
||||||
|
if (finalState.progress === "error") {
|
||||||
|
log(`INSTALL FAILED: ${JSON.stringify(finalState.progress_detail ?? finalState, null, 2)}`);
|
||||||
|
throw new Error(`VyOS install failed for ${opts.hostname}`);
|
||||||
|
}
|
||||||
|
const ip = finalState.ip ?? "";
|
||||||
|
log(`Install complete (${opts.hostname}). IP: ${ip}`);
|
||||||
|
|
||||||
|
// The driver force-reboots; the VM PXE boots, dispatch says installed ->
|
||||||
|
// localboot exit -> GRUB -> VyOS. nftables reject rules do not reappear
|
||||||
|
// (guest reboot, not a libvirt restart), but clearing is harmless.
|
||||||
|
deleteNftablesRejectRules();
|
||||||
|
await waitForSsh(ip, SSH_USER, SSH_TIMEOUT_MS, sshKeyPathGlobal);
|
||||||
|
return ip;
|
||||||
|
}
|
||||||
|
|
||||||
|
let sshKeyPathGlobal = "";
|
||||||
|
|
||||||
|
describe("VyOS provisioning", () => {
|
||||||
|
let bastionApp: { close: () => Promise<void> };
|
||||||
|
let testDir: string;
|
||||||
|
let vmMac: string;
|
||||||
|
let vmIp: string;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
const { pubKey, keyPath } = findSshKey();
|
||||||
|
sshKeyPathGlobal = keyPath;
|
||||||
|
|
||||||
|
log("Setting up PXE test network...");
|
||||||
|
ensurePxeNetwork();
|
||||||
|
|
||||||
|
testDir = join(tmpdir(), `lab-vyos-test-${Date.now()}`);
|
||||||
|
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||||
|
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||||
|
mkdirSync(join(testDir, "logs"), { recursive: true });
|
||||||
|
|
||||||
|
log("Starting bastion...");
|
||||||
|
const { createApp } = await import("../../src/bastion/src/server.js");
|
||||||
|
const { loadConfig } = await import("../../src/bastion/src/config.js");
|
||||||
|
const { generateDnsmasqConf, startDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
|
||||||
|
const { generateDiscoverKickstart } = await import("../../src/bastion/src/services/kickstart-generator.js");
|
||||||
|
const { renderBootIpxe } = await import("../../src/bastion/src/templates/boot.ipxe.js");
|
||||||
|
const { prepareVyosArtifacts } = await import("../../src/bastion/src/main.js");
|
||||||
|
|
||||||
|
const config = loadConfig({
|
||||||
|
bastionDir: testDir,
|
||||||
|
httpPort: HTTP_PORT,
|
||||||
|
iface: "virbr-pxe",
|
||||||
|
serverIp: BASTION_IP,
|
||||||
|
network: `${PXE_SUBNET}.0`,
|
||||||
|
gateway: BASTION_IP,
|
||||||
|
dhcpMode: "full",
|
||||||
|
dhcpRangeStart: DHCP_RANGE_START,
|
||||||
|
dhcpRangeEnd: DHCP_RANGE_END,
|
||||||
|
domain: "pxe-test.local",
|
||||||
|
sshKeys: [pubKey],
|
||||||
|
adminUser: "lab",
|
||||||
|
});
|
||||||
|
|
||||||
|
// iPXE binary
|
||||||
|
const ipxeSrc = "/usr/share/ipxe/ipxe-snponly-x86_64.efi";
|
||||||
|
if (!existsSync(ipxeSrc)) {
|
||||||
|
throw new Error(`iPXE not found: ${ipxeSrc}. Install: sudo dnf install ipxe-bootimgs-x86`);
|
||||||
|
}
|
||||||
|
copyFileSync(ipxeSrc, join(config.tftpDir, "ipxe.efi"));
|
||||||
|
try { symlinkSync(join(config.tftpDir, "ipxe.efi"), join(config.httpDir, "ipxe.efi")); } catch { /* exists */ }
|
||||||
|
|
||||||
|
const cacheDir = "/var/lib/libvirt/images/lab-pxe-cache";
|
||||||
|
execSync(`mkdir -p "${cacheDir}"`, { stdio: "pipe" });
|
||||||
|
|
||||||
|
// Fedora kernel+initrd for DISCOVERY (OS-neutral, same as pxe test)
|
||||||
|
const kernel = join(cacheDir, `vmlinuz-${config.fedoraVersion}`);
|
||||||
|
const initrd = join(cacheDir, `initrd-${config.fedoraVersion}.img`);
|
||||||
|
if (!existsSync(kernel)) {
|
||||||
|
log(`Downloading Fedora ${config.fedoraVersion} kernel (discovery)...`);
|
||||||
|
execSync(`curl -# -L -f -o "${kernel}" "${config.fedoraMirror}/images/pxeboot/vmlinuz"`, { stdio: "inherit", timeout: 300_000 });
|
||||||
|
}
|
||||||
|
if (!existsSync(initrd)) {
|
||||||
|
log(`Downloading Fedora ${config.fedoraVersion} initrd (discovery)...`);
|
||||||
|
execSync(`curl -# -L -f -o "${initrd}" "${config.fedoraMirror}/images/pxeboot/initrd.img"`, { stdio: "inherit", timeout: 300_000 });
|
||||||
|
}
|
||||||
|
copyFileSync(kernel, join(config.httpDir, "vmlinuz"));
|
||||||
|
copyFileSync(initrd, join(config.httpDir, "initrd.img"));
|
||||||
|
|
||||||
|
// VyOS netboot artifacts — cache the three extracted files across runs
|
||||||
|
const vyosCache = {
|
||||||
|
kernel: join(cacheDir, "vyos-vmlinuz"),
|
||||||
|
initrd: join(cacheDir, "vyos-initrd"),
|
||||||
|
squashfs: join(cacheDir, "vyos-filesystem.squashfs"),
|
||||||
|
};
|
||||||
|
if (Object.values(vyosCache).every((p) => existsSync(p))) {
|
||||||
|
log("VyOS netboot artifacts cached");
|
||||||
|
copyFileSync(vyosCache.kernel, join(config.httpDir, "vyos-vmlinuz"));
|
||||||
|
copyFileSync(vyosCache.initrd, join(config.httpDir, "vyos-initrd"));
|
||||||
|
copyFileSync(vyosCache.squashfs, join(config.httpDir, "vyos-filesystem.squashfs"));
|
||||||
|
} else {
|
||||||
|
log("Extracting VyOS artifacts from ISO (downloads ~600MB on first run)...");
|
||||||
|
prepareVyosArtifacts(config);
|
||||||
|
copyFileSync(join(config.httpDir, "vyos-vmlinuz"), vyosCache.kernel);
|
||||||
|
copyFileSync(join(config.httpDir, "vyos-initrd"), vyosCache.initrd);
|
||||||
|
copyFileSync(join(config.httpDir, "vyos-filesystem.squashfs"), vyosCache.squashfs);
|
||||||
|
}
|
||||||
|
|
||||||
|
writeFileSync(join(config.httpDir, "discover.ks"), generateDiscoverKickstart(config));
|
||||||
|
writeFileSync(join(config.httpDir, "boot.ipxe"), renderBootIpxe({ serverIp: config.serverIp, httpPort: config.httpPort }));
|
||||||
|
generateDnsmasqConf(config);
|
||||||
|
|
||||||
|
const { app, syslog } = createApp(config);
|
||||||
|
bastionApp = app;
|
||||||
|
await app.listen({ port: config.httpPort, host: "0.0.0.0" });
|
||||||
|
syslog.start();
|
||||||
|
log(`Bastion listening on :${HTTP_PORT}`);
|
||||||
|
|
||||||
|
log("Starting dnsmasq...");
|
||||||
|
startDnsmasq(config).catch((err) => {
|
||||||
|
log(`dnsmasq failed (expected without root): ${err instanceof Error ? err.message : String(err)}`);
|
||||||
|
});
|
||||||
|
await sleep(1000);
|
||||||
|
|
||||||
|
log("Creating PXE VM...");
|
||||||
|
// Two decoy NICs ahead of the PXE NIC, on a network with no route to the
|
||||||
|
// bastion. This reproduces the real VP2440 topology: live-boot scans for
|
||||||
|
// "the first connected interface", and without BOOTIF it picks a decoy,
|
||||||
|
// times out on DHCP/fetch, and dies with "Unable to find a live file
|
||||||
|
// system on the network". A single-NIC VM cannot catch that.
|
||||||
|
createPxeVm({
|
||||||
|
name: VM_NAME,
|
||||||
|
memory: VM_MEMORY,
|
||||||
|
vcpus: VM_VCPUS,
|
||||||
|
diskSize: VM_DISK_GB,
|
||||||
|
network: PXE_NETWORK_NAME,
|
||||||
|
decoyNics: 2,
|
||||||
|
});
|
||||||
|
const mac = getVmMac(VM_NAME, PXE_NETWORK_NAME);
|
||||||
|
if (!mac) throw new Error("Could not determine VM MAC address");
|
||||||
|
vmMac = mac;
|
||||||
|
log(`VM MAC: ${vmMac}`);
|
||||||
|
|
||||||
|
log("Waiting for discovery...");
|
||||||
|
type MachinesResponse = { discovered: Record<string, unknown> };
|
||||||
|
await pollApi<MachinesResponse>(
|
||||||
|
`http://${BASTION_IP}:${HTTP_PORT}/api/machines`,
|
||||||
|
(data) => vmMac in data.discovered,
|
||||||
|
DISCOVERY_TIMEOUT_MS,
|
||||||
|
);
|
||||||
|
log("VM discovered. Running fresh VyOS install (round 1)...");
|
||||||
|
|
||||||
|
await sleep(15_000); // discovery reboot cycle
|
||||||
|
vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R1 });
|
||||||
|
log("Round 1 (fresh install) complete.");
|
||||||
|
}, DISCOVERY_TIMEOUT_MS + INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 300_000);
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
log("Cleaning up...");
|
||||||
|
if (bastionApp) await bastionApp.close().catch(() => {});
|
||||||
|
const { stopDnsmasq } = await import("../../src/bastion/src/services/dnsmasq.js");
|
||||||
|
stopDnsmasq();
|
||||||
|
destroyPxeVm(VM_NAME);
|
||||||
|
destroyPxeNetwork();
|
||||||
|
if (testDir) rmSync(testDir, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("machine is installed with a real IP (WI-1: ready-at parsing)", async () => {
|
||||||
|
const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/machines`);
|
||||||
|
const data = (await res.json()) as { installed: Record<string, { ip: string; os?: string }> };
|
||||||
|
const machine = data.installed[vmMac];
|
||||||
|
expect(machine).toBeDefined();
|
||||||
|
expect(machine.ip).toMatch(/^\d+\.\d+\.\d+\.\d+$/);
|
||||||
|
expect(machine.os).toBe("vyos-rolling");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("install logs were streamed live (WI-2)", async () => {
|
||||||
|
const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`);
|
||||||
|
const data = (await res.json()) as LogsResponse;
|
||||||
|
expect(data.log_total).toBeGreaterThan(0);
|
||||||
|
const lines = (data.log_lines ?? []).map((l) => l.line).join("\n");
|
||||||
|
// Installer transcript lines and driver messages both flow through /api/log
|
||||||
|
expect(lines).toMatch(/Welcome to VyOS installation|>>> answered|base config:/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("SSH works as the vyos user with the injected key", () => {
|
||||||
|
const result = sshExec(vmIp, SSH_USER, "whoami", { keyPath: sshKeyPathGlobal });
|
||||||
|
expect(result.exitCode).toBe(0);
|
||||||
|
expect(result.stdout.trim()).toBe("vyos");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("generated config was adopted (hostname + ssh key)", () => {
|
||||||
|
const result = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal });
|
||||||
|
expect(result.exitCode).toBe(0);
|
||||||
|
expect(result.stdout).toContain(`host-name "${HOSTNAME_R1}"`);
|
||||||
|
expect(result.stdout).toContain("public-keys");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("boot-order step ran and reported (WI-3)", async () => {
|
||||||
|
const res = await fetch(`http://${BASTION_IP}:${HTTP_PORT}/api/logs/${encodeURIComponent(vmMac)}`);
|
||||||
|
const data = (await res.json()) as LogsResponse;
|
||||||
|
const lines = (data.log_lines ?? []).map((l) => l.line).join("\n");
|
||||||
|
expect(lines).toContain("boot order:");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("provisioning metadata persisted to /config (WI-4)", () => {
|
||||||
|
const result = sshExec(vmIp, SSH_USER, "cat /config/lab-provisioned 2>/dev/null || cat /opt/vyatta/etc/config/lab-provisioned", { keyPath: sshKeyPathGlobal });
|
||||||
|
expect(result.exitCode).toBe(0);
|
||||||
|
expect(result.stdout).toContain(`hostname=${HOSTNAME_R1}`);
|
||||||
|
expect(result.stdout).toContain("role=vanilla");
|
||||||
|
expect(result.stdout).toContain(`bastion=http://${BASTION_IP}:${HTTP_PORT}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reinstall preserves config and /config data (round 2)", async () => {
|
||||||
|
// Drop a marker in /config — the installer's previous-installation copy
|
||||||
|
// must carry it (and the whole old config) into the new image.
|
||||||
|
// `sync` is REQUIRED: rebootPxeVm uses `virsh destroy` (a hard power-cut),
|
||||||
|
// so an unsynced write never reaches the disk and the marker vanishes for
|
||||||
|
// reasons that have nothing to do with the installer.
|
||||||
|
const marker = sshExec(vmIp, SSH_USER, "echo LAB-MARKER-R2 > /config/lab-marker && sync && cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
|
||||||
|
expect(marker.exitCode).toBe(0);
|
||||||
|
expect(marker.stdout).toContain("LAB-MARKER-R2");
|
||||||
|
|
||||||
|
// Queue with a DIFFERENT hostname: with preserve semantics the previous
|
||||||
|
// config must win, so the hostname must NOT change.
|
||||||
|
vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R2 });
|
||||||
|
|
||||||
|
// Assert the config carry-forward first — it is the primary preservation
|
||||||
|
// signal and does not depend on the marker mechanism above.
|
||||||
|
const cfg = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal });
|
||||||
|
expect(cfg.stdout).toContain(`host-name "${HOSTNAME_R1}"`); // old config carried
|
||||||
|
expect(cfg.stdout).not.toContain(`host-name "${HOSTNAME_R2}"`);
|
||||||
|
|
||||||
|
const markerAfter = sshExec(vmIp, SSH_USER, "cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
|
||||||
|
expect(markerAfter.exitCode).toBe(0);
|
||||||
|
expect(markerAfter.stdout).toContain("LAB-MARKER-R2");
|
||||||
|
}, INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 60_000);
|
||||||
|
|
||||||
|
it("freshConfig makes the generated config win, data still kept (round 3)", async () => {
|
||||||
|
// Re-assert the marker is on disk and synced before the next power-cut.
|
||||||
|
const pre = sshExec(vmIp, SSH_USER, "sync && cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
|
||||||
|
expect(pre.stdout).toContain("LAB-MARKER-R2");
|
||||||
|
|
||||||
|
vmIp = await installRound({ mac: vmMac, hostname: HOSTNAME_R3, freshConfig: true });
|
||||||
|
|
||||||
|
const cfg = sshExec(vmIp, SSH_USER, "cat /opt/vyatta/etc/config/config.boot", { keyPath: sshKeyPathGlobal });
|
||||||
|
expect(cfg.stdout).toContain(`host-name "${HOSTNAME_R3}"`); // generated config won
|
||||||
|
|
||||||
|
// The marker file (non-config data under /config) still survives —
|
||||||
|
// freshConfig replaces only config.boot, not the carried data.
|
||||||
|
const markerAfter = sshExec(vmIp, SSH_USER, "cat /config/lab-marker", { keyPath: sshKeyPathGlobal });
|
||||||
|
expect(markerAfter.exitCode).toBe(0);
|
||||||
|
expect(markerAfter.stdout).toContain("LAB-MARKER-R2");
|
||||||
|
}, INSTALL_TIMEOUT_MS + SSH_TIMEOUT_MS + 60_000);
|
||||||
|
});
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
{
|
{
|
||||||
"files": [],
|
"files": [],
|
||||||
"references": [
|
"references": [
|
||||||
|
{ "path": "src/core" },
|
||||||
{ "path": "src/shared" },
|
{ "path": "src/shared" },
|
||||||
{ "path": "src/bastion" },
|
{ "path": "src/bastion" },
|
||||||
{ "path": "src/cli" },
|
{ "path": "src/cli" },
|
||||||
|
|||||||
4
labsim/.gitignore
vendored
Normal file
4
labsim/.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
# runtime artifacts, not source
|
||||||
|
*.log
|
||||||
|
labsim_matrix_lib.py
|
||||||
|
__pycache__/
|
||||||
104
labsim/README.md
Normal file
104
labsim/README.md
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
# labsim — libvirt replica of the lab network
|
||||||
|
|
||||||
|
A throwaway copy of the production VLAN topology for testing routing, firewall
|
||||||
|
rules and failover **without touching the real network**. Same VLAN IDs and
|
||||||
|
roles as UniFi, deliberately different IP ranges so nothing can be confused for
|
||||||
|
production.
|
||||||
|
|
||||||
|
## Topology
|
||||||
|
|
||||||
|
Each VLAN is its own isolated libvirt network with one tiny Alpine VM on it.
|
||||||
|
|
||||||
|
| VLAN | Name | Sim subnet | VM address | Mirrors production |
|
||||||
|
|-----:|------|------------|-----------|--------------------|
|
||||||
|
| 1 | management | 172.31.1.0/24 | 172.31.1.10 | 192.168.1.0/24 |
|
||||||
|
| 2 | k8s | 172.31.2.0/24 | 172.31.2.10 | 192.168.8.0/23 |
|
||||||
|
| 3 | kvm | 172.31.3.0/24 | 172.31.3.10 | 192.168.3.0/24 |
|
||||||
|
| 9 | private | 172.31.9.0/24 | 172.31.9.10 | 10.0.9.0/23 |
|
||||||
|
| 10 | lot | 172.31.10.0/24 | 172.31.10.10 | 10.0.0.0/23 |
|
||||||
|
| 200 | roomates | 172.31.200.0/24 | 172.31.200.10 | 192.168.2.0/24 |
|
||||||
|
|
||||||
|
The sim subnet always encodes the VLAN id: `172.31.<vlan>.0/24`.
|
||||||
|
|
||||||
|
Address plan, identical on every VLAN:
|
||||||
|
|
||||||
|
| Address | Role |
|
||||||
|
|---------|------|
|
||||||
|
| `.1` | gateway under test — a router VM you add (not created by default) |
|
||||||
|
| `.2` | host bridge — how you reach the VMs from this workstation |
|
||||||
|
| `.10` | the VLAN's micro VM |
|
||||||
|
| `.254` | reserved for a VRRP VIP, mirroring production |
|
||||||
|
|
||||||
|
The host sits at `.2` purely so you can SSH in. It is deliberately **not** the
|
||||||
|
VMs' default route — that is `.1` — so inter-VLAN tests fail loudly when no
|
||||||
|
router is present instead of being silently served by the host's own routing
|
||||||
|
table. libvirt also installs reject rules that stop these networks forwarding
|
||||||
|
to each other, so traffic between VLANs only works once a router VM bridges
|
||||||
|
them.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./labsim-up.sh # bring up every VLAN (idempotent)
|
||||||
|
./labsim-up.sh 2 3 # only VLANs 2 and 3
|
||||||
|
./labsim-down.sh # destroy VMs + networks, keep the base image
|
||||||
|
./labsim-down.sh --purge # also delete the downloaded Alpine image
|
||||||
|
```
|
||||||
|
|
||||||
|
Each VM: 256 MB, 1 vCPU, a copy-on-write overlay on one shared 176 MB Alpine
|
||||||
|
image (so six VMs cost a few MB of disk, not 1 GB).
|
||||||
|
|
||||||
|
## Access
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ssh alpine@172.31.2.10 # normal user (password: labsim)
|
||||||
|
ssh root@172.31.2.10 # privileged — this image has no sudo
|
||||||
|
curl http://172.31.2.10/ # hello-world page naming the VLAN
|
||||||
|
```
|
||||||
|
|
||||||
|
Console, when the network is the thing that is broken:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo virsh console labsim-2-k8s # root / labsim
|
||||||
|
```
|
||||||
|
|
||||||
|
## Watching it
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./labsim-matrix.py --watch 2 # terminal grid, changed cells highlighted
|
||||||
|
./monitoring-up.sh # topology page + Prometheus + Grafana
|
||||||
|
```
|
||||||
|
|
||||||
|
- **http://localhost:9101/** — live mesh: a node per VLAN, the router in the
|
||||||
|
middle, one line per pair coloured green/red with the ICMP RTT on it. Hover a
|
||||||
|
line for per-direction detail. Refreshes every 5s. This is the one to watch
|
||||||
|
while changing firewall rules.
|
||||||
|
- **http://localhost:3000/d/labsim-matrix** — Grafana (anonymous, no login) for
|
||||||
|
*history*: when did a path flip, and how has latency moved.
|
||||||
|
- **http://localhost:9101/metrics** — `labsim_reachable{src,dst,proto}` and
|
||||||
|
`labsim_rtt_ms{src,dst}`.
|
||||||
|
|
||||||
|
## Notes for whoever extends this
|
||||||
|
|
||||||
|
Things that cost time the first time round, all verified on this image:
|
||||||
|
|
||||||
|
- **No `sudo`.** Alpine ships `doas`; cloud-init's `sudo:` directive is inert
|
||||||
|
here. Use `root@` for privileged work.
|
||||||
|
- **cloud-init leaves users locked** (`!*` in `/etc/shadow`) unless
|
||||||
|
`lock_passwd: false`, and sshd then refuses key auth for that user.
|
||||||
|
- **One failing `runcmd` aborts every command after it.** Each entry is
|
||||||
|
`|| true` for that reason.
|
||||||
|
- **busybox here has no `httpd` applet**, and the VMs have no internet to
|
||||||
|
`apk add` one — so the hello-world server is `python3 -m http.server`
|
||||||
|
(python3 is already present because cloud-init depends on it).
|
||||||
|
- **`start-stop-daemon --exec /usr/bin/python3` matches cloud-init's own
|
||||||
|
python3** at boot and refuses to start anything.
|
||||||
|
- **busybox `pgrep -f PATTERN` matches its own argv**, so a "skip if already
|
||||||
|
running" guard always fires. Verified: `guard_exit=0` with nothing listening.
|
||||||
|
|
||||||
|
## Not modelled (yet)
|
||||||
|
|
||||||
|
VLANs are separate L2 segments rather than one 802.1Q trunk, so this exercises
|
||||||
|
inter-VLAN routing but not a `bond0.<vif>` trunk config specifically. A router
|
||||||
|
VM would attach one NIC per VLAN. Adding a tagged-trunk variant is the obvious
|
||||||
|
next step if the bond/vif config itself needs testing.
|
||||||
61
labsim/labsim-down.sh
Executable file
61
labsim/labsim-down.sh
Executable file
@@ -0,0 +1,61 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Tear down the lab network simulation.
|
||||||
|
#
|
||||||
|
# By default this destroys VMs and networks but KEEPS the downloaded base
|
||||||
|
# image, so the next bring-up is fast. Pass --purge to remove that too.
|
||||||
|
#
|
||||||
|
# Usage: ./labsim-down.sh [--purge] [vlan-id ...]
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
source "$SCRIPT_DIR/lib.sh"
|
||||||
|
source "$SCRIPT_DIR/ovs.sh"
|
||||||
|
|
||||||
|
PURGE=false
|
||||||
|
ARGS=()
|
||||||
|
for a in "$@"; do
|
||||||
|
case "$a" in
|
||||||
|
--purge) PURGE=true ;;
|
||||||
|
*) ARGS+=("$a") ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
selected_vlans "${ARGS[@]+"${ARGS[@]}"}"
|
||||||
|
|
||||||
|
for entry in "${SELECTED[@]}"; do
|
||||||
|
IFS=: read -r vid name prefix _r <<<"$entry"
|
||||||
|
vm="$(vm_name "$vid" "$name")"
|
||||||
|
|
||||||
|
if virsh_q dominfo "$vm" >/dev/null 2>&1; then
|
||||||
|
log "destroying VM $vm"
|
||||||
|
virsh_q destroy "$vm" >/dev/null 2>&1 || true
|
||||||
|
virsh_q undefine "$vm" --nvram >/dev/null 2>&1 || virsh_q undefine "$vm" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
sudo rm -f "$IMG_DIR/${vm}.qcow2" "$IMG_DIR/${vm}-seed.iso"
|
||||||
|
|
||||||
|
done
|
||||||
|
|
||||||
|
# Legacy per-VLAN Linux-bridge networks from before the OVS migration. If
|
||||||
|
# these survive they keep a duplicate <prefix>.2/24 on a dead bridge, and the
|
||||||
|
# kernel may prefer that route over the OVS host leg — which looks exactly
|
||||||
|
# like "the VM is unreachable" while ping -I hostvN works fine.
|
||||||
|
for entry in "${SELECTED[@]}"; do
|
||||||
|
IFS=: read -r vid _n _p _r <<<"$entry"
|
||||||
|
legacy="labsim-vlan${vid}"
|
||||||
|
if virsh_q net-info "$legacy" >/dev/null 2>&1; then
|
||||||
|
log "removing legacy network $legacy"
|
||||||
|
virsh_q net-destroy "$legacy" >/dev/null 2>&1 || true
|
||||||
|
virsh_q net-undefine "$legacy" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
log "removing OVS fabric"
|
||||||
|
ovs_down
|
||||||
|
|
||||||
|
if [ "$PURGE" = true ]; then
|
||||||
|
log "purging base image $BASE_IMAGE"
|
||||||
|
sudo rm -f "$BASE_IMAGE"
|
||||||
|
sudo rmdir "$IMG_DIR" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "environment is DOWN"
|
||||||
157
labsim/labsim-exporter.py
Executable file
157
labsim/labsim-exporter.py
Executable file
@@ -0,0 +1,157 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Prometheus exporter for the labsim connectivity matrix.
|
||||||
|
|
||||||
|
Runs the same sweep as labsim-matrix.py on an interval and exposes it as
|
||||||
|
metrics, so Grafana can show the mesh as a heatmap and — more usefully — a
|
||||||
|
history of exactly when a cell flipped after a firewall change.
|
||||||
|
|
||||||
|
labsim_reachable{src,dst,proto} 1 = reachable, 0 = blocked
|
||||||
|
labsim_sweep_seconds how long the last sweep took
|
||||||
|
labsim_sweep_total sweeps completed since start
|
||||||
|
labsim_up 1 while the exporter is alive
|
||||||
|
|
||||||
|
Deliberately stdlib-only (http.server + threads): this runs on the workstation
|
||||||
|
next to libvirt, and adding a dependency to watch a lab network is silly.
|
||||||
|
|
||||||
|
./labsim-exporter.py --port 9101 --interval 15
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import http.server
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
import labsim_matrix_lib as m # thin import shim, see below
|
||||||
|
|
||||||
|
|
||||||
|
class Collector:
|
||||||
|
def __init__(self, interval: int, timeout: int) -> None:
|
||||||
|
self.interval = interval
|
||||||
|
self.timeout = timeout
|
||||||
|
self.vlans = m.load_vlans()
|
||||||
|
self.lock = threading.Lock()
|
||||||
|
self.results: dict = {}
|
||||||
|
self.duration = 0.0
|
||||||
|
self.sweeps = 0
|
||||||
|
|
||||||
|
def loop(self) -> None:
|
||||||
|
while True:
|
||||||
|
started = time.time()
|
||||||
|
try:
|
||||||
|
results = m.sweep(self.vlans, self.timeout)
|
||||||
|
with self.lock:
|
||||||
|
self.results = results
|
||||||
|
self.duration = time.time() - started
|
||||||
|
self.sweeps += 1
|
||||||
|
except Exception: # noqa: BLE001 - never let the loop die
|
||||||
|
pass
|
||||||
|
time.sleep(max(1.0, self.interval - (time.time() - started)))
|
||||||
|
|
||||||
|
def snapshot(self) -> dict:
|
||||||
|
"""Everything the topology page needs, in one JSON payload."""
|
||||||
|
with self.lock:
|
||||||
|
results, duration = dict(self.results), self.duration
|
||||||
|
reach = total = 0
|
||||||
|
for data in results.values():
|
||||||
|
if "__error__" in data:
|
||||||
|
continue
|
||||||
|
for protos in data.values():
|
||||||
|
for proto, ok in protos.items():
|
||||||
|
if proto == "rtt_ms":
|
||||||
|
continue
|
||||||
|
total += 1
|
||||||
|
if ok:
|
||||||
|
reach += 1
|
||||||
|
return {"vlans": self.vlans, "results": results, "reachable": reach,
|
||||||
|
"total": total, "sweep_seconds": duration}
|
||||||
|
|
||||||
|
def render(self) -> str:
|
||||||
|
with self.lock:
|
||||||
|
results, duration, sweeps = dict(self.results), self.duration, self.sweeps
|
||||||
|
|
||||||
|
out = [
|
||||||
|
"# HELP labsim_reachable 1 if dst is reachable from src over proto",
|
||||||
|
"# TYPE labsim_reachable gauge",
|
||||||
|
]
|
||||||
|
rtts = []
|
||||||
|
for src, data in results.items():
|
||||||
|
if "__error__" in data:
|
||||||
|
continue
|
||||||
|
for dst, protos in data.items():
|
||||||
|
for proto, ok in protos.items():
|
||||||
|
if proto == "rtt_ms":
|
||||||
|
if isinstance(ok, (int, float)):
|
||||||
|
rtts.append((src, dst, ok))
|
||||||
|
continue
|
||||||
|
out.append(
|
||||||
|
f'labsim_reachable{{src="{src}",dst="{dst}",proto="{proto}"}} {1 if ok else 0}')
|
||||||
|
out += ["# HELP labsim_rtt_ms ICMP round-trip time",
|
||||||
|
"# TYPE labsim_rtt_ms gauge"]
|
||||||
|
for src, dst, val in rtts:
|
||||||
|
out.append(f'labsim_rtt_ms{{src="{src}",dst="{dst}"}} {val}')
|
||||||
|
out += [
|
||||||
|
"# HELP labsim_sweep_seconds duration of the last sweep",
|
||||||
|
"# TYPE labsim_sweep_seconds gauge",
|
||||||
|
f"labsim_sweep_seconds {duration:.3f}",
|
||||||
|
"# HELP labsim_sweep_total sweeps completed",
|
||||||
|
"# TYPE labsim_sweep_total counter",
|
||||||
|
f"labsim_sweep_total {sweeps}",
|
||||||
|
"# HELP labsim_up exporter liveness",
|
||||||
|
"# TYPE labsim_up gauge",
|
||||||
|
"labsim_up 1",
|
||||||
|
]
|
||||||
|
return "\n".join(out) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--port", type=int, default=9101)
|
||||||
|
ap.add_argument("--interval", type=int, default=15)
|
||||||
|
ap.add_argument("--timeout", type=int, default=30)
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
collector = Collector(args.interval, args.timeout)
|
||||||
|
threading.Thread(target=collector.loop, daemon=True).start()
|
||||||
|
|
||||||
|
here = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
|
||||||
|
class Handler(http.server.BaseHTTPRequestHandler):
|
||||||
|
def _send(self, body: bytes, ctype: str) -> None:
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", ctype)
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.send_header("Cache-Control", "no-store")
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def do_GET(self) -> None: # noqa: N802 - stdlib API
|
||||||
|
path = self.path.split("?")[0].rstrip("/")
|
||||||
|
if path in ("", "/topology"):
|
||||||
|
# Live topology view — the thing you actually watch.
|
||||||
|
try:
|
||||||
|
with open(os.path.join(here, "topology.html"), "rb") as fh:
|
||||||
|
self._send(fh.read(), "text/html; charset=utf-8")
|
||||||
|
except OSError:
|
||||||
|
self.send_error(500, "topology.html missing")
|
||||||
|
elif path == "/api/matrix":
|
||||||
|
self._send(json.dumps(collector.snapshot()).encode(), "application/json")
|
||||||
|
elif path == "/metrics":
|
||||||
|
self._send(collector.render().encode(), "text/plain; version=0.0.4")
|
||||||
|
else:
|
||||||
|
self.send_error(404)
|
||||||
|
|
||||||
|
def log_message(self, *_args) -> None: # keep the console quiet
|
||||||
|
return
|
||||||
|
|
||||||
|
srv = http.server.ThreadingHTTPServer(("0.0.0.0", args.port), Handler)
|
||||||
|
print(f"labsim topology http://localhost:{args.port}/")
|
||||||
|
print(f"labsim metrics http://localhost:{args.port}/metrics (sweep every {args.interval}s)")
|
||||||
|
srv.serve_forever()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
202
labsim/labsim-matrix.py
Executable file
202
labsim/labsim-matrix.py
Executable file
@@ -0,0 +1,202 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Full-mesh connectivity matrix for the labsim VLANs.
|
||||||
|
|
||||||
|
Probes every VLAN VM from every other VLAN VM (ICMP + TCP/22 + TCP/80) and
|
||||||
|
prints a grid. Use --watch to keep it live: cells that changed since the last
|
||||||
|
sweep are highlighted, so adding or removing a VyOS firewall rule shows up
|
||||||
|
within one refresh.
|
||||||
|
|
||||||
|
Deliberately dependency-free on the guests: the probe runs with python3, which
|
||||||
|
is already installed there (cloud-init needs it), so nothing has to be
|
||||||
|
installed on VMs that have no internet.
|
||||||
|
|
||||||
|
./labsim-matrix.py # one sweep
|
||||||
|
./labsim-matrix.py --watch # live, refresh every 5s
|
||||||
|
./labsim-matrix.py --watch 2 # live, every 2s
|
||||||
|
./labsim-matrix.py --proto icmp # single protocol
|
||||||
|
./labsim-matrix.py --json # machine-readable
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import concurrent.futures
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
CONF = os.path.join(HERE, "vlans.conf")
|
||||||
|
|
||||||
|
GREEN, RED, GREY, YELLOW, BOLD, RESET = (
|
||||||
|
"\033[0;32m", "\033[0;31m", "\033[0;90m", "\033[1;33m", "\033[1m", "\033[0m")
|
||||||
|
|
||||||
|
PROTOS = ("icmp", "tcp22", "tcp80")
|
||||||
|
|
||||||
|
# Runs ON the guest. Keep it stdlib-only and quick — a hung probe delays the
|
||||||
|
# whole sweep, so every check is hard-bounded by a timeout.
|
||||||
|
PROBE = r'''
|
||||||
|
import json, re, socket, subprocess, sys
|
||||||
|
targets = json.load(sys.stdin)
|
||||||
|
out = {}
|
||||||
|
for name, ip in targets.items():
|
||||||
|
res = {}
|
||||||
|
try:
|
||||||
|
p = subprocess.run(["ping", "-c", "1", "-W", "1", ip],
|
||||||
|
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=4)
|
||||||
|
res["icmp"] = p.returncode == 0
|
||||||
|
# RTT as well as pass/fail: a path that is up but slow is a different
|
||||||
|
# problem from one that is down, and the grid alone cannot show it.
|
||||||
|
res["rtt_ms"] = None
|
||||||
|
if res["icmp"]:
|
||||||
|
m = re.search(r"time[=<]\s*([0-9.]+)\s*ms", p.stdout.decode("utf-8", "replace"))
|
||||||
|
if m:
|
||||||
|
res["rtt_ms"] = float(m.group(1))
|
||||||
|
except Exception:
|
||||||
|
res["icmp"] = False
|
||||||
|
res["rtt_ms"] = None
|
||||||
|
for port in (22, 80):
|
||||||
|
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
|
s.settimeout(1.5)
|
||||||
|
try:
|
||||||
|
s.connect((ip, port)); res["tcp%d" % port] = True
|
||||||
|
except Exception:
|
||||||
|
res["tcp%d" % port] = False
|
||||||
|
finally:
|
||||||
|
try: s.close()
|
||||||
|
except Exception: pass
|
||||||
|
out[name] = res
|
||||||
|
print(json.dumps(out))
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
def load_vlans() -> list[dict]:
|
||||||
|
vlans = []
|
||||||
|
with open(CONF) as fh:
|
||||||
|
for line in fh:
|
||||||
|
line = line.strip()
|
||||||
|
if not line or line.startswith("#"):
|
||||||
|
continue
|
||||||
|
vid, name, prefix, real = line.split(":", 3)
|
||||||
|
vlans.append({"vid": vid, "name": name, "ip": f"{prefix}.10",
|
||||||
|
"label": f"{vid}:{name}", "real": real})
|
||||||
|
return vlans
|
||||||
|
|
||||||
|
|
||||||
|
def probe_from(src: dict, targets: list[dict], timeout: int) -> tuple[str, dict]:
|
||||||
|
"""SSH once into src and probe every target from there."""
|
||||||
|
payload = json.dumps({t["label"]: t["ip"] for t in targets if t["label"] != src["label"]})
|
||||||
|
cmd = [
|
||||||
|
"ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null",
|
||||||
|
"-o", "BatchMode=yes", "-o", "ConnectTimeout=5", "-o", "LogLevel=ERROR",
|
||||||
|
f"alpine@{src['ip']}", "python3", "-",
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
# The probe script goes on stdin, the target list follows it — the guest
|
||||||
|
# reads the script from argv-less stdin, so send both in one stream.
|
||||||
|
proc = subprocess.run(
|
||||||
|
cmd, input=PROBE.replace("json.load(sys.stdin)", f"json.loads({payload!r})"),
|
||||||
|
capture_output=True, text=True, timeout=timeout)
|
||||||
|
if proc.returncode != 0:
|
||||||
|
return src["label"], {"__error__": (proc.stderr or "ssh failed").strip()[:60]}
|
||||||
|
return src["label"], json.loads(proc.stdout)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return src["label"], {"__error__": "probe timed out"}
|
||||||
|
except Exception as exc: # noqa: BLE001 - report, never crash the sweep
|
||||||
|
return src["label"], {"__error__": f"{type(exc).__name__}: {exc}"[:60]}
|
||||||
|
|
||||||
|
|
||||||
|
def sweep(vlans: list[dict], timeout: int) -> dict:
|
||||||
|
results: dict = {}
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(max_workers=len(vlans)) as pool:
|
||||||
|
futures = [pool.submit(probe_from, v, vlans, timeout) for v in vlans]
|
||||||
|
for fut in concurrent.futures.as_completed(futures):
|
||||||
|
label, data = fut.result()
|
||||||
|
results[label] = data
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def cell(ok: bool | None, changed: bool) -> str:
|
||||||
|
if ok is None:
|
||||||
|
return f"{GREY} · {RESET}"
|
||||||
|
mark = "ok " if ok else "-- "
|
||||||
|
colour = GREEN if ok else RED
|
||||||
|
if changed:
|
||||||
|
return f"{YELLOW}{BOLD}{'OK*' if ok else 'XX*':<4}{RESET}"
|
||||||
|
return f"{colour}{mark}{RESET}"
|
||||||
|
|
||||||
|
|
||||||
|
def render(vlans: list[dict], results: dict, prev: dict | None, protos: tuple[str, ...]) -> None:
|
||||||
|
labels = [v["label"] for v in vlans]
|
||||||
|
width = max(len(x) for x in labels) + 2
|
||||||
|
|
||||||
|
for proto in protos:
|
||||||
|
print(f"\n{BOLD}{proto.upper()}{RESET} (rows = source, columns = destination)")
|
||||||
|
header = " " * width + "".join(f"{lbl:<{width}}" for lbl in labels)
|
||||||
|
print(f"{GREY}{header}{RESET}")
|
||||||
|
|
||||||
|
for src in vlans:
|
||||||
|
row = f"{src['label']:<{width}}"
|
||||||
|
data = results.get(src["label"], {})
|
||||||
|
if "__error__" in data:
|
||||||
|
print(row + f"{RED}{data['__error__']}{RESET}")
|
||||||
|
continue
|
||||||
|
for dst in vlans:
|
||||||
|
if dst["label"] == src["label"]:
|
||||||
|
row += f"{GREY}{'·':<{width}}{RESET}"
|
||||||
|
continue
|
||||||
|
ok = data.get(dst["label"], {}).get(proto)
|
||||||
|
was = (prev or {}).get(src["label"], {}).get(dst["label"], {}).get(proto)
|
||||||
|
changed = prev is not None and was is not None and was != ok
|
||||||
|
txt = cell(ok, changed)
|
||||||
|
row += txt + " " * (width - 4)
|
||||||
|
print(row)
|
||||||
|
|
||||||
|
reach = sum(1 for s in results.values() if "__error__" not in s
|
||||||
|
for d in s.values() for p in protos if d.get(p) is True)
|
||||||
|
total = sum(1 for s in results.values() if "__error__" not in s
|
||||||
|
for _d in s.values() for _p in protos)
|
||||||
|
print(f"\n reachable: {reach}/{total} "
|
||||||
|
f"{GREEN}ok{RESET}=allowed {RED}--{RESET}=blocked/no route "
|
||||||
|
f"{YELLOW}*{RESET}=changed since last sweep")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__,
|
||||||
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
|
ap.add_argument("--watch", nargs="?", const=5, type=int, metavar="SECONDS",
|
||||||
|
help="refresh continuously (default every 5s)")
|
||||||
|
ap.add_argument("--proto", choices=PROTOS, help="only this protocol")
|
||||||
|
ap.add_argument("--json", action="store_true", help="emit raw JSON and exit")
|
||||||
|
ap.add_argument("--timeout", type=int, default=30, help="per-host probe timeout")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
vlans = load_vlans()
|
||||||
|
protos = (args.proto,) if args.proto else PROTOS
|
||||||
|
|
||||||
|
if args.json:
|
||||||
|
print(json.dumps(sweep(vlans, args.timeout), indent=2))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
prev = None
|
||||||
|
while True:
|
||||||
|
started = time.time()
|
||||||
|
results = sweep(vlans, args.timeout)
|
||||||
|
if args.watch:
|
||||||
|
os.system("clear")
|
||||||
|
print(f"{BOLD}labsim connectivity matrix{RESET} "
|
||||||
|
f"{time.strftime('%H:%M:%S')} (refresh {args.watch}s, Ctrl-C to stop)")
|
||||||
|
render(vlans, results, prev, protos)
|
||||||
|
if not args.watch:
|
||||||
|
return 0
|
||||||
|
prev = results
|
||||||
|
time.sleep(max(0.0, args.watch - (time.time() - started)))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
sys.exit(main())
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print()
|
||||||
|
sys.exit(130)
|
||||||
70
labsim/labsim-up.sh
Executable file
70
labsim/labsim-up.sh
Executable file
@@ -0,0 +1,70 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Bring up the lab network simulation: one isolated libvirt network per VLAN,
|
||||||
|
# each with a single tiny Alpine VM offering SSH + a hello-world HTTP page.
|
||||||
|
#
|
||||||
|
# Idempotent: re-running only creates what is missing. Safe to run repeatedly.
|
||||||
|
#
|
||||||
|
# Usage: ./labsim-up.sh [vlan-id ...] (default: every VLAN in vlans.conf)
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
source "$SCRIPT_DIR/lib.sh"
|
||||||
|
source "$SCRIPT_DIR/ovs.sh"
|
||||||
|
|
||||||
|
require_tools
|
||||||
|
[ -f "$BASE_IMAGE" ] || die "base image missing: $BASE_IMAGE (see README)"
|
||||||
|
|
||||||
|
SSH_PUB="$(find_ssh_pubkey)"
|
||||||
|
log "Using SSH key: ${SSH_PUB%% *} ...${SSH_PUB##* }"
|
||||||
|
|
||||||
|
selected_vlans "$@"
|
||||||
|
|
||||||
|
# --- switch fabric ------------------------------------------------------
|
||||||
|
log "bringing up OVS fabric ($OVS_BR) with host legs per VLAN"
|
||||||
|
ovs_up
|
||||||
|
|
||||||
|
# --- VMs ------------------------------------------------------------------
|
||||||
|
for entry in "${SELECTED[@]}"; do
|
||||||
|
IFS=: read -r vid name prefix real <<<"$entry"
|
||||||
|
vm="$(vm_name "$vid" "$name")"
|
||||||
|
ip="${prefix}.10"
|
||||||
|
|
||||||
|
if virsh_q dominfo "$vm" >/dev/null 2>&1; then
|
||||||
|
state="$(virsh_q domstate "$vm" 2>/dev/null | head -1 | tr -d '\n')"
|
||||||
|
if [ "$state" = "running" ]; then
|
||||||
|
log "VM $vm already running ($ip)"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
log "VM $vm exists but is $state — starting"
|
||||||
|
virsh_q start "$vm" >/dev/null
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "creating VM $vm ($ip on vlan $vid/$name)"
|
||||||
|
|
||||||
|
disk="$IMG_DIR/${vm}.qcow2"
|
||||||
|
seed="$IMG_DIR/${vm}-seed.iso"
|
||||||
|
|
||||||
|
# Copy-on-write overlay: each VM costs a few MB, not 176.
|
||||||
|
sudo qemu-img create -q -f qcow2 -F qcow2 -b "$BASE_IMAGE" "$disk" "$VM_DISK" >/dev/null
|
||||||
|
|
||||||
|
build_seed "$seed" "$vm" "$vid" "$name" "$prefix" "$ip" "$real" "$SSH_PUB"
|
||||||
|
|
||||||
|
sudo virt-install \
|
||||||
|
--connect "$LIBVIRT_URI" \
|
||||||
|
--name "$vm" \
|
||||||
|
--memory "$VM_MEM" --vcpus "$VM_CPUS" \
|
||||||
|
--disk "path=$disk,format=qcow2,bus=virtio" \
|
||||||
|
--disk "path=$seed,device=cdrom,readonly=on" \
|
||||||
|
--network "network=$OVS_NET,portgroup=vlan${vid},model=virtio" \
|
||||||
|
--os-variant alpinelinux3.18 \
|
||||||
|
--graphics none --noautoconsole --import >/dev/null
|
||||||
|
done
|
||||||
|
|
||||||
|
echo
|
||||||
|
log "waiting for VMs to answer on SSH + HTTP..."
|
||||||
|
wait_ready
|
||||||
|
echo
|
||||||
|
status_table
|
||||||
|
echo
|
||||||
|
log "environment is UP. Tear down with: $SCRIPT_DIR/labsim-down.sh"
|
||||||
216
labsim/lib.sh
Normal file
216
labsim/lib.sh
Normal file
@@ -0,0 +1,216 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Shared helpers for the lab network simulation.
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
|
||||||
|
LIBVIRT_URI="${LIBVIRT_URI:-qemu:///system}"
|
||||||
|
IMG_DIR="${IMG_DIR:-/var/lib/libvirt/images/labsim}"
|
||||||
|
BASE_IMAGE="${BASE_IMAGE:-$IMG_DIR/alpine-base.qcow2}"
|
||||||
|
ALPINE_URL="${ALPINE_URL:-https://dl-cdn.alpinelinux.org/alpine/latest-stable/releases/cloud/generic_alpine-3.24.1-x86_64-bios-cloudinit-r0.qcow2}"
|
||||||
|
|
||||||
|
VM_MEM="${VM_MEM:-256}" # MB — Alpine is happy here
|
||||||
|
VM_CPUS="${VM_CPUS:-1}"
|
||||||
|
VM_DISK="${VM_DISK:-1G}"
|
||||||
|
PREFIX="labsim"
|
||||||
|
|
||||||
|
CONF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/vlans.conf"
|
||||||
|
|
||||||
|
log() { printf '\033[0;36m[labsim]\033[0m %s\n' "$*"; }
|
||||||
|
warn() { printf '\033[1;33m[labsim]\033[0m %s\n' "$*" >&2; }
|
||||||
|
die() { printf '\033[0;31m[labsim]\033[0m %s\n' "$*" >&2; exit 1; }
|
||||||
|
|
||||||
|
virsh_q() { sudo virsh --connect "$LIBVIRT_URI" "$@"; }
|
||||||
|
|
||||||
|
net_name() { echo "${PREFIX}-vlan$1"; }
|
||||||
|
vm_name() { echo "${PREFIX}-$1-$2"; } # labsim-2-k8s
|
||||||
|
# Linux bridge names are capped at 15 chars — keep it short and unique.
|
||||||
|
br_name() { echo "vbr-ls$1"; }
|
||||||
|
|
||||||
|
require_tools() {
|
||||||
|
for t in virsh virt-install qemu-img genisoimage; do
|
||||||
|
command -v "$t" >/dev/null 2>&1 || die "missing required tool: $t"
|
||||||
|
done
|
||||||
|
sudo -n true 2>/dev/null || warn "sudo may prompt for a password"
|
||||||
|
}
|
||||||
|
|
||||||
|
find_ssh_pubkey() {
|
||||||
|
local home="${SUDO_USER:+/home/$SUDO_USER}"
|
||||||
|
home="${home:-$HOME}"
|
||||||
|
for n in id_ed25519 id_ecdsa id_rsa; do
|
||||||
|
[ -f "$home/.ssh/$n.pub" ] && { cat "$home/.ssh/$n.pub"; return; }
|
||||||
|
done
|
||||||
|
die "no SSH public key found in $home/.ssh"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Populate SELECTED[] from argv (VLAN ids) or the whole config.
|
||||||
|
selected_vlans() {
|
||||||
|
SELECTED=()
|
||||||
|
local want=("$@")
|
||||||
|
while IFS= read -r line; do
|
||||||
|
[[ "$line" =~ ^[[:space:]]*# ]] && continue
|
||||||
|
[[ -z "${line// }" ]] && continue
|
||||||
|
local vid="${line%%:*}"
|
||||||
|
if [ ${#want[@]} -eq 0 ]; then
|
||||||
|
SELECTED+=("$line")
|
||||||
|
else
|
||||||
|
for w in "${want[@]}"; do [ "$w" = "$vid" ] && SELECTED+=("$line"); done
|
||||||
|
fi
|
||||||
|
done < "$CONF"
|
||||||
|
[ ${#SELECTED[@]} -gt 0 ] || die "no VLANs selected (checked $CONF)"
|
||||||
|
}
|
||||||
|
|
||||||
|
# cloud-init NoCloud seed: static addressing + SSH key + hello-world HTTP.
|
||||||
|
build_seed() {
|
||||||
|
local iso="$1" vm="$2" vid="$3" name="$4" prefix="$5" ip="$6" real="$7" pubkey="$8"
|
||||||
|
local tmp; tmp="$(mktemp -d)"
|
||||||
|
|
||||||
|
cat > "$tmp/meta-data" <<EOF
|
||||||
|
instance-id: $vm
|
||||||
|
local-hostname: $vm
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# Alpine's cloud-init does not reliably apply netplan-style network-config,
|
||||||
|
# and these networks have no DHCP server on purpose — so configure the
|
||||||
|
# interface the Alpine-native way instead (verified: hostname applied but no
|
||||||
|
# address, i.e. the seed was read and network-config was ignored).
|
||||||
|
#
|
||||||
|
# The default route deliberately points at the router under test (.1), not
|
||||||
|
# the host (.2), so a broken/absent router shows up as a failed test rather
|
||||||
|
# than being silently papered over by host routing. post-up ... || true keeps
|
||||||
|
# the interface up even while no router exists yet.
|
||||||
|
cat > "$tmp/network-config" <<EOF
|
||||||
|
version: 1
|
||||||
|
config:
|
||||||
|
- type: physical
|
||||||
|
name: eth0
|
||||||
|
subnets:
|
||||||
|
- type: static
|
||||||
|
address: $ip
|
||||||
|
netmask: 255.255.255.0
|
||||||
|
# Default route via the router under test. Without this the VMs can
|
||||||
|
# reach their own /24 and their gateway, but nothing beyond it — which
|
||||||
|
# looks exactly like "the router is broken" in the matrix.
|
||||||
|
gateway: ${prefix}.1
|
||||||
|
EOF
|
||||||
|
|
||||||
|
cat > "$tmp/user-data" <<EOF
|
||||||
|
#cloud-config
|
||||||
|
hostname: $vm
|
||||||
|
users:
|
||||||
|
- name: alpine
|
||||||
|
# NOTE: this Alpine image ships no sudo (and cloud-init's sudo: directive
|
||||||
|
# is therefore inert). For privileged work in these VMs, ssh as root —
|
||||||
|
# the key is installed there too.
|
||||||
|
shell: /bin/ash
|
||||||
|
# Without this cloud-init leaves the account locked ("!*" in /etc/shadow)
|
||||||
|
# and sshd refuses key auth for it — verified on the first build.
|
||||||
|
lock_passwd: false
|
||||||
|
plain_text_passwd: labsim
|
||||||
|
ssh_authorized_keys:
|
||||||
|
- $pubkey
|
||||||
|
ssh_authorized_keys:
|
||||||
|
- $pubkey
|
||||||
|
disable_root: false
|
||||||
|
chpasswd:
|
||||||
|
list: |
|
||||||
|
root:labsim
|
||||||
|
expire: false
|
||||||
|
write_files:
|
||||||
|
- path: /etc/network/interfaces
|
||||||
|
content: |
|
||||||
|
auto lo
|
||||||
|
iface lo inet loopback
|
||||||
|
auto eth0
|
||||||
|
iface eth0 inet static
|
||||||
|
address $ip
|
||||||
|
netmask 255.255.255.0
|
||||||
|
post-up ip route add default via ${prefix}.1 || true
|
||||||
|
- path: /var/www/index.html
|
||||||
|
content: |
|
||||||
|
<html><body>
|
||||||
|
<h1>labsim vlan $vid — $name</h1>
|
||||||
|
<p>host: $vm</p>
|
||||||
|
<p>address: $ip/24</p>
|
||||||
|
<p>gateway under test: ${prefix}.1</p>
|
||||||
|
<p>mirrors production: $real</p>
|
||||||
|
</body></html>
|
||||||
|
- path: /etc/local.d/labsim-http.start
|
||||||
|
permissions: '0755'
|
||||||
|
content: |
|
||||||
|
#!/bin/sh
|
||||||
|
# This image's busybox has no httpd applet ("applet not found"), and the
|
||||||
|
# VMs are isolated so apk cannot fetch one. python3 is already present
|
||||||
|
# (cloud-init depends on it), so serve with http.server — no packages,
|
||||||
|
# no internet.
|
||||||
|
#
|
||||||
|
# Two traps already hit here, both silent:
|
||||||
|
# - start-stop-daemon --exec /usr/bin/python3 matches cloud-init's OWN
|
||||||
|
# python3 at boot, says "already running", starts nothing.
|
||||||
|
# - busybox pgrep -f PATTERN matches its own argv, so a
|
||||||
|
# "skip if running" guard always fires (verified: guard_exit=0 with
|
||||||
|
# nothing listening).
|
||||||
|
# So: no guard, no start-stop-daemon. Binding twice is harmless — the
|
||||||
|
# second just fails to bind.
|
||||||
|
nohup /usr/bin/python3 -m http.server 80 --directory /var/www \\
|
||||||
|
>/var/log/labsim-http.log 2>&1 &
|
||||||
|
runcmd:
|
||||||
|
# cloud-init's network-config (v1, above) already applies the address, so do
|
||||||
|
# NOT restart networking here — it fails, and one failing runcmd aborts every
|
||||||
|
# command after it, which is what silently left httpd unstarted. Each command
|
||||||
|
# is || true for the same reason.
|
||||||
|
- [ sh, -c, "rc-update add sshd default || true" ]
|
||||||
|
- [ sh, -c, "rc-update add local default || true" ]
|
||||||
|
- [ sh, -c, "/etc/local.d/labsim-http.start || true" ]
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# Validate before building the ISO. The heredoc above is intentionally
|
||||||
|
# unquoted (it interpolates $ip/$prefix), which means backticks or $( ) in
|
||||||
|
# ANY line — including comments — get executed by the host shell and their
|
||||||
|
# output silently corrupts the YAML. Cheap check, expensive bug.
|
||||||
|
python3 -c "import yaml,sys; yaml.safe_load(open(sys.argv[1]))" "$tmp/user-data" \
|
||||||
|
|| die "generated user-data is not valid YAML (backticks or \$( ) in build_seed?): $tmp/user-data"
|
||||||
|
|
||||||
|
sudo genisoimage -quiet -output "$iso" -volid cidata -joliet -rock \
|
||||||
|
"$tmp/user-data" "$tmp/meta-data" "$tmp/network-config"
|
||||||
|
rm -rf "$tmp"
|
||||||
|
}
|
||||||
|
|
||||||
|
ssh_to() {
|
||||||
|
local ip="$1"; shift
|
||||||
|
timeout 12 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||||
|
-o ConnectTimeout=5 -o BatchMode=yes -o LogLevel=ERROR \
|
||||||
|
"alpine@$ip" "$@" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
wait_ready() {
|
||||||
|
local deadline=$((SECONDS + 240)) pending=1
|
||||||
|
while [ $SECONDS -lt $deadline ]; do
|
||||||
|
pending=0
|
||||||
|
for entry in "${SELECTED[@]}"; do
|
||||||
|
IFS=: read -r vid _n prefix _r <<<"$entry"
|
||||||
|
# Wait for BOTH: sshd is up well before cloud-init's runcmd starts the
|
||||||
|
# web server, so checking SSH alone reports "ready" then shows HTTP FAIL.
|
||||||
|
ssh_to "${prefix}.10" true >/dev/null 2>&1 \
|
||||||
|
&& curl -sS -o /dev/null --max-time 4 "http://${prefix}.10/" 2>/dev/null \
|
||||||
|
|| pending=$((pending + 1))
|
||||||
|
done
|
||||||
|
[ $pending -eq 0 ] && { log "all ${#SELECTED[@]} VMs reachable"; return 0; }
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
warn "$pending VM(s) still not answering SSH after 240s — see status below"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
status_table() {
|
||||||
|
printf ' %-18s %-6s %-16s %-9s %-7s %s\n' VM VLAN ADDRESS STATE SSH HTTP
|
||||||
|
printf ' %-18s %-6s %-16s %-9s %-7s %s\n' ------------------ ------ ---------------- --------- ------- ----
|
||||||
|
for entry in "${SELECTED[@]}"; do
|
||||||
|
IFS=: read -r vid name prefix _r <<<"$entry"
|
||||||
|
local vm ip state ssh http
|
||||||
|
vm="$(vm_name "$vid" "$name")"; ip="${prefix}.10"
|
||||||
|
state="$(virsh_q domstate "$vm" 2>/dev/null | head -1 | tr -d '\n')"
|
||||||
|
[ -z "$state" ] && state="absent"
|
||||||
|
ssh_to "$ip" true >/dev/null 2>&1 && ssh=ok || ssh=FAIL
|
||||||
|
if curl -sS -o /dev/null --max-time 5 "http://$ip/" 2>/dev/null; then http=ok; else http=FAIL; fi
|
||||||
|
printf ' %-18s %-6s %-16s %-9s %-7s %s\n' "$vm" "$vid" "$ip" "$state" "$ssh" "$http"
|
||||||
|
done
|
||||||
|
}
|
||||||
82
labsim/monitoring-up.sh
Executable file
82
labsim/monitoring-up.sh
Executable file
@@ -0,0 +1,82 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Prometheus + Grafana for the labsim connectivity matrix.
|
||||||
|
#
|
||||||
|
# Grafana runs with anonymous auth as Admin — NO LOGIN. That is deliberate for
|
||||||
|
# a throwaway lab on localhost; do not copy this into anything reachable.
|
||||||
|
#
|
||||||
|
# ./monitoring-up.sh start exporter + prometheus + grafana
|
||||||
|
# ./monitoring-up.sh --down stop and remove them
|
||||||
|
#
|
||||||
|
# Grafana: http://localhost:3000 (dashboard "labsim — VLAN connectivity matrix")
|
||||||
|
# Prometheus: http://localhost:9090
|
||||||
|
# Exporter: http://localhost:9101/metrics
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
source "$SCRIPT_DIR/lib.sh"
|
||||||
|
|
||||||
|
GRAFANA_PORT="${GRAFANA_PORT:-3000}"
|
||||||
|
PROM_PORT="${PROM_PORT:-9090}"
|
||||||
|
EXPORTER_PORT="${EXPORTER_PORT:-9101}"
|
||||||
|
NET="labsim-mon"
|
||||||
|
|
||||||
|
if [ "${1:-}" = "--down" ]; then
|
||||||
|
pkill -f "labsim-exporter.py" 2>/dev/null || true
|
||||||
|
podman rm -f labsim-grafana labsim-prometheus >/dev/null 2>&1 || true
|
||||||
|
podman network rm -f "$NET" >/dev/null 2>&1 || true
|
||||||
|
log "monitoring stopped"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
command -v podman >/dev/null 2>&1 || die "podman not installed"
|
||||||
|
|
||||||
|
# --- exporter (on the host: it needs SSH access to the VMs) ----------------
|
||||||
|
if pgrep -f "labsim-exporter.py" >/dev/null 2>&1; then
|
||||||
|
log "exporter already running on :$EXPORTER_PORT"
|
||||||
|
else
|
||||||
|
log "starting exporter on :$EXPORTER_PORT"
|
||||||
|
nohup "$SCRIPT_DIR/labsim-exporter.py" --port "$EXPORTER_PORT" --interval 15 \
|
||||||
|
> /tmp/labsim-exporter.log 2>&1 &
|
||||||
|
sleep 3
|
||||||
|
fi
|
||||||
|
curl -sS --max-time 5 "http://127.0.0.1:${EXPORTER_PORT}/metrics" >/dev/null \
|
||||||
|
|| die "exporter not answering on :$EXPORTER_PORT (see /tmp/labsim-exporter.log)"
|
||||||
|
|
||||||
|
podman network exists "$NET" 2>/dev/null || podman network create "$NET" >/dev/null
|
||||||
|
|
||||||
|
# --- prometheus -----------------------------------------------------------
|
||||||
|
podman rm -f labsim-prometheus >/dev/null 2>&1 || true
|
||||||
|
log "starting prometheus on :$PROM_PORT"
|
||||||
|
podman run -d --name labsim-prometheus --network "$NET" \
|
||||||
|
-p "${PROM_PORT}:9090" \
|
||||||
|
-v "$SCRIPT_DIR/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro,Z" \
|
||||||
|
--add-host "host.containers.internal:host-gateway" \
|
||||||
|
docker.io/prom/prometheus:latest >/dev/null
|
||||||
|
|
||||||
|
# --- grafana (anonymous, no login) ----------------------------------------
|
||||||
|
podman rm -f labsim-grafana >/dev/null 2>&1 || true
|
||||||
|
log "starting grafana on :$GRAFANA_PORT (anonymous auth — no password)"
|
||||||
|
podman run -d --name labsim-grafana --network "$NET" \
|
||||||
|
-p "${GRAFANA_PORT}:3000" \
|
||||||
|
-e GF_AUTH_ANONYMOUS_ENABLED=true \
|
||||||
|
-e GF_AUTH_ANONYMOUS_ORG_ROLE=Admin \
|
||||||
|
-e GF_AUTH_DISABLE_LOGIN_FORM=true \
|
||||||
|
-e GF_AUTH_BASIC_ENABLED=false \
|
||||||
|
-e GF_SECURITY_ALLOW_EMBEDDING=true \
|
||||||
|
-e GF_USERS_DEFAULT_THEME=dark \
|
||||||
|
-v "$SCRIPT_DIR/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro,Z" \
|
||||||
|
docker.io/grafana/grafana:latest >/dev/null
|
||||||
|
|
||||||
|
log "waiting for grafana..."
|
||||||
|
for _ in $(seq 1 40); do
|
||||||
|
if curl -sS --max-time 3 "http://127.0.0.1:${GRAFANA_PORT}/api/health" >/dev/null 2>&1; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 3
|
||||||
|
done
|
||||||
|
|
||||||
|
echo
|
||||||
|
log "Topology: http://localhost:${EXPORTER_PORT}/ <- live mesh, red/green + RTT"
|
||||||
|
log "Grafana: http://localhost:${GRAFANA_PORT}/d/labsim-matrix (no login, history)"
|
||||||
|
log "Prometheus: http://localhost:${PROM_PORT}"
|
||||||
|
log "Exporter: http://localhost:${EXPORTER_PORT}/metrics"
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
providers:
|
||||||
|
- name: labsim
|
||||||
|
folder: ''
|
||||||
|
type: file
|
||||||
|
disableDeletion: false
|
||||||
|
updateIntervalSeconds: 10
|
||||||
|
options:
|
||||||
|
path: /etc/grafana/provisioning/dashboards
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
{
|
||||||
|
"uid": "labsim-matrix",
|
||||||
|
"title": "labsim — VLAN connectivity matrix",
|
||||||
|
"tags": ["labsim"],
|
||||||
|
"timezone": "browser",
|
||||||
|
"refresh": "10s",
|
||||||
|
"time": { "from": "now-30m", "to": "now" },
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Reachable paths",
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
|
||||||
|
"targets": [ { "expr": "sum(labsim_reachable)", "refId": "A" } ],
|
||||||
|
"fieldConfig": { "defaults": { "thresholds": { "mode": "absolute",
|
||||||
|
"steps": [ { "color": "red", "value": null }, { "color": "green", "value": 90 } ] } } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Blocked paths",
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
|
||||||
|
"targets": [ { "expr": "count(labsim_reachable == 0) or vector(0)", "refId": "A" } ],
|
||||||
|
"fieldConfig": { "defaults": { "thresholds": { "mode": "absolute",
|
||||||
|
"steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 } ] } } }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Sweep duration (s)",
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
|
||||||
|
"targets": [ { "expr": "labsim_sweep_seconds", "refId": "A" } ]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Sweeps",
|
||||||
|
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
|
||||||
|
"targets": [ { "expr": "labsim_sweep_total", "refId": "A" } ]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "heatmap",
|
||||||
|
"title": "ICMP matrix (src → dst) — green = reachable",
|
||||||
|
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
|
||||||
|
"targets": [ { "expr": "labsim_reachable{proto=\"icmp\"}",
|
||||||
|
"legendFormat": "{{src}} → {{dst}}", "refId": "A" } ]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "state-timeline",
|
||||||
|
"title": "Every path over time — a firewall change shows up here immediately",
|
||||||
|
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 14 },
|
||||||
|
"targets": [ { "expr": "labsim_reachable",
|
||||||
|
"legendFormat": "{{proto}} {{src}} → {{dst}}", "refId": "A" } ],
|
||||||
|
"fieldConfig": { "defaults": {
|
||||||
|
"mappings": [ { "type": "value", "options": {
|
||||||
|
"0": { "text": "blocked", "color": "red", "index": 0 },
|
||||||
|
"1": { "text": "ok", "color": "green", "index": 1 } } } ] } },
|
||||||
|
"options": { "mergeValues": true, "showValue": "never" }
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"version": 1
|
||||||
|
}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
apiVersion: 1
|
||||||
|
datasources:
|
||||||
|
- name: Prometheus
|
||||||
|
type: prometheus
|
||||||
|
access: proxy
|
||||||
|
url: http://labsim-prometheus:9090
|
||||||
|
isDefault: true
|
||||||
9
labsim/monitoring/prometheus.yml
Normal file
9
labsim/monitoring/prometheus.yml
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
# Scrapes the labsim connectivity exporter running on the host.
|
||||||
|
global:
|
||||||
|
scrape_interval: 15s
|
||||||
|
evaluation_interval: 15s
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: labsim
|
||||||
|
static_configs:
|
||||||
|
- targets: ['host.containers.internal:9101']
|
||||||
161
labsim/ovs.sh
Normal file
161
labsim/ovs.sh
Normal file
@@ -0,0 +1,161 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Open vSwitch fabric for labsim — the "switch" the whole sim hangs off.
|
||||||
|
#
|
||||||
|
# Why OVS and not a Linux bridge: a Linux bridge cannot do LACP at all, and its
|
||||||
|
# VLAN support is awkward to drive from libvirt. OVS gives real 802.1Q access
|
||||||
|
# and trunk ports plus real LACP bonds, so a router VM can run the SAME bond0 +
|
||||||
|
# vif config as the production VP2440s instead of an approximation.
|
||||||
|
#
|
||||||
|
# Layout:
|
||||||
|
# ovs-labsim the switch
|
||||||
|
# ├─ vm ports access ports, tag=<vlan> (micro VM per VLAN)
|
||||||
|
# ├─ hostv<vlan> internal ports, tag=<vlan> (host leg, for SSH)
|
||||||
|
# └─ lag-vyos LACP bond, trunk of all VLANs (router under test)
|
||||||
|
# shellcheck disable=SC2034
|
||||||
|
|
||||||
|
OVS_BR="${OVS_BR:-ovs-labsim}"
|
||||||
|
OVS_NET="${OVS_NET:-labsim-ovs}" # libvirt network wrapping the bridge
|
||||||
|
LAG_NAME="${LAG_NAME:-lag-vyos}"
|
||||||
|
|
||||||
|
ovs() { sudo ovs-vsctl "$@"; }
|
||||||
|
|
||||||
|
ovs_require() {
|
||||||
|
command -v ovs-vsctl >/dev/null 2>&1 || die "openvswitch not installed (dnf install openvswitch)"
|
||||||
|
systemctl is-active --quiet openvswitch || sudo systemctl start openvswitch \
|
||||||
|
|| die "could not start openvswitch"
|
||||||
|
}
|
||||||
|
|
||||||
|
# All VLAN ids from the config, comma separated — used for trunk ports.
|
||||||
|
vlan_id_list() {
|
||||||
|
local ids=()
|
||||||
|
for entry in "${SELECTED[@]}"; do ids+=("${entry%%:*}"); done
|
||||||
|
(IFS=,; echo "${ids[*]}")
|
||||||
|
}
|
||||||
|
|
||||||
|
ovs_up() {
|
||||||
|
ovs_require
|
||||||
|
ovs --may-exist add-br "$OVS_BR"
|
||||||
|
|
||||||
|
# Host leg per VLAN: an OVS internal port carrying that VLAN's tag, given the
|
||||||
|
# .2 address. This is how you SSH to the VMs. It is deliberately NOT their
|
||||||
|
# default route (.1 is), so inter-VLAN tests exercise the router, not the
|
||||||
|
# host's routing table.
|
||||||
|
for entry in "${SELECTED[@]}"; do
|
||||||
|
IFS=: read -r vid _name prefix _real <<<"$entry"
|
||||||
|
local port="hostv${vid}"
|
||||||
|
ovs --may-exist add-port "$OVS_BR" "$port" tag="$vid" \
|
||||||
|
-- set interface "$port" type=internal
|
||||||
|
sudo ip link set "$port" up 2>/dev/null || true
|
||||||
|
sudo ip addr replace "${prefix}.2/24" dev "$port"
|
||||||
|
done
|
||||||
|
|
||||||
|
ovs_define_libvirt_net
|
||||||
|
}
|
||||||
|
|
||||||
|
# A libvirt network that hands out OVS ports: one portgroup per VLAN (access)
|
||||||
|
# plus a trunk portgroup for the router.
|
||||||
|
ovs_define_libvirt_net() {
|
||||||
|
local pg="" ids
|
||||||
|
for entry in "${SELECTED[@]}"; do
|
||||||
|
IFS=: read -r vid name _p _r <<<"$entry"
|
||||||
|
pg+=" <portgroup name='vlan${vid}'>
|
||||||
|
<vlan><tag id='${vid}'/></vlan>
|
||||||
|
</portgroup>
|
||||||
|
"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Trunk: VLAN 1 native/untagged, everything else tagged — the production
|
||||||
|
# shape. libvirt expresses this declaratively via nativeMode='untagged'
|
||||||
|
# (see libvirt formatnetwork.html), so it does not need fixing up by hand.
|
||||||
|
# It also matters functionally: LACPDUs are untagged, and a trunk with no
|
||||||
|
# native VLAN has nowhere to put them.
|
||||||
|
local trunk=" <portgroup name='trunk'>
|
||||||
|
<vlan trunk='yes'>
|
||||||
|
"
|
||||||
|
for entry in "${SELECTED[@]}"; do
|
||||||
|
IFS=: read -r vid _n _p _r <<<"$entry"
|
||||||
|
if [ "$vid" = "1" ]; then
|
||||||
|
trunk+=" <tag id='1' nativeMode='untagged'/>
|
||||||
|
"
|
||||||
|
else
|
||||||
|
trunk+=" <tag id='${vid}'/>
|
||||||
|
"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
trunk+=" </vlan>
|
||||||
|
</portgroup>
|
||||||
|
"
|
||||||
|
|
||||||
|
local xml="<network>
|
||||||
|
<name>${OVS_NET}</name>
|
||||||
|
<forward mode='bridge'/>
|
||||||
|
<bridge name='${OVS_BR}'/>
|
||||||
|
<virtualport type='openvswitch'/>
|
||||||
|
${pg}${trunk}</network>"
|
||||||
|
|
||||||
|
if virsh_q net-info "$OVS_NET" >/dev/null 2>&1; then
|
||||||
|
virsh_q net-destroy "$OVS_NET" >/dev/null 2>&1 || true
|
||||||
|
virsh_q net-undefine "$OVS_NET" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
echo "$xml" | virsh_q net-define /dev/stdin >/dev/null
|
||||||
|
virsh_q net-start "$OVS_NET" >/dev/null
|
||||||
|
log "libvirt network $OVS_NET bound to $OVS_BR (access portgroups + trunk)"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Replace the router VM's two individual OVS ports with a single LACP bond.
|
||||||
|
# libvirt attaches each NIC separately; only ovs-vsctl can bond them, and the
|
||||||
|
# taps only exist once the VM is running — so this runs post-start.
|
||||||
|
ovs_bond_router() {
|
||||||
|
local vm="$1"
|
||||||
|
local taps
|
||||||
|
# NB: domiflist indents its rows, so anchor on the FIELD not the line —
|
||||||
|
# /^vnet/ silently matches nothing and the bond never gets built.
|
||||||
|
taps="$(virsh_q domiflist "$vm" 2>/dev/null | awk '$1 ~ /^vnet/ {print $1}')"
|
||||||
|
local count; count="$(echo "$taps" | grep -c .)"
|
||||||
|
[ "$count" -eq 2 ] || { warn "router $vm has $count tap(s), expected 2 — skipping bond"; return 1; }
|
||||||
|
|
||||||
|
# Already bonded? (idempotent re-runs)
|
||||||
|
if ovs list-ports "$OVS_BR" 2>/dev/null | grep -qx "$LAG_NAME"; then
|
||||||
|
log "LACP bond $LAG_NAME already present"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local t1 t2; t1="$(echo "$taps" | sed -n 1p)"; t2="$(echo "$taps" | sed -n 2p)"
|
||||||
|
log "bonding $t1 + $t2 into $LAG_NAME (LACP active, balance-tcp)"
|
||||||
|
ovs del-port "$OVS_BR" "$t1" 2>/dev/null || true
|
||||||
|
ovs del-port "$OVS_BR" "$t2" 2>/dev/null || true
|
||||||
|
|
||||||
|
# bond_mode=balance-tcp is REQUIRED: OVS defaults a bond to active-backup,
|
||||||
|
# which does not speak LACP at all (confirmed on ovs-discuss). It is also the
|
||||||
|
# equivalent of VyOS's 802.3ad + layer2+3 hashing.
|
||||||
|
#
|
||||||
|
# lacp-fallback-ab breaks a genuine deadlock: OVS keeps members disabled
|
||||||
|
# until LACP negotiates, while the partner needs carrier before it will send
|
||||||
|
# LACPDUs. Falling back to active-backup brings the links up so negotiation
|
||||||
|
# can start.
|
||||||
|
#
|
||||||
|
# native-untagged + tag=1 carries the untagged LACPDUs and the management
|
||||||
|
# VLAN, matching production. libvirt's portgroup VLAN config does NOT apply
|
||||||
|
# here — the bond is a port libvirt never created — so set it inline.
|
||||||
|
local tagged; tagged="$(vlan_id_list | tr ',' '\n' | grep -vx 1 | paste -sd, -)"
|
||||||
|
ovs add-bond "$OVS_BR" "$LAG_NAME" "$t1" "$t2" \
|
||||||
|
lacp=active bond_mode=balance-tcp \
|
||||||
|
vlan_mode=native-untagged tag=1 trunks="$tagged" \
|
||||||
|
-- set port "$LAG_NAME" other_config:lacp-time=fast \
|
||||||
|
-- set port "$LAG_NAME" other_config:lacp-fallback-ab=true
|
||||||
|
}
|
||||||
|
|
||||||
|
ovs_bond_status() {
|
||||||
|
echo "--- ovs bond ---"
|
||||||
|
sudo ovs-appctl bond/show "$LAG_NAME" 2>/dev/null | grep -E "bond_mode|lacp_status|^member|may_enable" || echo "(no bond)"
|
||||||
|
echo "--- lacp ---"
|
||||||
|
sudo ovs-appctl lacp/show "$LAG_NAME" 2>/dev/null | grep -E "status|aggregation key|^member|attached" || true
|
||||||
|
}
|
||||||
|
|
||||||
|
ovs_down() {
|
||||||
|
virsh_q net-destroy "$OVS_NET" >/dev/null 2>&1 || true
|
||||||
|
virsh_q net-undefine "$OVS_NET" >/dev/null 2>&1 || true
|
||||||
|
if command -v ovs-vsctl >/dev/null 2>&1; then
|
||||||
|
ovs --if-exists del-br "$OVS_BR" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
162
labsim/router-install.py
Executable file
162
labsim/router-install.py
Executable file
@@ -0,0 +1,162 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Drive the labsim VyOS router over its serial console.
|
||||||
|
|
||||||
|
Three phases:
|
||||||
|
--phase live wait for the live system and log in
|
||||||
|
--phase install run `install image` unattended
|
||||||
|
--phase configure apply bond0 (LACP) + per-VLAN gateway addresses
|
||||||
|
|
||||||
|
The installer prompt list is the same one the bastion's install driver answers
|
||||||
|
(src/bastion/src/templates/vyos-install.py.ts). Two of them are easy to miss and
|
||||||
|
both hang forever rather than failing: the reinstall-only "copy data to the new
|
||||||
|
image?", and "choose two disks for RAID-1 mirroring?" — every RAID prompt
|
||||||
|
defaults to YES.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
import pexpect
|
||||||
|
|
||||||
|
PASSWORD = "vyos"
|
||||||
|
PROMPT = r"[\$#] $"
|
||||||
|
|
||||||
|
|
||||||
|
def console(vm: str, timeout: int = 60) -> pexpect.spawn:
|
||||||
|
c = pexpect.spawn(f"sudo virsh console {vm} --force", encoding="utf-8", timeout=timeout)
|
||||||
|
c.expect("Connected to domain", timeout=30)
|
||||||
|
return c
|
||||||
|
|
||||||
|
|
||||||
|
def login(c: pexpect.spawn, timeout: int = 300) -> None:
|
||||||
|
"""Get to a shell prompt, whether we land at a login or an open session."""
|
||||||
|
deadline = time.time() + timeout
|
||||||
|
while time.time() < deadline:
|
||||||
|
c.sendline("")
|
||||||
|
i = c.expect(["login:", PROMPT, pexpect.TIMEOUT], timeout=20)
|
||||||
|
if i == 0:
|
||||||
|
c.sendline("vyos")
|
||||||
|
c.expect("assword:", timeout=20)
|
||||||
|
c.sendline(PASSWORD)
|
||||||
|
j = c.expect([PROMPT, "incorrect", pexpect.TIMEOUT], timeout=30)
|
||||||
|
if j == 0:
|
||||||
|
return
|
||||||
|
elif i == 1:
|
||||||
|
return
|
||||||
|
raise SystemExit("timed out waiting for a VyOS shell")
|
||||||
|
|
||||||
|
|
||||||
|
def run(c: pexpect.spawn, cmd: str, timeout: int = 60) -> str:
|
||||||
|
c.sendline(cmd)
|
||||||
|
c.expect(PROMPT, timeout=timeout)
|
||||||
|
return c.before or ""
|
||||||
|
|
||||||
|
|
||||||
|
def phase_install(c: pexpect.spawn) -> None:
|
||||||
|
"""Answer `install image` end to end."""
|
||||||
|
rules: list[tuple[str, str]] = [
|
||||||
|
(r"Would you like to continue\?", "yes"),
|
||||||
|
(r"What would you like to name this image\?", ""),
|
||||||
|
(r"Please confirm password for the .vyos. user:", PASSWORD),
|
||||||
|
(r"Please enter a password for the .vyos. user:", PASSWORD),
|
||||||
|
(r"What console should be used by default", "K"),
|
||||||
|
# every RAID variant defaults to YES — decline them all
|
||||||
|
(r"Would you like to [^?]*RAID-1 mirroring", "no"),
|
||||||
|
(r"Installation will delete all data on (?:the drive|both drives)\. Continue\?", "yes"),
|
||||||
|
(r"Which one should be used for installation\?", "/dev/vda"),
|
||||||
|
(r"Would you like to use all the free space on the drive\?", "yes"),
|
||||||
|
(r"Which file would you like as boot config\?", "1"),
|
||||||
|
# reinstall-only; unanswered it blocks on stdin until the world ends
|
||||||
|
(r"Would you like to copy data to the new image\?", "yes"),
|
||||||
|
(r"From which image would you like to save config information\?", "1"),
|
||||||
|
]
|
||||||
|
patterns = [r for r, _ in rules] + [r"The image installed successfully",
|
||||||
|
r"Unable to install VyOS", pexpect.TIMEOUT]
|
||||||
|
|
||||||
|
c.sendline("install image")
|
||||||
|
for _ in range(60):
|
||||||
|
i = c.expect(patterns, timeout=180)
|
||||||
|
if i < len(rules):
|
||||||
|
c.sendline(rules[i][1])
|
||||||
|
continue
|
||||||
|
if i == len(rules):
|
||||||
|
print(" installer: success")
|
||||||
|
return
|
||||||
|
if i == len(rules) + 1:
|
||||||
|
raise SystemExit("installer reported failure")
|
||||||
|
raise SystemExit("installer went quiet (unanswered prompt?)")
|
||||||
|
raise SystemExit("installer exceeded expected prompt count")
|
||||||
|
|
||||||
|
|
||||||
|
def phase_configure(c: pexpect.spawn, vlans: list[tuple[str, str, str]]) -> None:
|
||||||
|
"""bond0 over eth0+eth1 with LACP, then a gateway address per VLAN."""
|
||||||
|
# Production shape: VLAN 1 (management) is the NATIVE/untagged VLAN on the
|
||||||
|
# bond, everything else is a tagged vif. This matters beyond fidelity —
|
||||||
|
# LACPDUs are untagged, so a trunk with no native VLAN has nowhere to put
|
||||||
|
# them and the bond never negotiates.
|
||||||
|
native = [v for v in vlans if v[0] == "1"]
|
||||||
|
tagged = [v for v in vlans if v[0] != "1"]
|
||||||
|
|
||||||
|
cmds = [
|
||||||
|
"configure",
|
||||||
|
"set interfaces bonding bond0 mode '802.3ad'",
|
||||||
|
"set interfaces bonding bond0 hash-policy 'layer2+3'",
|
||||||
|
"set interfaces bonding bond0 lacp-rate 'fast'",
|
||||||
|
"set interfaces bonding bond0 member interface 'eth0'",
|
||||||
|
"set interfaces bonding bond0 member interface 'eth1'",
|
||||||
|
"set service ssh port '22'",
|
||||||
|
"set system login user vyos authentication plaintext-password 'vyos'",
|
||||||
|
]
|
||||||
|
for vid, name, prefix in native:
|
||||||
|
cmds.append(f"set interfaces bonding bond0 address '{prefix}.1/24'")
|
||||||
|
cmds.append(f"set interfaces bonding bond0 description '{name} (native)'")
|
||||||
|
for vid, name, prefix in tagged:
|
||||||
|
cmds.append(f"set interfaces bonding bond0 vif {vid} address '{prefix}.1/24'")
|
||||||
|
cmds.append(f"set interfaces bonding bond0 vif {vid} description '{name}'")
|
||||||
|
cmds += ["commit", "save", "exit"]
|
||||||
|
|
||||||
|
for cmd in cmds:
|
||||||
|
out = run(c, cmd, timeout=180)
|
||||||
|
low = out.lower()
|
||||||
|
if "invalid" in low or "syntax error" in low or "commit failed" in low:
|
||||||
|
print(f" !! {cmd}\n{out.strip()[-300:]}")
|
||||||
|
raise SystemExit(f"config command rejected: {cmd}")
|
||||||
|
print(" config committed and saved")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--vm", required=True)
|
||||||
|
ap.add_argument("--phase", required=True, choices=["live", "install", "configure"])
|
||||||
|
ap.add_argument("--vlans", default="", help="space separated vid:name:prefix:real entries")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
c = console(args.vm)
|
||||||
|
try:
|
||||||
|
login(c)
|
||||||
|
if args.phase == "live":
|
||||||
|
print(" live system reachable")
|
||||||
|
elif args.phase == "install":
|
||||||
|
phase_install(c)
|
||||||
|
else:
|
||||||
|
vlans = []
|
||||||
|
for entry in args.vlans.split():
|
||||||
|
parts = entry.split(":")
|
||||||
|
if len(parts) >= 3:
|
||||||
|
vlans.append((parts[0], parts[1], parts[2]))
|
||||||
|
if not vlans:
|
||||||
|
raise SystemExit("no VLANs passed to configure")
|
||||||
|
phase_configure(c, vlans)
|
||||||
|
return 0
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
c.sendline("")
|
||||||
|
c.close(force=True)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
124
labsim/router-up.sh
Executable file
124
labsim/router-up.sh
Executable file
@@ -0,0 +1,124 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Add the VyOS router under test to labsim.
|
||||||
|
#
|
||||||
|
# Mirrors the production VP2440 pair: TWO NICs bonded with LACP carrying a
|
||||||
|
# trunk of every VLAN, then bond0.<vlan> sub-interfaces holding the .1 gateway
|
||||||
|
# address on each. That is the same config shape the real firewalls run, so a
|
||||||
|
# rule tested here means something.
|
||||||
|
#
|
||||||
|
# NIC model is e1000e, NOT virtio, and that is load-bearing: with virtio the
|
||||||
|
# guest's bonding driver reports its slaves "MII Status: down" despite
|
||||||
|
# carrier=1 and never emits a single LACPDU, so the bond sits in
|
||||||
|
# AD_STATE_DEFAULTED forever. Known issue — see the netdev thread "bonding
|
||||||
|
# (IEEE 802.3ad) not working with qemu/virtio"; e1000e fixes it with no other
|
||||||
|
# change. 802.3ad also requires the MII link monitor, which virtio cannot back.
|
||||||
|
#
|
||||||
|
# host OVS "switch" VyOS VM
|
||||||
|
# hostv<vlan> (.2) ──────── ovs-labsim ──── lag-vyos ═════ eth0 + eth1
|
||||||
|
# (tagged) (LACP, trunk) └─ bond0.<vlan> = .1
|
||||||
|
#
|
||||||
|
# Usage: ./router-up.sh build + install + configure
|
||||||
|
# ./router-up.sh --status show bond/LACP + interface state
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
source "$SCRIPT_DIR/lib.sh"
|
||||||
|
source "$SCRIPT_DIR/ovs.sh"
|
||||||
|
|
||||||
|
ROUTER_VM="${ROUTER_VM:-labsim-vyos}"
|
||||||
|
ROUTER_MEM="${ROUTER_MEM:-2048}"
|
||||||
|
ROUTER_CPUS="${ROUTER_CPUS:-2}"
|
||||||
|
ROUTER_DISK_GB="${ROUTER_DISK_GB:-8}"
|
||||||
|
VYOS_ISO="${VYOS_ISO:-$IMG_DIR/vyos.iso}"
|
||||||
|
VYOS_CACHE="/var/lib/libvirt/images/lab-pxe-cache"
|
||||||
|
|
||||||
|
selected_vlans
|
||||||
|
|
||||||
|
if [ "${1:-}" = "--status" ]; then
|
||||||
|
ovs_bond_status
|
||||||
|
echo "--- vyos gateway addresses (probed from each host leg) ---"
|
||||||
|
for entry in "${SELECTED[@]}"; do
|
||||||
|
IFS=: read -r vid _n prefix _r <<<"$entry"
|
||||||
|
printf ' vlan %-5s %-16s ' "$vid" "${prefix}.1"
|
||||||
|
ping -c1 -W2 "${prefix}.1" >/dev/null 2>&1 && echo up || echo down
|
||||||
|
done
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
ovs_require
|
||||||
|
|
||||||
|
# --- ISO ------------------------------------------------------------------
|
||||||
|
if [ ! -f "$VYOS_ISO" ]; then
|
||||||
|
# Reuse the bastion's cached nightly if it is already on this box.
|
||||||
|
if [ -f "$VYOS_CACHE/vyos.iso" ]; then
|
||||||
|
log "reusing cached VyOS ISO"
|
||||||
|
sudo cp "$VYOS_CACHE/vyos.iso" "$VYOS_ISO"
|
||||||
|
else
|
||||||
|
log "resolving latest VyOS nightly ISO..."
|
||||||
|
url="$(curl -sSL https://api.github.com/repos/vyos/vyos-nightly-build/releases/latest \
|
||||||
|
| python3 -c "import json,sys;print(next(a['browser_download_url'] for a in json.load(sys.stdin)['assets'] if a['name'].endswith('generic-amd64.iso')))")"
|
||||||
|
log "downloading $url"
|
||||||
|
sudo curl -sSL --max-time 1800 -o "$VYOS_ISO" "$url"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
[ -f "$VYOS_ISO" ] || die "no VyOS ISO at $VYOS_ISO"
|
||||||
|
|
||||||
|
# --- VM -------------------------------------------------------------------
|
||||||
|
if virsh_q dominfo "$ROUTER_VM" >/dev/null 2>&1; then
|
||||||
|
log "router VM $ROUTER_VM exists"
|
||||||
|
virsh_q start "$ROUTER_VM" >/dev/null 2>&1 || true
|
||||||
|
else
|
||||||
|
log "creating router VM $ROUTER_VM (2 NICs on the trunk, for LACP)"
|
||||||
|
sudo qemu-img create -q -f qcow2 "$IMG_DIR/${ROUTER_VM}.qcow2" "${ROUTER_DISK_GB}G" >/dev/null
|
||||||
|
|
||||||
|
# Two trunk NICs — OVS bonds them after boot (libvirt cannot create bonds).
|
||||||
|
sudo virt-install \
|
||||||
|
--connect "$LIBVIRT_URI" \
|
||||||
|
--name "$ROUTER_VM" \
|
||||||
|
--memory "$ROUTER_MEM" --vcpus "$ROUTER_CPUS" \
|
||||||
|
--disk "path=$IMG_DIR/${ROUTER_VM}.qcow2,format=qcow2,bus=virtio" \
|
||||||
|
--disk "path=$VYOS_ISO,device=cdrom,readonly=on" \
|
||||||
|
--network "network=$OVS_NET,portgroup=trunk,model=e1000e,trustGuestRxFilters=yes" \
|
||||||
|
--network "network=$OVS_NET,portgroup=trunk,model=e1000e,trustGuestRxFilters=yes" \
|
||||||
|
--boot cdrom,hd \
|
||||||
|
--os-variant debian12 \
|
||||||
|
--graphics none --noautoconsole --import >/dev/null
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "waiting for the live system to boot (VyOS live login)..."
|
||||||
|
python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase live || die "live boot failed"
|
||||||
|
|
||||||
|
log "installing VyOS to disk (unattended over the console)..."
|
||||||
|
python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase install || die "install failed"
|
||||||
|
|
||||||
|
# Boot the INSTALLED system from here on. Without this the VM was created with
|
||||||
|
# --boot cdrom,hd and every restart re-runs the ISO, so the live system comes
|
||||||
|
# back with no config and every `commit; save` silently evaporates.
|
||||||
|
log "switching boot to disk and ejecting the install media..."
|
||||||
|
virsh_q destroy "$ROUTER_VM" >/dev/null 2>&1 || true
|
||||||
|
sleep 2
|
||||||
|
sudo virt-xml "$ROUTER_VM" --edit --boot hd >/dev/null
|
||||||
|
sudo virt-xml "$ROUTER_VM" --remove-device --disk device=cdrom >/dev/null 2>&1 || true
|
||||||
|
virsh_q start "$ROUTER_VM" >/dev/null
|
||||||
|
sleep 10
|
||||||
|
|
||||||
|
# Bond the taps only now: they are recreated by the restart above, so bonding
|
||||||
|
# before this would bond stale interfaces.
|
||||||
|
ovs_bond_router "$ROUTER_VM"
|
||||||
|
|
||||||
|
log "applying router config (bond0 LACP + VLAN gateways)..."
|
||||||
|
python3 "$SCRIPT_DIR/router-install.py" --vm "$ROUTER_VM" --phase configure \
|
||||||
|
--vlans "$(printf '%s\n' "${SELECTED[@]}" | tr '\n' ' ')" || die "configure failed"
|
||||||
|
|
||||||
|
log "waiting for LACP to negotiate..."
|
||||||
|
for _ in $(seq 1 30); do
|
||||||
|
if sudo ovs-appctl lacp/show "$LAG_NAME" 2>/dev/null | grep -q "current attached"; then
|
||||||
|
log "LACP negotiated"; break
|
||||||
|
fi
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
|
||||||
|
echo
|
||||||
|
ovs_bond_status
|
||||||
|
echo
|
||||||
|
log "router is up. Check reachability with: $SCRIPT_DIR/labsim-matrix.py --watch 2"
|
||||||
181
labsim/topology.html
Normal file
181
labsim/topology.html
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<title>labsim — live VLAN topology</title>
|
||||||
|
<style>
|
||||||
|
:root {
|
||||||
|
--bg:#0e1116; --panel:#161b22; --line:#30363d; --text:#e6edf3; --dim:#8b949e;
|
||||||
|
--ok:#3fb950; --bad:#f85149; --warn:#d29922; --router:#58a6ff;
|
||||||
|
}
|
||||||
|
* { box-sizing:border-box; }
|
||||||
|
body { margin:0; background:var(--bg); color:var(--text);
|
||||||
|
font:14px/1.5 ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif; }
|
||||||
|
header { display:flex; align-items:baseline; gap:16px; flex-wrap:wrap;
|
||||||
|
padding:14px 20px; border-bottom:1px solid var(--line); }
|
||||||
|
h1 { font-size:16px; margin:0; font-weight:650; letter-spacing:.2px; }
|
||||||
|
.meta { color:var(--dim); font-size:12px; }
|
||||||
|
.pill { padding:2px 8px; border-radius:999px; font-size:12px; font-weight:600; }
|
||||||
|
.pill.ok { background:rgba(63,185,80,.15); color:var(--ok); }
|
||||||
|
.pill.bad { background:rgba(248,81,73,.15); color:var(--bad); }
|
||||||
|
main { display:grid; grid-template-columns:minmax(0,1.35fr) minmax(320px,.65fr);
|
||||||
|
gap:16px; padding:16px 20px; align-items:start; }
|
||||||
|
@media (max-width:1000px){ main { grid-template-columns:1fr; } }
|
||||||
|
.card { background:var(--panel); border:1px solid var(--line); border-radius:10px; padding:14px; }
|
||||||
|
.card h2 { margin:0 0 10px; font-size:13px; font-weight:600; color:var(--dim);
|
||||||
|
text-transform:uppercase; letter-spacing:.6px; }
|
||||||
|
svg { width:100%; height:auto; display:block; }
|
||||||
|
.edge { stroke-width:2.5; transition:stroke .25s, opacity .25s; }
|
||||||
|
.edge.ok { stroke:var(--ok); opacity:.55; }
|
||||||
|
.edge.bad { stroke:var(--bad); opacity:.95; stroke-dasharray:7 5; }
|
||||||
|
.edge:hover { opacity:1; stroke-width:4; }
|
||||||
|
.node circle { fill:#0d1117; stroke-width:2.5; }
|
||||||
|
.node text { text-anchor:middle; font-size:11px; font-weight:600; fill:var(--text); }
|
||||||
|
.node .sub { font-size:9.5px; font-weight:400; fill:var(--dim); }
|
||||||
|
.rtt { font-size:9px; fill:var(--dim); text-anchor:middle; }
|
||||||
|
table { width:100%; border-collapse:collapse; font-size:12.5px; }
|
||||||
|
th,td { text-align:left; padding:5px 8px; border-bottom:1px solid var(--line); }
|
||||||
|
th { color:var(--dim); font-weight:600; font-size:11px; text-transform:uppercase; }
|
||||||
|
td.n { text-align:right; font-variant-numeric:tabular-nums; }
|
||||||
|
.b-ok { color:var(--ok); } .b-bad { color:var(--bad); }
|
||||||
|
.empty { color:var(--dim); padding:10px 4px; }
|
||||||
|
.legend { display:flex; gap:14px; align-items:center; color:var(--dim);
|
||||||
|
font-size:11.5px; margin-top:10px; flex-wrap:wrap; }
|
||||||
|
.swatch { display:inline-block; width:22px; height:0; border-top:2.5px solid; margin-right:5px;
|
||||||
|
vertical-align:middle; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<header>
|
||||||
|
<h1>labsim — live VLAN topology</h1>
|
||||||
|
<span id="summary" class="pill ok">…</span>
|
||||||
|
<span class="meta">every path is probed <em>from</em> a VM <em>to</em> every other VM, through the VyOS router</span>
|
||||||
|
<span class="meta" id="clock" style="margin-left:auto"></span>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<main>
|
||||||
|
<section class="card">
|
||||||
|
<h2>Mesh — line colour is reachability, label is ICMP RTT</h2>
|
||||||
|
<svg id="topo" viewBox="0 0 720 560" role="img" aria-label="VLAN topology"></svg>
|
||||||
|
<div class="legend">
|
||||||
|
<span><i class="swatch" style="border-color:var(--ok)"></i>reachable</span>
|
||||||
|
<span><i class="swatch" style="border-color:var(--bad); border-top-style:dashed"></i>blocked</span>
|
||||||
|
<span>hover a line for detail · node ring turns red if anything to/from it is blocked</span>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<aside style="display:grid; gap:16px">
|
||||||
|
<section class="card">
|
||||||
|
<h2>Blocked paths</h2>
|
||||||
|
<div id="blocked"></div>
|
||||||
|
</section>
|
||||||
|
<section class="card">
|
||||||
|
<h2>Latency (ICMP, ms)</h2>
|
||||||
|
<table><thead><tr><th>path</th><th class="n">rtt</th></tr></thead>
|
||||||
|
<tbody id="lat"></tbody></table>
|
||||||
|
</section>
|
||||||
|
</aside>
|
||||||
|
</main>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
const REFRESH_MS = 5000;
|
||||||
|
const CX = 360, CY = 250, R = 185;
|
||||||
|
|
||||||
|
function polar(i, n) {
|
||||||
|
const a = (i / n) * Math.PI * 2 - Math.PI / 2;
|
||||||
|
return { x: CX + R * Math.cos(a), y: CY + R * Math.sin(a) };
|
||||||
|
}
|
||||||
|
|
||||||
|
function render(data) {
|
||||||
|
const vlans = data.vlans, res = data.results;
|
||||||
|
const svg = document.getElementById('topo');
|
||||||
|
const n = vlans.length;
|
||||||
|
const pos = vlans.map((_, i) => polar(i, n));
|
||||||
|
let out = '';
|
||||||
|
|
||||||
|
// Router in the middle — every inter-VLAN packet really does traverse it.
|
||||||
|
out += `<circle cx="${CX}" cy="${CY}" r="40" fill="#0d1117" stroke="var(--router)" stroke-width="2.5"/>`;
|
||||||
|
out += `<text x="${CX}" y="${CY-6}" text-anchor="middle" font-size="12" font-weight="700" fill="var(--router)">VyOS</text>`;
|
||||||
|
out += `<text x="${CX}" y="${CY+9}" text-anchor="middle" font-size="8.5" fill="var(--dim)">bond0</text>`;
|
||||||
|
out += `<text x="${CX}" y="${CY+20}" text-anchor="middle" font-size="8.5" fill="var(--dim)">LACP</text>`;
|
||||||
|
|
||||||
|
const bad = new Set();
|
||||||
|
// One line per unordered pair; a pair is bad if EITHER direction fails.
|
||||||
|
for (let i = 0; i < n; i++) {
|
||||||
|
for (let j = i + 1; j < n; j++) {
|
||||||
|
const a = vlans[i].label, b = vlans[j].label;
|
||||||
|
const ab = (res[a] || {})[b] || {}, ba = (res[b] || {})[a] || {};
|
||||||
|
const okAB = ab.icmp === true, okBA = ba.icmp === true;
|
||||||
|
const ok = okAB && okBA;
|
||||||
|
if (!ok) { bad.add(a); bad.add(b); }
|
||||||
|
const rtts = [ab.rtt_ms, ba.rtt_ms].filter(v => typeof v === 'number');
|
||||||
|
const rtt = rtts.length ? (rtts.reduce((s,v)=>s+v,0)/rtts.length) : null;
|
||||||
|
// Place the label ~32% along the edge, not at the midpoint: diagonals of
|
||||||
|
// a 6-node mesh all cross the centre, so midpoint labels stack on top of
|
||||||
|
// the router node. Plus a small perpendicular nudge off the line itself.
|
||||||
|
const dx = pos[j].x - pos[i].x, dy = pos[j].y - pos[i].y;
|
||||||
|
const len = Math.hypot(dx, dy) || 1;
|
||||||
|
const t = 0.32;
|
||||||
|
const mx = pos[i].x + dx * t + (-dy / len) * 8;
|
||||||
|
const my = pos[i].y + dy * t + ( dx / len) * 8;
|
||||||
|
const tip = `${a} ↔ ${b}\n→ ${okAB ? 'ok' : 'BLOCKED'} ← ${okBA ? 'ok' : 'BLOCKED'}` +
|
||||||
|
(rtt !== null ? `\nrtt ${rtt.toFixed(2)} ms` : '');
|
||||||
|
out += `<line class="edge ${ok?'ok':'bad'}" x1="${pos[i].x}" y1="${pos[i].y}" x2="${pos[j].x}" y2="${pos[j].y}"><title>${tip}</title></line>`;
|
||||||
|
if (ok && rtt !== null)
|
||||||
|
out += `<text class="rtt" x="${mx}" y="${my}">${rtt.toFixed(2)}</text>`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
vlans.forEach((v, i) => {
|
||||||
|
const p = pos[i], isBad = bad.has(v.label);
|
||||||
|
out += `<g class="node"><circle cx="${p.x}" cy="${p.y}" r="30" stroke="${isBad?'var(--bad)':'var(--ok)'}"/>` +
|
||||||
|
`<text x="${p.x}" y="${p.y-2}">${v.name}</text>` +
|
||||||
|
`<text class="sub" x="${p.x}" y="${p.y+11}">vlan ${v.vid}</text>` +
|
||||||
|
`<text class="sub" x="${p.x}" y="${p.y+47}">${v.ip}</text></g>`;
|
||||||
|
});
|
||||||
|
svg.innerHTML = out;
|
||||||
|
|
||||||
|
// Blocked list — the thing you actually act on.
|
||||||
|
const rows = [];
|
||||||
|
for (const src of vlans) for (const dst of vlans) {
|
||||||
|
if (src.label === dst.label) continue;
|
||||||
|
const d = (res[src.label] || {})[dst.label] || {};
|
||||||
|
for (const proto of ['icmp','tcp22','tcp80'])
|
||||||
|
if (d[proto] === false) rows.push(`${src.label} → ${dst.label} <span style="color:var(--dim)">(${proto})</span>`);
|
||||||
|
}
|
||||||
|
document.getElementById('blocked').innerHTML = rows.length
|
||||||
|
? `<table><tbody>${rows.map(r=>`<tr><td class="b-bad">${r}</td></tr>`).join('')}</tbody></table>`
|
||||||
|
: `<div class="empty">none — all ${vlans.length*(vlans.length-1)*3} paths open</div>`;
|
||||||
|
|
||||||
|
// Latency table, slowest first.
|
||||||
|
const lat = [];
|
||||||
|
for (const src of vlans) for (const dst of vlans) {
|
||||||
|
if (src.label === dst.label) continue;
|
||||||
|
const d = (res[src.label] || {})[dst.label] || {};
|
||||||
|
if (typeof d.rtt_ms === 'number') lat.push([`${src.label} → ${dst.label}`, d.rtt_ms]);
|
||||||
|
}
|
||||||
|
lat.sort((a,b) => b[1]-a[1]);
|
||||||
|
document.getElementById('lat').innerHTML = lat.slice(0,12)
|
||||||
|
.map(([k,v]) => `<tr><td>${k}</td><td class="n">${v.toFixed(2)}</td></tr>`).join('')
|
||||||
|
|| `<tr><td class="empty" colspan="2">no RTT data</td></tr>`;
|
||||||
|
|
||||||
|
const total = data.total, reach = data.reachable;
|
||||||
|
const pill = document.getElementById('summary');
|
||||||
|
pill.textContent = `${reach}/${total} paths open`;
|
||||||
|
pill.className = 'pill ' + (reach === total ? 'ok' : 'bad');
|
||||||
|
document.getElementById('clock').textContent =
|
||||||
|
`updated ${new Date().toLocaleTimeString()} · sweep ${data.sweep_seconds.toFixed(2)}s · refresh ${REFRESH_MS/1000}s`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function tick() {
|
||||||
|
try {
|
||||||
|
const r = await fetch('/api/matrix', {cache:'no-store'});
|
||||||
|
render(await r.json());
|
||||||
|
} catch (e) {
|
||||||
|
document.getElementById('clock').textContent = 'exporter unreachable — ' + e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tick(); setInterval(tick, REFRESH_MS);
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
21
labsim/vlans.conf
Normal file
21
labsim/vlans.conf
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
# Lab network simulation — VLAN map.
|
||||||
|
#
|
||||||
|
# Mirrors the real UniFi topology (same VLAN IDs, same roles) but with
|
||||||
|
# deliberately DIFFERENT IP ranges so nothing here can collide with, or be
|
||||||
|
# confused for, production. The sim subnet always encodes the VLAN id:
|
||||||
|
#
|
||||||
|
# 172.31.<vlan-id>.0/24
|
||||||
|
#
|
||||||
|
# Per-subnet address plan (same shape on every VLAN):
|
||||||
|
# .1 gateway under test (VyOS/router VM — not created by default)
|
||||||
|
# .2 host bridge (how you SSH in from this workstation)
|
||||||
|
# .10 the micro VM for this VLAN
|
||||||
|
# .254 VRRP VIP (reserved, mirrors production)
|
||||||
|
#
|
||||||
|
# Format: vlan_id:name:sim_subnet_prefix:real_subnet(for reference)
|
||||||
|
1:management:172.31.1:192.168.1.0/24
|
||||||
|
2:k8s:172.31.2:192.168.8.0/23
|
||||||
|
3:kvm:172.31.3:192.168.3.0/24
|
||||||
|
9:private:172.31.9:10.0.9.0/23
|
||||||
|
10:lot:172.31.10:10.0.0.0/23
|
||||||
|
200:roomates:172.31.200:192.168.2.0/24
|
||||||
Reference in New Issue
Block a user