Files
lab/bastion/src/labd/src/routes/bastions.ts
Michal cb9d99dd69
Some checks failed
CI/CD / lint (pull_request) Failing after 10s
CI/CD / test (pull_request) Failing after 10s
CI/CD / typecheck (pull_request) Failing after 23s
CI/CD / build (pull_request) Has been skipped
CI/CD / publish-rpm (pull_request) Has been skipped
CI/CD / publish-deb (pull_request) Has been skipped
feat(bastion): unattended VyOS network install with HA (bond + VRRP)
VyOS ships no unattended installer (install_image() is unconditionally
interactive; --no-prompt is wired only to 'add'), so the automation is
injected through live-config's hooks component: iPXE boots the live
kernel with fetch= and live-config.hooks=, the hook fetches a generated
per-MAC Python driver, and the driver builds config.boot, stages the
rootfs, and drives the interactive installer over a pty.

Bastion:
- vyos-boot.ipxe template (no 'nonetworking' — breaks the hook fetch;
  no console=ttyS0 — 30s/systemd-phase on UART-less boards)
- /vyos/autoinstall.sh + /vyos/install.py routes (per-MAC driver with
  the config spec baked in as base64)
- vyos-config-spec: bond0 (802.3ad) + tagged VLANs + VRRP groups
  (vrid = VLAN id) + sync group + hw-id pinning by MAC + SSH keys;
  config built from the image's own config.boot.default via
  vyos.configtree, version footer reattached via component_version
- prepareVyosArtifacts: extract kernel/initrd/squashfs from the nightly
  ISO with xorriso; initrd picked by size from regular files only;
  ISO URL "latest" resolves the newest vyos-nightly-build GH release
  (downloads.vyos.io no longer serves direct ISOs)

Verified end-to-end in a libvirt VM against the real nightly ISO —
installed system boots with bond/VRRP/hw-id config applied and no
migrations. Fixes found by the VM run, encoded in code comments:
config.boot.default lives at /usr/share/vyos at hook time; fetch= boot
has no medium so the rootfs is symlinked to the installer's expected
path; reboot must be --force (the hook is a child of the still-starting
live-config unit); installer disk answers are full /dev paths; zram0
passes the 2GB min-disk filter so the disk is always pinned.

CLI/labd: vyos spec threaded through provision install (--vyos-* and
--vlan/--vlan-vip flags with guards), labd install route, protocol
command-install, and the bastion's direct /api/install.

268 unit tests pass; no new lint errors in touched files.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DMVzWZgiKW2wquf5z8S1yH
2026-08-10 21:45:22 +01:00

391 lines
14 KiB
TypeScript

// Bastion management routes.
// GET /api/bastions — list connected bastions
// GET /api/machines — aggregated machines from all bastions
// POST /api/machines/install — queue install on correct bastion
// DELETE /api/machines/:mac — forget machine on correct bastion
// POST /api/machines/role — update role on correct bastion
// GET /api/machines/:mac/logs — get provision logs from correct bastion
import type { FastifyInstance } from "fastify";
import type { DbClient } from "../server.js";
import { bastionRegistry } from "../services/bastion-registry.js";
import { generateRequestId } from "@lab/shared";
import type { VyosInstallSpec } from "@lab/shared";
const COMMAND_TIMEOUT_MS = 15_000;
/** Send a command to a bastion and wait for the response. */
function sendCommand(
bastionId: string,
msg: Record<string, unknown>,
): Promise<{ status: string; data?: unknown; error?: string | undefined }> {
const bastion = bastionRegistry.getById(bastionId);
if (!bastion) {
return Promise.reject(new Error(`Bastion ${bastionId} not connected`));
}
const requestId = generateRequestId();
const fullMsg = { ...msg, requestId };
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
cleanup();
reject(new Error("Command timed out"));
}, COMMAND_TIMEOUT_MS);
const handler = (data: Buffer) => {
try {
const parsed = JSON.parse(data.toString()) as { type: string; requestId?: string; status?: string; data?: unknown; error?: string };
if (parsed.type === "command-response" && parsed.requestId === requestId) {
cleanup();
resolve({ status: parsed.status ?? "ok", data: parsed.data, error: parsed.error });
}
} catch { /* not our message */ }
};
const cleanup = () => {
clearTimeout(timeout);
bastion.socket.off("message", handler);
};
bastion.socket.on("message", handler);
bastion.socket.send(JSON.stringify(fullMsg));
});
}
export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void {
// List all bastions (DB records enriched with online status from registry)
app.get("/api/bastions", async () => {
const dbBastions = await db.bastion.findMany() as Array<{
id: string; hostname: string; network: string; serverIp: string;
status: string; lastHeartbeat: Date | null; createdAt: Date;
}>;
return dbBastions.map((b) => {
const connected = bastionRegistry.getById(b.id);
return {
id: b.id,
hostname: b.hostname,
network: b.network,
serverIp: b.serverIp,
status: connected ? "online" : "offline",
lastHeartbeat: connected?.lastHeartbeat ?? b.lastHeartbeat,
connectedAt: connected?.connectedAt,
machineCount: connected
? Object.keys(connected.state.discovered).length +
Object.keys(connected.state.install_queue).length +
Object.keys(connected.state.installed).length
: 0,
createdAt: b.createdAt,
};
});
});
// Aggregated machines from all connected bastions + DB fallback
app.get("/api/machines", async () => {
const live = bastionRegistry.getAggregatedState();
try {
const dbServers = (await db.server.findMany({})) as Array<{
mac: string | null; hostname: string; role: string; ip: string | null;
status: string; labels: Record<string, unknown>;
}>;
for (const s of dbServers) {
if (!s.mac) continue;
const mac = s.mac.toLowerCase();
// DB knows this machine has been installed at some point if it has a real
// hostname+role (not just product-name-as-hostname and role="unknown").
// Status alone is unreliable: a rediscovery can re-set it without erasing the
// install identity. If the bastion restarted and lost its installed map, the
// machine will only show up in live.discovered — promote it here so the CLI
// still sees hostname/role/IP.
const dbKnowsInstalled =
s.role !== "unknown" && s.role !== "" &&
s.hostname !== "" && s.hostname !== s.mac;
if (dbKnowsInstalled && !(mac in live.installed) && !(mac in live.install_queue)) {
const hw = live.discovered[mac];
live.installed[mac] = {
hostname: s.hostname,
role: s.role,
ip: s.ip ?? "",
installed_at: "",
bastionId: hw?.bastionId ?? "db",
...(hw ? {
product: hw.product,
manufacturer: hw.manufacturer,
cpu_model: hw.cpu_model,
cpu_cores: hw.cpu_cores,
memory_gb: hw.memory_gb,
arch: hw.arch,
} : {}),
};
delete live.discovered[mac];
continue;
}
// Unknown-to-live MAC: fall back to whatever the DB says.
if (!(mac in live.discovered) && !(mac in live.install_queue) && !(mac in live.installed)) {
if (s.status === "online" || s.status === "offline") {
live.installed[mac] = {
hostname: s.hostname,
role: s.role,
ip: s.ip ?? "",
installed_at: "",
bastionId: "db",
};
} else {
live.discovered[mac] = {
mac,
product: String(s.labels?.product ?? "unknown"),
board: "unknown",
serial: "unknown",
manufacturer: String(s.labels?.manufacturer ?? "unknown"),
cpu_model: String(s.labels?.cpu ?? "unknown"),
cpu_cores: Number(s.labels?.cores ?? 0),
memory_gb: Number(s.labels?.memory_gb ?? 0),
arch: String(s.labels?.arch ?? "unknown"),
disks: [],
nics: [],
first_seen: "",
last_seen: "",
bastionId: "db",
};
}
}
}
} catch {
// DB unavailable — return live state only
}
return live;
});
// Queue install — route to correct bastion by MAC
app.post<{
Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string; vyos?: VyosInstallSpec };
}>("/api/machines/install", async (request, reply) => {
const { mac, hostname, disk, role, os, vyos } = request.body ?? {};
if (!mac || !hostname) {
return reply.code(400).send({ error: "mac and hostname are required" });
}
// Find bastion that knows this MAC, or let caller specify
const bastion = bastionRegistry.findBastionByMac(mac);
if (!bastion) {
// If only one bastion is connected, use it
const all = bastionRegistry.getAll();
if (all.length === 0) {
return reply.code(503).send({ error: "No bastions connected" });
}
if (all.length === 1) {
try {
const result = await sendCommand(all[0]!.bastionId, {
type: "command-install",
mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
...(vyos ? { vyos } : {}),
});
return reply.code(result.status === "ok" ? 200 : 500).send(result);
} catch (err) {
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
}
}
return reply.code(404).send({ error: `MAC ${mac} not found on any bastion` });
}
try {
const result = await sendCommand(bastion.bastionId, {
type: "command-install",
mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
...(vyos ? { vyos } : {}),
});
return reply.code(result.status === "ok" ? 200 : 500).send(result);
} catch (err) {
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
}
});
// Register an already-installed machine — route to correct bastion (or single bastion)
app.post<{
Body: { mac?: string; hostname?: string; role?: string; ip?: string };
}>("/api/machines/register", async (request, reply) => {
const { mac, hostname, role, ip } = request.body ?? {};
if (!mac || !hostname) {
return reply.code(400).send({ error: "mac and hostname are required" });
}
const normalized = mac.toLowerCase().replace(/-/g, ":");
// Find bastion that knows this MAC, or use single connected bastion
const bastion = bastionRegistry.findBastionByMac(normalized);
const target = bastion ?? (bastionRegistry.getAll().length === 1 ? bastionRegistry.getAll()[0] : null);
if (!target) {
const all = bastionRegistry.getAll();
if (all.length === 0) {
return reply.code(503).send({ error: "No bastions connected" });
}
return reply.code(404).send({ error: `MAC ${normalized} not found on any bastion and multiple bastions connected` });
}
try {
const result = await sendCommand(target.bastionId, {
type: "command-register",
mac: normalized,
hostname,
role: role ?? "worker",
ip: ip ?? "",
});
return reply.code(result.status === "ok" ? 200 : 500).send(result);
} catch (err) {
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
}
});
// Queue debug/rescue mode — route to correct bastion by MAC
app.post<{
Body: { mac?: string; pxeBoot?: boolean };
}>("/api/machines/debug", async (request, reply) => {
const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":");
const pxeBoot = request.body?.pxeBoot ?? false;
if (!mac) {
return reply.code(400).send({ error: "mac is required" });
}
const bastion = bastionRegistry.findBastionByMac(mac);
if (!bastion) {
const all = bastionRegistry.getAll();
if (all.length === 0) {
return reply.code(503).send({ error: "No bastions connected" });
}
if (all.length === 1) {
try {
const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac, pxeBoot });
return reply.code(result.status === "ok" ? 200 : 500).send(result);
} catch (err) {
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
}
}
return reply.code(404).send({ error: `MAC ${mac} not found on any bastion` });
}
try {
const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac, pxeBoot });
return reply.code(result.status === "ok" ? 200 : 500).send(result);
} catch (err) {
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
}
});
// Forget machine
app.delete<{ Params: { mac: string } }>("/api/machines/:mac", async (request, reply) => {
const mac = request.params.mac.toLowerCase().replace(/-/g, ":");
const bastion = bastionRegistry.findBastionByMac(mac);
if (!bastion) {
return reply.code(404).send({ error: `MAC ${mac} not found on any bastion` });
}
try {
const result = await sendCommand(bastion.bastionId, { type: "command-forget", mac });
return reply.send(result);
} catch (err) {
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
}
});
// Update hardware info (discovery data) for a machine
app.post<{
Body: {
mac?: string; product?: string; board?: string; serial?: string;
manufacturer?: string; cpu_model?: string; cpu_cores?: number;
memory_gb?: number; arch?: string;
disks?: Array<{ name: string; size_gb: number; model: string }>;
nics?: Array<{ name: string; mac: string; state: string }>;
};
}>("/api/machines/discover", async (request, reply) => {
const data = request.body ?? {};
const mac = (data.mac ?? "").toLowerCase().replace(/-/g, ":");
if (!mac) {
return reply.code(400).send({ error: "mac is required" });
}
const bastion = bastionRegistry.findBastionByMac(mac);
const target = bastion ?? (bastionRegistry.getAll().length === 1 ? bastionRegistry.getAll()[0] : null);
if (!target) {
return reply.code(503).send({ error: "No bastion found for this MAC" });
}
try {
const result = await sendCommand(target.bastionId, { type: "command-discover", ...data, mac });
return reply.code(result.status === "ok" ? 200 : 500).send(result);
} catch (err) {
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
}
});
// Update role
app.post<{
Body: { mac?: string; role?: string };
}>("/api/machines/role", async (request, reply) => {
const { mac, role } = request.body ?? {};
if (!mac || !role) {
return reply.code(400).send({ error: "mac and role are required" });
}
const normalized = mac.toLowerCase().replace(/-/g, ":");
const bastion = bastionRegistry.findBastionByMac(normalized);
if (!bastion) {
return reply.code(404).send({ error: `MAC ${normalized} not found on any bastion` });
}
try {
const result = await sendCommand(bastion.bastionId, { type: "command-role-update", mac: normalized, role });
return reply.send(result);
} catch (err) {
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
}
});
// Machine logs (snapshot from bastion's state)
app.get<{ Params: { mac: string } }>("/api/machines/:mac/logs", async (request, reply) => {
const mac = request.params.mac.toLowerCase().replace(/-/g, ":");
const bastion = bastionRegistry.findBastionByMac(mac);
if (!bastion) {
return reply.code(404).send({ error: `MAC ${mac} not found` });
}
const queued = bastion.state.install_queue[mac];
const installed = bastion.state.installed[mac];
// Active install takes priority over old installed state (reprovision case)
if (queued) {
return {
mac,
hostname: queued.hostname,
status: queued.progress ? "installing" : "queued",
progress: queued.progress,
progress_detail: queued.progress_detail,
progress_at: queued.progress_at,
role: queued.role,
os: queued.os,
log: queued.log,
};
}
if (installed) {
return {
mac,
hostname: installed.hostname,
status: "installed",
role: installed.role,
ip: installed.ip,
installed_at: installed.installed_at,
};
}
return reply.code(404).send({ error: `MAC ${mac} not found in install queue or installed` });
});
}