feat: install logging, error trapping, PXE/ISO integration tests

Kickstart installs on real hardware failed silently — no error reporting, only 3 progress callbacks, zero log streaming. This overhaul makes every install fully observable. Kickstart improvements: - Error trapping in %pre and %post (trap ERR sends failure details to bastion) - 12+ granular progress stages (was 3): SSH, hostname, k3s prep, EFI boot, metadata - Background log streamer: tails %post output and batch-sends to /api/log - bastion_log() function for explicit log lines from kickstart scripts Bastion API: - POST /api/log — receives raw log lines from kickstart (single or batch) - InstallLogBuffer — per-MAC ring buffer (2000 lines) + file persistence - GET /api/logs/:mac — now returns log_lines + log_total alongside stages - SSE /api/logs/:mac/follow — uses named events (event: stage vs event: log) - Progress events forwarded to labd via bastion-progress WebSocket message - Post-provision k3s logs routed through progressBus (was console-only) dnsmasq fixes found during VM testing: - HTTP Boot filename: ipxe-real.efi → ipxe.efi (leftover from old 2-stage approach) - pxe-service directives: only in proxy mode (breaks OVMF PXE in full mode) - PXEClient vendor class echo for UEFI firmware compatibility Integration tests: - PXE boot test: blank UEFI VM → dnsmasq → HTTP Boot → iPXE → bastion → install - ISO boot test: blank VM boots from bastion-generated ISO → same flow - Shared helpers: pxe-network (no DHCP, nftables fix), pxe-vm (UEFI + ISO boot) - test-provision.sh: runs both PXE + ISO tests with prerequisite checks - 250GB sparse QCOW2 disk (LVM layout needs ~204GB) 201 unit tests passing (11 new). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-26 22:26:33 +00:00
parent ffc4a782d2
commit 46b017d77e
189 changed files with 16241 additions and 432 deletions
--- a/bastion/src/labd/package.json
+++ b/bastion/src/labd/package.json
@@ -16,18 +16,29 @@
    "clean": "rimraf dist",
    "dev": "tsx src/main.ts",
    "db:push": "prisma db push",
-    "db:migrate": "prisma migrate dev",
-    "db:generate": "prisma generate"
+    "db:generate": "prisma generate",
+    "db:migrate:dev": "prisma migrate dev",
+    "db:migrate:deploy": "prisma migrate deploy",
+    "db:migrate:reset": "prisma migrate reset",
+    "db:seed": "tsx prisma/seed.ts",
+    "db:studio": "prisma studio"
  },
  "dependencies": {
+    "@fastify/rate-limit": "^10.3.0",
+    "@fastify/websocket": "^11.0.2",
    "@lab/shared": "workspace:*",
    "@prisma/client": "^6.9.0",
    "fastify": "^5.3.3",
-    "@fastify/websocket": "^11.0.2",
-    "winston": "^3.17.0"
+    "winston": "^3.17.0",
+    "ws": "^8.19.0",
+    "zod": "^4.3.6"
+  },
+  "prisma": {
+    "seed": "tsx prisma/seed.ts"
  },
  "devDependencies": {
    "@types/node": "^22.14.1",
+    "@types/ws": "^8.18.1",
    "prisma": "^6.9.0",
    "rimraf": "^6.1.3",
    "tsx": "^4.21.0",
--- a/bastion/src/labd/prisma/schema.prisma
+++ b/bastion/src/labd/prisma/schema.prisma
@@ -133,6 +133,17 @@ model PulumiRun {
  @@index([stackName])
 }

+model Bastion {
+  id            String    @id @default(uuid())
+  hostname      String    @unique
+  network       String
+  serverIp      String
+  status        String    @default("offline") // online, offline
+  lastHeartbeat DateTime?
+  createdAt     DateTime  @default(now())
+  updatedAt     DateTime  @updatedAt
+}
+
 model Cluster {
  id            String   @id @default(uuid())
  name          String   @unique
--- a/bastion/src/labd/prisma/seed.ts
+++ b/bastion/src/labd/prisma/seed.ts
@@ -0,0 +1,113 @@
+import { PrismaClient } from "@prisma/client";
+
+const db = new PrismaClient();
+
+async function main() {
+  console.log("Seeding database with default RBAC roles and permissions...");
+
+  // --- Admin role: full wildcard access ---
+  const admin = await db.role.upsert({
+    where: { name: "admin" },
+    update: {
+      description: "Full administrative access to all resources",
+    },
+    create: {
+      name: "admin",
+      description: "Full administrative access to all resources",
+      permissions: {
+        create: [
+          {
+            type: "allow",
+            action: "*",
+            cloud: "*",
+            environment: "*",
+            server: "*",
+          },
+        ],
+      },
+    },
+  });
+  console.log(`  Upserted role: ${admin.name} (${admin.id})`);
+
+  // --- Viewer role: read-only access ---
+  const viewer = await db.role.upsert({
+    where: { name: "viewer" },
+    update: {
+      description: "Read-only access to all resources",
+    },
+    create: {
+      name: "viewer",
+      description: "Read-only access to all resources",
+      permissions: {
+        create: [
+          {
+            type: "allow",
+            action: "read",
+            cloud: "*",
+            environment: "*",
+            server: "*",
+          },
+        ],
+      },
+    },
+  });
+  console.log(`  Upserted role: ${viewer.name} (${viewer.id})`);
+
+  // --- Operator role: read/exec/kubectl allowed, destroy denied ---
+  const operator = await db.role.upsert({
+    where: { name: "operator" },
+    update: {
+      description:
+        "Operational access: read, exec, and kubectl — destroy denied",
+    },
+    create: {
+      name: "operator",
+      description:
+        "Operational access: read, exec, and kubectl — destroy denied",
+      permissions: {
+        create: [
+          {
+            type: "allow",
+            action: "read",
+            cloud: "*",
+            environment: "*",
+            server: "*",
+          },
+          {
+            type: "allow",
+            action: "exec",
+            cloud: "*",
+            environment: "*",
+            server: "*",
+          },
+          {
+            type: "allow",
+            action: "kubectl",
+            cloud: "*",
+            environment: "*",
+            server: "*",
+          },
+          {
+            type: "deny",
+            action: "destroy",
+            cloud: "*",
+            environment: "*",
+            server: "*",
+          },
+        ],
+      },
+    },
+  });
+  console.log(`  Upserted role: ${operator.name} (${operator.id})`);
+
+  console.log("Seed complete.");
+}
+
+main()
+  .catch((error) => {
+    console.error("Seed failed:", error);
+    process.exit(1);
+  })
+  .finally(async () => {
+    await db.$disconnect();
+  });
--- a/bastion/src/labd/src/main.ts
+++ b/bastion/src/labd/src/main.ts
@@ -4,59 +4,54 @@
 import { loadConfig } from "./config.js";
 import { createApp } from "./server.js";
 import { logger } from "./services/logger.js";
+import { setupGracefulShutdown } from "./services/shutdown.js";

 async function main(): Promise<void> {
  const config = loadConfig();

  // Initialize Prisma client (wrapped in try/catch for when DB isn't available)
-  let db;
+  let db: import("./server.js").DbClient;
  try {
    const { PrismaClient } = await import("@prisma/client");
-    const prisma = new PrismaClient({
-      datasources: config.databaseUrl
-        ? { db: { url: config.databaseUrl } }
-        : undefined,
-    });
+    const prisma = config.databaseUrl
+      ? new PrismaClient({ datasources: { db: { url: config.databaseUrl } } })
+      : new PrismaClient();
    await prisma.$connect();
    logger.info("Database connected");
-    db = prisma;
+    db = prisma as unknown as import("./server.js").DbClient;
  } catch (err) {
    const message = err instanceof Error ? err.message : String(err);
    logger.warn(`Database not available: ${message}`);
    logger.warn("Running without database -- some features will be unavailable");

    // Create a stub db client that returns errors for all operations
+    const dbError = (): never => {
+      throw new Error("Database not connected");
+    };
    db = {
-      $queryRaw: async () => {
-        throw new Error("Database not connected");
-      },
+      $queryRaw: () => dbError(),
+      $disconnect: async () => {},
      server: {
-        findMany: async () => {
-          throw new Error("Database not connected");
-        },
-        findUnique: async () => {
-          throw new Error("Database not connected");
-        },
+        findMany: () => dbError(),
+        findUnique: () => dbError(),
      },
      joinToken: {
-        findUnique: async () => {
-          throw new Error("Database not connected");
-        },
-        findMany: async () => {
-          throw new Error("Database not connected");
-        },
-        create: async () => {
-          throw new Error("Database not connected");
-        },
-        update: async () => {
-          throw new Error("Database not connected");
-        },
+        findUnique: () => dbError(),
+        findMany: () => dbError(),
+        create: () => dbError(),
+        update: () => dbError(),
+      },
+      bastion: {
+        upsert: () => dbError(),
+        findMany: () => dbError(),
+        findUnique: () => dbError(),
+        update: () => dbError(),
      },
    };
  }

  // Create Fastify app
-  const { app } = createApp(config, db);
+  const { app } = await createApp(config, db);

  // Start server
  try {
@@ -68,18 +63,7 @@ async function main(): Promise<void> {
  }

  // Graceful shutdown
-  const shutdown = async (): Promise<void> => {
-    logger.info("Shutting down...");
-    await app.close();
-    if (db !== null && "$disconnect" in db) {
-      await (db as { $disconnect: () => Promise<void> }).$disconnect();
-    }
-    logger.info("Goodbye");
-    process.exit(0);
-  };
-
-  process.on("SIGINT", () => void shutdown());
-  process.on("SIGTERM", () => void shutdown());
+  setupGracefulShutdown({ app, db });

  // Keep process alive
  await new Promise(() => {});
--- a/bastion/src/labd/src/middleware/rate-limit.ts
+++ b/bastion/src/labd/src/middleware/rate-limit.ts
@@ -0,0 +1,50 @@
+// Rate limiting middleware for labd API.
+// Applies global rate limits and stricter limits for sensitive routes.
+
+import type { FastifyInstance } from "fastify";
+import rateLimit from "@fastify/rate-limit";
+import { logger } from "../services/logger.js";
+
+/** Routes that require stricter rate limiting. */
+const SENSITIVE_ROUTE_LIMITS: Record<string, number> = {
+  "/api/auth/enroll": 10,
+  "/api/tokens": 20,
+};
+
+/**
+ * Register the @fastify/rate-limit plugin with global defaults
+ * and apply stricter limits to sensitive routes.
+ */
+export async function setupRateLimiting(
+  app: FastifyInstance,
+): Promise<void> {
+  await app.register(rateLimit, {
+    max: 100,
+    timeWindow: "1 minute",
+    keyGenerator: (request) => request.ip,
+    errorResponseBuilder: (_request, context) => ({
+      error: "Too many requests",
+      code: "RATE_LIMITED",
+      retryAfter: context.after,
+    }),
+  });
+
+  // Apply stricter per-route limits for sensitive endpoints.
+  app.addHook("onRoute", (routeOptions) => {
+    const url = routeOptions.url;
+
+    for (const [prefix, max] of Object.entries(SENSITIVE_ROUTE_LIMITS)) {
+      if (url.startsWith(prefix)) {
+        routeOptions.config = {
+          ...routeOptions.config,
+          rateLimit: {
+            max,
+            timeWindow: "1 minute",
+          },
+        };
+        logger.info(`Rate limit: ${url} -> ${max} req/min`);
+        break;
+      }
+    }
+  });
+}
--- a/bastion/src/labd/src/routes/agents.ts
+++ b/bastion/src/labd/src/routes/agents.ts
@@ -0,0 +1,20 @@
+// Agent connection routes.
+// GET /api/agents — list currently connected agents (excludes raw socket)
+
+import type { FastifyInstance } from "fastify";
+import { agentRegistry } from "../services/agent-registry.js";
+
+export function registerAgentRoutes(app: FastifyInstance): void {
+  app.get("/api/agents", async (_request, reply) => {
+    const agents = agentRegistry.getAllConnected().map((agent) => ({
+      serverId: agent.serverId,
+      hostname: agent.hostname,
+      connectedAt: agent.connectedAt,
+      lastHeartbeat: agent.lastHeartbeat,
+      version: agent.version,
+      certFingerprint: agent.certFingerprint,
+    }));
+
+    return reply.send(agents);
+  });
+}
--- a/bastion/src/labd/src/routes/bastions.ts
+++ b/bastion/src/labd/src/routes/bastions.ts
@@ -0,0 +1,207 @@
+// Bastion management routes.
+// GET  /api/bastions          — list connected bastions
+// GET  /api/machines          — aggregated machines from all bastions
+// POST /api/machines/install  — queue install on correct bastion
+// DELETE /api/machines/:mac   — forget machine on correct bastion
+// POST /api/machines/role     — update role on correct bastion
+// GET  /api/machines/:mac/logs — get provision logs from correct bastion
+
+import type { FastifyInstance } from "fastify";
+import type { DbClient } from "../server.js";
+import { bastionRegistry } from "../services/bastion-registry.js";
+import { generateRequestId } from "@lab/shared";
+
+const COMMAND_TIMEOUT_MS = 15_000;
+
+/** Send a command to a bastion and wait for the response. */
+function sendCommand(
+  bastionId: string,
+  msg: Record<string, unknown>,
+): Promise<{ status: string; data?: unknown; error?: string | undefined }> {
+  const bastion = bastionRegistry.getById(bastionId);
+  if (!bastion) {
+    return Promise.reject(new Error(`Bastion ${bastionId} not connected`));
+  }
+
+  const requestId = generateRequestId();
+  const fullMsg = { ...msg, requestId };
+
+  return new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      cleanup();
+      reject(new Error("Command timed out"));
+    }, COMMAND_TIMEOUT_MS);
+
+    const handler = (data: Buffer) => {
+      try {
+        const parsed = JSON.parse(data.toString()) as { type: string; requestId?: string; status?: string; data?: unknown; error?: string };
+        if (parsed.type === "command-response" && parsed.requestId === requestId) {
+          cleanup();
+          resolve({ status: parsed.status ?? "ok", data: parsed.data, error: parsed.error });
+        }
+      } catch { /* not our message */ }
+    };
+
+    const cleanup = () => {
+      clearTimeout(timeout);
+      bastion.socket.off("message", handler);
+    };
+
+    bastion.socket.on("message", handler);
+    bastion.socket.send(JSON.stringify(fullMsg));
+  });
+}
+
+export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void {
+  // List all bastions (DB records enriched with online status from registry)
+  app.get("/api/bastions", async () => {
+    const dbBastions = await db.bastion.findMany() as Array<{
+      id: string; hostname: string; network: string; serverIp: string;
+      status: string; lastHeartbeat: Date | null; createdAt: Date;
+    }>;
+
+    return dbBastions.map((b) => {
+      const connected = bastionRegistry.getById(b.id);
+      return {
+        id: b.id,
+        hostname: b.hostname,
+        network: b.network,
+        serverIp: b.serverIp,
+        status: connected ? "online" : "offline",
+        lastHeartbeat: connected?.lastHeartbeat ?? b.lastHeartbeat,
+        connectedAt: connected?.connectedAt,
+        machineCount: connected
+          ? Object.keys(connected.state.discovered).length +
+            Object.keys(connected.state.install_queue).length +
+            Object.keys(connected.state.installed).length
+          : 0,
+        createdAt: b.createdAt,
+      };
+    });
+  });
+
+  // Aggregated machines from all connected bastions
+  app.get("/api/machines", async () => {
+    return bastionRegistry.getAggregatedState();
+  });
+
+  // Queue install — route to correct bastion by MAC
+  app.post<{
+    Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string };
+  }>("/api/machines/install", async (request, reply) => {
+    const { mac, hostname, disk, role, os } = request.body ?? {};
+    if (!mac || !hostname) {
+      return reply.code(400).send({ error: "mac and hostname are required" });
+    }
+
+    // Find bastion that knows this MAC, or let caller specify
+    const bastion = bastionRegistry.findBastionByMac(mac);
+    if (!bastion) {
+      // If only one bastion is connected, use it
+      const all = bastionRegistry.getAll();
+      if (all.length === 0) {
+        return reply.code(503).send({ error: "No bastions connected" });
+      }
+      if (all.length === 1) {
+        try {
+          const result = await sendCommand(all[0]!.bastionId, {
+            type: "command-install",
+            mac, hostname, disk: disk ?? "/dev/sda", role: role ?? "infra", os: os ?? "fedora-43",
+          });
+          return reply.code(result.status === "ok" ? 200 : 500).send(result);
+        } catch (err) {
+          return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
+        }
+      }
+      return reply.code(404).send({ error: `MAC ${mac} not found on any bastion` });
+    }
+
+    try {
+      const result = await sendCommand(bastion.bastionId, {
+        type: "command-install",
+        mac, hostname, disk: disk ?? "/dev/sda", role: role ?? "infra", os: os ?? "fedora-43",
+      });
+      return reply.code(result.status === "ok" ? 200 : 500).send(result);
+    } catch (err) {
+      return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
+    }
+  });
+
+  // Forget machine
+  app.delete<{ Params: { mac: string } }>("/api/machines/:mac", async (request, reply) => {
+    const mac = request.params.mac.toLowerCase().replace(/-/g, ":");
+    const bastion = bastionRegistry.findBastionByMac(mac);
+    if (!bastion) {
+      return reply.code(404).send({ error: `MAC ${mac} not found on any bastion` });
+    }
+
+    try {
+      const result = await sendCommand(bastion.bastionId, { type: "command-forget", mac });
+      return reply.send(result);
+    } catch (err) {
+      return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
+    }
+  });
+
+  // Update role
+  app.post<{
+    Body: { mac?: string; role?: string };
+  }>("/api/machines/role", async (request, reply) => {
+    const { mac, role } = request.body ?? {};
+    if (!mac || !role) {
+      return reply.code(400).send({ error: "mac and role are required" });
+    }
+
+    const normalized = mac.toLowerCase().replace(/-/g, ":");
+    const bastion = bastionRegistry.findBastionByMac(normalized);
+    if (!bastion) {
+      return reply.code(404).send({ error: `MAC ${normalized} not found on any bastion` });
+    }
+
+    try {
+      const result = await sendCommand(bastion.bastionId, { type: "command-role-update", mac: normalized, role });
+      return reply.send(result);
+    } catch (err) {
+      return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
+    }
+  });
+
+  // Machine logs (snapshot from bastion's state)
+  app.get<{ Params: { mac: string } }>("/api/machines/:mac/logs", async (request, reply) => {
+    const mac = request.params.mac.toLowerCase().replace(/-/g, ":");
+    const bastion = bastionRegistry.findBastionByMac(mac);
+    if (!bastion) {
+      return reply.code(404).send({ error: `MAC ${mac} not found` });
+    }
+
+    const queued = bastion.state.install_queue[mac];
+    const installed = bastion.state.installed[mac];
+
+    if (installed) {
+      return {
+        mac,
+        hostname: installed.hostname,
+        status: "installed",
+        role: installed.role,
+        ip: installed.ip,
+        installed_at: installed.installed_at,
+      };
+    }
+
+    if (queued) {
+      return {
+        mac,
+        hostname: queued.hostname,
+        status: queued.progress ? "installing" : "queued",
+        progress: queued.progress,
+        progress_detail: queued.progress_detail,
+        progress_at: queued.progress_at,
+        role: queued.role,
+        os: queued.os,
+        log: queued.log,
+      };
+    }
+
+    return reply.code(404).send({ error: `MAC ${mac} not found in install queue or installed` });
+  });
+}
--- a/bastion/src/labd/src/routes/health.ts
+++ b/bastion/src/labd/src/routes/health.ts
@@ -1,10 +1,85 @@
 // Health check routes.

 import type { FastifyInstance } from "fastify";
+import { performance } from "node:perf_hooks";
 import type { DbClient } from "../server.js";
+import { agentRegistry } from "../services/agent-registry.js";
+import { isShuttingDownNow } from "../services/shutdown.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface ComponentStatus {
+  status: "up" | "down" | "degraded";
+  latency?: number;
+  message?: string;
+}
+
+export interface HealthStatus {
+  status: "healthy" | "degraded" | "unhealthy";
+  version: string;
+  uptime: number;
+  timestamp: string;
+  components: {
+    database: ComponentStatus;
+    agents: ComponentStatus;
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+async function checkDatabase(db: DbClient): Promise<ComponentStatus> {
+  const start = performance.now();
+  try {
+    await db.$queryRaw`SELECT 1`;
+    const latency = Math.round((performance.now() - start) * 100) / 100;
+    return { status: "up", latency };
+  } catch (err) {
+    const latency = Math.round((performance.now() - start) * 100) / 100;
+    const message = err instanceof Error ? err.message : "Unknown error";
+    return { status: "down", latency, message };
+  }
+}
+
+function checkAgents(): ComponentStatus {
+  const count = agentRegistry.getConnectedCount();
+  return {
+    status: count > 0 ? "up" : "degraded",
+    message: `${count} agent(s) connected`,
+  };
+}
+
+function aggregateStatus(
+  components: HealthStatus["components"],
+): { status: HealthStatus["status"]; statusCode: number } {
+  const statuses = Object.values(components);
+  if (statuses.some((c) => c.status === "down")) {
+    return { status: "unhealthy", statusCode: 503 };
+  }
+  if (statuses.some((c) => c.status === "degraded")) {
+    return { status: "degraded", statusCode: 200 };
+  }
+  return { status: "healthy", statusCode: 200 };
+}
+
+// ---------------------------------------------------------------------------
+// Route registration
+// ---------------------------------------------------------------------------

 export function registerHealthRoutes(app: FastifyInstance, db: DbClient): void {
+  // ---- existing /healthz (preserved for backward compat) ------------------
  app.get("/healthz", async (_request, reply) => {
+    if (isShuttingDownNow()) {
+      return reply.code(503).send({
+        status: "shutting_down",
+        uptime: process.uptime(),
+        timestamp: new Date().toISOString(),
+      });
+    }
+
    let dbOk = false;
    try {
      await db.$queryRaw`SELECT 1`;
@@ -25,4 +100,45 @@ export function registerHealthRoutes(app: FastifyInstance, db: DbClient): void {
      },
    });
  });
+
+  // ---- GET /health — simple probe for k8s --------------------------------
+  app.get("/health", async (_request, reply) => {
+    return reply.code(200).send({ status: "ok" });
+  });
+
+  // ---- GET /health/detailed — full component check -----------------------
+  app.get("/health/detailed", async (_request, reply) => {
+    const database = await checkDatabase(db);
+    const agents = checkAgents();
+
+    const components = { database, agents };
+    const { status, statusCode } = aggregateStatus(components);
+
+    const body: HealthStatus = {
+      status,
+      version: process.env["LABD_VERSION"] ?? "0.1.0",
+      uptime: process.uptime(),
+      timestamp: new Date().toISOString(),
+      components,
+    };
+
+    return reply.code(statusCode).send(body);
+  });
+
+  // ---- GET /health/live — liveness probe ---------------------------------
+  app.get("/health/live", async (_request, reply) => {
+    return reply.code(200).send({ status: "alive" });
+  });
+
+  // ---- GET /health/ready — readiness probe (needs DB) --------------------
+  app.get("/health/ready", async (_request, reply) => {
+    const database = await checkDatabase(db);
+    if (database.status === "down") {
+      return reply.code(503).send({
+        status: "not_ready",
+        reason: database.message,
+      });
+    }
+    return reply.code(200).send({ status: "ready" });
+  });
 }
--- a/bastion/src/labd/src/server.ts
+++ b/bastion/src/labd/src/server.ts
@@ -7,28 +7,43 @@ import { logger } from "./services/logger.js";
 import { registerHealthRoutes } from "./routes/health.js";
 import { registerServerRoutes } from "./routes/servers.js";
 import { registerAuthRoutes } from "./routes/auth.js";
+import { registerAgentRoutes } from "./routes/agents.js";
+import { registerBastionRoutes } from "./routes/bastions.js";
+import { setupRateLimiting } from "./middleware/rate-limit.js";
+import { bastionRegistry } from "./services/bastion-registry.js";
+import { isBastionMessage } from "@lab/shared";

 export interface DbClient {
-  $queryRaw: (query: TemplateStringsArray) => Promise<unknown>;
+  $queryRaw: (...args: unknown[]) => Promise<unknown>;
+  $disconnect?: () => Promise<void>;
  server: {
-    findMany: (args?: unknown) => Promise<unknown[]>;
-    findUnique: (args: unknown) => Promise<unknown>;
+    findMany: (...args: unknown[]) => Promise<unknown[]>;
+    findUnique: (...args: unknown[]) => Promise<unknown>;
  };
  joinToken: {
-    findUnique: (args: unknown) => Promise<unknown>;
-    findMany: (args?: unknown) => Promise<unknown[]>;
-    create: (args: unknown) => Promise<unknown>;
-    update: (args: unknown) => Promise<unknown>;
+    findUnique: (...args: unknown[]) => Promise<unknown>;
+    findMany: (...args: unknown[]) => Promise<unknown[]>;
+    create: (...args: unknown[]) => Promise<unknown>;
+    update: (...args: unknown[]) => Promise<unknown>;
+  };
+  bastion: {
+    upsert: (...args: unknown[]) => Promise<unknown>;
+    findMany: (...args: unknown[]) => Promise<unknown[]>;
+    findUnique: (...args: unknown[]) => Promise<unknown>;
+    update: (...args: unknown[]) => Promise<unknown>;
  };
 }

-export function createApp(_config: LabdConfig, db: DbClient): {
+export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
  app: ReturnType<typeof Fastify>;
-} {
+}> {
  const app = Fastify({
    logger: false, // We use winston instead
  });

+  // Register rate limiting before routes
+  await setupRateLimiting(app);
+
  // Register WebSocket support
  void app.register(websocket);

@@ -36,6 +51,8 @@ export function createApp(_config: LabdConfig, db: DbClient): {
  registerHealthRoutes(app, db);
  registerServerRoutes(app, db);
  registerAuthRoutes(app, db);
+  registerAgentRoutes(app);
+  registerBastionRoutes(app, db);

  // WebSocket handler for agent connections
  app.register(async (fastify) => {
@@ -54,6 +71,148 @@ export function createApp(_config: LabdConfig, db: DbClient): {
    });
  });

+  // WebSocket handler for bastion connections
+  app.register(async (fastify) => {
+    fastify.get("/ws/bastion", { websocket: true }, (socket, _request) => {
+      let bastionId: string | null = null;
+
+      logger.info("Bastion WebSocket connection established");
+
+      socket.on("message", (data: Buffer) => {
+        try {
+          const raw = data.toString();
+          const msg: unknown = JSON.parse(raw);
+
+          if (!isBastionMessage(msg)) {
+            logger.warn(`Unknown bastion message: ${(msg as { type?: string }).type}`);
+            return;
+          }
+
+          switch (msg.type) {
+            case "bastion-enroll": {
+              // Validate the join token
+              void (async () => {
+                try {
+                  const joinToken = await db.joinToken.findUnique({ where: { token: msg.token } }) as {
+                    id: string; type: string; usedBy: string | null; revokedAt: Date | null; expiresAt: Date | null;
+                  } | null;
+
+                  if (!joinToken || joinToken.revokedAt !== null) {
+                    logger.warn(`Bastion enrollment rejected: invalid/revoked token from ${msg.hostname}`);
+                    socket.send(JSON.stringify({ type: "error", error: "Invalid or revoked token" }));
+                    socket.close();
+                    return;
+                  }
+                  if (joinToken.expiresAt !== null && joinToken.expiresAt < new Date()) {
+                    logger.warn(`Bastion enrollment rejected: expired token from ${msg.hostname}`);
+                    socket.send(JSON.stringify({ type: "error", error: "Token expired" }));
+                    socket.close();
+                    return;
+                  }
+                  if (joinToken.type === "one-time" && joinToken.usedBy !== null) {
+                    logger.warn(`Bastion enrollment rejected: already-used token from ${msg.hostname}`);
+                    socket.send(JSON.stringify({ type: "error", error: "Token already used" }));
+                    socket.close();
+                    return;
+                  }
+
+                  // Mark token as used
+                  await db.joinToken.update({
+                    where: { id: joinToken.id },
+                    data: { usedBy: `bastion:${msg.hostname}`, usedAt: new Date() },
+                  });
+
+                  // Upsert bastion record
+                  const record = await db.bastion.upsert({
+                    where: { hostname: msg.hostname },
+                    create: { hostname: msg.hostname, network: msg.network, serverIp: msg.serverIp, status: "online" },
+                    update: { network: msg.network, serverIp: msg.serverIp, status: "online", lastHeartbeat: new Date() },
+                  }) as { id: string };
+
+                  bastionId = record.id;
+
+                  bastionRegistry.register({
+                    bastionId: record.id,
+                    hostname: msg.hostname,
+                    network: msg.network,
+                    serverIp: msg.serverIp,
+                    socket,
+                    connectedAt: new Date(),
+                    lastHeartbeat: new Date(),
+                    state: { discovered: {}, install_queue: {}, installed: {} },
+                  });
+
+                  socket.send(JSON.stringify({ type: "bastion-enrolled", bastionId: record.id }));
+                  logger.info(`BASTION ENROLLED: ${msg.hostname} (${msg.network}) as ${record.id.slice(0, 8)}...`);
+                } catch (err) {
+                  logger.error(`Bastion enrollment error: ${err instanceof Error ? err.message : String(err)}`);
+                  socket.close();
+                }
+              })();
+              break;
+            }
+
+            case "bastion-state-sync": {
+              if (!bastionId && msg.bastionId) {
+                // Reconnection with known bastionId — re-register
+                bastionId = msg.bastionId;
+                const existing = bastionRegistry.getById(bastionId);
+                if (!existing) {
+                  bastionRegistry.register({
+                    bastionId,
+                    hostname: "reconnecting",
+                    network: "",
+                    serverIp: "",
+                    socket,
+                    connectedAt: new Date(),
+                    lastHeartbeat: new Date(),
+                    state: msg.state,
+                  });
+                  // Update DB status
+                  void db.bastion.update({ where: { id: bastionId }, data: { status: "online", lastHeartbeat: new Date() } });
+                }
+              }
+              if (bastionId) {
+                bastionRegistry.updateState(bastionId, msg.state);
+                logger.info(`Bastion ${bastionId.slice(0, 8)} state sync: ${Object.keys(msg.state.discovered).length} discovered, ${Object.keys(msg.state.installed).length} installed`);
+              }
+              break;
+            }
+
+            case "bastion-heartbeat": {
+              if (bastionId) {
+                bastionRegistry.updateHeartbeat(bastionId);
+                socket.send(JSON.stringify({ type: "bastion-heartbeat-ack", serverTime: new Date().toISOString() }));
+              }
+              break;
+            }
+
+            case "bastion-progress": {
+              // Forward to any SSE subscribers (future)
+              logger.info(`Bastion progress: ${msg.mac} -> ${msg.stage}: ${msg.detail}`);
+              break;
+            }
+
+            case "command-response": {
+              // Handled by the pending command listener in bastions.ts routes
+              break;
+            }
+          }
+        } catch (err) {
+          logger.error(`Failed to parse bastion message: ${err instanceof Error ? err.message : String(err)}`);
+        }
+      });
+
+      socket.on("close", () => {
+        if (bastionId) {
+          logger.info(`Bastion ${bastionId.slice(0, 8)} disconnected`);
+          bastionRegistry.unregister(bastionId);
+          void db.bastion.update({ where: { id: bastionId }, data: { status: "offline" } }).catch(() => {});
+        }
+      });
+    });
+  });
+
  // Log all requests
  app.addHook("onRequest", async (request) => {
    logger.info(`HTTP: ${request.ip} ${request.method} ${request.url}`);
--- a/bastion/src/labd/src/services/agent-registry.ts
+++ b/bastion/src/labd/src/services/agent-registry.ts
@@ -0,0 +1,65 @@
+// In-memory registry of connected lab-agent WebSocket connections.
+// Tracks agents by serverId and hostname, emits lifecycle events.
+
+import { EventEmitter } from "node:events";
+import type { WebSocket } from "ws";
+
+export interface ConnectedAgent {
+  serverId: string;
+  hostname: string;
+  socket: WebSocket;
+  connectedAt: Date;
+  lastHeartbeat: Date;
+  version: string;
+  certFingerprint: string;
+}
+
+export class AgentRegistry extends EventEmitter {
+  private agents: Map<string, ConnectedAgent> = new Map();
+  private byHostname: Map<string, string> = new Map();
+
+  register(agent: ConnectedAgent): void {
+    this.agents.set(agent.serverId, agent);
+    this.byHostname.set(agent.hostname, agent.serverId);
+    this.emit("agent:connected", agent);
+  }
+
+  unregister(serverId: string): void {
+    const agent = this.agents.get(serverId);
+    if (agent !== undefined) {
+      this.byHostname.delete(agent.hostname);
+      this.agents.delete(serverId);
+      this.emit("agent:disconnected", agent);
+    }
+  }
+
+  getByServerId(serverId: string): ConnectedAgent | undefined {
+    return this.agents.get(serverId);
+  }
+
+  getByHostname(hostname: string): ConnectedAgent | undefined {
+    const serverId = this.byHostname.get(hostname);
+    if (serverId === undefined) {
+      return undefined;
+    }
+    return this.agents.get(serverId);
+  }
+
+  updateHeartbeat(serverId: string): void {
+    const agent = this.agents.get(serverId);
+    if (agent !== undefined) {
+      agent.lastHeartbeat = new Date();
+      this.emit("agent:heartbeat", agent);
+    }
+  }
+
+  getConnectedCount(): number {
+    return this.agents.size;
+  }
+
+  getAllConnected(): ConnectedAgent[] {
+    return [...this.agents.values()];
+  }
+}
+
+export const agentRegistry = new AgentRegistry();
--- a/bastion/src/labd/src/services/bastion-registry.ts
+++ b/bastion/src/labd/src/services/bastion-registry.ts
@@ -0,0 +1,107 @@
+// In-memory registry of connected bastion WebSocket connections.
+// Tracks bastions by ID, caches their BastionState, and provides aggregation.
+
+import { EventEmitter } from "node:events";
+import type { WebSocket } from "ws";
+import type { BastionState, HardwareInfo, InstallConfig, InstalledInfo } from "@lab/shared";
+
+export interface ConnectedBastion {
+  bastionId: string;
+  hostname: string;
+  network: string;
+  serverIp: string;
+  socket: WebSocket;
+  connectedAt: Date;
+  lastHeartbeat: Date;
+  state: BastionState;
+}
+
+export interface AggregatedState {
+  discovered: Record<string, HardwareInfo>;
+  install_queue: Record<string, InstallConfig>;
+  installed: Record<string, InstalledInfo>;
+}
+
+export class BastionRegistry extends EventEmitter {
+  private bastions = new Map<string, ConnectedBastion>();
+
+  register(bastion: ConnectedBastion): void {
+    this.bastions.set(bastion.bastionId, bastion);
+    this.emit("bastion:connected", bastion);
+  }
+
+  unregister(bastionId: string): void {
+    const bastion = this.bastions.get(bastionId);
+    if (bastion) {
+      this.bastions.delete(bastionId);
+      this.emit("bastion:disconnected", bastion);
+    }
+  }
+
+  getById(bastionId: string): ConnectedBastion | undefined {
+    return this.bastions.get(bastionId);
+  }
+
+  getAll(): ConnectedBastion[] {
+    return [...this.bastions.values()];
+  }
+
+  getConnectedCount(): number {
+    return this.bastions.size;
+  }
+
+  updateState(bastionId: string, state: BastionState): void {
+    const bastion = this.bastions.get(bastionId);
+    if (bastion) {
+      bastion.state = state;
+      this.emit("bastion:state-updated", bastion);
+    }
+  }
+
+  updateHeartbeat(bastionId: string): void {
+    const bastion = this.bastions.get(bastionId);
+    if (bastion) {
+      bastion.lastHeartbeat = new Date();
+    }
+  }
+
+  /** Find which bastion owns a given MAC address. */
+  findBastionByMac(mac: string): ConnectedBastion | undefined {
+    const normalized = mac.toLowerCase();
+    for (const bastion of this.bastions.values()) {
+      if (
+        normalized in bastion.state.discovered ||
+        normalized in bastion.state.install_queue ||
+        normalized in bastion.state.installed
+      ) {
+        return bastion;
+      }
+    }
+    return undefined;
+  }
+
+  /** Merge all bastion states into one, tagging each entry with bastionId. */
+  getAggregatedState(): AggregatedState {
+    const result: AggregatedState = {
+      discovered: {},
+      install_queue: {},
+      installed: {},
+    };
+
+    for (const bastion of this.bastions.values()) {
+      for (const [mac, hw] of Object.entries(bastion.state.discovered)) {
+        result.discovered[mac] = { ...hw, bastionId: bastion.bastionId };
+      }
+      for (const [mac, cfg] of Object.entries(bastion.state.install_queue)) {
+        result.install_queue[mac] = { ...cfg, bastionId: bastion.bastionId };
+      }
+      for (const [mac, info] of Object.entries(bastion.state.installed)) {
+        result.installed[mac] = { ...info, bastionId: bastion.bastionId };
+      }
+    }
+
+    return result;
+  }
+}
+
+export const bastionRegistry = new BastionRegistry();
--- a/bastion/src/labd/src/services/encryption.ts
+++ b/bastion/src/labd/src/services/encryption.ts
@@ -0,0 +1,73 @@
+// Encryption service for sensitive data (CA keys, kubeconfigs).
+// Uses AES-256-GCM with scrypt-derived keys.
+
+import {
+  createCipheriv,
+  createDecipheriv,
+  randomBytes,
+  scryptSync,
+} from "node:crypto";
+
+const ALGORITHM = "aes-256-gcm";
+const KEY_LENGTH = 32;
+const IV_LENGTH = 16;
+const SALT = "labctl-salt";
+
+export class EncryptionService {
+  private key: Buffer;
+
+  constructor(masterKey: string) {
+    this.key = scryptSync(masterKey, SALT, KEY_LENGTH);
+  }
+
+  encrypt(plaintext: string): string {
+    const iv = randomBytes(IV_LENGTH);
+    const cipher = createCipheriv(ALGORITHM, this.key, iv);
+
+    let encrypted = cipher.update(plaintext, "utf8", "base64");
+    encrypted += cipher.final("base64");
+
+    const authTag = cipher.getAuthTag();
+
+    // Format: iv:authTag:encrypted (all base64)
+    return [
+      iv.toString("base64"),
+      authTag.toString("base64"),
+      encrypted,
+    ].join(":");
+  }
+
+  decrypt(ciphertext: string): string {
+    const parts = ciphertext.split(":");
+    if (parts.length !== 3) {
+      throw new Error("Invalid ciphertext format: expected iv:authTag:encrypted");
+    }
+
+    const [ivB64, tagB64, encrypted] = parts as [string, string, string];
+    const iv = Buffer.from(ivB64, "base64");
+    const authTag = Buffer.from(tagB64, "base64");
+
+    const decipher = createDecipheriv(ALGORITHM, this.key, iv);
+    decipher.setAuthTag(authTag);
+
+    let decrypted = decipher.update(encrypted, "base64", "utf8");
+    decrypted += decipher.final("utf8");
+
+    return decrypted;
+  }
+}
+
+// --- Singleton ---
+
+let _instance: EncryptionService | undefined;
+
+export function getEncryptionService(): EncryptionService {
+  if (!_instance) {
+    const masterKey = process.env["CA_ENCRYPTION_KEY"];
+    if (!masterKey || masterKey.length < 32) {
+      throw new Error("CA_ENCRYPTION_KEY must be set and at least 32 characters");
+    }
+    _instance = new EncryptionService(masterKey);
+  }
+  return _instance;
+}
--- a/bastion/src/labd/src/services/message-router.ts
+++ b/bastion/src/labd/src/services/message-router.ts
@@ -0,0 +1,192 @@
+// WebSocket message routing between labd and connected agents.
+// Dispatches incoming agent messages to handlers, manages pending requests.
+
+import type { AgentMessage, ServerMessage, JournalOptions } from "@lab/shared";
+import { generateRequestId } from "@lab/shared";
+import type { ConnectedAgent } from "./agent-registry.js";
+
+export type MessageHandler = (
+  agent: ConnectedAgent,
+  message: AgentMessage,
+) => Promise<void>;
+
+export interface PendingRequest {
+  requestId: string;
+  serverId: string;
+  type: "exec" | "log";
+  resolve: (result: unknown) => void;
+  reject: (error: Error) => void;
+  timeout: NodeJS.Timeout;
+  onData?: (chunk: string) => void;
+}
+
+export class MessageRouter {
+  private pendingRequests: Map<string, PendingRequest> = new Map();
+  private handlers: Map<string, MessageHandler> = new Map();
+
+  registerHandler(type: string, handler: MessageHandler): void {
+    this.handlers.set(type, handler);
+  }
+
+  async handleMessage(
+    agent: ConnectedAgent,
+    message: AgentMessage,
+  ): Promise<void> {
+    // Dispatch to registered handler
+    const handler = this.handlers.get(message.type);
+    if (handler) {
+      await handler(agent, message);
+    }
+
+    // Handle responses to pending requests
+    if ("requestId" in message) {
+      const pending = this.pendingRequests.get(message.requestId);
+      if (!pending) return;
+
+      switch (message.type) {
+        case "exec-exit":
+          clearTimeout(pending.timeout);
+          pending.resolve({ exitCode: message.exitCode });
+          this.pendingRequests.delete(message.requestId);
+          break;
+        case "exec-stdout":
+        case "exec-stderr":
+          pending.onData?.(message.data);
+          break;
+        case "log-line":
+          pending.onData?.(message.line);
+          break;
+        case "log-end":
+          clearTimeout(pending.timeout);
+          pending.resolve({ completed: true });
+          this.pendingRequests.delete(message.requestId);
+          break;
+      }
+    }
+  }
+
+  async sendRequest(
+    agent: ConnectedAgent,
+    message: ServerMessage,
+    timeoutMs: number = 30_000,
+  ): Promise<unknown> {
+    return new Promise((resolve, reject) => {
+      const requestId =
+        "requestId" in message ? message.requestId : generateRequestId();
+
+      const timeoutHandle = setTimeout(() => {
+        this.pendingRequests.delete(requestId);
+        reject(new Error(`Request ${requestId} timed out after ${timeoutMs}ms`));
+      }, timeoutMs);
+
+      this.pendingRequests.set(requestId, {
+        requestId,
+        serverId: agent.serverId,
+        type: message.type === "log-subscribe" ? "log" : "exec",
+        resolve,
+        reject,
+        timeout: timeoutHandle,
+      });
+
+      agent.socket.send(JSON.stringify(message));
+    });
+  }
+
+  async sendRequestWithStreaming(
+    agent: ConnectedAgent,
+    message: ServerMessage,
+    onData: (chunk: string) => void,
+    timeoutMs: number = 30_000,
+  ): Promise<unknown> {
+    return new Promise((resolve, reject) => {
+      const requestId =
+        "requestId" in message ? message.requestId : generateRequestId();
+
+      const timeoutHandle = setTimeout(() => {
+        this.pendingRequests.delete(requestId);
+        reject(new Error(`Request ${requestId} timed out after ${timeoutMs}ms`));
+      }, timeoutMs);
+
+      this.pendingRequests.set(requestId, {
+        requestId,
+        serverId: agent.serverId,
+        type: message.type === "log-subscribe" ? "log" : "exec",
+        resolve,
+        reject,
+        timeout: timeoutHandle,
+        onData,
+      });
+
+      agent.socket.send(JSON.stringify(message));
+    });
+  }
+
+  subscribeToLogs(
+    agent: ConnectedAgent,
+    options: JournalOptions,
+    onLine: (line: string) => void,
+  ): { requestId: string; unsubscribe: () => void } {
+    const requestId = generateRequestId();
+
+    const timeoutHandle = setTimeout(() => {
+      // Log subscriptions don't time out by default, but we set a long one
+    }, 24 * 60 * 60 * 1000); // 24h max
+
+    this.pendingRequests.set(requestId, {
+      requestId,
+      serverId: agent.serverId,
+      type: "log",
+      resolve: () => {},
+      reject: () => {},
+      timeout: timeoutHandle,
+      onData: onLine,
+    });
+
+    agent.socket.send(
+      JSON.stringify({
+        type: "log-subscribe",
+        requestId,
+        options,
+      } satisfies ServerMessage),
+    );
+
+    return {
+      requestId,
+      unsubscribe: () => {
+        clearTimeout(timeoutHandle);
+        this.pendingRequests.delete(requestId);
+        agent.socket.send(
+          JSON.stringify({
+            type: "log-unsubscribe",
+            requestId,
+          } satisfies ServerMessage),
+        );
+      },
+    };
+  }
+
+  cancelRequest(requestId: string): boolean {
+    const pending = this.pendingRequests.get(requestId);
+    if (!pending) return false;
+    clearTimeout(pending.timeout);
+    pending.reject(new Error("Request cancelled"));
+    this.pendingRequests.delete(requestId);
+    return true;
+  }
+
+  cleanupAgent(serverId: string): void {
+    for (const [id, pending] of this.pendingRequests) {
+      if (pending.serverId === serverId) {
+        clearTimeout(pending.timeout);
+        pending.reject(new Error("Agent disconnected"));
+        this.pendingRequests.delete(id);
+      }
+    }
+  }
+
+  getPendingCount(): number {
+    return this.pendingRequests.size;
+  }
+}
+
+export const messageRouter = new MessageRouter();
--- a/bastion/src/labd/src/services/shutdown.ts
+++ b/bastion/src/labd/src/services/shutdown.ts
@@ -0,0 +1,98 @@
+// Graceful shutdown handling for labd.
+// Registers SIGTERM/SIGINT handlers, drains connections, and exits cleanly.
+
+import type { FastifyInstance } from "fastify";
+import { logger } from "./logger.js";
+import { agentRegistry } from "./agent-registry.js";
+import { messageRouter } from "./message-router.js";
+
+const FORCE_EXIT_TIMEOUT_MS = 15_000;
+
+let isShuttingDown = false;
+
+/**
+ * Returns true if the server is in the process of shutting down.
+ */
+export function isShuttingDownNow(): boolean {
+  return isShuttingDown;
+}
+
+export interface ShutdownResources {
+  app: FastifyInstance;
+  db?: { $disconnect?: () => Promise<void> };
+}
+
+/**
+ * Registers SIGTERM and SIGINT handlers for graceful shutdown.
+ * Idempotent: a second signal while shutdown is in progress is ignored.
+ */
+export function setupGracefulShutdown(resources: ShutdownResources): void {
+  const { app, db } = resources;
+
+  const shutdown = async (signal: string): Promise<void> => {
+    if (isShuttingDown) {
+      logger.info(`Received ${signal} again — shutdown already in progress, ignoring`);
+      return;
+    }
+
+    isShuttingDown = true;
+    logger.info(`Received ${signal}, starting graceful shutdown...`);
+
+    // Safety net: force exit after timeout
+    const forceExitTimer = setTimeout(() => {
+      logger.error("Graceful shutdown timed out — forcing exit");
+      process.exit(1);
+    }, FORCE_EXIT_TIMEOUT_MS);
+    forceExitTimer.unref();
+
+    try {
+      // 1. Stop accepting new connections
+      await app.close();
+      logger.info("HTTP server closed");
+
+      // 2. Notify connected agents and close their sockets
+      const agents = agentRegistry.getAllConnected();
+      if (agents.length > 0) {
+        logger.info(`Notifying ${agents.length} connected agent(s) of shutdown`);
+        for (const agent of agents) {
+          try {
+            agent.socket.send(
+              JSON.stringify({
+                type: "server-shutdown",
+                reconnectAfter: 5000,
+              }),
+            );
+            agent.socket.close();
+          } catch {
+            // Agent may already be disconnected
+          }
+        }
+      }
+
+      // 3. Clean up pending message-router requests for each agent
+      const pendingCount = messageRouter.getPendingCount();
+      if (pendingCount > 0) {
+        logger.info(`Cleaning up ${pendingCount} pending request(s)`);
+        for (const agent of agents) {
+          messageRouter.cleanupAgent(agent.serverId);
+        }
+      }
+
+      // 4. Disconnect database
+      if (db?.$disconnect) {
+        await db.$disconnect();
+        logger.info("Database disconnected");
+      }
+
+      logger.info("Graceful shutdown complete — goodbye");
+      process.exit(0);
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      logger.error(`Error during shutdown: ${message}`);
+      process.exit(1);
+    }
+  };
+
+  process.on("SIGTERM", () => void shutdown("SIGTERM"));
+  process.on("SIGINT", () => void shutdown("SIGINT"));
+}
--- a/bastion/src/labd/src/validation/index.ts
+++ b/bastion/src/labd/src/validation/index.ts
@@ -0,0 +1,13 @@
+export {
+  createTokenSchema,
+  enrollmentSchema,
+  serverFiltersSchema,
+  permissionPatternSchema,
+  createRoleSchema,
+  type CreateTokenInput,
+  type EnrollmentInput,
+  type ServerFiltersInput,
+  type CreateRoleInput,
+} from "./schemas.js";
+
+export { validateBody, validateQuery } from "./middleware.js";
--- a/bastion/src/labd/src/validation/middleware.ts
+++ b/bastion/src/labd/src/validation/middleware.ts
@@ -0,0 +1,32 @@
+// Fastify validation middleware using Zod schemas.
+
+import type { FastifyRequest, FastifyReply } from "fastify";
+import type { ZodSchema } from "zod";
+
+export function validateBody<T>(schema: ZodSchema<T>) {
+  return async (request: FastifyRequest, reply: FastifyReply): Promise<void> => {
+    const result = schema.safeParse(request.body);
+    if (!result.success) {
+      void reply.status(400).send({
+        error: "Validation failed",
+        details: result.error.flatten(),
+      });
+      return;
+    }
+    (request as unknown as { body: T }).body = result.data;
+  };
+}
+
+export function validateQuery<T>(schema: ZodSchema<T>) {
+  return async (request: FastifyRequest, reply: FastifyReply): Promise<void> => {
+    const result = schema.safeParse(request.query);
+    if (!result.success) {
+      void reply.status(400).send({
+        error: "Validation failed",
+        details: result.error.flatten(),
+      });
+      return;
+    }
+    (request as unknown as { query: T }).query = result.data;
+  };
+}
--- a/bastion/src/labd/src/validation/schemas.ts
+++ b/bastion/src/labd/src/validation/schemas.ts
@@ -0,0 +1,37 @@
+// Zod validation schemas for labd API requests.
+
+import { z } from "zod";
+
+export const createTokenSchema = z.object({
+  type: z.enum(["one-time", "reusable"]).default("one-time"),
+  label: z.string().max(255).optional(),
+  expiresInHours: z.number().positive().max(8760).optional(), // Max 1 year
+});
+export type CreateTokenInput = z.infer<typeof createTokenSchema>;
+
+export const enrollmentSchema = z.object({
+  token: z.string().min(1, "token is required"),
+  hostname: z.string().regex(/^[a-z0-9][a-z0-9.-]*$/i, "Invalid hostname format").max(253),
+  csr: z.string().optional(),
+});
+export type EnrollmentInput = z.infer<typeof enrollmentSchema>;
+
+export const serverFiltersSchema = z.object({
+  cloud: z.string().optional(),
+  environment: z.string().optional(),
+  status: z.enum(["online", "offline", "provisioning", "unknown"]).optional(),
+});
+export type ServerFiltersInput = z.infer<typeof serverFiltersSchema>;
+
+export const permissionPatternSchema = z.string().regex(
+  /^[a-z*]+:[a-z*]+:[a-z*]+:[a-z0-9.*-]+$/,
+  "Permission must match pattern action:cloud:environment:server",
+);
+
+export const createRoleSchema = z.object({
+  name: z.string().min(1).max(64).regex(/^[a-z][a-z0-9-]*$/, "Role name must be lowercase alphanumeric with hyphens"),
+  description: z.string().max(500).optional(),
+  allow: z.array(permissionPatternSchema).optional(),
+  deny: z.array(permissionPatternSchema).optional(),
+});
+export type CreateRoleInput = z.infer<typeof createRoleSchema>;
--- a/bastion/src/labd/tests/agent-registry.test.ts
+++ b/bastion/src/labd/tests/agent-registry.test.ts
@@ -0,0 +1,112 @@
+// Tests for AgentRegistry.
+
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import { AgentRegistry, type ConnectedAgent } from "../src/services/agent-registry.js";
+
+function makeAgent(overrides: Partial<ConnectedAgent> = {}): ConnectedAgent {
+  return {
+    serverId: "srv-1",
+    hostname: "worker-1",
+    socket: { send: vi.fn(), close: vi.fn(), readyState: 1 } as unknown as ConnectedAgent["socket"],
+    connectedAt: new Date(),
+    lastHeartbeat: new Date(),
+    version: "0.1.0",
+    certFingerprint: "abc123",
+    ...overrides,
+  };
+}
+
+describe("AgentRegistry", () => {
+  let registry: AgentRegistry;
+
+  beforeEach(() => {
+    registry = new AgentRegistry();
+  });
+
+  it("starts empty", () => {
+    expect(registry.getConnectedCount()).toBe(0);
+    expect(registry.getAllConnected()).toEqual([]);
+  });
+
+  it("registers and retrieves by serverId", () => {
+    const agent = makeAgent();
+    registry.register(agent);
+    expect(registry.getByServerId("srv-1")).toBe(agent);
+    expect(registry.getConnectedCount()).toBe(1);
+  });
+
+  it("retrieves by hostname", () => {
+    const agent = makeAgent({ hostname: "web-1" });
+    registry.register(agent);
+    expect(registry.getByHostname("web-1")).toBe(agent);
+  });
+
+  it("returns undefined for unknown serverId", () => {
+    expect(registry.getByServerId("nope")).toBeUndefined();
+  });
+
+  it("returns undefined for unknown hostname", () => {
+    expect(registry.getByHostname("nope")).toBeUndefined();
+  });
+
+  it("unregisters agent", () => {
+    const agent = makeAgent();
+    registry.register(agent);
+    registry.unregister("srv-1");
+    expect(registry.getByServerId("srv-1")).toBeUndefined();
+    expect(registry.getByHostname("worker-1")).toBeUndefined();
+    expect(registry.getConnectedCount()).toBe(0);
+  });
+
+  it("unregister is no-op for unknown serverId", () => {
+    registry.unregister("nonexistent"); // should not throw
+    expect(registry.getConnectedCount()).toBe(0);
+  });
+
+  it("updates heartbeat", () => {
+    const agent = makeAgent();
+    const oldTime = new Date(2020, 0, 1);
+    agent.lastHeartbeat = oldTime;
+    registry.register(agent);
+    registry.updateHeartbeat("srv-1");
+    expect(agent.lastHeartbeat.getTime()).toBeGreaterThan(oldTime.getTime());
+  });
+
+  it("emits agent:connected on register", () => {
+    const handler = vi.fn();
+    registry.on("agent:connected", handler);
+    const agent = makeAgent();
+    registry.register(agent);
+    expect(handler).toHaveBeenCalledWith(agent);
+  });
+
+  it("emits agent:disconnected on unregister", () => {
+    const handler = vi.fn();
+    registry.on("agent:disconnected", handler);
+    const agent = makeAgent();
+    registry.register(agent);
+    registry.unregister("srv-1");
+    expect(handler).toHaveBeenCalledWith(agent);
+  });
+
+  it("emits agent:heartbeat on updateHeartbeat", () => {
+    const handler = vi.fn();
+    registry.on("agent:heartbeat", handler);
+    const agent = makeAgent();
+    registry.register(agent);
+    registry.updateHeartbeat("srv-1");
+    expect(handler).toHaveBeenCalledWith(agent);
+  });
+
+  it("handles multiple agents", () => {
+    const a1 = makeAgent({ serverId: "s1", hostname: "h1" });
+    const a2 = makeAgent({ serverId: "s2", hostname: "h2" });
+    registry.register(a1);
+    registry.register(a2);
+    expect(registry.getConnectedCount()).toBe(2);
+    expect(registry.getAllConnected()).toHaveLength(2);
+    registry.unregister("s1");
+    expect(registry.getConnectedCount()).toBe(1);
+    expect(registry.getByHostname("h2")).toBe(a2);
+  });
+});
--- a/bastion/src/labd/tests/auth-routes.test.ts
+++ b/bastion/src/labd/tests/auth-routes.test.ts
@@ -0,0 +1,208 @@
+// Tests for auth and token management routes.
+
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import Fastify from "fastify";
+import { registerAuthRoutes } from "../src/routes/auth.js";
+
+function createMockDb() {
+  const tokens: Map<string, Record<string, unknown>> = new Map();
+
+  return {
+    $queryRaw: vi.fn(),
+    server: { findMany: vi.fn(), findUnique: vi.fn() },
+    joinToken: {
+      findUnique: vi.fn(async (args: { where: { token?: string; id?: string } }) => {
+        const key = args.where.token ?? args.where.id;
+        return key ? tokens.get(key) ?? null : null;
+      }),
+      findMany: vi.fn(async () => [...tokens.values()]),
+      create: vi.fn(async (args: { data: Record<string, unknown> }) => {
+        const id = `tok-${tokens.size + 1}`;
+        const record = {
+          id,
+          ...args.data,
+          usedBy: null,
+          usedAt: null,
+          revokedAt: null,
+          createdAt: new Date(),
+        };
+        tokens.set(record.token as string, record);
+        tokens.set(id, record);
+        return record;
+      }),
+      update: vi.fn(async (args: { where: { id: string }; data: Record<string, unknown> }) => {
+        const existing = tokens.get(args.where.id);
+        if (existing) Object.assign(existing, args.data);
+        return existing;
+      }),
+    },
+    _tokens: tokens,
+  };
+}
+
+describe("auth routes", () => {
+  let app: ReturnType<typeof Fastify>;
+  let db: ReturnType<typeof createMockDb>;
+
+  beforeEach(async () => {
+    app = Fastify({ logger: false });
+    db = createMockDb();
+    registerAuthRoutes(app, db);
+    await app.ready();
+  });
+
+  describe("POST /api/tokens", () => {
+    it("creates a one-time token", async () => {
+      const resp = await app.inject({
+        method: "POST",
+        url: "/api/tokens",
+        payload: { label: "test-token" },
+      });
+      expect(resp.statusCode).toBe(201);
+      const body = resp.json();
+      expect(body.token).toBeDefined();
+      expect(body.type).toBe("one-time");
+      expect(body.label).toBe("test-token");
+    });
+
+    it("creates a reusable token", async () => {
+      const resp = await app.inject({
+        method: "POST",
+        url: "/api/tokens",
+        payload: { type: "reusable", label: "asg-token" },
+      });
+      expect(resp.statusCode).toBe(201);
+      expect(resp.json().type).toBe("reusable");
+    });
+
+    it("rejects invalid type", async () => {
+      const resp = await app.inject({
+        method: "POST",
+        url: "/api/tokens",
+        payload: { type: "invalid" },
+      });
+      expect(resp.statusCode).toBe(400);
+    });
+
+    it("creates token with expiry", async () => {
+      const resp = await app.inject({
+        method: "POST",
+        url: "/api/tokens",
+        payload: { expiresInHours: 24 },
+      });
+      expect(resp.statusCode).toBe(201);
+      expect(resp.json().expiresAt).toBeDefined();
+    });
+  });
+
+  describe("GET /api/tokens", () => {
+    it("lists tokens", async () => {
+      // Create a token first
+      await app.inject({
+        method: "POST",
+        url: "/api/tokens",
+        payload: { label: "t1" },
+      });
+
+      const resp = await app.inject({ method: "GET", url: "/api/tokens" });
+      expect(resp.statusCode).toBe(200);
+      expect(Array.isArray(resp.json())).toBe(true);
+    });
+  });
+
+  describe("POST /api/auth/enroll", () => {
+    it("rejects missing token", async () => {
+      const resp = await app.inject({
+        method: "POST",
+        url: "/api/auth/enroll",
+        payload: { hostname: "w1" },
+      });
+      expect(resp.statusCode).toBe(400);
+      expect(resp.json().error).toContain("token");
+    });
+
+    it("rejects missing hostname", async () => {
+      const resp = await app.inject({
+        method: "POST",
+        url: "/api/auth/enroll",
+        payload: { token: "some-token" },
+      });
+      expect(resp.statusCode).toBe(400);
+      expect(resp.json().error).toContain("hostname");
+    });
+
+    it("rejects invalid token", async () => {
+      const resp = await app.inject({
+        method: "POST",
+        url: "/api/auth/enroll",
+        payload: { token: "nonexistent", hostname: "w1" },
+      });
+      expect(resp.statusCode).toBe(401);
+    });
+
+    it("accepts valid token", async () => {
+      // Create a token
+      const createResp = await app.inject({
+        method: "POST",
+        url: "/api/tokens",
+        payload: { label: "test" },
+      });
+      const tokenValue = createResp.json().token;
+
+      const resp = await app.inject({
+        method: "POST",
+        url: "/api/auth/enroll",
+        payload: { token: tokenValue, hostname: "worker-1" },
+      });
+      expect(resp.statusCode).toBe(200);
+      expect(resp.json().status).toBe("enrolled");
+      expect(resp.json().hostname).toBe("worker-1");
+    });
+
+    it("rejects revoked token", async () => {
+      // Create and revoke a token
+      const createResp = await app.inject({
+        method: "POST",
+        url: "/api/tokens",
+        payload: { label: "revoked" },
+      });
+      const token = createResp.json();
+
+      // Manually set revokedAt
+      const record = db._tokens.get(token.token);
+      if (record) record.revokedAt = new Date();
+
+      const resp = await app.inject({
+        method: "POST",
+        url: "/api/auth/enroll",
+        payload: { token: token.token, hostname: "w1" },
+      });
+      expect(resp.statusCode).toBe(401);
+      expect(resp.json().error).toContain("revoked");
+    });
+
+    it("rejects expired token", async () => {
+      const createResp = await app.inject({
+        method: "POST",
+        url: "/api/tokens",
+        payload: { label: "expired" },
+      });
+      const token = createResp.json();
+
+      // Manually set past expiry (keep revokedAt null so it hits expiry check)
+      const record = db._tokens.get(token.token);
+      if (record) {
+        record.expiresAt = new Date(Date.now() - 60000);
+        record.revokedAt = null;
+      }
+
+      const resp = await app.inject({
+        method: "POST",
+        url: "/api/auth/enroll",
+        payload: { token: token.token, hostname: "w1" },
+      });
+      expect(resp.statusCode).toBe(401);
+      expect(resp.json().error).toContain("expired");
+    });
+  });
+});
--- a/bastion/src/labd/tests/encryption.test.ts
+++ b/bastion/src/labd/tests/encryption.test.ts
@@ -0,0 +1,63 @@
+// Tests for EncryptionService.
+
+import { describe, it, expect } from "vitest";
+import { EncryptionService } from "../src/services/encryption.js";
+
+const MASTER_KEY = "a".repeat(32);
+
+describe("EncryptionService", () => {
+  it("encrypt/decrypt roundtrip", () => {
+    const svc = new EncryptionService(MASTER_KEY);
+    const plaintext = "hello world secret data";
+    const ciphertext = svc.encrypt(plaintext);
+    expect(ciphertext).not.toBe(plaintext);
+    expect(svc.decrypt(ciphertext)).toBe(plaintext);
+  });
+
+  it("encrypts empty string", () => {
+    const svc = new EncryptionService(MASTER_KEY);
+    expect(svc.decrypt(svc.encrypt(""))).toBe("");
+  });
+
+  it("encrypts unicode", () => {
+    const svc = new EncryptionService(MASTER_KEY);
+    const text = "Héllo 世界 🌍";
+    expect(svc.decrypt(svc.encrypt(text))).toBe(text);
+  });
+
+  it("encrypts large data", () => {
+    const svc = new EncryptionService(MASTER_KEY);
+    const text = "x".repeat(100_000);
+    expect(svc.decrypt(svc.encrypt(text))).toBe(text);
+  });
+
+  it("different IVs produce different ciphertext", () => {
+    const svc = new EncryptionService(MASTER_KEY);
+    const a = svc.encrypt("same");
+    const b = svc.encrypt("same");
+    expect(a).not.toBe(b); // Random IV each time
+    expect(svc.decrypt(a)).toBe("same");
+    expect(svc.decrypt(b)).toBe("same");
+  });
+
+  it("different keys produce different ciphertext", () => {
+    const svc1 = new EncryptionService("a".repeat(32));
+    const svc2 = new EncryptionService("b".repeat(32));
+    const ct1 = svc1.encrypt("secret");
+    expect(() => svc2.decrypt(ct1)).toThrow(); // Auth tag mismatch
+  });
+
+  it("rejects tampered ciphertext", () => {
+    const svc = new EncryptionService(MASTER_KEY);
+    const ct = svc.encrypt("data");
+    const parts = ct.split(":");
+    parts[2] = "AAAA" + parts[2]!.slice(4); // corrupt encrypted data
+    expect(() => svc.decrypt(parts.join(":"))).toThrow();
+  });
+
+  it("rejects invalid format", () => {
+    const svc = new EncryptionService(MASTER_KEY);
+    expect(() => svc.decrypt("not:enough")).toThrow("Invalid ciphertext format");
+    expect(() => svc.decrypt("")).toThrow("Invalid ciphertext format");
+  });
+});
--- a/bastion/src/labd/tests/validation.test.ts
+++ b/bastion/src/labd/tests/validation.test.ts
@@ -0,0 +1,123 @@
+// Tests for Zod validation schemas.
+
+import { describe, it, expect } from "vitest";
+import {
+  createTokenSchema,
+  enrollmentSchema,
+  serverFiltersSchema,
+  createRoleSchema,
+  permissionPatternSchema,
+} from "../src/validation/schemas.js";
+
+describe("createTokenSchema", () => {
+  it("accepts minimal input", () => {
+    const result = createTokenSchema.safeParse({});
+    expect(result.success).toBe(true);
+    if (result.success) {
+      expect(result.data.type).toBe("one-time"); // default
+    }
+  });
+
+  it("accepts full input", () => {
+    const result = createTokenSchema.safeParse({
+      type: "reusable",
+      label: "asg-token",
+      expiresInHours: 24,
+    });
+    expect(result.success).toBe(true);
+  });
+
+  it("rejects invalid type", () => {
+    const result = createTokenSchema.safeParse({ type: "invalid" });
+    expect(result.success).toBe(false);
+  });
+
+  it("rejects negative expiry", () => {
+    const result = createTokenSchema.safeParse({ expiresInHours: -1 });
+    expect(result.success).toBe(false);
+  });
+
+  it("rejects expiry over 1 year", () => {
+    const result = createTokenSchema.safeParse({ expiresInHours: 9000 });
+    expect(result.success).toBe(false);
+  });
+});
+
+describe("enrollmentSchema", () => {
+  it("accepts valid enrollment", () => {
+    const result = enrollmentSchema.safeParse({
+      token: "abc123",
+      hostname: "worker-1.ad.itaz.eu",
+    });
+    expect(result.success).toBe(true);
+  });
+
+  it("rejects empty token", () => {
+    const result = enrollmentSchema.safeParse({
+      token: "",
+      hostname: "w1",
+    });
+    expect(result.success).toBe(false);
+  });
+
+  it("rejects invalid hostname", () => {
+    const result = enrollmentSchema.safeParse({
+      token: "abc",
+      hostname: "-invalid",
+    });
+    expect(result.success).toBe(false);
+  });
+});
+
+describe("serverFiltersSchema", () => {
+  it("accepts empty filters", () => {
+    const result = serverFiltersSchema.safeParse({});
+    expect(result.success).toBe(true);
+  });
+
+  it("accepts valid status", () => {
+    const result = serverFiltersSchema.safeParse({ status: "online" });
+    expect(result.success).toBe(true);
+  });
+
+  it("rejects invalid status", () => {
+    const result = serverFiltersSchema.safeParse({ status: "banana" });
+    expect(result.success).toBe(false);
+  });
+});
+
+describe("permissionPatternSchema", () => {
+  it("accepts valid patterns", () => {
+    expect(permissionPatternSchema.safeParse("read:*:*:*").success).toBe(true);
+    expect(permissionPatternSchema.safeParse("exec:baremetal:lab:*").success).toBe(true);
+    expect(permissionPatternSchema.safeParse("*:*:*:worker-1").success).toBe(true);
+  });
+
+  it("rejects invalid patterns", () => {
+    expect(permissionPatternSchema.safeParse("read").success).toBe(false);
+    expect(permissionPatternSchema.safeParse("read:*").success).toBe(false);
+    expect(permissionPatternSchema.safeParse("READ:*:*:*").success).toBe(false);
+  });
+});
+
+describe("createRoleSchema", () => {
+  it("accepts valid role", () => {
+    const result = createRoleSchema.safeParse({
+      name: "deployer",
+      description: "Can deploy apps",
+      allow: ["exec:*:*:*", "read:*:*:*"],
+      deny: ["destroy:*:*:*"],
+    });
+    expect(result.success).toBe(true);
+  });
+
+  it("rejects uppercase role name", () => {
+    const result = createRoleSchema.safeParse({ name: "Admin" });
+    expect(result.success).toBe(false);
+  });
+
+  it("rejects empty role name", () => {
+    const result = createRoleSchema.safeParse({ name: "" });
+    expect(result.success).toBe(false);
+  });
+});