feat: install logging, error trapping, PXE/ISO integration tests

Kickstart installs on real hardware failed silently — no error reporting,
only 3 progress callbacks, zero log streaming. This overhaul makes every
install fully observable.

Kickstart improvements:
- Error trapping in %pre and %post (trap ERR sends failure details to bastion)
- 12+ granular progress stages (was 3): SSH, hostname, k3s prep, EFI boot, metadata
- Background log streamer: tails %post output and batch-sends to /api/log
- bastion_log() function for explicit log lines from kickstart scripts

Bastion API:
- POST /api/log — receives raw log lines from kickstart (single or batch)
- InstallLogBuffer — per-MAC ring buffer (2000 lines) + file persistence
- GET /api/logs/:mac — now returns log_lines + log_total alongside stages
- SSE /api/logs/:mac/follow — uses named events (event: stage vs event: log)
- Progress events forwarded to labd via bastion-progress WebSocket message
- Post-provision k3s logs routed through progressBus (was console-only)

dnsmasq fixes found during VM testing:
- HTTP Boot filename: ipxe-real.efi → ipxe.efi (leftover from old 2-stage approach)
- pxe-service directives: only in proxy mode (breaks OVMF PXE in full mode)
- PXEClient vendor class echo for UEFI firmware compatibility

Integration tests:
- PXE boot test: blank UEFI VM → dnsmasq → HTTP Boot → iPXE → bastion → install
- ISO boot test: blank VM boots from bastion-generated ISO → same flow
- Shared helpers: pxe-network (no DHCP, nftables fix), pxe-vm (UEFI + ISO boot)
- test-provision.sh: runs both PXE + ISO tests with prerequisite checks
- 250GB sparse QCOW2 disk (LVM layout needs ~204GB)

201 unit tests passing (11 new).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

bastion/src/shared/src/errors/index.ts

@@ -0,0 +1,86 @@
+// Error handling and reporting utilities for the lab platform.
+
+// --- Base error class ---
+
+export class LabError extends Error {
+  readonly code: string;
+  readonly statusCode: number;
+  readonly details: Record<string, unknown> | undefined;
+
+  constructor(
+    message: string,
+    code: string,
+    statusCode: number = 500,
+    details?: Record<string, unknown>,
+  ) {
+    super(message);
+    this.name = this.constructor.name;
+    this.code = code;
+    this.statusCode = statusCode;
+    this.details = details;
+  }
+
+  toJSON(): { error: string; code: string; statusCode: number; details: Record<string, unknown> | undefined } {
+    return {
+      error: this.message,
+      code: this.code,
+      statusCode: this.statusCode,
+      details: this.details,
+    };
+  }
+}
+
+// --- Subclasses ---
+
+export class NotFoundError extends LabError {
+  constructor(resourceType: string, identifier: string) {
+    super(
+      `${resourceType} '${identifier}' not found`,
+      "NOT_FOUND",
+      404,
+      { resourceType, identifier },
+    );
+  }
+}
+
+export class PermissionDeniedError extends LabError {
+  constructor(action: string, resource: string, reason?: string) {
+    const message = reason
+      ? `Permission denied: cannot ${action} on ${resource} — ${reason}`
+      : `Permission denied: cannot ${action} on ${resource}`;
+    super(
+      message,
+      "PERMISSION_DENIED",
+      403,
+      { action, resource, ...(reason !== undefined ? { reason } : {}) },
+    );
+  }
+}
+
+export class ValidationError extends LabError {
+  constructor(message: string, field?: string) {
+    super(
+      message,
+      "VALIDATION_ERROR",
+      400,
+      field !== undefined ? { field } : undefined,
+    );
+  }
+}
+
+export class AgentNotConnectedError extends LabError {
+  constructor(serverName: string) {
+    super(
+      `Agent on server '${serverName}' is not connected`,
+      "AGENT_NOT_CONNECTED",
+      503,
+      { serverName },
+    );
+  }
+}
+
+// --- Type guard ---
+
+export function isLabError(err: unknown): err is LabError {
+  return err instanceof LabError;
+}

bastion/src/shared/src/index.ts

@@ -1,4 +1,7 @@
 export type {
+  OsId,
+  Arch,
+  Role,
   HardwareInfo,
   InstallConfig,
   InstalledInfo,
@@ -6,4 +9,37 @@ export type {
   BastionConfig,
 } from "./types/index.js";
 
+export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./types/index.js";
+export type { RoleInfo } from "./types/index.js";
+
 export { APP_NAME, APP_VERSION } from "./constants/index.js";
+
+export {
+  type AgentMessage,
+  type ServerMessage,
+  type BastionMessage,
+  type LabdBastionMessage,
+  type JournalOptions,
+  type AgentMessageType,
+  type ServerMessageType,
+  type BastionMessageType,
+  type LabdBastionMessageType,
+  isAgentMessage,
+  isServerMessage,
+  isBastionMessage,
+  isLabdBastionMessage,
+  parseAgentMessage,
+  parseServerMessage,
+  parseBastionMessage,
+  parseLabdBastionMessage,
+  generateRequestId,
+} from "./protocol/index.js";
+
+export {
+  LabError,
+  NotFoundError,
+  PermissionDeniedError,
+  ValidationError,
+  AgentNotConnectedError,
+  isLabError,
+} from "./errors/index.js";

bastion/src/shared/src/protocol/index.ts

@@ -0,0 +1,170 @@
+// Protocol types for agent-labd WebSocket communication.
+
+import { randomUUID } from "node:crypto";
+
+// --- Agent -> labd messages ---
+
+export type AgentMessage =
+  | { type: "heartbeat"; hostname: string; uptime: number; version: string; memUsage: number; cpuUsage: number }
+  | { type: "exec-stdout"; requestId: string; data: string }
+  | { type: "exec-stderr"; requestId: string; data: string }
+  | { type: "exec-exit"; requestId: string; exitCode: number }
+  | { type: "log-line"; requestId: string; line: string }
+  | { type: "log-end"; requestId: string }
+  | { type: "enrollment-request"; joinToken: string; hostname: string; csr: string }
+  | { type: "rotation-request"; currentFingerprint: string; newCsr: string };
+
+// --- labd -> Agent messages ---
+
+export type ServerMessage =
+  | { type: "exec"; requestId: string; command: string; args: string[]; timeout: number; tty: boolean }
+  | { type: "exec-stdin"; requestId: string; data: string }
+  | { type: "exec-signal"; requestId: string; signal: "SIGTERM" | "SIGKILL" | "SIGINT" }
+  | { type: "log-subscribe"; requestId: string; options: JournalOptions }
+  | { type: "log-unsubscribe"; requestId: string }
+  | { type: "enrollment-response"; status: "success" | "error"; certificatePem?: string; error?: string }
+  | { type: "heartbeat-ack"; serverTime: string }
+  | { type: "server-shutdown"; reconnectAfter: number };
+
+// --- Supporting types ---
+
+export interface JournalOptions {
+  follow?: boolean;
+  lines?: number;
+  unit?: string;
+  since?: string;
+  priority?: string;
+  kernel?: boolean;
+  file?: string;
+}
+
+// --- Message types for discriminated union access ---
+
+export type AgentMessageType = AgentMessage["type"];
+export type ServerMessageType = ServerMessage["type"];
+
+// --- Type guards ---
+
+const AGENT_MESSAGE_TYPES = new Set<string>([
+  "heartbeat", "exec-stdout", "exec-stderr", "exec-exit",
+  "log-line", "log-end", "enrollment-request", "rotation-request",
+]);
+
+const SERVER_MESSAGE_TYPES = new Set<string>([
+  "exec", "exec-stdin", "exec-signal", "log-subscribe",
+  "log-unsubscribe", "enrollment-response", "heartbeat-ack", "server-shutdown",
+]);
+
+export function isAgentMessage(msg: unknown): msg is AgentMessage {
+  return (
+    typeof msg === "object" &&
+    msg !== null &&
+    "type" in msg &&
+    typeof (msg as { type: unknown }).type === "string" &&
+    AGENT_MESSAGE_TYPES.has((msg as { type: string }).type)
+  );
+}
+
+export function isServerMessage(msg: unknown): msg is ServerMessage {
+  return (
+    typeof msg === "object" &&
+    msg !== null &&
+    "type" in msg &&
+    typeof (msg as { type: unknown }).type === "string" &&
+    SERVER_MESSAGE_TYPES.has((msg as { type: string }).type)
+  );
+}
+
+// --- Parsing utilities ---
+
+export function parseAgentMessage(data: string): AgentMessage {
+  const msg: unknown = JSON.parse(data);
+  if (!isAgentMessage(msg)) {
+    throw new Error(`Invalid agent message: ${(msg as { type?: string }).type ?? "unknown"}`);
+  }
+  return msg;
+}
+
+export function parseServerMessage(data: string): ServerMessage {
+  const msg: unknown = JSON.parse(data);
+  if (!isServerMessage(msg)) {
+    throw new Error(`Invalid server message: ${(msg as { type?: string }).type ?? "unknown"}`);
+  }
+  return msg;
+}
+
+// --- Bastion -> labd messages ---
+
+export type BastionMessage =
+  | { type: "bastion-enroll"; token: string; hostname: string; network: string; serverIp: string }
+  | { type: "bastion-heartbeat"; bastionId: string; uptime: number; machineCount: number }
+  | { type: "bastion-state-sync"; bastionId: string; state: import("../types/state.js").BastionState }
+  | { type: "bastion-progress"; bastionId: string; mac: string; stage: string; detail: string; timestamp: string }
+  | { type: "command-response"; requestId: string; status: "ok" | "error"; data?: unknown; error?: string };
+
+// --- labd -> Bastion messages ---
+
+export type LabdBastionMessage =
+  | { type: "bastion-enrolled"; bastionId: string }
+  | { type: "bastion-heartbeat-ack"; serverTime: string }
+  | { type: "command-install"; requestId: string; mac: string; hostname: string; disk?: string; role: string; os: string }
+  | { type: "command-forget"; requestId: string; mac: string }
+  | { type: "command-role-update"; requestId: string; mac: string; role: string }
+  | { type: "server-shutdown"; reconnectAfter: number };
+
+export type BastionMessageType = BastionMessage["type"];
+export type LabdBastionMessageType = LabdBastionMessage["type"];
+
+// --- Bastion type guards ---
+
+const BASTION_MESSAGE_TYPES = new Set<string>([
+  "bastion-enroll", "bastion-heartbeat", "bastion-state-sync",
+  "bastion-progress", "command-response",
+]);
+
+const LABD_BASTION_MESSAGE_TYPES = new Set<string>([
+  "bastion-enrolled", "bastion-heartbeat-ack", "command-install",
+  "command-forget", "command-role-update", "server-shutdown",
+]);
+
+export function isBastionMessage(msg: unknown): msg is BastionMessage {
+  return (
+    typeof msg === "object" &&
+    msg !== null &&
+    "type" in msg &&
+    typeof (msg as { type: unknown }).type === "string" &&
+    BASTION_MESSAGE_TYPES.has((msg as { type: string }).type)
+  );
+}
+
+export function isLabdBastionMessage(msg: unknown): msg is LabdBastionMessage {
+  return (
+    typeof msg === "object" &&
+    msg !== null &&
+    "type" in msg &&
+    typeof (msg as { type: unknown }).type === "string" &&
+    LABD_BASTION_MESSAGE_TYPES.has((msg as { type: string }).type)
+  );
+}
+
+export function parseBastionMessage(data: string): BastionMessage {
+  const msg: unknown = JSON.parse(data);
+  if (!isBastionMessage(msg)) {
+    throw new Error(`Invalid bastion message: ${(msg as { type?: string }).type ?? "unknown"}`);
+  }
+  return msg;
+}
+
+export function parseLabdBastionMessage(data: string): LabdBastionMessage {
+  const msg: unknown = JSON.parse(data);
+  if (!isLabdBastionMessage(msg)) {
+    throw new Error(`Invalid labd-bastion message: ${(msg as { type?: string }).type ?? "unknown"}`);
+  }
+  return msg;
+}
+
+// --- Request ID utility ---
+
+export function generateRequestId(): string {
+  return randomUUID();
+}

bastion/src/shared/src/types/config.ts

@@ -11,9 +11,15 @@ export interface BastionConfig {
   dhcpMode: "proxy" | "full";
   dhcpRangeStart: string;
   dhcpRangeEnd: string;
+  // Ubuntu support
+  ubuntuVersion: string;
+  ubuntuMirror: string;
   // Flags
   skipDnsmasq?: boolean | undefined;
   skipArtifacts?: boolean | undefined;
+  // Labd registration (optional — if unset, bastion runs standalone)
+  labdUrl?: string | undefined;
+  bastionJoinToken?: string | undefined;
   // Derived at runtime
   iface: string;
   serverIp: string;

bastion/src/shared/src/types/index.ts

@@ -1,8 +1,14 @@
 export type {
+  OsId,
+  Arch,
+  Role,
   HardwareInfo,
   InstallConfig,
   InstalledInfo,
   BastionState,
 } from "./state.js";
 
+export { SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY, isValidOsId } from "./state.js";
+export type { RoleInfo } from "./state.js";
+
 export type { BastionConfig } from "./config.js";

bastion/src/shared/src/types/state.ts

@@ -1,5 +1,14 @@
 // State types for discovered machines, install queue, and installed machines.
 
+export type OsId = "fedora-43" | "ubuntu-26.04";
+export type Arch = "x86_64" | "aarch64";
+
+export const SUPPORTED_OS: readonly OsId[] = ["fedora-43", "ubuntu-26.04"] as const;
+
+export function isValidOsId(value: string): value is OsId {
+  return (SUPPORTED_OS as readonly string[]).includes(value);
+}
+
 export interface HardwareInfo {
   mac: string;
   product: string;
@@ -14,23 +23,77 @@ export interface HardwareInfo {
   nics: Array<{ name: string; mac: string; state: string }>;
   first_seen: string;
   last_seen: string;
+  bastionId?: string;  // set when aggregated through labd
+}
+
+export type Role = "vanilla" | "worker" | "infra" | "labcontroller";
+export const SUPPORTED_ROLES: readonly Role[] = ["vanilla", "worker", "infra", "labcontroller"] as const;
+
+export interface RoleInfo {
+  name: Role;
+  description: string;
+  parent?: Role;       // inherits from this role
+  k3s: boolean;        // installs k3s
+  apps: string[];      // apps auto-deployed after k3s
+}
+
+export const ROLE_REGISTRY: readonly RoleInfo[] = [
+  {
+    name: "vanilla",
+    description: "OS only — no k3s, no cluster services",
+    k3s: false,
+    apps: [],
+  },
+  {
+    name: "worker",
+    description: "k3s agent + Longhorn storage — joins existing cluster",
+    parent: "vanilla",
+    k3s: true,
+    apps: [],
+  },
+  {
+    name: "infra",
+    description: "k3s server + etcd — control plane node",
+    parent: "vanilla",
+    k3s: true,
+    apps: [],
+  },
+  {
+    name: "labcontroller",
+    description: "infra + bastion + labd + CockroachDB — self-sufficient provisioning node",
+    parent: "infra",
+    k3s: true,
+    apps: ["cockroachdb", "labd", "bastion"],
+  },
+] as const;
+
+export interface ProgressLogEntry {
+  stage: string;
+  detail: string;
+  timestamp: string;
 }
 
 export interface InstallConfig {
   hostname: string;
   disk: string;
-  role: "worker" | "infra";
+  role: Role;
+  os?: OsId;          // defaults to "fedora-43" for backward compat
+  arch?: Arch;         // detected from HardwareInfo or overridden
   queued_at: string;
   progress?: string;
   progress_at?: string;
   progress_detail?: string;
+  log?: ProgressLogEntry[];   // full progress history
+  bastionId?: string;  // set when aggregated through labd
 }
 
 export interface InstalledInfo {
   hostname: string;
   role: string;
+  os?: OsId;
   ip: string;
   installed_at: string;
+  bastionId?: string;  // set when aggregated through labd
 }
 
 export interface BastionState {

bastion/src/shared/tests/errors.test.ts

@@ -0,0 +1,85 @@
+// Tests for error hierarchy.
+
+import { describe, it, expect } from "vitest";
+import {
+  LabError,
+  NotFoundError,
+  PermissionDeniedError,
+  ValidationError,
+  AgentNotConnectedError,
+  isLabError,
+} from "../src/errors/index.js";
+
+describe("LabError", () => {
+  it("preserves stack trace", () => {
+    const err = new LabError("test", "TEST", 500);
+    expect(err.stack).toContain("errors.test");
+  });
+
+  it("sets name to constructor name", () => {
+    expect(new LabError("x", "X", 500).name).toBe("LabError");
+    expect(new NotFoundError("Server", "w1").name).toBe("NotFoundError");
+  });
+
+  it("toJSON returns structured error", () => {
+    const err = new LabError("msg", "CODE", 418, { key: "val" });
+    const json = err.toJSON();
+    expect(json).toEqual({
+      error: "msg",
+      code: "CODE",
+      statusCode: 418,
+      details: { key: "val" },
+    });
+  });
+});
+
+describe("NotFoundError", () => {
+  it("has 404 status", () => {
+    const err = new NotFoundError("Server", "worker-1");
+    expect(err.statusCode).toBe(404);
+    expect(err.code).toBe("NOT_FOUND");
+    expect(err.message).toContain("Server");
+    expect(err.message).toContain("worker-1");
+  });
+});
+
+describe("PermissionDeniedError", () => {
+  it("has 403 status", () => {
+    const err = new PermissionDeniedError("exec", "server/w1", "no permission");
+    expect(err.statusCode).toBe(403);
+    expect(err.code).toBe("PERMISSION_DENIED");
+    expect(err.message).toContain("exec");
+  });
+});
+
+describe("ValidationError", () => {
+  it("has 400 status", () => {
+    const err = new ValidationError("bad input", "hostname");
+    expect(err.statusCode).toBe(400);
+    expect(err.code).toBe("VALIDATION_ERROR");
+  });
+});
+
+describe("AgentNotConnectedError", () => {
+  it("has 503 status", () => {
+    const err = new AgentNotConnectedError("worker-1");
+    expect(err.statusCode).toBe(503);
+    expect(err.message).toContain("worker-1");
+  });
+});
+
+describe("isLabError", () => {
+  it("returns true for LabError subclasses", () => {
+    expect(isLabError(new NotFoundError("x", "y"))).toBe(true);
+    expect(isLabError(new PermissionDeniedError("a", "b"))).toBe(true);
+  });
+
+  it("returns false for plain Error", () => {
+    expect(isLabError(new Error("nope"))).toBe(false);
+  });
+
+  it("returns false for non-errors", () => {
+    expect(isLabError("string")).toBe(false);
+    expect(isLabError(null)).toBe(false);
+  });
+});

bastion/src/shared/tests/protocol.test.ts

@@ -0,0 +1,109 @@
+// Tests for WebSocket protocol types, type guards, and parsing.
+
+import { describe, it, expect } from "vitest";
+import {
+  isAgentMessage,
+  isServerMessage,
+  parseAgentMessage,
+  parseServerMessage,
+  generateRequestId,
+} from "../src/protocol/index.js";
+
+describe("protocol type guards", () => {
+  it("isAgentMessage accepts valid heartbeat", () => {
+    expect(
+      isAgentMessage({
+        type: "heartbeat",
+        hostname: "worker-1",
+        uptime: 100,
+        version: "0.1.0",
+        memUsage: 1024,
+        cpuUsage: 0.5,
+      }),
+    ).toBe(true);
+  });
+
+  it("isAgentMessage accepts valid exec-exit", () => {
+    expect(
+      isAgentMessage({ type: "exec-exit", requestId: "abc", exitCode: 0 }),
+    ).toBe(true);
+  });
+
+  it("isAgentMessage rejects unknown type", () => {
+    expect(isAgentMessage({ type: "unknown-type" })).toBe(false);
+  });
+
+  it("isAgentMessage rejects non-object", () => {
+    expect(isAgentMessage("heartbeat")).toBe(false);
+    expect(isAgentMessage(null)).toBe(false);
+    expect(isAgentMessage(42)).toBe(false);
+  });
+
+  it("isServerMessage accepts valid exec", () => {
+    expect(
+      isServerMessage({
+        type: "exec",
+        requestId: "abc",
+        command: "ls",
+        args: ["-la"],
+        timeout: 30000,
+        tty: false,
+      }),
+    ).toBe(true);
+  });
+
+  it("isServerMessage accepts server-shutdown", () => {
+    expect(
+      isServerMessage({ type: "server-shutdown", reconnectAfter: 5000 }),
+    ).toBe(true);
+  });
+
+  it("isServerMessage rejects agent message types", () => {
+    expect(isServerMessage({ type: "heartbeat" })).toBe(false);
+  });
+});
+
+describe("parseAgentMessage", () => {
+  it("parses valid JSON", () => {
+    const msg = parseAgentMessage(
+      JSON.stringify({ type: "heartbeat", hostname: "w1", uptime: 1, version: "0.1.0", memUsage: 0, cpuUsage: 0 }),
+    );
+    expect(msg.type).toBe("heartbeat");
+  });
+
+  it("throws on invalid JSON", () => {
+    expect(() => parseAgentMessage("not json")).toThrow();
+  });
+
+  it("throws on invalid message type", () => {
+    expect(() => parseAgentMessage(JSON.stringify({ type: "bogus" }))).toThrow(
+      "Invalid agent message",
+    );
+  });
+});
+
+describe("parseServerMessage", () => {
+  it("parses valid JSON", () => {
+    const msg = parseServerMessage(
+      JSON.stringify({ type: "heartbeat-ack", serverTime: "2026-01-01T00:00:00Z" }),
+    );
+    expect(msg.type).toBe("heartbeat-ack");
+  });
+
+  it("throws on agent message type", () => {
+    expect(() =>
+      parseServerMessage(JSON.stringify({ type: "heartbeat" })),
+    ).toThrow("Invalid server message");
+  });
+});
+
+describe("generateRequestId", () => {
+  it("returns a string", () => {
+    expect(typeof generateRequestId()).toBe("string");
+  });
+
+  it("returns unique IDs", () => {
+    const ids = new Set(Array.from({ length: 100 }, () => generateRequestId()));
+    expect(ids.size).toBe(100);
+  });
+});