Files
lab/bastion/src/bastion/tests/dispatch.test.ts
Michal 46b017d77e
Some checks failed
CI/CD / lint (pull_request) Failing after 13s
CI/CD / test (pull_request) Failing after 10s
CI/CD / typecheck (pull_request) Failing after 36s
CI/CD / build (pull_request) Has been skipped
CI/CD / publish-rpm (pull_request) Has been skipped
CI/CD / publish-deb (pull_request) Has been skipped
feat: install logging, error trapping, PXE/ISO integration tests
Kickstart installs on real hardware failed silently — no error reporting,
only 3 progress callbacks, zero log streaming. This overhaul makes every
install fully observable.

Kickstart improvements:
- Error trapping in %pre and %post (trap ERR sends failure details to bastion)
- 12+ granular progress stages (was 3): SSH, hostname, k3s prep, EFI boot, metadata
- Background log streamer: tails %post output and batch-sends to /api/log
- bastion_log() function for explicit log lines from kickstart scripts

Bastion API:
- POST /api/log — receives raw log lines from kickstart (single or batch)
- InstallLogBuffer — per-MAC ring buffer (2000 lines) + file persistence
- GET /api/logs/:mac — now returns log_lines + log_total alongside stages
- SSE /api/logs/:mac/follow — uses named events (event: stage vs event: log)
- Progress events forwarded to labd via bastion-progress WebSocket message
- Post-provision k3s logs routed through progressBus (was console-only)

dnsmasq fixes found during VM testing:
- HTTP Boot filename: ipxe-real.efi → ipxe.efi (leftover from old 2-stage approach)
- pxe-service directives: only in proxy mode (breaks OVMF PXE in full mode)
- PXEClient vendor class echo for UEFI firmware compatibility

Integration tests:
- PXE boot test: blank UEFI VM → dnsmasq → HTTP Boot → iPXE → bastion → install
- ISO boot test: blank VM boots from bastion-generated ISO → same flow
- Shared helpers: pxe-network (no DHCP, nftables fix), pxe-vm (UEFI + ISO boot)
- test-provision.sh: runs both PXE + ISO tests with prerequisite checks
- 250GB sparse QCOW2 disk (LVM layout needs ~204GB)

201 unit tests passing (11 new).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-26 22:26:33 +00:00

329 lines
9.8 KiB
TypeScript

import { describe, it, expect, beforeEach, afterEach } from "vitest";
import { mkdirSync, rmSync } from "node:fs";
import { join } from "node:path";
import { tmpdir } from "node:os";
import type { BastionConfig } from "@lab/shared";
import { createApp } from "../src/server.js";
import type { FastifyInstance } from "fastify";
import type { StateManager } from "../src/services/state.js";
import type { InstallLogBuffer } from "../src/services/install-log.js";
function createTestConfig(testDir: string): BastionConfig {
return {
fedoraVersion: "43",
arch: "x86_64",
httpPort: 0,
timezone: "Europe/London",
locale: "en_GB.UTF-8",
bastionDir: testDir,
domain: "test.local",
dhcpMode: "proxy",
dhcpRangeStart: "",
dhcpRangeEnd: "",
ubuntuVersion: "26.04",
ubuntuMirror: "https://releases.ubuntu.com/26.04",
iface: "eth0",
serverIp: "10.0.0.1",
network: "10.0.0.0",
gateway: "10.0.0.1",
sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"],
adminUser: "testadmin",
skipDnsmasq: true,
skipArtifacts: true,
fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
tftpDir: join(testDir, "tftp"),
httpDir: join(testDir, "http"),
stateFile: join(testDir, "state.json"),
};
}
describe("dispatch routes", () => {
let testDir: string;
let app: FastifyInstance;
let state: StateManager;
let installLog: InstallLogBuffer;
beforeEach(() => {
testDir = join(tmpdir(), `bastion-dispatch-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
mkdirSync(testDir, { recursive: true });
mkdirSync(join(testDir, "http"), { recursive: true });
mkdirSync(join(testDir, "tftp"), { recursive: true });
const config = createTestConfig(testDir);
const result = createApp(config);
app = result.app;
state = result.state;
installLog = result.installLog;
});
afterEach(async () => {
await app.close();
rmSync(testDir, { recursive: true, force: true });
});
it("unknown MAC returns discovery iPXE script", async () => {
const response = await app.inject({
method: "GET",
url: "/dispatch?mac=aa:bb:cc:dd:ee:ff",
});
expect(response.statusCode).toBe(200);
expect(response.headers["content-type"]).toContain("text/plain");
const body = response.body;
expect(body).toContain("#!ipxe");
expect(body).toContain("DISCOVERY MODE");
expect(body).toContain("discover.ks");
});
it("MAC in install_queue returns install iPXE script", async () => {
const mac = "aa:bb:cc:dd:ee:ff";
state.update((s) => {
s.install_queue[mac] = {
hostname: "worker-1",
disk: "/dev/sda",
role: "worker",
queued_at: new Date().toISOString(),
};
});
const response = await app.inject({
method: "GET",
url: `/dispatch?mac=${mac}`,
});
expect(response.statusCode).toBe(200);
const body = response.body;
expect(body).toContain("#!ipxe");
expect(body).toContain("INSTALLING");
expect(body).toContain("worker-1");
expect(body).toContain(`ks?mac=${mac}`);
});
it("MAC in installed returns local boot (exit) script", async () => {
const mac = "aa:bb:cc:dd:ee:ff";
state.update((s) => {
s.installed[mac] = {
hostname: "installed-node",
role: "worker",
ip: "10.0.0.50",
installed_at: new Date().toISOString(),
};
});
const response = await app.inject({
method: "GET",
url: `/dispatch?mac=${mac}`,
});
expect(response.statusCode).toBe(200);
const body = response.body;
expect(body).toContain("#!ipxe");
expect(body).toContain("installed-node");
expect(body).toContain("Already installed");
expect(body).toContain("exit");
});
it("progress endpoint updates state", async () => {
const mac = "aa:bb:cc:dd:ee:ff";
state.update((s) => {
s.install_queue[mac] = {
hostname: "worker-1",
disk: "/dev/sda",
role: "worker",
queued_at: new Date().toISOString(),
};
});
const response = await app.inject({
method: "POST",
url: "/api/progress",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
mac,
stage: "post-install",
detail: "configuring system",
}),
});
expect(response.statusCode).toBe(200);
const result = JSON.parse(response.body);
expect(result.status).toBe("ok");
// Verify state was updated
const currentState = state.load();
expect(currentState.install_queue[mac]?.progress).toBe("post-install");
expect(currentState.install_queue[mac]?.progress_detail).toBe("configuring system");
});
it("progress endpoint with 'complete' stage moves machine to installed", async () => {
const mac = "aa:bb:cc:dd:ee:ff";
state.update((s) => {
s.install_queue[mac] = {
hostname: "worker-1",
disk: "/dev/sda",
role: "worker",
queued_at: new Date().toISOString(),
};
});
const response = await app.inject({
method: "POST",
url: "/api/progress",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
mac,
stage: "complete",
detail: "ready at 10.0.0.50",
}),
});
expect(response.statusCode).toBe(200);
const currentState = state.load();
expect(currentState.install_queue[mac]).toBeUndefined();
expect(currentState.installed[mac]).toBeDefined();
expect(currentState.installed[mac]?.hostname).toBe("worker-1");
expect(currentState.installed[mac]?.ip).toBe("10.0.0.50");
});
it("DELETE /api/machines/:mac removes machine from state", async () => {
const mac = "aa:bb:cc:dd:ee:ff";
state.update((s) => {
s.discovered[mac] = {
mac,
product: "TestBox",
board: "TestBoard",
serial: "SN123",
manufacturer: "TestCorp",
cpu_model: "Test CPU",
cpu_cores: 4,
memory_gb: 16,
arch: "x86_64",
disks: [],
nics: [],
first_seen: new Date().toISOString(),
last_seen: new Date().toISOString(),
};
});
const response = await app.inject({
method: "DELETE",
url: `/api/machines/${encodeURIComponent(mac)}`,
});
expect(response.statusCode).toBe(200);
const result = JSON.parse(response.body);
expect(result.status).toBe("forgotten");
const currentState = state.load();
expect(currentState.discovered[mac]).toBeUndefined();
});
it("DELETE /api/machines/:mac returns 404 for unknown machine", async () => {
const response = await app.inject({
method: "DELETE",
url: "/api/machines/ff:ff:ff:ff:ff:ff",
});
expect(response.statusCode).toBe(404);
const result = JSON.parse(response.body);
expect(result.error).toBe("machine not found");
});
it("POST /api/log accepts a single line", async () => {
const mac = "aa:bb:cc:dd:ee:ff";
const response = await app.inject({
method: "POST",
url: "/api/log",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ mac, line: "hello from kickstart" }),
});
expect(response.statusCode).toBe(200);
const result = JSON.parse(response.body);
expect(result.status).toBe("ok");
expect(result.lines).toBe(1);
// Verify line is stored
const lines = installLog.getLines(mac);
expect(lines).toHaveLength(1);
expect(lines[0]!.line).toBe("hello from kickstart");
});
it("POST /api/log accepts multiple lines", async () => {
const mac = "aa:bb:cc:dd:ee:ff";
const response = await app.inject({
method: "POST",
url: "/api/log",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ mac, lines: ["line 1", "line 2", "line 3"] }),
});
expect(response.statusCode).toBe(200);
const result = JSON.parse(response.body);
expect(result.lines).toBe(3);
const lines = installLog.getLines(mac);
expect(lines).toHaveLength(3);
});
it("GET /api/logs/:mac includes log lines for installing machine", async () => {
const mac = "aa:bb:cc:dd:ee:ff";
state.update((s) => {
s.install_queue[mac] = {
hostname: "test-node",
disk: "/dev/sda",
role: "worker",
queued_at: new Date().toISOString(),
};
});
// Add some log lines
installLog.append(mac, ["log line 1", "log line 2"], "test-node");
const response = await app.inject({
method: "GET",
url: `/api/logs/${encodeURIComponent(mac)}`,
});
expect(response.statusCode).toBe(200);
const result = JSON.parse(response.body);
expect(result.status).toBe("installing");
expect(result.log_lines).toHaveLength(2);
expect(result.log_total).toBe(2);
expect(result.log_lines[0].line).toBe("log line 1");
});
it("progress endpoint with 'error' stage keeps machine in install_queue", async () => {
const mac = "aa:bb:cc:dd:ee:ff";
state.update((s) => {
s.install_queue[mac] = {
hostname: "failing-node",
disk: "/dev/sda",
role: "worker",
queued_at: new Date().toISOString(),
};
});
const response = await app.inject({
method: "POST",
url: "/api/progress",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
mac,
stage: "error",
detail: "%post failed at line 42",
}),
});
expect(response.statusCode).toBe(200);
// Machine should still be in install_queue (not moved to installed)
const currentState = state.load();
expect(currentState.install_queue[mac]).toBeDefined();
expect(currentState.install_queue[mac]?.progress).toBe("error");
expect(currentState.install_queue[mac]?.progress_detail).toBe("%post failed at line 42");
expect(currentState.installed[mac]).toBeUndefined();
});
});