Some checks failed
CI/CD / lint (pull_request) Failing after 13s
CI/CD / test (pull_request) Failing after 10s
CI/CD / typecheck (pull_request) Failing after 36s
CI/CD / build (pull_request) Has been skipped
CI/CD / publish-rpm (pull_request) Has been skipped
CI/CD / publish-deb (pull_request) Has been skipped
Kickstart installs on real hardware failed silently — no error reporting, only 3 progress callbacks, zero log streaming. This overhaul makes every install fully observable. Kickstart improvements: - Error trapping in %pre and %post (trap ERR sends failure details to bastion) - 12+ granular progress stages (was 3): SSH, hostname, k3s prep, EFI boot, metadata - Background log streamer: tails %post output and batch-sends to /api/log - bastion_log() function for explicit log lines from kickstart scripts Bastion API: - POST /api/log — receives raw log lines from kickstart (single or batch) - InstallLogBuffer — per-MAC ring buffer (2000 lines) + file persistence - GET /api/logs/:mac — now returns log_lines + log_total alongside stages - SSE /api/logs/:mac/follow — uses named events (event: stage vs event: log) - Progress events forwarded to labd via bastion-progress WebSocket message - Post-provision k3s logs routed through progressBus (was console-only) dnsmasq fixes found during VM testing: - HTTP Boot filename: ipxe-real.efi → ipxe.efi (leftover from old 2-stage approach) - pxe-service directives: only in proxy mode (breaks OVMF PXE in full mode) - PXEClient vendor class echo for UEFI firmware compatibility Integration tests: - PXE boot test: blank UEFI VM → dnsmasq → HTTP Boot → iPXE → bastion → install - ISO boot test: blank VM boots from bastion-generated ISO → same flow - Shared helpers: pxe-network (no DHCP, nftables fix), pxe-vm (UEFI + ISO boot) - test-provision.sh: runs both PXE + ISO tests with prerequisite checks - 250GB sparse QCOW2 disk (LVM layout needs ~204GB) 201 unit tests passing (11 new). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
329 lines
9.8 KiB
TypeScript
329 lines
9.8 KiB
TypeScript
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
import { mkdirSync, rmSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import type { BastionConfig } from "@lab/shared";
|
|
import { createApp } from "../src/server.js";
|
|
import type { FastifyInstance } from "fastify";
|
|
import type { StateManager } from "../src/services/state.js";
|
|
import type { InstallLogBuffer } from "../src/services/install-log.js";
|
|
|
|
function createTestConfig(testDir: string): BastionConfig {
|
|
return {
|
|
fedoraVersion: "43",
|
|
arch: "x86_64",
|
|
httpPort: 0,
|
|
timezone: "Europe/London",
|
|
locale: "en_GB.UTF-8",
|
|
bastionDir: testDir,
|
|
domain: "test.local",
|
|
dhcpMode: "proxy",
|
|
dhcpRangeStart: "",
|
|
dhcpRangeEnd: "",
|
|
ubuntuVersion: "26.04",
|
|
ubuntuMirror: "https://releases.ubuntu.com/26.04",
|
|
iface: "eth0",
|
|
serverIp: "10.0.0.1",
|
|
network: "10.0.0.0",
|
|
gateway: "10.0.0.1",
|
|
sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"],
|
|
adminUser: "testadmin",
|
|
skipDnsmasq: true,
|
|
skipArtifacts: true,
|
|
fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
|
|
tftpDir: join(testDir, "tftp"),
|
|
httpDir: join(testDir, "http"),
|
|
stateFile: join(testDir, "state.json"),
|
|
};
|
|
}
|
|
|
|
describe("dispatch routes", () => {
|
|
let testDir: string;
|
|
let app: FastifyInstance;
|
|
let state: StateManager;
|
|
let installLog: InstallLogBuffer;
|
|
|
|
beforeEach(() => {
|
|
testDir = join(tmpdir(), `bastion-dispatch-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
|
mkdirSync(testDir, { recursive: true });
|
|
mkdirSync(join(testDir, "http"), { recursive: true });
|
|
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
|
|
|
const config = createTestConfig(testDir);
|
|
const result = createApp(config);
|
|
app = result.app;
|
|
state = result.state;
|
|
installLog = result.installLog;
|
|
});
|
|
|
|
afterEach(async () => {
|
|
await app.close();
|
|
rmSync(testDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it("unknown MAC returns discovery iPXE script", async () => {
|
|
const response = await app.inject({
|
|
method: "GET",
|
|
url: "/dispatch?mac=aa:bb:cc:dd:ee:ff",
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
expect(response.headers["content-type"]).toContain("text/plain");
|
|
const body = response.body;
|
|
expect(body).toContain("#!ipxe");
|
|
expect(body).toContain("DISCOVERY MODE");
|
|
expect(body).toContain("discover.ks");
|
|
});
|
|
|
|
it("MAC in install_queue returns install iPXE script", async () => {
|
|
const mac = "aa:bb:cc:dd:ee:ff";
|
|
state.update((s) => {
|
|
s.install_queue[mac] = {
|
|
hostname: "worker-1",
|
|
disk: "/dev/sda",
|
|
role: "worker",
|
|
queued_at: new Date().toISOString(),
|
|
};
|
|
});
|
|
|
|
const response = await app.inject({
|
|
method: "GET",
|
|
url: `/dispatch?mac=${mac}`,
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
const body = response.body;
|
|
expect(body).toContain("#!ipxe");
|
|
expect(body).toContain("INSTALLING");
|
|
expect(body).toContain("worker-1");
|
|
expect(body).toContain(`ks?mac=${mac}`);
|
|
});
|
|
|
|
it("MAC in installed returns local boot (exit) script", async () => {
|
|
const mac = "aa:bb:cc:dd:ee:ff";
|
|
state.update((s) => {
|
|
s.installed[mac] = {
|
|
hostname: "installed-node",
|
|
role: "worker",
|
|
ip: "10.0.0.50",
|
|
installed_at: new Date().toISOString(),
|
|
};
|
|
});
|
|
|
|
const response = await app.inject({
|
|
method: "GET",
|
|
url: `/dispatch?mac=${mac}`,
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
const body = response.body;
|
|
expect(body).toContain("#!ipxe");
|
|
expect(body).toContain("installed-node");
|
|
expect(body).toContain("Already installed");
|
|
expect(body).toContain("exit");
|
|
});
|
|
|
|
it("progress endpoint updates state", async () => {
|
|
const mac = "aa:bb:cc:dd:ee:ff";
|
|
state.update((s) => {
|
|
s.install_queue[mac] = {
|
|
hostname: "worker-1",
|
|
disk: "/dev/sda",
|
|
role: "worker",
|
|
queued_at: new Date().toISOString(),
|
|
};
|
|
});
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: "/api/progress",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({
|
|
mac,
|
|
stage: "post-install",
|
|
detail: "configuring system",
|
|
}),
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
const result = JSON.parse(response.body);
|
|
expect(result.status).toBe("ok");
|
|
|
|
// Verify state was updated
|
|
const currentState = state.load();
|
|
expect(currentState.install_queue[mac]?.progress).toBe("post-install");
|
|
expect(currentState.install_queue[mac]?.progress_detail).toBe("configuring system");
|
|
});
|
|
|
|
it("progress endpoint with 'complete' stage moves machine to installed", async () => {
|
|
const mac = "aa:bb:cc:dd:ee:ff";
|
|
state.update((s) => {
|
|
s.install_queue[mac] = {
|
|
hostname: "worker-1",
|
|
disk: "/dev/sda",
|
|
role: "worker",
|
|
queued_at: new Date().toISOString(),
|
|
};
|
|
});
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: "/api/progress",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({
|
|
mac,
|
|
stage: "complete",
|
|
detail: "ready at 10.0.0.50",
|
|
}),
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
|
|
const currentState = state.load();
|
|
expect(currentState.install_queue[mac]).toBeUndefined();
|
|
expect(currentState.installed[mac]).toBeDefined();
|
|
expect(currentState.installed[mac]?.hostname).toBe("worker-1");
|
|
expect(currentState.installed[mac]?.ip).toBe("10.0.0.50");
|
|
});
|
|
|
|
it("DELETE /api/machines/:mac removes machine from state", async () => {
|
|
const mac = "aa:bb:cc:dd:ee:ff";
|
|
state.update((s) => {
|
|
s.discovered[mac] = {
|
|
mac,
|
|
product: "TestBox",
|
|
board: "TestBoard",
|
|
serial: "SN123",
|
|
manufacturer: "TestCorp",
|
|
cpu_model: "Test CPU",
|
|
cpu_cores: 4,
|
|
memory_gb: 16,
|
|
arch: "x86_64",
|
|
disks: [],
|
|
nics: [],
|
|
first_seen: new Date().toISOString(),
|
|
last_seen: new Date().toISOString(),
|
|
};
|
|
});
|
|
|
|
const response = await app.inject({
|
|
method: "DELETE",
|
|
url: `/api/machines/${encodeURIComponent(mac)}`,
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
const result = JSON.parse(response.body);
|
|
expect(result.status).toBe("forgotten");
|
|
|
|
const currentState = state.load();
|
|
expect(currentState.discovered[mac]).toBeUndefined();
|
|
});
|
|
|
|
it("DELETE /api/machines/:mac returns 404 for unknown machine", async () => {
|
|
const response = await app.inject({
|
|
method: "DELETE",
|
|
url: "/api/machines/ff:ff:ff:ff:ff:ff",
|
|
});
|
|
|
|
expect(response.statusCode).toBe(404);
|
|
const result = JSON.parse(response.body);
|
|
expect(result.error).toBe("machine not found");
|
|
});
|
|
|
|
it("POST /api/log accepts a single line", async () => {
|
|
const mac = "aa:bb:cc:dd:ee:ff";
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: "/api/log",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ mac, line: "hello from kickstart" }),
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
const result = JSON.parse(response.body);
|
|
expect(result.status).toBe("ok");
|
|
expect(result.lines).toBe(1);
|
|
|
|
// Verify line is stored
|
|
const lines = installLog.getLines(mac);
|
|
expect(lines).toHaveLength(1);
|
|
expect(lines[0]!.line).toBe("hello from kickstart");
|
|
});
|
|
|
|
it("POST /api/log accepts multiple lines", async () => {
|
|
const mac = "aa:bb:cc:dd:ee:ff";
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: "/api/log",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ mac, lines: ["line 1", "line 2", "line 3"] }),
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
const result = JSON.parse(response.body);
|
|
expect(result.lines).toBe(3);
|
|
|
|
const lines = installLog.getLines(mac);
|
|
expect(lines).toHaveLength(3);
|
|
});
|
|
|
|
it("GET /api/logs/:mac includes log lines for installing machine", async () => {
|
|
const mac = "aa:bb:cc:dd:ee:ff";
|
|
state.update((s) => {
|
|
s.install_queue[mac] = {
|
|
hostname: "test-node",
|
|
disk: "/dev/sda",
|
|
role: "worker",
|
|
queued_at: new Date().toISOString(),
|
|
};
|
|
});
|
|
|
|
// Add some log lines
|
|
installLog.append(mac, ["log line 1", "log line 2"], "test-node");
|
|
|
|
const response = await app.inject({
|
|
method: "GET",
|
|
url: `/api/logs/${encodeURIComponent(mac)}`,
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
const result = JSON.parse(response.body);
|
|
expect(result.status).toBe("installing");
|
|
expect(result.log_lines).toHaveLength(2);
|
|
expect(result.log_total).toBe(2);
|
|
expect(result.log_lines[0].line).toBe("log line 1");
|
|
});
|
|
|
|
it("progress endpoint with 'error' stage keeps machine in install_queue", async () => {
|
|
const mac = "aa:bb:cc:dd:ee:ff";
|
|
state.update((s) => {
|
|
s.install_queue[mac] = {
|
|
hostname: "failing-node",
|
|
disk: "/dev/sda",
|
|
role: "worker",
|
|
queued_at: new Date().toISOString(),
|
|
};
|
|
});
|
|
|
|
const response = await app.inject({
|
|
method: "POST",
|
|
url: "/api/progress",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({
|
|
mac,
|
|
stage: "error",
|
|
detail: "%post failed at line 42",
|
|
}),
|
|
});
|
|
|
|
expect(response.statusCode).toBe(200);
|
|
|
|
// Machine should still be in install_queue (not moved to installed)
|
|
const currentState = state.load();
|
|
expect(currentState.install_queue[mac]).toBeDefined();
|
|
expect(currentState.install_queue[mac]?.progress).toBe("error");
|
|
expect(currentState.install_queue[mac]?.progress_detail).toBe("%post failed at line 42");
|
|
expect(currentState.installed[mac]).toBeUndefined();
|
|
});
|
|
});
|