feat: install logging, error trapping, PXE/ISO integration tests
Some checks failed
CI/CD / lint (pull_request) Failing after 13s
CI/CD / test (pull_request) Failing after 10s
CI/CD / typecheck (pull_request) Failing after 36s
CI/CD / build (pull_request) Has been skipped
CI/CD / publish-rpm (pull_request) Has been skipped
CI/CD / publish-deb (pull_request) Has been skipped

Kickstart installs on real hardware failed silently — no error reporting,
only 3 progress callbacks, zero log streaming. This overhaul makes every
install fully observable.

Kickstart improvements:
- Error trapping in %pre and %post (trap ERR sends failure details to bastion)
- 12+ granular progress stages (was 3): SSH, hostname, k3s prep, EFI boot, metadata
- Background log streamer: tails %post output and batch-sends to /api/log
- bastion_log() function for explicit log lines from kickstart scripts

Bastion API:
- POST /api/log — receives raw log lines from kickstart (single or batch)
- InstallLogBuffer — per-MAC ring buffer (2000 lines) + file persistence
- GET /api/logs/:mac — now returns log_lines + log_total alongside stages
- SSE /api/logs/:mac/follow — uses named events (event: stage vs event: log)
- Progress events forwarded to labd via bastion-progress WebSocket message
- Post-provision k3s logs routed through progressBus (was console-only)

dnsmasq fixes found during VM testing:
- HTTP Boot filename: ipxe-real.efi → ipxe.efi (leftover from old 2-stage approach)
- pxe-service directives: only in proxy mode (breaks OVMF PXE in full mode)
- PXEClient vendor class echo for UEFI firmware compatibility

Integration tests:
- PXE boot test: blank UEFI VM → dnsmasq → HTTP Boot → iPXE → bastion → install
- ISO boot test: blank VM boots from bastion-generated ISO → same flow
- Shared helpers: pxe-network (no DHCP, nftables fix), pxe-vm (UEFI + ISO boot)
- test-provision.sh: runs both PXE + ISO tests with prerequisite checks
- 250GB sparse QCOW2 disk (LVM layout needs ~204GB)

201 unit tests passing (11 new).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Michal
2026-03-26 22:26:33 +00:00
parent ffc4a782d2
commit 46b017d77e
189 changed files with 16241 additions and 432 deletions

View File

@@ -154,8 +154,8 @@ function generateFish(root: CmdInfo): string {
const allCmds = collectCommands(root);
// Helper function for fish: test if exactly the given subcommand chain is present
emit('# Helper: test if a subcommand chain is active');
// Helper: test if EXACTLY the given subcommand chain is present (for subcommand suggestions)
emit('# Helper: test if exactly a subcommand chain is active (no extra positional args)');
emit(`function __${BIN}_using_cmd`);
emit(' set -l tokens (commandline -opc)');
emit(' set -l expected $argv');
@@ -181,6 +181,65 @@ function generateFish(root: CmdInfo): string {
emit('end');
emit('');
// Helper: test if command chain STARTS WITH the given prefix (for options that apply after args)
emit('# Helper: test if command starts with a subcommand chain (options still apply after args)');
emit(`function __${BIN}_in_cmd`);
emit(' set -l tokens (commandline -opc)');
emit(' set -l expected $argv');
emit(' set -l depth (count $expected)');
emit(' set -l found 0');
emit(' for tok in $tokens[2..]');
emit(' if string match -q -- "-*" $tok');
emit(' continue');
emit(' end');
emit(' set found (math $found + 1)');
emit(' if test $found -le $depth');
emit(' if test "$tok" != "$expected[$found]"');
emit(' return 1');
emit(' end');
emit(' end');
emit(' end');
emit(' test $found -ge $depth');
emit('end');
emit('');
// Dynamic completions: fetch machine data from bastion API
emit('# Dynamic: fetch machine hostnames from bastion (installed + queued)');
emit(`function __${BIN}_installed_hosts`);
emit(' curl -s http://localhost:8080/api/machines 2>/dev/null | ');
emit(" python3 -c 'import sys,json; d=json.load(sys.stdin); hosts=[v.get(\"hostname\",\"\") for v in {**d.get(\"install_queue\",{}), **d.get(\"installed\",{})}.values() if v.get(\"hostname\")]; [print(h) for h in set(hosts)]' 2>/dev/null");
emit('end');
emit('');
emit('# Dynamic: fetch all known MAC addresses (discovered + queue + installed)');
emit(`function __${BIN}_known_macs`);
emit(' curl -s http://localhost:8080/api/machines 2>/dev/null | ');
emit(" python3 -c 'import sys,json; d=json.load(sys.stdin); [print(k) for k in {**d.get(\"discovered\",{}), **d.get(\"install_queue\",{}), **d.get(\"installed\",{})}]' 2>/dev/null");
emit('end');
emit('');
emit('# Dynamic: fetch hostnames and MACs from all states');
emit(`function __${BIN}_hosts_and_macs`);
emit(' curl -s http://localhost:8080/api/machines 2>/dev/null | ');
emit(" python3 -c 'import sys,json; d=json.load(sys.stdin); a={**d.get(\"discovered\",{}), **d.get(\"install_queue\",{}), **d.get(\"installed\",{})}; macs=list(a.keys()); hosts=[v.get(\"hostname\",\"\") for v in {**d.get(\"install_queue\",{}), **d.get(\"installed\",{})}.values() if v.get(\"hostname\")]; [print(x) for x in set(macs+hosts)]' 2>/dev/null");
emit('end');
emit('');
// Target completions for commands that accept hostname/IP/MAC
emit('# Target argument completions');
// app k3s — takes hostname/IP
emit(`complete -c ${BIN} -n "__${BIN}_using_cmd app k3s install" -a "(__${BIN}_installed_hosts)" -d 'installed host'`);
emit(`complete -c ${BIN} -n "__${BIN}_using_cmd app k3s health" -a "(__${BIN}_installed_hosts)" -d 'installed host'`);
emit(`complete -c ${BIN} -n "__${BIN}_using_cmd app labcontroller deploy" -a "(__${BIN}_installed_hosts)" -d 'installed host'`);
emit(`complete -c ${BIN} -n "__${BIN}_using_cmd app labcontroller status" -a "(__${BIN}_installed_hosts)" -d 'installed host'`);
// provision install — takes MAC then hostname
emit(`complete -c ${BIN} -n "__${BIN}_using_cmd provision install" -a "(__${BIN}_known_macs)" -d 'MAC address'`);
// provision reprovision/forget/logs — takes MAC or hostname
emit(`complete -c ${BIN} -n "__${BIN}_using_cmd provision reprovision" -a "(__${BIN}_hosts_and_macs)" -d 'host or MAC'`);
emit(`complete -c ${BIN} -n "__${BIN}_using_cmd provision forget" -a "(__${BIN}_hosts_and_macs)" -d 'host or MAC'`);
emit(`complete -c ${BIN} -n "__${BIN}_using_cmd provision logs" -a "(__${BIN}_hosts_and_macs)" -d 'host or MAC'`);
emit('');
// Top-level commands
const topCmds = root.subcommands.filter((c) => !c.hidden);
emit('# Top-level commands');
@@ -204,9 +263,9 @@ function generateFish(root: CmdInfo): string {
emit('');
}
// Options for this command
// Options for this command (use __in_cmd so options complete even after positional args)
if (cmd.options.length > 0) {
const condition = `__${BIN}_using_cmd ${path.join(' ')}`;
const condition = `__${BIN}_in_cmd ${path.join(' ')}`;
emit(`# ${path.join(' ')} options`);
for (const opt of cmd.options) {
const parts = [`complete -c ${BIN} -n "${condition}"`];