From 46b017d77e678ac4db25f298ea20162b4e7d8d15 Mon Sep 17 00:00:00 2001 From: Michal Date: Thu, 26 Mar 2026 22:26:33 +0000 Subject: [PATCH] feat: install logging, error trapping, PXE/ISO integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kickstart installs on real hardware failed silently — no error reporting, only 3 progress callbacks, zero log streaming. This overhaul makes every install fully observable. Kickstart improvements: - Error trapping in %pre and %post (trap ERR sends failure details to bastion) - 12+ granular progress stages (was 3): SSH, hostname, k3s prep, EFI boot, metadata - Background log streamer: tails %post output and batch-sends to /api/log - bastion_log() function for explicit log lines from kickstart scripts Bastion API: - POST /api/log — receives raw log lines from kickstart (single or batch) - InstallLogBuffer — per-MAC ring buffer (2000 lines) + file persistence - GET /api/logs/:mac — now returns log_lines + log_total alongside stages - SSE /api/logs/:mac/follow — uses named events (event: stage vs event: log) - Progress events forwarded to labd via bastion-progress WebSocket message - Post-provision k3s logs routed through progressBus (was console-only) dnsmasq fixes found during VM testing: - HTTP Boot filename: ipxe-real.efi → ipxe.efi (leftover from old 2-stage approach) - pxe-service directives: only in proxy mode (breaks OVMF PXE in full mode) - PXEClient vendor class echo for UEFI firmware compatibility Integration tests: - PXE boot test: blank UEFI VM → dnsmasq → HTTP Boot → iPXE → bastion → install - ISO boot test: blank VM boots from bastion-generated ISO → same flow - Shared helpers: pxe-network (no DHCP, nftables fix), pxe-vm (UEFI + ISO boot) - test-provision.sh: runs both PXE + ISO tests with prerequisite checks - 250GB sparse QCOW2 disk (LVM layout needs ~204GB) 201 unit tests passing (11 new). Co-Authored-By: Claude Opus 4.6 (1M context) --- .env.example | 12 + .gitignore | 25 + .mcp.json | 12 + .taskmaster/config.json | 19 +- .taskmaster/state.json | 6 + .taskmaster/tasks/tasks.json | 180 +++++ .taskmaster/templates/example_prd.txt | 47 ++ .taskmaster/templates/example_prd_rpg.txt | 511 +++++++++++++ STATUS.md | 244 +++++++ bastion/.dockerignore | 8 + .../.taskmaster/docs/pulumi-k3s-refactor.md | 132 ++++ bastion/.taskmaster/docs/resource-tracking.md | 172 +++++ bastion/Dockerfile.bastion | 93 +++ bastion/Dockerfile.labd | 73 ++ bastion/completions/labctl.bash | 64 +- bastion/completions/labctl.fish | 163 ++++- bastion/deploy/k3s/configmap.yaml | 1 + bastion/deploy/k3s/deployment.yaml | 27 +- bastion/deploy/k8s/labd/base/configmap.yaml | 8 + bastion/deploy/k8s/labd/base/deployment.yaml | 44 ++ bastion/deploy/k8s/labd/base/hpa.yaml | 18 + .../deploy/k8s/labd/base/kustomization.yaml | 14 + bastion/deploy/k8s/labd/base/pdb.yaml | 9 + bastion/deploy/k8s/labd/base/service.yaml | 12 + bastion/package.json | 9 +- bastion/pnpm-lock.yaml | 679 ++++++++++++++++++ bastion/scripts/build-bastion.sh | 109 +-- bastion/scripts/build-labd.sh | 118 +++ bastion/scripts/generate-completions.ts | 67 +- bastion/scripts/test-integration.sh | 71 ++ bastion/scripts/test-provision.sh | 131 ++++ bastion/src/bastion/package.json | 11 +- bastion/src/bastion/src/config.ts | 8 + bastion/src/bastion/src/main.ts | 93 ++- bastion/src/bastion/src/routes/api.ts | 207 +++++- bastion/src/bastion/src/routes/boot-iso.ts | 249 +++++++ bastion/src/bastion/src/routes/dispatch.ts | 31 +- bastion/src/bastion/src/routes/kickstart.ts | 39 +- bastion/src/bastion/src/server.ts | 12 +- .../src/bastion/src/services/install-log.ts | 86 +++ .../src/bastion/src/services/iso-builder.ts | 437 +++++++++++ .../src/services/kickstart-generator.ts | 4 +- .../bastion/src/services/labd-connection.ts | 252 +++++++ .../bastion/src/services/post-provision.ts | 233 ++++++ .../bastion/src/services/progress-events.ts | 28 + bastion/src/bastion/src/services/state.ts | 12 + .../src/bastion/src/templates/dnsmasq.conf.ts | 12 +- .../src/bastion/src/templates/install.ks.ts | 173 ++++- .../src/templates/ubuntu-autoinstall.ts | 299 ++++++++ .../bastion/src/templates/ubuntu-boot.ipxe.ts | 24 + bastion/src/bastion/tests/dispatch.test.ts | 101 +++ bastion/src/bastion/tests/kickstart.test.ts | 52 +- bastion/src/bastion/tsconfig.json | 3 +- bastion/src/cli/package.json | 7 +- bastion/src/cli/src/api/client.ts | 161 +++++ bastion/src/cli/src/api/config.ts | 47 ++ bastion/src/cli/src/api/errors.ts | 59 ++ bastion/src/cli/src/api/index.ts | 18 + bastion/src/cli/src/api/types.ts | 96 +++ bastion/src/cli/src/api/websocket.ts | 160 +++++ bastion/src/cli/src/commands/app.ts | 403 +++++++++++ bastion/src/cli/src/commands/config.ts | 76 ++ bastion/src/cli/src/commands/doctor.ts | 126 ++++ bastion/src/cli/src/commands/forget.ts | 26 +- bastion/src/cli/src/commands/install.ts | 69 +- bastion/src/cli/src/commands/labcontroller.ts | 298 ++++++++ bastion/src/cli/src/commands/list.ts | 13 +- bastion/src/cli/src/commands/login.ts | 120 ++++ bastion/src/cli/src/commands/logs.ts | 85 +++ bastion/src/cli/src/commands/makeiso.ts | 114 +++ bastion/src/cli/src/commands/reprovision.ts | 205 ++++-- bastion/src/cli/src/commands/serve.ts | 124 ++-- bastion/src/cli/src/commands/status.ts | 83 +-- bastion/src/cli/src/config/index.ts | 111 +++ bastion/src/cli/src/index.ts | 88 ++- bastion/src/cli/src/utils/index.ts | 27 + bastion/src/cli/src/utils/prompts.ts | 48 ++ bastion/src/cli/src/utils/resource.ts | 129 ++++ bastion/src/cli/src/utils/table.ts | 267 +++++++ bastion/src/cli/tests/api-errors.test.ts | 56 ++ bastion/src/cli/tests/config.test.ts | 53 ++ bastion/src/cli/tests/resource.test.ts | 71 ++ bastion/src/cli/tests/smoke-bastion.test.ts | 197 +++++ bastion/src/cli/tsconfig.json | 3 +- bastion/src/lab-agent/package.json | 24 + bastion/src/lab-agent/src/main.ts | 10 + .../src/lab-agent/src/services/connection.ts | 157 ++++ .../src/lab-agent/src/services/executor.ts | 161 +++++ bastion/src/lab-agent/src/services/logger.ts | 38 + bastion/src/lab-agent/tests/executor.test.ts | 111 +++ bastion/src/lab-agent/tsconfig.json | 12 + bastion/src/labd/package.json | 19 +- bastion/src/labd/prisma/schema.prisma | 11 + bastion/src/labd/prisma/seed.ts | 113 +++ bastion/src/labd/src/main.ts | 66 +- bastion/src/labd/src/middleware/rate-limit.ts | 50 ++ bastion/src/labd/src/routes/agents.ts | 20 + bastion/src/labd/src/routes/bastions.ts | 207 ++++++ bastion/src/labd/src/routes/health.ts | 116 +++ bastion/src/labd/src/server.ts | 177 ++++- .../src/labd/src/services/agent-registry.ts | 65 ++ .../src/labd/src/services/bastion-registry.ts | 107 +++ bastion/src/labd/src/services/encryption.ts | 73 ++ .../src/labd/src/services/message-router.ts | 192 +++++ bastion/src/labd/src/services/shutdown.ts | 98 +++ bastion/src/labd/src/validation/index.ts | 13 + bastion/src/labd/src/validation/middleware.ts | 32 + bastion/src/labd/src/validation/schemas.ts | 37 + bastion/src/labd/tests/agent-registry.test.ts | 112 +++ bastion/src/labd/tests/auth-routes.test.ts | 208 ++++++ bastion/src/labd/tests/encryption.test.ts | 63 ++ bastion/src/labd/tests/validation.test.ts | 123 ++++ bastion/src/modules/modules/k3s/module.yaml | 6 + .../src/modules/modules/k3s/src/configure.ts | 117 +++ .../modules/k3s/src/groups/hardening.ts | 22 + .../modules/k3s/src/groups/host-prep.ts | 26 + .../modules/modules/k3s/src/groups/index.ts | 5 + .../modules/k3s/src/groups/k3s-agent.ts | 20 + .../modules/k3s/src/groups/k3s-server.ts | 24 + .../modules/k3s/src/groups/networking.ts | 22 + bastion/src/modules/modules/k3s/src/health.ts | 56 ++ .../modules/k3s/src/health/api-health.ts | 8 + .../modules/k3s/src/health/cilium-status.ts | 16 + .../modules/modules/k3s/src/health/index.ts | 6 + .../modules/k3s/src/health/k3s-service.ts | 9 + .../modules/k3s/src/health/node-ready.ts | 11 + .../modules/k3s/src/health/pod-status.ts | 20 + .../k3s/src/health/secrets-encryption.ts | 8 + bastion/src/modules/modules/k3s/src/index.ts | 32 + .../src/modules/modules/k3s/src/install.ts | 275 +++++++ .../src/modules/modules/k3s/src/k3s-module.ts | 112 +++ .../k3s/src/operations/audit-policy.ts | 43 ++ .../modules/k3s/src/operations/cert-check.ts | 30 + .../modules/k3s/src/operations/cilium.ts | 78 ++ .../modules/k3s/src/operations/cni-cleanup.ts | 57 ++ .../modules/k3s/src/operations/dns-fix.ts | 50 ++ .../modules/k3s/src/operations/firewall.ts | 38 + .../modules/k3s/src/operations/index.ts | 15 + .../modules/k3s/src/operations/k3s-config.ts | 66 ++ .../modules/k3s/src/operations/k3s-install.ts | 71 ++ .../k3s/src/operations/kernel-modules.ts | 39 + .../k3s/src/operations/log-rotation.ts | 25 + .../k3s/src/operations/network-policy.ts | 50 ++ .../k3s/src/operations/pod-security.ts | 21 + .../modules/k3s/src/operations/selinux.ts | 22 + .../modules/k3s/src/operations/swap.ts | 22 + .../modules/k3s/src/operations/sysctl.ts | 30 + bastion/src/modules/modules/k3s/src/types.ts | 61 ++ bastion/src/modules/modules/k3s/src/utils.ts | 102 +++ .../src/modules/modules/k3s/tests/helpers.ts | 63 ++ .../modules/modules/k3s/tests/install.test.ts | 134 ++++ .../modules/k3s/tests/operations.test.ts | 350 +++++++++ .../modules/modules/k3s/tests/smoke.test.ts | 125 ++++ .../modules/modules/labcontroller/module.yaml | 6 + .../modules/labcontroller/src/bastion.ts | 90 +++ .../modules/labcontroller/src/cockroachdb.ts | 172 +++++ .../modules/labcontroller/src/deploy.ts | 18 + .../modules/labcontroller/src/index.ts | 39 + .../modules/modules/labcontroller/src/labd.ts | 81 +++ bastion/src/modules/package.json | 21 + bastion/src/modules/src/index.ts | 23 + bastion/src/modules/src/registry.ts | 30 + bastion/src/modules/src/runner.ts | 61 ++ bastion/src/modules/src/ssh.d.ts | 18 + bastion/src/modules/src/ssh.d.ts.map | 1 + bastion/src/modules/src/ssh.js | 111 +++ bastion/src/modules/src/ssh.js.map | 1 + bastion/src/modules/src/ssh.ts | 156 ++++ bastion/src/modules/src/types.ts | 38 + bastion/src/modules/tsconfig.json | 13 + bastion/src/shared/src/errors/index.ts | 86 +++ bastion/src/shared/src/index.ts | 36 + bastion/src/shared/src/protocol/index.ts | 170 +++++ bastion/src/shared/src/types/config.ts | 6 + bastion/src/shared/src/types/index.ts | 6 + bastion/src/shared/src/types/state.ts | 65 +- bastion/src/shared/tests/errors.test.ts | 85 +++ bastion/src/shared/tests/protocol.test.ts | 109 +++ bastion/tests/integration/helpers/libvirt.ts | 219 ++++++ bastion/tests/integration/helpers/network.ts | 68 ++ .../tests/integration/helpers/pxe-network.ts | 95 +++ bastion/tests/integration/helpers/pxe-vm.ts | 146 ++++ bastion/tests/integration/helpers/ssh.ts | 106 +++ .../tests/integration/iso-provision.test.ts | 318 ++++++++ .../tests/integration/k3s-single-node.test.ts | 375 ++++++++++ .../tests/integration/pxe-provision.test.ts | 396 ++++++++++ bastion/tests/integration/vitest.config.ts | 17 + bastion/tsconfig.json | 3 +- bastion/vitest.config.ts | 2 +- 189 files changed, 16241 insertions(+), 432 deletions(-) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 .mcp.json create mode 100644 .taskmaster/state.json create mode 100644 .taskmaster/tasks/tasks.json create mode 100644 .taskmaster/templates/example_prd.txt create mode 100644 .taskmaster/templates/example_prd_rpg.txt create mode 100644 STATUS.md create mode 100644 bastion/.dockerignore create mode 100644 bastion/.taskmaster/docs/pulumi-k3s-refactor.md create mode 100644 bastion/.taskmaster/docs/resource-tracking.md create mode 100644 bastion/Dockerfile.bastion create mode 100644 bastion/Dockerfile.labd create mode 100644 bastion/deploy/k8s/labd/base/configmap.yaml create mode 100644 bastion/deploy/k8s/labd/base/deployment.yaml create mode 100644 bastion/deploy/k8s/labd/base/hpa.yaml create mode 100644 bastion/deploy/k8s/labd/base/kustomization.yaml create mode 100644 bastion/deploy/k8s/labd/base/pdb.yaml create mode 100644 bastion/deploy/k8s/labd/base/service.yaml create mode 100755 bastion/scripts/build-labd.sh create mode 100755 bastion/scripts/test-integration.sh create mode 100755 bastion/scripts/test-provision.sh create mode 100644 bastion/src/bastion/src/routes/boot-iso.ts create mode 100644 bastion/src/bastion/src/services/install-log.ts create mode 100644 bastion/src/bastion/src/services/iso-builder.ts create mode 100644 bastion/src/bastion/src/services/labd-connection.ts create mode 100644 bastion/src/bastion/src/services/post-provision.ts create mode 100644 bastion/src/bastion/src/services/progress-events.ts create mode 100644 bastion/src/bastion/src/templates/ubuntu-autoinstall.ts create mode 100644 bastion/src/bastion/src/templates/ubuntu-boot.ipxe.ts create mode 100644 bastion/src/cli/src/api/client.ts create mode 100644 bastion/src/cli/src/api/config.ts create mode 100644 bastion/src/cli/src/api/errors.ts create mode 100644 bastion/src/cli/src/api/index.ts create mode 100644 bastion/src/cli/src/api/types.ts create mode 100644 bastion/src/cli/src/api/websocket.ts create mode 100644 bastion/src/cli/src/commands/app.ts create mode 100644 bastion/src/cli/src/commands/config.ts create mode 100644 bastion/src/cli/src/commands/doctor.ts create mode 100644 bastion/src/cli/src/commands/labcontroller.ts create mode 100644 bastion/src/cli/src/commands/login.ts create mode 100644 bastion/src/cli/src/commands/logs.ts create mode 100644 bastion/src/cli/src/commands/makeiso.ts create mode 100644 bastion/src/cli/src/config/index.ts create mode 100644 bastion/src/cli/src/utils/index.ts create mode 100644 bastion/src/cli/src/utils/prompts.ts create mode 100644 bastion/src/cli/src/utils/resource.ts create mode 100644 bastion/src/cli/src/utils/table.ts create mode 100644 bastion/src/cli/tests/api-errors.test.ts create mode 100644 bastion/src/cli/tests/config.test.ts create mode 100644 bastion/src/cli/tests/resource.test.ts create mode 100644 bastion/src/cli/tests/smoke-bastion.test.ts create mode 100644 bastion/src/lab-agent/package.json create mode 100644 bastion/src/lab-agent/src/main.ts create mode 100644 bastion/src/lab-agent/src/services/connection.ts create mode 100644 bastion/src/lab-agent/src/services/executor.ts create mode 100644 bastion/src/lab-agent/src/services/logger.ts create mode 100644 bastion/src/lab-agent/tests/executor.test.ts create mode 100644 bastion/src/lab-agent/tsconfig.json create mode 100644 bastion/src/labd/prisma/seed.ts create mode 100644 bastion/src/labd/src/middleware/rate-limit.ts create mode 100644 bastion/src/labd/src/routes/agents.ts create mode 100644 bastion/src/labd/src/routes/bastions.ts create mode 100644 bastion/src/labd/src/services/agent-registry.ts create mode 100644 bastion/src/labd/src/services/bastion-registry.ts create mode 100644 bastion/src/labd/src/services/encryption.ts create mode 100644 bastion/src/labd/src/services/message-router.ts create mode 100644 bastion/src/labd/src/services/shutdown.ts create mode 100644 bastion/src/labd/src/validation/index.ts create mode 100644 bastion/src/labd/src/validation/middleware.ts create mode 100644 bastion/src/labd/src/validation/schemas.ts create mode 100644 bastion/src/labd/tests/agent-registry.test.ts create mode 100644 bastion/src/labd/tests/auth-routes.test.ts create mode 100644 bastion/src/labd/tests/encryption.test.ts create mode 100644 bastion/src/labd/tests/validation.test.ts create mode 100644 bastion/src/modules/modules/k3s/module.yaml create mode 100644 bastion/src/modules/modules/k3s/src/configure.ts create mode 100644 bastion/src/modules/modules/k3s/src/groups/hardening.ts create mode 100644 bastion/src/modules/modules/k3s/src/groups/host-prep.ts create mode 100644 bastion/src/modules/modules/k3s/src/groups/index.ts create mode 100644 bastion/src/modules/modules/k3s/src/groups/k3s-agent.ts create mode 100644 bastion/src/modules/modules/k3s/src/groups/k3s-server.ts create mode 100644 bastion/src/modules/modules/k3s/src/groups/networking.ts create mode 100644 bastion/src/modules/modules/k3s/src/health.ts create mode 100644 bastion/src/modules/modules/k3s/src/health/api-health.ts create mode 100644 bastion/src/modules/modules/k3s/src/health/cilium-status.ts create mode 100644 bastion/src/modules/modules/k3s/src/health/index.ts create mode 100644 bastion/src/modules/modules/k3s/src/health/k3s-service.ts create mode 100644 bastion/src/modules/modules/k3s/src/health/node-ready.ts create mode 100644 bastion/src/modules/modules/k3s/src/health/pod-status.ts create mode 100644 bastion/src/modules/modules/k3s/src/health/secrets-encryption.ts create mode 100644 bastion/src/modules/modules/k3s/src/index.ts create mode 100644 bastion/src/modules/modules/k3s/src/install.ts create mode 100644 bastion/src/modules/modules/k3s/src/k3s-module.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/audit-policy.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/cert-check.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/cilium.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/cni-cleanup.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/dns-fix.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/firewall.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/index.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/k3s-config.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/k3s-install.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/kernel-modules.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/log-rotation.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/network-policy.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/pod-security.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/selinux.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/swap.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/sysctl.ts create mode 100644 bastion/src/modules/modules/k3s/src/types.ts create mode 100644 bastion/src/modules/modules/k3s/src/utils.ts create mode 100644 bastion/src/modules/modules/k3s/tests/helpers.ts create mode 100644 bastion/src/modules/modules/k3s/tests/install.test.ts create mode 100644 bastion/src/modules/modules/k3s/tests/operations.test.ts create mode 100644 bastion/src/modules/modules/k3s/tests/smoke.test.ts create mode 100644 bastion/src/modules/modules/labcontroller/module.yaml create mode 100644 bastion/src/modules/modules/labcontroller/src/bastion.ts create mode 100644 bastion/src/modules/modules/labcontroller/src/cockroachdb.ts create mode 100644 bastion/src/modules/modules/labcontroller/src/deploy.ts create mode 100644 bastion/src/modules/modules/labcontroller/src/index.ts create mode 100644 bastion/src/modules/modules/labcontroller/src/labd.ts create mode 100644 bastion/src/modules/package.json create mode 100644 bastion/src/modules/src/index.ts create mode 100644 bastion/src/modules/src/registry.ts create mode 100644 bastion/src/modules/src/runner.ts create mode 100644 bastion/src/modules/src/ssh.d.ts create mode 100644 bastion/src/modules/src/ssh.d.ts.map create mode 100644 bastion/src/modules/src/ssh.js create mode 100644 bastion/src/modules/src/ssh.js.map create mode 100644 bastion/src/modules/src/ssh.ts create mode 100644 bastion/src/modules/src/types.ts create mode 100644 bastion/src/modules/tsconfig.json create mode 100644 bastion/src/shared/src/errors/index.ts create mode 100644 bastion/src/shared/src/protocol/index.ts create mode 100644 bastion/src/shared/tests/errors.test.ts create mode 100644 bastion/src/shared/tests/protocol.test.ts create mode 100644 bastion/tests/integration/helpers/libvirt.ts create mode 100644 bastion/tests/integration/helpers/network.ts create mode 100644 bastion/tests/integration/helpers/pxe-network.ts create mode 100644 bastion/tests/integration/helpers/pxe-vm.ts create mode 100644 bastion/tests/integration/helpers/ssh.ts create mode 100644 bastion/tests/integration/iso-provision.test.ts create mode 100644 bastion/tests/integration/k3s-single-node.test.ts create mode 100644 bastion/tests/integration/pxe-provision.test.ts create mode 100644 bastion/tests/integration/vitest.config.ts diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..60bd23e --- /dev/null +++ b/.env.example @@ -0,0 +1,12 @@ +# API Keys (Required to enable respective provider) +ANTHROPIC_API_KEY="your_anthropic_api_key_here" # Required: Format: sk-ant-api03-... +PERPLEXITY_API_KEY="your_perplexity_api_key_here" # Optional: Format: pplx-... +OPENAI_API_KEY="your_openai_api_key_here" # Optional, for OpenAI models. Format: sk-proj-... +GOOGLE_API_KEY="your_google_api_key_here" # Optional, for Google Gemini models. +MISTRAL_API_KEY="your_mistral_key_here" # Optional, for Mistral AI models. +XAI_API_KEY="YOUR_XAI_KEY_HERE" # Optional, for xAI AI models. +GROQ_API_KEY="YOUR_GROQ_KEY_HERE" # Optional, for Groq models. +OPENROUTER_API_KEY="YOUR_OPENROUTER_KEY_HERE" # Optional, for OpenRouter models. +AZURE_OPENAI_API_KEY="your_azure_key_here" # Optional, for Azure OpenAI models (requires endpoint in .taskmaster/config.json). +OLLAMA_API_KEY="your_ollama_api_key_here" # Optional: For remote Ollama servers that require authentication. +GITHUB_API_KEY="your_github_api_key_here" # Optional: For GitHub import/export features. Format: ghp_... or github_pat_... \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f270674 --- /dev/null +++ b/.gitignore @@ -0,0 +1,25 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +dev-debug.log + +# Dependency directories +node_modules/ + +# Environment variables +.env + +# Editor directories and files +.idea +.vscode +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? + +# OS specific +.DS_Store diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..f505dc7 --- /dev/null +++ b/.mcp.json @@ -0,0 +1,12 @@ +{ + "mcpServers": { + "labctl": { + "command": "mcpctl", + "args": [ + "mcp", + "-p", + "labctl" + ] + } + } +} diff --git a/.taskmaster/config.json b/.taskmaster/config.json index 0f790da..f026c1d 100644 --- a/.taskmaster/config.json +++ b/.taskmaster/config.json @@ -1,22 +1,21 @@ { "models": { "main": { - "provider": "anthropic", - "modelId": "claude-sonnet-4-20250514", - "maxTokens": 64000, + "provider": "claude-code", + "modelId": "opus", + "maxTokens": 32000, "temperature": 0.2 }, "research": { - "provider": "anthropic", - "modelId": "claude-sonnet-4-20250514", - "maxTokens": 64000, + "provider": "claude-code", + "modelId": "opus", + "maxTokens": 32000, "temperature": 0.2 }, - "resolution": "main", "fallback": { - "provider": "anthropic", - "modelId": "claude-3-7-sonnet-20250219", - "maxTokens": 120000, + "provider": "claude-code", + "modelId": "sonnet", + "maxTokens": 64000, "temperature": 0.2 } }, diff --git a/.taskmaster/state.json b/.taskmaster/state.json new file mode 100644 index 0000000..e0fdc3a --- /dev/null +++ b/.taskmaster/state.json @@ -0,0 +1,6 @@ +{ + "currentTag": "master", + "lastSwitched": "2026-03-18T00:17:54.213Z", + "branchTagMapping": {}, + "migrationNoticeShown": true +} \ No newline at end of file diff --git a/.taskmaster/tasks/tasks.json b/.taskmaster/tasks/tasks.json new file mode 100644 index 0000000..57fc906 --- /dev/null +++ b/.taskmaster/tasks/tasks.json @@ -0,0 +1,180 @@ +{ + "master": { + "tasks": [ + { + "id": 72, + "title": "Expand Prisma Schema with Resource Relationships", + "description": "Add Network, ServerNic, ServerDisk, and ClusterMember models to the Prisma schema. Add bastionId foreign key to Server model to track which bastion owns each server.", + "details": "Edit `bastion/src/labd/prisma/schema.prisma` to add:\n\n1. **Server model changes**:\n - Add `bastionId String?` with relation to Bastion\n - Add `hardwareInfo Json?` for storing raw HardwareInfo\n - Add `os String?` for installed OS\n\n2. **Network model**:\n```prisma\nmodel Network {\n id String @id @default(uuid())\n name String @unique\n cidr String\n vlan Int?\n gateway String?\n domain String?\n dhcpEnabled Boolean @default(false)\n createdAt DateTime @default(now())\n updatedAt DateTime @updatedAt\n \n nics ServerNic[]\n}\n```\n\n3. **ServerNic model**:\n```prisma\nmodel ServerNic {\n id String @id @default(uuid())\n serverId String\n server Server @relation(fields: [serverId], references: [id], onDelete: Cascade)\n networkId String?\n network Network? @relation(fields: [networkId], references: [id])\n mac String\n ip String?\n name String\n state String @default(\"DOWN\")\n \n @@unique([serverId, mac])\n @@index([networkId])\n}\n```\n\n4. **ServerDisk model**:\n```prisma\nmodel ServerDisk {\n id String @id @default(uuid())\n serverId String\n server Server @relation(fields: [serverId], references: [id], onDelete: Cascade)\n name String\n sizeGb Float\n model String?\n \n @@unique([serverId, name])\n}\n```\n\n5. **ClusterMember model**:\n```prisma\nmodel ClusterMember {\n id String @id @default(uuid())\n clusterId String\n cluster Cluster @relation(fields: [clusterId], references: [id], onDelete: Cascade)\n serverId String\n server Server @relation(fields: [serverId], references: [id], onDelete: Cascade)\n role String @default(\"worker\") // control-plane, worker\n joinedAt DateTime @default(now())\n \n @@unique([clusterId, serverId])\n @@index([clusterId])\n @@index([serverId])\n}\n```\n\n6. Update Server model with relations to nics, disks, clusterMemberships, and bastion.\n\nRun `pnpm prisma generate` and `pnpm prisma migrate dev --name add-resource-models`.", + "testStrategy": "1. Run `pnpm prisma validate` to verify schema syntax\n2. Run `pnpm prisma generate` to confirm client generation\n3. Create migration and verify it applies cleanly to local CockroachDB\n4. Write unit tests that create/read/delete each new model\n5. Verify cascade deletes work (deleting Server removes its NICs and Disks)", + "priority": "high", + "dependencies": [], + "status": "pending", + "subtasks": [] + }, + { + "id": 73, + "title": "Implement State Persistence Service in labd", + "description": "Create a new service in labd that persists bastion state syncs to the Server table in CockroachDB. When bastion-state-sync messages arrive, upsert machines into Server with their hardware info, status, and ownership.", + "details": "Create `bastion/src/labd/src/services/state-persistence.ts`:\n\n```typescript\nimport type { PrismaClient } from \"@prisma/client\";\nimport type { BastionState, HardwareInfo, InstallConfig, InstalledInfo } from \"@lab/shared\";\nimport { logger } from \"./logger.js\";\n\nexport class StatePersistence {\n constructor(private readonly db: PrismaClient) {}\n\n async syncBastionState(bastionId: string, state: BastionState): Promise {\n // Process discovered machines\n for (const [mac, hw] of Object.entries(state.discovered)) {\n await this.upsertDiscoveredServer(bastionId, mac, hw);\n }\n \n // Process queued machines (update status to provisioning)\n for (const [mac, cfg] of Object.entries(state.install_queue)) {\n await this.upsertQueuedServer(bastionId, mac, cfg);\n }\n \n // Process installed machines\n for (const [mac, info] of Object.entries(state.installed)) {\n await this.upsertInstalledServer(bastionId, mac, info);\n }\n }\n\n private async upsertDiscoveredServer(bastionId: string, mac: string, hw: HardwareInfo): Promise {\n const normalized = mac.toLowerCase();\n \n await this.db.server.upsert({\n where: { mac: normalized },\n create: {\n hostname: `unknown-${normalized.replace(/:/g, \"\").slice(-6)}`,\n mac: normalized,\n bastionId,\n status: \"discovered\",\n hardwareInfo: hw as any,\n labels: {\n arch: hw.arch,\n cpu_model: hw.cpu_model,\n cpu_cores: hw.cpu_cores,\n memory_gb: hw.memory_gb,\n },\n },\n update: {\n bastionId,\n status: \"discovered\", // only if not already provisioning/installed\n hardwareInfo: hw as any,\n },\n });\n \n // Sync NICs and Disks\n await this.syncServerHardware(normalized, hw);\n }\n \n private async syncServerHardware(mac: string, hw: HardwareInfo): Promise {\n const server = await this.db.server.findUnique({ where: { mac } });\n if (!server) return;\n \n // Upsert NICs\n for (const nic of hw.nics) {\n await this.db.serverNic.upsert({\n where: { serverId_mac: { serverId: server.id, mac: nic.mac.toLowerCase() } },\n create: { serverId: server.id, mac: nic.mac.toLowerCase(), name: nic.name, state: nic.state },\n update: { name: nic.name, state: nic.state },\n });\n }\n \n // Upsert Disks\n for (const disk of hw.disks) {\n await this.db.serverDisk.upsert({\n where: { serverId_name: { serverId: server.id, name: disk.name } },\n create: { serverId: server.id, name: disk.name, sizeGb: disk.size_gb, model: disk.model },\n update: { sizeGb: disk.size_gb, model: disk.model },\n });\n }\n }\n \n // Similar methods for upsertQueuedServer and upsertInstalledServer...\n}\n```\n\nIntegrate into `server.ts` WebSocket handler by calling `statePersistence.syncBastionState()` when `bastion-state-sync` messages arrive.", + "testStrategy": "1. Unit test StatePersistence with mocked PrismaClient\n2. Integration test: simulate bastion-state-sync message, verify Server rows created\n3. Test idempotency: send same state twice, verify no duplicates\n4. Test status transitions: discovered -> provisioning -> installed\n5. Verify hardware info (NICs, Disks) is correctly persisted", + "priority": "high", + "dependencies": [ + 72 + ], + "status": "pending", + "subtasks": [] + }, + { + "id": 74, + "title": "Add State Loading from labd on Bastion Startup", + "description": "Modify bastion startup to request its persisted state from labd before using the local JSON cache. This ensures bastions restore their state after pod restarts.", + "details": "1. Add new labd API endpoint `GET /api/bastions/:id/state` that returns the aggregated state for a specific bastion from the Server table:\n\n```typescript\n// bastion/src/labd/src/routes/bastions.ts\napp.get<{ Params: { id: string } }>(\"/api/bastions/:id/state\", async (request, reply) => {\n const { id } = request.params;\n \n const servers = await db.server.findMany({\n where: { bastionId: id },\n include: { nics: true, disks: true },\n });\n \n // Transform back to BastionState format\n const state: BastionState = { discovered: {}, install_queue: {}, installed: {} };\n for (const server of servers) {\n const mac = server.mac;\n if (!mac) continue;\n \n switch (server.status) {\n case \"discovered\":\n state.discovered[mac] = transformToHardwareInfo(server);\n break;\n case \"provisioning\":\n state.install_queue[mac] = transformToInstallConfig(server);\n break;\n case \"installed\":\n state.installed[mac] = transformToInstalledInfo(server);\n break;\n }\n }\n \n return reply.send(state);\n});\n```\n\n2. Modify `BastionConnection.connect()` in `labd-connection.ts` to fetch state after enrollment:\n\n```typescript\nprivate async loadRemoteState(): Promise {\n if (!this.bastionId || !this.config.labdUrl) return null;\n try {\n const resp = await fetch(`${this.config.labdUrl}/api/bastions/${this.bastionId}/state`);\n if (resp.ok) return await resp.json();\n } catch { /* fall back to local */ }\n return null;\n}\n```\n\n3. In bastion `main.ts`, after establishing labd connection, merge remote state with local state (remote takes precedence for installed machines, local wins for in-progress installs).", + "testStrategy": "1. Integration test: start bastion, let it persist state, restart bastion, verify state restored\n2. Test merge logic: local has in-progress install, remote has discovered - verify install preserved\n3. Test offline mode: labd unavailable, bastion falls back to local JSON\n4. Test fresh start: no local state, no remote state - bastion starts with empty state", + "priority": "high", + "dependencies": [ + 73 + ], + "status": "pending", + "subtasks": [] + }, + { + "id": 75, + "title": "Fix Bastion --dir Environment Variable Default", + "description": "Fix the bug where CLI's --dir default overrides the BASTION_DIR environment variable. The CLI option should use the env var as its default.", + "details": "Edit `bastion/src/cli/src/commands/serve.ts`:\n\n```typescript\n// Before (line 14):\n.option(\"--dir \", \"Bastion data directory\", \"/tmp/lab-bastion\")\n\n// After:\n.option(\n \"--dir \",\n \"Bastion data directory\",\n process.env[\"BASTION_DIR\"] ?? \"/tmp/lab-bastion\"\n)\n```\n\nThis ensures:\n1. If `BASTION_DIR` env var is set (e.g., in k8s deployment), it's used as default\n2. Explicit `--dir` flag still overrides both\n3. Falls back to `/tmp/lab-bastion` if neither is set\n\nAlso update the k8s deployment manifest `bastion/deploy/k3s/deployment.yaml` to ensure `BASTION_DIR=/data` is properly set.", + "testStrategy": "1. Unit test: verify option default reads from process.env\n2. Integration test: set BASTION_DIR, run labctl without --dir, verify correct dir used\n3. Integration test: set BASTION_DIR, run labctl with --dir /custom, verify /custom used\n4. Test no env var: verify default /tmp/lab-bastion used", + "priority": "high", + "dependencies": [], + "status": "pending", + "subtasks": [] + }, + { + "id": 76, + "title": "Create Resource Type Registry with Aliases", + "description": "Create a centralized resource type registry that maps resource names, plurals, and short aliases to canonical types. This enables kubectl-style resource resolution.", + "details": "Create `bastion/src/cli/src/utils/resources.ts`:\n\n```typescript\nexport interface ResourceDefinition {\n kind: string; // Canonical type: \"Server\", \"Cluster\", etc.\n singular: string; // \"server\"\n plural: string; // \"servers\"\n aliases: string[]; // [\"srv\"]\n apiPath: string; // \"/api/servers\"\n columns: TableColumn[]; // Default columns for 'get' output\n wideColumns?: TableColumn[]; // Extra columns for -o wide\n}\n\nconst RESOURCE_DEFINITIONS: ResourceDefinition[] = [\n {\n kind: \"Server\",\n singular: \"server\",\n plural: \"servers\",\n aliases: [\"srv\"],\n apiPath: \"/api/servers\",\n columns: serverColumns,\n wideColumns: serverWideColumns,\n },\n {\n kind: \"Cluster\",\n singular: \"cluster\",\n plural: \"clusters\",\n aliases: [],\n apiPath: \"/api/clusters\",\n columns: clusterColumns,\n },\n {\n kind: \"Network\",\n singular: \"network\",\n plural: \"networks\",\n aliases: [\"net\"],\n apiPath: \"/api/networks\",\n columns: networkColumns,\n },\n // ... bastion, role, user, token, audit\n];\n\nconst aliasMap = new Map();\nfor (const def of RESOURCE_DEFINITIONS) {\n aliasMap.set(def.singular, def);\n aliasMap.set(def.plural, def);\n for (const alias of def.aliases) {\n aliasMap.set(alias, def);\n }\n}\n\nexport function resolveResourceType(input: string): ResourceDefinition {\n const normalized = input.toLowerCase();\n const def = aliasMap.get(normalized);\n if (!def) {\n const valid = RESOURCE_DEFINITIONS.map(d => d.plural).join(\", \");\n throw new Error(`Unknown resource type \"${input}\". Valid types: ${valid}`);\n }\n return def;\n}\n\nexport function resolveResourceIdentifier(input: string): {\n type: ResourceDefinition;\n name?: string;\n} {\n // Handle \"server/labmaster\" or just \"servers\"\n const parts = input.split(\"/\");\n const type = resolveResourceType(parts[0]);\n const name = parts.length > 1 ? parts.slice(1).join(\"/\") : undefined;\n return { type, name };\n}\n```\n\nUpdate `bastion/src/cli/src/utils/resource.ts` to use the new registry.", + "testStrategy": "1. Unit test resolveResourceType with all aliases: server, servers, srv -> Server\n2. Test unknown resource type throws descriptive error\n3. Test case insensitivity: SERVER, Server, server all resolve correctly\n4. Test resolveResourceIdentifier parses \"server/labmaster\" correctly", + "priority": "high", + "dependencies": [], + "status": "pending", + "subtasks": [] + }, + { + "id": 77, + "title": "Implement 'labctl get' Command", + "description": "Create the core 'labctl get [name]' command that lists resources with filtering and output format support. This is the foundation of the kubectl-style CLI.", + "details": "Create `bastion/src/cli/src/commands/get.ts`:\n\n```typescript\nimport { Command } from \"commander\";\nimport { resolveResourceType, type ResourceDefinition } from \"../utils/resources.js\";\nimport { getLabdClient } from \"../api/config.js\";\nimport { formatOutput, type TableColumn } from \"../utils/table.js\";\n\nexport function registerGetCommand(program: Command): void {\n program\n .command(\"get [name]\")\n .description(\"List resources or get a specific resource by name\")\n .option(\"--status \", \"Filter by status\")\n .option(\"--role \", \"Filter by role (servers only)\")\n .option(\"--cloud \", \"Filter by cloud\")\n .option(\"--env \", \"Filter by environment\")\n .option(\"-l, --label