Compare commits
1 Commits
main
...
816736793d
| Author | SHA1 | Date | |
|---|---|---|---|
|
816736793d |
14
.gitignore
vendored
14
.gitignore
vendored
@@ -23,17 +23,3 @@ node_modules/
|
||||
|
||||
# OS specific
|
||||
.DS_Store
|
||||
|
||||
# Task files
|
||||
# tasks.json
|
||||
# tasks/
|
||||
|
||||
# Asahi build artifacts (large)
|
||||
bastion/.asahi-cache/
|
||||
bastion/asahi-repo/*.zip
|
||||
|
||||
# Regenerated by labsim/dualstack-lab.sh; derived state, not source.
|
||||
labsim/dualstack-evidence/
|
||||
|
||||
# Runtime snapshots from labsim/cilium-ipam-switch.sh
|
||||
labsim/.ipam-switch-state/
|
||||
|
||||
19
CLAUDE.md
19
CLAUDE.md
@@ -1,19 +0,0 @@
|
||||
## Skill routing
|
||||
|
||||
When the user's request matches an available skill, ALWAYS invoke it using the Skill
|
||||
tool as your FIRST action. Do NOT answer directly, do NOT use other tools first.
|
||||
The skill has specialized workflows that produce better results than ad-hoc answers.
|
||||
|
||||
Key routing rules:
|
||||
- Product ideas, "is this worth building", brainstorming → invoke gstack-office-hours
|
||||
- Bugs, errors, "why is this broken", 500 errors → invoke gstack-investigate
|
||||
- Ship, deploy, push, create PR → invoke gstack-ship
|
||||
- QA, test the site, find bugs → invoke gstack-qa
|
||||
- Code review, check my diff → invoke gstack-review
|
||||
- Update docs after shipping → invoke gstack-document-release
|
||||
- Weekly retro → invoke gstack-retro
|
||||
- Design system, brand → invoke gstack-design-consultation
|
||||
- Visual audit, design polish → invoke gstack-design-review
|
||||
- Architecture review → invoke gstack-plan-eng-review
|
||||
- Save progress, checkpoint, resume → invoke gstack-checkpoint
|
||||
- Code quality, health check → invoke gstack-health
|
||||
47
TODOS.md
47
TODOS.md
@@ -1,47 +0,0 @@
|
||||
# TODOS
|
||||
|
||||
## P1 — Ship with Phase 1
|
||||
|
||||
### v2.0 Architecture Document Update
|
||||
Update `bastion/docs/ARCHITECTURE.md` to cover v2.0: driver model, fleet system,
|
||||
Pulumi integration, Vault secrets, Deno evaluator, new CLI grammar. The existing
|
||||
doc covers v1.0 comprehensively (432 lines). v2.0 adds 5+ major subsystems.
|
||||
**Effort:** M (human: 1 week / CC: 1-2 days)
|
||||
**Depends on:** Phase 1 complete
|
||||
**Source:** CEO review 2026-04-01
|
||||
|
||||
## P2 — Post-v2.0 Core
|
||||
|
||||
### SSH Emergency Mode (scoped)
|
||||
SSH-based operations limited to: (1) earliest necessary box provisioning before agent
|
||||
is installed, and (2) emergency debugging/fixing operations that can't be done via agent.
|
||||
NOT a general-purpose DeploymentTarget alternative. The v1.0 `recheck` and `fix-ssh-root.sh`
|
||||
patterns are the model. Agent stays the primary management path.
|
||||
**Effort:** S (human: 1 week / CC: 1 day)
|
||||
**Depends on:** Phase 2 complete (DeploymentTarget interface exists)
|
||||
**Source:** CEO review 2026-04-01
|
||||
|
||||
### Prometheus Metrics Endpoint
|
||||
Add `/metrics` endpoint to labd: resource counts by status, apply duration histograms,
|
||||
driver operation latency, fleet pipeline completion rates. Standard Prometheus scraping
|
||||
for Grafana dashboards and alerting.
|
||||
**Effort:** S (human: 2-3 days / CC: 2-3 hours)
|
||||
**Depends on:** Phase 1 (labd exists with resource store)
|
||||
**Source:** CEO review 2026-04-01 (observability gap)
|
||||
|
||||
## P3 — Future Enhancements
|
||||
|
||||
### Infrastructure Graph Visualization
|
||||
Visual representation of resource dependencies, environment topology, fleet status.
|
||||
Could be a web UI or terminal-based (like `kubectl tree`).
|
||||
**Source:** CEO review 2026-04-01
|
||||
|
||||
### `labctl import` for Existing Cloud Resources
|
||||
Discover and import existing AWS/GCP resources into the state store.
|
||||
Pulumi's import functionality could be leveraged.
|
||||
**Source:** CEO review 2026-04-01
|
||||
|
||||
### Built-in Secrets Rotation
|
||||
Automatic rotation of managed secrets (database passwords, API keys).
|
||||
Vault handles rotation but a labctl-native workflow could simplify.
|
||||
**Source:** CEO review 2026-04-01
|
||||
@@ -11,7 +11,6 @@ WORKDIR /app
|
||||
# Copy workspace config and package manifests first (layer cache)
|
||||
COPY pnpm-workspace.yaml pnpm-lock.yaml package.json tsconfig.base.json tsconfig.json ./
|
||||
COPY src/shared/package.json src/shared/tsconfig.json src/shared/
|
||||
COPY src/core/package.json src/core/tsconfig.json src/core/
|
||||
COPY src/labd/package.json src/labd/tsconfig.json src/labd/
|
||||
|
||||
# Install all dependencies (dev included -- needed for build)
|
||||
@@ -23,13 +22,10 @@ RUN pnpm --filter @lab/labd exec prisma generate
|
||||
|
||||
# Copy source code
|
||||
COPY src/shared/src/ src/shared/src/
|
||||
COPY src/core/src/ src/core/src/
|
||||
COPY src/labd/src/ src/labd/src/
|
||||
|
||||
# Build TypeScript (shared + core before labd via project references)
|
||||
RUN pnpm --filter @lab/shared build \
|
||||
&& pnpm --filter @lab/core build \
|
||||
&& pnpm --filter @lab/labd build
|
||||
# Build TypeScript (shared first via project references)
|
||||
RUN pnpm --filter @lab/shared build && pnpm --filter @lab/labd build
|
||||
|
||||
# Hoist the generated Prisma client so stage 2 can COPY it from a stable path
|
||||
RUN mkdir -p /app/_prisma && \
|
||||
@@ -45,7 +41,6 @@ WORKDIR /app
|
||||
# Copy workspace config and package manifests
|
||||
COPY pnpm-workspace.yaml pnpm-lock.yaml package.json ./
|
||||
COPY src/shared/package.json src/shared/
|
||||
COPY src/core/package.json src/core/
|
||||
COPY src/labd/package.json src/labd/
|
||||
|
||||
# Install production dependencies only
|
||||
@@ -53,7 +48,6 @@ RUN pnpm install --frozen-lockfile --prod 2>/dev/null || pnpm install --prod
|
||||
|
||||
# Copy built output from builder
|
||||
COPY --from=builder /app/src/shared/dist/ src/shared/dist/
|
||||
COPY --from=builder /app/src/core/dist/ src/core/dist/
|
||||
COPY --from=builder /app/src/labd/dist/ src/labd/dist/
|
||||
|
||||
# Copy Prisma schema + generated client into pnpm store location
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
{
|
||||
"os_list": [
|
||||
{
|
||||
"name": "Fedora Asahi Lab (infra)",
|
||||
"default_os_name": "Fedora Linux Lab",
|
||||
"boot_object": "m1n1.bin",
|
||||
"next_object": "m1n1/boot.bin",
|
||||
"package": "fedora-asahi-lab.zip",
|
||||
"supported_fw": [
|
||||
"12.3",
|
||||
"12.3.1",
|
||||
"13.5"
|
||||
],
|
||||
"partitions": [
|
||||
{
|
||||
"name": "EFI",
|
||||
"type": "EFI",
|
||||
"size": "524288000B",
|
||||
"format": "fat",
|
||||
"volume_id": "0x804be8a6",
|
||||
"copy_firmware": true,
|
||||
"copy_installer_data": true,
|
||||
"source": "esp"
|
||||
},
|
||||
{
|
||||
"name": "Boot",
|
||||
"type": "Linux",
|
||||
"size": "1073741824B",
|
||||
"image": "boot.img"
|
||||
},
|
||||
{
|
||||
"name": "Root",
|
||||
"type": "Linux",
|
||||
"size": "4626296832B",
|
||||
"expand": false,
|
||||
"image": "root.img"
|
||||
},
|
||||
{
|
||||
"name": "Data",
|
||||
"type": "Linux",
|
||||
"size": "1073741824B",
|
||||
"expand": true
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
4
bastion/bastion/.gitignore
vendored
4
bastion/bastion/.gitignore
vendored
@@ -1,4 +0,0 @@
|
||||
|
||||
# Asahi build artifacts (large)
|
||||
.asahi-cache/
|
||||
asahi-repo/*.zip
|
||||
@@ -49,9 +49,6 @@ _labctl() {
|
||||
"app k3s list")
|
||||
COMPREPLY=($(compgen -W "--user -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"app k3s kubeconfig")
|
||||
COMPREPLY=($(compgen -W "--user --context --print -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"init bastion")
|
||||
COMPREPLY=($(compgen -W "standalone -h --help" -- "$cur"))
|
||||
return ;;
|
||||
@@ -59,32 +56,23 @@ _labctl() {
|
||||
COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision install")
|
||||
COMPREPLY=($(compgen -W "--role --os --disk --vyos-mgmt --vyos-mgmt-address --vyos-bond --vyos-bond-address --vyos-bond-vrrp --vlan-vip --vyos-vrrp-priority --vyos-mgmt-vlan --vlan --vyos-password --vyos-hwid --vyos-fresh-config -h --help" -- "$cur"))
|
||||
COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision reprovision")
|
||||
COMPREPLY=($(compgen -W "--role --os --disk --user -h --help" -- "$cur"))
|
||||
COMPREPLY=($(compgen -W "--role --os --disk -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision debug")
|
||||
COMPREPLY=($(compgen -W "--pxe-boot -h --help" -- "$cur"))
|
||||
COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision forget")
|
||||
COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision register")
|
||||
COMPREPLY=($(compgen -W "--role --ip -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision asahi")
|
||||
COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision logs")
|
||||
COMPREPLY=($(compgen -W "-f --follow -h --help" -- "$cur"))
|
||||
COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision makeiso")
|
||||
COMPREPLY=($(compgen -W "--arch --local --out -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision recheck")
|
||||
COMPREPLY=($(compgen -W "--user --target -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"config list")
|
||||
COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
|
||||
return ;;
|
||||
@@ -101,7 +89,7 @@ _labctl() {
|
||||
COMPREPLY=($(compgen -W "deploy status -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"app k3s")
|
||||
COMPREPLY=($(compgen -W "install health list kubeconfig -h --help" -- "$cur"))
|
||||
COMPREPLY=($(compgen -W "install health list -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"version")
|
||||
COMPREPLY=($(compgen -W "-h --help" -- "$cur"))
|
||||
@@ -110,7 +98,7 @@ _labctl() {
|
||||
COMPREPLY=($(compgen -W "bastion -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"provision")
|
||||
COMPREPLY=($(compgen -W "list install reprovision debug forget register asahi logs makeiso recheck -h --help" -- "$cur"))
|
||||
COMPREPLY=($(compgen -W "list install reprovision debug forget logs makeiso -h --help" -- "$cur"))
|
||||
return ;;
|
||||
"config")
|
||||
COMPREPLY=($(compgen -W "list get set path -h --help" -- "$cur"))
|
||||
|
||||
@@ -124,54 +124,24 @@ complete -c labctl -n "__labctl_using_cmd provision" -a install -d 'Queue a disc
|
||||
complete -c labctl -n "__labctl_using_cmd provision" -a reprovision -d 'Queue install + SSH reboot into PXE (target: hostname, MAC, or IP)'
|
||||
complete -c labctl -n "__labctl_using_cmd provision" -a debug -d 'PXE boot into Fedora rescue mode for debugging (target: hostname, MAC, or IP)'
|
||||
complete -c labctl -n "__labctl_using_cmd provision" -a forget -d 'Remove a machine from bastion state'
|
||||
complete -c labctl -n "__labctl_using_cmd provision" -a register -d 'Register an already-installed machine (e.g. after state loss)'
|
||||
complete -c labctl -n "__labctl_using_cmd provision" -a asahi -d 'Show instructions to provision an Apple Silicon Mac with Asahi Linux'
|
||||
complete -c labctl -n "__labctl_using_cmd provision" -a logs -d 'Show provisioning logs for a machine (hostname, MAC, or IP)'
|
||||
complete -c labctl -n "__labctl_using_cmd provision" -a makeiso -d 'Generate a UEFI-bootable iPXE ISO for network provisioning'
|
||||
complete -c labctl -n "__labctl_using_cmd provision" -a recheck -d 'Refresh hardware info for all installed machines via SSH'
|
||||
|
||||
# provision install options
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling'
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04'
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l disk -d 'Target disk device (auto-detect if omitted)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt -d 'VyOS: untagged interface the machine PXE boots from (default eth0)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-address -d 'VyOS: CIDR for the management interface, or \'dhcp\' (default dhcp)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond -d 'VyOS: comma-separated LACP bond members (must exclude the PXE NIC)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond-address -d 'VyOS: address on the untagged bond (trunk native VLAN)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-bond-vrrp -d 'VyOS: VRRP VIP floated on the untagged bond' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vlan-vip -d 'VyOS: VRRP VIP for a --vlan entry (repeatable)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-vrrp-priority -d 'VyOS: VRRP priority for all groups on this box (higher = master)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-mgmt-vlan -d 'VyOS: tagged management VLAN on the PXE port' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vlan -d 'VyOS: tagged VLAN sub-interface on the bond (repeatable)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-password -d 'VyOS: password for the \'vyos\' user' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-hwid -d 'VyOS: pin an interface name to a MAC via hw-id (repeatable)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision install" -l vyos-fresh-config -d 'VyOS: on reinstall, overwrite the preserved config with the generated one'
|
||||
|
||||
# provision reprovision options
|
||||
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l role -d 'Machine role (see below)' -xa 'vanilla worker infra labcontroller'
|
||||
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04 vyos-rolling'
|
||||
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l os -d 'Operating system' -xa 'fedora-43 ubuntu-26.04'
|
||||
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l disk -d 'Target disk device (auto-detect if omitted)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision reprovision" -l user -d 'SSH user for the reboot (default: vyos for VyOS machines, else current user)' -x
|
||||
|
||||
# provision debug options
|
||||
complete -c labctl -n "__labctl_in_cmd provision debug" -l pxe-boot -d 'Boot installed system via PXE (kernel+initrd from network, root from NVMe)'
|
||||
|
||||
# provision register options
|
||||
complete -c labctl -n "__labctl_in_cmd provision register" -l role -d 'Machine role' -xa 'vanilla worker infra labcontroller'
|
||||
complete -c labctl -n "__labctl_in_cmd provision register" -l ip -d 'Machine IP address' -x
|
||||
|
||||
# provision logs options
|
||||
complete -c labctl -n "__labctl_in_cmd provision logs" -s f -l follow -d 'Follow log output in real-time'
|
||||
|
||||
# provision makeiso options
|
||||
complete -c labctl -n "__labctl_in_cmd provision makeiso" -l arch -d 'Target architecture(s)' -xa 'x86_64 aarch64'
|
||||
complete -c labctl -n "__labctl_in_cmd provision makeiso" -l local -d 'Build ISO locally instead of using bastion-hosted URL'
|
||||
complete -c labctl -n "__labctl_in_cmd provision makeiso" -l out -d 'Output path for local ISO build' -x
|
||||
|
||||
# provision recheck options
|
||||
complete -c labctl -n "__labctl_in_cmd provision recheck" -l user -d 'SSH user' -x
|
||||
complete -c labctl -n "__labctl_in_cmd provision recheck" -l target -d 'Only recheck a specific machine (by hostname or MAC)' -x
|
||||
|
||||
# config subcommands
|
||||
complete -c labctl -n "__labctl_using_cmd config" -a list -d 'Show all configuration values'
|
||||
complete -c labctl -n "__labctl_using_cmd config" -a get -d 'Get a configuration value'
|
||||
@@ -203,7 +173,6 @@ complete -c labctl -n "__labctl_in_cmd app labcontroller status" -l user -d 'SSH
|
||||
complete -c labctl -n "__labctl_using_cmd app k3s" -a install -d 'Install k3s on a target machine (hostname, IP, or MAC)'
|
||||
complete -c labctl -n "__labctl_using_cmd app k3s" -a health -d 'Check k3s health (all hosts if no target given)'
|
||||
complete -c labctl -n "__labctl_using_cmd app k3s" -a list -d 'List installed machines and their k3s status'
|
||||
complete -c labctl -n "__labctl_using_cmd app k3s" -a kubeconfig -d 'Fetch kubeconfig from a target and merge into ~/.kube/config'
|
||||
|
||||
# app k3s install options
|
||||
complete -c labctl -n "__labctl_in_cmd app k3s install" -l role -d 'k3s role: infra (server) or worker (agent)' -x
|
||||
@@ -217,8 +186,3 @@ complete -c labctl -n "__labctl_in_cmd app k3s health" -l user -d 'SSH user' -x
|
||||
# app k3s list options
|
||||
complete -c labctl -n "__labctl_in_cmd app k3s list" -l user -d 'SSH user' -x
|
||||
|
||||
# app k3s kubeconfig options
|
||||
complete -c labctl -n "__labctl_in_cmd app k3s kubeconfig" -l user -d 'SSH user' -x
|
||||
complete -c labctl -n "__labctl_in_cmd app k3s kubeconfig" -l context -d 'Context name (defaults to hostname)' -x
|
||||
complete -c labctl -n "__labctl_in_cmd app k3s kubeconfig" -l print -d 'Print kubeconfig to stdout instead of merging'
|
||||
|
||||
|
||||
@@ -1,431 +0,0 @@
|
||||
# Lab Platform Architecture
|
||||
|
||||
## Overview
|
||||
|
||||
A bare-metal and hybrid cloud infrastructure platform for automated machine provisioning, Kubernetes cluster management, and fleet operations. The platform discovers hardware via PXE boot, installs operating systems unattended, deploys k3s clusters, and provides centralized management through a CLI and API.
|
||||
|
||||
**Components:**
|
||||
- **bastion** -- PXE boot server (DHCP/TFTP/HTTP) for machine discovery and OS installation
|
||||
- **labd** -- Master daemon for multi-bastion aggregation, persistent state, agent management
|
||||
- **labctl** -- CLI tool for operators (kubectl-style interface)
|
||||
- **lab-agent** -- Daemon on provisioned servers for remote execution and monitoring
|
||||
- **modules** -- Declarative configuration system (k3s, labcontroller)
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
labctl (CLI)
|
||||
|
|
||||
labd (master daemon)
|
||||
/ | \
|
||||
bastion1 bastion2 ... (PXE provisioning)
|
||||
/ \ |
|
||||
[machines] [machines] (bare metal)
|
||||
| |
|
||||
lab-agent lab-agent (remote exec)
|
||||
```
|
||||
|
||||
### Communication Patterns
|
||||
|
||||
| Path | Protocol | Auth |
|
||||
|------|----------|------|
|
||||
| labctl -> labd | HTTP/HTTPS | mTLS cert (future: token) |
|
||||
| bastion -> labd | WebSocket | Join token enrollment |
|
||||
| lab-agent -> labd | WebSocket | mTLS certificate |
|
||||
| machine -> bastion | HTTP | None (local network) |
|
||||
| Anaconda -> bastion | HTTP + UDP syslog | None (install-time) |
|
||||
| labctl -> bastion | HTTP | None (standalone mode) |
|
||||
|
||||
### Standalone vs Centralized
|
||||
|
||||
The bastion can operate in two modes:
|
||||
|
||||
1. **Standalone** -- single bastion, state in local JSON file, CLI talks directly to bastion HTTP API
|
||||
2. **Centralized** -- bastion registers with labd via WebSocket, state aggregated in CockroachDB, CLI talks to labd which routes commands to the correct bastion
|
||||
|
||||
---
|
||||
|
||||
## Machine Lifecycle
|
||||
|
||||
```
|
||||
PXE boot
|
||||
|
|
||||
+--------v--------+
|
||||
| DISCOVERED | Hardware inventory collected
|
||||
+---------+-------+
|
||||
|
|
||||
labctl provision install
|
||||
|
|
||||
+---------v-------+
|
||||
| INSTALL_QUEUE | Waiting for next PXE boot
|
||||
+---------+-------+
|
||||
|
|
||||
PXE boot (Anaconda)
|
||||
|
|
||||
+---------v-------+
|
||||
| INSTALLING | Progress: partitioning -> packages -> post-install
|
||||
+---------+-------+
|
||||
|
|
||||
+---------v-------+
|
||||
| INSTALLED | OS ready, SSH accessible
|
||||
+---------+-------+
|
||||
|
|
||||
labctl app k3s install
|
||||
|
|
||||
+---------v-------+
|
||||
| K3S RUNNING | Kubernetes node operational
|
||||
+--------+--------+
|
||||
|
|
||||
labctl provision reprovision
|
||||
|
|
||||
(back to INSTALL_QUEUE)
|
||||
```
|
||||
|
||||
Side paths:
|
||||
- **DEBUG** -- `labctl provision debug` boots Anaconda rescue mode for diagnostics
|
||||
- **FORGET** -- `labctl provision forget` removes machine from all state
|
||||
|
||||
---
|
||||
|
||||
## Packages
|
||||
|
||||
### Monorepo Structure
|
||||
|
||||
TypeScript ESM monorepo with pnpm workspaces. Six packages:
|
||||
|
||||
| Package | Role | Key Tech |
|
||||
|---------|------|----------|
|
||||
| `@lab/shared` | Types, protocol, constants | - |
|
||||
| `@lab/bastion` | PXE server | Fastify, dnsmasq |
|
||||
| `@lab/cli` | CLI binary | Commander.js |
|
||||
| `@lab/labd` | Master daemon | Fastify, Prisma, CockroachDB |
|
||||
| `@lab/agent` | Server agent | WebSocket |
|
||||
| `@lab/modules` | Config modules | SSH, k8s-client |
|
||||
|
||||
### @lab/shared
|
||||
|
||||
Core type system shared by all packages.
|
||||
|
||||
**State Model:**
|
||||
```typescript
|
||||
interface BastionState {
|
||||
discovered: Record<MAC, HardwareInfo>
|
||||
install_queue: Record<MAC, InstallConfig>
|
||||
installed: Record<MAC, InstalledInfo>
|
||||
debug: Record<MAC, DebugConfig>
|
||||
}
|
||||
```
|
||||
|
||||
**Roles:**
|
||||
- `vanilla` -- OS only, no k3s, no cluster services
|
||||
- `worker` -- k3s agent + Longhorn storage (joins existing cluster)
|
||||
- `infra` -- k3s server + etcd (control plane node)
|
||||
- `labcontroller` -- infra + bastion + labd + CockroachDB (self-sufficient)
|
||||
|
||||
**OS Support:**
|
||||
- `fedora-43` -- Anaconda kickstart installer
|
||||
- `ubuntu-26.04` -- cloud-init autoinstall
|
||||
|
||||
**Protocol:** Discriminated union message types for WebSocket communication between agents, bastions, and labd. Type guards and parsers for runtime validation.
|
||||
|
||||
### @lab/bastion
|
||||
|
||||
PXE boot server that handles the physical provisioning lifecycle.
|
||||
|
||||
**Services:**
|
||||
- `StateManager` -- JSON file persistence with immutable update pattern
|
||||
- `SyslogListener` -- UDP syslog receiver (port 5514) for Anaconda install logs
|
||||
- `InstallLogBuffer` -- In-memory ring buffer + disk persistence per machine
|
||||
- `BastionConnection` -- WebSocket client to labd for centralized mode
|
||||
- dnsmasq management (spawn, config generation, proxy/full DHCP)
|
||||
- Network auto-detection (interface, IP, subnet, gateway)
|
||||
- ISO builder (xorriso + mtools for non-PXE machines)
|
||||
|
||||
**HTTP Routes:**
|
||||
|
||||
| Endpoint | Purpose |
|
||||
|----------|---------|
|
||||
| `GET /dispatch?mac=` | Dynamic iPXE script (discover/install/debug/local-boot) |
|
||||
| `GET /ks?mac=` | Per-machine Anaconda kickstart |
|
||||
| `GET /debug.ks` | Rescue mode kickstart |
|
||||
| `GET /debug-setup.sh` | nc listener setup script for rescue shell |
|
||||
| `GET /discover.ks` | Hardware discovery kickstart |
|
||||
| `POST /api/discover` | Hardware inventory report |
|
||||
| `POST /api/install` | Queue machine for install |
|
||||
| `POST /api/progress` | Install progress callback |
|
||||
| `POST /api/log` | Raw log line ingestion |
|
||||
| `POST /api/debug` | Queue debug/rescue mode |
|
||||
| `GET /api/machines` | List all machines |
|
||||
| `GET /api/logs/:mac` | Install logs + progress |
|
||||
| `GET /api/logs/:mac/follow` | SSE stream of progress events |
|
||||
| `DELETE /api/machines/:mac` | Forget machine |
|
||||
|
||||
**Templates:**
|
||||
- `boot.ipxe.ts` -- iPXE scripts for each boot mode (discover, install, debug, pxe-boot-debug, local-boot)
|
||||
- `install.ks.ts` -- Full Fedora kickstart with LVM, SSH, k3s prereqs, progress callbacks, SysRq keys
|
||||
- `debug.ks.ts` -- Minimal rescue kickstart (SSH via inst.sshd)
|
||||
- `ubuntu-autoinstall.ts` -- cloud-init for Ubuntu
|
||||
- `dnsmasq.conf.ts` -- DHCP/TFTP configuration
|
||||
|
||||
**Boot Dispatch Logic:**
|
||||
```
|
||||
1. debug[mac]? -> renderDebugIpxe (auto-clear after serving)
|
||||
2. install_queue[mac]? -> renderInstallIpxe
|
||||
3. installed[mac]? -> renderLocalBootIpxe (exit to disk)
|
||||
4. unknown -> renderDiscoverIpxe
|
||||
```
|
||||
|
||||
### @lab/labd
|
||||
|
||||
Central management daemon. Aggregates multiple bastions, stores persistent state in CockroachDB, relays commands, manages agent fleet.
|
||||
|
||||
**Database (Prisma + CockroachDB):**
|
||||
- `Server` -- hostname, MAC, IP, role, status, cloud, environment, labels
|
||||
- `Bastion` -- hostname, network, serverIp, lastHeartbeat
|
||||
- `Agent` -- certificate, enrollment, heartbeat
|
||||
- `Cluster` -- name, cloud, environment, kubeconfig (encrypted)
|
||||
- `User` / `Role` / `Permission` -- RBAC (action:cloud:env:server matrix)
|
||||
- `JoinToken` -- one-time/reusable enrollment tokens
|
||||
- `AuditLog` -- action, resource, result, timestamp
|
||||
|
||||
**Key Services:**
|
||||
- `BastionRegistry` -- in-memory registry of connected bastions, state aggregation, MAC-to-bastion routing
|
||||
- `AgentRegistry` -- connected agents, heartbeat tracking
|
||||
- `MessageRouter` -- command relay between CLI/agents and bastions
|
||||
|
||||
**Command Routing:**
|
||||
```
|
||||
CLI: labctl provision install <mac> <hostname>
|
||||
-> POST /api/machines/install
|
||||
-> labd finds bastion that knows this MAC
|
||||
-> WebSocket: {type: "command-install", mac, hostname, disk, role}
|
||||
-> bastion updates install_queue
|
||||
-> WebSocket: {type: "command-response", status: "ok"}
|
||||
-> HTTP response to CLI
|
||||
```
|
||||
|
||||
### @lab/cli (labctl)
|
||||
|
||||
Operator CLI. Commander.js binary, distributed as RPM/DEB or standalone bun-compiled executable.
|
||||
|
||||
**Command Groups:**
|
||||
|
||||
```
|
||||
labctl init bastion standalone start|stop|status
|
||||
labctl provision list|install|reprovision|forget|debug|logs|makeiso
|
||||
labctl app k3s install|health|list
|
||||
labctl config list|get|set|path
|
||||
labctl login
|
||||
labctl doctor
|
||||
labctl roles
|
||||
```
|
||||
|
||||
**Key Features:**
|
||||
- Target resolution: hostname, MAC, or IP -> machine lookup
|
||||
- SSH reboot into PXE for reprovision/debug (efibootmgr --bootnext)
|
||||
- Follow mode: `labctl provision logs <target> -f` (5s polling)
|
||||
- Shell completions: bash, fish
|
||||
|
||||
### @lab/modules
|
||||
|
||||
Declarative configuration modules with three-phase lifecycle: install -> configure -> health.
|
||||
|
||||
**k3s Module:**
|
||||
- 5 operation groups: host-prep, networking, k3s-server, k3s-agent, hardening
|
||||
- 15+ individual operations: kernel modules, sysctl, firewall, Cilium CNI, SELinux, audit policy, pod security, cert checks
|
||||
- Health checks: service running, node ready, API health, pod status, Cilium status, secrets encryption
|
||||
- SSH execution backend with progress callbacks
|
||||
|
||||
### @lab/agent
|
||||
|
||||
Daemon on provisioned servers. WebSocket to labd for:
|
||||
- Heartbeat (hostname, uptime, CPU/mem usage)
|
||||
- Command execution (with stdout/stderr streaming)
|
||||
- Log streaming (journalctl relay)
|
||||
- mTLS certificate enrollment and rotation
|
||||
|
||||
---
|
||||
|
||||
## Disk Layout
|
||||
|
||||
### LVM Partitioning (labvg)
|
||||
|
||||
All roles share a common LVM layout. The kickstart `%pre` auto-detects the install disk (NVMe preferred, then SATA, skipping USB/removable).
|
||||
|
||||
| Volume | Size | FS | Reprovision |
|
||||
|--------|------|-----|-------------|
|
||||
| `/boot/efi` | 600 MB | vfat | Reused |
|
||||
| `/boot` | 3 GB | ext4 | Reused |
|
||||
| `swap` | 27 GB | swap | Recreated |
|
||||
| `/` (root) | 33 GB | xfs | Recreated |
|
||||
| `/var` | 100 GB | xfs | Recreated |
|
||||
| `/var/log` | 10 GB | xfs | Recreated |
|
||||
| `/home` | 10 GB | xfs | **Preserved** |
|
||||
| `/srv` | 20 GB | xfs | **Preserved** |
|
||||
| `/var/lib/longhorn` | remaining | xfs | **Preserved** (worker) |
|
||||
| `/var/lib/rancher` | 20 GB | xfs | **Preserved** (infra) |
|
||||
| `/tmp` | 4 GB | tmpfs | - |
|
||||
|
||||
Reprovision detection: if `labvg` VG exists, reuse EFI/boot partitions and preserve data volumes.
|
||||
|
||||
---
|
||||
|
||||
## Kickstart Features
|
||||
|
||||
The Fedora kickstart template (`install.ks.ts`) includes:
|
||||
|
||||
- **Dynamic disk detection** -- `%pre` probes NVMe/SATA/virtio, skips USB/removable, supports both fresh install and reprovision
|
||||
- **Progress callbacks** -- `curl -sf POST /api/progress` at each stage (partitioning, post-install substeps, complete)
|
||||
- **Anaconda syslog forwarding** -- `logging --host --port` streams real-time install logs to bastion
|
||||
- **SSH hardening** -- key-only auth, root login via pubkey only, admin user with passwordless sudo
|
||||
- **Network-first boot order** -- `efibootmgr` reorders boot entries so PXE is always first (bastion controls every reboot)
|
||||
- **SysRq magic keys** -- `kernel.sysrq=1` for emergency reboot via KVM keyboard
|
||||
- **Role-specific setup:**
|
||||
- `vanilla`: chronyd only
|
||||
- `worker`/`infra`: kernel modules (br_netfilter, overlay), sysctl (ip_forward, inotify), firewalld disabled, k3s binary installed
|
||||
- `infra`: k3s server binary pre-installed
|
||||
|
||||
**What is NOT in the kickstart:**
|
||||
- `console=ttyS0` -- causes 30s-per-step boot timeout on hardware without physical serial UART (discovered 2026-03-30, see docs/pxe-boot-debugging-2026-03-30.md)
|
||||
- Background log streamer (`tail -f`) -- prevents Anaconda from syncing filesystem, causes %post writes to not persist
|
||||
|
||||
---
|
||||
|
||||
## Deployment
|
||||
|
||||
### Container Images
|
||||
|
||||
**bastion** (`Dockerfile.bastion`):
|
||||
- Base: Fedora 43 (needs dnsmasq, iPXE)
|
||||
- Multi-stage: Alpine build -> Fedora runtime
|
||||
- iPXE rebuilt from source (SNP driver for EFI)
|
||||
- hostNetwork in k8s (DHCP needs raw sockets)
|
||||
- Capabilities: NET_ADMIN, NET_RAW
|
||||
|
||||
**labd** (`Dockerfile.labd`):
|
||||
- Base: Alpine (minimal)
|
||||
- Multi-stage build with Prisma client generation
|
||||
- Runs as non-root `node` user
|
||||
|
||||
### Kubernetes (k3s)
|
||||
|
||||
```
|
||||
Namespace: lab-infra
|
||||
Deployment: bastion (hostNetwork, PVC for /data, host SSH keys)
|
||||
ConfigMap: bastion-config (env vars)
|
||||
Secret: bastion-join-token
|
||||
PVC: bastion-state (local-path)
|
||||
|
||||
Namespace: lab-system
|
||||
Deployment: labd
|
||||
Service: labd (NodePort 30100)
|
||||
StatefulSet: cockroachdb-0
|
||||
```
|
||||
|
||||
### CLI Distribution
|
||||
|
||||
Built with `nfpm` as RPM/DEB. Includes:
|
||||
- `/usr/bin/labctl` (bun-compiled standalone binary)
|
||||
- `/usr/share/bash-completion/completions/labctl`
|
||||
- `/usr/share/fish/vendor_completions.d/labctl.fish`
|
||||
|
||||
Config: `~/.labctl/config.yaml` with `labdUrl`, output format, default cloud/environment.
|
||||
|
||||
---
|
||||
|
||||
## Build & Release
|
||||
|
||||
```bash
|
||||
# Development
|
||||
pnpm install && pnpm build # Compile all packages
|
||||
pnpm test:run # Unit tests (vitest)
|
||||
npx tsc --noEmit # Type check
|
||||
|
||||
# Deploy
|
||||
bash scripts/deploy.sh all # Build containers + RPM, push, restart pods
|
||||
bash scripts/deploy.sh bastion # Just bastion
|
||||
bash scripts/deploy.sh labd # Just labd
|
||||
bash scripts/deploy.sh labctl # Just CLI (local RPM install)
|
||||
|
||||
# Container builds
|
||||
bash scripts/build-bastion.sh --platforms linux/amd64 --push latest
|
||||
bash scripts/build-labd.sh --platforms linux/amd64 --push latest
|
||||
bash scripts/build-rpm.sh # RPM + DEB packages
|
||||
|
||||
# Integration tests (require libvirt, sudo)
|
||||
sudo tests/integration/run-pxe-test.sh
|
||||
```
|
||||
|
||||
Registry: `mysources.co.uk` (Gitea at 10.0.0.194:3012)
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
### Unit Tests
|
||||
- Kickstart rendering (ksvalidator syntax check, partition layout, role-specific sections)
|
||||
- State management (load, save, update, debug field)
|
||||
- Dispatch routing (correct iPXE script for each machine state)
|
||||
- Syslog listener (UDP receive, IP->MAC resolution, RFC 3164 parsing)
|
||||
|
||||
### Integration Tests (libvirt VMs)
|
||||
- **pxe-provision.test.ts** -- Full end-to-end: create VM -> PXE discovery -> queue install -> Anaconda install -> SSH verification -> systemd health -> SELinux enforcing -> boot order check
|
||||
- **iso-provision.test.ts** -- ISO boot for non-PXE machines
|
||||
- **k3s-single-node.test.ts** -- Post-provision k3s installation and health
|
||||
- VM screenshot capture during boot for debugging
|
||||
|
||||
---
|
||||
|
||||
## Security
|
||||
|
||||
- **mTLS** for agent-labd communication (certificate enrollment via join tokens)
|
||||
- **SSH key-only auth** on provisioned machines (no password auth)
|
||||
- **SELinux enforcing** verified in integration tests
|
||||
- **RBAC** (planned): action:cloud:environment:server permission matrix
|
||||
- **Audit logging** (planned): every mutation tracked in CockroachDB
|
||||
- **Network-first boot order** prevents machines from booting without bastion approval
|
||||
- **SysRq keys** enabled for emergency reboot without SSH access
|
||||
|
||||
---
|
||||
|
||||
## Known Issues & Lessons Learned
|
||||
|
||||
### Serial Console Boot Delay (2026-03-30)
|
||||
`console=ttyS0,115200n8` in kernel cmdline causes 30-second timeout at every systemd boot phase on hardware without a physical serial UART. Root cause: systemd blocks writing to non-existent UART. Fix: removed from kickstart entirely.
|
||||
|
||||
### Anaconda %post Log Streamer
|
||||
Background `tail -f` in kickstart `%post` prevents Anaconda from syncing the filesystem. All file writes in %post appear to succeed but are lost on reboot. Fix: removed background log streamer, replaced with Anaconda's built-in `logging --host --port` syslog forwarding.
|
||||
|
||||
### Disk Auto-Detection
|
||||
Hardcoded `/dev/sda` default broke NVMe-only machines. Fix: default to empty string (auto-detect) which triggers the `%pre` disk probe logic.
|
||||
|
||||
### Anaconda Rescue Mode Limitations
|
||||
`%pre` and `%post` sections do not execute in `inst.rescue` mode. SSH in rescue mode is provided by Anaconda's `inst.sshd` kernel parameter + `sshpw` kickstart directive. Manual setup via `curl bastion:8080/debug-setup.sh | bash` for nc listener.
|
||||
|
||||
---
|
||||
|
||||
## Planned Work (Taskmaster)
|
||||
|
||||
13 tasks in queue, all pending:
|
||||
|
||||
1. **#72** Expand Prisma schema with resource relationships (Network, ServerNic, ServerDisk, ClusterMember)
|
||||
2. **#73** State persistence service (bastion state -> CockroachDB)
|
||||
3. **#74** State loading from labd on bastion startup
|
||||
4. **#75** Fix bastion --dir env var default
|
||||
5. **#76** Resource type registry with aliases (kubectl-style)
|
||||
6. **#77** `labctl get <resource>` command
|
||||
7. **#78** `labctl describe <resource>` command
|
||||
8. **#79** `labctl create/delete` commands
|
||||
9. **#80** Refactor provision commands to kubectl-style
|
||||
10. **#81** Server and resource API endpoints in labd
|
||||
11. **#82** RBAC permission checks in CLI
|
||||
12. **#83** Audit logging for resource operations
|
||||
13. **#84** Update CLI entry point and help text
|
||||
|
||||
Additional items not in taskmaster:
|
||||
- Ubuntu autoinstall disk auto-detect (still defaults to /dev/sda)
|
||||
- Verify `inst.sshd` works end-to-end in rescue mode
|
||||
- k3s cluster join vs new cluster distinction in `labctl app k3s install`
|
||||
- arm64 container build (iPXE cross-compilation broken)
|
||||
@@ -1,91 +0,0 @@
|
||||
# PXE Boot Debugging Session — 2026-03-30
|
||||
|
||||
## Problem
|
||||
Beelink SER Mini Pro (AMD Ryzen 7 255, Radeon 780M, 64GB DDR5, 1TB NVMe) boots Fedora 43 100x slower than normal after PXE kickstart install. Every systemd boot phase takes ~30 seconds. The Anaconda installer/rescue mode boots fast on the same hardware.
|
||||
|
||||
## Root Cause
|
||||
**`console=ttyS0,115200n8` in kernel cmdline** — added via kickstart `bootloader --append` during install.
|
||||
|
||||
This mini PC has **no physical serial UART**. When systemd writes to ttyS0, each log write blocks for ~30 seconds waiting for the non-existent UART hardware. Since systemd logs at every phase transition, the total boot time was 10+ minutes.
|
||||
|
||||
The Anaconda installer was unaffected because it uses a different init flow that doesn't go through the same systemd phase transitions.
|
||||
|
||||
## How We Found It
|
||||
Hours of systematic elimination:
|
||||
|
||||
| What we tried | Result | Ruled out |
|
||||
|---|---|---|
|
||||
| `modprobe.blacklist=amdgpu` | No change | GPU driver |
|
||||
| `amd_iommu=off` | No change | IOMMU |
|
||||
| Rebuild initramfs without plymouth/drm/fips | No change | Initramfs bloat |
|
||||
| systemd-boot instead of GRUB | Still slow | Bootloader |
|
||||
| PXE-boot kernel+initrd (skip local GRUB entirely) | Still slow | Local bootloader/firmware |
|
||||
| Disable TPM in BIOS | No change | TPM |
|
||||
| Remove `resume=` + resume dracut module | No change | Hibernate resume |
|
||||
| Manual LVM activation in rescue shell | **Fast** | NVMe/LVM themselves |
|
||||
| Remove `console=ttyS0,115200n8` from GRUB | **FAST BOOT** | **This was it** |
|
||||
|
||||
The key breakthrough was noticing the timestamps showed **exactly 30-second gaps** between boot phases — a timeout pattern, not general slowness. Then realising the serial console was added during install and had never been tested without.
|
||||
|
||||
## What Was Fixed (PR #4, merged)
|
||||
|
||||
### 1. Removed serial console from kickstart
|
||||
- Removed `console=ttyS0,115200n8` from `bootloader --append`
|
||||
- Removed `serial-getty@ttyS0.service` enablement
|
||||
- Removed rsyslog serial forwarding
|
||||
|
||||
### 2. Enabled Anaconda syslog forwarding
|
||||
- Uncommented `logging --host --port` directive in kickstart
|
||||
- Bastion's SyslogListener was already built — just needed IP→MAC resolution improvement
|
||||
- Added `registerIp()` calls from kickstart fetch and progress callbacks
|
||||
- Added syslog listener unit tests
|
||||
|
||||
### 3. Fixed disk auto-detection
|
||||
- Default disk changed from `/dev/sda` to `""` (auto-detect) in labd route and bastion command handler
|
||||
- The kickstart `%pre` auto-detect logic probes nvme0n1, sda, sdb, vda in order
|
||||
- Without this fix, NVMe-only machines (like the SER Mini Pro) fail immediately
|
||||
|
||||
### 4. SysRq magic keys
|
||||
- Added `kernel.sysrq=1` sysctl to kickstart `%post`
|
||||
- Enables Alt+SysRq+REISUB via JetKVM for emergency reboot of stuck machines
|
||||
|
||||
### 5. Simplified debug command
|
||||
- Removed `--sshd` flag (SSH always available via `inst.sshd` + `sshpw` in rescue mode)
|
||||
- Added `/debug-setup.sh` HTTP endpoint for nc listener setup from rescue shell
|
||||
- Cleaned up `sshd` field from DebugConfig, protocol types, all routes
|
||||
|
||||
### 6. Added `labctl provision logs -f`
|
||||
- Follow mode with 5-second polling for real-time install monitoring
|
||||
|
||||
## What Works
|
||||
|
||||
- **PXE discovery → install → boot** — full flow works end-to-end
|
||||
- **Anaconda syslog forwarding** — install logs stream to bastion
|
||||
- **Progress callbacks** — stage-by-stage install tracking via curl
|
||||
- **Auto disk detection** — works for NVMe and SATA
|
||||
- **Debug rescue mode** — `labctl provision debug <target>` boots Anaconda rescue with SSH
|
||||
- **Network-first boot order** — bastion controls every reboot via efibootmgr
|
||||
- **SysRq keys** — emergency reboot via JetKVM keyboard
|
||||
|
||||
## What Doesn't Work / Known Issues
|
||||
|
||||
- **`--sshd` in rescue mode** — Anaconda rescue mode skips both `%pre` and `%post` kickstart sections. `inst.sshd` + `sshpw` should provide SSH access, but hasn't been verified end-to-end yet. The `/debug-setup.sh` curl workaround exists for nc.
|
||||
- **arm64 container build** — iPXE cross-compilation fails on arm64 (GCC flag incompatibility). Workaround: build with `--platforms linux/amd64` only.
|
||||
- **Integration test SSH timeout** — VM boots fine but SSH times out due to libvirt nftables reject rules after VM restart. Test infrastructure issue, not a code bug.
|
||||
|
||||
## What Was Skipped / Left To Do
|
||||
|
||||
1. **Syslog UDP port in k3s** — works because bastion uses `hostNetwork: true`, but should be documented properly
|
||||
2. **Background log streamer** — the old `tail -f` approach broke Anaconda filesystem sync. Replaced with syslog forwarding. If more granular %post logging is needed, a synchronous log push at end of %post would be safe.
|
||||
3. **Per-machine hardware overrides** — turned out not to be needed (serial console was the only "special" setting, and removing it is universal)
|
||||
4. **Ubuntu autoinstall disk default** — `ubuntu-autoinstall.ts` still has `disk || "/dev/sda"` fallback (line 38), should be changed to auto-detect
|
||||
5. **Verify `inst.sshd` works in rescue mode** — test SSH with password "debug" next time debug mode is used
|
||||
6. **Re-enable TPM in BIOS** — was disabled during debugging, should be factory-reset (user plans to reset BIOS to factory)
|
||||
|
||||
## Key Learnings
|
||||
|
||||
1. **`console=ttyS0` on hardware without UART = 30s timeout per boot phase.** Never add serial console to kernel cmdline unless the hardware has a verified physical UART.
|
||||
2. **Exactly-N-second gaps in boot logs = timeout, not slowness.** Look for the timeout source, not performance issues.
|
||||
3. **The bisection approach works.** Systematically removing features one at a time found the root cause. But it took hours because the serial console was added early and seemed harmless.
|
||||
4. **Anaconda rescue mode is limited.** It skips `%pre` and `%post`, so you can't automate setup via kickstart. Use `inst.sshd` + `sshpw` for SSH, and serve helper scripts via HTTP for everything else.
|
||||
5. **Default disk paths break NVMe machines.** Always default to auto-detect (empty string) rather than `/dev/sda`.
|
||||
@@ -21,14 +21,8 @@
|
||||
"test:integration:pxe:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'PXE boot'",
|
||||
"test:integration:iso": "vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'",
|
||||
"test:integration:iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ISO boot'",
|
||||
"test:integration:vyos": "vitest run -c tests/integration/vitest.config.ts -t 'VyOS provisioning'",
|
||||
"test:integration:vyos:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'VyOS provisioning'",
|
||||
"test:integration:arm-iso": "vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'",
|
||||
"test:integration:arm-iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'",
|
||||
"test:integration:asahi": "vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'",
|
||||
"test:integration:asahi:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'asahi firstboot'",
|
||||
"test:integration:asahi-validate": "vitest run -c tests/integration/vitest.config.ts -t 'asahi.*validation'",
|
||||
"test:integration:asahi-validate:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'asahi.*validation'"
|
||||
"test:integration:arm-iso:host": "sudo -E $(which npx) vitest run -c tests/integration/vitest.config.ts -t 'ARM ISO'"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0",
|
||||
|
||||
1847
bastion/pnpm-lock.yaml
generated
1847
bastion/pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -1,302 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Build a custom Fedora Asahi Remix rootfs with lab firstboot LVM setup.
|
||||
#
|
||||
# Downloads the upstream Fedora Asahi Remix Server package, injects our
|
||||
# firstboot script + systemd service, and repackages it for the bastion.
|
||||
#
|
||||
# Requirements: root, curl, unzip, mount (loop), zip
|
||||
# Output: bastion/asahi-repo/ directory with package + installer_data.json
|
||||
#
|
||||
# Usage: sudo ./scripts/build-asahi-rootfs.sh [--bastion-ip IP] [--http-port PORT]
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
ASAHI_DIR="$PROJECT_DIR/asahi-repo"
|
||||
CACHE_DIR="$PROJECT_DIR/.asahi-cache"
|
||||
WORK_DIR=""
|
||||
|
||||
# Defaults
|
||||
BASTION_IP="${BASTION_IP:-192.168.8.23}"
|
||||
HTTP_PORT="${HTTP_PORT:-8080}"
|
||||
ROLE="${ROLE:-infra}"
|
||||
HOSTNAME="${HOSTNAME:-mac-studio}"
|
||||
MAC="${MAC:-00:00:00:00:00:00}"
|
||||
ADMIN_USER="${ADMIN_USER:-michal}"
|
||||
|
||||
# Parse args
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--bastion-ip) BASTION_IP="$2"; shift 2 ;;
|
||||
--http-port) HTTP_PORT="$2"; shift 2 ;;
|
||||
--role) ROLE="$2"; shift 2 ;;
|
||||
--hostname) HOSTNAME="$2"; shift 2 ;;
|
||||
--mac) MAC="$2"; shift 2 ;;
|
||||
--admin-user) ADMIN_USER="$2"; shift 2 ;;
|
||||
*) echo "Unknown option: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ── Resolve upstream package URL ─────────────────────────────────
|
||||
echo "==> Fetching Asahi installer data..."
|
||||
INSTALLER_DATA=$(curl -sfL "https://cdn.asahilinux.org/installer/installer_data.json")
|
||||
|
||||
# Find the Server variant package URL
|
||||
SERVER_URL=$(echo "$INSTALLER_DATA" | python3 -c "
|
||||
import sys, json
|
||||
data = json.load(sys.stdin)
|
||||
for os in data.get('os_list', []):
|
||||
name = os.get('name', '').lower()
|
||||
if 'server' in name and 'uefi' not in name and not os.get('expert'):
|
||||
print(os['package'])
|
||||
break
|
||||
" 2>/dev/null)
|
||||
|
||||
if [ -z "$SERVER_URL" ]; then
|
||||
echo "ERROR: Could not find Fedora Asahi Remix Server in installer data."
|
||||
echo "Available variants:"
|
||||
echo "$INSTALLER_DATA" | python3 -c "
|
||||
import sys, json
|
||||
data = json.load(sys.stdin)
|
||||
for os in data.get('os_list', []):
|
||||
print(f\" - {os.get('name', '?')}\")" 2>/dev/null
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PACKAGE_NAME=$(basename "$SERVER_URL")
|
||||
echo " Variant: Fedora Asahi Remix Server"
|
||||
echo " Package: $PACKAGE_NAME"
|
||||
|
||||
# Also extract the partition layout and supported_fw from upstream
|
||||
UPSTREAM_CONFIG=$(echo "$INSTALLER_DATA" | python3 -c "
|
||||
import sys, json
|
||||
data = json.load(sys.stdin)
|
||||
for os in data.get('os_list', []):
|
||||
name = os.get('name', '').lower()
|
||||
if 'server' in name and 'uefi' not in name and not os.get('expert'):
|
||||
json.dump(os, sys.stdout)
|
||||
break
|
||||
")
|
||||
|
||||
# ── Download upstream package ────────────────────────────────────
|
||||
mkdir -p "$CACHE_DIR" "$ASAHI_DIR"
|
||||
|
||||
CACHED_PKG="$CACHE_DIR/$PACKAGE_NAME"
|
||||
if [ -f "$CACHED_PKG" ]; then
|
||||
echo "==> Using cached package: $CACHED_PKG"
|
||||
else
|
||||
echo "==> Downloading $SERVER_URL..."
|
||||
curl -# -L -o "$CACHED_PKG" "$SERVER_URL"
|
||||
fi
|
||||
|
||||
# ── Extract and modify rootfs ────────────────────────────────────
|
||||
WORK_DIR=$(mktemp -d)
|
||||
trap 'echo "==> Cleaning up..."; umount "$WORK_DIR/rootfs" 2>/dev/null || true; rm -rf "$WORK_DIR"' EXIT
|
||||
|
||||
echo "==> Extracting package..."
|
||||
unzip -q -o "$CACHED_PKG" -d "$WORK_DIR/pkg"
|
||||
|
||||
# List contents
|
||||
echo " Package contents:"
|
||||
ls -lh "$WORK_DIR/pkg/" | grep -v ^total | while read -r line; do echo " $line"; done
|
||||
|
||||
# Find root.img
|
||||
ROOT_IMG=$(find "$WORK_DIR/pkg" -name "root.img" -type f | head -1)
|
||||
if [ -z "$ROOT_IMG" ]; then
|
||||
echo "ERROR: root.img not found in package."
|
||||
echo "Contents: $(ls "$WORK_DIR/pkg/")"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==> Mounting root.img..."
|
||||
mkdir -p "$WORK_DIR/rootfs"
|
||||
mount -o loop "$ROOT_IMG" "$WORK_DIR/rootfs"
|
||||
|
||||
# ── Read SSH keys from the system ────────────────────────────────
|
||||
SSH_KEYS=""
|
||||
REAL_USER="${SUDO_USER:-$USER}"
|
||||
REAL_HOME=$(eval echo "~$REAL_USER")
|
||||
for keyfile in "$REAL_HOME/.ssh/id_ed25519.pub" "$REAL_HOME/.ssh/id_ecdsa.pub" "$REAL_HOME/.ssh/id_rsa.pub"; do
|
||||
if [ -f "$keyfile" ]; then
|
||||
SSH_KEYS=$(cat "$keyfile")
|
||||
echo " SSH key: $keyfile"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$SSH_KEYS" ]; then
|
||||
echo "WARNING: No SSH public key found. You'll need to add keys manually."
|
||||
fi
|
||||
|
||||
# ── Generate firstboot script from bastion ───────────────────────
|
||||
echo "==> Generating firstboot script..."
|
||||
|
||||
# Try to get the script from a running bastion, fall back to local generation
|
||||
FIRSTBOOT_SCRIPT=""
|
||||
FIRSTBOOT_URL="http://$BASTION_IP:$HTTP_PORT/asahi/firstboot.sh?hostname=$HOSTNAME&role=$ROLE&mac=$MAC&user=$ADMIN_USER"
|
||||
FIRSTBOOT_SCRIPT=$(curl -sf "$FIRSTBOOT_URL" 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$FIRSTBOOT_SCRIPT" ]; then
|
||||
echo " Bastion not reachable, generating script locally..."
|
||||
# Generate a basic firstboot script inline
|
||||
FIRSTBOOT_SCRIPT=$(cd "$PROJECT_DIR" && node -e "
|
||||
const { renderFirstbootScript } = require('./src/bastion/dist/templates/asahi-firstboot.sh.js');
|
||||
process.stdout.write(renderFirstbootScript({
|
||||
hostname: '$HOSTNAME',
|
||||
role: '$ROLE',
|
||||
serverIp: '$BASTION_IP',
|
||||
httpPort: $HTTP_PORT,
|
||||
sshKeys: $([ -n "$SSH_KEYS" ] && echo "[\"$SSH_KEYS\"]" || echo "[]"),
|
||||
adminUser: '$ADMIN_USER',
|
||||
mac: '$MAC',
|
||||
}));
|
||||
" 2>/dev/null) || {
|
||||
echo " ERROR: Could not generate firstboot script. Build the project first: npm run build"
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
|
||||
# ── Inject files into rootfs ─────────────────────────────────────
|
||||
echo "==> Injecting lab configuration into rootfs..."
|
||||
|
||||
# Firstboot script
|
||||
mkdir -p "$WORK_DIR/rootfs/usr/local/bin"
|
||||
echo "$FIRSTBOOT_SCRIPT" > "$WORK_DIR/rootfs/usr/local/bin/lab-firstboot.sh"
|
||||
chmod 755 "$WORK_DIR/rootfs/usr/local/bin/lab-firstboot.sh"
|
||||
echo " Installed: /usr/local/bin/lab-firstboot.sh"
|
||||
|
||||
# Systemd service
|
||||
mkdir -p "$WORK_DIR/rootfs/etc/systemd/system"
|
||||
cat > "$WORK_DIR/rootfs/etc/systemd/system/lab-firstboot.service" << 'UNIT'
|
||||
[Unit]
|
||||
Description=Lab first-boot LVM setup
|
||||
After=local-fs.target network-online.target
|
||||
Wants=network-online.target
|
||||
ConditionPathExists=!/etc/lab-lvm-setup-done
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/lab-firstboot.sh
|
||||
RemainAfterExit=yes
|
||||
StandardOutput=journal+console
|
||||
StandardError=journal+console
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
UNIT
|
||||
echo " Installed: /etc/systemd/system/lab-firstboot.service"
|
||||
|
||||
# Enable the service
|
||||
mkdir -p "$WORK_DIR/rootfs/etc/systemd/system/multi-user.target.wants"
|
||||
ln -sf /etc/systemd/system/lab-firstboot.service \
|
||||
"$WORK_DIR/rootfs/etc/systemd/system/multi-user.target.wants/lab-firstboot.service"
|
||||
echo " Enabled: lab-firstboot.service"
|
||||
|
||||
# SSH authorized keys for root (for initial access before firstboot runs user creation)
|
||||
if [ -n "$SSH_KEYS" ]; then
|
||||
mkdir -p "$WORK_DIR/rootfs/root/.ssh"
|
||||
chmod 700 "$WORK_DIR/rootfs/root/.ssh"
|
||||
echo "$SSH_KEYS" > "$WORK_DIR/rootfs/root/.ssh/authorized_keys"
|
||||
chmod 600 "$WORK_DIR/rootfs/root/.ssh/authorized_keys"
|
||||
echo " Installed: /root/.ssh/authorized_keys"
|
||||
fi
|
||||
|
||||
# Ensure lvm2 and xfsprogs are installed (should be in server image already)
|
||||
echo " Checking required packages..."
|
||||
if [ -f "$WORK_DIR/rootfs/usr/sbin/pvcreate" ] || [ -f "$WORK_DIR/rootfs/usr/bin/pvcreate" ]; then
|
||||
echo " lvm2: present"
|
||||
else
|
||||
echo " WARNING: lvm2 not found in rootfs. LVM setup may fail."
|
||||
fi
|
||||
if [ -f "$WORK_DIR/rootfs/usr/sbin/mkfs.xfs" ] || [ -f "$WORK_DIR/rootfs/usr/bin/mkfs.xfs" ]; then
|
||||
echo " xfsprogs: present"
|
||||
else
|
||||
echo " WARNING: xfsprogs not found in rootfs. LVM setup may fail."
|
||||
fi
|
||||
|
||||
# ── Unmount and repackage ────────────────────────────────────────
|
||||
echo "==> Unmounting rootfs..."
|
||||
umount "$WORK_DIR/rootfs"
|
||||
|
||||
echo "==> Repackaging..."
|
||||
OUTPUT_PKG="$ASAHI_DIR/fedora-asahi-lab.zip"
|
||||
rm -f "$OUTPUT_PKG"
|
||||
(cd "$WORK_DIR/pkg" && zip -q "$OUTPUT_PKG" *)
|
||||
echo " Output: $OUTPUT_PKG ($(du -sh "$OUTPUT_PKG" | cut -f1))"
|
||||
|
||||
# ── Generate installer_data.json ─────────────────────────────────
|
||||
echo "==> Generating installer_data.json..."
|
||||
|
||||
# Parse upstream config to get supported_fw, boot_object, next_object, and partition details
|
||||
python3 << PYEOF > "$ASAHI_DIR/installer_data.json"
|
||||
import json, sys
|
||||
|
||||
upstream = json.loads('''$UPSTREAM_CONFIG''')
|
||||
|
||||
# Build our custom installer data based on upstream
|
||||
# Keep EFI and Boot partitions identical, modify Root to not expand,
|
||||
# add Data partition that expands for LVM.
|
||||
partitions = []
|
||||
for p in upstream.get('partitions', []):
|
||||
if p.get('type') == 'EFI':
|
||||
partitions.append(p)
|
||||
elif p.get('name') == 'Boot':
|
||||
partitions.append(p)
|
||||
elif p.get('name') == 'Root':
|
||||
# Fixed size root, no expand
|
||||
root_p = dict(p)
|
||||
root_p['expand'] = False
|
||||
# Keep the original size (it's the minimum needed for the rootfs)
|
||||
partitions.append(root_p)
|
||||
|
||||
# Add Data partition for LVM
|
||||
partitions.append({
|
||||
"name": "Data",
|
||||
"type": "Linux",
|
||||
"size": "1073741824B", # 1GB minimum, will expand
|
||||
"expand": True
|
||||
})
|
||||
|
||||
data = {
|
||||
"os_list": [{
|
||||
"name": "Fedora Asahi Lab (${ROLE})",
|
||||
"default_os_name": "Fedora Linux Lab",
|
||||
"boot_object": upstream.get("boot_object", "m1n1.bin"),
|
||||
"next_object": upstream.get("next_object", "m1n1/boot.bin"),
|
||||
"package": "fedora-asahi-lab.zip",
|
||||
"supported_fw": upstream.get("supported_fw", ["13.5"]),
|
||||
"partitions": partitions,
|
||||
}]
|
||||
}
|
||||
|
||||
json.dump(data, sys.stdout, indent=2)
|
||||
print()
|
||||
PYEOF
|
||||
|
||||
echo " Generated: $ASAHI_DIR/installer_data.json"
|
||||
|
||||
# Pretty-print the partition layout
|
||||
echo ""
|
||||
echo " Partition layout:"
|
||||
python3 -c "
|
||||
import json
|
||||
with open('$ASAHI_DIR/installer_data.json') as f:
|
||||
data = json.load(f)
|
||||
for p in data['os_list'][0]['partitions']:
|
||||
size = p.get('size', '?')
|
||||
expand = ' (expand)' if p.get('expand') else ''
|
||||
image = f\" [{p['image']}]\" if 'image' in p else ''
|
||||
print(f\" {p['name']:8s} {p['type']:8s} {size:>16s}{expand}{image}\")
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "==> Build complete!"
|
||||
echo ""
|
||||
echo " Package: $ASAHI_DIR/fedora-asahi-lab.zip"
|
||||
echo " Config: $ASAHI_DIR/installer_data.json"
|
||||
echo ""
|
||||
echo " To serve from bastion, copy to the bastion's HTTP directory"
|
||||
echo " or configure REPO_BASE to point here."
|
||||
echo ""
|
||||
echo " To install on Mac Studio:"
|
||||
echo " curl http://$BASTION_IP:$HTTP_PORT/asahi | sh"
|
||||
@@ -99,22 +99,16 @@ if [ "$PUSH" = true ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# Use --tls-verify=false for plain HTTP registries (e.g. 10.0.0.194:3012)
|
||||
TLS_FLAG=""
|
||||
if [[ "$REGISTRY" =~ ^[0-9] ]] || [[ "$REGISTRY" =~ ^localhost ]]; then
|
||||
TLS_FLAG="--tls-verify=false"
|
||||
fi
|
||||
|
||||
echo "==> Logging in to $REGISTRY..."
|
||||
podman login $TLS_FLAG -u michal -p "$GITEA_TOKEN" "$REGISTRY"
|
||||
podman login -u michal -p "$GITEA_TOKEN" "$REGISTRY"
|
||||
|
||||
echo "==> Pushing $FULL_IMAGE:$TAG..."
|
||||
podman manifest push --all $TLS_FLAG "$MANIFEST" "docker://$FULL_IMAGE:$TAG"
|
||||
podman manifest push --all "$MANIFEST" "docker://$FULL_IMAGE:$TAG"
|
||||
|
||||
# Also tag as :latest if not already
|
||||
if [ "$TAG" != "latest" ]; then
|
||||
echo "==> Also pushing as :latest..."
|
||||
podman manifest push --all $TLS_FLAG "$MANIFEST" "docker://$FULL_IMAGE:latest"
|
||||
podman manifest push --all "$MANIFEST" "docker://$FULL_IMAGE:latest"
|
||||
fi
|
||||
|
||||
# Link package to repository if script exists
|
||||
|
||||
@@ -92,21 +92,15 @@ if [ "$PUSH" = true ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# Use --tls-verify=false for plain HTTP registries (e.g. 10.0.0.194:3012)
|
||||
TLS_FLAG=""
|
||||
if [[ "$REGISTRY" =~ ^[0-9] ]] || [[ "$REGISTRY" =~ ^localhost ]]; then
|
||||
TLS_FLAG="--tls-verify=false"
|
||||
fi
|
||||
|
||||
echo "==> Logging in to $REGISTRY..."
|
||||
podman login $TLS_FLAG -u michal -p "$GITEA_TOKEN" "$REGISTRY"
|
||||
podman login -u michal -p "$GITEA_TOKEN" "$REGISTRY"
|
||||
|
||||
echo "==> Pushing $FULL_IMAGE:$TAG..."
|
||||
podman manifest push --all $TLS_FLAG "$MANIFEST" "docker://$FULL_IMAGE:$TAG"
|
||||
podman manifest push --all "$MANIFEST" "docker://$FULL_IMAGE:$TAG"
|
||||
|
||||
if [ "$TAG" != "latest" ]; then
|
||||
echo "==> Also pushing as :latest..."
|
||||
podman manifest push --all $TLS_FLAG "$MANIFEST" "docker://$FULL_IMAGE:latest"
|
||||
podman manifest push --all "$MANIFEST" "docker://$FULL_IMAGE:latest"
|
||||
fi
|
||||
|
||||
if [ -f "$SCRIPT_DIR/link-package.sh" ]; then
|
||||
|
||||
@@ -24,21 +24,6 @@ deploy_bastion() {
|
||||
kubectl rollout restart deployment/bastion -n lab-infra
|
||||
kubectl rollout status deployment/bastion -n lab-infra --timeout=180s
|
||||
echo "✓ Bastion deployed"
|
||||
|
||||
# Sync Asahi rootfs package to bastion pod's persistent volume
|
||||
if [ -d "$PROJECT_DIR/asahi-repo" ] && [ -f "$PROJECT_DIR/asahi-repo/fedora-asahi-lab.zip" ]; then
|
||||
echo ""
|
||||
echo "=== Syncing Asahi rootfs to bastion pod ==="
|
||||
BASTION_POD=$(kubectl get pods -n lab-infra -l app=bastion -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
|
||||
if [ -n "$BASTION_POD" ]; then
|
||||
kubectl exec -n lab-infra "$BASTION_POD" -- mkdir -p /data/asahi-repo
|
||||
kubectl cp "$PROJECT_DIR/asahi-repo/installer_data.json" "lab-infra/$BASTION_POD:/data/asahi-repo/installer_data.json"
|
||||
kubectl cp "$PROJECT_DIR/asahi-repo/fedora-asahi-lab.zip" "lab-infra/$BASTION_POD:/data/asahi-repo/fedora-asahi-lab.zip"
|
||||
echo "✓ Asahi rootfs synced ($(du -sh "$PROJECT_DIR/asahi-repo/fedora-asahi-lab.zip" | cut -f1))"
|
||||
else
|
||||
echo "WARNING: Could not find bastion pod — Asahi rootfs not synced"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
deploy_labd() {
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Fix root SSH access on all provisioned machines.
|
||||
# Tries root, lab, michal users to find one that works,
|
||||
# then ensures root has the SSH key and PermitRootLogin is enabled.
|
||||
set -euo pipefail
|
||||
|
||||
SSH_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDMJ3FkUGbG174eoO5RjZd2eNV680FM5pgp0AgpW/QwlJExK3qxMk0DJSr4ICmzGUx4yujAXcrqU1otcOMPzzFzwc5heWpSmlNHU3TIW6NHEt0sF9ZTAbGLw2zSw3si5UouqFkCcENA40mePFJqY+Q9R8N1uvLgu4m/do+Zrn/mk5Ewc1V7OCRE5Acrnaec4T7LTB0BuVXcjPUfAmZ0q5fI+bKPR1q2Kc3+IeGhVkBuZ9OJVeXXhnpedm0uEbLeriK/jUYKYw/1QhsNDM8Tyty+UIGr9QVnWwzCMHB+wuQcDYC9mPGTqg0fYwX8Mp8xMi1PPxdsh1G7bj/cpWMAF43KswWORF2ul8ICGbaE1zEgIYXO790SuBjpBHhaC6Iegqi58hmCuP+a9893q/EU9HyrWTJHCZXC5E4kP1MsM57KrhEpszM6I3sW9f9zMTPd5QsCXFi4si4OMwX4kYNVu3fQGQPpseDPlTTSrT6uUdqj4Irm0c1m9cYTmK0vYgsM3ss= michal@fedora"
|
||||
|
||||
SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5"
|
||||
USERS_TO_TRY=(root lab michal)
|
||||
|
||||
# Machines: hostname ip
|
||||
MACHINES=(
|
||||
"labmaster 192.168.8.11"
|
||||
"worker0-k8s0 192.168.8.23"
|
||||
"worker1-k8s0 192.168.8.13"
|
||||
"worker2-k8s0 192.168.8.25"
|
||||
"spark-2935 192.168.8.12"
|
||||
)
|
||||
|
||||
BOLD="\033[1m"
|
||||
GREEN="\033[0;32m"
|
||||
RED="\033[0;31m"
|
||||
DIM="\033[2m"
|
||||
RESET="\033[0m"
|
||||
|
||||
# Script to run on each machine (via sudo if needed)
|
||||
read -r -d '' FIX_SCRIPT << 'FIXEOF' || true
|
||||
#!/bin/bash
|
||||
set -e
|
||||
KEY="$1"
|
||||
|
||||
# 1. Ensure root .ssh dir exists
|
||||
mkdir -p /root/.ssh
|
||||
chmod 700 /root/.ssh
|
||||
touch /root/.ssh/authorized_keys
|
||||
chmod 600 /root/.ssh/authorized_keys
|
||||
|
||||
# 2. Add key if not present
|
||||
if ! grep -qF "$KEY" /root/.ssh/authorized_keys 2>/dev/null; then
|
||||
echo "$KEY" >> /root/.ssh/authorized_keys
|
||||
echo "KEY_ADDED"
|
||||
else
|
||||
echo "KEY_EXISTS"
|
||||
fi
|
||||
|
||||
# 3. Fix sshd_config for root login with keys
|
||||
SSHD_CONF="/etc/ssh/sshd_config"
|
||||
CHANGED=0
|
||||
|
||||
# Ensure PermitRootLogin allows key auth
|
||||
CURRENT=$(grep -E "^PermitRootLogin" "$SSHD_CONF" 2>/dev/null | tail -1 || true)
|
||||
if [ "$CURRENT" = "PermitRootLogin prohibit-password" ] || [ "$CURRENT" = "PermitRootLogin without-password" ]; then
|
||||
echo "SSHD_OK"
|
||||
elif [ "$CURRENT" = "PermitRootLogin yes" ]; then
|
||||
echo "SSHD_OK"
|
||||
else
|
||||
# Remove any existing PermitRootLogin lines
|
||||
sed -i '/^#*PermitRootLogin/d' "$SSHD_CONF"
|
||||
echo "PermitRootLogin prohibit-password" >> "$SSHD_CONF"
|
||||
CHANGED=1
|
||||
echo "SSHD_FIXED"
|
||||
fi
|
||||
|
||||
# Ensure PubkeyAuthentication is enabled
|
||||
if grep -qE "^PubkeyAuthentication no" "$SSHD_CONF" 2>/dev/null; then
|
||||
sed -i 's/^PubkeyAuthentication no/PubkeyAuthentication yes/' "$SSHD_CONF"
|
||||
CHANGED=1
|
||||
echo "PUBKEY_FIXED"
|
||||
else
|
||||
echo "PUBKEY_OK"
|
||||
fi
|
||||
|
||||
# Restart sshd if changed
|
||||
if [ "$CHANGED" -eq 1 ]; then
|
||||
systemctl restart sshd 2>/dev/null || systemctl restart ssh 2>/dev/null || true
|
||||
echo "SSHD_RESTARTED"
|
||||
fi
|
||||
|
||||
# 4. Verify root can be reached
|
||||
echo "DONE"
|
||||
FIXEOF
|
||||
|
||||
echo ""
|
||||
echo -e "${BOLD}Fixing root SSH access on all machines...${RESET}"
|
||||
echo ""
|
||||
|
||||
for entry in "${MACHINES[@]}"; do
|
||||
read -r hostname ip <<< "$entry"
|
||||
printf " %-24s ${DIM}(%s)${RESET} " "$hostname" "$ip"
|
||||
|
||||
# Try each user until one works
|
||||
WORKING_USER=""
|
||||
for user in "${USERS_TO_TRY[@]}"; do
|
||||
if ssh $SSH_OPTS "$user@$ip" "true" 2>/dev/null; then
|
||||
WORKING_USER="$user"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$WORKING_USER" ]; then
|
||||
echo -e "${RED}UNREACHABLE${RESET} (tried: ${USERS_TO_TRY[*]})"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Run fix script (with sudo if not root)
|
||||
if [ "$WORKING_USER" = "root" ]; then
|
||||
RESULT=$(ssh $SSH_OPTS "root@$ip" "bash -s -- '$SSH_KEY'" <<< "$FIX_SCRIPT" 2>&1)
|
||||
else
|
||||
RESULT=$(ssh $SSH_OPTS "$WORKING_USER@$ip" "sudo bash -s -- '$SSH_KEY'" <<< "$FIX_SCRIPT" 2>&1)
|
||||
fi
|
||||
|
||||
# Parse result
|
||||
DETAILS=""
|
||||
if echo "$RESULT" | grep -q "KEY_ADDED"; then DETAILS="key added"; fi
|
||||
if echo "$RESULT" | grep -q "KEY_EXISTS"; then DETAILS="key ok"; fi
|
||||
if echo "$RESULT" | grep -q "SSHD_FIXED"; then DETAILS="$DETAILS, sshd fixed"; fi
|
||||
if echo "$RESULT" | grep -q "SSHD_OK"; then DETAILS="$DETAILS, sshd ok"; fi
|
||||
if echo "$RESULT" | grep -q "SSHD_RESTARTED"; then DETAILS="$DETAILS, restarted"; fi
|
||||
|
||||
# Verify root works now
|
||||
if ssh $SSH_OPTS "root@$ip" "true" 2>/dev/null; then
|
||||
echo -e "${GREEN}OK${RESET} ${DIM}(via $WORKING_USER: $DETAILS)${RESET}"
|
||||
else
|
||||
echo -e "${RED}PARTIAL${RESET} ${DIM}(via $WORKING_USER: $DETAILS -- root still blocked)${RESET}"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo -e "${BOLD}Done.${RESET} Verify: labctl provision recheck --user root"
|
||||
echo ""
|
||||
@@ -20,15 +20,6 @@ export function loadConfig(overrides: Partial<BastionConfig> = {}): BastionConfi
|
||||
const ubuntuMirror = overrides.ubuntuMirror ?? process.env["UBUNTU_MIRROR"]
|
||||
?? `https://releases.ubuntu.com/${ubuntuVersion}`;
|
||||
|
||||
// "latest" resolves the newest nightly ISO from the vyos-nightly-build GitHub
|
||||
// releases at startup. downloads.vyos.io no longer serves direct rolling ISOs
|
||||
// (it returns the vyos.io site, and nightly builds sit behind a signup form);
|
||||
// GitHub releases are the remaining free, unauthenticated direct source.
|
||||
// LTS ISOs are subscription-only. Set VYOS_ISO_URL to pin a specific build.
|
||||
const vyosIsoUrl = overrides.vyosIsoUrl ?? process.env["VYOS_ISO_URL"] ?? "latest";
|
||||
const vyosDefaultPassword = overrides.vyosDefaultPassword
|
||||
?? process.env["VYOS_DEFAULT_PASSWORD"] ?? "vyos";
|
||||
|
||||
const fedoraMirror = `https://download.fedoraproject.org/pub/fedora/linux/releases/${fedoraVersion}/Everything/${arch}/os`;
|
||||
const tftpDir = `${bastionDir}/tftp`;
|
||||
const httpDir = `${bastionDir}/http`;
|
||||
@@ -47,8 +38,6 @@ export function loadConfig(overrides: Partial<BastionConfig> = {}): BastionConfi
|
||||
dhcpRangeEnd,
|
||||
ubuntuVersion,
|
||||
ubuntuMirror,
|
||||
vyosIsoUrl,
|
||||
vyosDefaultPassword,
|
||||
// These are populated at runtime by the network service
|
||||
iface: overrides.iface ?? "",
|
||||
serverIp: overrides.serverIp ?? "",
|
||||
|
||||
@@ -40,125 +40,6 @@ function download(url: string, dest: string, label: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the largest regular-file initrd from an `xorriso -lsl` listing.
|
||||
*
|
||||
* /live carries decoys: a 0-byte initrd.img placeholder on some images, or an
|
||||
* initrd.img SYMLINK to the real version-suffixed file on others. Parsing is
|
||||
* field-based (ls -l layout: perms links uid gid size month day time 'name')
|
||||
* and considers only lines whose mode string marks a regular file — symlinks
|
||||
* report their link size, not the target's, and must not win.
|
||||
*/
|
||||
export function pickLargestInitrd(
|
||||
listing: string,
|
||||
): { name: string; size: number } | undefined {
|
||||
let best: { name: string; size: number } | undefined;
|
||||
for (const line of listing.split("\n")) {
|
||||
if (!line.startsWith("-")) continue; // regular files only
|
||||
const quoted = /'([^']+)'/.exec(line);
|
||||
const fields = line.trim().split(/\s+/);
|
||||
const size = parseInt(fields[4] ?? "", 10);
|
||||
const name = quoted?.[1] ?? "";
|
||||
if (!name.startsWith("initrd")) continue;
|
||||
if (!Number.isFinite(size) || size <= 0) continue;
|
||||
if (best === undefined || size > best.size) {
|
||||
best = { name, size };
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
const VYOS_NIGHTLY_RELEASES =
|
||||
"https://api.github.com/repos/vyos/vyos-nightly-build/releases/latest";
|
||||
|
||||
/**
|
||||
* Resolve the configured VyOS ISO URL, expanding the "latest" sentinel.
|
||||
*
|
||||
* The nightly asset filename embeds a build date, so there is no stable
|
||||
* "latest.iso" path to hardcode — the newest release has to be looked up.
|
||||
* Any other value is used verbatim, which is how VYOS_ISO_URL pins a build
|
||||
* or points at a locally mirrored copy.
|
||||
*/
|
||||
function resolveVyosIsoUrl(configured: string): string {
|
||||
if (configured !== "latest") return configured;
|
||||
|
||||
const body = execSync(`curl -sSfL "${VYOS_NIGHTLY_RELEASES}"`, {
|
||||
encoding: "utf-8",
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
const release = JSON.parse(body) as {
|
||||
tag_name?: string;
|
||||
assets?: Array<{ name: string; browser_download_url: string }>;
|
||||
};
|
||||
|
||||
const asset = (release.assets ?? []).find((a) =>
|
||||
/generic-amd64\.iso$/.test(a.name),
|
||||
);
|
||||
if (!asset) {
|
||||
throw new Error(
|
||||
`No generic-amd64 ISO asset in VyOS nightly release ${release.tag_name ?? "?"}`,
|
||||
);
|
||||
}
|
||||
|
||||
logger.info(` VyOS ISO resolved to ${asset.name} (${release.tag_name ?? "?"})`);
|
||||
return asset.browser_download_url;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract VyOS netboot artifacts from the release ISO.
|
||||
*
|
||||
* VyOS publishes no netboot bundle, so kernel/initrd/squashfs have to come out
|
||||
* of the ISO. xorriso is already in the bastion image (used for boot.iso) and
|
||||
* extracts without root or a loop mount.
|
||||
*
|
||||
* The initrd needs care: /live contains an empty initrd.img placeholder
|
||||
* alongside the real one, which carries a version-suffixed name. Booting the
|
||||
* 0-byte file fails with no useful diagnostic, so pick the largest initrd*.
|
||||
*/
|
||||
export function prepareVyosArtifacts(config: BastionConfig): void {
|
||||
const kernel = `${config.httpDir}/vyos-vmlinuz`;
|
||||
const initrd = `${config.httpDir}/vyos-initrd`;
|
||||
const squashfs = `${config.httpDir}/vyos-filesystem.squashfs`;
|
||||
|
||||
if (existsSync(kernel) && existsSync(initrd) && existsSync(squashfs)) {
|
||||
logger.info(" VyOS netboot artifacts -- cached");
|
||||
return;
|
||||
}
|
||||
|
||||
const iso = `${config.bastionDir}/vyos.iso`;
|
||||
download(resolveVyosIsoUrl(config.vyosIsoUrl), iso, "VyOS ISO");
|
||||
|
||||
const extract = (isoPath: string, dest: string, label: string): void => {
|
||||
execSync(
|
||||
`xorriso -osirrox on -indev "${iso}" -extract "${isoPath}" "${dest}"`,
|
||||
{ stdio: "pipe" },
|
||||
);
|
||||
logger.info(` ${label} -- extracted from ${isoPath}`);
|
||||
};
|
||||
|
||||
extract("/live/vmlinuz", kernel, "VyOS kernel");
|
||||
extract("/live/filesystem.squashfs", squashfs, "VyOS squashfs");
|
||||
|
||||
// Pick the real initrd by size from the ISO's own directory listing.
|
||||
const listing = execSync(`xorriso -indev "${iso}" -lsl /live/ --`, {
|
||||
encoding: "utf-8",
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
const best = pickLargestInitrd(listing);
|
||||
if (best === undefined) {
|
||||
throw new Error("No non-empty initrd found in /live on the VyOS ISO");
|
||||
}
|
||||
extract(`/live/${best.name}`, initrd, `VyOS initrd (${best.name}, ${best.size} bytes)`);
|
||||
|
||||
// The ISO is only needed to produce the three artifacts above.
|
||||
try {
|
||||
unlinkSync(iso);
|
||||
} catch {
|
||||
// Non-fatal: leaving it costs disk but nothing else.
|
||||
}
|
||||
}
|
||||
|
||||
function symlinkSafe(target: string, linkPath: string): void {
|
||||
try {
|
||||
symlinkSync(target, linkPath);
|
||||
@@ -301,17 +182,6 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
|
||||
logger.warn(`Ubuntu ${config.ubuntuVersion} artifacts not available -- Ubuntu provisioning disabled`);
|
||||
}
|
||||
|
||||
// VyOS netboot artifacts (non-fatal — same policy as Ubuntu)
|
||||
try {
|
||||
logger.info("Preparing VyOS netboot artifacts...");
|
||||
prepareVyosArtifacts(config);
|
||||
} catch (err) {
|
||||
logger.warn(
|
||||
`VyOS artifacts not available -- VyOS provisioning disabled ` +
|
||||
`(${err instanceof Error ? err.message : String(err)})`,
|
||||
);
|
||||
}
|
||||
|
||||
// Symlink iPXE binaries into HTTP dir for UEFI HTTP Boot
|
||||
for (const name of ["ipxe.efi", "ipxe-arm64.efi"]) {
|
||||
const src = `${config.tftpDir}/${name}`;
|
||||
@@ -387,11 +257,10 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
|
||||
state.update((s) => {
|
||||
s.install_queue[msg.mac] = {
|
||||
hostname: msg.hostname,
|
||||
disk: msg.disk ?? "",
|
||||
disk: msg.disk ?? "/dev/sda",
|
||||
role: msg.role as import("@lab/shared").Role,
|
||||
os: msg.os as import("@lab/shared").OsId,
|
||||
queued_at: new Date().toISOString(),
|
||||
...(msg.vyos ? { vyos: msg.vyos } : {}),
|
||||
};
|
||||
});
|
||||
return { status: "ok", data: { mac: msg.mac, hostname: msg.hostname } };
|
||||
@@ -400,7 +269,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
|
||||
labdConn.onCommand("command-debug", async (msg) => {
|
||||
if (msg.type !== "command-debug") throw new Error("unexpected");
|
||||
const mac = msg.mac.toLowerCase();
|
||||
const pxeBoot = msg.pxeBoot ?? false;
|
||||
const sshd = msg.sshd ?? false;
|
||||
const currentState = state.load();
|
||||
const hostname =
|
||||
currentState.installed[mac]?.hostname ??
|
||||
@@ -408,7 +277,7 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
|
||||
currentState.discovered[mac]?.product ??
|
||||
mac;
|
||||
state.update((s) => {
|
||||
s.debug[mac] = { hostname, queued_at: new Date().toISOString(), pxeBoot };
|
||||
s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd };
|
||||
});
|
||||
return { status: "ok", data: { mac, hostname } };
|
||||
});
|
||||
@@ -425,47 +294,6 @@ export async function startBastion(overrides: Partial<BastionConfig> = {}): Prom
|
||||
return { status: "ok", data: { mac } };
|
||||
});
|
||||
|
||||
labdConn.onCommand("command-register", async (msg) => {
|
||||
if (msg.type !== "command-register") throw new Error("unexpected");
|
||||
const mac = msg.mac.toLowerCase();
|
||||
state.update((s) => {
|
||||
s.installed[mac] = {
|
||||
hostname: msg.hostname,
|
||||
role: msg.role,
|
||||
ip: msg.ip,
|
||||
installed_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
logger.info(`MACHINE REGISTERED: ${mac} -> ${msg.hostname} (${msg.role}) ip=${msg.ip}`);
|
||||
return { status: "ok", data: { mac, hostname: msg.hostname } };
|
||||
});
|
||||
|
||||
labdConn.onCommand("command-discover", async (msg) => {
|
||||
if (msg.type !== "command-discover") throw new Error("unexpected");
|
||||
const mac = (msg.mac as string).toLowerCase();
|
||||
const now = new Date().toISOString();
|
||||
const existing = state.load().discovered[mac];
|
||||
state.update((s) => {
|
||||
s.discovered[mac] = {
|
||||
mac,
|
||||
product: (msg.product as string) ?? "unknown",
|
||||
board: (msg.board as string) ?? "unknown",
|
||||
serial: (msg.serial as string) ?? "unknown",
|
||||
manufacturer: (msg.manufacturer as string) ?? "unknown",
|
||||
cpu_model: (msg.cpu_model as string) ?? "unknown",
|
||||
cpu_cores: (msg.cpu_cores as number) ?? 0,
|
||||
memory_gb: (msg.memory_gb as number) ?? 0,
|
||||
arch: (msg.arch as string) ?? "unknown",
|
||||
disks: (msg.disks as Array<{ name: string; size_gb: number; model: string }>) ?? [],
|
||||
nics: (msg.nics as Array<{ name: string; mac: string; state: string }>) ?? [],
|
||||
first_seen: existing?.first_seen ?? now,
|
||||
last_seen: now,
|
||||
};
|
||||
});
|
||||
logger.info(`HARDWARE UPDATED: ${mac} -- ${msg.manufacturer ?? "?"} ${msg.product ?? "?"} (${msg.cpu_model ?? "?"}, ${msg.cpu_cores ?? "?"} cores, ${msg.memory_gb ?? "?"}GB RAM)`);
|
||||
return { status: "ok", data: { mac } };
|
||||
});
|
||||
|
||||
labdConn.onCommand("command-role-update", async (msg) => {
|
||||
if (msg.type !== "command-role-update") throw new Error("unexpected");
|
||||
const mac = msg.mac.toLowerCase();
|
||||
|
||||
@@ -5,28 +5,19 @@
|
||||
// /api/discover - receive hardware discovery reports from PXE-booted machines
|
||||
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import type { HardwareInfo, InstalledInfo, Role, VyosInstallSpec } from "@lab/shared";
|
||||
import { isValidOsId, SUPPORTED_ROLES, SUPPORTED_OS } from "@lab/shared";
|
||||
import type { HardwareInfo, InstalledInfo, Role } from "@lab/shared";
|
||||
import { isValidOsId, SUPPORTED_ROLES } from "@lab/shared";
|
||||
import type { StateManager } from "../services/state.js";
|
||||
import { logger } from "../services/logger.js";
|
||||
import { triggerPostProvisionK3s } from "../services/post-provision.js";
|
||||
import { progressBus } from "../services/progress-events.js";
|
||||
import type { ProgressEvent } from "../services/progress-events.js";
|
||||
import type { InstallLogBuffer } from "../services/install-log.js";
|
||||
import type { SyslogListener } from "../services/syslog-listener.js";
|
||||
|
||||
/**
|
||||
* Seconds after dispatch with zero progress before a machine is called stalled.
|
||||
* Generous: the slowest legitimate gap is fetching a ~600MB VyOS squashfs over
|
||||
* HTTP before the hook can report anything.
|
||||
*/
|
||||
const STALL_THRESHOLD_S = 8 * 60;
|
||||
|
||||
export function registerApiRoutes(
|
||||
app: FastifyInstance,
|
||||
state: StateManager,
|
||||
installLog: InstallLogBuffer,
|
||||
syslog: SyslogListener,
|
||||
): void {
|
||||
// List all machines
|
||||
app.get("/api/machines", async (_request, reply) => {
|
||||
@@ -41,10 +32,9 @@ export function registerApiRoutes(
|
||||
disk?: string;
|
||||
role?: string;
|
||||
os?: string;
|
||||
vyos?: VyosInstallSpec;
|
||||
};
|
||||
}>("/api/install", async (request, reply) => {
|
||||
const { mac: rawMac, hostname, disk, role, os, vyos } = request.body ?? {};
|
||||
const { mac: rawMac, hostname, disk, role, os } = request.body ?? {};
|
||||
const mac = (rawMac ?? "").toLowerCase().replace(/-/g, ":");
|
||||
|
||||
if (mac === "") {
|
||||
@@ -58,7 +48,7 @@ export function registerApiRoutes(
|
||||
|
||||
const osId = os ?? "fedora-43";
|
||||
if (!isValidOsId(osId)) {
|
||||
return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: ${SUPPORTED_OS.join(", ")}` });
|
||||
return reply.status(400).send({ error: `invalid os: '${osId}'. Supported: fedora-43, ubuntu-26.04` });
|
||||
}
|
||||
|
||||
state.update((s) => {
|
||||
@@ -68,7 +58,6 @@ export function registerApiRoutes(
|
||||
role: validRole as Role,
|
||||
os: osId,
|
||||
queued_at: new Date().toISOString(),
|
||||
...(vyos ? { vyos } : {}),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -95,11 +84,6 @@ export function registerApiRoutes(
|
||||
const { mac: rawMac, stage, detail } = request.body ?? {};
|
||||
const mac = (rawMac ?? "unknown").toLowerCase();
|
||||
const stageName = stage ?? "unknown";
|
||||
|
||||
// Register IP → MAC for syslog routing
|
||||
if (mac !== "unknown") {
|
||||
syslog.registerIp(request.ip, mac);
|
||||
}
|
||||
const detailStr = detail ?? "";
|
||||
|
||||
const GREEN = "\x1b[0;32m";
|
||||
@@ -148,36 +132,20 @@ export function registerApiRoutes(
|
||||
? detailStr.replace("ready at ", "").trim()
|
||||
: "";
|
||||
|
||||
const hw = s.discovered[mac];
|
||||
const installedInfo: InstalledInfo = {
|
||||
hostname: cfg?.hostname ?? "?",
|
||||
role: cfg?.role ?? "?",
|
||||
...(cfg?.os !== undefined ? { os: cfg.os } : {}),
|
||||
ip,
|
||||
installed_at: new Date().toISOString(),
|
||||
// Preserve hardware info from discovery
|
||||
...(hw ? {
|
||||
product: hw.product,
|
||||
manufacturer: hw.manufacturer,
|
||||
cpu_model: hw.cpu_model,
|
||||
cpu_cores: hw.cpu_cores,
|
||||
memory_gb: hw.memory_gb,
|
||||
arch: hw.arch,
|
||||
} : {}),
|
||||
};
|
||||
s.installed[mac] = installedInfo;
|
||||
|
||||
// VyOS: the only login user is "vyos", and a router never runs k3s —
|
||||
// without this guard a non-vanilla role + recorded IP would trigger
|
||||
// the k3s post-provision against a VyOS box.
|
||||
const isVyos = (installedInfo.os ?? "").startsWith("vyos");
|
||||
const admin = isVyos
|
||||
? "vyos"
|
||||
: installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "lab" : "root";
|
||||
const admin = installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "michal" : "root";
|
||||
console.log(`\n \x1b[0;32m\x1b[1m ssh ${admin}@${ip}\x1b[0m\n`); // eslint-disable-line no-console
|
||||
|
||||
// Auto-install k3s for non-vanilla roles
|
||||
if (!isVyos && installedInfo.role !== "vanilla" && ip !== "") {
|
||||
if (installedInfo.role !== "vanilla" && ip !== "") {
|
||||
void triggerPostProvisionK3s(installedInfo.hostname, ip, installedInfo.role, admin, mac);
|
||||
}
|
||||
}
|
||||
@@ -223,10 +191,10 @@ export function registerApiRoutes(
|
||||
|
||||
// Queue debug/rescue mode for a machine
|
||||
app.post<{
|
||||
Body: { mac?: string; pxeBoot?: boolean };
|
||||
Body: { mac?: string; sshd?: boolean };
|
||||
}>("/api/debug", async (request, reply) => {
|
||||
const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":");
|
||||
const pxeBoot = request.body?.pxeBoot ?? false;
|
||||
const sshd = request.body?.sshd ?? false;
|
||||
if (mac === "") {
|
||||
return reply.status(400).send({ error: "mac is required" });
|
||||
}
|
||||
@@ -240,7 +208,7 @@ export function registerApiRoutes(
|
||||
mac;
|
||||
|
||||
state.update((s) => {
|
||||
s.debug[mac] = { hostname, queued_at: new Date().toISOString(), pxeBoot };
|
||||
s.debug[mac] = { hostname, queued_at: new Date().toISOString(), sshd };
|
||||
});
|
||||
|
||||
logger.info(`DEBUG QUEUED: ${mac} -> ${hostname}`);
|
||||
@@ -340,67 +308,6 @@ export function registerApiRoutes(
|
||||
return reply.send({ status: "ok", mac, new: isNew });
|
||||
});
|
||||
|
||||
// Register an already-installed machine (e.g. re-add after state loss)
|
||||
app.post<{
|
||||
Body: {
|
||||
mac?: string;
|
||||
hostname?: string;
|
||||
role?: string;
|
||||
ip?: string;
|
||||
};
|
||||
}>("/api/register", async (request, reply) => {
|
||||
const { mac: rawMac, hostname, role, ip } = request.body ?? {};
|
||||
const mac = (rawMac ?? "").toLowerCase().replace(/-/g, ":");
|
||||
|
||||
if (mac === "") {
|
||||
return reply.status(400).send({ error: "mac is required" });
|
||||
}
|
||||
if (!hostname) {
|
||||
return reply.status(400).send({ error: "hostname is required" });
|
||||
}
|
||||
|
||||
const validRole = role ?? "worker";
|
||||
if (!(SUPPORTED_ROLES as readonly string[]).includes(validRole)) {
|
||||
return reply.status(400).send({ error: `invalid role: '${validRole}'. Supported: ${SUPPORTED_ROLES.join(", ")}` });
|
||||
}
|
||||
|
||||
state.update((s) => {
|
||||
s.installed[mac] = {
|
||||
hostname,
|
||||
role: validRole,
|
||||
ip: ip ?? "",
|
||||
installed_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
logger.info(`MACHINE REGISTERED: ${mac} -> hostname=${hostname} role=${validRole} ip=${ip ?? ""}`);
|
||||
|
||||
return reply.send({
|
||||
status: "registered",
|
||||
mac,
|
||||
hostname,
|
||||
role: validRole,
|
||||
ip: ip ?? "",
|
||||
});
|
||||
});
|
||||
|
||||
// Simple machine state query (used by ks-auto for ISO boot dispatch)
|
||||
app.get<{
|
||||
Params: { mac: string };
|
||||
}>("/api/machine-state/:mac", async (request, reply) => {
|
||||
const mac = request.params.mac.toLowerCase().replace(/-/g, ":");
|
||||
const currentState = state.load();
|
||||
|
||||
if (currentState.debug[mac]) return reply.send("debug");
|
||||
if (currentState.install_queue[mac]) {
|
||||
const progress = currentState.install_queue[mac].progress;
|
||||
return reply.send(progress ? "installing" : "queued");
|
||||
}
|
||||
if (currentState.installed[mac]) return reply.send("installed");
|
||||
if (currentState.discovered[mac]) return reply.send("discovered");
|
||||
return reply.send("unknown");
|
||||
});
|
||||
|
||||
// Update a machine's role (e.g. promote infra -> labcontroller)
|
||||
app.post<{
|
||||
Body: {
|
||||
@@ -449,15 +356,6 @@ export function registerApiRoutes(
|
||||
const installedEntry = currentState.installed[mac];
|
||||
|
||||
if (queueEntry) {
|
||||
// A machine that was handed an install script but has reported nothing
|
||||
// since is wedged BEFORE the installer environment came up — a bad
|
||||
// kernel/initrd, no network in the initramfs, or the wrong NIC picked.
|
||||
// Surfacing it here is what makes that diagnosable without a console.
|
||||
const since = queueEntry.progress_at ?? queueEntry.dispatched_at;
|
||||
const stalledForS = since !== undefined && queueEntry.progress === undefined
|
||||
? Math.floor((Date.now() - new Date(since).getTime()) / 1000)
|
||||
: 0;
|
||||
|
||||
return reply.send({
|
||||
mac,
|
||||
hostname: queueEntry.hostname,
|
||||
@@ -465,9 +363,6 @@ export function registerApiRoutes(
|
||||
progress: queueEntry.progress ?? "queued",
|
||||
progress_detail: queueEntry.progress_detail ?? "",
|
||||
progress_at: queueEntry.progress_at ?? queueEntry.queued_at,
|
||||
dispatched_at: queueEntry.dispatched_at,
|
||||
stalled_for_s: stalledForS,
|
||||
stalled: stalledForS > STALL_THRESHOLD_S,
|
||||
role: queueEntry.role,
|
||||
os: queueEntry.os,
|
||||
stages: queueEntry.log ?? [],
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
// Routes for Asahi Linux provisioning.
|
||||
// GET /asahi — wrapper script (curl bastion:8080/asahi | sh)
|
||||
// GET /asahi/installer_data.json — custom installer config (built or fallback)
|
||||
// GET /asahi/repo/* — serves built rootfs package (fedora-asahi-lab.zip)
|
||||
// GET /asahi/firstboot.sh — first-boot LVM setup script (for manual use)
|
||||
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import fastifyStatic from "@fastify/static";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import type { BastionConfig } from "@lab/shared";
|
||||
import { renderFirstbootScript, renderFirstbootUnit } from "../templates/asahi-firstboot.sh.js";
|
||||
import type { Role } from "@lab/shared";
|
||||
|
||||
/** Find the asahi-repo directory (built by scripts/build-asahi-rootfs.sh). */
|
||||
function findAsahiRepo(config: BastionConfig): string | null {
|
||||
// Check relative to bastionDir (container deploy)
|
||||
const inBastionDir = join(config.bastionDir, "asahi-repo");
|
||||
if (existsSync(inBastionDir)) return inBastionDir;
|
||||
|
||||
// Check /data/asahi-repo (PVC mount in k3s container)
|
||||
if (existsSync("/data/asahi-repo")) return "/data/asahi-repo";
|
||||
|
||||
// Check relative to project root (dev mode)
|
||||
try {
|
||||
const thisDir = dirname(fileURLToPath(import.meta.url));
|
||||
const projectRoot = join(thisDir, "..", "..", "..", "..");
|
||||
const inProjectRoot = join(projectRoot, "asahi-repo");
|
||||
if (existsSync(inProjectRoot)) return inProjectRoot;
|
||||
} catch { /* import.meta.url not available in tests */ }
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function registerAsahiRoutes(app: FastifyInstance, config: BastionConfig): void {
|
||||
const repoDir = findAsahiRepo(config);
|
||||
|
||||
// Serve built rootfs package files (fedora-asahi-lab.zip, etc.)
|
||||
if (repoDir) {
|
||||
app.register(fastifyStatic, {
|
||||
root: repoDir,
|
||||
prefix: "/asahi/repo/",
|
||||
decorateReply: false,
|
||||
});
|
||||
}
|
||||
|
||||
// Wrapper script — user runs: curl http://bastion:8080/asahi | sh
|
||||
app.get("/asahi", async (_request, reply) => {
|
||||
const script = `#!/bin/bash
|
||||
# Lab Asahi provisioner — sets up Apple Silicon machines with lab LVM layout.
|
||||
# This wraps the standard Asahi installer with custom installer_data.json
|
||||
# that creates a separate LVM data partition.
|
||||
set -euo pipefail
|
||||
|
||||
BASTION="http://${config.serverIp}:${config.httpPort}"
|
||||
|
||||
echo ""
|
||||
echo " ╔══════════════════════════════════════════════╗"
|
||||
echo " ║ Lab Asahi Provisioner ║"
|
||||
echo " ║ Bastion: \${BASTION} ║"
|
||||
echo " ╚══════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
|
||||
# Check we're on macOS
|
||||
if [ "$(uname)" != "Darwin" ]; then
|
||||
echo "ERROR: This script must be run from macOS on the target Mac."
|
||||
echo " It uses the Asahi Linux installer to set up Apple Silicon boot."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Download the standard Asahi installer
|
||||
echo "Downloading Asahi Linux installer..."
|
||||
WORKDIR=$(mktemp -d)
|
||||
cd "$WORKDIR"
|
||||
|
||||
INSTALLER_BASE="https://cdn.asahilinux.org/installer"
|
||||
PKG_VER=$(curl -s "\${INSTALLER_BASE}/latest")
|
||||
echo " Version: \${PKG_VER}"
|
||||
|
||||
curl -# -L -o "installer-\${PKG_VER}.tar.gz" "\${INSTALLER_BASE}/installer-\${PKG_VER}.tar.gz"
|
||||
|
||||
echo " Extracting..."
|
||||
tar xf "installer-\${PKG_VER}.tar.gz"
|
||||
|
||||
# Download our custom installer_data.json (installer reads it as a local file)
|
||||
echo " Downloading custom installer data from bastion..."
|
||||
curl -sfL -o installer_data.json "\${BASTION}/asahi/installer_data.json"
|
||||
|
||||
# Pre-download the rootfs package (avoids Python HTTP streaming issues on macOS)
|
||||
echo " Downloading rootfs package from bastion..."
|
||||
mkdir -p os
|
||||
curl -# -L -o os/fedora-asahi-lab.zip "\${BASTION}/asahi/repo/fedora-asahi-lab.zip"
|
||||
|
||||
# Point installer to local directory (REPO_BASE + /os/ + package name)
|
||||
export REPO_BASE="\${PWD}"
|
||||
|
||||
echo ""
|
||||
echo " Using custom partition layout + rootfs from bastion."
|
||||
echo " This will create:"
|
||||
echo " - Standard Asahi boot infrastructure (m1n1 + U-Boot)"
|
||||
echo " - Fedora Asahi Remix root partition"
|
||||
echo " - LVM data partition (remaining space)"
|
||||
echo ""
|
||||
echo " After first boot, SSH in and set up LVM:"
|
||||
echo " ssh lab@<ip> 'curl -sf \${BASTION}/asahi/firstboot.sh | sudo bash'"
|
||||
echo ""
|
||||
|
||||
# Run the installer
|
||||
if [ "$USER" != "root" ]; then
|
||||
echo "The installer needs root. Enter your sudo password if prompted."
|
||||
exec caffeinate -dis sudo -E ./install.sh "$@"
|
||||
else
|
||||
exec caffeinate -dis ./install.sh "$@"
|
||||
fi
|
||||
`;
|
||||
return reply.type("text/x-shellscript").send(script);
|
||||
});
|
||||
|
||||
// Custom installer_data.json — serves built config or fallback
|
||||
app.get("/asahi/installer_data.json", async (_request, reply) => {
|
||||
// Prefer the built installer_data.json (from build-asahi-rootfs.sh)
|
||||
if (repoDir) {
|
||||
const builtConfig = join(repoDir, "installer_data.json");
|
||||
if (existsSync(builtConfig)) {
|
||||
const data = JSON.parse(readFileSync(builtConfig, "utf-8"));
|
||||
return reply.type("application/json").send(data);
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: minimal config (won't have boot.img, for testing only)
|
||||
return reply.type("application/json").send({
|
||||
os_list: [{
|
||||
name: "Fedora Asahi Lab",
|
||||
default_os_name: "Fedora Linux with Lab LVM",
|
||||
boot_object: "m1n1.bin",
|
||||
next_object: "m1n1/boot.bin",
|
||||
package: "fedora-asahi-lab.zip",
|
||||
supported_fw: ["13.5"],
|
||||
partitions: [
|
||||
{ name: "EFI", type: "EFI", size: "524288000B", format: "fat",
|
||||
copy_firmware: true, copy_installer_data: true, source: "esp" },
|
||||
{ name: "Root", type: "Linux", size: "5368709120B", image: "root.img", expand: false },
|
||||
{ name: "Data", type: "Linux", size: "1073741824B", expand: true },
|
||||
],
|
||||
}],
|
||||
});
|
||||
});
|
||||
|
||||
// First-boot script — for manual download or embedding in rootfs
|
||||
app.get<{
|
||||
Querystring: { hostname?: string; role?: string; mac?: string; user?: string };
|
||||
}>("/asahi/firstboot.sh", async (request, reply) => {
|
||||
const hostname = request.query.hostname ?? "unknown";
|
||||
const role = (request.query.role ?? "infra") as Role;
|
||||
const mac = request.query.mac ?? "unknown";
|
||||
const user = request.query.user ?? "lab";
|
||||
|
||||
const script = renderFirstbootScript({
|
||||
hostname,
|
||||
role,
|
||||
serverIp: config.serverIp,
|
||||
httpPort: config.httpPort,
|
||||
sshKeys: config.sshKeys ?? [],
|
||||
adminUser: user,
|
||||
mac,
|
||||
});
|
||||
|
||||
return reply.type("text/x-shellscript").send(script);
|
||||
});
|
||||
|
||||
// Systemd unit file for first-boot service
|
||||
app.get("/asahi/firstboot.service", async (_request, reply) => {
|
||||
return reply.type("text/plain").send(renderFirstbootUnit());
|
||||
});
|
||||
}
|
||||
@@ -137,7 +137,7 @@ function generateIso(config: BastionConfig, outputPath: string): void {
|
||||
"# Map iPXE arch names to Fedora mirror paths (arm64 -> aarch64)",
|
||||
"set fedarch ${buildarch}",
|
||||
"iseq ${buildarch} arm64 && set fedarch aarch64 ||",
|
||||
`kernel file:/vmlinuz-\${buildarch} inst.ks=${bastionUrl}/ks-auto inst.repo=${FEDORA_MIRROR_BASE}/${config.fedoraVersion}/Everything/\${fedarch}/os inst.text || goto no_kernel`,
|
||||
`kernel file:/vmlinuz-\${buildarch} inst.ks=${bastionUrl}/discover.ks inst.repo=${FEDORA_MIRROR_BASE}/${config.fedoraVersion}/Everything/\${fedarch}/os inst.text || goto no_kernel`,
|
||||
`initrd file:/initrd-\${buildarch} || goto no_kernel`,
|
||||
"boot || shell",
|
||||
"",
|
||||
|
||||
@@ -11,11 +11,9 @@ import {
|
||||
renderDiscoverIpxe,
|
||||
renderInstallIpxe,
|
||||
renderDebugIpxe,
|
||||
renderPxeBootDebugIpxe,
|
||||
renderLocalBootIpxe,
|
||||
} from "../templates/boot.ipxe.js";
|
||||
import { renderUbuntuInstallIpxe } from "../templates/ubuntu-boot.ipxe.js";
|
||||
import { renderVyosInstallIpxe } from "../templates/vyos-boot.ipxe.js";
|
||||
import { renderDebugKickstart } from "../templates/debug.ks.js";
|
||||
import { logger } from "../services/logger.js";
|
||||
|
||||
@@ -24,44 +22,21 @@ export function registerDispatchRoutes(
|
||||
config: BastionConfig,
|
||||
state: StateManager,
|
||||
): void {
|
||||
// Serve debug/rescue kickstart (minimal: SSH keys + network for inst.sshd)
|
||||
app.get<{ Querystring: { mac?: string } }>("/debug.ks", async (_request, reply) => {
|
||||
// Serve debug/rescue kickstart (minimal: SSH keys + network)
|
||||
app.get<{ Querystring: { mac?: string; sshd?: string } }>("/debug.ks", async (request, reply) => {
|
||||
const mac = (request.query.mac ?? "").toLowerCase().replace(/-/g, ":");
|
||||
const currentState = state.load();
|
||||
const wantSshd = request.query.sshd === "1" || currentState.debug[mac]?.sshd === true;
|
||||
|
||||
const ks = renderDebugKickstart({
|
||||
sshKeys: config.sshKeys ?? [],
|
||||
sshd: wantSshd,
|
||||
serverIp: config.serverIp,
|
||||
httpPort: config.httpPort,
|
||||
});
|
||||
return reply.type("text/plain").send(ks);
|
||||
});
|
||||
|
||||
// Shell script for manual debug setup (nc listener + IP reporting)
|
||||
// Usage from rescue shell: curl http://bastion:port/debug-setup.sh | bash
|
||||
app.get("/debug-setup.sh", async (_request, reply) => {
|
||||
const script = `#!/bin/bash
|
||||
# Lab Bastion debug setup — run from rescue shell
|
||||
set -x
|
||||
|
||||
IP_ADDR=$(ip -4 addr show | awk '/inet / && !/127.0.0/ {split($2,a,"/"); print a[1]; exit}')
|
||||
MAC_ADDR=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')
|
||||
|
||||
# Start persistent nc listener for remote shell
|
||||
(while true; do nc -l -p 2323 -e /bin/bash 2>/dev/null; done) &
|
||||
echo "nc shell listener on port 2323"
|
||||
|
||||
# Report IP to bastion
|
||||
curl -sf -X POST "http://${config.serverIp}:${config.httpPort}/api/progress" \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d "{\\"mac\\":\\"$MAC_ADDR\\",\\"stage\\":\\"debug-ready\\",\\"detail\\":\\"nc $IP_ADDR 2323\\"}" 2>/dev/null || true
|
||||
|
||||
echo ""
|
||||
echo "=== Debug environment ready ==="
|
||||
echo " nc $IP_ADDR 2323 (remote shell)"
|
||||
echo " ssh root@$IP_ADDR (password: debug)"
|
||||
echo "==============================="
|
||||
`;
|
||||
return reply.type("text/plain").send(script);
|
||||
});
|
||||
|
||||
app.get<{ Querystring: { mac?: string } }>("/dispatch", async (request, reply) => {
|
||||
const mac = (request.query.mac ?? "").toLowerCase().replace(/-/g, ":");
|
||||
const currentState = state.load();
|
||||
@@ -70,27 +45,17 @@ echo "==============================="
|
||||
const debugEntry = currentState.debug[mac];
|
||||
if (debugEntry) {
|
||||
const hostname = debugEntry.hostname ?? "debug";
|
||||
logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (rescue mode)`);
|
||||
|
||||
state.update((s) => { delete s.debug[mac]; });
|
||||
|
||||
let script: string;
|
||||
if (debugEntry.pxeBoot) {
|
||||
logger.info(`PXE BOOT DEBUG: ${mac} -> ${hostname} (kernel+initrd from PXE, root from NVMe)`);
|
||||
script = renderPxeBootDebugIpxe({
|
||||
mac,
|
||||
hostname,
|
||||
serverIp: config.serverIp,
|
||||
httpPort: config.httpPort,
|
||||
});
|
||||
} else {
|
||||
logger.info(`DEBUG BOOT: ${mac} -> ${hostname} (rescue mode)`);
|
||||
script = renderDebugIpxe({
|
||||
mac,
|
||||
hostname,
|
||||
serverIp: config.serverIp,
|
||||
httpPort: config.httpPort,
|
||||
fedoraMirror: config.fedoraMirror,
|
||||
});
|
||||
}
|
||||
const script = renderDebugIpxe({
|
||||
mac,
|
||||
hostname,
|
||||
serverIp: config.serverIp,
|
||||
httpPort: config.httpPort,
|
||||
fedoraMirror: config.fedoraMirror,
|
||||
});
|
||||
return reply.type("text/plain").send(script);
|
||||
}
|
||||
|
||||
@@ -100,22 +65,8 @@ echo "==============================="
|
||||
const os = queueEntry.os ?? "fedora-43";
|
||||
logger.info(`INSTALL STARTED: ${mac} -> ${hostname} (${os})`);
|
||||
|
||||
// Stamp the handoff so a machine that boots the installer but never
|
||||
// reports can be spotted without a console.
|
||||
state.update((s) => {
|
||||
const entry = s.install_queue[mac];
|
||||
if (entry) entry.dispatched_at = new Date().toISOString();
|
||||
});
|
||||
|
||||
let script: string;
|
||||
if (os.startsWith("vyos")) {
|
||||
script = renderVyosInstallIpxe({
|
||||
mac,
|
||||
hostname,
|
||||
serverIp: config.serverIp,
|
||||
httpPort: config.httpPort,
|
||||
});
|
||||
} else if (os.startsWith("ubuntu")) {
|
||||
if (os.startsWith("ubuntu")) {
|
||||
script = renderUbuntuInstallIpxe({
|
||||
mac,
|
||||
hostname,
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import type { BastionConfig } from "@lab/shared";
|
||||
import type { StateManager } from "../services/state.js";
|
||||
import type { SyslogListener } from "../services/syslog-listener.js";
|
||||
import { generateInstallKickstart, generateDiscoverKickstart } from "../services/kickstart-generator.js";
|
||||
import { renderUbuntuAutoinstall, renderUbuntuMetaData, type UbuntuAutoinstallParams } from "../templates/ubuntu-autoinstall.js";
|
||||
|
||||
@@ -13,7 +12,6 @@ export function registerKickstartRoutes(
|
||||
app: FastifyInstance,
|
||||
config: BastionConfig,
|
||||
state: StateManager,
|
||||
syslog: SyslogListener,
|
||||
): void {
|
||||
// Per-MAC install kickstart
|
||||
app.get<{ Querystring: { mac?: string } }>("/ks", async (request, reply) => {
|
||||
@@ -21,11 +19,6 @@ export function registerKickstartRoutes(
|
||||
const currentState = state.load();
|
||||
const queueEntry = currentState.install_queue[mac];
|
||||
|
||||
// Register IP → MAC so syslog listener can route Anaconda logs
|
||||
if (mac) {
|
||||
syslog.registerIp(request.ip, mac);
|
||||
}
|
||||
|
||||
const ks = generateInstallKickstart(config, {
|
||||
hostname: queueEntry?.hostname ?? "lab-node",
|
||||
disk: queueEntry?.disk ?? "",
|
||||
@@ -41,150 +34,6 @@ export function registerKickstartRoutes(
|
||||
return reply.type("text/plain").send(ks);
|
||||
});
|
||||
|
||||
// Auto-detecting kickstart for ISO boot (no-network machines like R1 ARM).
|
||||
// %pre detects MAC, queries bastion state, writes dynamic kickstart to /tmp.
|
||||
// Main body %include's it — so Anaconda gets either discover or install content.
|
||||
app.get("/ks-auto", async (_request, reply) => {
|
||||
const bastionUrl = `http://${config.serverIp}:${config.httpPort}`;
|
||||
|
||||
const ks = `# Lab Bastion -- Auto-detect kickstart (ISO boot)
|
||||
# %pre detects MAC, queries bastion state, writes /tmp/dynamic.ks.
|
||||
# Main body %include's it to get either discovery reboot or full install.
|
||||
|
||||
%pre --erroronfail --log=/tmp/ks-auto.log
|
||||
#!/bin/bash
|
||||
set -x
|
||||
|
||||
# -- Detect MAC address --
|
||||
MAC=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')
|
||||
echo "Detected MAC: $MAC"
|
||||
|
||||
# -- Wait for network (Linux drivers may take a moment) --
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf "${bastionUrl}/healthz" >/dev/null 2>&1; then
|
||||
echo "Bastion reachable at ${bastionUrl}"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for network... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# -- Query bastion for machine state --
|
||||
STATE=$(curl -sf "${bastionUrl}/api/machine-state/$MAC" 2>/dev/null || echo "unknown")
|
||||
echo "Machine state: $STATE"
|
||||
|
||||
case "$STATE" in
|
||||
queued|installing)
|
||||
echo "=== Machine queued for install. Fetching install kickstart... ==="
|
||||
curl -sf "${bastionUrl}/ks?mac=$MAC" > /tmp/dynamic.ks
|
||||
if [ -s /tmp/dynamic.ks ]; then
|
||||
echo "Install kickstart downloaded ($(wc -l < /tmp/dynamic.ks) lines)"
|
||||
else
|
||||
echo "ERROR: Failed to download install kickstart"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Run any %pre scripts from the downloaded kickstart.
|
||||
# Anaconda only runs %pre from the top-level file, not from %include'd files.
|
||||
python3 -c "
|
||||
import re, subprocess
|
||||
content = open('/tmp/dynamic.ks').read()
|
||||
blocks = re.findall(r'%pre[^\\n]*\\n(.*?)%end', content, re.DOTALL)
|
||||
for i, script in enumerate(blocks):
|
||||
path = f'/tmp/inner-pre-{i}.sh'
|
||||
with open(path, 'w') as f:
|
||||
f.write(script)
|
||||
print(f'Running inner %pre script {i} ({len(script.splitlines())} lines)')
|
||||
subprocess.run(['bash', path], check=False)
|
||||
"
|
||||
;;
|
||||
|
||||
debug)
|
||||
echo "=== Debug mode ==="
|
||||
curl -sf "${bastionUrl}/debug.ks?mac=$MAC" > /tmp/dynamic.ks 2>/dev/null
|
||||
if [ ! -s /tmp/dynamic.ks ]; then
|
||||
echo "rescue" > /tmp/dynamic.ks
|
||||
fi
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "=== Running hardware discovery ==="
|
||||
# Collect hardware info
|
||||
PRODUCT=$(cat /sys/class/dmi/id/product_name 2>/dev/null || echo "unknown")
|
||||
BOARD=$(cat /sys/class/dmi/id/board_name 2>/dev/null || echo "unknown")
|
||||
SERIAL=$(cat /sys/class/dmi/id/product_serial 2>/dev/null || echo "unknown")
|
||||
MANUFACTURER=$(cat /sys/class/dmi/id/sys_vendor 2>/dev/null || echo "unknown")
|
||||
CPUMODEL=$(grep -m1 'model name' /proc/cpuinfo | cut -d: -f2 | sed 's/^ //')
|
||||
CPUCORES=$(grep -c '^processor' /proc/cpuinfo)
|
||||
MEMGB=$(awk '/MemTotal/ {printf "%d", $2/1024/1024}' /proc/meminfo)
|
||||
ARCHTYPE=$(uname -m)
|
||||
|
||||
DISKS_JSON=$(lsblk -Jb -o NAME,SIZE,TYPE,MODEL 2>/dev/null | python3 -c "
|
||||
import sys, json
|
||||
data = json.load(sys.stdin)
|
||||
disks = [d for d in data.get('blockdevices', []) if d.get('type') == 'disk']
|
||||
result = []
|
||||
for d in disks:
|
||||
size_gb = round(int(d.get('size', 0)) / 1073741824, 1)
|
||||
result.append({'name': d.get('name', '?'), 'size_gb': size_gb, 'model': (d.get('model') or 'unknown').strip()})
|
||||
print(json.dumps(result))
|
||||
" 2>/dev/null || echo '[]')
|
||||
|
||||
NICS_JSON=$(ip -j link show 2>/dev/null | python3 -c "
|
||||
import sys, json
|
||||
nics = json.load(sys.stdin)
|
||||
result = []
|
||||
for n in nics:
|
||||
if n.get('link_type') == 'loopback': continue
|
||||
result.append({'name': n.get('ifname', '?'), 'mac': n.get('address', '?'), 'state': n.get('operstate', '?')})
|
||||
print(json.dumps(result))
|
||||
" 2>/dev/null || echo '[]')
|
||||
|
||||
PAYLOAD=$(python3 -c "
|
||||
import json
|
||||
print(json.dumps({
|
||||
'mac': '$MAC', 'product': '$PRODUCT', 'board': '$BOARD', 'serial': '$SERIAL',
|
||||
'manufacturer': '$MANUFACTURER', 'cpu_model': '$CPUMODEL',
|
||||
'cpu_cores': int('$CPUCORES' or 0), 'memory_gb': int('$MEMGB' or 0),
|
||||
'arch': '$ARCHTYPE', 'disks': $DISKS_JSON, 'nics': $NICS_JSON
|
||||
}))
|
||||
")
|
||||
|
||||
curl -sf -X POST "${bastionUrl}/api/discover" \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d "$PAYLOAD" || true
|
||||
|
||||
echo ""
|
||||
echo "=== Discovery complete ==="
|
||||
echo "Machine MAC: $MAC"
|
||||
echo "Queue for install: labctl provision install $MAC <hostname> --role infra"
|
||||
echo "Then reboot to start installation."
|
||||
echo ""
|
||||
|
||||
# Write a minimal kickstart that just reboots
|
||||
cat > /tmp/dynamic.ks << 'DISCOVER_KS'
|
||||
# Discovery mode -- reboot to allow install queue
|
||||
reboot
|
||||
DISCOVER_KS
|
||||
|
||||
# Force reboot now (don't wait for Anaconda)
|
||||
sleep 3
|
||||
echo 1 > /proc/sys/kernel/sysrq
|
||||
echo b > /proc/sysrq-trigger
|
||||
sleep 5
|
||||
reboot -f
|
||||
;;
|
||||
esac
|
||||
|
||||
%end
|
||||
|
||||
# Include the dynamically chosen kickstart
|
||||
%include /tmp/dynamic.ks
|
||||
`;
|
||||
|
||||
return reply.type("text/plain").send(ks);
|
||||
});
|
||||
|
||||
// Ubuntu autoinstall user-data (cloud-init)
|
||||
app.get<{ Params: { mac: string } }>("/autoinstall/:mac/user-data", async (request, reply) => {
|
||||
const mac = request.params.mac.toLowerCase().replace(/-/g, ":");
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
// VyOS network install routes.
|
||||
//
|
||||
// VyOS has no unattended installer, so the automation is injected via
|
||||
// live-config's `hooks` component: the iPXE script passes
|
||||
// live-config.hooks=<.../vyos/autoinstall.sh>, live-config wgets it and runs it
|
||||
// as root, and that script fetches and executes the generated install driver.
|
||||
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import type { BastionConfig } from "@lab/shared";
|
||||
import type { StateManager } from "../services/state.js";
|
||||
import { buildVyosConfigSpec } from "../templates/vyos-config-spec.js";
|
||||
import { renderVyosInstallPy } from "../templates/vyos-install.py.js";
|
||||
import { logger } from "../services/logger.js";
|
||||
|
||||
function normalizeMac(value: string | undefined): string {
|
||||
return (value ?? "").toLowerCase().replace(/-/g, ":");
|
||||
}
|
||||
|
||||
export function registerVyosRoutes(
|
||||
app: FastifyInstance,
|
||||
config: BastionConfig,
|
||||
state: StateManager,
|
||||
): void {
|
||||
// live-config hook. Kept minimal: everything version-specific lives in the
|
||||
// generated Python. wget is guaranteed present -- live-config used it to
|
||||
// fetch this very script.
|
||||
app.get<{ Querystring: { mac?: string } }>("/vyos/autoinstall.sh", async (request, reply) => {
|
||||
const mac = normalizeMac(request.query.mac);
|
||||
const base = `http://${config.serverIp}:${config.httpPort}`;
|
||||
|
||||
logger.info(`VYOS AUTOINSTALL HOOK served to ${mac || "unknown MAC"}`);
|
||||
|
||||
const script = `#!/bin/sh
|
||||
# Lab PXE Bastion -- VyOS unattended install hook (run by live-config as root)
|
||||
set -eu
|
||||
|
||||
wget -q "${base}/vyos/install.py?mac=${mac}" -O /tmp/vyos-install.py
|
||||
exec python3 /tmp/vyos-install.py
|
||||
`;
|
||||
return reply.type("text/plain").send(script);
|
||||
});
|
||||
|
||||
// Per-MAC install driver, with the machine's config spec baked in.
|
||||
app.get<{ Querystring: { mac?: string } }>("/vyos/install.py", async (request, reply) => {
|
||||
const mac = normalizeMac(request.query.mac);
|
||||
const queueEntry = state.load().install_queue[mac];
|
||||
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: queueEntry?.hostname ?? "vyos",
|
||||
spec: queueEntry?.vyos,
|
||||
defaultPassword: config.vyosDefaultPassword,
|
||||
sshKeys: config.sshKeys,
|
||||
disk: queueEntry?.disk,
|
||||
});
|
||||
|
||||
logger.info(
|
||||
`VYOS INSTALL DRIVER served to ${mac} (${spec.hostname}, ` +
|
||||
`${spec.sets.length} config ops, disk="${spec.disk || "auto"}")`,
|
||||
);
|
||||
|
||||
const script = renderVyosInstallPy({
|
||||
spec,
|
||||
mac,
|
||||
serverIp: config.serverIp,
|
||||
httpPort: config.httpPort,
|
||||
role: queueEntry?.role ?? "vanilla",
|
||||
});
|
||||
|
||||
return reply.type("text/plain").send(script);
|
||||
});
|
||||
}
|
||||
@@ -11,8 +11,6 @@ import { logger } from "./services/logger.js";
|
||||
import { registerDispatchRoutes } from "./routes/dispatch.js";
|
||||
import { registerKickstartRoutes } from "./routes/kickstart.js";
|
||||
import { registerApiRoutes } from "./routes/api.js";
|
||||
import { registerAsahiRoutes } from "./routes/asahi.js";
|
||||
import { registerVyosRoutes } from "./routes/vyos.js";
|
||||
|
||||
|
||||
export function createApp(config: BastionConfig): { app: ReturnType<typeof Fastify>; state: StateManager; installLog: InstallLogBuffer; syslog: SyslogListener } {
|
||||
@@ -45,10 +43,8 @@ export function createApp(config: BastionConfig): { app: ReturnType<typeof Fasti
|
||||
|
||||
// Register route handlers
|
||||
registerDispatchRoutes(app, config, state);
|
||||
registerKickstartRoutes(app, config, state, syslog);
|
||||
registerApiRoutes(app, state, installLog, syslog);
|
||||
registerAsahiRoutes(app, config);
|
||||
registerVyosRoutes(app, config, state);
|
||||
registerKickstartRoutes(app, config, state);
|
||||
registerApiRoutes(app, state, installLog);
|
||||
// boot.iso is generated at startup and served as a static file from httpDir
|
||||
// (static serving supports HTTP Range requests, required by JetKVM streaming)
|
||||
|
||||
|
||||
@@ -165,8 +165,6 @@ export class BastionConnection {
|
||||
case "command-forget":
|
||||
case "command-role-update":
|
||||
case "command-debug":
|
||||
case "command-register":
|
||||
case "command-discover":
|
||||
void this.handleCommand(msg);
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -30,8 +30,6 @@ export class SyslogListener {
|
||||
private port: number;
|
||||
private installLog: InstallLogBuffer;
|
||||
private state: StateManager;
|
||||
/** Explicit IP → MAC mapping registered from kickstart/progress requests. */
|
||||
private ipToMac = new Map<string, string>();
|
||||
|
||||
constructor(port: number, installLog: InstallLogBuffer, state: StateManager) {
|
||||
this.port = port;
|
||||
@@ -39,21 +37,14 @@ export class SyslogListener {
|
||||
this.state = state;
|
||||
}
|
||||
|
||||
/** Register an IP → MAC mapping (called when we learn a machine's IP). */
|
||||
registerIp(ip: string, mac: string): void {
|
||||
this.ipToMac.set(ip, mac.toLowerCase());
|
||||
}
|
||||
|
||||
/** Resolve a source IP to a MAC address. */
|
||||
/** Resolve a source IP to a MAC address using the install queue. */
|
||||
private resolveIpToMac(ip: string): string | null {
|
||||
// Check explicit mapping first (most reliable)
|
||||
const explicit = this.ipToMac.get(ip);
|
||||
if (explicit) return explicit;
|
||||
|
||||
const currentState = this.state.load();
|
||||
|
||||
// Check install queue — machines being installed have an IP from DHCP
|
||||
for (const [mac, entry] of Object.entries(currentState.install_queue)) {
|
||||
// The progress callback sends IP in "complete" detail, but during install
|
||||
// we need to match by what we know. Check if any progress mentions this IP.
|
||||
if (entry.progress_detail?.includes(ip)) return mac;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,311 +0,0 @@
|
||||
// First-boot LVM setup script for Asahi-provisioned machines.
|
||||
// Embedded in the custom rootfs as a systemd service that runs once on first boot.
|
||||
// Creates the standard lab LVM layout on the data partition, matching install.ks.ts.
|
||||
|
||||
import type { Role } from "@lab/shared";
|
||||
|
||||
export interface AsahiFirstbootParams {
|
||||
hostname: string;
|
||||
role: Role;
|
||||
serverIp: string;
|
||||
httpPort: number;
|
||||
sshKeys: string[];
|
||||
adminUser: string;
|
||||
mac: string;
|
||||
}
|
||||
|
||||
export function renderFirstbootScript(params: AsahiFirstbootParams): string {
|
||||
const { hostname, role, serverIp, httpPort, sshKeys, adminUser, mac } = params;
|
||||
|
||||
const isWorker = role === "worker";
|
||||
const isInfra = role === "infra" || role === "labcontroller";
|
||||
|
||||
// Role-specific LV creation commands
|
||||
const roleLvLines: string[] = [];
|
||||
const roleFormatLines: string[] = [];
|
||||
const roleMountLines: string[] = [];
|
||||
const roleFstabLines: string[] = [];
|
||||
|
||||
if (isInfra) {
|
||||
roleLvLines.push('lvcreate -L 20480M -n rancher labvg -y');
|
||||
roleFormatLines.push('mkfs.xfs /dev/labvg/rancher');
|
||||
roleMountLines.push('mount_lv rancher /var/lib/rancher');
|
||||
roleFstabLines.push('echo "/dev/labvg/rancher /var/lib/rancher xfs defaults 0 0" >> /etc/fstab');
|
||||
}
|
||||
if (isWorker || isInfra) {
|
||||
roleLvLines.push('lvcreate -l 100%FREE -n longhorn labvg -y');
|
||||
roleFormatLines.push('mkfs.xfs /dev/labvg/longhorn');
|
||||
roleMountLines.push('mount_lv longhorn /var/lib/longhorn');
|
||||
roleFstabLines.push('echo "/dev/labvg/longhorn /var/lib/longhorn xfs defaults 0 0" >> /etc/fstab');
|
||||
}
|
||||
|
||||
// SSH key injection block (empty if no keys)
|
||||
const sshKeyBlock = sshKeys.length > 0
|
||||
? sshKeys.map(k => `echo '${k}' >> "$ADMIN_SSH/authorized_keys"`).join('\n')
|
||||
: 'true # no SSH keys configured';
|
||||
const rootSshKeyBlock = sshKeys.length > 0
|
||||
? sshKeys.map(k => `echo '${k}' >> /root/.ssh/authorized_keys`).join('\n')
|
||||
: 'true # no SSH keys configured';
|
||||
|
||||
// NOTE: All bash $ references use $VAR not \${VAR} to avoid TS template conflicts.
|
||||
// Where ${} is needed in bash, we use \\${...} to escape.
|
||||
return `#!/bin/bash
|
||||
# Lab first-boot LVM setup — generated by bastion
|
||||
# This script runs once on first boot via systemd, then disables itself.
|
||||
set -euo pipefail
|
||||
|
||||
MARKER="/etc/lab-lvm-setup-done"
|
||||
LOG="/var/log/lab-firstboot.log"
|
||||
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
echo "=== Lab first-boot LVM setup ==="
|
||||
date
|
||||
|
||||
# Already done?
|
||||
if [ -f "$MARKER" ]; then
|
||||
echo "LVM setup already completed, skipping."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Find the data partition ──────────────────────────────────────
|
||||
# The data partition/disk is a large block device that is NOT the root filesystem.
|
||||
# Handles: NVMe partitions, SCSI partitions, whole unpartitioned disks.
|
||||
ROOT_DEV=$(findmnt -n -o SOURCE / | sed 's/\\[.*\\]//') # strip btrfs subvol
|
||||
ROOT_DISK=$(lsblk -n -o PKNAME "$ROOT_DEV" 2>/dev/null | head -1)
|
||||
echo "Root device: $ROOT_DEV (disk: $ROOT_DISK)"
|
||||
|
||||
DATA_PART=""
|
||||
# Scan partitions first, then whole disks
|
||||
for part in /dev/nvme*n*p* /dev/sd*[0-9] /dev/vd*[0-9] /dev/nvme*n* /dev/sd[b-z] /dev/vd[b-z]; do
|
||||
[ -b "$part" ] || continue
|
||||
# Skip root device and root disk
|
||||
[ "$part" = "$ROOT_DEV" ] && continue
|
||||
PART_DISK=$(basename "$part" | sed 's/p[0-9]*$//' | sed 's/[0-9]*$//')
|
||||
[ "$PART_DISK" = "$ROOT_DISK" ] && continue
|
||||
# Skip small devices (<50GB) — EFI, boot, APFS stubs
|
||||
SIZE_BYTES=$(blockdev --getsize64 "$part" 2>/dev/null || echo 0)
|
||||
SIZE_GB=$((SIZE_BYTES / 1073741824))
|
||||
[ "$SIZE_GB" -lt 50 ] && continue
|
||||
# Use if unformatted or already LVM
|
||||
FSTYPE=$(blkid -o value -s TYPE "$part" 2>/dev/null || echo "")
|
||||
if [ -z "$FSTYPE" ] || [ "$FSTYPE" = "LVM2_member" ]; then
|
||||
DATA_PART="$part"
|
||||
echo "Found data device: $DATA_PART ($SIZE_GB GB)"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$DATA_PART" ]; then
|
||||
echo "ERROR: No suitable data partition found for LVM."
|
||||
echo "Expected a large (>50GB) unformatted partition."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Helper function ──────────────────────────────────────────────
|
||||
mount_lv() {
|
||||
local lv="$1" mp="$2"
|
||||
if lvs "labvg/$lv" &>/dev/null; then
|
||||
mkdir -p "$mp"
|
||||
mount "/dev/labvg/$lv" "$mp" 2>/dev/null || true
|
||||
echo " Mounted $lv -> $mp"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Write fstab function (idempotent) ────────────────────────────
|
||||
write_lab_fstab() {
|
||||
# Remove any previous lab LVM entries (clean slate)
|
||||
sed -i '/# lab-lvm:/d' /etc/fstab
|
||||
sed -i '/# Lab LVM volumes/d' /etc/fstab
|
||||
grep -v "/dev/labvg/" /etc/fstab > /etc/fstab.tmp && mv /etc/fstab.tmp /etc/fstab
|
||||
# Comment out non-LVM entries for mount points we manage
|
||||
for mp in "/var " "/var/log " "/home " "/srv "; do
|
||||
if grep -q "$mp" /etc/fstab; then
|
||||
awk -v m="$mp" '{if($0 !~ /^#/ && index($0,m)) print "# lab-lvm: " $0; else print}' /etc/fstab > /etc/fstab.tmp
|
||||
mv /etc/fstab.tmp /etc/fstab
|
||||
fi
|
||||
done
|
||||
# Add fresh LVM entries
|
||||
echo "# Lab LVM volumes" >> /etc/fstab
|
||||
echo "/dev/labvg/swap none swap defaults 0 0" >> /etc/fstab
|
||||
echo "/dev/labvg/var /var xfs defaults 0 0" >> /etc/fstab
|
||||
echo "/dev/labvg/varlog /var/log xfs defaults 0 0" >> /etc/fstab
|
||||
echo "/dev/labvg/home /home xfs defaults 0 0" >> /etc/fstab
|
||||
echo "/dev/labvg/srv /srv xfs defaults 0 0" >> /etc/fstab
|
||||
${roleFstabLines.join('\n ')}
|
||||
}
|
||||
|
||||
# ── Check for existing VG ────────────────────────────────────────
|
||||
if vgs labvg &>/dev/null; then
|
||||
echo "Volume group 'labvg' already exists — reprovision detected."
|
||||
echo "Activating existing volumes..."
|
||||
vgchange -ay labvg
|
||||
|
||||
mount_lv var /var
|
||||
mount_lv varlog /var/log
|
||||
mount_lv home /home
|
||||
mount_lv srv /srv
|
||||
${roleMountLines.map(l => ` ${l}`).join('\n')}
|
||||
|
||||
# Enable swap
|
||||
if lvs labvg/swap &>/dev/null; then
|
||||
swapon /dev/labvg/swap 2>/dev/null || true
|
||||
echo " Enabled swap"
|
||||
fi
|
||||
|
||||
# Ensure fstab entries exist — comment out conflicting btrfs subvol entries
|
||||
write_lab_fstab
|
||||
|
||||
echo "Existing LVM volumes re-mounted."
|
||||
else
|
||||
# ── Fresh install: create LVM ────────────────────────────────────
|
||||
echo "Creating LVM on $DATA_PART..."
|
||||
|
||||
pvcreate "$DATA_PART"
|
||||
vgcreate labvg "$DATA_PART"
|
||||
|
||||
# Create LVs — sizes match install.ks.ts (in MiB)
|
||||
echo "Creating logical volumes..."
|
||||
lvcreate -L 27648M -n swap labvg -y # 27GB swap
|
||||
lvcreate -L 102400M -n var labvg -y # 100GB /var
|
||||
lvcreate -L 10240M -n varlog labvg -y # 10GB /var/log
|
||||
lvcreate -L 10240M -n home labvg -y # 10GB /home
|
||||
lvcreate -L 20480M -n srv labvg -y # 20GB /srv
|
||||
${roleLvLines.join('\n')}
|
||||
|
||||
# Format
|
||||
echo "Formatting volumes..."
|
||||
mkswap /dev/labvg/swap
|
||||
mkfs.xfs /dev/labvg/var
|
||||
mkfs.xfs /dev/labvg/varlog
|
||||
mkfs.xfs /dev/labvg/home
|
||||
mkfs.xfs /dev/labvg/srv
|
||||
${roleFormatLines.join('\n')}
|
||||
|
||||
# Migrate and mount volumes that can be switched live.
|
||||
# Copy existing content first so we don't shadow files (e.g. /home/user/.ssh).
|
||||
for LV_MOUNT in "home /home" "srv /srv"; do
|
||||
LV_NAME=$(echo "$LV_MOUNT" | awk '{print $1}')
|
||||
MOUNT_PT=$(echo "$LV_MOUNT" | awk '{print $2}')
|
||||
STAGING="/mnt/labvg-$LV_NAME-staging"
|
||||
mkdir -p "$STAGING"
|
||||
mount "/dev/labvg/$LV_NAME" "$STAGING"
|
||||
cp -a "$MOUNT_PT"/. "$STAGING/" 2>/dev/null || true
|
||||
umount "$STAGING"
|
||||
rmdir "$STAGING"
|
||||
mount_lv "$LV_NAME" "$MOUNT_PT"
|
||||
done
|
||||
|
||||
# Mount role-specific volumes (empty, no content to preserve)
|
||||
set +e
|
||||
${roleMountLines.join('\n')}
|
||||
set -e
|
||||
|
||||
# Copy existing /var content into the LV for next boot
|
||||
echo "Preparing /var LV for next boot..."
|
||||
TMPVAR="/mnt/labvg-var-staging"
|
||||
mkdir -p "$TMPVAR"
|
||||
mount /dev/labvg/var "$TMPVAR"
|
||||
cp -a /var/. "$TMPVAR/" 2>/dev/null || true
|
||||
umount "$TMPVAR"
|
||||
rmdir "$TMPVAR"
|
||||
|
||||
# Same for /var/log
|
||||
TMPVARLOG="/mnt/labvg-varlog-staging"
|
||||
mkdir -p "$TMPVARLOG"
|
||||
mount /dev/labvg/varlog "$TMPVARLOG"
|
||||
cp -a /var/log/. "$TMPVARLOG/" 2>/dev/null || true
|
||||
umount "$TMPVARLOG"
|
||||
rmdir "$TMPVARLOG"
|
||||
|
||||
echo "NOTE: /var and /var/log will switch to LVM on next reboot."
|
||||
|
||||
# Enable swap
|
||||
swapon /dev/labvg/swap 2>/dev/null || true
|
||||
|
||||
write_lab_fstab
|
||||
|
||||
echo "LVM setup complete."
|
||||
lvs labvg
|
||||
|
||||
fi # end if/else for reprovision vs fresh install
|
||||
|
||||
# ── Set hostname (use configured value, or keep existing) ────────
|
||||
CONF_HOSTNAME="${hostname}"
|
||||
if [ "$CONF_HOSTNAME" != "unknown" ] && [ -n "$CONF_HOSTNAME" ]; then
|
||||
hostnamectl set-hostname "$CONF_HOSTNAME"
|
||||
fi
|
||||
ACTUAL_HOSTNAME=$(hostname)
|
||||
|
||||
# ── Detect MAC address ───────────────────────────────────────────
|
||||
CONF_MAC="${mac}"
|
||||
if [ "$CONF_MAC" = "unknown" ] || [ -z "$CONF_MAC" ]; then
|
||||
CONF_MAC=$(ip -o link show | grep -v "lo:" | grep "state UP" | head -1 | grep -oP 'link/ether \\K[^ ]+' || echo "unknown")
|
||||
fi
|
||||
|
||||
# ── Configure admin user ─────────────────────────────────────────
|
||||
ADMIN="${adminUser}"
|
||||
if ! id "$ADMIN" &>/dev/null; then
|
||||
useradd -m -G wheel "$ADMIN"
|
||||
echo "$ADMIN ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$ADMIN
|
||||
chmod 440 /etc/sudoers.d/$ADMIN
|
||||
fi
|
||||
ADMIN_SSH="/home/$ADMIN/.ssh"
|
||||
mkdir -p "$ADMIN_SSH"
|
||||
chmod 700 "$ADMIN_SSH"
|
||||
${sshKeyBlock}
|
||||
chmod 600 "$ADMIN_SSH/authorized_keys"
|
||||
chown -R $ADMIN:$ADMIN "$ADMIN_SSH"
|
||||
|
||||
# Also authorize root
|
||||
mkdir -p /root/.ssh
|
||||
chmod 700 /root/.ssh
|
||||
${rootSshKeyBlock}
|
||||
chmod 600 /root/.ssh/authorized_keys
|
||||
|
||||
# ── Harden SSH (takes effect on next sshd restart/reboot) ────────
|
||||
sed -i 's/^#*PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config
|
||||
sed -i 's/^#*PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config
|
||||
|
||||
# ── Write provisioning metadata ──────────────────────────────────
|
||||
cat > /etc/lab-provisioned << LABMETA
|
||||
hostname=$ACTUAL_HOSTNAME
|
||||
role=${role}
|
||||
mac=$CONF_MAC
|
||||
provisioned_at=$(date -Iseconds)
|
||||
method=asahi-firstboot
|
||||
LABMETA
|
||||
|
||||
# ── Register with bastion ─────────────────────────────────────────
|
||||
IP=$(hostname -I | awk '{print $1}')
|
||||
echo "Registering with bastion at ${serverIp}:${httpPort}..."
|
||||
curl -sf -X POST "http://${serverIp}:${httpPort}/api/register" \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d "{\\"mac\\":\\"$CONF_MAC\\",\\"hostname\\":\\"$ACTUAL_HOSTNAME\\",\\"role\\":\\"${role}\\",\\"ip\\":\\"$IP\\"}" \\
|
||||
2>/dev/null && echo " Registered as $ACTUAL_HOSTNAME ($IP)" \\
|
||||
|| echo " WARNING: Could not reach bastion — register manually with: labctl provision register $CONF_MAC $ACTUAL_HOSTNAME --role ${role} --ip $IP"
|
||||
|
||||
# ── Mark done ────────────────────────────────────────────────────
|
||||
touch "$MARKER"
|
||||
echo "=== First-boot setup complete ==="
|
||||
`;
|
||||
}
|
||||
|
||||
/** Systemd unit file for the first-boot service */
|
||||
export function renderFirstbootUnit(): string {
|
||||
return `[Unit]
|
||||
Description=Lab first-boot LVM setup
|
||||
After=local-fs.target network-online.target
|
||||
Wants=network-online.target
|
||||
ConditionPathExists=!/etc/lab-lvm-setup-done
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/lab-firstboot.sh
|
||||
RemainAfterExit=yes
|
||||
StandardOutput=journal+console
|
||||
StandardError=journal+console
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
`;
|
||||
}
|
||||
@@ -102,34 +102,6 @@ boot
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* iPXE script for PXE-boot debug mode -- boots the installed system's root
|
||||
* filesystem using the bastion's PXE kernel+initrd instead of local GRUB.
|
||||
* Workaround for UEFI firmware bugs that make local disk boot slow.
|
||||
*/
|
||||
export function renderPxeBootDebugIpxe(params: {
|
||||
mac: string;
|
||||
hostname: string;
|
||||
serverIp: string;
|
||||
httpPort: number;
|
||||
}): string {
|
||||
return `#!ipxe
|
||||
|
||||
echo
|
||||
echo =============================================
|
||||
echo Lab PXE Bastion - PXE BOOT (debug)
|
||||
echo Target: ${params.hostname}
|
||||
echo MAC: ${params.mac}
|
||||
echo Kernel+initrd from PXE, root from NVMe
|
||||
echo =============================================
|
||||
echo
|
||||
|
||||
kernel http://${params.serverIp}:${params.httpPort}/vmlinuz root=/dev/mapper/labvg-root ro rd.lvm.lv=labvg/root rd.lvm.lv=labvg/swap console=tty0
|
||||
initrd http://${params.serverIp}:${params.httpPort}/initrd.img
|
||||
boot
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* iPXE script for already-installed machines -- exits to boot from local disk.
|
||||
*/
|
||||
|
||||
@@ -1,33 +1,76 @@
|
||||
// Debug/rescue kickstart template.
|
||||
// Minimal kickstart for Anaconda rescue mode.
|
||||
//
|
||||
// SSH access: Anaconda's inst.sshd starts sshd automatically.
|
||||
// The sshpw directive sets the password, sshkey adds authorized keys.
|
||||
// %pre/%post do NOT run in rescue mode — don't put setup code there.
|
||||
// When sshd=true: generates host keys, starts sshd, reports IP to bastion.
|
||||
// No dependency on mounted filesystems — fully self-contained.
|
||||
|
||||
export interface DebugKickstartParams {
|
||||
sshKeys: string[];
|
||||
sshd?: boolean;
|
||||
serverIp?: string;
|
||||
httpPort?: number;
|
||||
}
|
||||
|
||||
export function renderDebugKickstart(params: DebugKickstartParams): string {
|
||||
const sshpw = "sshpw --username=root --plaintext lab-root-pw";
|
||||
const sshkeyLine = params.sshKeys.length > 0
|
||||
? `sshkey --username=root "${params.sshKeys[0]}"`
|
||||
: "";
|
||||
|
||||
const sshdSetup = params.sshd ? `
|
||||
%post --nochroot --log=/tmp/debug-sshd.log
|
||||
#!/bin/bash
|
||||
set -x
|
||||
|
||||
# Generate host keys (self-contained, no mounted FS needed)
|
||||
ssh-keygen -t ed25519 -f /tmp/ssh_host_ed25519_key -N "" -q
|
||||
ssh-keygen -t rsa -f /tmp/ssh_host_rsa_key -N "" -q
|
||||
|
||||
# Write minimal sshd config
|
||||
cat > /tmp/sshd_config << 'SSHCFG'
|
||||
HostKey /tmp/ssh_host_ed25519_key
|
||||
HostKey /tmp/ssh_host_rsa_key
|
||||
PermitRootLogin yes
|
||||
PasswordAuthentication yes
|
||||
PubkeyAuthentication yes
|
||||
AuthorizedKeysFile /root/.ssh/authorized_keys
|
||||
SSHCFG
|
||||
|
||||
# Set root password for SSH access
|
||||
echo "root:debug" | chpasswd
|
||||
|
||||
# Set up SSH authorized keys
|
||||
mkdir -p /root/.ssh && chmod 700 /root/.ssh
|
||||
${params.sshKeys.map(k => `echo '${k}' >> /root/.ssh/authorized_keys`).join("\n")}
|
||||
chmod 600 /root/.ssh/authorized_keys 2>/dev/null || true
|
||||
|
||||
# Start sshd
|
||||
/usr/sbin/sshd -f /tmp/sshd_config -p 22
|
||||
echo "sshd started on port 22"
|
||||
|
||||
# Start persistent nc listener for remote shell
|
||||
(while true; do nc -l -p 2323 -e /bin/bash 2>/dev/null; done) &
|
||||
echo "nc shell listener on port 2323"
|
||||
|
||||
# Report IP to bastion
|
||||
sleep 2
|
||||
IP_ADDR=$(ip -4 addr show | awk '/inet / && !/127.0.0/ {split($2,a,"/"); print a[1]; exit}')
|
||||
MAC_ADDR=$(ip link show | awk '/ether/ && !/00:00:00:00/ {print $2; exit}')
|
||||
curl -sf -X POST "http://${params.serverIp}:${params.httpPort}/api/progress" \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d "{\\"mac\\":\\"$MAC_ADDR\\",\\"stage\\":\\"debug-ready\\",\\"detail\\":\\"ssh root@$IP_ADDR (pw: debug) | nc $IP_ADDR 2323\\"}" 2>/dev/null || true
|
||||
|
||||
echo "Debug environment ready: ssh root@$IP_ADDR or nc $IP_ADDR 2323"
|
||||
%end
|
||||
` : "";
|
||||
|
||||
return `# Lab Bastion -- Debug/Rescue Kickstart
|
||||
# Minimal: SSH + network for Anaconda rescue mode
|
||||
#
|
||||
# SSH is started by Anaconda (inst.sshd kernel param).
|
||||
# Password: debug | SSH keys from bastion config.
|
||||
# %pre/%post do NOT run in rescue mode.
|
||||
|
||||
lang en_US.UTF-8
|
||||
keyboard uk
|
||||
network --bootproto=dhcp --activate
|
||||
|
||||
sshpw --username=root --plaintext debug
|
||||
${sshpw}
|
||||
${sshkeyLine}
|
||||
`;
|
||||
${sshdSetup}`;
|
||||
}
|
||||
|
||||
@@ -40,11 +40,6 @@ export function renderInstallKickstart(params: InstallKickstartParams): string {
|
||||
const now = new Date().toISOString();
|
||||
const hasLonghorn = role === "worker";
|
||||
const hasRancher = role === "infra";
|
||||
// k8s roles get a dedicated 120G image-store LV. 2026-08 incident: the old
|
||||
// 20G LV idled at 85% used, so a single ~5G image pull tripped imagefs
|
||||
// eviction. Must be sized here — longhorn's --grow consumes all remaining
|
||||
// VG space, making post-install lvextend impossible on worker nodes.
|
||||
const hasRancherLv = role === "infra" || role === "worker";
|
||||
const isVanilla = role === "vanilla";
|
||||
|
||||
// -- Auth section --
|
||||
@@ -118,9 +113,9 @@ done
|
||||
? `logvol /var/lib/longhorn --vgname=${vg} --name=longhorn --fstype=xfs --grow --size=1`
|
||||
: "";
|
||||
|
||||
// -- Rancher LV for fresh install (k8s roles: worker + infra) --
|
||||
const rancherFreshLine = hasRancherLv
|
||||
? `logvol /var/lib/rancher --vgname=${vg} --name=rancher --fstype=xfs --size=122880`
|
||||
// -- Rancher LV for fresh install (infra role) --
|
||||
const rancherFreshLine = hasRancher
|
||||
? `logvol /var/lib/rancher --vgname=${vg} --name=rancher --fstype=xfs --size=20480`
|
||||
: "";
|
||||
|
||||
return `# Lab Bastion -- Fedora ${fedoraVersion} server install
|
||||
@@ -134,20 +129,15 @@ lang ${locale}
|
||||
keyboard uk
|
||||
timezone ${timezone} --utc
|
||||
|
||||
# --ipv6=auto, not =dhcp: "auto" follows the router advertisement, so the RA's
|
||||
# managed-flag is what steers the node to DHCPv6, and a VLAN with no DHCPv6 yet
|
||||
# still installs instead of blocking on a lease that will never come. Failing an
|
||||
# OS install because IPv6 was not ready would be a worse trade than a node that
|
||||
# briefly has no v6. The address itself comes from a kea DHCPv6 reservation
|
||||
# keyed on MAC -- the same source of truth as the v4 address.
|
||||
network --bootproto=dhcp --ipv6=auto --activate --hostname=${fqdn}
|
||||
network --bootproto=dhcp --activate --hostname=${fqdn}
|
||||
|
||||
${auth}
|
||||
${userDirective}
|
||||
|
||||
bootloader --append="console=tty0"
|
||||
bootloader --append="console=tty0 console=ttyS0,115200n8"
|
||||
|
||||
logging --host=${serverIp} --port=${syslogPort}
|
||||
# logging --host=${serverIp} --port=${syslogPort}
|
||||
# Disabled: syslog UDP port needs to be exposed in k3s service/hostPort first
|
||||
|
||||
url --mirrorlist=https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-$releasever&arch=$basearch
|
||||
|
||||
@@ -352,7 +342,17 @@ echo "tmpfs /tmp tmpfs defaults,noatime,nosuid,nodev,size=4G 0 0" >> /etc/fstab
|
||||
|
||||
${isVanilla ? `# -- vanilla role: skip k3s kernel/sysctl/firewall setup --
|
||||
# -- Enable chronyd for time sync --
|
||||
systemctl enable chronyd || true` : `# -- Kernel modules for k3s --
|
||||
systemctl enable chronyd || true
|
||||
|
||||
# -- Serial console (for debugging — auto-login as root on ttyS0) --
|
||||
# AWS EC2 compatible: ttyS0 @ 115200n8
|
||||
systemctl enable serial-getty@ttyS0.service || true
|
||||
|
||||
# -- Forward all system logs to serial console --
|
||||
cat > /etc/rsyslog.d/serial-console.conf << 'RSYSLOG'
|
||||
*.* /dev/ttyS0
|
||||
RSYSLOG
|
||||
systemctl enable rsyslog || true` : `# -- Kernel modules for k3s --
|
||||
cat > /etc/modules-load.d/k3s.conf << 'MODULES'
|
||||
br_netfilter
|
||||
overlay
|
||||
@@ -372,21 +372,6 @@ fs.inotify.max_user_watches = 1048576
|
||||
SYSCTL
|
||||
sysctl --system || true
|
||||
|
||||
# -- IPv6 link-local address generation: EUI-64, fleet-wide --
|
||||
# A cluster node takes its IPv6 from a MAC-keyed DHCPv6 reservation. kea can only
|
||||
# match that reservation when it can recover the node's MAC, and for a modern
|
||||
# client that sends a DUID-UUID (no MAC in it) the only place kea can find one is
|
||||
# an EUI-64 link-local. NetworkManager's default is stable-privacy (RFC 7217),
|
||||
# whose link-local hides the MAC -- so a node on the default silently never gets
|
||||
# its reserved address, and with a reservations-only subnet it gets nothing at
|
||||
# all. Proven on 2026-09-06: worker0/worker2 (eui64) bound; worker1/spark
|
||||
# (default) did not, until flipped. Setting it here means a new node is correct
|
||||
# from first boot, before its connection is ever activated.
|
||||
cat > /etc/NetworkManager/conf.d/10-ipv6-eui64.conf << 'NMEUI64'
|
||||
[connection]
|
||||
ipv6.addr-gen-mode=eui64
|
||||
NMEUI64
|
||||
|
||||
# -- Disable firewalld permanently (k3s/Cilium manage iptables directly) --
|
||||
# Note: no '--now' — systemd is not running in the Anaconda chroot
|
||||
systemctl disable firewalld || true
|
||||
@@ -411,9 +396,6 @@ fi
|
||||
|
||||
bastion_progress "post-install" "3-bootorder done"
|
||||
|
||||
# -- Enable SysRq magic keys (for emergency reboot via Alt+SysRq+REISUB) --
|
||||
echo "kernel.sysrq=1" > /etc/sysctl.d/90-sysrq.conf
|
||||
|
||||
# -- Provisioning metadata --
|
||||
cat > /etc/lab-provisioned << PROVEOF
|
||||
hostname: ${fqdn}
|
||||
|
||||
@@ -40,39 +40,34 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
|
||||
// Build the LVM layout to match Fedora kickstart sizes
|
||||
const extraLvs: string[] = [];
|
||||
if (hasLonghorn) {
|
||||
// 6 spaces for the list item, 8 for its keys -- these are siblings of the
|
||||
// lv-home/lv-srv entries in storage.config, which sit at 6. At 8 the
|
||||
// rendered document is not valid YAML at all ("expected <block end>, but
|
||||
// found '-'"), so an Ubuntu node with the longhorn role could never have
|
||||
// installed. Ubuntu + longhorn is exactly the worker shape.
|
||||
extraLvs.push(` - id: lv-longhorn
|
||||
name: longhorn
|
||||
type: lvm_partition
|
||||
volgroup: vg0
|
||||
size: -1
|
||||
- id: fs-longhorn
|
||||
type: format
|
||||
volume: lv-longhorn
|
||||
fstype: xfs
|
||||
- id: mount-longhorn
|
||||
type: mount
|
||||
device: fs-longhorn
|
||||
path: /var/lib/longhorn`);
|
||||
extraLvs.push(` - id: lv-longhorn
|
||||
name: longhorn
|
||||
type: lvm_partition
|
||||
volgroup: vg0
|
||||
size: -1
|
||||
- id: fs-longhorn
|
||||
type: format
|
||||
volume: lv-longhorn
|
||||
fstype: xfs
|
||||
- id: mount-longhorn
|
||||
type: mount
|
||||
device: fs-longhorn
|
||||
path: /var/lib/longhorn`);
|
||||
}
|
||||
if (hasRancher) {
|
||||
extraLvs.push(` - id: lv-rancher
|
||||
name: rancher
|
||||
type: lvm_partition
|
||||
volgroup: vg0
|
||||
size: 20G
|
||||
- id: fs-rancher
|
||||
type: format
|
||||
volume: lv-rancher
|
||||
fstype: xfs
|
||||
- id: mount-rancher
|
||||
type: mount
|
||||
device: fs-rancher
|
||||
path: /var/lib/rancher`);
|
||||
extraLvs.push(` - id: lv-rancher
|
||||
name: rancher
|
||||
type: lvm_partition
|
||||
volgroup: vg0
|
||||
size: 20G
|
||||
- id: fs-rancher
|
||||
type: format
|
||||
volume: lv-rancher
|
||||
fstype: xfs
|
||||
- id: mount-rancher
|
||||
type: mount
|
||||
device: fs-rancher
|
||||
path: /var/lib/rancher`);
|
||||
}
|
||||
|
||||
const extraLvsBlock = extraLvs.length > 0 ? "\n" + extraLvs.join("\n") : "";
|
||||
@@ -86,11 +81,6 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
|
||||
`curtin in-target -- bash -c 'cat > /etc/modules-load.d/k3s.conf << EOF\nbr_netfilter\noverlay\nip_conntrack\nEOF'`,
|
||||
// Sysctl for k3s networking
|
||||
`curtin in-target -- bash -c 'cat > /etc/sysctl.d/90-k3s.conf << EOF\nnet.bridge.bridge-nf-call-iptables = 1\nnet.bridge.bridge-nf-call-ip6tables = 1\nnet.ipv4.ip_forward = 1\nnet.ipv6.conf.all.forwarding = 1\nfs.inotify.max_user_instances = 524288\nfs.inotify.max_user_watches = 1048576\nEOF'`,
|
||||
// IPv6 link-local = EUI-64, so a MAC-keyed DHCPv6 reservation can match: kea
|
||||
// recovers the node's MAC from an EUI-64 link-local when the client sends a
|
||||
// DUID-UUID (no MAC in it). NM's stable-privacy default hides the MAC and the
|
||||
// node silently never gets its reserved address. Proven 2026-09-06.
|
||||
`curtin in-target -- bash -c 'cat > /etc/NetworkManager/conf.d/10-ipv6-eui64.conf << EOF\n[connection]\nipv6.addr-gen-mode=eui64\nEOF'`,
|
||||
// Disable ufw firewall
|
||||
`curtin in-target -- systemctl disable ufw || true`,
|
||||
// Enable chrony/ntp
|
||||
@@ -131,20 +121,7 @@ export function renderUbuntuAutoinstall(params: UbuntuAutoinstallParams): string
|
||||
`curtin in-target -- bash -c 'IP_ADDR=$(ip -4 addr show | awk "/inet / && !/127.0.0/ {split(\\$2,a,\\"/\\"); print a[1]; exit}"); curl -sf -X POST "http://${serverIp}:${httpPort}/api/progress" -H "Content-Type: application/json" -d "{\\"mac\\":\\"$(ip link show | awk "/ether/ && !/00:00:00:00/ {print \\$2; exit}")\\",\\"stage\\":\\"complete\\",\\"detail\\":\\"ready at $IP_ADDR\\"}" || true'`,
|
||||
);
|
||||
|
||||
// JSON.stringify, not `"${c}"`. JSON is a subset of YAML, so this produces a
|
||||
// correctly escaped double-quoted scalar for free -- and the naive version
|
||||
// was broken in two ways at once, for every role:
|
||||
//
|
||||
// * embedded double quotes ended the scalar early
|
||||
// (`echo "tmpfs /tmp ..." >> /etc/fstab` -> "expected <block end>")
|
||||
// * the heredocs contain REAL newlines, and YAML folds newlines inside a
|
||||
// double-quoted scalar into spaces -- so even where it parsed, the
|
||||
// heredoc arrived at the target as one long line and wrote a file with
|
||||
// no line breaks.
|
||||
//
|
||||
// JSON escaping turns the newlines into \n, which YAML unescapes back to
|
||||
// real newlines on parse, so the heredoc survives intact.
|
||||
const lateCommandsYaml = lateCommands.map((c) => ` - ${JSON.stringify(c)}`).join("\n");
|
||||
const lateCommandsYaml = lateCommands.map((c) => ` - "${c}"`).join("\n");
|
||||
|
||||
return `#cloud-config
|
||||
autoinstall:
|
||||
@@ -162,30 +139,6 @@ autoinstall:
|
||||
allow-pw: false
|
||||
authorized-keys:
|
||||
${sshKeysYaml}
|
||||
# Both address families. Without dhcp6 the installer's default is IPv4-only,
|
||||
# so a node provisioned into a dual-stack cluster comes up with no IPv6, k3s
|
||||
# has no v6 node-ip to bind, and it joins as an IPv4-only member of a
|
||||
# dual-stack cluster -- which surfaces later as pods on that node being
|
||||
# unreachable over v6 while the node itself reads Ready.
|
||||
#
|
||||
# The address itself comes from a kea DHCPv6 reservation keyed on MAC, the
|
||||
# same source of truth as the v4 address, so nothing here needs to know it.
|
||||
#
|
||||
# optional: true matters -- it lets the install proceed if the v6 lease is
|
||||
# slow or the VLAN has no DHCPv6 yet, rather than blocking on a timeout. The
|
||||
# node still needs the address before k3s starts, but that is a later step's
|
||||
# problem and failing the OS install over it would be worse.
|
||||
# (No backticks in this comment: it lives inside a TS template literal, and a
|
||||
# backtick here ends the literal and breaks the build.)
|
||||
network:
|
||||
version: 2
|
||||
ethernets:
|
||||
primary:
|
||||
match:
|
||||
name: "en*"
|
||||
dhcp4: true
|
||||
dhcp6: true
|
||||
optional: true
|
||||
storage:
|
||||
config:
|
||||
- id: disk0
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
// iPXE boot script template for VyOS network install.
|
||||
//
|
||||
// VyOS ships no unattended installer: `install image` is unconditionally
|
||||
// interactive (image_installer.py's install action takes no arguments, and
|
||||
// --no-prompt is wired only to `add`). So PXE boots the *live* system and the
|
||||
// automation is injected through live-config's `hooks` component, which fetches
|
||||
// a script over HTTP and runs it as root late in live boot.
|
||||
//
|
||||
// Unlike the Fedora/Ubuntu paths this boots a live image rather than an
|
||||
// installer, so there is no kickstart/autoinstall equivalent — see
|
||||
// routes/vyos.ts for the hook that actually drives the install.
|
||||
|
||||
export function renderVyosInstallIpxe(params: {
|
||||
mac: string;
|
||||
hostname: string;
|
||||
serverIp: string;
|
||||
httpPort: number;
|
||||
}): string {
|
||||
const base = `http://${params.serverIp}:${params.httpPort}`;
|
||||
|
||||
// Pin the boot NIC by MAC. live-boot otherwise scans for the first
|
||||
// *connected* interface, and on a multi-NIC box that race is lost by
|
||||
// whichever port negotiates slowest: on the Protectli VP2440 the SFP+
|
||||
// pair links first, so live-boot picked the fiber ports (which have no
|
||||
// DHCP), burned 15s per port, and gave up with "Unable to find a live
|
||||
// file system on the network" -- while the copper port that actually PXE
|
||||
// booted came up at 4.6s and was never tried.
|
||||
//
|
||||
// live-boot's Device_from_bootif() strips the "01-" and matches the MAC
|
||||
// against /sys/class/net/*. params.mac is the dispatch key, i.e. exactly
|
||||
// the NIC that PXE booted -- more reliable than iPXE's ${net0} on a box
|
||||
// where the booting NIC may not be net0.
|
||||
const bootif = `01-${params.mac.toLowerCase().replace(/:/g, "-")}`;
|
||||
|
||||
// Deliberately NOT passing `nonetworking` (present in VyOS's own PXE docs):
|
||||
// live-config's hook component needs networking up to fetch the hook over
|
||||
// HTTP. Also no `console=ttyS0` — on hardware without a physical UART that
|
||||
// costs 30s at every systemd boot phase.
|
||||
return `#!ipxe
|
||||
|
||||
echo
|
||||
echo =============================================
|
||||
echo Lab PXE Bastion - INSTALLING VyOS
|
||||
echo Target: ${params.hostname}
|
||||
echo MAC: ${params.mac}
|
||||
echo =============================================
|
||||
echo
|
||||
|
||||
kernel ${base}/vyos-vmlinuz boot=live nopersistence noautologin BOOTIF=${bootif} fetch=${base}/vyos-filesystem.squashfs live-config.hooks=${base}/vyos/autoinstall.sh?mac=${params.mac}
|
||||
initrd ${base}/vyos-initrd
|
||||
boot
|
||||
`;
|
||||
}
|
||||
@@ -1,352 +0,0 @@
|
||||
// Builds the VyOS configuration spec applied by the autoinstall hook.
|
||||
//
|
||||
// We deliberately do NOT emit a config.boot file as text. A config.boot carries a
|
||||
// `vyos-config-version` trailer; without a trailer matching the running image,
|
||||
// VyOS runs its migration scripts from version 0 on first boot. Instead the hook
|
||||
// loads the image's own /opt/vyatta/etc/config.boot.default through vyos.configtree
|
||||
// and applies these set operations on top, so syntax and version trailer always
|
||||
// match the exact image being installed.
|
||||
|
||||
import type { VyosInstallSpec } from "@lab/shared";
|
||||
|
||||
export interface VyosSetOp {
|
||||
path: string[];
|
||||
value?: string;
|
||||
/** false appends to a multi-value node (e.g. bond members) instead of replacing. */
|
||||
replace?: boolean;
|
||||
}
|
||||
|
||||
export interface VyosConfigSpec {
|
||||
hostname: string;
|
||||
/** "" means accept the installer default (the running image's version string). */
|
||||
imageName: string;
|
||||
password: string;
|
||||
console: "K" | "S";
|
||||
/** Target disk name (e.g. "nvme0n1"); "" accepts the installer's first-disk default. */
|
||||
disk: string;
|
||||
/**
|
||||
* IP the driver should report in the "complete" callback ("ready at <ip>" —
|
||||
* the exact format routes/api.ts parses installed.ip from). The mgmt
|
||||
* address when static; "" means detect the live DHCP address at runtime.
|
||||
*/
|
||||
reportAddress: string;
|
||||
/** Whether to accept RAID-1 when the installer finds more than one disk. */
|
||||
raid: boolean;
|
||||
/** Overwrite the installed config.boot with the generated one on reinstall. */
|
||||
freshConfig: boolean;
|
||||
sets: VyosSetOp[];
|
||||
/** Paths that are VyOS tag nodes — must be marked as such in the ConfigTree. */
|
||||
tags: string[][];
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalise a target disk to the form the installer expects.
|
||||
*
|
||||
* find_disks() enumerates via `lsblk -Jbp` (-p = full paths), so its valid
|
||||
* responses are "/dev/mmcblk0"-style. A bare "mmcblk0" is rejected by
|
||||
* ask_input()'s valid_responses check and re-prompts forever.
|
||||
*/
|
||||
function normalizeDiskPath(value: string | undefined): string {
|
||||
const raw = (value ?? "").trim();
|
||||
if (raw === "") return "";
|
||||
return raw.startsWith("/dev/") ? raw : `/dev/${raw}`;
|
||||
}
|
||||
|
||||
/** Sentinel marking a value that lives in Pulumi config, not in the bundle. */
|
||||
const SECRET_PREFIX = "@secret:";
|
||||
|
||||
/**
|
||||
* Enable the VyOS HTTP API so the router is manageable the moment it boots.
|
||||
*
|
||||
* This belongs at install time rather than in the Pulumi model: the model is
|
||||
* applied THROUGH this API, so a router that lacks it cannot be brought under
|
||||
* management without a hand-run change on a live firewall. It is also why the
|
||||
* model excludes `service https` outright -- a provider able to rewrite its own
|
||||
* transport can lock itself out permanently.
|
||||
*
|
||||
* `listen-address` is always set. Leaving it unbound would expose a
|
||||
* config-write endpoint on every segment the router touches, the WAN included.
|
||||
*/
|
||||
function apiSets(apiKey: string, listenAddress: string): VyosSetOp[] {
|
||||
const sets: VyosSetOp[] = [
|
||||
{ path: ["service", "https", "api", "keys", "id", "pulumi", "key"], value: apiKey },
|
||||
{ path: ["service", "https", "api", "rest"] },
|
||||
];
|
||||
if (listenAddress !== "") {
|
||||
sets.push({ path: ["service", "https", "listen-address"], value: listenAddress });
|
||||
}
|
||||
return sets;
|
||||
}
|
||||
|
||||
/** Tag nodes introduced by the API config, needed by the installer's ConfigTree. */
|
||||
const API_TAGS: string[][] = [["service", "https", "api", "keys", "id"]];
|
||||
|
||||
/**
|
||||
* The address to bind the API to: an explicit choice, else the management
|
||||
* address with its prefix length stripped. Under DHCP there is no address to
|
||||
* bind at build time, so the caller must pass one or the listener stays unbound
|
||||
* and the API is not enabled at all.
|
||||
*/
|
||||
function apiListenAddress(spec: VyosInstallSpec, mgmtAddress: string): string {
|
||||
if (spec.apiListenAddress !== undefined && spec.apiListenAddress !== "") {
|
||||
return spec.apiListenAddress;
|
||||
}
|
||||
return mgmtAddress.includes("/") ? (mgmtAddress.split("/")[0] ?? "") : "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Use a Pulumi-rendered bundle as the router's config verbatim.
|
||||
*
|
||||
* Secret-valued nodes are dropped rather than installed with their sentinel
|
||||
* text: writing `@secret:pppoePassword` into config.boot would look configured
|
||||
* while being wrong, which is worse than being absent. The router comes up
|
||||
* without those values and the first `pulumi up` fills them in.
|
||||
*
|
||||
* `system host-name` is forced to the hostname the install was asked for. The
|
||||
* bundle carries the name of whichever router it was exported from, and
|
||||
* installing vyos001's hostname onto vyos002 would collide on the network.
|
||||
*/
|
||||
function buildFromBundle(
|
||||
params: { hostname: string; defaultPassword: string; disk?: string | undefined },
|
||||
spec: VyosInstallSpec,
|
||||
bundle: NonNullable<VyosInstallSpec["bundle"]>,
|
||||
mgmtAddress: string,
|
||||
): VyosConfigSpec {
|
||||
const sets: VyosSetOp[] = [];
|
||||
const dropped: string[] = [];
|
||||
for (const op of bundle.sets) {
|
||||
if (op.value !== undefined && op.value.startsWith(SECRET_PREFIX)) {
|
||||
dropped.push(op.path.join(" "));
|
||||
continue;
|
||||
}
|
||||
if (op.path.length === 2 && op.path[0] === "system" && op.path[1] === "host-name") {
|
||||
continue;
|
||||
}
|
||||
sets.push({
|
||||
path: op.path,
|
||||
...(op.value === undefined ? {} : { value: op.value }),
|
||||
...(op.replace === undefined ? {} : { replace: op.replace }),
|
||||
});
|
||||
}
|
||||
sets.unshift({ path: ["system", "host-name"], value: params.hostname });
|
||||
|
||||
if (dropped.length > 0) {
|
||||
console.warn(
|
||||
`vyos ${params.hostname}: ${dropped.length} secret-valued node(s) left unset by the ` +
|
||||
`bundle; run \`pulumi up\` to supply them: ${dropped.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
const tags = [...bundle.tags];
|
||||
const api = enableApi(spec, params.hostname, mgmtAddress);
|
||||
if (api.length > 0) {
|
||||
sets.push(...api);
|
||||
tags.push(...API_TAGS);
|
||||
}
|
||||
|
||||
return {
|
||||
hostname: params.hostname,
|
||||
imageName: "",
|
||||
password: spec.password ?? params.defaultPassword,
|
||||
console: "K",
|
||||
disk: normalizeDiskPath(params.disk),
|
||||
reportAddress: mgmtAddress.includes("/") ? (mgmtAddress.split("/")[0] ?? "") : "",
|
||||
raid: false,
|
||||
freshConfig: spec.freshConfig ?? false,
|
||||
sets,
|
||||
tags,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The API config for this install, or nothing when it cannot be enabled safely.
|
||||
*
|
||||
* Refusing to enable it unbound is deliberate. Under DHCP there is no address
|
||||
* known at build time, and the alternative -- binding to every interface --
|
||||
* would publish a config-write endpoint on the WAN. Better to leave the router
|
||||
* SSH-only and say so than to open it everywhere.
|
||||
*/
|
||||
function enableApi(spec: VyosInstallSpec, hostname: string, mgmtAddress: string): VyosSetOp[] {
|
||||
if (spec.apiKey === undefined || spec.apiKey === "") return [];
|
||||
const listen = apiListenAddress(spec, mgmtAddress);
|
||||
if (listen === "") {
|
||||
console.warn(
|
||||
`vyos ${hostname}: --vyos-api-key given but no address to bind to ` +
|
||||
`(management is "${mgmtAddress}"). Pass --vyos-api-listen <addr>; the HTTP API ` +
|
||||
`has NOT been enabled, so Pulumi cannot manage this router yet.`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
return apiSets(spec.apiKey, listen);
|
||||
}
|
||||
|
||||
export function buildVyosConfigSpec(params: {
|
||||
hostname: string;
|
||||
spec?: VyosInstallSpec | undefined;
|
||||
defaultPassword: string;
|
||||
sshKeys?: string[] | undefined;
|
||||
disk?: string | undefined;
|
||||
}): VyosConfigSpec {
|
||||
const spec = params.spec ?? {};
|
||||
const mgmt = spec.mgmtInterface ?? "eth0";
|
||||
const mgmtAddress = spec.mgmtAddress ?? "dhcp";
|
||||
|
||||
// A rendered bundle replaces the derived config entirely. Deriving a second
|
||||
// opinion alongside it is the drift the bundle exists to prevent: Pulumi and
|
||||
// labctl would each believe they knew the router's config, and the box would
|
||||
// end up with whichever ran last.
|
||||
if (spec.bundle !== undefined) {
|
||||
return buildFromBundle(params, spec, spec.bundle, mgmtAddress);
|
||||
}
|
||||
const bondMembers = spec.bondMembers ?? [];
|
||||
const vlans = spec.vlans ?? [];
|
||||
|
||||
const sets: VyosSetOp[] = [];
|
||||
const tags: string[][] = [
|
||||
["interfaces", "ethernet"],
|
||||
["system", "login", "user"],
|
||||
];
|
||||
|
||||
const hwIds = spec.hwIds ?? {};
|
||||
const pinHwId = (iface: string): void => {
|
||||
const mac = hwIds[iface];
|
||||
if (mac !== undefined && mac !== "") {
|
||||
sets.push({ path: ["interfaces", "ethernet", iface, "hw-id"], value: mac });
|
||||
}
|
||||
};
|
||||
|
||||
sets.push({ path: ["system", "host-name"], value: params.hostname });
|
||||
|
||||
// Management interface — the NIC that PXE booted, left untagged and unbonded.
|
||||
sets.push({ path: ["interfaces", "ethernet", mgmt, "address"], value: mgmtAddress });
|
||||
pinHwId(mgmt);
|
||||
|
||||
// Tagged management VLAN on the PXE port. Emitted regardless of bonding, so
|
||||
// the box stays reachable on the management VLAN while still booting untagged
|
||||
// on whichever VLAN the bastion's proxy DHCP serves.
|
||||
const mgmtVlan = spec.mgmtVlan;
|
||||
if (mgmtVlan !== undefined) {
|
||||
tags.push(["interfaces", "ethernet", mgmt, "vif"]);
|
||||
const vif = ["interfaces", "ethernet", mgmt, "vif", String(mgmtVlan.id)];
|
||||
sets.push({ path: [...vif, "address"], value: mgmtVlan.address });
|
||||
if (mgmtVlan.description !== undefined && mgmtVlan.description !== "") {
|
||||
sets.push({ path: [...vif, "description"], value: mgmtVlan.description });
|
||||
}
|
||||
}
|
||||
|
||||
// LACP bond. Members must exclude the PXE NIC; firmware PXE cannot run over LACP.
|
||||
const bonded = bondMembers.length > 0;
|
||||
if (bonded) {
|
||||
tags.push(["interfaces", "bonding"]);
|
||||
sets.push({ path: ["interfaces", "bonding", "bond0", "mode"], value: "802.3ad" });
|
||||
sets.push({ path: ["interfaces", "bonding", "bond0", "hash-policy"], value: "layer2+3" });
|
||||
for (const member of bondMembers) {
|
||||
sets.push({
|
||||
path: ["interfaces", "bonding", "bond0", "member", "interface"],
|
||||
value: member,
|
||||
replace: false,
|
||||
});
|
||||
pinHwId(member);
|
||||
}
|
||||
// Address on the trunk's native/untagged VLAN.
|
||||
if (spec.bondAddress !== undefined && spec.bondAddress !== "") {
|
||||
sets.push({ path: ["interfaces", "bonding", "bond0", "address"], value: spec.bondAddress });
|
||||
}
|
||||
}
|
||||
|
||||
// VRRP groups accumulate here; emitted (plus a sync group) after the VLANs.
|
||||
// interface accepts dotted vifs (constraint regex `[0-9]+(.\d+)?`), address
|
||||
// is a tag node (the VIP is the tag value itself), vrid range is 1-255.
|
||||
const vrrpGroups: Array<{ name: string; iface: string; vrid: number; vip: string }> = [];
|
||||
if (bonded && spec.bondVrrp !== undefined && spec.bondVrrp !== "") {
|
||||
// vrid 1 for the untagged group: the native VLAN is never a vif, so this
|
||||
// cannot collide with a vlan-id-derived vrid.
|
||||
vrrpGroups.push({ name: "native", iface: "bond0", vrid: 1, vip: spec.bondVrrp });
|
||||
}
|
||||
|
||||
// Tagged VLAN sub-interfaces hang off the bond when there is one, else off mgmt.
|
||||
const parent = bonded
|
||||
? ["interfaces", "bonding", "bond0"]
|
||||
: ["interfaces", "ethernet", mgmt];
|
||||
if (vlans.length > 0) {
|
||||
tags.push([...parent, "vif"]);
|
||||
const parentName = bonded ? "bond0" : mgmt;
|
||||
for (const vlan of vlans) {
|
||||
const vif = [...parent, "vif", String(vlan.id)];
|
||||
sets.push({ path: [...vif, "address"], value: vlan.address });
|
||||
if (vlan.description !== undefined && vlan.description !== "") {
|
||||
sets.push({ path: [...vif, "description"], value: vlan.description });
|
||||
}
|
||||
if (vlan.vrrp !== undefined && vlan.vrrp !== "") {
|
||||
vrrpGroups.push({
|
||||
name: `vlan${vlan.id}`,
|
||||
iface: `${parentName}.${vlan.id}`,
|
||||
vrid: vlan.id,
|
||||
vip: vlan.vrrp,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Emit VRRP groups plus one sync group so all VLANs fail over together —
|
||||
// without it a single-link event could split mastership across the pair.
|
||||
if (vrrpGroups.length > 0) {
|
||||
tags.push(["high-availability", "vrrp", "group"]);
|
||||
tags.push(["high-availability", "vrrp", "sync-group"]);
|
||||
const priority = String(spec.vrrpPriority ?? 100);
|
||||
for (const g of vrrpGroups) {
|
||||
const base = ["high-availability", "vrrp", "group", g.name];
|
||||
sets.push({ path: [...base, "interface"], value: g.iface });
|
||||
sets.push({ path: [...base, "vrid"], value: String(g.vrid) });
|
||||
sets.push({ path: [...base, "priority"], value: priority });
|
||||
// address is a tag node: the VIP is the path's final segment, no value.
|
||||
sets.push({ path: [...base, "address", g.vip] });
|
||||
tags.push([...base, "address"]);
|
||||
sets.push({
|
||||
path: ["high-availability", "vrrp", "sync-group", "MAIN", "member"],
|
||||
value: g.name,
|
||||
replace: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
sets.push({ path: ["service", "ssh", "port"], value: "22" });
|
||||
|
||||
const sshKeys = params.sshKeys ?? [];
|
||||
if (sshKeys.length > 0) {
|
||||
tags.push(["system", "login", "user", "vyos", "authentication", "public-keys"]);
|
||||
sshKeys.forEach((entry, index) => {
|
||||
const parts = entry.trim().split(/\s+/);
|
||||
const type = parts[0] ?? "";
|
||||
const key = parts[1] ?? "";
|
||||
if (!type.startsWith("ssh-") && !type.startsWith("ecdsa-")) return;
|
||||
if (!key) return;
|
||||
const name = parts[2] ?? `lab-key-${index}`;
|
||||
const base = ["system", "login", "user", "vyos", "authentication", "public-keys", name];
|
||||
sets.push({ path: [...base, "type"], value: type });
|
||||
sets.push({ path: [...base, "key"], value: key });
|
||||
});
|
||||
}
|
||||
|
||||
// Enabled here too, not just for bundle installs: every VyOS this bastion
|
||||
// provisions should be manageable from first boot.
|
||||
const api = enableApi(spec, params.hostname, mgmtAddress);
|
||||
if (api.length > 0) {
|
||||
sets.push(...api);
|
||||
tags.push(...API_TAGS);
|
||||
}
|
||||
|
||||
return {
|
||||
hostname: params.hostname,
|
||||
imageName: "",
|
||||
password: spec.password ?? params.defaultPassword,
|
||||
console: "K",
|
||||
disk: normalizeDiskPath(params.disk),
|
||||
// Static mgmt address wins; under DHCP the driver detects the live IP.
|
||||
reportAddress: mgmtAddress.includes("/") ? (mgmtAddress.split("/")[0] ?? "") : "",
|
||||
raid: false,
|
||||
freshConfig: spec.freshConfig ?? false,
|
||||
sets,
|
||||
tags,
|
||||
};
|
||||
}
|
||||
@@ -1,514 +0,0 @@
|
||||
// Renders the Python program that performs the unattended VyOS install.
|
||||
//
|
||||
// It runs as root inside the live system, fetched and executed by live-config's
|
||||
// `hooks` component (see vyos-boot.ipxe.ts). It does three things:
|
||||
// 1. builds config.boot from the image's own default via vyos.configtree
|
||||
// 2. drives the interactive `install image` through a pty
|
||||
// 3. reports progress back to the bastion, then reboots
|
||||
//
|
||||
// A pty is used rather than piping stdin because the installer reads the
|
||||
// password through getpass(), which opens /dev/tty directly and would ignore a
|
||||
// pipe. Prompts are matched by text rather than replayed positionally: the
|
||||
// installer skips the boot-config question when it finds a previous
|
||||
// installation, so a fixed answer sequence desyncs on reinstall.
|
||||
|
||||
import type { VyosConfigSpec } from "./vyos-config-spec.js";
|
||||
|
||||
export function renderVyosInstallPy(params: {
|
||||
spec: VyosConfigSpec;
|
||||
mac: string;
|
||||
serverIp: string;
|
||||
httpPort: number;
|
||||
role: string;
|
||||
}): string {
|
||||
// Base64 so arbitrary values (passwords, descriptions, SSH keys) can never
|
||||
// terminate the Python string literal that carries them.
|
||||
const specB64 = Buffer.from(JSON.stringify(params.spec), "utf-8").toString("base64");
|
||||
|
||||
return `#!/usr/bin/env python3
|
||||
"""Unattended VyOS install driver -- generated by the lab PXE bastion."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import pty
|
||||
import re
|
||||
import select
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
SPEC = json.loads(base64.b64decode("${specB64}").decode("utf-8"))
|
||||
BASTION = "http://${params.serverIp}:${params.httpPort}"
|
||||
MAC = "${params.mac}"
|
||||
ROLE = ${JSON.stringify(params.role ?? "vanilla")}
|
||||
|
||||
INSTALLER = "/usr/libexec/vyos/op_mode/image_installer.py"
|
||||
CONFIG_DIR = "/opt/vyatta/etc/config"
|
||||
# The installer copies the rootfs from the boot MEDIUM path -- which only a
|
||||
# CD/USB boot provides. With fetch= (HTTP netboot) nothing is mounted there
|
||||
# (verified in VM: Errno 2), so the squashfs must be linked or re-fetched into
|
||||
# place before 'install image' runs.
|
||||
ROOTFS_EXPECTED = "/usr/lib/live/mount/medium/live/filesystem.squashfs"
|
||||
SQUASHFS_URL = "http://${params.serverIp}:${params.httpPort}/vyos-filesystem.squashfs"
|
||||
# The live-config hook runs BEFORE vyos-router creates the /opt/vyatta compat
|
||||
# path, so the squashfs's own location must be tried too (verified in VM: only
|
||||
# /usr/share/vyos/config.boot.default exists at hook time).
|
||||
DEFAULT_CONFIG_CANDIDATES = [
|
||||
"/opt/vyatta/etc/config.boot.default",
|
||||
"/usr/share/vyos/config.boot.default",
|
||||
]
|
||||
STALL_TIMEOUT = 900 # seconds without installer output before giving up
|
||||
|
||||
|
||||
def detect_ip():
|
||||
"""Best-effort local IP as seen on the route toward the bastion.
|
||||
|
||||
Matches Fedora's semantics (IP captured during install): under DHCP the
|
||||
installed system will renew on the same NIC/subnet the live env used.
|
||||
"""
|
||||
import socket
|
||||
try:
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
s.connect(("${params.serverIp}", ${params.httpPort}))
|
||||
ip = s.getsockname()[0]
|
||||
s.close()
|
||||
return ip
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
class LogStreamer:
|
||||
"""Stream install output to the bastion's /api/log so 'labctl provision
|
||||
logs -f' works live for VyOS, like Anaconda's syslog does for Fedora.
|
||||
|
||||
Strictly best-effort: a failed POST drops the batch and must never stall
|
||||
the pty read loop or fail the install.
|
||||
"""
|
||||
|
||||
ANSI = re.compile(rb"\\x1b\\[[0-9;?]*[a-zA-Z]|\\x1b[=>]|\\r")
|
||||
|
||||
def __init__(self):
|
||||
self.partial = b""
|
||||
self.pending = []
|
||||
self.last_flush = time.time()
|
||||
|
||||
def feed(self, chunk):
|
||||
"""Raw pty bytes: split into lines, strip ANSI noise, queue."""
|
||||
self.partial += chunk
|
||||
while b"\\n" in self.partial:
|
||||
raw, self.partial = self.partial.split(b"\\n", 1)
|
||||
text = self.ANSI.sub(b"", raw).decode("utf-8", "replace").rstrip()
|
||||
if text:
|
||||
self.pending.append(text)
|
||||
self.maybe_flush()
|
||||
|
||||
def line(self, text):
|
||||
"""A driver-originated message (already a clean string)."""
|
||||
self.pending.append(text)
|
||||
self.maybe_flush()
|
||||
|
||||
def maybe_flush(self):
|
||||
if len(self.pending) >= 20 or (self.pending and time.time() - self.last_flush >= 2):
|
||||
self.flush()
|
||||
|
||||
def flush(self):
|
||||
if not self.pending:
|
||||
return
|
||||
batch, self.pending = self.pending[:200], self.pending[200:]
|
||||
self.last_flush = time.time()
|
||||
try:
|
||||
body = json.dumps({"mac": MAC, "lines": batch}).encode()
|
||||
req = urllib.request.Request(
|
||||
BASTION + "/api/log",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
urllib.request.urlopen(req, timeout=5).read()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
STREAM = LogStreamer()
|
||||
|
||||
|
||||
def say(msg):
|
||||
"""Print locally and stream to the bastion log buffer."""
|
||||
print(msg)
|
||||
STREAM.line(str(msg))
|
||||
|
||||
|
||||
def report(stage, detail=""):
|
||||
"""Best-effort progress callback; never fatal."""
|
||||
STREAM.flush()
|
||||
try:
|
||||
body = json.dumps({"mac": MAC, "stage": stage, "detail": detail}).encode()
|
||||
req = urllib.request.Request(
|
||||
BASTION + "/api/progress",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
urllib.request.urlopen(req, timeout=5).read()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def build_config():
|
||||
"""Apply our set operations onto the image's own default config.
|
||||
|
||||
Using config.boot.default as the base keeps the vyos-config-version trailer
|
||||
consistent with the running image, so first boot does not run migrations.
|
||||
"""
|
||||
from vyos.configtree import ConfigTree
|
||||
|
||||
default_config = next(
|
||||
(p for p in DEFAULT_CONFIG_CANDIDATES if os.path.exists(p)), None)
|
||||
if default_config is None:
|
||||
raise FileNotFoundError(
|
||||
"no config.boot.default found (tried %s)" % ", ".join(DEFAULT_CONFIG_CANDIDATES))
|
||||
say("base config: %s" % default_config)
|
||||
|
||||
with open(default_config) as handle:
|
||||
config = ConfigTree(handle.read())
|
||||
|
||||
for op in SPEC["sets"]:
|
||||
replace = op.get("replace", True)
|
||||
if "value" in op and op["value"] is not None:
|
||||
config.set(op["path"], value=op["value"], replace=replace)
|
||||
else:
|
||||
config.set(op["path"])
|
||||
|
||||
# Tag nodes must be marked after the nodes exist, as the installer itself does.
|
||||
for tag in SPEC["tags"]:
|
||||
try:
|
||||
config.set_tag(tag)
|
||||
except Exception as err:
|
||||
say("warning: set_tag %s failed: %s" % (tag, err))
|
||||
|
||||
os.makedirs(CONFIG_DIR, exist_ok=True)
|
||||
target = os.path.join(CONFIG_DIR, "config.boot")
|
||||
|
||||
# Re-attach the vyos-config-version footer: ConfigTree.to_string() emits
|
||||
# only the config body, and a config without the footer is treated as
|
||||
# ancient -- the boot migrator then runs every migration over it and (as
|
||||
# observed in the VM test) crashes in system/31-to-32. Building the footer
|
||||
# from the running system pins it to the exact image being installed.
|
||||
body = config.to_string()
|
||||
try:
|
||||
from vyos.component_version import version_info_from_system
|
||||
info = version_info_from_system()
|
||||
info.update_config_body(body)
|
||||
info.write(target)
|
||||
say("wrote %s (footer: %s)" % (target, info.release))
|
||||
except Exception as err:
|
||||
say("warning: version footer failed (%s); writing bare config" % err)
|
||||
with open(target, "w") as handle:
|
||||
handle.write(body)
|
||||
return target
|
||||
|
||||
|
||||
def find_live_squashfs():
|
||||
"""Locate the squashfs live-boot fetched, without walking into the mounted
|
||||
rootfs or overlay (each would mean traversing the entire OS tree)."""
|
||||
explicit = [
|
||||
"/run/live/medium/live/filesystem.squashfs",
|
||||
"/lib/live/mount/medium/live/filesystem.squashfs",
|
||||
]
|
||||
for path in explicit:
|
||||
if os.path.isfile(path) and os.path.getsize(path) > 0:
|
||||
return path
|
||||
for root in ("/run/live", "/lib/live/mount", "/usr/lib/live/mount"):
|
||||
for dirpath, dirs, files in os.walk(root):
|
||||
depth = dirpath.count(os.sep) - root.count(os.sep)
|
||||
dirs[:] = [d for d in dirs
|
||||
if d not in ("rootfs", "overlay")
|
||||
and not d.endswith(".squashfs")
|
||||
and depth < 3]
|
||||
if "filesystem.squashfs" in files:
|
||||
path = os.path.join(dirpath, "filesystem.squashfs")
|
||||
if os.path.isfile(path) and os.path.getsize(path) > 0:
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def ensure_rootfs():
|
||||
"""Make FILE_ROOTFS_SRC exist so the installer can copy the system image."""
|
||||
if os.path.isfile(ROOTFS_EXPECTED) and os.path.getsize(ROOTFS_EXPECTED) > 0:
|
||||
return
|
||||
src = find_live_squashfs()
|
||||
if src is None:
|
||||
say("squashfs not in live mounts; re-fetching %s" % SQUASHFS_URL)
|
||||
src = "/tmp/filesystem.squashfs"
|
||||
urllib.request.urlretrieve(SQUASHFS_URL, src)
|
||||
os.makedirs(os.path.dirname(ROOTFS_EXPECTED), exist_ok=True)
|
||||
if os.path.lexists(ROOTFS_EXPECTED):
|
||||
os.remove(ROOTFS_EXPECTED)
|
||||
os.symlink(src, ROOTFS_EXPECTED)
|
||||
say("rootfs source: %s -> %s" % (ROOTFS_EXPECTED, src))
|
||||
|
||||
|
||||
def build_rules():
|
||||
"""Prompt -> response table for the interactive installer."""
|
||||
password = SPEC["password"].encode() + b"\\n"
|
||||
image_name = SPEC["imageName"].encode() + b"\\n"
|
||||
disk = SPEC["disk"].encode() + b"\\n"
|
||||
console = SPEC["console"].encode() + b"\\n"
|
||||
raid = (b"yes\\n" if SPEC["raid"] else b"no\\n")
|
||||
|
||||
return [
|
||||
(re.compile(rb"Would you like to continue\\?"), b"yes\\n"),
|
||||
(re.compile(rb"What would you like to name this image\\?"), image_name),
|
||||
(re.compile(rb"Please confirm password for the .vyos. user:"), password),
|
||||
(re.compile(rb"Please enter a password for the .vyos. user:"), password),
|
||||
(re.compile(rb"What console should be used by default"), console),
|
||||
# Three RAID variants: "configure RAID-1 mirroring?", "...on them?",
|
||||
# and "choose two disks for RAID-1 mirroring?" -- all default to YES,
|
||||
# so a missed one both hangs the install and risks an unwanted mirror.
|
||||
(re.compile(rb"Would you like to [^?]*RAID-1 mirroring"), raid),
|
||||
(re.compile(rb"Installation will delete all data on (?:the drive|both drives)\\. Continue\\?"), b"yes\\n"),
|
||||
(re.compile(rb"Which one should be used for installation\\?"), disk),
|
||||
(re.compile(rb"Would you like to use all the free space on the drive\\?"), b"yes\\n"),
|
||||
(re.compile(rb"Which file would you like as boot config\\?"), b"1\\n"),
|
||||
# Reinstall path only (search_previous_installation): carrying the old
|
||||
# /config and SSH host keys forward is VyOS's "reinstall without losing
|
||||
# data". Always yes -- freshConfig replaces config.boot afterwards, so
|
||||
# answering no here would also discard non-config data under /config.
|
||||
(re.compile(rb"Would you like to copy data to the new image\\?"), b"yes\\n"),
|
||||
(re.compile(rb"Would you like to copy the encrypted config to the new image\\?"), b"yes\\n"),
|
||||
# More than one previous image found -- take the first offered.
|
||||
(re.compile(rb"From which image would you like to save config information\\?"), b"1\\n"),
|
||||
(re.compile(rb"From which image would you like to copy the encrypted config\\?"), b"1\\n"),
|
||||
]
|
||||
|
||||
|
||||
def run_installer():
|
||||
"""Drive image_installer.py over a pty, answering prompts as they appear."""
|
||||
rules = build_rules()
|
||||
master, slave = pty.openpty()
|
||||
|
||||
proc = subprocess.Popen(
|
||||
[INSTALLER, "--action", "install"],
|
||||
stdin=slave,
|
||||
stdout=slave,
|
||||
stderr=slave,
|
||||
close_fds=True,
|
||||
preexec_fn=os.setsid,
|
||||
)
|
||||
os.close(slave)
|
||||
|
||||
buf = b""
|
||||
transcript = b"" # rolling tail of everything the installer printed
|
||||
last_output = time.time()
|
||||
|
||||
while True:
|
||||
ready, _, _ = select.select([master], [], [], 1.0)
|
||||
|
||||
if ready:
|
||||
try:
|
||||
chunk = os.read(master, 4096)
|
||||
except OSError:
|
||||
break
|
||||
if not chunk:
|
||||
break
|
||||
|
||||
sys.stdout.buffer.write(chunk)
|
||||
sys.stdout.buffer.flush()
|
||||
buf += chunk
|
||||
transcript = (transcript + chunk)[-8000:]
|
||||
STREAM.feed(chunk)
|
||||
last_output = time.time()
|
||||
|
||||
# Answer every prompt currently in the buffer, earliest first, so
|
||||
# ordering is preserved even when the installer skips questions --
|
||||
# and so a single chunk carrying two prompts gets both answers.
|
||||
while True:
|
||||
best = None
|
||||
for pattern, response in rules:
|
||||
found = pattern.search(buf)
|
||||
if found and (best is None or found.start() < best[0].start()):
|
||||
best = (found, response)
|
||||
if best is None:
|
||||
break
|
||||
found, response = best
|
||||
os.write(master, response)
|
||||
transcript = (transcript + b"\\n>>> answered: " + response)[-8000:]
|
||||
STREAM.line(">>> answered: " + response.decode("utf-8", "replace").strip())
|
||||
buf = buf[found.end():]
|
||||
|
||||
# Bound memory if the installer emits a lot without prompting.
|
||||
if len(buf) > 65536:
|
||||
buf = buf[-8192:]
|
||||
|
||||
elif proc.poll() is not None:
|
||||
break
|
||||
|
||||
STREAM.maybe_flush()
|
||||
|
||||
if time.time() - last_output > STALL_TIMEOUT:
|
||||
proc.kill()
|
||||
raise SystemExit("installer produced no output for %ds" % STALL_TIMEOUT)
|
||||
|
||||
os.close(master)
|
||||
return proc.wait(), transcript.decode("utf-8", "replace")
|
||||
|
||||
|
||||
def ensure_network_boot_first():
|
||||
"""Keep network boot first so the bastion intercepts every reboot.
|
||||
|
||||
Port of the Fedora kickstart's %post efibootmgr step (install.ks.ts) --
|
||||
what makes reprovision-by-reboot work. Best-effort: skipped on BIOS boots
|
||||
or when efibootmgr is absent. Runs from the live env after the installer;
|
||||
efibootmgr edits NVRAM, not the disk, so installer cleanup is irrelevant.
|
||||
"""
|
||||
import shutil
|
||||
if not os.path.isdir("/sys/firmware/efi") or shutil.which("efibootmgr") is None:
|
||||
say("boot order: skipped (BIOS boot or efibootmgr missing)")
|
||||
return
|
||||
try:
|
||||
out = subprocess.run(["efibootmgr"], capture_output=True, text=True, timeout=30).stdout
|
||||
order = []
|
||||
network_entry = None
|
||||
for line in out.splitlines():
|
||||
m = re.match(r"^BootOrder:\\s*(.*)$", line)
|
||||
if m:
|
||||
order = [x.strip() for x in m.group(1).split(",") if x.strip()]
|
||||
continue
|
||||
m = re.match(r"^Boot([0-9A-Fa-f]{4})\\*?\\s+(.*)$", line)
|
||||
if m and network_entry is None:
|
||||
if re.search(r"network|pxe|ipv4|ipv6|http", m.group(2), re.IGNORECASE):
|
||||
network_entry = m.group(1).upper()
|
||||
if network_entry is None or not order:
|
||||
say("boot order: no network boot entry found; leaving as is")
|
||||
return
|
||||
new_order = [network_entry] + [x for x in order if x.upper() != network_entry]
|
||||
if [x.upper() for x in order] == [x.upper() for x in new_order]:
|
||||
say("boot order: network entry Boot%s already first" % network_entry)
|
||||
return
|
||||
subprocess.run(["efibootmgr", "-o", ",".join(new_order)],
|
||||
capture_output=True, timeout=30)
|
||||
say("boot order: moved network entry Boot%s first" % network_entry)
|
||||
except Exception as err:
|
||||
say("warning: boot order adjustment failed: %s" % err)
|
||||
|
||||
|
||||
def with_target_mounted(fn):
|
||||
"""Mount the installed root partition, call fn(rw_dir), always unmount.
|
||||
|
||||
The installer has unmounted and cleaned the target by the time this runs,
|
||||
so the block device is free. The partition holding boot/<image>/rw is the
|
||||
VyOS root; the glob also yields the installed image's rw dir directly.
|
||||
"""
|
||||
import glob
|
||||
disk = SPEC["disk"]
|
||||
if not disk:
|
||||
# No pinned disk (installer picked the default) -- enumerate all disks.
|
||||
candidates = ["/dev/" + b for b in os.listdir("/sys/block")
|
||||
if not b.startswith(("loop", "ram", "zram", "sr"))]
|
||||
else:
|
||||
candidates = [disk]
|
||||
|
||||
mnt = "/mnt/lab-target"
|
||||
os.makedirs(mnt, exist_ok=True)
|
||||
for dev in candidates:
|
||||
name = os.path.basename(dev)
|
||||
parts = sorted(p for p in os.listdir("/sys/block/%s" % name)
|
||||
if p.startswith(name)) if os.path.isdir("/sys/block/%s" % name) else []
|
||||
for part in parts:
|
||||
pdev = "/dev/" + part
|
||||
if subprocess.run(["mount", pdev, mnt], capture_output=True).returncode != 0:
|
||||
continue
|
||||
try:
|
||||
rw_dirs = glob.glob(os.path.join(mnt, "boot", "*", "rw"))
|
||||
if rw_dirs:
|
||||
fn(rw_dirs[0])
|
||||
return True
|
||||
finally:
|
||||
subprocess.run(["umount", mnt], capture_output=True)
|
||||
return False
|
||||
|
||||
|
||||
def post_install_target_steps():
|
||||
"""Metadata + optional fresh-config overwrite inside the installed image."""
|
||||
def apply(rw_dir):
|
||||
config_dir = os.path.join(rw_dir, "opt/vyatta/etc/config")
|
||||
os.makedirs(config_dir, exist_ok=True)
|
||||
|
||||
# /config/lab-provisioned -- survives VyOS image upgrades. Mirrors the
|
||||
# Fedora kickstart's /etc/lab-provisioned.
|
||||
try:
|
||||
with open(os.path.join(config_dir, "lab-provisioned"), "w") as handle:
|
||||
handle.write("hostname=%s\\n" % SPEC["hostname"])
|
||||
handle.write("role=%s\\n" % ROLE)
|
||||
handle.write("provisioned=%s\\n" % time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()))
|
||||
handle.write("bastion=%s\\n" % BASTION)
|
||||
say("wrote /config/lab-provisioned")
|
||||
except Exception as err:
|
||||
say("warning: lab-provisioned metadata failed: %s" % err)
|
||||
|
||||
# freshConfig: make the bastion-generated config win over the previous
|
||||
# installation's carried-forward config. Explicit intent -- failure is
|
||||
# fatal (raised out of with_target_mounted).
|
||||
if SPEC.get("freshConfig"):
|
||||
import shutil
|
||||
shutil.copyfile(os.path.join(CONFIG_DIR, "config.boot"),
|
||||
os.path.join(config_dir, "config.boot"))
|
||||
say("freshConfig: replaced installed config.boot with generated config")
|
||||
|
||||
mounted = with_target_mounted(apply)
|
||||
if not mounted:
|
||||
if SPEC.get("freshConfig"):
|
||||
raise RuntimeError("freshConfig requested but installed root partition not found")
|
||||
say("warning: installed root partition not found; skipping metadata")
|
||||
|
||||
|
||||
def main():
|
||||
report("vyos-install", "building config.boot")
|
||||
try:
|
||||
build_config()
|
||||
except Exception as err:
|
||||
report("error", "config generation failed: %s" % err)
|
||||
raise
|
||||
|
||||
report("vyos-install", "staging rootfs for installer")
|
||||
try:
|
||||
ensure_rootfs()
|
||||
except Exception as err:
|
||||
report("error", "rootfs staging failed: %s" % err)
|
||||
raise
|
||||
|
||||
report("vyos-install", "running install image")
|
||||
code, transcript = run_installer()
|
||||
|
||||
if code != 0:
|
||||
# Surface the installer's last words in bastion progress -- the console
|
||||
# they were printed on is usually invisible during unattended installs.
|
||||
report("error", "install image exited %d | tail: %s" % (code, transcript[-4000:]))
|
||||
raise SystemExit(code)
|
||||
|
||||
report("post-install", "boot order + metadata")
|
||||
ensure_network_boot_first()
|
||||
try:
|
||||
post_install_target_steps()
|
||||
except Exception as err:
|
||||
report("error", "post-install target steps failed: %s" % err)
|
||||
raise
|
||||
|
||||
# "complete" is the stage the bastion uses to move a machine out of the
|
||||
# install queue into installed state, and "ready at <ip>" is the exact
|
||||
# detail format it parses installed.ip from -- see routes/api.ts.
|
||||
ip = SPEC.get("reportAddress") or detect_ip()
|
||||
report("complete", "ready at %s" % ip if ip else "VyOS installed, rebooting")
|
||||
os.system("sync")
|
||||
# --force: this driver is a child of live-config.service, whose start job is
|
||||
# still running -- a normal reboot deadlocks waiting for it (verified in VM:
|
||||
# shutdown blocked >1min on "start job is running for live-config"). The
|
||||
# installer has already unmounted and cleaned the target, so an immediate
|
||||
# reboot is safe.
|
||||
os.system("systemctl reboot --force")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
`;
|
||||
}
|
||||
@@ -1,225 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { mkdirSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import type { BastionConfig } from "@lab/shared";
|
||||
import { createApp } from "../src/server.js";
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import { renderFirstbootScript, renderFirstbootUnit } from "../src/templates/asahi-firstboot.sh.js";
|
||||
|
||||
function createTestConfig(testDir: string): BastionConfig {
|
||||
return {
|
||||
fedoraVersion: "43",
|
||||
arch: "x86_64",
|
||||
httpPort: 0,
|
||||
timezone: "Europe/London",
|
||||
locale: "en_GB.UTF-8",
|
||||
bastionDir: testDir,
|
||||
domain: "test.local",
|
||||
dhcpMode: "proxy",
|
||||
dhcpRangeStart: "",
|
||||
dhcpRangeEnd: "",
|
||||
ubuntuVersion: "26.04",
|
||||
ubuntuMirror: "https://releases.ubuntu.com/26.04",
|
||||
iface: "eth0",
|
||||
serverIp: "192.168.8.1",
|
||||
network: "192.168.8.0",
|
||||
gateway: "192.168.8.1",
|
||||
sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@lab"],
|
||||
adminUser: "michal",
|
||||
syslogPort: 15514,
|
||||
skipDnsmasq: true,
|
||||
skipArtifacts: true,
|
||||
fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
|
||||
tftpDir: join(testDir, "tftp"),
|
||||
httpDir: join(testDir, "http"),
|
||||
stateFile: join(testDir, "state.json"),
|
||||
};
|
||||
}
|
||||
|
||||
describe("asahi routes", () => {
|
||||
let testDir: string;
|
||||
let app: FastifyInstance;
|
||||
|
||||
beforeEach(() => {
|
||||
testDir = join(tmpdir(), `bastion-asahi-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
||||
mkdirSync(testDir, { recursive: true });
|
||||
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||
|
||||
const config = createTestConfig(testDir);
|
||||
const result = createApp(config);
|
||||
app = result.app;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await app.close();
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("GET /asahi returns wrapper shell script", async () => {
|
||||
const resp = await app.inject({ method: "GET", url: "/asahi" });
|
||||
expect(resp.statusCode).toBe(200);
|
||||
expect(resp.headers["content-type"]).toContain("text/x-shellscript");
|
||||
expect(resp.body).toContain("#!/bin/bash");
|
||||
expect(resp.body).toContain("installer_data.json");
|
||||
expect(resp.body).toContain("192.168.8.1");
|
||||
expect(resp.body).toContain("install.sh");
|
||||
});
|
||||
|
||||
it("GET /asahi/installer_data.json returns valid config", async () => {
|
||||
const resp = await app.inject({ method: "GET", url: "/asahi/installer_data.json" });
|
||||
expect(resp.statusCode).toBe(200);
|
||||
const data = JSON.parse(resp.body);
|
||||
|
||||
expect(data.os_list).toHaveLength(1);
|
||||
const os = data.os_list[0];
|
||||
expect(os.name).toContain("Fedora Asahi Lab");
|
||||
|
||||
// 3 partitions (fallback) or 4 (built: EFI + Boot + Root + Data)
|
||||
expect(os.partitions.length).toBeGreaterThanOrEqual(3);
|
||||
expect(os.partitions[0].type).toBe("EFI");
|
||||
// Last partition should be the expanding Data partition
|
||||
const lastPart = os.partitions[os.partitions.length - 1];
|
||||
expect(lastPart.type).toBe("Linux");
|
||||
expect(lastPart.expand).toBe(true);
|
||||
// Root partition (second-to-last) should NOT expand
|
||||
const rootPart = os.partitions[os.partitions.length - 2];
|
||||
expect(rootPart.expand).toBe(false);
|
||||
expect(rootPart.image).toBe("root.img");
|
||||
});
|
||||
|
||||
it("GET /asahi/firstboot.sh returns parameterized script", async () => {
|
||||
const resp = await app.inject({
|
||||
method: "GET",
|
||||
url: "/asahi/firstboot.sh?hostname=mac-studio&role=infra&mac=00:11:22:33:44:55",
|
||||
});
|
||||
expect(resp.statusCode).toBe(200);
|
||||
expect(resp.body).toContain("#!/bin/bash");
|
||||
expect(resp.body).toContain("mac-studio");
|
||||
expect(resp.body).toContain("labvg");
|
||||
expect(resp.body).toContain("rancher"); // infra gets rancher LV
|
||||
expect(resp.body).toContain("longhorn"); // infra also gets longhorn
|
||||
expect(resp.body).toContain("ssh-ed25519"); // SSH key injected
|
||||
});
|
||||
|
||||
it("GET /asahi/firstboot.service returns systemd unit", async () => {
|
||||
const resp = await app.inject({ method: "GET", url: "/asahi/firstboot.service" });
|
||||
expect(resp.statusCode).toBe(200);
|
||||
expect(resp.body).toContain("[Unit]");
|
||||
expect(resp.body).toContain("lab-firstboot.sh");
|
||||
expect(resp.body).toContain("ConditionPathExists=!/etc/lab-lvm-setup-done");
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderFirstbootScript", () => {
|
||||
const baseParams = {
|
||||
hostname: "test-node",
|
||||
serverIp: "10.0.0.1",
|
||||
httpPort: 8080,
|
||||
sshKeys: ["ssh-ed25519 AAAA... user@host"],
|
||||
adminUser: "testadmin",
|
||||
mac: "aa:bb:cc:dd:ee:ff",
|
||||
};
|
||||
|
||||
it("generates valid bash with shebang", () => {
|
||||
const script = renderFirstbootScript({ ...baseParams, role: "worker" });
|
||||
expect(script.startsWith("#!/bin/bash")).toBe(true);
|
||||
});
|
||||
|
||||
it("includes LVM creation commands", () => {
|
||||
const script = renderFirstbootScript({ ...baseParams, role: "infra" });
|
||||
expect(script).toContain("pvcreate");
|
||||
expect(script).toContain("vgcreate labvg");
|
||||
expect(script).toContain("lvcreate");
|
||||
});
|
||||
|
||||
it("uses correct LV sizes from kickstart layout", () => {
|
||||
const script = renderFirstbootScript({ ...baseParams, role: "infra" });
|
||||
expect(script).toContain("27648M"); // swap
|
||||
expect(script).toContain("102400M"); // /var
|
||||
expect(script).toContain("10240M"); // /var/log and /home
|
||||
expect(script).toContain("20480M"); // /srv and /rancher
|
||||
});
|
||||
|
||||
it("includes rancher LV for infra role", () => {
|
||||
const script = renderFirstbootScript({ ...baseParams, role: "infra" });
|
||||
expect(script).toContain("rancher");
|
||||
expect(script).toContain("/var/lib/rancher");
|
||||
});
|
||||
|
||||
it("includes longhorn for worker role", () => {
|
||||
const script = renderFirstbootScript({ ...baseParams, role: "worker" });
|
||||
expect(script).toContain("longhorn");
|
||||
expect(script).toContain("/var/lib/longhorn");
|
||||
// Worker should NOT have rancher
|
||||
expect(script).not.toContain("rancher");
|
||||
});
|
||||
|
||||
it("includes longhorn for infra role", () => {
|
||||
const script = renderFirstbootScript({ ...baseParams, role: "infra" });
|
||||
expect(script).toContain("longhorn");
|
||||
expect(script).toContain("/var/lib/longhorn");
|
||||
});
|
||||
|
||||
it("vanilla role gets no role-specific LVs", () => {
|
||||
const script = renderFirstbootScript({ ...baseParams, role: "vanilla" });
|
||||
expect(script).not.toContain("rancher");
|
||||
expect(script).not.toContain("longhorn");
|
||||
});
|
||||
|
||||
it("handles reprovision (existing labvg)", () => {
|
||||
const script = renderFirstbootScript({ ...baseParams, role: "infra" });
|
||||
expect(script).toContain("reprovision detected");
|
||||
expect(script).toContain("vgchange -ay labvg");
|
||||
expect(script).toContain("mount_lv var /var");
|
||||
});
|
||||
|
||||
it("injects SSH keys for admin user and root", () => {
|
||||
const script = renderFirstbootScript({ ...baseParams, role: "worker" });
|
||||
expect(script).toContain("ssh-ed25519 AAAA...");
|
||||
expect(script).toContain("testadmin");
|
||||
expect(script).toContain("/root/.ssh/authorized_keys");
|
||||
});
|
||||
|
||||
it("sets hostname", () => {
|
||||
const script = renderFirstbootScript({ ...baseParams, role: "worker" });
|
||||
expect(script).toContain('CONF_HOSTNAME="test-node"');
|
||||
expect(script).toContain("hostnamectl set-hostname");
|
||||
});
|
||||
|
||||
it("includes bastion self-registration", () => {
|
||||
const script = renderFirstbootScript({ ...baseParams, role: "worker" });
|
||||
expect(script).toContain("/api/register");
|
||||
expect(script).toContain("aa:bb:cc:dd:ee:ff");
|
||||
expect(script).toContain("test-node");
|
||||
});
|
||||
|
||||
it("writes provisioning metadata", () => {
|
||||
const script = renderFirstbootScript({ ...baseParams, role: "infra" });
|
||||
expect(script).toContain("/etc/lab-provisioned");
|
||||
expect(script).toContain("method=asahi-firstboot");
|
||||
});
|
||||
|
||||
it("creates marker file to prevent re-run", () => {
|
||||
const script = renderFirstbootScript({ ...baseParams, role: "worker" });
|
||||
expect(script).toContain("/etc/lab-lvm-setup-done");
|
||||
expect(script).toContain('touch "$MARKER"');
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderFirstbootUnit", () => {
|
||||
it("generates valid systemd unit", () => {
|
||||
const unit = renderFirstbootUnit();
|
||||
expect(unit).toContain("[Unit]");
|
||||
expect(unit).toContain("[Service]");
|
||||
expect(unit).toContain("[Install]");
|
||||
expect(unit).toContain("Type=oneshot");
|
||||
expect(unit).toContain("WantedBy=multi-user.target");
|
||||
});
|
||||
|
||||
it("only runs when marker is missing", () => {
|
||||
const unit = renderFirstbootUnit();
|
||||
expect(unit).toContain("ConditionPathExists=!/etc/lab-lvm-setup-done");
|
||||
});
|
||||
});
|
||||
@@ -22,15 +22,12 @@ function createTestConfig(testDir: string): BastionConfig {
|
||||
dhcpRangeEnd: "",
|
||||
ubuntuVersion: "26.04",
|
||||
ubuntuMirror: "https://releases.ubuntu.com/26.04",
|
||||
vyosIsoUrl: "https://downloads.vyos.io/rolling/current/generic/vyos-rolling-latest.iso",
|
||||
vyosDefaultPassword: "vyos",
|
||||
iface: "eth0",
|
||||
serverIp: "10.0.0.1",
|
||||
network: "10.0.0.0",
|
||||
gateway: "10.0.0.1",
|
||||
sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST test@test"],
|
||||
adminUser: "testadmin",
|
||||
syslogPort: 15514,
|
||||
skipDnsmasq: true,
|
||||
skipArtifacts: true,
|
||||
fedoraMirror: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os",
|
||||
|
||||
@@ -96,9 +96,9 @@ describe("renderInstallKickstart", () => {
|
||||
expect(ks).toContain("/api/progress");
|
||||
});
|
||||
|
||||
it("infra role has 120G /var/lib/rancher partition", () => {
|
||||
it("infra role has /var/lib/rancher partition", () => {
|
||||
const ks = renderInstallKickstart(baseParams({ role: "infra" }));
|
||||
expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=122880");
|
||||
expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=20480");
|
||||
});
|
||||
|
||||
it("infra role has k3s install", () => {
|
||||
@@ -106,14 +106,10 @@ describe("renderInstallKickstart", () => {
|
||||
expect(ks).toContain("curl -sfL https://get.k3s.io | INSTALL_K3S_SKIP_START=true sh -");
|
||||
});
|
||||
|
||||
it("worker role has 120G /var/lib/rancher partition (imageFs must be sized before longhorn --grow)", () => {
|
||||
it("worker role does NOT have /var/lib/rancher partition in fresh install", () => {
|
||||
const ks = renderInstallKickstart(baseParams({ role: "worker" }));
|
||||
expect(ks).toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=122880");
|
||||
});
|
||||
|
||||
it("vanilla role does NOT have /var/lib/rancher partition in fresh install", () => {
|
||||
const ks = renderInstallKickstart(baseParams({ role: "vanilla" }));
|
||||
expect(ks).not.toContain("--name=rancher --fstype=xfs");
|
||||
// Worker should not have the fresh-install rancher partition line
|
||||
expect(ks).not.toContain("logvol /var/lib/rancher --vgname=labvg --name=rancher --fstype=xfs --size=20480");
|
||||
});
|
||||
|
||||
it("worker role does NOT have k3s install", () => {
|
||||
@@ -210,8 +206,10 @@ describe("renderInstallKickstart", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("does not include serial console (causes 30s boot timeout on hardware without UART)", () => {
|
||||
it("forwards system logs to serial console", () => {
|
||||
const ks = renderInstallKickstart(baseParams({ role: "vanilla" }));
|
||||
expect(ks).not.toContain("ttyS0");
|
||||
expect(ks).toContain("serial-console.conf");
|
||||
expect(ks).toContain("/dev/ttyS0");
|
||||
expect(ks).toContain("rsyslog");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { createSocket } from "node:dgram";
|
||||
import { mkdtempSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { SyslogListener } from "../src/services/syslog-listener.js";
|
||||
import { InstallLogBuffer } from "../src/services/install-log.js";
|
||||
import { StateManager } from "../src/services/state.js";
|
||||
|
||||
function sendUdpSyslog(port: number, message: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const client = createSocket("udp4");
|
||||
const buf = Buffer.from(message);
|
||||
client.send(buf, 0, buf.length, port, "127.0.0.1", (err) => {
|
||||
client.close();
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
describe("SyslogListener", () => {
|
||||
let tmpDir: string;
|
||||
let state: StateManager;
|
||||
let installLog: InstallLogBuffer;
|
||||
let syslog: SyslogListener;
|
||||
const PORT = 15514; // use non-privileged port for testing
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), "syslog-test-"));
|
||||
state = new StateManager(join(tmpDir, "state.json"));
|
||||
state.init();
|
||||
installLog = new InstallLogBuffer(tmpDir);
|
||||
syslog = new SyslogListener(PORT, installLog, state);
|
||||
syslog.start();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
syslog.stop();
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("receives and stores syslog messages for registered IP", async () => {
|
||||
const mac = "aa:bb:cc:dd:ee:ff";
|
||||
// Queue a machine so hostname can be resolved
|
||||
state.update((s) => {
|
||||
s.install_queue[mac] = {
|
||||
hostname: "testnode",
|
||||
disk: "/dev/sda",
|
||||
role: "worker",
|
||||
os: "fedora-43",
|
||||
queued_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
// Register IP → MAC mapping
|
||||
syslog.registerIp("127.0.0.1", mac);
|
||||
|
||||
// Send a syslog message (RFC 3164 format)
|
||||
await sendUdpSyslog(PORT, "<13>Mar 30 01:30:00 localhost anaconda[1234]: Installing package vim-enhanced");
|
||||
|
||||
// Wait for UDP delivery
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
|
||||
const lines = installLog.getLines(mac);
|
||||
expect(lines.length).toBeGreaterThan(0);
|
||||
expect(lines[0]!.line).toContain("anaconda");
|
||||
expect(lines[0]!.line).toContain("Installing package vim-enhanced");
|
||||
});
|
||||
|
||||
it("ignores messages from unknown IPs", async () => {
|
||||
// Don't register any IP mapping
|
||||
await sendUdpSyslog(PORT, "<13>Mar 30 01:30:00 localhost anaconda[1234]: test message");
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
|
||||
// No MAC to check, but the listener should not crash
|
||||
// and no logs should be stored for any MAC
|
||||
expect(installLog.lineCount("unknown")).toBe(0);
|
||||
});
|
||||
|
||||
it("resolves IP from installed machines state", async () => {
|
||||
const mac = "11:22:33:44:55:66";
|
||||
state.update((s) => {
|
||||
s.installed[mac] = {
|
||||
hostname: "installed-node",
|
||||
role: "worker",
|
||||
ip: "127.0.0.1",
|
||||
installed_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
await sendUdpSyslog(PORT, "<14>Mar 30 02:00:00 installed-node sshd[5678]: Accepted publickey for root");
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
|
||||
const lines = installLog.getLines(mac);
|
||||
expect(lines.length).toBeGreaterThan(0);
|
||||
expect(lines[0]!.line).toContain("sshd");
|
||||
});
|
||||
|
||||
it("parses various syslog formats", async () => {
|
||||
const mac = "aa:bb:cc:dd:ee:ff";
|
||||
syslog.registerIp("127.0.0.1", mac);
|
||||
state.update((s) => {
|
||||
s.install_queue[mac] = {
|
||||
hostname: "testnode",
|
||||
disk: "/dev/sda",
|
||||
role: "worker",
|
||||
os: "fedora-43",
|
||||
queued_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
// Message without PID
|
||||
await sendUdpSyslog(PORT, "<13>Mar 30 01:30:00 localhost kernel: NVMe device ready");
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
|
||||
const lines = installLog.getLines(mac);
|
||||
expect(lines.length).toBeGreaterThan(0);
|
||||
expect(lines[0]!.line).toContain("kernel");
|
||||
});
|
||||
});
|
||||
@@ -1,87 +0,0 @@
|
||||
// The Ubuntu autoinstall document must be valid YAML for EVERY role.
|
||||
//
|
||||
// There was no test here, and the template shipped a document that did not
|
||||
// parse: the longhorn/rancher LVM entries were indented 8 spaces while their
|
||||
// siblings in storage.config sit at 6, giving "expected <block end>, but found
|
||||
// '-'". Every role that gets a longhorn volume -- which is the worker shape --
|
||||
// rendered an uninstallable document. A `toContain` assertion would not have
|
||||
// caught that; only parsing does.
|
||||
//
|
||||
// Parsed with python3's yaml rather than a new npm dependency, mirroring how
|
||||
// kickstart.test.ts shells out to `ksvalidator`: the point is to check the
|
||||
// artefact with a real parser, not to grow the dependency tree.
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { writeFileSync, unlinkSync } from "node:fs";
|
||||
import { renderUbuntuAutoinstall } from "../src/templates/ubuntu-autoinstall.js";
|
||||
|
||||
const base = {
|
||||
hostname: "n6",
|
||||
disk: "/dev/sda",
|
||||
domain: "ad.itaz.eu",
|
||||
ubuntuVersion: "24.04",
|
||||
timezone: "Europe/London",
|
||||
locale: "en_GB.UTF-8",
|
||||
serverIp: "10.0.0.1",
|
||||
httpPort: 8080,
|
||||
sshKeys: ["ssh-ed25519 AAAAtest test@lab"],
|
||||
adminUser: "root",
|
||||
};
|
||||
|
||||
/** Parse with python3's yaml and return the document as JSON. */
|
||||
function parseYaml(text: string, label: string): Record<string, any> {
|
||||
const tmp = `/tmp/autoinstall-test-${label}.yaml`;
|
||||
writeFileSync(tmp, text);
|
||||
try {
|
||||
const out = execFileSync(
|
||||
"python3",
|
||||
["-c", "import sys,yaml,json; json.dump(yaml.safe_load(open(sys.argv[1])), sys.stdout)", tmp],
|
||||
{ encoding: "utf-8" },
|
||||
);
|
||||
return JSON.parse(out);
|
||||
} catch (err: unknown) {
|
||||
const msg = err instanceof Error ? (err as { stderr?: string }).stderr ?? err.message : String(err);
|
||||
throw new Error(`autoinstall YAML did not parse for ${label}: ${msg}`);
|
||||
} finally {
|
||||
try { unlinkSync(tmp); } catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
|
||||
describe("renderUbuntuAutoinstall", () => {
|
||||
for (const role of ["vanilla", "worker", "infra"]) {
|
||||
it(`renders parseable YAML for role=${role}`, () => {
|
||||
const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role }), role);
|
||||
expect(doc.autoinstall).toBeDefined();
|
||||
expect(doc.autoinstall.version).toBe(1);
|
||||
// storage.config must be a flat list; the indentation bug produced a
|
||||
// nested map here, which is how it went unnoticed.
|
||||
expect(Array.isArray(doc.autoinstall.storage.config)).toBe(true);
|
||||
});
|
||||
}
|
||||
|
||||
it("gives the longhorn role its volume as a sibling entry, not a nested map", () => {
|
||||
const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" }), "longhorn");
|
||||
const ids = doc.autoinstall.storage.config.map((e: { id: string }) => e.id);
|
||||
expect(ids).toContain("lv-longhorn");
|
||||
expect(ids).toContain("mount-longhorn");
|
||||
});
|
||||
|
||||
it("sets EUI-64 link-local so MAC-keyed DHCPv6 reservations can match", () => {
|
||||
const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" }), "eui64");
|
||||
const late = (doc.autoinstall["late-commands"] as string[]).join("\n");
|
||||
expect(late).toContain("10-ipv6-eui64.conf");
|
||||
expect(late).toContain("ipv6.addr-gen-mode=eui64");
|
||||
});
|
||||
|
||||
it("requests both address families on the primary NIC", () => {
|
||||
const doc = parseYaml(renderUbuntuAutoinstall({ ...base, role: "worker" }), "net");
|
||||
const eth = doc.autoinstall.network.ethernets.primary;
|
||||
expect(eth.dhcp4).toBe(true);
|
||||
// Without this a node provisioned into a dual-stack cluster comes up with
|
||||
// no IPv6 and joins as an IPv4-only member.
|
||||
expect(eth.dhcp6).toBe(true);
|
||||
// The install must not block waiting for a v6 lease that may never come.
|
||||
expect(eth.optional).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,155 +0,0 @@
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
import type { VyosBundle } from "@lab/shared";
|
||||
import { buildVyosConfigSpec } from "../src/templates/vyos-config-spec.js";
|
||||
|
||||
/**
|
||||
* A bundle is what makes "one config, two apply paths" true rather than
|
||||
* aspirational: `pulumi up` POSTs the subtree model to a running router, labctl
|
||||
* writes the same model into config.boot during a PXE install. These tests pin
|
||||
* the properties that keep the two honest.
|
||||
*/
|
||||
const bundle: VyosBundle = {
|
||||
sets: [
|
||||
{ path: ["system", "host-name"], value: "vyos001" },
|
||||
{ path: ["interfaces", "bonding", "bond0", "address"], value: "192.168.1.252/24" },
|
||||
{ path: ["interfaces", "bonding", "bond0", "member", "interface"], value: "eth1", replace: false },
|
||||
{ path: ["interfaces", "bonding", "bond0", "vif", "53", "disable"] },
|
||||
{ path: ["interfaces", "pppoe", "pppoe0", "authentication", "password"], value: "@secret:pppoePassword" },
|
||||
{ path: ["interfaces", "pppoe", "pppoe0", "mtu"], value: "1492" },
|
||||
],
|
||||
tags: [["interfaces", "bonding", "bond0"], ["interfaces", "ethernet"]],
|
||||
};
|
||||
|
||||
const build = (hostname: string, extra: Record<string, unknown> = {}) =>
|
||||
buildVyosConfigSpec({
|
||||
hostname,
|
||||
spec: { bundle, ...extra },
|
||||
defaultPassword: "changeme",
|
||||
});
|
||||
|
||||
describe("vyos config spec from a Pulumi bundle", () => {
|
||||
it("applies non-secret nodes verbatim, preserving valuelessness and replace:false", () => {
|
||||
const spec = build("vyos001");
|
||||
|
||||
expect(spec.sets).toContainEqual({
|
||||
path: ["interfaces", "bonding", "bond0", "address"],
|
||||
value: "192.168.1.252/24",
|
||||
});
|
||||
// A multi-value node must keep replace:false or the second bond member
|
||||
// overwrites the first.
|
||||
expect(spec.sets).toContainEqual({
|
||||
path: ["interfaces", "bonding", "bond0", "member", "interface"],
|
||||
value: "eth1",
|
||||
replace: false,
|
||||
});
|
||||
// A valueless node must not acquire a value on the way through.
|
||||
expect(spec.sets).toContainEqual({
|
||||
path: ["interfaces", "bonding", "bond0", "vif", "53", "disable"],
|
||||
});
|
||||
expect(spec.tags).toEqual(bundle.tags);
|
||||
});
|
||||
|
||||
it("drops secret-valued nodes instead of installing the sentinel text", () => {
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const spec = build("vyos001");
|
||||
|
||||
const values = spec.sets.map((s) => s.value ?? "");
|
||||
expect(values.some((v) => v.startsWith("@secret:"))).toBe(false);
|
||||
expect(spec.sets.some((s) => s.path.includes("authentication"))).toBe(false);
|
||||
// Silently dropping the WAN credential would leave someone debugging a dead
|
||||
// PPPoE link, so it has to be said out loud.
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining("pulumi up"));
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it("forces the hostname the install was asked for, not the bundle's", () => {
|
||||
// The bundle is exported from one router and reused for its peer; taking the
|
||||
// hostname from it would put two vyos001s on the network.
|
||||
const spec = build("vyos002");
|
||||
const hostnames = spec.sets.filter(
|
||||
(s) => s.path.length === 2 && s.path[0] === "system" && s.path[1] === "host-name",
|
||||
);
|
||||
expect(hostnames).toEqual([{ path: ["system", "host-name"], value: "vyos002" }]);
|
||||
});
|
||||
|
||||
it("still honours installer inputs, which are not router config", () => {
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "vyos001",
|
||||
spec: { bundle, password: "s3cret", freshConfig: true },
|
||||
defaultPassword: "changeme",
|
||||
disk: "nvme0n1",
|
||||
});
|
||||
expect(spec.password).toBe("s3cret");
|
||||
expect(spec.freshConfig).toBe(true);
|
||||
expect(spec.disk).toBe("/dev/nvme0n1");
|
||||
});
|
||||
|
||||
it("enables the HTTP API at install so Pulumi can manage the router from first boot", () => {
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "vyos001",
|
||||
spec: { bundle, apiKey: "k3y", apiListenAddress: "10.0.1.252" },
|
||||
defaultPassword: "changeme",
|
||||
});
|
||||
|
||||
expect(spec.sets).toContainEqual({
|
||||
path: ["service", "https", "api", "keys", "id", "pulumi", "key"],
|
||||
value: "k3y",
|
||||
});
|
||||
expect(spec.sets).toContainEqual({ path: ["service", "https", "api", "rest"] });
|
||||
expect(spec.sets).toContainEqual({
|
||||
path: ["service", "https", "listen-address"],
|
||||
value: "10.0.1.252",
|
||||
});
|
||||
// The key id is a tag node; without this the installer's ConfigTree rejects it.
|
||||
expect(spec.tags).toContainEqual(["service", "https", "api", "keys", "id"]);
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it("binds the API to the static management address when none is given", () => {
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "vyos001",
|
||||
spec: { apiKey: "k3y", mgmtAddress: "192.168.1.252/24" },
|
||||
defaultPassword: "changeme",
|
||||
});
|
||||
expect(spec.sets).toContainEqual({
|
||||
path: ["service", "https", "listen-address"],
|
||||
value: "192.168.1.252",
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses to enable the API unbound rather than exposing it on the WAN", () => {
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
// Management is DHCP, so there is no address to bind at build time. Binding
|
||||
// to everything would put a config-write endpoint on the WAN.
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "vyos001",
|
||||
spec: { apiKey: "k3y", mgmtAddress: "dhcp" },
|
||||
defaultPassword: "changeme",
|
||||
});
|
||||
expect(spec.sets.some((s) => s.path[0] === "service" && s.path[1] === "https")).toBe(false);
|
||||
expect(warn).toHaveBeenCalledWith(expect.stringContaining("has NOT been enabled"));
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
it("does not enable the API when no key is supplied", () => {
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "vyos001",
|
||||
spec: { mgmtAddress: "192.168.1.252/24" },
|
||||
defaultPassword: "changeme",
|
||||
});
|
||||
expect(spec.sets.some((s) => s.path[0] === "service" && s.path[1] === "https")).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores the derived path entirely when a bundle is present", () => {
|
||||
// Belt and braces: even if topology flags reach this far (the CLI rejects
|
||||
// them), the bundle must win rather than merge.
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "vyos001",
|
||||
spec: { bundle, bondMembers: ["eth2", "eth3"], vlans: [{ id: 99, address: "10.9.9.1/24" }] },
|
||||
defaultPassword: "changeme",
|
||||
});
|
||||
expect(spec.sets.some((s) => s.path.includes("99"))).toBe(false);
|
||||
expect(spec.sets.filter((s) => s.value === "eth2" || s.value === "eth3")).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,548 +0,0 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { mkdirSync, rmSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import type { BastionConfig } from "@lab/shared";
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import { createApp } from "../src/server.js";
|
||||
import type { StateManager } from "../src/services/state.js";
|
||||
import { buildVyosConfigSpec } from "../src/templates/vyos-config-spec.js";
|
||||
import { renderVyosInstallPy } from "../src/templates/vyos-install.py.js";
|
||||
|
||||
function createTestConfig(testDir: string): BastionConfig {
|
||||
return {
|
||||
fedoraVersion: "43",
|
||||
arch: "x86_64",
|
||||
httpPort: 0,
|
||||
timezone: "Europe/London",
|
||||
locale: "en_GB.UTF-8",
|
||||
bastionDir: testDir,
|
||||
domain: "test.local",
|
||||
dhcpMode: "proxy",
|
||||
dhcpRangeStart: "",
|
||||
dhcpRangeEnd: "",
|
||||
ubuntuVersion: "26.04",
|
||||
ubuntuMirror: "https://releases.ubuntu.com/26.04",
|
||||
vyosIsoUrl: "https://example.invalid/vyos.iso",
|
||||
vyosDefaultPassword: "test-pw",
|
||||
iface: "eth0",
|
||||
serverIp: "10.0.0.1",
|
||||
network: "10.0.0.0",
|
||||
gateway: "10.0.0.1",
|
||||
sshKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITEST lab@test"],
|
||||
adminUser: "testadmin",
|
||||
syslogPort: 15515,
|
||||
skipDnsmasq: true,
|
||||
skipArtifacts: true,
|
||||
fedoraMirror: "https://example.invalid/fedora",
|
||||
tftpDir: join(testDir, "tftp"),
|
||||
httpDir: join(testDir, "http"),
|
||||
stateFile: join(testDir, "state.json"),
|
||||
};
|
||||
}
|
||||
|
||||
/** Pull the base64 spec back out of the generated Python driver. */
|
||||
function decodeSpecFrom(python: string): Record<string, unknown> {
|
||||
const match = /base64\.b64decode\("([^"]+)"\)/.exec(python);
|
||||
if (!match?.[1]) throw new Error("no base64 spec found in generated driver");
|
||||
return JSON.parse(Buffer.from(match[1], "base64").toString("utf-8"));
|
||||
}
|
||||
|
||||
describe("vyos config spec", () => {
|
||||
it("puts VLANs on the bond when members are given", () => {
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "fw1",
|
||||
defaultPassword: "pw",
|
||||
spec: {
|
||||
mgmtInterface: "eth0",
|
||||
mgmtAddress: "10.0.8.2/24",
|
||||
bondMembers: ["eth2", "eth3"],
|
||||
vlans: [{ id: 10, address: "10.0.10.1/24", description: "k8s" }],
|
||||
},
|
||||
});
|
||||
|
||||
const paths = spec.sets.map((s) => s.path.join(" "));
|
||||
expect(paths).toContain("interfaces bonding bond0 mode");
|
||||
expect(paths).toContain("interfaces bonding bond0 vif 10 address");
|
||||
// VLANs must hang off the bond, not the management NIC.
|
||||
expect(paths).not.toContain("interfaces ethernet eth0 vif 10 address");
|
||||
|
||||
// Bond members are a multi-value node — appending, not replacing, is what
|
||||
// keeps the second member from overwriting the first.
|
||||
const members = spec.sets.filter(
|
||||
(s) => s.path.join(" ") === "interfaces bonding bond0 member interface",
|
||||
);
|
||||
expect(members.map((m) => m.value)).toEqual(["eth2", "eth3"]);
|
||||
expect(members.every((m) => m.replace === false)).toBe(true);
|
||||
});
|
||||
|
||||
it("falls back to VLANs on the management NIC when unbonded", () => {
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "fw2",
|
||||
defaultPassword: "pw",
|
||||
spec: { mgmtInterface: "eth1", vlans: [{ id: 20, address: "10.0.20.1/24" }] },
|
||||
});
|
||||
|
||||
const paths = spec.sets.map((s) => s.path.join(" "));
|
||||
expect(paths).toContain("interfaces ethernet eth1 vif 20 address");
|
||||
});
|
||||
|
||||
it("normalises the target disk to a full /dev path", () => {
|
||||
// find_disks() enumerates with `lsblk -Jbp`, so valid responses are full
|
||||
// paths; a bare name fails valid_responses and re-prompts forever.
|
||||
expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw", disk: "/dev/mmcblk0" }).disk)
|
||||
.toBe("/dev/mmcblk0");
|
||||
expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw", disk: "mmcblk0" }).disk)
|
||||
.toBe("/dev/mmcblk0");
|
||||
expect(buildVyosConfigSpec({ hostname: "fw3", defaultPassword: "pw" }).disk).toBe("");
|
||||
});
|
||||
|
||||
it("defaults to dhcp on eth0 and never opts into RAID", () => {
|
||||
const spec = buildVyosConfigSpec({ hostname: "fw4", defaultPassword: "pw" });
|
||||
const address = spec.sets.find(
|
||||
(s) => s.path.join(" ") === "interfaces ethernet eth0 address",
|
||||
);
|
||||
expect(address?.value).toBe("dhcp");
|
||||
// The installer's RAID prompt defaults to yes; a second disk must not
|
||||
// silently produce a mirror.
|
||||
expect(spec.raid).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("vyos routes", () => {
|
||||
let testDir: string;
|
||||
let app: FastifyInstance;
|
||||
let state: StateManager;
|
||||
const mac = "aa:bb:cc:11:22:33";
|
||||
|
||||
beforeEach(() => {
|
||||
testDir = join(tmpdir(), `bastion-vyos-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
||||
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||
|
||||
const result = createApp(createTestConfig(testDir));
|
||||
app = result.app;
|
||||
state = result.state;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await app.close();
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("dispatches a queued vyos machine to the live-boot script", async () => {
|
||||
state.update((s) => {
|
||||
s.install_queue[mac] = {
|
||||
hostname: "fw1",
|
||||
disk: "/dev/nvme0n1",
|
||||
role: "worker",
|
||||
os: "vyos-rolling",
|
||||
queued_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "GET", url: `/dispatch?mac=${mac}` });
|
||||
expect(response.statusCode).toBe(200);
|
||||
|
||||
expect(response.body).toContain("/vyos-vmlinuz");
|
||||
expect(response.body).toContain("fetch=http://10.0.0.1:0/vyos-filesystem.squashfs");
|
||||
expect(response.body).toContain(`live-config.hooks=http://10.0.0.1:0/vyos/autoinstall.sh?mac=${mac}`);
|
||||
|
||||
// `nonetworking` appears in VyOS's own PXE docs but breaks the hook fetch,
|
||||
// and console=ttyS0 costs 30s per systemd phase on boards with no UART.
|
||||
expect(response.body).not.toContain("nonetworking");
|
||||
expect(response.body).not.toContain("console=ttyS0");
|
||||
});
|
||||
|
||||
it("serves a hook that fetches and executes the install driver", async () => {
|
||||
const response = await app.inject({ method: "GET", url: `/vyos/autoinstall.sh?mac=${mac}` });
|
||||
expect(response.statusCode).toBe(200);
|
||||
expect(response.body).toContain(`/vyos/install.py?mac=${mac}`);
|
||||
expect(response.body).toContain("python3 /tmp/vyos-install.py");
|
||||
});
|
||||
|
||||
it("bakes the machine's config into the generated install driver", async () => {
|
||||
state.update((s) => {
|
||||
s.install_queue[mac] = {
|
||||
hostname: "fw1",
|
||||
disk: "/dev/nvme0n1",
|
||||
role: "worker",
|
||||
os: "vyos-rolling",
|
||||
queued_at: new Date().toISOString(),
|
||||
vyos: {
|
||||
mgmtInterface: "eth0",
|
||||
mgmtAddress: "10.0.8.2/24",
|
||||
bondMembers: ["eth2", "eth3"],
|
||||
vlans: [{ id: 10, address: "10.0.10.1/24" }],
|
||||
password: "s3cret",
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "GET", url: `/vyos/install.py?mac=${mac}` });
|
||||
expect(response.statusCode).toBe(200);
|
||||
|
||||
// Builds config from the image's own default so the vyos-config-version
|
||||
// trailer matches and first boot skips migrations.
|
||||
expect(response.body).toContain("/opt/vyatta/etc/config.boot.default");
|
||||
expect(response.body).toContain("/usr/libexec/vyos/op_mode/image_installer.py");
|
||||
// "complete" is what moves the machine out of the install queue.
|
||||
expect(response.body).toContain('report("complete"');
|
||||
|
||||
const spec = decodeSpecFrom(response.body);
|
||||
expect(spec["hostname"]).toBe("fw1");
|
||||
expect(spec["password"]).toBe("s3cret");
|
||||
expect(spec["disk"]).toBe("/dev/nvme0n1");
|
||||
|
||||
const paths = (spec["sets"] as Array<{ path: string[] }>).map((s) => s.path.join(" "));
|
||||
expect(paths).toContain("interfaces bonding bond0 vif 10 address");
|
||||
expect(paths).toContain("system host-name");
|
||||
});
|
||||
|
||||
it("falls back to the bastion default password when none is set", async () => {
|
||||
state.update((s) => {
|
||||
s.install_queue[mac] = {
|
||||
hostname: "fw9",
|
||||
disk: "",
|
||||
role: "worker",
|
||||
os: "vyos-rolling",
|
||||
queued_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
|
||||
const response = await app.inject({ method: "GET", url: `/vyos/install.py?mac=${mac}` });
|
||||
const spec = decodeSpecFrom(response.body);
|
||||
expect(spec["password"]).toBe("test-pw");
|
||||
// Empty disk means "accept the installer's first-disk default".
|
||||
expect(spec["disk"]).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("vyos hw-id pinning", () => {
|
||||
it("emits hw-id for the mgmt interface and each bond member", () => {
|
||||
// Discovery sees enp2s0/enp1s0f0np0 under Fedora, but VyOS enumerates its
|
||||
// own eth<N>. Pinning by MAC is what makes the mapping deterministic.
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "fw1",
|
||||
defaultPassword: "pw",
|
||||
spec: {
|
||||
mgmtInterface: "eth2",
|
||||
bondMembers: ["eth0", "eth1"],
|
||||
hwIds: {
|
||||
eth2: "64:62:66:25:96:47",
|
||||
eth0: "64:62:66:25:96:45",
|
||||
eth1: "64:62:66:25:96:46",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const hw = spec.sets.filter((s) => s.path[s.path.length - 1] === "hw-id");
|
||||
expect(hw.map((s) => [s.path[2], s.value])).toEqual([
|
||||
["eth2", "64:62:66:25:96:47"],
|
||||
["eth0", "64:62:66:25:96:45"],
|
||||
["eth1", "64:62:66:25:96:46"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("omits hw-id entirely when no mapping is given", () => {
|
||||
const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" });
|
||||
expect(spec.sets.some((s) => s.path.includes("hw-id"))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("vyos management VLAN", () => {
|
||||
it("puts the mgmt VLAN on the PXE port while the bond carries routed VLANs", () => {
|
||||
// Trunked PXE port: boots untagged on the VLAN the bastion serves, stays
|
||||
// reachable on the tagged management VLAN.
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "vyos001",
|
||||
defaultPassword: "pw",
|
||||
spec: {
|
||||
mgmtInterface: "eth2",
|
||||
mgmtAddress: "dhcp",
|
||||
mgmtVlan: { id: 3, address: "192.168.3.4/24", description: "kvm" },
|
||||
bondMembers: ["eth0", "eth1"],
|
||||
vlans: [{ id: 2, address: "192.168.8.2/23" }],
|
||||
},
|
||||
});
|
||||
|
||||
const paths = spec.sets.map((s) => s.path.join(" "));
|
||||
expect(paths).toContain("interfaces ethernet eth2 vif 3 address");
|
||||
expect(paths).toContain("interfaces bonding bond0 vif 2 address");
|
||||
// The mgmt VLAN must not land on the bond.
|
||||
expect(paths).not.toContain("interfaces bonding bond0 vif 3 address");
|
||||
expect(spec.tags.map((t) => t.join(" "))).toContain("interfaces ethernet eth2 vif");
|
||||
});
|
||||
});
|
||||
|
||||
describe("vyos VRRP HA", () => {
|
||||
const haSpec = {
|
||||
mgmtInterface: "eth2",
|
||||
mgmtAddress: "dhcp",
|
||||
bondMembers: ["eth0", "eth1"],
|
||||
bondAddress: "192.168.1.252/24",
|
||||
bondVrrp: "192.168.1.254/24",
|
||||
vrrpPriority: 200,
|
||||
vlans: [
|
||||
{ id: 3, address: "192.168.3.4/24", vrrp: "192.168.3.254/24" },
|
||||
{ id: 200, address: "192.168.2.252/24" }, // no VIP on this one
|
||||
],
|
||||
};
|
||||
|
||||
it("emits a vrrp group per VIP with vrid = VLAN id and dotted vif interface", () => {
|
||||
const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw", spec: haSpec });
|
||||
const paths = spec.sets.map((s) => `${s.path.join(" ")}${s.value !== undefined ? "=" + s.value : ""}`);
|
||||
|
||||
expect(paths).toContain("interfaces bonding bond0 address=192.168.1.252/24");
|
||||
// untagged bond group: vrid 1, interface bond0 itself
|
||||
expect(paths).toContain("high-availability vrrp group native interface=bond0");
|
||||
expect(paths).toContain("high-availability vrrp group native vrid=1");
|
||||
// address is a tag node -- VIP is the final path segment, no value
|
||||
expect(paths).toContain("high-availability vrrp group native address 192.168.1.254/24");
|
||||
// VLAN group: vrid = VLAN id, dotted vif
|
||||
expect(paths).toContain("high-availability vrrp group vlan3 interface=bond0.3");
|
||||
expect(paths).toContain("high-availability vrrp group vlan3 vrid=3");
|
||||
expect(paths).toContain("high-availability vrrp group vlan3 address 192.168.3.254/24");
|
||||
// VLAN without a VIP gets no group
|
||||
expect(paths.some((p) => p.includes("group vlan200"))).toBe(false);
|
||||
});
|
||||
|
||||
it("applies the box-wide priority and one sync group over all groups", () => {
|
||||
const spec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw", spec: haSpec });
|
||||
const prio = spec.sets.filter((s) => s.path[s.path.length - 1] === "priority"
|
||||
&& s.path[0] === "high-availability");
|
||||
expect(prio).toHaveLength(2);
|
||||
expect(prio.every((s) => s.value === "200")).toBe(true);
|
||||
|
||||
// sync group binds the pair: all groups fail over together
|
||||
const members = spec.sets.filter(
|
||||
(s) => s.path.join(" ") === "high-availability vrrp sync-group MAIN member",
|
||||
);
|
||||
expect(members.map((m) => m.value)).toEqual(["native", "vlan3"]);
|
||||
expect(members.every((m) => m.replace === false)).toBe(true);
|
||||
});
|
||||
|
||||
it("emits no high-availability nodes when no VIPs are given", () => {
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "fw1",
|
||||
defaultPassword: "pw",
|
||||
spec: { bondMembers: ["eth0", "eth1"], vlans: [{ id: 3, address: "192.168.3.4/24" }] },
|
||||
});
|
||||
expect(spec.sets.some((s) => s.path[0] === "high-availability")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("pickLargestInitrd", async () => {
|
||||
const { pickLargestInitrd } = await import("../src/main.js");
|
||||
|
||||
// Verbatim from `xorriso -lsl /live/` on vyos-2026.08.05-0033-rolling.
|
||||
const realListing = `total 8
|
||||
-r--r--r-- 1 0 0 22255 Aug 5 01:33 'filesystem.packages'
|
||||
-r--r--r-- 1 0 0 6 Aug 5 01:33 'filesystem.packages-remove'
|
||||
-r--r--r-- 1 0 0 541192192 Aug 5 01:33 'filesystem.squashfs'
|
||||
-r--r--r-- 1 0 0 50352547 Aug 5 01:33 'initrd.img'
|
||||
-r--r--r-- 1 0 0 50352547 Aug 5 01:33 'initrd.img-6.18.41-vyos'
|
||||
-r--r--r-- 1 0 0 20 Aug 5 01:33 'packages.txt'
|
||||
-r--r--r-- 1 0 0 9135104 Aug 2 19:54 'vmlinuz'
|
||||
-r--r--r-- 1 0 0 9135104 Aug 2 19:54 'vmlinuz-6.18.41-vyos'
|
||||
`;
|
||||
|
||||
it("picks a full-size initrd from a real nightly listing", () => {
|
||||
expect(pickLargestInitrd(realListing)).toEqual({ name: "initrd.img", size: 50352547 });
|
||||
});
|
||||
|
||||
it("ignores 0-byte decoys and symlinks (which report link size, not target size)", () => {
|
||||
const listing = `total 8
|
||||
-r--r--r-- 1 0 0 0 Aug 5 01:33 'initrd.img'
|
||||
lrwxrwxrwx 1 0 0 24 Aug 5 01:33 'initrd.img-link' -> 'initrd.img-6.18.41-vyos'
|
||||
-r--r--r-- 1 0 0 50352547 Aug 5 01:33 'initrd.img-6.18.41-vyos'
|
||||
`;
|
||||
expect(pickLargestInitrd(listing)).toEqual({ name: "initrd.img-6.18.41-vyos", size: 50352547 });
|
||||
});
|
||||
|
||||
it("returns undefined when only decoys exist", () => {
|
||||
expect(pickLargestInitrd("-r--r--r-- 1 0 0 0 Aug 5 01:33 'initrd.img'\n")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("vyos fedora-parity features", () => {
|
||||
it("computes reportAddress from a static mgmt address, empty for dhcp", () => {
|
||||
const staticSpec = buildVyosConfigSpec({
|
||||
hostname: "fw1", defaultPassword: "pw",
|
||||
spec: { mgmtAddress: "192.168.8.2/23" },
|
||||
});
|
||||
expect(staticSpec.reportAddress).toBe("192.168.8.2");
|
||||
|
||||
const dhcpSpec = buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" });
|
||||
expect(dhcpSpec.reportAddress).toBe("");
|
||||
});
|
||||
|
||||
it("defaults freshConfig off (reinstall preserves the on-disk config)", () => {
|
||||
expect(buildVyosConfigSpec({ hostname: "fw1", defaultPassword: "pw" }).freshConfig).toBe(false);
|
||||
expect(buildVyosConfigSpec({
|
||||
hostname: "fw1", defaultPassword: "pw", spec: { freshConfig: true },
|
||||
}).freshConfig).toBe(true);
|
||||
});
|
||||
|
||||
it("driver streams logs to /api/log and reports 'ready at' on completion", async () => {
|
||||
const testDir = join(tmpdir(), `bastion-vyos-parity-${Date.now()}`);
|
||||
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||
const { app: parityApp, state: parityState } = createApp(createTestConfig(testDir));
|
||||
try {
|
||||
parityState.update((s) => {
|
||||
s.install_queue["aa:bb:cc:44:55:66"] = {
|
||||
hostname: "fw9", disk: "/dev/vda", role: "vanilla",
|
||||
os: "vyos-rolling", queued_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
const response = await parityApp.inject({
|
||||
method: "GET", url: "/vyos/install.py?mac=aa:bb:cc:44:55:66",
|
||||
});
|
||||
expect(response.body).toContain("/api/log");
|
||||
expect(response.body).toContain('"lines": batch');
|
||||
expect(response.body).toContain('report("complete", "ready at %s"');
|
||||
expect(response.body).toContain("ensure_network_boot_first");
|
||||
expect(response.body).toContain("lab-provisioned");
|
||||
expect(response.body).toContain('ROLE = "vanilla"');
|
||||
} finally {
|
||||
await parityApp.close();
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("complete with 'ready at' records installed.ip for a vyos machine", async () => {
|
||||
const testDir = join(tmpdir(), `bastion-vyos-complete-${Date.now()}`);
|
||||
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||
const { app: cApp, state: cState } = createApp(createTestConfig(testDir));
|
||||
try {
|
||||
const mac2 = "aa:bb:cc:77:88:99";
|
||||
cState.update((s) => {
|
||||
s.install_queue[mac2] = {
|
||||
hostname: "fw1", disk: "/dev/vda", role: "vanilla",
|
||||
os: "vyos-rolling", queued_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
const response = await cApp.inject({
|
||||
method: "POST", url: "/api/progress",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ mac: mac2, stage: "complete", detail: "ready at 192.168.8.2" }),
|
||||
});
|
||||
expect(response.statusCode).toBe(200);
|
||||
const installed = cState.load().installed[mac2];
|
||||
expect(installed?.ip).toBe("192.168.8.2");
|
||||
expect(installed?.os).toBe("vyos-rolling");
|
||||
} finally {
|
||||
await cApp.close();
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("vyos installer prompt coverage", () => {
|
||||
// Every interactive prompt image_installer.py can emit, copied verbatim from
|
||||
// the MSG_* constants (including the reinstall-only search_previous_installation
|
||||
// ones). An unanswered prompt does not fail loudly -- the installer simply
|
||||
// blocks on stdin until the driver's stall timeout, which is how the reinstall
|
||||
// path silently hung for 15 minutes in the VM test.
|
||||
const PROMPTS: Record<string, string> = {
|
||||
continue: "Would you like to continue? [y/N] ",
|
||||
imageName: "What would you like to name this image? (Default: 1.5-rolling) ",
|
||||
password: 'Please enter a password for the "vyos" user: ',
|
||||
passwordConfirm: 'Please confirm password for the "vyos" user: ',
|
||||
console: "What console should be used by default? (K: KVM, S: Serial)? (Default: K) ",
|
||||
raidConfigure: "Would you like to configure RAID-1 mirroring? [Y/n] ",
|
||||
raidFoundDisks: "Would you like to configure RAID-1 mirroring on them? [Y/n] ",
|
||||
raidChooseDisks: "Would you like to choose two disks for RAID-1 mirroring? [Y/n] ",
|
||||
diskSelect: "Which one should be used for installation? (Default: /dev/vda) ",
|
||||
diskConfirm: "Installation will delete all data on the drive. Continue? [y/N] ",
|
||||
raidConfirm: "Installation will delete all data on both drives. Continue? [y/N] ",
|
||||
rootSizeAll: "Would you like to use all the free space on the drive? [Y/n] ",
|
||||
bootConfig: "Which file would you like as boot config? ",
|
||||
copyData: "Would you like to copy data to the new image? [Y/n] ",
|
||||
chooseCopyData: "From which image would you like to save config information? ",
|
||||
copyEncData: "Would you like to copy the encrypted config to the new image? [Y/n] ",
|
||||
chooseCopyEncData: "From which image would you like to copy the encrypted config? ",
|
||||
};
|
||||
|
||||
it("answers every installer prompt exactly once", () => {
|
||||
const { execFileSync } = require("node:child_process") as typeof import("node:child_process");
|
||||
const { writeFileSync, unlinkSync, mkdtempSync } = require("node:fs") as typeof import("node:fs");
|
||||
|
||||
// Skip cleanly where python3 is unavailable (same spirit as the
|
||||
// ksvalidator-backed kickstart test).
|
||||
try {
|
||||
execFileSync("python3", ["--version"], { stdio: "pipe" });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
const spec = buildVyosConfigSpec({
|
||||
hostname: "fw1", defaultPassword: "pw", disk: "/dev/vda",
|
||||
});
|
||||
const driver = renderVyosInstallPy({
|
||||
spec, mac: "aa:bb:cc:11:22:33", serverIp: "10.0.0.1", httpPort: 8080, role: "vanilla",
|
||||
});
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), "vyos-rules-"));
|
||||
const driverPath = join(dir, "driver.py");
|
||||
const checkPath = join(dir, "check.py");
|
||||
writeFileSync(driverPath, driver);
|
||||
writeFileSync(checkPath, `
|
||||
import importlib.util, json, sys
|
||||
spec = importlib.util.spec_from_file_location("drv", ${JSON.stringify(driverPath)})
|
||||
drv = importlib.util.module_from_spec(spec); spec.loader.exec_module(drv)
|
||||
rules = drv.build_rules()
|
||||
prompts = json.loads(sys.argv[1])
|
||||
out = {}
|
||||
for label, text in prompts.items():
|
||||
out[label] = len([r for p, r in rules if p.search(text.encode())])
|
||||
print(json.dumps(out))
|
||||
`);
|
||||
|
||||
try {
|
||||
const stdout = execFileSync("python3", [checkPath, JSON.stringify(PROMPTS)], {
|
||||
encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
const counts = JSON.parse(stdout) as Record<string, number>;
|
||||
const unanswered = Object.entries(counts).filter(([, n]) => n !== 1);
|
||||
expect(unanswered).toEqual([]);
|
||||
} finally {
|
||||
try { unlinkSync(driverPath); unlinkSync(checkPath); } catch { /* best effort */ }
|
||||
try { rmSync(dir, { recursive: true, force: true }); } catch { /* best effort */ }
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("vyos boot NIC pinning", () => {
|
||||
it("pins the boot interface by MAC via BOOTIF", async () => {
|
||||
// Without this, live-boot picks the first *connected* NIC. On the VP2440
|
||||
// the SFP+ pair links before the copper PXE port, so live-boot tried the
|
||||
// fiber ports (no DHCP), timed out 15s each, and failed with "Unable to
|
||||
// find a live file system on the network".
|
||||
const testDir = join(tmpdir(), `bastion-vyos-bootif-${Date.now()}`);
|
||||
mkdirSync(join(testDir, "http"), { recursive: true });
|
||||
mkdirSync(join(testDir, "tftp"), { recursive: true });
|
||||
const { app: a, state: st } = createApp(createTestConfig(testDir));
|
||||
try {
|
||||
const m = "64:62:66:25:96:47";
|
||||
st.update((s) => {
|
||||
s.install_queue[m] = {
|
||||
hostname: "vyos001", disk: "/dev/mmcblk0", role: "vanilla",
|
||||
os: "vyos-rolling", queued_at: new Date().toISOString(),
|
||||
};
|
||||
});
|
||||
const res = await a.inject({ method: "GET", url: `/dispatch?mac=${m}` });
|
||||
// live-boot's Device_from_bootif() expects 01-<mac with dashes>
|
||||
expect(res.body).toContain("BOOTIF=01-64-62-66-25-96-47");
|
||||
// and it must be on the kernel line, before fetch= is attempted
|
||||
const kernelLine = res.body.split("\n").find((l) => l.startsWith("kernel "));
|
||||
expect(kernelLine).toContain("BOOTIF=01-64-62-66-25-96-47");
|
||||
expect(kernelLine).toContain("fetch=");
|
||||
} finally {
|
||||
await a.close();
|
||||
rmSync(testDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -90,29 +90,12 @@ export class LabdClient {
|
||||
|
||||
async installMachine(opts: {
|
||||
mac: string; hostname: string; disk?: string; role?: string; os?: string;
|
||||
vyos?: import("@lab/shared").VyosInstallSpec;
|
||||
}): Promise<{ status: string; data?: unknown; error?: string }> {
|
||||
return this.request("POST", "/api/machines/install", { body: opts });
|
||||
}
|
||||
|
||||
async registerMachine(opts: {
|
||||
mac: string; hostname: string; role?: string; ip?: string;
|
||||
}): Promise<{ status: string; data?: unknown; error?: string }> {
|
||||
return this.request("POST", "/api/machines/register", { body: opts });
|
||||
}
|
||||
|
||||
async debugMachine(mac: string, opts?: { pxeBoot?: boolean }): Promise<{ status: string; data?: { mac: string; hostname: string }; error?: string }> {
|
||||
return this.request("POST", "/api/machines/debug", { body: { mac, pxeBoot: opts?.pxeBoot } });
|
||||
}
|
||||
|
||||
async discoverMachine(data: {
|
||||
mac: string; product?: string; board?: string; serial?: string;
|
||||
manufacturer?: string; cpu_model?: string; cpu_cores?: number;
|
||||
memory_gb?: number; arch?: string;
|
||||
disks?: Array<{ name: string; size_gb: number; model: string }>;
|
||||
nics?: Array<{ name: string; mac: string; state: string }>;
|
||||
}): Promise<{ status: string; error?: string }> {
|
||||
return this.request("POST", "/api/machines/discover", { body: data });
|
||||
async debugMachine(mac: string, opts?: { sshd?: boolean }): Promise<{ status: string; data?: { mac: string; hostname: string }; error?: string }> {
|
||||
return this.request("POST", "/api/machines/debug", { body: { mac, sshd: opts?.sshd } });
|
||||
}
|
||||
|
||||
async forgetMachine(mac: string): Promise<{ status: string }> {
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
// CLI command: labctl app k3s install/health <target>
|
||||
// Install or check k3s on a target machine via SSH.
|
||||
|
||||
import { existsSync, writeFileSync, mkdirSync } from "node:fs";
|
||||
import { existsSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { execSync } from "node:child_process";
|
||||
import type { Command } from "commander";
|
||||
import type { BastionState } from "@lab/shared";
|
||||
import { K3sModule, sshExec } from "@lab/modules";
|
||||
@@ -70,7 +69,7 @@ export function registerAppCommand(program: Command): void {
|
||||
.command("install <target>")
|
||||
.description("Install k3s on a target machine (hostname, IP, or MAC)")
|
||||
.option("--role <role>", "k3s role: infra (server) or worker (agent)", "infra")
|
||||
.option("--user <user>", "SSH user", "root")
|
||||
.option("--user <user>", "SSH user", "michal")
|
||||
.option("--k3s-server <url>", "k3s server URL (required for worker role)")
|
||||
.option("--k3s-token <token>", "k3s join token (required for worker role)")
|
||||
.action(async (target: string, opts: {
|
||||
@@ -164,7 +163,7 @@ export function registerAppCommand(program: Command): void {
|
||||
k3sCmd
|
||||
.command("health [target]")
|
||||
.description("Check k3s health (all hosts if no target given)")
|
||||
.option("--user <user>", "SSH user", "root")
|
||||
.option("--user <user>", "SSH user", "michal")
|
||||
.action(async (target: string | undefined, opts: { user: string }) => {
|
||||
const sshKey = findSshKey();
|
||||
|
||||
@@ -304,7 +303,7 @@ export function registerAppCommand(program: Command): void {
|
||||
k3sCmd
|
||||
.command("list")
|
||||
.description("List installed machines and their k3s status")
|
||||
.option("--user <user>", "SSH user", "root")
|
||||
.option("--user <user>", "SSH user", "michal")
|
||||
.action(async (opts: { user: string }) => {
|
||||
let state: BastionState;
|
||||
try {
|
||||
@@ -401,88 +400,4 @@ export function registerAppCommand(program: Command): void {
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
k3sCmd
|
||||
.command("kubeconfig <target>")
|
||||
.description("Fetch kubeconfig from a target and merge into ~/.kube/config")
|
||||
.option("--user <user>", "SSH user", "root")
|
||||
.option("--context <name>", "Context name (defaults to hostname)")
|
||||
.option("--print", "Print kubeconfig to stdout instead of merging")
|
||||
.action(async (target: string, opts: {
|
||||
user: string;
|
||||
context?: string;
|
||||
print?: boolean;
|
||||
}) => {
|
||||
const state = await fetchState();
|
||||
const resolved = resolveTarget(target, state);
|
||||
|
||||
if (!resolved) {
|
||||
console.error(`Cannot resolve target: ${target}`);
|
||||
console.error("Provide an IP address, hostname, or MAC of an installed machine.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const sshKey = findSshKey();
|
||||
|
||||
// Fetch kubeconfig via SSH
|
||||
let raw: string;
|
||||
try {
|
||||
const result = await sshExec(resolved.ip, opts.user, "cat /etc/rancher/k3s/k3s.yaml", {
|
||||
...(sshKey ? { keyPath: sshKey } : {}),
|
||||
timeoutMs: 10_000,
|
||||
});
|
||||
raw = result.stdout;
|
||||
} catch (err) {
|
||||
console.error(`Failed to fetch kubeconfig: ${err instanceof Error ? err.message : String(err)}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const contextName = opts.context ?? resolved.hostname;
|
||||
|
||||
// Rewrite: replace 127.0.0.1 with actual IP, rename cluster/user/context
|
||||
const rewritten = raw
|
||||
.replace(/server:\s*https:\/\/127\.0\.0\.1:/, `server: https://${resolved.ip}:`)
|
||||
.replace(/name:\s*default/g, `name: ${contextName}`)
|
||||
.replace(/cluster:\s*default/g, `cluster: ${contextName}`)
|
||||
.replace(/user:\s*default/g, `user: ${contextName}`)
|
||||
.replace(/current-context:\s*default/, `current-context: ${contextName}`);
|
||||
|
||||
if (opts.print) {
|
||||
process.stdout.write(rewritten);
|
||||
return;
|
||||
}
|
||||
|
||||
// Merge into ~/.kube/config using kubectl
|
||||
const kubeDir = join(homedir(), ".kube");
|
||||
mkdirSync(kubeDir, { recursive: true });
|
||||
const mainConfig = join(kubeDir, "config");
|
||||
const tmpFile = join(kubeDir, `.labctl-${contextName}.tmp`);
|
||||
|
||||
writeFileSync(tmpFile, rewritten, { mode: 0o600 });
|
||||
|
||||
try {
|
||||
if (existsSync(mainConfig)) {
|
||||
const merged = execSync(
|
||||
`KUBECONFIG="${mainConfig}:${tmpFile}" kubectl config view --flatten`,
|
||||
{ encoding: "utf-8" },
|
||||
);
|
||||
writeFileSync(mainConfig, merged, { mode: 0o600 });
|
||||
} else {
|
||||
writeFileSync(mainConfig, rewritten, { mode: 0o600 });
|
||||
}
|
||||
|
||||
// Set current context
|
||||
execSync(`kubectl config use-context ${contextName}`, { stdio: "pipe" });
|
||||
|
||||
console.log(`Merged kubeconfig for ${contextName} (${resolved.ip})`);
|
||||
console.log(`Context set to: ${contextName}`);
|
||||
console.log(`\nSwitch contexts: kubectl config use-context <name>`);
|
||||
} catch (err) {
|
||||
console.error(`Failed to merge kubeconfig: ${err instanceof Error ? err.message : String(err)}`);
|
||||
console.error(`Standalone config saved at: ${tmpFile}`);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
try { const { unlinkSync } = await import("node:fs"); unlinkSync(tmpFile); } catch { /* ignore */ }
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
// CLI command: provision asahi
|
||||
// Prints the curl command to run on the Mac Studio (macOS) to install
|
||||
// Fedora Asahi Remix with lab LVM layout.
|
||||
|
||||
import type { Command } from "commander";
|
||||
import { getLabdClient } from "../api/config.js";
|
||||
|
||||
export function registerAsahiCommand(parent: Command): void {
|
||||
parent
|
||||
.command("asahi")
|
||||
.description("Show instructions to provision an Apple Silicon Mac with Asahi Linux")
|
||||
.action(async () => {
|
||||
// Try to get bastion info to determine the correct URL
|
||||
let bastionUrl = "";
|
||||
try {
|
||||
const bastions = await getLabdClient().getBastions();
|
||||
const online = bastions.find(b => b.status === "online");
|
||||
if (online) {
|
||||
bastionUrl = `http://${online.serverIp}:8080`;
|
||||
}
|
||||
} catch { /* labd not reachable */ }
|
||||
|
||||
if (!bastionUrl) {
|
||||
// Fall back to config
|
||||
const { loadConfig } = await import("../config/index.js");
|
||||
const config = loadConfig();
|
||||
bastionUrl = config.labdUrl ?? "http://<bastion-ip>:8080";
|
||||
// Convert labd URL to bastion URL (labd is on different port/host)
|
||||
bastionUrl = bastionUrl.replace(/:\d+$/, ":8080");
|
||||
}
|
||||
|
||||
const BOLD = "\x1b[1m";
|
||||
const CYAN = "\x1b[36m";
|
||||
const DIM = "\x1b[2m";
|
||||
const RESET = "\x1b[0m";
|
||||
|
||||
console.log("");
|
||||
console.log(`${BOLD} Asahi Linux Provisioning${RESET}`);
|
||||
console.log(`${DIM} For Apple Silicon Macs (Mac Studio, MacBook, etc.)${RESET}`);
|
||||
console.log("");
|
||||
console.log(` Run this command ${BOLD}on the Mac${RESET} (from macOS Terminal):`);
|
||||
console.log("");
|
||||
console.log(` ${CYAN}${BOLD}curl ${bastionUrl}/asahi | sh${RESET}`);
|
||||
console.log("");
|
||||
console.log(` The installer will ask a few interactive questions:`);
|
||||
console.log(` ${BOLD}1.${RESET} Action: press ${BOLD}r${RESET} to resize macOS`);
|
||||
console.log(` ${BOLD}2.${RESET} How much space for Linux: choose maximum`);
|
||||
console.log(` ${BOLD}3.${RESET} Confirm the resize operation`);
|
||||
console.log(` ${BOLD}4.${RESET} macOS password for firmware authentication`);
|
||||
console.log("");
|
||||
console.log(` After that, everything is automatic:`);
|
||||
console.log(` - Asahi boot infrastructure (m1n1 + U-Boot)`);
|
||||
console.log(` - Fedora Asahi Remix root partition`);
|
||||
console.log(` - LVM data partition (remaining space)`);
|
||||
console.log("");
|
||||
console.log(` On first boot, LVM volumes are created automatically:`);
|
||||
console.log(` ${DIM}labvg/swap (27GB), labvg/var (100GB), labvg/varlog (10GB),`);
|
||||
console.log(` labvg/home (10GB), labvg/srv (20GB), labvg/rancher (20GB),`);
|
||||
console.log(` labvg/longhorn (remaining space)${RESET}`);
|
||||
console.log("");
|
||||
console.log(` After first boot, SSH in and run the firstboot script:`);
|
||||
console.log(` ${BOLD}ssh root@<ip> 'curl -sf ${bastionUrl}/asahi/firstboot.sh | bash'${RESET}`);
|
||||
console.log("");
|
||||
console.log(` This sets up LVM, detects hostname/MAC, and self-registers.`);
|
||||
console.log(` Then install k3s:`);
|
||||
console.log(` ${BOLD}labctl app k3s install <hostname> --role infra${RESET}`);
|
||||
console.log("");
|
||||
});
|
||||
}
|
||||
@@ -48,9 +48,9 @@ export function registerDebugCommand(parent: Command): void {
|
||||
parent
|
||||
.command("debug <target>")
|
||||
.description("PXE boot into Fedora rescue mode for debugging (target: hostname, MAC, or IP)")
|
||||
.option("--pxe-boot", "Boot installed system via PXE (kernel+initrd from network, root from NVMe)")
|
||||
.option("--sshd", "Start SSH + nc listener automatically, report IP to bastion")
|
||||
.showHelpAfterError(true)
|
||||
.action(async (target: string, opts: { pxeBoot?: boolean }) => {
|
||||
.action(async (target: string, opts: { sshd?: boolean }) => {
|
||||
const client = getLabdClient();
|
||||
|
||||
// Resolve target from labd aggregated state
|
||||
@@ -74,7 +74,7 @@ export function registerDebugCommand(parent: Command): void {
|
||||
console.log(`Queuing debug mode for ${hostname} (${mac})...`);
|
||||
|
||||
try {
|
||||
const result = await client.debugMachine(mac, { pxeBoot: opts.pxeBoot === true });
|
||||
const result = await client.debugMachine(mac, { sshd: opts.sshd });
|
||||
if (result.error) {
|
||||
console.error(`Failed: ${result.error}`);
|
||||
process.exit(1);
|
||||
@@ -103,7 +103,6 @@ export function registerDebugCommand(parent: Command): void {
|
||||
|
||||
const sshArgs = [
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "ConnectTimeout=10",
|
||||
...(sshKey !== undefined ? ["-i", sshKey] : []),
|
||||
`${effectiveUser}@${ip}`,
|
||||
@@ -118,39 +117,38 @@ export function registerDebugCommand(parent: Command): void {
|
||||
}
|
||||
}
|
||||
|
||||
// Determine bastion URL from labd config for the setup script URL
|
||||
const bastionUrl = process.env["LABD_URL"]
|
||||
? process.env["LABD_URL"].replace(/\/ws\/bastion$/, "").replace(/^wss?:/, "http:")
|
||||
: "http://<bastion-ip>:8080";
|
||||
|
||||
console.log(`
|
||||
Debug mode queued for ${hostname} (${mac}).
|
||||
Reboot the machine to enter Fedora rescue mode.
|
||||
|
||||
SSH access (started by Anaconda):
|
||||
ssh root@<ip> (password: debug)
|
||||
|
||||
For nc remote shell, run from rescue shell:
|
||||
curl ${bastionUrl}/debug-setup.sh | bash
|
||||
|
||||
Once in rescue shell:
|
||||
|
||||
# Activate LVM and mount installed system
|
||||
vgchange -ay
|
||||
mkdir -p /mnt/sysroot
|
||||
mount /dev/<vg>/root /mnt/sysroot
|
||||
cat /mnt/sysroot/etc/fstab
|
||||
mount /dev/<vg>/var /mnt/sysroot/var
|
||||
mount /dev/<vg>/home /mnt/sysroot/home
|
||||
# Activate LVM
|
||||
vgchange -ay labvg
|
||||
|
||||
# Boot installed system in a container
|
||||
# Mount root + other volumes
|
||||
mkdir -p /mnt/sysroot
|
||||
mount /dev/labvg/root /mnt/sysroot
|
||||
cat /mnt/sysroot/etc/fstab # check what else to mount
|
||||
mount /dev/labvg/var /mnt/sysroot/var
|
||||
mount /dev/labvg/home /mnt/sysroot/home
|
||||
|
||||
# Boot the installed system in a container
|
||||
/mnt/sysroot/usr/bin/systemd-nspawn -D /mnt/sysroot --boot
|
||||
|
||||
# Or chroot for quick fixes
|
||||
# Or just chroot for quick fixes
|
||||
mount --bind /dev /mnt/sysroot/dev
|
||||
mount --bind /proc /mnt/sysroot/proc
|
||||
mount --bind /sys /mnt/sysroot/sys
|
||||
chroot /mnt/sysroot
|
||||
|
||||
# Check initramfs size
|
||||
ls -lh /mnt/sysroot/boot/initramfs-*.img
|
||||
|
||||
# Rebuild initramfs without amdgpu
|
||||
chroot /mnt/sysroot
|
||||
echo 'omit_drivers+=" amdgpu "' > /etc/dracut.conf.d/omit-amdgpu.conf
|
||||
dracut -f --regenerate-all
|
||||
`);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,61 +1,10 @@
|
||||
// CLI command: provision install
|
||||
// Queue a discovered machine for OS installation via labd.
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { Command, Option, InvalidArgumentError } from "commander";
|
||||
import { Command, Option } from "commander";
|
||||
import { isValidOsId, SUPPORTED_OS, SUPPORTED_ROLES, ROLE_REGISTRY } from "@lab/shared";
|
||||
import type { VyosBundle, VyosInstallSpec, VyosVlanSpec } from "@lab/shared";
|
||||
import { getLabdClient } from "../api/config.js";
|
||||
|
||||
/**
|
||||
* Load one router's config out of a Pulumi-rendered bundle.
|
||||
*
|
||||
* The bundle is produced by `kubernetes-deployment` (npm run vyos:bundle) and
|
||||
* holds every router it manages, keyed by name. Selecting by hostname here is
|
||||
* what keeps bring-up and `pulumi up` describing the same box: labctl replays
|
||||
* the declared config rather than deriving its own.
|
||||
*/
|
||||
export function loadVyosBundle(path: string, hostname: string): VyosBundle {
|
||||
let parsed: { version?: number; routers?: Record<string, VyosBundle> };
|
||||
try {
|
||||
parsed = JSON.parse(readFileSync(path, "utf8"));
|
||||
} catch (e) {
|
||||
throw new InvalidArgumentError(`Cannot read VyOS bundle ${path}: ${(e as Error).message}`);
|
||||
}
|
||||
if (parsed.version !== 1) {
|
||||
throw new InvalidArgumentError(
|
||||
`VyOS bundle ${path} has version ${parsed.version ?? "<none>"}; this labctl understands 1`,
|
||||
);
|
||||
}
|
||||
const router = parsed.routers?.[hostname];
|
||||
if (router === undefined) {
|
||||
const known = Object.keys(parsed.routers ?? {}).join(", ") || "<none>";
|
||||
throw new InvalidArgumentError(
|
||||
`VyOS bundle ${path} has no entry for "${hostname}" (has: ${known})`,
|
||||
);
|
||||
}
|
||||
return router;
|
||||
}
|
||||
|
||||
/** Parse a repeated --vlan flag: "<id>:<cidr>[:<description>]". */
|
||||
export function parseVlan(value: string, previous: VyosVlanSpec[] = []): VyosVlanSpec[] {
|
||||
const parts = value.split(":");
|
||||
const id = Number(parts[0]);
|
||||
const address = parts[1] ?? "";
|
||||
// InvalidArgumentError makes commander print a clean message instead of
|
||||
// dumping a stack trace at the operator.
|
||||
if (!Number.isInteger(id) || id < 1 || id > 4094) {
|
||||
throw new InvalidArgumentError(`Invalid VLAN id in "${value}" (expected 1-4094)`);
|
||||
}
|
||||
if (!address.includes("/")) {
|
||||
throw new InvalidArgumentError(
|
||||
`Invalid VLAN address in "${value}" (expected CIDR, e.g. 10.0.10.1/24)`,
|
||||
);
|
||||
}
|
||||
const description = parts.slice(2).join(":");
|
||||
return [...previous, { id, address, ...(description ? { description } : {}) }];
|
||||
}
|
||||
|
||||
function roleTable(): string {
|
||||
const lines: string[] = ["", "Available roles:"];
|
||||
for (const r of ROLE_REGISTRY) {
|
||||
@@ -66,38 +15,6 @@ function roleTable(): string {
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
/** Parse a repeated --vlan-vip flag: "<id>:<cidr>" — VRRP VIP for a --vlan entry. */
|
||||
export function parseVlanVip(
|
||||
value: string,
|
||||
previous: Record<number, string> = {},
|
||||
): Record<number, string> {
|
||||
const index = value.indexOf(":");
|
||||
const id = Number(index === -1 ? Number.NaN : value.slice(0, index));
|
||||
const cidr = index === -1 ? "" : value.slice(index + 1).trim();
|
||||
if (!Number.isInteger(id) || id < 1 || id > 4094 || !cidr.includes("/")) {
|
||||
throw new InvalidArgumentError(
|
||||
`Invalid VLAN VIP "${value}" (expected <id>:<cidr>, e.g. 3:192.168.3.254/24)`,
|
||||
);
|
||||
}
|
||||
return { ...previous, [id]: cidr };
|
||||
}
|
||||
|
||||
/** Parse a repeated --vyos-hwid flag: "<iface>=<mac>". */
|
||||
export function parseHwId(
|
||||
value: string,
|
||||
previous: Record<string, string> = {},
|
||||
): Record<string, string> {
|
||||
const index = value.indexOf("=");
|
||||
const iface = index === -1 ? "" : value.slice(0, index).trim();
|
||||
const mac = index === -1 ? "" : value.slice(index + 1).trim().toLowerCase();
|
||||
if (iface === "" || !/^([0-9a-f]{2}:){5}[0-9a-f]{2}$/.test(mac)) {
|
||||
throw new InvalidArgumentError(
|
||||
`Invalid hw-id "${value}" (expected <iface>=<mac>, e.g. eth2=64:62:66:25:96:47)`,
|
||||
);
|
||||
}
|
||||
return { ...previous, [iface]: mac };
|
||||
}
|
||||
|
||||
export function registerInstallCommand(parent: Command): void {
|
||||
parent
|
||||
.command("install <mac> <hostname>")
|
||||
@@ -107,51 +24,10 @@ export function registerInstallCommand(parent: Command): void {
|
||||
.addOption(new Option("--role <role>", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker"))
|
||||
.addOption(new Option("--os <os>", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43"))
|
||||
.option("--disk <device>", "Target disk device (auto-detect if omitted)")
|
||||
.option("--vyos-mgmt <iface>", "VyOS: untagged interface the machine PXE boots from (default eth0)")
|
||||
.option("--vyos-mgmt-address <addr>", "VyOS: CIDR for the management interface, or 'dhcp' (default dhcp)")
|
||||
.option("--vyos-bond <ifaces>", "VyOS: comma-separated LACP bond members (must exclude the PXE NIC)")
|
||||
.option("--vyos-bond-address <cidr>", "VyOS: address on the untagged bond (trunk native VLAN)")
|
||||
.option("--vyos-bond-vrrp <cidr>", "VyOS: VRRP VIP floated on the untagged bond")
|
||||
.option("--vlan-vip <id:cidr>", "VyOS: VRRP VIP for a --vlan entry (repeatable)", parseVlanVip)
|
||||
.option("--vyos-vrrp-priority <n>", "VyOS: VRRP priority for all groups on this box (higher = master)")
|
||||
.option("--vyos-mgmt-vlan <id:cidr[:desc]>", "VyOS: tagged management VLAN on the PXE port")
|
||||
.option("--vlan <id:cidr[:desc]>", "VyOS: tagged VLAN sub-interface on the bond (repeatable)", parseVlan)
|
||||
.option("--vyos-password <password>", "VyOS: password for the 'vyos' user")
|
||||
.option("--vyos-hwid <iface=mac>", "VyOS: pin an interface name to a MAC via hw-id (repeatable)", parseHwId)
|
||||
.option("--vyos-fresh-config", "VyOS: on reinstall, overwrite the preserved config with the generated one")
|
||||
.option(
|
||||
"--vyos-bundle <path>",
|
||||
"VyOS: apply a Pulumi-rendered bundle verbatim (kubernetes-deployment/infra/vyos/vyos-bundle.json). " +
|
||||
"Replaces the derived --vyos-bond/--vlan/... config; secret values are left unset for `pulumi up`.",
|
||||
)
|
||||
.option(
|
||||
"--vyos-api-key <key>",
|
||||
"VyOS: enable the HTTP API with this key so Pulumi can manage the router from first boot",
|
||||
)
|
||||
.option(
|
||||
"--vyos-api-listen <addr>",
|
||||
"VyOS: address the HTTP API binds to (default: the static management address). " +
|
||||
"Required when management is DHCP; the API is never bound to all interfaces.",
|
||||
)
|
||||
.action(async (mac: string, hostname: string, opts: {
|
||||
role: string;
|
||||
os: string;
|
||||
disk?: string;
|
||||
vyosMgmt?: string;
|
||||
vyosMgmtAddress?: string;
|
||||
vyosBond?: string;
|
||||
vyosBondAddress?: string;
|
||||
vyosBondVrrp?: string;
|
||||
vlan?: VyosVlanSpec[];
|
||||
vlanVip?: Record<number, string>;
|
||||
vyosVrrpPriority?: string;
|
||||
vyosMgmtVlan?: string;
|
||||
vyosPassword?: string;
|
||||
vyosHwid?: Record<string, string>;
|
||||
vyosFreshConfig?: boolean;
|
||||
vyosBundle?: string;
|
||||
vyosApiKey?: string;
|
||||
vyosApiListen?: string;
|
||||
}) => {
|
||||
if (!isValidOsId(opts.os)) {
|
||||
console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`);
|
||||
@@ -163,89 +39,6 @@ export function registerInstallCommand(parent: Command): void {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const bondMembers = opts.vyosBond !== undefined && opts.vyosBond !== ""
|
||||
? opts.vyosBond.split(",").map((s) => s.trim()).filter((s) => s.length > 0)
|
||||
: [];
|
||||
|
||||
// Attach --vlan-vip entries to their --vlan definitions. A VIP for a VLAN
|
||||
// that was never defined is a typo that would otherwise vanish silently.
|
||||
const vips = opts.vlanVip ?? {};
|
||||
const vlans = (opts.vlan ?? []).map((v) =>
|
||||
vips[v.id] !== undefined ? { ...v, vrrp: vips[v.id] as string } : v,
|
||||
);
|
||||
for (const id of Object.keys(vips)) {
|
||||
if (!vlans.some((v) => String(v.id) === id)) {
|
||||
console.error(`--vlan-vip ${id}:... has no matching --vlan ${id}:... entry`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
const vrrpPriority = opts.vyosVrrpPriority !== undefined && opts.vyosVrrpPriority !== ""
|
||||
? Number(opts.vyosVrrpPriority)
|
||||
: undefined;
|
||||
if (vrrpPriority !== undefined
|
||||
&& (!Number.isInteger(vrrpPriority) || vrrpPriority < 1 || vrrpPriority > 255)) {
|
||||
console.error(`--vyos-vrrp-priority must be an integer 1-255 (got ${opts.vyosVrrpPriority})`);
|
||||
process.exit(1);
|
||||
}
|
||||
const vyos: VyosInstallSpec = {
|
||||
...(opts.vyosMgmt !== undefined && opts.vyosMgmt !== ""
|
||||
? { mgmtInterface: opts.vyosMgmt } : {}),
|
||||
...(opts.vyosMgmtAddress !== undefined && opts.vyosMgmtAddress !== ""
|
||||
? { mgmtAddress: opts.vyosMgmtAddress } : {}),
|
||||
...(bondMembers.length > 0 ? { bondMembers } : {}),
|
||||
...(opts.vyosBondAddress !== undefined && opts.vyosBondAddress !== ""
|
||||
? { bondAddress: opts.vyosBondAddress } : {}),
|
||||
...(opts.vyosBondVrrp !== undefined && opts.vyosBondVrrp !== ""
|
||||
? { bondVrrp: opts.vyosBondVrrp } : {}),
|
||||
...(vrrpPriority !== undefined ? { vrrpPriority } : {}),
|
||||
...(vlans.length > 0 ? { vlans } : {}),
|
||||
...(opts.vyosPassword !== undefined && opts.vyosPassword !== ""
|
||||
? { password: opts.vyosPassword } : {}),
|
||||
...(opts.vyosHwid !== undefined && Object.keys(opts.vyosHwid).length > 0
|
||||
? { hwIds: opts.vyosHwid } : {}),
|
||||
...(opts.vyosMgmtVlan !== undefined && opts.vyosMgmtVlan !== ""
|
||||
? { mgmtVlan: parseVlan(opts.vyosMgmtVlan)[0] as VyosVlanSpec } : {}),
|
||||
...(opts.vyosFreshConfig === true ? { freshConfig: true } : {}),
|
||||
...(opts.vyosBundle !== undefined && opts.vyosBundle !== ""
|
||||
? { bundle: loadVyosBundle(opts.vyosBundle, hostname) } : {}),
|
||||
...(opts.vyosApiKey !== undefined && opts.vyosApiKey !== ""
|
||||
? { apiKey: opts.vyosApiKey } : {}),
|
||||
...(opts.vyosApiListen !== undefined && opts.vyosApiListen !== ""
|
||||
? { apiListenAddress: opts.vyosApiListen } : {}),
|
||||
};
|
||||
const hasVyosOptions = Object.keys(vyos).length > 0;
|
||||
|
||||
// A bundle already describes the whole router. Accepting derived topology
|
||||
// flags alongside it would silently discard them (the bundle wins in
|
||||
// buildVyosConfigSpec), so say so rather than appear to honour both.
|
||||
if (vyos.bundle !== undefined) {
|
||||
const derived = ["mgmtInterface", "mgmtAddress", "bondMembers", "bondAddress",
|
||||
"bondVrrp", "vrrpPriority", "vlans", "mgmtVlan"] as const;
|
||||
const conflicting = derived.filter((k) => vyos[k] !== undefined);
|
||||
if (conflicting.length > 0) {
|
||||
console.error(
|
||||
`--vyos-bundle describes the whole router; these would be ignored: ${conflicting.join(", ")}`,
|
||||
);
|
||||
console.error("Remove them, or change the bundle in kubernetes-deployment and re-render.");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (hasVyosOptions && !opts.os.startsWith("vyos")) {
|
||||
console.error(`VyOS options require --os vyos-rolling (got --os ${opts.os})`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Firmware PXE cannot run over LACP, so the NIC that boots the installer
|
||||
// must stay out of the bond — otherwise the next reinstall has no path in.
|
||||
const mgmt = vyos.mgmtInterface ?? "eth0";
|
||||
if (bondMembers.includes(mgmt)) {
|
||||
console.error(`--vyos-bond must not include the PXE/management interface "${mgmt}"`);
|
||||
console.error("PXE cannot boot over an LACP bond; keep that NIC unbonded.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await getLabdClient().installMachine({
|
||||
mac,
|
||||
@@ -253,14 +46,11 @@ export function registerInstallCommand(parent: Command): void {
|
||||
role: opts.role,
|
||||
os: opts.os,
|
||||
...(opts.disk ? { disk: opts.disk } : {}),
|
||||
...(hasVyosOptions ? { vyos } : {}),
|
||||
});
|
||||
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
console.log("");
|
||||
const osLabel = opts.os.startsWith("ubuntu")
|
||||
? "Ubuntu"
|
||||
: opts.os.startsWith("vyos") ? "VyOS" : "Fedora";
|
||||
const osLabel = opts.os.startsWith("ubuntu") ? "Ubuntu" : "Fedora";
|
||||
console.log(`Power on the machine to start ${osLabel} installation.`);
|
||||
|
||||
const roleInfo = ROLE_REGISTRY.find(r => r.name === opts.role);
|
||||
|
||||
@@ -38,7 +38,7 @@ export function registerLabcontrollerCommands(appCmd: Command): void {
|
||||
lcCmd
|
||||
.command("deploy <target>")
|
||||
.description("Deploy labcontroller stack to a k3s node")
|
||||
.option("--user <user>", "SSH user", "root")
|
||||
.option("--user <user>", "SSH user", "michal")
|
||||
.option("--crdb-replicas <n>", "CockroachDB replicas", "1")
|
||||
.action(async (target: string, opts: {
|
||||
user: string;
|
||||
@@ -193,7 +193,7 @@ export function registerLabcontrollerCommands(appCmd: Command): void {
|
||||
lcCmd
|
||||
.command("status [target]")
|
||||
.description("Check labcontroller deployment status (all hosts if no target)")
|
||||
.option("--user <user>", "SSH user", "root")
|
||||
.option("--user <user>", "SSH user", "michal")
|
||||
.action(async (target: string | undefined, opts: { user: string }) => {
|
||||
const sshKey = findSshKey();
|
||||
const sshOpts = sshKey ? { keyPath: sshKey } : {};
|
||||
|
||||
@@ -69,10 +69,10 @@ export function registerListCommand(parent: Command): void {
|
||||
const hostname = inst?.hostname ?? queued?.hostname ?? "-";
|
||||
const role = inst?.role ?? queued?.role ?? "-";
|
||||
const ip = inst?.ip ?? "-";
|
||||
const cpu = hw?.cpu_model ?? inst?.cpu_model ?? "-";
|
||||
const cores = (hw?.cpu_cores ?? inst?.cpu_cores) != null ? String(hw?.cpu_cores ?? inst?.cpu_cores) : "-";
|
||||
const ram = (hw?.memory_gb ?? inst?.memory_gb) != null ? `${hw?.memory_gb ?? inst?.memory_gb}GB` : "-";
|
||||
const product = hw?.product ?? inst?.product ?? "-";
|
||||
const cpu = hw?.cpu_model ?? "-";
|
||||
const cores = hw?.cpu_cores != null ? String(hw.cpu_cores) : "-";
|
||||
const ram = hw?.memory_gb != null ? `${hw.memory_gb}GB` : "-";
|
||||
const product = hw?.product ?? "-";
|
||||
|
||||
const color = statusColor(status);
|
||||
|
||||
|
||||
@@ -39,25 +39,19 @@ export function registerLogsCommand(parent: Command): void {
|
||||
parent
|
||||
.command("logs <target>")
|
||||
.description("Show provisioning logs for a machine (hostname, MAC, or IP)")
|
||||
.option("-f, --follow", "Follow log output in real-time")
|
||||
.action(async (target: string, opts: { follow?: boolean }) => {
|
||||
.action(async (target: string) => {
|
||||
const mac = await resolveToMac(target);
|
||||
|
||||
const BOLD = "\x1b[1m";
|
||||
const GREEN = "\x1b[32m";
|
||||
const YELLOW = "\x1b[33m";
|
||||
const RED = "\x1b[31m";
|
||||
const DIM = "\x1b[2m";
|
||||
const RESET = "\x1b[0m";
|
||||
|
||||
if (opts.follow) {
|
||||
await followLogs(mac, { BOLD, GREEN, YELLOW, RED, DIM, RESET });
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const data = await getLabdClient().getMachineLogs(mac);
|
||||
|
||||
const BOLD = "\x1b[1m";
|
||||
const GREEN = "\x1b[32m";
|
||||
const YELLOW = "\x1b[33m";
|
||||
const RED = "\x1b[31m";
|
||||
const DIM = "\x1b[2m";
|
||||
const RESET = "\x1b[0m";
|
||||
|
||||
console.log(`${BOLD}${data["hostname"]}${RESET} (${mac})`);
|
||||
console.log(` Status: ${data["status"] === "installed" ? GREEN : YELLOW}${data["status"]}${RESET}`);
|
||||
console.log(` Role: ${data["role"]}`);
|
||||
@@ -89,64 +83,3 @@ export function registerLogsCommand(parent: Command): void {
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Follow logs by polling labd. */
|
||||
async function followLogs(
|
||||
mac: string,
|
||||
colors: { BOLD: string; GREEN: string; YELLOW: string; RED: string; DIM: string; RESET: string },
|
||||
): Promise<void> {
|
||||
const { BOLD, GREEN, YELLOW, RED, DIM, RESET } = colors;
|
||||
const client = getLabdClient();
|
||||
|
||||
console.log(`${DIM}Following logs for ${mac} (Ctrl+C to stop)${RESET}`);
|
||||
console.log("");
|
||||
|
||||
let lastStageCount = 0;
|
||||
let lastStatus = "";
|
||||
let sawInstalling = false;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
const data = await client.getMachineLogs(mac);
|
||||
const status = String(data["status"] ?? "");
|
||||
const log = data["log"] as Array<{ stage: string; detail: string; timestamp: string }> | undefined;
|
||||
|
||||
// Print header once or on status change
|
||||
if (status !== lastStatus) {
|
||||
const hostname = String(data["hostname"] ?? mac);
|
||||
const statusColor = status === "installed" ? GREEN : YELLOW;
|
||||
console.log(` ${BOLD}${hostname}${RESET} ${statusColor}${status}${RESET}`);
|
||||
lastStatus = status;
|
||||
}
|
||||
|
||||
if (status === "installing" || status === "queued") {
|
||||
sawInstalling = true;
|
||||
}
|
||||
|
||||
// Print new stages
|
||||
if (log && log.length > lastStageCount) {
|
||||
for (let i = lastStageCount; i < log.length; i++) {
|
||||
const entry = log[i]!;
|
||||
const time = entry.timestamp.slice(11, 19);
|
||||
const color = entry.stage === "complete" ? GREEN : entry.stage === "error" ? RED : YELLOW;
|
||||
const detail = entry.detail ? ` ${DIM}-- ${entry.detail}${RESET}` : "";
|
||||
console.log(` ${DIM}${time}${RESET} ${color}${entry.stage}${RESET}${detail}`);
|
||||
}
|
||||
lastStageCount = log.length;
|
||||
}
|
||||
|
||||
// Only exit on "installed" if we actually saw the install happen
|
||||
// (avoids exiting immediately when following a reprovision that hasn't started yet)
|
||||
if (status === "installed" && sawInstalling) {
|
||||
const ip = data["ip"] ?? "";
|
||||
console.log("");
|
||||
console.log(` ${GREEN}${BOLD}Install complete!${RESET}${ip ? ` ${DIM}ssh lab@${ip}${RESET}` : ""}`);
|
||||
process.exit(0);
|
||||
}
|
||||
} catch {
|
||||
// Machine may not be in logs yet (still queued)
|
||||
}
|
||||
|
||||
await new Promise((r) => setTimeout(r, 5000));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
// CLI command: provision recheck
|
||||
// SSH into all installed machines, collect hardware info, update bastion state.
|
||||
|
||||
import type { Command } from "commander";
|
||||
import { sshExec } from "@lab/modules";
|
||||
import { getLabdClient } from "../api/config.js";
|
||||
|
||||
const BOLD = "\x1b[1m";
|
||||
const GREEN = "\x1b[0;32m";
|
||||
const RED = "\x1b[0;31m";
|
||||
const DIM = "\x1b[2m";
|
||||
const RESET = "\x1b[0m";
|
||||
|
||||
const SSH_OPTS = { timeoutMs: 30_000 };
|
||||
|
||||
// Shell script that collects hardware info as JSON.
|
||||
// Kept simple — no Python, pure shell + awk.
|
||||
const HW_COLLECT_SCRIPT = [
|
||||
'P=$(cat /sys/class/dmi/id/product_name 2>/dev/null || echo unknown)',
|
||||
'B=$(cat /sys/class/dmi/id/board_name 2>/dev/null || echo unknown)',
|
||||
'S=$(cat /sys/class/dmi/id/product_serial 2>/dev/null || echo unknown)',
|
||||
'M=$(cat /sys/class/dmi/id/sys_vendor 2>/dev/null || echo unknown)',
|
||||
'C=$(grep -m1 "model name" /proc/cpuinfo 2>/dev/null | cut -d: -f2 | sed "s/^ //" || grep -m1 Model /proc/cpuinfo 2>/dev/null | cut -d: -f2 | sed "s/^ //" || echo unknown)',
|
||||
'N=$(grep -c "^processor" /proc/cpuinfo 2>/dev/null || echo 0)',
|
||||
'R=$(awk "/MemTotal/ {printf \\"%d\\", \\$2/1024/1024}" /proc/meminfo 2>/dev/null || echo 0)',
|
||||
'A=$(uname -m)',
|
||||
'printf \'{"product":"%s","board":"%s","serial":"%s","manufacturer":"%s","cpu_model":"%s","cpu_cores":%s,"memory_gb":%s,"arch":"%s"}\\n\' "$P" "$B" "$S" "$M" "$C" "$N" "$R" "$A"',
|
||||
].join("; ");
|
||||
|
||||
export function registerRecheckCommand(parent: Command): void {
|
||||
parent
|
||||
.command("recheck")
|
||||
.description("Refresh hardware info for all installed machines via SSH")
|
||||
.option("--user <user>", "SSH user", "root")
|
||||
.option("--target <hostname>", "Only recheck a specific machine (by hostname or MAC)")
|
||||
.action(async (opts: { user: string; target?: string }) => {
|
||||
const client = getLabdClient();
|
||||
let state;
|
||||
try {
|
||||
state = await client.getMachines();
|
||||
} catch (err) {
|
||||
console.error(`Cannot reach labd: ${err instanceof Error ? err.message : String(err)}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Build list of machines to check
|
||||
const targets: Array<{ mac: string; hostname: string; ip: string; sshUser: string }> = [];
|
||||
const userIsDefault = opts.user === "root";
|
||||
for (const [mac, info] of Object.entries(state.installed)) {
|
||||
if (!info.ip) continue;
|
||||
if (opts.target && info.hostname !== opts.target && mac !== opts.target) continue;
|
||||
// VyOS boxes only have the "vyos" login; honor an explicit --user.
|
||||
const sshUser = userIsDefault && (info.os ?? "").startsWith("vyos") ? "vyos" : opts.user;
|
||||
targets.push({ mac, hostname: info.hostname, ip: info.ip, sshUser });
|
||||
}
|
||||
|
||||
if (targets.length === 0) {
|
||||
console.log("No installed machines with IPs to check.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`\n${BOLD}Rechecking ${targets.length} machine(s)...${RESET}\n`);
|
||||
|
||||
let updated = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const { mac, hostname, ip, sshUser } of targets) {
|
||||
process.stdout.write(` ${hostname.padEnd(24)} ${DIM}(${ip})${RESET} `);
|
||||
|
||||
try {
|
||||
const t0 = Date.now();
|
||||
const result = await sshExec(ip, sshUser, HW_COLLECT_SCRIPT, SSH_OPTS);
|
||||
const elapsed = Date.now() - t0;
|
||||
if (result.exitCode !== 0) {
|
||||
console.log(`${RED}SSH failed (exit ${result.exitCode}, ${elapsed}ms)${RESET}`);
|
||||
if (result.stderr) console.log(` ${DIM}${result.stderr.substring(0, 200)}${RESET}`);
|
||||
console.log(`${RED}SSH failed (exit ${result.exitCode})${RESET}`);
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const hwData = JSON.parse(result.stdout.trim());
|
||||
await client.discoverMachine({ mac, ...hwData });
|
||||
const cpu = hwData.cpu_model || "?";
|
||||
const cores = hwData.cpu_cores || "?";
|
||||
const mem = hwData.memory_gb || "?";
|
||||
console.log(`${GREEN}OK${RESET} ${DIM}${cpu}, ${cores} cores, ${mem}GB${RESET}`);
|
||||
updated++;
|
||||
} catch (err) {
|
||||
console.log(`${RED}FAIL${RESET} ${DIM}${err instanceof Error ? err.message : String(err)}${RESET}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n${BOLD}Done:${RESET} ${updated} updated, ${failed} failed\n`);
|
||||
});
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
// CLI command: provision register
|
||||
// Register an already-installed machine that is missing from bastion state.
|
||||
|
||||
import { Command, Option } from "commander";
|
||||
import { SUPPORTED_ROLES } from "@lab/shared";
|
||||
import { getLabdClient } from "../api/config.js";
|
||||
|
||||
export function registerRegisterCommand(parent: Command): void {
|
||||
parent
|
||||
.command("register <mac> <hostname>")
|
||||
.description("Register an already-installed machine (e.g. after state loss)")
|
||||
.addOption(new Option("--role <role>", "Machine role").choices([...SUPPORTED_ROLES]).default("worker"))
|
||||
.option("--ip <address>", "Machine IP address")
|
||||
.action(async (mac: string, hostname: string, opts: {
|
||||
role: string;
|
||||
ip?: string;
|
||||
}) => {
|
||||
try {
|
||||
const result = await getLabdClient().registerMachine({
|
||||
mac,
|
||||
hostname,
|
||||
role: opts.role,
|
||||
...(opts.ip ? { ip: opts.ip } : {}),
|
||||
});
|
||||
|
||||
if (result.error) {
|
||||
console.error(`Failed: ${result.error}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Registered ${mac} as ${hostname} (role=${opts.role}${opts.ip ? `, ip=${opts.ip}` : ""})`);
|
||||
} catch (err) {
|
||||
console.error(`Failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -24,12 +24,12 @@ function roleTable(): string {
|
||||
function resolveTarget(
|
||||
target: string,
|
||||
state: BastionState,
|
||||
): { mac: string; hostname: string; ip: string; os?: string } | null {
|
||||
): { mac: string; hostname: string; ip: string } | null {
|
||||
const normalized = target.toLowerCase().replace(/-/g, ":");
|
||||
|
||||
if (state.installed[normalized]) {
|
||||
const info = state.installed[normalized];
|
||||
return { mac: normalized, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
|
||||
return { mac: normalized, hostname: info.hostname, ip: info.ip };
|
||||
}
|
||||
|
||||
if (state.discovered[normalized]) {
|
||||
@@ -38,13 +38,13 @@ function resolveTarget(
|
||||
|
||||
for (const [mac, info] of Object.entries(state.installed)) {
|
||||
if (info.hostname === target || info.hostname.startsWith(target + ".")) {
|
||||
return { mac, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
|
||||
return { mac, hostname: info.hostname, ip: info.ip };
|
||||
}
|
||||
}
|
||||
|
||||
for (const [mac, info] of Object.entries(state.installed)) {
|
||||
if (info.ip === target) {
|
||||
return { mac, hostname: info.hostname, ip: info.ip, ...(info.os !== undefined ? { os: info.os } : {}) };
|
||||
return { mac, hostname: info.hostname, ip: info.ip };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,12 +60,10 @@ export function registerReprovisionCommand(parent: Command): void {
|
||||
.addOption(new Option("--role <role>", "Machine role (see below)").choices([...SUPPORTED_ROLES]).default("worker"))
|
||||
.addOption(new Option("--os <os>", "Operating system").choices([...SUPPORTED_OS]).default("fedora-43"))
|
||||
.option("--disk <device>", "Target disk device (auto-detect if omitted)")
|
||||
.option("--user <user>", "SSH user for the reboot (default: vyos for VyOS machines, else current user)")
|
||||
.action(async (target: string, hostnameOverride: string | undefined, opts: {
|
||||
role: string;
|
||||
os: string;
|
||||
disk?: string;
|
||||
user?: string;
|
||||
}) => {
|
||||
if (!isValidOsId(opts.os)) {
|
||||
console.error(`Unknown OS: ${opts.os}. Supported: ${SUPPORTED_OS.join(", ")}`);
|
||||
@@ -125,11 +123,7 @@ export function registerReprovisionCommand(parent: Command): void {
|
||||
return;
|
||||
}
|
||||
|
||||
// SSH user: explicit flag > the machine's current OS (VyOS boxes only
|
||||
// have the "vyos" login) > the invoking user.
|
||||
const currentOsIsVyos = (resolved.os ?? "").startsWith("vyos");
|
||||
const adminUser = opts.user
|
||||
?? (currentOsIsVyos ? "vyos" : (process.env["SUDO_USER"] ?? process.env["USER"] ?? ""));
|
||||
const adminUser = process.env["SUDO_USER"] ?? process.env["USER"] ?? "";
|
||||
const effectiveUser = adminUser === "root" ? "" : adminUser;
|
||||
|
||||
if (effectiveUser === "") {
|
||||
@@ -150,7 +144,6 @@ export function registerReprovisionCommand(parent: Command): void {
|
||||
|
||||
const sshArgs = [
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "ConnectTimeout=10",
|
||||
...(sshKey !== undefined ? ["-i", sshKey] : []),
|
||||
`${effectiveUser}@${ip}`,
|
||||
|
||||
@@ -11,7 +11,7 @@ export function registerStartCommand(parent: Command): void {
|
||||
.command("start")
|
||||
.description("Start the bastion server (HTTP + dnsmasq PXE)")
|
||||
.option("--port <port>", "HTTP port", "8080")
|
||||
.option("--dir <dir>", "Bastion data directory", process.env["BASTION_DIR"] ?? "/tmp/lab-bastion")
|
||||
.option("--dir <dir>", "Bastion data directory", "/tmp/lab-bastion")
|
||||
.option("--domain <domain>", "Internal domain for hostnames", "ad.itaz.eu")
|
||||
.option("--dhcp-mode <mode>", "DHCP mode: proxy or full", "proxy")
|
||||
.option("--fedora <version>", "Fedora version", "43")
|
||||
|
||||
@@ -8,7 +8,7 @@ export function registerStopCommand(parent: Command): void {
|
||||
parent
|
||||
.command("stop")
|
||||
.description("Stop a running bastion server")
|
||||
.option("--dir <dir>", "Bastion data directory", process.env["BASTION_DIR"] ?? "/tmp/lab-bastion")
|
||||
.option("--dir <dir>", "Bastion data directory", "/tmp/lab-bastion")
|
||||
.action((opts: { dir: string }) => {
|
||||
const pidFile = `${opts.dir}/bastion.pid`;
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// CLI entry point for lab-bastion.
|
||||
// Commands:
|
||||
// init bastion standalone start/stop/status
|
||||
// provision list/install/reprovision/forget/register
|
||||
// provision list/install/reprovision/forget
|
||||
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Command, Option } from "commander";
|
||||
@@ -16,11 +16,8 @@ import { registerListCommand } from "./commands/list.js";
|
||||
import { registerReprovisionCommand } from "./commands/reprovision.js";
|
||||
import { registerDebugCommand } from "./commands/debug.js";
|
||||
import { registerForgetCommand } from "./commands/forget.js";
|
||||
import { registerRegisterCommand } from "./commands/register.js";
|
||||
import { registerAsahiCommand } from "./commands/asahi.js";
|
||||
import { registerLogsCommand } from "./commands/logs.js";
|
||||
import { registerMakeIsoCommand } from "./commands/makeiso.js";
|
||||
import { registerRecheckCommand } from "./commands/recheck.js";
|
||||
import { registerConfigCommand } from "./commands/config.js";
|
||||
import { registerLoginCommand } from "./commands/login.js";
|
||||
import { registerDoctorCommand } from "./commands/doctor.js";
|
||||
@@ -101,11 +98,8 @@ export function createProgram(): Command {
|
||||
registerReprovisionCommand(provisionCmd);
|
||||
registerDebugCommand(provisionCmd);
|
||||
registerForgetCommand(provisionCmd);
|
||||
registerRegisterCommand(provisionCmd);
|
||||
registerAsahiCommand(provisionCmd);
|
||||
registerLogsCommand(provisionCmd);
|
||||
registerMakeIsoCommand(provisionCmd);
|
||||
registerRecheckCommand(provisionCmd);
|
||||
|
||||
// config list/get/set/path
|
||||
registerConfigCommand(program);
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
// Tests for VyOS install option parsing.
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { parseVlan } from "../src/commands/install.js";
|
||||
|
||||
describe("parseVlan", () => {
|
||||
it("parses id and CIDR", () => {
|
||||
expect(parseVlan("10:10.0.10.1/24")).toEqual([{ id: 10, address: "10.0.10.1/24" }]);
|
||||
});
|
||||
|
||||
it("accumulates across repeated flags", () => {
|
||||
const first = parseVlan("10:10.0.10.1/24");
|
||||
const both = parseVlan("20:10.0.20.1/24", first);
|
||||
expect(both).toHaveLength(2);
|
||||
expect(both[1]).toEqual({ id: 20, address: "10.0.20.1/24" });
|
||||
});
|
||||
|
||||
it("keeps a description, including one containing colons", () => {
|
||||
expect(parseVlan("30:10.0.30.1/24:mgmt:secondary")).toEqual([
|
||||
{ id: 30, address: "10.0.30.1/24", description: "mgmt:secondary" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects an address that is not CIDR", () => {
|
||||
// A bare address would produce a VyOS config that fails to commit on first
|
||||
// boot, long after the operator has stopped watching.
|
||||
expect(() => parseVlan("10:10.0.10.1")).toThrow(/CIDR/);
|
||||
});
|
||||
|
||||
it("rejects out-of-range and non-numeric VLAN ids", () => {
|
||||
expect(() => parseVlan("0:10.0.10.1/24")).toThrow(/1-4094/);
|
||||
expect(() => parseVlan("4095:10.0.10.1/24")).toThrow(/1-4094/);
|
||||
expect(() => parseVlan("abc:10.0.10.1/24")).toThrow(/1-4094/);
|
||||
});
|
||||
});
|
||||
@@ -137,7 +137,7 @@ describe("bastion smoke tests", () => {
|
||||
|
||||
// Wait for the server to start (look for the banner)
|
||||
const startedAt = Date.now();
|
||||
const maxWait = 15_000;
|
||||
const maxWait = 10_000;
|
||||
while (Date.now() - startedAt < maxWait) {
|
||||
if (stdout.includes("Waiting for PXE boot requests")) break;
|
||||
await sleep(200);
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
{
|
||||
"name": "@lab/core",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"import": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc --build",
|
||||
"clean": "rimraf dist",
|
||||
"test": "vitest",
|
||||
"test:run": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@pulumi/pulumi": "^3.0.0"
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
// Audit event types for the labctl platform.
|
||||
// Every mutation is tracked with correlation IDs for causal chains.
|
||||
|
||||
export type AuditEventKind =
|
||||
| "resource_created"
|
||||
| "resource_updated"
|
||||
| "resource_deleted"
|
||||
| "resource_state_change"
|
||||
| "plan_generated"
|
||||
| "apply_started"
|
||||
| "apply_step"
|
||||
| "apply_completed"
|
||||
| "driver_translate"
|
||||
| "driver_execute"
|
||||
| "driver_error"
|
||||
| "fleet_discovery"
|
||||
| "fleet_classification"
|
||||
| "fleet_approval"
|
||||
| "fleet_auto_approve"
|
||||
| "pipeline_started"
|
||||
| "pipeline_step_started"
|
||||
| "pipeline_step_completed"
|
||||
| "pipeline_completed"
|
||||
| "deploy_started"
|
||||
| "deploy_completed"
|
||||
| "deploy_failed"
|
||||
| "drift_detected"
|
||||
| "drift_corrected"
|
||||
| "sync_triggered"
|
||||
| "sync_completed"
|
||||
| "auth_login"
|
||||
| "auth_logout"
|
||||
| "auth_bootstrap"
|
||||
| "rbac_decision"
|
||||
| "impersonation"
|
||||
| "server_started"
|
||||
| "controller_started"
|
||||
| "agent_connected"
|
||||
| "agent_disconnected"
|
||||
| "bastion_registered";
|
||||
|
||||
export type AuditSource =
|
||||
| "cli"
|
||||
| "labd"
|
||||
| "agent"
|
||||
| "driver"
|
||||
| "fleet-controller"
|
||||
| "sync-controller";
|
||||
|
||||
export type AuditResult = "success" | "failure" | "denied" | "skipped";
|
||||
|
||||
export interface AuditEvent {
|
||||
id: string;
|
||||
timestamp: Date;
|
||||
eventKind: AuditEventKind;
|
||||
source: AuditSource;
|
||||
verified: boolean;
|
||||
|
||||
userId?: string;
|
||||
userName?: string;
|
||||
sessionId?: string;
|
||||
environmentName?: string;
|
||||
accountName?: string;
|
||||
|
||||
resourceKind?: string;
|
||||
resourceName?: string;
|
||||
|
||||
correlationId: string;
|
||||
parentEventId?: string;
|
||||
|
||||
details: Record<string, unknown>;
|
||||
result: AuditResult;
|
||||
error?: string;
|
||||
durationMs?: number;
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
// Auth types for the labctl platform.
|
||||
// Bearer token auth for CLI/SDK. mTLS stays for agent/bastion.
|
||||
|
||||
export type UserRole = "USER" | "ADMIN";
|
||||
|
||||
export interface User {
|
||||
id: string;
|
||||
email: string;
|
||||
name?: string;
|
||||
role: UserRole;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export interface Session {
|
||||
id: string;
|
||||
userId: string;
|
||||
token: string;
|
||||
expiresAt: Date;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export interface Group {
|
||||
id: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
export type SubjectKind = "User" | "Group" | "ServiceAccount";
|
||||
|
||||
export interface RoleBinding {
|
||||
role: "view" | "edit" | "create" | "delete" | "run" | "admin";
|
||||
resource: string;
|
||||
name?: string;
|
||||
environment?: string;
|
||||
action?: string;
|
||||
}
|
||||
|
||||
export interface RbacSubject {
|
||||
kind: SubjectKind;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface RbacDefinition {
|
||||
id: string;
|
||||
name: string;
|
||||
subjects: RbacSubject[];
|
||||
roleBindings: RoleBinding[];
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
// Environment and Account types.
|
||||
// An Environment is a logical boundary (production, staging, dev).
|
||||
// An Account is a configured driver instance with credentials.
|
||||
|
||||
export interface Environment {
|
||||
id: string;
|
||||
name: string;
|
||||
status: "active" | "archived";
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export interface Account {
|
||||
id: string;
|
||||
name: string;
|
||||
driver: string;
|
||||
config: Record<string, unknown>;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export interface Binding {
|
||||
id: string;
|
||||
environmentId: string;
|
||||
accountId: string;
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
// @lab/core — foundation types for the labctl platform.
|
||||
// Phase 1 stub: resource types, auth types, audit types, Output<T>.
|
||||
// Phase 5 adds: CompositeResource, evaluator integration, full SDK.
|
||||
|
||||
export * from "./resource.js";
|
||||
export * from "./environment.js";
|
||||
export * from "./audit.js";
|
||||
export * from "./auth.js";
|
||||
export { Output, output, all, interpolate, secret } from "./output.js";
|
||||
@@ -1,5 +0,0 @@
|
||||
// Re-export Pulumi's Output<T> type for use across the platform.
|
||||
// Cloud drivers use this for future values (endpoints, IPs, kubeconfigs).
|
||||
// Phase 1: type re-export only. Phase 5 adds full evaluator integration.
|
||||
|
||||
export { Output, output, all, interpolate, secret } from "@pulumi/pulumi";
|
||||
@@ -1,83 +0,0 @@
|
||||
// Core resource types for the labctl platform.
|
||||
// Every managed thing (Server, Database, App, Cluster) is a Resource.
|
||||
|
||||
export type ResourceOrigin = "file" | "cli" | "fleet" | "imported";
|
||||
export type ResourceManagedBy = "gitops" | "manual" | "auto";
|
||||
|
||||
export type ResourceStatus =
|
||||
| "pending"
|
||||
| "creating"
|
||||
| "ready"
|
||||
| "updating"
|
||||
| "deleting"
|
||||
| "error"
|
||||
| "unknown";
|
||||
|
||||
export interface ResourceMetadata {
|
||||
kind: string;
|
||||
name: string;
|
||||
environmentId: string;
|
||||
accountId: string;
|
||||
origin: ResourceOrigin;
|
||||
managedBy: ResourceManagedBy;
|
||||
sourceRef?: string;
|
||||
}
|
||||
|
||||
export interface ResourceState {
|
||||
status: ResourceStatus;
|
||||
message?: string;
|
||||
lastReconciled?: Date;
|
||||
platformRef?: string;
|
||||
}
|
||||
|
||||
export interface Resource<TSpec = Record<string, unknown>> {
|
||||
id: string;
|
||||
metadata: ResourceMetadata;
|
||||
desiredSpec: TSpec;
|
||||
actualSpec?: TSpec;
|
||||
state: ResourceState;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
// Well-known resource kinds. Drivers register additional kinds.
|
||||
export const RESOURCE_KINDS = {
|
||||
SERVER: "server",
|
||||
DATABASE: "database",
|
||||
CACHE: "cache",
|
||||
CLUSTER: "cluster",
|
||||
APP: "app",
|
||||
SERVICE: "service",
|
||||
CRONJOB: "cronjob",
|
||||
NETWORK: "network",
|
||||
LOADBALANCER: "loadbalancer",
|
||||
DNSZONE: "dnszone",
|
||||
CERTIFICATE: "certificate",
|
||||
OBJECTSTORE: "objectstore",
|
||||
QUEUE: "queue",
|
||||
SECRET: "secret",
|
||||
FLEET: "fleet",
|
||||
} as const;
|
||||
|
||||
export type ResourceKind = (typeof RESOURCE_KINDS)[keyof typeof RESOURCE_KINDS];
|
||||
|
||||
// Resource aliases for CLI (kubectl-style shortnames)
|
||||
export const RESOURCE_ALIASES: Record<string, string> = {
|
||||
srv: "server",
|
||||
db: "database",
|
||||
cl: "cluster",
|
||||
svc: "service",
|
||||
cj: "cronjob",
|
||||
lb: "loadbalancer",
|
||||
dns: "dnszone",
|
||||
cert: "certificate",
|
||||
os: "objectstore",
|
||||
mq: "queue",
|
||||
sec: "secret",
|
||||
fl: "fleet",
|
||||
};
|
||||
|
||||
export function resolveResourceKind(input: string): string {
|
||||
const lower = input.toLowerCase();
|
||||
return RESOURCE_ALIASES[lower] ?? lower;
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "src",
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
@@ -26,10 +26,8 @@
|
||||
"dependencies": {
|
||||
"@fastify/rate-limit": "^10.3.0",
|
||||
"@fastify/websocket": "^11.0.2",
|
||||
"@lab/core": "workspace:^",
|
||||
"@lab/shared": "workspace:*",
|
||||
"@prisma/client": "^6.9.0",
|
||||
"bcryptjs": "^3.0.3",
|
||||
"fastify": "^5.3.3",
|
||||
"winston": "^3.17.0",
|
||||
"ws": "^8.19.0",
|
||||
@@ -39,7 +37,6 @@
|
||||
"seed": "tsx prisma/seed.ts"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bcryptjs": "^3.0.0",
|
||||
"@types/node": "^22.14.1",
|
||||
"@types/ws": "^8.18.1",
|
||||
"prisma": "^6.9.0",
|
||||
|
||||
@@ -7,241 +7,23 @@ datasource db {
|
||||
url = env("DATABASE_URL")
|
||||
}
|
||||
|
||||
// ── Auth (mcpctl pattern: email/password + bearer token sessions) ──
|
||||
|
||||
model User {
|
||||
id String @id @default(cuid())
|
||||
email String @unique
|
||||
password String // bcrypt
|
||||
name String?
|
||||
role UserRole @default(USER)
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
sessions Session[]
|
||||
auditLogs AuditEvent[]
|
||||
groups GroupMember[]
|
||||
}
|
||||
|
||||
enum UserRole {
|
||||
USER
|
||||
ADMIN
|
||||
}
|
||||
|
||||
model Session {
|
||||
id String @id @default(cuid())
|
||||
userId String
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
token String @unique
|
||||
expiresAt DateTime
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
@@index([userId])
|
||||
@@index([token])
|
||||
}
|
||||
|
||||
model Group {
|
||||
id String @id @default(cuid())
|
||||
name String @unique
|
||||
description String?
|
||||
createdAt DateTime @default(now())
|
||||
members GroupMember[]
|
||||
}
|
||||
|
||||
model GroupMember {
|
||||
id String @id @default(cuid())
|
||||
groupId String
|
||||
group Group @relation(fields: [groupId], references: [id], onDelete: Cascade)
|
||||
userId String
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@unique([groupId, userId])
|
||||
}
|
||||
|
||||
model ServiceAccount {
|
||||
id String @id @default(cuid())
|
||||
name String @unique
|
||||
token String @unique
|
||||
createdAt DateTime @default(now())
|
||||
}
|
||||
|
||||
// ── RBAC (mcpctl pattern: named definitions with JSON subjects/bindings) ──
|
||||
|
||||
model RbacDefinition {
|
||||
id String @id @default(cuid())
|
||||
name String @unique
|
||||
subjects Json // [{kind: "User"|"Group"|"ServiceAccount", name: string}]
|
||||
roleBindings Json // [{role, resource, name?, environment?, action?}]
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
}
|
||||
|
||||
// ── Audit (mcpctl pattern: fire-and-forget with correlation IDs) ──
|
||||
|
||||
model AuditEvent {
|
||||
id String @id @default(cuid())
|
||||
timestamp DateTime @default(now())
|
||||
eventKind String
|
||||
source String // cli | labd | agent | driver | fleet-controller | sync-controller
|
||||
verified Boolean @default(false)
|
||||
|
||||
userId String?
|
||||
user User? @relation(fields: [userId], references: [id])
|
||||
userName String?
|
||||
sessionId String?
|
||||
environmentName String?
|
||||
accountName String?
|
||||
|
||||
resourceKind String?
|
||||
resourceName String?
|
||||
|
||||
correlationId String
|
||||
parentEventId String?
|
||||
|
||||
details Json @default("{}")
|
||||
result String // success | failure | denied | skipped
|
||||
error String?
|
||||
durationMs Int?
|
||||
|
||||
@@index([correlationId])
|
||||
@@index([eventKind, timestamp])
|
||||
@@index([environmentName, timestamp])
|
||||
@@index([resourceKind, resourceName])
|
||||
@@index([userId, timestamp])
|
||||
}
|
||||
|
||||
// ── Core infrastructure ──
|
||||
|
||||
model Environment {
|
||||
id String @id @default(cuid())
|
||||
name String @unique
|
||||
status String @default("active") // active | archived
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
bindings Binding[]
|
||||
resources Resource[]
|
||||
}
|
||||
|
||||
model Account {
|
||||
id String @id @default(cuid())
|
||||
name String @unique
|
||||
driver String // baremetal-pxe | aws | gcp | kubernetes | ovh
|
||||
config Json @default("{}")
|
||||
// Credentials stored in Infisical, referenced by secretPath
|
||||
secretPath String?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
bindings Binding[]
|
||||
resources Resource[]
|
||||
}
|
||||
|
||||
model Binding {
|
||||
id String @id @default(cuid())
|
||||
environmentId String
|
||||
environment Environment @relation(fields: [environmentId], references: [id], onDelete: Cascade)
|
||||
accountId String
|
||||
account Account @relation(fields: [accountId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@unique([environmentId, accountId])
|
||||
}
|
||||
|
||||
model Resource {
|
||||
id String @id @default(cuid())
|
||||
kind String
|
||||
name String
|
||||
environmentId String
|
||||
environment Environment @relation(fields: [environmentId], references: [id])
|
||||
accountId String
|
||||
account Account @relation(fields: [accountId], references: [id])
|
||||
origin String @default("cli") // file | cli | fleet | imported
|
||||
managedBy String @default("manual") // gitops | manual | auto
|
||||
sourceRef String?
|
||||
desiredSpec Json @default("{}")
|
||||
actualSpec Json?
|
||||
platformRef String?
|
||||
status String @default("pending") // pending | creating | ready | updating | deleting | error
|
||||
statusMessage String?
|
||||
lastReconciled DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
@@unique([kind, name, environmentId])
|
||||
@@index([environmentId])
|
||||
@@index([accountId])
|
||||
@@index([kind, status])
|
||||
}
|
||||
|
||||
model Secret {
|
||||
id String @id @default(cuid())
|
||||
name String @unique
|
||||
// Encrypted data — application-layer encryption as fallback if Infisical unavailable
|
||||
data Json @default("{}")
|
||||
version Int @default(1)
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
}
|
||||
|
||||
// ── Fleet ──
|
||||
|
||||
model Fleet {
|
||||
id String @id @default(cuid())
|
||||
name String
|
||||
environmentId String
|
||||
accountId String
|
||||
selector Json // fact-matching rules
|
||||
onboardPipeline Json // step definitions
|
||||
offboardPipeline Json?
|
||||
approvalConfig Json?
|
||||
status String @default("active")
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
members FleetMember[]
|
||||
}
|
||||
|
||||
model FleetMember {
|
||||
id String @id @default(cuid())
|
||||
fleetId String
|
||||
fleet Fleet @relation(fields: [fleetId], references: [id], onDelete: Cascade)
|
||||
serverId String
|
||||
status String // discovered | pending | onboarding | active | offboarding | removed
|
||||
joinedAt DateTime @default(now())
|
||||
|
||||
@@index([fleetId])
|
||||
}
|
||||
|
||||
// ── Git sources (for sync controller) ──
|
||||
|
||||
model GitSource {
|
||||
id String @id @default(cuid())
|
||||
name String @unique
|
||||
repo String
|
||||
branch String @default("main")
|
||||
path String @default("environments/")
|
||||
lastSync DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
}
|
||||
|
||||
// ── Existing v1.0 models (kept for bastion/agent compatibility) ──
|
||||
|
||||
model Server {
|
||||
id String @id @default(uuid())
|
||||
hostname String @unique
|
||||
mac String? @unique
|
||||
cloud String @default("baremetal")
|
||||
environment String @default("default")
|
||||
role String @default("worker")
|
||||
labels Json @default("{}")
|
||||
id String @id @default(uuid())
|
||||
hostname String @unique
|
||||
mac String? @unique
|
||||
cloud String @default("baremetal")
|
||||
environment String @default("default")
|
||||
role String @default("worker")
|
||||
labels Json @default("{}")
|
||||
ip String?
|
||||
agentVersion String?
|
||||
status String @default("unknown")
|
||||
status String @default("unknown") // unknown, online, offline, provisioning
|
||||
lastHeartbeat DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
agent Agent?
|
||||
agent Agent?
|
||||
auditLogs AuditLog[]
|
||||
}
|
||||
|
||||
model Agent {
|
||||
@@ -251,29 +33,112 @@ model Agent {
|
||||
certificatePem String?
|
||||
enrolledAt DateTime @default(now())
|
||||
lastSeen DateTime?
|
||||
facts Json? // hardware facts reported by agent
|
||||
|
||||
@@index([serverId])
|
||||
}
|
||||
|
||||
model User {
|
||||
id String @id @default(uuid())
|
||||
username String @unique
|
||||
displayName String?
|
||||
certFingerprint String? @unique
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
roleBindings UserRole[]
|
||||
auditLogs AuditLog[]
|
||||
}
|
||||
|
||||
model Role {
|
||||
id String @id @default(uuid())
|
||||
name String @unique
|
||||
description String?
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
permissions Permission[]
|
||||
userBindings UserRole[]
|
||||
}
|
||||
|
||||
model Permission {
|
||||
id String @id @default(uuid())
|
||||
roleId String
|
||||
role Role @relation(fields: [roleId], references: [id], onDelete: Cascade)
|
||||
type String @default("allow") // allow or deny
|
||||
action String // read, exec, apply, destroy, manage, admin, kubectl, *
|
||||
cloud String @default("*")
|
||||
environment String @default("*")
|
||||
server String @default("*")
|
||||
|
||||
@@index([roleId])
|
||||
}
|
||||
|
||||
model UserRole {
|
||||
id String @id @default(uuid())
|
||||
userId String
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
roleId String
|
||||
role Role @relation(fields: [roleId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@unique([userId, roleId])
|
||||
@@index([userId])
|
||||
@@index([roleId])
|
||||
}
|
||||
|
||||
model JoinToken {
|
||||
id String @id @default(uuid())
|
||||
token String @unique
|
||||
type String @default("one-time")
|
||||
type String @default("one-time") // one-time or reusable
|
||||
label String?
|
||||
usedBy String?
|
||||
usedBy String? // server hostname that used it
|
||||
usedAt DateTime?
|
||||
revokedAt DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
expiresAt DateTime?
|
||||
}
|
||||
|
||||
model AuditLog {
|
||||
id String @id @default(uuid())
|
||||
userId String?
|
||||
user User? @relation(fields: [userId], references: [id])
|
||||
serverId String?
|
||||
server Server? @relation(fields: [serverId], references: [id])
|
||||
sessionId String?
|
||||
action String // exec, kubectl, apply, login, rbac-denied, etc.
|
||||
resourceType String? // server, cluster, role, app, etc.
|
||||
resourceName String?
|
||||
args String? // sanitized command args
|
||||
result String @default("success") // success, denied, error
|
||||
durationMs Int?
|
||||
sourceIp String?
|
||||
timestamp DateTime @default(now())
|
||||
|
||||
@@index([userId])
|
||||
@@index([serverId])
|
||||
@@index([sessionId])
|
||||
@@index([timestamp])
|
||||
@@index([action])
|
||||
}
|
||||
|
||||
model PulumiRun {
|
||||
id String @id @default(uuid())
|
||||
userId String
|
||||
stackName String
|
||||
action String // up, preview, destroy
|
||||
status String @default("pending") // pending, running, succeeded, failed
|
||||
output String?
|
||||
startedAt DateTime @default(now())
|
||||
completedAt DateTime?
|
||||
|
||||
@@index([userId])
|
||||
@@index([stackName])
|
||||
}
|
||||
|
||||
model Bastion {
|
||||
id String @id @default(uuid())
|
||||
hostname String @unique
|
||||
network String
|
||||
serverIp String
|
||||
status String @default("offline")
|
||||
status String @default("offline") // online, offline
|
||||
lastHeartbeat DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
@@ -284,7 +149,7 @@ model Cluster {
|
||||
name String @unique
|
||||
cloud String @default("baremetal")
|
||||
environment String @default("default")
|
||||
kubeconfigEnc String?
|
||||
kubeconfigEnc String? // encrypted kubeconfig
|
||||
labels Json @default("{}")
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
// Bearer token auth middleware for Fastify.
|
||||
// Validates Authorization header, resolves user identity, attaches to request.
|
||||
|
||||
import type { FastifyRequest, FastifyReply } from "fastify";
|
||||
import type { AuthService } from "../services/auth.js";
|
||||
|
||||
declare module "fastify" {
|
||||
interface FastifyRequest {
|
||||
userId?: string;
|
||||
userEmail?: string;
|
||||
userRole?: string;
|
||||
}
|
||||
}
|
||||
|
||||
// Paths that don't require authentication
|
||||
const PUBLIC_PATHS = new Set([
|
||||
"/health",
|
||||
"/api/auth/login",
|
||||
"/ws/bastion",
|
||||
"/ws/agent",
|
||||
"/api/auth/enroll",
|
||||
]);
|
||||
|
||||
export function createBearerAuthMiddleware(authService: AuthService) {
|
||||
return async function bearerAuth(
|
||||
request: FastifyRequest,
|
||||
reply: FastifyReply,
|
||||
): Promise<void> {
|
||||
// Skip auth for public paths
|
||||
if (PUBLIC_PATHS.has(request.url.split("?")[0] ?? "")) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip auth for WebSocket upgrade requests (handled by their own auth)
|
||||
if (request.headers.upgrade === "websocket") {
|
||||
return;
|
||||
}
|
||||
|
||||
const authHeader = request.headers.authorization;
|
||||
if (!authHeader) {
|
||||
void reply.code(401).send({ error: "Authorization header required" });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!authHeader.startsWith("Bearer ")) {
|
||||
void reply.code(401).send({ error: "Invalid authorization format, expected: Bearer <token>" });
|
||||
return;
|
||||
}
|
||||
|
||||
const token = authHeader.slice(7);
|
||||
if (token.length === 0) {
|
||||
void reply.code(401).send({ error: "Empty bearer token" });
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const identity = await authService.validateToken(token);
|
||||
request.userId = identity.userId;
|
||||
request.userEmail = identity.email;
|
||||
request.userRole = identity.role;
|
||||
} catch {
|
||||
void reply.code(401).send({ error: "Invalid or expired token. Run: labctl login" });
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -10,7 +10,6 @@ import type { FastifyInstance } from "fastify";
|
||||
import type { DbClient } from "../server.js";
|
||||
import { bastionRegistry } from "../services/bastion-registry.js";
|
||||
import { generateRequestId } from "@lab/shared";
|
||||
import type { VyosInstallSpec } from "@lab/shared";
|
||||
|
||||
const COMMAND_TIMEOUT_MS = 15_000;
|
||||
|
||||
@@ -85,6 +84,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
app.get("/api/machines", async () => {
|
||||
const live = bastionRegistry.getAggregatedState();
|
||||
|
||||
// Merge DB records for machines not currently in any bastion's live state
|
||||
try {
|
||||
const dbServers = (await db.server.findMany({})) as Array<{
|
||||
mac: string | null; hostname: string; role: string; ip: string | null;
|
||||
@@ -93,49 +93,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
for (const s of dbServers) {
|
||||
if (!s.mac) continue;
|
||||
const mac = s.mac.toLowerCase();
|
||||
|
||||
// DB knows this machine has been installed at some point if it has a real
|
||||
// hostname+role (not just product-name-as-hostname and role="unknown").
|
||||
// Status alone is unreliable: a rediscovery can re-set it without erasing the
|
||||
// install identity. If the bastion restarted and lost its installed map, the
|
||||
// machine will only show up in live.discovered — promote it here so the CLI
|
||||
// still sees hostname/role/IP.
|
||||
const dbKnowsInstalled =
|
||||
s.role !== "unknown" && s.role !== "" &&
|
||||
s.hostname !== "" && s.hostname !== s.mac;
|
||||
|
||||
if (dbKnowsInstalled && !(mac in live.installed) && !(mac in live.install_queue)) {
|
||||
const hw = live.discovered[mac];
|
||||
live.installed[mac] = {
|
||||
hostname: s.hostname,
|
||||
role: s.role,
|
||||
ip: s.ip ?? "",
|
||||
installed_at: "",
|
||||
bastionId: hw?.bastionId ?? "db",
|
||||
...(hw ? {
|
||||
product: hw.product,
|
||||
manufacturer: hw.manufacturer,
|
||||
cpu_model: hw.cpu_model,
|
||||
cpu_cores: hw.cpu_cores,
|
||||
memory_gb: hw.memory_gb,
|
||||
arch: hw.arch,
|
||||
} : {}),
|
||||
};
|
||||
delete live.discovered[mac];
|
||||
continue;
|
||||
}
|
||||
|
||||
// Unknown-to-live MAC: fall back to whatever the DB says.
|
||||
// Only add from DB if not already in live state
|
||||
if (!(mac in live.discovered) && !(mac in live.install_queue) && !(mac in live.installed)) {
|
||||
if (s.status === "online" || s.status === "offline") {
|
||||
live.installed[mac] = {
|
||||
hostname: s.hostname,
|
||||
role: s.role,
|
||||
ip: s.ip ?? "",
|
||||
installed_at: "",
|
||||
bastionId: "db",
|
||||
};
|
||||
} else {
|
||||
if (s.status === "discovered") {
|
||||
live.discovered[mac] = {
|
||||
mac,
|
||||
product: String(s.labels?.product ?? "unknown"),
|
||||
@@ -152,6 +112,14 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
last_seen: "",
|
||||
bastionId: "db",
|
||||
};
|
||||
} else if (s.status === "online" || s.status === "offline") {
|
||||
live.installed[mac] = {
|
||||
hostname: s.hostname,
|
||||
role: s.role,
|
||||
ip: s.ip ?? "",
|
||||
installed_at: "",
|
||||
bastionId: "db",
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -164,9 +132,9 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
|
||||
// Queue install — route to correct bastion by MAC
|
||||
app.post<{
|
||||
Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string; vyos?: VyosInstallSpec };
|
||||
Body: { mac?: string; hostname?: string; disk?: string; role?: string; os?: string };
|
||||
}>("/api/machines/install", async (request, reply) => {
|
||||
const { mac, hostname, disk, role, os, vyos } = request.body ?? {};
|
||||
const { mac, hostname, disk, role, os } = request.body ?? {};
|
||||
if (!mac || !hostname) {
|
||||
return reply.code(400).send({ error: "mac and hostname are required" });
|
||||
}
|
||||
@@ -183,8 +151,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
try {
|
||||
const result = await sendCommand(all[0]!.bastionId, {
|
||||
type: "command-install",
|
||||
mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
|
||||
...(vyos ? { vyos } : {}),
|
||||
mac, hostname, disk: disk ?? "/dev/sda", role: role ?? "infra", os: os ?? "fedora-43",
|
||||
});
|
||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||
} catch (err) {
|
||||
@@ -197,45 +164,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
try {
|
||||
const result = await sendCommand(bastion.bastionId, {
|
||||
type: "command-install",
|
||||
mac, hostname, disk: disk ?? "", role: role ?? "infra", os: os ?? "fedora-43",
|
||||
...(vyos ? { vyos } : {}),
|
||||
});
|
||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||
} catch (err) {
|
||||
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// Register an already-installed machine — route to correct bastion (or single bastion)
|
||||
app.post<{
|
||||
Body: { mac?: string; hostname?: string; role?: string; ip?: string };
|
||||
}>("/api/machines/register", async (request, reply) => {
|
||||
const { mac, hostname, role, ip } = request.body ?? {};
|
||||
if (!mac || !hostname) {
|
||||
return reply.code(400).send({ error: "mac and hostname are required" });
|
||||
}
|
||||
|
||||
const normalized = mac.toLowerCase().replace(/-/g, ":");
|
||||
|
||||
// Find bastion that knows this MAC, or use single connected bastion
|
||||
const bastion = bastionRegistry.findBastionByMac(normalized);
|
||||
const target = bastion ?? (bastionRegistry.getAll().length === 1 ? bastionRegistry.getAll()[0] : null);
|
||||
|
||||
if (!target) {
|
||||
const all = bastionRegistry.getAll();
|
||||
if (all.length === 0) {
|
||||
return reply.code(503).send({ error: "No bastions connected" });
|
||||
}
|
||||
return reply.code(404).send({ error: `MAC ${normalized} not found on any bastion and multiple bastions connected` });
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await sendCommand(target.bastionId, {
|
||||
type: "command-register",
|
||||
mac: normalized,
|
||||
hostname,
|
||||
role: role ?? "worker",
|
||||
ip: ip ?? "",
|
||||
mac, hostname, disk: disk ?? "/dev/sda", role: role ?? "infra", os: os ?? "fedora-43",
|
||||
});
|
||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||
} catch (err) {
|
||||
@@ -245,10 +174,10 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
|
||||
// Queue debug/rescue mode — route to correct bastion by MAC
|
||||
app.post<{
|
||||
Body: { mac?: string; pxeBoot?: boolean };
|
||||
Body: { mac?: string; sshd?: boolean };
|
||||
}>("/api/machines/debug", async (request, reply) => {
|
||||
const mac = (request.body?.mac ?? "").toLowerCase().replace(/-/g, ":");
|
||||
const pxeBoot = request.body?.pxeBoot ?? false;
|
||||
const sshd = request.body?.sshd ?? false;
|
||||
if (!mac) {
|
||||
return reply.code(400).send({ error: "mac is required" });
|
||||
}
|
||||
@@ -261,7 +190,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
}
|
||||
if (all.length === 1) {
|
||||
try {
|
||||
const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac, pxeBoot });
|
||||
const result = await sendCommand(all[0]!.bastionId, { type: "command-debug", mac, sshd });
|
||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||
} catch (err) {
|
||||
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
|
||||
@@ -271,7 +200,7 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac, pxeBoot });
|
||||
const result = await sendCommand(bastion.bastionId, { type: "command-debug", mac, sshd });
|
||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||
} catch (err) {
|
||||
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
|
||||
@@ -294,37 +223,6 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
}
|
||||
});
|
||||
|
||||
// Update hardware info (discovery data) for a machine
|
||||
app.post<{
|
||||
Body: {
|
||||
mac?: string; product?: string; board?: string; serial?: string;
|
||||
manufacturer?: string; cpu_model?: string; cpu_cores?: number;
|
||||
memory_gb?: number; arch?: string;
|
||||
disks?: Array<{ name: string; size_gb: number; model: string }>;
|
||||
nics?: Array<{ name: string; mac: string; state: string }>;
|
||||
};
|
||||
}>("/api/machines/discover", async (request, reply) => {
|
||||
const data = request.body ?? {};
|
||||
const mac = (data.mac ?? "").toLowerCase().replace(/-/g, ":");
|
||||
if (!mac) {
|
||||
return reply.code(400).send({ error: "mac is required" });
|
||||
}
|
||||
|
||||
const bastion = bastionRegistry.findBastionByMac(mac);
|
||||
const target = bastion ?? (bastionRegistry.getAll().length === 1 ? bastionRegistry.getAll()[0] : null);
|
||||
|
||||
if (!target) {
|
||||
return reply.code(503).send({ error: "No bastion found for this MAC" });
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await sendCommand(target.bastionId, { type: "command-discover", ...data, mac });
|
||||
return reply.code(result.status === "ok" ? 200 : 500).send(result);
|
||||
} catch (err) {
|
||||
return reply.code(500).send({ error: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
});
|
||||
|
||||
// Update role
|
||||
app.post<{
|
||||
Body: { mac?: string; role?: string };
|
||||
@@ -359,7 +257,17 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
const queued = bastion.state.install_queue[mac];
|
||||
const installed = bastion.state.installed[mac];
|
||||
|
||||
// Active install takes priority over old installed state (reprovision case)
|
||||
if (installed) {
|
||||
return {
|
||||
mac,
|
||||
hostname: installed.hostname,
|
||||
status: "installed",
|
||||
role: installed.role,
|
||||
ip: installed.ip,
|
||||
installed_at: installed.installed_at,
|
||||
};
|
||||
}
|
||||
|
||||
if (queued) {
|
||||
return {
|
||||
mac,
|
||||
@@ -374,17 +282,6 @@ export function registerBastionRoutes(app: FastifyInstance, db: DbClient): void
|
||||
};
|
||||
}
|
||||
|
||||
if (installed) {
|
||||
return {
|
||||
mac,
|
||||
hostname: installed.hostname,
|
||||
status: "installed",
|
||||
role: installed.role,
|
||||
ip: installed.ip,
|
||||
installed_at: installed.installed_at,
|
||||
};
|
||||
}
|
||||
|
||||
return reply.code(404).send({ error: `MAC ${mac} not found in install queue or installed` });
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,191 +0,0 @@
|
||||
// Environment and Account management routes.
|
||||
// GET/POST /api/environments — list/create environments
|
||||
// GET/POST /api/accounts — list/create accounts
|
||||
// POST /api/accounts/bind — bind account to environment
|
||||
// GET /api/bindings — list bindings
|
||||
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import type { PrismaClient, Prisma } from "@prisma/client";
|
||||
import type { RbacService } from "../services/rbac.js";
|
||||
import type { AuditService } from "../services/audit.js";
|
||||
|
||||
export function registerEnvironmentRoutes(
|
||||
app: FastifyInstance,
|
||||
db: PrismaClient,
|
||||
rbacService: RbacService,
|
||||
auditService: AuditService,
|
||||
): void {
|
||||
// List environments
|
||||
app.get("/api/environments", async (_request, reply) => {
|
||||
const envs = await db.environment.findMany({ orderBy: { name: "asc" } });
|
||||
return reply.send(envs);
|
||||
});
|
||||
|
||||
// Create environment
|
||||
app.post<{
|
||||
Body: { name?: string };
|
||||
}>("/api/environments", async (request, reply) => {
|
||||
const { name } = request.body ?? {};
|
||||
if (!name) {
|
||||
return reply.code(400).send({ error: "name is required" });
|
||||
}
|
||||
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "admin",
|
||||
resource: "environments",
|
||||
});
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
try {
|
||||
const env = await db.environment.create({ data: { name } });
|
||||
auditService.emit({
|
||||
eventKind: "resource_created",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userId: request.userId ?? null,
|
||||
resourceKind: "environment",
|
||||
resourceName: name,
|
||||
result: "success",
|
||||
});
|
||||
return reply.code(201).send(env);
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message.includes("Unique constraint")) {
|
||||
return reply.code(409).send({ error: `Environment '${name}' already exists` });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
|
||||
// List accounts
|
||||
app.get("/api/accounts", async (_request, reply) => {
|
||||
const accounts = await db.account.findMany({
|
||||
orderBy: { name: "asc" },
|
||||
select: { id: true, name: true, driver: true, config: true, createdAt: true, updatedAt: true },
|
||||
});
|
||||
return reply.send(accounts);
|
||||
});
|
||||
|
||||
// Create account
|
||||
app.post<{
|
||||
Body: { name?: string; driver?: string; config?: Record<string, unknown> };
|
||||
}>("/api/accounts", async (request, reply) => {
|
||||
const { name, driver, config } = request.body ?? {};
|
||||
if (!name || !driver) {
|
||||
return reply.code(400).send({ error: "name and driver are required" });
|
||||
}
|
||||
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "admin",
|
||||
resource: "accounts",
|
||||
});
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
try {
|
||||
const account = await db.account.create({
|
||||
data: { name, driver, config: (config ?? {}) as Prisma.InputJsonValue },
|
||||
});
|
||||
auditService.emit({
|
||||
eventKind: "resource_created",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userId: request.userId ?? null,
|
||||
resourceKind: "account",
|
||||
resourceName: name,
|
||||
result: "success",
|
||||
details: { driver },
|
||||
});
|
||||
return reply.code(201).send(account);
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message.includes("Unique constraint")) {
|
||||
return reply.code(409).send({ error: `Account '${name}' already exists` });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
|
||||
// Bind account to environment
|
||||
app.post<{
|
||||
Body: { environmentId?: string; accountId?: string };
|
||||
}>("/api/accounts/bind", async (request, reply) => {
|
||||
const { environmentId, accountId } = request.body ?? {};
|
||||
if (!environmentId || !accountId) {
|
||||
return reply.code(400).send({ error: "environmentId and accountId are required" });
|
||||
}
|
||||
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "admin",
|
||||
resource: "accounts",
|
||||
});
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
try {
|
||||
const binding = await db.binding.create({
|
||||
data: { environmentId, accountId },
|
||||
});
|
||||
return reply.code(201).send(binding);
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message.includes("Unique constraint")) {
|
||||
return reply.code(409).send({ error: "This account is already bound to this environment" });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
|
||||
// List bindings
|
||||
app.get("/api/bindings", async (_request, reply) => {
|
||||
const bindings = await db.binding.findMany({
|
||||
include: { environment: true, account: true },
|
||||
});
|
||||
return reply.send(bindings);
|
||||
});
|
||||
|
||||
// Audit event query
|
||||
app.get<{
|
||||
Querystring: {
|
||||
last?: string;
|
||||
kind?: string;
|
||||
env?: string;
|
||||
correlation?: string;
|
||||
limit?: string;
|
||||
};
|
||||
}>("/api/events", async (request, reply) => {
|
||||
const { last, kind, env, correlation, limit } = request.query as { last?: string; kind?: string; env?: string; correlation?: string; limit?: string };
|
||||
|
||||
const where: Record<string, unknown> = {};
|
||||
|
||||
if (last) {
|
||||
const match = last.match(/^(\d+)(h|d|m)$/);
|
||||
if (match) {
|
||||
const [, num, unit] = match;
|
||||
const ms = { h: 3_600_000, d: 86_400_000, m: 60_000 }[unit!]!;
|
||||
where.timestamp = { gte: new Date(Date.now() - parseInt(num!) * ms) };
|
||||
}
|
||||
}
|
||||
if (kind) where.eventKind = kind;
|
||||
if (env) where.environmentName = env;
|
||||
if (correlation) where.correlationId = correlation;
|
||||
|
||||
const events = await db.auditEvent.findMany({
|
||||
where,
|
||||
orderBy: { timestamp: "desc" },
|
||||
take: Math.min(parseInt(limit ?? "100"), 500),
|
||||
});
|
||||
|
||||
return reply.send(events);
|
||||
});
|
||||
}
|
||||
@@ -1,196 +0,0 @@
|
||||
// Resource CRUD routes with RBAC enforcement.
|
||||
// GET /api/resources — list (filtered by RBAC scope)
|
||||
// GET /api/resources/:id — get
|
||||
// POST /api/resources — create
|
||||
// PUT /api/resources/:id — update
|
||||
// DELETE /api/resources/:id — delete (marks as deleting)
|
||||
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import type { ResourceStore, CreateResourceInput } from "../services/resource-store.js";
|
||||
import type { RbacService } from "../services/rbac.js";
|
||||
import type { AuditService } from "../services/audit.js";
|
||||
import { resolveResourceKind } from "@lab/core";
|
||||
|
||||
export function registerResourceRoutes(
|
||||
app: FastifyInstance,
|
||||
resourceStore: ResourceStore,
|
||||
rbacService: RbacService,
|
||||
auditService: AuditService,
|
||||
): void {
|
||||
// List resources (filtered by kind, environment, status)
|
||||
app.get<{
|
||||
Querystring: { kind?: string; environment?: string; status?: string };
|
||||
}>("/api/resources", async (request, reply) => {
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "view",
|
||||
resource: request.query.kind ? resolveResourceKind(request.query.kind) : undefined,
|
||||
});
|
||||
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
const resources = await resourceStore.list({
|
||||
kind: request.query.kind ? resolveResourceKind(request.query.kind) : undefined,
|
||||
environmentId: request.query.environment,
|
||||
status: request.query.status,
|
||||
});
|
||||
|
||||
return reply.send(resources);
|
||||
});
|
||||
|
||||
// Get single resource
|
||||
app.get<{
|
||||
Params: { id: string };
|
||||
}>("/api/resources/:id", async (request, reply) => {
|
||||
const resource = await resourceStore.get(request.params.id);
|
||||
if (!resource) {
|
||||
return reply.code(404).send({ error: "Resource not found" });
|
||||
}
|
||||
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "view",
|
||||
resource: resource.kind,
|
||||
name: resource.name,
|
||||
});
|
||||
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
return reply.send(resource);
|
||||
});
|
||||
|
||||
// Create resource
|
||||
app.post<{
|
||||
Body: CreateResourceInput;
|
||||
}>("/api/resources", async (request, reply) => {
|
||||
const input = request.body;
|
||||
if (!input?.kind || !input?.name || !input?.environmentId || !input?.accountId) {
|
||||
return reply.code(400).send({ error: "kind, name, environmentId, and accountId are required" });
|
||||
}
|
||||
|
||||
const kind = resolveResourceKind(input.kind);
|
||||
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "create",
|
||||
resource: kind,
|
||||
});
|
||||
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
const correlationId = auditService.createCorrelation();
|
||||
|
||||
try {
|
||||
const resource = await resourceStore.create({ ...input, kind });
|
||||
|
||||
auditService.emit({
|
||||
eventKind: "resource_created",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userId: request.userId ?? null,
|
||||
userName: request.userEmail ?? null,
|
||||
resourceKind: kind,
|
||||
resourceName: input.name,
|
||||
correlationId,
|
||||
result: "success",
|
||||
});
|
||||
|
||||
return reply.code(201).send(resource);
|
||||
} catch (err) {
|
||||
// Prisma unique constraint violation
|
||||
if (err instanceof Error && err.message.includes("Unique constraint")) {
|
||||
return reply.code(409).send({ error: `Resource ${kind}/${input.name} already exists in this environment` });
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
|
||||
// Update resource
|
||||
app.put<{
|
||||
Params: { id: string };
|
||||
Body: { desiredSpec?: Record<string, unknown>; status?: string };
|
||||
}>("/api/resources/:id", async (request, reply) => {
|
||||
const resource = await resourceStore.get(request.params.id);
|
||||
if (!resource) {
|
||||
return reply.code(404).send({ error: "Resource not found" });
|
||||
}
|
||||
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "edit",
|
||||
resource: resource.kind,
|
||||
name: resource.name,
|
||||
});
|
||||
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
const updated = await resourceStore.update(request.params.id, request.body);
|
||||
|
||||
auditService.emit({
|
||||
eventKind: "resource_updated",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userId: request.userId ?? null,
|
||||
userName: request.userEmail ?? null,
|
||||
resourceKind: resource.kind,
|
||||
resourceName: resource.name,
|
||||
result: "success",
|
||||
});
|
||||
|
||||
return reply.send(updated);
|
||||
});
|
||||
|
||||
// Delete resource (marks as deleting)
|
||||
app.delete<{
|
||||
Params: { id: string };
|
||||
}>("/api/resources/:id", async (request, reply) => {
|
||||
const resource = await resourceStore.get(request.params.id);
|
||||
if (!resource) {
|
||||
return reply.code(404).send({ error: "Resource not found" });
|
||||
}
|
||||
|
||||
const rbac = await rbacService.check({
|
||||
userId: request.userId!,
|
||||
userEmail: request.userEmail!,
|
||||
userRole: request.userRole!,
|
||||
action: "delete",
|
||||
resource: resource.kind,
|
||||
name: resource.name,
|
||||
});
|
||||
|
||||
if (!rbac.allowed) {
|
||||
return reply.code(403).send({ error: rbac.reason });
|
||||
}
|
||||
|
||||
await resourceStore.delete(request.params.id);
|
||||
|
||||
auditService.emit({
|
||||
eventKind: "resource_deleted",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userId: request.userId ?? null,
|
||||
userName: request.userEmail ?? null,
|
||||
resourceKind: resource.kind,
|
||||
resourceName: resource.name,
|
||||
result: "success",
|
||||
});
|
||||
|
||||
return reply.send({ status: "deleting", id: request.params.id });
|
||||
});
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
// v2 Auth routes: bearer token login/logout.
|
||||
// POST /api/auth/login — email + password → session token
|
||||
// POST /api/auth/logout — revoke session
|
||||
|
||||
import type { FastifyInstance } from "fastify";
|
||||
import type { AuthService } from "../services/auth.js";
|
||||
import type { AuditService } from "../services/audit.js";
|
||||
import { AuthError } from "../services/auth.js";
|
||||
|
||||
export function registerV2AuthRoutes(
|
||||
app: FastifyInstance,
|
||||
authService: AuthService,
|
||||
auditService: AuditService,
|
||||
): void {
|
||||
app.post<{
|
||||
Body: { email?: string; password?: string };
|
||||
}>("/api/auth/login", async (request, reply) => {
|
||||
const { email, password } = request.body ?? {};
|
||||
|
||||
if (!email || !password) {
|
||||
return reply.code(400).send({ error: "email and password are required" });
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await authService.login(email, password);
|
||||
|
||||
auditService.emit({
|
||||
eventKind: result.isBootstrap ? "auth_bootstrap" : "auth_login",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userId: result.userId,
|
||||
userName: email,
|
||||
result: "success",
|
||||
details: { isBootstrap: result.isBootstrap },
|
||||
});
|
||||
|
||||
return reply.send({
|
||||
token: result.token,
|
||||
expiresAt: result.expiresAt.toISOString(),
|
||||
isBootstrap: result.isBootstrap,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof AuthError) {
|
||||
auditService.emit({
|
||||
eventKind: "auth_login",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userName: email,
|
||||
result: "failure",
|
||||
error: err.message,
|
||||
});
|
||||
return reply.code(401).send({ error: err.message });
|
||||
}
|
||||
return reply.code(500).send({ error: "Login failed" });
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/api/auth/logout", async (request, reply) => {
|
||||
const token = request.headers.authorization?.slice(7);
|
||||
if (!token) {
|
||||
return reply.code(400).send({ error: "Authorization header required" });
|
||||
}
|
||||
|
||||
try {
|
||||
await authService.logout(token);
|
||||
auditService.emit({
|
||||
eventKind: "auth_logout",
|
||||
source: "labd",
|
||||
verified: true,
|
||||
userId: request.userId ?? null,
|
||||
result: "success",
|
||||
});
|
||||
return reply.send({ status: "logged_out" });
|
||||
} catch (err) {
|
||||
if (err instanceof AuthError) {
|
||||
return reply.code(400).send({ error: err.message });
|
||||
}
|
||||
return reply.code(500).send({ error: "Logout failed" });
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
import Fastify from "fastify";
|
||||
import websocket from "@fastify/websocket";
|
||||
import type { PrismaClient } from "@prisma/client";
|
||||
import type { LabdConfig } from "./config.js";
|
||||
import { logger } from "./services/logger.js";
|
||||
import { registerHealthRoutes } from "./routes/health.js";
|
||||
@@ -10,16 +9,8 @@ import { registerServerRoutes } from "./routes/servers.js";
|
||||
import { registerAuthRoutes } from "./routes/auth.js";
|
||||
import { registerAgentRoutes } from "./routes/agents.js";
|
||||
import { registerBastionRoutes } from "./routes/bastions.js";
|
||||
import { registerV2AuthRoutes } from "./routes/v2-auth.js";
|
||||
import { registerEnvironmentRoutes } from "./routes/environments.js";
|
||||
import { registerResourceRoutes } from "./routes/resources.js";
|
||||
import { setupRateLimiting } from "./middleware/rate-limit.js";
|
||||
import { createBearerAuthMiddleware } from "./middleware/bearer-auth.js";
|
||||
import { bastionRegistry } from "./services/bastion-registry.js";
|
||||
import { AuthService } from "./services/auth.js";
|
||||
import { RbacService } from "./services/rbac.js";
|
||||
import { ResourceStore } from "./services/resource-store.js";
|
||||
import { AuditService } from "./services/audit.js";
|
||||
import { isBastionMessage } from "@lab/shared";
|
||||
|
||||
export interface DbClient {
|
||||
@@ -46,7 +37,6 @@ export interface DbClient {
|
||||
|
||||
export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
|
||||
app: ReturnType<typeof Fastify>;
|
||||
auditService: AuditService;
|
||||
}> {
|
||||
const app = Fastify({
|
||||
logger: false, // We use winston instead
|
||||
@@ -58,39 +48,13 @@ export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
|
||||
// Register WebSocket support
|
||||
void app.register(websocket);
|
||||
|
||||
// v2 services. The structural DbClient is a subset of the real PrismaClient;
|
||||
// at runtime db IS the PrismaClient instance, so the cast is safe. Tests that
|
||||
// exercise v2 routes provide a PrismaClient-shaped mock (see auth-bootstrap,
|
||||
// rbac-deny, audit-correlation tests).
|
||||
const prisma = db as unknown as PrismaClient;
|
||||
const authService = new AuthService(prisma);
|
||||
const rbacService = new RbacService(prisma);
|
||||
const resourceStore = new ResourceStore(prisma);
|
||||
const auditService = new AuditService(prisma);
|
||||
auditService.start();
|
||||
|
||||
// Register v1 (legacy) route handlers
|
||||
// Register route handlers
|
||||
registerHealthRoutes(app, db);
|
||||
registerServerRoutes(app, db);
|
||||
registerAuthRoutes(app, db);
|
||||
registerAgentRoutes(app);
|
||||
registerBastionRoutes(app, db);
|
||||
|
||||
// v2 routes live in a scope with bearer-auth as preHandler. Public paths
|
||||
// (login, /health, websockets) are skipped inside the middleware itself.
|
||||
// v1 routes above are unaffected — they're registered on the root scope.
|
||||
await app.register(async (scope) => {
|
||||
scope.addHook("preHandler", createBearerAuthMiddleware(authService));
|
||||
registerV2AuthRoutes(scope, authService, auditService);
|
||||
registerEnvironmentRoutes(scope, prisma, rbacService, auditService);
|
||||
registerResourceRoutes(scope, resourceStore, rbacService, auditService);
|
||||
});
|
||||
|
||||
// Flush pending audit events on shutdown so we never lose the last batch.
|
||||
app.addHook("onClose", async () => {
|
||||
auditService.stop();
|
||||
});
|
||||
|
||||
// WebSocket handler for agent connections
|
||||
app.register(async (fastify) => {
|
||||
fastify.get("/ws/agent", { websocket: true }, (socket, _request) => {
|
||||
@@ -228,9 +192,7 @@ export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
|
||||
labels: { cpu: hw.cpu_model, cores: hw.cpu_cores, memory_gb: hw.memory_gb, arch: hw.arch, product: hw.product, manufacturer: hw.manufacturer },
|
||||
},
|
||||
update: {
|
||||
// Leave status alone — a previously "online"/"offline" record
|
||||
// must not be downgraded to "discovered" just because the bastion
|
||||
// restarted and re-discovered the MAC via DHCP/PXE.
|
||||
status: "discovered",
|
||||
lastHeartbeat: new Date(),
|
||||
labels: { cpu: hw.cpu_model, cores: hw.cpu_cores, memory_gb: hw.memory_gb, arch: hw.arch, product: hw.product, manufacturer: hw.manufacturer },
|
||||
},
|
||||
@@ -303,5 +265,5 @@ export async function createApp(_config: LabdConfig, db: DbClient): Promise<{
|
||||
logger.info(`HTTP: ${request.ip} ${request.method} ${request.url}`);
|
||||
});
|
||||
|
||||
return { app, auditService };
|
||||
return { app };
|
||||
}
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
// Audit service: fire-and-forget event collection with batching.
|
||||
// Batches 50 events or flushes every 5 seconds, whichever comes first.
|
||||
// Failures never block the operation being audited.
|
||||
|
||||
import { randomBytes } from "node:crypto";
|
||||
import type { PrismaClient, Prisma } from "@prisma/client";
|
||||
import { logger } from "./logger.js";
|
||||
|
||||
const BATCH_SIZE = 50;
|
||||
const FLUSH_INTERVAL_MS = 5_000;
|
||||
|
||||
export interface AuditEventInput {
|
||||
eventKind: string;
|
||||
source: string;
|
||||
verified?: boolean;
|
||||
userId?: string | null;
|
||||
userName?: string | null;
|
||||
sessionId?: string | null;
|
||||
environmentName?: string | null;
|
||||
accountName?: string | null;
|
||||
resourceKind?: string | null;
|
||||
resourceName?: string | null;
|
||||
correlationId?: string | null;
|
||||
parentEventId?: string | null;
|
||||
details?: Record<string, unknown>;
|
||||
result: string;
|
||||
error?: string | null;
|
||||
durationMs?: number | null;
|
||||
}
|
||||
|
||||
export class AuditService {
|
||||
private batch: AuditEventInput[] = [];
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
constructor(private readonly db: PrismaClient) {}
|
||||
|
||||
start(): void {
|
||||
this.timer = setInterval(() => {
|
||||
void this.flush();
|
||||
}, FLUSH_INTERVAL_MS);
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (this.timer) {
|
||||
clearInterval(this.timer);
|
||||
this.timer = null;
|
||||
}
|
||||
void this.flush();
|
||||
}
|
||||
|
||||
emit(event: AuditEventInput): void {
|
||||
// Generate correlation ID if not provided
|
||||
if (!event.correlationId) {
|
||||
event.correlationId = `corr_${randomBytes(8).toString("hex")}`;
|
||||
}
|
||||
|
||||
this.batch.push(event);
|
||||
|
||||
if (this.batch.length >= BATCH_SIZE) {
|
||||
void this.flush();
|
||||
}
|
||||
}
|
||||
|
||||
/** Create a correlation context for a chain of related events. */
|
||||
createCorrelation(): string {
|
||||
return `corr_${randomBytes(8).toString("hex")}`;
|
||||
}
|
||||
|
||||
/** Flush all pending events synchronously. Tests await this; production
|
||||
* relies on the interval timer or stop() during shutdown. */
|
||||
async flushPending(): Promise<void> {
|
||||
await this.flush();
|
||||
}
|
||||
|
||||
private async flush(): Promise<void> {
|
||||
if (this.batch.length === 0) return;
|
||||
|
||||
const events = this.batch.splice(0);
|
||||
try {
|
||||
await this.db.auditEvent.createMany({
|
||||
data: events.map((e) => ({
|
||||
eventKind: e.eventKind,
|
||||
source: e.source,
|
||||
verified: e.verified ?? false,
|
||||
userId: e.userId ?? null,
|
||||
userName: e.userName ?? null,
|
||||
sessionId: e.sessionId ?? null,
|
||||
environmentName: e.environmentName ?? null,
|
||||
accountName: e.accountName ?? null,
|
||||
resourceKind: e.resourceKind ?? null,
|
||||
resourceName: e.resourceName ?? null,
|
||||
correlationId: e.correlationId ?? `corr_${randomBytes(8).toString("hex")}`,
|
||||
parentEventId: e.parentEventId ?? null,
|
||||
details: (e.details ?? {}) as Prisma.InputJsonValue,
|
||||
result: e.result,
|
||||
error: e.error ?? null,
|
||||
durationMs: e.durationMs ?? null,
|
||||
})),
|
||||
});
|
||||
logger.info(`AUDIT: flushed ${events.length} events`);
|
||||
} catch (err) {
|
||||
// Fire-and-forget: audit failures never block operations
|
||||
logger.warn(`AUDIT: failed to flush ${events.length} events: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
// Auth service: bearer token authentication with bootstrap flow.
|
||||
// First login creates the admin user. Subsequent logins return session tokens.
|
||||
|
||||
import { randomBytes } from "node:crypto";
|
||||
import bcrypt from "bcryptjs";
|
||||
import type { PrismaClient } from "@prisma/client";
|
||||
import { logger } from "./logger.js";
|
||||
|
||||
const SESSION_EXPIRY_DAYS = 30;
|
||||
const BCRYPT_ROUNDS = 12;
|
||||
|
||||
export interface LoginResult {
|
||||
token: string;
|
||||
expiresAt: Date;
|
||||
userId: string;
|
||||
isBootstrap: boolean;
|
||||
}
|
||||
|
||||
export class AuthService {
|
||||
constructor(private readonly db: PrismaClient) {}
|
||||
|
||||
async login(email: string, password: string): Promise<LoginResult> {
|
||||
const userCount = await this.db.user.count();
|
||||
|
||||
// Bootstrap: first login creates admin user
|
||||
if (userCount === 0) {
|
||||
return this.bootstrap(email, password);
|
||||
}
|
||||
|
||||
const user = await this.db.user.findUnique({ where: { email } });
|
||||
if (!user) {
|
||||
// Same error for unknown user and wrong password (no enumeration)
|
||||
throw new AuthError("Invalid email or password");
|
||||
}
|
||||
|
||||
const valid = await bcrypt.compare(password, user.password);
|
||||
if (!valid) {
|
||||
throw new AuthError("Invalid email or password");
|
||||
}
|
||||
|
||||
const session = await this.createSession(user.id);
|
||||
logger.info(`AUTH LOGIN: ${email} (${user.id.slice(0, 8)}...)`);
|
||||
|
||||
return {
|
||||
token: session.token,
|
||||
expiresAt: session.expiresAt,
|
||||
userId: user.id,
|
||||
isBootstrap: false,
|
||||
};
|
||||
}
|
||||
|
||||
async logout(token: string): Promise<void> {
|
||||
const session = await this.db.session.findUnique({ where: { token } });
|
||||
if (!session) {
|
||||
throw new AuthError("Invalid session");
|
||||
}
|
||||
await this.db.session.delete({ where: { id: session.id } });
|
||||
logger.info(`AUTH LOGOUT: session ${session.id.slice(0, 8)}...`);
|
||||
}
|
||||
|
||||
async validateToken(token: string): Promise<{ userId: string; email: string; role: string }> {
|
||||
const session = await this.db.session.findUnique({
|
||||
where: { token },
|
||||
include: { user: true },
|
||||
});
|
||||
|
||||
if (!session) {
|
||||
throw new AuthError("Invalid token");
|
||||
}
|
||||
if (session.expiresAt < new Date()) {
|
||||
await this.db.session.delete({ where: { id: session.id } });
|
||||
throw new AuthError("Token expired");
|
||||
}
|
||||
|
||||
return {
|
||||
userId: session.user.id,
|
||||
email: session.user.email,
|
||||
role: session.user.role,
|
||||
};
|
||||
}
|
||||
|
||||
private async bootstrap(email: string, password: string): Promise<LoginResult> {
|
||||
const hashed = await bcrypt.hash(password, BCRYPT_ROUNDS);
|
||||
const user = await this.db.user.create({
|
||||
data: {
|
||||
email,
|
||||
password: hashed,
|
||||
role: "ADMIN",
|
||||
name: email.split("@")[0] ?? null,
|
||||
},
|
||||
});
|
||||
|
||||
const session = await this.createSession(user.id);
|
||||
logger.info(`AUTH BOOTSTRAP: created admin user ${email} (${user.id.slice(0, 8)}...)`);
|
||||
|
||||
return {
|
||||
token: session.token,
|
||||
expiresAt: session.expiresAt,
|
||||
userId: user.id,
|
||||
isBootstrap: true,
|
||||
};
|
||||
}
|
||||
|
||||
private async createSession(userId: string) {
|
||||
const token = randomBytes(32).toString("hex");
|
||||
const expiresAt = new Date(Date.now() + SESSION_EXPIRY_DAYS * 24 * 60 * 60 * 1000);
|
||||
|
||||
return this.db.session.create({
|
||||
data: { userId, token, expiresAt },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export class AuthError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "AuthError";
|
||||
}
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
// RBAC service: environment-scoped permission checks.
|
||||
// Uses named RbacDefinition records with JSON subjects and roleBindings.
|
||||
//
|
||||
// Resolution flow:
|
||||
// 1. Find all RbacDefinitions where subjects match the current user/groups
|
||||
// 2. Collect all roleBindings from matching definitions
|
||||
// 3. Check if any binding grants the requested action on the requested resource
|
||||
|
||||
import type { PrismaClient } from "@prisma/client";
|
||||
import { logger } from "./logger.js";
|
||||
|
||||
export interface RbacCheck {
|
||||
userId: string;
|
||||
userEmail: string;
|
||||
userRole: string;
|
||||
action: string; // "view" | "edit" | "create" | "delete" | "run" | "admin"
|
||||
resource?: string | undefined; // "servers" | "databases" | "clusters" | "*"
|
||||
name?: string | undefined; // specific resource name
|
||||
environment?: string | undefined; // specific environment name
|
||||
}
|
||||
|
||||
export interface RbacResult {
|
||||
allowed: boolean;
|
||||
reason: string;
|
||||
matchedDefinition?: string;
|
||||
}
|
||||
|
||||
interface StoredSubject {
|
||||
kind: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
interface StoredBinding {
|
||||
role: string;
|
||||
resource?: string;
|
||||
name?: string;
|
||||
environment?: string;
|
||||
action?: string;
|
||||
}
|
||||
|
||||
export class RbacService {
|
||||
constructor(private readonly db: PrismaClient) {}
|
||||
|
||||
async check(req: RbacCheck): Promise<RbacResult> {
|
||||
// Admin users bypass RBAC
|
||||
if (req.userRole === "ADMIN") {
|
||||
return { allowed: true, reason: "admin role" };
|
||||
}
|
||||
|
||||
// Collect user's group memberships
|
||||
const memberships = await this.db.groupMember.findMany({
|
||||
where: { userId: req.userId },
|
||||
include: { group: true },
|
||||
});
|
||||
const groupNames = memberships.map((m) => m.group.name);
|
||||
|
||||
// Find all RBAC definitions
|
||||
const definitions = await this.db.rbacDefinition.findMany();
|
||||
|
||||
for (const def of definitions) {
|
||||
const subjects = def.subjects as unknown as StoredSubject[];
|
||||
const bindings = def.roleBindings as unknown as StoredBinding[];
|
||||
|
||||
// Check if this definition's subjects match the user
|
||||
const subjectMatch = subjects.some((s) => {
|
||||
if (s.kind === "User" && s.name === req.userEmail) return true;
|
||||
if (s.kind === "Group" && groupNames.includes(s.name)) return true;
|
||||
return false;
|
||||
});
|
||||
|
||||
if (!subjectMatch) continue;
|
||||
|
||||
// Check if any binding grants the requested permission
|
||||
for (const binding of bindings) {
|
||||
if (this.bindingMatches(binding, req)) {
|
||||
logger.info(`RBAC ALLOW: ${req.userEmail} ${req.action} ${req.resource ?? "*"}${req.name ? `/${req.name}` : ""} via ${def.name}`);
|
||||
return {
|
||||
allowed: true,
|
||||
reason: `granted by ${def.name}`,
|
||||
matchedDefinition: def.name,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(`RBAC DENY: ${req.userEmail} ${req.action} ${req.resource ?? "*"}${req.name ? `/${req.name}` : ""}`);
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `no matching role binding for ${req.action} on ${req.resource ?? "*"}`,
|
||||
};
|
||||
}
|
||||
|
||||
private bindingMatches(binding: StoredBinding, req: RbacCheck): boolean {
|
||||
// Check role grants the action
|
||||
if (!this.roleGrantsAction(binding.role, req.action)) return false;
|
||||
|
||||
// Check resource scope
|
||||
if (binding.resource && binding.resource !== "*" && binding.resource !== req.resource) return false;
|
||||
|
||||
// Check name scope
|
||||
if (binding.name && binding.name !== req.name) return false;
|
||||
|
||||
// Check environment scope
|
||||
if (binding.environment && binding.environment !== req.environment) return false;
|
||||
|
||||
// Check operation scope (for "run" role with specific actions)
|
||||
if (binding.action && binding.action !== "*" && binding.action !== req.action) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private roleGrantsAction(role: string, action: string): boolean {
|
||||
const grants: Record<string, string[]> = {
|
||||
admin: ["view", "edit", "create", "delete", "run", "admin"],
|
||||
edit: ["view", "edit", "create", "delete"],
|
||||
create: ["create"],
|
||||
delete: ["delete"],
|
||||
view: ["view"],
|
||||
run: ["run"],
|
||||
};
|
||||
return grants[role]?.includes(action) ?? false;
|
||||
}
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
// Resource store: CRUD for generic resources with origin/managedBy tracking.
|
||||
// All mutations go through this service so RBAC and audit are applied consistently.
|
||||
|
||||
import type { PrismaClient, Resource as PrismaResource, Prisma } from "@prisma/client";
|
||||
import { logger } from "./logger.js";
|
||||
|
||||
export interface CreateResourceInput {
|
||||
kind: string;
|
||||
name: string;
|
||||
environmentId: string;
|
||||
accountId: string;
|
||||
origin?: string;
|
||||
managedBy?: string;
|
||||
sourceRef?: string;
|
||||
desiredSpec: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface UpdateResourceInput {
|
||||
desiredSpec?: Record<string, unknown>;
|
||||
status?: string;
|
||||
statusMessage?: string;
|
||||
actualSpec?: Record<string, unknown>;
|
||||
platformRef?: string;
|
||||
}
|
||||
|
||||
export interface ListResourcesFilter {
|
||||
kind?: string | undefined;
|
||||
environmentId?: string | undefined;
|
||||
accountId?: string | undefined;
|
||||
status?: string | undefined;
|
||||
}
|
||||
|
||||
export class ResourceStore {
|
||||
constructor(private readonly db: PrismaClient) {}
|
||||
|
||||
async create(input: CreateResourceInput): Promise<PrismaResource> {
|
||||
const resource = await this.db.resource.create({
|
||||
data: {
|
||||
kind: input.kind,
|
||||
name: input.name,
|
||||
environmentId: input.environmentId,
|
||||
accountId: input.accountId,
|
||||
origin: input.origin ?? "cli",
|
||||
managedBy: input.managedBy ?? "manual",
|
||||
sourceRef: input.sourceRef ?? null,
|
||||
desiredSpec: input.desiredSpec as Prisma.InputJsonValue,
|
||||
status: "pending",
|
||||
},
|
||||
});
|
||||
|
||||
logger.info(`RESOURCE CREATED: ${input.kind}/${input.name} in env ${input.environmentId.slice(0, 8)}...`);
|
||||
return resource;
|
||||
}
|
||||
|
||||
async get(id: string): Promise<PrismaResource | null> {
|
||||
return this.db.resource.findUnique({ where: { id } });
|
||||
}
|
||||
|
||||
async getByKindNameEnv(kind: string, name: string, environmentId: string): Promise<PrismaResource | null> {
|
||||
return this.db.resource.findUnique({
|
||||
where: { kind_name_environmentId: { kind, name, environmentId } },
|
||||
});
|
||||
}
|
||||
|
||||
async list(filter: ListResourcesFilter = {}): Promise<PrismaResource[]> {
|
||||
return this.db.resource.findMany({
|
||||
where: {
|
||||
...(filter.kind ? { kind: filter.kind } : {}),
|
||||
...(filter.environmentId ? { environmentId: filter.environmentId } : {}),
|
||||
...(filter.accountId ? { accountId: filter.accountId } : {}),
|
||||
...(filter.status ? { status: filter.status } : {}),
|
||||
},
|
||||
orderBy: { createdAt: "desc" },
|
||||
});
|
||||
}
|
||||
|
||||
async update(id: string, input: UpdateResourceInput): Promise<PrismaResource> {
|
||||
const data: Prisma.ResourceUpdateInput = {};
|
||||
if (input.desiredSpec !== undefined) data.desiredSpec = input.desiredSpec as Prisma.InputJsonValue;
|
||||
if (input.status !== undefined) data.status = input.status;
|
||||
if (input.statusMessage !== undefined) data.statusMessage = input.statusMessage;
|
||||
if (input.actualSpec !== undefined) data.actualSpec = input.actualSpec as Prisma.InputJsonValue;
|
||||
if (input.platformRef !== undefined) data.platformRef = input.platformRef;
|
||||
if (input.status === "ready") data.lastReconciled = new Date();
|
||||
|
||||
const resource = await this.db.resource.update({ where: { id }, data });
|
||||
|
||||
logger.info(`RESOURCE UPDATED: ${resource.kind}/${resource.name} -> ${input.status ?? "spec change"}`);
|
||||
return resource;
|
||||
}
|
||||
|
||||
async delete(id: string): Promise<void> {
|
||||
const resource = await this.db.resource.findUnique({ where: { id } });
|
||||
if (!resource) return;
|
||||
|
||||
// Mark as deleting first (driver handles actual deletion)
|
||||
await this.db.resource.update({
|
||||
where: { id },
|
||||
data: { status: "deleting" },
|
||||
});
|
||||
|
||||
logger.info(`RESOURCE DELETING: ${resource.kind}/${resource.name}`);
|
||||
}
|
||||
|
||||
async hardDelete(id: string): Promise<void> {
|
||||
await this.db.resource.delete({ where: { id } });
|
||||
}
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import Fastify from "fastify";
|
||||
import { registerBastionRoutes } from "../src/routes/bastions.js";
|
||||
import { bastionRegistry } from "../src/services/bastion-registry.js";
|
||||
import type { DbClient } from "../src/server.js";
|
||||
import type { BastionState } from "@lab/shared";
|
||||
|
||||
function createMockDb(servers: unknown[] = []): DbClient {
|
||||
return {
|
||||
$queryRaw: vi.fn().mockResolvedValue([{ "?column?": 1 }]),
|
||||
server: {
|
||||
findMany: vi.fn().mockResolvedValue(servers),
|
||||
findUnique: vi.fn().mockResolvedValue(null),
|
||||
upsert: vi.fn().mockResolvedValue({}),
|
||||
},
|
||||
joinToken: {
|
||||
findUnique: vi.fn().mockResolvedValue(null),
|
||||
findMany: vi.fn().mockResolvedValue([]),
|
||||
create: vi.fn().mockResolvedValue({ id: "t" }),
|
||||
update: vi.fn().mockResolvedValue({}),
|
||||
},
|
||||
bastion: {
|
||||
upsert: vi.fn().mockResolvedValue({}),
|
||||
findMany: vi.fn().mockResolvedValue([]),
|
||||
findUnique: vi.fn().mockResolvedValue(null),
|
||||
update: vi.fn().mockResolvedValue({}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function registerFakeBastion(bastionId: string, state: BastionState): void {
|
||||
bastionRegistry.register({
|
||||
bastionId,
|
||||
hostname: "fake",
|
||||
network: "192.168.8.0/24",
|
||||
serverIp: "192.168.8.11",
|
||||
// socket is referenced only on commands, not during aggregation
|
||||
socket: { on: () => undefined, off: () => undefined, send: () => undefined, close: () => undefined } as never,
|
||||
connectedAt: new Date(),
|
||||
lastHeartbeat: new Date(),
|
||||
state,
|
||||
});
|
||||
}
|
||||
|
||||
describe("GET /api/machines aggregation", () => {
|
||||
beforeEach(() => {
|
||||
for (const b of bastionRegistry.getAll()) bastionRegistry.unregister(b.bastionId);
|
||||
});
|
||||
|
||||
it("promotes a live-discovered MAC to installed when the DB has a real hostname+role for it", async () => {
|
||||
// Simulates the worker0-k8s0 bug: bastion restarted, lost its installed map,
|
||||
// rediscovered the machine via DHCP/PXE. DB still has hostname=worker0-k8s0,
|
||||
// role=infra, ip=192.168.8.23. Without the fix, the CLI sees a "discovered"
|
||||
// row with no hostname/role/IP. With the fix, the row is promoted to
|
||||
// "installed" with full identity preserved.
|
||||
const mac = "78:55:36:08:28:fb";
|
||||
registerFakeBastion("b1", {
|
||||
discovered: {
|
||||
[mac]: {
|
||||
mac, product: "SER", board: "SER", serial: "x", manufacturer: "AZW",
|
||||
cpu_model: "AMD Ryzen 7 255", cpu_cores: 16, memory_gb: 58, arch: "x86_64",
|
||||
disks: [], nics: [], first_seen: "", last_seen: "",
|
||||
},
|
||||
},
|
||||
install_queue: {},
|
||||
installed: {},
|
||||
debug: {},
|
||||
});
|
||||
|
||||
const app = Fastify({ logger: false });
|
||||
const db = createMockDb([
|
||||
{ mac, hostname: "worker0-k8s0", role: "infra", ip: "192.168.8.23", status: "discovered", labels: {} },
|
||||
]);
|
||||
registerBastionRoutes(app, db);
|
||||
|
||||
const res = await app.inject({ method: "GET", url: "/api/machines" });
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = JSON.parse(res.body);
|
||||
|
||||
expect(body.discovered[mac]).toBeUndefined();
|
||||
expect(body.installed[mac]).toMatchObject({
|
||||
hostname: "worker0-k8s0",
|
||||
role: "infra",
|
||||
ip: "192.168.8.23",
|
||||
cpu_model: "AMD Ryzen 7 255",
|
||||
cpu_cores: 16,
|
||||
memory_gb: 58,
|
||||
});
|
||||
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it("leaves a fresh-discovery MAC in discovered when DB only has a discovery-shaped record", async () => {
|
||||
const mac = "aa:bb:cc:dd:ee:ff";
|
||||
registerFakeBastion("b1", {
|
||||
discovered: {
|
||||
[mac]: {
|
||||
mac, product: "SER", board: "SER", serial: "x", manufacturer: "AZW",
|
||||
cpu_model: "AMD Ryzen 7", cpu_cores: 8, memory_gb: 32, arch: "x86_64",
|
||||
disks: [], nics: [], first_seen: "", last_seen: "",
|
||||
},
|
||||
},
|
||||
install_queue: {},
|
||||
installed: {},
|
||||
debug: {},
|
||||
});
|
||||
|
||||
const app = Fastify({ logger: false });
|
||||
// Matches what labd writes on first discovery: hostname=product, role="unknown"
|
||||
const db = createMockDb([
|
||||
{ mac, hostname: "SER", role: "unknown", ip: null, status: "discovered", labels: {} },
|
||||
]);
|
||||
registerBastionRoutes(app, db);
|
||||
|
||||
const res = await app.inject({ method: "GET", url: "/api/machines" });
|
||||
const body = JSON.parse(res.body);
|
||||
|
||||
expect(body.discovered[mac]).toBeDefined();
|
||||
expect(body.installed[mac]).toBeUndefined();
|
||||
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it("falls back to DB for MACs not in any live bucket", async () => {
|
||||
const mac = "11:22:33:44:55:66";
|
||||
// No bastions connected
|
||||
const app = Fastify({ logger: false });
|
||||
const db = createMockDb([
|
||||
{ mac, hostname: "worker1-k8s0", role: "infra", ip: "192.168.8.13", status: "online", labels: {} },
|
||||
]);
|
||||
registerBastionRoutes(app, db);
|
||||
|
||||
const res = await app.inject({ method: "GET", url: "/api/machines" });
|
||||
const body = JSON.parse(res.body);
|
||||
|
||||
expect(body.installed[mac]).toMatchObject({
|
||||
hostname: "worker1-k8s0",
|
||||
role: "infra",
|
||||
ip: "192.168.8.13",
|
||||
});
|
||||
|
||||
await app.close();
|
||||
});
|
||||
});
|
||||
@@ -1,425 +0,0 @@
|
||||
// End-to-end smoke tests for the v2.0 Phase 1 surface (auth bootstrap, RBAC,
|
||||
// audit correlation). These exercise the wiring in createApp(): the bearer
|
||||
// auth middleware, the v2 routes scope, and the AuditService lifecycle.
|
||||
//
|
||||
// We don't spin up CockroachDB. Instead we provide a PrismaClient-shaped
|
||||
// in-memory mock that matches the surface the v2 services actually touch.
|
||||
// Tests follow the project convention of using mock DBs + Fastify.inject().
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
|
||||
import bcrypt from "bcryptjs";
|
||||
import { createApp } from "../src/server.js";
|
||||
import type { DbClient } from "../src/server.js";
|
||||
import type { AuditService } from "../src/services/audit.js";
|
||||
|
||||
const TEST_CONFIG = { port: 0, host: "127.0.0.1", databaseUrl: "", caDir: "/tmp", logLevel: "silent" };
|
||||
|
||||
interface UserRow { id: string; email: string; password: string; role: string; name: string | null; }
|
||||
interface SessionRow { id: string; userId: string; token: string; expiresAt: Date; user?: UserRow; }
|
||||
interface RbacDefRow { id: string; name: string; subjects: unknown; roleBindings: unknown; }
|
||||
interface AuditEventRow {
|
||||
id: string;
|
||||
eventKind: string;
|
||||
source: string;
|
||||
verified: boolean;
|
||||
userId: string | null;
|
||||
userName: string | null;
|
||||
environmentName: string | null;
|
||||
resourceKind: string | null;
|
||||
correlationId: string | null;
|
||||
parentEventId: string | null;
|
||||
details: unknown;
|
||||
result: string;
|
||||
error: string | null;
|
||||
durationMs: number | null;
|
||||
timestamp: Date;
|
||||
}
|
||||
|
||||
interface Stores {
|
||||
users: Map<string, UserRow>;
|
||||
sessions: Map<string, SessionRow>;
|
||||
groupMembers: Array<{ userId: string; group: { name: string } }>;
|
||||
rbacDefs: RbacDefRow[];
|
||||
auditEvents: AuditEventRow[];
|
||||
resources: Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
function makeStores(): Stores {
|
||||
return {
|
||||
users: new Map(),
|
||||
sessions: new Map(),
|
||||
groupMembers: [],
|
||||
rbacDefs: [],
|
||||
auditEvents: [],
|
||||
resources: [],
|
||||
};
|
||||
}
|
||||
|
||||
function makeMockDb(s: Stores): DbClient {
|
||||
let idCounter = 0;
|
||||
const newId = (prefix: string): string => `${prefix}-${++idCounter}`;
|
||||
|
||||
return {
|
||||
$queryRaw: vi.fn(async () => [{ "?column?": 1 }]),
|
||||
server: { findMany: vi.fn(async () => []), findUnique: vi.fn(), upsert: vi.fn() },
|
||||
joinToken: { findUnique: vi.fn(), findMany: vi.fn(), create: vi.fn(), update: vi.fn() },
|
||||
bastion: { upsert: vi.fn(), findMany: vi.fn(), findUnique: vi.fn(), update: vi.fn() },
|
||||
|
||||
user: {
|
||||
count: vi.fn(async () => s.users.size),
|
||||
findUnique: vi.fn(async (args: { where: { email?: string; id?: string } }) => {
|
||||
if (args.where.email) {
|
||||
for (const u of s.users.values()) if (u.email === args.where.email) return u;
|
||||
}
|
||||
if (args.where.id) return s.users.get(args.where.id) ?? null;
|
||||
return null;
|
||||
}),
|
||||
create: vi.fn(async (args: { data: Omit<UserRow, "id"> }) => {
|
||||
const id = newId("user");
|
||||
const row: UserRow = { id, ...args.data };
|
||||
s.users.set(id, row);
|
||||
return row;
|
||||
}),
|
||||
},
|
||||
session: {
|
||||
findUnique: vi.fn(async (args: { where: { token?: string; id?: string }; include?: { user?: boolean } }) => {
|
||||
let session: SessionRow | undefined;
|
||||
if (args.where.token) {
|
||||
for (const sess of s.sessions.values()) if (sess.token === args.where.token) { session = sess; break; }
|
||||
} else if (args.where.id) {
|
||||
session = s.sessions.get(args.where.id);
|
||||
}
|
||||
if (!session) return null;
|
||||
if (args.include?.user) {
|
||||
return { ...session, user: s.users.get(session.userId)! };
|
||||
}
|
||||
return session;
|
||||
}),
|
||||
create: vi.fn(async (args: { data: { userId: string; token: string; expiresAt: Date } }) => {
|
||||
const id = newId("sess");
|
||||
const row: SessionRow = { id, ...args.data };
|
||||
s.sessions.set(id, row);
|
||||
return row;
|
||||
}),
|
||||
delete: vi.fn(async (args: { where: { id: string } }) => {
|
||||
s.sessions.delete(args.where.id);
|
||||
return null;
|
||||
}),
|
||||
},
|
||||
groupMember: {
|
||||
findMany: vi.fn(async (args: { where: { userId: string } }) =>
|
||||
s.groupMembers.filter((m) => m.userId === args.where.userId),
|
||||
),
|
||||
},
|
||||
rbacDefinition: {
|
||||
findMany: vi.fn(async () => s.rbacDefs),
|
||||
},
|
||||
auditEvent: {
|
||||
createMany: vi.fn(async (args: { data: Array<Omit<AuditEventRow, "id" | "timestamp">> }) => {
|
||||
const ts = new Date();
|
||||
for (const e of args.data) {
|
||||
s.auditEvents.push({ id: newId("evt"), timestamp: ts, ...e });
|
||||
}
|
||||
return { count: args.data.length };
|
||||
}),
|
||||
findMany: vi.fn(async (args: { where?: Record<string, unknown>; orderBy?: unknown; take?: number }) => {
|
||||
const where = args.where ?? {};
|
||||
const filtered = s.auditEvents.filter((e) => {
|
||||
if (where["eventKind"] && e.eventKind !== where["eventKind"]) return false;
|
||||
if (where["correlationId"] && e.correlationId !== where["correlationId"]) return false;
|
||||
if (where["environmentName"] && e.environmentName !== where["environmentName"]) return false;
|
||||
return true;
|
||||
});
|
||||
return filtered.slice(0, args.take ?? 100);
|
||||
}),
|
||||
},
|
||||
resource: {
|
||||
findMany: vi.fn(async () => s.resources),
|
||||
findUnique: vi.fn(),
|
||||
create: vi.fn(),
|
||||
update: vi.fn(),
|
||||
delete: vi.fn(),
|
||||
},
|
||||
environment: { findMany: vi.fn(async () => []), findUnique: vi.fn(), create: vi.fn() },
|
||||
account: { findMany: vi.fn(async () => []), findUnique: vi.fn(), create: vi.fn() },
|
||||
binding: { findMany: vi.fn(async () => []), create: vi.fn() },
|
||||
} as unknown as DbClient;
|
||||
}
|
||||
|
||||
async function buildApp(s: Stores) {
|
||||
const db = makeMockDb(s);
|
||||
const result = await createApp(TEST_CONFIG, db);
|
||||
await result.app.ready();
|
||||
return result;
|
||||
}
|
||||
|
||||
describe("v2 auth: bootstrap flow", () => {
|
||||
let stores: Stores;
|
||||
let app: Awaited<ReturnType<typeof buildApp>>["app"];
|
||||
let auditService: AuditService;
|
||||
|
||||
beforeEach(async () => {
|
||||
stores = makeStores();
|
||||
const built = await buildApp(stores);
|
||||
app = built.app;
|
||||
auditService = built.auditService;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await app.close(); // triggers auditService.stop()
|
||||
});
|
||||
|
||||
it("first login with no users seeds the admin and returns a session token", async () => {
|
||||
expect(stores.users.size).toBe(0);
|
||||
|
||||
const resp = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/auth/login",
|
||||
payload: { email: "admin@itaz.eu", password: "s3cret-pw" },
|
||||
});
|
||||
|
||||
expect(resp.statusCode).toBe(200);
|
||||
const body = resp.json();
|
||||
expect(body.isBootstrap).toBe(true);
|
||||
expect(body.token).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect(typeof body.expiresAt).toBe("string");
|
||||
|
||||
expect(stores.users.size).toBe(1);
|
||||
const created = [...stores.users.values()][0]!;
|
||||
expect(created.email).toBe("admin@itaz.eu");
|
||||
expect(created.role).toBe("ADMIN");
|
||||
// Password is hashed, not stored plaintext.
|
||||
expect(created.password).not.toBe("s3cret-pw");
|
||||
expect(await bcrypt.compare("s3cret-pw", created.password)).toBe(true);
|
||||
|
||||
// Bootstrap emits an audit event.
|
||||
await auditService.flushPending();
|
||||
const bootstrapEvents = stores.auditEvents.filter((e) => e.eventKind === "auth_bootstrap");
|
||||
expect(bootstrapEvents).toHaveLength(1);
|
||||
expect(bootstrapEvents[0]!.result).toBe("success");
|
||||
expect(bootstrapEvents[0]!.userName).toBe("admin@itaz.eu");
|
||||
});
|
||||
|
||||
it("returns 400 for missing credentials", async () => {
|
||||
const resp = await app.inject({ method: "POST", url: "/api/auth/login", payload: {} });
|
||||
expect(resp.statusCode).toBe(400);
|
||||
});
|
||||
|
||||
it("second login uses normal flow (no isBootstrap)", async () => {
|
||||
// Bootstrap once
|
||||
await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/auth/login",
|
||||
payload: { email: "admin@itaz.eu", password: "s3cret-pw" },
|
||||
});
|
||||
expect(stores.users.size).toBe(1);
|
||||
|
||||
// Login again
|
||||
const resp = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/auth/login",
|
||||
payload: { email: "admin@itaz.eu", password: "s3cret-pw" },
|
||||
});
|
||||
|
||||
expect(resp.statusCode).toBe(200);
|
||||
expect(resp.json().isBootstrap).toBe(false);
|
||||
expect(stores.users.size).toBe(1); // no new user
|
||||
});
|
||||
|
||||
it("rejects wrong password with 401", async () => {
|
||||
// Seed admin
|
||||
await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/auth/login",
|
||||
payload: { email: "admin@itaz.eu", password: "s3cret-pw" },
|
||||
});
|
||||
|
||||
const resp = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/auth/login",
|
||||
payload: { email: "admin@itaz.eu", password: "wrong" },
|
||||
});
|
||||
expect(resp.statusCode).toBe(401);
|
||||
|
||||
// Failed login is also audited.
|
||||
await auditService.flushPending();
|
||||
const fails = stores.auditEvents.filter((e) => e.eventKind === "auth_login" && e.result === "failure");
|
||||
expect(fails).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("v2 RBAC: env-scoped denial", () => {
|
||||
let stores: Stores;
|
||||
let app: Awaited<ReturnType<typeof buildApp>>["app"];
|
||||
|
||||
async function seedSession(role: string): Promise<string> {
|
||||
stores.users.set("u-1", {
|
||||
id: "u-1",
|
||||
email: `${role.toLowerCase()}@itaz.eu`,
|
||||
password: "x",
|
||||
role,
|
||||
name: null,
|
||||
});
|
||||
const token = "test-token-" + role;
|
||||
stores.sessions.set("s-1", {
|
||||
id: "s-1",
|
||||
userId: "u-1",
|
||||
token,
|
||||
expiresAt: new Date(Date.now() + 86_400_000),
|
||||
});
|
||||
return token;
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
stores = makeStores();
|
||||
app = (await buildApp(stores)).app;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it("non-admin user with no role bindings gets 403 on /api/resources", async () => {
|
||||
const token = await seedSession("EDITOR"); // not admin, no bindings
|
||||
|
||||
const resp = await app.inject({
|
||||
method: "GET",
|
||||
url: "/api/resources",
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
});
|
||||
|
||||
expect(resp.statusCode).toBe(403);
|
||||
expect(resp.json().error).toMatch(/no matching role binding/);
|
||||
});
|
||||
|
||||
it("missing/empty bearer token gets 401 (auth, not RBAC)", async () => {
|
||||
const r1 = await app.inject({ method: "GET", url: "/api/resources" });
|
||||
expect(r1.statusCode).toBe(401);
|
||||
|
||||
const r2 = await app.inject({
|
||||
method: "GET",
|
||||
url: "/api/resources",
|
||||
headers: { authorization: "Bearer " },
|
||||
});
|
||||
expect(r2.statusCode).toBe(401);
|
||||
});
|
||||
|
||||
it("invalid bearer token gets 401", async () => {
|
||||
const resp = await app.inject({
|
||||
method: "GET",
|
||||
url: "/api/resources",
|
||||
headers: { authorization: "Bearer not-a-real-token" },
|
||||
});
|
||||
expect(resp.statusCode).toBe(401);
|
||||
});
|
||||
|
||||
it("admin role bypasses RBAC", async () => {
|
||||
const token = await seedSession("ADMIN");
|
||||
|
||||
const resp = await app.inject({
|
||||
method: "GET",
|
||||
url: "/api/resources",
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
});
|
||||
|
||||
expect(resp.statusCode).toBe(200);
|
||||
expect(resp.json()).toEqual([]);
|
||||
});
|
||||
|
||||
it("user with binding for env A is denied for resources in env B", async () => {
|
||||
const token = await seedSession("EDITOR");
|
||||
stores.groupMembers.push({ userId: "u-1", group: { name: "team-a" } });
|
||||
stores.rbacDefs.push({
|
||||
id: "rbac-1",
|
||||
name: "team-a-edit-on-env-a",
|
||||
subjects: [{ kind: "Group", name: "team-a" }],
|
||||
roleBindings: [{ role: "edit", environment: "env-a" }],
|
||||
});
|
||||
|
||||
// List in env-a → should pass RBAC (no env query so it's global view, but
|
||||
// the binding scope is environment-specific → for global list the binding
|
||||
// doesn't apply when an environment scope is set on the binding).
|
||||
// Smoke test the targeted denial: trying to create in env-b is rejected.
|
||||
const respB = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/resources",
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
payload: { kind: "database", name: "x", environmentId: "env-b", accountId: "acc-1" },
|
||||
});
|
||||
|
||||
expect(respB.statusCode).toBe(403);
|
||||
expect(respB.json().error).toMatch(/no matching role binding/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("v2 audit: correlation chain visible via /api/events", () => {
|
||||
let stores: Stores;
|
||||
let app: Awaited<ReturnType<typeof buildApp>>["app"];
|
||||
let auditService: AuditService;
|
||||
|
||||
beforeEach(async () => {
|
||||
stores = makeStores();
|
||||
const built = await buildApp(stores);
|
||||
app = built.app;
|
||||
auditService = built.auditService;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it("emitted audit events are queryable by correlation id", async () => {
|
||||
// Seed admin so /api/events is accessible (it sits behind bearer auth)
|
||||
const loginResp = await app.inject({
|
||||
method: "POST",
|
||||
url: "/api/auth/login",
|
||||
payload: { email: "admin@itaz.eu", password: "pw" },
|
||||
});
|
||||
const token = loginResp.json().token;
|
||||
|
||||
// Force flush so the bootstrap event is in the DB
|
||||
await auditService.flushPending();
|
||||
|
||||
expect(stores.auditEvents.length).toBeGreaterThan(0);
|
||||
const bootstrap = stores.auditEvents.find((e) => e.eventKind === "auth_bootstrap")!;
|
||||
expect(bootstrap.correlationId).toMatch(/^corr_[a-f0-9]{16}$/);
|
||||
|
||||
// Query /api/events filtered by correlation id
|
||||
const queryResp = await app.inject({
|
||||
method: "GET",
|
||||
url: `/api/events?correlation=${bootstrap.correlationId}`,
|
||||
headers: { authorization: `Bearer ${token}` },
|
||||
});
|
||||
|
||||
expect(queryResp.statusCode).toBe(200);
|
||||
const events = queryResp.json() as Array<{ correlationId: string; eventKind: string }>;
|
||||
expect(events.length).toBe(1);
|
||||
expect(events[0]!.eventKind).toBe("auth_bootstrap");
|
||||
expect(events[0]!.correlationId).toBe(bootstrap.correlationId);
|
||||
});
|
||||
|
||||
it("explicit parent/child correlation chain is preserved across emits", async () => {
|
||||
const correlationId = auditService.createCorrelation();
|
||||
|
||||
auditService.emit({
|
||||
eventKind: "test_parent",
|
||||
source: "test",
|
||||
result: "success",
|
||||
correlationId,
|
||||
});
|
||||
auditService.emit({
|
||||
eventKind: "test_child",
|
||||
source: "test",
|
||||
result: "success",
|
||||
correlationId,
|
||||
parentEventId: "evt-1",
|
||||
});
|
||||
|
||||
await auditService.flushPending();
|
||||
|
||||
const chain = stores.auditEvents.filter((e) => e.correlationId === correlationId);
|
||||
expect(chain).toHaveLength(2);
|
||||
expect(chain.map((e) => e.eventKind).sort()).toEqual(["test_child", "test_parent"]);
|
||||
expect(chain.find((e) => e.eventKind === "test_child")!.parentEventId).toBe("evt-1");
|
||||
});
|
||||
});
|
||||
@@ -1,42 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// Render /etc/rancher/k3s/config.yaml using the PRODUCTION generator.
|
||||
//
|
||||
// Exists so labsim (and anything else) can produce the exact config.yaml a node
|
||||
// would get from `labctl install`, without an SSH/OperationContext. Driving the
|
||||
// rehearsal through this means the sim tests the same code path production runs
|
||||
// -- if generateServerConfig ever changes shape, the sim moves with it.
|
||||
//
|
||||
// All input via env, so a sim's cloud-init or a shell can call it plainly:
|
||||
//
|
||||
// ROLE=infra HOSTNAME=k8s1 IP=172.31.2.11 render-config.ts # cluster-init server
|
||||
// ROLE=infra HOSTNAME=k8s2 IP=172.31.2.12 \
|
||||
// K3S_SERVER_URL=https://172.31.2.11:6443 K3S_TOKEN=... render-config.ts # joining server
|
||||
// ROLE=worker HOSTNAME=k8s4 IP=172.31.2.14 K3S_SERVER_URL=... K3S_TOKEN=... render-config.ts # agent
|
||||
//
|
||||
// Dual-stack is opt-in and matches K3sConfig exactly: set IPV6, CLUSTER_CIDR,
|
||||
// SERVICE_CIDR (comma-separated families) and the generator emits the dual
|
||||
// node-ip + CIDRs. Omit them and the output is byte-identical to a v4-only node.
|
||||
import { generateServerConfig, generateAgentConfig } from "../src/operations/k3s-config.js";
|
||||
import type { K3sConfig } from "../src/types.js";
|
||||
import type { Role } from "@lab/shared";
|
||||
|
||||
const env = process.env;
|
||||
const role = (env.ROLE ?? "worker") as Role;
|
||||
const isServer = role === "infra" || role === "labcontroller";
|
||||
|
||||
const splitCsv = (v: string | undefined): string[] | undefined =>
|
||||
v ? v.split(",").map((s) => s.trim()).filter(Boolean) : undefined;
|
||||
|
||||
const cfg: K3sConfig = {
|
||||
hostname: env.HOSTNAME ?? "node",
|
||||
ip: env.IP ?? (() => { throw new Error("IP is required"); })(),
|
||||
role,
|
||||
k3sServerUrl: env.K3S_SERVER_URL,
|
||||
k3sToken: env.K3S_TOKEN,
|
||||
tlsSans: splitCsv(env.TLS_SANS),
|
||||
ipv6: env.IPV6,
|
||||
clusterCidr: splitCsv(env.CLUSTER_CIDR),
|
||||
serviceCidr: splitCsv(env.SERVICE_CIDR),
|
||||
};
|
||||
|
||||
process.stdout.write(isServer ? generateServerConfig(cfg) : generateAgentConfig(cfg));
|
||||
@@ -1,22 +1,18 @@
|
||||
// Hardening: Pod Security Standards, certificate check, journald cap, storage.
|
||||
// Hardening: Pod Security Standards, certificate check, log rotation.
|
||||
|
||||
import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
|
||||
import { runSequential } from "../utils.js";
|
||||
import { applyPodSecurityStandards } from "../operations/pod-security.js";
|
||||
import { checkCertExpiry } from "../operations/cert-check.js";
|
||||
import { configureLogRotation } from "../operations/log-rotation.js";
|
||||
import { configureJournaldLimits } from "../operations/journald-limits.js";
|
||||
import { configureLonghornDisk } from "../operations/longhorn-disk.js";
|
||||
|
||||
export const hardeningGroup: OperationGroup = {
|
||||
name: "hardening",
|
||||
description: "Pod security, certificate check, journald cap, storage",
|
||||
description: "Pod security, certificate check, log rotation",
|
||||
operations: [
|
||||
{ name: "Apply Pod Security Standards", fn: applyPodSecurityStandards },
|
||||
{ name: "Check certificate expiry", fn: checkCertExpiry },
|
||||
{ name: "Decommission file-based audit logs", fn: configureLogRotation },
|
||||
{ name: "Configure journald disk cap", fn: configureJournaldLimits },
|
||||
{ name: "Configure Longhorn disk", fn: configureLonghornDisk },
|
||||
{ name: "Configure log rotation", fn: configureLogRotation },
|
||||
],
|
||||
};
|
||||
|
||||
|
||||
@@ -1,26 +1,22 @@
|
||||
// Host preparation: kernel modules, sysctl, swap, storage, firewall, SELinux.
|
||||
// Host preparation: kernel modules, sysctl, swap, firewall, SELinux.
|
||||
|
||||
import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
|
||||
import { runSequential } from "../utils.js";
|
||||
import { loadKernelModules } from "../operations/kernel-modules.js";
|
||||
import { applyCisHardening } from "../operations/sysctl.js";
|
||||
import { enableSwap } from "../operations/swap.js";
|
||||
import { growRancherLv } from "../operations/rancher-storage.js";
|
||||
import { disableSwap } from "../operations/swap.js";
|
||||
import { disableFirewall } from "../operations/firewall.js";
|
||||
import { setSelinuxPermissive } from "../operations/selinux.js";
|
||||
import { enableIscsi } from "../operations/iscsi.js";
|
||||
|
||||
export const hostPrepGroup: OperationGroup = {
|
||||
name: "host-prep",
|
||||
description: "Prepare host for k3s: kernel modules, sysctl, swap, imageFs sizing, firewall, SELinux, iSCSI",
|
||||
description: "Prepare host for k3s: kernel modules, sysctl, swap, firewall, SELinux",
|
||||
operations: [
|
||||
{ name: "Load kernel modules", fn: loadKernelModules },
|
||||
{ name: "Apply CIS sysctl", fn: applyCisHardening },
|
||||
{ name: "Enable swap", fn: enableSwap },
|
||||
{ name: "Grow rancher LV", fn: growRancherLv },
|
||||
{ name: "Disable swap", fn: disableSwap },
|
||||
{ name: "Disable firewall", fn: disableFirewall },
|
||||
{ name: "Set SELinux permissive", fn: setSelinuxPermissive },
|
||||
{ name: "Enable iSCSI", fn: enableIscsi },
|
||||
],
|
||||
};
|
||||
|
||||
|
||||
@@ -3,8 +3,6 @@
|
||||
import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
|
||||
import { runSequential } from "../utils.js";
|
||||
import { installCilium } from "../operations/cilium.js";
|
||||
import { installMultus } from "../operations/multus.js";
|
||||
import { installVlanSetup } from "../operations/vlan-setup.js";
|
||||
import { fixCoreDnsUpstream } from "../operations/dns-fix.js";
|
||||
import { applyDefaultNetworkPolicies } from "../operations/network-policy.js";
|
||||
|
||||
@@ -13,11 +11,6 @@ export const networkingGroup: OperationGroup = {
|
||||
description: "Install Cilium CNI, fix DNS, apply network policies",
|
||||
operations: [
|
||||
{ name: "Install Cilium CNI", fn: installCilium },
|
||||
// Multus + vlan-setup: give pods a second interface on VLAN 10 (macvlan)
|
||||
// for LAN device discovery (Matter/HomeKit mDNS). Must follow Cilium
|
||||
// (needs cni.exclusive=false + bpf.vlanBypass={10} from installCilium).
|
||||
{ name: "Install Multus CNI", fn: installMultus },
|
||||
{ name: "Install vlan-setup (lan10 + CNI plugins)", fn: installVlanSetup },
|
||||
{ name: "Fix CoreDNS upstream", fn: fixCoreDnsUpstream },
|
||||
{ name: "Apply network policies", fn: applyDefaultNetworkPolicies },
|
||||
],
|
||||
|
||||
@@ -76,6 +76,7 @@ sed -i 's/^SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config 2>/dev/nul
|
||||
# ── 5b. Create k3s config directory ──
|
||||
echo "[5/10] Writing k3s server configuration..."
|
||||
mkdir -p /etc/rancher/k3s
|
||||
mkdir -p /var/log/kubernetes
|
||||
|
||||
cat > /etc/rancher/k3s/config.yaml << 'K3S_CONFIG'
|
||||
# k3s server configuration — CIS hardened
|
||||
@@ -90,10 +91,13 @@ disable:
|
||||
- servicelb
|
||||
- traefik
|
||||
|
||||
# API server hardening (audit-log-path=- routes audit to journald via stdout)
|
||||
# API server hardening
|
||||
kube-apiserver-arg:
|
||||
- "anonymous-auth=false"
|
||||
- "audit-log-path=-"
|
||||
- "audit-log-path=/var/log/kubernetes/audit.log"
|
||||
- "audit-log-maxage=30"
|
||||
- "audit-log-maxbackup=10"
|
||||
- "audit-log-maxsize=100"
|
||||
- "audit-policy-file=/etc/rancher/k3s/audit-policy.yaml"
|
||||
- "enable-admission-plugins=NodeRestriction,PodSecurity"
|
||||
- "request-timeout=300s"
|
||||
@@ -215,9 +219,7 @@ echo " Using network device: \$DEFAULT_DEV"
|
||||
|
||||
KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install \\
|
||||
--set kubeProxyReplacement=true \\
|
||||
--set ipam.mode=cluster-pool \\
|
||||
--set ipam.operator.clusterPoolIPv4PodCIDRList='{10.42.0.0/16}' \\
|
||||
--set ipam.operator.clusterPoolIPv4MaskSize=24 \\
|
||||
--set ipam.mode=kubernetes \\
|
||||
--set devices="\$DEFAULT_DEV" \\
|
||||
--set nodePort.directRoutingDevice="\$DEFAULT_DEV"
|
||||
|
||||
|
||||
@@ -78,10 +78,9 @@ export class K3sModule implements Module {
|
||||
return toModuleResult("install", [...prepResults, ...k3sResults], start);
|
||||
}
|
||||
|
||||
// Phase 3: Networking (initial server only — joining servers get Cilium via daemonset)
|
||||
// Phase 3: Networking (server only — agents don't install Cilium)
|
||||
let netResults: OperationResult[] = [];
|
||||
const isJoiningServer = isServer && !!opCtx.config.k3sServerUrl;
|
||||
if (isServer && !isJoiningServer) {
|
||||
if (isServer) {
|
||||
netResults = await runNetworking(opCtx);
|
||||
}
|
||||
|
||||
|
||||
@@ -35,25 +35,21 @@ export const installCilium: Operation = async (ctx): Promise<OperationResult> =>
|
||||
}
|
||||
details.push(`Installed cilium CLI ${version} (${cliArch})`);
|
||||
|
||||
// Detect default network device (avoid tailscale/wireguard)
|
||||
const devResult = await ctx.ssh.exec(
|
||||
"ip -4 route show default | awk '{print $5}' | head -1",
|
||||
sshOpts(ctx),
|
||||
);
|
||||
const defaultDev = devResult.stdout.trim();
|
||||
details.push(`Network device: ${defaultDev}`);
|
||||
|
||||
// Install Cilium
|
||||
// - No hardcoded devices: Cilium auto-detects per node (heterogeneous NICs like eno1 vs enP7s7)
|
||||
// - k8sServiceHost/Port: k3s agents proxy the API on 127.0.0.1:6444 (not 6443)
|
||||
// - cni.exclusive=false: required so Multus can install its CNI config alongside
|
||||
// Cilium (Cilium otherwise deletes any non-Cilium CNI conf).
|
||||
// - bpf.vlanBypass={10}: allow VLAN 10 (LoT) tagged traffic through the eBPF
|
||||
// host VLAN filter, so pods on a macvlan/VLAN-10 interface receive multicast
|
||||
// (Matter/mDNS ff02::fb + 224.0.0.251). Without this Cilium drops it
|
||||
// ("VLAN traffic disallowed by VLAN filter", bpf_host.c).
|
||||
const installResult = await ctx.ssh.exec(
|
||||
`KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install \
|
||||
--set kubeProxyReplacement=true \
|
||||
--set ipam.mode=cluster-pool \
|
||||
--set ipam.operator.clusterPoolIPv4PodCIDRList='{10.42.0.0/16}' \
|
||||
--set ipam.operator.clusterPoolIPv4MaskSize=24 \
|
||||
--set k8sServiceHost=127.0.0.1 \
|
||||
--set k8sServicePort=6444 \
|
||||
--set cni.exclusive=false \
|
||||
--set bpf.vlanBypass="{10}"`,
|
||||
--set ipam.mode=kubernetes \
|
||||
--set devices="${defaultDev}" \
|
||||
--set nodePort.directRoutingDevice="${defaultDev}"`,
|
||||
{ timeoutMs: 300_000 },
|
||||
);
|
||||
if (installResult.exitCode !== 0) {
|
||||
|
||||
@@ -1,194 +0,0 @@
|
||||
// Recover a broken etcd member by removing it from the cluster, wiping its
|
||||
// local state, and restarting k3s so it rejoins as a fresh member.
|
||||
//
|
||||
// Use case: a node panics on startup with
|
||||
// "tocommit(N+1) is out of range [lastIndex(N)]. Was the raft log corrupted,
|
||||
// truncated, or lost?"
|
||||
// This means the local raft WAL is missing the last entry the leader thinks
|
||||
// the follower acknowledged (lost write, unclean shutdown, etc). The fix is
|
||||
// always the same and well-documented; this codifies it so we don't fumble
|
||||
// the procedure under pressure.
|
||||
//
|
||||
// Preconditions:
|
||||
// - At least one healthy peer is reachable so the cluster has quorum after
|
||||
// we remove the broken member. (For a 3-node cluster: 2 healthy. For a
|
||||
// 5-node: 3 healthy.) If quorum would be lost, this function refuses.
|
||||
// - SSH access to both the broken node and a healthy peer.
|
||||
// - etcdctl available on the healthy peer (k3s does not bundle it; the
|
||||
// procedure installs it on demand on Fedora).
|
||||
|
||||
import type { SshClient } from "../types.js";
|
||||
|
||||
const ETCD_TLS = {
|
||||
ca: "/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt",
|
||||
cert: "/var/lib/rancher/k3s/server/tls/etcd/server-client.crt",
|
||||
key: "/var/lib/rancher/k3s/server/tls/etcd/server-client.key",
|
||||
} as const;
|
||||
|
||||
const SSH_TIMEOUT = 60_000;
|
||||
|
||||
export interface RecoverEtcdMemberOptions {
|
||||
/** SSH client for the broken node (the one panicking). */
|
||||
broken: SshClient;
|
||||
/** SSH client for any healthy server peer in the same cluster. */
|
||||
peer: SshClient;
|
||||
/** Hostname (k8s node name) of the broken node. Used to find its etcd member id. */
|
||||
brokenHostname: string;
|
||||
/** Logger for progress output. */
|
||||
log?: (msg: string) => void;
|
||||
}
|
||||
|
||||
export interface RecoverEtcdMemberResult {
|
||||
success: boolean;
|
||||
changed: boolean;
|
||||
message: string;
|
||||
/** New etcd member id assigned after rejoin (when known). */
|
||||
newMemberId?: string;
|
||||
/** Old etcd member id that was removed. */
|
||||
removedMemberId?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
function etcdctl(subcmd: string): string {
|
||||
return [
|
||||
"ETCDCTL_API=3 etcdctl",
|
||||
`--cacert=${ETCD_TLS.ca}`,
|
||||
`--cert=${ETCD_TLS.cert}`,
|
||||
`--key=${ETCD_TLS.key}`,
|
||||
"--endpoints=https://127.0.0.1:2379",
|
||||
"--command-timeout=10s",
|
||||
subcmd,
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
async function ensureEtcdctl(peer: SshClient): Promise<void> {
|
||||
const probe = await peer.exec("command -v etcdctl 2>/dev/null", { timeoutMs: 5_000 });
|
||||
if (probe.exitCode === 0 && probe.stdout.trim()) return;
|
||||
// Best-effort install on Fedora. If the host isn't dnf-based, surface the
|
||||
// error to the caller via the next etcdctl invocation.
|
||||
await peer.exec("dnf install -y etcd 2>&1", { timeoutMs: 120_000 });
|
||||
}
|
||||
|
||||
async function getMemberList(peer: SshClient): Promise<Array<{ id: string; name: string }>> {
|
||||
const result = await peer.exec(etcdctl("member list"), { timeoutMs: SSH_TIMEOUT });
|
||||
if (result.exitCode !== 0) {
|
||||
throw new Error(`etcdctl member list failed: ${result.stderr || result.stdout}`);
|
||||
}
|
||||
// Format: <hex-id>, started, <name>, <peer-urls>, <client-urls>, <isLearner>
|
||||
return result.stdout
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const [id, , name] = line.split(",").map((p) => p.trim());
|
||||
return { id: id ?? "", name: name ?? "" };
|
||||
})
|
||||
.filter((m) => m.id);
|
||||
}
|
||||
|
||||
export async function recoverEtcdMember(
|
||||
opts: RecoverEtcdMemberOptions,
|
||||
): Promise<RecoverEtcdMemberResult> {
|
||||
const log = opts.log ?? (() => {});
|
||||
|
||||
try {
|
||||
log(`Looking up etcd member id for ${opts.brokenHostname} via peer...`);
|
||||
await ensureEtcdctl(opts.peer);
|
||||
|
||||
const members = await getMemberList(opts.peer);
|
||||
if (members.length < 3) {
|
||||
return {
|
||||
success: false,
|
||||
changed: false,
|
||||
message: "Refusing to remove a member from a cluster with <3 members (quorum would be lost)",
|
||||
error: `member count = ${members.length}`,
|
||||
};
|
||||
}
|
||||
|
||||
// Member names are <hostname>-<random-suffix>; match by hostname prefix.
|
||||
const broken = members.find((m) => m.name.startsWith(opts.brokenHostname));
|
||||
if (!broken) {
|
||||
return {
|
||||
success: false,
|
||||
changed: false,
|
||||
message: `No etcd member found matching hostname ${opts.brokenHostname}`,
|
||||
error: `members: ${members.map((m) => m.name).join(", ")}`,
|
||||
};
|
||||
}
|
||||
log(`Broken member: ${broken.id} (${broken.name})`);
|
||||
|
||||
log("Step 1/4: stopping k3s on broken node");
|
||||
await opts.broken.exec("systemctl stop k3s 2>&1", { timeoutMs: SSH_TIMEOUT });
|
||||
|
||||
log("Step 2/4: removing broken etcd member from cluster");
|
||||
const remove = await opts.peer.exec(
|
||||
etcdctl(`member remove ${broken.id}`),
|
||||
{ timeoutMs: SSH_TIMEOUT },
|
||||
);
|
||||
if (remove.exitCode !== 0) {
|
||||
return {
|
||||
success: false,
|
||||
changed: false,
|
||||
message: "etcdctl member remove failed",
|
||||
error: remove.stderr || remove.stdout,
|
||||
removedMemberId: broken.id,
|
||||
};
|
||||
}
|
||||
|
||||
log("Step 3/4: archiving corrupt etcd state and stale TLS/cred dirs on broken node");
|
||||
const ts = Math.floor(Date.now() / 1000);
|
||||
await opts.broken.exec(
|
||||
[
|
||||
`mv /var/lib/rancher/k3s/server/db /var/lib/rancher/k3s/server/db.corrupt-${ts} 2>/dev/null || true`,
|
||||
"rm -rf /var/lib/rancher/k3s/server/tls /var/lib/rancher/k3s/server/cred",
|
||||
].join(" && "),
|
||||
{ timeoutMs: SSH_TIMEOUT },
|
||||
);
|
||||
|
||||
log("Step 4/4: starting k3s on broken node — it will rejoin");
|
||||
await opts.broken.exec("systemctl start k3s 2>&1", { timeoutMs: SSH_TIMEOUT });
|
||||
|
||||
// Poll for rejoin. The new member-id is what the cluster assigns on join.
|
||||
let newMemberId: string | undefined;
|
||||
for (let i = 0; i < 60; i++) {
|
||||
await new Promise((r) => setTimeout(r, 5_000));
|
||||
try {
|
||||
const after = await getMemberList(opts.peer);
|
||||
const rejoined = after.find(
|
||||
(m) => m.name.startsWith(opts.brokenHostname) && m.id !== broken.id,
|
||||
);
|
||||
if (rejoined) {
|
||||
newMemberId = rejoined.id;
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
// peer may briefly be unreachable mid-rejoin — keep polling
|
||||
}
|
||||
}
|
||||
|
||||
if (!newMemberId) {
|
||||
return {
|
||||
success: false,
|
||||
changed: true,
|
||||
message: "k3s started but new member did not appear in cluster within 5 minutes",
|
||||
removedMemberId: broken.id,
|
||||
};
|
||||
}
|
||||
|
||||
log(`Rejoined as ${newMemberId}`);
|
||||
return {
|
||||
success: true,
|
||||
changed: true,
|
||||
message: `Recovered: removed ${broken.id}, rejoined as ${newMemberId}`,
|
||||
removedMemberId: broken.id,
|
||||
newMemberId,
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
success: false,
|
||||
changed: false,
|
||||
message: "Recovery failed",
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,26 +1,15 @@
|
||||
export { loadKernelModules } from "./kernel-modules.js";
|
||||
export { applyCisHardening } from "./sysctl.js";
|
||||
export { enableSwap } from "./swap.js";
|
||||
export { growRancherLv } from "./rancher-storage.js";
|
||||
export { enableIscsi } from "./iscsi.js";
|
||||
export { disableSwap } from "./swap.js";
|
||||
export { disableFirewall } from "./firewall.js";
|
||||
export { setSelinuxPermissive } from "./selinux.js";
|
||||
export { writeK3sConfig, generateServerConfig, generateAgentConfig } from "./k3s-config.js";
|
||||
export { writeK3sConfig } from "./k3s-config.js";
|
||||
export { writeAuditPolicy } from "./audit-policy.js";
|
||||
export { cleanupStaleCni } from "./cni-cleanup.js";
|
||||
export { installK3sBinary } from "./k3s-install.js";
|
||||
export { installCilium } from "./cilium.js";
|
||||
export { installMultus } from "./multus.js";
|
||||
export { installVlanSetup } from "./vlan-setup.js";
|
||||
export { fixCoreDnsUpstream } from "./dns-fix.js";
|
||||
export { configureLogRotation } from "./log-rotation.js";
|
||||
export { configureJournaldLimits } from "./journald-limits.js";
|
||||
export { applyDefaultNetworkPolicies } from "./network-policy.js";
|
||||
export { applyPodSecurityStandards } from "./pod-security.js";
|
||||
export { checkCertExpiry } from "./cert-check.js";
|
||||
export { configureLonghornDisk } from "./longhorn-disk.js";
|
||||
export { recoverEtcdMember } from "./etcd-recover.js";
|
||||
export type {
|
||||
RecoverEtcdMemberOptions,
|
||||
RecoverEtcdMemberResult,
|
||||
} from "./etcd-recover.js";
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
// Install and enable iSCSI initiator (required by Longhorn storage).
|
||||
// Fedora: iscsi-initiator-utils, Ubuntu: open-iscsi
|
||||
|
||||
import type { Operation, OperationResult } from "../types.js";
|
||||
import { sshOpts } from "../utils.js";
|
||||
|
||||
export const enableIscsi: Operation = async (ctx): Promise<OperationResult> => {
|
||||
// Check if iscsid is already running
|
||||
const check = await ctx.ssh.exec("systemctl is-active iscsid 2>/dev/null", sshOpts(ctx));
|
||||
if (check.stdout.trim() === "active") {
|
||||
return { success: true, changed: false, message: "iSCSI already active" };
|
||||
}
|
||||
|
||||
// Install the package (detect distro)
|
||||
const osRelease = await ctx.ssh.exec("cat /etc/os-release", sshOpts(ctx));
|
||||
const osLower = osRelease.stdout.toLowerCase();
|
||||
const isFedora = osLower.includes("fedora") || osLower.includes("rhel") || osLower.includes("centos");
|
||||
|
||||
const pkg = isFedora ? "iscsi-initiator-utils" : "open-iscsi";
|
||||
const installCmd = isFedora ? `sudo dnf install -y ${pkg}` : `sudo apt-get install -y ${pkg}`;
|
||||
|
||||
const install = await ctx.ssh.exec(installCmd, { timeoutMs: 120_000 });
|
||||
if (install.exitCode !== 0) {
|
||||
return { success: false, changed: false, message: `Failed to install ${pkg}`, error: install.stderr.trim() };
|
||||
}
|
||||
|
||||
// Enable and start
|
||||
await ctx.ssh.exec("sudo systemctl enable --now iscsid", sshOpts(ctx));
|
||||
|
||||
return { success: true, changed: true, message: `Installed ${pkg} and enabled iscsid` };
|
||||
};
|
||||
@@ -1,33 +0,0 @@
|
||||
// Cap journald disk usage so audit logs (which now flow through journald via
|
||||
// kube-apiserver's stdout) cannot fill /var/log. Default journald uses up to
|
||||
// 10% of the filesystem, capped at 4 GB. In a /var/log of ~10 GB shared with
|
||||
// other services, that's still room for audit volume to evict useful logs.
|
||||
// 2 GB / 200 MB-per-file is a comfortable middle.
|
||||
|
||||
import type { Operation, OperationResult } from "../types.js";
|
||||
import { sshOpts, writeRemoteFile } from "../utils.js";
|
||||
|
||||
const DROPIN_CONTENT = `[Journal]
|
||||
SystemMaxUse=2G
|
||||
SystemKeepFree=1G
|
||||
SystemMaxFileSize=200M
|
||||
`;
|
||||
|
||||
const DROPIN_PATH = "/etc/systemd/journald.conf.d/10-k3s-audit-cap.conf";
|
||||
|
||||
export const configureJournaldLimits: Operation = async (ctx): Promise<OperationResult> => {
|
||||
const changed = await writeRemoteFile(ctx, DROPIN_PATH, DROPIN_CONTENT);
|
||||
if (changed) {
|
||||
// Reload journald so the new limit applies without a reboot.
|
||||
await ctx.ssh.exec(
|
||||
"systemctl kill --signal=SIGUSR2 systemd-journald 2>/dev/null; " +
|
||||
"systemctl restart systemd-journald 2>&1 || true",
|
||||
sshOpts(ctx),
|
||||
);
|
||||
}
|
||||
return {
|
||||
success: true,
|
||||
changed,
|
||||
message: changed ? "journald limits configured (2 GB cap)" : "journald limits already configured",
|
||||
};
|
||||
};
|
||||
@@ -7,65 +7,10 @@ function isServerRole(role: string): boolean {
|
||||
return role === "infra" || role === "labcontroller";
|
||||
}
|
||||
|
||||
/**
|
||||
* The address-family block: `cluster-cidr`, `service-cidr` and `node-ip`.
|
||||
*
|
||||
* Emitted ONLY when the corresponding config is supplied, and that is
|
||||
* deliberate. With no `ipv6` and no CIDRs this returns "", so the generated
|
||||
* file is byte-identical to what every existing node already has -- no diff,
|
||||
* so `writeRemoteFile` reports unchanged and nothing restarts k3s. Dual-stack
|
||||
* is therefore opt-in per node rather than a flag day.
|
||||
*
|
||||
* `node-ip` is only written once there is a second family to name. k3s
|
||||
* auto-detects a sensible IPv4 on its own, and writing it out unconditionally
|
||||
* would rewrite the config of five healthy nodes to tell them what they had
|
||||
* already worked out.
|
||||
*/
|
||||
function addressFamilyLines(config: K3sConfig, opts: { cidrs: boolean }): string {
|
||||
const lines: string[] = [];
|
||||
if (opts.cidrs && config.clusterCidr?.length) {
|
||||
lines.push(`cluster-cidr: "${config.clusterCidr.join(",")}"`);
|
||||
}
|
||||
if (opts.cidrs && config.serviceCidr?.length) {
|
||||
lines.push(`service-cidr: "${config.serviceCidr.join(",")}"`);
|
||||
}
|
||||
if (config.ipv6) {
|
||||
// Order matters to k3s: the FIRST entry is the primary family, and the
|
||||
// supported single-to-dual-stack conversion is the one that preserves it.
|
||||
// IPv4 stays primary so existing Services keep their ClusterIP.
|
||||
lines.push(`node-ip: "${config.ip},${config.ipv6}"`);
|
||||
}
|
||||
return lines.length ? `${lines.join("\n")}\n` : "";
|
||||
}
|
||||
|
||||
// Exported so the exact production config.yaml can be rendered outside an SSH
|
||||
// context -- notably by the labsim 3-server-etcd rehearsal, which must drive its
|
||||
// nodes through THIS generator rather than a parallel set of INSTALL_K3S_EXEC
|
||||
// flags, or it proves a mechanism production does not run.
|
||||
export function generateServerConfig(config: K3sConfig): string {
|
||||
// The IPv6 address goes in the cert too. Without it, anything that reaches
|
||||
// this apiserver over v6 -- a peer server joining, or kubectl against the v6
|
||||
// address -- fails TLS verification, and the error names the certificate
|
||||
// rather than the missing SAN, which is a long way from the cause.
|
||||
const tlsSans = [
|
||||
config.hostname,
|
||||
config.ip,
|
||||
...(config.ipv6 ? [config.ipv6] : []),
|
||||
...(config.tlsSans ?? []),
|
||||
];
|
||||
const isJoining = !!config.k3sServerUrl;
|
||||
const clusterLines = isJoining
|
||||
? `server: "${config.k3sServerUrl}"\ntoken: "${config.k3sToken}"`
|
||||
: "cluster-init: true";
|
||||
// audit-log-path=- routes audit events to k3s.service's stdout, which systemd
|
||||
// forwards to journald. journald enforces its own size caps (see
|
||||
// configureJournaldLimits) so audit volume cannot fill the disk. File-based
|
||||
// audit logs led to /var/log/kubernetes growing to 7+ GB because apiserver's
|
||||
// own rotation produced files that any logrotate glob would double-rotate
|
||||
// and never expire.
|
||||
return `# k3s server configuration — CIS hardened, etcd HA
|
||||
${clusterLines}
|
||||
${addressFamilyLines(config, { cidrs: true })}protect-kernel-defaults: true
|
||||
function generateServerConfig(config: K3sConfig): string {
|
||||
const tlsSans = [config.hostname, config.ip, ...(config.tlsSans ?? [])];
|
||||
return `# k3s server configuration — CIS hardened
|
||||
protect-kernel-defaults: true
|
||||
secrets-encryption: true
|
||||
write-kubeconfig-mode: "0640"
|
||||
|
||||
@@ -75,12 +20,12 @@ disable:
|
||||
- servicelb
|
||||
- traefik
|
||||
|
||||
node-label:
|
||||
- "node.longhorn.io/create-default-disk=config"
|
||||
|
||||
kube-apiserver-arg:
|
||||
- "anonymous-auth=false"
|
||||
- "audit-log-path=-"
|
||||
- "audit-log-path=/var/log/kubernetes/audit.log"
|
||||
- "audit-log-maxage=30"
|
||||
- "audit-log-maxbackup=10"
|
||||
- "audit-log-maxsize=100"
|
||||
- "audit-policy-file=/etc/rancher/k3s/audit-policy.yaml"
|
||||
- "enable-admission-plugins=NodeRestriction,PodSecurity"
|
||||
- "request-timeout=300s"
|
||||
@@ -95,16 +40,8 @@ ${tlsSans.map((s) => ` - "${s}"`).join("\n")}
|
||||
`;
|
||||
}
|
||||
|
||||
// Takes the config now: an agent needs its own dual `node-ip` just as much as a
|
||||
// server does. Without one it joins as an IPv4-only node into a dual-stack
|
||||
// cluster, gets no IPv6 pod CIDR, and the failure surfaces later as pods on that
|
||||
// node being unreachable over v6 while the node itself reads Ready.
|
||||
// It takes no cluster/service CIDRs -- those are server-side only.
|
||||
export function generateAgentConfig(config: K3sConfig): string {
|
||||
return `${addressFamilyLines(config, { cidrs: false })}protect-kernel-defaults: true
|
||||
node-label:
|
||||
- "node-role.kubernetes.io/worker=true"
|
||||
- "node.longhorn.io/create-default-disk=config"
|
||||
function generateAgentConfig(): string {
|
||||
return `protect-kernel-defaults: true
|
||||
kubelet-arg:
|
||||
- "protect-kernel-defaults=true"
|
||||
- "streaming-connection-idle-timeout=5m"
|
||||
@@ -113,41 +50,11 @@ kubelet-arg:
|
||||
}
|
||||
|
||||
export const writeK3sConfig: Operation = async (ctx): Promise<OperationResult> => {
|
||||
// Refuse to name an address the node does not have.
|
||||
//
|
||||
// Most of this estate is SSH-onboard, not PXE-provisioned: Asahi cannot PXE
|
||||
// at all, and the DGX Sparks run NVIDIA's own OS and must never be
|
||||
// reinstalled. For those nodes the install templates govern nothing and this
|
||||
// module is the ONLY labctl touchpoint, so nothing upstream can guarantee the
|
||||
// vendor OS actually took a DHCPv6 lease.
|
||||
//
|
||||
// Writing node-ip for a missing address does not fail here -- it fails later,
|
||||
// when k3s will not start, with an error about binding rather than about
|
||||
// addressing. Checking costs one ssh round trip and turns a confusing
|
||||
// start-up failure into a sentence naming the address and the node.
|
||||
if (ctx.config.ipv6) {
|
||||
const probe = await ctx.ssh.exec(
|
||||
`ip -6 -o addr show 2>/dev/null | grep -qF " ${ctx.config.ipv6}/" && echo present || true`,
|
||||
sshOpts(ctx),
|
||||
);
|
||||
if (!probe.stdout.includes("present")) {
|
||||
return {
|
||||
success: false,
|
||||
changed: false,
|
||||
message: `Node does not have IPv6 ${ctx.config.ipv6}`,
|
||||
error:
|
||||
`k3s config would set node-ip to ${ctx.config.ipv6}, but that address is not on any ` +
|
||||
`interface. k3s resolves node-ip at start-up and would fail to bind. Check the node ` +
|
||||
`took its DHCPv6 lease (a kea reservation keyed on its MAC) before retrying.`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
await ctx.ssh.exec("mkdir -p /etc/rancher/k3s", sshOpts(ctx));
|
||||
await ctx.ssh.exec("mkdir -p /etc/rancher/k3s /var/log/kubernetes", sshOpts(ctx));
|
||||
|
||||
const content = isServerRole(ctx.config.role)
|
||||
? generateServerConfig(ctx.config)
|
||||
: generateAgentConfig(ctx.config);
|
||||
: generateAgentConfig();
|
||||
|
||||
const changed = await writeRemoteFile(ctx, "/etc/rancher/k3s/config.yaml", content);
|
||||
|
||||
|
||||
@@ -15,21 +15,8 @@ export const installK3sBinary: Operation = async (ctx): Promise<OperationResult>
|
||||
const alreadyInstalled = version.exitCode === 0;
|
||||
|
||||
if (isServer) {
|
||||
// Clean stale server state when joining an existing cluster
|
||||
// (TLS certs from a previous run cause "newer than datastore" fatal error)
|
||||
if (ctx.config.k3sServerUrl && ctx.config.k3sToken) {
|
||||
await ctx.ssh.exec(
|
||||
"rm -rf /var/lib/rancher/k3s/server/tls /var/lib/rancher/k3s/server/cred /var/lib/rancher/k3s/server/db",
|
||||
sshOpts(ctx),
|
||||
);
|
||||
}
|
||||
|
||||
// If joining an existing cluster, pass K3S_URL and K3S_TOKEN
|
||||
const joinEnv = ctx.config.k3sServerUrl && ctx.config.k3sToken
|
||||
? `K3S_URL="${ctx.config.k3sServerUrl}" K3S_TOKEN="${ctx.config.k3sToken}"`
|
||||
: "";
|
||||
const result = await ctx.ssh.exec(
|
||||
`curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server" INSTALL_K3S_SKIP_SELINUX_RPM=true ${joinEnv} sh -`,
|
||||
'curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server" INSTALL_K3S_SKIP_SELINUX_RPM=true sh -',
|
||||
{ timeoutMs: 300_000 },
|
||||
);
|
||||
if (result.exitCode !== 0) {
|
||||
|
||||
@@ -1,44 +1,25 @@
|
||||
// Decommission file-based k8s audit logging in favor of journald.
|
||||
//
|
||||
// Earlier versions wrote audit events to /var/log/kubernetes/audit.log and
|
||||
// rotated them with a logrotate rule. Two failure modes followed: kube-apiserver
|
||||
// rotated internally (audit-{ts}.log), the *.log glob in logrotate
|
||||
// double-rotated those (-{date}), and the resulting filename matched no
|
||||
// retention policy, so the directory grew unbounded (we observed 7+ GB).
|
||||
//
|
||||
// k3s now sets audit-log-path=- so audit goes to stdout → journald, which
|
||||
// enforces SystemMaxUse caps. This operation removes the obsolete logrotate
|
||||
// rule and reaps any audit files left behind by the old setup. Idempotent: on
|
||||
// fresh installs everything is already absent and the operation is a no-op.
|
||||
// Configure log rotation for k3s.
|
||||
|
||||
import type { Operation, OperationResult } from "../types.js";
|
||||
import { sshOpts } from "../utils.js";
|
||||
import { writeRemoteFile } from "../utils.js";
|
||||
|
||||
const REMOVE_LOGROTATE = "rm -f /etc/logrotate.d/k3s";
|
||||
|
||||
// Bounded by a max-depth and explicit name pattern so we never reach outside
|
||||
// the deprecated audit-log directory.
|
||||
const REAP_OLD_AUDIT_FILES =
|
||||
"find /var/log/kubernetes -maxdepth 1 -type f " +
|
||||
"\\( -name 'audit*.log*' -o -name 'audit-*.log' \\) " +
|
||||
"-delete 2>/dev/null; " +
|
||||
"rmdir /var/log/kubernetes 2>/dev/null; true";
|
||||
const LOGROTATE_CONFIG = `/var/log/kubernetes/*.log {
|
||||
daily
|
||||
rotate 14
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
copytruncate
|
||||
maxsize 100M
|
||||
}`;
|
||||
|
||||
export const configureLogRotation: Operation = async (ctx): Promise<OperationResult> => {
|
||||
const before = await ctx.ssh.exec(
|
||||
"test -e /etc/logrotate.d/k3s -o -d /var/log/kubernetes && echo present || echo absent",
|
||||
sshOpts(ctx),
|
||||
);
|
||||
const wasPresent = before.stdout.trim() === "present";
|
||||
|
||||
await ctx.ssh.exec(REMOVE_LOGROTATE, sshOpts(ctx));
|
||||
await ctx.ssh.exec(REAP_OLD_AUDIT_FILES, sshOpts(ctx));
|
||||
const changed = await writeRemoteFile(ctx, "/etc/logrotate.d/k3s", LOGROTATE_CONFIG);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
changed: wasPresent,
|
||||
message: wasPresent
|
||||
? "Removed legacy file-based audit logging (now via journald)"
|
||||
: "No legacy audit log artifacts present",
|
||||
changed,
|
||||
message: changed ? "Log rotation configured" : "Log rotation already configured",
|
||||
};
|
||||
};
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
// Annotate nodes with Longhorn default disk config when /var/lib/longhorn exists.
|
||||
// The label is set in k3s config (node-label), but the annotation must be applied via kubectl.
|
||||
|
||||
import type { Operation, OperationResult } from "../types.js";
|
||||
import { sshOpts } from "../utils.js";
|
||||
import { sshExec as remoteSshExec } from "../../../../src/ssh.js";
|
||||
|
||||
export const configureLonghornDisk: Operation = async (ctx): Promise<OperationResult> => {
|
||||
// Check if /var/lib/longhorn exists on this node
|
||||
const check = await ctx.ssh.exec("test -d /var/lib/longhorn && echo yes || echo no", sshOpts(ctx));
|
||||
if (check.stdout.trim() !== "yes") {
|
||||
return { success: true, changed: false, message: "No /var/lib/longhorn directory — skipping Longhorn disk config" };
|
||||
}
|
||||
|
||||
// Find the node name (hostname as registered in k3s)
|
||||
const nodeNameResult = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
|
||||
const nodeName = nodeNameResult.stdout.trim();
|
||||
|
||||
const annotation = JSON.stringify([{ path: "/var/lib/longhorn", allowScheduling: true }]);
|
||||
|
||||
// Try kubectl locally first (works on server nodes)
|
||||
const result = await ctx.ssh.exec(
|
||||
`k3s kubectl annotate node "${nodeName}" "node.longhorn.io/default-disks-config=${annotation}" --overwrite 2>&1 || true`,
|
||||
sshOpts(ctx),
|
||||
);
|
||||
|
||||
if (result.stdout.includes("annotated") || result.stdout.includes("unchanged")) {
|
||||
return { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName}` };
|
||||
}
|
||||
|
||||
// For worker/agent nodes without local kubectl: apply via the server
|
||||
if (ctx.config.k3sServerUrl) {
|
||||
// The CLI has SSH access to the server — use sshExec from there
|
||||
const serverHost = new URL(ctx.config.k3sServerUrl).hostname;
|
||||
try {
|
||||
const remoteResult = await remoteSshExec(
|
||||
serverHost, "root",
|
||||
`k3s kubectl annotate node "${nodeName}" "node.longhorn.io/default-disks-config=${annotation}" --overwrite`,
|
||||
{ ...(ctx.ssh.keyPath ? { keyPath: ctx.ssh.keyPath } : {}), timeoutMs: 15_000 },
|
||||
);
|
||||
if (remoteResult.stdout.includes("annotated") || remoteResult.stdout.includes("unchanged")) {
|
||||
return { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName} (via server)` };
|
||||
}
|
||||
} catch {
|
||||
// Fall through to manual instruction
|
||||
}
|
||||
}
|
||||
|
||||
return { success: true, changed: false, message: "Longhorn disk label set (annotation requires server kubectl)" };
|
||||
};
|
||||
@@ -1,34 +0,0 @@
|
||||
// Install Multus CNI (thick plugin) — the meta-CNI that lets pods attach an
|
||||
// extra interface (macvlan on VLAN 10) alongside Cilium, via a
|
||||
// NetworkAttachmentDefinition. Required for Home Assistant's LAN presence
|
||||
// (Matter/HomeKit mDNS discovery). Cilium must be installed with
|
||||
// cni.exclusive=false first (see cilium.ts) or it deletes Multus's CNI conf.
|
||||
|
||||
import type { Operation, OperationResult } from "../types.js";
|
||||
import { sshOpts } from "../utils.js";
|
||||
|
||||
const MULTUS_VERSION = "v4.1.4";
|
||||
const MULTUS_MANIFEST = `https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/${MULTUS_VERSION}/deployments/multus-daemonset-thick.yml`;
|
||||
|
||||
export const installMultus: Operation = async (ctx): Promise<OperationResult> => {
|
||||
const K = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml";
|
||||
|
||||
// Idempotent: skip if the Multus DaemonSet is already present.
|
||||
const check = await ctx.ssh.exec(
|
||||
`${K} kubectl -n kube-system get ds kube-multus-ds -o name 2>/dev/null`,
|
||||
sshOpts(ctx),
|
||||
);
|
||||
if (check.exitCode === 0 && check.stdout.includes("kube-multus-ds")) {
|
||||
return { success: true, changed: false, message: `Multus already installed (${MULTUS_VERSION})` };
|
||||
}
|
||||
|
||||
const apply = await ctx.ssh.exec(
|
||||
`${K} kubectl apply -f ${MULTUS_MANIFEST}`,
|
||||
{ ...sshOpts(ctx), timeoutMs: 120_000 },
|
||||
);
|
||||
if (apply.exitCode !== 0) {
|
||||
return { success: false, changed: false, message: "Failed to apply Multus manifest", error: apply.stderr };
|
||||
}
|
||||
|
||||
return { success: true, changed: true, message: `Installed Multus ${MULTUS_VERSION} (thick)` };
|
||||
};
|
||||
@@ -1,48 +0,0 @@
|
||||
// Grow the labvg/rancher LV (k3s image store / imageFs) to 120G.
|
||||
// 2026-08 incident: the original 20G LV sat at 85% used from steady-state
|
||||
// images alone, so one ~5G image pull tripped imagefs eviction and evicted
|
||||
// unrelated pods. Fresh installs are sized at 120G by the kickstart; this op
|
||||
// covers nodes installed before that change and vanilla nodes converted to
|
||||
// k8s later. Never removes or shrinks anything — if the VG lacks free space
|
||||
// (e.g. a longhorn --grow LV consumed it), it reports and moves on.
|
||||
|
||||
import type { Operation, OperationResult } from "../types.js";
|
||||
import { sshOpts } from "../utils.js";
|
||||
|
||||
const RANCHER_LV = "labvg/rancher";
|
||||
const TARGET_MIB = 122880; // 120G
|
||||
|
||||
export const growRancherLv: Operation = async (ctx): Promise<OperationResult> => {
|
||||
const lv = await ctx.ssh.exec(
|
||||
`lvs --noheadings --units m --nosuffix -o lv_size ${RANCHER_LV} 2>/dev/null || true`,
|
||||
sshOpts(ctx),
|
||||
);
|
||||
const sizeMib = Number.parseFloat(lv.stdout.trim());
|
||||
if (Number.isNaN(sizeMib)) {
|
||||
return { success: true, changed: false, message: "No labvg/rancher LV — imageFs shares /var, skipping" };
|
||||
}
|
||||
if (sizeMib >= TARGET_MIB) {
|
||||
return { success: true, changed: false, message: `rancher LV already ${Math.round(sizeMib / 1024)}G` };
|
||||
}
|
||||
|
||||
const vg = await ctx.ssh.exec(`vgs --noheadings --units m --nosuffix -o vg_free labvg`, sshOpts(ctx));
|
||||
const freeMib = Number.parseFloat(vg.stdout.trim());
|
||||
const neededMib = TARGET_MIB - sizeMib;
|
||||
if (Number.isNaN(freeMib) || freeMib < neededMib) {
|
||||
return {
|
||||
success: true,
|
||||
changed: false,
|
||||
message: `VG labvg has ${Math.floor((Number.isNaN(freeMib) ? 0 : freeMib) / 1024)}G free — ` +
|
||||
`need ${Math.ceil(neededMib / 1024)}G to grow rancher LV to 120G (manual LV rebuild required)`,
|
||||
};
|
||||
}
|
||||
|
||||
await ctx.ssh.exec(`lvextend -L ${TARGET_MIB}m /dev/${RANCHER_LV}`, sshOpts(ctx));
|
||||
await ctx.ssh.exec(`xfs_growfs /var/lib/rancher`, sshOpts(ctx));
|
||||
|
||||
return {
|
||||
success: true,
|
||||
changed: true,
|
||||
message: `rancher LV grown ${Math.round(sizeMib / 1024)}G → 120G`,
|
||||
};
|
||||
};
|
||||
@@ -1,40 +1,22 @@
|
||||
// Enable swap so memory pressure spills to disk instead of OOM-killing.
|
||||
// kubelet runs with failSwapOn=false (k3s default); zram stays the fast tier,
|
||||
// the labvg-swap LV is the overflow tier. Replaces the old CIS-style
|
||||
// disableSwap op — a kernel OOM kill of a node daemon is worse than slow swap.
|
||||
// Disable swap (CIS requirement for k3s).
|
||||
|
||||
import type { Operation, OperationResult } from "../types.js";
|
||||
import { sshOpts } from "../utils.js";
|
||||
|
||||
const SWAP_DEV = "/dev/mapper/labvg-swap";
|
||||
export const disableSwap: Operation = async (ctx): Promise<OperationResult> => {
|
||||
const check = await ctx.ssh.exec("swapon --show --noheadings", sshOpts(ctx));
|
||||
const active = check.stdout.trim().length > 0;
|
||||
|
||||
export const enableSwap: Operation = async (ctx): Promise<OperationResult> => {
|
||||
const lv = await ctx.ssh.exec(`test -b ${SWAP_DEV} && echo yes || echo no`, sshOpts(ctx));
|
||||
if (lv.stdout.trim() !== "yes") {
|
||||
return { success: true, changed: false, message: "No labvg-swap LV — skipping swap enable" };
|
||||
if (active) {
|
||||
await ctx.ssh.exec("swapoff -a", sshOpts(ctx));
|
||||
}
|
||||
|
||||
const active = await ctx.ssh.exec(
|
||||
`grep -q "^$(readlink -f ${SWAP_DEV}) " /proc/swaps && echo on || echo off`,
|
||||
sshOpts(ctx),
|
||||
);
|
||||
const wasOff = active.stdout.trim() !== "on";
|
||||
|
||||
if (wasOff) {
|
||||
// Format if the LV was never (or wrongly) initialised, then activate
|
||||
await ctx.ssh.exec(`blkid ${SWAP_DEV} | grep -q 'TYPE="swap"' || mkswap ${SWAP_DEV}`, sshOpts(ctx));
|
||||
await ctx.ssh.exec(`swapon ${SWAP_DEV}`, sshOpts(ctx));
|
||||
}
|
||||
|
||||
// Persist across reboots (idempotent)
|
||||
await ctx.ssh.exec(
|
||||
`grep -q "labvg-swap" /etc/fstab || echo "${SWAP_DEV} none swap defaults 0 0" >> /etc/fstab`,
|
||||
sshOpts(ctx),
|
||||
);
|
||||
// Remove swap entries from fstab permanently
|
||||
await ctx.ssh.exec("sed -i '/\\sswap\\s/d' /etc/fstab", sshOpts(ctx));
|
||||
|
||||
return {
|
||||
success: true,
|
||||
changed: wasOff,
|
||||
message: wasOff ? "LV swap enabled" : "LV swap already active",
|
||||
changed: active,
|
||||
message: active ? "Swap disabled" : "Swap already disabled",
|
||||
};
|
||||
};
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
// vlan-setup DaemonSet — the node-level half of the macvlan/VLAN-10 story.
|
||||
// On every node it (1) installs the reference CNI plugins (macvlan/ipvlan/
|
||||
// static/host-local/vlan/tuning) into /opt/cni/bin if missing, and (2) creates
|
||||
// a `lan10` VLAN-10 sub-interface on the primary NIC that macvlan
|
||||
// NetworkAttachmentDefinitions use as their master. Idempotent + self-healing
|
||||
// (re-creates lan10 if it disappears). Paired with Multus (multus.ts) + Cilium
|
||||
// bpf.vlanBypass={10} (cilium.ts).
|
||||
|
||||
import type { Operation, OperationResult } from "../types.js";
|
||||
import { sshOpts } from "../utils.js";
|
||||
|
||||
const MANIFEST = `apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: macvlan-sys
|
||||
labels:
|
||||
pod-security.kubernetes.io/enforce: privileged
|
||||
pod-security.kubernetes.io/audit: privileged
|
||||
pod-security.kubernetes.io/warn: privileged
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: vlan-setup
|
||||
namespace: macvlan-sys
|
||||
spec:
|
||||
selector:
|
||||
matchLabels: { app: vlan-setup }
|
||||
template:
|
||||
metadata:
|
||||
labels: { app: vlan-setup }
|
||||
spec:
|
||||
hostNetwork: true
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
containers:
|
||||
- name: vlan
|
||||
image: nicolaka/netshoot
|
||||
securityContext:
|
||||
privileged: true
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -x
|
||||
# install reference CNI plugins (macvlan/ipvlan/static/host-local) if missing
|
||||
if [ ! -f /host/opt/cni/bin/macvlan ] || [ ! -f /host/opt/cni/bin/ipvlan ]; then
|
||||
case "$(uname -m)" in x86_64) A=amd64;; aarch64) A=arm64;; *) A=amd64;; esac
|
||||
curl -sSL "https://github.com/containernetworking/plugins/releases/download/v1.5.1/cni-plugins-linux-$A-v1.5.1.tgz" -o /tmp/cni.tgz
|
||||
tar -xzf /tmp/cni.tgz -C /host/opt/cni/bin ./macvlan ./ipvlan ./static ./host-local ./vlan ./tuning
|
||||
fi
|
||||
# detect the primary NIC (default route dev, else the one holding 192.168.8.x)
|
||||
NIC="$(ip -o -4 route show default 2>/dev/null | awk '{print $5; exit}')"
|
||||
[ -z "$NIC" ] && NIC="$(ip -o -4 addr show 2>/dev/null | awk '/192\\.168\\.8\\./{print $2; exit}')"
|
||||
echo "primary NIC = $NIC"
|
||||
while true; do
|
||||
if [ -n "$NIC" ]; then
|
||||
ip link show lan10 >/dev/null 2>&1 || ip link add link "$NIC" name lan10 type vlan id 10
|
||||
ip link set lan10 up
|
||||
# NIC-driver workarounds for VLAN multicast RX
|
||||
ip link set "$NIC" allmulticast on 2>/dev/null
|
||||
ethtool -K "$NIC" rxvlan off rx-vlan-filter off 2>/dev/null
|
||||
fi
|
||||
sleep 30
|
||||
done
|
||||
volumeMounts:
|
||||
- name: cnibin
|
||||
mountPath: /host/opt/cni/bin
|
||||
volumes:
|
||||
- name: cnibin
|
||||
hostPath:
|
||||
path: /opt/cni/bin
|
||||
`;
|
||||
|
||||
export const installVlanSetup: Operation = async (ctx): Promise<OperationResult> => {
|
||||
const K = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml";
|
||||
|
||||
const check = await ctx.ssh.exec(
|
||||
`${K} kubectl -n macvlan-sys get ds vlan-setup -o name 2>/dev/null`,
|
||||
sshOpts(ctx),
|
||||
);
|
||||
if (check.exitCode === 0 && check.stdout.includes("vlan-setup")) {
|
||||
return { success: true, changed: false, message: "vlan-setup DaemonSet already installed" };
|
||||
}
|
||||
|
||||
const b64 = Buffer.from(MANIFEST).toString("base64");
|
||||
const apply = await ctx.ssh.exec(
|
||||
`echo ${b64} | base64 -d | ${K} kubectl apply -f -`,
|
||||
{ ...sshOpts(ctx), timeoutMs: 60_000 },
|
||||
);
|
||||
if (apply.exitCode !== 0) {
|
||||
return { success: false, changed: false, message: "Failed to apply vlan-setup DaemonSet", error: apply.stderr };
|
||||
}
|
||||
|
||||
return { success: true, changed: true, message: "Installed vlan-setup DaemonSet (lan10 + CNI plugins)" };
|
||||
};
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user