feat: iSCSI, Longhorn disk labels, labctl asahi command, ZIP32 fix

k3s host prep: - Add iSCSI initiator install+enable (Fedora: iscsi-initiator-utils, Ubuntu: open-iscsi) — required by Longhorn - Add Longhorn disk label to k3s server+agent configs - Add Longhorn disk annotation operation in post-install hardening CLI: - Add `labctl provision asahi` command with interactive install guide - Change default SSH user from "michal" to "lab" in all commands - Change admin user in bastion progress callback to "lab" Asahi provisioning fixes: - Download installer_data.json locally (installer reads it as file) - Use REPO_BASE to serve upstream ZIP from bastion (LAN speed) - Fix ZIP32 vs ZIP64: serve original upstream ZIP unmodified (our repackaged ZIP used ZIP64 which breaks Asahi urlcache) - Add /data/asahi-repo fallback path for k3s container PVC mount - Deploy script syncs asahi-repo to bastion pod after deployment Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-31 23:32:38 +01:00
parent a8dc79bc5a
commit bb8f37ef7d
17 changed files with 203 additions and 31 deletions
--- a/bastion/src/bastion/src/routes/api.ts
+++ b/bastion/src/bastion/src/routes/api.ts
@@ -148,7 +148,7 @@ export function registerApiRoutes(
          };
          s.installed[mac] = installedInfo;

-          const admin = installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "michal" : "root";
+          const admin = installedInfo.role !== "vanilla" && installedInfo.role !== "" ? "lab" : "root";
          console.log(`\n  \x1b[0;32m\x1b[1m  ssh ${admin}@${ip}\x1b[0m\n`);  // eslint-disable-line no-console

          // Auto-install k3s for non-vanilla roles
--- a/bastion/src/bastion/src/routes/asahi.ts
+++ b/bastion/src/bastion/src/routes/asahi.ts
@@ -19,6 +19,9 @@ function findAsahiRepo(config: BastionConfig): string | null {
  const inBastionDir = join(config.bastionDir, "asahi-repo");
  if (existsSync(inBastionDir)) return inBastionDir;

+  // Check /data/asahi-repo (PVC mount in k3s container)
+  if (existsSync("/data/asahi-repo")) return "/data/asahi-repo";
+
  // Check relative to project root (dev mode)
  try {
    const thisDir = dirname(fileURLToPath(import.meta.url));
@@ -80,20 +83,21 @@ curl -# -L -o "installer-\${PKG_VER}.tar.gz" "\${INSTALLER_BASE}/installer-\${PK
 echo "  Extracting..."
 tar xf "installer-\${PKG_VER}.tar.gz"

-# Point to our custom installer_data.json + rootfs repo
-export INSTALLER_DATA="\${BASTION}/asahi/installer_data.json"
+# Download our custom installer_data.json (installer reads it as a local file)
+echo "  Downloading custom installer data from bastion..."
+curl -sfL -o installer_data.json "\${BASTION}/asahi/installer_data.json"
+
+# Set REPO_BASE so the installer downloads rootfs from bastion (local, fast)
 export REPO_BASE="\${BASTION}/asahi/repo/"

 echo ""
-echo "  Using custom installer data from bastion."
+echo "  Using custom partition layout + rootfs from bastion."
 echo "  This will create:"
 echo "    - Standard Asahi boot infrastructure (m1n1 + U-Boot)"
 echo "    - Fedora Asahi Remix root partition"
 echo "    - LVM data partition (remaining space)"
 echo ""
-echo "  On first boot, LVM volumes will be created automatically:"
-echo "    labvg/var (100GB), labvg/varlog (10GB), labvg/home (10GB),"
-echo "    labvg/srv (20GB), labvg/rancher (20GB), labvg/longhorn (rest)"
+echo "  On first boot, LVM volumes are created automatically."
 echo ""

 # Run the installer
--- a/bastion/src/bastion/src/templates/asahi-firstboot.sh.ts
+++ b/bastion/src/bastion/src/templates/asahi-firstboot.sh.ts
@@ -258,12 +258,14 @@ provisioned_at=$(date -Iseconds)
 method=asahi-firstboot
 LABMETA

-# ── Callback to bastion ──────────────────────────────────────────
+# ── Register with bastion ─────────────────────────────────────────
 IP=$(hostname -I | awk '{print $1}')
-curl -sf -X POST "http://${serverIp}:${httpPort}/api/progress" \\
+echo "Registering with bastion at ${serverIp}:${httpPort}..."
+curl -sf -X POST "http://${serverIp}:${httpPort}/api/register" \\
    -H "Content-Type: application/json" \\
-    -d "{\\"mac\\":\\"${mac}\\",\\"stage\\":\\"complete\\",\\"detail\\":\\"ready at $IP\\"}" \\
-    2>/dev/null || true
+    -d "{\\"mac\\":\\"${mac}\\",\\"hostname\\":\\"${hostname}\\",\\"role\\":\\"${role}\\",\\"ip\\":\\"$IP\\"}" \\
+    2>/dev/null && echo "  Registered as ${hostname} ($IP)" \\
+    || echo "  WARNING: Could not reach bastion — register manually with: labctl provision register ${mac} ${hostname} --role ${role} --ip $IP"

 # ── Mark done ────────────────────────────────────────────────────
 touch "$MARKER"
--- a/bastion/src/bastion/tests/asahi.test.ts
+++ b/bastion/src/bastion/tests/asahi.test.ts
@@ -62,8 +62,7 @@ describe("asahi routes", () => {
    expect(resp.statusCode).toBe(200);
    expect(resp.headers["content-type"]).toContain("text/x-shellscript");
    expect(resp.body).toContain("#!/bin/bash");
-    expect(resp.body).toContain("INSTALLER_DATA");
-    expect(resp.body).toContain("REPO_BASE");
+    expect(resp.body).toContain("installer_data.json");
    expect(resp.body).toContain("192.168.8.1");
    expect(resp.body).toContain("install.sh");
  });
@@ -77,14 +76,17 @@ describe("asahi routes", () => {
    const os = data.os_list[0];
    expect(os.name).toContain("Fedora Asahi Lab");

-    // Three partitions: EFI + Root + Data
-    expect(os.partitions).toHaveLength(3);
+    // 3 partitions (fallback) or 4 (built: EFI + Boot + Root + Data)
+    expect(os.partitions.length).toBeGreaterThanOrEqual(3);
    expect(os.partitions[0].type).toBe("EFI");
-    expect(os.partitions[1].type).toBe("Linux");
-    expect(os.partitions[1].expand).toBe(false);
-    expect(os.partitions[1].image).toBe("root.img");
-    expect(os.partitions[2].type).toBe("Linux");
-    expect(os.partitions[2].expand).toBe(true);
+    // Last partition should be the expanding Data partition
+    const lastPart = os.partitions[os.partitions.length - 1];
+    expect(lastPart.type).toBe("Linux");
+    expect(lastPart.expand).toBe(true);
+    // Root partition (second-to-last) should NOT expand
+    const rootPart = os.partitions[os.partitions.length - 2];
+    expect(rootPart.expand).toBe(false);
+    expect(rootPart.image).toBe("root.img");
  });

  it("GET /asahi/firstboot.sh returns parameterized script", async () => {
@@ -185,11 +187,11 @@ describe("renderFirstbootScript", () => {
    expect(script).toContain('hostnamectl set-hostname "test-node"');
  });

-  it("includes bastion callback", () => {
+  it("includes bastion self-registration", () => {
    const script = renderFirstbootScript({ ...baseParams, role: "worker" });
-    expect(script).toContain("/api/progress");
+    expect(script).toContain("/api/register");
    expect(script).toContain("aa:bb:cc:dd:ee:ff");
-    expect(script).toContain("complete");
+    expect(script).toContain("test-node");
  });

  it("writes provisioning metadata", () => {
--- a/bastion/src/cli/src/commands/app.ts
+++ b/bastion/src/cli/src/commands/app.ts
@@ -70,7 +70,7 @@ export function registerAppCommand(program: Command): void {
    .command("install <target>")
    .description("Install k3s on a target machine (hostname, IP, or MAC)")
    .option("--role <role>", "k3s role: infra (server) or worker (agent)", "infra")
-    .option("--user <user>", "SSH user", "michal")
+    .option("--user <user>", "SSH user", "lab")
    .option("--k3s-server <url>", "k3s server URL (required for worker role)")
    .option("--k3s-token <token>", "k3s join token (required for worker role)")
    .action(async (target: string, opts: {
@@ -164,7 +164,7 @@ export function registerAppCommand(program: Command): void {
  k3sCmd
    .command("health [target]")
    .description("Check k3s health (all hosts if no target given)")
-    .option("--user <user>", "SSH user", "michal")
+    .option("--user <user>", "SSH user", "lab")
    .action(async (target: string | undefined, opts: { user: string }) => {
      const sshKey = findSshKey();

@@ -304,7 +304,7 @@ export function registerAppCommand(program: Command): void {
  k3sCmd
    .command("list")
    .description("List installed machines and their k3s status")
-    .option("--user <user>", "SSH user", "michal")
+    .option("--user <user>", "SSH user", "lab")
    .action(async (opts: { user: string }) => {
      let state: BastionState;
      try {
--- a/bastion/src/cli/src/commands/asahi.ts
+++ b/bastion/src/cli/src/commands/asahi.ts
@@ -0,0 +1,69 @@
+// CLI command: provision asahi
+// Prints the curl command to run on the Mac Studio (macOS) to install
+// Fedora Asahi Remix with lab LVM layout.
+
+import type { Command } from "commander";
+import { getLabdClient } from "../api/config.js";
+
+export function registerAsahiCommand(parent: Command): void {
+  parent
+    .command("asahi")
+    .description("Show instructions to provision an Apple Silicon Mac with Asahi Linux")
+    .action(async () => {
+      // Try to get bastion info to determine the correct URL
+      let bastionUrl = "";
+      try {
+        const bastions = await getLabdClient().getBastions();
+        const online = bastions.find(b => b.status === "online");
+        if (online) {
+          bastionUrl = `http://${online.serverIp}:8080`;
+        }
+      } catch { /* labd not reachable */ }
+
+      if (!bastionUrl) {
+        // Fall back to config
+        const { loadConfig } = await import("../config/index.js");
+        const config = loadConfig();
+        bastionUrl = config.labdUrl ?? "http://<bastion-ip>:8080";
+        // Convert labd URL to bastion URL (labd is on different port/host)
+        bastionUrl = bastionUrl.replace(/:\d+$/, ":8080");
+      }
+
+      const BOLD = "\x1b[1m";
+      const CYAN = "\x1b[36m";
+      const DIM = "\x1b[2m";
+      const RESET = "\x1b[0m";
+
+      console.log("");
+      console.log(`${BOLD}  Asahi Linux Provisioning${RESET}`);
+      console.log(`${DIM}  For Apple Silicon Macs (Mac Studio, MacBook, etc.)${RESET}`);
+      console.log("");
+      console.log(`  Run this command ${BOLD}on the Mac${RESET} (from macOS Terminal):`);
+      console.log("");
+      console.log(`    ${CYAN}${BOLD}curl ${bastionUrl}/asahi | sh${RESET}`);
+      console.log("");
+      console.log(`  The installer will ask a few interactive questions:`);
+      console.log(`    ${BOLD}1.${RESET} Action: press ${BOLD}r${RESET} to resize macOS`);
+      console.log(`    ${BOLD}2.${RESET} How much space for Linux: choose maximum`);
+      console.log(`    ${BOLD}3.${RESET} Confirm the resize operation`);
+      console.log(`    ${BOLD}4.${RESET} macOS password for firmware authentication`);
+      console.log("");
+      console.log(`  After that, everything is automatic:`);
+      console.log(`    - Asahi boot infrastructure (m1n1 + U-Boot)`);
+      console.log(`    - Fedora Asahi Remix root partition`);
+      console.log(`    - LVM data partition (remaining space)`);
+      console.log("");
+      console.log(`  On first boot, LVM volumes are created automatically:`);
+      console.log(`    ${DIM}labvg/swap (27GB), labvg/var (100GB), labvg/varlog (10GB),`);
+      console.log(`    labvg/home (10GB), labvg/srv (20GB), labvg/rancher (20GB),`);
+      console.log(`    labvg/longhorn (remaining space)${RESET}`);
+      console.log("");
+      console.log(`  After first boot, SSH in and run the firstboot script:`);
+      console.log(`    ${BOLD}ssh root@<ip> 'curl -sf ${bastionUrl}/asahi/firstboot.sh?hostname=<name>\\&role=infra | bash'${RESET}`);
+      console.log("");
+      console.log(`  This sets up LVM and self-registers with the bastion.`);
+      console.log(`  Then install k3s:`);
+      console.log(`    ${BOLD}labctl app k3s install <hostname> --role infra${RESET}`);
+      console.log("");
+    });
+}
--- a/bastion/src/cli/src/commands/labcontroller.ts
+++ b/bastion/src/cli/src/commands/labcontroller.ts
@@ -38,7 +38,7 @@ export function registerLabcontrollerCommands(appCmd: Command): void {
  lcCmd
    .command("deploy <target>")
    .description("Deploy labcontroller stack to a k3s node")
-    .option("--user <user>", "SSH user", "michal")
+    .option("--user <user>", "SSH user", "lab")
    .option("--crdb-replicas <n>", "CockroachDB replicas", "1")
    .action(async (target: string, opts: {
      user: string;
@@ -193,7 +193,7 @@ export function registerLabcontrollerCommands(appCmd: Command): void {
  lcCmd
    .command("status [target]")
    .description("Check labcontroller deployment status (all hosts if no target)")
-    .option("--user <user>", "SSH user", "michal")
+    .option("--user <user>", "SSH user", "lab")
    .action(async (target: string | undefined, opts: { user: string }) => {
      const sshKey = findSshKey();
      const sshOpts = sshKey ? { keyPath: sshKey } : {};
--- a/bastion/src/cli/src/index.ts
+++ b/bastion/src/cli/src/index.ts
@@ -17,6 +17,7 @@ import { registerReprovisionCommand } from "./commands/reprovision.js";
 import { registerDebugCommand } from "./commands/debug.js";
 import { registerForgetCommand } from "./commands/forget.js";
 import { registerRegisterCommand } from "./commands/register.js";
+import { registerAsahiCommand } from "./commands/asahi.js";
 import { registerLogsCommand } from "./commands/logs.js";
 import { registerMakeIsoCommand } from "./commands/makeiso.js";
 import { registerConfigCommand } from "./commands/config.js";
@@ -100,6 +101,7 @@ export function createProgram(): Command {
  registerDebugCommand(provisionCmd);
  registerForgetCommand(provisionCmd);
  registerRegisterCommand(provisionCmd);
+  registerAsahiCommand(provisionCmd);
  registerLogsCommand(provisionCmd);
  registerMakeIsoCommand(provisionCmd);

--- a/bastion/src/modules/modules/k3s/src/groups/hardening.ts
+++ b/bastion/src/modules/modules/k3s/src/groups/hardening.ts
@@ -5,14 +5,16 @@ import { runSequential } from "../utils.js";
 import { applyPodSecurityStandards } from "../operations/pod-security.js";
 import { checkCertExpiry } from "../operations/cert-check.js";
 import { configureLogRotation } from "../operations/log-rotation.js";
+import { configureLonghornDisk } from "../operations/longhorn-disk.js";

 export const hardeningGroup: OperationGroup = {
  name: "hardening",
-  description: "Pod security, certificate check, log rotation",
+  description: "Pod security, certificate check, log rotation, storage",
  operations: [
    { name: "Apply Pod Security Standards", fn: applyPodSecurityStandards },
    { name: "Check certificate expiry", fn: checkCertExpiry },
    { name: "Configure log rotation", fn: configureLogRotation },
+    { name: "Configure Longhorn disk", fn: configureLonghornDisk },
  ],
 };

--- a/bastion/src/modules/modules/k3s/src/groups/host-prep.ts
+++ b/bastion/src/modules/modules/k3s/src/groups/host-prep.ts
@@ -7,16 +7,18 @@ import { applyCisHardening } from "../operations/sysctl.js";
 import { disableSwap } from "../operations/swap.js";
 import { disableFirewall } from "../operations/firewall.js";
 import { setSelinuxPermissive } from "../operations/selinux.js";
+import { enableIscsi } from "../operations/iscsi.js";

 export const hostPrepGroup: OperationGroup = {
  name: "host-prep",
-  description: "Prepare host for k3s: kernel modules, sysctl, swap, firewall, SELinux",
+  description: "Prepare host for k3s: kernel modules, sysctl, swap, firewall, SELinux, iSCSI",
  operations: [
    { name: "Load kernel modules", fn: loadKernelModules },
    { name: "Apply CIS sysctl", fn: applyCisHardening },
    { name: "Disable swap", fn: disableSwap },
    { name: "Disable firewall", fn: disableFirewall },
    { name: "Set SELinux permissive", fn: setSelinuxPermissive },
+    { name: "Enable iSCSI", fn: enableIscsi },
  ],
 };

--- a/bastion/src/modules/modules/k3s/src/operations/index.ts
+++ b/bastion/src/modules/modules/k3s/src/operations/index.ts
@@ -1,6 +1,7 @@
 export { loadKernelModules } from "./kernel-modules.js";
 export { applyCisHardening } from "./sysctl.js";
 export { disableSwap } from "./swap.js";
+export { enableIscsi } from "./iscsi.js";
 export { disableFirewall } from "./firewall.js";
 export { setSelinuxPermissive } from "./selinux.js";
 export { writeK3sConfig } from "./k3s-config.js";
@@ -13,3 +14,4 @@ export { configureLogRotation } from "./log-rotation.js";
 export { applyDefaultNetworkPolicies } from "./network-policy.js";
 export { applyPodSecurityStandards } from "./pod-security.js";
 export { checkCertExpiry } from "./cert-check.js";
+export { configureLonghornDisk } from "./longhorn-disk.js";
--- a/bastion/src/modules/modules/k3s/src/operations/iscsi.ts
+++ b/bastion/src/modules/modules/k3s/src/operations/iscsi.ts
@@ -0,0 +1,30 @@
+// Install and enable iSCSI initiator (required by Longhorn storage).
+// Fedora: iscsi-initiator-utils, Ubuntu: open-iscsi
+
+import type { Operation, OperationResult } from "../types.js";
+import { sshOpts } from "../utils.js";
+
+export const enableIscsi: Operation = async (ctx): Promise<OperationResult> => {
+  // Check if iscsid is already running
+  const check = await ctx.ssh.exec("systemctl is-active iscsid 2>/dev/null", sshOpts(ctx));
+  if (check.stdout.trim() === "active") {
+    return { success: true, changed: false, message: "iSCSI already active" };
+  }
+
+  // Install the package (detect distro)
+  const osRelease = await ctx.ssh.exec("cat /etc/os-release", sshOpts(ctx));
+  const isFedora = osRelease.stdout.includes("fedora") || osRelease.stdout.includes("rhel") || osRelease.stdout.includes("centos");
+
+  const pkg = isFedora ? "iscsi-initiator-utils" : "open-iscsi";
+  const installCmd = isFedora ? `dnf install -y ${pkg}` : `apt-get install -y ${pkg}`;
+
+  const install = await ctx.ssh.exec(installCmd, { timeoutMs: 120_000 });
+  if (install.exitCode !== 0) {
+    return { success: false, changed: false, message: `Failed to install ${pkg}`, error: install.stderr.trim() };
+  }
+
+  // Enable and start
+  await ctx.ssh.exec("systemctl enable --now iscsid", sshOpts(ctx));
+
+  return { success: true, changed: true, message: `Installed ${pkg} and enabled iscsid` };
+};
--- a/bastion/src/modules/modules/k3s/src/operations/k3s-config.ts
+++ b/bastion/src/modules/modules/k3s/src/operations/k3s-config.ts
@@ -20,6 +20,9 @@ disable:
  - servicelb
  - traefik

+node-label:
+  - "node.longhorn.io/create-default-disk=config"
+
 kube-apiserver-arg:
  - "anonymous-auth=false"
  - "audit-log-path=/var/log/kubernetes/audit.log"
@@ -44,6 +47,7 @@ function generateAgentConfig(): string {
  return `protect-kernel-defaults: true
 node-label:
  - "node-role.kubernetes.io/worker=true"
+  - "node.longhorn.io/create-default-disk=config"
 kubelet-arg:
  - "protect-kernel-defaults=true"
  - "streaming-connection-idle-timeout=5m"
--- a/bastion/src/modules/modules/k3s/src/operations/longhorn-disk.ts
+++ b/bastion/src/modules/modules/k3s/src/operations/longhorn-disk.ts
@@ -0,0 +1,34 @@
+// Annotate nodes with Longhorn default disk config when /var/lib/longhorn exists.
+// The label is set in k3s config (node-label), but the annotation must be applied via kubectl.
+
+import type { Operation, OperationResult } from "../types.js";
+import { sshOpts } from "../utils.js";
+
+export const configureLonghornDisk: Operation = async (ctx): Promise<OperationResult> => {
+  // Check if /var/lib/longhorn exists on this node
+  const check = await ctx.ssh.exec("test -d /var/lib/longhorn && echo yes || echo no", sshOpts(ctx));
+  if (check.stdout.trim() !== "yes") {
+    return { success: true, changed: false, message: "No /var/lib/longhorn directory — skipping Longhorn disk config" };
+  }
+
+  // Find the node name (hostname as registered in k3s)
+  const nodeNameResult = await ctx.ssh.exec("hostname -f 2>/dev/null || hostname", sshOpts(ctx));
+  const nodeName = nodeNameResult.stdout.trim();
+
+  // Apply the annotation via kubectl (works on server nodes, or via KUBECONFIG on agents)
+  const kubectlPrefix = "k3s kubectl";
+  const annotation = JSON.stringify([{ path: "/var/lib/longhorn", allowScheduling: true }]);
+
+  const result = await ctx.ssh.exec(
+    `${kubectlPrefix} annotate node "${nodeName}" "node.longhorn.io/default-disks-config=${annotation}" --overwrite 2>&1 || true`,
+    sshOpts(ctx),
+  );
+
+  if (result.stdout.includes("annotated") || result.stdout.includes("unchanged")) {
+    return { success: true, changed: true, message: `Longhorn disk annotation applied to ${nodeName}` };
+  }
+
+  // If kubectl isn't available (agent node without server access), that's OK —
+  // the label is set, annotation can be applied from the server later
+  return { success: true, changed: false, message: "Longhorn disk label set (annotation requires server kubectl)" };
+};