labctl: k3s config can carry both address families

Phase 2d of cluster dual-stack. This is the file that makes a NEW node correct
by construction: it already owns /etc/rancher/k3s/config.yaml for every node, so
once it emits dual values, a joining node is dual-stack with nothing else to do.
Before this there was nowhere for a node's IPv6 to come from -- K3sConfig had no
v6 field and generateAgentConfig() took no arguments at all.

The property that matters most is the negative one: with no ipv6 and no CIDRs,
addressFamilyLines() returns "" and the output is BYTE-IDENTICAL to what the
five existing nodes already have. Verified by diffing the generator's output for
worker0's parameters against worker0's live config.yaml -- identical. So this
change is inert until dual-stack config is supplied, rather than a flag day that
rewrites five healthy nodes and restarts the cluster to tell it what it knew.

node-ip is written only once there is a second family to name. k3s auto-detects
a sensible IPv4 by itself, and emitting it unconditionally would be that same
pointless rewrite. IPv4 stays first: the supported single-to-dual-stack
conversion preserves the primary family, which is what lets existing Services
keep their ClusterIP.

Agents get their own node-ip and no CIDRs. An agent without one joins a
dual-stack cluster as IPv4-only, gets no IPv6 pod CIDR, and fails later as pods
unreachable over v6 while the node itself reads Ready.

The IPv6 address is also added as a TLS SAN. Without it anything reaching the
apiserver over v6 -- a peer server joining, kubectl against the v6 address --
fails verification with an error naming the certificate rather than the missing
SAN.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DMVzWZgiKW2wquf5z8S1yH

bastion/src/modules/modules/k3s/src/operations/k3s-config.ts

@@ -7,8 +7,48 @@ function isServerRole(role: string): boolean {
   return role === "infra" || role === "labcontroller";
 }
 
+/**
+ * The address-family block: `cluster-cidr`, `service-cidr` and `node-ip`.
+ *
+ * Emitted ONLY when the corresponding config is supplied, and that is
+ * deliberate. With no `ipv6` and no CIDRs this returns "", so the generated
+ * file is byte-identical to what every existing node already has -- no diff,
+ * so `writeRemoteFile` reports unchanged and nothing restarts k3s. Dual-stack
+ * is therefore opt-in per node rather than a flag day.
+ *
+ * `node-ip` is only written once there is a second family to name. k3s
+ * auto-detects a sensible IPv4 on its own, and writing it out unconditionally
+ * would rewrite the config of five healthy nodes to tell them what they had
+ * already worked out.
+ */
+function addressFamilyLines(config: K3sConfig, opts: { cidrs: boolean }): string {
+  const lines: string[] = [];
+  if (opts.cidrs && config.clusterCidr?.length) {
+    lines.push(`cluster-cidr: "${config.clusterCidr.join(",")}"`);
+  }
+  if (opts.cidrs && config.serviceCidr?.length) {
+    lines.push(`service-cidr: "${config.serviceCidr.join(",")}"`);
+  }
+  if (config.ipv6) {
+    // Order matters to k3s: the FIRST entry is the primary family, and the
+    // supported single-to-dual-stack conversion is the one that preserves it.
+    // IPv4 stays primary so existing Services keep their ClusterIP.
+    lines.push(`node-ip: "${config.ip},${config.ipv6}"`);
+  }
+  return lines.length ? `${lines.join("\n")}\n` : "";
+}
+
 function generateServerConfig(config: K3sConfig): string {
-  const tlsSans = [config.hostname, config.ip, ...(config.tlsSans ?? [])];
+  // The IPv6 address goes in the cert too. Without it, anything that reaches
+  // this apiserver over v6 -- a peer server joining, or kubectl against the v6
+  // address -- fails TLS verification, and the error names the certificate
+  // rather than the missing SAN, which is a long way from the cause.
+  const tlsSans = [
+    config.hostname,
+    config.ip,
+    ...(config.ipv6 ? [config.ipv6] : []),
+    ...(config.tlsSans ?? []),
+  ];
   const isJoining = !!config.k3sServerUrl;
   const clusterLines = isJoining
     ? `server: "${config.k3sServerUrl}"\ntoken: "${config.k3sToken}"`
@@ -21,7 +61,7 @@ function generateServerConfig(config: K3sConfig): string {
   // and never expire.
   return `# k3s server configuration — CIS hardened, etcd HA
 ${clusterLines}
-protect-kernel-defaults: true
+${addressFamilyLines(config, { cidrs: true })}protect-kernel-defaults: true
 secrets-encryption: true
 write-kubeconfig-mode: "0640"
 
@@ -51,8 +91,13 @@ ${tlsSans.map((s) => `  - "${s}"`).join("\n")}
 `;
 }
 
-function generateAgentConfig(): string {
+// Takes the config now: an agent needs its own dual `node-ip` just as much as a
-  return `protect-kernel-defaults: true
+// server does. Without one it joins as an IPv4-only node into a dual-stack
+// cluster, gets no IPv6 pod CIDR, and the failure surfaces later as pods on that
+// node being unreachable over v6 while the node itself reads Ready.
+// It takes no cluster/service CIDRs -- those are server-side only.
+function generateAgentConfig(config: K3sConfig): string {
+  return `${addressFamilyLines(config, { cidrs: false })}protect-kernel-defaults: true
 node-label:
   - "node-role.kubernetes.io/worker=true"
   - "node.longhorn.io/create-default-disk=config"
@@ -68,7 +113,7 @@ export const writeK3sConfig: Operation = async (ctx): Promise<OperationResult> =
 
   const content = isServerRole(ctx.config.role)
     ? generateServerConfig(ctx.config)
-    : generateAgentConfig();
+    : generateAgentConfig(ctx.config);
 
   const changed = await writeRemoteFile(ctx, "/etc/rancher/k3s/config.yaml", content);
 

bastion/src/modules/modules/k3s/src/types.ts

@@ -16,6 +16,33 @@ export interface K3sConfig {
 
   // Additional TLS SANs for API server certificate
   tlsSans?: string[] | undefined;
+
+  /**
+   * IPv6 address of this node on the cluster VLAN. Its PRESENCE is what makes a
+   * node dual-stack: supply it and `node-ip` is emitted as `<v4>,<v6>`; omit it
+   * and the generated config is byte-identical to the IPv4-only one.
+   *
+   * It must be a real address on the node before k3s starts. k3s resolves
+   * node-ip at boot, so a SLAAC or DHCPv6 lease that has not landed yet leaves
+   * the node with no usable v6 identity. The estate takes it from a kea DHCPv6
+   * reservation keyed on MAC, the same way it takes its IPv4 address.
+   */
+  ipv6?: string | undefined;
+
+  /**
+   * Pod and Service ranges, one entry per address family. Passed rather than
+   * hardcoded so the ranges stay configuration -- and so labsim can rehearse
+   * with its own addresses through this same generator, instead of a parallel
+   * set of INSTALL_K3S_EXEC flags that would prove a different mechanism.
+   *
+   * k3s validates the two together and crash-loops on a mismatch (a dual
+   * cluster-cidr with a single-family service-cidr, say), which is a safe
+   * failure but an avoidable one: set both or neither.
+   *
+   * Servers only -- agents take neither.
+   */
+  clusterCidr?: string[] | undefined;
+  serviceCidr?: string[] | undefined;
 }
 
 /** SSH execution interface injected into operations. */

bastion/src/modules/modules/k3s/tests/operations.test.ts

@@ -268,6 +268,87 @@ describe("writeK3sConfig", () => {
     expect(writeCall).toContain("protect-kernel-defaults: true");
     expect(writeCall).not.toContain("secrets-encryption");
   });
+
+  // --- dual-stack ---
+  //
+  // The property that matters most is the NEGATIVE one: with no ipv6 and no
+  // CIDRs the output must be byte-identical to what the five existing nodes
+  // already have on disk. If it is not, rolling this out rewrites every node's
+  // config.yaml and restarts a healthy cluster to tell it what it already knew.
+
+  const writtenBy = async (config: Parameters<typeof mockCtx>[0]) => {
+    const ctx = mockCtx(config);
+    ctx.ssh.exec
+      .mockResolvedValueOnce(OK)
+      .mockResolvedValueOnce(stdout("__LABCTL_NOT_FOUND__"))
+      .mockResolvedValueOnce(OK);
+    await writeK3sConfig(ctx);
+    return ctx.ssh.exec.mock.calls[2]![0] as string;
+  };
+
+  it("emits no address-family lines at all when single-stack", async () => {
+    const server = await writtenBy({ hostname: "n1.lab", ip: "10.0.1.1", role: "infra" });
+    expect(server).not.toContain("node-ip");
+    expect(server).not.toContain("cluster-cidr");
+    expect(server).not.toContain("service-cidr");
+
+    const agent = await writtenBy({ role: "worker" });
+    expect(agent).not.toContain("node-ip");
+    // Nothing inserted ahead of, or between, the lines the agent config has
+    // always opened with.
+    expect(agent).toContain("protect-kernel-defaults: true\nnode-label:");
+  });
+
+  it("emits dual node-ip and CIDRs on a server, IPv4 first", async () => {
+    const out = await writtenBy({
+      hostname: "n1.lab",
+      ip: "192.168.8.23",
+      role: "infra",
+      ipv6: "2001:470:187e:2::23",
+      clusterCidr: ["10.42.0.0/16", "2001:470:187e:1000::/56"],
+      serviceCidr: ["10.43.0.0/16", "2001:470:187e:1fff::/112"],
+    });
+    expect(out).toContain('node-ip: "192.168.8.23,2001:470:187e:2::23"');
+    expect(out).toContain('cluster-cidr: "10.42.0.0/16,2001:470:187e:1000::/56"');
+    expect(out).toContain('service-cidr: "10.43.0.0/16,2001:470:187e:1fff::/112"');
+    // IPv4 must stay the primary family -- that is the supported conversion
+    // path and what lets existing Services keep their ClusterIP.
+    expect(out.indexOf("10.42.0.0/16")).toBeLessThan(out.indexOf("2001:470:187e:1000::/56"));
+    // still a valid server config
+    expect(out).toContain("cluster-init: true");
+    expect(out).toContain("secrets-encryption: true");
+    // the v6 address must be a TLS SAN, or a peer joining over v6 fails
+    // verification with an error that names the cert, not the missing SAN
+    expect(out).toContain('  - "2001:470:187e:2::23"');
+  });
+
+  it("gives an AGENT its own dual node-ip but no CIDRs", async () => {
+    const out = await writtenBy({
+      ip: "192.168.8.12",
+      role: "worker",
+      ipv6: "2001:470:187e:2::12",
+      // deliberately supplied: an agent must ignore them
+      clusterCidr: ["10.42.0.0/16", "2001:470:187e:1000::/56"],
+      serviceCidr: ["10.43.0.0/16", "2001:470:187e:1fff::/112"],
+    });
+    expect(out).toContain('node-ip: "192.168.8.12,2001:470:187e:2::12"');
+    expect(out).not.toContain("cluster-cidr");
+    expect(out).not.toContain("service-cidr");
+  });
+
+  it("does not write node-ip for a v4-only node even when CIDRs are given", async () => {
+    // Guards the flag-day risk: supplying ranges alone must not start rewriting
+    // node identity on nodes that have no IPv6 yet.
+    const out = await writtenBy({
+      hostname: "n1.lab",
+      ip: "10.0.1.1",
+      role: "infra",
+      clusterCidr: ["10.42.0.0/16"],
+      serviceCidr: ["10.43.0.0/16"],
+    });
+    expect(out).toContain('cluster-cidr: "10.42.0.0/16"');
+    expect(out).not.toContain("node-ip");
+  });
 });
 
 // --- CNI Cleanup ---