diff --git a/bastion/src/modules/modules/k3s/src/operations/k3s-config.ts b/bastion/src/modules/modules/k3s/src/operations/k3s-config.ts index 0ef334c..3c5d3be 100644 --- a/bastion/src/modules/modules/k3s/src/operations/k3s-config.ts +++ b/bastion/src/modules/modules/k3s/src/operations/k3s-config.ts @@ -7,8 +7,48 @@ function isServerRole(role: string): boolean { return role === "infra" || role === "labcontroller"; } +/** + * The address-family block: `cluster-cidr`, `service-cidr` and `node-ip`. + * + * Emitted ONLY when the corresponding config is supplied, and that is + * deliberate. With no `ipv6` and no CIDRs this returns "", so the generated + * file is byte-identical to what every existing node already has -- no diff, + * so `writeRemoteFile` reports unchanged and nothing restarts k3s. Dual-stack + * is therefore opt-in per node rather than a flag day. + * + * `node-ip` is only written once there is a second family to name. k3s + * auto-detects a sensible IPv4 on its own, and writing it out unconditionally + * would rewrite the config of five healthy nodes to tell them what they had + * already worked out. + */ +function addressFamilyLines(config: K3sConfig, opts: { cidrs: boolean }): string { + const lines: string[] = []; + if (opts.cidrs && config.clusterCidr?.length) { + lines.push(`cluster-cidr: "${config.clusterCidr.join(",")}"`); + } + if (opts.cidrs && config.serviceCidr?.length) { + lines.push(`service-cidr: "${config.serviceCidr.join(",")}"`); + } + if (config.ipv6) { + // Order matters to k3s: the FIRST entry is the primary family, and the + // supported single-to-dual-stack conversion is the one that preserves it. + // IPv4 stays primary so existing Services keep their ClusterIP. + lines.push(`node-ip: "${config.ip},${config.ipv6}"`); + } + return lines.length ? `${lines.join("\n")}\n` : ""; +} + function generateServerConfig(config: K3sConfig): string { - const tlsSans = [config.hostname, config.ip, ...(config.tlsSans ?? [])]; + // The IPv6 address goes in the cert too. Without it, anything that reaches + // this apiserver over v6 -- a peer server joining, or kubectl against the v6 + // address -- fails TLS verification, and the error names the certificate + // rather than the missing SAN, which is a long way from the cause. + const tlsSans = [ + config.hostname, + config.ip, + ...(config.ipv6 ? [config.ipv6] : []), + ...(config.tlsSans ?? []), + ]; const isJoining = !!config.k3sServerUrl; const clusterLines = isJoining ? `server: "${config.k3sServerUrl}"\ntoken: "${config.k3sToken}"` @@ -21,7 +61,7 @@ function generateServerConfig(config: K3sConfig): string { // and never expire. return `# k3s server configuration — CIS hardened, etcd HA ${clusterLines} -protect-kernel-defaults: true +${addressFamilyLines(config, { cidrs: true })}protect-kernel-defaults: true secrets-encryption: true write-kubeconfig-mode: "0640" @@ -51,8 +91,13 @@ ${tlsSans.map((s) => ` - "${s}"`).join("\n")} `; } -function generateAgentConfig(): string { - return `protect-kernel-defaults: true +// Takes the config now: an agent needs its own dual `node-ip` just as much as a +// server does. Without one it joins as an IPv4-only node into a dual-stack +// cluster, gets no IPv6 pod CIDR, and the failure surfaces later as pods on that +// node being unreachable over v6 while the node itself reads Ready. +// It takes no cluster/service CIDRs -- those are server-side only. +function generateAgentConfig(config: K3sConfig): string { + return `${addressFamilyLines(config, { cidrs: false })}protect-kernel-defaults: true node-label: - "node-role.kubernetes.io/worker=true" - "node.longhorn.io/create-default-disk=config" @@ -68,7 +113,7 @@ export const writeK3sConfig: Operation = async (ctx): Promise = const content = isServerRole(ctx.config.role) ? generateServerConfig(ctx.config) - : generateAgentConfig(); + : generateAgentConfig(ctx.config); const changed = await writeRemoteFile(ctx, "/etc/rancher/k3s/config.yaml", content); diff --git a/bastion/src/modules/modules/k3s/src/types.ts b/bastion/src/modules/modules/k3s/src/types.ts index 4304334..1b000fd 100644 --- a/bastion/src/modules/modules/k3s/src/types.ts +++ b/bastion/src/modules/modules/k3s/src/types.ts @@ -16,6 +16,33 @@ export interface K3sConfig { // Additional TLS SANs for API server certificate tlsSans?: string[] | undefined; + + /** + * IPv6 address of this node on the cluster VLAN. Its PRESENCE is what makes a + * node dual-stack: supply it and `node-ip` is emitted as `,`; omit it + * and the generated config is byte-identical to the IPv4-only one. + * + * It must be a real address on the node before k3s starts. k3s resolves + * node-ip at boot, so a SLAAC or DHCPv6 lease that has not landed yet leaves + * the node with no usable v6 identity. The estate takes it from a kea DHCPv6 + * reservation keyed on MAC, the same way it takes its IPv4 address. + */ + ipv6?: string | undefined; + + /** + * Pod and Service ranges, one entry per address family. Passed rather than + * hardcoded so the ranges stay configuration -- and so labsim can rehearse + * with its own addresses through this same generator, instead of a parallel + * set of INSTALL_K3S_EXEC flags that would prove a different mechanism. + * + * k3s validates the two together and crash-loops on a mismatch (a dual + * cluster-cidr with a single-family service-cidr, say), which is a safe + * failure but an avoidable one: set both or neither. + * + * Servers only -- agents take neither. + */ + clusterCidr?: string[] | undefined; + serviceCidr?: string[] | undefined; } /** SSH execution interface injected into operations. */ diff --git a/bastion/src/modules/modules/k3s/tests/operations.test.ts b/bastion/src/modules/modules/k3s/tests/operations.test.ts index 3502242..c5ea639 100644 --- a/bastion/src/modules/modules/k3s/tests/operations.test.ts +++ b/bastion/src/modules/modules/k3s/tests/operations.test.ts @@ -268,6 +268,87 @@ describe("writeK3sConfig", () => { expect(writeCall).toContain("protect-kernel-defaults: true"); expect(writeCall).not.toContain("secrets-encryption"); }); + + // --- dual-stack --- + // + // The property that matters most is the NEGATIVE one: with no ipv6 and no + // CIDRs the output must be byte-identical to what the five existing nodes + // already have on disk. If it is not, rolling this out rewrites every node's + // config.yaml and restarts a healthy cluster to tell it what it already knew. + + const writtenBy = async (config: Parameters[0]) => { + const ctx = mockCtx(config); + ctx.ssh.exec + .mockResolvedValueOnce(OK) + .mockResolvedValueOnce(stdout("__LABCTL_NOT_FOUND__")) + .mockResolvedValueOnce(OK); + await writeK3sConfig(ctx); + return ctx.ssh.exec.mock.calls[2]![0] as string; + }; + + it("emits no address-family lines at all when single-stack", async () => { + const server = await writtenBy({ hostname: "n1.lab", ip: "10.0.1.1", role: "infra" }); + expect(server).not.toContain("node-ip"); + expect(server).not.toContain("cluster-cidr"); + expect(server).not.toContain("service-cidr"); + + const agent = await writtenBy({ role: "worker" }); + expect(agent).not.toContain("node-ip"); + // Nothing inserted ahead of, or between, the lines the agent config has + // always opened with. + expect(agent).toContain("protect-kernel-defaults: true\nnode-label:"); + }); + + it("emits dual node-ip and CIDRs on a server, IPv4 first", async () => { + const out = await writtenBy({ + hostname: "n1.lab", + ip: "192.168.8.23", + role: "infra", + ipv6: "2001:470:187e:2::23", + clusterCidr: ["10.42.0.0/16", "2001:470:187e:1000::/56"], + serviceCidr: ["10.43.0.0/16", "2001:470:187e:1fff::/112"], + }); + expect(out).toContain('node-ip: "192.168.8.23,2001:470:187e:2::23"'); + expect(out).toContain('cluster-cidr: "10.42.0.0/16,2001:470:187e:1000::/56"'); + expect(out).toContain('service-cidr: "10.43.0.0/16,2001:470:187e:1fff::/112"'); + // IPv4 must stay the primary family -- that is the supported conversion + // path and what lets existing Services keep their ClusterIP. + expect(out.indexOf("10.42.0.0/16")).toBeLessThan(out.indexOf("2001:470:187e:1000::/56")); + // still a valid server config + expect(out).toContain("cluster-init: true"); + expect(out).toContain("secrets-encryption: true"); + // the v6 address must be a TLS SAN, or a peer joining over v6 fails + // verification with an error that names the cert, not the missing SAN + expect(out).toContain(' - "2001:470:187e:2::23"'); + }); + + it("gives an AGENT its own dual node-ip but no CIDRs", async () => { + const out = await writtenBy({ + ip: "192.168.8.12", + role: "worker", + ipv6: "2001:470:187e:2::12", + // deliberately supplied: an agent must ignore them + clusterCidr: ["10.42.0.0/16", "2001:470:187e:1000::/56"], + serviceCidr: ["10.43.0.0/16", "2001:470:187e:1fff::/112"], + }); + expect(out).toContain('node-ip: "192.168.8.12,2001:470:187e:2::12"'); + expect(out).not.toContain("cluster-cidr"); + expect(out).not.toContain("service-cidr"); + }); + + it("does not write node-ip for a v4-only node even when CIDRs are given", async () => { + // Guards the flag-day risk: supplying ranges alone must not start rewriting + // node identity on nodes that have no IPv6 yet. + const out = await writtenBy({ + hostname: "n1.lab", + ip: "10.0.1.1", + role: "infra", + clusterCidr: ["10.42.0.0/16"], + serviceCidr: ["10.43.0.0/16"], + }); + expect(out).toContain('cluster-cidr: "10.42.0.0/16"'); + expect(out).not.toContain("node-ip"); + }); }); // --- CNI Cleanup ---