From 4f9a6f64e4295f641336f9430e71555967bfbd27 Mon Sep 17 00:00:00 2001 From: Michal Date: Tue, 7 Jul 2026 15:37:20 +0100 Subject: [PATCH] k3s/networking: codify Multus + vlan-setup as lab operations The macvlan/VLAN-10 foundation HA depends on (Multus meta-CNI + a lan10 sub-interface + reference CNI plugins) was applied by hand during the HA migration. Codify both as idempotent lab operations in the networking group, after installCilium (which already sets cni.exclusive=false + bpf.vlanBypass={10}). A fresh cluster now reproduces the full macvlan stack. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../modules/k3s/src/groups/networking.ts | 7 ++ .../modules/k3s/src/operations/index.ts | 2 + .../modules/k3s/src/operations/multus.ts | 34 +++++++ .../modules/k3s/src/operations/vlan-setup.ts | 96 +++++++++++++++++++ 4 files changed, 139 insertions(+) create mode 100644 bastion/src/modules/modules/k3s/src/operations/multus.ts create mode 100644 bastion/src/modules/modules/k3s/src/operations/vlan-setup.ts diff --git a/bastion/src/modules/modules/k3s/src/groups/networking.ts b/bastion/src/modules/modules/k3s/src/groups/networking.ts index fa786db..469216b 100644 --- a/bastion/src/modules/modules/k3s/src/groups/networking.ts +++ b/bastion/src/modules/modules/k3s/src/groups/networking.ts @@ -3,6 +3,8 @@ import type { OperationContext, OperationResult, OperationGroup } from "../types.js"; import { runSequential } from "../utils.js"; import { installCilium } from "../operations/cilium.js"; +import { installMultus } from "../operations/multus.js"; +import { installVlanSetup } from "../operations/vlan-setup.js"; import { fixCoreDnsUpstream } from "../operations/dns-fix.js"; import { applyDefaultNetworkPolicies } from "../operations/network-policy.js"; @@ -11,6 +13,11 @@ export const networkingGroup: OperationGroup = { description: "Install Cilium CNI, fix DNS, apply network policies", operations: [ { name: "Install Cilium CNI", fn: installCilium }, + // Multus + vlan-setup: give pods a second interface on VLAN 10 (macvlan) + // for LAN device discovery (Matter/HomeKit mDNS). Must follow Cilium + // (needs cni.exclusive=false + bpf.vlanBypass={10} from installCilium). + { name: "Install Multus CNI", fn: installMultus }, + { name: "Install vlan-setup (lan10 + CNI plugins)", fn: installVlanSetup }, { name: "Fix CoreDNS upstream", fn: fixCoreDnsUpstream }, { name: "Apply network policies", fn: applyDefaultNetworkPolicies }, ], diff --git a/bastion/src/modules/modules/k3s/src/operations/index.ts b/bastion/src/modules/modules/k3s/src/operations/index.ts index ac669b3..f6434dc 100644 --- a/bastion/src/modules/modules/k3s/src/operations/index.ts +++ b/bastion/src/modules/modules/k3s/src/operations/index.ts @@ -9,6 +9,8 @@ export { writeAuditPolicy } from "./audit-policy.js"; export { cleanupStaleCni } from "./cni-cleanup.js"; export { installK3sBinary } from "./k3s-install.js"; export { installCilium } from "./cilium.js"; +export { installMultus } from "./multus.js"; +export { installVlanSetup } from "./vlan-setup.js"; export { fixCoreDnsUpstream } from "./dns-fix.js"; export { configureLogRotation } from "./log-rotation.js"; export { configureJournaldLimits } from "./journald-limits.js"; diff --git a/bastion/src/modules/modules/k3s/src/operations/multus.ts b/bastion/src/modules/modules/k3s/src/operations/multus.ts new file mode 100644 index 0000000..630be91 --- /dev/null +++ b/bastion/src/modules/modules/k3s/src/operations/multus.ts @@ -0,0 +1,34 @@ +// Install Multus CNI (thick plugin) — the meta-CNI that lets pods attach an +// extra interface (macvlan on VLAN 10) alongside Cilium, via a +// NetworkAttachmentDefinition. Required for Home Assistant's LAN presence +// (Matter/HomeKit mDNS discovery). Cilium must be installed with +// cni.exclusive=false first (see cilium.ts) or it deletes Multus's CNI conf. + +import type { Operation, OperationResult } from "../types.js"; +import { sshOpts } from "../utils.js"; + +const MULTUS_VERSION = "v4.1.4"; +const MULTUS_MANIFEST = `https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/${MULTUS_VERSION}/deployments/multus-daemonset-thick.yml`; + +export const installMultus: Operation = async (ctx): Promise => { + const K = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml"; + + // Idempotent: skip if the Multus DaemonSet is already present. + const check = await ctx.ssh.exec( + `${K} kubectl -n kube-system get ds kube-multus-ds -o name 2>/dev/null`, + sshOpts(ctx), + ); + if (check.exitCode === 0 && check.stdout.includes("kube-multus-ds")) { + return { success: true, changed: false, message: `Multus already installed (${MULTUS_VERSION})` }; + } + + const apply = await ctx.ssh.exec( + `${K} kubectl apply -f ${MULTUS_MANIFEST}`, + { ...sshOpts(ctx), timeoutMs: 120_000 }, + ); + if (apply.exitCode !== 0) { + return { success: false, changed: false, message: "Failed to apply Multus manifest", error: apply.stderr }; + } + + return { success: true, changed: true, message: `Installed Multus ${MULTUS_VERSION} (thick)` }; +}; diff --git a/bastion/src/modules/modules/k3s/src/operations/vlan-setup.ts b/bastion/src/modules/modules/k3s/src/operations/vlan-setup.ts new file mode 100644 index 0000000..6e4ff5f --- /dev/null +++ b/bastion/src/modules/modules/k3s/src/operations/vlan-setup.ts @@ -0,0 +1,96 @@ +// vlan-setup DaemonSet — the node-level half of the macvlan/VLAN-10 story. +// On every node it (1) installs the reference CNI plugins (macvlan/ipvlan/ +// static/host-local/vlan/tuning) into /opt/cni/bin if missing, and (2) creates +// a `lan10` VLAN-10 sub-interface on the primary NIC that macvlan +// NetworkAttachmentDefinitions use as their master. Idempotent + self-healing +// (re-creates lan10 if it disappears). Paired with Multus (multus.ts) + Cilium +// bpf.vlanBypass={10} (cilium.ts). + +import type { Operation, OperationResult } from "../types.js"; +import { sshOpts } from "../utils.js"; + +const MANIFEST = `apiVersion: v1 +kind: Namespace +metadata: + name: macvlan-sys + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: vlan-setup + namespace: macvlan-sys +spec: + selector: + matchLabels: { app: vlan-setup } + template: + metadata: + labels: { app: vlan-setup } + spec: + hostNetwork: true + tolerations: + - operator: Exists + containers: + - name: vlan + image: nicolaka/netshoot + securityContext: + privileged: true + command: + - sh + - -c + - | + set -x + # install reference CNI plugins (macvlan/ipvlan/static/host-local) if missing + if [ ! -f /host/opt/cni/bin/macvlan ] || [ ! -f /host/opt/cni/bin/ipvlan ]; then + case "$(uname -m)" in x86_64) A=amd64;; aarch64) A=arm64;; *) A=amd64;; esac + curl -sSL "https://github.com/containernetworking/plugins/releases/download/v1.5.1/cni-plugins-linux-$A-v1.5.1.tgz" -o /tmp/cni.tgz + tar -xzf /tmp/cni.tgz -C /host/opt/cni/bin ./macvlan ./ipvlan ./static ./host-local ./vlan ./tuning + fi + # detect the primary NIC (default route dev, else the one holding 192.168.8.x) + NIC="$(ip -o -4 route show default 2>/dev/null | awk '{print $5; exit}')" + [ -z "$NIC" ] && NIC="$(ip -o -4 addr show 2>/dev/null | awk '/192\\.168\\.8\\./{print $2; exit}')" + echo "primary NIC = $NIC" + while true; do + if [ -n "$NIC" ]; then + ip link show lan10 >/dev/null 2>&1 || ip link add link "$NIC" name lan10 type vlan id 10 + ip link set lan10 up + # NIC-driver workarounds for VLAN multicast RX + ip link set "$NIC" allmulticast on 2>/dev/null + ethtool -K "$NIC" rxvlan off rx-vlan-filter off 2>/dev/null + fi + sleep 30 + done + volumeMounts: + - name: cnibin + mountPath: /host/opt/cni/bin + volumes: + - name: cnibin + hostPath: + path: /opt/cni/bin +`; + +export const installVlanSetup: Operation = async (ctx): Promise => { + const K = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml"; + + const check = await ctx.ssh.exec( + `${K} kubectl -n macvlan-sys get ds vlan-setup -o name 2>/dev/null`, + sshOpts(ctx), + ); + if (check.exitCode === 0 && check.stdout.includes("vlan-setup")) { + return { success: true, changed: false, message: "vlan-setup DaemonSet already installed" }; + } + + const b64 = Buffer.from(MANIFEST).toString("base64"); + const apply = await ctx.ssh.exec( + `echo ${b64} | base64 -d | ${K} kubectl apply -f -`, + { ...sshOpts(ctx), timeoutMs: 60_000 }, + ); + if (apply.exitCode !== 0) { + return { success: false, changed: false, message: "Failed to apply vlan-setup DaemonSet", error: apply.stderr }; + } + + return { success: true, changed: true, message: "Installed vlan-setup DaemonSet (lan10 + CNI plugins)" }; +};