k3s/networking: codify Multus + vlan-setup as lab operations

The macvlan/VLAN-10 foundation HA depends on (Multus meta-CNI + a lan10
sub-interface + reference CNI plugins) was applied by hand during the HA
migration. Codify both as idempotent lab operations in the networking group,
after installCilium (which already sets cni.exclusive=false + bpf.vlanBypass={10}).
A fresh cluster now reproduces the full macvlan stack.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

bastion/src/modules/modules/k3s/src/groups/networking.ts

@@ -3,6 +3,8 @@
 import type { OperationContext, OperationResult, OperationGroup } from "../types.js";
 import { runSequential } from "../utils.js";
 import { installCilium } from "../operations/cilium.js";
+import { installMultus } from "../operations/multus.js";
+import { installVlanSetup } from "../operations/vlan-setup.js";
 import { fixCoreDnsUpstream } from "../operations/dns-fix.js";
 import { applyDefaultNetworkPolicies } from "../operations/network-policy.js";
 
@@ -11,6 +13,11 @@ export const networkingGroup: OperationGroup = {
   description: "Install Cilium CNI, fix DNS, apply network policies",
   operations: [
     { name: "Install Cilium CNI", fn: installCilium },
+    // Multus + vlan-setup: give pods a second interface on VLAN 10 (macvlan)
+    // for LAN device discovery (Matter/HomeKit mDNS). Must follow Cilium
+    // (needs cni.exclusive=false + bpf.vlanBypass={10} from installCilium).
+    { name: "Install Multus CNI", fn: installMultus },
+    { name: "Install vlan-setup (lan10 + CNI plugins)", fn: installVlanSetup },
     { name: "Fix CoreDNS upstream", fn: fixCoreDnsUpstream },
     { name: "Apply network policies", fn: applyDefaultNetworkPolicies },
   ],

bastion/src/modules/modules/k3s/src/operations/cilium.ts

@@ -38,12 +38,20 @@ export const installCilium: Operation = async (ctx): Promise<OperationResult> =>
   // Install Cilium
   // - No hardcoded devices: Cilium auto-detects per node (heterogeneous NICs like eno1 vs enP7s7)
   // - k8sServiceHost/Port: k3s agents proxy the API on 127.0.0.1:6444 (not 6443)
+  // - cni.exclusive=false: required so Multus can install its CNI config alongside
+  //   Cilium (Cilium otherwise deletes any non-Cilium CNI conf).
+  // - bpf.vlanBypass={10}: allow VLAN 10 (LoT) tagged traffic through the eBPF
+  //   host VLAN filter, so pods on a macvlan/VLAN-10 interface receive multicast
+  //   (Matter/mDNS ff02::fb + 224.0.0.251). Without this Cilium drops it
+  //   ("VLAN traffic disallowed by VLAN filter", bpf_host.c).
   const installResult = await ctx.ssh.exec(
     `KUBECONFIG=/etc/rancher/k3s/k3s.yaml cilium install \
       --set kubeProxyReplacement=true \
       --set ipam.mode=kubernetes \
       --set k8sServiceHost=127.0.0.1 \
-      --set k8sServicePort=6444`,
+      --set k8sServicePort=6444 \
+      --set cni.exclusive=false \
+      --set bpf.vlanBypass="{10}"`,
     { timeoutMs: 300_000 },
   );
   if (installResult.exitCode !== 0) {

bastion/src/modules/modules/k3s/src/operations/index.ts

@@ -9,6 +9,8 @@ export { writeAuditPolicy } from "./audit-policy.js";
 export { cleanupStaleCni } from "./cni-cleanup.js";
 export { installK3sBinary } from "./k3s-install.js";
 export { installCilium } from "./cilium.js";
+export { installMultus } from "./multus.js";
+export { installVlanSetup } from "./vlan-setup.js";
 export { fixCoreDnsUpstream } from "./dns-fix.js";
 export { configureLogRotation } from "./log-rotation.js";
 export { configureJournaldLimits } from "./journald-limits.js";

bastion/src/modules/modules/k3s/src/operations/multus.ts

@@ -0,0 +1,34 @@
+// Install Multus CNI (thick plugin) — the meta-CNI that lets pods attach an
+// extra interface (macvlan on VLAN 10) alongside Cilium, via a
+// NetworkAttachmentDefinition. Required for Home Assistant's LAN presence
+// (Matter/HomeKit mDNS discovery). Cilium must be installed with
+// cni.exclusive=false first (see cilium.ts) or it deletes Multus's CNI conf.
+
+import type { Operation, OperationResult } from "../types.js";
+import { sshOpts } from "../utils.js";
+
+const MULTUS_VERSION = "v4.1.4";
+const MULTUS_MANIFEST = `https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/${MULTUS_VERSION}/deployments/multus-daemonset-thick.yml`;
+
+export const installMultus: Operation = async (ctx): Promise<OperationResult> => {
+  const K = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml";
+
+  // Idempotent: skip if the Multus DaemonSet is already present.
+  const check = await ctx.ssh.exec(
+    `${K} kubectl -n kube-system get ds kube-multus-ds -o name 2>/dev/null`,
+    sshOpts(ctx),
+  );
+  if (check.exitCode === 0 && check.stdout.includes("kube-multus-ds")) {
+    return { success: true, changed: false, message: `Multus already installed (${MULTUS_VERSION})` };
+  }
+
+  const apply = await ctx.ssh.exec(
+    `${K} kubectl apply -f ${MULTUS_MANIFEST}`,
+    { ...sshOpts(ctx), timeoutMs: 120_000 },
+  );
+  if (apply.exitCode !== 0) {
+    return { success: false, changed: false, message: "Failed to apply Multus manifest", error: apply.stderr };
+  }
+
+  return { success: true, changed: true, message: `Installed Multus ${MULTUS_VERSION} (thick)` };
+};

bastion/src/modules/modules/k3s/src/operations/vlan-setup.ts

@@ -0,0 +1,96 @@
+// vlan-setup DaemonSet — the node-level half of the macvlan/VLAN-10 story.
+// On every node it (1) installs the reference CNI plugins (macvlan/ipvlan/
+// static/host-local/vlan/tuning) into /opt/cni/bin if missing, and (2) creates
+// a `lan10` VLAN-10 sub-interface on the primary NIC that macvlan
+// NetworkAttachmentDefinitions use as their master. Idempotent + self-healing
+// (re-creates lan10 if it disappears). Paired with Multus (multus.ts) + Cilium
+// bpf.vlanBypass={10} (cilium.ts).
+
+import type { Operation, OperationResult } from "../types.js";
+import { sshOpts } from "../utils.js";
+
+const MANIFEST = `apiVersion: v1
+kind: Namespace
+metadata:
+  name: macvlan-sys
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: vlan-setup
+  namespace: macvlan-sys
+spec:
+  selector:
+    matchLabels: { app: vlan-setup }
+  template:
+    metadata:
+      labels: { app: vlan-setup }
+    spec:
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+      containers:
+      - name: vlan
+        image: nicolaka/netshoot
+        securityContext:
+          privileged: true
+        command:
+        - sh
+        - -c
+        - |
+          set -x
+          # install reference CNI plugins (macvlan/ipvlan/static/host-local) if missing
+          if [ ! -f /host/opt/cni/bin/macvlan ] || [ ! -f /host/opt/cni/bin/ipvlan ]; then
+            case "$(uname -m)" in x86_64) A=amd64;; aarch64) A=arm64;; *) A=amd64;; esac
+            curl -sSL "https://github.com/containernetworking/plugins/releases/download/v1.5.1/cni-plugins-linux-$A-v1.5.1.tgz" -o /tmp/cni.tgz
+            tar -xzf /tmp/cni.tgz -C /host/opt/cni/bin ./macvlan ./ipvlan ./static ./host-local ./vlan ./tuning
+          fi
+          # detect the primary NIC (default route dev, else the one holding 192.168.8.x)
+          NIC="$(ip -o -4 route show default 2>/dev/null | awk '{print $5; exit}')"
+          [ -z "$NIC" ] && NIC="$(ip -o -4 addr show 2>/dev/null | awk '/192\\.168\\.8\\./{print $2; exit}')"
+          echo "primary NIC = $NIC"
+          while true; do
+            if [ -n "$NIC" ]; then
+              ip link show lan10 >/dev/null 2>&1 || ip link add link "$NIC" name lan10 type vlan id 10
+              ip link set lan10 up
+              # NIC-driver workarounds for VLAN multicast RX
+              ip link set "$NIC" allmulticast on 2>/dev/null
+              ethtool -K "$NIC" rxvlan off rx-vlan-filter off 2>/dev/null
+            fi
+            sleep 30
+          done
+        volumeMounts:
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+      volumes:
+      - name: cnibin
+        hostPath:
+          path: /opt/cni/bin
+`;
+
+export const installVlanSetup: Operation = async (ctx): Promise<OperationResult> => {
+  const K = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml";
+
+  const check = await ctx.ssh.exec(
+    `${K} kubectl -n macvlan-sys get ds vlan-setup -o name 2>/dev/null`,
+    sshOpts(ctx),
+  );
+  if (check.exitCode === 0 && check.stdout.includes("vlan-setup")) {
+    return { success: true, changed: false, message: "vlan-setup DaemonSet already installed" };
+  }
+
+  const b64 = Buffer.from(MANIFEST).toString("base64");
+  const apply = await ctx.ssh.exec(
+    `echo ${b64} | base64 -d | ${K} kubectl apply -f -`,
+    { ...sshOpts(ctx), timeoutMs: 60_000 },
+  );
+  if (apply.exitCode !== 0) {
+    return { success: false, changed: false, message: "Failed to apply vlan-setup DaemonSet", error: apply.stderr };
+  }
+
+  return { success: true, changed: true, message: "Installed vlan-setup DaemonSet (lan10 + CNI plugins)" };
+};