labsim: default-deny firewall policy, proven in the sim

Policy: internal VLANs reach each other and the internet; the internet
initiates nothing inward. That was already the effect of the IPv4 ruleset, but
built as a blacklist -- default-action accept plus explicit drops per WAN
interface. Identical behaviour right up until a WAN is added, at which point it
is open and nothing looks wrong. This expresses it as a whitelist.

Two findings from the sim, both of which would have been outages in production:

`set` on a rule number is ADDITIVE. The sim already had a rule 10 carrying
inbound/outbound interface constraints; `set ... rule 10 state established`
ANDed onto it, producing a stateful-accept that applied to one interface pair
only. Return traffic from the internet then matched no rule and hit the default
drop, so LAN hosts could reach nothing outbound. The generator now deletes each
filter before rebuilding it, so the code owns the subtree. It is one commit, so
nftables is rebuilt atomically -- there is no window without a firewall.

DHCP lease renewal is unicast UDP to port 68 and conntrack does not reliably
cover it. Without an explicit rule the WAN keeps working until the lease
expires and then dies -- a delayed failure that looks nothing like a firewall
change. Also added a loopback accept for both families, absent from the v6
policy since it went default-deny.

Verified in labsim: inter-VLAN ok, LAN-to-internet ok, internet-to-router
dropped, and internet-to-LAN dropped with the drop counter incrementing by
exactly the packets sent, after routing the test through the router rather than
around it via the hypervisor.

Also extends the drift check to the firewall subtree, which it did not cover --
so it had been reporting "in sync" while that subtree was uncaptured.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DMVzWZgiKW2wquf5z8S1yH

labsim/sim-net-config.py

@@ -177,6 +177,118 @@ def wan(drop_scaffold: bool) -> list[str]:
     return out
 
 
+# ---------------------------------------------------------------------------
+# Firewall. The policy is: internal VLANs talk to each other and to the
+# internet; the internet initiates nothing inward.
+#
+# That was already the *effect* of the previous IPv4 ruleset, but it was built
+# as a blacklist -- `default-action accept` plus explicit drops on each WAN
+# interface. The result is identical right up until someone adds a WAN, at
+# which point it is wide open and nothing looks wrong. This is the same policy
+# expressed as a whitelist, so a new interface is closed until it is named.
+# ---------------------------------------------------------------------------
+LAN_IFACES = ["bond0", "bond0.2", "bond0.3", "bond0.9", "bond0.10", "bond0.200"]
+LAN_GROUP = "LAN"
+
+
+def firewall(wan_dhcp_if: str | None = f"bond0.{WAN_DHCP_VLAN}") -> list[str]:
+    """wan_dhcp_if=None on a router with no DHCP WAN -- a firewall rule naming
+    an interface that does not exist is rejected at commit."""
+    out = [f"# --- firewall: LAN-to-anywhere, internet-to-nothing ---"]
+    # Delete each filter before rebuilding it. `set` on a rule number is
+    # ADDITIVE: if a rule 10 already exists carrying an inbound-interface
+    # constraint, `set ... rule 10 state established` silently ANDs onto it,
+    # and you get a stateful-accept rule that only applies to one interface
+    # pair. Observed in labsim: return traffic from the internet matched
+    # neither that rule nor the LAN rule and hit the default drop, so LAN
+    # hosts could reach nothing outbound. Everything here is one commit, so
+    # nftables is rebuilt atomically -- there is no window with no firewall.
+    out += [f"delete firewall {fam} {hook} filter"
+            for fam in ("ipv4", "ipv6") for hook in ("forward", "input")]
+    out += [f"set firewall group interface-group {LAN_GROUP} interface {i}"
+            for i in LAN_IFACES]
+    out += [
+        "",
+        # INPUT -- traffic terminating ON the router.
+        # Loopback first. Under a default-drop input policy, services talking to
+        # 127.0.0.1 are filtered like anything else, and the failures are
+        # bizarre and hard to attribute. Nothing off-box can forge iif lo.
+        "set firewall ipv4 input filter rule 5 action accept",
+        "set firewall ipv4 input filter rule 5 description 'loopback'",
+        "set firewall ipv4 input filter rule 5 inbound-interface name lo",
+        "set firewall ipv4 input filter rule 10 action accept",
+        "set firewall ipv4 input filter rule 10 description 'established/related'",
+        "set firewall ipv4 input filter rule 10 state established",
+        "set firewall ipv4 input filter rule 10 state related",
+        # One rule covers VRRP, conntrack-sync, kea HA, SSH, DNS and BGP,
+        # because every one of them arrives on a LAN interface. Enumerating the
+        # protocols instead would mean a new firewall rule every time the pair
+        # gains a feature -- and a lockout the day someone forgets.
+        f"set firewall ipv4 input filter rule 20 action accept",
+        f"set firewall ipv4 input filter rule 20 description 'trusted LAN to the router'",
+        f"set firewall ipv4 input filter rule 20 inbound-interface group {LAN_GROUP}",
+        # DHCP client. Lease RENEWAL is unicast UDP to port 68 and conntrack
+        # does not reliably cover it, so without this the WAN keeps working
+        # until the lease expires and then dies -- a delayed failure that looks
+        # nothing like a firewall change.
+        "set firewall ipv4 input filter default-action drop",
+        "",
+        # FORWARD -- traffic passing THROUGH the router.
+        "set firewall ipv4 forward filter rule 10 action accept",
+        "set firewall ipv4 forward filter rule 10 description 'established/related'",
+        "set firewall ipv4 forward filter rule 10 state established",
+        "set firewall ipv4 forward filter rule 10 state related",
+        # Inter-VLAN *and* LAN-to-internet in one rule: both are "came in on a
+        # LAN interface". Deliberately no restriction between internal VLANs --
+        # segmenting them is a separate decision, not a side effect of this one.
+        f"set firewall ipv4 forward filter rule 20 action accept",
+        f"set firewall ipv4 forward filter rule 20 description 'LAN to anywhere (inter-VLAN + internet)'",
+        f"set firewall ipv4 forward filter rule 20 inbound-interface group {LAN_GROUP}",
+        "set firewall ipv4 forward filter default-action drop",
+        "",
+        # IPv6 already runs default-deny. It only lacks the loopback rule.
+        "set firewall ipv6 input filter rule 5 action accept",
+        "set firewall ipv6 input filter rule 5 description 'loopback'",
+        "set firewall ipv6 input filter rule 5 inbound-interface name lo",
+        "set firewall ipv6 input filter rule 10 action accept",
+        "set firewall ipv6 input filter rule 10 description 'replies to our own traffic'",
+        "set firewall ipv6 input filter rule 10 state established",
+        "set firewall ipv6 input filter rule 10 state related",
+        # RFC 4890: filtering ICMPv6 wholesale breaks ND and PMTUD, which
+        # presents as "IPv6 works until something large", not as a block.
+        "set firewall ipv6 input filter rule 20 action accept",
+        "set firewall ipv6 input filter rule 20 description 'ICMPv6 - ND/RA/PMTUD'",
+        "set firewall ipv6 input filter rule 20 protocol icmpv6",
+        f"set firewall ipv6 input filter rule 30 action accept",
+        f"set firewall ipv6 input filter rule 30 description 'trusted LAN to the router'",
+        f"set firewall ipv6 input filter rule 30 inbound-interface group {LAN_GROUP}",
+        "set firewall ipv6 input filter default-action drop",
+        "set firewall ipv6 forward filter rule 10 action accept",
+        "set firewall ipv6 forward filter rule 10 description 'replies to our own traffic'",
+        "set firewall ipv6 forward filter rule 10 state established",
+        "set firewall ipv6 forward filter rule 10 state related",
+        "set firewall ipv6 forward filter rule 20 action accept",
+        "set firewall ipv6 forward filter rule 20 description 'ICMPv6 - ND/RA/PMTUD'",
+        "set firewall ipv6 forward filter rule 20 protocol icmpv6",
+        f"set firewall ipv6 forward filter rule 30 action accept",
+        f"set firewall ipv6 forward filter rule 30 description 'trusted LAN interfaces only'",
+        f"set firewall ipv6 forward filter rule 30 inbound-interface group {LAN_GROUP}",
+        "set firewall ipv6 forward filter default-action drop",
+        "",
+    ]
+    if wan_dhcp_if:
+        dhcp = [
+            "set firewall ipv4 input filter rule 140 action accept",
+            "set firewall ipv4 input filter rule 140 description 'DHCP client lease renewal'",
+            "set firewall ipv4 input filter rule 140 protocol udp",
+            "set firewall ipv4 input filter rule 140 destination port 68",
+            f"set firewall ipv4 input filter rule 140 inbound-interface name {wan_dhcp_if}",
+        ]
+        i = out.index("set firewall ipv4 input filter default-action drop")
+        out[i:i] = dhcp
+    return out
+
+
 def isp_dhcp(wan_if: str, uplink_if: str) -> list[str]:
     """The 10gig-equivalent ISP: hands out a lease, NATs to the real internet."""
     return [
@@ -250,9 +362,15 @@ def build(role: str, drop_scaffold: bool, wan_if: str, uplink_if: str) -> list[s
         # against a single access concentrator is a different failure mode than
         # anything production has. VRRP/conntrack failover is still exercised --
         # see README, "known gaps".
-        return [f"# labsim routing -- {role}", ""] + bgp(role) + wan(drop_scaffold)
+        return ([f"# labsim routing -- {role}", ""]
+                + bgp(role) + wan(drop_scaffold) + firewall())
     if role == "secondary":
-        return [f"# labsim routing -- {role}", ""] + bgp(role)
+        # The backup has no WAN in the sim, so it has no DHCP client to
+        # exempt -- but it gets the same policy otherwise, because after a VRRP
+        # failover it IS the router and a divergent ruleset would only be
+        # discovered during the failover.
+        return ([f"# labsim routing -- {role}", ""]
+                + bgp(role) + firewall(wan_dhcp_if=None))
     if role == "isp-dhcp":
         return isp_dhcp(wan_if, uplink_if)
     return isp_pppoe(wan_if, uplink_if)