c729275961
Some checks failed
The sim's eBGP was IPv4-only; extend it to the v6 family so labsim can rehearse
Cilium BGP advertising a public-style v6 LoadBalancer /64 to the VyOS pair before
it touches production (approved plan: public IPv6 via Cilium BGP + Gateway API).
- sim-net-config.py: bgp6 inside bgp() -- ipv6-unicast peer-group K8S6 over
fd00:2::11/12/13, prefix-list6 (le 128, /128 host routes), K8S-IN6/OUT6 route-maps
(export deny -- never hand the cluster a default). ULA fd61:1e00::/64 as the sim
LB range so nothing leaks into the real HE /48. All v6 lines pass the
sim-net-apply.sh whitelist.
- sim-ha-config.py: VLANS6 = {2: fd00:2/64} -> routers hold fd00:2::252/253 on
bond0.2 so they can peer the nodes over v6 (no v6 VRRP VIP; BGP peers the per-box
address, as production).
- k8s-up.sh: nodes get fd00:2::11/12/13 + dual-stack k3s (dual node-ip, dual
cluster/service CIDR). Peers line up with sim-net-config K8S_NODES_V6.
Config-gen verified for all three. Live chunk (Cilium v6+BGP+Gateway install,
bring-up, datapath/#26847 proof) is next.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DMVzWZgiKW2wquf5z8S1yH
482 lines
27 KiB
Python
Executable File
482 lines
27 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Generate the labsim routing + WAN config: BGP, dual WAN, and the two ISP VMs.
|
|
|
|
`sim-ha-config.py` covers the LAN side of the sim routers (addresses, VRRP,
|
|
conntrack-sync, DHCP). This covers everything that makes the sim a rehearsal for
|
|
production routing rather than just a LAN:
|
|
|
|
* eBGP between the sim routers and the k3s nodes, carrying the service range
|
|
* dual WAN -- DHCP on VLAN 53, PPPoE on VLAN 51 -- with health-checked failover
|
|
* the two ISP VMs that terminate those WANs and NAT to the real internet
|
|
|
|
All of it previously existed only as running state on the VMs, applied by hand
|
|
over SSH. Rebuilding a VM lost the rehearsal, and nothing recorded *why* any of
|
|
it was shaped the way it is. That is the entire reason this file exists.
|
|
|
|
./sim-net-config.py --role primary > r1-net.conf
|
|
./console-apply.py --vm labsim-vyos --config r1-net.conf
|
|
|
|
or apply all four at once with ./sim-net-apply.sh.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# BGP. Numbers match production so what is proven here ports over unchanged.
|
|
# ---------------------------------------------------------------------------
|
|
ROUTER_AS = 65000
|
|
CLUSTER_AS = 65001
|
|
# The service range Cilium advertises. Chosen against a survey of third-party
|
|
# RFC1918 defaults (docker-desktop, tailscale, k3s, EKS...) so it cannot collide
|
|
# with something we adopt later. Production uses the same /22 -- keep them equal.
|
|
SERVICE_CIDR = "10.61.0.0/22"
|
|
K8S_VLAN = 2
|
|
K8S_NODES = ["172.31.2.11", "172.31.2.12", "172.31.2.13"]
|
|
PEER_GROUP = "K8S"
|
|
PFX_LIST = "K8S-SERVICE-IPS"
|
|
RM_IN, RM_OUT = "K8S-IN", "K8S-OUT"
|
|
|
|
# IPv6 unicast. Production advertises a public Gateway LoadBalancer /64 out of
|
|
# the HE /48 (2001:470:187e:1e00::/64) and peers over VLAN 2 IPv6. The sim proves
|
|
# the MECHANISM -- ipv6-unicast eBGP, prefix-list6, /128 host routes, ECMP -- with
|
|
# a ULA so nothing here can leak into the real /48. Peering is over VLAN 2 v6
|
|
# (nodes' fd00:2::1x <-> routers' fd00:2::25x, set in sim-ha-config.py /
|
|
# k8s-up.sh), directly connected exactly as v4.
|
|
SERVICE_CIDR_V6 = "fd61:1e00::/64" # sim analog of prod :1e00::/64 LB pool
|
|
K8S_NODES_V6 = ["fd00:2::11", "fd00:2::12", "fd00:2::13"]
|
|
PEER_GROUP_V6 = "K8S6"
|
|
PFX_LIST_V6 = "K8S-SERVICE-IPS-V6"
|
|
RM_IN_V6, RM_OUT_V6 = "K8S-IN6", "K8S-OUT6"
|
|
# One route per node; ECMP across all three. 4 leaves headroom for a fourth node
|
|
# without a config change.
|
|
MAX_PATHS = 4
|
|
# A safety valve, not a capacity plan: a misconfigured Cilium that starts
|
|
# advertising pod CIDRs should tear the session down, not quietly fill the FIB.
|
|
MAX_PREFIX = 100
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dual WAN. The sim ISPs deliberately use TEST-NET-3 (203.0.113.0/24) and
|
|
# TEST-NET-2 (198.51.100.0/24) from RFC 5737: documentation ranges that are
|
|
# guaranteed never to be real destinations, so a leaked sim route cannot
|
|
# blackhole something that matters.
|
|
# ---------------------------------------------------------------------------
|
|
WAN_DHCP_VLAN = 53 # "10gig-equivalent" -- the primary in production
|
|
WAN_PPPOE_VLAN = 51 # "Vodafone-equivalent" -- the backup
|
|
ISP_DHCP_NET = "203.0.113.0/24"
|
|
ISP_DHCP_GW = "203.0.113.1"
|
|
ISP_DHCP_POOL = ("203.0.113.100", "203.0.113.150")
|
|
ISP_PPPOE_NET = "198.51.100.0/24"
|
|
ISP_PPPOE_GW = "198.51.100.1"
|
|
ISP_PPPOE_POOL = ("198.51.100.100", "198.51.100.150")
|
|
# Sim-only fake credentials. Both ends are in this file on purpose: they
|
|
# authenticate nothing real, and splitting them across a secret store would make
|
|
# the sim unreproducible for no security gain. The PRODUCTION PPPoE password
|
|
# lives in /config/wan-secrets on the router and is never in git.
|
|
PPPOE_USER, PPPOE_PASS = "simdsl", "simpass"
|
|
PPPOE_MTU = 1492 # 1500 - 8 bytes of PPPoE header
|
|
PPPOE_AC = "sim-isp"
|
|
|
|
# Failover probe targets. NOT 8.8.8.8/8.8.4.4: those are `system name-server`,
|
|
# so a probe failure and a DNS failure would be the same event and the router
|
|
# would flap the WAN every time DNS hiccuped.
|
|
PROBE_TARGETS = ["9.9.9.9", "208.67.222.222"]
|
|
# The bug this shape fixes (WI-8, found here, fixed in production): `ping -I
|
|
# bond0.53` binds the SOURCE address but does not make the kernel use that
|
|
# interface's gateway. On a cold boot where PPPoE won the default route, probes
|
|
# for the 10 gig egressed via PPPoE, succeeded, and the 10 gig was still never
|
|
# selected -- the house ran on the backup line silently. Pinning each target as
|
|
# a /32 via `dhcp-interface` forces the probe onto the line being tested.
|
|
WAN_DHCP_DISTANCE = 210 # NOT `no-default-route`, which blanks new_routers
|
|
PPPOE_DISTANCE = 10 # in the lease file, leaving failover no gateway
|
|
# to install and silently handing the default
|
|
# route to the backup line.
|
|
|
|
# One MAC, cloned onto BOTH routers' bond0.53, mirroring production's use of the
|
|
# retired USG's WAN2 MAC to keep its DHCP lease. The sim did not model a shared
|
|
# MAC at all, which is exactly why bond0.53 has to stay on the config plane --
|
|
# only VyOS config can move a MAC between boxes. Locally-administered, sim-only.
|
|
WAN_DHCP_MAC = "02:53:10:61:00:53"
|
|
|
|
SIM_LAN = "172.31.0.0/16"
|
|
|
|
# The sim routers' own libvirt-NAT uplink, from before the ISP VMs existed. It
|
|
# is a third default route that does not exist in production and quietly masks
|
|
# WAN failures during a failover test. `--drop-scaffold` removes it.
|
|
SCAFFOLD_IF = "eth2"
|
|
SCAFFOLD_NAT_RULE = 100
|
|
|
|
|
|
def bgp(role: str) -> list[str]:
|
|
"""eBGP toward the k3s nodes. Identical on both routers except router-id."""
|
|
octet = 252 if role == "primary" else 253
|
|
out = [
|
|
f"# --- BGP: AS{ROUTER_AS} <-> AS{CLUSTER_AS} (k3s/Cilium) ---",
|
|
# FRR enforces RFC 8212: an eBGP session with no policy establishes but
|
|
# exchanges ZERO prefixes, silently. Both directions need a policy or
|
|
# the session looks perfectly healthy and carries nothing.
|
|
f"set policy prefix-list {PFX_LIST} rule 10 action permit",
|
|
f"set policy prefix-list {PFX_LIST} rule 10 prefix {SERVICE_CIDR}",
|
|
# `le 32` because Cilium advertises individual /32 service addresses out
|
|
# of the pool, not the aggregate.
|
|
f"set policy prefix-list {PFX_LIST} rule 10 le 32",
|
|
f"set policy route-map {RM_IN} rule 10 action permit",
|
|
f"set policy route-map {RM_IN} rule 10 match ip address prefix-list {PFX_LIST}",
|
|
# Deny everything outbound. The cluster must never learn a default route
|
|
# from us -- Cilium would install it and blackhole pod egress.
|
|
f"set policy route-map {RM_OUT} rule 10 action deny",
|
|
f"set protocols bgp system-as {ROUTER_AS}",
|
|
f"set protocols bgp parameters router-id 172.31.{K8S_VLAN}.{octet}",
|
|
f"set protocols bgp address-family ipv4-unicast maximum-paths ebgp {MAX_PATHS}",
|
|
f"set protocols bgp peer-group {PEER_GROUP} remote-as {CLUSTER_AS}",
|
|
f"set protocols bgp peer-group {PEER_GROUP} address-family ipv4-unicast route-map import {RM_IN}",
|
|
f"set protocols bgp peer-group {PEER_GROUP} address-family ipv4-unicast route-map export {RM_OUT}",
|
|
f"set protocols bgp peer-group {PEER_GROUP} address-family ipv4-unicast maximum-prefix {MAX_PREFIX}",
|
|
]
|
|
out += [f"set protocols bgp neighbor {n} peer-group {PEER_GROUP}" for n in K8S_NODES]
|
|
# --- IPv6 unicast: same policy shape, over a separate v6 peer-group ---
|
|
# `prefix-list6` + `match ipv6 address` are the v6 spellings; `le 128` because
|
|
# Cilium advertises each Gateway LoadBalancer address as a /128, not the
|
|
# aggregate. Separate peer-group because the neighbors are v6 addresses; the
|
|
# export deny is the same safety property (never hand the cluster a default).
|
|
out += [
|
|
"",
|
|
f"set policy prefix-list6 {PFX_LIST_V6} rule 10 action permit",
|
|
f"set policy prefix-list6 {PFX_LIST_V6} rule 10 prefix {SERVICE_CIDR_V6}",
|
|
f"set policy prefix-list6 {PFX_LIST_V6} rule 10 le 128",
|
|
f"set policy route-map {RM_IN_V6} rule 10 action permit",
|
|
f"set policy route-map {RM_IN_V6} rule 10 match ipv6 address prefix-list {PFX_LIST_V6}",
|
|
f"set policy route-map {RM_OUT_V6} rule 10 action deny",
|
|
f"set protocols bgp address-family ipv6-unicast maximum-paths ebgp {MAX_PATHS}",
|
|
f"set protocols bgp peer-group {PEER_GROUP_V6} remote-as {CLUSTER_AS}",
|
|
f"set protocols bgp peer-group {PEER_GROUP_V6} address-family ipv6-unicast route-map import {RM_IN_V6}",
|
|
f"set protocols bgp peer-group {PEER_GROUP_V6} address-family ipv6-unicast route-map export {RM_OUT_V6}",
|
|
f"set protocols bgp peer-group {PEER_GROUP_V6} address-family ipv6-unicast maximum-prefix {MAX_PREFIX}",
|
|
]
|
|
out += [f"set protocols bgp neighbor {n} peer-group {PEER_GROUP_V6}" for n in K8S_NODES_V6]
|
|
out.append("")
|
|
return out
|
|
|
|
|
|
def wan(drop_scaffold: bool, role: str = "primary") -> list[str]:
|
|
"""Dual WAN + health-checked failover. IDENTICAL on both routers.
|
|
|
|
It used to be primary-only, on the grounds that "two PPPoE clients sharing
|
|
one credential against a single access concentrator is a different failure
|
|
mode than anything production has". That was backwards: production has
|
|
exactly that, and by omitting it the sim could not test the one thing most
|
|
likely to go wrong. The secondary having no WAN is also why it ended up with
|
|
zero NAT rules while the primary had ten -- the pair was not comparable.
|
|
|
|
Both routers therefore get the same WAN config. What differs is the RESTING
|
|
STATE, and only for the DHCP line:
|
|
|
|
bond0.53 `disable` on BOTH. Its lease is bound to a cloned MAC, and two
|
|
boxes claiming one MAC is the fault this whole design exists to
|
|
prevent. vrrp-wan-reconcile removes `disable` on the master.
|
|
pppoe0 enabled on BOTH, never `disable`d. `disable` unlinks
|
|
/etc/ppp/peers/pppoe0, which is pppd's own options file, so the
|
|
promotion path destroyed what it needed. Dialling is gated at
|
|
the systemd unit instead -- see migration/ppp-vrrp-gate.conf.
|
|
"""
|
|
out = [
|
|
"# --- WAN: DHCP (primary) + PPPoE (backup), health-checked ---",
|
|
f"set interfaces bonding bond0 vif {WAN_PPPOE_VLAN} description "
|
|
f"'WAN1 Vodafone-equivalent (sim ISP PPPoE)'",
|
|
f"set interfaces bonding bond0 vif {WAN_DHCP_VLAN} address dhcp",
|
|
f"set interfaces bonding bond0 vif {WAN_DHCP_VLAN} description "
|
|
f"'WAN3 10gig-equivalent (sim ISP DHCP)'",
|
|
f"set interfaces bonding bond0 vif {WAN_DHCP_VLAN} dhcp-options "
|
|
f"default-route-distance {WAN_DHCP_DISTANCE}",
|
|
# The cloned MAC. Production clones the old USG's WAN2 MAC so the ISP
|
|
# keeps handing back the same lease; the sim did not model a shared MAC
|
|
# at all, which is precisely why bond0.53 must stay on the config plane.
|
|
# Modelling it lets the sim prove the lease returns to the new master.
|
|
f"set interfaces bonding bond0 vif {WAN_DHCP_VLAN} mac {WAN_DHCP_MAC}",
|
|
# Safe at rest on BOTH routers: only the VIP holder enables it.
|
|
f"set interfaces bonding bond0 vif {WAN_DHCP_VLAN} disable",
|
|
f"set interfaces pppoe pppoe0 source-interface bond0.{WAN_PPPOE_VLAN}",
|
|
f"set interfaces pppoe pppoe0 authentication username {PPPOE_USER}",
|
|
f"set interfaces pppoe pppoe0 authentication password {PPPOE_PASS}",
|
|
f"set interfaces pppoe pppoe0 default-route-distance {PPPOE_DISTANCE}",
|
|
f"set interfaces pppoe pppoe0 mtu {PPPOE_MTU}",
|
|
# The ISP's resolvers would otherwise overwrite ours in resolv.conf every
|
|
# time the session comes up.
|
|
"set interfaces pppoe pppoe0 no-peer-dns",
|
|
"",
|
|
"# Failover: prefer the DHCP WAN, fall back to PPPoE when probes fail.",
|
|
f"set protocols failover route 0.0.0.0/0 dhcp-interface bond0.{WAN_DHCP_VLAN} metric 1",
|
|
f"set protocols failover route 0.0.0.0/0 dhcp-interface bond0.{WAN_DHCP_VLAN} check type icmp",
|
|
f"set protocols failover route 0.0.0.0/0 dhcp-interface bond0.{WAN_DHCP_VLAN} check timeout 5",
|
|
# any-available, not all: one unreachable public resolver is a normal
|
|
# internet event, not a reason to abandon a working 10 gig line.
|
|
f"set protocols failover route 0.0.0.0/0 dhcp-interface bond0.{WAN_DHCP_VLAN} check policy any-available",
|
|
]
|
|
for t in PROBE_TARGETS:
|
|
out.append(f"set protocols failover route 0.0.0.0/0 dhcp-interface "
|
|
f"bond0.{WAN_DHCP_VLAN} check target {t}")
|
|
out.append("")
|
|
out.append("# Pin the probe targets to the line under test (WI-8 -- see above).")
|
|
for t in PROBE_TARGETS:
|
|
out.append(f"set protocols static route {t}/32 dhcp-interface bond0.{WAN_DHCP_VLAN}")
|
|
out += [
|
|
"",
|
|
"# Masquerade out of whichever WAN currently holds the default route.",
|
|
f"set nat source rule 110 outbound-interface name bond0.{WAN_DHCP_VLAN}",
|
|
f"set nat source rule 110 source address {SIM_LAN}",
|
|
"set nat source rule 110 translation address masquerade",
|
|
"set nat source rule 120 outbound-interface name pppoe0",
|
|
f"set nat source rule 120 source address {SIM_LAN}",
|
|
"set nat source rule 120 translation address masquerade",
|
|
"",
|
|
]
|
|
if drop_scaffold:
|
|
out += [
|
|
"# Remove the pre-ISP-VM libvirt-NAT uplink: a third default route",
|
|
"# that has no production equivalent and hides real WAN failures.",
|
|
f"delete interfaces ethernet {SCAFFOLD_IF} address",
|
|
f"delete nat source rule {SCAFFOLD_NAT_RULE}",
|
|
"",
|
|
]
|
|
return out
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Firewall. The policy is: internal VLANs talk to each other and to the
|
|
# internet; the internet initiates nothing inward.
|
|
#
|
|
# That was already the *effect* of the previous IPv4 ruleset, but it was built
|
|
# as a blacklist -- `default-action accept` plus explicit drops on each WAN
|
|
# interface. The result is identical right up until someone adds a WAN, at
|
|
# which point it is wide open and nothing looks wrong. This is the same policy
|
|
# expressed as a whitelist, so a new interface is closed until it is named.
|
|
# ---------------------------------------------------------------------------
|
|
# Management is `bond0.1`, NOT the bare `bond0`. Every VLAN is tagged and the
|
|
# bond parent carries no subnet at all -- see NATIVE_VLAN in ovs.sh for why.
|
|
#
|
|
# This line is the trap in that change. The address move is the visible part and
|
|
# the part you remember; leaving `bond0` here instead of `bond0.1` means the
|
|
# whole Management VLAN falls outside the LAN group, and with a default-deny
|
|
# ruleset that is every management session and all inter-VLAN routing for VLAN 1
|
|
# dropped the instant the commit lands -- on a router you reach through itself.
|
|
LAN_IFACES = ["bond0.1", "bond0.2", "bond0.3", "bond0.9", "bond0.10", "bond0.200"]
|
|
LAN_GROUP = "LAN"
|
|
|
|
|
|
def firewall(wan_dhcp_if: str | None = f"bond0.{WAN_DHCP_VLAN}") -> list[str]:
|
|
"""wan_dhcp_if=None on a router with no DHCP WAN -- a firewall rule naming
|
|
an interface that does not exist is rejected at commit."""
|
|
out = [f"# --- firewall: LAN-to-anywhere, internet-to-nothing ---"]
|
|
# Delete each filter before rebuilding it. `set` on a rule number is
|
|
# ADDITIVE: if a rule 10 already exists carrying an inbound-interface
|
|
# constraint, `set ... rule 10 state established` silently ANDs onto it,
|
|
# and you get a stateful-accept rule that only applies to one interface
|
|
# pair. Observed in labsim: return traffic from the internet matched
|
|
# neither that rule nor the LAN rule and hit the default drop, so LAN
|
|
# hosts could reach nothing outbound. Everything here is one commit, so
|
|
# nftables is rebuilt atomically -- there is no window with no firewall.
|
|
out += [f"delete firewall {fam} {hook} filter"
|
|
for fam in ("ipv4", "ipv6") for hook in ("forward", "input")]
|
|
out += [f"set firewall group interface-group {LAN_GROUP} interface {i}"
|
|
for i in LAN_IFACES]
|
|
out += [
|
|
"",
|
|
# INPUT -- traffic terminating ON the router.
|
|
# Loopback first. Under a default-drop input policy, services talking to
|
|
# 127.0.0.1 are filtered like anything else, and the failures are
|
|
# bizarre and hard to attribute. Nothing off-box can forge iif lo.
|
|
"set firewall ipv4 input filter rule 5 action accept",
|
|
"set firewall ipv4 input filter rule 5 description 'loopback'",
|
|
"set firewall ipv4 input filter rule 5 inbound-interface name lo",
|
|
"set firewall ipv4 input filter rule 10 action accept",
|
|
"set firewall ipv4 input filter rule 10 description 'established/related'",
|
|
"set firewall ipv4 input filter rule 10 state established",
|
|
"set firewall ipv4 input filter rule 10 state related",
|
|
# One rule covers VRRP, conntrack-sync, kea HA, SSH, DNS and BGP,
|
|
# because every one of them arrives on a LAN interface. Enumerating the
|
|
# protocols instead would mean a new firewall rule every time the pair
|
|
# gains a feature -- and a lockout the day someone forgets.
|
|
f"set firewall ipv4 input filter rule 20 action accept",
|
|
f"set firewall ipv4 input filter rule 20 description 'trusted LAN to the router'",
|
|
f"set firewall ipv4 input filter rule 20 inbound-interface group {LAN_GROUP}",
|
|
# DHCP client. Lease RENEWAL is unicast UDP to port 68 and conntrack
|
|
# does not reliably cover it, so without this the WAN keeps working
|
|
# until the lease expires and then dies -- a delayed failure that looks
|
|
# nothing like a firewall change.
|
|
"set firewall ipv4 input filter default-action drop",
|
|
"",
|
|
# FORWARD -- traffic passing THROUGH the router.
|
|
"set firewall ipv4 forward filter rule 10 action accept",
|
|
"set firewall ipv4 forward filter rule 10 description 'established/related'",
|
|
"set firewall ipv4 forward filter rule 10 state established",
|
|
"set firewall ipv4 forward filter rule 10 state related",
|
|
# Inter-VLAN *and* LAN-to-internet in one rule: both are "came in on a
|
|
# LAN interface". Deliberately no restriction between internal VLANs --
|
|
# segmenting them is a separate decision, not a side effect of this one.
|
|
f"set firewall ipv4 forward filter rule 20 action accept",
|
|
f"set firewall ipv4 forward filter rule 20 description 'LAN to anywhere (inter-VLAN + internet)'",
|
|
f"set firewall ipv4 forward filter rule 20 inbound-interface group {LAN_GROUP}",
|
|
"set firewall ipv4 forward filter default-action drop",
|
|
"",
|
|
# IPv6 already runs default-deny. It only lacks the loopback rule.
|
|
"set firewall ipv6 input filter rule 5 action accept",
|
|
"set firewall ipv6 input filter rule 5 description 'loopback'",
|
|
"set firewall ipv6 input filter rule 5 inbound-interface name lo",
|
|
"set firewall ipv6 input filter rule 10 action accept",
|
|
"set firewall ipv6 input filter rule 10 description 'replies to our own traffic'",
|
|
"set firewall ipv6 input filter rule 10 state established",
|
|
"set firewall ipv6 input filter rule 10 state related",
|
|
# RFC 4890: filtering ICMPv6 wholesale breaks ND and PMTUD, which
|
|
# presents as "IPv6 works until something large", not as a block.
|
|
"set firewall ipv6 input filter rule 20 action accept",
|
|
"set firewall ipv6 input filter rule 20 description 'ICMPv6 - ND/RA/PMTUD'",
|
|
"set firewall ipv6 input filter rule 20 protocol icmpv6",
|
|
f"set firewall ipv6 input filter rule 30 action accept",
|
|
f"set firewall ipv6 input filter rule 30 description 'trusted LAN to the router'",
|
|
f"set firewall ipv6 input filter rule 30 inbound-interface group {LAN_GROUP}",
|
|
"set firewall ipv6 input filter default-action drop",
|
|
"set firewall ipv6 forward filter rule 10 action accept",
|
|
"set firewall ipv6 forward filter rule 10 description 'replies to our own traffic'",
|
|
"set firewall ipv6 forward filter rule 10 state established",
|
|
"set firewall ipv6 forward filter rule 10 state related",
|
|
"set firewall ipv6 forward filter rule 20 action accept",
|
|
"set firewall ipv6 forward filter rule 20 description 'ICMPv6 - ND/RA/PMTUD'",
|
|
"set firewall ipv6 forward filter rule 20 protocol icmpv6",
|
|
f"set firewall ipv6 forward filter rule 30 action accept",
|
|
f"set firewall ipv6 forward filter rule 30 description 'trusted LAN interfaces only'",
|
|
f"set firewall ipv6 forward filter rule 30 inbound-interface group {LAN_GROUP}",
|
|
"set firewall ipv6 forward filter default-action drop",
|
|
"",
|
|
]
|
|
if wan_dhcp_if:
|
|
dhcp = [
|
|
"set firewall ipv4 input filter rule 140 action accept",
|
|
"set firewall ipv4 input filter rule 140 description 'DHCP client lease renewal'",
|
|
"set firewall ipv4 input filter rule 140 protocol udp",
|
|
"set firewall ipv4 input filter rule 140 destination port 68",
|
|
f"set firewall ipv4 input filter rule 140 inbound-interface name {wan_dhcp_if}",
|
|
]
|
|
i = out.index("set firewall ipv4 input filter default-action drop")
|
|
out[i:i] = dhcp
|
|
return out
|
|
|
|
|
|
def isp_dhcp(wan_if: str, uplink_if: str) -> list[str]:
|
|
"""The 10gig-equivalent ISP: hands out a lease, NATs to the real internet."""
|
|
return [
|
|
f"# --- sim ISP: DHCP WAN on VLAN {WAN_DHCP_VLAN} ---",
|
|
"set system host-name isp-dhcp",
|
|
f"set interfaces ethernet {wan_if} address {ISP_DHCP_GW}/24",
|
|
f"set interfaces ethernet {wan_if} description "
|
|
f"'sim ISP - 10gig-equivalent WAN on VLAN{WAN_DHCP_VLAN}'",
|
|
f"set interfaces ethernet {uplink_if} address dhcp",
|
|
f"set interfaces ethernet {uplink_if} description 'uplink to the real internet'",
|
|
f"set service dhcp-server shared-network-name WAN{WAN_DHCP_VLAN} "
|
|
f"subnet {ISP_DHCP_NET} subnet-id 1",
|
|
f"set service dhcp-server shared-network-name WAN{WAN_DHCP_VLAN} "
|
|
f"subnet {ISP_DHCP_NET} option default-router {ISP_DHCP_GW}",
|
|
f"set service dhcp-server shared-network-name WAN{WAN_DHCP_VLAN} "
|
|
f"subnet {ISP_DHCP_NET} option name-server 8.8.8.8",
|
|
f"set service dhcp-server shared-network-name WAN{WAN_DHCP_VLAN} "
|
|
f"subnet {ISP_DHCP_NET} range CUST start {ISP_DHCP_POOL[0]}",
|
|
f"set service dhcp-server shared-network-name WAN{WAN_DHCP_VLAN} "
|
|
f"subnet {ISP_DHCP_NET} range CUST stop {ISP_DHCP_POOL[1]}",
|
|
"",
|
|
] + _isp_common(uplink_if, ISP_DHCP_NET, "sim ISP: NAT customers to the real internet")
|
|
|
|
|
|
def isp_pppoe(wan_if: str, uplink_if: str) -> list[str]:
|
|
"""The Vodafone-equivalent ISP: terminates PPPoE, NATs to the real internet."""
|
|
return [
|
|
f"# --- sim ISP: PPPoE WAN on VLAN {WAN_PPPOE_VLAN} ---",
|
|
"set system host-name isp-pppoe",
|
|
f"set interfaces ethernet {wan_if} description "
|
|
f"'sim ISP - Vodafone-equivalent WAN on VLAN{WAN_PPPOE_VLAN} (PPPoE)'",
|
|
f"set interfaces ethernet {uplink_if} address dhcp",
|
|
f"set interfaces ethernet {uplink_if} description 'uplink to the real internet'",
|
|
f"set service pppoe-server access-concentrator {PPPOE_AC}",
|
|
f"set service pppoe-server interface {wan_if}",
|
|
f"set service pppoe-server gateway-address {ISP_PPPOE_GW}",
|
|
"set service pppoe-server authentication mode local",
|
|
f"set service pppoe-server authentication local-users username {PPPOE_USER} "
|
|
f"password {PPPOE_PASS}",
|
|
f"set service pppoe-server client-ip-pool CUST range "
|
|
f"{ISP_PPPOE_POOL[0]}-{ISP_PPPOE_POOL[1]}",
|
|
"set service pppoe-server default-pool CUST",
|
|
"set service pppoe-server name-server 8.8.8.8",
|
|
"",
|
|
] + _isp_common(uplink_if, ISP_PPPOE_NET, "sim ISP: NAT PPPoE customers to the real internet")
|
|
|
|
|
|
def _isp_common(uplink_if: str, customer_net: str, desc: str) -> list[str]:
|
|
return [
|
|
"set nat source rule 100 description " + f"'{desc}'",
|
|
f"set nat source rule 100 outbound-interface name {uplink_if}",
|
|
f"set nat source rule 100 source address {customer_net}",
|
|
"set nat source rule 100 translation address masquerade",
|
|
"",
|
|
# An ISP that drops return traffic is not simulating an ISP. The forward
|
|
# chain defaults to accept here on purpose -- these VMs model the
|
|
# internet, and the thing under test is the router's firewall, not this.
|
|
"set firewall ipv4 forward filter default-action accept",
|
|
"set firewall ipv4 forward filter rule 10 action accept",
|
|
"set firewall ipv4 forward filter rule 10 state established",
|
|
"set firewall ipv4 forward filter rule 10 state related",
|
|
"set firewall ipv4 forward filter rule 10 description conntrack-engage",
|
|
"",
|
|
]
|
|
|
|
|
|
def build(role: str, drop_scaffold: bool, wan_if: str, uplink_if: str) -> list[str]:
|
|
if role in ("primary", "secondary"):
|
|
# BOTH routers get the identical WAN. The sim used to give it to the
|
|
# primary only, reasoning that two PPPoE clients sharing one credential
|
|
# was "a different failure mode than anything production has" -- but
|
|
# that IS production, and omitting it meant the failover path was the
|
|
# one path the sim could not exercise. It also left the pair
|
|
# incomparable: ten NAT rules on one box, none on the other.
|
|
#
|
|
# Safety comes from resting state, not from asymmetry: bond0.53 is
|
|
# `disable`d on both (cloned MAC), pppoe0 is enabled on both but gated
|
|
# at the systemd unit. See wan() and migration/ppp-vrrp-gate.conf.
|
|
return ([f"# labsim routing -- {role}", ""]
|
|
+ bgp(role) + wan(drop_scaffold, role)
|
|
+ firewall(wan_dhcp_if=f"bond0.{WAN_DHCP_VLAN}"))
|
|
if role == "isp-dhcp":
|
|
return isp_dhcp(wan_if, uplink_if)
|
|
return isp_pppoe(wan_if, uplink_if)
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--role", required=True,
|
|
choices=("primary", "secondary", "isp-dhcp", "isp-pppoe"))
|
|
ap.add_argument("--drop-scaffold", action="store_true",
|
|
help="also remove the pre-ISP-VM libvirt-NAT uplink (primary only)")
|
|
# The ISP VMs' interface names depend on PCI enumeration order, which is not
|
|
# stable across a rebuild: isp-dhcp came up as eth0/eth1 and isp-pppoe as
|
|
# eth2/eth3 from identical XML. Check with `show interfaces` before applying
|
|
# rather than trusting these defaults.
|
|
ap.add_argument("--wan-if", default=None, help="ISP VM: customer-facing NIC")
|
|
ap.add_argument("--uplink-if", default=None, help="ISP VM: internet-facing NIC")
|
|
args = ap.parse_args()
|
|
|
|
defaults = {"isp-dhcp": ("eth0", "eth1"), "isp-pppoe": ("eth2", "eth3")}
|
|
w, u = defaults.get(args.role, ("", ""))
|
|
w, u = args.wan_if or w, args.uplink_if or u
|
|
|
|
if args.drop_scaffold and args.role != "primary":
|
|
print("--drop-scaffold only applies to --role primary", file=sys.stderr)
|
|
return 2
|
|
|
|
lines = build(args.role, args.drop_scaffold, w, u)
|
|
sys.stdout.write("\n".join(lines) + "\n")
|
|
n = len([l for l in lines if l.startswith(("set ", "delete "))])
|
|
print(f"{args.role}: {n} commands", file=sys.stderr)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|