diff --git a/labsim/README.md b/labsim/README.md index 497d2ee..292954c 100644 --- a/labsim/README.md +++ b/labsim/README.md @@ -143,6 +143,49 @@ unreserved MAC gets an unreserved address. - **http://localhost:9101/metrics** — `labsim_reachable{src,dst,proto}` and `labsim_rtt_ms{src,dst}`. +## Routing: BGP, dual WAN, and the ISP VMs + +`sim-ha-config.py` covers the LAN side of the routers. `sim-net-config.py` +covers everything that makes this a rehearsal for production *routing*: + +| role | VM | what it generates | +|---|---|---| +| `primary` | `labsim-vyos` | BGP + dual WAN + health-checked failover | +| `secondary` | `labsim-vyos2` | BGP only | +| `isp-dhcp` | `labsim-isp-dhcp` | 10gig-equivalent ISP on VLAN 53 | +| `isp-pppoe` | `labsim-isp-pppoe` | Vodafone-equivalent PPPoE ISP on VLAN 51 | + +Both ISP VMs are VyOS with two NICs: one on the OVS trunk facing the sim +router, one on libvirt's `default` network, NATing customers to the real +internet. They use RFC 5737 documentation ranges (`203.0.113.0/24`, +`198.51.100.0/24`) so a leaked sim route cannot blackhole anything real. + +```sh +./sim-net-apply.sh check # VM state vs what the code says — run this first +./sim-net-apply.sh apply # push generated config over the serial console +``` + +`check` is the important one. All of this previously existed only as running +state, applied by hand over SSH; rebuilding a VM lost it, and nothing recorded +why any of it was shaped the way it was. + +### Known gaps vs production + +- **WAN is on the primary router only.** Production has WAN on both. Two PPPoE + clients sharing one credential against a single access concentrator is a + failure mode production does not have, so the sim does not model it. VRRP and + conntrack failover are still exercised. +- **ISP VM interface names are not stable across a rebuild** — `isp-dhcp` came + up as `eth0`/`eth1` and `isp-pppoe` as `eth2`/`eth3` from identical XML. + Check `show interfaces` and pass `--wan-if` / `--uplink-if` rather than + trusting the defaults. +- **`eth2` on the primary router** is a libvirt-NAT uplink predating the ISP + VMs: a third default route with no production equivalent that masks real WAN + failures during a failover test. `--drop-scaffold` removes it. +- **Committing on `isp-pppoe` drops the router's PPPoE session**, and the + client does not redial promptly. After any change there, check `pppoe0` on + the router and `sudo systemctl restart ppp@pppoe0` if it is missing. + ## Notes for whoever extends this Things that cost time the first time round, all verified on this image: diff --git a/labsim/sim-net-apply.sh b/labsim/sim-net-apply.sh new file mode 100755 index 0000000..d001120 --- /dev/null +++ b/labsim/sim-net-apply.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Apply -- or drift-check -- the labsim routing config on all four VMs. +# +# ./sim-net-apply.sh check what the VMs run vs what sim-net-config.py says +# ./sim-net-apply.sh apply push the generated config over the serial console +# +# `check` is the one you want most of the time. The whole failure mode this +# guards against is somebody (including me) fixing something on a VM over SSH +# and never writing it down, so the next rebuild silently loses it. +# +# Applied over the serial console rather than SSH because a freshly installed +# sim router holds the same addresses as its peer -- there is a window where it +# is not safely reachable over the network at all. See console-apply.py. +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ACTION="${1:-check}" +WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT + +# role : vm : address : regex selecting the subtrees this generator owns +TARGETS=( + "primary:labsim-vyos:172.31.1.252:^set (protocols (bgp|failover|static)|policy (prefix-list|route-map)|nat source rule 1[12]0|interfaces (pppoe|bonding bond0 vif 5[13]))" + "secondary:labsim-vyos2:172.31.1.253:^set (protocols bgp|policy (prefix-list|route-map))" + "isp-dhcp:labsim-isp-dhcp:192.168.122.136:^set (interfaces ethernet|nat source|service dhcp-server|firewall ipv4 forward|system host-name)" + "isp-pppoe:labsim-isp-pppoe:192.168.122.63:^set (interfaces ethernet|nat source|service pppoe-server|firewall ipv4 forward|system host-name)" +) +# Sim-only credential; these VMs hold nothing real and are not reachable from +# outside the hypervisor. +SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR -o PreferredAuthentications=password -o ConnectTimeout=5) +live() { timeout 30 sshpass -p vyos ssh "${SSH_OPTS[@]}" "vyos@$1" \ + "/opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands" 2>/dev/null; } +norm() { sed "s/'//g" | grep -v 'hw-id\|offload' | sort -u; } + +rc=0 +for t in "${TARGETS[@]}"; do + IFS=: read -r role vm addr rx <<<"$t" + "$HERE/sim-net-config.py" --role "$role" >"$WORK/$role.conf" 2>/dev/null || { + printf ' %-11s GENERATE FAILED\n' "$role"; rc=1; continue; } + + if [ "$ACTION" = apply ]; then + printf ' %-11s applying to %s over console...\n' "$role" "$vm" + "$HERE/console-apply.py" --vm "$vm" --config "$WORK/$role.conf" || rc=1 + continue + fi + + if ! live "$addr" >"$WORK/$role.live" || [ ! -s "$WORK/$role.live" ]; then + printf ' %-11s UNREACHABLE (%s)\n' "$role" "$addr"; rc=1; continue + fi + grep -E '^set ' "$WORK/$role.conf" | norm >"$WORK/$role.g" + grep -E "$rx" "$WORK/$role.live" | norm >"$WORK/$role.l" + if d="$(diff "$WORK/$role.g" "$WORK/$role.l")" && [ -z "$d" ]; then + printf ' %-11s in sync (%s commands)\n' "$role" "$(wc -l <"$WORK/$role.g")" + else + printf ' %-11s DRIFT — "<" only in code, ">" only on the VM:\n' "$role" + printf '%s\n' "$d" | sed 's/^/ /' + rc=1 + fi +done +exit $rc diff --git a/labsim/sim-net-config.py b/labsim/sim-net-config.py new file mode 100755 index 0000000..7813b7e --- /dev/null +++ b/labsim/sim-net-config.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +"""Generate the labsim routing + WAN config: BGP, dual WAN, and the two ISP VMs. + +`sim-ha-config.py` covers the LAN side of the sim routers (addresses, VRRP, +conntrack-sync, DHCP). This covers everything that makes the sim a rehearsal for +production routing rather than just a LAN: + + * eBGP between the sim routers and the k3s nodes, carrying the service range + * dual WAN -- DHCP on VLAN 53, PPPoE on VLAN 51 -- with health-checked failover + * the two ISP VMs that terminate those WANs and NAT to the real internet + +All of it previously existed only as running state on the VMs, applied by hand +over SSH. Rebuilding a VM lost the rehearsal, and nothing recorded *why* any of +it was shaped the way it is. That is the entire reason this file exists. + + ./sim-net-config.py --role primary > r1-net.conf + ./console-apply.py --vm labsim-vyos --config r1-net.conf + +or apply all four at once with ./sim-net-apply.sh. +""" +from __future__ import annotations + +import argparse +import sys + +# --------------------------------------------------------------------------- +# BGP. Numbers match production so what is proven here ports over unchanged. +# --------------------------------------------------------------------------- +ROUTER_AS = 65000 +CLUSTER_AS = 65001 +# The service range Cilium advertises. Chosen against a survey of third-party +# RFC1918 defaults (docker-desktop, tailscale, k3s, EKS...) so it cannot collide +# with something we adopt later. Production uses the same /22 -- keep them equal. +SERVICE_CIDR = "10.61.0.0/22" +K8S_VLAN = 2 +K8S_NODES = ["172.31.2.11", "172.31.2.12", "172.31.2.13"] +PEER_GROUP = "K8S" +PFX_LIST = "K8S-SERVICE-IPS" +RM_IN, RM_OUT = "K8S-IN", "K8S-OUT" +# One route per node; ECMP across all three. 4 leaves headroom for a fourth node +# without a config change. +MAX_PATHS = 4 +# A safety valve, not a capacity plan: a misconfigured Cilium that starts +# advertising pod CIDRs should tear the session down, not quietly fill the FIB. +MAX_PREFIX = 100 + +# --------------------------------------------------------------------------- +# Dual WAN. The sim ISPs deliberately use TEST-NET-3 (203.0.113.0/24) and +# TEST-NET-2 (198.51.100.0/24) from RFC 5737: documentation ranges that are +# guaranteed never to be real destinations, so a leaked sim route cannot +# blackhole something that matters. +# --------------------------------------------------------------------------- +WAN_DHCP_VLAN = 53 # "10gig-equivalent" -- the primary in production +WAN_PPPOE_VLAN = 51 # "Vodafone-equivalent" -- the backup +ISP_DHCP_NET = "203.0.113.0/24" +ISP_DHCP_GW = "203.0.113.1" +ISP_DHCP_POOL = ("203.0.113.100", "203.0.113.150") +ISP_PPPOE_NET = "198.51.100.0/24" +ISP_PPPOE_GW = "198.51.100.1" +ISP_PPPOE_POOL = ("198.51.100.100", "198.51.100.150") +# Sim-only fake credentials. Both ends are in this file on purpose: they +# authenticate nothing real, and splitting them across a secret store would make +# the sim unreproducible for no security gain. The PRODUCTION PPPoE password +# lives in /config/wan-secrets on the router and is never in git. +PPPOE_USER, PPPOE_PASS = "simdsl", "simpass" +PPPOE_MTU = 1492 # 1500 - 8 bytes of PPPoE header +PPPOE_AC = "sim-isp" + +# Failover probe targets. NOT 8.8.8.8/8.8.4.4: those are `system name-server`, +# so a probe failure and a DNS failure would be the same event and the router +# would flap the WAN every time DNS hiccuped. +PROBE_TARGETS = ["9.9.9.9", "208.67.222.222"] +# The bug this shape fixes (WI-8, found here, fixed in production): `ping -I +# bond0.53` binds the SOURCE address but does not make the kernel use that +# interface's gateway. On a cold boot where PPPoE won the default route, probes +# for the 10 gig egressed via PPPoE, succeeded, and the 10 gig was still never +# selected -- the house ran on the backup line silently. Pinning each target as +# a /32 via `dhcp-interface` forces the probe onto the line being tested. +WAN_DHCP_DISTANCE = 210 # NOT `no-default-route`, which blanks new_routers +PPPOE_DISTANCE = 10 # in the lease file, leaving failover no gateway + # to install and silently handing the default + # route to the backup line. +SIM_LAN = "172.31.0.0/16" + +# The sim routers' own libvirt-NAT uplink, from before the ISP VMs existed. It +# is a third default route that does not exist in production and quietly masks +# WAN failures during a failover test. `--drop-scaffold` removes it. +SCAFFOLD_IF = "eth2" +SCAFFOLD_NAT_RULE = 100 + + +def bgp(role: str) -> list[str]: + """eBGP toward the k3s nodes. Identical on both routers except router-id.""" + octet = 252 if role == "primary" else 253 + out = [ + f"# --- BGP: AS{ROUTER_AS} <-> AS{CLUSTER_AS} (k3s/Cilium) ---", + # FRR enforces RFC 8212: an eBGP session with no policy establishes but + # exchanges ZERO prefixes, silently. Both directions need a policy or + # the session looks perfectly healthy and carries nothing. + f"set policy prefix-list {PFX_LIST} rule 10 action permit", + f"set policy prefix-list {PFX_LIST} rule 10 prefix {SERVICE_CIDR}", + # `le 32` because Cilium advertises individual /32 service addresses out + # of the pool, not the aggregate. + f"set policy prefix-list {PFX_LIST} rule 10 le 32", + f"set policy route-map {RM_IN} rule 10 action permit", + f"set policy route-map {RM_IN} rule 10 match ip address prefix-list {PFX_LIST}", + # Deny everything outbound. The cluster must never learn a default route + # from us -- Cilium would install it and blackhole pod egress. + f"set policy route-map {RM_OUT} rule 10 action deny", + f"set protocols bgp system-as {ROUTER_AS}", + f"set protocols bgp parameters router-id 172.31.{K8S_VLAN}.{octet}", + f"set protocols bgp address-family ipv4-unicast maximum-paths ebgp {MAX_PATHS}", + f"set protocols bgp peer-group {PEER_GROUP} remote-as {CLUSTER_AS}", + f"set protocols bgp peer-group {PEER_GROUP} address-family ipv4-unicast route-map import {RM_IN}", + f"set protocols bgp peer-group {PEER_GROUP} address-family ipv4-unicast route-map export {RM_OUT}", + f"set protocols bgp peer-group {PEER_GROUP} address-family ipv4-unicast maximum-prefix {MAX_PREFIX}", + ] + out += [f"set protocols bgp neighbor {n} peer-group {PEER_GROUP}" for n in K8S_NODES] + out.append("") + return out + + +def wan(drop_scaffold: bool) -> list[str]: + """Dual WAN + health-checked failover. Primary router only -- see README.""" + out = [ + "# --- WAN: DHCP (primary) + PPPoE (backup), health-checked ---", + f"set interfaces bonding bond0 vif {WAN_PPPOE_VLAN} description " + f"'WAN1 Vodafone-equivalent (sim ISP PPPoE)'", + f"set interfaces bonding bond0 vif {WAN_DHCP_VLAN} address dhcp", + f"set interfaces bonding bond0 vif {WAN_DHCP_VLAN} description " + f"'WAN3 10gig-equivalent (sim ISP DHCP)'", + f"set interfaces bonding bond0 vif {WAN_DHCP_VLAN} dhcp-options " + f"default-route-distance {WAN_DHCP_DISTANCE}", + f"set interfaces pppoe pppoe0 source-interface bond0.{WAN_PPPOE_VLAN}", + f"set interfaces pppoe pppoe0 authentication username {PPPOE_USER}", + f"set interfaces pppoe pppoe0 authentication password {PPPOE_PASS}", + f"set interfaces pppoe pppoe0 default-route-distance {PPPOE_DISTANCE}", + f"set interfaces pppoe pppoe0 mtu {PPPOE_MTU}", + # The ISP's resolvers would otherwise overwrite ours in resolv.conf every + # time the session comes up. + "set interfaces pppoe pppoe0 no-peer-dns", + "", + "# Failover: prefer the DHCP WAN, fall back to PPPoE when probes fail.", + f"set protocols failover route 0.0.0.0/0 dhcp-interface bond0.{WAN_DHCP_VLAN} metric 1", + f"set protocols failover route 0.0.0.0/0 dhcp-interface bond0.{WAN_DHCP_VLAN} check type icmp", + f"set protocols failover route 0.0.0.0/0 dhcp-interface bond0.{WAN_DHCP_VLAN} check timeout 5", + # any-available, not all: one unreachable public resolver is a normal + # internet event, not a reason to abandon a working 10 gig line. + f"set protocols failover route 0.0.0.0/0 dhcp-interface bond0.{WAN_DHCP_VLAN} check policy any-available", + ] + for t in PROBE_TARGETS: + out.append(f"set protocols failover route 0.0.0.0/0 dhcp-interface " + f"bond0.{WAN_DHCP_VLAN} check target {t}") + out.append("") + out.append("# Pin the probe targets to the line under test (WI-8 -- see above).") + for t in PROBE_TARGETS: + out.append(f"set protocols static route {t}/32 dhcp-interface bond0.{WAN_DHCP_VLAN}") + out += [ + "", + "# Masquerade out of whichever WAN currently holds the default route.", + f"set nat source rule 110 outbound-interface name bond0.{WAN_DHCP_VLAN}", + f"set nat source rule 110 source address {SIM_LAN}", + "set nat source rule 110 translation address masquerade", + "set nat source rule 120 outbound-interface name pppoe0", + f"set nat source rule 120 source address {SIM_LAN}", + "set nat source rule 120 translation address masquerade", + "", + ] + if drop_scaffold: + out += [ + "# Remove the pre-ISP-VM libvirt-NAT uplink: a third default route", + "# that has no production equivalent and hides real WAN failures.", + f"delete interfaces ethernet {SCAFFOLD_IF} address", + f"delete nat source rule {SCAFFOLD_NAT_RULE}", + "", + ] + return out + + +def isp_dhcp(wan_if: str, uplink_if: str) -> list[str]: + """The 10gig-equivalent ISP: hands out a lease, NATs to the real internet.""" + return [ + f"# --- sim ISP: DHCP WAN on VLAN {WAN_DHCP_VLAN} ---", + "set system host-name isp-dhcp", + f"set interfaces ethernet {wan_if} address {ISP_DHCP_GW}/24", + f"set interfaces ethernet {wan_if} description " + f"'sim ISP - 10gig-equivalent WAN on VLAN{WAN_DHCP_VLAN}'", + f"set interfaces ethernet {uplink_if} address dhcp", + f"set interfaces ethernet {uplink_if} description 'uplink to the real internet'", + f"set service dhcp-server shared-network-name WAN{WAN_DHCP_VLAN} " + f"subnet {ISP_DHCP_NET} subnet-id 1", + f"set service dhcp-server shared-network-name WAN{WAN_DHCP_VLAN} " + f"subnet {ISP_DHCP_NET} option default-router {ISP_DHCP_GW}", + f"set service dhcp-server shared-network-name WAN{WAN_DHCP_VLAN} " + f"subnet {ISP_DHCP_NET} option name-server 8.8.8.8", + f"set service dhcp-server shared-network-name WAN{WAN_DHCP_VLAN} " + f"subnet {ISP_DHCP_NET} range CUST start {ISP_DHCP_POOL[0]}", + f"set service dhcp-server shared-network-name WAN{WAN_DHCP_VLAN} " + f"subnet {ISP_DHCP_NET} range CUST stop {ISP_DHCP_POOL[1]}", + "", + ] + _isp_common(uplink_if, ISP_DHCP_NET, "sim ISP: NAT customers to the real internet") + + +def isp_pppoe(wan_if: str, uplink_if: str) -> list[str]: + """The Vodafone-equivalent ISP: terminates PPPoE, NATs to the real internet.""" + return [ + f"# --- sim ISP: PPPoE WAN on VLAN {WAN_PPPOE_VLAN} ---", + "set system host-name isp-pppoe", + f"set interfaces ethernet {wan_if} description " + f"'sim ISP - Vodafone-equivalent WAN on VLAN{WAN_PPPOE_VLAN} (PPPoE)'", + f"set interfaces ethernet {uplink_if} address dhcp", + f"set interfaces ethernet {uplink_if} description 'uplink to the real internet'", + f"set service pppoe-server access-concentrator {PPPOE_AC}", + f"set service pppoe-server interface {wan_if}", + f"set service pppoe-server gateway-address {ISP_PPPOE_GW}", + "set service pppoe-server authentication mode local", + f"set service pppoe-server authentication local-users username {PPPOE_USER} " + f"password {PPPOE_PASS}", + f"set service pppoe-server client-ip-pool CUST range " + f"{ISP_PPPOE_POOL[0]}-{ISP_PPPOE_POOL[1]}", + "set service pppoe-server default-pool CUST", + "set service pppoe-server name-server 8.8.8.8", + "", + ] + _isp_common(uplink_if, ISP_PPPOE_NET, "sim ISP: NAT PPPoE customers to the real internet") + + +def _isp_common(uplink_if: str, customer_net: str, desc: str) -> list[str]: + return [ + "set nat source rule 100 description " + f"'{desc}'", + f"set nat source rule 100 outbound-interface name {uplink_if}", + f"set nat source rule 100 source address {customer_net}", + "set nat source rule 100 translation address masquerade", + "", + # An ISP that drops return traffic is not simulating an ISP. The forward + # chain defaults to accept here on purpose -- these VMs model the + # internet, and the thing under test is the router's firewall, not this. + "set firewall ipv4 forward filter default-action accept", + "set firewall ipv4 forward filter rule 10 action accept", + "set firewall ipv4 forward filter rule 10 state established", + "set firewall ipv4 forward filter rule 10 state related", + "set firewall ipv4 forward filter rule 10 description conntrack-engage", + "", + ] + + +def build(role: str, drop_scaffold: bool, wan_if: str, uplink_if: str) -> list[str]: + if role == "primary": + # WAN lives on the primary only. Production has WAN on both routers; + # the sim does not, because two PPPoE clients sharing one credential + # against a single access concentrator is a different failure mode than + # anything production has. VRRP/conntrack failover is still exercised -- + # see README, "known gaps". + return [f"# labsim routing -- {role}", ""] + bgp(role) + wan(drop_scaffold) + if role == "secondary": + return [f"# labsim routing -- {role}", ""] + bgp(role) + if role == "isp-dhcp": + return isp_dhcp(wan_if, uplink_if) + return isp_pppoe(wan_if, uplink_if) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--role", required=True, + choices=("primary", "secondary", "isp-dhcp", "isp-pppoe")) + ap.add_argument("--drop-scaffold", action="store_true", + help="also remove the pre-ISP-VM libvirt-NAT uplink (primary only)") + # The ISP VMs' interface names depend on PCI enumeration order, which is not + # stable across a rebuild: isp-dhcp came up as eth0/eth1 and isp-pppoe as + # eth2/eth3 from identical XML. Check with `show interfaces` before applying + # rather than trusting these defaults. + ap.add_argument("--wan-if", default=None, help="ISP VM: customer-facing NIC") + ap.add_argument("--uplink-if", default=None, help="ISP VM: internet-facing NIC") + args = ap.parse_args() + + defaults = {"isp-dhcp": ("eth0", "eth1"), "isp-pppoe": ("eth2", "eth3")} + w, u = defaults.get(args.role, ("", "")) + w, u = args.wan_if or w, args.uplink_if or u + + if args.drop_scaffold and args.role != "primary": + print("--drop-scaffold only applies to --role primary", file=sys.stderr) + return 2 + + lines = build(args.role, args.drop_scaffold, w, u) + sys.stdout.write("\n".join(lines) + "\n") + n = len([l for l in lines if l.startswith(("set ", "delete "))]) + print(f"{args.role}: {n} commands", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main())