diff --git a/migration/CUTOVER.md b/migration/CUTOVER.md index d666063..6dfd26f 100644 --- a/migration/CUTOVER.md +++ b/migration/CUTOVER.md @@ -136,9 +136,23 @@ reference, and `translation port` rejects a port list. ## What will probably go wrong first -**PPPoE.** It is the one thing that could not be tested in advance — the line -permits a single session and the USG held it until you unplugged it. If the -WAN check fails: +**The WAN.** There are two, and they behave differently: + +| | line | VLAN | transport | notes | +|---|---|---|---|---| +| WAN2 | 10 gig ISP | **53** | DHCP, public `87.192.101.48/21` | primary, distance 1 | +| WAN1 | Vodafone | **51** | PPPoE, ~900/700 Mbit | failover, distance 10 | + +VyOS clones the USG's WAN2 MAC (`f0:9f:c2:12:9b:4f`) on `bond0.53`, which is how +it keeps the existing public lease rather than asking for a new one. **Only +vyos001 carries the WAN** — the MAC must be unique, so vyos002 routes the LAN +and holds the VIPs but has no internet path until the WAN is moved deliberately. + +PPPoE is no longer an unknown: it was proven on the USG before cutover +(`pppoe0` came up with `90.241.226.213`, MTU 1492). What remains untested is +VyOS dialling it, and whether the ISP hands the same lease to the cloned MAC. + +If the WAN check fails: ``` show interfaces pppoe pppoe0 diff --git a/migration/vyos-mode-delta.py b/migration/vyos-mode-delta.py index befccdb..983c9dc 100755 --- a/migration/vyos-mode-delta.py +++ b/migration/vyos-mode-delta.py @@ -34,8 +34,28 @@ _spec = importlib.util.spec_from_file_location( unifi_to_vyos = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(unifi_to_vyos) -WAN_VIF = "bond0.51" # WAN arrives trunked as vlan-only network 51 -WAN_IF = "pppoe0" +# Two WANs, established by reading the live USG rather than the UniFi fields +# (which report wan_type=dhcp for both and are simply wrong): +# +# WAN1 Vodafone, PPPoE on the USG's eth0, ~900/700 Mbit. Verified working: +# pppoe0 came up with 90.241.226.213 peer 84.65.128.1, MTU 1492. +# WAN2 10 gig ISP, plain DHCP on the USG's eth2, public 87.192.101.48/21 +# gw 87.192.96.1. This is what carries traffic today. +# +# Both reach the USG as untagged access ports but are carried across the switch +# fabric as vlan-only networks 51 and 53, so VyOS picks them up as bond vifs. +WAN_PPPOE_VIF = "bond0.51" # Vodafone +WAN_PPPOE_IF = "pppoe0" +WAN_DHCP_VIF = "bond0.53" # 10 gig ISP + +# The DHCP lease is bound to the MAC, so cloning the USG's WAN2 MAC is how VyOS +# keeps 87.192.101.48 instead of negotiating a fresh lease -- or getting none, +# if the ISP hands out one per line. Only ONE box may carry this at a time. +WAN_DHCP_MAC = "f0:9f:c2:12:9b:4f" + +# Route distances: the 10 gig line wins, Vodafone is failover. +DIST_DHCP, DIST_PPPOE = 1, 10 + PLACEHOLDER = "@@WAN_PASSWORD@@" @@ -45,7 +65,7 @@ def vrrp_group(vlan: int) -> str: return "native" if vlan == 1 else f"vlan{vlan}" -def build_delta(inv: dict, priority: int, wan_user: str) -> list[str]: +def build_delta(inv: dict, priority: int, wan_user: str, with_wan: bool) -> list[str]: out: list[str] = [] nets = [n for n in inv["networks"] if n["dhcp_enabled"] and n["subnet"]] nets.sort(key=unifi_to_vyos.vlan_of) @@ -81,139 +101,170 @@ def build_delta(inv: dict, priority: int, wan_user: str) -> list[str]: out += [ "", - "# --- WAN ---------------------------------------------------", - "# Vodafone line, PPPoE over the trunked WAN VLAN. This is the one", - "# part that cannot be tested before cutover: the line permits a", - "# single session and the USG holds it until it is unplugged.", - "#", - "# The WAN vif must be created first. Neither firewall has vif 51 today", - "# (only 2, 3, 9, 10 and 200 exist), and pppoe source-interface refers to", - "# an interface that has to already be configured -- without this the", - "# commit fails and takes the whole switch with it. No address on it:", - "# PPPoE rides the VLAN, it does not need L3 of its own.", - f"set interfaces bonding bond0 vif {WAN_VIF.split('.')[1]} description 'WAN (Vodafone, PPPoE)'", - "", - f"set interfaces pppoe {WAN_IF} source-interface {WAN_VIF}", - f"set interfaces pppoe {WAN_IF} authentication username '{wan_user}'", - f"set interfaces pppoe {WAN_IF} authentication password '{PLACEHOLDER}'", - f"set interfaces pppoe {WAN_IF} mtu 1492", - # The peer's resolvers would otherwise overwrite /etc/resolv.conf and - # cost the box its ability to resolve internal ad.itaz.eu names. - f"set interfaces pppoe {WAN_IF} no-peer-dns", - "", - "# The static default route exists only for unifi mode, where the USG", - "# is the next hop. PPPoE supplies the default here; leaving both at", - "# the same distance would be ambiguous.", - "delete protocols static route 0.0.0.0/0", - "", - "# --- NAT ---------------------------------------------------", - f"set nat source rule 100 outbound-interface name {WAN_IF}", - "set nat source rule 100 translation address masquerade", - "set nat source rule 100 description 'LAN out to the internet'", ] - # Port forwards, straight from UniFi. - for i, p in enumerate(inv["port_forwards"]): - if not p.get("enabled"): - continue - rule = 100 + i * 10 - proto = p["proto"] # tcp | udp | tcp_udp -- all valid VyOS values + if with_wan: out += [ + "# --- WAN -----------------------------------------------", + "# Both vifs must be created before anything references them.", + "# Neither firewall has vif 51 or 53 today (only 2, 3, 9, 10, 200),", + "# and pppoe source-interface points at an interface that must", + "# already exist -- without this the commit fails and, since the", + "# delta commits as one unit, takes the whole switch with it.", + f"set interfaces bonding bond0 vif {WAN_PPPOE_VIF.split('.')[1]} description 'WAN1 Vodafone (PPPoE)'", + f"set interfaces bonding bond0 vif {WAN_DHCP_VIF.split('.')[1]} description 'WAN2 10gig ISP (DHCP)'", "", - f"set nat destination rule {rule} description '{p['name']}'", - f"set nat destination rule {rule} inbound-interface name {WAN_IF}", - f"set nat destination rule {rule} protocol {proto}", - f"set nat destination rule {rule} destination port '{p['dst_port']}'", - f"set nat destination rule {rule} translation address {p['fwd']}", + "# WAN2, the 10 gig line -- primary. The cloned MAC is what keeps", + "# the existing public lease (87.192.101.48) instead of asking for", + "# a new one. Only the box carrying the WAN may set this.", + f"set interfaces bonding bond0 vif {WAN_DHCP_VIF.split('.')[1]} mac '{WAN_DHCP_MAC}'", + f"set interfaces bonding bond0 vif {WAN_DHCP_VIF.split('.')[1]} address dhcp", + f"set interfaces bonding bond0 vif {WAN_DHCP_VIF.split('.')[1]} dhcp-options default-route-distance {DIST_DHCP}", + "", + "# WAN1, Vodafone -- failover at a higher distance. Verified working", + "# on the USG: pppoe0 came up with a public address, MTU 1492.", + f"set interfaces pppoe {WAN_PPPOE_IF} source-interface {WAN_PPPOE_VIF}", + f"set interfaces pppoe {WAN_PPPOE_IF} authentication username '{wan_user}'", + f"set interfaces pppoe {WAN_PPPOE_IF} authentication password '{PLACEHOLDER}'", + f"set interfaces pppoe {WAN_PPPOE_IF} mtu 1492", + f"set interfaces pppoe {WAN_PPPOE_IF} default-route-distance {DIST_PPPOE}", + # The peer's resolvers would otherwise overwrite resolv.conf. + f"set interfaces pppoe {WAN_PPPOE_IF} no-peer-dns", + "", + "# The static default route exists only for unifi mode, where the", + "# USG is the next hop. Both WANs supply one here.", + "delete protocols static route 0.0.0.0/0", + "", + "# --- NAT -----------------------------------------------", + f"set nat source rule 100 outbound-interface name {WAN_DHCP_VIF}", + "set nat source rule 100 translation address masquerade", + "set nat source rule 100 description 'LAN out via the 10gig line'", + f"set nat source rule 110 outbound-interface name {WAN_PPPOE_IF}", + "set nat source rule 110 translation address masquerade", + "set nat source rule 110 description 'LAN out via Vodafone (failover)'", ] - # `destination port` accepts a comma list but `translation port` does - # NOT -- "16881,6881 is not a valid service name" -- because mapping a - # list onto a list is ambiguous. Every forward here maps a port to - # itself, and omitting translation port makes VyOS preserve the - # original, which is exactly right. Only emit it when it genuinely - # differs, and refuse rather than guess when a differing list appears. - if p["fwd_port"] != p["dst_port"]: - if "," in str(p["fwd_port"]) or "," in str(p["dst_port"]): - raise SystemExit( - f"port forward '{p['name']}' remaps a LIST of ports " - f"({p['dst_port']} -> {p['fwd_port']}). VyOS cannot express " - f"that in one rule; split it into one rule per port by hand.") - out.append(f"set nat destination rule {rule} translation port '{p['fwd_port']}'") - - out += [ - "", - "# --- firewall ----------------------------------------------", - "# VyOS defaults to accepting everything. The USG has an implicit", - "# WAN drop, so migrating the port forwards alone would leave the", - "# router's own services and the whole LAN reachable from the WAN.", - "#", - "# Scoped to the WAN interface rather than a global default-action", - "# drop: that way a mistake here cannot lock anyone out over the LAN,", - "# which is the only path back in during a cutover.", - "", - "# Traffic TO the router.", - "set firewall ipv4 input filter default-action accept", - "set firewall ipv4 input filter rule 100 action accept", - "set firewall ipv4 input filter rule 100 state established", - "set firewall ipv4 input filter rule 100 state related", - "set firewall ipv4 input filter rule 100 description 'established/related'", - ] - # The two WAN_LOCAL accepts carried over from UniFi. - out += [ - "", - "set firewall ipv4 input filter rule 110 action accept", - "set firewall ipv4 input filter rule 110 protocol esp", - f"set firewall ipv4 input filter rule 110 inbound-interface name {WAN_IF}", - "set firewall ipv4 input filter rule 110 description 'VPN accept ESP (from UniFi WAN_LOCAL)'", - "", - "set firewall ipv4 input filter rule 120 action accept", - "set firewall ipv4 input filter rule 120 protocol udp", - "set firewall ipv4 input filter rule 120 destination port '500,4500'", - f"set firewall ipv4 input filter rule 120 inbound-interface name {WAN_IF}", - "set firewall ipv4 input filter rule 120 description 'VPN accept UDP500/4500 (from UniFi WAN_LOCAL)'", - "", - "set firewall ipv4 input filter rule 130 action accept", - "set firewall ipv4 input filter rule 130 protocol icmp", - f"set firewall ipv4 input filter rule 130 inbound-interface name {WAN_IF}", - "set firewall ipv4 input filter rule 130 description 'ICMP to the router (path MTU discovery)'", - "", - "# Everything else arriving from the WAN is dropped. LAN is untouched.", - "set firewall ipv4 input filter rule 900 action drop", - f"set firewall ipv4 input filter rule 900 inbound-interface name {WAN_IF}", - "set firewall ipv4 input filter rule 900 description 'drop all other WAN-to-router'", - "", - "# Traffic THROUGH the router.", - "set firewall ipv4 forward filter default-action accept", - "set firewall ipv4 forward filter rule 100 action accept", - "set firewall ipv4 forward filter rule 100 state established", - "set firewall ipv4 forward filter rule 100 state related", - "set firewall ipv4 forward filter rule 100 description 'established/related'", - ] - - # Destination NAT happens before the forward filter, so these rules must - # match the translated destination, not the WAN address. - for i, p in enumerate(inv["port_forwards"]): - if not p.get("enabled"): - continue - rule = 200 + i * 10 + else: out += [ + "# --- WAN -----------------------------------------------", + "# This box carries NO WAN. Only one of the pair may hold the", + "# cloned WAN MAC, so the backup routes the LAN and holds the VIPs", + "# if the master dies, but has no internet path until the WAN is", + "# moved to it deliberately.", "", - f"set firewall ipv4 forward filter rule {rule} action accept", - f"set firewall ipv4 forward filter rule {rule} inbound-interface name {WAN_IF}", - f"set firewall ipv4 forward filter rule {rule} protocol {p['proto']}", - f"set firewall ipv4 forward filter rule {rule} destination address {p['fwd']}", - f"set firewall ipv4 forward filter rule {rule} destination port '{p['fwd_port']}'", - f"set firewall ipv4 forward filter rule {rule} description 'port forward: {p['name']}'", ] - out += [ - "", - "# New inbound connections from the WAN that are not a port forward.", - "set firewall ipv4 forward filter rule 900 action drop", - f"set firewall ipv4 forward filter rule 900 inbound-interface name {WAN_IF}", - "set firewall ipv4 forward filter rule 900 description 'drop unsolicited WAN-to-LAN'", - "", - ] + if with_wan: + # NAT and the WAN firewall only mean anything on the box that has a + # WAN. On the other one they would reference interfaces that do not + # exist and fail the commit. + # Port forwards, straight from UniFi. + for i, p in enumerate(inv["port_forwards"]): + if not p.get("enabled"): + continue + rule = 100 + i * 10 + proto = p["proto"] # tcp | udp | tcp_udp -- all valid VyOS values + out += [ + "", + f"set nat destination rule {rule} description '{p['name']}'", + f"set nat destination rule {rule} inbound-interface name {WAN_DHCP_VIF}", + f"set nat destination rule {rule} protocol {proto}", + f"set nat destination rule {rule} destination port '{p['dst_port']}'", + f"set nat destination rule {rule} translation address {p['fwd']}", + ] + # `destination port` accepts a comma list but `translation port` does + # NOT -- "16881,6881 is not a valid service name" -- because mapping a + # list onto a list is ambiguous. Every forward here maps a port to + # itself, and omitting translation port makes VyOS preserve the + # original, which is exactly right. Only emit it when it genuinely + # differs, and refuse rather than guess when a differing list appears. + if p["fwd_port"] != p["dst_port"]: + if "," in str(p["fwd_port"]) or "," in str(p["dst_port"]): + raise SystemExit( + f"port forward '{p['name']}' remaps a LIST of ports " + f"({p['dst_port']} -> {p['fwd_port']}). VyOS cannot express " + f"that in one rule; split it into one rule per port by hand.") + out.append(f"set nat destination rule {rule} translation port '{p['fwd_port']}'") + + out += [ + "", + "# --- firewall ----------------------------------------------", + "# VyOS defaults to accepting everything. The USG has an implicit", + "# WAN drop, so migrating the port forwards alone would leave the", + "# router's own services and the whole LAN reachable from the WAN.", + "#", + "# Scoped to the WAN interface rather than a global default-action", + "# drop: that way a mistake here cannot lock anyone out over the LAN,", + "# which is the only path back in during a cutover.", + "", + "# Traffic TO the router.", + "set firewall ipv4 input filter default-action accept", + "set firewall ipv4 input filter rule 100 action accept", + "set firewall ipv4 input filter rule 100 state established", + "set firewall ipv4 input filter rule 100 state related", + "set firewall ipv4 input filter rule 100 description 'established/related'", + ] + # The two WAN_LOCAL accepts carried over from UniFi. + out += [ + "", + "set firewall ipv4 input filter rule 110 action accept", + "set firewall ipv4 input filter rule 110 protocol esp", + f"set firewall ipv4 input filter rule 110 inbound-interface name {WAN_DHCP_VIF}", + "set firewall ipv4 input filter rule 110 description 'VPN accept ESP (from UniFi WAN_LOCAL)'", + "", + "set firewall ipv4 input filter rule 120 action accept", + "set firewall ipv4 input filter rule 120 protocol udp", + "set firewall ipv4 input filter rule 120 destination port '500,4500'", + f"set firewall ipv4 input filter rule 120 inbound-interface name {WAN_DHCP_VIF}", + "set firewall ipv4 input filter rule 120 description 'VPN accept UDP500/4500 (from UniFi WAN_LOCAL)'", + "", + "set firewall ipv4 input filter rule 130 action accept", + "set firewall ipv4 input filter rule 130 protocol icmp", + f"set firewall ipv4 input filter rule 130 inbound-interface name {WAN_DHCP_VIF}", + "set firewall ipv4 input filter rule 130 description 'ICMP to the router (path MTU discovery)'", + "", + "# Everything else arriving from the WAN is dropped. LAN is untouched.", + "set firewall ipv4 input filter rule 900 action drop", + f"set firewall ipv4 input filter rule 900 inbound-interface name {WAN_DHCP_VIF}", + f"set firewall ipv4 input filter rule 910 action drop", + f"set firewall ipv4 input filter rule 910 inbound-interface name {WAN_PPPOE_IF}", + "set firewall ipv4 input filter rule 910 description 'drop all other WAN-to-router (Vodafone)'", + "set firewall ipv4 input filter rule 900 description 'drop all other WAN-to-router'", + "", + "# Traffic THROUGH the router.", + "set firewall ipv4 forward filter default-action accept", + "set firewall ipv4 forward filter rule 100 action accept", + "set firewall ipv4 forward filter rule 100 state established", + "set firewall ipv4 forward filter rule 100 state related", + "set firewall ipv4 forward filter rule 100 description 'established/related'", + ] + + # Destination NAT happens before the forward filter, so these rules must + # match the translated destination, not the WAN address. + for i, p in enumerate(inv["port_forwards"]): + if not p.get("enabled"): + continue + rule = 200 + i * 10 + out += [ + "", + f"set firewall ipv4 forward filter rule {rule} action accept", + f"set firewall ipv4 forward filter rule {rule} inbound-interface name {WAN_DHCP_VIF}", + f"set firewall ipv4 forward filter rule {rule} protocol {p['proto']}", + f"set firewall ipv4 forward filter rule {rule} destination address {p['fwd']}", + f"set firewall ipv4 forward filter rule {rule} destination port '{p['fwd_port']}'", + f"set firewall ipv4 forward filter rule {rule} description 'port forward: {p['name']}'", + ] + + out += [ + "", + "# New inbound connections from the WAN that are not a port forward.", + "set firewall ipv4 forward filter rule 900 action drop", + f"set firewall ipv4 forward filter rule 900 inbound-interface name {WAN_DHCP_VIF}", + f"set firewall ipv4 forward filter rule 910 action drop", + f"set firewall ipv4 forward filter rule 910 inbound-interface name {WAN_PPPOE_IF}", + "set firewall ipv4 forward filter rule 910 description 'drop unsolicited WAN-to-LAN (Vodafone)'", + "set firewall ipv4 forward filter rule 900 description 'drop unsolicited WAN-to-LAN'", + "", + ] # DHCP + DNS, from the same generator labsim proved. dhcp_lines, stats = unifi_to_vyos.build(inv, "prod") @@ -232,6 +283,8 @@ def main() -> int: help="VRRP priority: 200 for the master, 100 for the backup") ap.add_argument("--inventory", default=os.path.join(HERE, "export", "inventory.json")) ap.add_argument("--raw-networkconf", default=os.path.join(HERE, "export", "rest_networkconf.json")) + ap.add_argument("--with-wan", action="store_true", + help="configure the WAN on this box. Only ONE of the pair may have\n it, because the cloned WAN MAC must be unique.") ap.add_argument("-o", "--out") ap.add_argument("--emit-secrets", metavar="PATH", help="write the PPPoE credential to PATH with mode 0600 and exit") @@ -259,11 +312,12 @@ def main() -> int: file=sys.stderr) return 0 - lines = build_delta(inv, args.priority, wan["wan_username"]) + lines = build_delta(inv, args.priority, wan["wan_username"], args.with_wan) text = "\n".join(lines) + "\n" - if PLACEHOLDER not in text: - print("BUG: password placeholder missing from the delta", file=sys.stderr) + # Only a WAN-carrying delta has a credential to placeholder-substitute. + if args.with_wan and PLACEHOLDER not in text: + print("BUG: password placeholder missing from a WAN delta", file=sys.stderr) return 1 if wan.get("x_wan_password") and wan["x_wan_password"] in text: print("BUG: the WAN password leaked into the delta", file=sys.stderr)