From ccdd1e7e49682e4173aa28ff504254f7b2dbbc92 Mon Sep 17 00:00:00 2001 From: Michal Date: Mon, 17 Aug 2026 00:30:23 +0100 Subject: [PATCH] fix(migration): both boxes carry WAN and NAT; the backup just holds it down "No NAT? How are we supposed to get internet?" -- a fair question that exposed a worse design than I had admitted. Internet did work, but only via vyos001: NAT and the entire WAN were gated behind --with-wan, so vyos002 would have held the LAN VIPs and routed between VLANs with no path to the outside at all. Failover would have preserved addressing and lost the internet. The fix rests on a checked fact rather than an assumption: VyOS WARNS but still commits when a NAT rule names an interface that does not exist ("Interface bond0.53 for source NAT rule 900 does not exist!"). Verified on a real VyOS before relying on it. So both boxes now get the identical WAN, NAT, port-forward and firewall config, and the backup's two WAN interfaces are simply set `disable`. The cloned WAN MAC is therefore never live on two boxes at once, while everything needed to route and masquerade is already in place. The two deltas are now byte-identical apart from VRRP priority, own/peer addresses, DHCP HA role, the conntrack /30 -- and the two disable lines. Taking over the internet path becomes deleting two lines rather than reconstructing NAT under pressure: delete interfaces bonding bond0 vif 53 disable delete interfaces pppoe pppoe0 disable Both boxes now: 21 NAT rules, 58 firewall rules, full PPPoE. Backup delta validated against a real VyOS config with the disable lines present -- commits clean. Runbook updated with the takeover procedure and the warning that it must only be done when vyos001 is genuinely down. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DMVzWZgiKW2wquf5z8S1yH --- migration/CUTOVER.md | 18 +++++++++++++++--- migration/vyos-mode-delta.py | 33 ++++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/migration/CUTOVER.md b/migration/CUTOVER.md index 6dfd26f..8c8aa8c 100644 --- a/migration/CUTOVER.md +++ b/migration/CUTOVER.md @@ -144,9 +144,21 @@ reference, and `translation port` rejects a port list. | WAN1 | Vodafone | **51** | PPPoE, ~900/700 Mbit | failover, distance 10 | VyOS clones the USG's WAN2 MAC (`f0:9f:c2:12:9b:4f`) on `bond0.53`, which is how -it keeps the existing public lease rather than asking for a new one. **Only -vyos001 carries the WAN** — the MAC must be unique, so vyos002 routes the LAN -and holds the VIPs but has no internet path until the WAN is moved deliberately. +it keeps the existing public lease rather than asking for a new one. + +**Both boxes carry the identical WAN and NAT config.** vyos002's WAN interfaces +are simply held administratively down, so the cloned MAC is never live on two +boxes at once. To move the internet path to vyos002: + +``` +configure +delete interfaces bonding bond0 vif 53 disable +delete interfaces pppoe pppoe0 disable +commit; save +``` + +Two lines. Do it only when vyos001 is genuinely down or disconnected — two boxes +holding that MAC at once is exactly what the disable prevents. PPPoE is no longer an unknown: it was proven on the USG before cutover (`pppoe0` came up with `90.241.226.213`, MTU 1492). What remains untested is diff --git a/migration/vyos-mode-delta.py b/migration/vyos-mode-delta.py index 66d06dc..fc376bf 100755 --- a/migration/vyos-mode-delta.py +++ b/migration/vyos-mode-delta.py @@ -156,7 +156,7 @@ def build_delta(inv: dict, priority: int, wan_user: str, with_wan: bool, "set firewall ipv4 forward filter rule 10 description 'stateful tracking'", ] - if with_wan: + if True: # WAN config on BOTH boxes; see the disable block below out += [ "# --- WAN -----------------------------------------------", "# Both vifs must be created before anything references them.", @@ -196,20 +196,31 @@ def build_delta(inv: dict, priority: int, wan_user: str, with_wan: bool, "set nat source rule 110 translation address masquerade", "set nat source rule 110 description 'LAN out via Vodafone (failover)'", ] - else: + + if not with_wan: + # The backup carries the identical WAN and NAT config but with the + # interfaces administratively DOWN. The cloned MAC is therefore never + # live on two boxes at once, while everything needed to route and + # masquerade is already present -- taking over is enabling two + # interfaces, not rebuilding a config under pressure. + # + # NAT rules naming a down interface are harmless: VyOS warns at commit + # ("Interface ... does not exist!") and commits anyway, verified. out += [ - "# --- WAN -----------------------------------------------", - "# This box carries NO WAN. Only one of the pair may hold the", - "# cloned WAN MAC, so the backup routes the LAN and holds the VIPs", - "# if the master dies, but has no internet path until the WAN is", - "# moved to it deliberately.", "", + "# --- WAN held DOWN on this box -----------------------------", + "# Enable these two to take over the internet path:", + f"# set interfaces bonding bond0 vif {WAN_DHCP_VIF.split('.')[1]} disable <- delete this", + f"# set interfaces pppoe {WAN_PPPOE_IF} disable <- and this", + f"set interfaces bonding bond0 vif {WAN_DHCP_VIF.split('.')[1]} disable", + f"set interfaces pppoe {WAN_PPPOE_IF} disable", ] - if with_wan: - # NAT and the WAN firewall only mean anything on the box that has a - # WAN. On the other one they would reference interfaces that do not - # exist and fail the commit. + if True: + # Port forwards and the WAN firewall go on BOTH boxes. They name + # interfaces that are present-but-disabled on the backup, which VyOS + # accepts (it warns and commits). Putting them here means a failover is + # enabling an interface, not reconstructing NAT under pressure. # Port forwards, straight from UniFi. for i, p in enumerate(inv["port_forwards"]): if not p.get("enabled"):