lab/labsim/labsim-he-endpoint.sh

#!/bin/bash
# Build a fake Hurricane Electric 6in4 endpoint inside labsim.
#
# WHY THIS EXISTS
# The production HE tunnel was never rehearsed. The override that introduced it
# says so in its own reason text -- "the sim has no public IPv4 and no HE
# endpoint, so there is nothing to tunnel to" -- and that gap is why IPv6 was
# the one half of the WAN story with no matrix behind it. Then the WAN became
# HA and IPv6 did not follow, which nobody caught, because nothing tests it.
#
# THE PROBLEM THIS HAD TO SOLVE
# The sim's two WANs are isolated islands. Verified:
#
#     203.0.113.1 from 203.0.113.107 : OK          <- 10 gig analogue
#     203.0.113.1 from 198.51.100.137 : unreachable
#     198.51.100.1 from 198.51.100.137 : OK        <- PPPoE analogue
#     198.51.100.1 from 203.0.113.107 : unreachable
#
# A 6in4 tunnel has ONE remote address, and production never changes it -- so an
# endpoint reachable over only one WAN could not rehearse the case that matters
# most: the WITHIN-box fall back from the 10 gig to PPPoE, where he-tunnel-follow
# re-points the tunnel and calls the HE API. That is exactly where the 2026-09-06
# near-miss lived.
#
# So the sim needs a minimal "internet": both ISP boxes already sit on the
# libvirt default network (192.168.122.0/24) and both forward, so that becomes
# the backbone, and HE lives on a single address behind it, reachable over
# either WAN. No new VMs, no new networks.
#
#     192.0.2.10  "HE"  -- on isp-dhcp, reached from the PPPoE island via
#                          192.168.122.136, and directly from the 10 gig island
#
# EVERYTHING HERE IS KERNEL-LEVEL, not VyOS config. The ISP boxes are scaffold,
# not the thing under test: `ip` commands leave no config to drift, no commit to
# fail, and a reboot cleans up. The ROUTER side is deliberately the opposite --
# it goes through real VyOS config, because "will VyOS commit a tunnel whose
# source-address does not exist on this box?" is one of the questions.
#
#   ./labsim-he-endpoint.sh up        build it
#   ./labsim-he-endpoint.sh down      tear it down
#   ./labsim-he-endpoint.sh status    what is live
#   ./labsim-he-endpoint.sh calls     how many times the HE API was called
#   ./labsim-he-endpoint.sh point IP  point the endpoint by hand (sim bookkeeping)
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
DHCP_ISP="${DHCP_ISP:-192.168.122.136}"   # owns the 10 gig segment, hosts "HE"
PPPOE_ISP="${PPPOE_ISP:-192.168.122.63}"  # owns the PPPoE segment
PW="${VYOS_PW:-vyos}"

# TEST-NET-1 for the endpoint, and the documentation prefix for v6. Production
# uses 2001:470:187e::/48 from HE; the sim mirrors its SHAPE
# (2001:db8:187e:<vlan>::/64) so the scheme is exercised, not just the tunnel.
HE_ADDR="${HE_ADDR:-192.0.2.10}"
HE_LINK6="${HE_LINK6:-2001:db8:1f1c:f6::1}"   # HE side of the tunnel /64
RT_LINK6="${RT_LINK6:-2001:db8:1f1c:f6::2}"   # router side
SITE6="${SITE6:-2001:db8:187e::/48}"          # routed to the router side
TENGIG_NET="${TENGIG_NET:-203.0.113.0/24}"
# Any valid address that will never be a router WAN -- see the tunnel creation
# below for why this cannot be 0.0.0.0.
PLACEHOLDER_REMOTE="${PLACEHOLDER_REMOTE:-203.0.113.1}"
PPPOE_NET="${PPPOE_NET:-198.51.100.0/24}"

SSH=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
     -o LogLevel=ERROR -o ConnectTimeout=6 -o PreferredAuthentications=password)
dhcp_isp()  { timeout 40 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$DHCP_ISP" "$@" 2>/dev/null; }
pppoe_isp() { timeout 40 sshpass -p "$PW" ssh "${SSH[@]}" "vyos@$PPPOE_ISP" "$@" 2>/dev/null; }

log()  { printf '\033[0;36m[he-sim]\033[0m %s\n' "$*"; }
die()  { printf '\033[0;31m[he-sim]\033[0m %s\n' "$*" >&2; exit 1; }

# --- the stub tunnelbroker API ---------------------------------------------
# HE's real endpoint is a dyndns-style updater that re-points the tunnel's remote
# address. This is that, in 40 lines, so the sim can exercise the HE-SIDE half of
# a failover -- the half production can never safely test.
#
# It logs every call to /run/he-sim-api.log, which is what lets the matrix assert
# the invariant that matters: a ROUTER-level failover must call this ZERO times,
# because the 10 gig address follows the cloned MAC to the other box unchanged.
API_PY='
import http.server, subprocess, urllib.parse, datetime, sys, re

TUN  = "he-sim"
LOCAL    = sys.argv[1] if len(sys.argv) > 1 else "192.0.2.10"
V6_LOCAL = sys.argv[2] if len(sys.argv) > 2 else "2001:db8:1f1c:f6::1/64"
V6_PEER  = sys.argv[3] if len(sys.argv) > 3 else "2001:db8:1f1c:f6::2"
SITE6    = sys.argv[4] if len(sys.argv) > 4 else "2001:db8:187e::/48"
LOG  = "/run/he-sim-api.log"

def note(msg):
    with open(LOG, "a") as f:
        f.write("%s %s\n" % (datetime.datetime.now().isoformat(timespec="seconds"), msg))

def current_remote():
    out = subprocess.run(["ip", "tunnel", "show", TUN], capture_output=True, text=True).stdout
    m = re.search(r"remote ([0-9.]+)", out)
    return m.group(1) if m else None

class H(http.server.BaseHTTPRequestHandler):
    def reply(self, body):
        b = body.encode()
        self.send_response(200)
        self.send_header("Content-Type", "text/plain")
        self.send_header("Content-Length", str(len(b)))
        self.end_headers()
        self.wfile.write(b)

    def handle_update(self, qs):
        q = urllib.parse.parse_qs(qs)
        ip = (q.get("myip") or [""])[0]
        if not ip:
            note("REFUSED no myip"); return self.reply("nohost")
        cur = current_remote()
        if cur == ip:
            note("nochg %s" % ip); return self.reply("nochg %s" % ip)
        rc = subprocess.run(["ip", "tunnel", "change", TUN, "mode", "sit",
                             "local", LOCAL, "remote", ip],
                            capture_output=True, text=True)
        if rc.returncode != 0:
            # Recreate rather than report a success we did not achieve. This is
            # the path that a multipoint tunnel takes; keeping it means a stub
            # that cannot silently no-op.
            note("change failed (%s) -- recreating" % rc.stderr.strip())
            subprocess.run(["ip", "tunnel", "del", TUN], check=False)
            subprocess.run(["ip", "tunnel", "add", TUN, "mode", "sit",
                            "local", LOCAL, "remote", ip, "ttl", "64"], check=False)
            subprocess.run(["ip", "link", "set", TUN, "up", "mtu", "1480"], check=False)
            subprocess.run(["ip", "-6", "addr", "replace", V6_LOCAL, "dev", TUN], check=False)
            subprocess.run(["ip", "-6", "route", "replace", SITE6, "via", V6_PEER,
                            "dev", TUN], check=False)
        # Verify rather than trust: read the remote back.
        got = current_remote()
        if got != ip:
            note("FAILED to point %s at %s (reads %s)" % (TUN, ip, got))
            return self.reply("dnserr")
        note("good %s (was %s)" % (ip, cur))
        self.reply("good %s" % ip)

    def do_GET(self):
        u = urllib.parse.urlparse(self.path)
        if u.path == "/nic/update": self.handle_update(u.query)
        else: self.reply("badauth")

    def do_POST(self):
        n = int(self.headers.get("Content-Length") or 0)
        self.handle_update(self.rfile.read(n).decode())

    def log_message(self, *a): pass

http.server.HTTPServer(("0.0.0.0", 80), H).serve_forever()
'

up() {
    log "backbone: teaching each ISP box how to reach the other island"
    # isp-dhcp owns HE and must be able to answer a router that arrived over
    # PPPoE, so it needs a route back to that island via the backbone.
    dhcp_isp "sudo ip route replace $PPPOE_NET via $PPPOE_ISP" \
        || die "could not add the PPPoE-island route on isp-dhcp"
    # isp-pppoe must forward its clients' traffic for HE across the backbone.
    pppoe_isp "sudo ip route replace $HE_ADDR/32 via $DHCP_ISP" \
        || die "could not add the HE route on isp-pppoe"

    log "HE endpoint: $HE_ADDR on isp-dhcp"
    # A dummy interface, not a loopback alias: `ip tunnel` wants a real local
    # address and a dummy is the honest way to have one that is not tied to
    # either WAN segment -- which is the point, HE is neither.
    dhcp_isp "sudo modprobe dummy 2>/dev/null;
              sudo ip link add he-lo type dummy 2>/dev/null;
              sudo ip link set he-lo up;
              sudo ip addr replace $HE_ADDR/32 dev he-lo"

    log "6in4 tunnel he-sim: local $HE_ADDR, remote set by the API on demand"
    # A PLACEHOLDER remote, not 0.0.0.0. A sit tunnel created with `remote any`
    # is multipoint (6rd-shaped), and `ip tunnel change` then refuses to convert
    # it to point-to-point -- "add tunnel he-sim failed: Invalid argument". The
    # API's update silently did nothing, so the stub logged "good", production's
    # he-tunnel-follow logged success, and the tunnel still pointed nowhere.
    # Created point-to-point from the start, `change` works.
    dhcp_isp "sudo ip tunnel del he-sim 2>/dev/null;
              sudo ip tunnel add he-sim mode sit local $HE_ADDR remote $PLACEHOLDER_REMOTE ttl 64;
              sudo ip link set he-sim up mtu 1480;
              sudo ip -6 addr replace $HE_LINK6/64 dev he-sim;
              sudo ip -6 route replace $SITE6 via $RT_LINK6 dev he-sim;
              sudo sysctl -qw net.ipv6.conf.all.forwarding=1"

    log "stub tunnelbroker API on $HE_ADDR:80"
    printf '%s' "$API_PY" | dhcp_isp "cat > /tmp/he-sim-api.py"
    # Launch from a script FILE, not an inline ssh command. The remote login
    # shell is vbash, and a multi-line inlined `sudo setsid nohup ... &` through
    # it silently ran nothing at all: no process, no /run/he-sim-api.out, and a
    # `pgrep -f he-sim-api.py` status check that reported "running" because the
    # unbracketed pattern matched its OWN ssh command line. Two self-inflicted
    # illusions stacked on each other.
    #
    # This repo already learned this once -- see isp_session_control() in
    # labsim-pppoe-ha-test.sh, where driving vbash inline made every iteration
    # of the T4 matrix test the wrong policy while printing the right one.
    printf '%s\n' \
        '#!/bin/sh' \
        '# started detached so it outlives the ssh session that launched it' \
        'pkill -f "he-sim-api[.]py" 2>/dev/null' \
        'rm -f /run/he-sim-api.log /run/he-sim-api.out' \
        "exec setsid python3 /tmp/he-sim-api.py $HE_ADDR '$HE_LINK6/64' $RT_LINK6 $SITE6 >/run/he-sim-api.out 2>&1 </dev/null &" \
        | dhcp_isp "cat > /tmp/he-sim-start.sh"
    dhcp_isp "chmod +x /tmp/he-sim-start.sh && sudo /tmp/he-sim-start.sh" >/dev/null
    # Poll for the bind rather than sleeping a guessed interval.
    local i probe=""
    for i in $(seq 1 10); do
        sleep 1
        probe="$(dhcp_isp "curl -sS --max-time 3 'http://$HE_ADDR/nic/update' 2>&1")"
        [ "$probe" = nohost ] && break
    done
    case "$probe" in
        nohost) log "API answering (returned 'nohost' for a call with no myip -- correct)" ;;
        *)      die "stub API not answering on $HE_ADDR:80 (got: ${probe:-<nothing>})" ;;
    esac

    log "up. Router side is NOT configured by this script -- that is real VyOS"
    log "config and belongs to the matrix; see labsim-ipv6-ha-test.sh --setup."
}

down() {
    log "tearing down"
    dhcp_isp "sudo pkill -f 'he-sim-api[.]py' 2>/dev/null;
              sudo ip tunnel del he-sim 2>/dev/null;
              sudo ip link del he-lo 2>/dev/null;
              sudo ip route del $PPPOE_NET via $PPPOE_ISP 2>/dev/null" >/dev/null
    pppoe_isp "sudo ip route del $HE_ADDR/32 via $DHCP_ISP 2>/dev/null" >/dev/null
    log "down"
}

status() {
    printf '  HE address : %s\n' "$(dhcp_isp "ip -4 -br addr show he-lo 2>/dev/null | awk '{print \$3}'" || echo '<absent>')"
    printf '  he-sim     : %s\n' "$(dhcp_isp "ip tunnel show he-sim 2>/dev/null" || echo '<absent>')"
    printf '  he-sim v6  : %s\n' "$(dhcp_isp "ip -6 -br addr show he-sim 2>/dev/null | awk '{print \$3}'" || echo '-')"
    # Bracketed so the pattern cannot match the ssh command line carrying it --
    # unbracketed, this reported "running" while nothing was listening at all.
    printf '  API        : %s\n' "$(dhcp_isp "pgrep -f 'he-sim-api[.]py' >/dev/null && echo running || echo stopped")"
    printf '  API bound  : %s\n' "$(dhcp_isp "curl -sS --max-time 3 'http://$HE_ADDR/nic/update' 2>/dev/null" || echo 'NOT ANSWERING')"
    printf '  API calls  : %s\n' "$(dhcp_isp "grep -c . /run/he-sim-api.log 2>/dev/null" || echo 0)"
    printf '  route back : %s\n' "$(dhcp_isp "ip route show $PPPOE_NET 2>/dev/null" || echo '<none>')"
}

# Count of endpoint-CHANGING calls. `nochg` does not count: production's
# he-tunnel-follow re-sends the same address happily and HE treats it as a
# no-op, so only a real move is evidence that something re-pointed the tunnel.
calls() { dhcp_isp "grep -c ' good ' /run/he-sim-api.log 2>/dev/null" | tr -d ' \n'; }

# Point the endpoint at an address WITHOUT going through the API, and without
# counting as an API call.
#
# Needed because rebuilding the sim endpoint resets its remote, while the
# routers' he-tunnel-follow still reads "in sync" and therefore never re-asserts
# -- it only calls HE when its own LOCAL source changes, and has no way to learn
# that the far end drifted. The real HE does not forget, so this is sim
# bookkeeping, not a behaviour production needs. Keeping it out of the call
# counter is the point: the matrix asserts on that counter.
point() {
    local ip="$1"
    [ -n "$ip" ] || die "usage: $0 point <ipv4>"
    dhcp_isp "sudo ip tunnel change he-sim mode sit local $HE_ADDR remote $ip 2>/dev/null \
              || { sudo ip tunnel del he-sim 2>/dev/null;
                   sudo ip tunnel add he-sim mode sit local $HE_ADDR remote $ip ttl 64;
                   sudo ip link set he-sim up mtu 1480;
                   sudo ip -6 addr replace $HE_LINK6/64 dev he-sim;
                   sudo ip -6 route replace $SITE6 via $RT_LINK6 dev he-sim; }"
    local got
    got="$(dhcp_isp "ip tunnel show he-sim 2>/dev/null | sed -nE 's/.* remote ([0-9.]+).*/\\1/p'" | tr -d ' \n')"
    [ "$got" = "$ip" ] || die "endpoint still points at ${got:-nothing}, wanted $ip"
    log "endpoint now points at $ip"
}

case "${1:-status}" in
    up)     up ;;
    down)   down ;;
    status) status ;;
    calls)  calls; echo ;;
    point)  point "${2:-}" ;;
    *) die "usage: $0 {up|down|status|calls}" ;;
esac