fix: remote connectivity — USB tether TX, bus-state failover, WireGuard re-establishment, coord deadline

.gitignore

@@ -18,6 +18,8 @@ host/machine_app_runner
 host/test_clock_guard
 host/test_crypto_kat
 host/test_demote_veto
+host/test_wg_regain_policy
+host/test_coord_frame_budget
 host/test_aux_channel
 # cppcheck
 *.ctu-info

components/microlink/components/wireguard_lwip/src/wireguard.h

@@ -139,6 +139,14 @@ struct wireguard_peer {
 	// This is the latest received IP/port
 	ip_addr_t ip;
 	u16_t port;
+	// Handshake second leg (safety peers while DERP-only): the source of the
+	// most recent DIRECT disco packet from this peer. Our initiations are
+	// mirrored there, so a peer that is provably reachable direct can complete
+	// the handshake even when the relay leg is slow or dead and the peer itself
+	// has nothing to send (an idle wireguard-go never initiates). ip any = none.
+	ip_addr_t hs_cand_ip;
+	u16_t hs_cand_port;
+	uint32_t hs_cand_ms;
 	// keep-alive interval in seconds, 0 is disable
 	uint16_t keepalive_interval;
 

components/microlink/components/wireguard_lwip/src/wireguardif.c

@@ -84,6 +84,7 @@ volatile uint32_t wireguardif_pbuf_alloc_fails = 0;
 // no_valid_keys   = subsequent sends with no session at all (surface as errno 128).
 volatile uint32_t wireguardif_tx_keypair_expired = 0;
 volatile uint32_t wireguardif_tx_no_valid_keys = 0;
+volatile uint32_t wireguardif_hs_cand_sends = 0;
 
 #define WIREGUARDIF_TIMER_MSECS 400
 
@@ -169,6 +170,23 @@ static err_t wireguardif_peer_output(struct netif *netif, struct pbuf *q, struct
 			if (data) {
 				pbuf_copy_partial(q, data, q->tot_len, 0);
 				err_t result = device->derp_output_fn(peer->public_key, data, q->tot_len, device->derp_output_ctx);
+				// Second leg for handshake INITIATIONS only: the same bytes to the
+				// most recent direct disco source of this peer (bench 2026-09-19:
+				// a remote sat DERP-only for 7+ min with no valid keypair, its
+				// initiations relayed only, while the machine host pinged it
+				// directly every few seconds; one host-initiated handshake healed
+				// it). One initiation, two legs -> either response completes it,
+				// and a direct response re-adopts the direct endpoint.
+				if (device->udp_output_fn && !ip_addr_isany(&peer->hs_cand_ip) && peer->hs_cand_port != 0 &&
+				    !wireguard_expired(peer->hs_cand_ms, HS_CAND_FRESH_MS / 1000) &&
+				    wireguard_get_message_type(data, q->tot_len) == MESSAGE_HANDSHAKE_INITIATION) {
+					uint32_t cand_ip = ip4_addr_get_u32(ip_2_ip4(&peer->hs_cand_ip));
+					err_t leg2 = device->udp_output_fn(cand_ip, peer->hs_cand_port, data, q->tot_len, device->udp_output_ctx);
+					wireguardif_hs_cand_sends++;
+					if (result != ERR_OK && leg2 == ERR_OK) {
+						result = ERR_OK;
+					}
+				}
 				mem_free(data);
 				return result;
 			}
@@ -1020,6 +1038,48 @@ static err_t wireguardif_lookup_peer(struct netif *netif, u8_t peer_index, struc
 	return result;
 }
 
+// hs_cand_{ip,port,ms} are a multi-word record written from the disco task and
+// read on the TCPIP thread (peer_output, which runs under the core lock). Plain
+// stores carry no cross-core ordering guarantee, so the record is updated and
+// read under LOCK_TCPIP_CORE — the same idiom as netif_set_link_up/down here.
+// Callers already on the TCPIP thread hold the lock; the guard is a no-op then.
+err_t wireguardif_set_hs_candidate(struct netif *netif, u8_t peer_index, const ip_addr_t *ip, u16_t port) {
+	struct wireguard_peer *peer;
+	bool nl = wireguardif_core_lock_needed();
+	if (nl) LOCK_TCPIP_CORE(); // covers the slot lookup (valid flag) as well as the record
+	err_t result = wireguardif_lookup_peer(netif, peer_index, &peer);
+	if (result == ERR_OK) {
+		if (ip && port != 0) {
+			ip_addr_copy(peer->hs_cand_ip, *ip);
+			peer->hs_cand_ms = wireguard_sys_now();
+			peer->hs_cand_port = port;
+		} else {
+			ip_addr_set_any(false, &peer->hs_cand_ip);
+			peer->hs_cand_port = 0;
+		}
+	}
+	if (nl) UNLOCK_TCPIP_CORE();
+	return result;
+}
+
+err_t wireguardif_get_hs_candidate(struct netif *netif, u8_t peer_index, uint32_t *ip_host, u16_t *port) {
+	struct wireguard_peer *peer;
+	if (!ip_host || !port) {
+		return ERR_ARG;
+	}
+	bool nl = wireguardif_core_lock_needed();
+	if (nl) LOCK_TCPIP_CORE();
+	err_t result = wireguardif_lookup_peer(netif, peer_index, &peer);
+	if (result == ERR_OK) {
+		bool fresh = !ip_addr_isany(&peer->hs_cand_ip) && peer->hs_cand_port != 0 &&
+		             !wireguard_expired(peer->hs_cand_ms, HS_CAND_FRESH_MS / 1000);
+		*ip_host = fresh ? lwip_ntohl(ip4_addr_get_u32(ip_2_ip4(&peer->hs_cand_ip))) : 0;
+		*port = fresh ? peer->hs_cand_port : 0;
+	}
+	if (nl) UNLOCK_TCPIP_CORE();
+	return result;
+}
+
 err_t wireguardif_connect(struct netif *netif, u8_t peer_index) {
 	struct wireguard_peer *peer;
 	err_t result = wireguardif_lookup_peer(netif, peer_index, &peer);

components/microlink/components/wireguard_lwip/src/wireguardif.h

@@ -128,6 +128,13 @@ err_t wireguardif_update_endpoint(struct netif *netif, u8_t peer_index, const ip
 // Try and connect to the given peer
 err_t wireguardif_connect(struct netif *netif, u8_t peer_index);
 
+// Remember where a DIRECT disco packet from this peer came from. While the
+// peer has no direct WG endpoint (DERP-only), handshake initiations are also
+// sent there; a response from it re-adopts the direct endpoint. Safety peers.
+err_t wireguardif_set_hs_candidate(struct netif *netif, u8_t peer_index, const ip_addr_t *ip, u16_t port);
+// Read it back (host-order IPv4, 0 = none/stale). For the disco fan-out.
+err_t wireguardif_get_hs_candidate(struct netif *netif, u8_t peer_index, uint32_t *ip_host, u16_t *port);
+
 // Stop trying to connect to the given peer
 err_t wireguardif_disconnect(struct netif *netif, u8_t peer_index);
 
@@ -158,6 +165,9 @@ err_t wireguardif_peer_handshake_age(struct netif *netif, u8_t peer_index, u32_t
 // vs no-valid-keys (every such send surfaces as ENOTCONN/errno 128).
 extern volatile uint32_t wireguardif_tx_keypair_expired;
 extern volatile uint32_t wireguardif_tx_no_valid_keys;
+// Handshake initiations mirrored to a direct disco-source candidate while DERP-only.
+extern volatile uint32_t wireguardif_hs_cand_sends;
+#define HS_CAND_FRESH_MS 60000u /* a handshake second-leg candidate (direct disco source) older than this is stale */
 
 // Register a DERP relay output callback for peers without direct endpoints
 // This callback is invoked when a WireGuard packet needs to be sent to a peer

components/microlink/include/microlink.h

@@ -62,6 +62,20 @@ extern "C"
     bool online;
     bool direct_path; /* true if communicating via direct UDP */
     uint16_t derp_region; /* learned DERP home region (0 = unknown) — diagnostic */
+    /* Path-recovery diagnostics (bench 2026-09-19): why a peer is or is not
+     * being reached directly. All 0 when unknown. */
+    int endpoint_count; /* disco candidate endpoints held for this peer */
+    uint32_t best_ip; /* chosen direct endpoint (host order), 0 = none */
+    uint16_t best_port;
+    uint32_t hs_cand_ip; /* WG handshake second-leg candidate (host order), 0 = none/stale */
+    uint16_t hs_cand_port;
+    uint32_t ping_age_ms; /* since our last disco ping to it (0xFFFFFFFF = never) */
+    uint32_t backoff_ms; /* remaining direct-upgrade flap backoff */
+    bool wg_up; /* WireGuard session currently has a usable keypair */
+    bool wg_active; /* wireguardif: actively (re)initiating handshakes */
+    bool wg_send_hs; /* wireguardif: handshake requested by the data path */
+    bool wg_hs_pending; /* wireguardif: an initiation is outstanding */
+    uint32_t wg_init_age_ms; /* since our last handshake initiation TX (0xFFFFFFFF = never) */
   } microlink_peer_info_t;
 
   /* Connection state */

components/microlink/include/microlink_internal.h

@@ -614,6 +614,7 @@ extern "C"
 
     /* DISCO state (rate limiting) */
     uint64_t last_ping_sent_ms; /* Last DISCO ping we sent */
+    uint64_t last_safety_connect_ms; /* last keyless-regain wireguardif_connect() (paced to REKEY_TIMEOUT) */
     uint64_t last_pong_recv_ms; /* Last DISCO pong we received (any path) */
     uint64_t last_direct_pong_recv_ms; /* Last pong received DIRECT — the
                                         * pong-dead demote trigger keys on

components/microlink/include/ml_coord_frame_budget.h

@@ -0,0 +1,113 @@
+// SPDX-FileCopyrightText: 2026 Polymath Robotics
+// SPDX-License-Identifier: Apache-2.0
+
+/* Wall-clock budget for finishing ONE partial coordination (Noise) frame —
+ * pure, host-testable (host/test_coord_frame_budget.c). Used by
+ * ml_coord.c coord_recv_ex().
+ *
+ * Issue #127: the old bound was a retry COUNT (300 x 10 ms, "~3 s"), but each
+ * retry first blocked for the socket's SO_RCVTIMEO — 2 s in long-poll, 60 s
+ * while fetching the MapResponse — so a stalled-but-open peer held the coord
+ * task for 300 x 2 s = 10 min, starving the ML_CTRL_WATCHDOG_MS (120 s) check
+ * that only runs between reads.
+ *
+ * Two limits, both counted from the FIRST byte of the frame:
+ *  - NO-PROGRESS window (10 s): every byte received restarts it. A stalled
+ *    peer (TCP black hole, dead relay) is abandoned 10 s after its last byte —
+ *    the #127 case — while a slow-but-live link keeps its frame as long as
+ *    bytes keep arriving. Bytes arriving is exactly the liveness the control
+ *    watchdog wants to see, so extending on progress does not hide a dead
+ *    control plane from it.
+ *  - HARD CAP scaled by frame size: max(10 s, frame_len / 1 KiB/s). Control
+ *    frames (<= 4 KB) get the 10 s floor; a full 64 KB MapResponse frame gets
+ *    64 s, i.e. a link must sustain ~8 kbit/s to keep it. Below that the
+ *    5 Hz safety heartbeat is marginal anyway, and the cap bounds how long a
+ *    trickling peer can hold the coord task (previously unbounded).
+ *  - Not armed (no byte consumed yet) = an idle stream, not a partial frame:
+ *    the caller keeps using the socket timeout and may simply retry later.
+ *  - Once armed, one blocking wait may last at most the time to the nearer of
+ *    the two deadlines — never the socket's own timeout.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define ML_COORD_FRAME_NO_PROGRESS_MS 10000u /* abandon after this long without a byte */
+#define ML_COORD_FRAME_MIN_BUDGET_MS 10000u /* hard-cap floor (control frames <= 4 KB) */
+#define ML_COORD_FRAME_MIN_RATE_BPS 1024u /* bytes/s a live link must sustain for the hard cap */
+
+typedef struct
+{
+  bool armed;
+  int64_t total_us; /* hard cap length for this frame (from frame_len) */
+  int64_t first_byte_us; /* when the frame started (arming instant) */
+  int64_t hard_deadline_us; /* first byte + total_us */
+  int64_t progress_deadline_us; /* last byte + NO_PROGRESS */
+} ml_coord_frame_budget_t;
+
+/* Hard cap for a frame of frame_len bytes, in ms. */
+static inline uint32_t ml_coord_frame_budget_total_ms(size_t frame_len)
+{
+  uint64_t by_size = ((uint64_t)frame_len * 1000u) / ML_COORD_FRAME_MIN_RATE_BPS;
+  return (by_size > ML_COORD_FRAME_MIN_BUDGET_MS) ? (uint32_t)by_size : ML_COORD_FRAME_MIN_BUDGET_MS;
+}
+
+/* frame_len = total bytes this read must complete (the caller's len). */
+static inline void ml_coord_frame_budget_init(ml_coord_frame_budget_t * b, size_t frame_len)
+{
+  b->armed = false;
+  b->total_us = (int64_t)ml_coord_frame_budget_total_ms(frame_len) * 1000LL;
+  b->first_byte_us = 0;
+  b->hard_deadline_us = 0;
+  b->progress_deadline_us = 0;
+}
+
+/* A frame's length is often known only after its header: re-size the hard cap
+ * for the WHOLE frame (header + payload) while keeping the first-byte instant,
+ * so header and payload run under ONE budget, not two. Never shrinks below the
+ * length the budget was initialised with. */
+static inline void ml_coord_frame_budget_set_frame_len(ml_coord_frame_budget_t * b, size_t frame_len)
+{
+  int64_t total = (int64_t)ml_coord_frame_budget_total_ms(frame_len) * 1000LL;
+  if (total > b->total_us) b->total_us = total;
+  if (b->armed) b->hard_deadline_us = b->first_byte_us + b->total_us;
+}
+
+/* Call after every read that consumed n > 0 bytes: arms on the first byte,
+ * restarts the no-progress window on every byte, never moves the hard cap. */
+static inline void ml_coord_frame_budget_on_bytes(ml_coord_frame_budget_t * b, int64_t now_us)
+{
+  if (!b->armed) {
+    b->armed = true;
+    b->first_byte_us = now_us;
+    b->hard_deadline_us = now_us + b->total_us;
+  }
+  b->progress_deadline_us = now_us + (int64_t)ML_COORD_FRAME_NO_PROGRESS_MS * 1000LL;
+}
+
+static inline bool ml_coord_frame_budget_armed(const ml_coord_frame_budget_t * b)
+{
+  return b->armed;
+}
+
+static inline int64_t ml_coord_frame_budget_deadline_us(const ml_coord_frame_budget_t * b)
+{
+  return (b->progress_deadline_us < b->hard_deadline_us) ? b->progress_deadline_us : b->hard_deadline_us;
+}
+
+/* True once an armed budget has run out (either limit): the frame must be abandoned. */
+static inline bool ml_coord_frame_budget_expired(const ml_coord_frame_budget_t * b, int64_t now_us)
+{
+  return b->armed && now_us >= ml_coord_frame_budget_deadline_us(b);
+}
+
+/* Longest one blocking wait for more bytes may last. 0 when not armed or expired. */
+static inline int64_t ml_coord_frame_budget_wait_us(const ml_coord_frame_budget_t * b, int64_t now_us)
+{
+  if (!b->armed) return 0;
+  int64_t rem = ml_coord_frame_budget_deadline_us(b) - now_us;
+  return rem > 0 ? rem : 0;
+}

components/microlink/include/ml_wg_regain_policy.h

@@ -0,0 +1,50 @@
+// SPDX-FileCopyrightText: 2026 Polymath Robotics
+// SPDX-License-Identifier: Apache-2.0
+
+/* WireGuard re-establishment decision on a direct-path (re)gain — pure,
+ * host-testable (host/test_wg_regain_policy.c).
+ *
+ * Context (ml_wg_mgr.c process_disco_pong): a direct pong just installed an
+ * endpoint for a peer that has NO valid WireGuard session. Two kinds of peer
+ * want two different reactions:
+ *
+ *  - SAFETY peer (the machine host): connect() with WireGuard's own
+ *    REKEY_TIMEOUT retries. Its wireguard-go initiates only when it has data,
+ *    and it has nothing to send while we are silent — a one-shot that then
+ *    waits for the peer to initiate deadlocks (bench 2026-09-19: relay dead,
+ *    stuck indefinitely). Paced to REKEY_TIMEOUT: pongs arrive several times a
+ *    second and every connect() resets the pending handshake, so a slow path
+ *    could otherwise keep answering an already-superseded initiation. Inside
+ *    the pace window the answer is NONE — never the one-shot, which clears
+ *    peer->active and would cancel the retries just armed.
+ *
+ *  - BULK tailnet peer: one initiation, active cleared. If it has us trimmed
+ *    it will never answer and retries would run forever.
+ */
+
+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#define ML_WG_SAFETY_CONNECT_PACE_MS 5000u /* = WireGuard REKEY_TIMEOUT */
+
+typedef enum
+{
+  ML_WG_REGAIN_NONE = 0, /* leave the pending handshake alone           */
+  ML_WG_REGAIN_CONNECT_RETRYING, /* wireguardif_connect(): active, auto-retries  */
+  ML_WG_REGAIN_ONE_SHOT_INIT, /* single initiation, active cleared afterwards */
+} ml_wg_regain_action_t;
+
+/* last_safety_connect_ms == 0 means "never connected". */
+static inline ml_wg_regain_action_t ml_wg_regain_action(
+  bool is_safety_peer, bool tried_initial_handshake, uint64_t now_ms, uint64_t last_safety_connect_ms)
+{
+  if (is_safety_peer) {
+    if (last_safety_connect_ms == 0u || (now_ms - last_safety_connect_ms) >= ML_WG_SAFETY_CONNECT_PACE_MS) {
+      return ML_WG_REGAIN_CONNECT_RETRYING;
+    }
+    return ML_WG_REGAIN_NONE;
+  }
+  return tried_initial_handshake ? ML_WG_REGAIN_NONE : ML_WG_REGAIN_ONE_SHOT_INIT;
+}

components/microlink/src/microlink.c

@@ -21,9 +21,11 @@
 #include "esp_timer.h"
 #include "esp_wifi.h"
 #include "lwip/sockets.h"
+#include "lwip/sys.h"
 #include "microlink_internal.h"
 #include "nvs.h"
 #include "nvs_flash.h"
+#include "wireguardif.h" /* peer path-recovery diagnostics in microlink_get_peer_info */
 
 static const char * TAG = "microlink";
 
@@ -848,6 +850,48 @@ esp_err_t microlink_get_peer_info(const microlink_t * ml, int index, microlink_p
   info->online = p->active;
   info->direct_path = p->has_direct_path;
   info->derp_region = p->derp_region;
+  info->endpoint_count = p->endpoint_count;
+  info->best_ip = p->best_ip;
+  info->best_port = (uint16_t)p->best_port;
+  info->hs_cand_ip = 0;
+  info->hs_cand_port = 0;
+  info->wg_up = false;
+  info->wg_active = false;
+  info->wg_send_hs = false;
+  info->wg_hs_pending = false;
+  info->wg_init_age_ms = 0xFFFFFFFFu; /* "never" until a WG peer says otherwise */
+  uint64_t now = ml_get_time_ms();
+  info->ping_age_ms = (p->last_ping_sent_ms != 0) ? (uint32_t)(now - p->last_ping_sent_ms) : 0xFFFFFFFFu;
+  info->backoff_ms = (p->direct_backoff_until > now) ? (uint32_t)(p->direct_backoff_until - now) : 0u;
+  if (ml->wg_netif != NULL && p->wg_peer_index >= 0) {
+    struct netif * netif = (struct netif *)ml->wg_netif;
+    ip_addr_t cur_ip;
+    u16_t cur_port;
+    info->wg_up = (wireguardif_peer_is_up(netif, (u8_t)p->wg_peer_index, &cur_ip, &cur_port) == ERR_OK);
+    /* Diagnostic snapshot of plain scalars owned by the TCPIP thread, read
+     * without the core lock on purpose: a torn read can only yield a stale
+     * bool or age for one JSON sample, never a fault, and taking the lock from
+     * the HTTP task for telemetry is what stalled the safety loop before. */
+    struct wireguard_device * dev = (struct wireguard_device *)netif->state;
+    if (dev != NULL && p->wg_peer_index < WIREGUARD_MAX_PEERS) {
+      const struct wireguard_peer * wp = &dev->peers[p->wg_peer_index];
+      info->wg_active = wp->active;
+      info->wg_send_hs = wp->send_handshake;
+      info->wg_hs_pending = wp->handshake.valid;
+      /* Handshake second-leg candidate: same lock-free telemetry snapshot (the
+       * locked wireguardif_get_hs_candidate() is for the disco task's decisions,
+       * not for the HTTP task — see the comment above). */
+      if (
+        !ip_addr_isany(&wp->hs_cand_ip) && wp->hs_cand_port != 0 &&
+        !wireguard_expired(wp->hs_cand_ms, HS_CAND_FRESH_MS / 1000))
+      {
+        info->hs_cand_ip = lwip_ntohl(ip4_addr_get_u32(ip_2_ip4(&wp->hs_cand_ip)));
+        info->hs_cand_port = wp->hs_cand_port;
+      }
+      uint32_t wg_now = sys_now(); /* wireguardif stamps with lwIP sys_now(), not esp_timer */
+      info->wg_init_age_ms = (wp->last_initiation_tx != 0) ? (wg_now - wp->last_initiation_tx) : 0xFFFFFFFFu;
+    }
+  }
   return ESP_OK;
 }