diag(safety): attribute lockstep mismatches; time NVS flushes

components/microlink/include/microlink_internal.h

@@ -1258,8 +1258,8 @@ extern "C"
   /* Deferred flash flush of the peer cache (writes are debounced: saves only
  * update the PSRAM working copy; call this ~once per wg_mgr pass). */
   esp_err_t ml_peer_nvs_flush_if_due(uint64_t now_ms, bool ingest_busy);
-  /* Flush timing diag: out[0]=last ms, out[1]=max ms, out[2]=count. */
-  void ml_peer_nvs_get_flush_diag(uint32_t out[3]);
+  /* Flush timing diag: out[0]=last ms, out[1]=max ms, out[2]=count, out[3]=start uptime ms of the last flush. */
+  void ml_peer_nvs_get_flush_diag(uint32_t out[4]);
   /* Mark a peer (by VPN IP, host order) as never-LRU-evicted from the cache.
  * Bounded set (priority peer, fleet server, app-pinned operator remotes):
  * these keys feed the boot-time WG preseed that answers cold inbound

components/microlink/src/ml_config_httpd.c

@@ -1178,7 +1178,7 @@ static esp_err_t handler_monitor(httpd_req_t * req)
              * 2=vpn_ip 3=disco_key 4=hostname 5=region 6=endpoints */
             for (int i = 0; i < 7; i++) cJSON_AddItemToArray(sfa, cJSON_CreateNumber(sf[i]));
           }
-          uint32_t fl[3] = {0};
+          uint32_t fl[4] = {0};
           ml_peer_nvs_get_flush_diag(fl);
           cJSON_AddNumberToObject(json, "nvs_flush_last_ms", fl[0]);
           cJSON_AddNumberToObject(json, "nvs_flush_max_ms", fl[1]);

components/microlink/src/ml_peer_nvs.c

@@ -16,11 +16,14 @@
  * Reference: microlink v1 microlink_peer_registry.c
  */
 
+#include <stdatomic.h>
 #include <stdio.h>
 #include <string.h>
 
 #include "esp_log.h"
 #include "esp_timer.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
 #include "microlink_internal.h"
 #include "nvs.h"
 #include "nvs_flash.h"
@@ -252,16 +255,42 @@ void ml_peer_nvs_set_protected(uint32_t vpn_ip)
 /* Flush timing diag (§7 R6): the flash commit suspends flash-resident
  * execution on BOTH cores — these gauges make that cost measurable and the
  * fix falsifiable (flush_count flat in steady state = §7a working). */
-static uint32_t s_diag_flush_last_ms;
-static uint32_t s_diag_flush_max_ms;
-static uint32_t s_diag_flush_count;
+/* Flush diag record. Every word is atomic (no data race, each load returns a
+ * value some store wrote), and the record as a whole is published under a
+ * seqlock (s_diag_flush_seq odd while the writer is mid-update) so
+ * ml_peer_nvs_get_flush_diag() never pairs e.g. a new count with the previous
+ * flush's duration. Single writer (ml_wg_mgr). */
+static atomic_uint_fast32_t s_diag_flush_seq;
+static atomic_uint_fast32_t s_diag_flush_last_ms;
+static atomic_uint_fast32_t s_diag_flush_max_ms;
+static atomic_uint_fast32_t s_diag_flush_count;
+static atomic_uint_fast32_t
+  s_diag_flush_at_ms; /* uptime ms at the START of the last flush (soak item 5: overlap test) */
 static uint64_t s_defer_start_ms; /* nonzero while an ingest-busy deferral runs */
 
-void ml_peer_nvs_get_flush_diag(uint32_t out[3])
+void ml_peer_nvs_get_flush_diag(uint32_t out[4])
 {
-  out[0] = s_diag_flush_last_ms;
-  out[1] = s_diag_flush_max_ms;
-  out[2] = s_diag_flush_count;
+  for (int attempt = 0; attempt < 8; attempt++) {
+    uint32_t s1 = (uint32_t)atomic_load(&s_diag_flush_seq);
+    if ((s1 & 1u) != 0u) {
+      taskYIELD();
+      continue;
+    }
+    out[0] = (uint32_t)atomic_load(&s_diag_flush_last_ms);
+    out[1] = (uint32_t)atomic_load(&s_diag_flush_max_ms);
+    out[2] = (uint32_t)atomic_load(&s_diag_flush_count);
+    out[3] = (uint32_t)atomic_load(&s_diag_flush_at_ms);
+    /* Textbook seqlock reader: the acquire fence keeps the payload loads above
+     * the re-read of the sequence (an acquire LOAD alone only pins what follows
+     * it); the writer's seq_cst fetch_adds order its stores on the other side. */
+    atomic_thread_fence(memory_order_acquire);
+    if ((uint32_t)atomic_load(&s_diag_flush_seq) == s1) return;
+  }
+  /* 8 collisions (flushes are >= 5 s apart, so effectively never): best-effort copy so out[] is always written */
+  out[0] = (uint32_t)atomic_load(&s_diag_flush_last_ms);
+  out[1] = (uint32_t)atomic_load(&s_diag_flush_max_ms);
+  out[2] = (uint32_t)atomic_load(&s_diag_flush_count);
+  out[3] = (uint32_t)atomic_load(&s_diag_flush_at_ms);
 }
 
 esp_err_t ml_peer_nvs_flush_if_due(uint64_t now_ms, bool ingest_busy)
@@ -290,9 +319,12 @@ esp_err_t ml_peer_nvs_flush_if_due(uint64_t now_ms, bool ingest_busy)
   int64_t t0 = esp_timer_get_time();
   esp_err_t r = flush_table();
   uint32_t dur = (uint32_t)((esp_timer_get_time() - t0) / 1000);
-  s_diag_flush_last_ms = dur;
-  if (dur > s_diag_flush_max_ms) s_diag_flush_max_ms = dur;
-  s_diag_flush_count++;
+  (void)atomic_fetch_add(&s_diag_flush_seq, 1u); /* odd: record in flux */
+  atomic_store(&s_diag_flush_last_ms, dur);
+  atomic_store(&s_diag_flush_at_ms, (uint32_t)(t0 / 1000));
+  if (dur > (uint32_t)atomic_load(&s_diag_flush_max_ms)) atomic_store(&s_diag_flush_max_ms, dur);
+  (void)atomic_fetch_add(&s_diag_flush_count, 1u);
+  (void)atomic_fetch_add(&s_diag_flush_seq, 1u); /* even: consistent */
   s_last_flush_ms = now_ms;
   if (r == ESP_OK) {
     s_dirty = false; /* keep the dirt on a failed flush */

firmware/components/dcs_support/src/dcs_admin_pages.c

@@ -230,14 +230,16 @@ static esp_err_t page_state(httpd_req_t * req)
    */
   enum
   {
-    JSON_CAP = 4352 /* + 5 usb_tx_* counters (<= ~120 B). eth-watchdog fields + bonded-remote stop_only + operator list
+    JSON_CAP = 4864 /* + 5 usb_tx_* counters (<= ~120 B). eth-watchdog fields + bonded-remote stop_only + operator list
                        + instantaneous internal-heap fields (heap_free_int/heap_lfb_int).
                        remote_stop_id + restart_state add <= 47 B worst case against
                        ~940 B live headroom (measured 2026-08-09). derp_region_locked
                        adds <= 27 B. Region auto-negotiation surfacing (source string
                        + auto_applied + counters + mbb_state + switches_1h) adds
                        <= ~160 B worst case — bumped 3776 -> 4096 to keep comfortable
-                       headroom rather than shave the measured margin. */
+                       headroom rather than shave the measured margin. Lockstep-mismatch
+                       attribution (pstop_mm_timeout/content/last, pstop_core_lat_max_ms,
+                       nvs_pf, nvs_dcs) adds <= ~230 B worst case: 4352 -> 4864. */
   };
 
   char * buf = heap_caps_malloc(JSON_CAP, MALLOC_CAP_SPIRAM);
@@ -249,6 +251,12 @@ static esp_err_t page_state(httpd_req_t * req)
   const int cap = JSON_CAP;
   ml_usb_tx_diag_t usb_tx;
   ml_usb_tx_get_diag(&usb_tx); /* zeros until the tether has ever started */
+  extern void ml_peer_nvs_get_flush_diag(uint32_t out[4]); /* peer-cache flash flush: last/max/count/start ms */
+  uint32_t pf[4] = {0};
+  ml_peer_nvs_get_flush_diag(pf); /* seqlock-consistent copy (ml_peer_nvs.c) */
+  uint32_t mm[7] = {0};
+  dcs_pstop_mm_snapshot(mm); /* seqlock-consistent copy of the comparator's record */
+  const uint64_t nvs_w = (uint64_t)atomic_load(&g_dcs_nvs_write);
   int n = snprintf(
     buf,
     cap,
@@ -279,6 +287,8 @@ static esp_err_t page_state(httpd_req_t * req)
     "\"derp_mbb_state\":%d,\"derp_switches_1h\":%lu,"
     "\"pstop_peer_ip\":%lu,\"pstop_peer_port\":%lu,"
     "\"pstop_sent\":%lu,\"pstop_replies\":%lu,\"pstop_last_msg\":%lu,\"pstop_mismatch\":%lu,"
+    "\"pstop_mm_timeout\":%lu,\"pstop_mm_content\":%lu,\"pstop_mm_last\":[%lu,%lu,%lu],"
+    "\"pstop_core_lat_max_ms\":[%lu,%lu],\"nvs_pf\":[%lu,%lu,%lu],\"nvs_dcs\":[%lu,%lu,%lu],"
     "\"pstop_send_fail\":%lu,\"pstop_sf_nomem\":%lu,\"pstop_sf_route\":%lu,"
     "\"pstop_sf_txdrv\":%lu,\"pstop_sf_txdrv_recovered\":%lu,\"pstop_sf_other\":%lu,"
     "\"pstop_sf_enotconn\":%lu,\"pstop_sf_enotconn_kicks\":%lu,\"pstop_sf_errno\":%d,\"pstop_"
@@ -371,6 +381,19 @@ static esp_err_t page_state(httpd_req_t * req)
     (unsigned long)atomic_load(&g_dcs_pstop_replies),
     (unsigned long)atomic_load(&g_dcs_pstop_last_msg),
     (unsigned long)atomic_load(&g_dcs_pstop_mismatch),
+    (unsigned long)mm[0],
+    (unsigned long)mm[1],
+    (unsigned long)mm[2], /* packed detail — layout in dcs_internal.h */
+    (unsigned long)mm[3], /* late core's actual publish latency ms */
+    (unsigned long)mm[4], /* last event uptime ms */
+    (unsigned long)mm[5],
+    (unsigned long)mm[6],
+    (unsigned long)pf[3], /* nvs_pf: start uptime ms, duration ms, max duration ms */
+    (unsigned long)pf[0],
+    (unsigned long)pf[1],
+    (unsigned long)(uint32_t)(nvs_w >> 32), /* nvs_dcs: start uptime ms, duration ms (one 64-bit word), max ms */
+    (unsigned long)(uint32_t)(nvs_w & 0xFFFFFFFFu),
+    (unsigned long)atomic_load(&g_dcs_nvs_write_max),
     (unsigned long)atomic_load(&g_dcs_pstop_send_fail),
     (unsigned long)atomic_load(&g_dcs_pstop_sf_nomem),
     (unsigned long)atomic_load(&g_dcs_pstop_sf_route),

firmware/components/dcs_support/src/dcs_internal.h

@@ -171,6 +171,24 @@ extern "C"
   extern atomic_uint_fast32_t g_dcs_pstop_replies; /* machine replies received */
   extern atomic_uint_fast32_t g_dcs_pstop_last_msg; /* last message TYPE from the machine (PSTOP_MESSAGE_*) */
   extern atomic_uint_fast32_t g_dcs_pstop_mismatch;
+  /* pstop_mismatch attribution (soak item 5; comparator-written; /state.json pstop_mm_*): [0] timeout-class
+   * count (a core missed CORE_PUBLISH_TIMEOUT), [1] content-class count (both published, frames differed),
+   * [2] last event packed = kind<<28 (1 timeout, 2 content) | late_core_mask<<26 (bit0 core0, bit1 core1) |
+   * slot<<24 | first_differing_byte<<16 (0xFF n/a) | verdict0<<8 | verdict1 (on a timeout record the LATE core's
+   * verdict byte is from its previous publish — the late mask says which), [3] slowest late core's actual
+   * notify->publish ms for a timeout record (0 = not landed yet / n/a for a content record), [4] last event
+   * uptime ms, [5],[6] worst notify->publish ms per core this boot, including late publishes. */
+  extern atomic_uint_fast32_t g_dcs_pstop_mm[7];
+  /* Seqlock for g_dcs_pstop_mm: the comparator (single writer) bumps it before and after the 7 stores, so it is
+   * odd while the record is in flux. Readers use dcs_pstop_mm_snapshot() and never see two events mixed. */
+  extern atomic_uint_fast32_t g_dcs_pstop_mm_seq;
+  void dcs_pstop_mm_snapshot(uint32_t out[7]);
+  /* Last dcs-side NVS write (dcs_nvs.c times EVERY read-write handle open->close; both cores stall for the
+   * flash op), ONE 64-bit word so start and duration are always from the same write: start uptime ms << 32 |
+   * duration ms; plus the max duration this boot. Peer-cache flushes have their own diag
+   * (ml_peer_nvs_get_flush_diag). /state.json nvs_dcs = [start, duration, max] / nvs_pf. */
+  extern atomic_uint_fast64_t g_dcs_nvs_write;
+  extern atomic_uint_fast32_t g_dcs_nvs_write_max;
   extern atomic_uint_fast32_t g_dcs_pstop_send_fail;
   /* send_fail split by cause (errno at the failing sendto): ENOMEM =
    * TX-queue/pbuf pressure (typically DERP relay backpressure), route =