campaign: score the ANE direct embed lane by aggregate throughput

The row set is half 12-38-token rows and half near-512-token rows, so a
median of per-row rates sat between two regimes and rewarded shaving the
per-row fixed cost that dominates the short rows. Every active token over
every millisecond weights rows by the work they carry. The runner reads the
objective from the field name every embed harness here uses, so that name
stays and aggregate_tok_s carries the same value under the honest one.
Baseline re-measured with the harness under the new ruler: 9,222 tok/s at
512, min cosine 0.9990908 across shapes, load 8 at admission.

Author: ualtinok

bench/campaign/ane-direct-embed-harness.sh

@@ -21,7 +21,7 @@ import tempfile
 from pathlib import Path
 from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple
 
-BASELINE_TOK_S = 8_941.75223984836
+BASELINE_TOK_S = 9_222.454033809034
 ROW_SET_SHA256 = "f4889a38df77b9940ce973c4d9b82857d0c401987ae8e77b5ca25e6062808c39"
 MODEL_REVISION = "e7f32e3c00f91d699e8c43b53106206bcc72bb22"
 MODEL_COMPONENT_SHA256 = {
@@ -99,6 +99,7 @@ def initial_payload(note: str) -> Dict[str, Any]:
         "samples": [],
         "paired_runs": [],
         "median_tok_s": None,
+        "aggregate_tok_s": None,
         "min_cosine": None,
         "sequence_1024_tok_s": None,
         "sequence_2048_tok_s": None,
@@ -604,8 +605,21 @@ def validate_report(
     report_load = payload.get("one_minute_load_average")
     if not finite_number(report_load) or float(report_load) < 0.0:
         raise CandidateRejected("candidate report omitted a valid one-minute load average")
+    # The row set is half 12-38-token rows and half near-512-token rows. A median
+    # of per-row rates lands between those two regimes and rewards shaving the
+    # per-row fixed cost that dominates the short rows. Aggregate throughput —
+    # every active token over every millisecond spent — weights the rows by the
+    # work they carry, which is what the lane exists to serve.
+    total_active = sum(int(row["active_tokens"]) for row in row_measurements)
+    total_wall_ms = sum(float(row["warm_wall_ms_median"]) for row in row_measurements)
+    aggregate = total_active * 1000.0 / total_wall_ms
     return {
-        "median_tok_s": float(statistics.median(row_rates)),
+        # The campaign runner reads the objective from the field every embed
+        # harness in this repository names `median_tok_s`; the name is a wire
+        # contract, so it stays, and `aggregate_tok_s` carries the same value
+        # under the name that describes it.
+        "median_tok_s": aggregate,
+        "aggregate_tok_s": aggregate,
         "row_tok_s": row_rates,
         "rows": row_measurements,
         "min_cosine": float(reported_min),
@@ -728,6 +742,7 @@ def run_harness(workspace_arg: str, runner_arg: str, result_arg: str) -> int:
             "samples": objective["row_tok_s"],
             "paired_runs": measurements,
             "median_tok_s": objective["median_tok_s"],
+            "aggregate_tok_s": objective["aggregate_tok_s"],
             "min_cosine": min_cosine,
             "sequence_1024_tok_s": measurements[1]["median_tok_s"],
             "sequence_2048_tok_s": measurements[2]["median_tok_s"],

bench/campaign/ane-direct-embed-registration.jsonc

@@ -7,16 +7,16 @@
 {
   "machine": "m5-local",
   "harness_path": "bench/campaign/ane-direct-embed-harness.sh",
-  "harness_sha256": "25fd123291206020008e56bf6f7ef573cc0f18c7f16cc68258c313eb4877510f",
+  "harness_sha256": "a3f9546e2402ffb5ce077a915cb52d609e2f964de732807e40b0d68291185933",
   "parameters": {},
   "parity_command": {
-    "harness_sha256": "25fd123291206020008e56bf6f7ef573cc0f18c7f16cc68258c313eb4877510f",
+    "harness_sha256": "a3f9546e2402ffb5ce077a915cb52d609e2f964de732807e40b0d68291185933",
     "program": "{harness}",
     "args": ["{workspace}", "{candidate_runner}", "{result}"],
     "env": {
       "HF_HUB_OFFLINE": "1",
       "TRANSFORMERS_OFFLINE": "1",
-      "SYNAPSE_CAMPAIGN_BASELINE_TOK_S": "8941.75223984836",
+      "SYNAPSE_CAMPAIGN_BASELINE_TOK_S": "9222.454033809034",
       "SYNAPSE_CAMPAIGN_MODEL": "<RIG>/.cache/huggingface/hub/models--Alibaba-NLP--gte-modernbert-base/snapshots/e7f32e3c00f91d699e8c43b53106206bcc72bb22",
       "SYNAPSE_CAMPAIGN_MODEL_SHA256": "3e85899d5728cb7de79781c0c3acfb91ccef9f875f1f7e0b3c9f3dd4b6a724ba",
       "SYNAPSE_CAMPAIGN_ROWS_SHA256": "f4889a38df77b9940ce973c4d9b82857d0c401987ae8e77b5ca25e6062808c39",
@@ -30,13 +30,13 @@
     }
   },
   "objective_command": {
-    "harness_sha256": "25fd123291206020008e56bf6f7ef573cc0f18c7f16cc68258c313eb4877510f",
+    "harness_sha256": "a3f9546e2402ffb5ce077a915cb52d609e2f964de732807e40b0d68291185933",
     "program": "{harness}",
     "args": ["{workspace}", "{candidate_runner}", "{result}"],
     "env": {
       "HF_HUB_OFFLINE": "1",
       "TRANSFORMERS_OFFLINE": "1",
-      "SYNAPSE_CAMPAIGN_BASELINE_TOK_S": "8941.75223984836",
+      "SYNAPSE_CAMPAIGN_BASELINE_TOK_S": "9222.454033809034",
       "SYNAPSE_CAMPAIGN_MODEL": "<RIG>/.cache/huggingface/hub/models--Alibaba-NLP--gte-modernbert-base/snapshots/e7f32e3c00f91d699e8c43b53106206bcc72bb22",
       "SYNAPSE_CAMPAIGN_MODEL_SHA256": "3e85899d5728cb7de79781c0c3acfb91ccef9f875f1f7e0b3c9f3dd4b6a724ba",
       "SYNAPSE_CAMPAIGN_ROWS_SHA256": "f4889a38df77b9940ce973c4d9b82857d0c401987ae8e77b5ca25e6062808c39",
@@ -55,6 +55,7 @@
       "parity_passed": {"type": "boolean"},
       "determinism_passed": {"type": "boolean"},
       "median_tok_s": {"type": "number"},
+      "aggregate_tok_s": {"type": "number"},
       "min_cosine": {"type": "number"},
       "sequence_1024_tok_s": {"type": "number"},
       "sequence_2048_tok_s": {"type": "number"}