Fix tps-utilization to normalize per pareto curve, not submission-wide

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.15.21
+  hooks:
+    # Run the linter.
+    - id: ruff-format
+    - id: ruff-check
+      args: [ --fix ]

src/endpoints_submission_cli/commands/common.py

@@ -45,6 +45,7 @@ def _write_cli_metadata(submission_dir: Path, command: str) -> None:
     }
     (submission_dir / "cli_metadata.json").write_text(json.dumps(meta, indent=2))
 
+
 _console = Console(stderr=True)
 _stdout_console = Console()
 

src/endpoints_submission_cli/submissions/builder.py

@@ -126,8 +126,12 @@ def build_submission_folder(
             _write_system_description(submission_dir, system_id, runs[0]["system_info"])
             written_systems.add(system_id)
         _write_pareto_entries(
-            submission_dir, system_id, model, runs,
-            system_max_concurrency[system_id], max_tps_by_model[model],
+            submission_dir,
+            system_id,
+            model,
+            runs,
+            system_max_concurrency[system_id],
+            max_tps_by_model[model],
         )
 
     # Copy src/ for Standardized division submissions (mirrors documentation/ handling)
@@ -453,7 +457,7 @@ def _write_pareto_entries(
             if rel_path in ("results.json", "accuracy/results.json"):
                 content = truncate_responses(content)
             dest_rel = (
-                rel_path[len(_acc_prefix):]
+                rel_path[len(_acc_prefix) :]
                 if run_type == "accuracy" and rel_path.startswith(_acc_prefix)
                 else rel_path
             )

src/submission_checker/checker.py

@@ -135,15 +135,15 @@ def run(self) -> Report:
         for system_json in system_jsons:
             report.results.extend(self._check_system(system_json, pareto_dir))
 
-
-        # Submission-wide: tps_utilization must match system_tps / max(system_tps).
+        # Per-curve: tps_utilization must match system_tps / max(system_tps)
+        # within each <system_desc_id>/<benchmark_model> pareto curve.
         report.results.extend(self._check_tps_utilization(pareto_dir))
 
         # §15: at least one model must carry accuracy results — either as accuracy_scores
         # embedded in a results.json, or as a standalone accuracy/results.json.
-        has_full_accuracy = any(
-            True for _ in pareto_dir.rglob("accuracy/results.json")
-        ) or any(_results_has_accuracy_scores(p) for p in pareto_dir.rglob("results.json"))
+        has_full_accuracy = any(True for _ in pareto_dir.rglob("accuracy/results.json")) or any(
+            _results_has_accuracy_scores(p) for p in pareto_dir.rglob("results.json")
+        )
 
         if has_full_accuracy:
             report.results.append(
@@ -174,13 +174,21 @@ def run(self) -> Report:
     def _check_tps_utilization(self, pareto_dir: Path) -> list[CheckResult]:
         """Verify each run's ``tps_utilization`` equals ``system_tps / max(system_tps)``.
 
-        ``tps_utilization`` normalises a run to the peak ``system_tps`` across the
-        whole submission, so this is a cross-run check. Stored values are compared
-        to the recomputed expectation within an absolute tolerance of
-        ``_TPS_UTILIZATION_ABS_TOL``. Structurally invalid metadata (missing or
-        non-numeric fields) is left to the per-file ``run-metadata-valid`` check.
+        ``tps_utilization`` normalises a run to the peak ``system_tps`` of the
+        system+model curve it belongs to — NOT across the whole submission.
+        Normalising submission-wide is wrong when a submission contains more than
+        one system: e.g. an ``MI355X_1x`` and an ``MI355X_8x`` config sharing a
+        folder would force every 1x point to be divided by the 8x peak, so the
+        smaller system can never match its stored (per-curve) values.
+
+        Stored values are compared to the recomputed expectation within an
+        absolute tolerance of ``_TPS_UTILIZATION_ABS_TOL``. Structurally invalid
+        metadata (missing or non-numeric fields) is left to the per-file
+        ``run-metadata-valid`` check.
         """
-        entries: list[tuple[Path, float, float]] = []
+        # Group run metadata by its pareto curve: the first two path components
+        # under pareto_dir are <system_desc_id>/<benchmark_model>.
+        curves: dict[tuple[str, ...], list[tuple[Path, float, float]]] = {}
         for md_path in sorted(pareto_dir.rglob("run_metadata.json")):
             try:
                 data = json.loads(md_path.read_text())
@@ -194,37 +202,38 @@ def _check_tps_utilization(self, pareto_dir: Path) -> list[CheckResult]:
                 and isinstance(util, (int, float))
                 and not isinstance(util, bool)
             ):
-                entries.append((md_path, float(tps), float(util)))
-
-        if not entries:
-            return []
-        max_tps = max(tps for _, tps, _ in entries)
-        if max_tps <= 0:
-            return []
+                rel = md_path.relative_to(pareto_dir).parts
+                curve = rel[:2] if len(rel) >= 2 else rel
+                curves.setdefault(curve, []).append((md_path, float(tps), float(util)))
 
         results: list[CheckResult] = []
-        for md_path, tps, util in entries:
-            expected = tps / max_tps
-            if abs(util - expected) <= _TPS_UTILIZATION_ABS_TOL:
-                results.append(
-                    _ok(
-                        "tps-utilization",
-                        f"tps_utilization {util:.4f} matches expected {expected:.4f}",
-                        md_path,
-                        "#8.1",
+        for curve in sorted(curves):
+            entries = curves[curve]
+            max_tps = max(tps for _, tps, _ in entries)
+            if max_tps <= 0:
+                continue
+            for md_path, tps, util in entries:
+                expected = tps / max_tps
+                if abs(util - expected) <= _TPS_UTILIZATION_ABS_TOL:
+                    results.append(
+                        _ok(
+                            "tps-utilization",
+                            f"tps_utilization {util:.4f} matches expected {expected:.4f}",
+                            md_path,
+                            "#8.1",
+                        )
                     )
-                )
-            else:
-                results.append(
-                    _err(
-                        "tps-utilization",
-                        f"tps_utilization {util} != expected {expected:.4f}"
-                        f" (system_tps {tps} / submission max {max_tps};"
-                        f" abs tol {_TPS_UTILIZATION_ABS_TOL})",
-                        md_path,
-                        "#8.1",
+                else:
+                    results.append(
+                        _err(
+                            "tps-utilization",
+                            f"tps_utilization {util} != expected {expected:.4f}"
+                            f" (system_tps {tps} / curve max {max_tps} for"
+                            f" {'/'.join(curve)}; abs tol {_TPS_UTILIZATION_ABS_TOL})",
+                            md_path,
+                            "#8.1",
+                        )
                     )
-                )
         return results
 
     # ------------------------------------------------------------------

src/submission_checker/models/aggregate/point_result.py

@@ -18,7 +18,7 @@
 # Dataset → minimum completed query count (§6.4).
 # Values equal the full dataset size; every sample must be run for a valid submission.
 MIN_QUERY_COUNT: dict[str, int] = {
-   "open_orca": 24576,
+    "open_orca": 24576,
     "cnn_dailymail": 13368,
     "aime25": 30,
     "gpqa": 198,

tests/endpoints_submission_cli/runs/test_parser.py

@@ -278,9 +278,7 @@ def test_no_metadata_file_no_crash(self, run_folder: Path, tmp_path: Path) -> No
         with tarfile.open(dest) as tar:
             assert not any(n.endswith("run_metadata.json") for n in tar.getnames())
 
-    def test_no_run_date_leaves_metadata_untouched(
-        self, run_folder: Path, tmp_path: Path
-    ) -> None:
+    def test_no_run_date_leaves_metadata_untouched(self, run_folder: Path, tmp_path: Path) -> None:
         (run_folder / "run_metadata.json").write_text(json.dumps({"run_date": "1999-01-01"}))
         dest = tmp_path / "out.tar.gz"
         build_archive(run_folder, dest)  # no run_date passed

tests/endpoints_submission_cli/submissions/test_builder.py

@@ -511,9 +511,7 @@ def test_runtime_seeds_from_config(self, run_folder: Path, tmp_path: Path) -> No
         assert runtime["scheduler_random_seed"] == 42
         assert runtime["dataloader_random_seed"] == 42
 
-    def test_runtime_block_mirrors_config_runtime(
-        self, run_folder: Path, tmp_path: Path
-    ) -> None:
+    def test_runtime_block_mirrors_config_runtime(self, run_folder: Path, tmp_path: Path) -> None:
         """The whole settings.runtime block is dropped into runtime_settings.runtime."""
         archive = self._make_archive(
             run_folder,
@@ -648,7 +646,9 @@ def test_built_points_parse_through_checker_pointconfig(
             assert cfg.runtime_settings.runtime.dataloader_random_seed == 42
             # No per-point structural/seed errors for a compliant input.
             errors = [r for r in cfg._check_results if r.severity == Severity.ERROR]
-            assert not errors, f"{py.name} unexpected errors: {[(r.rule, r.message) for r in errors]}"
+            assert not errors, (
+                f"{py.name} unexpected errors: {[(r.rule, r.message) for r in errors]}"
+            )
 
     def test_full_checker_runs_without_parse_failures(
         self, run_folder: Path, tmp_path: Path
@@ -791,7 +791,9 @@ def test_dict_responses_truncated_in_bundle(self, run_folder: Path, tmp_path: Pa
         big = {f"uuid-{i:08d}": "hello world " * 5 for i in range(10_000)}
         sub_dir = build_submission_folder(
             [("run-001", self._archive(run_folder, tmp_path, big))],
-            "standardized", "available", tmp_path / "sub",
+            "standardized",
+            "available",
+            tmp_path / "sub",
         )
         written = self._perf_results(sub_dir)
         assert isinstance(written["responses"], dict)
@@ -802,7 +804,9 @@ def test_list_responses_truncated_in_bundle(self, run_folder: Path, tmp_path: Pa
         big = [{"text": "hello world", "idx": i} for i in range(10_000)]
         sub_dir = build_submission_folder(
             [("run-001", self._archive(run_folder, tmp_path, big))],
-            "standardized", "available", tmp_path / "sub",
+            "standardized",
+            "available",
+            tmp_path / "sub",
         )
         written = self._perf_results(sub_dir)
         assert isinstance(written["responses"], list)

tests/submission_checker/test_checker.py

@@ -366,8 +366,14 @@ def _make_run_metadata(concurrency: int) -> dict:
     }
     for group in ("ttft", "tpot", "request"):
         for stat, val in (
-            ("min", 1.0), ("average", 2.0), ("p50", 2.0), ("p90", 3.0),
-            ("p95", 3.5), ("p99", 4.0), ("p999", 4.5), ("max", 5.0),
+            ("min", 1.0),
+            ("average", 2.0),
+            ("p50", 2.0),
+            ("p90", 3.0),
+            ("p95", 3.5),
+            ("p99", 4.0),
+            ("p999", 4.5),
+            ("max", 5.0),
         ):
             md[f"measured_latency_{group}_{stat}"] = val
     return md
@@ -560,6 +566,49 @@ def test_tps_utilization_out_of_tolerance_errors(self, tmp_path):
         report = _check(root)
         assert _errors(report, "tps-utilization")
 
+    def _add_curve(self, root, system_id, model, values: dict[int, tuple[float, float]]):
+        """Add a second <system_id>/<model> pareto curve with per-point (tps, util)."""
+        (root / "systems" / f"{system_id}.json").write_text(json.dumps(_SYSTEM_DESC))
+        model_dir = root / "pareto" / system_id / model
+        (model_dir / "points").mkdir(parents=True)
+        results_dir = model_dir / "results"
+        results_dir.mkdir(parents=True)
+        for c, (tps, util) in values.items():
+            (model_dir / "points" / f"point_{c}.yaml").write_text(yaml.dump(_make_run_yaml(c)))
+            rd = results_dir / f"point_{c}"
+            rd.mkdir()
+            (rd / "results_summary.json").write_text(json.dumps(_SUMMARY))
+            (rd / "config.yaml").write_text(yaml.dump({"concurrency": c}))
+            md = _make_run_metadata(c)
+            md["system_tps"], md["tps_utilization"] = tps, util
+            (rd / "run_metadata.json").write_text(json.dumps(md))
+            acc = rd / "accuracy"
+            acc.mkdir()
+            (acc / "results.json").write_text(json.dumps(_ACCURACY))
+
+    def test_tps_utilization_normalized_per_curve(self, tmp_path):
+        """Each system/model curve normalizes to its OWN peak, not a submission-wide max.
+
+        Regression test: a small system (peak 200) alongside a large one (peak 2000)
+        must not be forced to divide by the large system's peak. With the old
+        submission-wide max this errored on every small-system point.
+        """
+        root = _build_submission(tmp_path, system_id="sys-small", concurrencies=[16, 38])
+        self._set_tps(root, {16: (100.0, 0.5), 38: (200.0, 1.0)})  # own peak 200
+        self._add_curve(root, "sys-big", "llama3-70b", {16: (1000.0, 0.5), 38: (2000.0, 1.0)})
+        report = _check(root)
+        assert not _errors(report, "tps-utilization")
+        assert any(r.rule == "tps-utilization" for r in report.results)
+
+    def test_tps_utilization_per_curve_detects_error(self, tmp_path):
+        """A wrongly-normalized value is still caught within its own curve."""
+        root = _build_submission(tmp_path, system_id="sys-small", concurrencies=[16, 38])
+        self._set_tps(root, {16: (100.0, 0.5), 38: (200.0, 1.0)})  # correct
+        # sys-big point 16 expects 0.5 but stores 0.9 (off by 0.4 > tol)
+        self._add_curve(root, "sys-big", "llama3-70b", {16: (1000.0, 0.9), 38: (2000.0, 1.0)})
+        report = _check(root)
+        assert _errors(report, "tps-utilization")
+
     def test_missing_config_yaml(self, tmp_path):
         """result-file-present error when config.yaml is absent from a result dir."""
         root = _build_submission(tmp_path)

tests/submission_checker/test_checks_aggregate.py

@@ -51,7 +51,10 @@ def _config_with_dataset(dataset: str, concurrency: int = 64) -> PointConfig:
     return PointConfig(
         concurrency=concurrency,
         dataset=dataset,
-        runtime_settings=RuntimeSettings(min_duration_ms=1_200_000, runtime=RuntimeSettings.Runtime(scheduler_random_seed=42, dataloader_random_seed=42)),
+        runtime_settings=RuntimeSettings(
+            min_duration_ms=1_200_000,
+            runtime=RuntimeSettings.Runtime(scheduler_random_seed=42, dataloader_random_seed=42),
+        ),
     )
 
 
@@ -273,7 +276,12 @@ def test_tps_per_user_zero_concurrency_errors(self, tmp_path):
         config = PointConfig(
             concurrency=0,
             dataset="mlperf-perf-dataset-v1",
-            runtime_settings=RuntimeSettings(min_duration_ms=1_200_000, runtime=RuntimeSettings.Runtime(scheduler_random_seed=42, dataloader_random_seed=42)),
+            runtime_settings=RuntimeSettings(
+                min_duration_ms=1_200_000,
+                runtime=RuntimeSettings.Runtime(
+                    scheduler_random_seed=42, dataloader_random_seed=42
+                ),
+            ),
         )
         run_result = PointResult.model_validate(
             {"config": config, "summary": _summary(), "yaml_path": tmp_path / "run_64.yaml"},
@@ -439,12 +447,16 @@ def test_inconsistent_datasets(self, tmp_path):
         c1 = PointConfig(
             concurrency=64,
             dataset="open_orca",
-            runtime_settings=RuntimeSettings(runtime=RuntimeSettings.Runtime(scheduler_random_seed=42, dataloader_random_seed=42)),
+            runtime_settings=RuntimeSettings(
+                runtime=RuntimeSettings.Runtime(scheduler_random_seed=42, dataloader_random_seed=42)
+            ),
         )
         c2 = PointConfig(
             concurrency=128,
             dataset="cnn_dailymail",
-            runtime_settings=RuntimeSettings(runtime=RuntimeSettings.Runtime(scheduler_random_seed=42, dataloader_random_seed=42)),
+            runtime_settings=RuntimeSettings(
+                runtime=RuntimeSettings.Runtime(scheduler_random_seed=42, dataloader_random_seed=42)
+            ),
         )
         s = _summary()
         ctx = _model_ctx(tmp_path, loaded_points=[(c1, s), (c2, s)])
@@ -565,14 +577,11 @@ def test_scalar_score_gated_as_single_metric_passes(self, tmp_path):
             for r in ctx._check_results
         )
         assert any(
-            r.rule == "accuracy-gate"
-            and r.severity == Severity.INFO
-            and "exact_match" in r.message
+            r.rule == "accuracy-gate" and r.severity == Severity.INFO and "exact_match" in r.message
             for r in ctx._check_results
         )
         assert not any(
-            r.rule == "accuracy-gate" and r.severity == Severity.ERROR
-            for r in ctx._check_results
+            r.rule == "accuracy-gate" and r.severity == Severity.ERROR for r in ctx._check_results
         )
 
     def test_scalar_score_gated_as_single_metric_fails(self, tmp_path):
@@ -592,8 +601,7 @@ def test_scalar_score_not_mapped_for_multimetric_model(self, tmp_path):
         ar = AccuracyResult({"cnn_dailymail::llama3_8b": {"num_samples": 13368, "score": 39.0}})
         ctx = _model_ctx(tmp_path, accuracy_result=ar, model_name="Llama-3_1-8B-Instruct")
         assert not any(
-            r.rule == "accuracy-gate" and "unnamed scalar" in r.message
-            for r in ctx._check_results
+            r.rule == "accuracy-gate" and "unnamed scalar" in r.message for r in ctx._check_results
         )
         assert not any(
             r.rule == "accuracy-gate" and r.severity in (Severity.INFO, Severity.ERROR)
@@ -656,6 +664,8 @@ def test_multidataset_aggregate_score_below_threshold_fails(self, tmp_path):
         ctx = _model_ctx(tmp_path, accuracy_result=ar, model_name="gpt-oss-120b")
         # mean 0.50 → 50% < 82.2987 → error
         assert any(
-            r.rule == "accuracy-gate" and r.severity == Severity.ERROR and "exact_match" in r.message
+            r.rule == "accuracy-gate"
+            and r.severity == Severity.ERROR
+            and "exact_match" in r.message
             for r in ctx._check_results
         )