From 77f0899c7e37333f24bd4f036b6030be4af4372a Mon Sep 17 00:00:00 2001 From: aas008 Date: Tue, 6 Oct 2026 16:30:53 -0400 Subject: [PATCH 01/10] feat(manual-import): add turn_index column to manual run CSV output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add turn_index as empty string for aggregate rows so manually-imported CSVs are schema-compatible with the dashboard's per-turn logic (PR #101). Per-turn rows (turn_index=0/1/2) are generated by the Forge pipeline when request-level turn data is preserved in the benchmark JSON. Update README: 52 → 53 columns, add turn_index to column table (#7), clarify that manual import produces aggregate rows only. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/README.md | 97 ++++++++++--------- .../vllm/import_manual_runs_json_v2.py | 1 + 2 files changed, 50 insertions(+), 48 deletions(-) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index e5e17d7..e20a24d 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -112,7 +112,7 @@ python import_manual_runs_json_v2.py \ --csv-file "gemma-multiturn.csv" ``` -> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. +> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. `turn_index` is always empty in manual-import output (aggregate rows only) — per-turn rows are generated by the Forge pipeline when `turn_index` data is available in the benchmark JSON. ## Appending to Consolidated Dashboard @@ -124,7 +124,7 @@ tail -n +2 my-benchmark.csv >> ../../../consolidated_dashboard.csv ## Output CSV Columns -The script outputs 52 columns compatible with the performance dashboard: +The script outputs 53 columns compatible with the performance dashboard: | # | Column | Description | | --- | ------------------------- | ---------------------------------------------------- | @@ -134,52 +134,53 @@ The script outputs 52 columns compatible with the performance dashboard: | 4 | `version` | Framework version | | 5 | `prompt toks` | Configured prompt token count | | 6 | `output toks` | Configured output token count | -| 7 | `TP` | Tensor parallelism size | -| 8 | `measured concurrency` | Actual measured concurrency | -| 9 | `intended concurrency` | Requested concurrency (streams) | -| 10 | `measured rps` | Requests per second | -| 11 | `output_tok/sec` | Output tokens per second | -| 12 | `total_tok/sec` | Total tokens per second | -| 13 | `prompt_token_count_mean` | Mean prompt token count | -| 14 | `prompt_token_count_p99` | P99 prompt token count | -| 15 | `output_token_count_mean` | Mean output token count | -| 16 | `output_token_count_p99` | P99 output token count | -| 17 | `ttft_median` | Time to first token - median (ms) | -| 18 | `ttft_p95` | Time to first token - P95 (ms) | -| 19 | `ttft_p1` | Time to first token - P1 (ms) | -| 20 | `ttft_p999` | Time to first token - P99.9 (ms) | -| 21 | `tpot_median` | Time per output token - median (ms) | -| 22 | `tpot_p95` | Time per output token - P95 (ms) | -| 23 | `tpot_p99` | Time per output token - P99 (ms) | -| 24 | `tpot_p999` | Time per output token - P99.9 (ms) | -| 25 | `tpot_p1` | Time per output token - P1 (ms) | -| 26 | `itl_median` | Inter-token latency - median (ms) | -| 27 | `itl_p95` | Inter-token latency - P95 (ms) | -| 28 | `itl_p999` | Inter-token latency - P99.9 (ms) | -| 29 | `itl_p1` | Inter-token latency - P1 (ms) | -| 30 | `request_latency_median` | End-to-end request latency - median (s) | -| 31 | `request_latency_min` | End-to-end request latency - minimum (s) | -| 32 | `request_latency_max` | End-to-end request latency - maximum (s) | -| 33 | `successful_requests` | Number of successful requests | -| 34 | `errored_requests` | Number of errored requests | -| 35 | `uuid` | Unique benchmark run ID | -| 36 | `ttft_mean` | Time to first token - mean (ms) | -| 37 | `ttft_p99` | Time to first token - P99 (ms) | -| 38 | `itl_mean` | Inter-token latency - mean (ms) | -| 39 | `itl_p99` | Inter-token latency - P99 (ms) | -| 40 | `runtime_args` | Server configuration arguments | -| 41 | `guidellm_start_time_ms` | Benchmark start time (epoch ms) | -| 42 | `guidellm_end_time_ms` | Benchmark end time (epoch ms) | -| 43 | `image_tag` | Container image used | -| 44 | `guidellm_version` | guidellm version used | -| 45 | `DP` | Data parallelism size (empty for TP runs) | -| 46 | `dataset` | Real dataset name (empty for synthetic runs) | -| 47 | `spec_decoding` | Speculative decoding method (empty if none) | -| 48 | `prefix_caching` | Prefix caching status (`yes`, `no`, or empty) | -| 49 | `turns` | Conversation turns for multiturn benchmarks | -| 50 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | -| 51 | `prefix_count` | Prefix count (auto-detected from JSON) | -| 52 | `request_type` | GuideLLM API endpoint type (auto-detected from JSON) | +| 7 | `turn_index` | Per-turn breakdown index (empty for aggregate rows) | +| 9 | `TP` | Tensor parallelism size | +| 9 | `measured concurrency` | Actual measured concurrency | +| 10 | `intended concurrency` | Requested concurrency (streams) | +| 11 | `measured rps` | Requests per second | +| 12 | `output_tok/sec` | Output tokens per second | +| 13 | `total_tok/sec` | Total tokens per second | +| 14 | `prompt_token_count_mean` | Mean prompt token count | +| 15 | `prompt_token_count_p99` | P99 prompt token count | +| 16 | `output_token_count_mean` | Mean output token count | +| 17 | `output_token_count_p99` | P99 output token count | +| 18 | `ttft_median` | Time to first token - median (ms) | +| 19 | `ttft_p95` | Time to first token - P95 (ms) | +| 20 | `ttft_p1` | Time to first token - P1 (ms) | +| 21 | `ttft_p999` | Time to first token - P99.9 (ms) | +| 22 | `tpot_median` | Time per output token - median (ms) | +| 23 | `tpot_p95` | Time per output token - P95 (ms) | +| 24 | `tpot_p99` | Time per output token - P99 (ms) | +| 25 | `tpot_p999` | Time per output token - P99.9 (ms) | +| 26 | `tpot_p1` | Time per output token - P1 (ms) | +| 27 | `itl_median` | Inter-token latency - median (ms) | +| 28 | `itl_p95` | Inter-token latency - P95 (ms) | +| 29 | `itl_p999` | Inter-token latency - P99.9 (ms) | +| 30 | `itl_p1` | Inter-token latency - P1 (ms) | +| 31 | `request_latency_median` | End-to-end request latency - median (s) | +| 32 | `request_latency_min` | End-to-end request latency - minimum (s) | +| 33 | `request_latency_max` | End-to-end request latency - maximum (s) | +| 34 | `successful_requests` | Number of successful requests | +| 35 | `errored_requests` | Number of errored requests | +| 36 | `uuid` | Unique benchmark run ID | +| 37 | `ttft_mean` | Time to first token - mean (ms) | +| 38 | `ttft_p99` | Time to first token - P99 (ms) | +| 39 | `itl_mean` | Inter-token latency - mean (ms) | +| 40 | `itl_p99` | Inter-token latency - P99 (ms) | +| 41 | `runtime_args` | Server configuration arguments | +| 42 | `guidellm_start_time_ms` | Benchmark start time (epoch ms) | +| 43 | `guidellm_end_time_ms` | Benchmark end time (epoch ms) | +| 44 | `image_tag` | Container image used | +| 45 | `guidellm_version` | guidellm version used | +| 46 | `DP` | Data parallelism size (empty for TP runs) | +| 47 | `dataset` | Real dataset name (empty for synthetic runs) | +| 48 | `spec_decoding` | Speculative decoding method (empty if none) | +| 49 | `prefix_caching` | Prefix caching status (`yes`, `no`, or empty) | +| 50 | `turns` | Conversation turns for multiturn benchmarks | +| 51 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | +| 52 | `prefix_count` | Prefix count (auto-detected from JSON) | +| 53 | `request_type` | GuideLLM API endpoint type (auto-detected from JSON) | ## Notes diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index 9058c33..d30ceb6 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -205,6 +205,7 @@ def get_percentile(metrics_dict, key): "dataset": dataset, "spec_decoding": spec_decoding, "prefix_caching": prefix_caching, + "turn_index": "", "turns": turns, "prefix_tokens": detected_prefix_tokens if detected_prefix_tokens is not None From cb3ee0d8a2b5e7fcbdf95e7973a37cdef1899f21 Mon Sep 17 00:00:00 2001 From: aas008 Date: Tue, 6 Oct 2026 16:32:45 -0400 Subject: [PATCH 02/10] fix: correct turn_index position in README column table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Row numbering was corrupted by the previous renumber script — TP and measured concurrency both landed at row 9. Fix: turn_index=7, TP=8, measured concurrency=9, continuing sequentially to request_type=53. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index e20a24d..949dd45 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -135,7 +135,7 @@ The script outputs 53 columns compatible with the performance dashboard: | 5 | `prompt toks` | Configured prompt token count | | 6 | `output toks` | Configured output token count | | 7 | `turn_index` | Per-turn breakdown index (empty for aggregate rows) | -| 9 | `TP` | Tensor parallelism size | +| 8 | `TP` | Tensor parallelism size | | 9 | `measured concurrency` | Actual measured concurrency | | 10 | `intended concurrency` | Requested concurrency (streams) | | 11 | `measured rps` | Requests per second | From e2f4d3c3c1efa956cbfa65872840f0b49e16c44b Mon Sep 17 00:00:00 2001 From: aas008 Date: Tue, 6 Oct 2026 16:35:09 -0400 Subject: [PATCH 03/10] fix: move turn_index adjacent to turns in README column table The script emits turn_index right before turns in the output dict. The README should reflect actual output order, not the RHAIIS schema field position (which pandas aligns by name on concat anyway). turn_index: #49, turns: #50, prefix_tokens: #51, prefix_count: #52, request_type: #53. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/README.md | 86 +++++++++++++++--------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index 949dd45..11a9448 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -134,49 +134,49 @@ The script outputs 53 columns compatible with the performance dashboard: | 4 | `version` | Framework version | | 5 | `prompt toks` | Configured prompt token count | | 6 | `output toks` | Configured output token count | -| 7 | `turn_index` | Per-turn breakdown index (empty for aggregate rows) | -| 8 | `TP` | Tensor parallelism size | -| 9 | `measured concurrency` | Actual measured concurrency | -| 10 | `intended concurrency` | Requested concurrency (streams) | -| 11 | `measured rps` | Requests per second | -| 12 | `output_tok/sec` | Output tokens per second | -| 13 | `total_tok/sec` | Total tokens per second | -| 14 | `prompt_token_count_mean` | Mean prompt token count | -| 15 | `prompt_token_count_p99` | P99 prompt token count | -| 16 | `output_token_count_mean` | Mean output token count | -| 17 | `output_token_count_p99` | P99 output token count | -| 18 | `ttft_median` | Time to first token - median (ms) | -| 19 | `ttft_p95` | Time to first token - P95 (ms) | -| 20 | `ttft_p1` | Time to first token - P1 (ms) | -| 21 | `ttft_p999` | Time to first token - P99.9 (ms) | -| 22 | `tpot_median` | Time per output token - median (ms) | -| 23 | `tpot_p95` | Time per output token - P95 (ms) | -| 24 | `tpot_p99` | Time per output token - P99 (ms) | -| 25 | `tpot_p999` | Time per output token - P99.9 (ms) | -| 26 | `tpot_p1` | Time per output token - P1 (ms) | -| 27 | `itl_median` | Inter-token latency - median (ms) | -| 28 | `itl_p95` | Inter-token latency - P95 (ms) | -| 29 | `itl_p999` | Inter-token latency - P99.9 (ms) | -| 30 | `itl_p1` | Inter-token latency - P1 (ms) | -| 31 | `request_latency_median` | End-to-end request latency - median (s) | -| 32 | `request_latency_min` | End-to-end request latency - minimum (s) | -| 33 | `request_latency_max` | End-to-end request latency - maximum (s) | -| 34 | `successful_requests` | Number of successful requests | -| 35 | `errored_requests` | Number of errored requests | -| 36 | `uuid` | Unique benchmark run ID | -| 37 | `ttft_mean` | Time to first token - mean (ms) | -| 38 | `ttft_p99` | Time to first token - P99 (ms) | -| 39 | `itl_mean` | Inter-token latency - mean (ms) | -| 40 | `itl_p99` | Inter-token latency - P99 (ms) | -| 41 | `runtime_args` | Server configuration arguments | -| 42 | `guidellm_start_time_ms` | Benchmark start time (epoch ms) | -| 43 | `guidellm_end_time_ms` | Benchmark end time (epoch ms) | -| 44 | `image_tag` | Container image used | -| 45 | `guidellm_version` | guidellm version used | -| 46 | `DP` | Data parallelism size (empty for TP runs) | -| 47 | `dataset` | Real dataset name (empty for synthetic runs) | -| 48 | `spec_decoding` | Speculative decoding method (empty if none) | -| 49 | `prefix_caching` | Prefix caching status (`yes`, `no`, or empty) | +| 7 | `TP` | Tensor parallelism size | +| 8 | `measured concurrency` | Actual measured concurrency | +| 9 | `intended concurrency` | Requested concurrency (streams) | +| 10 | `measured rps` | Requests per second | +| 11 | `output_tok/sec` | Output tokens per second | +| 12 | `total_tok/sec` | Total tokens per second | +| 13 | `prompt_token_count_mean` | Mean prompt token count | +| 14 | `prompt_token_count_p99` | P99 prompt token count | +| 15 | `output_token_count_mean` | Mean output token count | +| 16 | `output_token_count_p99` | P99 output token count | +| 17 | `ttft_median` | Time to first token - median (ms) | +| 18 | `ttft_p95` | Time to first token - P95 (ms) | +| 19 | `ttft_p1` | Time to first token - P1 (ms) | +| 20 | `ttft_p999` | Time to first token - P99.9 (ms) | +| 21 | `tpot_median` | Time per output token - median (ms) | +| 22 | `tpot_p95` | Time per output token - P95 (ms) | +| 23 | `tpot_p99` | Time per output token - P99 (ms) | +| 24 | `tpot_p999` | Time per output token - P99.9 (ms) | +| 25 | `tpot_p1` | Time per output token - P1 (ms) | +| 26 | `itl_median` | Inter-token latency - median (ms) | +| 27 | `itl_p95` | Inter-token latency - P95 (ms) | +| 28 | `itl_p999` | Inter-token latency - P99.9 (ms) | +| 29 | `itl_p1` | Inter-token latency - P1 (ms) | +| 30 | `request_latency_median` | End-to-end request latency - median (s) | +| 31 | `request_latency_min` | End-to-end request latency - minimum (s) | +| 32 | `request_latency_max` | End-to-end request latency - maximum (s) | +| 33 | `successful_requests` | Number of successful requests | +| 34 | `errored_requests` | Number of errored requests | +| 35 | `uuid` | Unique benchmark run ID | +| 36 | `ttft_mean` | Time to first token - mean (ms) | +| 37 | `ttft_p99` | Time to first token - P99 (ms) | +| 38 | `itl_mean` | Inter-token latency - mean (ms) | +| 39 | `itl_p99` | Inter-token latency - P99 (ms) | +| 40 | `runtime_args` | Server configuration arguments | +| 41 | `guidellm_start_time_ms` | Benchmark start time (epoch ms) | +| 42 | `guidellm_end_time_ms` | Benchmark end time (epoch ms) | +| 43 | `image_tag` | Container image used | +| 44 | `guidellm_version` | guidellm version used | +| 45 | `DP` | Data parallelism size (empty for TP runs) | +| 46 | `dataset` | Real dataset name (empty for synthetic runs) | +| 47 | `spec_decoding` | Speculative decoding method (empty if none) | +| 48 | `prefix_caching` | Prefix caching status (`yes`, `no`, or empty) | +| 49 | `turn_index` | Per-turn breakdown index (empty for aggregate rows) | | 50 | `turns` | Conversation turns for multiturn benchmarks | | 51 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | | 52 | `prefix_count` | Prefix count (auto-detected from JSON) | From e1baa5c7715474398e80661cb55ad0d349b76bdb Mon Sep 17 00:00:00 2001 From: aas008 Date: Tue, 6 Oct 2026 18:00:08 -0400 Subject: [PATCH 04/10] fix: add turn_index to fieldnames so it is written to CSV output The turn_index key was added to the result dict but missing from the fieldnames list passed to combined_df[fieldnames], so pandas silently dropped it. CSV output now has 53 columns as the README states. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/import_manual_runs_json_v2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index d30ceb6..9712de1 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -513,6 +513,7 @@ def main(): "dataset", "spec_decoding", "prefix_caching", + "turn_index", "turns", "prefix_tokens", "prefix_count", From aedc0e72e1793c9391cf9da515ad9657aed00d1a Mon Sep 17 00:00:00 2001 From: aas008 Date: Wed, 7 Oct 2026 11:37:38 -0400 Subject: [PATCH 05/10] feat(manual-import): generate per-turn rows from request-level turn_index data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the guidellm JSON contains requests.successful with turn_index fields (multi-turn benchmarks), the script now emits: - one aggregate row per concurrency level (turn_index = empty, unchanged) - one per-turn row per turn per concurrency level (turn_index = 0, 1, 2, …) Per-turn rows carry the same metadata as the aggregate row plus per-turn stats: TTFT/ITL/TPOT median+p95+p99+p1+p999+mean, request latency, output throughput, token counts. Benchmarks without turn_index data are unaffected. Add _percentile() helper and extract_per_turn_rows() function. Update README to clarify that per-turn rows are now emitted automatically. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/README.md | 2 +- .../vllm/import_manual_runs_json_v2.py | 124 +++++++++++++++++- 2 files changed, 124 insertions(+), 2 deletions(-) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index 11a9448..5d89f52 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -112,7 +112,7 @@ python import_manual_runs_json_v2.py \ --csv-file "gemma-multiturn.csv" ``` -> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. `turn_index` is always empty in manual-import output (aggregate rows only) — per-turn rows are generated by the Forge pipeline when `turn_index` data is available in the benchmark JSON. +> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. For multi-turn benchmarks, the script emits one aggregate row per concurrency level (`turn_index` = empty) **plus** one per-turn row per turn per concurrency level (`turn_index` = 0, 1, 2, …) when `requests.successful` contains `turn_index` data. Non-multi-turn benchmarks emit aggregate rows only. ## Appending to Consolidated Dashboard diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index 9712de1..cdf56c4 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -13,6 +13,125 @@ import pandas as pd +def _percentile(values, p): + """Return the p-th percentile of a sorted list, or None if empty.""" + if not values: + return None + values = sorted(values) + idx = (len(values) - 1) * p / 100 + lo, hi = int(idx), min(int(idx) + 1, len(values) - 1) + return values[lo] + (values[hi] - values[lo]) * (idx - lo) + + +def extract_per_turn_rows(benchmark, base_row): + """Extract per-turn breakdown rows from request-level data in a benchmark section. + + For each turn_index found in requests.successful, computes per-turn stats + (TTFT, ITL, TPOT, latency, throughput) and returns one row dict per turn. + Returns an empty list if no turn_index data is present. + + Args: + benchmark: A single benchmark dict from the guidellm JSON. + base_row: The aggregate row dict produced by process_benchmark_section — + used as the metadata template for per-turn rows. + + Returns: + list[dict]: Per-turn row dicts ready to append to all_run_data. + """ + requests = benchmark.get("requests", {}) + successful = requests.get("successful", []) + errored = requests.get("errored", []) + + if not successful and not errored: + return [] + + # Discover all turn indices from both buckets + all_turns = set() + for req in successful + errored: + info = req.get("info", {}) + ti = info.get("turn_index") + if ti is not None: + try: + all_turns.add(str(int(float(ti)))) + except (ValueError, TypeError): + pass + + if not all_turns: + return [] + + # Group successful requests by turn_index + by_turn = {} + for req in successful: + info = req.get("info", {}) + ti = info.get("turn_index") + if ti is None: + continue + try: + key = str(int(float(ti))) + except (ValueError, TypeError): + continue + by_turn.setdefault(key, []).append(req) + + # Group errored requests by turn_index for error counts + err_by_turn = {} + for req in errored: + info = req.get("info", {}) + ti = info.get("turn_index") + if ti is None: + continue + try: + key = str(int(float(ti))) + except (ValueError, TypeError): + continue + err_by_turn.setdefault(key, []).append(req) + + per_turn_rows = [] + for turn_key in sorted(all_turns, key=lambda x: int(x)): + reqs = by_turn.get(turn_key, []) + n_err = len(err_by_turn.get(turn_key, [])) + + ttfts = [r.get("time_to_first_token_ms") for r in reqs if r.get("time_to_first_token_ms") is not None] + itls = [r.get("inter_token_latency_ms") for r in reqs if r.get("inter_token_latency_ms") is not None] + tpots = [r.get("time_per_output_token_ms") for r in reqs if r.get("time_per_output_token_ms") is not None] + lats = [r.get("request_latency") for r in reqs if r.get("request_latency") is not None] + out_tps = [r.get("output_tokens_per_second") for r in reqs if r.get("output_tokens_per_second") is not None] + total_tps = [r.get("tokens_per_second") for r in reqs if r.get("tokens_per_second") is not None] + out_toks = [r.get("output_tokens") for r in reqs if r.get("output_tokens") is not None] + prompt_toks = [r.get("prompt_tokens") for r in reqs if r.get("prompt_tokens") is not None] + + row = dict(base_row) + row["turn_index"] = int(turn_key) + row["successful_requests"] = len(reqs) + row["errored_requests"] = n_err + row["ttft_median"] = _percentile(ttfts, 50) + row["ttft_p95"] = _percentile(ttfts, 95) + row["ttft_p99"] = _percentile(ttfts, 99) + row["ttft_p1"] = _percentile(ttfts, 1) + row["ttft_p999"] = _percentile(ttfts, 99.9) + row["ttft_mean"] = (sum(ttfts) / len(ttfts)) if ttfts else None + row["itl_median"] = _percentile(itls, 50) + row["itl_p95"] = _percentile(itls, 95) + row["itl_p99"] = _percentile(itls, 99) + row["itl_p1"] = _percentile(itls, 1) + row["itl_p999"] = _percentile(itls, 99.9) + row["itl_mean"] = (sum(itls) / len(itls)) if itls else None + row["tpot_median"] = _percentile(tpots, 50) + row["tpot_p95"] = _percentile(tpots, 95) + row["tpot_p99"] = _percentile(tpots, 99) + row["tpot_p1"] = _percentile(tpots, 1) + row["tpot_p999"] = _percentile(tpots, 99.9) + row["request_latency_median"] = _percentile(lats, 50) + row["request_latency_min"] = min(lats) if lats else None + row["request_latency_max"] = max(lats) if lats else None + row["output_tok/sec"] = (sum(out_tps) / len(out_tps)) if out_tps else None + row["total_tok/sec"] = (sum(total_tps) / len(total_tps)) if total_tps else None + row["output_token_count_mean"] = (sum(out_toks) / len(out_toks)) if out_toks else None + row["prompt_token_count_mean"] = (sum(prompt_toks) / len(prompt_toks)) if prompt_toks else None + per_turn_rows.append(row) + + return per_turn_rows + + def process_benchmark_section( benchmark, accelerator, @@ -329,11 +448,14 @@ def parse_guidellm_json( ) if row_data: all_run_data.append(row_data) + per_turn = extract_per_turn_rows(benchmark, row_data) + all_run_data.extend(per_turn) streams = ( benchmark.get("config", {}).get("strategy", {}).get("streams", "?") ) + turn_msg = f", {len(per_turn)} per-turn rows" if per_turn else "" print( - f" Processed benchmark {i + 1}/{len(benchmarks)} (streams={streams})" + f" Processed benchmark {i + 1}/{len(benchmarks)} (streams={streams}{turn_msg})" ) if all_run_data: From 9494fa36c53042020c7be910765335d17a2ec477 Mon Sep 17 00:00:00 2001 From: aas008 Date: Wed, 7 Oct 2026 11:39:58 -0400 Subject: [PATCH 06/10] refactor: align per-turn stats with Forge PR #215 implementation - _percentile(): use same k/f/c linear interpolation as Forge - Add _median(), _mean(), _sorted_field() helpers matching Forge's _turn_metric_values() helpers exactly - Add len(all_turns) <= 1 guard: skip per-turn extraction for single-turn runs (matches Forge PR #215 behavior) - Replace _percentile(50) calls with _median() for correctness Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../vllm/import_manual_runs_json_v2.py | 105 ++++++++++-------- 1 file changed, 61 insertions(+), 44 deletions(-) diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index cdf56c4..f59fde5 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -14,13 +14,32 @@ def _percentile(values, p): - """Return the p-th percentile of a sorted list, or None if empty.""" + """Return the p-th percentile of a pre-sorted list using linear interpolation. + + Matches the implementation in Forge PR #215 _percentile(). + """ if not values: return None - values = sorted(values) - idx = (len(values) - 1) * p / 100 - lo, hi = int(idx), min(int(idx) + 1, len(values) - 1) - return values[lo] + (values[hi] - values[lo]) * (idx - lo) + k = (len(values) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(values) - 1) + return values[f] + (k - f) * (values[c] - values[f]) + + +def _median(values): + """Return the median of a pre-sorted list. Matches Forge PR #215 _median().""" + n = len(values) + if not n: + return None + return values[n // 2] if n % 2 else (values[n // 2 - 1] + values[n // 2]) / 2 + + +def _mean(values): + return sum(values) / len(values) if values else None + + +def _sorted_field(reqs, field): + return sorted(r[field] for r in reqs if r.get(field) is not None) def extract_per_turn_rows(benchmark, base_row): @@ -56,14 +75,14 @@ def extract_per_turn_rows(benchmark, base_row): except (ValueError, TypeError): pass - if not all_turns: + # Match Forge PR #215: skip extraction for single-turn runs + if len(all_turns) <= 1: return [] - # Group successful requests by turn_index + # Group requests by turn_index by_turn = {} for req in successful: - info = req.get("info", {}) - ti = info.get("turn_index") + ti = req.get("info", {}).get("turn_index") if ti is None: continue try: @@ -72,11 +91,9 @@ def extract_per_turn_rows(benchmark, base_row): continue by_turn.setdefault(key, []).append(req) - # Group errored requests by turn_index for error counts err_by_turn = {} for req in errored: - info = req.get("info", {}) - ti = info.get("turn_index") + ti = req.get("info", {}).get("turn_index") if ti is None: continue try: @@ -90,43 +107,43 @@ def extract_per_turn_rows(benchmark, base_row): reqs = by_turn.get(turn_key, []) n_err = len(err_by_turn.get(turn_key, [])) - ttfts = [r.get("time_to_first_token_ms") for r in reqs if r.get("time_to_first_token_ms") is not None] - itls = [r.get("inter_token_latency_ms") for r in reqs if r.get("inter_token_latency_ms") is not None] - tpots = [r.get("time_per_output_token_ms") for r in reqs if r.get("time_per_output_token_ms") is not None] - lats = [r.get("request_latency") for r in reqs if r.get("request_latency") is not None] - out_tps = [r.get("output_tokens_per_second") for r in reqs if r.get("output_tokens_per_second") is not None] - total_tps = [r.get("tokens_per_second") for r in reqs if r.get("tokens_per_second") is not None] - out_toks = [r.get("output_tokens") for r in reqs if r.get("output_tokens") is not None] - prompt_toks = [r.get("prompt_tokens") for r in reqs if r.get("prompt_tokens") is not None] + ttft = _sorted_field(reqs, "time_to_first_token_ms") + itl = _sorted_field(reqs, "inter_token_latency_ms") + tpot = _sorted_field(reqs, "time_per_output_token_ms") + lat = _sorted_field(reqs, "request_latency") + out_tps = _sorted_field(reqs, "output_tokens_per_second") + total_tps = _sorted_field(reqs, "tokens_per_second") + out_toks = _sorted_field(reqs, "output_tokens") + prompt_toks = _sorted_field(reqs, "prompt_tokens") row = dict(base_row) row["turn_index"] = int(turn_key) row["successful_requests"] = len(reqs) row["errored_requests"] = n_err - row["ttft_median"] = _percentile(ttfts, 50) - row["ttft_p95"] = _percentile(ttfts, 95) - row["ttft_p99"] = _percentile(ttfts, 99) - row["ttft_p1"] = _percentile(ttfts, 1) - row["ttft_p999"] = _percentile(ttfts, 99.9) - row["ttft_mean"] = (sum(ttfts) / len(ttfts)) if ttfts else None - row["itl_median"] = _percentile(itls, 50) - row["itl_p95"] = _percentile(itls, 95) - row["itl_p99"] = _percentile(itls, 99) - row["itl_p1"] = _percentile(itls, 1) - row["itl_p999"] = _percentile(itls, 99.9) - row["itl_mean"] = (sum(itls) / len(itls)) if itls else None - row["tpot_median"] = _percentile(tpots, 50) - row["tpot_p95"] = _percentile(tpots, 95) - row["tpot_p99"] = _percentile(tpots, 99) - row["tpot_p1"] = _percentile(tpots, 1) - row["tpot_p999"] = _percentile(tpots, 99.9) - row["request_latency_median"] = _percentile(lats, 50) - row["request_latency_min"] = min(lats) if lats else None - row["request_latency_max"] = max(lats) if lats else None - row["output_tok/sec"] = (sum(out_tps) / len(out_tps)) if out_tps else None - row["total_tok/sec"] = (sum(total_tps) / len(total_tps)) if total_tps else None - row["output_token_count_mean"] = (sum(out_toks) / len(out_toks)) if out_toks else None - row["prompt_token_count_mean"] = (sum(prompt_toks) / len(prompt_toks)) if prompt_toks else None + row["ttft_median"] = _median(ttft) + row["ttft_p95"] = _percentile(ttft, 95) + row["ttft_p99"] = _percentile(ttft, 99) + row["ttft_p1"] = _percentile(ttft, 1) + row["ttft_p999"] = _percentile(ttft, 99.9) + row["ttft_mean"] = _mean(ttft) + row["itl_median"] = _median(itl) + row["itl_p95"] = _percentile(itl, 95) + row["itl_p99"] = _percentile(itl, 99) + row["itl_p1"] = _percentile(itl, 1) + row["itl_p999"] = _percentile(itl, 99.9) + row["itl_mean"] = _mean(itl) + row["tpot_median"] = _median(tpot) + row["tpot_p95"] = _percentile(tpot, 95) + row["tpot_p99"] = _percentile(tpot, 99) + row["tpot_p1"] = _percentile(tpot, 1) + row["tpot_p999"] = _percentile(tpot, 99.9) + row["request_latency_median"] = _median(lat) + row["request_latency_min"] = min(lat) if lat else None + row["request_latency_max"] = max(lat) if lat else None + row["output_tok/sec"] = _mean(out_tps) + row["total_tok/sec"] = _mean(total_tps) + row["output_token_count_mean"] = _mean(out_toks) + row["prompt_token_count_mean"] = _mean(prompt_toks) per_turn_rows.append(row) return per_turn_rows From b39d1cf5debe3fd7108da8098479dc02c1de81d8 Mon Sep 17 00:00:00 2001 From: aas008 Date: Wed, 7 Oct 2026 14:39:16 -0400 Subject: [PATCH 07/10] feat(manual-import): add optional --mlflow-run-id and --mlflow-experiment-id flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When provided, these values are stamped on every output row and included in the fieldnames list so the dashboard's Filtered Data table renders a clickable MLflow artifact link (added to the schema in PR #83). Both flags are optional and default to empty string — no breaking change. Closes #107 Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/README.md | 6 +++++- .../vllm/import_manual_runs_json_v2.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index 5d89f52..7c9e494 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -46,6 +46,8 @@ python import_manual_runs_json_v2.py \ | `--dataset` | No | Real dataset name (for real-dataset runs) | `mlperf-gpt-oss`, `sharegpt` | | `--spec-decoding` | No | Speculative decoding method used | `eagle3`, `ngram` | | `--prefix-caching` | No | Whether prefix caching was enabled | `yes`, `no` | +| `--mlflow-run-id` | No | MLflow run UUID — enables clickable MLflow links in dashboard | `c6aa48a0d312448380621e1bf00a8a5d` | +| `--mlflow-experiment-id` | No | MLflow experiment ID (used with `--mlflow-run-id`) | `264` | ## Examples @@ -124,7 +126,7 @@ tail -n +2 my-benchmark.csv >> ../../../consolidated_dashboard.csv ## Output CSV Columns -The script outputs 53 columns compatible with the performance dashboard: +The script outputs 55 columns compatible with the performance dashboard: | # | Column | Description | | --- | ------------------------- | ---------------------------------------------------- | @@ -181,6 +183,8 @@ The script outputs 53 columns compatible with the performance dashboard: | 51 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | | 52 | `prefix_count` | Prefix count (auto-detected from JSON) | | 53 | `request_type` | GuideLLM API endpoint type (auto-detected from JSON) | +| 54 | `mlflow_run_id` | MLflow run UUID (from `--mlflow-run-id`, empty if not provided) | +| 55 | `mlflow_experiment_id` | MLflow experiment ID (from `--mlflow-experiment-id`, empty if not provided) | ## Notes diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index f59fde5..e1ec5ea 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -559,6 +559,19 @@ def main(): help="Whether prefix caching is enabled ('yes' or 'no'). " "Leave empty if not applicable.", ) + parser.add_argument( + "--mlflow-run-id", + default="", + help="MLflow run UUID (optional). When provided, the Filtered Data table " + "in the dashboard shows a clickable MLflow artifact link. " + "Example: c6aa48a0d312448380621e1bf00a8a5d", + ) + parser.add_argument( + "--mlflow-experiment-id", + default="", + help="MLflow experiment ID (optional, used together with --mlflow-run-id). " + "Example: 264", + ) parser.add_argument( "--csv-file", default="new_benchmarks.csv", @@ -593,6 +606,10 @@ def main(): ) if new_data_df is not None and not new_data_df.empty: + # Stamp MLflow identifiers on every row (empty string when not provided) + new_data_df["mlflow_run_id"] = args.mlflow_run_id + new_data_df["mlflow_experiment_id"] = args.mlflow_experiment_id + if os.path.exists(args.csv_file): print(f"Appending {len(new_data_df)} new rows to {args.csv_file}...") existing_df = pd.read_csv(args.csv_file) @@ -657,6 +674,8 @@ def main(): "prefix_tokens", "prefix_count", "request_type", + "mlflow_run_id", + "mlflow_experiment_id", ] for col in fieldnames: From 5ee1e2532d0284fb3ecdbb1b3258fe6e69525a81 Mon Sep 17 00:00:00 2001 From: aas008 Date: Thu, 15 Oct 2026 15:56:05 -0400 Subject: [PATCH 08/10] fix: address review findings in per-turn manual import Compute output/prompt token p99 per turn from request-level data instead of inheriting aggregate values, clear measured rps on per-turn rows (it is not computed per turn), use contextlib.suppress to satisfy ruff SIM105, and correct the README emission rule. Also filter per-turn breakdown rows out of the RHAIIS comparison loader in llmd_dashboard.py, mirroring the aggregate-row filter added to dashboard.py, so multi-turn runs are not double-counted. Co-Authored-By: Claude Code --- llmd_dashboard.py | 7 +++++++ manual_runs/scripts/vllm/README.md | 2 +- manual_runs/scripts/vllm/import_manual_runs_json_v2.py | 8 +++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/llmd_dashboard.py b/llmd_dashboard.py index 5e3db9d..693b8ff 100644 --- a/llmd_dashboard.py +++ b/llmd_dashboard.py @@ -1640,6 +1640,13 @@ def load_rhaiis_data(file_path: str = "consolidated_dashboard.csv") -> pd.DataFr for col in df.select_dtypes(include=["object"]).columns: df[col] = df[col].str.strip() # type: ignore[assignment] + # Keep only aggregate rows: per-turn breakdown rows (turn_index 0, 1, 2…) + # from multi-turn benchmarks would double-count runs in comparisons. + # Mirrors the aggregate-row filter in dashboard.py. + if "turn_index" in df.columns: + _ti = df["turn_index"].astype(str).str.strip() + df = df[df["turn_index"].isna() | (_ti == "") | (_ti == "nan")].copy() + # Assign profile based on prompt/output tokens (same as llmd_data) if "prompt toks" in df.columns and "output toks" in df.columns: df["profile"] = df.apply(assign_profile, axis=1) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index 7c9e494..894b0a0 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -114,7 +114,7 @@ python import_manual_runs_json_v2.py \ --csv-file "gemma-multiturn.csv" ``` -> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. For multi-turn benchmarks, the script emits one aggregate row per concurrency level (`turn_index` = empty) **plus** one per-turn row per turn per concurrency level (`turn_index` = 0, 1, 2, …) when `requests.successful` contains `turn_index` data. Non-multi-turn benchmarks emit aggregate rows only. +> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. For multi-turn benchmarks, the script emits one aggregate row per concurrency level (`turn_index` = empty) **plus** one per-turn row per turn per concurrency level (`turn_index` = 0, 1, 2, …) when `requests.successful`/`requests.errored` contain at least two distinct valid `turn_index` values. Other benchmarks emit aggregate rows only. ## Appending to Consolidated Dashboard diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index e1ec5ea..a16fbb3 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -7,6 +7,7 @@ """ import argparse +import contextlib import json import os @@ -70,10 +71,8 @@ def extract_per_turn_rows(benchmark, base_row): info = req.get("info", {}) ti = info.get("turn_index") if ti is not None: - try: + with contextlib.suppress(ValueError, TypeError): all_turns.add(str(int(float(ti)))) - except (ValueError, TypeError): - pass # Match Forge PR #215: skip extraction for single-turn runs if len(all_turns) <= 1: @@ -144,6 +143,9 @@ def extract_per_turn_rows(benchmark, base_row): row["total_tok/sec"] = _mean(total_tps) row["output_token_count_mean"] = _mean(out_toks) row["prompt_token_count_mean"] = _mean(prompt_toks) + row["output_token_count_p99"] = _percentile(out_toks, 99) + row["prompt_token_count_p99"] = _percentile(prompt_toks, 99) + row["measured rps"] = None per_turn_rows.append(row) return per_turn_rows From 58c353ddff521569e987585a8649a66c3f8ed80d Mon Sep 17 00:00:00 2001 From: aas008 Date: Thu, 15 Oct 2026 16:04:04 -0400 Subject: [PATCH 09/10] fix: resolve remaining pre-commit hook failures Rename the local requests dict to requests_data so bandit does not flag its .get() calls as requests-library usage without a timeout, split the _median docstring so its first line omits the function name (pydocstyle D402), realign the README tables for prettier, and add a type: ignore for the pandas-stubs str.strip() mismatch in the RHAIIS loader filter. Co-Authored-By: Claude Code --- llmd_dashboard.py | 2 +- manual_runs/scripts/vllm/README.md | 144 +++++++++--------- .../vllm/import_manual_runs_json_v2.py | 13 +- 3 files changed, 82 insertions(+), 77 deletions(-) diff --git a/llmd_dashboard.py b/llmd_dashboard.py index 693b8ff..f6ac845 100644 --- a/llmd_dashboard.py +++ b/llmd_dashboard.py @@ -1644,7 +1644,7 @@ def load_rhaiis_data(file_path: str = "consolidated_dashboard.csv") -> pd.DataFr # from multi-turn benchmarks would double-count runs in comparisons. # Mirrors the aggregate-row filter in dashboard.py. if "turn_index" in df.columns: - _ti = df["turn_index"].astype(str).str.strip() + _ti = df["turn_index"].astype(str).str.strip() # type: ignore[misc] df = df[df["turn_index"].isna() | (_ti == "") | (_ti == "nan")].copy() # Assign profile based on prompt/output tokens (same as llmd_data) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index 894b0a0..3f071c4 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -32,22 +32,22 @@ python import_manual_runs_json_v2.py \ ## Arguments -| Argument | Required | Description | Example | -| -------------------- | -------- | ----------------------------------------------- | --------------------------------------- | -| `json_file` | Yes | Path to guidellm JSON output file | `llama-70b-vllm.json` | -| `--model` | Yes | Model name (HuggingFace format) | `meta-llama/Llama-3.3-70B-Instruct` | -| `--version` | Yes | Framework/version identifier | `vLLM-0.13.0`, `TRT-LLM-1.2.0rc2.post1` | -| `--tp` | Yes | Tensor parallelism size | `4`, `8` | -| `--accelerator` | Yes | GPU/accelerator type | `H200`, `MI300X`, `TPU` | -| `--runtime-args` | Yes | Server runtime configuration | See examples below | -| `--image-tag` | Yes | Container image tag | `vllm/vllm-openai:v0.13.0` | -| `--guidellm-version` | Yes | guidellm version used | `v0.5.1`, `v0.3.0` | -| `--csv-file` | No | Output CSV path (default: `new_benchmarks.csv`) | `llama-70b-vllm.csv` | -| `--dataset` | No | Real dataset name (for real-dataset runs) | `mlperf-gpt-oss`, `sharegpt` | -| `--spec-decoding` | No | Speculative decoding method used | `eagle3`, `ngram` | -| `--prefix-caching` | No | Whether prefix caching was enabled | `yes`, `no` | -| `--mlflow-run-id` | No | MLflow run UUID — enables clickable MLflow links in dashboard | `c6aa48a0d312448380621e1bf00a8a5d` | -| `--mlflow-experiment-id` | No | MLflow experiment ID (used with `--mlflow-run-id`) | `264` | +| Argument | Required | Description | Example | +| ------------------------ | -------- | ------------------------------------------------------------- | --------------------------------------- | +| `json_file` | Yes | Path to guidellm JSON output file | `llama-70b-vllm.json` | +| `--model` | Yes | Model name (HuggingFace format) | `meta-llama/Llama-3.3-70B-Instruct` | +| `--version` | Yes | Framework/version identifier | `vLLM-0.13.0`, `TRT-LLM-1.2.0rc2.post1` | +| `--tp` | Yes | Tensor parallelism size | `4`, `8` | +| `--accelerator` | Yes | GPU/accelerator type | `H200`, `MI300X`, `TPU` | +| `--runtime-args` | Yes | Server runtime configuration | See examples below | +| `--image-tag` | Yes | Container image tag | `vllm/vllm-openai:v0.13.0` | +| `--guidellm-version` | Yes | guidellm version used | `v0.5.1`, `v0.3.0` | +| `--csv-file` | No | Output CSV path (default: `new_benchmarks.csv`) | `llama-70b-vllm.csv` | +| `--dataset` | No | Real dataset name (for real-dataset runs) | `mlperf-gpt-oss`, `sharegpt` | +| `--spec-decoding` | No | Speculative decoding method used | `eagle3`, `ngram` | +| `--prefix-caching` | No | Whether prefix caching was enabled | `yes`, `no` | +| `--mlflow-run-id` | No | MLflow run UUID — enables clickable MLflow links in dashboard | `c6aa48a0d312448380621e1bf00a8a5d` | +| `--mlflow-experiment-id` | No | MLflow experiment ID (used with `--mlflow-run-id`) | `264` | ## Examples @@ -128,62 +128,62 @@ tail -n +2 my-benchmark.csv >> ../../../consolidated_dashboard.csv The script outputs 55 columns compatible with the performance dashboard: -| # | Column | Description | -| --- | ------------------------- | ---------------------------------------------------- | -| 1 | `run` | Unique run identifier (`{accelerator}-{model}-{tp}`) | -| 2 | `accelerator` | GPU type (H200, MI300X, etc.) | -| 3 | `model` | Model name | -| 4 | `version` | Framework version | -| 5 | `prompt toks` | Configured prompt token count | -| 6 | `output toks` | Configured output token count | -| 7 | `TP` | Tensor parallelism size | -| 8 | `measured concurrency` | Actual measured concurrency | -| 9 | `intended concurrency` | Requested concurrency (streams) | -| 10 | `measured rps` | Requests per second | -| 11 | `output_tok/sec` | Output tokens per second | -| 12 | `total_tok/sec` | Total tokens per second | -| 13 | `prompt_token_count_mean` | Mean prompt token count | -| 14 | `prompt_token_count_p99` | P99 prompt token count | -| 15 | `output_token_count_mean` | Mean output token count | -| 16 | `output_token_count_p99` | P99 output token count | -| 17 | `ttft_median` | Time to first token - median (ms) | -| 18 | `ttft_p95` | Time to first token - P95 (ms) | -| 19 | `ttft_p1` | Time to first token - P1 (ms) | -| 20 | `ttft_p999` | Time to first token - P99.9 (ms) | -| 21 | `tpot_median` | Time per output token - median (ms) | -| 22 | `tpot_p95` | Time per output token - P95 (ms) | -| 23 | `tpot_p99` | Time per output token - P99 (ms) | -| 24 | `tpot_p999` | Time per output token - P99.9 (ms) | -| 25 | `tpot_p1` | Time per output token - P1 (ms) | -| 26 | `itl_median` | Inter-token latency - median (ms) | -| 27 | `itl_p95` | Inter-token latency - P95 (ms) | -| 28 | `itl_p999` | Inter-token latency - P99.9 (ms) | -| 29 | `itl_p1` | Inter-token latency - P1 (ms) | -| 30 | `request_latency_median` | End-to-end request latency - median (s) | -| 31 | `request_latency_min` | End-to-end request latency - minimum (s) | -| 32 | `request_latency_max` | End-to-end request latency - maximum (s) | -| 33 | `successful_requests` | Number of successful requests | -| 34 | `errored_requests` | Number of errored requests | -| 35 | `uuid` | Unique benchmark run ID | -| 36 | `ttft_mean` | Time to first token - mean (ms) | -| 37 | `ttft_p99` | Time to first token - P99 (ms) | -| 38 | `itl_mean` | Inter-token latency - mean (ms) | -| 39 | `itl_p99` | Inter-token latency - P99 (ms) | -| 40 | `runtime_args` | Server configuration arguments | -| 41 | `guidellm_start_time_ms` | Benchmark start time (epoch ms) | -| 42 | `guidellm_end_time_ms` | Benchmark end time (epoch ms) | -| 43 | `image_tag` | Container image used | -| 44 | `guidellm_version` | guidellm version used | -| 45 | `DP` | Data parallelism size (empty for TP runs) | -| 46 | `dataset` | Real dataset name (empty for synthetic runs) | -| 47 | `spec_decoding` | Speculative decoding method (empty if none) | -| 48 | `prefix_caching` | Prefix caching status (`yes`, `no`, or empty) | -| 49 | `turn_index` | Per-turn breakdown index (empty for aggregate rows) | -| 50 | `turns` | Conversation turns for multiturn benchmarks | -| 51 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | -| 52 | `prefix_count` | Prefix count (auto-detected from JSON) | -| 53 | `request_type` | GuideLLM API endpoint type (auto-detected from JSON) | -| 54 | `mlflow_run_id` | MLflow run UUID (from `--mlflow-run-id`, empty if not provided) | +| # | Column | Description | +| --- | ------------------------- | --------------------------------------------------------------------------- | +| 1 | `run` | Unique run identifier (`{accelerator}-{model}-{tp}`) | +| 2 | `accelerator` | GPU type (H200, MI300X, etc.) | +| 3 | `model` | Model name | +| 4 | `version` | Framework version | +| 5 | `prompt toks` | Configured prompt token count | +| 6 | `output toks` | Configured output token count | +| 7 | `TP` | Tensor parallelism size | +| 8 | `measured concurrency` | Actual measured concurrency | +| 9 | `intended concurrency` | Requested concurrency (streams) | +| 10 | `measured rps` | Requests per second | +| 11 | `output_tok/sec` | Output tokens per second | +| 12 | `total_tok/sec` | Total tokens per second | +| 13 | `prompt_token_count_mean` | Mean prompt token count | +| 14 | `prompt_token_count_p99` | P99 prompt token count | +| 15 | `output_token_count_mean` | Mean output token count | +| 16 | `output_token_count_p99` | P99 output token count | +| 17 | `ttft_median` | Time to first token - median (ms) | +| 18 | `ttft_p95` | Time to first token - P95 (ms) | +| 19 | `ttft_p1` | Time to first token - P1 (ms) | +| 20 | `ttft_p999` | Time to first token - P99.9 (ms) | +| 21 | `tpot_median` | Time per output token - median (ms) | +| 22 | `tpot_p95` | Time per output token - P95 (ms) | +| 23 | `tpot_p99` | Time per output token - P99 (ms) | +| 24 | `tpot_p999` | Time per output token - P99.9 (ms) | +| 25 | `tpot_p1` | Time per output token - P1 (ms) | +| 26 | `itl_median` | Inter-token latency - median (ms) | +| 27 | `itl_p95` | Inter-token latency - P95 (ms) | +| 28 | `itl_p999` | Inter-token latency - P99.9 (ms) | +| 29 | `itl_p1` | Inter-token latency - P1 (ms) | +| 30 | `request_latency_median` | End-to-end request latency - median (s) | +| 31 | `request_latency_min` | End-to-end request latency - minimum (s) | +| 32 | `request_latency_max` | End-to-end request latency - maximum (s) | +| 33 | `successful_requests` | Number of successful requests | +| 34 | `errored_requests` | Number of errored requests | +| 35 | `uuid` | Unique benchmark run ID | +| 36 | `ttft_mean` | Time to first token - mean (ms) | +| 37 | `ttft_p99` | Time to first token - P99 (ms) | +| 38 | `itl_mean` | Inter-token latency - mean (ms) | +| 39 | `itl_p99` | Inter-token latency - P99 (ms) | +| 40 | `runtime_args` | Server configuration arguments | +| 41 | `guidellm_start_time_ms` | Benchmark start time (epoch ms) | +| 42 | `guidellm_end_time_ms` | Benchmark end time (epoch ms) | +| 43 | `image_tag` | Container image used | +| 44 | `guidellm_version` | guidellm version used | +| 45 | `DP` | Data parallelism size (empty for TP runs) | +| 46 | `dataset` | Real dataset name (empty for synthetic runs) | +| 47 | `spec_decoding` | Speculative decoding method (empty if none) | +| 48 | `prefix_caching` | Prefix caching status (`yes`, `no`, or empty) | +| 49 | `turn_index` | Per-turn breakdown index (empty for aggregate rows) | +| 50 | `turns` | Conversation turns for multiturn benchmarks | +| 51 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | +| 52 | `prefix_count` | Prefix count (auto-detected from JSON) | +| 53 | `request_type` | GuideLLM API endpoint type (auto-detected from JSON) | +| 54 | `mlflow_run_id` | MLflow run UUID (from `--mlflow-run-id`, empty if not provided) | | 55 | `mlflow_experiment_id` | MLflow experiment ID (from `--mlflow-experiment-id`, empty if not provided) | ## Notes diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index a16fbb3..053c816 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -28,7 +28,10 @@ def _percentile(values, p): def _median(values): - """Return the median of a pre-sorted list. Matches Forge PR #215 _median().""" + """Return the median of a pre-sorted list. + + Matches the Forge PR #215 implementation. + """ n = len(values) if not n: return None @@ -58,9 +61,11 @@ def extract_per_turn_rows(benchmark, base_row): Returns: list[dict]: Per-turn row dicts ready to append to all_run_data. """ - requests = benchmark.get("requests", {}) - successful = requests.get("successful", []) - errored = requests.get("errored", []) + # Named requests_data (not requests) so bandit does not mistake the dict + # for the requests library (B113). + requests_data = benchmark.get("requests", {}) + successful = requests_data.get("successful", []) + errored = requests_data.get("errored", []) if not successful and not errored: return [] From e1cfb7bd6eeae28a40655b706a44d6550cfae595 Mon Sep 17 00:00:00 2001 From: aas008 Date: Tue, 20 Oct 2026 02:09:41 -0400 Subject: [PATCH 10/10] fix(manual-import): compute per-turn throughput over the measurement interval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-turn output_tok/sec and total_tok/sec previously averaged per-request rates, which reports a single request's rate rather than the turn's aggregate throughput — at concurrent load this understates the value by roughly the concurrency factor and is inconsistent with the GuideLLM "total" throughput the aggregate row reports in the same columns. Compute each turn's rate as the turn's tokens divided by the benchmark's scheduler measurement interval (the same window GuideLLM uses for its run-level total throughput), so per-turn values sum to the aggregate row. Document the semantics in the README per-turn note and add unit tests covering per-turn extraction (throughput interval math, per-turn p99, errored counts, single-turn guard, missing-duration fallback). Co-Authored-By: Claude Code --- manual_runs/scripts/vllm/README.md | 2 +- .../vllm/import_manual_runs_json_v2.py | 32 +++- tests/test_import_script_v2_per_turn.py | 142 ++++++++++++++++++ 3 files changed, 168 insertions(+), 8 deletions(-) create mode 100644 tests/test_import_script_v2_per_turn.py diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index 3f071c4..cce32e5 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -114,7 +114,7 @@ python import_manual_runs_json_v2.py \ --csv-file "gemma-multiturn.csv" ``` -> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. For multi-turn benchmarks, the script emits one aggregate row per concurrency level (`turn_index` = empty) **plus** one per-turn row per turn per concurrency level (`turn_index` = 0, 1, 2, …) when `requests.successful`/`requests.errored` contain at least two distinct valid `turn_index` values. Other benchmarks emit aggregate rows only. +> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. For multi-turn benchmarks, the script emits one aggregate row per concurrency level (`turn_index` = empty) **plus** one per-turn row per turn per concurrency level (`turn_index` = 0, 1, 2, …) when `requests.successful`/`requests.errored` contain at least two distinct valid `turn_index` values. Other benchmarks emit aggregate rows only. On per-turn rows, `output_tok/sec` and `total_tok/sec` report the turn's aggregate throughput over the benchmark's measurement interval (the turn's tokens divided by the scheduler run duration), matching the GuideLLM total-throughput semantics of the aggregate row; `measured rps` is left empty because it is not computed per turn. ## Appending to Consolidated Dashboard diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index 053c816..9d45513 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -49,9 +49,13 @@ def _sorted_field(reqs, field): def extract_per_turn_rows(benchmark, base_row): """Extract per-turn breakdown rows from request-level data in a benchmark section. - For each turn_index found in requests.successful, computes per-turn stats - (TTFT, ITL, TPOT, latency, throughput) and returns one row dict per turn. - Returns an empty list if no turn_index data is present. + For each turn_index found in requests.successful/requests.errored, computes + per-turn stats (TTFT, ITL, TPOT, latency, throughput) and returns one row + dict per turn. Throughput columns carry the turn's aggregate rate over the + benchmark's measurement interval (the turn's tokens divided by the + scheduler start-to-end duration), matching the GuideLLM "total" throughput + reported on the aggregate row. Returns an empty list when fewer than two + distinct valid turn indices are present. Args: benchmark: A single benchmark dict from the guidellm JSON. @@ -70,6 +74,15 @@ def extract_per_turn_rows(benchmark, base_row): if not successful and not errored: return [] + # Measurement interval for per-turn throughput: the same scheduler window + # GuideLLM uses for its run-level "total" throughput distributions. + scheduler_metrics = benchmark.get("scheduler_metrics", {}) + start_time = scheduler_metrics.get("start_time") + end_time = scheduler_metrics.get("end_time") + duration = None + if start_time is not None and end_time is not None and end_time > start_time: + duration = end_time - start_time + # Discover all turn indices from both buckets all_turns = set() for req in successful + errored: @@ -115,8 +128,6 @@ def extract_per_turn_rows(benchmark, base_row): itl = _sorted_field(reqs, "inter_token_latency_ms") tpot = _sorted_field(reqs, "time_per_output_token_ms") lat = _sorted_field(reqs, "request_latency") - out_tps = _sorted_field(reqs, "output_tokens_per_second") - total_tps = _sorted_field(reqs, "tokens_per_second") out_toks = _sorted_field(reqs, "output_tokens") prompt_toks = _sorted_field(reqs, "prompt_tokens") @@ -144,8 +155,15 @@ def extract_per_turn_rows(benchmark, base_row): row["request_latency_median"] = _median(lat) row["request_latency_min"] = min(lat) if lat else None row["request_latency_max"] = max(lat) if lat else None - row["output_tok/sec"] = _mean(out_tps) - row["total_tok/sec"] = _mean(total_tps) + # Throughput on per-turn rows is the turn's aggregate rate over the + # measurement interval (turn tokens / run duration), matching the + # GuideLLM "total" throughput on the aggregate row. The mean of + # per-request rates would report a single request's rate and understate + # concurrent load by roughly the concurrency factor. + row["output_tok/sec"] = sum(out_toks) / duration if duration else None + row["total_tok/sec"] = ( + (sum(out_toks) + sum(prompt_toks)) / duration if duration else None + ) row["output_token_count_mean"] = _mean(out_toks) row["prompt_token_count_mean"] = _mean(prompt_toks) row["output_token_count_p99"] = _percentile(out_toks, 99) diff --git a/tests/test_import_script_v2_per_turn.py b/tests/test_import_script_v2_per_turn.py new file mode 100644 index 0000000..9bb5dee --- /dev/null +++ b/tests/test_import_script_v2_per_turn.py @@ -0,0 +1,142 @@ +"""Unit tests for import_manual_runs_json_v2.py per-turn row extraction.""" + +from manual_runs.scripts.vllm.import_manual_runs_json_v2 import extract_per_turn_rows + +BASE_ROW = { + "run": "H200-test-model-8", + "measured rps": 5.0, + "output_tok/sec": 1000.0, + "total_tok/sec": 11000.0, + "prompt_token_count_p99": 1024.0, + "output_token_count_p99": 128.0, + "turn_index": "", +} + + +def _request(turn_index, output_tokens=100, prompt_tokens=1000): + """Build a guidellm request record for the given turn.""" + return { + "info": {"turn_index": turn_index}, + "prompt_tokens": prompt_tokens, + "output_tokens": output_tokens, + "time_to_first_token_ms": 50.0, + "inter_token_latency_ms": 10.0, + "time_per_output_token_ms": 10.0, + "request_latency": 1.0, + "output_tokens_per_second": 100.0, + "tokens_per_second": 1100.0, + } + + +def _benchmark(successful, errored=None, duration=10.0): + """Build a guidellm benchmark section with an optional run duration.""" + benchmark = { + "requests": {"successful": successful, "errored": errored or []}, + } + if duration is not None: + benchmark["scheduler_metrics"] = { + "start_time": 100.0, + "end_time": 100.0 + duration, + } + return benchmark + + +class TestExtractPerTurnRows: + """Test the extract_per_turn_rows function.""" + + def test_multi_turn_emits_one_row_per_turn(self): + """Two distinct turn indices produce one row per turn.""" + rows = extract_per_turn_rows( + _benchmark( + [_request(0) for _ in range(5)] + [_request(1) for _ in range(5)] + ), + dict(BASE_ROW), + ) + assert [row["turn_index"] for row in rows] == [0, 1] + + def test_throughput_is_turn_totals_over_measurement_interval(self): + """Per-turn throughput is turn tokens / run duration, not per-request rate. + + 5 requests x 100 output tokens over a 10s interval is 50 output tok/s + (and 550 total tok/s with 1000 prompt tokens each), not the 100 tok/s + per-request rate. + """ + rows = extract_per_turn_rows( + _benchmark( + [_request(0) for _ in range(5)] + [_request(1) for _ in range(5)] + ), + dict(BASE_ROW), + ) + for row in rows: + assert row["output_tok/sec"] == 50.0 + assert row["total_tok/sec"] == 550.0 + + def test_throughput_sums_to_aggregate_across_turns(self): + """Per-turn throughput values sum to the run's total throughput.""" + rows = extract_per_turn_rows( + _benchmark( + [_request(0, output_tokens=60) for _ in range(5)] + + [_request(1, output_tokens=40) for _ in range(10)] + ), + dict(BASE_ROW), + ) + # Turn 0: 5*60/10 = 30, turn 1: 10*40/10 = 40, total = 70 tok/s. + assert sum(row["output_tok/sec"] for row in rows) == 70.0 + + def test_token_count_p99_computed_per_turn(self): + """Token count p99 comes from the turn's requests, not the aggregate.""" + rows = extract_per_turn_rows( + _benchmark( + [_request(0, output_tokens=100) for _ in range(5)] + + [_request(1, output_tokens=200) for _ in range(5)] + ), + dict(BASE_ROW), + ) + assert rows[0]["output_token_count_p99"] == 100.0 + assert rows[1]["output_token_count_p99"] == 200.0 + + def test_measured_rps_cleared_on_per_turn_rows(self): + """measured rps is run-level context and is not inherited per turn.""" + rows = extract_per_turn_rows( + _benchmark([_request(0), _request(1)]), dict(BASE_ROW) + ) + assert all(row["measured rps"] is None for row in rows) + + def test_errored_requests_counted_per_turn(self): + """Errored requests are counted per turn and participate in discovery.""" + rows = extract_per_turn_rows( + _benchmark( + [_request(0) for _ in range(4)], + errored=[_request(0), _request(1)], + ), + dict(BASE_ROW), + ) + by_turn = {row["turn_index"]: row for row in rows} + assert by_turn[0]["successful_requests"] == 4 + assert by_turn[0]["errored_requests"] == 1 + assert by_turn[1]["errored_requests"] == 1 + assert by_turn[1]["successful_requests"] == 0 + + def test_single_turn_emits_no_rows(self): + """A single distinct turn index produces no per-turn rows.""" + rows = extract_per_turn_rows( + _benchmark([_request(0) for _ in range(5)]), dict(BASE_ROW) + ) + assert rows == [] + + def test_missing_turn_index_emits_no_rows(self): + """Requests without turn_index data produce no per-turn rows.""" + requests = [ + {k: v for k, v in _request(0).items() if k != "info"} for _ in range(5) + ] + rows = extract_per_turn_rows(_benchmark(requests), dict(BASE_ROW)) + assert rows == [] + + def test_missing_duration_leaves_throughput_empty(self): + """Without a scheduler measurement interval, throughput is not computed.""" + rows = extract_per_turn_rows( + _benchmark([_request(0), _request(1)], duration=None), dict(BASE_ROW) + ) + assert len(rows) == 2 + assert all(row["output_tok/sec"] is None for row in rows) + assert all(row["total_tok/sec"] is None for row in rows)