From 6c2b63701f21ed11d7d3b0b202c4b60a663f1cb7 Mon Sep 17 00:00:00 2001 From: aas008 Date: Wed, 7 Oct 2026 11:37:38 -0400 Subject: [PATCH 1/6] feat(manual-import): generate per-turn rows from request-level turn_index data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the guidellm JSON contains requests.successful with turn_index fields (multi-turn benchmarks), the script now emits: - one aggregate row per concurrency level (turn_index = empty, unchanged) - one per-turn row per turn per concurrency level (turn_index = 0, 1, 2, …) Per-turn rows carry the same metadata as the aggregate row plus per-turn stats: TTFT/ITL/TPOT median+p95+p99+p1+p999+mean, request latency, output throughput, token counts. Benchmarks without turn_index data are unaffected. Add _percentile() helper and extract_per_turn_rows() function. Update README to clarify that per-turn rows are now emitted automatically. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/README.md | 2 +- .../vllm/import_manual_runs_json_v2.py | 124 +++++++++++++++++- 2 files changed, 124 insertions(+), 2 deletions(-) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index 8273b6c..e945e70 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -116,7 +116,7 @@ python import_manual_runs_json_v2.py \ --csv-file "gemma-multiturn.csv" ``` -> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. `turn_index` is always empty in manual-import output (aggregate rows only) — per-turn rows are generated by the Forge pipeline when `turn_index` data is available in the benchmark JSON. +> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. For multi-turn benchmarks, the script emits one aggregate row per concurrency level (`turn_index` = empty) **plus** one per-turn row per turn per concurrency level (`turn_index` = 0, 1, 2, …) when `requests.successful` contains `turn_index` data. Non-multi-turn benchmarks emit aggregate rows only. ## Appending to Consolidated Dashboard diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index ca6a02a..3b7d754 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -13,6 +13,125 @@ import pandas as pd +def _percentile(values, p): + """Return the p-th percentile of a sorted list, or None if empty.""" + if not values: + return None + values = sorted(values) + idx = (len(values) - 1) * p / 100 + lo, hi = int(idx), min(int(idx) + 1, len(values) - 1) + return values[lo] + (values[hi] - values[lo]) * (idx - lo) + + +def extract_per_turn_rows(benchmark, base_row): + """Extract per-turn breakdown rows from request-level data in a benchmark section. + + For each turn_index found in requests.successful, computes per-turn stats + (TTFT, ITL, TPOT, latency, throughput) and returns one row dict per turn. + Returns an empty list if no turn_index data is present. + + Args: + benchmark: A single benchmark dict from the guidellm JSON. + base_row: The aggregate row dict produced by process_benchmark_section — + used as the metadata template for per-turn rows. + + Returns: + list[dict]: Per-turn row dicts ready to append to all_run_data. + """ + requests = benchmark.get("requests", {}) + successful = requests.get("successful", []) + errored = requests.get("errored", []) + + if not successful and not errored: + return [] + + # Discover all turn indices from both buckets + all_turns = set() + for req in successful + errored: + info = req.get("info", {}) + ti = info.get("turn_index") + if ti is not None: + try: + all_turns.add(str(int(float(ti)))) + except (ValueError, TypeError): + pass + + if not all_turns: + return [] + + # Group successful requests by turn_index + by_turn = {} + for req in successful: + info = req.get("info", {}) + ti = info.get("turn_index") + if ti is None: + continue + try: + key = str(int(float(ti))) + except (ValueError, TypeError): + continue + by_turn.setdefault(key, []).append(req) + + # Group errored requests by turn_index for error counts + err_by_turn = {} + for req in errored: + info = req.get("info", {}) + ti = info.get("turn_index") + if ti is None: + continue + try: + key = str(int(float(ti))) + except (ValueError, TypeError): + continue + err_by_turn.setdefault(key, []).append(req) + + per_turn_rows = [] + for turn_key in sorted(all_turns, key=lambda x: int(x)): + reqs = by_turn.get(turn_key, []) + n_err = len(err_by_turn.get(turn_key, [])) + + ttfts = [r.get("time_to_first_token_ms") for r in reqs if r.get("time_to_first_token_ms") is not None] + itls = [r.get("inter_token_latency_ms") for r in reqs if r.get("inter_token_latency_ms") is not None] + tpots = [r.get("time_per_output_token_ms") for r in reqs if r.get("time_per_output_token_ms") is not None] + lats = [r.get("request_latency") for r in reqs if r.get("request_latency") is not None] + out_tps = [r.get("output_tokens_per_second") for r in reqs if r.get("output_tokens_per_second") is not None] + total_tps = [r.get("tokens_per_second") for r in reqs if r.get("tokens_per_second") is not None] + out_toks = [r.get("output_tokens") for r in reqs if r.get("output_tokens") is not None] + prompt_toks = [r.get("prompt_tokens") for r in reqs if r.get("prompt_tokens") is not None] + + row = dict(base_row) + row["turn_index"] = int(turn_key) + row["successful_requests"] = len(reqs) + row["errored_requests"] = n_err + row["ttft_median"] = _percentile(ttfts, 50) + row["ttft_p95"] = _percentile(ttfts, 95) + row["ttft_p99"] = _percentile(ttfts, 99) + row["ttft_p1"] = _percentile(ttfts, 1) + row["ttft_p999"] = _percentile(ttfts, 99.9) + row["ttft_mean"] = (sum(ttfts) / len(ttfts)) if ttfts else None + row["itl_median"] = _percentile(itls, 50) + row["itl_p95"] = _percentile(itls, 95) + row["itl_p99"] = _percentile(itls, 99) + row["itl_p1"] = _percentile(itls, 1) + row["itl_p999"] = _percentile(itls, 99.9) + row["itl_mean"] = (sum(itls) / len(itls)) if itls else None + row["tpot_median"] = _percentile(tpots, 50) + row["tpot_p95"] = _percentile(tpots, 95) + row["tpot_p99"] = _percentile(tpots, 99) + row["tpot_p1"] = _percentile(tpots, 1) + row["tpot_p999"] = _percentile(tpots, 99.9) + row["request_latency_median"] = _percentile(lats, 50) + row["request_latency_min"] = min(lats) if lats else None + row["request_latency_max"] = max(lats) if lats else None + row["output_tok/sec"] = (sum(out_tps) / len(out_tps)) if out_tps else None + row["total_tok/sec"] = (sum(total_tps) / len(total_tps)) if total_tps else None + row["output_token_count_mean"] = (sum(out_toks) / len(out_toks)) if out_toks else None + row["prompt_token_count_mean"] = (sum(prompt_toks) / len(prompt_toks)) if prompt_toks else None + per_turn_rows.append(row) + + return per_turn_rows + + def process_benchmark_section( benchmark, accelerator, @@ -332,11 +451,14 @@ def parse_guidellm_json( ) if row_data: all_run_data.append(row_data) + per_turn = extract_per_turn_rows(benchmark, row_data) + all_run_data.extend(per_turn) streams = ( benchmark.get("config", {}).get("strategy", {}).get("streams", "?") ) + turn_msg = f", {len(per_turn)} per-turn rows" if per_turn else "" print( - f" Processed benchmark {i + 1}/{len(benchmarks)} (streams={streams})" + f" Processed benchmark {i + 1}/{len(benchmarks)} (streams={streams}{turn_msg})" ) if all_run_data: From 8af00eb7c29be5405feb2da9aa892df7de5391d0 Mon Sep 17 00:00:00 2001 From: aas008 Date: Tue, 6 Oct 2026 18:00:08 -0400 Subject: [PATCH 2/6] fix: add turn_index to fieldnames so it is written to CSV output The turn_index key was added to the result dict but missing from the fieldnames list passed to combined_df[fieldnames], so pandas silently dropped it. CSV output now has 53 columns as the README states. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/import_manual_runs_json_v2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index 3b7d754..f71e46b 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -641,6 +641,7 @@ def main(): "dataset", "spec_decoding", "prefix_caching", + "turn_index", "turns", "prefix_tokens", "prefix_count", From 57d7e14863912bd085536eb2f5a3187375f0c68b Mon Sep 17 00:00:00 2001 From: aas008 Date: Wed, 7 Oct 2026 11:39:58 -0400 Subject: [PATCH 3/6] refactor: align per-turn stats with Forge PR #215 implementation - _percentile(): use same k/f/c linear interpolation as Forge - Add _median(), _mean(), _sorted_field() helpers matching Forge's _turn_metric_values() helpers exactly - Add len(all_turns) <= 1 guard: skip per-turn extraction for single-turn runs (matches Forge PR #215 behavior) - Replace _percentile(50) calls with _median() for correctness Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../vllm/import_manual_runs_json_v2.py | 105 ++++++++++-------- 1 file changed, 61 insertions(+), 44 deletions(-) diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index f71e46b..e58a3c1 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -14,13 +14,32 @@ def _percentile(values, p): - """Return the p-th percentile of a sorted list, or None if empty.""" + """Return the p-th percentile of a pre-sorted list using linear interpolation. + + Matches the implementation in Forge PR #215 _percentile(). + """ if not values: return None - values = sorted(values) - idx = (len(values) - 1) * p / 100 - lo, hi = int(idx), min(int(idx) + 1, len(values) - 1) - return values[lo] + (values[hi] - values[lo]) * (idx - lo) + k = (len(values) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(values) - 1) + return values[f] + (k - f) * (values[c] - values[f]) + + +def _median(values): + """Return the median of a pre-sorted list. Matches Forge PR #215 _median().""" + n = len(values) + if not n: + return None + return values[n // 2] if n % 2 else (values[n // 2 - 1] + values[n // 2]) / 2 + + +def _mean(values): + return sum(values) / len(values) if values else None + + +def _sorted_field(reqs, field): + return sorted(r[field] for r in reqs if r.get(field) is not None) def extract_per_turn_rows(benchmark, base_row): @@ -56,14 +75,14 @@ def extract_per_turn_rows(benchmark, base_row): except (ValueError, TypeError): pass - if not all_turns: + # Match Forge PR #215: skip extraction for single-turn runs + if len(all_turns) <= 1: return [] - # Group successful requests by turn_index + # Group requests by turn_index by_turn = {} for req in successful: - info = req.get("info", {}) - ti = info.get("turn_index") + ti = req.get("info", {}).get("turn_index") if ti is None: continue try: @@ -72,11 +91,9 @@ def extract_per_turn_rows(benchmark, base_row): continue by_turn.setdefault(key, []).append(req) - # Group errored requests by turn_index for error counts err_by_turn = {} for req in errored: - info = req.get("info", {}) - ti = info.get("turn_index") + ti = req.get("info", {}).get("turn_index") if ti is None: continue try: @@ -90,43 +107,43 @@ def extract_per_turn_rows(benchmark, base_row): reqs = by_turn.get(turn_key, []) n_err = len(err_by_turn.get(turn_key, [])) - ttfts = [r.get("time_to_first_token_ms") for r in reqs if r.get("time_to_first_token_ms") is not None] - itls = [r.get("inter_token_latency_ms") for r in reqs if r.get("inter_token_latency_ms") is not None] - tpots = [r.get("time_per_output_token_ms") for r in reqs if r.get("time_per_output_token_ms") is not None] - lats = [r.get("request_latency") for r in reqs if r.get("request_latency") is not None] - out_tps = [r.get("output_tokens_per_second") for r in reqs if r.get("output_tokens_per_second") is not None] - total_tps = [r.get("tokens_per_second") for r in reqs if r.get("tokens_per_second") is not None] - out_toks = [r.get("output_tokens") for r in reqs if r.get("output_tokens") is not None] - prompt_toks = [r.get("prompt_tokens") for r in reqs if r.get("prompt_tokens") is not None] + ttft = _sorted_field(reqs, "time_to_first_token_ms") + itl = _sorted_field(reqs, "inter_token_latency_ms") + tpot = _sorted_field(reqs, "time_per_output_token_ms") + lat = _sorted_field(reqs, "request_latency") + out_tps = _sorted_field(reqs, "output_tokens_per_second") + total_tps = _sorted_field(reqs, "tokens_per_second") + out_toks = _sorted_field(reqs, "output_tokens") + prompt_toks = _sorted_field(reqs, "prompt_tokens") row = dict(base_row) row["turn_index"] = int(turn_key) row["successful_requests"] = len(reqs) row["errored_requests"] = n_err - row["ttft_median"] = _percentile(ttfts, 50) - row["ttft_p95"] = _percentile(ttfts, 95) - row["ttft_p99"] = _percentile(ttfts, 99) - row["ttft_p1"] = _percentile(ttfts, 1) - row["ttft_p999"] = _percentile(ttfts, 99.9) - row["ttft_mean"] = (sum(ttfts) / len(ttfts)) if ttfts else None - row["itl_median"] = _percentile(itls, 50) - row["itl_p95"] = _percentile(itls, 95) - row["itl_p99"] = _percentile(itls, 99) - row["itl_p1"] = _percentile(itls, 1) - row["itl_p999"] = _percentile(itls, 99.9) - row["itl_mean"] = (sum(itls) / len(itls)) if itls else None - row["tpot_median"] = _percentile(tpots, 50) - row["tpot_p95"] = _percentile(tpots, 95) - row["tpot_p99"] = _percentile(tpots, 99) - row["tpot_p1"] = _percentile(tpots, 1) - row["tpot_p999"] = _percentile(tpots, 99.9) - row["request_latency_median"] = _percentile(lats, 50) - row["request_latency_min"] = min(lats) if lats else None - row["request_latency_max"] = max(lats) if lats else None - row["output_tok/sec"] = (sum(out_tps) / len(out_tps)) if out_tps else None - row["total_tok/sec"] = (sum(total_tps) / len(total_tps)) if total_tps else None - row["output_token_count_mean"] = (sum(out_toks) / len(out_toks)) if out_toks else None - row["prompt_token_count_mean"] = (sum(prompt_toks) / len(prompt_toks)) if prompt_toks else None + row["ttft_median"] = _median(ttft) + row["ttft_p95"] = _percentile(ttft, 95) + row["ttft_p99"] = _percentile(ttft, 99) + row["ttft_p1"] = _percentile(ttft, 1) + row["ttft_p999"] = _percentile(ttft, 99.9) + row["ttft_mean"] = _mean(ttft) + row["itl_median"] = _median(itl) + row["itl_p95"] = _percentile(itl, 95) + row["itl_p99"] = _percentile(itl, 99) + row["itl_p1"] = _percentile(itl, 1) + row["itl_p999"] = _percentile(itl, 99.9) + row["itl_mean"] = _mean(itl) + row["tpot_median"] = _median(tpot) + row["tpot_p95"] = _percentile(tpot, 95) + row["tpot_p99"] = _percentile(tpot, 99) + row["tpot_p1"] = _percentile(tpot, 1) + row["tpot_p999"] = _percentile(tpot, 99.9) + row["request_latency_median"] = _median(lat) + row["request_latency_min"] = min(lat) if lat else None + row["request_latency_max"] = max(lat) if lat else None + row["output_tok/sec"] = _mean(out_tps) + row["total_tok/sec"] = _mean(total_tps) + row["output_token_count_mean"] = _mean(out_toks) + row["prompt_token_count_mean"] = _mean(prompt_toks) per_turn_rows.append(row) return per_turn_rows From a424aa08afe911d6c35d752a04fe137411d74213 Mon Sep 17 00:00:00 2001 From: aas008 Date: Wed, 7 Oct 2026 14:39:16 -0400 Subject: [PATCH 4/6] feat(manual-import): add optional --mlflow-run-id and --mlflow-experiment-id flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When provided, these values are stamped on every output row and included in the fieldnames list so the dashboard's Filtered Data table renders a clickable MLflow artifact link (added to the schema in PR #83). Both flags are optional and default to empty string — no breaking change. Closes #107 Co-Authored-By: Claude Sonnet 4.6 (1M context) --- manual_runs/scripts/vllm/README.md | 6 +++++- .../vllm/import_manual_runs_json_v2.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index e945e70..9b4ce4b 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -46,6 +46,8 @@ python import_manual_runs_json_v2.py \ | `--dataset` | No | Real dataset name (for real-dataset runs) | `mlperf-gpt-oss`, `sharegpt` | | `--spec-decoding` | No | Speculative decoding method used | `eagle3`, `ngram` | | `--prefix-caching` | No | Whether prefix caching was enabled | `yes`, `no` | +| `--mlflow-run-id` | No | MLflow run UUID — enables clickable MLflow links in dashboard | `c6aa48a0d312448380621e1bf00a8a5d` | +| `--mlflow-experiment-id` | No | MLflow experiment ID (used with `--mlflow-run-id`) | `264` | For labeled runs, include the label suffix in `--version` (for example, `--version vLLM-0.28.0-pcon-mnbt`). The dashboard derives the release and label @@ -139,7 +141,7 @@ values remain aligned with the dashboard column order. ## Output CSV Columns -The script outputs 53 columns compatible with the performance dashboard: +The script outputs 55 columns compatible with the performance dashboard: | # | Column | Description | | --- | ------------------------- | ---------------------------------------------------- | @@ -196,6 +198,8 @@ The script outputs 53 columns compatible with the performance dashboard: | 51 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | | 52 | `prefix_count` | Prefix count (auto-detected from JSON) | | 53 | `request_type` | GuideLLM API endpoint type (auto-detected from JSON) | +| 54 | `mlflow_run_id` | MLflow run UUID (from `--mlflow-run-id`, empty if not provided) | +| 55 | `mlflow_experiment_id` | MLflow experiment ID (from `--mlflow-experiment-id`, empty if not provided) | ## Notes diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index e58a3c1..8d46bc2 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -565,6 +565,19 @@ def main(): help="Whether prefix caching is enabled ('yes' or 'no'). " "Leave empty if not applicable.", ) + parser.add_argument( + "--mlflow-run-id", + default="", + help="MLflow run UUID (optional). When provided, the Filtered Data table " + "in the dashboard shows a clickable MLflow artifact link. " + "Example: c6aa48a0d312448380621e1bf00a8a5d", + ) + parser.add_argument( + "--mlflow-experiment-id", + default="", + help="MLflow experiment ID (optional, used together with --mlflow-run-id). " + "Example: 264", + ) parser.add_argument( "--csv-file", default="new_benchmarks.csv", @@ -599,6 +612,10 @@ def main(): ) if new_data_df is not None and not new_data_df.empty: + # Stamp MLflow identifiers on every row (empty string when not provided) + new_data_df["mlflow_run_id"] = args.mlflow_run_id + new_data_df["mlflow_experiment_id"] = args.mlflow_experiment_id + if os.path.exists(args.csv_file): print(f"Appending {len(new_data_df)} new rows to {args.csv_file}...") existing_df = pd.read_csv(args.csv_file) @@ -663,6 +680,8 @@ def main(): "prefix_tokens", "prefix_count", "request_type", + "mlflow_run_id", + "mlflow_experiment_id", ] for col in fieldnames: From 1be4906e85c67ef0b92122df90e28d274e90ed56 Mon Sep 17 00:00:00 2001 From: aas008 Date: Thu, 15 Oct 2026 15:56:05 -0400 Subject: [PATCH 5/6] fix: address review findings in per-turn manual import Compute output/prompt token p99 per turn from request-level data instead of inheriting aggregate values, clear measured rps on per-turn rows (it is not computed per turn), use contextlib.suppress to satisfy ruff SIM105, and correct the README emission rule. Also filter per-turn breakdown rows out of the RHAIIS comparison loader in llmd_dashboard.py, mirroring the aggregate-row filter added to dashboard.py, so multi-turn runs are not double-counted. Co-Authored-By: Claude Code --- llmd_dashboard.py | 7 +++++++ manual_runs/scripts/vllm/README.md | 2 +- manual_runs/scripts/vllm/import_manual_runs_json_v2.py | 8 +++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/llmd_dashboard.py b/llmd_dashboard.py index 5e3db9d..693b8ff 100644 --- a/llmd_dashboard.py +++ b/llmd_dashboard.py @@ -1640,6 +1640,13 @@ def load_rhaiis_data(file_path: str = "consolidated_dashboard.csv") -> pd.DataFr for col in df.select_dtypes(include=["object"]).columns: df[col] = df[col].str.strip() # type: ignore[assignment] + # Keep only aggregate rows: per-turn breakdown rows (turn_index 0, 1, 2…) + # from multi-turn benchmarks would double-count runs in comparisons. + # Mirrors the aggregate-row filter in dashboard.py. + if "turn_index" in df.columns: + _ti = df["turn_index"].astype(str).str.strip() + df = df[df["turn_index"].isna() | (_ti == "") | (_ti == "nan")].copy() + # Assign profile based on prompt/output tokens (same as llmd_data) if "prompt toks" in df.columns and "output toks" in df.columns: df["profile"] = df.apply(assign_profile, axis=1) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index 9b4ce4b..815e7e1 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -118,7 +118,7 @@ python import_manual_runs_json_v2.py \ --csv-file "gemma-multiturn.csv" ``` -> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. For multi-turn benchmarks, the script emits one aggregate row per concurrency level (`turn_index` = empty) **plus** one per-turn row per turn per concurrency level (`turn_index` = 0, 1, 2, …) when `requests.successful` contains `turn_index` data. Non-multi-turn benchmarks emit aggregate rows only. +> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. For multi-turn benchmarks, the script emits one aggregate row per concurrency level (`turn_index` = empty) **plus** one per-turn row per turn per concurrency level (`turn_index` = 0, 1, 2, …) when `requests.successful`/`requests.errored` contain at least two distinct valid `turn_index` values. Other benchmarks emit aggregate rows only. ## Appending to Consolidated Dashboard diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index 8d46bc2..227df80 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -7,6 +7,7 @@ """ import argparse +import contextlib import json import os @@ -70,10 +71,8 @@ def extract_per_turn_rows(benchmark, base_row): info = req.get("info", {}) ti = info.get("turn_index") if ti is not None: - try: + with contextlib.suppress(ValueError, TypeError): all_turns.add(str(int(float(ti)))) - except (ValueError, TypeError): - pass # Match Forge PR #215: skip extraction for single-turn runs if len(all_turns) <= 1: @@ -144,6 +143,9 @@ def extract_per_turn_rows(benchmark, base_row): row["total_tok/sec"] = _mean(total_tps) row["output_token_count_mean"] = _mean(out_toks) row["prompt_token_count_mean"] = _mean(prompt_toks) + row["output_token_count_p99"] = _percentile(out_toks, 99) + row["prompt_token_count_p99"] = _percentile(prompt_toks, 99) + row["measured rps"] = None per_turn_rows.append(row) return per_turn_rows From 4549b5c2bf14a03b8c2456c55b063f0b64dcae28 Mon Sep 17 00:00:00 2001 From: aas008 Date: Thu, 15 Oct 2026 16:04:04 -0400 Subject: [PATCH 6/6] fix: resolve remaining pre-commit hook failures Rename the local requests dict to requests_data so bandit does not flag its .get() calls as requests-library usage without a timeout, split the _median docstring so its first line omits the function name (pydocstyle D402), realign the README tables for prettier, and add a type: ignore for the pandas-stubs str.strip() mismatch in the RHAIIS loader filter. Co-Authored-By: Claude Code --- llmd_dashboard.py | 2 +- manual_runs/scripts/vllm/README.md | 144 +++++++++--------- .../vllm/import_manual_runs_json_v2.py | 13 +- 3 files changed, 82 insertions(+), 77 deletions(-) diff --git a/llmd_dashboard.py b/llmd_dashboard.py index 693b8ff..f6ac845 100644 --- a/llmd_dashboard.py +++ b/llmd_dashboard.py @@ -1644,7 +1644,7 @@ def load_rhaiis_data(file_path: str = "consolidated_dashboard.csv") -> pd.DataFr # from multi-turn benchmarks would double-count runs in comparisons. # Mirrors the aggregate-row filter in dashboard.py. if "turn_index" in df.columns: - _ti = df["turn_index"].astype(str).str.strip() + _ti = df["turn_index"].astype(str).str.strip() # type: ignore[misc] df = df[df["turn_index"].isna() | (_ti == "") | (_ti == "nan")].copy() # Assign profile based on prompt/output tokens (same as llmd_data) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index 815e7e1..cf657a1 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -32,22 +32,22 @@ python import_manual_runs_json_v2.py \ ## Arguments -| Argument | Required | Description | Example | -| -------------------- | -------- | ----------------------------------------------- | --------------------------------------- | -| `json_file` | Yes | Path to guidellm JSON output file | `llama-70b-vllm.json` | -| `--model` | Yes | Model name (HuggingFace format) | `meta-llama/Llama-3.3-70B-Instruct` | -| `--version` | Yes | Framework version, optionally suffixed with a run label | `vLLM-0.13.0`, `vLLM-0.28.0-pcon-mnbt` | -| `--tp` | Yes | Tensor parallelism size | `4`, `8` | -| `--accelerator` | Yes | GPU/accelerator type | `H200`, `MI300X`, `TPU` | -| `--runtime-args` | Yes | Server runtime configuration | See examples below | -| `--image-tag` | Yes | Container image tag | `vllm/vllm-openai:v0.13.0` | -| `--guidellm-version` | Yes | guidellm version used | `v0.5.1`, `v0.3.0` | -| `--csv-file` | No | Output CSV path (default: `new_benchmarks.csv`) | `llama-70b-vllm.csv` | -| `--dataset` | No | Real dataset name (for real-dataset runs) | `mlperf-gpt-oss`, `sharegpt` | -| `--spec-decoding` | No | Speculative decoding method used | `eagle3`, `ngram` | -| `--prefix-caching` | No | Whether prefix caching was enabled | `yes`, `no` | -| `--mlflow-run-id` | No | MLflow run UUID — enables clickable MLflow links in dashboard | `c6aa48a0d312448380621e1bf00a8a5d` | -| `--mlflow-experiment-id` | No | MLflow experiment ID (used with `--mlflow-run-id`) | `264` | +| Argument | Required | Description | Example | +| ------------------------ | -------- | ------------------------------------------------------------- | -------------------------------------- | +| `json_file` | Yes | Path to guidellm JSON output file | `llama-70b-vllm.json` | +| `--model` | Yes | Model name (HuggingFace format) | `meta-llama/Llama-3.3-70B-Instruct` | +| `--version` | Yes | Framework version, optionally suffixed with a run label | `vLLM-0.13.0`, `vLLM-0.28.0-pcon-mnbt` | +| `--tp` | Yes | Tensor parallelism size | `4`, `8` | +| `--accelerator` | Yes | GPU/accelerator type | `H200`, `MI300X`, `TPU` | +| `--runtime-args` | Yes | Server runtime configuration | See examples below | +| `--image-tag` | Yes | Container image tag | `vllm/vllm-openai:v0.13.0` | +| `--guidellm-version` | Yes | guidellm version used | `v0.5.1`, `v0.3.0` | +| `--csv-file` | No | Output CSV path (default: `new_benchmarks.csv`) | `llama-70b-vllm.csv` | +| `--dataset` | No | Real dataset name (for real-dataset runs) | `mlperf-gpt-oss`, `sharegpt` | +| `--spec-decoding` | No | Speculative decoding method used | `eagle3`, `ngram` | +| `--prefix-caching` | No | Whether prefix caching was enabled | `yes`, `no` | +| `--mlflow-run-id` | No | MLflow run UUID — enables clickable MLflow links in dashboard | `c6aa48a0d312448380621e1bf00a8a5d` | +| `--mlflow-experiment-id` | No | MLflow experiment ID (used with `--mlflow-run-id`) | `264` | For labeled runs, include the label suffix in `--version` (for example, `--version vLLM-0.28.0-pcon-mnbt`). The dashboard derives the release and label @@ -143,62 +143,62 @@ values remain aligned with the dashboard column order. The script outputs 55 columns compatible with the performance dashboard: -| # | Column | Description | -| --- | ------------------------- | ---------------------------------------------------- | -| 1 | `run` | Unique run identifier (`{accelerator}-{model}-{tp}`) | -| 2 | `accelerator` | GPU type (H200, MI300X, etc.) | -| 3 | `model` | Model name | -| 4 | `version` | Framework version, optionally suffixed with a run label | -| 5 | `prompt toks` | Configured prompt token count | -| 6 | `output toks` | Configured output token count | -| 7 | `TP` | Tensor parallelism size | -| 8 | `measured concurrency` | Actual measured concurrency | -| 9 | `intended concurrency` | Requested concurrency (streams) | -| 10 | `measured rps` | Requests per second | -| 11 | `output_tok/sec` | Output tokens per second | -| 12 | `total_tok/sec` | Total tokens per second | -| 13 | `prompt_token_count_mean` | Mean prompt token count | -| 14 | `prompt_token_count_p99` | P99 prompt token count | -| 15 | `output_token_count_mean` | Mean output token count | -| 16 | `output_token_count_p99` | P99 output token count | -| 17 | `ttft_median` | Time to first token - median (ms) | -| 18 | `ttft_p95` | Time to first token - P95 (ms) | -| 19 | `ttft_p1` | Time to first token - P1 (ms) | -| 20 | `ttft_p999` | Time to first token - P99.9 (ms) | -| 21 | `tpot_median` | Time per output token - median (ms) | -| 22 | `tpot_p95` | Time per output token - P95 (ms) | -| 23 | `tpot_p99` | Time per output token - P99 (ms) | -| 24 | `tpot_p999` | Time per output token - P99.9 (ms) | -| 25 | `tpot_p1` | Time per output token - P1 (ms) | -| 26 | `itl_median` | Inter-token latency - median (ms) | -| 27 | `itl_p95` | Inter-token latency - P95 (ms) | -| 28 | `itl_p999` | Inter-token latency - P99.9 (ms) | -| 29 | `itl_p1` | Inter-token latency - P1 (ms) | -| 30 | `request_latency_median` | End-to-end request latency - median (s) | -| 31 | `request_latency_min` | End-to-end request latency - minimum (s) | -| 32 | `request_latency_max` | End-to-end request latency - maximum (s) | -| 33 | `successful_requests` | Number of successful requests | -| 34 | `errored_requests` | Number of errored requests | -| 35 | `uuid` | Unique benchmark run ID | -| 36 | `ttft_mean` | Time to first token - mean (ms) | -| 37 | `ttft_p99` | Time to first token - P99 (ms) | -| 38 | `itl_mean` | Inter-token latency - mean (ms) | -| 39 | `itl_p99` | Inter-token latency - P99 (ms) | -| 40 | `runtime_args` | Server configuration arguments | -| 41 | `guidellm_start_time_ms` | Benchmark start time (epoch ms) | -| 42 | `guidellm_end_time_ms` | Benchmark end time (epoch ms) | -| 43 | `image_tag` | Container image used | -| 44 | `guidellm_version` | guidellm version used | -| 45 | `DP` | Data parallelism size (empty for TP runs) | -| 46 | `dataset` | Real dataset name (empty for synthetic runs) | -| 47 | `spec_decoding` | Speculative decoding method (empty if none) | -| 48 | `prefix_caching` | Prefix caching status (`yes`, `no`, or empty) | -| 49 | `turn_index` | Per-turn breakdown index (empty for aggregate rows) | -| 50 | `turns` | Conversation turns for multiturn benchmarks | -| 51 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | -| 52 | `prefix_count` | Prefix count (auto-detected from JSON) | -| 53 | `request_type` | GuideLLM API endpoint type (auto-detected from JSON) | -| 54 | `mlflow_run_id` | MLflow run UUID (from `--mlflow-run-id`, empty if not provided) | +| # | Column | Description | +| --- | ------------------------- | --------------------------------------------------------------------------- | +| 1 | `run` | Unique run identifier (`{accelerator}-{model}-{tp}`) | +| 2 | `accelerator` | GPU type (H200, MI300X, etc.) | +| 3 | `model` | Model name | +| 4 | `version` | Framework version, optionally suffixed with a run label | +| 5 | `prompt toks` | Configured prompt token count | +| 6 | `output toks` | Configured output token count | +| 7 | `TP` | Tensor parallelism size | +| 8 | `measured concurrency` | Actual measured concurrency | +| 9 | `intended concurrency` | Requested concurrency (streams) | +| 10 | `measured rps` | Requests per second | +| 11 | `output_tok/sec` | Output tokens per second | +| 12 | `total_tok/sec` | Total tokens per second | +| 13 | `prompt_token_count_mean` | Mean prompt token count | +| 14 | `prompt_token_count_p99` | P99 prompt token count | +| 15 | `output_token_count_mean` | Mean output token count | +| 16 | `output_token_count_p99` | P99 output token count | +| 17 | `ttft_median` | Time to first token - median (ms) | +| 18 | `ttft_p95` | Time to first token - P95 (ms) | +| 19 | `ttft_p1` | Time to first token - P1 (ms) | +| 20 | `ttft_p999` | Time to first token - P99.9 (ms) | +| 21 | `tpot_median` | Time per output token - median (ms) | +| 22 | `tpot_p95` | Time per output token - P95 (ms) | +| 23 | `tpot_p99` | Time per output token - P99 (ms) | +| 24 | `tpot_p999` | Time per output token - P99.9 (ms) | +| 25 | `tpot_p1` | Time per output token - P1 (ms) | +| 26 | `itl_median` | Inter-token latency - median (ms) | +| 27 | `itl_p95` | Inter-token latency - P95 (ms) | +| 28 | `itl_p999` | Inter-token latency - P99.9 (ms) | +| 29 | `itl_p1` | Inter-token latency - P1 (ms) | +| 30 | `request_latency_median` | End-to-end request latency - median (s) | +| 31 | `request_latency_min` | End-to-end request latency - minimum (s) | +| 32 | `request_latency_max` | End-to-end request latency - maximum (s) | +| 33 | `successful_requests` | Number of successful requests | +| 34 | `errored_requests` | Number of errored requests | +| 35 | `uuid` | Unique benchmark run ID | +| 36 | `ttft_mean` | Time to first token - mean (ms) | +| 37 | `ttft_p99` | Time to first token - P99 (ms) | +| 38 | `itl_mean` | Inter-token latency - mean (ms) | +| 39 | `itl_p99` | Inter-token latency - P99 (ms) | +| 40 | `runtime_args` | Server configuration arguments | +| 41 | `guidellm_start_time_ms` | Benchmark start time (epoch ms) | +| 42 | `guidellm_end_time_ms` | Benchmark end time (epoch ms) | +| 43 | `image_tag` | Container image used | +| 44 | `guidellm_version` | guidellm version used | +| 45 | `DP` | Data parallelism size (empty for TP runs) | +| 46 | `dataset` | Real dataset name (empty for synthetic runs) | +| 47 | `spec_decoding` | Speculative decoding method (empty if none) | +| 48 | `prefix_caching` | Prefix caching status (`yes`, `no`, or empty) | +| 49 | `turn_index` | Per-turn breakdown index (empty for aggregate rows) | +| 50 | `turns` | Conversation turns for multiturn benchmarks | +| 51 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | +| 52 | `prefix_count` | Prefix count (auto-detected from JSON) | +| 53 | `request_type` | GuideLLM API endpoint type (auto-detected from JSON) | +| 54 | `mlflow_run_id` | MLflow run UUID (from `--mlflow-run-id`, empty if not provided) | | 55 | `mlflow_experiment_id` | MLflow experiment ID (from `--mlflow-experiment-id`, empty if not provided) | ## Notes diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index 227df80..6b464d2 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -28,7 +28,10 @@ def _percentile(values, p): def _median(values): - """Return the median of a pre-sorted list. Matches Forge PR #215 _median().""" + """Return the median of a pre-sorted list. + + Matches the Forge PR #215 implementation. + """ n = len(values) if not n: return None @@ -58,9 +61,11 @@ def extract_per_turn_rows(benchmark, base_row): Returns: list[dict]: Per-turn row dicts ready to append to all_run_data. """ - requests = benchmark.get("requests", {}) - successful = requests.get("successful", []) - errored = requests.get("errored", []) + # Named requests_data (not requests) so bandit does not mistake the dict + # for the requests library (B113). + requests_data = benchmark.get("requests", {}) + successful = requests_data.get("successful", []) + errored = requests_data.get("errored", []) if not successful and not errored: return []