diff --git a/llmd_dashboard.py b/llmd_dashboard.py index 5e3db9d..f6ac845 100644 --- a/llmd_dashboard.py +++ b/llmd_dashboard.py @@ -1640,6 +1640,13 @@ def load_rhaiis_data(file_path: str = "consolidated_dashboard.csv") -> pd.DataFr for col in df.select_dtypes(include=["object"]).columns: df[col] = df[col].str.strip() # type: ignore[assignment] + # Keep only aggregate rows: per-turn breakdown rows (turn_index 0, 1, 2…) + # from multi-turn benchmarks would double-count runs in comparisons. + # Mirrors the aggregate-row filter in dashboard.py. + if "turn_index" in df.columns: + _ti = df["turn_index"].astype(str).str.strip() # type: ignore[misc] + df = df[df["turn_index"].isna() | (_ti == "") | (_ti == "nan")].copy() + # Assign profile based on prompt/output tokens (same as llmd_data) if "prompt toks" in df.columns and "output toks" in df.columns: df["profile"] = df.apply(assign_profile, axis=1) diff --git a/manual_runs/scripts/vllm/README.md b/manual_runs/scripts/vllm/README.md index 8273b6c..cf657a1 100644 --- a/manual_runs/scripts/vllm/README.md +++ b/manual_runs/scripts/vllm/README.md @@ -32,20 +32,22 @@ python import_manual_runs_json_v2.py \ ## Arguments -| Argument | Required | Description | Example | -| -------------------- | -------- | ----------------------------------------------- | --------------------------------------- | -| `json_file` | Yes | Path to guidellm JSON output file | `llama-70b-vllm.json` | -| `--model` | Yes | Model name (HuggingFace format) | `meta-llama/Llama-3.3-70B-Instruct` | -| `--version` | Yes | Framework version, optionally suffixed with a run label | `vLLM-0.13.0`, `vLLM-0.28.0-pcon-mnbt` | -| `--tp` | Yes | Tensor parallelism size | `4`, `8` | -| `--accelerator` | Yes | GPU/accelerator type | `H200`, `MI300X`, `TPU` | -| `--runtime-args` | Yes | Server runtime configuration | See examples below | -| `--image-tag` | Yes | Container image tag | `vllm/vllm-openai:v0.13.0` | -| `--guidellm-version` | Yes | guidellm version used | `v0.5.1`, `v0.3.0` | -| `--csv-file` | No | Output CSV path (default: `new_benchmarks.csv`) | `llama-70b-vllm.csv` | -| `--dataset` | No | Real dataset name (for real-dataset runs) | `mlperf-gpt-oss`, `sharegpt` | -| `--spec-decoding` | No | Speculative decoding method used | `eagle3`, `ngram` | -| `--prefix-caching` | No | Whether prefix caching was enabled | `yes`, `no` | +| Argument | Required | Description | Example | +| ------------------------ | -------- | ------------------------------------------------------------- | -------------------------------------- | +| `json_file` | Yes | Path to guidellm JSON output file | `llama-70b-vllm.json` | +| `--model` | Yes | Model name (HuggingFace format) | `meta-llama/Llama-3.3-70B-Instruct` | +| `--version` | Yes | Framework version, optionally suffixed with a run label | `vLLM-0.13.0`, `vLLM-0.28.0-pcon-mnbt` | +| `--tp` | Yes | Tensor parallelism size | `4`, `8` | +| `--accelerator` | Yes | GPU/accelerator type | `H200`, `MI300X`, `TPU` | +| `--runtime-args` | Yes | Server runtime configuration | See examples below | +| `--image-tag` | Yes | Container image tag | `vllm/vllm-openai:v0.13.0` | +| `--guidellm-version` | Yes | guidellm version used | `v0.5.1`, `v0.3.0` | +| `--csv-file` | No | Output CSV path (default: `new_benchmarks.csv`) | `llama-70b-vllm.csv` | +| `--dataset` | No | Real dataset name (for real-dataset runs) | `mlperf-gpt-oss`, `sharegpt` | +| `--spec-decoding` | No | Speculative decoding method used | `eagle3`, `ngram` | +| `--prefix-caching` | No | Whether prefix caching was enabled | `yes`, `no` | +| `--mlflow-run-id` | No | MLflow run UUID — enables clickable MLflow links in dashboard | `c6aa48a0d312448380621e1bf00a8a5d` | +| `--mlflow-experiment-id` | No | MLflow experiment ID (used with `--mlflow-run-id`) | `264` | For labeled runs, include the label suffix in `--version` (for example, `--version vLLM-0.28.0-pcon-mnbt`). The dashboard derives the release and label @@ -116,7 +118,7 @@ python import_manual_runs_json_v2.py \ --csv-file "gemma-multiturn.csv" ``` -> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. `turn_index` is always empty in manual-import output (aggregate rows only) — per-turn rows are generated by the Forge pipeline when `turn_index` data is available in the benchmark JSON. +> **Note:** `turns`, `prefix_tokens`, `prefix_count`, and `request_type` are auto-detected from the guidellm JSON. For multi-turn benchmarks, the script emits one aggregate row per concurrency level (`turn_index` = empty) **plus** one per-turn row per turn per concurrency level (`turn_index` = 0, 1, 2, …) when `requests.successful`/`requests.errored` contain at least two distinct valid `turn_index` values. Other benchmarks emit aggregate rows only. ## Appending to Consolidated Dashboard @@ -139,63 +141,65 @@ values remain aligned with the dashboard column order. ## Output CSV Columns -The script outputs 53 columns compatible with the performance dashboard: - -| # | Column | Description | -| --- | ------------------------- | ---------------------------------------------------- | -| 1 | `run` | Unique run identifier (`{accelerator}-{model}-{tp}`) | -| 2 | `accelerator` | GPU type (H200, MI300X, etc.) | -| 3 | `model` | Model name | -| 4 | `version` | Framework version, optionally suffixed with a run label | -| 5 | `prompt toks` | Configured prompt token count | -| 6 | `output toks` | Configured output token count | -| 7 | `TP` | Tensor parallelism size | -| 8 | `measured concurrency` | Actual measured concurrency | -| 9 | `intended concurrency` | Requested concurrency (streams) | -| 10 | `measured rps` | Requests per second | -| 11 | `output_tok/sec` | Output tokens per second | -| 12 | `total_tok/sec` | Total tokens per second | -| 13 | `prompt_token_count_mean` | Mean prompt token count | -| 14 | `prompt_token_count_p99` | P99 prompt token count | -| 15 | `output_token_count_mean` | Mean output token count | -| 16 | `output_token_count_p99` | P99 output token count | -| 17 | `ttft_median` | Time to first token - median (ms) | -| 18 | `ttft_p95` | Time to first token - P95 (ms) | -| 19 | `ttft_p1` | Time to first token - P1 (ms) | -| 20 | `ttft_p999` | Time to first token - P99.9 (ms) | -| 21 | `tpot_median` | Time per output token - median (ms) | -| 22 | `tpot_p95` | Time per output token - P95 (ms) | -| 23 | `tpot_p99` | Time per output token - P99 (ms) | -| 24 | `tpot_p999` | Time per output token - P99.9 (ms) | -| 25 | `tpot_p1` | Time per output token - P1 (ms) | -| 26 | `itl_median` | Inter-token latency - median (ms) | -| 27 | `itl_p95` | Inter-token latency - P95 (ms) | -| 28 | `itl_p999` | Inter-token latency - P99.9 (ms) | -| 29 | `itl_p1` | Inter-token latency - P1 (ms) | -| 30 | `request_latency_median` | End-to-end request latency - median (s) | -| 31 | `request_latency_min` | End-to-end request latency - minimum (s) | -| 32 | `request_latency_max` | End-to-end request latency - maximum (s) | -| 33 | `successful_requests` | Number of successful requests | -| 34 | `errored_requests` | Number of errored requests | -| 35 | `uuid` | Unique benchmark run ID | -| 36 | `ttft_mean` | Time to first token - mean (ms) | -| 37 | `ttft_p99` | Time to first token - P99 (ms) | -| 38 | `itl_mean` | Inter-token latency - mean (ms) | -| 39 | `itl_p99` | Inter-token latency - P99 (ms) | -| 40 | `runtime_args` | Server configuration arguments | -| 41 | `guidellm_start_time_ms` | Benchmark start time (epoch ms) | -| 42 | `guidellm_end_time_ms` | Benchmark end time (epoch ms) | -| 43 | `image_tag` | Container image used | -| 44 | `guidellm_version` | guidellm version used | -| 45 | `DP` | Data parallelism size (empty for TP runs) | -| 46 | `dataset` | Real dataset name (empty for synthetic runs) | -| 47 | `spec_decoding` | Speculative decoding method (empty if none) | -| 48 | `prefix_caching` | Prefix caching status (`yes`, `no`, or empty) | -| 49 | `turn_index` | Per-turn breakdown index (empty for aggregate rows) | -| 50 | `turns` | Conversation turns for multiturn benchmarks | -| 51 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | -| 52 | `prefix_count` | Prefix count (auto-detected from JSON) | -| 53 | `request_type` | GuideLLM API endpoint type (auto-detected from JSON) | +The script outputs 55 columns compatible with the performance dashboard: + +| # | Column | Description | +| --- | ------------------------- | --------------------------------------------------------------------------- | +| 1 | `run` | Unique run identifier (`{accelerator}-{model}-{tp}`) | +| 2 | `accelerator` | GPU type (H200, MI300X, etc.) | +| 3 | `model` | Model name | +| 4 | `version` | Framework version, optionally suffixed with a run label | +| 5 | `prompt toks` | Configured prompt token count | +| 6 | `output toks` | Configured output token count | +| 7 | `TP` | Tensor parallelism size | +| 8 | `measured concurrency` | Actual measured concurrency | +| 9 | `intended concurrency` | Requested concurrency (streams) | +| 10 | `measured rps` | Requests per second | +| 11 | `output_tok/sec` | Output tokens per second | +| 12 | `total_tok/sec` | Total tokens per second | +| 13 | `prompt_token_count_mean` | Mean prompt token count | +| 14 | `prompt_token_count_p99` | P99 prompt token count | +| 15 | `output_token_count_mean` | Mean output token count | +| 16 | `output_token_count_p99` | P99 output token count | +| 17 | `ttft_median` | Time to first token - median (ms) | +| 18 | `ttft_p95` | Time to first token - P95 (ms) | +| 19 | `ttft_p1` | Time to first token - P1 (ms) | +| 20 | `ttft_p999` | Time to first token - P99.9 (ms) | +| 21 | `tpot_median` | Time per output token - median (ms) | +| 22 | `tpot_p95` | Time per output token - P95 (ms) | +| 23 | `tpot_p99` | Time per output token - P99 (ms) | +| 24 | `tpot_p999` | Time per output token - P99.9 (ms) | +| 25 | `tpot_p1` | Time per output token - P1 (ms) | +| 26 | `itl_median` | Inter-token latency - median (ms) | +| 27 | `itl_p95` | Inter-token latency - P95 (ms) | +| 28 | `itl_p999` | Inter-token latency - P99.9 (ms) | +| 29 | `itl_p1` | Inter-token latency - P1 (ms) | +| 30 | `request_latency_median` | End-to-end request latency - median (s) | +| 31 | `request_latency_min` | End-to-end request latency - minimum (s) | +| 32 | `request_latency_max` | End-to-end request latency - maximum (s) | +| 33 | `successful_requests` | Number of successful requests | +| 34 | `errored_requests` | Number of errored requests | +| 35 | `uuid` | Unique benchmark run ID | +| 36 | `ttft_mean` | Time to first token - mean (ms) | +| 37 | `ttft_p99` | Time to first token - P99 (ms) | +| 38 | `itl_mean` | Inter-token latency - mean (ms) | +| 39 | `itl_p99` | Inter-token latency - P99 (ms) | +| 40 | `runtime_args` | Server configuration arguments | +| 41 | `guidellm_start_time_ms` | Benchmark start time (epoch ms) | +| 42 | `guidellm_end_time_ms` | Benchmark end time (epoch ms) | +| 43 | `image_tag` | Container image used | +| 44 | `guidellm_version` | guidellm version used | +| 45 | `DP` | Data parallelism size (empty for TP runs) | +| 46 | `dataset` | Real dataset name (empty for synthetic runs) | +| 47 | `spec_decoding` | Speculative decoding method (empty if none) | +| 48 | `prefix_caching` | Prefix caching status (`yes`, `no`, or empty) | +| 49 | `turn_index` | Per-turn breakdown index (empty for aggregate rows) | +| 50 | `turns` | Conversation turns for multiturn benchmarks | +| 51 | `prefix_tokens` | Prefix token count (auto-detected from JSON) | +| 52 | `prefix_count` | Prefix count (auto-detected from JSON) | +| 53 | `request_type` | GuideLLM API endpoint type (auto-detected from JSON) | +| 54 | `mlflow_run_id` | MLflow run UUID (from `--mlflow-run-id`, empty if not provided) | +| 55 | `mlflow_experiment_id` | MLflow experiment ID (from `--mlflow-experiment-id`, empty if not provided) | ## Notes diff --git a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py index ca6a02a..6b464d2 100644 --- a/manual_runs/scripts/vllm/import_manual_runs_json_v2.py +++ b/manual_runs/scripts/vllm/import_manual_runs_json_v2.py @@ -7,12 +7,155 @@ """ import argparse +import contextlib import json import os import pandas as pd +def _percentile(values, p): + """Return the p-th percentile of a pre-sorted list using linear interpolation. + + Matches the implementation in Forge PR #215 _percentile(). + """ + if not values: + return None + k = (len(values) - 1) * p / 100.0 + f = int(k) + c = min(f + 1, len(values) - 1) + return values[f] + (k - f) * (values[c] - values[f]) + + +def _median(values): + """Return the median of a pre-sorted list. + + Matches the Forge PR #215 implementation. + """ + n = len(values) + if not n: + return None + return values[n // 2] if n % 2 else (values[n // 2 - 1] + values[n // 2]) / 2 + + +def _mean(values): + return sum(values) / len(values) if values else None + + +def _sorted_field(reqs, field): + return sorted(r[field] for r in reqs if r.get(field) is not None) + + +def extract_per_turn_rows(benchmark, base_row): + """Extract per-turn breakdown rows from request-level data in a benchmark section. + + For each turn_index found in requests.successful, computes per-turn stats + (TTFT, ITL, TPOT, latency, throughput) and returns one row dict per turn. + Returns an empty list if no turn_index data is present. + + Args: + benchmark: A single benchmark dict from the guidellm JSON. + base_row: The aggregate row dict produced by process_benchmark_section — + used as the metadata template for per-turn rows. + + Returns: + list[dict]: Per-turn row dicts ready to append to all_run_data. + """ + # Named requests_data (not requests) so bandit does not mistake the dict + # for the requests library (B113). + requests_data = benchmark.get("requests", {}) + successful = requests_data.get("successful", []) + errored = requests_data.get("errored", []) + + if not successful and not errored: + return [] + + # Discover all turn indices from both buckets + all_turns = set() + for req in successful + errored: + info = req.get("info", {}) + ti = info.get("turn_index") + if ti is not None: + with contextlib.suppress(ValueError, TypeError): + all_turns.add(str(int(float(ti)))) + + # Match Forge PR #215: skip extraction for single-turn runs + if len(all_turns) <= 1: + return [] + + # Group requests by turn_index + by_turn = {} + for req in successful: + ti = req.get("info", {}).get("turn_index") + if ti is None: + continue + try: + key = str(int(float(ti))) + except (ValueError, TypeError): + continue + by_turn.setdefault(key, []).append(req) + + err_by_turn = {} + for req in errored: + ti = req.get("info", {}).get("turn_index") + if ti is None: + continue + try: + key = str(int(float(ti))) + except (ValueError, TypeError): + continue + err_by_turn.setdefault(key, []).append(req) + + per_turn_rows = [] + for turn_key in sorted(all_turns, key=lambda x: int(x)): + reqs = by_turn.get(turn_key, []) + n_err = len(err_by_turn.get(turn_key, [])) + + ttft = _sorted_field(reqs, "time_to_first_token_ms") + itl = _sorted_field(reqs, "inter_token_latency_ms") + tpot = _sorted_field(reqs, "time_per_output_token_ms") + lat = _sorted_field(reqs, "request_latency") + out_tps = _sorted_field(reqs, "output_tokens_per_second") + total_tps = _sorted_field(reqs, "tokens_per_second") + out_toks = _sorted_field(reqs, "output_tokens") + prompt_toks = _sorted_field(reqs, "prompt_tokens") + + row = dict(base_row) + row["turn_index"] = int(turn_key) + row["successful_requests"] = len(reqs) + row["errored_requests"] = n_err + row["ttft_median"] = _median(ttft) + row["ttft_p95"] = _percentile(ttft, 95) + row["ttft_p99"] = _percentile(ttft, 99) + row["ttft_p1"] = _percentile(ttft, 1) + row["ttft_p999"] = _percentile(ttft, 99.9) + row["ttft_mean"] = _mean(ttft) + row["itl_median"] = _median(itl) + row["itl_p95"] = _percentile(itl, 95) + row["itl_p99"] = _percentile(itl, 99) + row["itl_p1"] = _percentile(itl, 1) + row["itl_p999"] = _percentile(itl, 99.9) + row["itl_mean"] = _mean(itl) + row["tpot_median"] = _median(tpot) + row["tpot_p95"] = _percentile(tpot, 95) + row["tpot_p99"] = _percentile(tpot, 99) + row["tpot_p1"] = _percentile(tpot, 1) + row["tpot_p999"] = _percentile(tpot, 99.9) + row["request_latency_median"] = _median(lat) + row["request_latency_min"] = min(lat) if lat else None + row["request_latency_max"] = max(lat) if lat else None + row["output_tok/sec"] = _mean(out_tps) + row["total_tok/sec"] = _mean(total_tps) + row["output_token_count_mean"] = _mean(out_toks) + row["prompt_token_count_mean"] = _mean(prompt_toks) + row["output_token_count_p99"] = _percentile(out_toks, 99) + row["prompt_token_count_p99"] = _percentile(prompt_toks, 99) + row["measured rps"] = None + per_turn_rows.append(row) + + return per_turn_rows + + def process_benchmark_section( benchmark, accelerator, @@ -332,11 +475,14 @@ def parse_guidellm_json( ) if row_data: all_run_data.append(row_data) + per_turn = extract_per_turn_rows(benchmark, row_data) + all_run_data.extend(per_turn) streams = ( benchmark.get("config", {}).get("strategy", {}).get("streams", "?") ) + turn_msg = f", {len(per_turn)} per-turn rows" if per_turn else "" print( - f" Processed benchmark {i + 1}/{len(benchmarks)} (streams={streams})" + f" Processed benchmark {i + 1}/{len(benchmarks)} (streams={streams}{turn_msg})" ) if all_run_data: @@ -426,6 +572,19 @@ def main(): help="Whether prefix caching is enabled ('yes' or 'no'). " "Leave empty if not applicable.", ) + parser.add_argument( + "--mlflow-run-id", + default="", + help="MLflow run UUID (optional). When provided, the Filtered Data table " + "in the dashboard shows a clickable MLflow artifact link. " + "Example: c6aa48a0d312448380621e1bf00a8a5d", + ) + parser.add_argument( + "--mlflow-experiment-id", + default="", + help="MLflow experiment ID (optional, used together with --mlflow-run-id). " + "Example: 264", + ) parser.add_argument( "--csv-file", default="new_benchmarks.csv", @@ -460,6 +619,10 @@ def main(): ) if new_data_df is not None and not new_data_df.empty: + # Stamp MLflow identifiers on every row (empty string when not provided) + new_data_df["mlflow_run_id"] = args.mlflow_run_id + new_data_df["mlflow_experiment_id"] = args.mlflow_experiment_id + if os.path.exists(args.csv_file): print(f"Appending {len(new_data_df)} new rows to {args.csv_file}...") existing_df = pd.read_csv(args.csv_file) @@ -519,10 +682,13 @@ def main(): "dataset", "spec_decoding", "prefix_caching", + "turn_index", "turns", "prefix_tokens", "prefix_count", "request_type", + "mlflow_run_id", + "mlflow_experiment_id", ] for col in fieldnames: