📈 {plot_name}
- -- 💡 Click image to view interactive version + 💡 Performance visualization with comprehensive metrics analysis
{% block header_subtitle %}Performance Analysis Report{% endblock %}
+Test Runs
+Total KPIs
+Curve KPIs
+Scalar KPIs
+| Test Run | +Labels | +
|---|---|
| + + {{ test.run_id }} + + | +
+ {% if test.labels %}
+
+ {% for key, value in test.labels.items() %}
+ {{ key }}: {{ value }}
+ {% endfor %}
+
+ {% else %}
+ -
+ {% endif %}
+ |
+
| KPI | +Value | +Unit | +Type | +Trend | +Description | +
|---|---|---|---|---|---|
| {{ kpi.name or kpi.kpi_id }} | ++ {% if kpi.is_curve %} + {% if kpi.values_count > 0 %} + {{ kpi.values_count }} points + {% else %} + No data + {% endif %} + {% else %} + {% if kpi.value is not none %} + {{ "%.3f"|format(kpi.value) if kpi.value is number else kpi.value }} + {% else %} + No data + {% endif %} + {% endif %} + | +{{ kpi.unit or '-' }} | ++ {% if kpi.is_curve %} + Curve + {% else %} + Scalar + {% endif %} + | ++ {% if kpi.higher_is_better %} + ↑ Higher Better + {% else %} + ↓ Lower Better + {% endif %} + | +{{ kpi.help or '-' }} | +
| {{ kpi.x_help or kpi.x_unit or 'X' }} | + {% for point in kpi.values_list %} +{{ point[0] }} | + {% endfor %} +
|---|---|
| {{ kpi.y_help or kpi.unit or 'Y' }} | + {% for point in kpi.values_list %} +{{ "%.3f"|format(point[1]) if point[1] is number else point[1] }} | + {% endfor %} +
No KPI data was found in the provided file.
+Overall Status
+Total KPIs
+Passed
+Regressions
+Skipped
+| Configuration | +Value | +
|---|---|
| Comparison Labels | +{{ config.comparison_labels | join(', ') }} | +
| Ignored Labels | +{{ config.ignored_labels | join(', ') }} | +
| Sorting Labels | +{{ config.sorting_labels | join(', ') }} | +
| KPI | +Current Value | +Change | +Algorithm | +Details | +
|---|---|---|---|---|
|
+ {{ result.kpi_id }}
+ {% if result.run_id %}
+ {{ result.run_id }} + {% endif %} + |
+ + {% if result.current_value %} + {% if result.is_curve %} + {% if result.current_value['values'] %} + {{ result.current_value['values']|length }} points + {% else %} + No data + {% endif %} + {% else %} + {% if result.current_value.value is not none %} + {{ "%.3f"|format(result.current_value.value) if result.current_value.value is number else result.current_value.value }} + {% else %} + No data + {% endif %} + {% endif %} + {% else %} + No data + {% endif %} + | ++ {% if result.details and result.details.relative_change is defined %} + + {{ "%+.1f%%"|format(result.details.relative_change * 100) }} + + {% else %} + - + {% endif %} + | ++ {{ result.details.algorithm if result.details and result.details.algorithm else 'Unknown' }} + | +
+ {% if result.details %}
+
+ Baselines: {{ result.baseline_count }} + {% if result.details.config %} + Threshold: {{ result.details.config.max_relative_regression if result.details.config.max_relative_regression is defined else 'N/A' }} + {% endif %} + + {% else %} + - + {% endif %} + |
+
| KPI | +Verdict | +Relative Change | +Direction | +Labels | +Details | +
|---|---|---|---|---|---|
|
+ {{ result.kpi_id }}
+ {% if result.run_id %}
+ {{ result.run_id }} + {% endif %} + |
+ + + {{ result.verdict }} + + | ++ {% if result.details and result.details.relative_change is not none %} + + {{ "%+.2f%%"|format(result.details.relative_change * 100) }} + + {% else %} + - + {% endif %} + | ++ {% if result.higher_is_better is defined %} + {% if result.higher_is_better %} + ↑ Higher better + {% else %} + ↓ Lower better + {% endif %} + {% else %} + - + {% endif %} + | +
+ {% if result.labels and result.labels.comparison_keys %}
+
+ {% for key, value in result.labels.comparison_keys.items() %}
+ {{ key }}: {{ value }}
+ {% endfor %}
+
+ {% else %}
+ -
+ {% endif %}
+ |
+
+
+
+
+
+
+ |
+
No regression analysis results were found in the provided file.
+| {{ key|title|replace('_', ' ') }} | +
+ {% if key in ['file_path', 'path', 'source'] %}
+ {{ value }}
+
+ {% elif key == 'comparison_keys' and value %}
+
+ {% for item in value %}
+ {{ item }}
+ {% endfor %}
+
+
+ {% elif key == 'ignored_keys' and value %}
+
+ {% for item in value %}
+ {{ item }}
+ {% endfor %}
+
+
+ {% elif key == 'relevant_common_keys' and value %}
+
+ {% for item in value %}
+ {{ item }}
+ {% endfor %}
+
+
+ {% elif key == 'relevant_distinct_keys' and value %}
+
+ {% for item in value %}
+ {% if item %}
+ {{ item }}
+ {% endif %}
+ {% endfor %}
+
+
+ {% elif key == 'relevant_distinct_labels' and value %}
+
+ {% for item in value %}
+ {% if item %}
+ {{ item }}
+ {% endif %}
+ {% endfor %}
+
+
+ {% elif key in ['relevant_count', 'irrelevant_count'] %}
+ {{ value }}
+
+ {% elif key in ['timestamp', 'modified', 'created'] %}
+
+ {% elif key in ['size', 'file_size'] %}
+ {{ value }}
+ {% elif key in ['type', 'format'] %}
+ {{ value }}
+ {% else %}
+ {{ value }}
+ {% endif %}
+ |
+
Input data source information not available
+ This information is populated by the regression analysis engine +| Label | -Value | -
|---|---|
| {escape(label.replace("_", " ").title())} | -{escape(str(value))} | -
| Field | -Value | -
|---|---|
| {display_key} | -{escape(str(value))} | -
| KPI | -Value | -Unit | -Direction | -Description | -
|---|---|---|---|---|
| {escape(kpi["kpi_id"].replace("guidellm_", "").replace("_", " ").title())} | -{escape(str(formatted_value))} | -{escape(str(kpi["unit"]))} | -{escape(direction_text)} | -{escape(str(kpi.get("help", "No description")))} | -
| KPI | -Data Points | -X-Axis | -Y-Axis | -Description | -
|---|---|---|---|---|
| {escape(kpi["kpi_id"].replace("guidellm_", "").replace("_", " ").title())} | -{escape(points_text)} | -{escape(str(kpi.get("x_unit", "unknown")))} - {escape(str(kpi.get("x_help", "")))} | -{escape(str(kpi.get("y_unit", kpi["unit"])))} - {escape(str(kpi.get("y_help", "")))} | -{escape(str(kpi.get("help", "No description")))} | -
Total Tests: {len(test_sections)} test configurations processed
-Test Paths:
-| Label | -Value | -
|---|---|
| {escape(label.replace("_", " ").title())} | -{escape(str(value))} | -
| KPI | -Value | -Unit | -Direction | -Description | -
|---|---|---|---|---|
| {escape(kpi["kpi_id"].replace("guidellm_", "").replace("_", " ").title())} | -{escape(str(formatted_value))} | -{escape(str(kpi["unit"]))} | -{escape(direction_text)} | -{escape(str(kpi.get("help", "No description")))} | -
| KPI | -Data Points | -X-Axis | -Y-Axis | -Description | -
|---|---|---|---|---|
| {escape(kpi["kpi_id"].replace("guidellm_", "").replace("_", " ").title())} | -{escape(points_text)} | -{escape(str(kpi.get("x_unit", "unknown")))} - {escape(str(kpi.get("x_help", "")))} | -{escape(str(kpi.get("y_unit", kpi["unit"])))} - {escape(str(kpi.get("y_help", "")))} | -{escape(str(kpi.get("help", "No description")))} | -
⚠️ Could not load interactive plot for {plot_name}
" + + # Create plot with optional PNG spoiler + result = _create_plot_with_optional_png_spoiler( + plot_html_content, png_path, plot_name, output_dir, html_path + ) + + result_size_kb = len(result.encode("utf-8")) / 1024 + logger.debug(f" ✅ Successfully embedded {plot_name} ({result_size_kb:.1f} KB)") + return result + + +# Filesystem-unsafe characters for path sanitization +_PATH_UNSAFE_CHARS = ["/", "\\", ":", "*", "?", "|", "<", ">", '"'] + + +def sanitize_for_path(text: str) -> str: + """Replace filesystem-unsafe characters in label values.""" + result = str(text) + for char in _PATH_UNSAFE_CHARS: + result = result.replace(char, "_") + return result + def _safe_get_curve_value(curves: dict, metric_name: str, index: int, default: Any = 0.0) -> Any: """ @@ -41,6 +235,30 @@ def _safe_get_curve_value(curves: dict, metric_name: str, index: int, default: A return default +def _custom_configuration_sort_key(config_name: str) -> tuple[int, str]: + """ + Create a sort key that ensures heterogeneous configurations appear before multi-turn. + + Args: + config_name: Configuration name from test_configuration column + + Returns: + Tuple of (priority, config_name) for sorting + """ + # Convert to lowercase for case-insensitive comparison + config_lower = config_name.lower() + + # Priority: 0 = heterogeneous (first), 1 = multi-turn (second), 2 = others (last) + if "heterogeneous" in config_lower: + priority = 0 + elif "multi-turn" in config_lower or "multi_turn" in config_lower: + priority = 1 + else: + priority = 2 + + return (priority, config_name) + + def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.DataFrame: """ Convert Caliper UnifiedResultRecord objects to pandas DataFrame for analysis. @@ -51,18 +269,18 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data Returns: DataFrame with all benchmark metrics and distinguishing labels """ - print(f"📊 Converting {len(records)} records to analysis dataframe...") + logger.info(f"📊 Converting {len(records)} records to analysis dataframe...") data = [] # Get parameters that vary across all records for legend names - print("🔍 Analyzing parameter variations for meaningful legend names...") + logger.info("🔍 Analyzing parameter variations for meaningful legend names...") varying_params = get_varying_parameters(records) if varying_params: - print(f" Found varying parameters: {', '.join(sorted(varying_params))}") + logger.info(f" Found varying parameters: {', '.join(sorted(varying_params))}") else: - print(" No varying parameters found - using default naming") + logger.info(" No varying parameters found - using default naming") - print("📝 Processing records and extracting metrics...") + logger.info("📝 Processing records and extracting metrics...") for record in records: # Skip records without GuideLLM data or missing benchmarks if not record.run_identity.get("guidellm") or record.metrics.get("no_benchmarks_found"): @@ -71,174 +289,126 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data # Create legend name using only varying parameters legend_name = create_legend_name(record, varying_params) - # Check if this is curve data (new format) or scalar data (old format) - has_curves = "performance_curves" in record.metrics + # Extract performance curves data request_rates = record.metrics.get("request_rate", []) - if has_curves and isinstance(request_rates, list) and len(request_rates) > 0: - # New format: expand performance curves into multiple data points - print( - f" 🔄 Expanding performance curves for {legend_name} ({len(request_rates)} points)" - ) - curves = record.metrics.get("performance_curves", {}) - - for i, rate in enumerate(request_rates): - # Create one row per rate point - row = { - # Identity and configuration - "test_configuration": legend_name, - "test_base_path": record.test_base_path, - "rate_point_index": i, - # All distinguishing labels as individual columns - **{f"label_{k}": v for k, v in record.distinguishing_labels.items()}, - # Core performance metrics from curves - "strategy": record.metrics.get("strategy", "unknown"), - "duration": record.metrics.get("duration", 0.0), - "request_concurrency": _safe_get_curve_value( - curves, - "request_concurrency", - i, - record.metrics.get("request_concurrency", 1.0), - ), - "request_rate": rate, - "completed_requests": _safe_get_curve_value(curves, "completed_requests", i, 0), - "failed_requests": _safe_get_curve_value(curves, "failed_requests", i, 0), - # Token metrics from curves - "tokens_per_second": _safe_get_curve_value(curves, "tokens_per_second", i, 0.0), - "input_tokens_per_second": _safe_get_curve_value( - curves, "input_tokens_per_second", i, 0.0 - ), - "output_tokens_per_second": _safe_get_curve_value( - curves, "output_tokens_per_second", i, 0.0 - ), - "input_tokens_per_request": record.metrics.get("input_tokens_per_request", 0.0), - "output_tokens_per_request": record.metrics.get( - "output_tokens_per_request", 0.0 - ), - "total_tokens_per_request": record.metrics.get("total_tokens_per_request", 0.0), - # Latency metrics from curves (convert seconds to ms where needed) - "request_latency_median_ms": _safe_get_curve_value( - curves, "request_latency_median", i, 0.0 - ) - * 1000, - "request_latency_p95_ms": _safe_get_curve_value( - curves, "request_latency_p95", i, 0.0 - ) - * 1000, - "ttft_median_ms": _safe_get_curve_value(curves, "ttft_median", i, 0.0), - "ttft_p10_ms": _safe_get_curve_value(curves, "ttft_p10", i, 0.0), - "ttft_p25_ms": _safe_get_curve_value(curves, "ttft_p25", i, 0.0), - "ttft_p50_ms": _safe_get_curve_value( - curves, "ttft_median", i, 0.0 - ), # p50 = median - "ttft_p75_ms": _safe_get_curve_value(curves, "ttft_p75", i, 0.0), - "ttft_p90_ms": _safe_get_curve_value(curves, "ttft_p90", i, 0.0), - "ttft_p95_ms": _safe_get_curve_value(curves, "ttft_p95", i, 0.0), - "itl_median_ms": _safe_get_curve_value(curves, "itl_median", i, 0.0), - "itl_p10_ms": _safe_get_curve_value(curves, "itl_p10", i, 0.0), - "itl_p25_ms": _safe_get_curve_value(curves, "itl_p25", i, 0.0), - "itl_p50_ms": _safe_get_curve_value( - curves, "itl_median", i, 0.0 - ), # p50 = median - "itl_p75_ms": _safe_get_curve_value(curves, "itl_p75", i, 0.0), - "itl_p90_ms": _safe_get_curve_value(curves, "itl_p90", i, 0.0), - "itl_p95_ms": _safe_get_curve_value(curves, "itl_p95", i, 0.0), - "tpot_median_ms": _safe_get_curve_value(curves, "tpot_median", i, 0.0), - "tpot_p95_ms": _safe_get_curve_value(curves, "tpot_p95", i, 0.0), - # Output token throughput percentiles (not in curves currently, use zeros) - "output_tokens_per_second_p10": 0.0, - "output_tokens_per_second_p25": 0.0, - "output_tokens_per_second_p50": _safe_get_curve_value( - curves, "output_tokens_per_second", i, 0.0 - ), - "output_tokens_per_second_p75": 0.0, - "output_tokens_per_second_p90": 0.0, - } - data.append(row) - else: - # Old format: single data point per record (backward compatibility) - print(f" 📊 Using scalar metrics for {legend_name}") - # Extract scalar request_rate if it's a single value - request_rate = ( - request_rates[0] - if isinstance(request_rates, list) and len(request_rates) > 0 - else record.metrics.get("request_rate", 0.0) - ) + if not ( + isinstance(request_rates, list) + and len(request_rates) > 0 + and "performance_curves" in record.metrics + ): + # Skip records that don't have the expected curve format + logger.info(f" ⚠️ Skipping {legend_name} - no performance curves found") + continue + + # Expand performance curves into multiple data points + logger.info( + f" 🔄 Expanding performance curves for {legend_name} ({len(request_rates)} points)" + ) + curves = record.metrics.get("performance_curves", {}) + for i, rate in enumerate(request_rates): + # Create one row per rate point row = { # Identity and configuration "test_configuration": legend_name, "test_base_path": record.test_base_path, - "rate_point_index": 0, + "rate_point_index": i, # All distinguishing labels as individual columns **{f"label_{k}": v for k, v in record.distinguishing_labels.items()}, - # Core performance metrics + # Core performance metrics from curves "strategy": record.metrics.get("strategy", "unknown"), "duration": record.metrics.get("duration", 0.0), - "request_concurrency": record.metrics.get("request_concurrency", 1.0), - "request_rate": request_rate, - "completed_requests": record.metrics.get("completed_requests", 0), - "failed_requests": record.metrics.get("failed_requests", 0), - # Token metrics - "tokens_per_second": record.metrics.get("tokens_per_second", 0.0), - "input_tokens_per_second": record.metrics.get("input_tokens_per_second", 0.0), - "output_tokens_per_second": record.metrics.get("output_tokens_per_second", 0.0), - "input_tokens_per_request": record.metrics.get("input_tokens_per_request", 0.0), - "output_tokens_per_request": record.metrics.get("output_tokens_per_request", 0.0), - "total_tokens_per_request": record.metrics.get("total_tokens_per_request", 0.0), - # Latency metrics (in ms for consistency with topsail) - "request_latency_median_ms": record.metrics.get("request_latency_median", 0.0) - * 1000, - "request_latency_p95_ms": record.metrics.get("request_latency_p95", 0.0) * 1000, - "ttft_median_ms": record.metrics.get("ttft_median", 0.0), - "ttft_p10_ms": record.metrics.get("ttft_p10", 0.0), - "ttft_p25_ms": record.metrics.get("ttft_p25", 0.0), - "ttft_p50_ms": record.metrics.get("ttft_p50", 0.0), - "ttft_p75_ms": record.metrics.get("ttft_p75", 0.0), - "ttft_p90_ms": record.metrics.get("ttft_p90", 0.0), - "ttft_p95_ms": record.metrics.get("ttft_p95", 0.0), - "itl_median_ms": record.metrics.get("itl_median", 0.0), - "itl_p10_ms": record.metrics.get("itl_p10", 0.0), - "itl_p25_ms": record.metrics.get("itl_p25", 0.0), - "itl_p50_ms": record.metrics.get("itl_p50", 0.0), - "itl_p75_ms": record.metrics.get("itl_p75", 0.0), - "itl_p90_ms": record.metrics.get("itl_p90", 0.0), - "itl_p95_ms": record.metrics.get("itl_p95", 0.0), - "tpot_median_ms": record.metrics.get("tpot_median", 0.0), - "tpot_p95_ms": record.metrics.get("tpot_p95", 0.0), - # Output token throughput percentiles - "output_tokens_per_second_p10": record.metrics.get( - "output_tokens_per_second_p10", 0.0 + "request_concurrency": _safe_get_curve_value( + curves, + "request_concurrency", + i, + record.metrics.get("request_concurrency", 1.0), ), - "output_tokens_per_second_p25": record.metrics.get( - "output_tokens_per_second_p25", 0.0 + "intended_concurrency": _safe_get_curve_value( + curves, + "intended_concurrency", + i, + record.metrics.get("request_concurrency", 1.0), ), - "output_tokens_per_second_p50": record.metrics.get( - "output_tokens_per_second_p50", 0.0 + "request_rate": rate, + "completed_requests": _safe_get_curve_value(curves, "completed_requests", i, 0), + "failed_requests": _safe_get_curve_value(curves, "failed_requests", i, 0), + # Token metrics from curves + "tokens_per_second": _safe_get_curve_value(curves, "tokens_per_second", i, 0.0), + "input_tokens_per_second": _safe_get_curve_value( + curves, "input_tokens_per_second", i, 0.0 ), - "output_tokens_per_second_p75": record.metrics.get( - "output_tokens_per_second_p75", 0.0 + "output_tokens_per_second": _safe_get_curve_value( + curves, "output_tokens_per_second", i, 0.0 ), - "output_tokens_per_second_p90": record.metrics.get( - "output_tokens_per_second_p90", 0.0 + "input_tokens_per_request": record.metrics.get("input_tokens_per_request", 0.0), + "output_tokens_per_request": record.metrics.get("output_tokens_per_request", 0.0), + "total_tokens_per_request": record.metrics.get("total_tokens_per_request", 0.0), + # Latency metrics from curves (convert seconds to ms where needed) + "request_latency_median_ms": _safe_get_curve_value( + curves, "request_latency_median", i, 0.0 + ) + * 1000, + "request_latency_p95_ms": _safe_get_curve_value( + curves, "request_latency_p95", i, 0.0 + ) + * 1000, + "ttft_median_ms": _safe_get_curve_value(curves, "ttft_median", i, 0.0), + "ttft_p10_ms": _safe_get_curve_value(curves, "ttft_p10", i, 0.0), + "ttft_p25_ms": _safe_get_curve_value(curves, "ttft_p25", i, 0.0), + "ttft_p50_ms": _safe_get_curve_value(curves, "ttft_median", i, 0.0), # p50 = median + "ttft_p75_ms": _safe_get_curve_value(curves, "ttft_p75", i, 0.0), + "ttft_p90_ms": _safe_get_curve_value(curves, "ttft_p90", i, 0.0), + "ttft_p95_ms": _safe_get_curve_value(curves, "ttft_p95", i, 0.0), + "itl_median_ms": _safe_get_curve_value(curves, "itl_median", i, 0.0), + "itl_p10_ms": _safe_get_curve_value(curves, "itl_p10", i, 0.0), + "itl_p25_ms": _safe_get_curve_value(curves, "itl_p25", i, 0.0), + "itl_p50_ms": _safe_get_curve_value(curves, "itl_median", i, 0.0), # p50 = median + "itl_p75_ms": _safe_get_curve_value(curves, "itl_p75", i, 0.0), + "itl_p90_ms": _safe_get_curve_value(curves, "itl_p90", i, 0.0), + "itl_p95_ms": _safe_get_curve_value(curves, "itl_p95", i, 0.0), + "tpot_median_ms": _safe_get_curve_value(curves, "tpot_median", i, 0.0), + "tpot_p95_ms": _safe_get_curve_value(curves, "tpot_p95", i, 0.0), + # Output token throughput percentiles (not in curves currently, use zeros) + "output_tokens_per_second_p10": 0.0, + "output_tokens_per_second_p25": 0.0, + "output_tokens_per_second_p50": _safe_get_curve_value( + curves, "output_tokens_per_second", i, 0.0 ), + "output_tokens_per_second_p75": 0.0, + "output_tokens_per_second_p90": 0.0, } data.append(row) if not data: - print("⚠️ No GuideLLM data found in records") + logger.info("⚠️ No GuideLLM data found in records") return pd.DataFrame() - print(f"✅ Successfully processed {len(data)} data points from {len(records)} records") + logger.info(f"✅ Successfully processed {len(data)} data points from {len(records)} records") df = pd.DataFrame(data) # Sort for consistent ordering - print("📋 Organizing data by configuration, concurrency, and request rate...") + logger.info("📋 Organizing data by configuration, concurrency, and request rate...") + # Add custom sort column to ensure heterogeneous comes before multi-turn + df["_config_sort_key"] = df["test_configuration"].apply( + lambda x: _custom_configuration_sort_key(x)[0] + ) + # Sort and fill any NaN values in numeric columns with 0 for consistent plotting df = df.sort_values( - ["test_configuration", "request_concurrency", "request_rate", "rate_point_index"] + [ + "_config_sort_key", + "test_configuration", + "intended_concurrency", + "request_rate", + "rate_point_index", + ] ) + # Remove the temporary sort column + df = df.drop(columns=["_config_sort_key"]) + # Fill NaN values in numeric columns with appropriate defaults numeric_columns = [ col @@ -249,8 +419,8 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data if col in df.columns: df[col] = pd.to_numeric(df[col], errors="coerce").fillna(0) - # Show what we found - configs = df["test_configuration"].unique() + # Show what we found - preserve custom sort order + configs = df["test_configuration"].drop_duplicates().tolist() total_records = len( [ r @@ -259,7 +429,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data ] ) - print( + logger.info( f"🎯 Expanded {total_records} benchmark records into {len(df)} data points across {len(configs)} configurations:" ) for config in configs: @@ -270,7 +440,7 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data if rate_points > 1 else f"{config_data['request_rate'].iloc[0]:.1f} req/s" ) - print(f" • {config}: {rate_points} rate points ({rate_range})") + logger.info(f" - {config}: {rate_points} rate points ({rate_range})") return df @@ -279,18 +449,21 @@ def create_dataframe_from_records(records: list[UnifiedResultRecord]) -> pd.Data def create_throughput_scaling_plot(df: pd.DataFrame, title_context: str = ""): """Create throughput scaling scatter plot.""" try: - print("📈 Creating throughput scaling scatter plot...") + logger.info("📈 Creating throughput scaling scatter plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for throughput scaling plot") + logger.info("⚠️ No data available for throughput scaling plot") return None title = f"Request Throughput vs Concurrency by Configuration{title_context}" + # Get ordered configuration list to maintain consistent legend order + config_order = df["test_configuration"].drop_duplicates().tolist() + fig = px.scatter( df, - x="request_concurrency", + x="intended_concurrency", y="request_rate", color="test_configuration", size="tokens_per_second", @@ -299,83 +472,119 @@ def create_throughput_scaling_plot(df: pd.DataFrame, title_context: str = ""): "request_latency_median_ms": ":.1f", "ttft_median_ms": ":.1f", "tokens_per_second": ":.0f", + "request_concurrency": ":.1f", # Show achieved concurrency in hover }, title=title, labels={ - "request_concurrency": "Concurrency Level", + "intended_concurrency": "Concurrency Level (Requested)", "request_rate": "Request Rate (req/s)", "test_configuration": "Configuration", + "request_concurrency": "Achieved Concurrency", }, + category_orders={"test_configuration": config_order}, ) fig.update_traces(textposition="top center") - fig.update_layout( - showlegend=True, width=800, height=500, font={"size": 12}, title_font_size=16 - ) + fig.update_layout(showlegend=True, **PLOT_CONFIG) fig.update_yaxes(rangemode="tozero") - print("✅ Throughput scaling plot created successfully") + logger.info("✅ Throughput scaling plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create throughput scaling plot: {e}") + logger.info(f"❌ Failed to create throughput scaling plot: {e}") return None def create_latency_vs_throughput_plot(df: pd.DataFrame, title_context: str = ""): """Create latency vs throughput scatter plot.""" try: - print("📈 Creating latency vs throughput trade-off plot...") + logger.info("📈 Creating latency vs throughput trade-off plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for latency vs throughput plot") + logger.info("⚠️ No data available for latency vs throughput plot") return None title = f"Latency vs Throughput Trade-off{title_context}" + # Get ordered configuration list to maintain consistent legend order + config_order = df["test_configuration"].drop_duplicates().tolist() + fig = px.scatter( df, x="request_rate", y="request_latency_median_ms", color="test_configuration", size="tokens_per_second", - hover_data={"strategy": True, "request_concurrency": True, "ttft_median_ms": ":.1f"}, + hover_data={ + "strategy": True, + "intended_concurrency": ":.1f", + "request_concurrency": ":.1f", + "ttft_median_ms": ":.1f", + }, title=title, labels={ "request_rate": "Request Rate (req/s)", "request_latency_median_ms": "Latency (ms)", "test_configuration": "Configuration", }, + category_orders={"test_configuration": config_order}, ) - fig.update_layout( - showlegend=True, width=800, height=500, font={"size": 12}, title_font_size=16 - ) + fig.update_layout(showlegend=True, **PLOT_CONFIG) - print("✅ Latency vs throughput plot created successfully") + logger.info("✅ Latency vs throughput plot created successfully") return fig except Exception as e: - print(f"❌ Failed to create latency vs throughput plot: {e}") + logger.info(f"❌ Failed to create latency vs throughput plot: {e}") return None def create_token_throughput_vs_concurrency_plot(df: pd.DataFrame, title_context: str = ""): """Create token throughput vs concurrency line plot.""" try: - print("📈 Creating token throughput vs concurrency scaling plot...") + logger.info("📈 Creating token throughput vs concurrency scaling plot...") import plotly.express as px if df.empty: - print("⚠️ No data available for token throughput vs concurrency plot") + logger.info("⚠️ No data available for token throughput vs concurrency plot") return None - title = f"Token Throughput vs Concurrency{title_context}Click on the image to open the interactive full-size view of the plot.
+ In the interactive view, click in the legend to hide a line, double click to see only this line.
{{ page_description }}
+{{ description }}
+ {{ embed_plot_for_report(png_path, html_path, plot_name, output_dir) }} +{{ description }}
+ {{ embed_plot_for_report(png_path, html_path, plot_name, output_dir) }} +