From ae9b8c6d6d6fdf71ba56857f6412a34eb7df956e Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Tue, 21 Jul 2026 13:57:23 -0400 Subject: [PATCH 01/40] Sync evaluator with VAKRA evaluator updates --- benchmarks/m3/evaluator/constant.py | 30 +- benchmarks/m3/evaluator/domain_mapping.json | 82 ------ benchmarks/m3/evaluator/evaluator.py | 238 +++++++-------- benchmarks/m3/evaluator/judge.py | 197 ++++++------- benchmarks/m3/evaluator/mcp_tools.py | 99 +++---- benchmarks/m3/evaluator/prompt.py | 13 +- benchmarks/m3/evaluator/scorer.py | 308 ++++++++++---------- benchmarks/m3/evaluator/utils.py | 12 +- 8 files changed, 422 insertions(+), 557 deletions(-) delete mode 100644 benchmarks/m3/evaluator/domain_mapping.json diff --git a/benchmarks/m3/evaluator/constant.py b/benchmarks/m3/evaluator/constant.py index c3799084..56f2ad49 100644 --- a/benchmarks/m3/evaluator/constant.py +++ b/benchmarks/m3/evaluator/constant.py @@ -1,19 +1,21 @@ from __future__ import annotations +N_TOOL_CALLS_PER_TURN=20 + # Predicted file keys -PRED_UUID = "uuid" -PRED_DOMAIN = "domain" -PRED_OUTPUT_KEY = "output" -PRED_OUTPUT_TURN_ID_KEY = "turn_id" -PRED_OUTPUT_QUERY_KEY = "query" -PRED_OUTPUT_ANSWER_KEY = "answer" -PRED_OUTPUT_SEQUENCE_KEY = "sequence" +PRED_UUID="uuid" +PRED_DOMAIN="domain" +PRED_OUTPUT_KEY="output" +PRED_OUTPUT_TURN_ID_KEY="turn_id" +PRED_OUTPUT_QUERY_KEY="query" +PRED_OUTPUT_ANSWER_KEY="answer" +PRED_OUTPUT_SEQUENCE_KEY= "sequence" # Ground Truth file keys -GT_UUID = "uuid" -GT_DOMAIN = "domain" -GT_OUTPUT_KEY = "output" -GT_OUTPUT_TURN_ID_KEY = "turn_id" -GT_OUTPUT_QUERY_KEY = "query" -GT_OUTPUT_ANSWER_KEY = "answer" -GT_OUTPUT_SEQUENCE_KEY = "sequence" +GT_UUID="uuid" +GT_DOMAIN="domain" +GT_OUTPUT_KEY="output" +GT_OUTPUT_TURN_ID_KEY="turn_id" +GT_OUTPUT_QUERY_KEY="query" +GT_OUTPUT_ANSWER_KEY="answer" +GT_OUTPUT_SEQUENCE_KEY= "sequence" diff --git a/benchmarks/m3/evaluator/domain_mapping.json b/benchmarks/m3/evaluator/domain_mapping.json deleted file mode 100644 index 60b07f34..00000000 --- a/benchmarks/m3/evaluator/domain_mapping.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "california_schools" : "Education & Students", - "student_club": "Education & Students", - "student_loan": "Education & Students", - "university": "Education & Students", - "cs_semester": "Education & Students", - "computer_student": "Education & Students", - "college_completion" : "Education & Students", - "movie" : "Movies, TV & Entertainment, Social Media", - "movie_platform": "Movies, TV & Entertainment, Social Media", - "movie_3": "Movies, TV & Entertainment, Social Media", - "movies_4": "Movies, TV & Entertainment, Social Media", - "simpson_episodes": "Movies, TV & Entertainment, Social Media", - "disney": "Movies, TV & Entertainment, Social Media", - "law_episode": "Movies, TV & Entertainment, Social Media", - "movielens": "Movies, TV & Entertainment, Social Media", - "social_media": "Movies, TV & Entertainment, Social Media", - "talkingdata": "Movies, TV & Entertainment, Social Media", - "music_tracker": "Movies, TV & Entertainment, Social Media", - "music_platform_2": "Movies, TV & Entertainment, Social Media", - "card_games": "Movies, TV & Entertainment, Social Media", - "video_games": "Movies, TV & Entertainment, Social Media", - "superhero": "Movies, TV & Entertainment, Social Media", - "formula_1": "Sports & Athletics", - "professional_basketball": "Sports & Athletics", - "european_football_1": "Sports & Athletics", - "european_football_2": "Sports & Athletics", - "olympics": "Sports & Athletics", - "ice_hockey_draft": "Sports & Athletics", - "hockey": "Sports & Athletics", - "soccer_2016": "Sports & Athletics", - "retails": "Retail, Sales & Commerce", - "retail_world": "Retail, Sales & Commerce", - "retail_complains": "Retail, Sales & Commerce", - "superstore": "Retail, Sales & Commerce", - "regional_sales": "Retail, Sales & Commerce", - "car_retails": "Retail, Sales & Commerce", - "book_publishing_company": "Retail, Sales & Commerce", - "sales": "Retail, Sales & Commerce", - "sales_in_weather": "Retail, Sales & Commerce", - "works_cycles": "Retail, Sales & Commerce", - "restaurant": "Food, Restaurants & Beverage", - "food_inspection": "Food, Restaurants & Beverage", - "food_inspection_2": "Food, Restaurants & Beverage", - "cookbook": "Food, Restaurants & Beverage", - "beer_factory": "Food, Restaurants & Beverage", - "craftbeer": "Food, Restaurants & Beverage", - "menu": "Food, Restaurants & Beverage", - "app_store": "Technology & Software", - "codebase_community": "Technology & Software", - "codebase_comments": "Technology & Software", - "software_company": "Technology & Software", - "image_and_language": "Technology & Software", - "toxicology": "Health & Medicine", - "synthea": "Health & Medicine", - "mental_health_survey": "Health & Medicine", - "thrombosis_prediction": "Health & Medicine", - "genes": "Health & Medicine", - "mondial_geo": "Geography & Demographics", - "world": "Geography & Demographics", - "address": "Geography & Demographics", - "world_development_indicators": "Geography & Demographics", - "airline": "Transportation & Mobility", - "trains": "Transportation & Mobility", - "bike_share_1": "Transportation & Mobility", - "cars": "Transportation & Mobility", - "shipping": "Transportation & Mobility", - "financial": "Finance & Economics", - "donor": "Finance & Economics", - "coinmarketcap": "Finance & Economics", - "debit_card_specializing": "Finance & Economics", - "legislator": "Government, Law & Public Services, Human Capital", - "chicago_crime": "Government, Law & Public Services, Human Capital", - "shooting": "Government, Law & Public Services, Human Capital", - "public_review_platform": "Government, Law & Public Services, Human Capital", - "human_resources": "Government, Law & Public Services, Human Capital", - "books": "Literature, Language & Publishing", - "authors": "Literature, Language & Publishing", - "shakespeare": "Literature, Language & Publishing", - "language_corpus": "Literature, Language & Publishing", - "citeseer": "Literature, Language & Publishing" -} \ No newline at end of file diff --git a/benchmarks/m3/evaluator/evaluator.py b/benchmarks/m3/evaluator/evaluator.py index 6908ea4f..2274b9bb 100644 --- a/benchmarks/m3/evaluator/evaluator.py +++ b/benchmarks/m3/evaluator/evaluator.py @@ -5,15 +5,15 @@ import json from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, Optional, Tuple, List +from tqdm import tqdm +from judge import CorrectnessJudge, GroundednessJudge, ExactMatchJudge from benchmark.mcp_client import ( MCPConnectionConfig, - create_client_and_connect, load_mcp_config, + create_client_and_connect, ) -from constant import PRED_OUTPUT_KEY, PRED_OUTPUT_SEQUENCE_KEY -from judge import CorrectnessJudge, ExactMatchJudge, GroundednessJudge from mcp_tools import ( execute_tools_batch, extract_toolcalls_for_mcp, @@ -25,10 +25,11 @@ TurnScorer, TurnScorerConfig, ) -from tqdm import tqdm -from utils import pair_dialogues_by_uuid, read_domain_file +from constant import PRED_OUTPUT_KEY, PRED_OUTPUT_SEQUENCE_KEY +from constant import N_TOOL_CALLS_PER_TURN +from utils import read_domain_file, pair_dialogues_by_uuid -CAPABILITY_MCP_TOOL_MAP = { +CAPABILITY_MCP_TOOL_MAP={ "capability_bi_apis": 1, "capability_dashboard_apis": 2, "capability_multihop_reasoning": 3, @@ -42,7 +43,8 @@ @dataclass(frozen=True) class CapabilityPolicy: - dialogue_aggregate: str = "mean" # "mean" | "sum" | "min" + + dialogue_aggregate: str = "mean" # "mean" | "sum" | "min" # --- MCP execution --- execute_mcp_tools: bool = True @@ -53,32 +55,35 @@ class CapabilityPolicy: exactmatch_judge: Any = ExactMatchJudge(config={}) + def build_default_capability_registry() -> Dict[str, CapabilityPolicy]: return { "capability_bi_apis": CapabilityPolicy( execute_mcp_tools=True, correctness_judge=CorrectnessJudge(config={}), groundedness_judge=GroundednessJudge(config={}), - exactmatch_judge=ExactMatchJudge(config={}), + exactmatch_judge=ExactMatchJudge(config={}) ), + "capability_dashboard_apis": CapabilityPolicy( execute_mcp_tools=True, correctness_judge=CorrectnessJudge(config={}), groundedness_judge=GroundednessJudge(config={}), - exactmatch_judge=ExactMatchJudge(config={}), + exactmatch_judge=ExactMatchJudge(config={}) ), + "capability_multihop_reasoning": CapabilityPolicy( execute_mcp_tools=True, correctness_judge=CorrectnessJudge(config={}), groundedness_judge=GroundednessJudge(config={}), - exactmatch_judge=ExactMatchJudge(config={}), + exactmatch_judge=ExactMatchJudge(config={}) ), "capability_multiturn": CapabilityPolicy( execute_mcp_tools=True, correctness_judge=CorrectnessJudge(config={}), groundedness_judge=GroundednessJudge(config={}), - exactmatch_judge=ExactMatchJudge(config={}), - ), + exactmatch_judge=ExactMatchJudge(config={}) + ), } @@ -102,10 +107,9 @@ def _prepend_get_data_to_batch( result.append(dialogue_tools) continue first_turn = list(dialogue_tools[0]) - if ( - skip_initialize_active_data - and first_turn - and first_turn[0].get("name") == "initialize_active_data" + if skip_initialize_active_data and first_turn and ( + first_turn[0].get("name") == "initialize_active_data" + or first_turn[0].get("name") == "get_data" ): first_turn = first_turn[1:] first_turn = [{"name": "get_data", "arguments": {"tool_universe_id": uuid}}] + first_turn @@ -141,33 +145,6 @@ def _update_dialogue_toolcall_for_get_data( # ----------------------------- -def _make_missing_dialogue_entry( - uuid: str, - capability_name: str, - domain: str, - policy: CapabilityPolicy, -) -> Dict[str, Any]: - return { - "uuid": uuid, - "score": 0.0, - "metadata": { - "capability": capability_name, - "domain": domain, - "policy": { - "dialogue_aggregate": policy.dialogue_aggregate, - "execute_mcp_tools": policy.execute_mcp_tools, - }, - "error": "missing_prediction", - }, - "details": { - "dialogue_score": 0.0, - "num_turns": 0, - "per_turn": [], - "aggregate": policy.dialogue_aggregate, - }, - } - - async def evaluate_domain( domain: str, gt_path: Path, @@ -187,12 +164,9 @@ async def evaluate_domain( pred_list = read_domain_file(pred_path) if pred_path.exists() else [] # Pair dialogues - paired, missing_pred, extra_pred = pair_dialogues_by_uuid(gt_list, pred_list) + paired, missing_pred, extra_pred = pair_dialogues_by_uuid(gt_list, pred_list) if len(pred_list) == 0: - zero_dialogues = [ - _make_missing_dialogue_entry(uuid, capability_name, domain, policy) for uuid in missing_pred - ] domain_out: Dict[str, Any] = { "domain": domain, "n_groundtruth": len(gt_list), @@ -200,16 +174,16 @@ async def evaluate_domain( "n_paired": len(paired), "missing_prediction_uuids": missing_pred, "extra_prediction_uuids": extra_pred, - "dialogues": zero_dialogues, - "summary": { - "num_samples": len(gt_list), - "num_correct": 0.0, - "mean_dialogue_score": 0.0, - "min_dialogue_score": 0.0, - "max_dialogue_score": 0.0, - }, - } - return domain_out, [0.0] * len(zero_dialogues) + "dialogues": [], + "summary":{ + "num_samples": len(gt_list), + "num_correct": 0.0, + "mean_dialogue_score": 0.0, + "min_dialogue_score": 0.0, + "max_dialogue_score": 0.0, + } + } + return domain_out, [] # Build scorers per domain turn_cfg = TurnScorerConfig( @@ -220,7 +194,7 @@ async def evaluate_domain( cfg=turn_cfg, correctness_judge=policy.correctness_judge, groundedness_judge=policy.groundedness_judge, - exactmatch_judge=policy.exactmatch_judge, + exactmatch_judge=policy.exactmatch_judge ) dialogue_scorer = DialogueScorer( turn_scorer=turn_scorer, @@ -249,25 +223,26 @@ async def evaluate_domain( schema_map = {tool.name: tool.inputSchema for tool in tools_result.tools} # Batch execute tools - batch_tools_pred = [extract_toolcalls_for_mcp(pr) for _, pr in paired] + batch_tools_pred = [extract_toolcalls_for_mcp(pr, limit_last_n=N_TOOL_CALLS_PER_TURN) for _, pr in paired] batch_tools_gt = [extract_toolcalls_for_mcp(gt) for gt, _ in paired] # capability_bi_apis: replace initialize_active_data with get_data (GT), # and prepend get_data (pred), then sync dialogue tool_call sequences if capability_name == "capability_bi_apis": uuids = [str(gt.get("uuid")) for gt, _ in paired] - batch_tools_gt = _prepend_get_data_to_batch( - batch_tools_gt, uuids, skip_initialize_active_data=True - ) - batch_tools_pred = _prepend_get_data_to_batch( - batch_tools_pred, uuids, skip_initialize_active_data=False - ) + batch_tools_gt = _prepend_get_data_to_batch(batch_tools_gt, uuids, skip_initialize_active_data=True) + batch_tools_pred = _prepend_get_data_to_batch(batch_tools_pred, uuids, skip_initialize_active_data=False) for (gt_raw, pr_raw), uuid in zip(paired, uuids): _update_dialogue_toolcall_for_get_data(gt_raw, uuid, skip_initialize_active_data=True) _update_dialogue_toolcall_for_get_data(pr_raw, uuid, skip_initialize_active_data=False) mcp_batch_responses_pred = await execute_tools_batch(session, batch_tools_pred, schema_map) - mcp_batch_responses_gt = await execute_tools_batch(session, batch_tools_gt, schema_map) + mcp_batch_responses_gt = await execute_tools_batch( + session, + batch_tools_gt, + schema_map, + is_gt=capability_name == "capability_bi_apis", + ) # Score each paired dialogue for idx, (gt_raw, pr_raw) in enumerate( @@ -276,48 +251,19 @@ async def evaluate_domain( uuid = str(gt_raw.get("uuid")) # Inject fresh responses so groundedness judge uses tool outputs - inject_mcp_responses( - pr_raw, mcp_batch_responses_pred[idx], type="pred", capability_name=capability_name - ) - inject_mcp_responses( - gt_raw, mcp_batch_responses_gt[idx], type="gt", capability_name=capability_name - ) + inject_mcp_responses(pr_raw, mcp_batch_responses_pred[idx], type="pred", capability_name=capability_name) + if capability_name != "capability_bi_apis" and mcp_batch_responses_gt is not None: + inject_mcp_responses(gt_raw, mcp_batch_responses_gt[idx], type="gt",capability_name=capability_name) # Score and store details dialogue_score, dialogue_details = dialogue_scorer.score( - gt_dialogue=gt_raw, pred_dialogue=pr_raw, pred_key=PRED_OUTPUT_KEY + gt_dialogue=gt_raw, + pred_dialogue=pr_raw, + pred_key=PRED_OUTPUT_KEY ) dialogue_scores.append(float(dialogue_score)) - domain_out["dialogues"].append( - { - "uuid": uuid, - "score": float(dialogue_score), - "metadata": { - "capability": capability_name, - "domain": domain, - "policy": { - "dialogue_aggregate": policy.dialogue_aggregate, - "execute_mcp_tools": policy.execute_mcp_tools, - }, - }, - "details": dialogue_details, - } - ) - else: - # No MCP tools - just score based on predictions as-is - for idx, (gt_raw, pr_raw) in enumerate( - tqdm(paired, desc=f"[{capability_name}][{domain}]", leave=False) - ): - uuid = str(gt_raw.get("uuid")) - - dialogue_score, dialogue_details = dialogue_scorer.score( - gt_dialogue=gt_raw, pred_dialogue=pr_raw, pred_key=PRED_OUTPUT_KEY - ) - dialogue_scores.append(float(dialogue_score)) - - domain_out["dialogues"].append( - { + domain_out["dialogues"].append({ "uuid": uuid, "score": float(dialogue_score), "metadata": { @@ -329,29 +275,47 @@ async def evaluate_domain( }, }, "details": dialogue_details, - } + }) + else: + # No MCP tools - just score based on predictions as-is + for idx, (gt_raw, pr_raw) in enumerate( + tqdm(paired, desc=f"[{capability_name}][{domain}]", leave=False) + ): + uuid = str(gt_raw.get("uuid")) + + dialogue_score, dialogue_details = dialogue_scorer.score( + gt_dialogue=gt_raw, + pred_dialogue=pr_raw, + pred_key=PRED_OUTPUT_KEY ) + dialogue_scores.append(float(dialogue_score)) - # Penalize missing predictions as zero-scored dialogues so per-domain and - # top-level summaries share a denominator of len(gt_list). - for uuid in missing_pred: - domain_out["dialogues"].append(_make_missing_dialogue_entry(uuid, capability_name, domain, policy)) - dialogue_scores.append(0.0) + domain_out["dialogues"].append({ + "uuid": uuid, + "score": float(dialogue_score), + "metadata": { + "capability": capability_name, + "domain": domain, + "policy": { + "dialogue_aggregate": policy.dialogue_aggregate, + "execute_mcp_tools": policy.execute_mcp_tools, + }, + }, + "details": dialogue_details, + }) # Domain summary domain_scores = [d["score"] for d in domain_out["dialogues"]] - num_samples = len(gt_list) domain_out["summary"] = { - "num_samples": num_samples, + "num_samples": len(gt_list), "num_correct": sum(domain_scores), - "mean_dialogue_score": (sum(domain_scores) / num_samples) if num_samples else 0.0, + "mean_dialogue_score": (sum(domain_scores) / len(domain_scores)) if domain_scores else 0.0, "min_dialogue_score": min(domain_scores) if domain_scores else 0.0, "max_dialogue_score": max(domain_scores) if domain_scores else 0.0, } return domain_out, dialogue_scores - def _load_existing_results(out_path: Path) -> Optional[Dict[str, Any]]: """ Load existing results file if it exists. @@ -385,14 +349,24 @@ def _write_intermediate_results( This allows recovery of partial results if evaluation is interrupted. """ # Calculate totals from completed domains - total_paired = sum(d["n_paired"] for d in results["domains"].values()) - total_missing = sum(len(d["missing_prediction_uuids"]) for d in results["domains"].values()) - total_extra = sum(len(d["extra_prediction_uuids"]) for d in results["domains"].values()) + total_paired = sum( + d["n_paired"] for d in results["domains"].values() + ) + total_missing = sum( + len(d["missing_prediction_uuids"]) + for d in results["domains"].values() + ) + total_extra = sum( + len(d["extra_prediction_uuids"]) + for d in results["domains"].values() + ) - total_samples = sum(d["summary"]["num_samples"] for d in results["domains"].values()) + total_samples = sum( + d["summary"]["num_samples"] for d in results["domains"].values() + ) # Update summary with current progress - num_correct = sum(all_dialogue_scores) if all_dialogue_scores else 0.0 + num_correct=(sum(all_dialogue_scores) if all_dialogue_scores else 0.0) results["summary"] = { "n_domains": len(results["domains"]), "n_paired_dialogues": total_paired, @@ -400,9 +374,16 @@ def _write_intermediate_results( "n_extra_predictions": total_extra, "n_samples": total_samples, "n_correct": num_correct, - "mean_dialogue_score": (num_correct / total_samples if total_samples else 0.0), - "min_dialogue_score": (min(all_dialogue_scores) if all_dialogue_scores else 0.0), - "max_dialogue_score": (max(all_dialogue_scores) if all_dialogue_scores else 0.0), + "mean_dialogue_score": (num_correct/total_samples + if all_dialogue_scores + else 0.0 + ), + "min_dialogue_score": ( + min(all_dialogue_scores) if all_dialogue_scores else 0.0 + ), + "max_dialogue_score": ( + max(all_dialogue_scores) if all_dialogue_scores else 0.0 + ), } # Write to disk @@ -438,7 +419,9 @@ def evaluate_capability( missing = selected_domains - {p.stem for p in all_gt_files} if missing: - raise ValueError(f"Requested domains not found in groundtruth_dir: {sorted(missing)}") + raise ValueError( + f"Requested domains not found in groundtruth_dir: {sorted(missing)}" + ) else: gt_files = all_gt_files @@ -498,7 +481,7 @@ def evaluate_capability( # Final write (summary already updated by last intermediate write) # This is technically redundant but ensures the final state is written - results = _write_intermediate_results(results, all_dialogue_scores, out_path) + results=_write_intermediate_results(results, all_dialogue_scores, out_path) print("=========================================================================") print("==================================[RESULTS]==============================") @@ -520,26 +503,23 @@ def evaluate_capability( # CLI # ----------------------------- - def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--capability_name", required=True, help="Capability name (must exist in registry)") ap.add_argument("--gt_root", required=True, help="Path to capability_name/groundtruth/") ap.add_argument("--pred_root", required=True, help="Path to capability_name/prediction/") - ap.add_argument( - "--output", default=None, help="Output results.json path (default: /results.json)" - ) + ap.add_argument("--output", default=None, help="Output results.json path (default: /results.json)") ap.add_argument( "--mcp-config", default="benchmark/mcp_connection_config.yaml", - help="Path to MCP connection config YAML file (default: benchmark/mcp_connection_config.yaml)", + help="Path to MCP connection config YAML file (default: benchmark/mcp_connection_config.yaml)" ) ap.add_argument( "--domains", nargs="+", default=None, help="Optional list of domain names to evaluate (without .json extension). " - "If omitted, all domains are evaluated.", + "If omitted, all domains are evaluated." ) args = ap.parse_args() diff --git a/benchmarks/m3/evaluator/judge.py b/benchmarks/m3/evaluator/judge.py index 36a4444f..362c8c01 100644 --- a/benchmarks/m3/evaluator/judge.py +++ b/benchmarks/m3/evaluator/judge.py @@ -1,40 +1,31 @@ from __future__ import annotations -import json -import os import re -from typing import Any, List, Optional - -from langchain_openai import ChatOpenAI -from prompt import CorrectnessPrompt, GroundednessPrompt +import os +import json +from copy import deepcopy +from prompt import GroundednessPrompt, CorrectnessPrompt from utils import JudgeInput, JudgeOutput +from constant import N_TOOL_CALLS_PER_TURN +from langchain_openai import ChatOpenAI +from typing import Any, Dict, Optional, Tuple _LABEL_RE = re.compile(r"\b(yes|partial|no|unsure)\b", re.IGNORECASE) _CONCLUSION_RE = re.compile(r"\s*(.*?)\s*", re.IGNORECASE | re.DOTALL) _SCORE_MAP = {"yes": 1.0, "partial": 0.0, "no": 0.0, "unsure": 0.0} -N_TOOL_CALLS_PER_TURN = 20 - - class JudgeOutputParseError(ValueError): pass - class JudgeValidationError(ValueError): """Raised when a judge returns an unexpected/invalid score.""" - pass - -class ChatModel(ChatOpenAI): - """ - openai/gpt-oss-120b chat model is being used as LLM-as-a-judge using langchain-openai. - Groq-backed OpenAI-compatible chat model. - """ +class ChatGroq(ChatOpenAI): + """Groq-backed OpenAI-compatible chat model.""" def __init__(self, config: dict): - # Set model with model or model_name model_name = config.get("model_name", "openai/gpt-oss-120b") end_point = config.get("end_point", "https://api.groq.com/openai") @@ -44,81 +35,62 @@ def __init__(self, config: dict): params = config.get("params", {}) - # Set default values for overriding fields - config = {} - config.setdefault("model", model_name) - config.setdefault("api_key", api_key) - config.setdefault("base_url", end_point.rstrip("/") + "/v1") - config.setdefault("temperature", 0) - - config.update(params) + groq_config = {} + groq_config.setdefault("model", model_name) + groq_config.setdefault("api_key", api_key) + groq_config.setdefault("base_url", end_point.rstrip("/") + "/v1") + groq_config.setdefault("temperature", 0) + groq_config.update(params) - super().__init__(**config) + super().__init__(**groq_config) +class ChatRits(ChatOpenAI): + """RITS chat model integration using langchain-openai.""" -class LiteLLMChatModel(ChatOpenAI): - """ - Azure/gpt-4.1 served through the IBM LiteLLM proxy, used as LLM-as-a-judge. - - Default judge backend (see ``_build_judge_llm``). The Groq ``ChatModel`` above - is retained and still selectable via ``JUDGE_BACKEND=groq`` (or gpt-oss). - Reads the same proxy env the agent's gpt4.1 profile uses: ``OPENAI_BASE_URL`` - + ``OPENAI_API_KEY``. Override the model with ``JUDGE_MODEL_NAME``. - """ - - def __init__(self, config: dict): - model_name = config.get("model_name") or os.getenv("JUDGE_MODEL_NAME", "Azure/gpt-4.1") - base_url = ( - config.get("end_point") - or os.getenv("JUDGE_BASE_URL") - or os.getenv("OPENAI_BASE_URL", "https://ete-litellm.bx.cloud9.ibm.com") - ) - api_key = os.getenv("JUDGE_API_KEY") or os.getenv("OPENAI_API_KEY") - if api_key is None or api_key == "": - raise ValueError("OPENAI_API_KEY (or JUDGE_API_KEY) is required for the LiteLLM judge") - + def __init__(self, config): + # Set model with model or model_name + model_name=config.get("model_name", "openai/gpt-oss-120b") + end_point=config.get("end_point","https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b") + rits_api_key = os.getenv("RITS_API_KEY") + if rits_api_key is None: + raise ValueError("rits_api_key is required") params = config.get("params", {}) - - # The LiteLLM proxy serves the OpenAI routes at the root, so use base_url - # as-is (no "/v1" suffix) — matching the agent's gpt4.1 profile. - cfg = { - "model": model_name, - "api_key": api_key, - "base_url": base_url.rstrip("/"), - "temperature": 0, - } - cfg.update(params) - - super().__init__(**cfg) - - -def _build_judge_llm(config: dict) -> ChatOpenAI: - """Select the judge backend. Default ``litellm`` -> Azure/gpt-4.1 via the - LiteLLM proxy; set ``JUDGE_BACKEND=groq`` (or gpt-oss) for the legacy - Groq-backed gpt-oss-120b judge. An explicit ``config["backend"]`` wins.""" - backend = (config.get("backend") or os.getenv("JUDGE_BACKEND", "litellm")).strip().lower() - if backend in ("groq", "gpt-oss", "gpt_oss", "oss"): - return ChatModel(config) - if backend in ("litellm", "azure", ""): - return LiteLLMChatModel(config) - raise ValueError( - f"Unknown judge backend {backend!r}. Set JUDGE_BACKEND (or config['backend']) to one of: " - "'litellm'/'azure' (default, Azure/gpt-4.1 via the LiteLLM proxy) or " - "'groq'/'gpt-oss' (legacy Groq gpt-oss-120b)." - ) - + # Set default values for overriding fields + rits_config = {} + rits_config.setdefault("model_name", model_name) + rits_config.setdefault("api_key", "/") + rits_config.setdefault("default_headers", {"RITS_API_KEY": rits_api_key}) + rits_config.setdefault("base_url", end_point + "/v1") + rits_config.update(params) + super().__init__(**rits_config) class LLMJudge: """ Interface you implement with your provider (OpenAI, vLLM, etc). Must be deterministic as much as possible (temperature=0). """ + def __init__(self, + config: dict = {}): + self.model_config=config + self.llm=self._build_llm(self.model_config) + + def _build_llm(self, config: dict) -> ChatOpenAI: + backend = ( + config.get("backend") + or config.get("provider") + or os.getenv("JUDGE_BACKEND") + ) + if backend is None: + backend = "rits" # Use RITS as default judge backend if not specified - def __init__(self, config: dict = {}): - self.model_config = config - self.llm = _build_judge_llm(self.model_config) + backend = backend.lower() + if backend == "rits": + return ChatRits(config) + if backend == "groq": + return ChatGroq(config) + raise ValueError(f"Unsupported judge backend: {backend!r}. Expected 'groq' or 'rits'.") - def invoke(self, prompt: str) -> str: + def invoke(self, prompt:str) -> str: res = self.llm.invoke(prompt).content return res @@ -131,35 +103,26 @@ class GroundednessJudge(LLMJudge): Check if the predicted answer is grounded in the answers in the turn. """ - def __init__(self, config: dict = {}): - # Allow overriding ONLY the groundedness judge model, independent of the - # shared JUDGE_MODEL_NAME used by the correctness judge. Falls back to the - # standard default in LiteLLMChatModel when unset. - override = os.getenv("GROUNDEDNESS_JUDGE_MODEL_NAME") - if override and not config.get("model_name"): - config = {**config, "model_name": override} - super().__init__(config) - _ws = re.compile(r"\s+") def _norm(self, s: str) -> str: return self._ws.sub(" ", s.strip().lower()) def judge(self, inp: JudgeInput) -> JudgeOutput: - tr = " ".join( - [self._norm(str(t)) for t in inp.pred_tool_responses[-N_TOOL_CALLS_PER_TURN:]] - ) # Concatenate the "tool_responses". Only top 20 tool_responses considered. - tr = f"QUERY: {inp.query} {tr}" # Add Query to the responses + tr=" ".join([self._norm(str(t)) for t in inp.pred_tool_responses[-N_TOOL_CALLS_PER_TURN:]]) # Concatenate the "tool_responses". Only top 20 tool_responses considered. + tr=f"QUERY: {inp.query} {tr}" # Add Query to the responses pr = self._norm(inp.pred_answer) - prompt = GroundednessPrompt.format(doc=tr, response=pr) + prompt=GroundednessPrompt.format( + doc=tr, + response=pr) try: - result = self.invoke(prompt) - result_parsed = self._parse_response(result) + result=self.invoke(prompt) + result_parsed= self._parse_response(result) return JudgeOutput(score=result_parsed["score"], explanation=result_parsed["explanation"]) except Exception as e: return JudgeOutput(score=0.0, explanation=f"Judge Error {e}.") - + def _find_label_and_line(self, text: str) -> tuple[Optional[str], Optional[int]]: lines = [ln.strip() for ln in text.split("\n") if ln.strip()] for i, ln in enumerate(lines[:10]): # scan first few meaningful lines @@ -170,7 +133,7 @@ def _find_label_and_line(self, text: str) -> tuple[Optional[str], Optional[int]] m = _LABEL_RE.search(text) return (m.group(1).lower(), None) if m else (None, None) - def _parse_response(self, raw: str) -> dict: + def _parse_response(self, raw: str)-> dict: """ Parse the judge output into: {"explanation": str, "score": int} @@ -188,7 +151,7 @@ def _parse_response(self, raw: str) -> dict: text = raw.strip() if not text: raise JudgeOutputParseError("empty output") - + # Prefer ... content if present m = _CONCLUSION_RE.search(text) if m: @@ -200,7 +163,7 @@ def _parse_response(self, raw: str) -> dict: if label is None: raise JudgeOutputParseError("No label found (expected: yes/partial/no/unsure)") - + explanation = self._extract_explanation(text, label, line_idx).strip() return {"explanation": explanation.strip(), "score": _SCORE_MAP[label]} @@ -227,20 +190,18 @@ def _extract_explanation(self, text: str, label: str, line_idx: Optional[int]) - m = _LABEL_RE.search(text) if not m: return "" - tail = text[m.end() :].lstrip(" \t:;-–—\n").strip() + tail = text[m.end():].lstrip(" \t:;-–—\n").strip() return tail.split("\n", 1)[0].strip() if tail else "" - - + class CorrectnessJudge(LLMJudge): """ Score the predicted answer as opposed to ground truth answer and query. """ - _ws = re.compile(r"\s+") def _norm(self, s: str) -> str: return self._ws.sub(" ", s.strip().lower()) - + def _parse_response(self, raw: str) -> dict: """ Parse the judge output into: @@ -319,7 +280,7 @@ def coerce_score(v: Any) -> int: return {"explanation": explanation.strip(), "score": score_int} - def _find_first_json_object(self, text: str) -> Optional[str]: + def _find_first_json_object(text: str) -> Optional[str]: """ Returns the substring of the first top-level JSON object found in `text`, or None if none is found. @@ -358,20 +319,22 @@ def _find_first_json_object(self, text: str) -> Optional[str]: return None def judge(self, inp: JudgeInput) -> JudgeOutput: - query = self._norm(inp.query) + query=self._norm(inp.query) gt = self._norm(inp.gt_answer) pr = self._norm(inp.pred_answer) if not gt and not pr: return JudgeOutput(score=1.0, explanation="Both empty.") - prompt = CorrectnessPrompt.format(question=query, answer=gt, prediction=pr) + prompt=CorrectnessPrompt.format( + question=query, + answer=gt, + prediction=pr) try: - result = self.invoke(prompt) - result_parsed = self._parse_response(result) + result=self.invoke(prompt) + result_parsed= self._parse_response(result) return JudgeOutput(score=result_parsed["score"], explanation=result_parsed["explanation"]) - except Exception: + except Exception as e: return JudgeOutput(score=0.0, explanation="Judge Error {e}.") - class ExactMatchJudge(LLMJudge): """ Ensures the tool_responses (tool_response) match exactly. @@ -402,12 +365,16 @@ def judge(self, inp: "JudgeInput") -> "JudgeOutput": missing = [e for e in expected_cmp if e not in actual_cmp] if not missing: - return JudgeOutput(score=1.0, explanation="All expected tool_responses are present in actual.") + return JudgeOutput( + score=1.0, + explanation="All expected tool_responses are present in actual." + ) else: return JudgeOutput( score=0.0, explanation=( - f"Missing expected tool_responses.\nMissing: {missing}\n" + "Missing expected tool_responses.\n" + f"Missing: {missing}\n" # f"Expected ({len(expected_cmp)}): {expected_cmp}\n" # f"Actual ({len(actual_cmp)}): {actual_cmp}" ), diff --git a/benchmarks/m3/evaluator/mcp_tools.py b/benchmarks/m3/evaluator/mcp_tools.py index fbbce3c3..201292be 100644 --- a/benchmarks/m3/evaluator/mcp_tools.py +++ b/benchmarks/m3/evaluator/mcp_tools.py @@ -1,17 +1,18 @@ from __future__ import annotations -import asyncio -import json import os +import json +import asyncio import warnings from abc import ABC, abstractmethod from contextlib import asynccontextmanager from dataclasses import dataclass from typing import Any, AsyncIterator, Dict, List, Optional, Sequence -from constant import PRED_OUTPUT_KEY, PRED_OUTPUT_SEQUENCE_KEY from mcp import ClientSession, StdioServerParameters from mcp.client.stdio import stdio_client + +from constant import PRED_OUTPUT_KEY, PRED_OUTPUT_SEQUENCE_KEY from utils import _ensure_list ToolSpec = Dict[str, Any] @@ -21,8 +22,10 @@ # Prediction dialogue helpers # ----------------------------- - -def extract_toolcalls_for_mcp(pred_dialogue: Dict[str, Any]) -> List[List[Dict[str, Any]]]: +def extract_toolcalls_for_mcp( + pred_dialogue: Dict[str, Any], + limit_last_n: Optional[int] = None, +) -> List[List[Dict[str, Any]]]: """ FastAPIMCPToolClient.call_mcp_tools expects: tools = [ @@ -47,16 +50,18 @@ def extract_toolcalls_for_mcp(pred_dialogue: Dict[str, Any]) -> List[List[Dict[s if not isinstance(raw_tc, dict): continue if isinstance(raw_tc, dict) and "name" in raw_tc: - turn_tools.append({"name": raw_tc["name"], "arguments": raw_tc.get("arguments", {})}) + turn_tools.append( + {"name": raw_tc["name"], "arguments": raw_tc.get("arguments", {})}) + if limit_last_n is not None: + turn_tools = turn_tools[-limit_last_n:] dialogue_tools.append(turn_tools) return dialogue_tools - def inject_mcp_responses( pred_entry: Dict[str, Any], mcp_dialogue_responses: List[List[Any]], - type: str, # "pred" / "gt" - capability_name: str, + type: str, # "pred" / "gt" + capability_name:str, ) -> None: """ Inject tool responses into each turn's sequence.tool_response. @@ -67,6 +72,7 @@ def inject_mcp_responses( - they should be the same length (1:1 aligned) for downstream scoring """ turns = _ensure_list(pred_entry.get(PRED_OUTPUT_KEY, [])) + query_tool_present=False for i, turn in enumerate(turns): if not isinstance(turn, dict): @@ -82,35 +88,33 @@ def inject_mcp_responses( turn_resps = mcp_dialogue_responses[i] if i < len(mcp_dialogue_responses) else [] turn_resps = _ensure_list(turn_resps) - query_tool_present = False + if type == "pred" and len(tool_calls) > len(turn_resps): + tool_calls = tool_calls[-len(turn_resps):] if turn_resps else [] + if "multiturn" in capability_name: for tool in tool_calls: if "query_" in tool["name"]: - query_tool_present = True + query_tool_present=True break # If tool_call exists, enforce 1:1 length alignment if tool_calls: n = len(tool_calls) - if query_tool_present: - truncated_responses = turn_resps[:n] + if (query_tool_present): + truncated_responses=turn_resps[:n] if type == "gt": for idx, tool in enumerate(tool_calls): if "query_" not in tool["name"]: seq["tool_response"][idx] = truncated_responses[idx] else: - seq["tool_response"][idx] = [ - item["text"] for item in seq["tool_response"][idx] - ] # Only text in chunks is retained - elif type == "pred": + seq["tool_response"][idx] = [item["text"] for item in seq["tool_response"][idx]] # Only text in chunks is retained + elif type=="pred": for idx, tool in enumerate(tool_calls): if "query_" in tool["name"]: try: - truncated_responses[idx] = [ - item["text"] for item in json.loads(truncated_responses[idx])["results"] - ] # Only text in chunks is retained - except Exception: - truncated_responses[idx] = [] # For incorrect tool calls + truncated_responses[idx] = [item["text"] for item in json.loads(truncated_responses[idx])["results"]] # Only text in chunks is retained + except Exception as e: + truncated_responses[idx] = [f"Error processing response: {str(e)}"] seq["tool_response"] = truncated_responses else: if len(turn_resps) >= n: @@ -126,14 +130,12 @@ def inject_mcp_responses( # If you require strict alignment only when tool_calls exist: if tool_calls: - assert len(seq["tool_call"]) == len(seq["tool_response"]) # noqa: S101 — runtime invariant on tool/response alignment - + assert len(seq["tool_call"]) == len(seq["tool_response"]) # ----------------------------- # Schema coercion helpers # ----------------------------- - def _coerce_value_to_schema(value: Any, schema: Optional[Dict[str, Any]]) -> Any: """ Coerce a single value to match a (subset of) JSON schema. @@ -214,9 +216,7 @@ def _coerce_args_to_schema(args: Dict[str, Any], input_schema: Optional[Dict[str if not input_schema or not isinstance(input_schema, dict): return args - properties = ( - input_schema.get("properties", {}) if isinstance(input_schema.get("properties", {}), dict) else {} - ) + properties = input_schema.get("properties", {}) if isinstance(input_schema.get("properties", {}), dict) else {} coerced: Dict[str, Any] = {} for k, v in args.items(): @@ -230,11 +230,11 @@ def _coerce_args_to_schema(args: Dict[str, Any], input_schema: Optional[Dict[str # Batch tool execution # ----------------------------- - async def execute_tools_batch( session: ClientSession, dialogue_tools_batch: List[List[List[Dict[str, Any]]]], schema_map: Optional[Dict[str, Any]], + is_gt: bool = False, ) -> List[List[List[Any]]]: """ Execute a batch of tool calls using a raw MCP session, with schema-based @@ -262,6 +262,14 @@ async def execute_tools_batch( for tool in turn_tools: tool_name = tool["name"] raw_args = tool.get("arguments", {}) or {} + if is_gt: + if (("input_value" in raw_args.keys()) or "data_label" in raw_args.keys()) and len(turn_responses)>0: + if "handle" in turn_responses[-1]: + previous_response = json.loads(turn_responses[-1]) + if "input_value" in raw_args.keys(): + raw_args["input_value"] = previous_response["results"] + if "data_label" in raw_args.keys(): + raw_args["data_label"] = previous_response["handle"] # Parse JSON string args if needed (JSON round-trip safety) if isinstance(raw_args, str): @@ -346,23 +354,19 @@ async def connect_to_mcp_server(self) -> AsyncIterator[ClientSession]: await session.initialize() yield session - async def call_mcp_tools( - self, session: ClientSession, tools: Sequence[ToolSpec], schema_map: Optional[Dict[str, Any]] - ) -> List[Any]: + async def call_mcp_tools(self, session: ClientSession, tools: Sequence[ToolSpec], schema_map: Optional[Dict[str, Any]]) -> List[Any]: """ Calls MCP tools using an already-connected and initialized ClientSession. """ raise NotImplementedError - async def run_tools( - self, tools: Optional[Sequence[ToolSpec]] = None, schema_map: Optional[Dict[str, Any]] = None - ) -> List[Any]: + async def run_tools(self, tools: Optional[Sequence[ToolSpec]] = None, schema_map: Optional[Dict[str, Any]] = None) -> List[Any]: """ Connect to MCP server and call the provided tools. """ async with self.connect_to_mcp_server() as session: return await self.call_mcp_tools(session, tools, schema_map) - + async def list_loaded_tools(self) -> List[Dict[str, Any]]: """ Connects to the MCP server and returns the list of tools that the server reports as loaded. @@ -382,16 +386,12 @@ async def list_loaded_tools(self) -> List[Dict[str, Any]]: for t in tool_list: # t may be dict or object name = t.get("name") if isinstance(t, dict) else getattr(t, "name", None) - input_schema = ( - t.get("inputSchema") if isinstance(t, dict) else getattr(t, "inputSchema", None) - ) + input_schema = t.get("inputSchema") if isinstance(t, dict) else getattr(t, "inputSchema", None) if isinstance(name, str) and isinstance(input_schema, dict): schema_map[name] = input_schema return tool_list, schema_map - def _coerce_args_to_schema( - self, args: Dict[str, Any], input_schema: Optional[Dict[str, Any]] - ) -> Dict[str, Any]: + def _coerce_args_to_schema(self, args: Dict[str, Any], input_schema: Optional[Dict[str, Any]]) -> Dict[str, Any]: """Delegate to module-level _coerce_args_to_schema.""" return _coerce_args_to_schema(args, input_schema) @@ -436,7 +436,7 @@ def build_server_params(self) -> StdioServerParameters: args=exec_args, env=None, ) - elif self.config.run_type == "local": + elif self.config.run_type =="local": return StdioServerParameters( command="python", args=[self.local_server_script], @@ -445,10 +445,8 @@ def build_server_params(self) -> StdioServerParameters: "MCP_DOMAIN": self.config.domain, }, ) - - async def call_mcp_tools( - self, session: ClientSession, tools: Sequence[ToolSpec], schema_map: Optional[Dict[str, Any]] - ) -> List[Any]: + + async def call_mcp_tools(self, session: ClientSession, tools: Sequence[ToolSpec], schema_map: Optional[Dict[str, Any]]) -> List[Any]: """ Calls MCP tools using an already-connected and initialized ClientSession. """ @@ -467,9 +465,7 @@ async def call_mcp_tools( raw_args = json.loads(raw_args) if not isinstance(raw_args, dict): - raise TypeError( - f"Tool arguments must be a dict (or JSON string), got {type(raw_args)} for {name}" - ) + raise TypeError(f"Tool arguments must be a dict (or JSON string), got {type(raw_args)} for {name}") # 2) Coerce args according to schema if available input_schema = schema_map.get(name) @@ -486,7 +482,6 @@ async def call_mcp_tools( return results - class PythonMCPToolClient(MCPToolClientBase): """ MCP client for Python tools. @@ -572,7 +567,6 @@ def build_server_params(self) -> StdioServerParameters: }, ) - def test(tools: List[ToolSpec]) -> List[Any]: async def _run(): client = FastAPIMCPToolClient( @@ -582,7 +576,6 @@ async def _run(): return asyncio.run(_run()) - if __name__ == "__main__": # This example gives a super huge output. # tools = [[[ diff --git a/benchmarks/m3/evaluator/prompt.py b/benchmarks/m3/evaluator/prompt.py index 6551c951..058b7a85 100644 --- a/benchmarks/m3/evaluator/prompt.py +++ b/benchmarks/m3/evaluator/prompt.py @@ -1,3 +1,4 @@ + # Auto-evalution prompt from "CRAG: Comprehensive RAG Benchmark" is used for correctness prompt. # https://github.com/facebookresearch/CRAG/blob/main/prompts/templates.py CorrectnessPrompt = """Assume you are a human expert in grading predictions given by a model. You are given a question and a model prediction. Judge if the prediction matches the ground truth answer by following these steps: @@ -31,7 +32,7 @@ """ -GroundednessPrompt = """ +GroundednessPrompt =""" The following tasks each contains document and a response. The response is supposed to rely on the document for its source of information, optionally using common sense knowledge and common sense inference, but it may fail this, and instead contain substantial claims that are not grounded in the document or common sense knowledge. Your task is to assess whether the response is entirely grounded in the document, grounded in the document plus common sense knowledge and reasoning, or ungrounded. To make this determination, perform the following steps: @@ -39,7 +40,7 @@ - Ignore non-substantial claims, such as greetings or self-descriptions such as "I'm a helpful assistant", - Try to formulate each claim in a stand-alone form with all pronouns and other references resolved; 2. Assess the grounding of each of these claims: - - If it is essentially a rephrasing of information from the document, or can be derived from such information by trivial common-sense reasoning, it is grounded, This is so even if it contradicts other parts of the document. + - If it is essentially a rephrasing of information from the document, or can be derived from such information by trivial common-sense reasoning, it is grounded, This is so even if it contradicts other parts of the document. - If it relies on, in additional to information from the document, additional non-trivial common sense knowledge or common sense reasoning, it is partially grounded, - If a claim is about the provided document, or about the agent's state of knowledge, with the effect of not being able to answer the user inquiry, it is grounded if and only if the required information is indeed lacking in the document. - If a claim cannot be derived directly from the document or indirectly with help of common sense knowledge and reasoning, it is ungrounded; @@ -57,11 +58,11 @@ - "no" is for ungrounded, - "unsure" is for the situations where the document, conversation or response contain ambiguities such that different interpretations lead to different conclusions about groundedness; - The second line contains an explanation of your answer as short as possible. - + Here is the document starting with and end with - + {doc} - + Here is the response starting with and end with . @@ -76,4 +77,4 @@ - "no" is for ungrounded, - "unsure" is for the situations where the document, conversation or response contain ambiguities such that different interpretations lead to different conclusions about groundedness; -""" +""" \ No newline at end of file diff --git a/benchmarks/m3/evaluator/scorer.py b/benchmarks/m3/evaluator/scorer.py index 9b1caa28..f5d1ed23 100644 --- a/benchmarks/m3/evaluator/scorer.py +++ b/benchmarks/m3/evaluator/scorer.py @@ -1,28 +1,29 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Sequence, Tuple - -from constant import ( - GT_OUTPUT_ANSWER_KEY, - GT_OUTPUT_KEY, - GT_OUTPUT_QUERY_KEY, - GT_OUTPUT_SEQUENCE_KEY, - GT_OUTPUT_TURN_ID_KEY, - PRED_OUTPUT_ANSWER_KEY, - PRED_OUTPUT_KEY, - PRED_OUTPUT_QUERY_KEY, - PRED_OUTPUT_SEQUENCE_KEY, - PRED_OUTPUT_TURN_ID_KEY, -) -from judge import CorrectnessJudge, ExactMatchJudge, GroundednessJudge, JudgeValidationError, LLMJudge -from utils import JudgeInput, ToolCall +from importlib import import_module +from importlib.util import find_spec +from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple +from utils import ToolCall +from judge import CorrectnessJudge, GroundednessJudge, LLMJudge, JudgeValidationError +from utils import JudgeInput +from constant import N_TOOL_CALLS_PER_TURN, PRED_OUTPUT_KEY, PRED_OUTPUT_TURN_ID_KEY, PRED_OUTPUT_QUERY_KEY, PRED_OUTPUT_ANSWER_KEY, PRED_OUTPUT_SEQUENCE_KEY, GT_OUTPUT_KEY, GT_OUTPUT_TURN_ID_KEY, GT_OUTPUT_QUERY_KEY, GT_OUTPUT_ANSWER_KEY, GT_OUTPUT_SEQUENCE_KEY + + +def _load_policy_judge() -> Optional[Any]: + if find_spec("policy_judge") is None: + return None + + policy_module = import_module("policy_judge") + judge_cls = getattr(policy_module, "PolicyAdherenceJudge", None) + if judge_cls is None: + judge_cls = getattr(policy_module, "PolicyAdheranceJudge") + return judge_cls() # ----------------------------- # Output Scorer # ----------------------------- - @dataclass(frozen=True) class TurnScorerConfig: """ @@ -33,39 +34,27 @@ class TurnScorerConfig: - ground truth answer and predicted answer are evaluated using an LLM-as-Judge """ - capability: str domain: str - class TurnScorer: """ Compares GT vs predicted tool trajectory per turn based on final answer. Produces a binary score to evaluate a turn. """ - - def __init__( - self, - cfg: TurnScorerConfig | None = None, - correctness_judge: Optional[LLMJudge] = None, - groundedness_judge: Optional[LLMJudge] = None, - exactmatch_judge: Optional[LLMJudge] = None, - ): + def __init__(self, cfg: TurnScorerConfig | None = None, + correctness_judge: Optional[LLMJudge] = None, + groundedness_judge: Optional[LLMJudge] = None, + exactmatch_judge: Optional[LLMJudge] = None): self.cfg = cfg or TurnScorerConfig() - self.correctness_judge = correctness_judge - self.groundedness_judge = groundedness_judge - self.exactmatch_judge = exactmatch_judge - - def compare( - self, - query: str, - additional_instructions: str, - gt_answer: str, - pred_answer: str, - gt: Sequence[ToolCall], - pred: Sequence[ToolCall], - gt_responses: Sequence[Any], - pred_responses: Sequence[Any], - ) -> Tuple[float, Dict[str, Any]]: + self.correctness_judge=correctness_judge + self.groundedness_judge=groundedness_judge + self.exactmatch_judge=exactmatch_judge + self.policy_judge=_load_policy_judge() + + def compare(self, query: str, additional_instructions:str + , gt_answer: str, pred_answer: str + , gt: Sequence[ToolCall], pred: Sequence[ToolCall] + , gt_responses: Sequence[Any], pred_responses: Sequence[Any]) -> Tuple[float, Dict[str, Any]]: # Run tools in pred and ascertain if predicted answer is grounded on tool call answers. # Use Judge to ascertain pred answer is answering the query. input = JudgeInput( @@ -80,34 +69,43 @@ def compare( gt_tool_responses=gt_responses, pred_tool_responses=pred_responses, ) - + # Scoring Turns - extra_steps = len(pred) - len(gt) - - # Check for policy adherance - # if "multiturn" in self.cfg.capability: - # policy = self.policy_judge.judge(inp=input) - # policy_score, policy_explanation = float(policy.score), policy.explanation - # if policy_score==0.0: - # score = policy_score - # details = { - # "gt_steps": len(gt), - # "pred_steps": len(pred), - # "extra_steps": max(0, extra_steps), - # "policy_adherance_score": policy_score, - # "exactmatch_score": None, - # "answer_score": None, - # "groundedness_score": None, - # "score_explanation": {"policy": policy_explanation, "answer": None, "exactmatch": None, "groundedness":None}, - # } - # return score, details - - exactmatch = self.exactmatch_judge.judge(inp=input) - exactmatch_score, exactmatch_explanation = float(exactmatch.score), exactmatch.explanation - if exactmatch_score == 0.0: - correctness = self.correctness_judge.judge(inp=input) - answer_score, answer_explanation = float(correctness.score), correctness.explanation - if answer_score == 0.0: + extra_steps=len(pred)-len(gt) + + # Check for policy adherence when an internal policy_judge.py file is present. + if self.policy_judge is not None and "multiturn" in self.cfg.capability: + policy = self.policy_judge.judge(inp=input) + policy_score, policy_explanation = float(policy.score), policy.explanation + if policy_score==0.0: + score = policy_score + details = { + "gt_steps": len(gt), + "pred_steps": len(pred), + "extra_steps": max(0, extra_steps), + "policy_adherance_score": policy_score, + "exactmatch_score": None, + "answer_score": None, + "groundedness_score": None, + "score_explanation": {"policy": policy_explanation, "answer": None, "exactmatch": None, "groundedness":None}, + } + return score, details + + unanswerable_no_tool_case = ( + len(input.pred_tool_calls) == 0 + and "i can not answer" in input.gt_answer.lower() + and len(input.gt_tool_calls) == 0 + ) + if unanswerable_no_tool_case: + exactmatch_score = 1.0 + exactmatch_explanation = "Special case applied - GT indicates unanswerable and no tool calls made, granting full exact match credit." + else: + exactmatch=self.exactmatch_judge.judge(inp=input) + exactmatch_score, exactmatch_explanation=float(exactmatch.score), exactmatch.explanation + if exactmatch_score==0.0: + correctness=self.correctness_judge.judge(inp=input) + answer_score, answer_explanation=float(correctness.score), correctness.explanation + if answer_score==0.0: score = answer_score details = { "gt_steps": len(gt), @@ -116,19 +114,33 @@ def compare( "exactmatch_score": exactmatch_score, "answer_score": answer_score, "groundedness_score": None, - "score_explanation": { - "answer": answer_explanation, - "exactmatch": exactmatch_explanation, - "groundedness": None, - }, + "score_explanation": {"answer": answer_explanation, "exactmatch": exactmatch_explanation, "groundedness":None}, } return score, details - elif answer_score == 1.0: + elif answer_score==1.0: + if ( + "multiturn" in self.cfg.capability + and "i can not answer" in input.gt_answer.lower() + and (len(input.pred_tool_calls) == 0 or input.pred_answer in ["", " "]) + ): + score = 1.0 + details = { + "gt_steps": len(gt), + "pred_steps": len(pred), + "extra_steps": max(0, extra_steps), + "exactmatch_score": exactmatch_score, + "answer_score": answer_score, + "groundedness_score": 1.0, + "score_explanation": { + "answer": answer_explanation, + "exactmatch": exactmatch_explanation, + "groundedness": "Special case applied - GT indicates unanswerable and no tool calls made, granting full groundedness credit.", + }, + } + return score, details + groundedness = self.groundedness_judge.judge(inp=input) - groundedness_score, groundedness_explanation = ( - float(groundedness.score), - groundedness.explanation, - ) + groundedness_score, groundedness_explanation = float(groundedness.score), groundedness.explanation score = groundedness_score details = { "gt_steps": len(gt), @@ -137,11 +149,7 @@ def compare( "exactmatch_score": exactmatch_score, "answer_score": answer_score, "groundedness_score": groundedness_score, - "score_explanation": { - "answer": answer_explanation, - "exactmatch": exactmatch_explanation, - "groundedness": groundedness_explanation, - }, + "score_explanation": {"answer": answer_explanation, "exactmatch": exactmatch_explanation, "groundedness":groundedness_explanation}, } return score, details else: @@ -149,7 +157,28 @@ def compare( f"Unexpected answer_score from correctness_judge: {answer_score!r}. " f"Expected 0.0 or 1.0. Explanation: {answer_explanation!r}" ) - elif exactmatch_score == 1.0: + elif exactmatch_score==1.0: + if ( + "multiturn" in self.cfg.capability + and "i can not answer" in input.gt_answer.lower() + and (len(input.pred_tool_calls) == 0 or input.pred_answer in ["", " "]) + ): + score = 1.0 + details = { + "gt_steps": len(gt), + "pred_steps": len(pred), + "extra_steps": max(0, extra_steps), + "exactmatch_score": exactmatch_score, + "answer_score": None, + "groundedness_score": 1.0, + "score_explanation": { + "answer": None, + "exactmatch": exactmatch_explanation, + "groundedness": "Special case applied - GT indicates unanswerable and no tool calls made, granting full groundedness credit.", + }, + } + return score, details + groundedness = self.groundedness_judge.judge(inp=input) groundedness_score, groundedness_explanation = float(groundedness.score), groundedness.explanation score = groundedness_score @@ -160,11 +189,7 @@ def compare( "exactmatch_score": exactmatch_score, "answer_score": None, "groundedness_score": groundedness_score, - "score_explanation": { - "answer": None, - "exactmatch": exactmatch_explanation, - "groundedness": groundedness_explanation, - }, + "score_explanation": {"answer": None, "exactmatch": exactmatch_explanation, "groundedness":groundedness_explanation}, } return score, details else: @@ -173,7 +198,6 @@ def compare( f"exactmatch_explanation={exactmatch_explanation!r}" ) - @dataclass(frozen=True) class DialogueScorerConfig: """ @@ -182,7 +206,6 @@ class DialogueScorerConfig: - "sum": sum of per-turn scores - "min": minimum per-turn score (strict) """ - aggregate: str = "mean" # "mean" | "sum" | "min" @@ -212,9 +235,9 @@ class DialogueScorer: "output": [ { "turn_id": 0, - "query": "...", + "query": "...", "predicted_answer": "100", - "sequence": + "sequence": { "tool_call": [{"name": "...", "arguments": {...}}], "tool_response": [[100]] @@ -246,43 +269,28 @@ def score( ) -> Tuple[float, Dict[str, Any]]: gt_turns: List[Dict[str, Any]] = list(gt_dialogue.get(gt_key, [])) pred_turns: List[Dict[str, Any]] = list(pred_dialogue.get(pred_key, [])) - additional_instructions = gt_dialogue.get("additional_instructions", "") - - assert len(pred_turns) == 1, f"Predicted Turns {len(pred_turns)} should have been 1." # noqa: S101 — runtime invariant for single-turn scoring + additional_instructions = gt_dialogue.get("additional_instructions","") + + assert len(pred_turns) == 1, f"Predicted Turns {len(pred_turns)} should have been 1." pred_by_id = {t.get(PRED_OUTPUT_TURN_ID_KEY): t for t in pred_turns if "turn_id" in t} per_turn: List[Dict[str, Any]] = [] turn_scores: List[float] = [] - for gt_turn in [gt_turns[-1]]: # Only the last turn evaluated against prediction + for gt_turn in [gt_turns[-1]]: # Only the last turn evaluated against prediction turn_id = gt_turn.get(GT_OUTPUT_TURN_ID_KEY) query = str(gt_turn.get(GT_OUTPUT_QUERY_KEY)) gt_answer = self._stringify_answer(gt_turn.get(GT_OUTPUT_ANSWER_KEY)) - gt_calls = gt_turn.get(GT_OUTPUT_SEQUENCE_KEY, {}).get("tool_call", []) - gt_responses = self._extract_tool_responses( - gt_turn.get(GT_OUTPUT_SEQUENCE_KEY, {}).get("tool_response", []) - ) + gt_calls = gt_turn.get(GT_OUTPUT_SEQUENCE_KEY, {}).get("tool_call",[]) + gt_responses = self._extract_tool_responses(gt_turn.get(GT_OUTPUT_SEQUENCE_KEY, {}).get("tool_response",[])) pred_turn = pred_by_id.get(turn_id, None) - if pred_turn is None: - per_turn.append( - { - "turn_id": turn_id, - "query": query, - "pred_answer": "", - "score": 0.0, - "metadata": {"error": f"missing predicted turn_id={turn_id}"}, - } - ) - turn_scores.append(0.0) - continue - pred_answer = self._stringify_pred_answer(pred_turn.get(PRED_OUTPUT_ANSWER_KEY, "")) - pred_calls = pred_turn.get(PRED_OUTPUT_SEQUENCE_KEY, {}).get("tool_call", []) - pred_responses = self._extract_tool_responses( - pred_turn.get(PRED_OUTPUT_SEQUENCE_KEY, {}).get("tool_response", []) - ) + pred_sequence = pred_turn.get(PRED_OUTPUT_SEQUENCE_KEY, {}) or {} + pred_calls_all = pred_sequence.get("tool_call", []) or [] + pred_calls = pred_calls_all[-N_TOOL_CALLS_PER_TURN:] if isinstance(pred_calls_all, list) else [] + pred_responses = self._extract_tool_responses(pred_sequence.get("tool_response", []))[-N_TOOL_CALLS_PER_TURN:] score, details = self.turn_scorer.compare( query=query, @@ -301,7 +309,7 @@ def score( "query": query, "pred_answer": pred_answer, "score": score, - "metadata": details, + "metadata":details, } ) turn_scores.append(float(score)) @@ -350,7 +358,7 @@ def _stringify_answer(self, answer_obj: Any) -> str: return ", ".join(flattened) - except Exception: # noqa: S110 — fall through to str() on any stringify error + except Exception: pass return str(answer_obj) @@ -396,7 +404,7 @@ def _flatten_2d(self, obj: Any) -> List[List[Any]]: return [obj] # type: ignore[list-item] # non-list -> wrap twice return [[obj]] - + if __name__ == "__main__": # ---- Example ground-truth dialogue ---- @@ -406,15 +414,16 @@ def _flatten_2d(self, obj: Any) -> List[List[Any]]: GT_OUTPUT_TURN_ID_KEY: 0, GT_OUTPUT_QUERY_KEY: "How many students have never been absent from school?", GT_OUTPUT_ANSWER_KEY: [[100]], - GT_OUTPUT_SEQUENCE_KEY: { - "tool_call": [ - { - "name": "get_count_names_by_month_v1_student_loan_count_names_by_month_get", - "arguments": {"month": "0"}, - } - ], - "tool_response": [[100]], - }, + GT_OUTPUT_SEQUENCE_KEY: + { + "tool_call": [ + { + "name": "get_count_names_by_month_v1_student_loan_count_names_by_month_get", + "arguments": {"month": "0"}, + } + ], + "tool_response": [[100]], + } } ] } @@ -426,17 +435,18 @@ def _flatten_2d(self, obj: Any) -> List[List[Any]]: PRED_OUTPUT_TURN_ID_KEY: 0, PRED_OUTPUT_QUERY_KEY: "How many students have never been absent from school?", PRED_OUTPUT_ANSWER_KEY: "100", - PRED_OUTPUT_SEQUENCE_KEY: { - "tool_call": [ - [ - { - "name": "get_count_names_by_month_v1_student_loan_count_names_by_month_get", - "arguments": {"month": "0"}, - } - ] - ], - "tool_response": [[500]], - }, + PRED_OUTPUT_SEQUENCE_KEY: + { + "tool_call": [ + [ + { + "name": "get_count_names_by_month_v1_student_loan_count_names_by_month_get", + "arguments": {"month": "0"}, + } + ] + ], + "tool_response": [[500]], + } } ] } @@ -447,17 +457,15 @@ def _flatten_2d(self, obj: Any) -> List[List[Any]]: answer_weight=0.5, trajectory_weight=0.5, extra_step_penalty=0.1, - ) + ) - turn_scorer = TurnScorer( - cfg=turn_cfg, - correctness_judge=CorrectnessJudge(), - groundedness_judge=GroundednessJudge(), - exactmatch_judge=ExactMatchJudge(config={}), - ) + turn_scorer = TurnScorer(cfg=turn_cfg, + correctness_judge=CorrectnessJudge(), + groundedness_judge=GroundednessJudge(), + exactmatch_judge=ExactMatchJudge(config={})) dialogue_cfg = DialogueScorerConfig( - aggregate="mean", # mean | sum | min + aggregate="mean", # mean | sum | min ) dialogue_scorer = DialogueScorer( diff --git a/benchmarks/m3/evaluator/utils.py b/benchmarks/m3/evaluator/utils.py index 9497c6d0..3ac6e8e9 100644 --- a/benchmarks/m3/evaluator/utils.py +++ b/benchmarks/m3/evaluator/utils.py @@ -4,13 +4,13 @@ import json from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, List, Literal, Mapping, Optional, Sequence, Tuple +from typing import Any, Dict, Iterable, List, Literal, Mapping, Optional, Sequence, Tuple + # ----------------------------- # General helpers # ----------------------------- - def _ensure_list(x: Any) -> List[Any]: return x if isinstance(x, list) else ([] if x is None else [x]) @@ -38,7 +38,6 @@ def pair_dialogues_by_uuid( paired.append((gt_map[u], pr_map[u])) return paired, missing_pred, extra_pred - # ----------------------------- # Data model # ----------------------------- @@ -105,7 +104,6 @@ def from_dict(d: Mapping[str, Any]) -> "Example": @dataclass(frozen=True) class PairedExample: """Ground truth + prediction matched by example ID.""" - gt: Example pred: Example @@ -114,7 +112,6 @@ class PairedExample: # Metrics / results # ----------------------------- - @dataclass class ScalarMetric: name: str @@ -160,13 +157,12 @@ def to_dict(self) -> Dict[str, Any]: # LLM Judge for correctness and groundedness # ----------------------------- - @dataclass(frozen=True) class JudgeInput: capability: str domain: str query: str - additional_instructions: str + additional_instructions: str gt_answer: str pred_answer: str gt_tool_calls: Sequence[ToolCall] @@ -174,9 +170,9 @@ class JudgeInput: gt_tool_responses: List[str] pred_tool_responses: List[str] - @dataclass(frozen=True) class JudgeOutput: # score should be normalized either 1 or 0 score: float explanation: Optional[str] = None + From 7914f9aa8836eb89460ad971a59c394c2a20cd39 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Tue, 21 Jul 2026 14:02:51 -0400 Subject: [PATCH 02/40] Minimize M3 evaluator sync diff --- benchmarks/m3/evaluator/README.md | 4 +- benchmarks/m3/evaluator/constant.py | 30 +-- benchmarks/m3/evaluator/domain_mapping.json | 82 ++++++++ benchmarks/m3/evaluator/evaluator.py | 220 +++++++++++--------- benchmarks/m3/evaluator/judge.py | 204 +++++++++++------- benchmarks/m3/evaluator/mcp_tools.py | 82 +++++--- benchmarks/m3/evaluator/prompt.py | 13 +- benchmarks/m3/evaluator/scorer.py | 213 ++++++++++++------- benchmarks/m3/evaluator/utils.py | 12 +- 9 files changed, 560 insertions(+), 300 deletions(-) create mode 100644 benchmarks/m3/evaluator/domain_mapping.json diff --git a/benchmarks/m3/evaluator/README.md b/benchmarks/m3/evaluator/README.md index 9d258194..a3528eb0 100644 --- a/benchmarks/m3/evaluator/README.md +++ b/benchmarks/m3/evaluator/README.md @@ -29,8 +29,8 @@ Each paired dialogue is scored in the following order: If exact match fails, an LLM judge compares the predicted answer against the ground-truth answer for the final turn. The correctness prompt is adapted from CRAG and is defined in [`prompt.py`](/vakra/evaluator/prompt.py). 3. **Groundedness check** If the answer is acceptable, another LLM judge verifies that the predicted answer is grounded in the executed tool outputs. -4. **PolicyAdheranceJudge** - NOT INCLUDED: For the Multi-hop multi-source capability, we use a PolicyAdherenceJudge that programmatically checks the agent's application of a policy. This judge is currently not included for evaluation. +4. **PolicyAdherenceJudge** + Optional: for the multi-hop multi-source capability, the evaluator can programmatically check policy application when `evaluator/policy_judge.py` is present. If your checkout does not include this file and you need policy evaluation, open a GitHub issue requesting access to the PolicyJudge file. Turn scores are then aggregated into a dialogue score. The default aggregation policy in this repo is `mean`. diff --git a/benchmarks/m3/evaluator/constant.py b/benchmarks/m3/evaluator/constant.py index 56f2ad49..62cf4291 100644 --- a/benchmarks/m3/evaluator/constant.py +++ b/benchmarks/m3/evaluator/constant.py @@ -1,21 +1,21 @@ from __future__ import annotations -N_TOOL_CALLS_PER_TURN=20 +N_TOOL_CALLS_PER_TURN = 20 # Predicted file keys -PRED_UUID="uuid" -PRED_DOMAIN="domain" -PRED_OUTPUT_KEY="output" -PRED_OUTPUT_TURN_ID_KEY="turn_id" -PRED_OUTPUT_QUERY_KEY="query" -PRED_OUTPUT_ANSWER_KEY="answer" -PRED_OUTPUT_SEQUENCE_KEY= "sequence" +PRED_UUID = "uuid" +PRED_DOMAIN = "domain" +PRED_OUTPUT_KEY = "output" +PRED_OUTPUT_TURN_ID_KEY = "turn_id" +PRED_OUTPUT_QUERY_KEY = "query" +PRED_OUTPUT_ANSWER_KEY = "answer" +PRED_OUTPUT_SEQUENCE_KEY = "sequence" # Ground Truth file keys -GT_UUID="uuid" -GT_DOMAIN="domain" -GT_OUTPUT_KEY="output" -GT_OUTPUT_TURN_ID_KEY="turn_id" -GT_OUTPUT_QUERY_KEY="query" -GT_OUTPUT_ANSWER_KEY="answer" -GT_OUTPUT_SEQUENCE_KEY= "sequence" +GT_UUID = "uuid" +GT_DOMAIN = "domain" +GT_OUTPUT_KEY = "output" +GT_OUTPUT_TURN_ID_KEY = "turn_id" +GT_OUTPUT_QUERY_KEY = "query" +GT_OUTPUT_ANSWER_KEY = "answer" +GT_OUTPUT_SEQUENCE_KEY = "sequence" diff --git a/benchmarks/m3/evaluator/domain_mapping.json b/benchmarks/m3/evaluator/domain_mapping.json new file mode 100644 index 00000000..60b07f34 --- /dev/null +++ b/benchmarks/m3/evaluator/domain_mapping.json @@ -0,0 +1,82 @@ +{ + "california_schools" : "Education & Students", + "student_club": "Education & Students", + "student_loan": "Education & Students", + "university": "Education & Students", + "cs_semester": "Education & Students", + "computer_student": "Education & Students", + "college_completion" : "Education & Students", + "movie" : "Movies, TV & Entertainment, Social Media", + "movie_platform": "Movies, TV & Entertainment, Social Media", + "movie_3": "Movies, TV & Entertainment, Social Media", + "movies_4": "Movies, TV & Entertainment, Social Media", + "simpson_episodes": "Movies, TV & Entertainment, Social Media", + "disney": "Movies, TV & Entertainment, Social Media", + "law_episode": "Movies, TV & Entertainment, Social Media", + "movielens": "Movies, TV & Entertainment, Social Media", + "social_media": "Movies, TV & Entertainment, Social Media", + "talkingdata": "Movies, TV & Entertainment, Social Media", + "music_tracker": "Movies, TV & Entertainment, Social Media", + "music_platform_2": "Movies, TV & Entertainment, Social Media", + "card_games": "Movies, TV & Entertainment, Social Media", + "video_games": "Movies, TV & Entertainment, Social Media", + "superhero": "Movies, TV & Entertainment, Social Media", + "formula_1": "Sports & Athletics", + "professional_basketball": "Sports & Athletics", + "european_football_1": "Sports & Athletics", + "european_football_2": "Sports & Athletics", + "olympics": "Sports & Athletics", + "ice_hockey_draft": "Sports & Athletics", + "hockey": "Sports & Athletics", + "soccer_2016": "Sports & Athletics", + "retails": "Retail, Sales & Commerce", + "retail_world": "Retail, Sales & Commerce", + "retail_complains": "Retail, Sales & Commerce", + "superstore": "Retail, Sales & Commerce", + "regional_sales": "Retail, Sales & Commerce", + "car_retails": "Retail, Sales & Commerce", + "book_publishing_company": "Retail, Sales & Commerce", + "sales": "Retail, Sales & Commerce", + "sales_in_weather": "Retail, Sales & Commerce", + "works_cycles": "Retail, Sales & Commerce", + "restaurant": "Food, Restaurants & Beverage", + "food_inspection": "Food, Restaurants & Beverage", + "food_inspection_2": "Food, Restaurants & Beverage", + "cookbook": "Food, Restaurants & Beverage", + "beer_factory": "Food, Restaurants & Beverage", + "craftbeer": "Food, Restaurants & Beverage", + "menu": "Food, Restaurants & Beverage", + "app_store": "Technology & Software", + "codebase_community": "Technology & Software", + "codebase_comments": "Technology & Software", + "software_company": "Technology & Software", + "image_and_language": "Technology & Software", + "toxicology": "Health & Medicine", + "synthea": "Health & Medicine", + "mental_health_survey": "Health & Medicine", + "thrombosis_prediction": "Health & Medicine", + "genes": "Health & Medicine", + "mondial_geo": "Geography & Demographics", + "world": "Geography & Demographics", + "address": "Geography & Demographics", + "world_development_indicators": "Geography & Demographics", + "airline": "Transportation & Mobility", + "trains": "Transportation & Mobility", + "bike_share_1": "Transportation & Mobility", + "cars": "Transportation & Mobility", + "shipping": "Transportation & Mobility", + "financial": "Finance & Economics", + "donor": "Finance & Economics", + "coinmarketcap": "Finance & Economics", + "debit_card_specializing": "Finance & Economics", + "legislator": "Government, Law & Public Services, Human Capital", + "chicago_crime": "Government, Law & Public Services, Human Capital", + "shooting": "Government, Law & Public Services, Human Capital", + "public_review_platform": "Government, Law & Public Services, Human Capital", + "human_resources": "Government, Law & Public Services, Human Capital", + "books": "Literature, Language & Publishing", + "authors": "Literature, Language & Publishing", + "shakespeare": "Literature, Language & Publishing", + "language_corpus": "Literature, Language & Publishing", + "citeseer": "Literature, Language & Publishing" +} \ No newline at end of file diff --git a/benchmarks/m3/evaluator/evaluator.py b/benchmarks/m3/evaluator/evaluator.py index 2274b9bb..8a52d70f 100644 --- a/benchmarks/m3/evaluator/evaluator.py +++ b/benchmarks/m3/evaluator/evaluator.py @@ -5,15 +5,15 @@ import json from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, Optional, Tuple, List -from tqdm import tqdm +from typing import Any, Dict, List, Optional, Tuple -from judge import CorrectnessJudge, GroundednessJudge, ExactMatchJudge from benchmark.mcp_client import ( MCPConnectionConfig, - load_mcp_config, create_client_and_connect, + load_mcp_config, ) +from constant import PRED_OUTPUT_KEY, PRED_OUTPUT_SEQUENCE_KEY +from judge import CorrectnessJudge, ExactMatchJudge, GroundednessJudge from mcp_tools import ( execute_tools_batch, extract_toolcalls_for_mcp, @@ -25,11 +25,11 @@ TurnScorer, TurnScorerConfig, ) -from constant import PRED_OUTPUT_KEY, PRED_OUTPUT_SEQUENCE_KEY +from tqdm import tqdm +from utils import pair_dialogues_by_uuid, read_domain_file from constant import N_TOOL_CALLS_PER_TURN -from utils import read_domain_file, pair_dialogues_by_uuid -CAPABILITY_MCP_TOOL_MAP={ +CAPABILITY_MCP_TOOL_MAP = { "capability_bi_apis": 1, "capability_dashboard_apis": 2, "capability_multihop_reasoning": 3, @@ -43,8 +43,7 @@ @dataclass(frozen=True) class CapabilityPolicy: - - dialogue_aggregate: str = "mean" # "mean" | "sum" | "min" + dialogue_aggregate: str = "mean" # "mean" | "sum" | "min" # --- MCP execution --- execute_mcp_tools: bool = True @@ -55,35 +54,32 @@ class CapabilityPolicy: exactmatch_judge: Any = ExactMatchJudge(config={}) - def build_default_capability_registry() -> Dict[str, CapabilityPolicy]: return { "capability_bi_apis": CapabilityPolicy( execute_mcp_tools=True, correctness_judge=CorrectnessJudge(config={}), groundedness_judge=GroundednessJudge(config={}), - exactmatch_judge=ExactMatchJudge(config={}) + exactmatch_judge=ExactMatchJudge(config={}), ), - "capability_dashboard_apis": CapabilityPolicy( execute_mcp_tools=True, correctness_judge=CorrectnessJudge(config={}), groundedness_judge=GroundednessJudge(config={}), - exactmatch_judge=ExactMatchJudge(config={}) + exactmatch_judge=ExactMatchJudge(config={}), ), - "capability_multihop_reasoning": CapabilityPolicy( execute_mcp_tools=True, correctness_judge=CorrectnessJudge(config={}), groundedness_judge=GroundednessJudge(config={}), - exactmatch_judge=ExactMatchJudge(config={}) + exactmatch_judge=ExactMatchJudge(config={}), ), "capability_multiturn": CapabilityPolicy( execute_mcp_tools=True, correctness_judge=CorrectnessJudge(config={}), groundedness_judge=GroundednessJudge(config={}), - exactmatch_judge=ExactMatchJudge(config={}) - ), + exactmatch_judge=ExactMatchJudge(config={}), + ), } @@ -145,6 +141,33 @@ def _update_dialogue_toolcall_for_get_data( # ----------------------------- +def _make_missing_dialogue_entry( + uuid: str, + capability_name: str, + domain: str, + policy: CapabilityPolicy, +) -> Dict[str, Any]: + return { + "uuid": uuid, + "score": 0.0, + "metadata": { + "capability": capability_name, + "domain": domain, + "policy": { + "dialogue_aggregate": policy.dialogue_aggregate, + "execute_mcp_tools": policy.execute_mcp_tools, + }, + "error": "missing_prediction", + }, + "details": { + "dialogue_score": 0.0, + "num_turns": 0, + "per_turn": [], + "aggregate": policy.dialogue_aggregate, + }, + } + + async def evaluate_domain( domain: str, gt_path: Path, @@ -164,9 +187,12 @@ async def evaluate_domain( pred_list = read_domain_file(pred_path) if pred_path.exists() else [] # Pair dialogues - paired, missing_pred, extra_pred = pair_dialogues_by_uuid(gt_list, pred_list) + paired, missing_pred, extra_pred = pair_dialogues_by_uuid(gt_list, pred_list) if len(pred_list) == 0: + zero_dialogues = [ + _make_missing_dialogue_entry(uuid, capability_name, domain, policy) for uuid in missing_pred + ] domain_out: Dict[str, Any] = { "domain": domain, "n_groundtruth": len(gt_list), @@ -174,16 +200,16 @@ async def evaluate_domain( "n_paired": len(paired), "missing_prediction_uuids": missing_pred, "extra_prediction_uuids": extra_pred, - "dialogues": [], - "summary":{ - "num_samples": len(gt_list), - "num_correct": 0.0, - "mean_dialogue_score": 0.0, - "min_dialogue_score": 0.0, - "max_dialogue_score": 0.0, - } - } - return domain_out, [] + "dialogues": zero_dialogues, + "summary": { + "num_samples": len(gt_list), + "num_correct": 0.0, + "mean_dialogue_score": 0.0, + "min_dialogue_score": 0.0, + "max_dialogue_score": 0.0, + }, + } + return domain_out, [0.0] * len(zero_dialogues) # Build scorers per domain turn_cfg = TurnScorerConfig( @@ -194,7 +220,7 @@ async def evaluate_domain( cfg=turn_cfg, correctness_judge=policy.correctness_judge, groundedness_judge=policy.groundedness_judge, - exactmatch_judge=policy.exactmatch_judge + exactmatch_judge=policy.exactmatch_judge, ) dialogue_scorer = DialogueScorer( turn_scorer=turn_scorer, @@ -230,8 +256,12 @@ async def evaluate_domain( # and prepend get_data (pred), then sync dialogue tool_call sequences if capability_name == "capability_bi_apis": uuids = [str(gt.get("uuid")) for gt, _ in paired] - batch_tools_gt = _prepend_get_data_to_batch(batch_tools_gt, uuids, skip_initialize_active_data=True) - batch_tools_pred = _prepend_get_data_to_batch(batch_tools_pred, uuids, skip_initialize_active_data=False) + batch_tools_gt = _prepend_get_data_to_batch( + batch_tools_gt, uuids, skip_initialize_active_data=True + ) + batch_tools_pred = _prepend_get_data_to_batch( + batch_tools_pred, uuids, skip_initialize_active_data=False + ) for (gt_raw, pr_raw), uuid in zip(paired, uuids): _update_dialogue_toolcall_for_get_data(gt_raw, uuid, skip_initialize_active_data=True) _update_dialogue_toolcall_for_get_data(pr_raw, uuid, skip_initialize_active_data=False) @@ -251,31 +281,35 @@ async def evaluate_domain( uuid = str(gt_raw.get("uuid")) # Inject fresh responses so groundedness judge uses tool outputs - inject_mcp_responses(pr_raw, mcp_batch_responses_pred[idx], type="pred", capability_name=capability_name) + inject_mcp_responses( + pr_raw, mcp_batch_responses_pred[idx], type="pred", capability_name=capability_name + ) if capability_name != "capability_bi_apis" and mcp_batch_responses_gt is not None: - inject_mcp_responses(gt_raw, mcp_batch_responses_gt[idx], type="gt",capability_name=capability_name) + inject_mcp_responses( + gt_raw, mcp_batch_responses_gt[idx], type="gt", capability_name=capability_name + ) # Score and store details dialogue_score, dialogue_details = dialogue_scorer.score( - gt_dialogue=gt_raw, - pred_dialogue=pr_raw, - pred_key=PRED_OUTPUT_KEY + gt_dialogue=gt_raw, pred_dialogue=pr_raw, pred_key=PRED_OUTPUT_KEY ) dialogue_scores.append(float(dialogue_score)) - domain_out["dialogues"].append({ - "uuid": uuid, - "score": float(dialogue_score), - "metadata": { - "capability": capability_name, - "domain": domain, - "policy": { - "dialogue_aggregate": policy.dialogue_aggregate, - "execute_mcp_tools": policy.execute_mcp_tools, + domain_out["dialogues"].append( + { + "uuid": uuid, + "score": float(dialogue_score), + "metadata": { + "capability": capability_name, + "domain": domain, + "policy": { + "dialogue_aggregate": policy.dialogue_aggregate, + "execute_mcp_tools": policy.execute_mcp_tools, + }, }, - }, - "details": dialogue_details, - }) + "details": dialogue_details, + } + ) else: # No MCP tools - just score based on predictions as-is for idx, (gt_raw, pr_raw) in enumerate( @@ -284,38 +318,46 @@ async def evaluate_domain( uuid = str(gt_raw.get("uuid")) dialogue_score, dialogue_details = dialogue_scorer.score( - gt_dialogue=gt_raw, - pred_dialogue=pr_raw, - pred_key=PRED_OUTPUT_KEY + gt_dialogue=gt_raw, pred_dialogue=pr_raw, pred_key=PRED_OUTPUT_KEY ) dialogue_scores.append(float(dialogue_score)) - domain_out["dialogues"].append({ - "uuid": uuid, - "score": float(dialogue_score), - "metadata": { - "capability": capability_name, - "domain": domain, - "policy": { - "dialogue_aggregate": policy.dialogue_aggregate, - "execute_mcp_tools": policy.execute_mcp_tools, + domain_out["dialogues"].append( + { + "uuid": uuid, + "score": float(dialogue_score), + "metadata": { + "capability": capability_name, + "domain": domain, + "policy": { + "dialogue_aggregate": policy.dialogue_aggregate, + "execute_mcp_tools": policy.execute_mcp_tools, + }, }, - }, - "details": dialogue_details, - }) + "details": dialogue_details, + } + ) + + # Penalize missing predictions as zero-scored dialogues so per-domain and + # top-level summaries share a denominator of len(gt_list). + for uuid in missing_pred: + domain_out["dialogues"].append(_make_missing_dialogue_entry(uuid, capability_name, domain, policy)) + dialogue_scores.append(0.0) # Domain summary domain_scores = [d["score"] for d in domain_out["dialogues"]] + num_samples = len(gt_list) domain_out["summary"] = { - "num_samples": len(gt_list), + "num_samples": num_samples, "num_correct": sum(domain_scores), - "mean_dialogue_score": (sum(domain_scores) / len(domain_scores)) if domain_scores else 0.0, + "mean_dialogue_score": (sum(domain_scores) / num_samples) if num_samples else 0.0, "min_dialogue_score": min(domain_scores) if domain_scores else 0.0, "max_dialogue_score": max(domain_scores) if domain_scores else 0.0, } return domain_out, dialogue_scores + def _load_existing_results(out_path: Path) -> Optional[Dict[str, Any]]: """ Load existing results file if it exists. @@ -349,24 +391,14 @@ def _write_intermediate_results( This allows recovery of partial results if evaluation is interrupted. """ # Calculate totals from completed domains - total_paired = sum( - d["n_paired"] for d in results["domains"].values() - ) - total_missing = sum( - len(d["missing_prediction_uuids"]) - for d in results["domains"].values() - ) - total_extra = sum( - len(d["extra_prediction_uuids"]) - for d in results["domains"].values() - ) + total_paired = sum(d["n_paired"] for d in results["domains"].values()) + total_missing = sum(len(d["missing_prediction_uuids"]) for d in results["domains"].values()) + total_extra = sum(len(d["extra_prediction_uuids"]) for d in results["domains"].values()) - total_samples = sum( - d["summary"]["num_samples"] for d in results["domains"].values() - ) + total_samples = sum(d["summary"]["num_samples"] for d in results["domains"].values()) # Update summary with current progress - num_correct=(sum(all_dialogue_scores) if all_dialogue_scores else 0.0) + num_correct = sum(all_dialogue_scores) if all_dialogue_scores else 0.0 results["summary"] = { "n_domains": len(results["domains"]), "n_paired_dialogues": total_paired, @@ -374,16 +406,9 @@ def _write_intermediate_results( "n_extra_predictions": total_extra, "n_samples": total_samples, "n_correct": num_correct, - "mean_dialogue_score": (num_correct/total_samples - if all_dialogue_scores - else 0.0 - ), - "min_dialogue_score": ( - min(all_dialogue_scores) if all_dialogue_scores else 0.0 - ), - "max_dialogue_score": ( - max(all_dialogue_scores) if all_dialogue_scores else 0.0 - ), + "mean_dialogue_score": (num_correct / total_samples if total_samples else 0.0), + "min_dialogue_score": (min(all_dialogue_scores) if all_dialogue_scores else 0.0), + "max_dialogue_score": (max(all_dialogue_scores) if all_dialogue_scores else 0.0), } # Write to disk @@ -419,9 +444,7 @@ def evaluate_capability( missing = selected_domains - {p.stem for p in all_gt_files} if missing: - raise ValueError( - f"Requested domains not found in groundtruth_dir: {sorted(missing)}" - ) + raise ValueError(f"Requested domains not found in groundtruth_dir: {sorted(missing)}") else: gt_files = all_gt_files @@ -481,7 +504,7 @@ def evaluate_capability( # Final write (summary already updated by last intermediate write) # This is technically redundant but ensures the final state is written - results=_write_intermediate_results(results, all_dialogue_scores, out_path) + results = _write_intermediate_results(results, all_dialogue_scores, out_path) print("=========================================================================") print("==================================[RESULTS]==============================") @@ -503,23 +526,26 @@ def evaluate_capability( # CLI # ----------------------------- + def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--capability_name", required=True, help="Capability name (must exist in registry)") ap.add_argument("--gt_root", required=True, help="Path to capability_name/groundtruth/") ap.add_argument("--pred_root", required=True, help="Path to capability_name/prediction/") - ap.add_argument("--output", default=None, help="Output results.json path (default: /results.json)") + ap.add_argument( + "--output", default=None, help="Output results.json path (default: /results.json)" + ) ap.add_argument( "--mcp-config", default="benchmark/mcp_connection_config.yaml", - help="Path to MCP connection config YAML file (default: benchmark/mcp_connection_config.yaml)" + help="Path to MCP connection config YAML file (default: benchmark/mcp_connection_config.yaml)", ) ap.add_argument( "--domains", nargs="+", default=None, help="Optional list of domain names to evaluate (without .json extension). " - "If omitted, all domains are evaluated." + "If omitted, all domains are evaluated.", ) args = ap.parse_args() diff --git a/benchmarks/m3/evaluator/judge.py b/benchmarks/m3/evaluator/judge.py index 362c8c01..521ec9f8 100644 --- a/benchmarks/m3/evaluator/judge.py +++ b/benchmarks/m3/evaluator/judge.py @@ -1,14 +1,14 @@ from __future__ import annotations -import re -import os import json -from copy import deepcopy -from prompt import GroundednessPrompt, CorrectnessPrompt -from utils import JudgeInput, JudgeOutput -from constant import N_TOOL_CALLS_PER_TURN +import os +import re +from typing import Any, List, Optional + from langchain_openai import ChatOpenAI -from typing import Any, Dict, Optional, Tuple +from prompt import CorrectnessPrompt, GroundednessPrompt +from constant import N_TOOL_CALLS_PER_TURN +from utils import JudgeInput, JudgeOutput _LABEL_RE = re.compile(r"\b(yes|partial|no|unsure)\b", re.IGNORECASE) _CONCLUSION_RE = re.compile(r"\s*(.*?)\s*", re.IGNORECASE | re.DOTALL) @@ -18,14 +18,21 @@ class JudgeOutputParseError(ValueError): pass + class JudgeValidationError(ValueError): """Raised when a judge returns an unexpected/invalid score.""" + pass -class ChatGroq(ChatOpenAI): - """Groq-backed OpenAI-compatible chat model.""" + +class ChatModel(ChatOpenAI): + """ + openai/gpt-oss-120b chat model is being used as LLM-as-a-judge using langchain-openai. + Groq-backed OpenAI-compatible chat model. + """ def __init__(self, config: dict): + # Set model with model or model_name model_name = config.get("model_name", "openai/gpt-oss-120b") end_point = config.get("end_point", "https://api.groq.com/openai") @@ -35,62 +42,108 @@ def __init__(self, config: dict): params = config.get("params", {}) - groq_config = {} - groq_config.setdefault("model", model_name) - groq_config.setdefault("api_key", api_key) - groq_config.setdefault("base_url", end_point.rstrip("/") + "/v1") - groq_config.setdefault("temperature", 0) - groq_config.update(params) + # Set default values for overriding fields + config = {} + config.setdefault("model", model_name) + config.setdefault("api_key", api_key) + config.setdefault("base_url", end_point.rstrip("/") + "/v1") + config.setdefault("temperature", 0) + + config.update(params) + + super().__init__(**config) - super().__init__(**groq_config) class ChatRits(ChatOpenAI): """RITS chat model integration using langchain-openai.""" - def __init__(self, config): - # Set model with model or model_name - model_name=config.get("model_name", "openai/gpt-oss-120b") - end_point=config.get("end_point","https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b") + def __init__(self, config: dict): + model_name = config.get("model_name", "openai/gpt-oss-120b") + end_point = config.get( + "end_point", + "https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b", + ) rits_api_key = os.getenv("RITS_API_KEY") - if rits_api_key is None: - raise ValueError("rits_api_key is required") + if rits_api_key is None or rits_api_key == "": + raise ValueError("RITS_API_KEY is required") + params = config.get("params", {}) - # Set default values for overriding fields rits_config = {} rits_config.setdefault("model_name", model_name) rits_config.setdefault("api_key", "/") rits_config.setdefault("default_headers", {"RITS_API_KEY": rits_api_key}) - rits_config.setdefault("base_url", end_point + "/v1") + rits_config.setdefault("base_url", end_point.rstrip("/") + "/v1") rits_config.update(params) + super().__init__(**rits_config) + +class LiteLLMChatModel(ChatOpenAI): + """ + Azure/gpt-4.1 served through the IBM LiteLLM proxy, used as LLM-as-a-judge. + + Default judge backend (see ``_build_judge_llm``). The Groq ``ChatModel`` above + is retained and still selectable via ``JUDGE_BACKEND=groq`` (or gpt-oss). + Reads the same proxy env the agent's gpt4.1 profile uses: ``OPENAI_BASE_URL`` + + ``OPENAI_API_KEY``. Override the model with ``JUDGE_MODEL_NAME``. + """ + + def __init__(self, config: dict): + model_name = config.get("model_name") or os.getenv("JUDGE_MODEL_NAME", "Azure/gpt-4.1") + base_url = ( + config.get("end_point") + or os.getenv("JUDGE_BASE_URL") + or os.getenv("OPENAI_BASE_URL", "https://ete-litellm.bx.cloud9.ibm.com") + ) + api_key = os.getenv("JUDGE_API_KEY") or os.getenv("OPENAI_API_KEY") + if api_key is None or api_key == "": + raise ValueError("OPENAI_API_KEY (or JUDGE_API_KEY) is required for the LiteLLM judge") + + params = config.get("params", {}) + + # The LiteLLM proxy serves the OpenAI routes at the root, so use base_url + # as-is (no "/v1" suffix) — matching the agent's gpt4.1 profile. + cfg = { + "model": model_name, + "api_key": api_key, + "base_url": base_url.rstrip("/"), + "temperature": 0, + } + cfg.update(params) + + super().__init__(**cfg) + + +def _build_judge_llm(config: dict) -> ChatOpenAI: + """Select the judge backend. Default ``litellm`` -> Azure/gpt-4.1 via the + LiteLLM proxy; set ``JUDGE_BACKEND=groq`` (or gpt-oss) for the legacy + Groq-backed gpt-oss-120b judge, or ``JUDGE_BACKEND=rits`` for RITS. + An explicit ``config["backend"]`` wins.""" + backend = (config.get("backend") or os.getenv("JUDGE_BACKEND", "litellm")).strip().lower() + if backend in ("groq", "gpt-oss", "gpt_oss", "oss"): + return ChatModel(config) + if backend == "rits": + return ChatRits(config) + if backend in ("litellm", "azure", ""): + return LiteLLMChatModel(config) + raise ValueError( + f"Unknown judge backend {backend!r}. Set JUDGE_BACKEND (or config['backend']) to one of: " + "'litellm'/'azure' (default, Azure/gpt-4.1 via the LiteLLM proxy) or " + "'groq'/'gpt-oss' (legacy Groq gpt-oss-120b) or 'rits'." + ) + + class LLMJudge: """ Interface you implement with your provider (OpenAI, vLLM, etc). Must be deterministic as much as possible (temperature=0). """ - def __init__(self, - config: dict = {}): - self.model_config=config - self.llm=self._build_llm(self.model_config) - - def _build_llm(self, config: dict) -> ChatOpenAI: - backend = ( - config.get("backend") - or config.get("provider") - or os.getenv("JUDGE_BACKEND") - ) - if backend is None: - backend = "rits" # Use RITS as default judge backend if not specified - backend = backend.lower() - if backend == "rits": - return ChatRits(config) - if backend == "groq": - return ChatGroq(config) - raise ValueError(f"Unsupported judge backend: {backend!r}. Expected 'groq' or 'rits'.") + def __init__(self, config: dict = {}): + self.model_config = config + self.llm = _build_judge_llm(self.model_config) - def invoke(self, prompt:str) -> str: + def invoke(self, prompt: str) -> str: res = self.llm.invoke(prompt).content return res @@ -103,26 +156,35 @@ class GroundednessJudge(LLMJudge): Check if the predicted answer is grounded in the answers in the turn. """ + def __init__(self, config: dict = {}): + # Allow overriding ONLY the groundedness judge model, independent of the + # shared JUDGE_MODEL_NAME used by the correctness judge. Falls back to the + # standard default in LiteLLMChatModel when unset. + override = os.getenv("GROUNDEDNESS_JUDGE_MODEL_NAME") + if override and not config.get("model_name"): + config = {**config, "model_name": override} + super().__init__(config) + _ws = re.compile(r"\s+") def _norm(self, s: str) -> str: return self._ws.sub(" ", s.strip().lower()) def judge(self, inp: JudgeInput) -> JudgeOutput: - tr=" ".join([self._norm(str(t)) for t in inp.pred_tool_responses[-N_TOOL_CALLS_PER_TURN:]]) # Concatenate the "tool_responses". Only top 20 tool_responses considered. - tr=f"QUERY: {inp.query} {tr}" # Add Query to the responses + tr = " ".join( + [self._norm(str(t)) for t in inp.pred_tool_responses[-N_TOOL_CALLS_PER_TURN:]] + ) # Concatenate the "tool_responses". Only top 20 tool_responses considered. + tr = f"QUERY: {inp.query} {tr}" # Add Query to the responses pr = self._norm(inp.pred_answer) - prompt=GroundednessPrompt.format( - doc=tr, - response=pr) + prompt = GroundednessPrompt.format(doc=tr, response=pr) try: - result=self.invoke(prompt) - result_parsed= self._parse_response(result) + result = self.invoke(prompt) + result_parsed = self._parse_response(result) return JudgeOutput(score=result_parsed["score"], explanation=result_parsed["explanation"]) except Exception as e: return JudgeOutput(score=0.0, explanation=f"Judge Error {e}.") - + def _find_label_and_line(self, text: str) -> tuple[Optional[str], Optional[int]]: lines = [ln.strip() for ln in text.split("\n") if ln.strip()] for i, ln in enumerate(lines[:10]): # scan first few meaningful lines @@ -133,7 +195,7 @@ def _find_label_and_line(self, text: str) -> tuple[Optional[str], Optional[int]] m = _LABEL_RE.search(text) return (m.group(1).lower(), None) if m else (None, None) - def _parse_response(self, raw: str)-> dict: + def _parse_response(self, raw: str) -> dict: """ Parse the judge output into: {"explanation": str, "score": int} @@ -151,7 +213,7 @@ def _parse_response(self, raw: str)-> dict: text = raw.strip() if not text: raise JudgeOutputParseError("empty output") - + # Prefer ... content if present m = _CONCLUSION_RE.search(text) if m: @@ -163,7 +225,7 @@ def _parse_response(self, raw: str)-> dict: if label is None: raise JudgeOutputParseError("No label found (expected: yes/partial/no/unsure)") - + explanation = self._extract_explanation(text, label, line_idx).strip() return {"explanation": explanation.strip(), "score": _SCORE_MAP[label]} @@ -190,18 +252,20 @@ def _extract_explanation(self, text: str, label: str, line_idx: Optional[int]) - m = _LABEL_RE.search(text) if not m: return "" - tail = text[m.end():].lstrip(" \t:;-–—\n").strip() + tail = text[m.end() :].lstrip(" \t:;-–—\n").strip() return tail.split("\n", 1)[0].strip() if tail else "" - + + class CorrectnessJudge(LLMJudge): """ Score the predicted answer as opposed to ground truth answer and query. """ + _ws = re.compile(r"\s+") def _norm(self, s: str) -> str: return self._ws.sub(" ", s.strip().lower()) - + def _parse_response(self, raw: str) -> dict: """ Parse the judge output into: @@ -280,7 +344,7 @@ def coerce_score(v: Any) -> int: return {"explanation": explanation.strip(), "score": score_int} - def _find_first_json_object(text: str) -> Optional[str]: + def _find_first_json_object(self, text: str) -> Optional[str]: """ Returns the substring of the first top-level JSON object found in `text`, or None if none is found. @@ -319,22 +383,20 @@ def _find_first_json_object(text: str) -> Optional[str]: return None def judge(self, inp: JudgeInput) -> JudgeOutput: - query=self._norm(inp.query) + query = self._norm(inp.query) gt = self._norm(inp.gt_answer) pr = self._norm(inp.pred_answer) if not gt and not pr: return JudgeOutput(score=1.0, explanation="Both empty.") - prompt=CorrectnessPrompt.format( - question=query, - answer=gt, - prediction=pr) + prompt = CorrectnessPrompt.format(question=query, answer=gt, prediction=pr) try: - result=self.invoke(prompt) - result_parsed= self._parse_response(result) + result = self.invoke(prompt) + result_parsed = self._parse_response(result) return JudgeOutput(score=result_parsed["score"], explanation=result_parsed["explanation"]) - except Exception as e: + except Exception: return JudgeOutput(score=0.0, explanation="Judge Error {e}.") + class ExactMatchJudge(LLMJudge): """ Ensures the tool_responses (tool_response) match exactly. @@ -365,16 +427,12 @@ def judge(self, inp: "JudgeInput") -> "JudgeOutput": missing = [e for e in expected_cmp if e not in actual_cmp] if not missing: - return JudgeOutput( - score=1.0, - explanation="All expected tool_responses are present in actual." - ) + return JudgeOutput(score=1.0, explanation="All expected tool_responses are present in actual.") else: return JudgeOutput( score=0.0, explanation=( - "Missing expected tool_responses.\n" - f"Missing: {missing}\n" + f"Missing expected tool_responses.\nMissing: {missing}\n" # f"Expected ({len(expected_cmp)}): {expected_cmp}\n" # f"Actual ({len(actual_cmp)}): {actual_cmp}" ), diff --git a/benchmarks/m3/evaluator/mcp_tools.py b/benchmarks/m3/evaluator/mcp_tools.py index 201292be..6cc97e29 100644 --- a/benchmarks/m3/evaluator/mcp_tools.py +++ b/benchmarks/m3/evaluator/mcp_tools.py @@ -1,18 +1,17 @@ from __future__ import annotations -import os -import json import asyncio +import json +import os import warnings from abc import ABC, abstractmethod from contextlib import asynccontextmanager from dataclasses import dataclass from typing import Any, AsyncIterator, Dict, List, Optional, Sequence +from constant import PRED_OUTPUT_KEY, PRED_OUTPUT_SEQUENCE_KEY from mcp import ClientSession, StdioServerParameters from mcp.client.stdio import stdio_client - -from constant import PRED_OUTPUT_KEY, PRED_OUTPUT_SEQUENCE_KEY from utils import _ensure_list ToolSpec = Dict[str, Any] @@ -22,6 +21,7 @@ # Prediction dialogue helpers # ----------------------------- + def extract_toolcalls_for_mcp( pred_dialogue: Dict[str, Any], limit_last_n: Optional[int] = None, @@ -50,18 +50,18 @@ def extract_toolcalls_for_mcp( if not isinstance(raw_tc, dict): continue if isinstance(raw_tc, dict) and "name" in raw_tc: - turn_tools.append( - {"name": raw_tc["name"], "arguments": raw_tc.get("arguments", {})}) + turn_tools.append({"name": raw_tc["name"], "arguments": raw_tc.get("arguments", {})}) if limit_last_n is not None: turn_tools = turn_tools[-limit_last_n:] dialogue_tools.append(turn_tools) return dialogue_tools + def inject_mcp_responses( pred_entry: Dict[str, Any], mcp_dialogue_responses: List[List[Any]], - type: str, # "pred" / "gt" - capability_name:str, + type: str, # "pred" / "gt" + capability_name: str, ) -> None: """ Inject tool responses into each turn's sequence.tool_response. @@ -72,7 +72,6 @@ def inject_mcp_responses( - they should be the same length (1:1 aligned) for downstream scoring """ turns = _ensure_list(pred_entry.get(PRED_OUTPUT_KEY, [])) - query_tool_present=False for i, turn in enumerate(turns): if not isinstance(turn, dict): @@ -91,30 +90,37 @@ def inject_mcp_responses( if type == "pred" and len(tool_calls) > len(turn_resps): tool_calls = tool_calls[-len(turn_resps):] if turn_resps else [] + query_tool_present = False if "multiturn" in capability_name: for tool in tool_calls: if "query_" in tool["name"]: - query_tool_present=True + query_tool_present = True break # If tool_call exists, enforce 1:1 length alignment if tool_calls: n = len(tool_calls) - if (query_tool_present): - truncated_responses=turn_resps[:n] + if query_tool_present: + truncated_responses = turn_resps[:n] if type == "gt": for idx, tool in enumerate(tool_calls): if "query_" not in tool["name"]: seq["tool_response"][idx] = truncated_responses[idx] else: - seq["tool_response"][idx] = [item["text"] for item in seq["tool_response"][idx]] # Only text in chunks is retained - elif type=="pred": + seq["tool_response"][idx] = [ + item["text"] for item in seq["tool_response"][idx] + ] # Only text in chunks is retained + elif type == "pred": for idx, tool in enumerate(tool_calls): if "query_" in tool["name"]: try: - truncated_responses[idx] = [item["text"] for item in json.loads(truncated_responses[idx])["results"]] # Only text in chunks is retained + truncated_responses[idx] = [ + item["text"] for item in json.loads(truncated_responses[idx])["results"] + ] # Only text in chunks is retained except Exception as e: - truncated_responses[idx] = [f"Error processing response: {str(e)}"] + truncated_responses[idx] = [ + f"Error processing response: {str(e)}" + ] seq["tool_response"] = truncated_responses else: if len(turn_resps) >= n: @@ -130,12 +136,14 @@ def inject_mcp_responses( # If you require strict alignment only when tool_calls exist: if tool_calls: - assert len(seq["tool_call"]) == len(seq["tool_response"]) + assert len(seq["tool_call"]) == len(seq["tool_response"]) # noqa: S101 — runtime invariant on tool/response alignment + # ----------------------------- # Schema coercion helpers # ----------------------------- + def _coerce_value_to_schema(value: Any, schema: Optional[Dict[str, Any]]) -> Any: """ Coerce a single value to match a (subset of) JSON schema. @@ -216,7 +224,9 @@ def _coerce_args_to_schema(args: Dict[str, Any], input_schema: Optional[Dict[str if not input_schema or not isinstance(input_schema, dict): return args - properties = input_schema.get("properties", {}) if isinstance(input_schema.get("properties", {}), dict) else {} + properties = ( + input_schema.get("properties", {}) if isinstance(input_schema.get("properties", {}), dict) else {} + ) coerced: Dict[str, Any] = {} for k, v in args.items(): @@ -230,6 +240,7 @@ def _coerce_args_to_schema(args: Dict[str, Any], input_schema: Optional[Dict[str # Batch tool execution # ----------------------------- + async def execute_tools_batch( session: ClientSession, dialogue_tools_batch: List[List[List[Dict[str, Any]]]], @@ -263,7 +274,7 @@ async def execute_tools_batch( tool_name = tool["name"] raw_args = tool.get("arguments", {}) or {} if is_gt: - if (("input_value" in raw_args.keys()) or "data_label" in raw_args.keys()) and len(turn_responses)>0: + if (("input_value" in raw_args.keys()) or "data_label" in raw_args.keys()) and len(turn_responses) > 0: if "handle" in turn_responses[-1]: previous_response = json.loads(turn_responses[-1]) if "input_value" in raw_args.keys(): @@ -354,19 +365,23 @@ async def connect_to_mcp_server(self) -> AsyncIterator[ClientSession]: await session.initialize() yield session - async def call_mcp_tools(self, session: ClientSession, tools: Sequence[ToolSpec], schema_map: Optional[Dict[str, Any]]) -> List[Any]: + async def call_mcp_tools( + self, session: ClientSession, tools: Sequence[ToolSpec], schema_map: Optional[Dict[str, Any]] + ) -> List[Any]: """ Calls MCP tools using an already-connected and initialized ClientSession. """ raise NotImplementedError - async def run_tools(self, tools: Optional[Sequence[ToolSpec]] = None, schema_map: Optional[Dict[str, Any]] = None) -> List[Any]: + async def run_tools( + self, tools: Optional[Sequence[ToolSpec]] = None, schema_map: Optional[Dict[str, Any]] = None + ) -> List[Any]: """ Connect to MCP server and call the provided tools. """ async with self.connect_to_mcp_server() as session: return await self.call_mcp_tools(session, tools, schema_map) - + async def list_loaded_tools(self) -> List[Dict[str, Any]]: """ Connects to the MCP server and returns the list of tools that the server reports as loaded. @@ -386,12 +401,16 @@ async def list_loaded_tools(self) -> List[Dict[str, Any]]: for t in tool_list: # t may be dict or object name = t.get("name") if isinstance(t, dict) else getattr(t, "name", None) - input_schema = t.get("inputSchema") if isinstance(t, dict) else getattr(t, "inputSchema", None) + input_schema = ( + t.get("inputSchema") if isinstance(t, dict) else getattr(t, "inputSchema", None) + ) if isinstance(name, str) and isinstance(input_schema, dict): schema_map[name] = input_schema return tool_list, schema_map - def _coerce_args_to_schema(self, args: Dict[str, Any], input_schema: Optional[Dict[str, Any]]) -> Dict[str, Any]: + def _coerce_args_to_schema( + self, args: Dict[str, Any], input_schema: Optional[Dict[str, Any]] + ) -> Dict[str, Any]: """Delegate to module-level _coerce_args_to_schema.""" return _coerce_args_to_schema(args, input_schema) @@ -436,7 +455,7 @@ def build_server_params(self) -> StdioServerParameters: args=exec_args, env=None, ) - elif self.config.run_type =="local": + elif self.config.run_type == "local": return StdioServerParameters( command="python", args=[self.local_server_script], @@ -445,8 +464,10 @@ def build_server_params(self) -> StdioServerParameters: "MCP_DOMAIN": self.config.domain, }, ) - - async def call_mcp_tools(self, session: ClientSession, tools: Sequence[ToolSpec], schema_map: Optional[Dict[str, Any]]) -> List[Any]: + + async def call_mcp_tools( + self, session: ClientSession, tools: Sequence[ToolSpec], schema_map: Optional[Dict[str, Any]] + ) -> List[Any]: """ Calls MCP tools using an already-connected and initialized ClientSession. """ @@ -465,7 +486,9 @@ async def call_mcp_tools(self, session: ClientSession, tools: Sequence[ToolSpec] raw_args = json.loads(raw_args) if not isinstance(raw_args, dict): - raise TypeError(f"Tool arguments must be a dict (or JSON string), got {type(raw_args)} for {name}") + raise TypeError( + f"Tool arguments must be a dict (or JSON string), got {type(raw_args)} for {name}" + ) # 2) Coerce args according to schema if available input_schema = schema_map.get(name) @@ -482,6 +505,7 @@ async def call_mcp_tools(self, session: ClientSession, tools: Sequence[ToolSpec] return results + class PythonMCPToolClient(MCPToolClientBase): """ MCP client for Python tools. @@ -567,6 +591,7 @@ def build_server_params(self) -> StdioServerParameters: }, ) + def test(tools: List[ToolSpec]) -> List[Any]: async def _run(): client = FastAPIMCPToolClient( @@ -576,6 +601,7 @@ async def _run(): return asyncio.run(_run()) + if __name__ == "__main__": # This example gives a super huge output. # tools = [[[ diff --git a/benchmarks/m3/evaluator/prompt.py b/benchmarks/m3/evaluator/prompt.py index 058b7a85..6551c951 100644 --- a/benchmarks/m3/evaluator/prompt.py +++ b/benchmarks/m3/evaluator/prompt.py @@ -1,4 +1,3 @@ - # Auto-evalution prompt from "CRAG: Comprehensive RAG Benchmark" is used for correctness prompt. # https://github.com/facebookresearch/CRAG/blob/main/prompts/templates.py CorrectnessPrompt = """Assume you are a human expert in grading predictions given by a model. You are given a question and a model prediction. Judge if the prediction matches the ground truth answer by following these steps: @@ -32,7 +31,7 @@ """ -GroundednessPrompt =""" +GroundednessPrompt = """ The following tasks each contains document and a response. The response is supposed to rely on the document for its source of information, optionally using common sense knowledge and common sense inference, but it may fail this, and instead contain substantial claims that are not grounded in the document or common sense knowledge. Your task is to assess whether the response is entirely grounded in the document, grounded in the document plus common sense knowledge and reasoning, or ungrounded. To make this determination, perform the following steps: @@ -40,7 +39,7 @@ - Ignore non-substantial claims, such as greetings or self-descriptions such as "I'm a helpful assistant", - Try to formulate each claim in a stand-alone form with all pronouns and other references resolved; 2. Assess the grounding of each of these claims: - - If it is essentially a rephrasing of information from the document, or can be derived from such information by trivial common-sense reasoning, it is grounded, This is so even if it contradicts other parts of the document. + - If it is essentially a rephrasing of information from the document, or can be derived from such information by trivial common-sense reasoning, it is grounded, This is so even if it contradicts other parts of the document. - If it relies on, in additional to information from the document, additional non-trivial common sense knowledge or common sense reasoning, it is partially grounded, - If a claim is about the provided document, or about the agent's state of knowledge, with the effect of not being able to answer the user inquiry, it is grounded if and only if the required information is indeed lacking in the document. - If a claim cannot be derived directly from the document or indirectly with help of common sense knowledge and reasoning, it is ungrounded; @@ -58,11 +57,11 @@ - "no" is for ungrounded, - "unsure" is for the situations where the document, conversation or response contain ambiguities such that different interpretations lead to different conclusions about groundedness; - The second line contains an explanation of your answer as short as possible. - + Here is the document starting with and end with - + {doc} - + Here is the response starting with and end with . @@ -77,4 +76,4 @@ - "no" is for ungrounded, - "unsure" is for the situations where the document, conversation or response contain ambiguities such that different interpretations lead to different conclusions about groundedness; -""" \ No newline at end of file +""" diff --git a/benchmarks/m3/evaluator/scorer.py b/benchmarks/m3/evaluator/scorer.py index f5d1ed23..0c67d0eb 100644 --- a/benchmarks/m3/evaluator/scorer.py +++ b/benchmarks/m3/evaluator/scorer.py @@ -3,11 +3,23 @@ from dataclasses import dataclass from importlib import import_module from importlib.util import find_spec -from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple -from utils import ToolCall -from judge import CorrectnessJudge, GroundednessJudge, LLMJudge, JudgeValidationError -from utils import JudgeInput -from constant import N_TOOL_CALLS_PER_TURN, PRED_OUTPUT_KEY, PRED_OUTPUT_TURN_ID_KEY, PRED_OUTPUT_QUERY_KEY, PRED_OUTPUT_ANSWER_KEY, PRED_OUTPUT_SEQUENCE_KEY, GT_OUTPUT_KEY, GT_OUTPUT_TURN_ID_KEY, GT_OUTPUT_QUERY_KEY, GT_OUTPUT_ANSWER_KEY, GT_OUTPUT_SEQUENCE_KEY +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from constant import ( + GT_OUTPUT_ANSWER_KEY, + GT_OUTPUT_KEY, + GT_OUTPUT_QUERY_KEY, + GT_OUTPUT_SEQUENCE_KEY, + GT_OUTPUT_TURN_ID_KEY, + PRED_OUTPUT_ANSWER_KEY, + PRED_OUTPUT_KEY, + PRED_OUTPUT_QUERY_KEY, + PRED_OUTPUT_SEQUENCE_KEY, + PRED_OUTPUT_TURN_ID_KEY, + N_TOOL_CALLS_PER_TURN, +) +from judge import CorrectnessJudge, ExactMatchJudge, GroundednessJudge, JudgeValidationError, LLMJudge +from utils import JudgeInput, ToolCall def _load_policy_judge() -> Optional[Any]: @@ -24,6 +36,7 @@ def _load_policy_judge() -> Optional[Any]: # Output Scorer # ----------------------------- + @dataclass(frozen=True) class TurnScorerConfig: """ @@ -34,27 +47,40 @@ class TurnScorerConfig: - ground truth answer and predicted answer are evaluated using an LLM-as-Judge """ + capability: str domain: str + class TurnScorer: """ Compares GT vs predicted tool trajectory per turn based on final answer. Produces a binary score to evaluate a turn. """ - def __init__(self, cfg: TurnScorerConfig | None = None, - correctness_judge: Optional[LLMJudge] = None, - groundedness_judge: Optional[LLMJudge] = None, - exactmatch_judge: Optional[LLMJudge] = None): + + def __init__( + self, + cfg: TurnScorerConfig | None = None, + correctness_judge: Optional[LLMJudge] = None, + groundedness_judge: Optional[LLMJudge] = None, + exactmatch_judge: Optional[LLMJudge] = None, + ): self.cfg = cfg or TurnScorerConfig() - self.correctness_judge=correctness_judge - self.groundedness_judge=groundedness_judge - self.exactmatch_judge=exactmatch_judge - self.policy_judge=_load_policy_judge() - - def compare(self, query: str, additional_instructions:str - , gt_answer: str, pred_answer: str - , gt: Sequence[ToolCall], pred: Sequence[ToolCall] - , gt_responses: Sequence[Any], pred_responses: Sequence[Any]) -> Tuple[float, Dict[str, Any]]: + self.correctness_judge = correctness_judge + self.groundedness_judge = groundedness_judge + self.exactmatch_judge = exactmatch_judge + self.policy_judge = _load_policy_judge() + + def compare( + self, + query: str, + additional_instructions: str, + gt_answer: str, + pred_answer: str, + gt: Sequence[ToolCall], + pred: Sequence[ToolCall], + gt_responses: Sequence[Any], + pred_responses: Sequence[Any], + ) -> Tuple[float, Dict[str, Any]]: # Run tools in pred and ascertain if predicted answer is grounded on tool call answers. # Use Judge to ascertain pred answer is answering the query. input = JudgeInput( @@ -69,15 +95,15 @@ def compare(self, query: str, additional_instructions:str gt_tool_responses=gt_responses, pred_tool_responses=pred_responses, ) - + # Scoring Turns - extra_steps=len(pred)-len(gt) + extra_steps = len(pred) - len(gt) # Check for policy adherence when an internal policy_judge.py file is present. if self.policy_judge is not None and "multiturn" in self.cfg.capability: policy = self.policy_judge.judge(inp=input) policy_score, policy_explanation = float(policy.score), policy.explanation - if policy_score==0.0: + if policy_score == 0.0: score = policy_score details = { "gt_steps": len(gt), @@ -87,7 +113,12 @@ def compare(self, query: str, additional_instructions:str "exactmatch_score": None, "answer_score": None, "groundedness_score": None, - "score_explanation": {"policy": policy_explanation, "answer": None, "exactmatch": None, "groundedness":None}, + "score_explanation": { + "policy": policy_explanation, + "answer": None, + "exactmatch": None, + "groundedness": None, + }, } return score, details @@ -100,12 +131,12 @@ def compare(self, query: str, additional_instructions:str exactmatch_score = 1.0 exactmatch_explanation = "Special case applied - GT indicates unanswerable and no tool calls made, granting full exact match credit." else: - exactmatch=self.exactmatch_judge.judge(inp=input) - exactmatch_score, exactmatch_explanation=float(exactmatch.score), exactmatch.explanation - if exactmatch_score==0.0: - correctness=self.correctness_judge.judge(inp=input) - answer_score, answer_explanation=float(correctness.score), correctness.explanation - if answer_score==0.0: + exactmatch = self.exactmatch_judge.judge(inp=input) + exactmatch_score, exactmatch_explanation = float(exactmatch.score), exactmatch.explanation + if exactmatch_score == 0.0: + correctness = self.correctness_judge.judge(inp=input) + answer_score, answer_explanation = float(correctness.score), correctness.explanation + if answer_score == 0.0: score = answer_score details = { "gt_steps": len(gt), @@ -114,10 +145,14 @@ def compare(self, query: str, additional_instructions:str "exactmatch_score": exactmatch_score, "answer_score": answer_score, "groundedness_score": None, - "score_explanation": {"answer": answer_explanation, "exactmatch": exactmatch_explanation, "groundedness":None}, + "score_explanation": { + "answer": answer_explanation, + "exactmatch": exactmatch_explanation, + "groundedness": None, + }, } return score, details - elif answer_score==1.0: + elif answer_score == 1.0: if ( "multiturn" in self.cfg.capability and "i can not answer" in input.gt_answer.lower() @@ -140,7 +175,10 @@ def compare(self, query: str, additional_instructions:str return score, details groundedness = self.groundedness_judge.judge(inp=input) - groundedness_score, groundedness_explanation = float(groundedness.score), groundedness.explanation + groundedness_score, groundedness_explanation = ( + float(groundedness.score), + groundedness.explanation, + ) score = groundedness_score details = { "gt_steps": len(gt), @@ -149,7 +187,11 @@ def compare(self, query: str, additional_instructions:str "exactmatch_score": exactmatch_score, "answer_score": answer_score, "groundedness_score": groundedness_score, - "score_explanation": {"answer": answer_explanation, "exactmatch": exactmatch_explanation, "groundedness":groundedness_explanation}, + "score_explanation": { + "answer": answer_explanation, + "exactmatch": exactmatch_explanation, + "groundedness": groundedness_explanation, + }, } return score, details else: @@ -157,7 +199,7 @@ def compare(self, query: str, additional_instructions:str f"Unexpected answer_score from correctness_judge: {answer_score!r}. " f"Expected 0.0 or 1.0. Explanation: {answer_explanation!r}" ) - elif exactmatch_score==1.0: + elif exactmatch_score == 1.0: if ( "multiturn" in self.cfg.capability and "i can not answer" in input.gt_answer.lower() @@ -189,7 +231,11 @@ def compare(self, query: str, additional_instructions:str "exactmatch_score": exactmatch_score, "answer_score": None, "groundedness_score": groundedness_score, - "score_explanation": {"answer": None, "exactmatch": exactmatch_explanation, "groundedness":groundedness_explanation}, + "score_explanation": { + "answer": None, + "exactmatch": exactmatch_explanation, + "groundedness": groundedness_explanation, + }, } return score, details else: @@ -198,6 +244,7 @@ def compare(self, query: str, additional_instructions:str f"exactmatch_explanation={exactmatch_explanation!r}" ) + @dataclass(frozen=True) class DialogueScorerConfig: """ @@ -206,6 +253,7 @@ class DialogueScorerConfig: - "sum": sum of per-turn scores - "min": minimum per-turn score (strict) """ + aggregate: str = "mean" # "mean" | "sum" | "min" @@ -235,9 +283,9 @@ class DialogueScorer: "output": [ { "turn_id": 0, - "query": "...", + "query": "...", "predicted_answer": "100", - "sequence": + "sequence": { "tool_call": [{"name": "...", "arguments": {...}}], "tool_response": [[100]] @@ -269,28 +317,45 @@ def score( ) -> Tuple[float, Dict[str, Any]]: gt_turns: List[Dict[str, Any]] = list(gt_dialogue.get(gt_key, [])) pred_turns: List[Dict[str, Any]] = list(pred_dialogue.get(pred_key, [])) - additional_instructions = gt_dialogue.get("additional_instructions","") - - assert len(pred_turns) == 1, f"Predicted Turns {len(pred_turns)} should have been 1." + additional_instructions = gt_dialogue.get("additional_instructions", "") + + assert len(pred_turns) == 1, f"Predicted Turns {len(pred_turns)} should have been 1." # noqa: S101 — runtime invariant for single-turn scoring pred_by_id = {t.get(PRED_OUTPUT_TURN_ID_KEY): t for t in pred_turns if "turn_id" in t} per_turn: List[Dict[str, Any]] = [] turn_scores: List[float] = [] - for gt_turn in [gt_turns[-1]]: # Only the last turn evaluated against prediction + for gt_turn in [gt_turns[-1]]: # Only the last turn evaluated against prediction turn_id = gt_turn.get(GT_OUTPUT_TURN_ID_KEY) query = str(gt_turn.get(GT_OUTPUT_QUERY_KEY)) gt_answer = self._stringify_answer(gt_turn.get(GT_OUTPUT_ANSWER_KEY)) - gt_calls = gt_turn.get(GT_OUTPUT_SEQUENCE_KEY, {}).get("tool_call",[]) - gt_responses = self._extract_tool_responses(gt_turn.get(GT_OUTPUT_SEQUENCE_KEY, {}).get("tool_response",[])) + gt_calls = gt_turn.get(GT_OUTPUT_SEQUENCE_KEY, {}).get("tool_call", []) + gt_responses = self._extract_tool_responses( + gt_turn.get(GT_OUTPUT_SEQUENCE_KEY, {}).get("tool_response", []) + ) pred_turn = pred_by_id.get(turn_id, None) + if pred_turn is None: + per_turn.append( + { + "turn_id": turn_id, + "query": query, + "pred_answer": "", + "score": 0.0, + "metadata": {"error": f"missing predicted turn_id={turn_id}"}, + } + ) + turn_scores.append(0.0) + continue + pred_answer = self._stringify_pred_answer(pred_turn.get(PRED_OUTPUT_ANSWER_KEY, "")) pred_sequence = pred_turn.get(PRED_OUTPUT_SEQUENCE_KEY, {}) or {} pred_calls_all = pred_sequence.get("tool_call", []) or [] pred_calls = pred_calls_all[-N_TOOL_CALLS_PER_TURN:] if isinstance(pred_calls_all, list) else [] - pred_responses = self._extract_tool_responses(pred_sequence.get("tool_response", []))[-N_TOOL_CALLS_PER_TURN:] + pred_responses = self._extract_tool_responses( + pred_sequence.get("tool_response", []) + )[-N_TOOL_CALLS_PER_TURN:] score, details = self.turn_scorer.compare( query=query, @@ -309,7 +374,7 @@ def score( "query": query, "pred_answer": pred_answer, "score": score, - "metadata":details, + "metadata": details, } ) turn_scores.append(float(score)) @@ -358,7 +423,7 @@ def _stringify_answer(self, answer_obj: Any) -> str: return ", ".join(flattened) - except Exception: + except Exception: # noqa: S110 — fall through to str() on any stringify error pass return str(answer_obj) @@ -404,7 +469,7 @@ def _flatten_2d(self, obj: Any) -> List[List[Any]]: return [obj] # type: ignore[list-item] # non-list -> wrap twice return [[obj]] - + if __name__ == "__main__": # ---- Example ground-truth dialogue ---- @@ -414,16 +479,15 @@ def _flatten_2d(self, obj: Any) -> List[List[Any]]: GT_OUTPUT_TURN_ID_KEY: 0, GT_OUTPUT_QUERY_KEY: "How many students have never been absent from school?", GT_OUTPUT_ANSWER_KEY: [[100]], - GT_OUTPUT_SEQUENCE_KEY: - { - "tool_call": [ - { - "name": "get_count_names_by_month_v1_student_loan_count_names_by_month_get", - "arguments": {"month": "0"}, - } - ], - "tool_response": [[100]], - } + GT_OUTPUT_SEQUENCE_KEY: { + "tool_call": [ + { + "name": "get_count_names_by_month_v1_student_loan_count_names_by_month_get", + "arguments": {"month": "0"}, + } + ], + "tool_response": [[100]], + }, } ] } @@ -435,18 +499,17 @@ def _flatten_2d(self, obj: Any) -> List[List[Any]]: PRED_OUTPUT_TURN_ID_KEY: 0, PRED_OUTPUT_QUERY_KEY: "How many students have never been absent from school?", PRED_OUTPUT_ANSWER_KEY: "100", - PRED_OUTPUT_SEQUENCE_KEY: - { - "tool_call": [ - [ - { - "name": "get_count_names_by_month_v1_student_loan_count_names_by_month_get", - "arguments": {"month": "0"}, - } - ] - ], - "tool_response": [[500]], - } + PRED_OUTPUT_SEQUENCE_KEY: { + "tool_call": [ + [ + { + "name": "get_count_names_by_month_v1_student_loan_count_names_by_month_get", + "arguments": {"month": "0"}, + } + ] + ], + "tool_response": [[500]], + }, } ] } @@ -457,15 +520,17 @@ def _flatten_2d(self, obj: Any) -> List[List[Any]]: answer_weight=0.5, trajectory_weight=0.5, extra_step_penalty=0.1, - ) + ) - turn_scorer = TurnScorer(cfg=turn_cfg, - correctness_judge=CorrectnessJudge(), - groundedness_judge=GroundednessJudge(), - exactmatch_judge=ExactMatchJudge(config={})) + turn_scorer = TurnScorer( + cfg=turn_cfg, + correctness_judge=CorrectnessJudge(), + groundedness_judge=GroundednessJudge(), + exactmatch_judge=ExactMatchJudge(config={}), + ) dialogue_cfg = DialogueScorerConfig( - aggregate="mean", # mean | sum | min + aggregate="mean", # mean | sum | min ) dialogue_scorer = DialogueScorer( diff --git a/benchmarks/m3/evaluator/utils.py b/benchmarks/m3/evaluator/utils.py index 3ac6e8e9..9497c6d0 100644 --- a/benchmarks/m3/evaluator/utils.py +++ b/benchmarks/m3/evaluator/utils.py @@ -4,13 +4,13 @@ import json from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, Iterable, List, Literal, Mapping, Optional, Sequence, Tuple - +from typing import Any, Dict, List, Literal, Mapping, Optional, Sequence, Tuple # ----------------------------- # General helpers # ----------------------------- + def _ensure_list(x: Any) -> List[Any]: return x if isinstance(x, list) else ([] if x is None else [x]) @@ -38,6 +38,7 @@ def pair_dialogues_by_uuid( paired.append((gt_map[u], pr_map[u])) return paired, missing_pred, extra_pred + # ----------------------------- # Data model # ----------------------------- @@ -104,6 +105,7 @@ def from_dict(d: Mapping[str, Any]) -> "Example": @dataclass(frozen=True) class PairedExample: """Ground truth + prediction matched by example ID.""" + gt: Example pred: Example @@ -112,6 +114,7 @@ class PairedExample: # Metrics / results # ----------------------------- + @dataclass class ScalarMetric: name: str @@ -157,12 +160,13 @@ def to_dict(self) -> Dict[str, Any]: # LLM Judge for correctness and groundedness # ----------------------------- + @dataclass(frozen=True) class JudgeInput: capability: str domain: str query: str - additional_instructions: str + additional_instructions: str gt_answer: str pred_answer: str gt_tool_calls: Sequence[ToolCall] @@ -170,9 +174,9 @@ class JudgeInput: gt_tool_responses: List[str] pred_tool_responses: List[str] + @dataclass(frozen=True) class JudgeOutput: # score should be normalized either 1 or 0 score: float explanation: Optional[str] = None - From 161cf100a4967985eeeb63e51c44c4cb355f9a75 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Tue, 21 Jul 2026 14:08:44 -0400 Subject: [PATCH 03/40] Make policy judge path explicit --- benchmarks/m3/evaluator/README.md | 3 ++- benchmarks/m3/evaluator/evaluator.py | 10 ++++++++++ benchmarks/m3/evaluator/scorer.py | 22 ++++++++++++++++------ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/benchmarks/m3/evaluator/README.md b/benchmarks/m3/evaluator/README.md index a3528eb0..dffa6c47 100644 --- a/benchmarks/m3/evaluator/README.md +++ b/benchmarks/m3/evaluator/README.md @@ -30,7 +30,7 @@ Each paired dialogue is scored in the following order: 3. **Groundedness check** If the answer is acceptable, another LLM judge verifies that the predicted answer is grounded in the executed tool outputs. 4. **PolicyAdherenceJudge** - Optional: for the multi-hop multi-source capability, the evaluator can programmatically check policy application when `evaluator/policy_judge.py` is present. If your checkout does not include this file and you need policy evaluation, open a GitHub issue requesting access to the PolicyJudge file. + Optional: for the multi-hop multi-source capability, the evaluator can programmatically check policy application when `--policy-judge-path /path/to/policy_judge.py` is provided. By default this argument is unset and policy judging is disabled. If you need policy evaluation, open a GitHub issue requesting access to the PolicyJudge file. Turn scores are then aggregated into a dialogue score. The default aggregation policy in this repo is `mean`. @@ -144,6 +144,7 @@ CLI arguments: - `--output`: output file path; if omitted, defaults to `results.json` next to the capability directory - `--mcp-config`: MCP connection YAML, defaulting to [`benchmark/mcp_connection_config.yaml`](/vakra/benchmark/mcp_connection_config.yaml) - `--domains`: optional list of domain names to evaluate +- `--policy-judge-path`: optional path to a PolicyJudge Python file. If omitted, policy judging is disabled. ## Output Format diff --git a/benchmarks/m3/evaluator/evaluator.py b/benchmarks/m3/evaluator/evaluator.py index 8a52d70f..ed157bb5 100644 --- a/benchmarks/m3/evaluator/evaluator.py +++ b/benchmarks/m3/evaluator/evaluator.py @@ -175,6 +175,7 @@ async def evaluate_domain( policy: CapabilityPolicy, mcp_config: Optional[MCPConnectionConfig], capability_name: str, + policy_judge_path: Optional[str] = None, ) -> Tuple[Dict[str, Any], List[float]]: """ Evaluate a single domain (async version). @@ -215,6 +216,7 @@ async def evaluate_domain( turn_cfg = TurnScorerConfig( capability=capability_name, domain=domain, + policy_judge_path=policy_judge_path, ) turn_scorer = TurnScorer( cfg=turn_cfg, @@ -428,6 +430,7 @@ def evaluate_capability( registry: Dict[str, CapabilityPolicy], mcp_config: Optional[MCPConnectionConfig] = None, selected_domains: Optional[set[str]] = None, + policy_judge_path: Optional[str] = None, ) -> Dict[str, Any]: if capability_name not in registry: raise ValueError( @@ -492,6 +495,7 @@ def evaluate_capability( policy=policy, mcp_config=mcp_config, capability_name=capability_name, + policy_judge_path=policy_judge_path, ) ) @@ -547,6 +551,11 @@ def main() -> None: help="Optional list of domain names to evaluate (without .json extension). " "If omitted, all domains are evaluated.", ) + ap.add_argument( + "--policy-judge-path", + default=None, + help="Optional path to a PolicyJudge Python file. If omitted, policy judging is disabled.", + ) args = ap.parse_args() capability_name = args.capability_name @@ -589,6 +598,7 @@ def main() -> None: registry=registry, mcp_config=mcp_config, selected_domains=selected_domains, + policy_judge_path=args.policy_judge_path, ) print(f"Wrote: {out_path}") diff --git a/benchmarks/m3/evaluator/scorer.py b/benchmarks/m3/evaluator/scorer.py index 0c67d0eb..33891ec6 100644 --- a/benchmarks/m3/evaluator/scorer.py +++ b/benchmarks/m3/evaluator/scorer.py @@ -1,8 +1,8 @@ from __future__ import annotations from dataclasses import dataclass -from importlib import import_module -from importlib.util import find_spec +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path from typing import Any, Dict, List, Optional, Sequence, Tuple from constant import ( @@ -22,11 +22,20 @@ from utils import JudgeInput, ToolCall -def _load_policy_judge() -> Optional[Any]: - if find_spec("policy_judge") is None: +def _load_policy_judge(policy_judge_path: Optional[str]) -> Optional[Any]: + if policy_judge_path is None: return None - policy_module = import_module("policy_judge") + path = Path(policy_judge_path).expanduser() + if not path.exists(): + raise FileNotFoundError(f"PolicyJudge file not found: {path}") + + spec = spec_from_file_location("policy_judge", path) + if spec is None or spec.loader is None: + raise ImportError(f"Could not load PolicyJudge file: {path}") + + policy_module = module_from_spec(spec) + spec.loader.exec_module(policy_module) judge_cls = getattr(policy_module, "PolicyAdherenceJudge", None) if judge_cls is None: judge_cls = getattr(policy_module, "PolicyAdheranceJudge") @@ -50,6 +59,7 @@ class TurnScorerConfig: capability: str domain: str + policy_judge_path: Optional[str] = None class TurnScorer: @@ -68,7 +78,7 @@ def __init__( self.correctness_judge = correctness_judge self.groundedness_judge = groundedness_judge self.exactmatch_judge = exactmatch_judge - self.policy_judge = _load_policy_judge() + self.policy_judge = _load_policy_judge(self.cfg.policy_judge_path) def compare( self, From ab5b6193a540c111e07117bbce038461de7e4cb4 Mon Sep 17 00:00:00 2001 From: Ankita Naik Date: Mon, 17 Aug 2026 18:26:03 -0400 Subject: [PATCH 04/40] Add PR regression testing workflow setup script This script sets up a PR regression testing environment by cloning necessary repositories, installing dependencies, and configuring a GitHub Actions runner. --- scripts/setup-pr-regression-workflow.sh | 289 ++++++++++++++++++++++++ 1 file changed, 289 insertions(+) create mode 100644 scripts/setup-pr-regression-workflow.sh diff --git a/scripts/setup-pr-regression-workflow.sh b/scripts/setup-pr-regression-workflow.sh new file mode 100644 index 00000000..17ab9547 --- /dev/null +++ b/scripts/setup-pr-regression-workflow.sh @@ -0,0 +1,289 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# ============================================================================= +# PR Regression Testing VM Bootstrap +# +# Modes: +# REPO_MODE=fork -> use temporary AnkitaNaik forks (default for now) +# REPO_MODE=upstream -> use cuga-project upstream repositories +# +# Final layout: +# +# ~/pr-regression-testing/ +# ├── cuga-agent/ +# ├── cuga-eval/ +# │ └── scripts/ +# │ └── run-eval.sh +# └── github-runner/ +# └── _work/ +# +# Required VM prerequisites: +# git curl tar python3 git-lfs +# +# Required environment variables: +# GITHUB_REPO_URL +# GITHUB_RUNNER_TOKEN +# +# Optional: +# REPO_MODE=fork|upstream +# ROOT_DIR +# RUNNER_NAME +# RUNNER_LABELS +# CUGA_AGENT_BRANCH +# CUGA_EVAL_BRANCH +# RUNNER_VERSION +# ============================================================================= + +REPO_MODE="${REPO_MODE:-fork}" +ROOT_DIR="${ROOT_DIR:-$HOME/pr-regression-testing}" + +RUNNER_NAME="${RUNNER_NAME:-$(hostname)}" +RUNNER_LABELS="${RUNNER_LABELS:-run-eval}" + +CUGA_AGENT_BRANCH="${CUGA_AGENT_BRANCH:-main}" +CUGA_EVAL_BRANCH="${CUGA_EVAL_BRANCH:-main}" + +case "${REPO_MODE}" in + fork) + DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/AnkitaNaik/cuga-agent.git" + DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/AnkitaNaik/cuga-eval.git" + ;; + upstream) + DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/cuga-project/cuga-agent.git" + DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/cuga-project/cuga-eval.git" + ;; + *) + echo "ERROR: REPO_MODE must be 'fork' or 'upstream'." >&2 + exit 2 + ;; +esac + +CUGA_AGENT_REPO_URL="${CUGA_AGENT_REPO_URL:-$DEFAULT_CUGA_AGENT_REPO_URL}" +CUGA_EVAL_REPO_URL="${CUGA_EVAL_REPO_URL:-$DEFAULT_CUGA_EVAL_REPO_URL}" + +CUGA_AGENT_DIR="${ROOT_DIR}/cuga-agent" +CUGA_EVAL_DIR="${ROOT_DIR}/cuga-eval" +GH_RUNNER_DIR="${ROOT_DIR}/github-runner" + +RUN_EVAL_SCRIPT="${CUGA_EVAL_DIR}/scripts/run-pr-regression-eval.sh" +x +log() { + printf '\n[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || die "Required command '$1' is missing." +} + +clone_or_update() { + local repo_url="$1" + local branch="$2" + local dest="$3" + local name="$4" + + if [[ -d "${dest}/.git" ]]; then + log "Updating ${name}" + git -C "${dest}" fetch --all --prune + git -C "${dest}" checkout "${branch}" + git -C "${dest}" pull --ff-only origin "${branch}" + else + log "Cloning ${name}" + git clone --branch "${branch}" "${repo_url}" "${dest}" + fi +} + +: "${GITHUB_REPO_URL:?Set GITHUB_REPO_URL, e.g. https://github.com/OWNER/REPO}" +: "${GITHUB_RUNNER_TOKEN:?Set GITHUB_RUNNER_TOKEN to the temporary runner token}" + +log "Repository mode: ${REPO_MODE}" +echo "cuga-agent: ${CUGA_AGENT_REPO_URL}" +echo "cuga-eval: ${CUGA_EVAL_REPO_URL}" + +log "Checking VM prerequisites" +need_cmd git +need_cmd curl +need_cmd tar +need_cmd python3 +need_cmd git-lfs + +python3 - <<'PY' +import sys +if not ((3, 12) <= sys.version_info[:2] < (3, 14)): + raise SystemExit( + f"ERROR: Python 3.12 or 3.13 is required; found " + f"{sys.version_info.major}.{sys.version_info.minor}" + ) +PY + +if ! command -v uv >/dev/null 2>&1; then + log "Installing uv" + curl -LsSf https://astral.sh/uv/install.sh | sh + export PATH="${HOME}/.local/bin:${HOME}/.cargo/bin:${PATH}" +fi +need_cmd uv + +mkdir -p "${ROOT_DIR}" + +clone_or_update \ + "${CUGA_AGENT_REPO_URL}" \ + "${CUGA_AGENT_BRANCH}" \ + "${CUGA_AGENT_DIR}" \ + "cuga-agent" + +clone_or_update \ + "${CUGA_EVAL_REPO_URL}" \ + "${CUGA_EVAL_BRANCH}" \ + "${CUGA_EVAL_DIR}" \ + "cuga-eval" + +log "Running cuga-eval/setup_cuga.sh" +cd "${CUGA_EVAL_DIR}" +chmod u+x ./setup_cuga.sh +bash ./setup_cuga.sh + +log "Creating/syncing cuga-eval environment" +uv venv +uv sync + +log "Setting up AppWorld" +git lfs install +chmod u+x ./setup_appworld.sh +bash ./setup_appworld.sh +uv sync --group appworld + +log "Checking run-eval.sh" +if [[ ! -f "${RUN_EVAL_SCRIPT}" ]]; then + cat >&2 </dev/null 2>&1; then + log "Runner already running" +else + log "Starting runner in background" + + nohup ./run.sh > "${GH_RUNNER_DIR}/runner.log" 2>&1 & + echo $! > "${GH_RUNNER_DIR}/runner.pid" +fi + +sleep 3 + +if ! pgrep -f "${GH_RUNNER_DIR}/bin/Runner.Listener" >/dev/null 2>&1; then + echo "ERROR: Runner did not remain active." >&2 + tail -n 50 "${GH_RUNNER_DIR}/runner.log" 2>/dev/null || true + exit 1 +fi + +echo +echo "======================================================================" +echo "PR regression testing VM is ready" +echo "======================================================================" +echo +echo "Repository mode:" +echo " ${REPO_MODE}" +echo +echo "Layout:" +echo " ${ROOT_DIR}/" +echo " ├── cuga-agent/" +echo " ├── cuga-eval/" +echo " │ └── scripts/run-eval.sh" +echo " └── github-runner/" +echo " └── _work/" +echo +echo "Repos:" +echo " cuga-agent: ${CUGA_AGENT_REPO_URL}" +echo " cuga-eval: ${CUGA_EVAL_REPO_URL}" +echo +echo "Runner logs:" +echo " tail -f ${GH_RUNNER_DIR}/runner.log" +echo +echo "Workflow target:" +echo " runs-on: [self-hosted, linux, ${RUNNER_LABELS}]" +echo +echo "Workflow script:" +echo " bash \"${RUN_EVAL_SCRIPT}\" \"\${COMMENT_BODY}\"" From f66fad02651ef4eff65c4bfbcb9739402bda4824 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 19:13:25 -0400 Subject: [PATCH 05/40] feat(pr-eval): Update pr-regression setup script and add eval.sh script. --- scripts/setup-pr-regression-workflow.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/setup-pr-regression-workflow.sh b/scripts/setup-pr-regression-workflow.sh index 17ab9547..6b5ff34b 100644 --- a/scripts/setup-pr-regression-workflow.sh +++ b/scripts/setup-pr-regression-workflow.sh @@ -14,7 +14,8 @@ set -Eeuo pipefail # ├── cuga-agent/ # ├── cuga-eval/ # │ └── scripts/ -# │ └── run-eval.sh +# │ ├── run-pr-regression-eval.sh +# │ └── run-pr-eval.sh -> run-pr-regression-eval.sh # └── github-runner/ # └── _work/ # @@ -32,6 +33,7 @@ set -Eeuo pipefail # RUNNER_LABELS # CUGA_AGENT_BRANCH # CUGA_EVAL_BRANCH +# RUN_EVAL_SCRIPT # RUNNER_VERSION # ============================================================================= @@ -39,7 +41,7 @@ REPO_MODE="${REPO_MODE:-fork}" ROOT_DIR="${ROOT_DIR:-$HOME/pr-regression-testing}" RUNNER_NAME="${RUNNER_NAME:-$(hostname)}" -RUNNER_LABELS="${RUNNER_LABELS:-run-eval}" +RUNNER_LABELS="${RUNNER_LABELS:-run-pr-eval}" CUGA_AGENT_BRANCH="${CUGA_AGENT_BRANCH:-main}" CUGA_EVAL_BRANCH="${CUGA_EVAL_BRANCH:-main}" @@ -66,8 +68,9 @@ CUGA_AGENT_DIR="${ROOT_DIR}/cuga-agent" CUGA_EVAL_DIR="${ROOT_DIR}/cuga-eval" GH_RUNNER_DIR="${ROOT_DIR}/github-runner" -RUN_EVAL_SCRIPT="${CUGA_EVAL_DIR}/scripts/run-pr-regression-eval.sh" -x +RUN_EVAL_SCRIPT="${RUN_EVAL_SCRIPT:-${CUGA_EVAL_DIR}/scripts/run-pr-regression-eval.sh}" +WORKFLOW_EVAL_SCRIPT="${CUGA_EVAL_DIR}/scripts/run-pr-eval.sh" + log() { printf '\n[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" } @@ -157,7 +160,7 @@ chmod u+x ./setup_appworld.sh bash ./setup_appworld.sh uv sync --group appworld -log "Checking run-eval.sh" +log "Checking PR regression evaluation entrypoint" if [[ ! -f "${RUN_EVAL_SCRIPT}" ]]; then cat >&2 < Date: Mon, 17 Aug 2026 19:15:42 -0400 Subject: [PATCH 06/40] Add main pr-regression runner. --- scripts/run-pr-regression-eval.sh | 184 ++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 scripts/run-pr-regression-eval.sh diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh new file mode 100644 index 00000000..536f4f78 --- /dev/null +++ b/scripts/run-pr-regression-eval.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +START_EPOCH="$(date +%s)" +START_TIME="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" + +log() { + printf '[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" +} + +on_exit() { + status=$? + end_epoch="$(date +%s)" + end_time="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" + duration=$((end_epoch - START_EPOCH + + minutes=$((duration / 60)) + seconds=$((duration % 60)))) + + + echo "######## REPORT START ########" + echo + echo "## Run summary" + echo "- Started: ${START_TIME}" + echo "- Finished: ${end_time}" + echo "- Duration: ${minutes}m ${seconds}s" + echo "- Exit code: ${status}" + + if [[ ${status} -eq 0 ]]; then + echo "- Result: SUCCESS" + else + echo "- Result: FAILED" + fi + echo "######## REPORT END ########" +} +trap on_exit EXIT + +: "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" + +DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" +DEFAULT_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" +DEFAULT_SPLIT_NAME="default" +DEFAULT_NUM_TASKS="4" + +MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" +TASK_IDS="${TASK_IDS:-$DEFAULT_TASK_IDS}" +SPLIT_NAME="${SPLIT_NAME:-$DEFAULT_SPLIT_NAME}" +NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" + +# Parse whitespace-separated key=value parameters from the PR comment. +# Supported aliases: +# model_name=... +# task_id=id1,id2 +# task_ids=id1,id2 +# split_name=... +# num_tasks=... +COMMENT_BODY="${1:-}" + +# Remove Windows carriage returns and newlines. +COMMENT_BODY="${COMMENT_BODY//$'\r'/ }" +COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" + +read -r -a TOKENS <<< "${COMMENT_BODY}" +echo "######## REPORT START ########" +for token in "${TOKENS[@]}"; do + case "${token}" in + /run-eval|/appworld-eval) + ;; + + model_name=*) + MODEL_NAME="${token#model_name=}" + ;; + + task_id=*|task_ids=*) + TASK_IDS="${token#*=}" + TASK_IDS="${TASK_IDS//,/ }" + ;; + + split_name=*) + SPLIT_NAME="${token#split_name=}" + ;; + + num_tasks=*) + NUM_TASKS="${token#num_tasks=}" + ;; + + "") + ;; + + *) + echo "ERROR: Unsupported parameter: ${token}" + echo "Supported parameters: model_name, task_id, task_ids, split_name, num_tasks" + exit 2 + ;; + esac +done + +if [[ -z "${MODEL_NAME}" ]]; then + echo "ERROR: model_name cannot be empty." + exit 2 +fi + +if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then + echo "ERROR: num_tasks must be a positive integer." + exit 2 +fi + +read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" + +if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then + echo "ERROR: At least one task_id is required." + exit 2 +fi +echo "######## REPORT END ########" + +# num_tasks is currently a dummy input, but limit the supplied task list to make +# the mock behavior predictable. +if [[ ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then + TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") +fi + +export DYNACONF_SERVER_PORTS__REGISTRY="${DYNACONF_SERVER_PORTS__REGISTRY:-8100}" +export AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-settings.rits.toml}" +export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" +export MODEL_NAME +export ENVIRONMENT_URL="${ENVIRONMENT_URL:-http://127.0.0.1:8000}" +export APIS_URL="${APIS_URL:-http://127.0.0.1:9000}" + +EVAL_REPO="/proj/cuga_models/cuga-eval" + +if [[ ! -d "${EVAL_REPO}" ]]; then + echo "ERROR: Evaluation repo not found: ${EVAL_REPO}" + exit 1 +fi + +cd "${EVAL_REPO}" + +echo "######## REPORT START ########" +echo "# PR Evaluation" +echo +echo "## Run metadata" +echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" +echo "- PR: ${PR_NUMBER:-unknown}" +echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" +echo "- Runner: ${RUNNER_NAME:-$(hostname)}" +echo +echo "## Evaluation configuration" +echo "- Model: ${MODEL_NAME}" +echo "- Split: ${SPLIT_NAME}" +echo "- Requested num_tasks: ${NUM_TASKS}" +echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" +echo "- Task IDs:" +for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" +done +echo +echo "######## REPORT END ########" + +log "Verifying required commands" +command -v uv >/dev/null 2>&1 || { + echo "ERROR: uv is not installed or is not in PATH." + exit 127 +} + +command -v curl >/dev/null 2>&1 || { + echo "ERROR: curl is not installed or is not in PATH." + exit 127 +} + +log "Verifying local benchmark services" +for endpoint in "${ENVIRONMENT_URL}" "${APIS_URL}"; do + if ! curl --silent --show-error --fail --max-time 5 "${endpoint}" >/dev/null; then + echo "ERROR: Required service is unavailable at ${endpoint}" + exit 1 + fi +done + +log "Starting AppWorld React evaluation" + +uv run --no-sync python -m benchmarks.appworld.appworld_eval_react \ + --agent react \ + --task-id "${TASK_ID_ARRAY[@]}" \ + --environment-url "${ENVIRONMENT_URL}" \ + --apis-url "${APIS_URL}" From edfe8fe524a31f77f7764a702f5d92ece93b5193 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 20:00:23 -0400 Subject: [PATCH 07/40] Update hard-coded paths in regression runner for VM compatability. --- scripts/run-pr-regression-eval.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) mode change 100644 => 100755 scripts/run-pr-regression-eval.sh diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh old mode 100644 new mode 100755 index 536f4f78..71ad8f64 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -12,11 +12,9 @@ on_exit() { status=$? end_epoch="$(date +%s)" end_time="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" - duration=$((end_epoch - START_EPOCH - + duration=$((end_epoch - START_EPOCH)) minutes=$((duration / 60)) - seconds=$((duration % 60)))) - + seconds=$((duration % 60)) echo "######## REPORT START ########" echo @@ -62,9 +60,10 @@ COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" read -r -a TOKENS <<< "${COMMENT_BODY}" echo "######## REPORT START ########" +echo "## Requested parameters" for token in "${TOKENS[@]}"; do case "${token}" in - /run-eval|/appworld-eval) + /run-pr-eval) ;; model_name=*) @@ -90,6 +89,7 @@ for token in "${TOKENS[@]}"; do *) echo "ERROR: Unsupported parameter: ${token}" echo "Supported parameters: model_name, task_id, task_ids, split_name, num_tasks" + echo "######## REPORT END ########" exit 2 ;; esac @@ -97,11 +97,13 @@ done if [[ -z "${MODEL_NAME}" ]]; then echo "ERROR: model_name cannot be empty." + echo "######## REPORT END ########" exit 2 fi if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then echo "ERROR: num_tasks must be a positive integer." + echo "######## REPORT END ########" exit 2 fi @@ -109,8 +111,13 @@ read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then echo "ERROR: At least one task_id is required." + echo "######## REPORT END ########" exit 2 fi +echo "- Model: ${MODEL_NAME}" +echo "- Split: ${SPLIT_NAME}" +echo "- Num tasks: ${NUM_TASKS}" +echo "- Task IDs: ${TASK_IDS}" echo "######## REPORT END ########" # num_tasks is currently a dummy input, but limit the supplied task list to make @@ -126,7 +133,8 @@ export MODEL_NAME export ENVIRONMENT_URL="${ENVIRONMENT_URL:-http://127.0.0.1:8000}" export APIS_URL="${APIS_URL:-http://127.0.0.1:9000}" -EVAL_REPO="/proj/cuga_models/cuga-eval" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EVAL_REPO="${EVAL_REPO:-$(cd "${SCRIPT_DIR}/.." && pwd)}" if [[ ! -d "${EVAL_REPO}" ]]; then echo "ERROR: Evaluation repo not found: ${EVAL_REPO}" From dccbf29471546ae6ea06c8c17a4f27db4f791496 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 20:25:03 -0400 Subject: [PATCH 08/40] Add support for RITS in the react agent. --- benchmarks/appworld/appworld_eval_react.py | 7 ++-- benchmarks/helpers/react_agent.py | 43 +++++++++++++++++----- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/benchmarks/appworld/appworld_eval_react.py b/benchmarks/appworld/appworld_eval_react.py index fbbd5a5b..a8d269cc 100644 --- a/benchmarks/appworld/appworld_eval_react.py +++ b/benchmarks/appworld/appworld_eval_react.py @@ -33,6 +33,7 @@ from datetime import datetime from typing import Any, Optional +import httpx from appworld import AppWorld, load_task_ids # pyright: ignore[reportAttributeAccessIssue] from cuga.backend.activity_tracker.tracker import ActivityTracker, Step from cuga.backend.cuga_graph.state.agent_state import VariablesManager @@ -181,13 +182,13 @@ def _print_appworld_summary(report: dict): avg_steps = report.get("avg_steps", 0) avg_duration = report.get("avg_duration", 0) - print("\n" + "=" * 80) - print("EVALUATION COMPLETE") - print("=" * 80) + print("######## REPORT START ########") + print("## EVALUATION COMPLETE") print(f"Total Tasks: {total}") print(f"Completed: {completed}/{total} ({success_rate:.1%})") print(f"Avg Steps: {avg_steps:.2f}") print(f"Avg Duration: {avg_duration:.2f}s") + print("######## REPORT END ########") def _render_initial_prompt(world: AppWorld) -> str: diff --git a/benchmarks/helpers/react_agent.py b/benchmarks/helpers/react_agent.py index be1c941c..2b9f8526 100644 --- a/benchmarks/helpers/react_agent.py +++ b/benchmarks/helpers/react_agent.py @@ -19,6 +19,7 @@ from pathlib import Path from typing import Any, Optional +from langchain_openai import ChatOpenAI from cuga.backend.cuga_graph.nodes.cuga_lite.providers.combined import ( CombinedToolProvider, ) @@ -44,6 +45,30 @@ class ReactInvokeResult: react_steps: int = 0 +class ChatRits(ChatOpenAI): + """RITS chat model integration using langchain-openai.""" + + def __init__(self, config: dict[str, Any]): + model_name = config.get("model_name", "openai/gpt-oss-120b-a100") + end_point = config.get( + "end_point", + "https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100", + ) + rits_api_key = config.get("api_key") or os.getenv("RITS_API_KEY") + if rits_api_key is None: + raise ValueError("RITS_API_KEY is required for settings.rits.toml") + + params = config.get("params", {}) + rits_config: dict[str, Any] = { + "model_name": model_name, + "api_key": "/", + "default_headers": {"RITS_API_KEY": rits_api_key}, + "base_url": end_point.rstrip("/") + "/v1", + } + rits_config.update(params) + super().__init__(**rits_config) + + class GenericReactAgent: """Prompt-template driven generic ReAct agent over benchmark tools.""" @@ -149,14 +174,6 @@ def _create_llm(self) -> Any: ) elif settings_config == "settings.openai.toml": - try: - from langchain_openai import ChatOpenAI - except ImportError as exc: - raise RuntimeError( - "langchain-openai is required for React agent with OpenAI. " - "Install with: pip install langchain-openai" - ) from exc - disable_ssl = os.getenv("CUGA_DISABLE_SSL", "").lower() in ("true", "1", "yes") ssl_verify = ( os.getenv("OPENAI_SSL_VERIFY", "true").lower() not in ("false", "0", "no") and not disable_ssl @@ -183,10 +200,18 @@ def _create_llm(self) -> Any: return ChatOpenAI(**llm_kwargs) + elif settings_config in ("settings.rits.toml", "settings.rits.proxy.toml"): + api_base = os.getenv("LITE_LLM_URL") or os.getenv("OPENAI_BASE_URL") or os.getenv("RITS_BASE_URL") + api_key = os.getenv("LITE_LLM_KEY") or os.getenv("OPENAI_API_KEY") or os.getenv("RITS_API_KEY") + model_name = os.getenv("MODEL_NAME") or self.model or "openai/gpt-oss-120b-a100" + logger.info(f"RITS config: model={model_name}, api_base={api_base}, api_key_present={bool(api_key)}") + return ChatRits(config={"model_name": model_name, "end_point": api_base, "api_key": api_key}) + else: raise RuntimeError( "Unsupported AGENT_SETTING_CONFIG for React agent. " - "Expected 'settings.groq.toml' or 'settings.openai.toml'." + "Expected 'settings.groq.toml', 'settings.openai.toml', " + "'settings.rits.toml', or 'settings.rits.proxy.toml'." ) def _get_llm(self) -> Any: From 85faa3d8222d5ff58366418683816d4ff650ae9e Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 20:36:11 -0400 Subject: [PATCH 09/40] Remove hard-coded port for AppWorld. --- scripts/run-pr-regression-eval.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 71ad8f64..e0935fc5 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -126,7 +126,6 @@ if [[ ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") fi -export DYNACONF_SERVER_PORTS__REGISTRY="${DYNACONF_SERVER_PORTS__REGISTRY:-8100}" export AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-settings.rits.toml}" export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" export MODEL_NAME From 70128b469bb980c56cda25e424c1b4ba4096b92c Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 20:53:25 -0400 Subject: [PATCH 10/40] Remove hard-coded ports. --- scripts/run-pr-regression-eval.sh | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index e0935fc5..8d6b1071 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -129,8 +129,6 @@ fi export AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-settings.rits.toml}" export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" export MODEL_NAME -export ENVIRONMENT_URL="${ENVIRONMENT_URL:-http://127.0.0.1:8000}" -export APIS_URL="${APIS_URL:-http://127.0.0.1:9000}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" EVAL_REPO="${EVAL_REPO:-$(cd "${SCRIPT_DIR}/.." && pwd)}" @@ -174,18 +172,11 @@ command -v curl >/dev/null 2>&1 || { exit 127 } -log "Verifying local benchmark services" -for endpoint in "${ENVIRONMENT_URL}" "${APIS_URL}"; do - if ! curl --silent --show-error --fail --max-time 5 "${endpoint}" >/dev/null; then - echo "ERROR: Required service is unavailable at ${endpoint}" - exit 1 - fi -done - log "Starting AppWorld React evaluation" -uv run --no-sync python -m benchmarks.appworld.appworld_eval_react \ - --agent react \ - --task-id "${TASK_ID_ARRAY[@]}" \ - --environment-url "${ENVIRONMENT_URL}" \ - --apis-url "${APIS_URL}" +EVAL_ARGS=( + --agent react + --task "${TASK_ID_ARRAY[@]}" +) + +bash benchmarks/appworld/eval.sh "${EVAL_ARGS[@]}" From 74e56e8a685ae7f52100c61576bd189d145162ac Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:01:24 -0400 Subject: [PATCH 11/40] Default using Langfuse to false.: --- scripts/run-pr-regression-eval.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 8d6b1071..8aa2e425 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -127,6 +127,7 @@ if [[ ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then fi export AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-settings.rits.toml}" +export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" export MODEL_NAME From 5bd72ff3f9cf77cbd787e9ca56bfbfec755a0eaa Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:12:49 -0400 Subject: [PATCH 12/40] Add support for all the agents for AppWorld. --- scripts/run-pr-regression-eval.sh | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 8aa2e425..37d28419 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -39,11 +39,13 @@ DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" DEFAULT_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" DEFAULT_SPLIT_NAME="default" DEFAULT_NUM_TASKS="4" +DEFAULT_AGENT="react" MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" TASK_IDS="${TASK_IDS:-$DEFAULT_TASK_IDS}" SPLIT_NAME="${SPLIT_NAME:-$DEFAULT_SPLIT_NAME}" NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" +AGENT="${AGENT:-$DEFAULT_AGENT}" # Parse whitespace-separated key=value parameters from the PR comment. # Supported aliases: @@ -52,6 +54,7 @@ NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" # task_ids=id1,id2 # split_name=... # num_tasks=... +# agent=react|cuga|codeact COMMENT_BODY="${1:-}" # Remove Windows carriage returns and newlines. @@ -83,12 +86,16 @@ for token in "${TOKENS[@]}"; do NUM_TASKS="${token#num_tasks=}" ;; + agent=*|agent_type=*) + AGENT="${token#*=}" + ;; + "") ;; *) echo "ERROR: Unsupported parameter: ${token}" - echo "Supported parameters: model_name, task_id, task_ids, split_name, num_tasks" + echo "Supported parameters: model_name, task_id, task_ids, split_name, num_tasks, agent" echo "######## REPORT END ########" exit 2 ;; @@ -107,6 +114,16 @@ if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then exit 2 fi +case "${AGENT}" in + react|cuga|codeact) + ;; + *) + echo "ERROR: agent must be one of: react, cuga, codeact." + echo "######## REPORT END ########" + exit 2 + ;; +esac + read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then @@ -115,6 +132,7 @@ if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then exit 2 fi echo "- Model: ${MODEL_NAME}" +echo "- Agent: ${AGENT}" echo "- Split: ${SPLIT_NAME}" echo "- Num tasks: ${NUM_TASKS}" echo "- Task IDs: ${TASK_IDS}" @@ -151,6 +169,7 @@ echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown) echo "- Runner: ${RUNNER_NAME:-$(hostname)}" echo echo "## Evaluation configuration" +echo "- Agent: ${AGENT}" echo "- Model: ${MODEL_NAME}" echo "- Split: ${SPLIT_NAME}" echo "- Requested num_tasks: ${NUM_TASKS}" @@ -173,10 +192,10 @@ command -v curl >/dev/null 2>&1 || { exit 127 } -log "Starting AppWorld React evaluation" +log "Starting AppWorld ${AGENT} evaluation" EVAL_ARGS=( - --agent react + --agent "${AGENT}" --task "${TASK_ID_ARRAY[@]}" ) From 70ac2c379d8e9489caf7ed69889c018e81a31ebb Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:27:39 -0400 Subject: [PATCH 13/40] Log the complete report in the PR comment. --- benchmarks/helpers/compare_report.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/helpers/compare_report.py b/benchmarks/helpers/compare_report.py index 58382cfd..6d19b50d 100644 --- a/benchmarks/helpers/compare_report.py +++ b/benchmarks/helpers/compare_report.py @@ -1536,7 +1536,8 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: fence_open = (lambda: "```text") if markdown else (lambda: "") fence_close = (lambda: "```") if markdown else (lambda: "") - lines = [h1("Evaluation Report"), ""] + lines = ["######## REPORT START ########"] + lines.extend([h1("Evaluation Report"), ""]) lines.append(h2("Summary")) lines.append("") if markdown: @@ -1825,6 +1826,7 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: markdown=markdown, ) ) + lines.extend("######## REPORT END ########") return "\n".join(lines) From 16424a211a19040b21465f63f0c8d174d38d6c26 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:28:01 -0400 Subject: [PATCH 14/40] Log the agent name. --- scripts/run-pr-regression-eval.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 37d28419..63b516ba 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -167,6 +167,7 @@ echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" echo "- PR: ${PR_NUMBER:-unknown}" echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" echo "- Runner: ${RUNNER_NAME:-$(hostname)}" +echo "- Agent: ${AGENT}" echo echo "## Evaluation configuration" echo "- Agent: ${AGENT}" From fcdc51cb5b812e6b670329d8ec494c4be5c3d955 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:31:15 -0400 Subject: [PATCH 15/40] fix(appworld): skip Langfuse fetch when tracing is disabled --- benchmarks/appworld/appworld_eval.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmarks/appworld/appworld_eval.py b/benchmarks/appworld/appworld_eval.py index 8c45a10f..7d34d6bd 100644 --- a/benchmarks/appworld/appworld_eval.py +++ b/benchmarks/appworld/appworld_eval.py @@ -335,8 +335,9 @@ async def run_agent_on_task( langfuse_data = None # Extract Langfuse data if trace_id is available - langfuse_handler = LangfuseTraceHandler(langfuse_trace_id) - langfuse_data = await langfuse_handler.get_langfuse_data() + if langfuse_trace_id: + langfuse_handler = LangfuseTraceHandler(langfuse_trace_id) + langfuse_data = await langfuse_handler.get_langfuse_data() if langfuse_trace_id: task_result.trace_id = langfuse_trace_id if langfuse_data: From f865b7e5999d56b0950c4bdfc39f398e7dd3581a Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:38:17 -0400 Subject: [PATCH 16/40] fix(pr-eval): improve collapsible PR report formatting --- benchmarks/helpers/compare_report.py | 13 ++++++------ scripts/run-pr-regression-eval.sh | 30 +++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/benchmarks/helpers/compare_report.py b/benchmarks/helpers/compare_report.py index 6d19b50d..a115ee1f 100644 --- a/benchmarks/helpers/compare_report.py +++ b/benchmarks/helpers/compare_report.py @@ -1826,7 +1826,7 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: markdown=markdown, ) ) - lines.extend("######## REPORT END ########") + lines.append("######## REPORT END ########") return "\n".join(lines) @@ -1864,14 +1864,15 @@ def main(): # the canonical bundle location at the end of the run, which is what the # user actually wants to navigate to. if args.output: - # Compare-mode and eval-mode both produce markdown for the saved file - # and re-render plain text for the terminal. + # Compare-mode and eval-mode both produce markdown for the saved file. + # For eval-mode, also print markdown to stdout because PR workflows + # capture stdout and post marked report blocks back to GitHub comments. if "command" in args and getattr(args, "command", None) == "eval": - plain = generate_eval_report(args.result_file, markdown=False) + stdout_report = report else: - plain = generate_report(config_results, markdown=False) + stdout_report = generate_report(config_results, markdown=False) Path(args.output).write_text(report) - print(plain) + print(stdout_report) else: # No file requested — just print whatever generate_* produced. print(report) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 63b516ba..57c150bc 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -8,6 +8,18 @@ log() { printf '[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" } +open_details() { + local title="$1" + echo "
" + echo "${title}" + echo +} + +close_details() { + echo + echo "
" +} + on_exit() { status=$? end_epoch="$(date +%s)" @@ -18,7 +30,7 @@ on_exit() { echo "######## REPORT START ########" echo - echo "## Run summary" + open_details "Run summary" echo "- Started: ${START_TIME}" echo "- Finished: ${end_time}" echo "- Duration: ${minutes}m ${seconds}s" @@ -29,6 +41,7 @@ on_exit() { else echo "- Result: FAILED" fi + close_details echo "######## REPORT END ########" } trap on_exit EXIT @@ -63,7 +76,7 @@ COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" read -r -a TOKENS <<< "${COMMENT_BODY}" echo "######## REPORT START ########" -echo "## Requested parameters" +open_details "Requested parameters" for token in "${TOKENS[@]}"; do case "${token}" in /run-pr-eval) @@ -96,6 +109,7 @@ for token in "${TOKENS[@]}"; do *) echo "ERROR: Unsupported parameter: ${token}" echo "Supported parameters: model_name, task_id, task_ids, split_name, num_tasks, agent" + close_details echo "######## REPORT END ########" exit 2 ;; @@ -104,12 +118,14 @@ done if [[ -z "${MODEL_NAME}" ]]; then echo "ERROR: model_name cannot be empty." + close_details echo "######## REPORT END ########" exit 2 fi if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then echo "ERROR: num_tasks must be a positive integer." + close_details echo "######## REPORT END ########" exit 2 fi @@ -119,6 +135,7 @@ case "${AGENT}" in ;; *) echo "ERROR: agent must be one of: react, cuga, codeact." + close_details echo "######## REPORT END ########" exit 2 ;; @@ -128,6 +145,7 @@ read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then echo "ERROR: At least one task_id is required." + close_details echo "######## REPORT END ########" exit 2 fi @@ -136,6 +154,7 @@ echo "- Agent: ${AGENT}" echo "- Split: ${SPLIT_NAME}" echo "- Num tasks: ${NUM_TASKS}" echo "- Task IDs: ${TASK_IDS}" +close_details echo "######## REPORT END ########" # num_tasks is currently a dummy input, but limit the supplied task list to make @@ -162,14 +181,15 @@ cd "${EVAL_REPO}" echo "######## REPORT START ########" echo "# PR Evaluation" echo -echo "## Run metadata" +open_details "Run metadata" echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" echo "- PR: ${PR_NUMBER:-unknown}" echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" echo "- Runner: ${RUNNER_NAME:-$(hostname)}" echo "- Agent: ${AGENT}" +close_details echo -echo "## Evaluation configuration" +open_details "Evaluation configuration" echo "- Agent: ${AGENT}" echo "- Model: ${MODEL_NAME}" echo "- Split: ${SPLIT_NAME}" @@ -179,7 +199,7 @@ echo "- Task IDs:" for task_id in "${TASK_ID_ARRAY[@]}"; do echo " - ${task_id}" done -echo +close_details echo "######## REPORT END ########" log "Verifying required commands" From b0e68375a952cbc222d80f0a2be9b7675550ebc3 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:45:00 -0400 Subject: [PATCH 17/40] Format the summary report. --- benchmarks/helpers/compare_report.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/benchmarks/helpers/compare_report.py b/benchmarks/helpers/compare_report.py index a115ee1f..4de3c44b 100644 --- a/benchmarks/helpers/compare_report.py +++ b/benchmarks/helpers/compare_report.py @@ -1537,9 +1537,16 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: fence_close = (lambda: "```") if markdown else (lambda: "") lines = ["######## REPORT START ########"] - lines.extend([h1("Evaluation Report"), ""]) - lines.append(h2("Summary")) - lines.append("") + if markdown: + lines.append("
") + lines.append("Evaluation Report") + lines.append("") + lines.append(h2("Summary")) + lines.append("") + else: + lines.extend([h1("Evaluation Report"), ""]) + lines.append(h2("Summary")) + lines.append("") if markdown: lines.append(f"- **Pass@1**: {parsed['passed']}/{parsed['total']} ({parsed['rate']:.1%})") lines.append(f"- **Total Tokens**: {_fmt(parsed['tokens'])}") @@ -1826,6 +1833,9 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: markdown=markdown, ) ) + if markdown: + lines.append("") + lines.append("
") lines.append("######## REPORT END ########") return "\n".join(lines) From 938c85daa594bf6ea5af47f0372c2087bbfbd807 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:52:38 -0400 Subject: [PATCH 18/40] fix(pr-eval): support RITS credentials for CUGA agent runs --- scripts/run-pr-regression-eval.sh | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 57c150bc..57cade61 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -46,19 +46,19 @@ on_exit() { } trap on_exit EXIT -: "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" - DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" DEFAULT_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" DEFAULT_SPLIT_NAME="default" DEFAULT_NUM_TASKS="4" DEFAULT_AGENT="react" +DEFAULT_AGENT_SETTING_CONFIG="settings.rits.toml" MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" TASK_IDS="${TASK_IDS:-$DEFAULT_TASK_IDS}" SPLIT_NAME="${SPLIT_NAME:-$DEFAULT_SPLIT_NAME}" NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" AGENT="${AGENT:-$DEFAULT_AGENT}" +AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-$DEFAULT_AGENT_SETTING_CONFIG}" # Parse whitespace-separated key=value parameters from the PR comment. # Supported aliases: @@ -141,6 +141,25 @@ case "${AGENT}" in ;; esac +case "${AGENT_SETTING_CONFIG}" in + settings.rits.toml|settings.rits.proxy.toml) + if [[ -z "${RITS_API_KEY:-}" ]]; then + echo "ERROR: RITS_API_KEY is required when AGENT_SETTING_CONFIG=${AGENT_SETTING_CONFIG}." + close_details + echo "######## REPORT END ########" + exit 2 + fi + ;; + *) + if [[ -z "${OPENAI_API_KEY:-}" ]]; then + echo "ERROR: OPENAI_API_KEY is required when AGENT_SETTING_CONFIG=${AGENT_SETTING_CONFIG}." + close_details + echo "######## REPORT END ########" + exit 2 + fi + ;; +esac + read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then @@ -163,7 +182,7 @@ if [[ ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") fi -export AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-settings.rits.toml}" +export AGENT_SETTING_CONFIG export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" export MODEL_NAME From 1f4c5ec18d7f388a8fc82bd00b490fa03bf23e5b Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 22:12:17 -0400 Subject: [PATCH 19/40] Fix logging. --- scripts/run-pr-regression-eval.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 57cade61..0b5dc93f 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -185,6 +185,11 @@ fi export AGENT_SETTING_CONFIG export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" +if [[ -z "${RITS_BASE_URL:-}" ]]; then + RITS_BASE_URL="${OPENAI_BASE_URL%/}" + [[ "${RITS_BASE_URL}" == */v1 ]] || RITS_BASE_URL="${RITS_BASE_URL}/v1" + export RITS_BASE_URL +fi export MODEL_NAME SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" From 72b7da6d4d87e5d4c737b1a924065e832ad0ce55 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 23:13:09 -0400 Subject: [PATCH 20/40] feat:Support benchmark selection in PR eval --- scripts/run-pr-regression-eval.sh | 92 +++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 24 deletions(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 0b5dc93f..4ba8558b 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -47,15 +47,15 @@ on_exit() { trap on_exit EXIT DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" -DEFAULT_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" -DEFAULT_SPLIT_NAME="default" +DEFAULT_APPWORLD_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" +DEFAULT_BENCHMARK="appworld" DEFAULT_NUM_TASKS="4" DEFAULT_AGENT="react" DEFAULT_AGENT_SETTING_CONFIG="settings.rits.toml" MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" -TASK_IDS="${TASK_IDS:-$DEFAULT_TASK_IDS}" -SPLIT_NAME="${SPLIT_NAME:-$DEFAULT_SPLIT_NAME}" +TASK_IDS="${TASK_IDS:-}" +BENCHMARK="${BENCHMARK:-$DEFAULT_BENCHMARK}" NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" AGENT="${AGENT:-$DEFAULT_AGENT}" AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-$DEFAULT_AGENT_SETTING_CONFIG}" @@ -65,7 +65,7 @@ AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-$DEFAULT_AGENT_SETTING_CONFIG}" # model_name=... # task_id=id1,id2 # task_ids=id1,id2 -# split_name=... +# benchmark=appworld|m3 # num_tasks=... # agent=react|cuga|codeact COMMENT_BODY="${1:-}" @@ -91,8 +91,8 @@ for token in "${TOKENS[@]}"; do TASK_IDS="${TASK_IDS//,/ }" ;; - split_name=*) - SPLIT_NAME="${token#split_name=}" + benchmark=*) + BENCHMARK="${token#benchmark=}" ;; num_tasks=*) @@ -108,7 +108,7 @@ for token in "${TOKENS[@]}"; do *) echo "ERROR: Unsupported parameter: ${token}" - echo "Supported parameters: model_name, task_id, task_ids, split_name, num_tasks, agent" + echo "Supported parameters: model_name, task_id, task_ids, benchmark, num_tasks, agent" close_details echo "######## REPORT END ########" exit 2 @@ -130,6 +130,9 @@ if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then exit 2 fi +AGENT="$(printf '%s' "${AGENT}" | tr '[:upper:]' '[:lower:]')" +BENCHMARK="$(printf '%s' "${BENCHMARK}" | tr '[:upper:]' '[:lower:]')" + case "${AGENT}" in react|cuga|codeact) ;; @@ -141,6 +144,17 @@ case "${AGENT}" in ;; esac +case "${BENCHMARK}" in + appworld|m3) + ;; + *) + echo "ERROR: benchmark must be one of: appworld, m3." + close_details + echo "######## REPORT END ########" + exit 2 + ;; +esac + case "${AGENT_SETTING_CONFIG}" in settings.rits.toml|settings.rits.proxy.toml) if [[ -z "${RITS_API_KEY:-}" ]]; then @@ -160,25 +174,33 @@ case "${AGENT_SETTING_CONFIG}" in ;; esac +if [[ -z "${TASK_IDS}" && "${BENCHMARK}" == "appworld" ]]; then + TASK_IDS="${DEFAULT_APPWORLD_TASK_IDS}" +fi + read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" -if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then - echo "ERROR: At least one task_id is required." +if [[ ${#TASK_ID_ARRAY[@]} -eq 0 && "${BENCHMARK}" == "appworld" ]]; then + echo "ERROR: At least one task_id is required for benchmark=appworld." close_details echo "######## REPORT END ########" exit 2 fi echo "- Model: ${MODEL_NAME}" echo "- Agent: ${AGENT}" -echo "- Split: ${SPLIT_NAME}" +echo "- Benchmark: ${BENCHMARK}" echo "- Num tasks: ${NUM_TASKS}" -echo "- Task IDs: ${TASK_IDS}" +if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + echo "- Task IDs: ${TASK_IDS}" +else + echo "- Task IDs: benchmark default" +fi close_details echo "######## REPORT END ########" -# num_tasks is currently a dummy input, but limit the supplied task list to make -# the mock behavior predictable. -if [[ ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then +# For AppWorld, num_tasks limits the explicit task list. M3 maps num_tasks to +# --max-samples-per-domain below. +if [[ "${BENCHMARK}" == "appworld" && ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") fi @@ -211,18 +233,31 @@ echo "- PR: ${PR_NUMBER:-unknown}" echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" echo "- Runner: ${RUNNER_NAME:-$(hostname)}" echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" close_details echo open_details "Evaluation configuration" echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" echo "- Model: ${MODEL_NAME}" -echo "- Split: ${SPLIT_NAME}" echo "- Requested num_tasks: ${NUM_TASKS}" -echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" -echo "- Task IDs:" -for task_id in "${TASK_ID_ARRAY[@]}"; do - echo " - ${task_id}" -done +if [[ "${BENCHMARK}" == "appworld" ]]; then + echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" + echo "- Task IDs:" + for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" + done +else + echo "- M3 max samples per domain: ${NUM_TASKS}" + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + echo "- M3 task filters:" + for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" + done + else + echo "- M3 task filters: benchmark default" + fi +fi close_details echo "######## REPORT END ########" @@ -237,11 +272,20 @@ command -v curl >/dev/null 2>&1 || { exit 127 } -log "Starting AppWorld ${AGENT} evaluation" +log "Starting ${BENCHMARK} ${AGENT} evaluation" EVAL_ARGS=( + --benchmark "${BENCHMARK}" --agent "${AGENT}" - --task "${TASK_ID_ARRAY[@]}" ) -bash benchmarks/appworld/eval.sh "${EVAL_ARGS[@]}" +if [[ "${BENCHMARK}" == "appworld" ]]; then + EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") +else + EVAL_ARGS+=(--m3-data --max-samples-per-domain "${NUM_TASKS}") + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") + fi +fi + +bash scripts/eval.sh "${EVAL_ARGS[@]}" From 283565caa7b3ee5b71673475d78777aa11cb525c Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 07:18:49 -0400 Subject: [PATCH 21/40] Update evaluation report. --- scripts/run-pr-regression-eval.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 4ba8558b..b149c9ce 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -225,15 +225,13 @@ fi cd "${EVAL_REPO}" echo "######## REPORT START ########" -echo "# PR Evaluation" +echo "## Evaluation Metrics" echo open_details "Run metadata" echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" echo "- PR: ${PR_NUMBER:-unknown}" echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" echo "- Runner: ${RUNNER_NAME:-$(hostname)}" -echo "- Agent: ${AGENT}" -echo "- Benchmark: ${BENCHMARK}" close_details echo open_details "Evaluation configuration" From 551f2306b8fd18dc03c5811ec251c45f2e9e857d Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 07:30:44 -0400 Subject: [PATCH 22/40] Add litellm support. --- scripts/run-pr-regression-eval.sh | 66 ++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index b149c9ce..de117e5d 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -51,14 +51,14 @@ DEFAULT_APPWORLD_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" DEFAULT_BENCHMARK="appworld" DEFAULT_NUM_TASKS="4" DEFAULT_AGENT="react" -DEFAULT_AGENT_SETTING_CONFIG="settings.rits.toml" +DEFAULT_PROVIDER="rits" MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" TASK_IDS="${TASK_IDS:-}" BENCHMARK="${BENCHMARK:-$DEFAULT_BENCHMARK}" NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" AGENT="${AGENT:-$DEFAULT_AGENT}" -AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-$DEFAULT_AGENT_SETTING_CONFIG}" +PROVIDER="${PROVIDER:-$DEFAULT_PROVIDER}" # Parse whitespace-separated key=value parameters from the PR comment. # Supported aliases: @@ -68,6 +68,7 @@ AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-$DEFAULT_AGENT_SETTING_CONFIG}" # benchmark=appworld|m3 # num_tasks=... # agent=react|cuga|codeact +# provider=rits|litellm COMMENT_BODY="${1:-}" # Remove Windows carriage returns and newlines. @@ -103,12 +104,16 @@ for token in "${TOKENS[@]}"; do AGENT="${token#*=}" ;; + provider=*) + PROVIDER="${token#provider=}" + ;; + "") ;; *) echo "ERROR: Unsupported parameter: ${token}" - echo "Supported parameters: model_name, task_id, task_ids, benchmark, num_tasks, agent" + echo "Supported parameters: model_name, task_id, task_ids, benchmark, num_tasks, agent, provider" close_details echo "######## REPORT END ########" exit 2 @@ -132,6 +137,7 @@ fi AGENT="$(printf '%s' "${AGENT}" | tr '[:upper:]' '[:lower:]')" BENCHMARK="$(printf '%s' "${BENCHMARK}" | tr '[:upper:]' '[:lower:]')" +PROVIDER="$(printf '%s' "${PROVIDER}" | tr '[:upper:]' '[:lower:]')" case "${AGENT}" in react|cuga|codeact) @@ -155,25 +161,57 @@ case "${BENCHMARK}" in ;; esac -case "${AGENT_SETTING_CONFIG}" in - settings.rits.toml|settings.rits.proxy.toml) +case "${PROVIDER}" in + rits) + AGENT_SETTING_CONFIG="settings.rits.toml" if [[ -z "${RITS_API_KEY:-}" ]]; then - echo "ERROR: RITS_API_KEY is required when AGENT_SETTING_CONFIG=${AGENT_SETTING_CONFIG}." + echo "ERROR: RITS_API_KEY is required when provider=rits." close_details echo "######## REPORT END ########" exit 2 fi ;; - *) + litellm) + if [[ "${AGENT}" == "cuga" ]]; then + AGENT_SETTING_CONFIG="settings.litellm.toml" + else + AGENT_SETTING_CONFIG="settings.openai.toml" + fi + OPENAI_BASE_URL="${LITE_LLM_URL:-${OPENAI_BASE_URL:-}}" + OPENAI_API_KEY="${LITE_LLM_KEY:-${OPENAI_API_KEY:-}}" if [[ -z "${OPENAI_API_KEY:-}" ]]; then - echo "ERROR: OPENAI_API_KEY is required when AGENT_SETTING_CONFIG=${AGENT_SETTING_CONFIG}." + echo "ERROR: LITE_LLM_KEY or OPENAI_API_KEY is required when provider=litellm." + close_details + echo "######## REPORT END ########" + exit 2 + fi + if [[ -z "${OPENAI_BASE_URL:-}" ]]; then + echo "ERROR: LITE_LLM_URL or OPENAI_BASE_URL is required when provider=litellm." close_details echo "######## REPORT END ########" exit 2 fi + export OPENAI_API_KEY OPENAI_BASE_URL + ;; + *) + echo "ERROR: provider must be one of: rits, litellm." + close_details + echo "######## REPORT END ########" + exit 2 ;; esac +if [[ "${PROVIDER}" == "rits" ]]; then + export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" + if [[ -z "${RITS_BASE_URL:-}" ]]; then + RITS_BASE_URL="${OPENAI_BASE_URL%/}" + [[ "${RITS_BASE_URL}" == */v1 ]] || RITS_BASE_URL="${RITS_BASE_URL}/v1" + export RITS_BASE_URL + fi +else + : +fi + if [[ -z "${TASK_IDS}" && "${BENCHMARK}" == "appworld" ]]; then TASK_IDS="${DEFAULT_APPWORLD_TASK_IDS}" fi @@ -189,6 +227,7 @@ fi echo "- Model: ${MODEL_NAME}" echo "- Agent: ${AGENT}" echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" echo "- Num tasks: ${NUM_TASKS}" if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then echo "- Task IDs: ${TASK_IDS}" @@ -206,12 +245,6 @@ fi export AGENT_SETTING_CONFIG export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" -export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" -if [[ -z "${RITS_BASE_URL:-}" ]]; then - RITS_BASE_URL="${OPENAI_BASE_URL%/}" - [[ "${RITS_BASE_URL}" == */v1 ]] || RITS_BASE_URL="${RITS_BASE_URL}/v1" - export RITS_BASE_URL -fi export MODEL_NAME SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -232,11 +265,16 @@ echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" echo "- PR: ${PR_NUMBER:-unknown}" echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" echo "- Runner: ${RUNNER_NAME:-$(hostname)}" +echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" close_details echo open_details "Evaluation configuration" echo "- Agent: ${AGENT}" echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" +echo "- Agent setting config: ${AGENT_SETTING_CONFIG}" echo "- Model: ${MODEL_NAME}" echo "- Requested num_tasks: ${NUM_TASKS}" if [[ "${BENCHMARK}" == "appworld" ]]; then From 7b9d2cf694fcb304201348b0a6964db05e1b8ea7 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 07:50:26 -0400 Subject: [PATCH 23/40] Add litellm URL. --- scripts/run-pr-regression-eval.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index de117e5d..13caceb7 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -177,7 +177,7 @@ case "${PROVIDER}" in else AGENT_SETTING_CONFIG="settings.openai.toml" fi - OPENAI_BASE_URL="${LITE_LLM_URL:-${OPENAI_BASE_URL:-}}" + OPENAI_BASE_URL="${LITE_LLM_URL:-${OPENAI_BASE_URL:-https://ete-litellm.ai-models.vpc-int.res.ibm.com/}}" OPENAI_API_KEY="${LITE_LLM_KEY:-${OPENAI_API_KEY:-}}" if [[ -z "${OPENAI_API_KEY:-}" ]]; then echo "ERROR: LITE_LLM_KEY or OPENAI_API_KEY is required when provider=litellm." From 040cf919406b27d705f8e38a2a78d7fc0c2edfa4 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 08:15:56 -0400 Subject: [PATCH 24/40] Reorganize files and add README. --- scripts/pr-regression-testing/README.md | 199 +++++++++++ .../run-pr-regression-eval.sh | 333 ++++++++++++++++++ .../setup-pr-regression-workflow.sh | 308 ++++++++++++++++ 3 files changed, 840 insertions(+) create mode 100644 scripts/pr-regression-testing/README.md create mode 100755 scripts/pr-regression-testing/run-pr-regression-eval.sh create mode 100644 scripts/pr-regression-testing/setup-pr-regression-workflow.sh diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md new file mode 100644 index 00000000..c3e5daed --- /dev/null +++ b/scripts/pr-regression-testing/README.md @@ -0,0 +1,199 @@ +# PR Regression Testing + +This folder contains the scripts used to set up and run PR-triggered regression evaluations from GitHub Actions comments. + +## Files + +- `setup-pr-regression-workflow.sh`: bootstraps the self-hosted GitHub Actions runner VM. +- `run-pr-regression-eval.sh`: parses `/run-pr-eval` PR comments and runs the requested benchmark. + +## Setup The PR Regression Flow + +1. Add the workflow file to the `cuga-agent` repository: + + ```text + .github/workflows/run-pr-eval.yml + ``` + +2. Configure the workflow to use the self-hosted runner label: + + ```yaml + runs-on: [self-hosted, linux, run-pr-eval] + ``` + +3. In GitHub, create a self-hosted runner token for the `cuga-agent` repository. + + Go to: + + ```text + cuga-agent repository -> Settings -> Actions -> Runners -> New self-hosted runner + ``` + + Copy the repository URL and temporary runner token from the generated `config.sh` command. + +4. On the runner VM, export the required setup variables: + + ```bash + export GITHUB_REPO_URL="https://github.com/OWNER/cuga-agent" + export GITHUB_RUNNER_TOKEN="YOUR_TEMPORARY_RUNNER_TOKEN" + ``` + +5. Optionally choose repo mode and branches: + + ```bash + export REPO_MODE="fork" + export CUGA_AGENT_BRANCH="main" + export CUGA_EVAL_BRANCH="main" + ``` + + Supported `REPO_MODE` values: + + - `fork`: uses `https://github.com/AnkitaNaik/cuga-agent.git` and `https://github.com/AnkitaNaik/cuga-eval.git`. + - `upstream`: uses `https://github.com/cuga-project/cuga-agent.git` and `https://github.com/cuga-project/cuga-eval.git`. + +6. Run the setup script on the VM as the normal runner user, not with `sudo`: + + ```bash + bash /path/to/cuga-eval/scripts/pr-regression-testing/setup-pr-regression-workflow.sh + ``` + + The script will: + + - verify required commands: `git`, `curl`, `tar`, `python3`, `git-lfs`, and `uv`; + - clone or update `cuga-agent` and `cuga-eval` under `~/pr-regression-testing`; + - run `cuga-eval/setup_cuga.sh`; + - create and sync the `uv` environment; + - run AppWorld setup; + - create the workflow compatibility symlink; + - download and register the GitHub Actions runner; + - start the runner in the background. + +7. Confirm the runner is online in GitHub: + + ```text + cuga-agent repository -> Settings -> Actions -> Runners + ``` + +8. Check runner logs on the VM if needed: + + ```bash + tail -f ~/pr-regression-testing/github-runner/runner.log + ``` + +## GitHub Secrets And Variables + +Configure these in the `cuga-agent` repository: + +```text +Settings -> Secrets and variables -> Actions +``` + +Required secrets: + +- `RITS_API_KEY`: required for `provider=rits`. +- `LITE_LLM_KEY`: required for `provider=litellm`, unless `OPENAI_API_KEY` is set instead. +- `OPENAI_API_KEY`: optional fallback key for OpenAI-compatible/LiteLLM flows. + +Optional repository variables: + +- `MODEL_NAME`: default model override used by the workflow environment. +- `OPENAI_BASE_URL`: default OpenAI-compatible endpoint override. +- `LITE_LLM_URL`: LiteLLM endpoint override. If omitted, `provider=litellm` defaults to `https://ete-litellm.ai-models.vpc-int.res.ibm.com/`. + +Provider defaults used by `run-pr-regression-eval.sh`: + +- `provider=rits` uses `RITS_API_KEY` and defaults `OPENAI_BASE_URL` to `https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100`. +- `provider=litellm` uses `LITE_LLM_KEY` first, then `OPENAI_API_KEY`; it uses `LITE_LLM_URL` first, then `OPENAI_BASE_URL`, then `https://ete-litellm.ai-models.vpc-int.res.ibm.com/`. Unless `model_name` is explicitly set in the PR comment, it uses `aws/gpt-oss-120b`. + +## Run PR Evaluations + +Create a comment on a pull request in `cuga-agent`: + +```text +/run-pr-eval +``` + +By default, this runs: + +- `benchmark=appworld` +- `agent=react` +- `provider=rits` +- `model_name=openai/gpt-oss-120b-a100` +- `num_tasks=4` +- default AppWorld task IDs: `9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1` + +For `provider=litellm`, the default model changes to `aws/gpt-oss-120b` unless `model_name` is explicitly provided. + +## Command Parameters + +The command accepts whitespace-separated `key=value` parameters: + +- `model_name`: model name to evaluate. +- `task_id`: one task ID, or comma-separated task IDs. +- `task_ids`: alias for `task_id`. +- `benchmark`: `appworld` or `m3`; defaults to `appworld`. +- `num_tasks`: positive integer; defaults to `4`. +- `agent`: `react`, `cuga`, or `codeact`; defaults to `react`. +- `provider`: `rits` or `litellm`; defaults to `rits`. + +Unsupported parameters cause the workflow to fail early with a list of supported parameters. + +## Examples + +Run the default AppWorld React-agent evaluation on RITS: + +```text +/run-pr-eval +``` + +Run one AppWorld task with the React agent on RITS: + +```text +/run-pr-eval provider=rits agent=react benchmark=appworld task_id=9aae7da_1 num_tasks=1 +``` + +Run one AppWorld task with the CUGA agent on LiteLLM: + +```text +/run-pr-eval provider=litellm agent=cuga benchmark=appworld task_id=9aae7da_1 num_tasks=1 +``` + +Run AppWorld with two explicit tasks: + +```text +/run-pr-eval benchmark=appworld task_id=9aae7da_1,365e0a3_1 num_tasks=2 +``` + +Run M3 with the default React agent: + +```text +/run-pr-eval benchmark=m3 num_tasks=1 +``` + +Run M3 with the CUGA agent and LiteLLM: + +```text +/run-pr-eval benchmark=m3 agent=cuga provider=litellm num_tasks=1 +``` + +Override the model: + +```text +/run-pr-eval provider=litellm agent=cuga benchmark=appworld model_name=aws/glm-5 task_id=9aae7da_1 num_tasks=1 +``` + +## Output + +The workflow posts a PR comment containing collapsible sections for: + +- requested parameters; +- run metadata; +- evaluation configuration; +- evaluation report; +- run summary. + +The full output is also uploaded as a GitHub Actions artifact and saved on the VM under: + +```text +~/pr-regression-testing/logs/ +``` diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh new file mode 100755 index 00000000..c3d48fce --- /dev/null +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -0,0 +1,333 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +START_EPOCH="$(date +%s)" +START_TIME="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" + +log() { + printf '[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" +} + +open_details() { + local title="$1" + echo "
" + echo "${title}" + echo +} + +close_details() { + echo + echo "
" +} + +on_exit() { + status=$? + end_epoch="$(date +%s)" + end_time="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" + duration=$((end_epoch - START_EPOCH)) + minutes=$((duration / 60)) + seconds=$((duration % 60)) + + echo "######## REPORT START ########" + echo + open_details "Run summary" + echo "- Started: ${START_TIME}" + echo "- Finished: ${end_time}" + echo "- Duration: ${minutes}m ${seconds}s" + echo "- Exit code: ${status}" + + if [[ ${status} -eq 0 ]]; then + echo "- Result: SUCCESS" + else + echo "- Result: FAILED" + fi + close_details + echo "######## REPORT END ########" +} +trap on_exit EXIT + +DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" +DEFAULT_LITELLM_MODEL_NAME="aws/gpt-oss-120b" +DEFAULT_APPWORLD_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" +DEFAULT_BENCHMARK="appworld" +DEFAULT_NUM_TASKS="4" +DEFAULT_AGENT="react" +DEFAULT_PROVIDER="rits" + +MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" +TASK_IDS="${TASK_IDS:-}" +BENCHMARK="${BENCHMARK:-$DEFAULT_BENCHMARK}" +NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" +AGENT="${AGENT:-$DEFAULT_AGENT}" +PROVIDER="${PROVIDER:-$DEFAULT_PROVIDER}" +MODEL_NAME_FROM_COMMENT=false + +# Parse whitespace-separated key=value parameters from the PR comment. +# Supported aliases: +# model_name=... +# task_id=id1,id2 +# task_ids=id1,id2 +# benchmark=appworld|m3 +# num_tasks=... +# agent=react|cuga|codeact +# provider=rits|litellm +COMMENT_BODY="${1:-}" + +# Remove Windows carriage returns and newlines. +COMMENT_BODY="${COMMENT_BODY//$'\r'/ }" +COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" + +read -r -a TOKENS <<< "${COMMENT_BODY}" +echo "######## REPORT START ########" +open_details "Requested parameters" +for token in "${TOKENS[@]}"; do + case "${token}" in + /run-pr-eval) + ;; + + model_name=*) + MODEL_NAME="${token#model_name=}" + MODEL_NAME_FROM_COMMENT=true + ;; + + task_id=*|task_ids=*) + TASK_IDS="${token#*=}" + TASK_IDS="${TASK_IDS//,/ }" + ;; + + benchmark=*) + BENCHMARK="${token#benchmark=}" + ;; + + num_tasks=*) + NUM_TASKS="${token#num_tasks=}" + ;; + + agent=*|agent_type=*) + AGENT="${token#*=}" + ;; + + provider=*) + PROVIDER="${token#provider=}" + ;; + + "") + ;; + + *) + echo "ERROR: Unsupported parameter: ${token}" + echo "Supported parameters: model_name, task_id, task_ids, benchmark, num_tasks, agent, provider" + close_details + echo "######## REPORT END ########" + exit 2 + ;; + esac +done + +if [[ -z "${MODEL_NAME}" ]]; then + echo "ERROR: model_name cannot be empty." + close_details + echo "######## REPORT END ########" + exit 2 +fi + +if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then + echo "ERROR: num_tasks must be a positive integer." + close_details + echo "######## REPORT END ########" + exit 2 +fi + +AGENT="$(printf '%s' "${AGENT}" | tr '[:upper:]' '[:lower:]')" +BENCHMARK="$(printf '%s' "${BENCHMARK}" | tr '[:upper:]' '[:lower:]')" +PROVIDER="$(printf '%s' "${PROVIDER}" | tr '[:upper:]' '[:lower:]')" + +case "${AGENT}" in + react|cuga|codeact) + ;; + *) + echo "ERROR: agent must be one of: react, cuga, codeact." + close_details + echo "######## REPORT END ########" + exit 2 + ;; +esac + +case "${BENCHMARK}" in + appworld|m3) + ;; + *) + echo "ERROR: benchmark must be one of: appworld, m3." + close_details + echo "######## REPORT END ########" + exit 2 + ;; +esac + +case "${PROVIDER}" in + rits) + AGENT_SETTING_CONFIG="settings.rits.toml" + if [[ -z "${RITS_API_KEY:-}" ]]; then + echo "ERROR: RITS_API_KEY is required when provider=rits." + close_details + echo "######## REPORT END ########" + exit 2 + fi + ;; + litellm) + if [[ "${AGENT}" == "cuga" ]]; then + AGENT_SETTING_CONFIG="settings.litellm.toml" + else + AGENT_SETTING_CONFIG="settings.openai.toml" + fi + if [[ "${MODEL_NAME_FROM_COMMENT}" == "false" ]]; then + MODEL_NAME="${DEFAULT_LITELLM_MODEL_NAME}" + fi + OPENAI_BASE_URL="${LITE_LLM_URL:-${OPENAI_BASE_URL:-https://ete-litellm.ai-models.vpc-int.res.ibm.com/}}" + OPENAI_API_KEY="${LITE_LLM_KEY:-${OPENAI_API_KEY:-}}" + if [[ -z "${OPENAI_API_KEY:-}" ]]; then + echo "ERROR: LITE_LLM_KEY or OPENAI_API_KEY is required when provider=litellm." + close_details + echo "######## REPORT END ########" + exit 2 + fi + if [[ -z "${OPENAI_BASE_URL:-}" ]]; then + echo "ERROR: LITE_LLM_URL or OPENAI_BASE_URL is required when provider=litellm." + close_details + echo "######## REPORT END ########" + exit 2 + fi + export OPENAI_API_KEY OPENAI_BASE_URL + ;; + *) + echo "ERROR: provider must be one of: rits, litellm." + close_details + echo "######## REPORT END ########" + exit 2 + ;; +esac + +if [[ "${PROVIDER}" == "rits" ]]; then + export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" + if [[ -z "${RITS_BASE_URL:-}" ]]; then + RITS_BASE_URL="${OPENAI_BASE_URL%/}" + [[ "${RITS_BASE_URL}" == */v1 ]] || RITS_BASE_URL="${RITS_BASE_URL}/v1" + export RITS_BASE_URL + fi +else + : +fi + +if [[ -z "${TASK_IDS}" && "${BENCHMARK}" == "appworld" ]]; then + TASK_IDS="${DEFAULT_APPWORLD_TASK_IDS}" +fi + +read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" + +if [[ ${#TASK_ID_ARRAY[@]} -eq 0 && "${BENCHMARK}" == "appworld" ]]; then + echo "ERROR: At least one task_id is required for benchmark=appworld." + close_details + echo "######## REPORT END ########" + exit 2 +fi +echo "- Model: ${MODEL_NAME}" +echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" +echo "- Num tasks: ${NUM_TASKS}" +if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + echo "- Task IDs: ${TASK_IDS}" +else + echo "- Task IDs: benchmark default" +fi +close_details +echo "######## REPORT END ########" + +# For AppWorld, num_tasks limits the explicit task list. M3 maps num_tasks to +# --max-samples-per-domain below. +if [[ "${BENCHMARK}" == "appworld" && ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then + TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") +fi + +export AGENT_SETTING_CONFIG +export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" +export MODEL_NAME + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EVAL_REPO="${EVAL_REPO:-$(cd "${SCRIPT_DIR}/.." && pwd)}" + +if [[ ! -d "${EVAL_REPO}" ]]; then + echo "ERROR: Evaluation repo not found: ${EVAL_REPO}" + exit 1 +fi + +cd "${EVAL_REPO}" + +echo "######## REPORT START ########" +echo "## Evaluation Metrics" +echo +open_details "Run metadata" +echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" +echo "- PR: ${PR_NUMBER:-unknown}" +echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" +echo "- Runner: ${RUNNER_NAME:-$(hostname)}" +echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" +close_details +echo +open_details "Evaluation configuration" +echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" +echo "- Agent setting config: ${AGENT_SETTING_CONFIG}" +echo "- Model: ${MODEL_NAME}" +echo "- Requested num_tasks: ${NUM_TASKS}" +if [[ "${BENCHMARK}" == "appworld" ]]; then + echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" + echo "- Task IDs:" + for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" + done +else + echo "- M3 max samples per domain: ${NUM_TASKS}" + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + echo "- M3 task filters:" + for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" + done + else + echo "- M3 task filters: benchmark default" + fi +fi +close_details +echo "######## REPORT END ########" + +log "Verifying required commands" +command -v uv >/dev/null 2>&1 || { + echo "ERROR: uv is not installed or is not in PATH." + exit 127 +} + +command -v curl >/dev/null 2>&1 || { + echo "ERROR: curl is not installed or is not in PATH." + exit 127 +} + +log "Starting ${BENCHMARK} ${AGENT} evaluation" + +EVAL_ARGS=( + --benchmark "${BENCHMARK}" + --agent "${AGENT}" +) + +if [[ "${BENCHMARK}" == "appworld" ]]; then + EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") +else + EVAL_ARGS+=(--m3-data --max-samples-per-domain "${NUM_TASKS}") + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") + fi +fi + +bash scripts/eval.sh "${EVAL_ARGS[@]}" diff --git a/scripts/pr-regression-testing/setup-pr-regression-workflow.sh b/scripts/pr-regression-testing/setup-pr-regression-workflow.sh new file mode 100644 index 00000000..b3c6f730 --- /dev/null +++ b/scripts/pr-regression-testing/setup-pr-regression-workflow.sh @@ -0,0 +1,308 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# ============================================================================= +# PR Regression Testing VM Bootstrap +# +# Modes: +# REPO_MODE=fork -> use temporary AnkitaNaik forks (default for now) +# REPO_MODE=upstream -> use cuga-project upstream repositories +# +# Final layout: +# +# ~/pr-regression-testing/ +# ├── cuga-agent/ +# ├── cuga-eval/ +# │ └── scripts/ +# │ ├── pr-regression-testing/ +# │ │ ├── setup-pr-regression-workflow.sh +# │ │ └── run-pr-regression-eval.sh +# │ └── run-pr-eval.sh -> pr-regression-testing/run-pr-regression-eval.sh +# └── github-runner/ +# └── _work/ +# +# Required VM prerequisites: +# git curl tar python3 git-lfs +# +# Required environment variables: +# GITHUB_REPO_URL +# GITHUB_RUNNER_TOKEN +# +# Optional: +# REPO_MODE=fork|upstream +# ROOT_DIR +# RUNNER_NAME +# RUNNER_LABELS +# CUGA_AGENT_BRANCH +# CUGA_EVAL_BRANCH +# RUN_EVAL_SCRIPT +# RUNNER_VERSION +# ============================================================================= + +REPO_MODE="${REPO_MODE:-fork}" +ROOT_DIR="${ROOT_DIR:-$HOME/pr-regression-testing}" + +RUNNER_NAME="${RUNNER_NAME:-$(hostname)}" +RUNNER_LABELS="${RUNNER_LABELS:-run-pr-eval}" + +CUGA_AGENT_BRANCH="${CUGA_AGENT_BRANCH:-main}" +CUGA_EVAL_BRANCH="${CUGA_EVAL_BRANCH:-main}" + +case "${REPO_MODE}" in + fork) + DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/AnkitaNaik/cuga-agent.git" + DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/AnkitaNaik/cuga-eval.git" + ;; + upstream) + DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/cuga-project/cuga-agent.git" + DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/cuga-project/cuga-eval.git" + ;; + *) + echo "ERROR: REPO_MODE must be 'fork' or 'upstream'." >&2 + exit 2 + ;; +esac + +CUGA_AGENT_REPO_URL="${CUGA_AGENT_REPO_URL:-$DEFAULT_CUGA_AGENT_REPO_URL}" +CUGA_EVAL_REPO_URL="${CUGA_EVAL_REPO_URL:-$DEFAULT_CUGA_EVAL_REPO_URL}" + +CUGA_AGENT_DIR="${ROOT_DIR}/cuga-agent" +CUGA_EVAL_DIR="${ROOT_DIR}/cuga-eval" +GH_RUNNER_DIR="${ROOT_DIR}/github-runner" + +RUN_EVAL_SCRIPT="${RUN_EVAL_SCRIPT:-${CUGA_EVAL_DIR}/scripts/pr-regression-testing/run-pr-regression-eval.sh}" +WORKFLOW_EVAL_SCRIPT="${CUGA_EVAL_DIR}/scripts/run-pr-eval.sh" + +log() { + printf '\n[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || die "Required command '$1' is missing." +} + +clone_or_update() { + local repo_url="$1" + local branch="$2" + local dest="$3" + local name="$4" + + if [[ -d "${dest}/.git" ]]; then + log "Updating ${name}" + git -C "${dest}" fetch --all --prune + git -C "${dest}" checkout "${branch}" + git -C "${dest}" pull --ff-only origin "${branch}" + else + log "Cloning ${name}" + git clone --branch "${branch}" "${repo_url}" "${dest}" + fi +} + +: "${GITHUB_REPO_URL:?Set GITHUB_REPO_URL, e.g. https://github.com/OWNER/REPO}" +: "${GITHUB_RUNNER_TOKEN:?Set GITHUB_RUNNER_TOKEN to the temporary runner token}" + +log "Repository mode: ${REPO_MODE}" +echo "cuga-agent: ${CUGA_AGENT_REPO_URL}" +echo "cuga-eval: ${CUGA_EVAL_REPO_URL}" + +log "Checking VM prerequisites" +need_cmd git +need_cmd curl +need_cmd tar +need_cmd python3 +need_cmd git-lfs + +python3 - <<'PY' +import sys +if not ((3, 12) <= sys.version_info[:2] < (3, 14)): + raise SystemExit( + f"ERROR: Python 3.12 or 3.13 is required; found " + f"{sys.version_info.major}.{sys.version_info.minor}" + ) +PY + +if ! command -v uv >/dev/null 2>&1; then + log "Installing uv" + curl -LsSf https://astral.sh/uv/install.sh | sh + export PATH="${HOME}/.local/bin:${HOME}/.cargo/bin:${PATH}" +fi +need_cmd uv + +mkdir -p "${ROOT_DIR}" + +clone_or_update \ + "${CUGA_AGENT_REPO_URL}" \ + "${CUGA_AGENT_BRANCH}" \ + "${CUGA_AGENT_DIR}" \ + "cuga-agent" + +clone_or_update \ + "${CUGA_EVAL_REPO_URL}" \ + "${CUGA_EVAL_BRANCH}" \ + "${CUGA_EVAL_DIR}" \ + "cuga-eval" + +log "Running cuga-eval/setup_cuga.sh" +cd "${CUGA_EVAL_DIR}" +chmod u+x ./setup_cuga.sh +bash ./setup_cuga.sh + +log "Creating/syncing cuga-eval environment" +uv venv +uv sync + +log "Setting up AppWorld" +git lfs install +chmod u+x ./setup_appworld.sh +bash ./setup_appworld.sh +uv sync --group appworld + +log "Checking PR regression evaluation entrypoint" +if [[ ! -f "${RUN_EVAL_SCRIPT}" ]]; then + cat >&2 </dev/null 2>&1; then + log "Runner already running" +else + log "Starting runner in background" + + nohup ./run.sh > "${GH_RUNNER_DIR}/runner.log" 2>&1 & + echo $! > "${GH_RUNNER_DIR}/runner.pid" +fi + +sleep 3 + +if ! pgrep -f "${GH_RUNNER_DIR}/bin/Runner.Listener" >/dev/null 2>&1; then + echo "ERROR: Runner did not remain active." >&2 + tail -n 50 "${GH_RUNNER_DIR}/runner.log" 2>/dev/null || true + exit 1 +fi + +echo +echo "======================================================================" +echo "PR regression testing VM is ready" +echo "======================================================================" +echo +echo "Repository mode:" +echo " ${REPO_MODE}" +echo +echo "Layout:" +echo " ${ROOT_DIR}/" +echo " ├── cuga-agent/" +echo " ├── cuga-eval/" +echo " │ └── scripts/run-eval.sh" +echo " └── github-runner/" +echo " └── _work/" +echo +echo "Repos:" +echo " cuga-agent: ${CUGA_AGENT_REPO_URL}" +echo " cuga-eval: ${CUGA_EVAL_REPO_URL}" +echo +echo "Runner logs:" +echo " tail -f ${GH_RUNNER_DIR}/runner.log" +echo +echo "Workflow target:" +echo " runs-on: [self-hosted, linux, ${RUNNER_LABELS}]" +echo +echo "Workflow script:" +echo " bash \"${RUN_EVAL_SCRIPT}\" \"\${COMMENT_BODY}\"" From 48ce183e96f2c7ce9e36fd86e6fec1ce8889a638 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 08:27:28 -0400 Subject: [PATCH 25/40] Fix script path. --- .../pr-regression-testing/run-pr-regression-eval.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index c3d48fce..07b88bdd 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -254,13 +254,24 @@ export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURE export MODEL_NAME SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -EVAL_REPO="${EVAL_REPO:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +if [[ -z "${EVAL_REPO:-}" ]]; then + if [[ -f "${SCRIPT_DIR}/../../scripts/eval.sh" ]]; then + EVAL_REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" + else + EVAL_REPO="$(cd "${SCRIPT_DIR}/.." && pwd)" + fi +fi if [[ ! -d "${EVAL_REPO}" ]]; then echo "ERROR: Evaluation repo not found: ${EVAL_REPO}" exit 1 fi +if [[ ! -f "${EVAL_REPO}/scripts/eval.sh" ]]; then + echo "ERROR: Evaluation script not found: ${EVAL_REPO}/scripts/eval.sh" + exit 1 +fi + cd "${EVAL_REPO}" echo "######## REPORT START ########" From 02cf5332bb78f0e7baffd359d7b7743106ed9f3c Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 08:27:53 -0400 Subject: [PATCH 26/40] Reorganize files. --- scripts/run-pr-regression-eval.sh | 327 ------------------------ scripts/setup-pr-regression-workflow.sh | 297 --------------------- 2 files changed, 624 deletions(-) delete mode 100755 scripts/run-pr-regression-eval.sh delete mode 100644 scripts/setup-pr-regression-workflow.sh diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh deleted file mode 100755 index 13caceb7..00000000 --- a/scripts/run-pr-regression-eval.sh +++ /dev/null @@ -1,327 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -START_EPOCH="$(date +%s)" -START_TIME="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" - -log() { - printf '[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" -} - -open_details() { - local title="$1" - echo "
" - echo "${title}" - echo -} - -close_details() { - echo - echo "
" -} - -on_exit() { - status=$? - end_epoch="$(date +%s)" - end_time="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" - duration=$((end_epoch - START_EPOCH)) - minutes=$((duration / 60)) - seconds=$((duration % 60)) - - echo "######## REPORT START ########" - echo - open_details "Run summary" - echo "- Started: ${START_TIME}" - echo "- Finished: ${end_time}" - echo "- Duration: ${minutes}m ${seconds}s" - echo "- Exit code: ${status}" - - if [[ ${status} -eq 0 ]]; then - echo "- Result: SUCCESS" - else - echo "- Result: FAILED" - fi - close_details - echo "######## REPORT END ########" -} -trap on_exit EXIT - -DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" -DEFAULT_APPWORLD_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" -DEFAULT_BENCHMARK="appworld" -DEFAULT_NUM_TASKS="4" -DEFAULT_AGENT="react" -DEFAULT_PROVIDER="rits" - -MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" -TASK_IDS="${TASK_IDS:-}" -BENCHMARK="${BENCHMARK:-$DEFAULT_BENCHMARK}" -NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" -AGENT="${AGENT:-$DEFAULT_AGENT}" -PROVIDER="${PROVIDER:-$DEFAULT_PROVIDER}" - -# Parse whitespace-separated key=value parameters from the PR comment. -# Supported aliases: -# model_name=... -# task_id=id1,id2 -# task_ids=id1,id2 -# benchmark=appworld|m3 -# num_tasks=... -# agent=react|cuga|codeact -# provider=rits|litellm -COMMENT_BODY="${1:-}" - -# Remove Windows carriage returns and newlines. -COMMENT_BODY="${COMMENT_BODY//$'\r'/ }" -COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" - -read -r -a TOKENS <<< "${COMMENT_BODY}" -echo "######## REPORT START ########" -open_details "Requested parameters" -for token in "${TOKENS[@]}"; do - case "${token}" in - /run-pr-eval) - ;; - - model_name=*) - MODEL_NAME="${token#model_name=}" - ;; - - task_id=*|task_ids=*) - TASK_IDS="${token#*=}" - TASK_IDS="${TASK_IDS//,/ }" - ;; - - benchmark=*) - BENCHMARK="${token#benchmark=}" - ;; - - num_tasks=*) - NUM_TASKS="${token#num_tasks=}" - ;; - - agent=*|agent_type=*) - AGENT="${token#*=}" - ;; - - provider=*) - PROVIDER="${token#provider=}" - ;; - - "") - ;; - - *) - echo "ERROR: Unsupported parameter: ${token}" - echo "Supported parameters: model_name, task_id, task_ids, benchmark, num_tasks, agent, provider" - close_details - echo "######## REPORT END ########" - exit 2 - ;; - esac -done - -if [[ -z "${MODEL_NAME}" ]]; then - echo "ERROR: model_name cannot be empty." - close_details - echo "######## REPORT END ########" - exit 2 -fi - -if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then - echo "ERROR: num_tasks must be a positive integer." - close_details - echo "######## REPORT END ########" - exit 2 -fi - -AGENT="$(printf '%s' "${AGENT}" | tr '[:upper:]' '[:lower:]')" -BENCHMARK="$(printf '%s' "${BENCHMARK}" | tr '[:upper:]' '[:lower:]')" -PROVIDER="$(printf '%s' "${PROVIDER}" | tr '[:upper:]' '[:lower:]')" - -case "${AGENT}" in - react|cuga|codeact) - ;; - *) - echo "ERROR: agent must be one of: react, cuga, codeact." - close_details - echo "######## REPORT END ########" - exit 2 - ;; -esac - -case "${BENCHMARK}" in - appworld|m3) - ;; - *) - echo "ERROR: benchmark must be one of: appworld, m3." - close_details - echo "######## REPORT END ########" - exit 2 - ;; -esac - -case "${PROVIDER}" in - rits) - AGENT_SETTING_CONFIG="settings.rits.toml" - if [[ -z "${RITS_API_KEY:-}" ]]; then - echo "ERROR: RITS_API_KEY is required when provider=rits." - close_details - echo "######## REPORT END ########" - exit 2 - fi - ;; - litellm) - if [[ "${AGENT}" == "cuga" ]]; then - AGENT_SETTING_CONFIG="settings.litellm.toml" - else - AGENT_SETTING_CONFIG="settings.openai.toml" - fi - OPENAI_BASE_URL="${LITE_LLM_URL:-${OPENAI_BASE_URL:-https://ete-litellm.ai-models.vpc-int.res.ibm.com/}}" - OPENAI_API_KEY="${LITE_LLM_KEY:-${OPENAI_API_KEY:-}}" - if [[ -z "${OPENAI_API_KEY:-}" ]]; then - echo "ERROR: LITE_LLM_KEY or OPENAI_API_KEY is required when provider=litellm." - close_details - echo "######## REPORT END ########" - exit 2 - fi - if [[ -z "${OPENAI_BASE_URL:-}" ]]; then - echo "ERROR: LITE_LLM_URL or OPENAI_BASE_URL is required when provider=litellm." - close_details - echo "######## REPORT END ########" - exit 2 - fi - export OPENAI_API_KEY OPENAI_BASE_URL - ;; - *) - echo "ERROR: provider must be one of: rits, litellm." - close_details - echo "######## REPORT END ########" - exit 2 - ;; -esac - -if [[ "${PROVIDER}" == "rits" ]]; then - export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" - if [[ -z "${RITS_BASE_URL:-}" ]]; then - RITS_BASE_URL="${OPENAI_BASE_URL%/}" - [[ "${RITS_BASE_URL}" == */v1 ]] || RITS_BASE_URL="${RITS_BASE_URL}/v1" - export RITS_BASE_URL - fi -else - : -fi - -if [[ -z "${TASK_IDS}" && "${BENCHMARK}" == "appworld" ]]; then - TASK_IDS="${DEFAULT_APPWORLD_TASK_IDS}" -fi - -read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" - -if [[ ${#TASK_ID_ARRAY[@]} -eq 0 && "${BENCHMARK}" == "appworld" ]]; then - echo "ERROR: At least one task_id is required for benchmark=appworld." - close_details - echo "######## REPORT END ########" - exit 2 -fi -echo "- Model: ${MODEL_NAME}" -echo "- Agent: ${AGENT}" -echo "- Benchmark: ${BENCHMARK}" -echo "- Provider: ${PROVIDER}" -echo "- Num tasks: ${NUM_TASKS}" -if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then - echo "- Task IDs: ${TASK_IDS}" -else - echo "- Task IDs: benchmark default" -fi -close_details -echo "######## REPORT END ########" - -# For AppWorld, num_tasks limits the explicit task list. M3 maps num_tasks to -# --max-samples-per-domain below. -if [[ "${BENCHMARK}" == "appworld" && ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then - TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") -fi - -export AGENT_SETTING_CONFIG -export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" -export MODEL_NAME - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -EVAL_REPO="${EVAL_REPO:-$(cd "${SCRIPT_DIR}/.." && pwd)}" - -if [[ ! -d "${EVAL_REPO}" ]]; then - echo "ERROR: Evaluation repo not found: ${EVAL_REPO}" - exit 1 -fi - -cd "${EVAL_REPO}" - -echo "######## REPORT START ########" -echo "## Evaluation Metrics" -echo -open_details "Run metadata" -echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" -echo "- PR: ${PR_NUMBER:-unknown}" -echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" -echo "- Runner: ${RUNNER_NAME:-$(hostname)}" -echo "- Agent: ${AGENT}" -echo "- Benchmark: ${BENCHMARK}" -echo "- Provider: ${PROVIDER}" -close_details -echo -open_details "Evaluation configuration" -echo "- Agent: ${AGENT}" -echo "- Benchmark: ${BENCHMARK}" -echo "- Provider: ${PROVIDER}" -echo "- Agent setting config: ${AGENT_SETTING_CONFIG}" -echo "- Model: ${MODEL_NAME}" -echo "- Requested num_tasks: ${NUM_TASKS}" -if [[ "${BENCHMARK}" == "appworld" ]]; then - echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" - echo "- Task IDs:" - for task_id in "${TASK_ID_ARRAY[@]}"; do - echo " - ${task_id}" - done -else - echo "- M3 max samples per domain: ${NUM_TASKS}" - if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then - echo "- M3 task filters:" - for task_id in "${TASK_ID_ARRAY[@]}"; do - echo " - ${task_id}" - done - else - echo "- M3 task filters: benchmark default" - fi -fi -close_details -echo "######## REPORT END ########" - -log "Verifying required commands" -command -v uv >/dev/null 2>&1 || { - echo "ERROR: uv is not installed or is not in PATH." - exit 127 -} - -command -v curl >/dev/null 2>&1 || { - echo "ERROR: curl is not installed or is not in PATH." - exit 127 -} - -log "Starting ${BENCHMARK} ${AGENT} evaluation" - -EVAL_ARGS=( - --benchmark "${BENCHMARK}" - --agent "${AGENT}" -) - -if [[ "${BENCHMARK}" == "appworld" ]]; then - EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") -else - EVAL_ARGS+=(--m3-data --max-samples-per-domain "${NUM_TASKS}") - if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then - EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") - fi -fi - -bash scripts/eval.sh "${EVAL_ARGS[@]}" diff --git a/scripts/setup-pr-regression-workflow.sh b/scripts/setup-pr-regression-workflow.sh deleted file mode 100644 index 6b5ff34b..00000000 --- a/scripts/setup-pr-regression-workflow.sh +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -# ============================================================================= -# PR Regression Testing VM Bootstrap -# -# Modes: -# REPO_MODE=fork -> use temporary AnkitaNaik forks (default for now) -# REPO_MODE=upstream -> use cuga-project upstream repositories -# -# Final layout: -# -# ~/pr-regression-testing/ -# ├── cuga-agent/ -# ├── cuga-eval/ -# │ └── scripts/ -# │ ├── run-pr-regression-eval.sh -# │ └── run-pr-eval.sh -> run-pr-regression-eval.sh -# └── github-runner/ -# └── _work/ -# -# Required VM prerequisites: -# git curl tar python3 git-lfs -# -# Required environment variables: -# GITHUB_REPO_URL -# GITHUB_RUNNER_TOKEN -# -# Optional: -# REPO_MODE=fork|upstream -# ROOT_DIR -# RUNNER_NAME -# RUNNER_LABELS -# CUGA_AGENT_BRANCH -# CUGA_EVAL_BRANCH -# RUN_EVAL_SCRIPT -# RUNNER_VERSION -# ============================================================================= - -REPO_MODE="${REPO_MODE:-fork}" -ROOT_DIR="${ROOT_DIR:-$HOME/pr-regression-testing}" - -RUNNER_NAME="${RUNNER_NAME:-$(hostname)}" -RUNNER_LABELS="${RUNNER_LABELS:-run-pr-eval}" - -CUGA_AGENT_BRANCH="${CUGA_AGENT_BRANCH:-main}" -CUGA_EVAL_BRANCH="${CUGA_EVAL_BRANCH:-main}" - -case "${REPO_MODE}" in - fork) - DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/AnkitaNaik/cuga-agent.git" - DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/AnkitaNaik/cuga-eval.git" - ;; - upstream) - DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/cuga-project/cuga-agent.git" - DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/cuga-project/cuga-eval.git" - ;; - *) - echo "ERROR: REPO_MODE must be 'fork' or 'upstream'." >&2 - exit 2 - ;; -esac - -CUGA_AGENT_REPO_URL="${CUGA_AGENT_REPO_URL:-$DEFAULT_CUGA_AGENT_REPO_URL}" -CUGA_EVAL_REPO_URL="${CUGA_EVAL_REPO_URL:-$DEFAULT_CUGA_EVAL_REPO_URL}" - -CUGA_AGENT_DIR="${ROOT_DIR}/cuga-agent" -CUGA_EVAL_DIR="${ROOT_DIR}/cuga-eval" -GH_RUNNER_DIR="${ROOT_DIR}/github-runner" - -RUN_EVAL_SCRIPT="${RUN_EVAL_SCRIPT:-${CUGA_EVAL_DIR}/scripts/run-pr-regression-eval.sh}" -WORKFLOW_EVAL_SCRIPT="${CUGA_EVAL_DIR}/scripts/run-pr-eval.sh" - -log() { - printf '\n[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" -} - -die() { - echo "ERROR: $*" >&2 - exit 1 -} - -need_cmd() { - command -v "$1" >/dev/null 2>&1 || die "Required command '$1' is missing." -} - -clone_or_update() { - local repo_url="$1" - local branch="$2" - local dest="$3" - local name="$4" - - if [[ -d "${dest}/.git" ]]; then - log "Updating ${name}" - git -C "${dest}" fetch --all --prune - git -C "${dest}" checkout "${branch}" - git -C "${dest}" pull --ff-only origin "${branch}" - else - log "Cloning ${name}" - git clone --branch "${branch}" "${repo_url}" "${dest}" - fi -} - -: "${GITHUB_REPO_URL:?Set GITHUB_REPO_URL, e.g. https://github.com/OWNER/REPO}" -: "${GITHUB_RUNNER_TOKEN:?Set GITHUB_RUNNER_TOKEN to the temporary runner token}" - -log "Repository mode: ${REPO_MODE}" -echo "cuga-agent: ${CUGA_AGENT_REPO_URL}" -echo "cuga-eval: ${CUGA_EVAL_REPO_URL}" - -log "Checking VM prerequisites" -need_cmd git -need_cmd curl -need_cmd tar -need_cmd python3 -need_cmd git-lfs - -python3 - <<'PY' -import sys -if not ((3, 12) <= sys.version_info[:2] < (3, 14)): - raise SystemExit( - f"ERROR: Python 3.12 or 3.13 is required; found " - f"{sys.version_info.major}.{sys.version_info.minor}" - ) -PY - -if ! command -v uv >/dev/null 2>&1; then - log "Installing uv" - curl -LsSf https://astral.sh/uv/install.sh | sh - export PATH="${HOME}/.local/bin:${HOME}/.cargo/bin:${PATH}" -fi -need_cmd uv - -mkdir -p "${ROOT_DIR}" - -clone_or_update \ - "${CUGA_AGENT_REPO_URL}" \ - "${CUGA_AGENT_BRANCH}" \ - "${CUGA_AGENT_DIR}" \ - "cuga-agent" - -clone_or_update \ - "${CUGA_EVAL_REPO_URL}" \ - "${CUGA_EVAL_BRANCH}" \ - "${CUGA_EVAL_DIR}" \ - "cuga-eval" - -log "Running cuga-eval/setup_cuga.sh" -cd "${CUGA_EVAL_DIR}" -chmod u+x ./setup_cuga.sh -bash ./setup_cuga.sh - -log "Creating/syncing cuga-eval environment" -uv venv -uv sync - -log "Setting up AppWorld" -git lfs install -chmod u+x ./setup_appworld.sh -bash ./setup_appworld.sh -uv sync --group appworld - -log "Checking PR regression evaluation entrypoint" -if [[ ! -f "${RUN_EVAL_SCRIPT}" ]]; then - cat >&2 </dev/null 2>&1; then - log "Runner already running" -else - log "Starting runner in background" - - nohup ./run.sh > "${GH_RUNNER_DIR}/runner.log" 2>&1 & - echo $! > "${GH_RUNNER_DIR}/runner.pid" -fi - -sleep 3 - -if ! pgrep -f "${GH_RUNNER_DIR}/bin/Runner.Listener" >/dev/null 2>&1; then - echo "ERROR: Runner did not remain active." >&2 - tail -n 50 "${GH_RUNNER_DIR}/runner.log" 2>/dev/null || true - exit 1 -fi - -echo -echo "======================================================================" -echo "PR regression testing VM is ready" -echo "======================================================================" -echo -echo "Repository mode:" -echo " ${REPO_MODE}" -echo -echo "Layout:" -echo " ${ROOT_DIR}/" -echo " ├── cuga-agent/" -echo " ├── cuga-eval/" -echo " │ └── scripts/run-eval.sh" -echo " └── github-runner/" -echo " └── _work/" -echo -echo "Repos:" -echo " cuga-agent: ${CUGA_AGENT_REPO_URL}" -echo " cuga-eval: ${CUGA_EVAL_REPO_URL}" -echo -echo "Runner logs:" -echo " tail -f ${GH_RUNNER_DIR}/runner.log" -echo -echo "Workflow target:" -echo " runs-on: [self-hosted, linux, ${RUNNER_LABELS}]" -echo -echo "Workflow script:" -echo " bash \"${RUN_EVAL_SCRIPT}\" \"\${COMMENT_BODY}\"" From 8c1f357e81ebd5829618377287a4e9b27d36b8e1 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 08:40:38 -0400 Subject: [PATCH 27/40] Update the README. --- scripts/pr-regression-testing/README.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index c3e5daed..a62d245e 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -5,7 +5,7 @@ This folder contains the scripts used to set up and run PR-triggered regression ## Files - `setup-pr-regression-workflow.sh`: bootstraps the self-hosted GitHub Actions runner VM. -- `run-pr-regression-eval.sh`: parses `/run-pr-eval` PR comments and runs the requested benchmark. +- `run-pr-regression-eval.sh`: parses `/run-pr-eval` PR comments and runs the requested benchmark using the `cuga-eval/scripts/eval.sh` script. ## Setup The PR Regression Flow @@ -15,13 +15,7 @@ This folder contains the scripts used to set up and run PR-triggered regression .github/workflows/run-pr-eval.yml ``` -2. Configure the workflow to use the self-hosted runner label: - - ```yaml - runs-on: [self-hosted, linux, run-pr-eval] - ``` - -3. In GitHub, create a self-hosted runner token for the `cuga-agent` repository. +2. In GitHub, create a self-hosted runner token for the `cuga-agent` repository. Go to: @@ -29,7 +23,13 @@ This folder contains the scripts used to set up and run PR-triggered regression cuga-agent repository -> Settings -> Actions -> Runners -> New self-hosted runner ``` - Copy the repository URL and temporary runner token from the generated `config.sh` command. + Copy temporary runner token from the generated `config.sh` command. + +3. Ensure the workflow file uses the self-hosted runner labels: + + ```yaml + runs-on: [self-hosted, linux, run-pr-eval] + ``` 4. On the runner VM, export the required setup variables: @@ -64,7 +64,6 @@ This folder contains the scripts used to set up and run PR-triggered regression - run `cuga-eval/setup_cuga.sh`; - create and sync the `uv` environment; - run AppWorld setup; - - create the workflow compatibility symlink; - download and register the GitHub Actions runner; - start the runner in the background. @@ -74,6 +73,8 @@ This folder contains the scripts used to set up and run PR-triggered regression cuga-agent repository -> Settings -> Actions -> Runners ``` + Ensure the runner is running with the labels `[self-hosted, linux, run-pr-eval]` i.e. same as workflow file. + 8. Check runner logs on the VM if needed: ```bash From 29981387d438f77b254f7bc5f74a3eff7d979da5 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 08:46:53 -0400 Subject: [PATCH 28/40] Update README with parameter name. --- scripts/pr-regression-testing/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index a62d245e..5f7ed5c3 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -36,6 +36,7 @@ This folder contains the scripts used to set up and run PR-triggered regression ```bash export GITHUB_REPO_URL="https://github.com/OWNER/cuga-agent" export GITHUB_RUNNER_TOKEN="YOUR_TEMPORARY_RUNNER_TOKEN" + export RUNNER_LABELS="run-pr-eval" ``` 5. Optionally choose repo mode and branches: From 01a0c898033668fae8fa403a0ee00a7337c93762 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Sat, 5 Sep 2026 09:00:46 -0400 Subject: [PATCH 29/40] Add support for AppWorld subsets. --- .../run-pr-regression-eval.sh | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index 07b88bdd..9660552b 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -56,6 +56,7 @@ DEFAULT_PROVIDER="rits" MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" TASK_IDS="${TASK_IDS:-}" +EVAL_KEY="${EVAL_KEY:-}" BENCHMARK="${BENCHMARK:-$DEFAULT_BENCHMARK}" NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" AGENT="${AGENT:-$DEFAULT_AGENT}" @@ -67,6 +68,7 @@ MODEL_NAME_FROM_COMMENT=false # model_name=... # task_id=id1,id2 # task_ids=id1,id2 +# eval_key=test_easy|test_med|test_hard|... # benchmark=appworld|m3 # num_tasks=... # agent=react|cuga|codeact @@ -95,6 +97,10 @@ for token in "${TOKENS[@]}"; do TASK_IDS="${TASK_IDS//,/ }" ;; + eval_key=*|eval-key=*) + EVAL_KEY="${token#*=}" + ;; + benchmark=*) BENCHMARK="${token#benchmark=}" ;; @@ -116,7 +122,7 @@ for token in "${TOKENS[@]}"; do *) echo "ERROR: Unsupported parameter: ${token}" - echo "Supported parameters: model_name, task_id, task_ids, benchmark, num_tasks, agent, provider" + echo "Supported parameters: model_name, task_id, task_ids, eval_key, benchmark, num_tasks, agent, provider" close_details echo "######## REPORT END ########" exit 2 @@ -141,6 +147,7 @@ fi AGENT="$(printf '%s' "${AGENT}" | tr '[:upper:]' '[:lower:]')" BENCHMARK="$(printf '%s' "${BENCHMARK}" | tr '[:upper:]' '[:lower:]')" PROVIDER="$(printf '%s' "${PROVIDER}" | tr '[:upper:]' '[:lower:]')" +EVAL_KEY="$(printf '%s' "${EVAL_KEY}" | tr '[:upper:]' '[:lower:]')" case "${AGENT}" in react|cuga|codeact) @@ -164,6 +171,13 @@ case "${BENCHMARK}" in ;; esac +if [[ -n "${EVAL_KEY}" && "${BENCHMARK}" != "appworld" ]]; then + echo "ERROR: eval_key is only supported for benchmark=appworld." + close_details + echo "######## REPORT END ########" + exit 2 +fi + case "${PROVIDER}" in rits) AGENT_SETTING_CONFIG="settings.rits.toml" @@ -218,14 +232,14 @@ else : fi -if [[ -z "${TASK_IDS}" && "${BENCHMARK}" == "appworld" ]]; then +if [[ -z "${TASK_IDS}" && -z "${EVAL_KEY}" && "${BENCHMARK}" == "appworld" ]]; then TASK_IDS="${DEFAULT_APPWORLD_TASK_IDS}" fi read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" -if [[ ${#TASK_ID_ARRAY[@]} -eq 0 && "${BENCHMARK}" == "appworld" ]]; then - echo "ERROR: At least one task_id is required for benchmark=appworld." +if [[ ${#TASK_ID_ARRAY[@]} -eq 0 && -z "${EVAL_KEY}" && "${BENCHMARK}" == "appworld" ]]; then + echo "ERROR: At least one task_id or eval_key is required for benchmark=appworld." close_details echo "######## REPORT END ########" exit 2 @@ -234,6 +248,9 @@ echo "- Model: ${MODEL_NAME}" echo "- Agent: ${AGENT}" echo "- Benchmark: ${BENCHMARK}" echo "- Provider: ${PROVIDER}" +if [[ -n "${EVAL_KEY}" ]]; then + echo "- Eval key: ${EVAL_KEY}" +fi echo "- Num tasks: ${NUM_TASKS}" if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then echo "- Task IDs: ${TASK_IDS}" @@ -295,11 +312,18 @@ echo "- Agent setting config: ${AGENT_SETTING_CONFIG}" echo "- Model: ${MODEL_NAME}" echo "- Requested num_tasks: ${NUM_TASKS}" if [[ "${BENCHMARK}" == "appworld" ]]; then - echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" - echo "- Task IDs:" - for task_id in "${TASK_ID_ARRAY[@]}"; do - echo " - ${task_id}" - done + if [[ -n "${EVAL_KEY}" ]]; then + echo "- Eval key: ${EVAL_KEY}" + fi + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" + echo "- Task IDs:" + for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" + done + else + echo "- Task IDs: selected by eval_key" + fi else echo "- M3 max samples per domain: ${NUM_TASKS}" if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then @@ -333,7 +357,12 @@ EVAL_ARGS=( ) if [[ "${BENCHMARK}" == "appworld" ]]; then - EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") + if [[ -n "${EVAL_KEY}" ]]; then + EVAL_ARGS+=(--eval-key "${EVAL_KEY}") + fi + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") + fi else EVAL_ARGS+=(--m3-data --max-samples-per-domain "${NUM_TASKS}") if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then From c5300815093d8b0ffb365ab8d8f7c25b954868d7 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Sat, 5 Sep 2026 09:39:54 -0400 Subject: [PATCH 30/40] Remove evaluator changes from PR --- benchmarks/m3/evaluator/README.md | 5 +- benchmarks/m3/evaluator/constant.py | 2 - benchmarks/m3/evaluator/evaluator.py | 34 ++----- benchmarks/m3/evaluator/judge.py | 35 ++------ benchmarks/m3/evaluator/mcp_tools.py | 25 +----- benchmarks/m3/evaluator/scorer.py | 129 +++++---------------------- 6 files changed, 42 insertions(+), 188 deletions(-) diff --git a/benchmarks/m3/evaluator/README.md b/benchmarks/m3/evaluator/README.md index dffa6c47..9d258194 100644 --- a/benchmarks/m3/evaluator/README.md +++ b/benchmarks/m3/evaluator/README.md @@ -29,8 +29,8 @@ Each paired dialogue is scored in the following order: If exact match fails, an LLM judge compares the predicted answer against the ground-truth answer for the final turn. The correctness prompt is adapted from CRAG and is defined in [`prompt.py`](/vakra/evaluator/prompt.py). 3. **Groundedness check** If the answer is acceptable, another LLM judge verifies that the predicted answer is grounded in the executed tool outputs. -4. **PolicyAdherenceJudge** - Optional: for the multi-hop multi-source capability, the evaluator can programmatically check policy application when `--policy-judge-path /path/to/policy_judge.py` is provided. By default this argument is unset and policy judging is disabled. If you need policy evaluation, open a GitHub issue requesting access to the PolicyJudge file. +4. **PolicyAdheranceJudge** + NOT INCLUDED: For the Multi-hop multi-source capability, we use a PolicyAdherenceJudge that programmatically checks the agent's application of a policy. This judge is currently not included for evaluation. Turn scores are then aggregated into a dialogue score. The default aggregation policy in this repo is `mean`. @@ -144,7 +144,6 @@ CLI arguments: - `--output`: output file path; if omitted, defaults to `results.json` next to the capability directory - `--mcp-config`: MCP connection YAML, defaulting to [`benchmark/mcp_connection_config.yaml`](/vakra/benchmark/mcp_connection_config.yaml) - `--domains`: optional list of domain names to evaluate -- `--policy-judge-path`: optional path to a PolicyJudge Python file. If omitted, policy judging is disabled. ## Output Format diff --git a/benchmarks/m3/evaluator/constant.py b/benchmarks/m3/evaluator/constant.py index 62cf4291..c3799084 100644 --- a/benchmarks/m3/evaluator/constant.py +++ b/benchmarks/m3/evaluator/constant.py @@ -1,7 +1,5 @@ from __future__ import annotations -N_TOOL_CALLS_PER_TURN = 20 - # Predicted file keys PRED_UUID = "uuid" PRED_DOMAIN = "domain" diff --git a/benchmarks/m3/evaluator/evaluator.py b/benchmarks/m3/evaluator/evaluator.py index ed157bb5..6908ea4f 100644 --- a/benchmarks/m3/evaluator/evaluator.py +++ b/benchmarks/m3/evaluator/evaluator.py @@ -27,7 +27,6 @@ ) from tqdm import tqdm from utils import pair_dialogues_by_uuid, read_domain_file -from constant import N_TOOL_CALLS_PER_TURN CAPABILITY_MCP_TOOL_MAP = { "capability_bi_apis": 1, @@ -103,9 +102,10 @@ def _prepend_get_data_to_batch( result.append(dialogue_tools) continue first_turn = list(dialogue_tools[0]) - if skip_initialize_active_data and first_turn and ( - first_turn[0].get("name") == "initialize_active_data" - or first_turn[0].get("name") == "get_data" + if ( + skip_initialize_active_data + and first_turn + and first_turn[0].get("name") == "initialize_active_data" ): first_turn = first_turn[1:] first_turn = [{"name": "get_data", "arguments": {"tool_universe_id": uuid}}] + first_turn @@ -175,7 +175,6 @@ async def evaluate_domain( policy: CapabilityPolicy, mcp_config: Optional[MCPConnectionConfig], capability_name: str, - policy_judge_path: Optional[str] = None, ) -> Tuple[Dict[str, Any], List[float]]: """ Evaluate a single domain (async version). @@ -216,7 +215,6 @@ async def evaluate_domain( turn_cfg = TurnScorerConfig( capability=capability_name, domain=domain, - policy_judge_path=policy_judge_path, ) turn_scorer = TurnScorer( cfg=turn_cfg, @@ -251,7 +249,7 @@ async def evaluate_domain( schema_map = {tool.name: tool.inputSchema for tool in tools_result.tools} # Batch execute tools - batch_tools_pred = [extract_toolcalls_for_mcp(pr, limit_last_n=N_TOOL_CALLS_PER_TURN) for _, pr in paired] + batch_tools_pred = [extract_toolcalls_for_mcp(pr) for _, pr in paired] batch_tools_gt = [extract_toolcalls_for_mcp(gt) for gt, _ in paired] # capability_bi_apis: replace initialize_active_data with get_data (GT), @@ -269,12 +267,7 @@ async def evaluate_domain( _update_dialogue_toolcall_for_get_data(pr_raw, uuid, skip_initialize_active_data=False) mcp_batch_responses_pred = await execute_tools_batch(session, batch_tools_pred, schema_map) - mcp_batch_responses_gt = await execute_tools_batch( - session, - batch_tools_gt, - schema_map, - is_gt=capability_name == "capability_bi_apis", - ) + mcp_batch_responses_gt = await execute_tools_batch(session, batch_tools_gt, schema_map) # Score each paired dialogue for idx, (gt_raw, pr_raw) in enumerate( @@ -286,10 +279,9 @@ async def evaluate_domain( inject_mcp_responses( pr_raw, mcp_batch_responses_pred[idx], type="pred", capability_name=capability_name ) - if capability_name != "capability_bi_apis" and mcp_batch_responses_gt is not None: - inject_mcp_responses( - gt_raw, mcp_batch_responses_gt[idx], type="gt", capability_name=capability_name - ) + inject_mcp_responses( + gt_raw, mcp_batch_responses_gt[idx], type="gt", capability_name=capability_name + ) # Score and store details dialogue_score, dialogue_details = dialogue_scorer.score( @@ -430,7 +422,6 @@ def evaluate_capability( registry: Dict[str, CapabilityPolicy], mcp_config: Optional[MCPConnectionConfig] = None, selected_domains: Optional[set[str]] = None, - policy_judge_path: Optional[str] = None, ) -> Dict[str, Any]: if capability_name not in registry: raise ValueError( @@ -495,7 +486,6 @@ def evaluate_capability( policy=policy, mcp_config=mcp_config, capability_name=capability_name, - policy_judge_path=policy_judge_path, ) ) @@ -551,11 +541,6 @@ def main() -> None: help="Optional list of domain names to evaluate (without .json extension). " "If omitted, all domains are evaluated.", ) - ap.add_argument( - "--policy-judge-path", - default=None, - help="Optional path to a PolicyJudge Python file. If omitted, policy judging is disabled.", - ) args = ap.parse_args() capability_name = args.capability_name @@ -598,7 +583,6 @@ def main() -> None: registry=registry, mcp_config=mcp_config, selected_domains=selected_domains, - policy_judge_path=args.policy_judge_path, ) print(f"Wrote: {out_path}") diff --git a/benchmarks/m3/evaluator/judge.py b/benchmarks/m3/evaluator/judge.py index 521ec9f8..36a4444f 100644 --- a/benchmarks/m3/evaluator/judge.py +++ b/benchmarks/m3/evaluator/judge.py @@ -7,7 +7,6 @@ from langchain_openai import ChatOpenAI from prompt import CorrectnessPrompt, GroundednessPrompt -from constant import N_TOOL_CALLS_PER_TURN from utils import JudgeInput, JudgeOutput _LABEL_RE = re.compile(r"\b(yes|partial|no|unsure)\b", re.IGNORECASE) @@ -15,6 +14,9 @@ _SCORE_MAP = {"yes": 1.0, "partial": 0.0, "no": 0.0, "unsure": 0.0} +N_TOOL_CALLS_PER_TURN = 20 + + class JudgeOutputParseError(ValueError): pass @@ -54,30 +56,6 @@ def __init__(self, config: dict): super().__init__(**config) -class ChatRits(ChatOpenAI): - """RITS chat model integration using langchain-openai.""" - - def __init__(self, config: dict): - model_name = config.get("model_name", "openai/gpt-oss-120b") - end_point = config.get( - "end_point", - "https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b", - ) - rits_api_key = os.getenv("RITS_API_KEY") - if rits_api_key is None or rits_api_key == "": - raise ValueError("RITS_API_KEY is required") - - params = config.get("params", {}) - rits_config = {} - rits_config.setdefault("model_name", model_name) - rits_config.setdefault("api_key", "/") - rits_config.setdefault("default_headers", {"RITS_API_KEY": rits_api_key}) - rits_config.setdefault("base_url", end_point.rstrip("/") + "/v1") - rits_config.update(params) - - super().__init__(**rits_config) - - class LiteLLMChatModel(ChatOpenAI): """ Azure/gpt-4.1 served through the IBM LiteLLM proxy, used as LLM-as-a-judge. @@ -117,19 +95,16 @@ def __init__(self, config: dict): def _build_judge_llm(config: dict) -> ChatOpenAI: """Select the judge backend. Default ``litellm`` -> Azure/gpt-4.1 via the LiteLLM proxy; set ``JUDGE_BACKEND=groq`` (or gpt-oss) for the legacy - Groq-backed gpt-oss-120b judge, or ``JUDGE_BACKEND=rits`` for RITS. - An explicit ``config["backend"]`` wins.""" + Groq-backed gpt-oss-120b judge. An explicit ``config["backend"]`` wins.""" backend = (config.get("backend") or os.getenv("JUDGE_BACKEND", "litellm")).strip().lower() if backend in ("groq", "gpt-oss", "gpt_oss", "oss"): return ChatModel(config) - if backend == "rits": - return ChatRits(config) if backend in ("litellm", "azure", ""): return LiteLLMChatModel(config) raise ValueError( f"Unknown judge backend {backend!r}. Set JUDGE_BACKEND (or config['backend']) to one of: " "'litellm'/'azure' (default, Azure/gpt-4.1 via the LiteLLM proxy) or " - "'groq'/'gpt-oss' (legacy Groq gpt-oss-120b) or 'rits'." + "'groq'/'gpt-oss' (legacy Groq gpt-oss-120b)." ) diff --git a/benchmarks/m3/evaluator/mcp_tools.py b/benchmarks/m3/evaluator/mcp_tools.py index 6cc97e29..fbbce3c3 100644 --- a/benchmarks/m3/evaluator/mcp_tools.py +++ b/benchmarks/m3/evaluator/mcp_tools.py @@ -22,10 +22,7 @@ # ----------------------------- -def extract_toolcalls_for_mcp( - pred_dialogue: Dict[str, Any], - limit_last_n: Optional[int] = None, -) -> List[List[Dict[str, Any]]]: +def extract_toolcalls_for_mcp(pred_dialogue: Dict[str, Any]) -> List[List[Dict[str, Any]]]: """ FastAPIMCPToolClient.call_mcp_tools expects: tools = [ @@ -51,8 +48,6 @@ def extract_toolcalls_for_mcp( continue if isinstance(raw_tc, dict) and "name" in raw_tc: turn_tools.append({"name": raw_tc["name"], "arguments": raw_tc.get("arguments", {})}) - if limit_last_n is not None: - turn_tools = turn_tools[-limit_last_n:] dialogue_tools.append(turn_tools) return dialogue_tools @@ -87,9 +82,6 @@ def inject_mcp_responses( turn_resps = mcp_dialogue_responses[i] if i < len(mcp_dialogue_responses) else [] turn_resps = _ensure_list(turn_resps) - if type == "pred" and len(tool_calls) > len(turn_resps): - tool_calls = tool_calls[-len(turn_resps):] if turn_resps else [] - query_tool_present = False if "multiturn" in capability_name: for tool in tool_calls: @@ -117,10 +109,8 @@ def inject_mcp_responses( truncated_responses[idx] = [ item["text"] for item in json.loads(truncated_responses[idx])["results"] ] # Only text in chunks is retained - except Exception as e: - truncated_responses[idx] = [ - f"Error processing response: {str(e)}" - ] + except Exception: + truncated_responses[idx] = [] # For incorrect tool calls seq["tool_response"] = truncated_responses else: if len(turn_resps) >= n: @@ -245,7 +235,6 @@ async def execute_tools_batch( session: ClientSession, dialogue_tools_batch: List[List[List[Dict[str, Any]]]], schema_map: Optional[Dict[str, Any]], - is_gt: bool = False, ) -> List[List[List[Any]]]: """ Execute a batch of tool calls using a raw MCP session, with schema-based @@ -273,14 +262,6 @@ async def execute_tools_batch( for tool in turn_tools: tool_name = tool["name"] raw_args = tool.get("arguments", {}) or {} - if is_gt: - if (("input_value" in raw_args.keys()) or "data_label" in raw_args.keys()) and len(turn_responses) > 0: - if "handle" in turn_responses[-1]: - previous_response = json.loads(turn_responses[-1]) - if "input_value" in raw_args.keys(): - raw_args["input_value"] = previous_response["results"] - if "data_label" in raw_args.keys(): - raw_args["data_label"] = previous_response["handle"] # Parse JSON string args if needed (JSON round-trip safety) if isinstance(raw_args, str): diff --git a/benchmarks/m3/evaluator/scorer.py b/benchmarks/m3/evaluator/scorer.py index 33891ec6..9b1caa28 100644 --- a/benchmarks/m3/evaluator/scorer.py +++ b/benchmarks/m3/evaluator/scorer.py @@ -1,8 +1,6 @@ from __future__ import annotations from dataclasses import dataclass -from importlib.util import module_from_spec, spec_from_file_location -from pathlib import Path from typing import Any, Dict, List, Optional, Sequence, Tuple from constant import ( @@ -16,31 +14,10 @@ PRED_OUTPUT_QUERY_KEY, PRED_OUTPUT_SEQUENCE_KEY, PRED_OUTPUT_TURN_ID_KEY, - N_TOOL_CALLS_PER_TURN, ) from judge import CorrectnessJudge, ExactMatchJudge, GroundednessJudge, JudgeValidationError, LLMJudge from utils import JudgeInput, ToolCall - -def _load_policy_judge(policy_judge_path: Optional[str]) -> Optional[Any]: - if policy_judge_path is None: - return None - - path = Path(policy_judge_path).expanduser() - if not path.exists(): - raise FileNotFoundError(f"PolicyJudge file not found: {path}") - - spec = spec_from_file_location("policy_judge", path) - if spec is None or spec.loader is None: - raise ImportError(f"Could not load PolicyJudge file: {path}") - - policy_module = module_from_spec(spec) - spec.loader.exec_module(policy_module) - judge_cls = getattr(policy_module, "PolicyAdherenceJudge", None) - if judge_cls is None: - judge_cls = getattr(policy_module, "PolicyAdheranceJudge") - return judge_cls() - # ----------------------------- # Output Scorer # ----------------------------- @@ -59,7 +36,6 @@ class TurnScorerConfig: capability: str domain: str - policy_judge_path: Optional[str] = None class TurnScorer: @@ -78,7 +54,6 @@ def __init__( self.correctness_judge = correctness_judge self.groundedness_judge = groundedness_judge self.exactmatch_judge = exactmatch_judge - self.policy_judge = _load_policy_judge(self.cfg.policy_judge_path) def compare( self, @@ -109,40 +84,26 @@ def compare( # Scoring Turns extra_steps = len(pred) - len(gt) - # Check for policy adherence when an internal policy_judge.py file is present. - if self.policy_judge is not None and "multiturn" in self.cfg.capability: - policy = self.policy_judge.judge(inp=input) - policy_score, policy_explanation = float(policy.score), policy.explanation - if policy_score == 0.0: - score = policy_score - details = { - "gt_steps": len(gt), - "pred_steps": len(pred), - "extra_steps": max(0, extra_steps), - "policy_adherance_score": policy_score, - "exactmatch_score": None, - "answer_score": None, - "groundedness_score": None, - "score_explanation": { - "policy": policy_explanation, - "answer": None, - "exactmatch": None, - "groundedness": None, - }, - } - return score, details - - unanswerable_no_tool_case = ( - len(input.pred_tool_calls) == 0 - and "i can not answer" in input.gt_answer.lower() - and len(input.gt_tool_calls) == 0 - ) - if unanswerable_no_tool_case: - exactmatch_score = 1.0 - exactmatch_explanation = "Special case applied - GT indicates unanswerable and no tool calls made, granting full exact match credit." - else: - exactmatch = self.exactmatch_judge.judge(inp=input) - exactmatch_score, exactmatch_explanation = float(exactmatch.score), exactmatch.explanation + # Check for policy adherance + # if "multiturn" in self.cfg.capability: + # policy = self.policy_judge.judge(inp=input) + # policy_score, policy_explanation = float(policy.score), policy.explanation + # if policy_score==0.0: + # score = policy_score + # details = { + # "gt_steps": len(gt), + # "pred_steps": len(pred), + # "extra_steps": max(0, extra_steps), + # "policy_adherance_score": policy_score, + # "exactmatch_score": None, + # "answer_score": None, + # "groundedness_score": None, + # "score_explanation": {"policy": policy_explanation, "answer": None, "exactmatch": None, "groundedness":None}, + # } + # return score, details + + exactmatch = self.exactmatch_judge.judge(inp=input) + exactmatch_score, exactmatch_explanation = float(exactmatch.score), exactmatch.explanation if exactmatch_score == 0.0: correctness = self.correctness_judge.judge(inp=input) answer_score, answer_explanation = float(correctness.score), correctness.explanation @@ -163,27 +124,6 @@ def compare( } return score, details elif answer_score == 1.0: - if ( - "multiturn" in self.cfg.capability - and "i can not answer" in input.gt_answer.lower() - and (len(input.pred_tool_calls) == 0 or input.pred_answer in ["", " "]) - ): - score = 1.0 - details = { - "gt_steps": len(gt), - "pred_steps": len(pred), - "extra_steps": max(0, extra_steps), - "exactmatch_score": exactmatch_score, - "answer_score": answer_score, - "groundedness_score": 1.0, - "score_explanation": { - "answer": answer_explanation, - "exactmatch": exactmatch_explanation, - "groundedness": "Special case applied - GT indicates unanswerable and no tool calls made, granting full groundedness credit.", - }, - } - return score, details - groundedness = self.groundedness_judge.judge(inp=input) groundedness_score, groundedness_explanation = ( float(groundedness.score), @@ -210,27 +150,6 @@ def compare( f"Expected 0.0 or 1.0. Explanation: {answer_explanation!r}" ) elif exactmatch_score == 1.0: - if ( - "multiturn" in self.cfg.capability - and "i can not answer" in input.gt_answer.lower() - and (len(input.pred_tool_calls) == 0 or input.pred_answer in ["", " "]) - ): - score = 1.0 - details = { - "gt_steps": len(gt), - "pred_steps": len(pred), - "extra_steps": max(0, extra_steps), - "exactmatch_score": exactmatch_score, - "answer_score": None, - "groundedness_score": 1.0, - "score_explanation": { - "answer": None, - "exactmatch": exactmatch_explanation, - "groundedness": "Special case applied - GT indicates unanswerable and no tool calls made, granting full groundedness credit.", - }, - } - return score, details - groundedness = self.groundedness_judge.judge(inp=input) groundedness_score, groundedness_explanation = float(groundedness.score), groundedness.explanation score = groundedness_score @@ -360,12 +279,10 @@ def score( continue pred_answer = self._stringify_pred_answer(pred_turn.get(PRED_OUTPUT_ANSWER_KEY, "")) - pred_sequence = pred_turn.get(PRED_OUTPUT_SEQUENCE_KEY, {}) or {} - pred_calls_all = pred_sequence.get("tool_call", []) or [] - pred_calls = pred_calls_all[-N_TOOL_CALLS_PER_TURN:] if isinstance(pred_calls_all, list) else [] + pred_calls = pred_turn.get(PRED_OUTPUT_SEQUENCE_KEY, {}).get("tool_call", []) pred_responses = self._extract_tool_responses( - pred_sequence.get("tool_response", []) - )[-N_TOOL_CALLS_PER_TURN:] + pred_turn.get(PRED_OUTPUT_SEQUENCE_KEY, {}).get("tool_response", []) + ) score, details = self.turn_scorer.compare( query=query, From 6ff41ebc0c1f34c9785e52877d3605ed3beebc7b Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 17 Sep 2026 00:11:34 -0400 Subject: [PATCH 31/40] fix(pr-eval): align runner input and report formatting Normalize num_tasks as base-10, accept sanitized workflow parameters, and document the SHA-pinned PR eval flow. Fix RITS React-agent defaults and env precedence, and make PR comment report formatting an explicit opt-in so local eval reports keep their normal heading and plain stdout behavior. --- benchmarks/appworld/appworld_eval.py | 13 +- benchmarks/appworld/appworld_eval_codeact.py | 13 +- benchmarks/appworld/appworld_eval_react.py | 13 +- benchmarks/appworld/eval.sh | 136 +--- benchmarks/bpo/eval.sh | 7 +- benchmarks/helpers/compare_report.py | 287 +------- benchmarks/helpers/react_agent.py | 17 +- .../helpers/tests/test_compare_report.py | 664 ++---------------- benchmarks/m3/eval.sh | 13 +- benchmarks/oak_health_insurance/eval.sh | 7 +- scripts/pr-regression-testing/README.md | 73 +- .../run-pr-regression-eval.sh | 123 +++- .../setup-pr-regression-workflow.sh | 5 +- 13 files changed, 339 insertions(+), 1032 deletions(-) diff --git a/benchmarks/appworld/appworld_eval.py b/benchmarks/appworld/appworld_eval.py index 7d34d6bd..8082c909 100644 --- a/benchmarks/appworld/appworld_eval.py +++ b/benchmarks/appworld/appworld_eval.py @@ -470,14 +470,21 @@ def _print_appworld_summary(report: dict): success_rate = report.get("success_rate", 0) avg_steps = report.get("avg_steps", 0) avg_duration = report.get("avg_duration", 0) + for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes") - print("\n" + "=" * 80) - print("EVALUATION COMPLETE") - print("=" * 80) + if for_pr_comment: + print("######## REPORT START ########") + print("## EVALUATION COMPLETE") + else: + print("\n" + "=" * 80) + print("EVALUATION COMPLETE") + print("=" * 80) print(f"Total Tasks: {total}") print(f"Completed: {completed}/{total} ({success_rate:.1%})") print(f"Avg Steps: {avg_steps:.2f}") print(f"Avg Duration: {avg_duration:.2f}s") + if for_pr_comment: + print("######## REPORT END ########") async def main(): diff --git a/benchmarks/appworld/appworld_eval_codeact.py b/benchmarks/appworld/appworld_eval_codeact.py index f92fa516..96f5a9e2 100644 --- a/benchmarks/appworld/appworld_eval_codeact.py +++ b/benchmarks/appworld/appworld_eval_codeact.py @@ -96,14 +96,21 @@ def _print_appworld_summary(report: dict): success_rate = report.get("success_rate", 0) avg_steps = report.get("avg_steps", 0) avg_duration = report.get("avg_duration", 0) + for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes") - print("\n" + "=" * 80) - print("EVALUATION COMPLETE") - print("=" * 80) + if for_pr_comment: + print("######## REPORT START ########") + print("## EVALUATION COMPLETE") + else: + print("\n" + "=" * 80) + print("EVALUATION COMPLETE") + print("=" * 80) print(f"Total Tasks: {total}") print(f"Completed: {completed}/{total} ({success_rate:.1%})") print(f"Avg Steps: {avg_steps:.2f}") print(f"Avg Duration: {avg_duration:.2f}s") + if for_pr_comment: + print("######## REPORT END ########") def _parse_prompt_to_messages(input_str: str) -> list[dict[str, Any]]: diff --git a/benchmarks/appworld/appworld_eval_react.py b/benchmarks/appworld/appworld_eval_react.py index a8d269cc..d79c357e 100644 --- a/benchmarks/appworld/appworld_eval_react.py +++ b/benchmarks/appworld/appworld_eval_react.py @@ -181,14 +181,21 @@ def _print_appworld_summary(report: dict): success_rate = report.get("success_rate", 0) avg_steps = report.get("avg_steps", 0) avg_duration = report.get("avg_duration", 0) + for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes") - print("######## REPORT START ########") - print("## EVALUATION COMPLETE") + if for_pr_comment: + print("######## REPORT START ########") + print("## EVALUATION COMPLETE") + else: + print("\n" + "=" * 80) + print("EVALUATION COMPLETE") + print("=" * 80) print(f"Total Tasks: {total}") print(f"Completed: {completed}/{total} ({success_rate:.1%})") print(f"Avg Steps: {avg_steps:.2f}") print(f"Avg Duration: {avg_duration:.2f}s") - print("######## REPORT END ########") + if for_pr_comment: + print("######## REPORT END ########") def _render_initial_prompt(world: AppWorld) -> str: diff --git a/benchmarks/appworld/eval.sh b/benchmarks/appworld/eval.sh index a7f7ec0f..54e77d49 100755 --- a/benchmarks/appworld/eval.sh +++ b/benchmarks/appworld/eval.sh @@ -34,10 +34,6 @@ for arg in "$@"; do echo " --model-profile Model profile (for bundle metadata)" echo " --agent Agent to run (cuga, react, codeact; default: cuga)" echo " --eval-key Task group key in eval_config.toml (e.g. test_med); recorded in bundle metadata" - echo " --leaderboard Tag this run for official AppWorld leaderboard submission (implies --sdk)" - echo " --force-retry Re-run listed tasks even if a clean partial already exists" - echo " --dry-run Print the evaluator command that would run and exit without starting servers" - echo " --status Show run status; also prints leaderboard status when the workspace has a leaderboard block" echo "" echo "Examples:" echo " ./eval.sh # Default evaluation (cuga)" @@ -52,7 +48,6 @@ done APPWORLD_PID="" REGISTRY_PID="" -APPWORLD_MCP_EXPANDED="" # Parse bundle / model-profile / sdk flags before any server startup so # --status / --stop / --background can short-circuit without side effects. @@ -73,9 +68,6 @@ while [[ $# -gt 0 ]]; do --restart) RESTART=true; shift ;; --status) STATUS=true; shift ;; --agent) AGENT="$2"; shift 2 ;; - --leaderboard) LEADERBOARD="$2"; LEADERBOARD_REQUESTED=true; PASSTHROUGH_ARGS+=(--leaderboard "$2"); USE_SDK=true; shift 2 ;; - --force-retry) FORCE_RETRY=true; PASSTHROUGH_ARGS+=(--force-retry); USE_SDK=true; shift ;; - --dry-run) DRY_RUN=true; shift ;; --verbose|-v|--quiet|-q) PASSTHROUGH_ARGS+=("$1"); shift ;; --task) PASSTHROUGH_ARGS+=("--task-id"); shift while [[ $# -gt 0 && ! "$1" =~ ^-- ]]; do @@ -85,59 +77,6 @@ while [[ $# -gt 0 ]]; do esac done -# --leaderboard and --force-retry are defined only by eval_appworld_sdk.py. The -# other evaluators use argparse.parse_args(), so passing either through would -# abort with "unrecognized arguments" *after* the AppWorld and registry servers -# have already booted. Reject it up front instead. Keyed on whether the flag was -# *passed*, not on its value — `--leaderboard ""` must not slip past the guard. -for sdk_only in ${LEADERBOARD_REQUESTED:+--leaderboard} ${FORCE_RETRY:+--force-retry}; do - if [[ "${AGENT:-cuga}" == "codeact" || "${AGENT:-cuga}" == "react" ]]; then - echo "Error: ${sdk_only} is SDK-only (CUGA lite). --agent ${AGENT} does not accept ${sdk_only}." >&2 - echo "Drop --agent, or drop ${sdk_only}." >&2 - exit 2 - fi -done - -# Single source of truth for the evaluator command. Both --dry-run and the live -# run below call this, so what --dry-run prints is exactly what would execute — -# the two used to be separate copies and had already drifted on the --agent flags. -# Sets EVAL_CMD (array) and EVAL_LABEL. Reads AGENT, USE_SDK, PASSTHROUGH_ARGS. -build_eval_command() { - EVAL_CMD=(uv run --no-sync python -m) - if [ "${AGENT:-cuga}" = "codeact" ]; then - EVAL_CMD+=(benchmarks.appworld.appworld_eval_codeact --agent codeact) - EVAL_LABEL="Using CodeAct agent (appworld_eval_codeact.py)" - elif [ "${AGENT:-cuga}" = "react" ]; then - EVAL_CMD+=(benchmarks.appworld.appworld_eval_react --agent react) - EVAL_LABEL="Using React agent (appworld_eval_react.py)" - elif [[ "$USE_SDK" == "true" ]]; then - EVAL_CMD+=(benchmarks.appworld.eval_appworld_sdk) - EVAL_LABEL="Using SDK evaluator (eval_appworld_sdk.py)" - else - EVAL_CMD+=(benchmarks.appworld.appworld_eval --agent "${AGENT:-cuga}") - EVAL_LABEL="Using default evaluator (appworld_eval.py)" - fi - EVAL_CMD+=("${PASSTHROUGH_ARGS[@]+"${PASSTHROUGH_ARGS[@]}"}") -} - -if [[ "${DRY_RUN:-false}" == "true" ]]; then - # Named experiments / resume use the SDK evaluator on the live path - # (see experiment_workspace_requested below). Apply the same rule here - # so --dry-run --experiment prints the module that would actually run. - if experiment_workspace_requested && [[ "${NO_BUNDLE:-false}" != "true" ]]; then - USE_SDK=true - fi - build_eval_command - echo "DISPATCH: ${EVAL_CMD[*]}" - exit 0 -fi - -if [[ "${STATUS:-false}" == "true" ]]; then - if bd=$(resolve_lifecycle_bundle_dir "appworld" 2>/dev/null) && grep -q '"leaderboard"' "$bd/metadata.json" 2>/dev/null; then - uv run --no-sync python -m benchmarks.appworld.leaderboard status --bundle-dir "$bd" || echo -e "${YELLOW:-}Warning: leaderboard status failed (see above)${NC:-}" - fi -fi - if handle_eval_lifecycle "appworld" "$0" "${PASSTHROUGH_ARGS[@]}"; then exit 0 fi @@ -167,8 +106,6 @@ cleanup() { # after the evaluator exits; clean it up here instead of a dedicated trap so # it can't leak on SIGKILL/early-abort without clobbering this EXIT trap. [ -n "${RUN_MARKER:-}" ] && rm -f "$RUN_MARKER" - # Per-run expanded MCP servers config (only created for non-default APIS ports). - [ -n "${APPWORLD_MCP_EXPANDED:-}" ] && rm -f "$APPWORLD_MCP_EXPANDED" exit $exit_code } @@ -192,33 +129,10 @@ APPWORLD_APIS_PORT="${APPWORLD_APIS_PORT:-${DYNACONF_SERVER_PORTS__APIS_URL:-911 export APPWORLD_APIS_PORT export DYNACONF_SERVER_PORTS__APIS_URL="$APPWORLD_APIS_PORT" -# Per-run token/timing receipt from CugaAgent.invoke() (cuga-agent#467). -# Only the --sdk evaluator (eval_appworld_sdk.py) calls agent.invoke() and can -# use it; the default graph-based appworld_eval.py ignores it harmlessly. -export DYNACONF_ADVANCED_FEATURES__RUN_RECEIPT=true - # Capture console output to a log file for reproducibility bundles CONSOLE_LOG="/tmp/appworld_console.log" exec > >(tee "$CONSOLE_LOG") 2>&1 -# mcp_servers_appworld.yaml hard-codes the app API server as localhost:9111 (one -# per registered app), so overriding APPWORLD_APIS_PORT alone moves the server but -# leaves the registry fetching tool specs from 9111 — it then loads 0 tools and the -# run aborts (#113). When a non-default port is in use, expand the config into a -# per-run temp copy with the real port and point MCP_SERVERS_FILE at it. This -# mirrors benchmarks/m3/eval_m3.py, which expands its registry YAML per run. -# (Done after the exec/tee above so the confirmation echo below lands in -# CONSOLE_LOG and is captured by reproducibility bundles, not just the terminal.) -if [ "$APPWORLD_APIS_PORT" != "9111" ]; then - APPWORLD_MCP_SRC="$SCRIPT_DIR/mcp_servers_appworld.yaml" - if [ -f "$APPWORLD_MCP_SRC" ]; then - APPWORLD_MCP_EXPANDED="$(mktemp "${TMPDIR:-/tmp}/appworld_mcp_servers.XXXXXX")" - sed "s#localhost:9111#localhost:${APPWORLD_APIS_PORT}#g" "$APPWORLD_MCP_SRC" > "$APPWORLD_MCP_EXPANDED" - export MCP_SERVERS_FILE="$APPWORLD_MCP_EXPANDED" - echo -e "${GREEN:-}✓${NC:-} Templated MCP servers config for APIS port ${APPWORLD_APIS_PORT}" - fi -fi - echo -e "${BLUE:-}╔════════════════════════════════════════════════════════════╗${NC:-}" echo -e "${BLUE:-}║ AppWorld Benchmark Evaluation ║${NC:-}" echo -e "${BLUE:-}╚════════════════════════════════════════════════════════════╝${NC:-}" @@ -301,10 +215,6 @@ if prepare_experiment_workspace "appworld"; then mark_run_state_started fi -if [[ -n "${LEADERBOARD_REQUESTED:-}" && -z "${WORKSPACE_BUNDLE_DIR:-}" ]]; then - echo -e "${YELLOW:-}Warning: --leaderboard without --experiment/--resume-experiment: no workspace, so resume, retry keys and the official evaluate are unavailable for this run${NC:-}" -fi - # Apply --model-profile after load_env + arg parsing. common.sh is sourced # unconditionally above; hard-fail loudly if that ever stops being true # instead of silently skipping model-profile application. @@ -332,9 +242,19 @@ RUN_MARKER=$(mktemp "${TMPDIR:-/tmp}/appworld_run_marker.XXXXXX") # Capture the evaluator's exit code instead of letting `set -e` abort here — an # abort would skip both bundling and the failure banner below. set +e -build_eval_command -echo -e "${BLUE:-}${EVAL_LABEL}${NC:-}" -"${EVAL_CMD[@]}" +if [ "${AGENT:-cuga}" = "codeact" ]; then + echo -e "${BLUE:-}Using CodeAct agent (appworld_eval_codeact.py)${NC:-}" + uv run --no-sync python -m benchmarks.appworld.appworld_eval_codeact --agent codeact "${PASSTHROUGH_ARGS[@]}" +elif [ "${AGENT:-cuga}" = "react" ]; then + echo -e "${BLUE:-}Using React agent (appworld_eval_react.py)${NC:-}" + uv run --no-sync python -m benchmarks.appworld.appworld_eval_react --agent react "${PASSTHROUGH_ARGS[@]}" +elif [[ "$USE_SDK" == "true" ]]; then + echo -e "${BLUE:-}Using SDK evaluator (eval_appworld_sdk.py)${NC:-}" + uv run --no-sync python -m benchmarks.appworld.eval_appworld_sdk "${PASSTHROUGH_ARGS[@]}" +else + echo -e "${BLUE:-}Using default evaluator (appworld_eval.py)${NC:-}" + uv run --no-sync python -m benchmarks.appworld.appworld_eval --agent "${AGENT:-cuga}" "${PASSTHROUGH_ARGS[@]}" +fi EVAL_EXIT=$? set -e @@ -382,36 +302,16 @@ if [ -n "$LATEST_RESULT" ] && [ "${NO_BUNDLE:-false}" != "true" ]; then fi FIN_EXTRA+=(--log-file /tmp/appworld.log --log-file /tmp/appworld_registry.log --log-file "$CONSOLE_LOG") finalize_experiment_workspace "appworld" "${FIN_EXTRA[@]}" - - # Run AFTER finalize: finalize_workspace_bundle regenerates - # /report.md from scratch (generate_eval_report), so a - # section appended before finalize would be overwritten. - if [ $EVAL_EXIT -eq 0 ] && grep -q '"leaderboard"' "$WORKSPACE_BUNDLE_DIR/metadata.json" 2>/dev/null; then - AW_EXP=$(uv run --no-sync python -c "import json,sys;print(json.load(open(sys.argv[1]))['leaderboard']['appworld_experiment'])" "$WORKSPACE_BUNDLE_DIR/metadata.json" 2>/dev/null) || AW_EXP="" - # Fall back to the eval_key this run recorded in metadata, so a - # resume that finishes without repeating --eval-key still writes - # appworld_official.json and the report.md section. - OFFICIAL_KEY="${EVAL_KEY:-}" - if [[ -z "$OFFICIAL_KEY" ]]; then - OFFICIAL_KEY=$(uv run --no-sync python -c "import json,sys;m=json.load(open(sys.argv[1]));print((m.get('run') or {}).get('eval_key') or '')" "$WORKSPACE_BUNDLE_DIR/metadata.json" 2>/dev/null) || OFFICIAL_KEY="" - fi - if [[ -z "$AW_EXP" ]]; then - echo -e "${YELLOW:-}Warning: could not read leaderboard.appworld_experiment from metadata.json; skipping official evaluate${NC:-}" - elif [[ -n "$OFFICIAL_KEY" ]]; then - echo -e "${YELLOW:-}Official AppWorld evaluate for key ${OFFICIAL_KEY}...${NC:-}" - uv run --no-sync python -m benchmarks.appworld.leaderboard evaluate "$AW_EXP" --key "$OFFICIAL_KEY" --bundle-dir "$WORKSPACE_BUNDLE_DIR" || echo -e "${YELLOW:-}Warning: official evaluate failed (see above)${NC:-}" - else - echo -e "${YELLOW:-}Note: no --eval-key for this run and none recorded in metadata.json; skipping the official evaluate.${NC:-}" - echo -e "${YELLOW:-} Run it manually: uv run --no-sync python -m benchmarks.appworld.leaderboard evaluate ${AW_EXP} --split --bundle-dir ${WORKSPACE_BUNDLE_DIR}${NC:-}" - fi - fi else echo -e "${YELLOW:-}Creating reproducibility bundle...${NC:-}" # Generate eval report REPORT_TMP=$(mktemp /tmp/appworld_eval_report_XXXXXX) - uv run --no-sync python -m benchmarks.helpers.compare_report eval \ - --result-file "$LATEST_RESULT" --output "$REPORT_TMP" + REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then + REPORT_ARGS+=(--for-pr-comment --stdout-markdown) + fi + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" BUNDLE_ARGS=(assemble --benchmark appworld --result-files "$LATEST_RESULT" diff --git a/benchmarks/bpo/eval.sh b/benchmarks/bpo/eval.sh index f4694dd9..746081c2 100755 --- a/benchmarks/bpo/eval.sh +++ b/benchmarks/bpo/eval.sh @@ -409,8 +409,11 @@ if [ $EVAL_EXIT_CODE -eq 0 ]; then # Generate eval report REPORT_TMP=$(mktemp /tmp/bpo_eval_report_XXXXXX) - uv run --no-sync python -m benchmarks.helpers.compare_report eval \ - --result-file "$LATEST_RESULT" --output "$REPORT_TMP" + REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then + REPORT_ARGS+=(--for-pr-comment --stdout-markdown) + fi + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" BUNDLE_ARGS=(assemble --benchmark bpo --result-files "$LATEST_RESULT" diff --git a/benchmarks/helpers/compare_report.py b/benchmarks/helpers/compare_report.py index 4de3c44b..5b396b7a 100644 --- a/benchmarks/helpers/compare_report.py +++ b/benchmarks/helpers/compare_report.py @@ -261,16 +261,6 @@ def _parse_sdk_results(data: dict) -> dict: total_cost = sum(r.get("total_cost", 0) or 0 for r in results) total_llm_calls = sum(r.get("total_llm_calls", 0) or 0 for r in results) total_cache_tokens = sum(r.get("total_cache_input_tokens", 0) or 0 for r in results) - # Run Receipt fields (cuga-eval#95 / cuga-agent#467) — absent (0) for - # results sourced from Langfuse or from benchmarks not yet opted in. - total_input_tokens = sum(r.get("input_tokens", 0) or 0 for r in results) - total_output_tokens = sum(r.get("output_tokens", 0) or 0 for r in results) - total_receipt_cache_read = sum(r.get("cache_read_tokens", 0) or 0 for r in results) - total_reasoning_tokens = sum(r.get("reasoning_tokens", 0) or 0 for r in results) - total_tool_call_count = sum(r.get("tool_call_count", 0) or 0 for r in results) - total_llm_time_s = sum(r.get("llm_time_s", 0) or 0 for r in results) - total_tool_time_s = sum(r.get("tool_time_s", 0) or 0 for r in results) - total_wall_time_s = sum(r.get("wall_time_s", 0) or 0 for r in results) tasks = {} total_duration = 0.0 @@ -287,17 +277,6 @@ def _parse_sdk_results(data: dict) -> dict: "cost": r.get("total_cost", 0) or 0, "llm_calls": r.get("total_llm_calls", 0) or 0, "cache_tokens": r.get("total_cache_input_tokens", 0) or 0, - # Receipt-only: left None (not 0) when absent so per-task tables - # render "--" via _fmt instead of a misleading 0 (PR #182 review). - "input_tokens": r.get("input_tokens"), - "output_tokens": r.get("output_tokens"), - "cache_read_tokens": r.get("cache_read_tokens", 0) or 0, - "reasoning_tokens": r.get("reasoning_tokens"), - "tool_call_count": r.get("tool_call_count", 0) or 0, - "llm_time_s": r.get("llm_time_s", 0) or 0, - "tool_time_s": r.get("tool_time_s", 0) or 0, - "wall_time_s": r.get("wall_time_s", 0) or 0, - "token_source": r.get("token_source"), "duration": dur, "steps": r.get("steps"), # AppWorld results carry a per-task difficulty band; preserved for @@ -328,16 +307,7 @@ def _parse_sdk_results(data: dict) -> dict: "cost": total_cost, "llm_calls": total_llm_calls, "cache_tokens": total_cache_tokens, - "input_tokens": total_input_tokens, - "output_tokens": total_output_tokens, - "cache_read_tokens": total_receipt_cache_read, - "reasoning_tokens": total_reasoning_tokens, - "tool_call_count": total_tool_call_count, - "llm_time_s": total_llm_time_s, - "tool_time_s": total_tool_time_s, - "wall_time_s": total_wall_time_s, "duration": total_duration if has_duration else None, - "has_receipt_data": any(t.get("token_source") == "receipt" for t in tasks.values()), "tasks": tasks, } @@ -351,18 +321,6 @@ def _parse_appworld_results(data: dict) -> dict: total_cost = sum(t.get("total_cost", 0) or 0 for t in task_results.values()) total_llm_calls = sum(t.get("total_llm_calls", 0) or 0 for t in task_results.values()) total_cache_tokens = sum(t.get("cache_input_tokens", 0) or 0 for t in task_results.values()) - # Run Receipt fields (cuga-eval#95 / cuga-agent#467) — mirrors the full - # 8-field set _parse_sdk_results carries, not just the 3 token fields, so - # a future receipt-producing AppWorld harness isn't silently truncated to - # zero on cache/tool/timing columns (CodeRabbit review, PR #182). - total_input_tokens = sum(t.get("input_tokens", 0) or 0 for t in task_results.values()) - total_output_tokens = sum(t.get("output_tokens", 0) or 0 for t in task_results.values()) - total_receipt_cache_read = sum(t.get("cache_read_tokens", 0) or 0 for t in task_results.values()) - total_reasoning_tokens = sum(t.get("reasoning_tokens", 0) or 0 for t in task_results.values()) - total_tool_call_count = sum(t.get("tool_call_count", 0) or 0 for t in task_results.values()) - total_llm_time_s = sum(t.get("llm_time_s", 0) or 0 for t in task_results.values()) - total_tool_time_s = sum(t.get("tool_time_s", 0) or 0 for t in task_results.values()) - total_wall_time_s = sum(t.get("wall_time_s", 0) or 0 for t in task_results.values()) total_duration = data.get("duration") or sum( t.get("full_execution_time", 0) or 0 for t in task_results.values() ) @@ -375,17 +333,6 @@ def _parse_appworld_results(data: dict) -> dict: "cost": t.get("total_cost", 0) or 0, "llm_calls": t.get("total_llm_calls", 0) or 0, "cache_tokens": t.get("cache_input_tokens", 0) or 0, - # Receipt-only: left None (not 0) when absent so per-task tables - # render "--" via _fmt instead of a misleading 0 (PR #182 review). - "input_tokens": t.get("input_tokens"), - "output_tokens": t.get("output_tokens"), - "cache_read_tokens": t.get("cache_read_tokens", 0) or 0, - "reasoning_tokens": t.get("reasoning_tokens"), - "tool_call_count": t.get("tool_call_count", 0) or 0, - "llm_time_s": t.get("llm_time_s", 0) or 0, - "tool_time_s": t.get("tool_time_s", 0) or 0, - "wall_time_s": t.get("wall_time_s", 0) or 0, - "token_source": t.get("token_source"), "duration": t.get("full_execution_time") or t.get("duration"), "steps": t.get("steps"), "difficulty": t.get("difficulty"), @@ -401,16 +348,7 @@ def _parse_appworld_results(data: dict) -> dict: "cost": total_cost, "llm_calls": total_llm_calls, "cache_tokens": total_cache_tokens, - "input_tokens": total_input_tokens, - "output_tokens": total_output_tokens, - "cache_read_tokens": total_receipt_cache_read, - "reasoning_tokens": total_reasoning_tokens, - "tool_call_count": total_tool_call_count, - "llm_time_s": total_llm_time_s, - "tool_time_s": total_tool_time_s, - "wall_time_s": total_wall_time_s, "duration": total_duration, - "has_receipt_data": any(t.get("token_source") == "receipt" for t in tasks.values()), "tasks": tasks, } @@ -608,48 +546,6 @@ def _aggregate_costs(tasks: dict) -> dict: } -_RECEIPT_COST_FIELDS = ( - "input_tokens", - "output_tokens", - "cache_read_tokens", - "reasoning_tokens", - "tool_call_count", - "llm_time_s", - "tool_time_s", - "wall_time_s", -) - - -def _aggregate_receipt_costs(tasks: dict) -> dict: - """Sum and average Run Receipt fields (cuga-eval#95 / cuga-agent#467) - across a dict of task dicts (as produced by ``_parse_sdk_results``). - - Uses each task's ``token_source == "receipt"`` marker (not field - truthiness) to decide which tasks actually carry receipt data — a - genuine receipt with every field at 0 must still count, and averages - must exclude legacy/non-receipt tasks from the denominator rather than - silently diluting them in. Returns every value as None (not 0) when no - task in *tasks* carries receipt data, so callers can render "--" instead - of a misleading zero for benchmarks/runs that never opted into - ``advanced_features.run_receipt``. - """ - receipt_tasks = [t for t in tasks.values() if t.get("token_source") == "receipt"] - if not receipt_tasks: - result: dict = {} - for field in _RECEIPT_COST_FIELDS: - result[f"total_{field}"] = None - result[f"avg_{field}"] = None - return result - - n = len(receipt_tasks) - result = {} - for field in _RECEIPT_COST_FIELDS: - total = sum(t.get(field, 0) or 0 for t in receipt_tasks) - result[f"total_{field}"] = total - result[f"avg_{field}"] = total / n - return result - - def _per_config_cost_stats(runs, task_filter=None) -> dict: """Mean tokens / LLM calls / duration *per task* across runs for the filtered task subset. Companion to ``_per_config_pass_stats`` so the @@ -844,18 +740,6 @@ def _stats_for_task(task_runs): "mean_tokens": _avg([r.get("tokens") for r in task_runs]), "mean_llm": _avg([r.get("llm_calls") for r in task_runs]), "mean_dur": _avg([r.get("duration") for r in task_runs]), - # Run Receipt fields (cuga-eval#95 / cuga-agent#467) — averaged only - # over runs that actually carry a receipt (token_source == "receipt"), - # so a mix of receipt and legacy/Langfuse runs for the same task - # doesn't dilute the average toward zero. _avg returns None when the - # filtered list is empty, which callers render as "--". - "mean_input": _avg([r.get("input_tokens") for r in task_runs if r.get("token_source") == "receipt"]), - "mean_output": _avg( - [r.get("output_tokens") for r in task_runs if r.get("token_source") == "receipt"] - ), - "mean_reasoning": _avg( - [r.get("reasoning_tokens") for r in task_runs if r.get("token_source") == "receipt"] - ), # Vakra scores (M3 only): mean dialogue score and mean per-judge # scores across runs, ignoring runs where a judge was skipped. "mean_match_rate": _avg([r.get("match_rate") for r in task_runs]), @@ -995,52 +879,6 @@ def _render_compare_report_sections( lines.append(fence_close()) lines.append("") - # ---- 2a-2. Run Receipt Breakdown: only rendered when at least one run - # carries Run Receipt data (cuga-eval#95 / cuga-agent#467) — bpo/oak/ - # appworld-default runs never will, so their reports are unchanged. - # Uses the run-level has_receipt_data marker (token_source == "receipt" - # on at least one task), not field truthiness, so a genuine all-zero - # receipt still counts. - any_receipt_data = any(r.get("has_receipt_data") for runs in model_data.values() for r in runs) - if any_receipt_data: - lines.append(h2("Run Receipt Breakdown")) - lines.append("") - if fence_open(): - lines.append(fence_open()) - receipt_header = ( - f"{'Configuration':<28} {'In Tok':>9} {'Out Tok':>9} {'Cache Rd':>9} " - f"{'Reason':>8} {'Tool#':>6} {'LLM(s)':>8} {'Tool(s)':>8} {'Wall(s)':>8}" - ) - lines.append(receipt_header) - lines.append("─" * len(receipt_header)) - for config_key, runs in model_data.items(): - display = _format_config_label(config_key) - receipt_runs = [r for r in runs if r.get("has_receipt_data")] - if not receipt_runs: - lines.append( - f"{display:<28} {_fmt(None):>9} {_fmt(None):>9} {_fmt(None):>9} " - f"{_fmt(None):>8} {_fmt(None):>6} {_fmt(None, 's'):>8} " - f"{_fmt(None, 's'):>8} {_fmt(None, 's'):>8}" - ) - continue - n = len(receipt_runs) - avg_in = sum(r.get("input_tokens", 0) or 0 for r in receipt_runs) / n - avg_out = sum(r.get("output_tokens", 0) or 0 for r in receipt_runs) / n - avg_cache = sum(r.get("cache_read_tokens", 0) or 0 for r in receipt_runs) / n - avg_reason = sum(r.get("reasoning_tokens", 0) or 0 for r in receipt_runs) / n - avg_tool_n = sum(r.get("tool_call_count", 0) or 0 for r in receipt_runs) / n - avg_llm_t = sum(r.get("llm_time_s", 0) or 0 for r in receipt_runs) / n - avg_tool_t = sum(r.get("tool_time_s", 0) or 0 for r in receipt_runs) / n - avg_wall_t = sum(r.get("wall_time_s", 0) or 0 for r in receipt_runs) / n - lines.append( - f"{display:<28} {_fmt(avg_in):>9} {_fmt(avg_out):>9} {_fmt(avg_cache):>9} " - f"{_fmt(avg_reason):>8} {_fmt(avg_tool_n):>6} {_fmt(avg_llm_t, 's'):>8} " - f"{_fmt(avg_tool_t, 's'):>8} {_fmt(avg_wall_t, 's'):>8}" - ) - if fence_close(): - lines.append(fence_close()) - lines.append("") - # ---- 2b. Per-group breakdowns (only when result files carry the relevant # metadata, so unrelated reports stay unchanged): # - difficulty (AppWorld) @@ -1171,8 +1009,7 @@ def _num_label(t, _m=task_meta): vakra_hdr = f"{'Dialog':>6} {'ExctM':>5} {'Answer':>6} {'Ground':>6} " if has_vakra else "" task_header = ( f"{prefix_hdr}{'Task':<{col_task_w}} {run_cols} {'Successes':>10} " - f"{'Rate':>6} {vakra_hdr}{'Tokens':>8} {'Input':>8} {'Output':>8} {'Reason':>7} " - f"{'LLM':>5} {'Time':>6}" + f"{'Rate':>6} {vakra_hdr}{'Tokens':>8} {'LLM':>5} {'Time':>6}" ) lines.append(task_header) lines.append("─" * len(task_header)) @@ -1184,12 +1021,6 @@ def _num_label(t, _m=task_meta): n_llm = 0 sum_dur = 0.0 n_dur = 0 - sum_input = 0.0 - n_input = 0 - sum_output = 0.0 - n_output = 0 - sum_reasoning = 0.0 - n_reasoning = 0 sum_match_rate = 0.0 n_match_rate = 0 sum_judge = dict.fromkeys(_JUDGE_KEYS, 0.0) @@ -1227,18 +1058,6 @@ def _num_label(t, _m=task_meta): if md is not None: sum_dur += md n_dur += 1 - mi = stats["mean_input"] - mo = stats["mean_output"] - mre = stats["mean_reasoning"] - if mi is not None: - sum_input += mi - n_input += 1 - if mo is not None: - sum_output += mo - n_output += 1 - if mre is not None: - sum_reasoning += mre - n_reasoning += 1 mr = stats["mean_match_rate"] if mr is not None: sum_match_rate += mr @@ -1267,8 +1086,7 @@ def _num_label(t, _m=task_meta): lines.append( f"{row_prefix}{task_disp:<{col_task_w}} {symbols} " f"{successes:>3}/{total:<3} {rate_pct:>5.1f}% {vakra_cols}" - f"{_fmt(mt):>8} {_fmt(mi):>8} {_fmt(mo):>8} {_fmt(mre):>7} " - f"{_fmt(ml):>5} {_fmt(md, 's'):>6}" + f"{_fmt(mt):>8} {_fmt(ml):>5} {_fmt(md, 's'):>6}" ) # AVERAGE row @@ -1279,9 +1097,6 @@ def _num_label(t, _m=task_meta): avg_tok = _fmt(sum_tokens / n_tokens) if n_tokens else "--" avg_llm = _fmt(sum_llm / n_llm) if n_llm else "--" avg_dur = _fmt(sum_dur / n_dur, "s") if n_dur else "--" - avg_input = _fmt(sum_input / n_input) if n_input else "--" - avg_output = _fmt(sum_output / n_output) if n_output else "--" - avg_reasoning = _fmt(sum_reasoning / n_reasoning) if n_reasoning else "--" lines.append("─" * len(task_header)) spacer = " ".join("──" for _ in range(n_runs)) avg_prefix = f"{'':<{cap_w}} {'':<{dom_w}} {'':>{num_w}} " if m3_mode else "" @@ -1299,8 +1114,7 @@ def _num_label(t, _m=task_meta): lines.append( f"{avg_prefix}{'AVERAGE':<{col_task_w}} {spacer} " f"{avg_successes:>3.1f}/{n_runs:<3} {avg_rate:>5.1f}% {avg_vakra_cols}" - f"{avg_tok:>8} {avg_input:>8} {avg_output:>8} {avg_reasoning:>7} " - f"{avg_llm:>5} {avg_dur:>6}" + f"{avg_tok:>8} {avg_llm:>5} {avg_dur:>6}" ) lines.append("") cons = (all_pass / maj_pass) if maj_pass else None @@ -1516,7 +1330,7 @@ def _bucket_m3_tasks(tasks: dict) -> tuple: return rows, True -def generate_eval_report(result_file: str, markdown: bool = True) -> str: +def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment: bool = False) -> str: """Generate a single-evaluation-run report. When the result file is M3-shaped (each task has ``m3_task_id`` + ``domain``), @@ -1524,27 +1338,27 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: rows are readable instead of an unattributed UUID. When ``markdown=False``, the same content is rendered as a plain-text monospace table for terminals (compare.sh's stdout path); ``markdown=True`` (default) is what gets saved - into the bundle's report.md. + into the bundle's report.md. ``for_pr_comment`` is an explicit opt-in for + the PR-regression workflow's collapsible, sentinel-wrapped markdown. """ parsed = parse_result_file(result_file) rows, grouped = _bucket_m3_tasks(parsed["tasks"]) cost = _aggregate_costs(parsed["tasks"]) - receipt_cost = _aggregate_receipt_costs(parsed["tasks"]) h1 = (lambda s: f"# {s}") if markdown else (lambda s: f"\n{s}\n{'=' * len(s)}") h2 = (lambda s: f"## {s}") if markdown else (lambda s: f"\n{s}\n{'-' * len(s)}") fence_open = (lambda: "```text") if markdown else (lambda: "") fence_close = (lambda: "```") if markdown else (lambda: "") - lines = ["######## REPORT START ########"] - if markdown: + if markdown and for_pr_comment: + lines = ["######## REPORT START ########"] lines.append("
") lines.append("Evaluation Report") lines.append("") lines.append(h2("Summary")) lines.append("") else: - lines.extend([h1("Evaluation Report"), ""]) + lines = [h1("Evaluation Report"), ""] lines.append(h2("Summary")) lines.append("") if markdown: @@ -1566,33 +1380,6 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: _append_content_filter_summary(lines, parsed["tasks"], markdown=markdown) lines.append("") - if receipt_cost["total_input_tokens"] is not None: - lines.append(h2("Run Receipt Breakdown")) - lines.append("") - receipt_rows = [ - ("Input Tokens", "total_input_tokens", "avg_input_tokens", ","), - ("Output Tokens", "total_output_tokens", "avg_output_tokens", ","), - ("Cache Read Tokens", "total_cache_read_tokens", "avg_cache_read_tokens", ","), - ("Reasoning Tokens", "total_reasoning_tokens", "avg_reasoning_tokens", ","), - ("Tool Calls", "total_tool_call_count", "avg_tool_call_count", ","), - ("LLM Time", "total_llm_time_s", "avg_llm_time_s", "s"), - ("Tool Time", "total_tool_time_s", "avg_tool_time_s", "s"), - ("Wall Time", "total_wall_time_s", "avg_wall_time_s", "s"), - ] - if markdown: - for label, total_key, avg_key, fmt in receipt_rows: - lines.append( - f"- **{label}**: {_fmt(receipt_cost[total_key], fmt)} total, " - f"{_fmt(receipt_cost[avg_key], fmt)} / task" - ) - else: - for label, total_key, avg_key, fmt in receipt_rows: - lines.append( - f" {label:<18} {_fmt(receipt_cost[total_key], fmt):>10} total " - f"{_fmt(receipt_cost[avg_key], fmt):>10} / task" - ) - lines.append("") - lines.append(h2("Per-Task Results")) lines.append("") @@ -1613,20 +1400,18 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: if has_vakra: lines.append( "| Task | Domain | # | Result | Dialogue | ExactMatch | Answer | Groundedness " - "| Tokens | Cost | LLM Calls | Cache Tokens | Input | Output | Reasoning | Duration | Steps |" + "| Tokens | Cost | LLM Calls | Cache Tokens | Duration | Steps |" ) lines.append( "|------|--------|---|--------|----------|------------|--------|--------------" - "|--------|------|-----------|--------------|-------|--------|-----------|----------|-------|" + "|--------|------|-----------|--------------|----------|-------|" ) else: lines.append( - "| Task | Domain | # | Result | Tokens | Cost | LLM Calls | Cache Tokens " - "| Input | Output | Reasoning | Duration | Steps |" + "| Task | Domain | # | Result | Tokens | Cost | LLM Calls | Cache Tokens | Duration | Steps |" ) lines.append( - "|------|--------|---|--------|--------|------|-----------|--------------" - "|-------|--------|-----------|----------|-------|" + "|------|--------|---|--------|--------|------|-----------|--------------|----------|-------|" ) current_key: tuple = (None, None) for row in rows: @@ -1658,8 +1443,6 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: f"| {tid_disp} | {dom_disp} | {ordn_disp} | {status} {vakra_cols}" f"| {_fmt(t['tokens'])} | {_fmt(t.get('cost'), '$')} " f"| {_fmt(t.get('llm_calls'))} | {_fmt(t.get('cache_tokens'))} " - f"| {_fmt(t.get('input_tokens'))} | {_fmt(t.get('output_tokens'))} " - f"| {_fmt(t.get('reasoning_tokens'))} " f"| {_fmt(t.get('duration'), 's')} | {_fmt(t.get('steps'))} |" ) else: @@ -1671,15 +1454,13 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: f" {col_task:<4} {'Domain':<{col_dom_w}} {'#':>2} " f"{'R':<2} {'Dialog':>6} {'ExctM':>5} {'Answer':>6} {'Ground':>6} " f"{'Tokens':>10} {'Cost':>7} {'LLM':>5} " - f"{'Cache':>10} {'Input':>9} {'Output':>9} {'Reason':>7} " - f"{'Duration':>9} {'Steps':>5}" + f"{'Cache':>10} {'Duration':>9} {'Steps':>5}" ) else: header = ( f" {col_task:<4} {'Domain':<{col_dom_w}} {'#':>2} " f"{'R':<2} {'Tokens':>10} {'Cost':>7} {'LLM':>5} " - f"{'Cache':>10} {'Input':>9} {'Output':>9} {'Reason':>7} " - f"{'Duration':>9} {'Steps':>5}" + f"{'Cache':>10} {'Duration':>9} {'Steps':>5}" ) lines.append(header) lines.append(" " + "─" * (len(header) - 2)) @@ -1717,33 +1498,21 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: f"{_fmt(t.get('cost'), '$'):>7} " f"{_fmt(t.get('llm_calls')):>5} " f"{_fmt(t.get('cache_tokens')):>10} " - f"{_fmt(t.get('input_tokens')):>9} " - f"{_fmt(t.get('output_tokens')):>9} " - f"{_fmt(t.get('reasoning_tokens')):>7} " f"{_fmt(t.get('duration'), 's'):>9} " f"{_fmt(t.get('steps')):>5}" ) else: # Legacy flat table (e.g. AppWorld where m3_task_id/domain aren't set). if markdown: - lines.append( - "| Task | Result | Tokens | Cost | LLM Calls | Cache Tokens " - "| Input | Output | Reasoning | Duration | Steps |" - ) - lines.append( - "|------|--------|--------|------|-----------|--------------" - "|-------|--------|-----------|----------|-------|" - ) + lines.append("| Task | Result | Tokens | Cost | LLM Calls | Cache Tokens | Duration | Steps |") + lines.append("|------|--------|--------|------|-----------|--------------|----------|-------|") for row in rows: t = row["data"] status = _task_result_mark(t) lines.append( f"| {row['label']} | {status} | {_fmt(t['tokens'])} " f"| {_fmt(t.get('cost'), '$')} | {_fmt(t.get('llm_calls'))} " - f"| {_fmt(t.get('cache_tokens'))} " - f"| {_fmt(t.get('input_tokens'))} | {_fmt(t.get('output_tokens'))} " - f"| {_fmt(t.get('reasoning_tokens'))} " - f"| {_fmt(t.get('duration'), 's')} " + f"| {_fmt(t.get('cache_tokens'))} | {_fmt(t.get('duration'), 's')} " f"| {_fmt(t.get('steps'))} |" ) else: @@ -1751,7 +1520,6 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: header = ( f" {'Task':<{col_task_w}} {'R':<2} {'Tokens':>10} " f"{'Cost':>7} {'LLM':>5} {'Cache':>10} " - f"{'Input':>9} {'Output':>9} {'Reason':>7} " f"{'Duration':>9} {'Steps':>5}" ) lines.append(header) @@ -1768,9 +1536,6 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: f"{_fmt(t.get('cost'), '$'):>7} " f"{_fmt(t.get('llm_calls')):>5} " f"{_fmt(t.get('cache_tokens')):>10} " - f"{_fmt(t.get('input_tokens')):>9} " - f"{_fmt(t.get('output_tokens')):>9} " - f"{_fmt(t.get('reasoning_tokens')):>7} " f"{_fmt(t.get('duration'), 's'):>9} " f"{_fmt(t.get('steps')):>5}" ) @@ -1833,10 +1598,10 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: markdown=markdown, ) ) - if markdown: + if markdown and for_pr_comment: lines.append("") lines.append("
") - lines.append("######## REPORT END ########") + lines.append("######## REPORT END ########") return "\n".join(lines) @@ -1851,8 +1616,10 @@ def main(): parser.add_argument("command") # consume "eval" parser.add_argument("--result-file", required=True) parser.add_argument("--output", "-o", default=None) + parser.add_argument("--for-pr-comment", action="store_true") + parser.add_argument("--stdout-markdown", action="store_true") args = parser.parse_args() - report = generate_eval_report(args.result_file) + report = generate_eval_report(args.result_file, for_pr_comment=args.for_pr_comment) else: parser = argparse.ArgumentParser(description="Generate comparison report") parser.add_argument("command") # consume "compare" @@ -1874,11 +1641,11 @@ def main(): # the canonical bundle location at the end of the run, which is what the # user actually wants to navigate to. if args.output: - # Compare-mode and eval-mode both produce markdown for the saved file. - # For eval-mode, also print markdown to stdout because PR workflows - # capture stdout and post marked report blocks back to GitHub comments. if "command" in args and getattr(args, "command", None) == "eval": - stdout_report = report + if getattr(args, "for_pr_comment", False) or getattr(args, "stdout_markdown", False): + stdout_report = report + else: + stdout_report = generate_eval_report(args.result_file, markdown=False) else: stdout_report = generate_report(config_results, markdown=False) Path(args.output).write_text(report) diff --git a/benchmarks/helpers/react_agent.py b/benchmarks/helpers/react_agent.py index 2b9f8526..6a11157c 100644 --- a/benchmarks/helpers/react_agent.py +++ b/benchmarks/helpers/react_agent.py @@ -50,20 +50,23 @@ class ChatRits(ChatOpenAI): def __init__(self, config: dict[str, Any]): model_name = config.get("model_name", "openai/gpt-oss-120b-a100") - end_point = config.get( - "end_point", - "https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100", + end_point = ( + config.get("end_point") + or "https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100" ) rits_api_key = config.get("api_key") or os.getenv("RITS_API_KEY") if rits_api_key is None: raise ValueError("RITS_API_KEY is required for settings.rits.toml") params = config.get("params", {}) + base_url = end_point.rstrip("/") + if not base_url.endswith("/v1"): + base_url = f"{base_url}/v1" rits_config: dict[str, Any] = { - "model_name": model_name, + "model": model_name, "api_key": "/", "default_headers": {"RITS_API_KEY": rits_api_key}, - "base_url": end_point.rstrip("/") + "/v1", + "base_url": base_url, } rits_config.update(params) super().__init__(**rits_config) @@ -201,8 +204,8 @@ def _create_llm(self) -> Any: return ChatOpenAI(**llm_kwargs) elif settings_config in ("settings.rits.toml", "settings.rits.proxy.toml"): - api_base = os.getenv("LITE_LLM_URL") or os.getenv("OPENAI_BASE_URL") or os.getenv("RITS_BASE_URL") - api_key = os.getenv("LITE_LLM_KEY") or os.getenv("OPENAI_API_KEY") or os.getenv("RITS_API_KEY") + api_base = os.getenv("RITS_BASE_URL") or os.getenv("LITE_LLM_URL") or os.getenv("OPENAI_BASE_URL") + api_key = os.getenv("RITS_API_KEY") or os.getenv("LITE_LLM_KEY") or os.getenv("OPENAI_API_KEY") model_name = os.getenv("MODEL_NAME") or self.model or "openai/gpt-oss-120b-a100" logger.info(f"RITS config: model={model_name}, api_base={api_base}, api_key_present={bool(api_key)}") return ChatRits(config={"model_name": model_name, "end_point": api_base, "api_key": api_key}) diff --git a/benchmarks/helpers/tests/test_compare_report.py b/benchmarks/helpers/tests/test_compare_report.py index 46f5a55d..0a67e51a 100644 --- a/benchmarks/helpers/tests/test_compare_report.py +++ b/benchmarks/helpers/tests/test_compare_report.py @@ -21,17 +21,16 @@ """ import json +import sys from pathlib import Path import pytest +from benchmarks.helpers import compare_report as compare_report_module from benchmarks.helpers.compare_report import ( - _aggregate_receipt_costs, _last_turn_judge_scores, _m3_capability_group, - _parse_appworld_results, _parse_sdk_results, - _stats_for_task, generate_eval_report, generate_report, ) @@ -221,6 +220,78 @@ def test_parse_sdk_results_includes_react_steps(): assert parsed["tasks"]["t1"]["steps"] == 3 +def test_eval_report_default_omits_pr_comment_wrapping(tmp_path): + result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False}) + + report = generate_eval_report(result_file) + + assert report.startswith("# Evaluation Report") + assert "######## REPORT START ########" not in report + assert "######## REPORT END ########" not in report + assert "Evaluation Report" not in report + + +def test_eval_report_pr_comment_wrapping_is_opt_in(tmp_path): + result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False}) + + report = generate_eval_report(result_file, for_pr_comment=True) + + assert report.startswith("######## REPORT START ########") + assert "Evaluation Report" in report + assert report.rstrip().endswith("######## REPORT END ########") + + +def test_eval_main_with_output_prints_plain_text_by_default(tmp_path, monkeypatch, capsys): + result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False}) + output_file = tmp_path / "report.md" + monkeypatch.setattr( + sys, + "argv", + [ + "compare_report.py", + "eval", + "--result-file", + result_file, + "--output", + str(output_file), + ], + ) + + compare_report_module.main() + + stdout = capsys.readouterr().out + saved_report = output_file.read_text() + assert stdout.startswith("\nEvaluation Report") + assert "######## REPORT START ########" not in stdout + assert saved_report.startswith("# Evaluation Report") + assert "######## REPORT START ########" not in saved_report + + +def test_eval_main_pr_comment_mode_prints_markdown_markers(tmp_path, monkeypatch, capsys): + result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False}) + output_file = tmp_path / "report.md" + monkeypatch.setattr( + sys, + "argv", + [ + "compare_report.py", + "eval", + "--result-file", + result_file, + "--output", + str(output_file), + "--for-pr-comment", + ], + ) + + compare_report_module.main() + + stdout = capsys.readouterr().out + saved_report = output_file.read_text() + assert stdout.startswith("######## REPORT START ########") + assert saved_report.startswith("######## REPORT START ########") + + def _m3_run(tmp_path: Path, name: str, tasks) -> str: """Write an M3-shape ``*_final_report.json`` and return its path. @@ -339,8 +410,8 @@ def test_eval_report_m3_without_vakra_omits_columns(tmp_path): assert "Dialogue" not in report assert "ExactMatch" not in report assert ( - "| Task | Domain | # | Result | Tokens | Cost | LLM Calls | Cache Tokens " - "| Input | Output | Reasoning | Duration | Steps |" in report + "| Task | Domain | # | Result | Tokens | Cost | LLM Calls | Cache Tokens | Duration | Steps |" + in report ) @@ -927,586 +998,3 @@ def test_vakra_scores_stay_attributed_within_grouped_sections(tmp_path): assert "1.00" in cuga_section assert "Dialog" in react_section assert "0.00" in react_section - - -def test_parse_sdk_results_extracts_receipt_fields(): - data = { - "metrics": {"total_tasks": 1, "passed": 1}, - "results": [ - { - "task_name": "t1", - "success": True, - "total_tokens": 150, - "input_tokens": 100, - "output_tokens": 50, - "cache_read_tokens": 20, - "reasoning_tokens": 5, - "tool_call_count": 2, - "llm_time_s": 1.5, - "tool_time_s": 0.5, - "wall_time_s": 2.5, - } - ], - } - parsed = _parse_sdk_results(data) - - assert parsed["input_tokens"] == 100 - assert parsed["output_tokens"] == 50 - assert parsed["cache_read_tokens"] == 20 - assert parsed["reasoning_tokens"] == 5 - assert parsed["tool_call_count"] == 2 - assert parsed["llm_time_s"] == 1.5 - assert parsed["tool_time_s"] == 0.5 - assert parsed["wall_time_s"] == 2.5 - assert parsed["tasks"]["t1"]["input_tokens"] == 100 - - -def test_parse_sdk_results_defaults_receipt_fields_to_zero_when_absent(): - """Run-level totals still sum to 0 when no result carries receipt fields - (they're plain sums). The per-task value, however, stays None -- it's a - receipt-only field that was never measured for this task, not a genuine - zero (Sergey review, PR #182 follow-up).""" - data = { - "metrics": {"total_tasks": 1, "passed": 1}, - "results": [{"task_name": "t1", "success": True, "total_tokens": 50}], - } - parsed = _parse_sdk_results(data) - - assert parsed["input_tokens"] == 0 - assert parsed["tasks"]["t1"]["input_tokens"] is None - - -def test_aggregate_receipt_costs_totals_and_averages(): - tasks = { - "t1": { - "token_source": "receipt", - "input_tokens": 100, - "output_tokens": 50, - "cache_read_tokens": 20, - "reasoning_tokens": 5, - "tool_call_count": 2, - "llm_time_s": 1.0, - "tool_time_s": 0.5, - "wall_time_s": 1.5, - }, - "t2": { - "token_source": "receipt", - "input_tokens": 200, - "output_tokens": 100, - "cache_read_tokens": 0, - "reasoning_tokens": 0, - "tool_call_count": 4, - "llm_time_s": 2.0, - "tool_time_s": 1.0, - "wall_time_s": 3.0, - }, - } - agg = _aggregate_receipt_costs(tasks) - - assert agg["total_input_tokens"] == 300 - assert agg["avg_input_tokens"] == 150 - assert agg["total_output_tokens"] == 150 - assert agg["total_tool_call_count"] == 6 - assert agg["avg_wall_time_s"] == 2.25 - - -def test_aggregate_receipt_costs_all_none_when_no_receipt_data(): - tasks = {"t1": {"tokens": 50}} # legacy shape, no receipt fields at all - agg = _aggregate_receipt_costs(tasks) - - assert agg["total_input_tokens"] is None - assert agg["avg_input_tokens"] is None - - -def test_aggregate_receipt_costs_detects_nontoken_receipt_data(): - """When a task has zero token-related fields but nonzero non-token fields - (tool_call_count, llm_time_s, tool_time_s, wall_time_s, cache_read_tokens, - reasoning_tokens), _aggregate_receipt_costs must recognize this as receipt - data and return real numbers, not None.""" - tasks = { - "t1": { - "token_source": "receipt", - "input_tokens": 0, - "output_tokens": 0, - "cache_read_tokens": 0, - "reasoning_tokens": 0, - "tool_call_count": 3, # nonzero non-token field - "llm_time_s": 1.5, # nonzero non-token field - "tool_time_s": 0.0, - "wall_time_s": 2.0, # nonzero non-token field - } - } - agg = _aggregate_receipt_costs(tasks) - - # All fields should have real numbers, not None - assert agg["total_tool_call_count"] == 3 - assert agg["avg_tool_call_count"] == 3 - assert agg["total_llm_time_s"] == 1.5 - assert agg["avg_llm_time_s"] == 1.5 - assert agg["total_wall_time_s"] == 2.0 - assert agg["avg_wall_time_s"] == 2.0 - # Token fields are still 0, not None - assert agg["total_input_tokens"] == 0 - assert agg["avg_input_tokens"] == 0 - - -def test_aggregate_receipt_costs_detects_genuine_all_zero_receipt(): - """A task marked token_source == "receipt" whose every receipt field is - legitimately 0 (e.g. a call that used 0 output tokens) must still be - detected as carrying receipt data — the marker, not field truthiness, - decides this (cuga-eval#182 CodeRabbit review).""" - tasks = { - "t1": { - "token_source": "receipt", - "input_tokens": 0, - "output_tokens": 0, - "cache_read_tokens": 0, - "reasoning_tokens": 0, - "tool_call_count": 0, - "llm_time_s": 0.0, - "tool_time_s": 0.0, - "wall_time_s": 0.0, - } - } - agg = _aggregate_receipt_costs(tasks) - - assert agg["total_input_tokens"] == 0 - assert agg["avg_input_tokens"] == 0 - assert agg["total_wall_time_s"] == 0.0 - assert agg["avg_wall_time_s"] == 0.0 - - -def test_aggregate_receipt_costs_averages_only_over_receipt_tagged_tasks(): - """A run with a mix of receipt-tagged tasks and legacy/non-receipt tasks - (no token_source, everything defaulted to 0) must average only over the - receipt-tagged subset — dividing by the full task count would silently - dilute the average toward zero (cuga-eval#182 CodeRabbit review).""" - tasks = { - "t1": { - "token_source": "receipt", - "input_tokens": 100, - "output_tokens": 50, - "cache_read_tokens": 0, - "reasoning_tokens": 0, - "tool_call_count": 1, - "llm_time_s": 1.0, - "tool_time_s": 0.0, - "wall_time_s": 1.0, - }, - "t2": { - "token_source": "receipt", - "input_tokens": 200, - "output_tokens": 100, - "cache_read_tokens": 0, - "reasoning_tokens": 0, - "tool_call_count": 1, - "llm_time_s": 1.0, - "tool_time_s": 0.0, - "wall_time_s": 1.0, - }, - "t3_legacy": { - # No token_source at all: pre-migration task, never opted in. - "input_tokens": 0, - "output_tokens": 0, - }, - } - agg = _aggregate_receipt_costs(tasks) - - # Average must divide by 2 (the receipt-tagged count), not 3. - assert agg["total_input_tokens"] == 300 - assert agg["avg_input_tokens"] == 150 - assert agg["total_output_tokens"] == 150 - assert agg["avg_output_tokens"] == 75 - - -def _write_sdk_result_file(tmp_path, results, name="results.json"): - path = tmp_path / name - path.write_text( - json.dumps({"metrics": {"total_tasks": len(results), "passed": len(results)}, "results": results}) - ) - return str(path) - - -def test_generate_eval_report_includes_receipt_breakdown_when_present(tmp_path): - results = [ - { - "task_name": "t1", - "success": True, - "total_tokens": 150, - "token_source": "receipt", - "input_tokens": 100, - "output_tokens": 50, - "cache_read_tokens": 20, - "reasoning_tokens": 5, - "tool_call_count": 2, - "llm_time_s": 1.5, - "tool_time_s": 0.5, - "wall_time_s": 2.5, - } - ] - report = generate_eval_report(_write_sdk_result_file(tmp_path, results)) - - assert "Run Receipt Breakdown" in report - assert "Input Tokens" in report - assert "100" in report - - -def test_generate_eval_report_omits_receipt_breakdown_when_absent(tmp_path): - results = [{"task_name": "t1", "success": True, "total_tokens": 150}] - report = generate_eval_report(_write_sdk_result_file(tmp_path, results)) - - assert "Run Receipt Breakdown" not in report - - -def test_generate_report_includes_receipt_breakdown_when_present(tmp_path): - results = [ - { - "task_name": "t1", - "success": True, - "total_tokens": 150, - "token_source": "receipt", - "input_tokens": 100, - "output_tokens": 50, - "cache_read_tokens": 20, - "reasoning_tokens": 5, - "tool_call_count": 2, - "llm_time_s": 1.5, - "tool_time_s": 0.5, - "wall_time_s": 2.5, - } - ] - result_file = _write_sdk_result_file(tmp_path, results) - report = generate_report({"gpt-oss-120b": [result_file]}) - - assert "Run Receipt Breakdown" in report - assert "In Tok" in report - - -def test_generate_report_omits_receipt_breakdown_when_absent(tmp_path): - results = [{"task_name": "t1", "success": True, "total_tokens": 150}] - result_file = _write_sdk_result_file(tmp_path, results) - report = generate_report({"gpt-oss-120b": [result_file]}) - - assert "Run Receipt Breakdown" not in report - - -def test_generate_report_detects_nontoken_receipt_data(tmp_path): - """Same class of bug as test_aggregate_receipt_costs_detects_nontoken_receipt_data, - but for _render_compare_report_sections's own any_receipt_data gate: a run - with zero input_tokens/output_tokens but nonzero tool_call_count/wall_time_s - must still trigger the "Run Receipt Breakdown" section (cuga-eval#95 final - review finding — the gate previously only looked at 2 of 8 receipt fields).""" - results = [ - { - "task_name": "t1", - "success": True, - "total_tokens": 0, - "token_source": "receipt", - "input_tokens": 0, - "output_tokens": 0, - "tool_call_count": 3, - "wall_time_s": 2.0, - } - ] - result_file = _write_sdk_result_file(tmp_path, results) - report = generate_report({"gpt-oss-120b": [result_file]}) - - assert "Run Receipt Breakdown" in report - - -def test_generate_report_detects_genuine_all_zero_receipt(tmp_path): - """A run whose single task has token_source == "receipt" but every - receipt field is legitimately 0 must still trigger the "Run Receipt - Breakdown" section — field truthiness alone would wrongly treat this as - "no receipt data" (cuga-eval#182 CodeRabbit review).""" - results = [ - { - "task_name": "t1", - "success": True, - "total_tokens": 0, - "token_source": "receipt", - "input_tokens": 0, - "output_tokens": 0, - "cache_read_tokens": 0, - "reasoning_tokens": 0, - "tool_call_count": 0, - "llm_time_s": 0.0, - "tool_time_s": 0.0, - "wall_time_s": 0.0, - } - ] - result_file = _write_sdk_result_file(tmp_path, results) - report = generate_report({"gpt-oss-120b": [result_file]}) - eval_report = generate_eval_report(result_file) - - assert "Run Receipt Breakdown" in report - assert "Run Receipt Breakdown" in eval_report - - -def test_generate_report_shows_dash_for_config_without_receipt_data(tmp_path): - """A/B comparison between a baseline bundle predating the receipt feature - (no receipt fields at all) and a new one that has them: the baseline's row - in the Run Receipt Breakdown table must render "--", not "0" (cuga-eval#95 - final review finding — a config with no receipt data looked identical to - one that measured real zeros).""" - with_receipt = [ - { - "task_name": "t1", - "success": True, - "total_tokens": 150, - "token_source": "receipt", - "input_tokens": 100, - "output_tokens": 50, - "cache_read_tokens": 20, - "reasoning_tokens": 5, - "tool_call_count": 2, - "llm_time_s": 1.5, - "tool_time_s": 0.5, - "wall_time_s": 2.5, - } - ] - without_receipt = [{"task_name": "t1", "success": True, "total_tokens": 150}] - - file_with = _write_sdk_result_file(tmp_path, with_receipt, name="with.json") - file_without = _write_sdk_result_file(tmp_path, without_receipt, name="without.json") - - report = generate_report({"model-with-receipt": [file_with], "model-without-receipt": [file_without]}) - - assert "Run Receipt Breakdown" in report - - breakdown = report.split("Run Receipt Breakdown", 1)[1] - with_line = next(line for line in breakdown.splitlines() if "model-with-receipt" in line) - without_line = next(line for line in breakdown.splitlines() if "model-without-receipt" in line) - - assert "100" in with_line - assert "--" not in with_line - - assert "--" in without_line - assert "0" not in without_line - - -# --------------------------------------------------------------------------- -# Per-task Input/Output/Reasoning columns (cuga-eval#95 follow-up). -# -# These are distinct from the "Run Receipt Breakdown" summary section tested -# above: the columns added here appear unconditionally on the genuinely -# per-task tables (Per-Task Results in generate_eval_report, Per-Task Details -# in generate_report), the same way `Cache Tokens` already does — they are -# not gated behind "does this result carry receipt data". -# --------------------------------------------------------------------------- - - -def test_eval_report_grouped_per_task_table_shows_input_output_reasoning(tmp_path): - """generate_eval_report's grouped (M3), non-Vakra per-task table gains - Input/Output/Reasoning columns with correct per-task values.""" - result_file = _m3_run( - tmp_path, - "m3_receipt.json", - { - "uuid-a": {"m3_task_id": 2, "domain": "hockey", "task_number": 1, "success": True}, - "uuid-b": {"m3_task_id": 2, "domain": "hockey", "task_number": 2, "success": False}, - }, - ) - # _m3_run doesn't plumb receipt fields, so patch them into the raw file. - payload = json.loads(Path(result_file).read_text()) - payload["results"][0].update({"input_tokens": 111, "output_tokens": 22, "reasoning_tokens": 3}) - payload["results"][1].update({"input_tokens": 444, "output_tokens": 55, "reasoning_tokens": 6}) - Path(result_file).write_text(json.dumps(payload)) - - report = generate_eval_report(result_file) - - assert ( - "| Task | Domain | # | Result | Tokens | Cost | LLM Calls | Cache Tokens " - "| Input | Output | Reasoning | Duration | Steps |" in report - ) - assert "| 111 | 22 | 3 |" in report - assert "| 444 | 55 | 6 |" in report - - -def test_eval_report_legacy_per_task_table_shows_input_output_reasoning(tmp_path): - """generate_eval_report's non-grouped/legacy per-task table (e.g. AppWorld) - gains Input/Output/Reasoning columns with correct per-task values.""" - results = [ - { - "task_name": "t1", - "success": True, - "total_tokens": 150, - "input_tokens": 100, - "output_tokens": 50, - "reasoning_tokens": 5, - } - ] - report = generate_eval_report(_write_sdk_result_file(tmp_path, results)) - - assert ( - "| Task | Result | Tokens | Cost | LLM Calls | Cache Tokens " - "| Input | Output | Reasoning | Duration | Steps |" in report - ) - assert "| 100 | 50 | 5 |" in report - - -def test_eval_report_per_task_table_shows_dash_when_receipt_fields_absent(tmp_path): - """Legacy/non-receipt result files (no input_tokens/output_tokens/ - reasoning_tokens at all) still render the per-task table without crashing; - those three receipt-only columns render `--` (never measured), not a - misleading 0 (Sergey review, PR #182 follow-up).""" - results = [{"task_name": "t1", "success": True, "total_tokens": 150}] - report = generate_eval_report(_write_sdk_result_file(tmp_path, results)) - - assert ( - "| Task | Result | Tokens | Cost | LLM Calls | Cache Tokens " - "| Input | Output | Reasoning | Duration | Steps |" in report - ) - assert "| -- | -- | -- |" in report - - -def test_stats_for_task_computes_mean_input_output_reasoning(): - task_runs = [ - {"token_source": "receipt", "input_tokens": 100, "output_tokens": 20, "reasoning_tokens": 2}, - {"token_source": "receipt", "input_tokens": 200, "output_tokens": 40, "reasoning_tokens": 6}, - ] - stats = _stats_for_task(task_runs) - - assert stats["mean_input"] == 150 - assert stats["mean_output"] == 30 - assert stats["mean_reasoning"] == 4 - - -def test_compare_report_per_task_details_shows_input_output_reasoning_and_average(tmp_path): - """generate_report's Per-Task Details table gains Input/Output/Reasoning - columns, correctly averaged per task and in the AVERAGE row.""" - results_r1 = [ - { - "task_name": "t1", - "success": True, - "total_tokens": 150, - "token_source": "receipt", - "input_tokens": 100, - "output_tokens": 20, - }, - { - "task_name": "t2", - "success": True, - "total_tokens": 150, - "token_source": "receipt", - "input_tokens": 300, - "output_tokens": 60, - }, - ] - results_r2 = [ - { - "task_name": "t1", - "success": True, - "total_tokens": 150, - "token_source": "receipt", - "input_tokens": 200, - "output_tokens": 40, - }, - { - "task_name": "t2", - "success": True, - "total_tokens": 150, - "token_source": "receipt", - "input_tokens": 300, - "output_tokens": 60, - }, - ] - run1 = _write_sdk_result_file(tmp_path, results_r1, name="r1.json") - run2 = _write_sdk_result_file(tmp_path, results_r2, name="r2.json") - - report = generate_report({"gpt-oss:cuga": [run1, run2]}) - - assert "Input" in report - assert "Output" in report - assert "Reason" in report - # t1: mean input = (100+200)/2 = 150.0, mean output = (20+40)/2 = 30.0. - # t2: mean input = 300.0, mean output = 60.0 (constant across both runs). - # AVERAGE row: mean input across tasks = (150+300)/2 = 225.0, mean output = (30+60)/2 = 45.0. - lines = report.splitlines() - t1_line = next(ln for ln in lines if ln.strip().startswith("t1 ")) - t2_line = next(ln for ln in lines if ln.strip().startswith("t2 ")) - avg_line = next(ln for ln in lines if "AVERAGE" in ln) - assert "150.0" in t1_line - assert "300.0" in t2_line - assert "225.0" in avg_line - assert "45.0" in avg_line - - -def test_eval_report_appworld_task_results_shape_shows_dashes_for_receipt_only_fields(tmp_path): - """Legacy AppWorld `task_results`-shaped result files (still produced today by - appworld_eval.py / appworld_eval_react.py / appworld_eval_codeact.py, parsed by - `_parse_appworld_results` rather than `_parse_sdk_results`) carry no - input_tokens/output_tokens/reasoning_tokens on their per-task dicts -- those - are receipt-only fields written solely by sdk_eval_helpers.py. They render - `--` (never measured), distinct from Cache Tokens, which is a genuinely - universal field (populated by every evaluator whenever Langfuse succeeds or - a receipt exists) and legitimately 0 here, not absent (Sergey review, PR - #182 follow-up).""" - result_file = _appworld_eval_run( - tmp_path, - "appworld_no_receipt.json", - { - "t1": { - "success": True, - "total_tokens": 150, - "total_llm_calls": 3, - "cache_input_tokens": 0, - "full_execution_time": 5.0, - "steps": 4, - } - }, - ) - - report = generate_eval_report(result_file) - - assert ( - "| Task | Result | Tokens | Cost | LLM Calls | Cache Tokens " - "| Input | Output | Reasoning | Duration | Steps |" in report - ) - row = next(ln for ln in report.splitlines() if ln.startswith("| t1 ")) - # Cache Tokens is a real measured 0; Input/Output/Reasoning were never - # measured for this result shape and must render `--`, not a misleading 0. - assert "| 0 | -- | -- | -- |" in row - - -def test_parse_appworld_results_preserves_all_eight_receipt_fields(): - """_parse_appworld_results must mirror _parse_sdk_results' full 8-field - Run Receipt handling (cache_read_tokens, tool_call_count, llm_time_s, - tool_time_s, wall_time_s), not just input/output/reasoning tokens -- - otherwise a receipt-producing AppWorld harness would have those 5 fields - silently truncated to zero in both the per-task dict and the run-level - totals (CodeRabbit review, PR #182).""" - data = { - "tasks_total": 1, - "tasks_completed": 1, - "task_results": { - "t1": { - "success": True, - "total_tokens": 150, - "input_tokens": 100, - "output_tokens": 50, - "cache_read_tokens": 20, - "reasoning_tokens": 5, - "tool_call_count": 2, - "llm_time_s": 1.5, - "tool_time_s": 0.5, - "wall_time_s": 2.5, - "token_source": "receipt", - } - }, - } - - parsed = _parse_appworld_results(data) - - task = parsed["tasks"]["t1"] - assert task["cache_read_tokens"] == 20 - assert task["tool_call_count"] == 2 - assert task["llm_time_s"] == 1.5 - assert task["tool_time_s"] == 0.5 - assert task["wall_time_s"] == 2.5 - assert parsed["cache_read_tokens"] == 20 - assert parsed["tool_call_count"] == 2 - assert parsed["llm_time_s"] == 1.5 - assert parsed["tool_time_s"] == 0.5 - assert parsed["wall_time_s"] == 2.5 - assert parsed["has_receipt_data"] is True diff --git a/benchmarks/m3/eval.sh b/benchmarks/m3/eval.sh index 1c79bb7c..eef2bb31 100755 --- a/benchmarks/m3/eval.sh +++ b/benchmarks/m3/eval.sh @@ -263,8 +263,11 @@ create_bundle() { # want the bundle, so don't let `set -e` abort here). local report_tmp report_tmp=$(mktemp /tmp/m3_eval_report_XXXXXX) - uv run --no-sync python -m benchmarks.helpers.compare_report eval \ - --result-file "$latest_result" --output "$report_tmp" || \ + local report_args=(eval --result-file "$latest_result" --output "$report_tmp") + if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then + report_args+=(--for-pr-comment --stdout-markdown) + fi + uv run --no-sync python -m benchmarks.helpers.compare_report "${report_args[@]}" || \ echo -e "${YELLOW:-}Report generation failed — bundling without report.${NC:-}" local bundle_args=(assemble --benchmark m3 @@ -359,16 +362,12 @@ REGISTRY_PORT="${REGISTRY_PORT:-${DYNACONF_SERVER_PORTS__REGISTRY:-8001}}" export REGISTRY_PORT export DYNACONF_SERVER_PORTS__REGISTRY="$REGISTRY_PORT" -# Per-run token/timing receipt from CugaAgent.invoke() (cuga-agent#467), -# preferred over the Langfuse HTTP fetch when present (cuga-eval#95). -export DYNACONF_ADVANCED_FEATURES__RUN_RECEIPT=true - # Capture the cuga-agent checkout's git state now, before the eval run starts # — not at bundle-assembly time (after the run finishes). If the checkout is # shared (e.g. someone switches branches in it for unrelated work) while a # long run is in flight, a live git query at the end would silently mislabel # the bundle with whatever happens to be checked out by then. -CUGA_REPO_PATH_RESOLVED="$(resolve_cuga_repo_path)" +CUGA_REPO_PATH_RESOLVED="${CUGA_REPO_PATH:-$HOME/workspace/cuga-agent}" CUGA_GIT_INFO_JSON="" if [ -d "$CUGA_REPO_PATH_RESOLVED" ]; then _cuga_commit=$(git -C "$CUGA_REPO_PATH_RESOLVED" rev-parse --short HEAD 2>/dev/null || echo "") diff --git a/benchmarks/oak_health_insurance/eval.sh b/benchmarks/oak_health_insurance/eval.sh index 747c4b7d..de7c3288 100755 --- a/benchmarks/oak_health_insurance/eval.sh +++ b/benchmarks/oak_health_insurance/eval.sh @@ -242,8 +242,11 @@ if [ $EVAL_EXIT -eq 0 ]; then # Generate eval report REPORT_TMP=$(mktemp /tmp/oak_eval_report_XXXXXX) - uv run --no-sync python -m benchmarks.helpers.compare_report eval \ - --result-file "$LATEST_RESULT" --output "$REPORT_TMP" + REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then + REPORT_ARGS+=(--for-pr-comment --stdout-markdown) + fi + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" BUNDLE_ARGS=(assemble --benchmark oak_health_insurance --result-files "$LATEST_RESULT" diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index 5f7ed5c3..8c633e13 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -5,7 +5,7 @@ This folder contains the scripts used to set up and run PR-triggered regression ## Files - `setup-pr-regression-workflow.sh`: bootstraps the self-hosted GitHub Actions runner VM. -- `run-pr-regression-eval.sh`: parses `/run-pr-eval` PR comments and runs the requested benchmark using the `cuga-eval/scripts/eval.sh` script. +- `run-pr-regression-eval.sh`: consumes validated `/run-pr-eval` parameters and runs the requested benchmark using the `cuga-eval/scripts/eval.sh` script. ## Setup The PR Regression Flow @@ -112,10 +112,10 @@ Provider defaults used by `run-pr-regression-eval.sh`: Create a comment on a pull request in `cuga-agent`: ```text -/run-pr-eval +/run-pr-eval sha= ``` -By default, this runs: +The `sha` value must be the current 40-character PR head commit SHA. The workflow validates it before checkout and does not forward it to the evaluation process. By default, the forwarded evaluation runs: - `benchmark=appworld` - `agent=react` @@ -130,72 +130,101 @@ For `provider=litellm`, the default model changes to `aws/gpt-oss-120b` unless ` The command accepts whitespace-separated `key=value` parameters: +- `sha`: required by the GitHub workflow; must be the current 40-character PR head SHA. - `model_name`: model name to evaluate. - `task_id`: one task ID, or comma-separated task IDs. - `task_ids`: alias for `task_id`. +- `eval_key`: AppWorld task group from `benchmarks/appworld/eval_config.toml`, such as `test_easy`, `test_med`, or `test_hard`. - `benchmark`: `appworld` or `m3`; defaults to `appworld`. - `num_tasks`: positive integer; defaults to `4`. - `agent`: `react`, `cuga`, or `codeact`; defaults to `react`. - `provider`: `rits` or `litellm`; defaults to `rits`. -Unsupported parameters cause the workflow to fail early with a list of supported parameters. +Unsupported parameters cause the workflow or runner script to fail early with a list of supported parameters. + +The workflow parser tolerates spaces around `=`, markdown code ticks, mixed-case values for `agent`, `benchmark`, and `provider`, and spaces after commas in task lists. For example, `provider = litellm` and `task_id = 9aae7da_1, 365e0a3_1` are accepted. The runner script also accepts sanitized CLI-style arguments such as `--benchmark appworld --agent react --provider rits`. + +Useful AppWorld `eval_key` values: + +- `test_easy` +- `test_med` +- `test_hard` +- `test_challenge_easy` +- `test_challenge_med` +- `test_challenge_hard` +- `test_normal_easy` +- `test_normal_med` +- `test_normal_hard` ## Examples Run the default AppWorld React-agent evaluation on RITS: ```text -/run-pr-eval +/run-pr-eval sha= ``` Run one AppWorld task with the React agent on RITS: ```text -/run-pr-eval provider=rits agent=react benchmark=appworld task_id=9aae7da_1 num_tasks=1 +/run-pr-eval sha= provider=rits agent=react benchmark=appworld task_id=9aae7da_1 num_tasks=1 ``` Run one AppWorld task with the CUGA agent on LiteLLM: ```text -/run-pr-eval provider=litellm agent=cuga benchmark=appworld task_id=9aae7da_1 num_tasks=1 +/run-pr-eval sha= provider=litellm agent=cuga benchmark=appworld task_id=9aae7da_1 num_tasks=1 ``` Run AppWorld with two explicit tasks: ```text -/run-pr-eval benchmark=appworld task_id=9aae7da_1,365e0a3_1 num_tasks=2 +/run-pr-eval sha= benchmark=appworld task_id=9aae7da_1,365e0a3_1 num_tasks=2 +``` + +Run AppWorld easy tasks from `eval_config.toml`: + +```text +/run-pr-eval sha= benchmark=appworld eval_key=test_easy +``` + +Run AppWorld medium tasks from `eval_config.toml`: + +```text +/run-pr-eval sha= benchmark=appworld eval_key=test_med +``` + +Run AppWorld hard tasks from `eval_config.toml`: + +```text +/run-pr-eval sha= benchmark=appworld eval_key=test_hard ``` Run M3 with the default React agent: ```text -/run-pr-eval benchmark=m3 num_tasks=1 +/run-pr-eval sha= benchmark=m3 num_tasks=1 ``` Run M3 with the CUGA agent and LiteLLM: ```text -/run-pr-eval benchmark=m3 agent=cuga provider=litellm num_tasks=1 +/run-pr-eval sha= benchmark=m3 agent=cuga provider=litellm num_tasks=1 ``` Override the model: ```text -/run-pr-eval provider=litellm agent=cuga benchmark=appworld model_name=aws/glm-5 task_id=9aae7da_1 num_tasks=1 +/run-pr-eval sha= provider=litellm agent=cuga benchmark=appworld model_name=aws/glm-5 task_id=9aae7da_1 num_tasks=1 ``` ## Output -The workflow posts a PR comment containing collapsible sections for: - -- requested parameters; -- run metadata; -- evaluation configuration; -- evaluation report; -- run summary. +The workflow posts a PR comment containing: -The full output is also uploaded as a GitHub Actions artifact and saved on the VM under: +- trusted run metadata; +- requested command and SHA; +- the evaluation report markdown emitted by the PR regression runner; +- a link to the uploaded workflow artifact. -```text -~/pr-regression-testing/logs/ -``` +The full output is also uploaded as a GitHub Actions artifact. During the workflow run, logs are written under the checked-out workspace in `pr-eval-logs/`. diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index 9660552b..c9db5d01 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -63,7 +63,21 @@ AGENT="${AGENT:-$DEFAULT_AGENT}" PROVIDER="${PROVIDER:-$DEFAULT_PROVIDER}" MODEL_NAME_FROM_COMMENT=false +usage_error() { + local message="$1" + echo "ERROR: ${message}" + echo "Supported parameters: model_name, task_id, task_ids, eval_key, benchmark, num_tasks, agent, provider, sha" + close_details + echo "######## REPORT END ########" + exit 2 +} + # Parse whitespace-separated key=value parameters from the PR comment. +# Common comment formatting is normalized first, so these are equivalent: +# provider=litellm +# provider = litellm +# provider= litellm +# provider =litellm # Supported aliases: # model_name=... # task_id=id1,id2 @@ -73,11 +87,90 @@ MODEL_NAME_FROM_COMMENT=false # num_tasks=... # agent=react|cuga|codeact # provider=rits|litellm -COMMENT_BODY="${1:-}" +COMMAND_TOKENS=() +if [[ $# -gt 0 ]]; then + while [[ $# -gt 0 ]]; do + case "$1" in + /run-pr-eval) + shift + ;; + --model-name|--model_name) + [[ -n "${2:-}" && "${2:-}" != --* ]] || { + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "$1 requires a value." + } + COMMAND_TOKENS+=("model_name=$2") + shift 2 + ;; + --task-id|--task-ids|--task|--tasks) + [[ -n "${2:-}" && "${2:-}" != --* ]] || { + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "$1 requires a value." + } + COMMAND_TOKENS+=("task_ids=$2") + shift 2 + ;; + --eval-key|--eval_key) + [[ -n "${2:-}" && "${2:-}" != --* ]] || { + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "$1 requires a value." + } + COMMAND_TOKENS+=("eval_key=$2") + shift 2 + ;; + --benchmark|--num-tasks|--num_tasks|--agent|--agent-type|--agent_type|--provider|--sha) + key="${1#--}" + key="${key//-/_}" + [[ -n "${2:-}" && "${2:-}" != --* ]] || { + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "$1 requires a value." + } + COMMAND_TOKENS+=("${key}=$2") + shift 2 + ;; + *=*) + COMMAND_TOKENS+=("$1") + shift + ;; + *) + if [[ $# -eq 1 && "$1" == *"/run-pr-eval"* ]]; then + COMMENT_BODY="$1" + shift + else + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "Unsupported argument: $1" + fi + ;; + esac + done +fi -# Remove Windows carriage returns and newlines. +if [[ ${#COMMAND_TOKENS[@]} -gt 0 ]]; then + COMMENT_BODY="/run-pr-eval ${COMMAND_TOKENS[*]}" +else + COMMENT_BODY="${COMMENT_BODY:-}" +fi + +# Remove Windows carriage returns, newlines, tabs, and markdown code ticks. COMMENT_BODY="${COMMENT_BODY//$'\r'/ }" COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" +COMMENT_BODY="${COMMENT_BODY//$'\t'/ }" +COMMENT_BODY="${COMMENT_BODY//\`/ }" +if [[ "${COMMENT_BODY}" == *"/run-pr-eval"* ]]; then + COMMENT_BODY="/run-pr-eval ${COMMENT_BODY#*/run-pr-eval}" +fi + +# Normalize spaces around "=" for supported keys, and tolerate spaces after +# commas in comma-separated values such as task_id=a, b. +COMMENT_BODY="$( + printf '%s' "${COMMENT_BODY}" | + sed -E 's/(^|[[:space:]])(model_name|task_id|task_ids|eval_key|eval-key|benchmark|num_tasks|agent|agent_type|provider|sha)[[:space:]]*=[[:space:]]*/\1\2=/g; s/,[[:space:]]+/,/g' +)" read -r -a TOKENS <<< "${COMMENT_BODY}" echo "######## REPORT START ########" @@ -117,31 +210,28 @@ for token in "${TOKENS[@]}"; do PROVIDER="${token#provider=}" ;; + sha=*) + ;; + "") ;; *) - echo "ERROR: Unsupported parameter: ${token}" - echo "Supported parameters: model_name, task_id, task_ids, eval_key, benchmark, num_tasks, agent, provider" - close_details - echo "######## REPORT END ########" - exit 2 + usage_error "Unsupported parameter: ${token}" ;; esac done if [[ -z "${MODEL_NAME}" ]]; then - echo "ERROR: model_name cannot be empty." - close_details - echo "######## REPORT END ########" - exit 2 + usage_error "model_name cannot be empty." fi -if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then - echo "ERROR: num_tasks must be a positive integer." - close_details - echo "######## REPORT END ########" - exit 2 +if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]]; then + usage_error "num_tasks must be a positive integer." +fi +NUM_TASKS=$((10#${NUM_TASKS})) +if (( NUM_TASKS < 1 )); then + usage_error "num_tasks must be a positive integer." fi AGENT="$(printf '%s' "${AGENT}" | tr '[:upper:]' '[:lower:]')" @@ -269,6 +359,7 @@ fi export AGENT_SETTING_CONFIG export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" export MODEL_NAME +export PR_EVAL_FOR_PR_COMMENT="${PR_EVAL_FOR_PR_COMMENT:-true}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [[ -z "${EVAL_REPO:-}" ]]; then diff --git a/scripts/pr-regression-testing/setup-pr-regression-workflow.sh b/scripts/pr-regression-testing/setup-pr-regression-workflow.sh index b3c6f730..c9bd9694 100644 --- a/scripts/pr-regression-testing/setup-pr-regression-workflow.sh +++ b/scripts/pr-regression-testing/setup-pr-regression-workflow.sh @@ -290,7 +290,7 @@ echo "Layout:" echo " ${ROOT_DIR}/" echo " ├── cuga-agent/" echo " ├── cuga-eval/" -echo " │ └── scripts/run-eval.sh" +echo " │ └── scripts/pr-regression-testing/run-pr-regression-eval.sh" echo " └── github-runner/" echo " └── _work/" echo @@ -306,3 +306,6 @@ echo " runs-on: [self-hosted, linux, ${RUNNER_LABELS}]" echo echo "Workflow script:" echo " bash \"${RUN_EVAL_SCRIPT}\" \"\${COMMENT_BODY}\"" +echo +echo "PR command format:" +echo " /run-pr-eval sha= benchmark=appworld provider=rits agent=react" From 335d01884af734f21a5a191ce1e2271842ba829c Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 17 Sep 2026 00:18:18 -0400 Subject: [PATCH 32/40] Revert merge changes. --- benchmarks/appworld/eval.sh | 129 ++++++++++++++++++++++++++++++++---- 1 file changed, 116 insertions(+), 13 deletions(-) diff --git a/benchmarks/appworld/eval.sh b/benchmarks/appworld/eval.sh index 54e77d49..c0e27f70 100755 --- a/benchmarks/appworld/eval.sh +++ b/benchmarks/appworld/eval.sh @@ -34,6 +34,10 @@ for arg in "$@"; do echo " --model-profile Model profile (for bundle metadata)" echo " --agent Agent to run (cuga, react, codeact; default: cuga)" echo " --eval-key Task group key in eval_config.toml (e.g. test_med); recorded in bundle metadata" + echo " --leaderboard Tag this run for official AppWorld leaderboard submission (implies --sdk)" + echo " --force-retry Re-run listed tasks even if a clean partial already exists" + echo " --dry-run Print the evaluator command that would run and exit without starting servers" + echo " --status Show run status; also prints leaderboard status when the workspace has a leaderboard block" echo "" echo "Examples:" echo " ./eval.sh # Default evaluation (cuga)" @@ -48,6 +52,7 @@ done APPWORLD_PID="" REGISTRY_PID="" +APPWORLD_MCP_EXPANDED="" # Parse bundle / model-profile / sdk flags before any server startup so # --status / --stop / --background can short-circuit without side effects. @@ -68,6 +73,9 @@ while [[ $# -gt 0 ]]; do --restart) RESTART=true; shift ;; --status) STATUS=true; shift ;; --agent) AGENT="$2"; shift 2 ;; + --leaderboard) LEADERBOARD="$2"; LEADERBOARD_REQUESTED=true; PASSTHROUGH_ARGS+=(--leaderboard "$2"); USE_SDK=true; shift 2 ;; + --force-retry) FORCE_RETRY=true; PASSTHROUGH_ARGS+=(--force-retry); USE_SDK=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; --verbose|-v|--quiet|-q) PASSTHROUGH_ARGS+=("$1"); shift ;; --task) PASSTHROUGH_ARGS+=("--task-id"); shift while [[ $# -gt 0 && ! "$1" =~ ^-- ]]; do @@ -77,6 +85,59 @@ while [[ $# -gt 0 ]]; do esac done +# --leaderboard and --force-retry are defined only by eval_appworld_sdk.py. The +# other evaluators use argparse.parse_args(), so passing either through would +# abort with "unrecognized arguments" *after* the AppWorld and registry servers +# have already booted. Reject it up front instead. Keyed on whether the flag was +# *passed*, not on its value — `--leaderboard ""` must not slip past the guard. +for sdk_only in ${LEADERBOARD_REQUESTED:+--leaderboard} ${FORCE_RETRY:+--force-retry}; do + if [[ "${AGENT:-cuga}" == "codeact" || "${AGENT:-cuga}" == "react" ]]; then + echo "Error: ${sdk_only} is SDK-only (CUGA lite). --agent ${AGENT} does not accept ${sdk_only}." >&2 + echo "Drop --agent, or drop ${sdk_only}." >&2 + exit 2 + fi +done + +# Single source of truth for the evaluator command. Both --dry-run and the live +# run below call this, so what --dry-run prints is exactly what would execute — +# the two used to be separate copies and had already drifted on the --agent flags. +# Sets EVAL_CMD (array) and EVAL_LABEL. Reads AGENT, USE_SDK, PASSTHROUGH_ARGS. +build_eval_command() { + EVAL_CMD=(uv run --no-sync python -m) + if [ "${AGENT:-cuga}" = "codeact" ]; then + EVAL_CMD+=(benchmarks.appworld.appworld_eval_codeact --agent codeact) + EVAL_LABEL="Using CodeAct agent (appworld_eval_codeact.py)" + elif [ "${AGENT:-cuga}" = "react" ]; then + EVAL_CMD+=(benchmarks.appworld.appworld_eval_react --agent react) + EVAL_LABEL="Using React agent (appworld_eval_react.py)" + elif [[ "$USE_SDK" == "true" ]]; then + EVAL_CMD+=(benchmarks.appworld.eval_appworld_sdk) + EVAL_LABEL="Using SDK evaluator (eval_appworld_sdk.py)" + else + EVAL_CMD+=(benchmarks.appworld.appworld_eval --agent "${AGENT:-cuga}") + EVAL_LABEL="Using default evaluator (appworld_eval.py)" + fi + EVAL_CMD+=("${PASSTHROUGH_ARGS[@]+"${PASSTHROUGH_ARGS[@]}"}") +} + +if [[ "${DRY_RUN:-false}" == "true" ]]; then + # Named experiments / resume use the SDK evaluator on the live path + # (see experiment_workspace_requested below). Apply the same rule here + # so --dry-run --experiment prints the module that would actually run. + if experiment_workspace_requested && [[ "${NO_BUNDLE:-false}" != "true" ]]; then + USE_SDK=true + fi + build_eval_command + echo "DISPATCH: ${EVAL_CMD[*]}" + exit 0 +fi + +if [[ "${STATUS:-false}" == "true" ]]; then + if bd=$(resolve_lifecycle_bundle_dir "appworld" 2>/dev/null) && grep -q '"leaderboard"' "$bd/metadata.json" 2>/dev/null; then + uv run --no-sync python -m benchmarks.appworld.leaderboard status --bundle-dir "$bd" || echo -e "${YELLOW:-}Warning: leaderboard status failed (see above)${NC:-}" + fi +fi + if handle_eval_lifecycle "appworld" "$0" "${PASSTHROUGH_ARGS[@]}"; then exit 0 fi @@ -106,6 +167,8 @@ cleanup() { # after the evaluator exits; clean it up here instead of a dedicated trap so # it can't leak on SIGKILL/early-abort without clobbering this EXIT trap. [ -n "${RUN_MARKER:-}" ] && rm -f "$RUN_MARKER" + # Per-run expanded MCP servers config (only created for non-default APIS ports). + [ -n "${APPWORLD_MCP_EXPANDED:-}" ] && rm -f "$APPWORLD_MCP_EXPANDED" exit $exit_code } @@ -129,10 +192,33 @@ APPWORLD_APIS_PORT="${APPWORLD_APIS_PORT:-${DYNACONF_SERVER_PORTS__APIS_URL:-911 export APPWORLD_APIS_PORT export DYNACONF_SERVER_PORTS__APIS_URL="$APPWORLD_APIS_PORT" +# Per-run token/timing receipt from CugaAgent.invoke() (cuga-agent#467). +# Only the --sdk evaluator (eval_appworld_sdk.py) calls agent.invoke() and can +# use it; the default graph-based appworld_eval.py ignores it harmlessly. +export DYNACONF_ADVANCED_FEATURES__RUN_RECEIPT=true + # Capture console output to a log file for reproducibility bundles CONSOLE_LOG="/tmp/appworld_console.log" exec > >(tee "$CONSOLE_LOG") 2>&1 +# mcp_servers_appworld.yaml hard-codes the app API server as localhost:9111 (one +# per registered app), so overriding APPWORLD_APIS_PORT alone moves the server but +# leaves the registry fetching tool specs from 9111 — it then loads 0 tools and the +# run aborts (#113). When a non-default port is in use, expand the config into a +# per-run temp copy with the real port and point MCP_SERVERS_FILE at it. This +# mirrors benchmarks/m3/eval_m3.py, which expands its registry YAML per run. +# (Done after the exec/tee above so the confirmation echo below lands in +# CONSOLE_LOG and is captured by reproducibility bundles, not just the terminal.) +if [ "$APPWORLD_APIS_PORT" != "9111" ]; then + APPWORLD_MCP_SRC="$SCRIPT_DIR/mcp_servers_appworld.yaml" + if [ -f "$APPWORLD_MCP_SRC" ]; then + APPWORLD_MCP_EXPANDED="$(mktemp "${TMPDIR:-/tmp}/appworld_mcp_servers.XXXXXX")" + sed "s#localhost:9111#localhost:${APPWORLD_APIS_PORT}#g" "$APPWORLD_MCP_SRC" > "$APPWORLD_MCP_EXPANDED" + export MCP_SERVERS_FILE="$APPWORLD_MCP_EXPANDED" + echo -e "${GREEN:-}✓${NC:-} Templated MCP servers config for APIS port ${APPWORLD_APIS_PORT}" + fi +fi + echo -e "${BLUE:-}╔════════════════════════════════════════════════════════════╗${NC:-}" echo -e "${BLUE:-}║ AppWorld Benchmark Evaluation ║${NC:-}" echo -e "${BLUE:-}╚════════════════════════════════════════════════════════════╝${NC:-}" @@ -215,6 +301,10 @@ if prepare_experiment_workspace "appworld"; then mark_run_state_started fi +if [[ -n "${LEADERBOARD_REQUESTED:-}" && -z "${WORKSPACE_BUNDLE_DIR:-}" ]]; then + echo -e "${YELLOW:-}Warning: --leaderboard without --experiment/--resume-experiment: no workspace, so resume, retry keys and the official evaluate are unavailable for this run${NC:-}" +fi + # Apply --model-profile after load_env + arg parsing. common.sh is sourced # unconditionally above; hard-fail loudly if that ever stops being true # instead of silently skipping model-profile application. @@ -242,19 +332,9 @@ RUN_MARKER=$(mktemp "${TMPDIR:-/tmp}/appworld_run_marker.XXXXXX") # Capture the evaluator's exit code instead of letting `set -e` abort here — an # abort would skip both bundling and the failure banner below. set +e -if [ "${AGENT:-cuga}" = "codeact" ]; then - echo -e "${BLUE:-}Using CodeAct agent (appworld_eval_codeact.py)${NC:-}" - uv run --no-sync python -m benchmarks.appworld.appworld_eval_codeact --agent codeact "${PASSTHROUGH_ARGS[@]}" -elif [ "${AGENT:-cuga}" = "react" ]; then - echo -e "${BLUE:-}Using React agent (appworld_eval_react.py)${NC:-}" - uv run --no-sync python -m benchmarks.appworld.appworld_eval_react --agent react "${PASSTHROUGH_ARGS[@]}" -elif [[ "$USE_SDK" == "true" ]]; then - echo -e "${BLUE:-}Using SDK evaluator (eval_appworld_sdk.py)${NC:-}" - uv run --no-sync python -m benchmarks.appworld.eval_appworld_sdk "${PASSTHROUGH_ARGS[@]}" -else - echo -e "${BLUE:-}Using default evaluator (appworld_eval.py)${NC:-}" - uv run --no-sync python -m benchmarks.appworld.appworld_eval --agent "${AGENT:-cuga}" "${PASSTHROUGH_ARGS[@]}" -fi +build_eval_command +echo -e "${BLUE:-}${EVAL_LABEL}${NC:-}" +"${EVAL_CMD[@]}" EVAL_EXIT=$? set -e @@ -302,6 +382,29 @@ if [ -n "$LATEST_RESULT" ] && [ "${NO_BUNDLE:-false}" != "true" ]; then fi FIN_EXTRA+=(--log-file /tmp/appworld.log --log-file /tmp/appworld_registry.log --log-file "$CONSOLE_LOG") finalize_experiment_workspace "appworld" "${FIN_EXTRA[@]}" + + # Run AFTER finalize: finalize_workspace_bundle regenerates + # /report.md from scratch (generate_eval_report), so a + # section appended before finalize would be overwritten. + if [ $EVAL_EXIT -eq 0 ] && grep -q '"leaderboard"' "$WORKSPACE_BUNDLE_DIR/metadata.json" 2>/dev/null; then + AW_EXP=$(uv run --no-sync python -c "import json,sys;print(json.load(open(sys.argv[1]))['leaderboard']['appworld_experiment'])" "$WORKSPACE_BUNDLE_DIR/metadata.json" 2>/dev/null) || AW_EXP="" + # Fall back to the eval_key this run recorded in metadata, so a + # resume that finishes without repeating --eval-key still writes + # appworld_official.json and the report.md section. + OFFICIAL_KEY="${EVAL_KEY:-}" + if [[ -z "$OFFICIAL_KEY" ]]; then + OFFICIAL_KEY=$(uv run --no-sync python -c "import json,sys;m=json.load(open(sys.argv[1]));print((m.get('run') or {}).get('eval_key') or '')" "$WORKSPACE_BUNDLE_DIR/metadata.json" 2>/dev/null) || OFFICIAL_KEY="" + fi + if [[ -z "$AW_EXP" ]]; then + echo -e "${YELLOW:-}Warning: could not read leaderboard.appworld_experiment from metadata.json; skipping official evaluate${NC:-}" + elif [[ -n "$OFFICIAL_KEY" ]]; then + echo -e "${YELLOW:-}Official AppWorld evaluate for key ${OFFICIAL_KEY}...${NC:-}" + uv run --no-sync python -m benchmarks.appworld.leaderboard evaluate "$AW_EXP" --key "$OFFICIAL_KEY" --bundle-dir "$WORKSPACE_BUNDLE_DIR" || echo -e "${YELLOW:-}Warning: official evaluate failed (see above)${NC:-}" + else + echo -e "${YELLOW:-}Note: no --eval-key for this run and none recorded in metadata.json; skipping the official evaluate.${NC:-}" + echo -e "${YELLOW:-} Run it manually: uv run --no-sync python -m benchmarks.appworld.leaderboard evaluate ${AW_EXP} --split --bundle-dir ${WORKSPACE_BUNDLE_DIR}${NC:-}" + fi + fi else echo -e "${YELLOW:-}Creating reproducibility bundle...${NC:-}" From 1d727f3a6a49d320739ea296ab48ff01b78fc7ef Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Fri, 18 Sep 2026 12:08:57 -0400 Subject: [PATCH 33/40] Rebase to main/ --- benchmarks/appworld/appworld_eval.py | 10 +- benchmarks/helpers/compare_report.py | 260 +++++++- .../helpers/tests/test_compare_report.py | 590 +++++++++++++++++- benchmarks/m3/eval.sh | 6 +- 4 files changed, 850 insertions(+), 16 deletions(-) diff --git a/benchmarks/appworld/appworld_eval.py b/benchmarks/appworld/appworld_eval.py index 8082c909..8f892663 100644 --- a/benchmarks/appworld/appworld_eval.py +++ b/benchmarks/appworld/appworld_eval.py @@ -177,6 +177,14 @@ def _get_registry_base_url() -> str: return "http://localhost:8001" +def _is_langfuse_tracing_enabled() -> bool: + env_value = os.getenv("DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING") + if env_value is not None: + return env_value.strip().lower() in {"1", "true", "yes", "on"} + + return bool(getattr(getattr(settings, "advanced_features", None), "langfuse_tracing", False)) + + async def run_agent_on_task( task_id: str, experiment_name: str = "api_cuga_agent", @@ -217,7 +225,7 @@ async def run_agent_on_task( # Without this, LangfuseCallbackHandler creates a new root trace for every # agent_loop_obj.run() call in the controller while-loop. _langfuse = None - if _LANGFUSE_AVAILABLE and _get_langfuse_client is not None: + if _is_langfuse_tracing_enabled() and _LANGFUSE_AVAILABLE and _get_langfuse_client is not None: try: _langfuse = _get_langfuse_client() _run_uid = uuid.uuid4().hex[:8] diff --git a/benchmarks/helpers/compare_report.py b/benchmarks/helpers/compare_report.py index 5b396b7a..1c1117e3 100644 --- a/benchmarks/helpers/compare_report.py +++ b/benchmarks/helpers/compare_report.py @@ -261,6 +261,16 @@ def _parse_sdk_results(data: dict) -> dict: total_cost = sum(r.get("total_cost", 0) or 0 for r in results) total_llm_calls = sum(r.get("total_llm_calls", 0) or 0 for r in results) total_cache_tokens = sum(r.get("total_cache_input_tokens", 0) or 0 for r in results) + # Run Receipt fields (cuga-eval#95 / cuga-agent#467) — absent (0) for + # results sourced from Langfuse or from benchmarks not yet opted in. + total_input_tokens = sum(r.get("input_tokens", 0) or 0 for r in results) + total_output_tokens = sum(r.get("output_tokens", 0) or 0 for r in results) + total_receipt_cache_read = sum(r.get("cache_read_tokens", 0) or 0 for r in results) + total_reasoning_tokens = sum(r.get("reasoning_tokens", 0) or 0 for r in results) + total_tool_call_count = sum(r.get("tool_call_count", 0) or 0 for r in results) + total_llm_time_s = sum(r.get("llm_time_s", 0) or 0 for r in results) + total_tool_time_s = sum(r.get("tool_time_s", 0) or 0 for r in results) + total_wall_time_s = sum(r.get("wall_time_s", 0) or 0 for r in results) tasks = {} total_duration = 0.0 @@ -277,6 +287,17 @@ def _parse_sdk_results(data: dict) -> dict: "cost": r.get("total_cost", 0) or 0, "llm_calls": r.get("total_llm_calls", 0) or 0, "cache_tokens": r.get("total_cache_input_tokens", 0) or 0, + # Receipt-only: left None (not 0) when absent so per-task tables + # render "--" via _fmt instead of a misleading 0 (PR #182 review). + "input_tokens": r.get("input_tokens"), + "output_tokens": r.get("output_tokens"), + "cache_read_tokens": r.get("cache_read_tokens", 0) or 0, + "reasoning_tokens": r.get("reasoning_tokens"), + "tool_call_count": r.get("tool_call_count", 0) or 0, + "llm_time_s": r.get("llm_time_s", 0) or 0, + "tool_time_s": r.get("tool_time_s", 0) or 0, + "wall_time_s": r.get("wall_time_s", 0) or 0, + "token_source": r.get("token_source"), "duration": dur, "steps": r.get("steps"), # AppWorld results carry a per-task difficulty band; preserved for @@ -307,7 +328,16 @@ def _parse_sdk_results(data: dict) -> dict: "cost": total_cost, "llm_calls": total_llm_calls, "cache_tokens": total_cache_tokens, + "input_tokens": total_input_tokens, + "output_tokens": total_output_tokens, + "cache_read_tokens": total_receipt_cache_read, + "reasoning_tokens": total_reasoning_tokens, + "tool_call_count": total_tool_call_count, + "llm_time_s": total_llm_time_s, + "tool_time_s": total_tool_time_s, + "wall_time_s": total_wall_time_s, "duration": total_duration if has_duration else None, + "has_receipt_data": any(t.get("token_source") == "receipt" for t in tasks.values()), "tasks": tasks, } @@ -321,6 +351,18 @@ def _parse_appworld_results(data: dict) -> dict: total_cost = sum(t.get("total_cost", 0) or 0 for t in task_results.values()) total_llm_calls = sum(t.get("total_llm_calls", 0) or 0 for t in task_results.values()) total_cache_tokens = sum(t.get("cache_input_tokens", 0) or 0 for t in task_results.values()) + # Run Receipt fields (cuga-eval#95 / cuga-agent#467) — mirrors the full + # 8-field set _parse_sdk_results carries, not just the 3 token fields, so + # a future receipt-producing AppWorld harness isn't silently truncated to + # zero on cache/tool/timing columns (CodeRabbit review, PR #182). + total_input_tokens = sum(t.get("input_tokens", 0) or 0 for t in task_results.values()) + total_output_tokens = sum(t.get("output_tokens", 0) or 0 for t in task_results.values()) + total_receipt_cache_read = sum(t.get("cache_read_tokens", 0) or 0 for t in task_results.values()) + total_reasoning_tokens = sum(t.get("reasoning_tokens", 0) or 0 for t in task_results.values()) + total_tool_call_count = sum(t.get("tool_call_count", 0) or 0 for t in task_results.values()) + total_llm_time_s = sum(t.get("llm_time_s", 0) or 0 for t in task_results.values()) + total_tool_time_s = sum(t.get("tool_time_s", 0) or 0 for t in task_results.values()) + total_wall_time_s = sum(t.get("wall_time_s", 0) or 0 for t in task_results.values()) total_duration = data.get("duration") or sum( t.get("full_execution_time", 0) or 0 for t in task_results.values() ) @@ -333,6 +375,17 @@ def _parse_appworld_results(data: dict) -> dict: "cost": t.get("total_cost", 0) or 0, "llm_calls": t.get("total_llm_calls", 0) or 0, "cache_tokens": t.get("cache_input_tokens", 0) or 0, + # Receipt-only: left None (not 0) when absent so per-task tables + # render "--" via _fmt instead of a misleading 0 (PR #182 review). + "input_tokens": t.get("input_tokens"), + "output_tokens": t.get("output_tokens"), + "cache_read_tokens": t.get("cache_read_tokens", 0) or 0, + "reasoning_tokens": t.get("reasoning_tokens"), + "tool_call_count": t.get("tool_call_count", 0) or 0, + "llm_time_s": t.get("llm_time_s", 0) or 0, + "tool_time_s": t.get("tool_time_s", 0) or 0, + "wall_time_s": t.get("wall_time_s", 0) or 0, + "token_source": t.get("token_source"), "duration": t.get("full_execution_time") or t.get("duration"), "steps": t.get("steps"), "difficulty": t.get("difficulty"), @@ -348,7 +401,16 @@ def _parse_appworld_results(data: dict) -> dict: "cost": total_cost, "llm_calls": total_llm_calls, "cache_tokens": total_cache_tokens, + "input_tokens": total_input_tokens, + "output_tokens": total_output_tokens, + "cache_read_tokens": total_receipt_cache_read, + "reasoning_tokens": total_reasoning_tokens, + "tool_call_count": total_tool_call_count, + "llm_time_s": total_llm_time_s, + "tool_time_s": total_tool_time_s, + "wall_time_s": total_wall_time_s, "duration": total_duration, + "has_receipt_data": any(t.get("token_source") == "receipt" for t in tasks.values()), "tasks": tasks, } @@ -546,6 +608,48 @@ def _aggregate_costs(tasks: dict) -> dict: } +_RECEIPT_COST_FIELDS = ( + "input_tokens", + "output_tokens", + "cache_read_tokens", + "reasoning_tokens", + "tool_call_count", + "llm_time_s", + "tool_time_s", + "wall_time_s", +) + + +def _aggregate_receipt_costs(tasks: dict) -> dict: + """Sum and average Run Receipt fields (cuga-eval#95 / cuga-agent#467) + across a dict of task dicts (as produced by ``_parse_sdk_results``). + + Uses each task's ``token_source == "receipt"`` marker (not field + truthiness) to decide which tasks actually carry receipt data — a + genuine receipt with every field at 0 must still count, and averages + must exclude legacy/non-receipt tasks from the denominator rather than + silently diluting them in. Returns every value as None (not 0) when no + task in *tasks* carries receipt data, so callers can render "--" instead + of a misleading zero for benchmarks/runs that never opted into + ``advanced_features.run_receipt``. + """ + receipt_tasks = [t for t in tasks.values() if t.get("token_source") == "receipt"] + if not receipt_tasks: + result: dict = {} + for field in _RECEIPT_COST_FIELDS: + result[f"total_{field}"] = None + result[f"avg_{field}"] = None + return result + + n = len(receipt_tasks) + result = {} + for field in _RECEIPT_COST_FIELDS: + total = sum(t.get(field, 0) or 0 for t in receipt_tasks) + result[f"total_{field}"] = total + result[f"avg_{field}"] = total / n + return result + + def _per_config_cost_stats(runs, task_filter=None) -> dict: """Mean tokens / LLM calls / duration *per task* across runs for the filtered task subset. Companion to ``_per_config_pass_stats`` so the @@ -740,6 +844,18 @@ def _stats_for_task(task_runs): "mean_tokens": _avg([r.get("tokens") for r in task_runs]), "mean_llm": _avg([r.get("llm_calls") for r in task_runs]), "mean_dur": _avg([r.get("duration") for r in task_runs]), + # Run Receipt fields (cuga-eval#95 / cuga-agent#467) — averaged only + # over runs that actually carry a receipt (token_source == "receipt"), + # so a mix of receipt and legacy/Langfuse runs for the same task + # doesn't dilute the average toward zero. _avg returns None when the + # filtered list is empty, which callers render as "--". + "mean_input": _avg([r.get("input_tokens") for r in task_runs if r.get("token_source") == "receipt"]), + "mean_output": _avg( + [r.get("output_tokens") for r in task_runs if r.get("token_source") == "receipt"] + ), + "mean_reasoning": _avg( + [r.get("reasoning_tokens") for r in task_runs if r.get("token_source") == "receipt"] + ), # Vakra scores (M3 only): mean dialogue score and mean per-judge # scores across runs, ignoring runs where a judge was skipped. "mean_match_rate": _avg([r.get("match_rate") for r in task_runs]), @@ -879,6 +995,52 @@ def _render_compare_report_sections( lines.append(fence_close()) lines.append("") + # ---- 2a-2. Run Receipt Breakdown: only rendered when at least one run + # carries Run Receipt data (cuga-eval#95 / cuga-agent#467) — bpo/oak/ + # appworld-default runs never will, so their reports are unchanged. + # Uses the run-level has_receipt_data marker (token_source == "receipt" + # on at least one task), not field truthiness, so a genuine all-zero + # receipt still counts. + any_receipt_data = any(r.get("has_receipt_data") for runs in model_data.values() for r in runs) + if any_receipt_data: + lines.append(h2("Run Receipt Breakdown")) + lines.append("") + if fence_open(): + lines.append(fence_open()) + receipt_header = ( + f"{'Configuration':<28} {'In Tok':>9} {'Out Tok':>9} {'Cache Rd':>9} " + f"{'Reason':>8} {'Tool#':>6} {'LLM(s)':>8} {'Tool(s)':>8} {'Wall(s)':>8}" + ) + lines.append(receipt_header) + lines.append("─" * len(receipt_header)) + for config_key, runs in model_data.items(): + display = _format_config_label(config_key) + receipt_runs = [r for r in runs if r.get("has_receipt_data")] + if not receipt_runs: + lines.append( + f"{display:<28} {_fmt(None):>9} {_fmt(None):>9} {_fmt(None):>9} " + f"{_fmt(None):>8} {_fmt(None):>6} {_fmt(None, 's'):>8} " + f"{_fmt(None, 's'):>8} {_fmt(None, 's'):>8}" + ) + continue + n = len(receipt_runs) + avg_in = sum(r.get("input_tokens", 0) or 0 for r in receipt_runs) / n + avg_out = sum(r.get("output_tokens", 0) or 0 for r in receipt_runs) / n + avg_cache = sum(r.get("cache_read_tokens", 0) or 0 for r in receipt_runs) / n + avg_reason = sum(r.get("reasoning_tokens", 0) or 0 for r in receipt_runs) / n + avg_tool_n = sum(r.get("tool_call_count", 0) or 0 for r in receipt_runs) / n + avg_llm_t = sum(r.get("llm_time_s", 0) or 0 for r in receipt_runs) / n + avg_tool_t = sum(r.get("tool_time_s", 0) or 0 for r in receipt_runs) / n + avg_wall_t = sum(r.get("wall_time_s", 0) or 0 for r in receipt_runs) / n + lines.append( + f"{display:<28} {_fmt(avg_in):>9} {_fmt(avg_out):>9} {_fmt(avg_cache):>9} " + f"{_fmt(avg_reason):>8} {_fmt(avg_tool_n):>6} {_fmt(avg_llm_t, 's'):>8} " + f"{_fmt(avg_tool_t, 's'):>8} {_fmt(avg_wall_t, 's'):>8}" + ) + if fence_close(): + lines.append(fence_close()) + lines.append("") + # ---- 2b. Per-group breakdowns (only when result files carry the relevant # metadata, so unrelated reports stay unchanged): # - difficulty (AppWorld) @@ -1009,7 +1171,8 @@ def _num_label(t, _m=task_meta): vakra_hdr = f"{'Dialog':>6} {'ExctM':>5} {'Answer':>6} {'Ground':>6} " if has_vakra else "" task_header = ( f"{prefix_hdr}{'Task':<{col_task_w}} {run_cols} {'Successes':>10} " - f"{'Rate':>6} {vakra_hdr}{'Tokens':>8} {'LLM':>5} {'Time':>6}" + f"{'Rate':>6} {vakra_hdr}{'Tokens':>8} {'Input':>8} {'Output':>8} {'Reason':>7} " + f"{'LLM':>5} {'Time':>6}" ) lines.append(task_header) lines.append("─" * len(task_header)) @@ -1021,6 +1184,12 @@ def _num_label(t, _m=task_meta): n_llm = 0 sum_dur = 0.0 n_dur = 0 + sum_input = 0.0 + n_input = 0 + sum_output = 0.0 + n_output = 0 + sum_reasoning = 0.0 + n_reasoning = 0 sum_match_rate = 0.0 n_match_rate = 0 sum_judge = dict.fromkeys(_JUDGE_KEYS, 0.0) @@ -1058,6 +1227,18 @@ def _num_label(t, _m=task_meta): if md is not None: sum_dur += md n_dur += 1 + mi = stats["mean_input"] + mo = stats["mean_output"] + mre = stats["mean_reasoning"] + if mi is not None: + sum_input += mi + n_input += 1 + if mo is not None: + sum_output += mo + n_output += 1 + if mre is not None: + sum_reasoning += mre + n_reasoning += 1 mr = stats["mean_match_rate"] if mr is not None: sum_match_rate += mr @@ -1086,7 +1267,8 @@ def _num_label(t, _m=task_meta): lines.append( f"{row_prefix}{task_disp:<{col_task_w}} {symbols} " f"{successes:>3}/{total:<3} {rate_pct:>5.1f}% {vakra_cols}" - f"{_fmt(mt):>8} {_fmt(ml):>5} {_fmt(md, 's'):>6}" + f"{_fmt(mt):>8} {_fmt(mi):>8} {_fmt(mo):>8} {_fmt(mre):>7} " + f"{_fmt(ml):>5} {_fmt(md, 's'):>6}" ) # AVERAGE row @@ -1097,6 +1279,9 @@ def _num_label(t, _m=task_meta): avg_tok = _fmt(sum_tokens / n_tokens) if n_tokens else "--" avg_llm = _fmt(sum_llm / n_llm) if n_llm else "--" avg_dur = _fmt(sum_dur / n_dur, "s") if n_dur else "--" + avg_input = _fmt(sum_input / n_input) if n_input else "--" + avg_output = _fmt(sum_output / n_output) if n_output else "--" + avg_reasoning = _fmt(sum_reasoning / n_reasoning) if n_reasoning else "--" lines.append("─" * len(task_header)) spacer = " ".join("──" for _ in range(n_runs)) avg_prefix = f"{'':<{cap_w}} {'':<{dom_w}} {'':>{num_w}} " if m3_mode else "" @@ -1114,7 +1299,8 @@ def _num_label(t, _m=task_meta): lines.append( f"{avg_prefix}{'AVERAGE':<{col_task_w}} {spacer} " f"{avg_successes:>3.1f}/{n_runs:<3} {avg_rate:>5.1f}% {avg_vakra_cols}" - f"{avg_tok:>8} {avg_llm:>5} {avg_dur:>6}" + f"{avg_tok:>8} {avg_input:>8} {avg_output:>8} {avg_reasoning:>7} " + f"{avg_llm:>5} {avg_dur:>6}" ) lines.append("") cons = (all_pass / maj_pass) if maj_pass else None @@ -1344,6 +1530,7 @@ def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment parsed = parse_result_file(result_file) rows, grouped = _bucket_m3_tasks(parsed["tasks"]) cost = _aggregate_costs(parsed["tasks"]) + receipt_cost = _aggregate_receipt_costs(parsed["tasks"]) h1 = (lambda s: f"# {s}") if markdown else (lambda s: f"\n{s}\n{'=' * len(s)}") h2 = (lambda s: f"## {s}") if markdown else (lambda s: f"\n{s}\n{'-' * len(s)}") @@ -1380,6 +1567,33 @@ def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment _append_content_filter_summary(lines, parsed["tasks"], markdown=markdown) lines.append("") + if receipt_cost["total_input_tokens"] is not None: + lines.append(h2("Run Receipt Breakdown")) + lines.append("") + receipt_rows = [ + ("Input Tokens", "total_input_tokens", "avg_input_tokens", ","), + ("Output Tokens", "total_output_tokens", "avg_output_tokens", ","), + ("Cache Read Tokens", "total_cache_read_tokens", "avg_cache_read_tokens", ","), + ("Reasoning Tokens", "total_reasoning_tokens", "avg_reasoning_tokens", ","), + ("Tool Calls", "total_tool_call_count", "avg_tool_call_count", ","), + ("LLM Time", "total_llm_time_s", "avg_llm_time_s", "s"), + ("Tool Time", "total_tool_time_s", "avg_tool_time_s", "s"), + ("Wall Time", "total_wall_time_s", "avg_wall_time_s", "s"), + ] + if markdown: + for label, total_key, avg_key, fmt in receipt_rows: + lines.append( + f"- **{label}**: {_fmt(receipt_cost[total_key], fmt)} total, " + f"{_fmt(receipt_cost[avg_key], fmt)} / task" + ) + else: + for label, total_key, avg_key, fmt in receipt_rows: + lines.append( + f" {label:<18} {_fmt(receipt_cost[total_key], fmt):>10} total " + f"{_fmt(receipt_cost[avg_key], fmt):>10} / task" + ) + lines.append("") + lines.append(h2("Per-Task Results")) lines.append("") @@ -1400,18 +1614,20 @@ def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment if has_vakra: lines.append( "| Task | Domain | # | Result | Dialogue | ExactMatch | Answer | Groundedness " - "| Tokens | Cost | LLM Calls | Cache Tokens | Duration | Steps |" + "| Tokens | Cost | LLM Calls | Cache Tokens | Input | Output | Reasoning | Duration | Steps |" ) lines.append( "|------|--------|---|--------|----------|------------|--------|--------------" - "|--------|------|-----------|--------------|----------|-------|" + "|--------|------|-----------|--------------|-------|--------|-----------|----------|-------|" ) else: lines.append( - "| Task | Domain | # | Result | Tokens | Cost | LLM Calls | Cache Tokens | Duration | Steps |" + "| Task | Domain | # | Result | Tokens | Cost | LLM Calls | Cache Tokens " + "| Input | Output | Reasoning | Duration | Steps |" ) lines.append( - "|------|--------|---|--------|--------|------|-----------|--------------|----------|-------|" + "|------|--------|---|--------|--------|------|-----------|--------------" + "|-------|--------|-----------|----------|-------|" ) current_key: tuple = (None, None) for row in rows: @@ -1443,6 +1659,8 @@ def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment f"| {tid_disp} | {dom_disp} | {ordn_disp} | {status} {vakra_cols}" f"| {_fmt(t['tokens'])} | {_fmt(t.get('cost'), '$')} " f"| {_fmt(t.get('llm_calls'))} | {_fmt(t.get('cache_tokens'))} " + f"| {_fmt(t.get('input_tokens'))} | {_fmt(t.get('output_tokens'))} " + f"| {_fmt(t.get('reasoning_tokens'))} " f"| {_fmt(t.get('duration'), 's')} | {_fmt(t.get('steps'))} |" ) else: @@ -1454,13 +1672,15 @@ def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment f" {col_task:<4} {'Domain':<{col_dom_w}} {'#':>2} " f"{'R':<2} {'Dialog':>6} {'ExctM':>5} {'Answer':>6} {'Ground':>6} " f"{'Tokens':>10} {'Cost':>7} {'LLM':>5} " - f"{'Cache':>10} {'Duration':>9} {'Steps':>5}" + f"{'Cache':>10} {'Input':>9} {'Output':>9} {'Reason':>7} " + f"{'Duration':>9} {'Steps':>5}" ) else: header = ( f" {col_task:<4} {'Domain':<{col_dom_w}} {'#':>2} " f"{'R':<2} {'Tokens':>10} {'Cost':>7} {'LLM':>5} " - f"{'Cache':>10} {'Duration':>9} {'Steps':>5}" + f"{'Cache':>10} {'Input':>9} {'Output':>9} {'Reason':>7} " + f"{'Duration':>9} {'Steps':>5}" ) lines.append(header) lines.append(" " + "─" * (len(header) - 2)) @@ -1498,21 +1718,33 @@ def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment f"{_fmt(t.get('cost'), '$'):>7} " f"{_fmt(t.get('llm_calls')):>5} " f"{_fmt(t.get('cache_tokens')):>10} " + f"{_fmt(t.get('input_tokens')):>9} " + f"{_fmt(t.get('output_tokens')):>9} " + f"{_fmt(t.get('reasoning_tokens')):>7} " f"{_fmt(t.get('duration'), 's'):>9} " f"{_fmt(t.get('steps')):>5}" ) else: # Legacy flat table (e.g. AppWorld where m3_task_id/domain aren't set). if markdown: - lines.append("| Task | Result | Tokens | Cost | LLM Calls | Cache Tokens | Duration | Steps |") - lines.append("|------|--------|--------|------|-----------|--------------|----------|-------|") + lines.append( + "| Task | Result | Tokens | Cost | LLM Calls | Cache Tokens " + "| Input | Output | Reasoning | Duration | Steps |" + ) + lines.append( + "|------|--------|--------|------|-----------|--------------" + "|-------|--------|-----------|----------|-------|" + ) for row in rows: t = row["data"] status = _task_result_mark(t) lines.append( f"| {row['label']} | {status} | {_fmt(t['tokens'])} " f"| {_fmt(t.get('cost'), '$')} | {_fmt(t.get('llm_calls'))} " - f"| {_fmt(t.get('cache_tokens'))} | {_fmt(t.get('duration'), 's')} " + f"| {_fmt(t.get('cache_tokens'))} " + f"| {_fmt(t.get('input_tokens'))} | {_fmt(t.get('output_tokens'))} " + f"| {_fmt(t.get('reasoning_tokens'))} " + f"| {_fmt(t.get('duration'), 's')} " f"| {_fmt(t.get('steps'))} |" ) else: @@ -1520,6 +1752,7 @@ def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment header = ( f" {'Task':<{col_task_w}} {'R':<2} {'Tokens':>10} " f"{'Cost':>7} {'LLM':>5} {'Cache':>10} " + f"{'Input':>9} {'Output':>9} {'Reason':>7} " f"{'Duration':>9} {'Steps':>5}" ) lines.append(header) @@ -1536,6 +1769,9 @@ def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment f"{_fmt(t.get('cost'), '$'):>7} " f"{_fmt(t.get('llm_calls')):>5} " f"{_fmt(t.get('cache_tokens')):>10} " + f"{_fmt(t.get('input_tokens')):>9} " + f"{_fmt(t.get('output_tokens')):>9} " + f"{_fmt(t.get('reasoning_tokens')):>7} " f"{_fmt(t.get('duration'), 's'):>9} " f"{_fmt(t.get('steps')):>5}" ) diff --git a/benchmarks/helpers/tests/test_compare_report.py b/benchmarks/helpers/tests/test_compare_report.py index 0a67e51a..93345b8d 100644 --- a/benchmarks/helpers/tests/test_compare_report.py +++ b/benchmarks/helpers/tests/test_compare_report.py @@ -28,9 +28,12 @@ from benchmarks.helpers import compare_report as compare_report_module from benchmarks.helpers.compare_report import ( + _aggregate_receipt_costs, _last_turn_judge_scores, _m3_capability_group, + _parse_appworld_results, _parse_sdk_results, + _stats_for_task, generate_eval_report, generate_report, ) @@ -410,8 +413,8 @@ def test_eval_report_m3_without_vakra_omits_columns(tmp_path): assert "Dialogue" not in report assert "ExactMatch" not in report assert ( - "| Task | Domain | # | Result | Tokens | Cost | LLM Calls | Cache Tokens | Duration | Steps |" - in report + "| Task | Domain | # | Result | Tokens | Cost | LLM Calls | Cache Tokens " + "| Input | Output | Reasoning | Duration | Steps |" in report ) @@ -998,3 +1001,586 @@ def test_vakra_scores_stay_attributed_within_grouped_sections(tmp_path): assert "1.00" in cuga_section assert "Dialog" in react_section assert "0.00" in react_section + + +def test_parse_sdk_results_extracts_receipt_fields(): + data = { + "metrics": {"total_tasks": 1, "passed": 1}, + "results": [ + { + "task_name": "t1", + "success": True, + "total_tokens": 150, + "input_tokens": 100, + "output_tokens": 50, + "cache_read_tokens": 20, + "reasoning_tokens": 5, + "tool_call_count": 2, + "llm_time_s": 1.5, + "tool_time_s": 0.5, + "wall_time_s": 2.5, + } + ], + } + parsed = _parse_sdk_results(data) + + assert parsed["input_tokens"] == 100 + assert parsed["output_tokens"] == 50 + assert parsed["cache_read_tokens"] == 20 + assert parsed["reasoning_tokens"] == 5 + assert parsed["tool_call_count"] == 2 + assert parsed["llm_time_s"] == 1.5 + assert parsed["tool_time_s"] == 0.5 + assert parsed["wall_time_s"] == 2.5 + assert parsed["tasks"]["t1"]["input_tokens"] == 100 + + +def test_parse_sdk_results_defaults_receipt_fields_to_zero_when_absent(): + """Run-level totals still sum to 0 when no result carries receipt fields + (they're plain sums). The per-task value, however, stays None -- it's a + receipt-only field that was never measured for this task, not a genuine + zero (Sergey review, PR #182 follow-up).""" + data = { + "metrics": {"total_tasks": 1, "passed": 1}, + "results": [{"task_name": "t1", "success": True, "total_tokens": 50}], + } + parsed = _parse_sdk_results(data) + + assert parsed["input_tokens"] == 0 + assert parsed["tasks"]["t1"]["input_tokens"] is None + + +def test_aggregate_receipt_costs_totals_and_averages(): + tasks = { + "t1": { + "token_source": "receipt", + "input_tokens": 100, + "output_tokens": 50, + "cache_read_tokens": 20, + "reasoning_tokens": 5, + "tool_call_count": 2, + "llm_time_s": 1.0, + "tool_time_s": 0.5, + "wall_time_s": 1.5, + }, + "t2": { + "token_source": "receipt", + "input_tokens": 200, + "output_tokens": 100, + "cache_read_tokens": 0, + "reasoning_tokens": 0, + "tool_call_count": 4, + "llm_time_s": 2.0, + "tool_time_s": 1.0, + "wall_time_s": 3.0, + }, + } + agg = _aggregate_receipt_costs(tasks) + + assert agg["total_input_tokens"] == 300 + assert agg["avg_input_tokens"] == 150 + assert agg["total_output_tokens"] == 150 + assert agg["total_tool_call_count"] == 6 + assert agg["avg_wall_time_s"] == 2.25 + + +def test_aggregate_receipt_costs_all_none_when_no_receipt_data(): + tasks = {"t1": {"tokens": 50}} # legacy shape, no receipt fields at all + agg = _aggregate_receipt_costs(tasks) + + assert agg["total_input_tokens"] is None + assert agg["avg_input_tokens"] is None + + +def test_aggregate_receipt_costs_detects_nontoken_receipt_data(): + """When a task has zero token-related fields but nonzero non-token fields + (tool_call_count, llm_time_s, tool_time_s, wall_time_s, cache_read_tokens, + reasoning_tokens), _aggregate_receipt_costs must recognize this as receipt + data and return real numbers, not None.""" + tasks = { + "t1": { + "token_source": "receipt", + "input_tokens": 0, + "output_tokens": 0, + "cache_read_tokens": 0, + "reasoning_tokens": 0, + "tool_call_count": 3, # nonzero non-token field + "llm_time_s": 1.5, # nonzero non-token field + "tool_time_s": 0.0, + "wall_time_s": 2.0, # nonzero non-token field + } + } + agg = _aggregate_receipt_costs(tasks) + + # All fields should have real numbers, not None + assert agg["total_tool_call_count"] == 3 + assert agg["avg_tool_call_count"] == 3 + assert agg["total_llm_time_s"] == 1.5 + assert agg["avg_llm_time_s"] == 1.5 + assert agg["total_wall_time_s"] == 2.0 + assert agg["avg_wall_time_s"] == 2.0 + # Token fields are still 0, not None + assert agg["total_input_tokens"] == 0 + assert agg["avg_input_tokens"] == 0 + + +def test_aggregate_receipt_costs_detects_genuine_all_zero_receipt(): + """A task marked token_source == "receipt" whose every receipt field is + legitimately 0 (e.g. a call that used 0 output tokens) must still be + detected as carrying receipt data — the marker, not field truthiness, + decides this (cuga-eval#182 CodeRabbit review).""" + tasks = { + "t1": { + "token_source": "receipt", + "input_tokens": 0, + "output_tokens": 0, + "cache_read_tokens": 0, + "reasoning_tokens": 0, + "tool_call_count": 0, + "llm_time_s": 0.0, + "tool_time_s": 0.0, + "wall_time_s": 0.0, + } + } + agg = _aggregate_receipt_costs(tasks) + + assert agg["total_input_tokens"] == 0 + assert agg["avg_input_tokens"] == 0 + assert agg["total_wall_time_s"] == 0.0 + assert agg["avg_wall_time_s"] == 0.0 + + +def test_aggregate_receipt_costs_averages_only_over_receipt_tagged_tasks(): + """A run with a mix of receipt-tagged tasks and legacy/non-receipt tasks + (no token_source, everything defaulted to 0) must average only over the + receipt-tagged subset — dividing by the full task count would silently + dilute the average toward zero (cuga-eval#182 CodeRabbit review).""" + tasks = { + "t1": { + "token_source": "receipt", + "input_tokens": 100, + "output_tokens": 50, + "cache_read_tokens": 0, + "reasoning_tokens": 0, + "tool_call_count": 1, + "llm_time_s": 1.0, + "tool_time_s": 0.0, + "wall_time_s": 1.0, + }, + "t2": { + "token_source": "receipt", + "input_tokens": 200, + "output_tokens": 100, + "cache_read_tokens": 0, + "reasoning_tokens": 0, + "tool_call_count": 1, + "llm_time_s": 1.0, + "tool_time_s": 0.0, + "wall_time_s": 1.0, + }, + "t3_legacy": { + # No token_source at all: pre-migration task, never opted in. + "input_tokens": 0, + "output_tokens": 0, + }, + } + agg = _aggregate_receipt_costs(tasks) + + # Average must divide by 2 (the receipt-tagged count), not 3. + assert agg["total_input_tokens"] == 300 + assert agg["avg_input_tokens"] == 150 + assert agg["total_output_tokens"] == 150 + assert agg["avg_output_tokens"] == 75 + + +def _write_sdk_result_file(tmp_path, results, name="results.json"): + path = tmp_path / name + path.write_text( + json.dumps({"metrics": {"total_tasks": len(results), "passed": len(results)}, "results": results}) + ) + return str(path) + + +def test_generate_eval_report_includes_receipt_breakdown_when_present(tmp_path): + results = [ + { + "task_name": "t1", + "success": True, + "total_tokens": 150, + "token_source": "receipt", + "input_tokens": 100, + "output_tokens": 50, + "cache_read_tokens": 20, + "reasoning_tokens": 5, + "tool_call_count": 2, + "llm_time_s": 1.5, + "tool_time_s": 0.5, + "wall_time_s": 2.5, + } + ] + report = generate_eval_report(_write_sdk_result_file(tmp_path, results)) + + assert "Run Receipt Breakdown" in report + assert "Input Tokens" in report + assert "100" in report + + +def test_generate_eval_report_omits_receipt_breakdown_when_absent(tmp_path): + results = [{"task_name": "t1", "success": True, "total_tokens": 150}] + report = generate_eval_report(_write_sdk_result_file(tmp_path, results)) + + assert "Run Receipt Breakdown" not in report + + +def test_generate_report_includes_receipt_breakdown_when_present(tmp_path): + results = [ + { + "task_name": "t1", + "success": True, + "total_tokens": 150, + "token_source": "receipt", + "input_tokens": 100, + "output_tokens": 50, + "cache_read_tokens": 20, + "reasoning_tokens": 5, + "tool_call_count": 2, + "llm_time_s": 1.5, + "tool_time_s": 0.5, + "wall_time_s": 2.5, + } + ] + result_file = _write_sdk_result_file(tmp_path, results) + report = generate_report({"gpt-oss-120b": [result_file]}) + + assert "Run Receipt Breakdown" in report + assert "In Tok" in report + + +def test_generate_report_omits_receipt_breakdown_when_absent(tmp_path): + results = [{"task_name": "t1", "success": True, "total_tokens": 150}] + result_file = _write_sdk_result_file(tmp_path, results) + report = generate_report({"gpt-oss-120b": [result_file]}) + + assert "Run Receipt Breakdown" not in report + + +def test_generate_report_detects_nontoken_receipt_data(tmp_path): + """Same class of bug as test_aggregate_receipt_costs_detects_nontoken_receipt_data, + but for _render_compare_report_sections's own any_receipt_data gate: a run + with zero input_tokens/output_tokens but nonzero tool_call_count/wall_time_s + must still trigger the "Run Receipt Breakdown" section (cuga-eval#95 final + review finding — the gate previously only looked at 2 of 8 receipt fields).""" + results = [ + { + "task_name": "t1", + "success": True, + "total_tokens": 0, + "token_source": "receipt", + "input_tokens": 0, + "output_tokens": 0, + "tool_call_count": 3, + "wall_time_s": 2.0, + } + ] + result_file = _write_sdk_result_file(tmp_path, results) + report = generate_report({"gpt-oss-120b": [result_file]}) + + assert "Run Receipt Breakdown" in report + + +def test_generate_report_detects_genuine_all_zero_receipt(tmp_path): + """A run whose single task has token_source == "receipt" but every + receipt field is legitimately 0 must still trigger the "Run Receipt + Breakdown" section — field truthiness alone would wrongly treat this as + "no receipt data" (cuga-eval#182 CodeRabbit review).""" + results = [ + { + "task_name": "t1", + "success": True, + "total_tokens": 0, + "token_source": "receipt", + "input_tokens": 0, + "output_tokens": 0, + "cache_read_tokens": 0, + "reasoning_tokens": 0, + "tool_call_count": 0, + "llm_time_s": 0.0, + "tool_time_s": 0.0, + "wall_time_s": 0.0, + } + ] + result_file = _write_sdk_result_file(tmp_path, results) + report = generate_report({"gpt-oss-120b": [result_file]}) + eval_report = generate_eval_report(result_file) + + assert "Run Receipt Breakdown" in report + assert "Run Receipt Breakdown" in eval_report + + +def test_generate_report_shows_dash_for_config_without_receipt_data(tmp_path): + """A/B comparison between a baseline bundle predating the receipt feature + (no receipt fields at all) and a new one that has them: the baseline's row + in the Run Receipt Breakdown table must render "--", not "0" (cuga-eval#95 + final review finding — a config with no receipt data looked identical to + one that measured real zeros).""" + with_receipt = [ + { + "task_name": "t1", + "success": True, + "total_tokens": 150, + "token_source": "receipt", + "input_tokens": 100, + "output_tokens": 50, + "cache_read_tokens": 20, + "reasoning_tokens": 5, + "tool_call_count": 2, + "llm_time_s": 1.5, + "tool_time_s": 0.5, + "wall_time_s": 2.5, + } + ] + without_receipt = [{"task_name": "t1", "success": True, "total_tokens": 150}] + + file_with = _write_sdk_result_file(tmp_path, with_receipt, name="with.json") + file_without = _write_sdk_result_file(tmp_path, without_receipt, name="without.json") + + report = generate_report({"model-with-receipt": [file_with], "model-without-receipt": [file_without]}) + + assert "Run Receipt Breakdown" in report + + breakdown = report.split("Run Receipt Breakdown", 1)[1] + with_line = next(line for line in breakdown.splitlines() if "model-with-receipt" in line) + without_line = next(line for line in breakdown.splitlines() if "model-without-receipt" in line) + + assert "100" in with_line + assert "--" not in with_line + + assert "--" in without_line + assert "0" not in without_line + + +# --------------------------------------------------------------------------- +# Per-task Input/Output/Reasoning columns (cuga-eval#95 follow-up). +# +# These are distinct from the "Run Receipt Breakdown" summary section tested +# above: the columns added here appear unconditionally on the genuinely +# per-task tables (Per-Task Results in generate_eval_report, Per-Task Details +# in generate_report), the same way `Cache Tokens` already does — they are +# not gated behind "does this result carry receipt data". +# --------------------------------------------------------------------------- + + +def test_eval_report_grouped_per_task_table_shows_input_output_reasoning(tmp_path): + """generate_eval_report's grouped (M3), non-Vakra per-task table gains + Input/Output/Reasoning columns with correct per-task values.""" + result_file = _m3_run( + tmp_path, + "m3_receipt.json", + { + "uuid-a": {"m3_task_id": 2, "domain": "hockey", "task_number": 1, "success": True}, + "uuid-b": {"m3_task_id": 2, "domain": "hockey", "task_number": 2, "success": False}, + }, + ) + # _m3_run doesn't plumb receipt fields, so patch them into the raw file. + payload = json.loads(Path(result_file).read_text()) + payload["results"][0].update({"input_tokens": 111, "output_tokens": 22, "reasoning_tokens": 3}) + payload["results"][1].update({"input_tokens": 444, "output_tokens": 55, "reasoning_tokens": 6}) + Path(result_file).write_text(json.dumps(payload)) + + report = generate_eval_report(result_file) + + assert ( + "| Task | Domain | # | Result | Tokens | Cost | LLM Calls | Cache Tokens " + "| Input | Output | Reasoning | Duration | Steps |" in report + ) + assert "| 111 | 22 | 3 |" in report + assert "| 444 | 55 | 6 |" in report + + +def test_eval_report_legacy_per_task_table_shows_input_output_reasoning(tmp_path): + """generate_eval_report's non-grouped/legacy per-task table (e.g. AppWorld) + gains Input/Output/Reasoning columns with correct per-task values.""" + results = [ + { + "task_name": "t1", + "success": True, + "total_tokens": 150, + "input_tokens": 100, + "output_tokens": 50, + "reasoning_tokens": 5, + } + ] + report = generate_eval_report(_write_sdk_result_file(tmp_path, results)) + + assert ( + "| Task | Result | Tokens | Cost | LLM Calls | Cache Tokens " + "| Input | Output | Reasoning | Duration | Steps |" in report + ) + assert "| 100 | 50 | 5 |" in report + + +def test_eval_report_per_task_table_shows_dash_when_receipt_fields_absent(tmp_path): + """Legacy/non-receipt result files (no input_tokens/output_tokens/ + reasoning_tokens at all) still render the per-task table without crashing; + those three receipt-only columns render `--` (never measured), not a + misleading 0 (Sergey review, PR #182 follow-up).""" + results = [{"task_name": "t1", "success": True, "total_tokens": 150}] + report = generate_eval_report(_write_sdk_result_file(tmp_path, results)) + + assert ( + "| Task | Result | Tokens | Cost | LLM Calls | Cache Tokens " + "| Input | Output | Reasoning | Duration | Steps |" in report + ) + assert "| -- | -- | -- |" in report + + +def test_stats_for_task_computes_mean_input_output_reasoning(): + task_runs = [ + {"token_source": "receipt", "input_tokens": 100, "output_tokens": 20, "reasoning_tokens": 2}, + {"token_source": "receipt", "input_tokens": 200, "output_tokens": 40, "reasoning_tokens": 6}, + ] + stats = _stats_for_task(task_runs) + + assert stats["mean_input"] == 150 + assert stats["mean_output"] == 30 + assert stats["mean_reasoning"] == 4 + + +def test_compare_report_per_task_details_shows_input_output_reasoning_and_average(tmp_path): + """generate_report's Per-Task Details table gains Input/Output/Reasoning + columns, correctly averaged per task and in the AVERAGE row.""" + results_r1 = [ + { + "task_name": "t1", + "success": True, + "total_tokens": 150, + "token_source": "receipt", + "input_tokens": 100, + "output_tokens": 20, + }, + { + "task_name": "t2", + "success": True, + "total_tokens": 150, + "token_source": "receipt", + "input_tokens": 300, + "output_tokens": 60, + }, + ] + results_r2 = [ + { + "task_name": "t1", + "success": True, + "total_tokens": 150, + "token_source": "receipt", + "input_tokens": 200, + "output_tokens": 40, + }, + { + "task_name": "t2", + "success": True, + "total_tokens": 150, + "token_source": "receipt", + "input_tokens": 300, + "output_tokens": 60, + }, + ] + run1 = _write_sdk_result_file(tmp_path, results_r1, name="r1.json") + run2 = _write_sdk_result_file(tmp_path, results_r2, name="r2.json") + + report = generate_report({"gpt-oss:cuga": [run1, run2]}) + + assert "Input" in report + assert "Output" in report + assert "Reason" in report + # t1: mean input = (100+200)/2 = 150.0, mean output = (20+40)/2 = 30.0. + # t2: mean input = 300.0, mean output = 60.0 (constant across both runs). + # AVERAGE row: mean input across tasks = (150+300)/2 = 225.0, mean output = (30+60)/2 = 45.0. + lines = report.splitlines() + t1_line = next(ln for ln in lines if ln.strip().startswith("t1 ")) + t2_line = next(ln for ln in lines if ln.strip().startswith("t2 ")) + avg_line = next(ln for ln in lines if "AVERAGE" in ln) + assert "150.0" in t1_line + assert "300.0" in t2_line + assert "225.0" in avg_line + assert "45.0" in avg_line + + +def test_eval_report_appworld_task_results_shape_shows_dashes_for_receipt_only_fields(tmp_path): + """Legacy AppWorld `task_results`-shaped result files (still produced today by + appworld_eval.py / appworld_eval_react.py / appworld_eval_codeact.py, parsed by + `_parse_appworld_results` rather than `_parse_sdk_results`) carry no + input_tokens/output_tokens/reasoning_tokens on their per-task dicts -- those + are receipt-only fields written solely by sdk_eval_helpers.py. They render + `--` (never measured), distinct from Cache Tokens, which is a genuinely + universal field (populated by every evaluator whenever Langfuse succeeds or + a receipt exists) and legitimately 0 here, not absent (Sergey review, PR + #182 follow-up).""" + result_file = _appworld_eval_run( + tmp_path, + "appworld_no_receipt.json", + { + "t1": { + "success": True, + "total_tokens": 150, + "total_llm_calls": 3, + "cache_input_tokens": 0, + "full_execution_time": 5.0, + "steps": 4, + } + }, + ) + + report = generate_eval_report(result_file) + + assert ( + "| Task | Result | Tokens | Cost | LLM Calls | Cache Tokens " + "| Input | Output | Reasoning | Duration | Steps |" in report + ) + row = next(ln for ln in report.splitlines() if ln.startswith("| t1 ")) + # Cache Tokens is a real measured 0; Input/Output/Reasoning were never + # measured for this result shape and must render `--`, not a misleading 0. + assert "| 0 | -- | -- | -- |" in row + + +def test_parse_appworld_results_preserves_all_eight_receipt_fields(): + """_parse_appworld_results must mirror _parse_sdk_results' full 8-field + Run Receipt handling (cache_read_tokens, tool_call_count, llm_time_s, + tool_time_s, wall_time_s), not just input/output/reasoning tokens -- + otherwise a receipt-producing AppWorld harness would have those 5 fields + silently truncated to zero in both the per-task dict and the run-level + totals (CodeRabbit review, PR #182).""" + data = { + "tasks_total": 1, + "tasks_completed": 1, + "task_results": { + "t1": { + "success": True, + "total_tokens": 150, + "input_tokens": 100, + "output_tokens": 50, + "cache_read_tokens": 20, + "reasoning_tokens": 5, + "tool_call_count": 2, + "llm_time_s": 1.5, + "tool_time_s": 0.5, + "wall_time_s": 2.5, + "token_source": "receipt", + } + }, + } + + parsed = _parse_appworld_results(data) + + task = parsed["tasks"]["t1"] + assert task["cache_read_tokens"] == 20 + assert task["tool_call_count"] == 2 + assert task["llm_time_s"] == 1.5 + assert task["tool_time_s"] == 0.5 + assert task["wall_time_s"] == 2.5 + assert parsed["cache_read_tokens"] == 20 + assert parsed["tool_call_count"] == 2 + assert parsed["llm_time_s"] == 1.5 + assert parsed["tool_time_s"] == 0.5 + assert parsed["wall_time_s"] == 2.5 + assert parsed["has_receipt_data"] is True diff --git a/benchmarks/m3/eval.sh b/benchmarks/m3/eval.sh index eef2bb31..bdc1f4e1 100755 --- a/benchmarks/m3/eval.sh +++ b/benchmarks/m3/eval.sh @@ -362,12 +362,16 @@ REGISTRY_PORT="${REGISTRY_PORT:-${DYNACONF_SERVER_PORTS__REGISTRY:-8001}}" export REGISTRY_PORT export DYNACONF_SERVER_PORTS__REGISTRY="$REGISTRY_PORT" +# Per-run token/timing receipt from CugaAgent.invoke() (cuga-agent#467), +# preferred over the Langfuse HTTP fetch when present (cuga-eval#95). +export DYNACONF_ADVANCED_FEATURES__RUN_RECEIPT=true + # Capture the cuga-agent checkout's git state now, before the eval run starts # — not at bundle-assembly time (after the run finishes). If the checkout is # shared (e.g. someone switches branches in it for unrelated work) while a # long run is in flight, a live git query at the end would silently mislabel # the bundle with whatever happens to be checked out by then. -CUGA_REPO_PATH_RESOLVED="${CUGA_REPO_PATH:-$HOME/workspace/cuga-agent}" +CUGA_REPO_PATH_RESOLVED="$(resolve_cuga_repo_path)" CUGA_GIT_INFO_JSON="" if [ -d "$CUGA_REPO_PATH_RESOLVED" ]; then _cuga_commit=$(git -C "$CUGA_REPO_PATH_RESOLVED" rev-parse --short HEAD 2>/dev/null || echo "") From af10e91905b7fc14e6dad2babc95493a8162bc48 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Fri, 18 Sep 2026 12:12:03 -0400 Subject: [PATCH 34/40] Fix PR eval reporting and receipt metadata handling --- .../run-pr-regression-eval.sh | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index c9db5d01..c9c5bd06 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -8,6 +8,58 @@ log() { printf '[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" } +extract_report_blocks() { + local input_path="$1" + local output_path="$2" + + if [[ ! -f "${input_path}" ]]; then + echo "No evaluation output artifact was available." > "${output_path}" + return 0 + fi + + awk ' + $0 == "######## REPORT START ########" { + in_report = 1 + block = "" + next + } + $0 == "######## REPORT END ########" { + if (in_report) { + sub(/^[[:space:]]+/, "", block) + sub(/[[:space:]]+$/, "", block) + if (length(block) > 0) { + if (printed) { + print "" + print "" + } + print block + printed = 1 + } + } + in_report = 0 + block = "" + next + } + in_report { + block = block $0 "\n" + } + END { + if (!printed) { + print "No report section found between REPORT START and REPORT END markers." + } + } + ' "${input_path}" > "${output_path}" +} + +if [[ "${1:-}" == "--extract-report" ]]; then + if [[ $# -ne 3 ]]; then + echo "Usage: $0 --extract-report " >&2 + exit 2 + fi + extract_report_blocks "$2" "$3" + exit 0 +fi + open_details() { local title="$1" echo "
" From 34882c0cdbdeb3e8ad0f23b8ebbd14dd707c1254 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Fri, 18 Sep 2026 15:39:33 -0400 Subject: [PATCH 35/40] Clarify PR eval provider environment contract --- scripts/pr-regression-testing/README.md | 12 +++++----- .../run-pr-regression-eval.sh | 22 ++++++------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index 8c633e13..611deb5f 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -93,19 +93,19 @@ Settings -> Secrets and variables -> Actions Required secrets: - `RITS_API_KEY`: required for `provider=rits`. -- `LITE_LLM_KEY`: required for `provider=litellm`, unless `OPENAI_API_KEY` is set instead. -- `OPENAI_API_KEY`: optional fallback key for OpenAI-compatible/LiteLLM flows. +- `LITE_LLM_KEY`: preferred secret for `provider=litellm`; the workflow injects it into `OPENAI_API_KEY`. +- `OPENAI_API_KEY`: optional fallback secret for `provider=litellm` when `LITE_LLM_KEY` is not set. Optional repository variables: - `MODEL_NAME`: default model override used by the workflow environment. -- `OPENAI_BASE_URL`: default OpenAI-compatible endpoint override. -- `LITE_LLM_URL`: LiteLLM endpoint override. If omitted, `provider=litellm` defaults to `https://ete-litellm.ai-models.vpc-int.res.ibm.com/`. +- `RITS_BASE_URL`: RITS endpoint override for `provider=rits`. +- `LITELLM_BASE_URL`: LiteLLM endpoint override for `provider=litellm`. Provider defaults used by `run-pr-regression-eval.sh`: -- `provider=rits` uses `RITS_API_KEY` and defaults `OPENAI_BASE_URL` to `https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100`. -- `provider=litellm` uses `LITE_LLM_KEY` first, then `OPENAI_API_KEY`; it uses `LITE_LLM_URL` first, then `OPENAI_BASE_URL`, then `https://ete-litellm.ai-models.vpc-int.res.ibm.com/`. Unless `model_name` is explicitly set in the PR comment, it uses `aws/gpt-oss-120b`. +- `provider=rits` selects `settings.rits.toml`, requires `RITS_API_KEY`, and passes `RITS_BASE_URL` to `models.py`. +- `provider=litellm` selects `settings.litellm.toml` for the CUGA agent or `settings.openai.toml` for non-CUGA agents, requires `OPENAI_API_KEY`, and passes `OPENAI_BASE_URL` to `models.py`. Unless `model_name` is explicitly set in the PR comment, it uses `aws/gpt-oss-120b`. ## Run PR Evaluations diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index c9c5bd06..2db04697 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -100,6 +100,8 @@ trap on_exit EXIT DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" DEFAULT_LITELLM_MODEL_NAME="aws/gpt-oss-120b" +DEFAULT_RITS_BASE_URL="https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100/v1" +DEFAULT_LITELLM_BASE_URL="https://ete-litellm.ai-models.vpc-int.res.ibm.com/" DEFAULT_APPWORLD_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" DEFAULT_BENCHMARK="appworld" DEFAULT_NUM_TASKS="4" @@ -329,6 +331,8 @@ case "${PROVIDER}" in echo "######## REPORT END ########" exit 2 fi + RITS_BASE_URL="${RITS_BASE_URL:-${DEFAULT_RITS_BASE_URL}}" + export RITS_API_KEY RITS_BASE_URL ;; litellm) if [[ "${AGENT}" == "cuga" ]]; then @@ -339,16 +343,15 @@ case "${PROVIDER}" in if [[ "${MODEL_NAME_FROM_COMMENT}" == "false" ]]; then MODEL_NAME="${DEFAULT_LITELLM_MODEL_NAME}" fi - OPENAI_BASE_URL="${LITE_LLM_URL:-${OPENAI_BASE_URL:-https://ete-litellm.ai-models.vpc-int.res.ibm.com/}}" - OPENAI_API_KEY="${LITE_LLM_KEY:-${OPENAI_API_KEY:-}}" + OPENAI_BASE_URL="${OPENAI_BASE_URL:-${DEFAULT_LITELLM_BASE_URL}}" if [[ -z "${OPENAI_API_KEY:-}" ]]; then - echo "ERROR: LITE_LLM_KEY or OPENAI_API_KEY is required when provider=litellm." + echo "ERROR: OPENAI_API_KEY is required when provider=litellm." close_details echo "######## REPORT END ########" exit 2 fi if [[ -z "${OPENAI_BASE_URL:-}" ]]; then - echo "ERROR: LITE_LLM_URL or OPENAI_BASE_URL is required when provider=litellm." + echo "ERROR: OPENAI_BASE_URL is required when provider=litellm." close_details echo "######## REPORT END ########" exit 2 @@ -363,17 +366,6 @@ case "${PROVIDER}" in ;; esac -if [[ "${PROVIDER}" == "rits" ]]; then - export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" - if [[ -z "${RITS_BASE_URL:-}" ]]; then - RITS_BASE_URL="${OPENAI_BASE_URL%/}" - [[ "${RITS_BASE_URL}" == */v1 ]] || RITS_BASE_URL="${RITS_BASE_URL}/v1" - export RITS_BASE_URL - fi -else - : -fi - if [[ -z "${TASK_IDS}" && -z "${EVAL_KEY}" && "${BENCHMARK}" == "appworld" ]]; then TASK_IDS="${DEFAULT_APPWORLD_TASK_IDS}" fi From f3cedfa1f661d76dd59e121603b77ec903c6a2c7 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Fri, 18 Sep 2026 16:35:50 -0400 Subject: [PATCH 36/40] Fix ReAct eval Ruff import issues --- benchmarks/appworld/appworld_eval_react.py | 1 - benchmarks/helpers/react_agent.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmarks/appworld/appworld_eval_react.py b/benchmarks/appworld/appworld_eval_react.py index d79c357e..0346fdb3 100644 --- a/benchmarks/appworld/appworld_eval_react.py +++ b/benchmarks/appworld/appworld_eval_react.py @@ -33,7 +33,6 @@ from datetime import datetime from typing import Any, Optional -import httpx from appworld import AppWorld, load_task_ids # pyright: ignore[reportAttributeAccessIssue] from cuga.backend.activity_tracker.tracker import ActivityTracker, Step from cuga.backend.cuga_graph.state.agent_state import VariablesManager diff --git a/benchmarks/helpers/react_agent.py b/benchmarks/helpers/react_agent.py index 6a11157c..e3477f17 100644 --- a/benchmarks/helpers/react_agent.py +++ b/benchmarks/helpers/react_agent.py @@ -19,11 +19,11 @@ from pathlib import Path from typing import Any, Optional -from langchain_openai import ChatOpenAI from cuga.backend.cuga_graph.nodes.cuga_lite.providers.combined import ( CombinedToolProvider, ) from langchain_core.messages import HumanMessage +from langchain_openai import ChatOpenAI from loguru import logger TOOL_BLOCK_RE = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE) From 4539cef1c7463305bd67dddb6c0ad11d4f397d87 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 24 Sep 2026 09:18:30 -0400 Subject: [PATCH 37/40] Keep PR eval comment formatting out of bundles --- benchmarks/appworld/eval.sh | 5 +++-- benchmarks/bpo/eval.sh | 5 +++-- benchmarks/m3/eval.sh | 8 +++++--- benchmarks/oak_health_insurance/eval.sh | 5 +++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/benchmarks/appworld/eval.sh b/benchmarks/appworld/eval.sh index c0e27f70..0629545c 100755 --- a/benchmarks/appworld/eval.sh +++ b/benchmarks/appworld/eval.sh @@ -411,10 +411,11 @@ if [ -n "$LATEST_RESULT" ] && [ "${NO_BUNDLE:-false}" != "true" ]; then # Generate eval report REPORT_TMP=$(mktemp /tmp/appworld_eval_report_XXXXXX) REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then - REPORT_ARGS+=(--for-pr-comment --stdout-markdown) + uv run --no-sync python -m benchmarks.helpers.compare_report \ + eval --result-file "$LATEST_RESULT" --for-pr-comment --stdout-markdown fi - uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" BUNDLE_ARGS=(assemble --benchmark appworld --result-files "$LATEST_RESULT" diff --git a/benchmarks/bpo/eval.sh b/benchmarks/bpo/eval.sh index 746081c2..569c9d60 100755 --- a/benchmarks/bpo/eval.sh +++ b/benchmarks/bpo/eval.sh @@ -410,10 +410,11 @@ if [ $EVAL_EXIT_CODE -eq 0 ]; then # Generate eval report REPORT_TMP=$(mktemp /tmp/bpo_eval_report_XXXXXX) REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then - REPORT_ARGS+=(--for-pr-comment --stdout-markdown) + uv run --no-sync python -m benchmarks.helpers.compare_report \ + eval --result-file "$LATEST_RESULT" --for-pr-comment --stdout-markdown fi - uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" BUNDLE_ARGS=(assemble --benchmark bpo --result-files "$LATEST_RESULT" diff --git a/benchmarks/m3/eval.sh b/benchmarks/m3/eval.sh index bdc1f4e1..ea417aad 100755 --- a/benchmarks/m3/eval.sh +++ b/benchmarks/m3/eval.sh @@ -264,11 +264,13 @@ create_bundle() { local report_tmp report_tmp=$(mktemp /tmp/m3_eval_report_XXXXXX) local report_args=(eval --result-file "$latest_result" --output "$report_tmp") - if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then - report_args+=(--for-pr-comment --stdout-markdown) - fi uv run --no-sync python -m benchmarks.helpers.compare_report "${report_args[@]}" || \ echo -e "${YELLOW:-}Report generation failed — bundling without report.${NC:-}" + if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then + uv run --no-sync python -m benchmarks.helpers.compare_report \ + eval --result-file "$latest_result" --for-pr-comment --stdout-markdown || \ + echo -e "${YELLOW:-}PR report generation failed — continuing with bundle.${NC:-}" + fi local bundle_args=(assemble --benchmark m3 --result-files "$latest_result" diff --git a/benchmarks/oak_health_insurance/eval.sh b/benchmarks/oak_health_insurance/eval.sh index de7c3288..d937dd59 100755 --- a/benchmarks/oak_health_insurance/eval.sh +++ b/benchmarks/oak_health_insurance/eval.sh @@ -243,10 +243,11 @@ if [ $EVAL_EXIT -eq 0 ]; then # Generate eval report REPORT_TMP=$(mktemp /tmp/oak_eval_report_XXXXXX) REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then - REPORT_ARGS+=(--for-pr-comment --stdout-markdown) + uv run --no-sync python -m benchmarks.helpers.compare_report \ + eval --result-file "$LATEST_RESULT" --for-pr-comment --stdout-markdown fi - uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" BUNDLE_ARGS=(assemble --benchmark oak_health_insurance --result-files "$LATEST_RESULT" From 0cd1cb77ba51e52f993ba867b40e3ab406bec41a Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 24 Sep 2026 09:35:03 -0400 Subject: [PATCH 38/40] Harden PR eval provider config handling --- benchmarks/helpers/react_agent.py | 23 ++++++++++++++++--- scripts/pr-regression-testing/README.md | 3 ++- .../run-pr-regression-eval.sh | 6 +++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/benchmarks/helpers/react_agent.py b/benchmarks/helpers/react_agent.py index e3477f17..2f46afce 100644 --- a/benchmarks/helpers/react_agent.py +++ b/benchmarks/helpers/react_agent.py @@ -23,9 +23,16 @@ CombinedToolProvider, ) from langchain_core.messages import HumanMessage -from langchain_openai import ChatOpenAI from loguru import logger +try: + from langchain_openai import ChatOpenAI as _ChatOpenAI +except ImportError as exc: + _ChatOpenAI = None + _CHAT_OPENAI_IMPORT_ERROR = exc +else: + _CHAT_OPENAI_IMPORT_ERROR = None + TOOL_BLOCK_RE = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE) PROMPT_ROLE_RE = re.compile(r"(USER|ASSISTANT|SYSTEM):\n", re.IGNORECASE) PROMPT_PATH = Path(__file__).with_name("react_agent_prompt.txt") @@ -45,10 +52,20 @@ class ReactInvokeResult: react_steps: int = 0 -class ChatRits(ChatOpenAI): +def _require_chat_openai(): + if _ChatOpenAI is None: + raise RuntimeError( + "langchain-openai is required for React agent with OpenAI-compatible or RITS providers. " + "Install with: pip install langchain-openai" + ) from _CHAT_OPENAI_IMPORT_ERROR + return _ChatOpenAI + + +class ChatRits(_ChatOpenAI if _ChatOpenAI is not None else object): """RITS chat model integration using langchain-openai.""" def __init__(self, config: dict[str, Any]): + _require_chat_openai() model_name = config.get("model_name", "openai/gpt-oss-120b-a100") end_point = ( config.get("end_point") @@ -201,7 +218,7 @@ def _create_llm(self) -> Any: llm_kwargs["http_async_client"] = httpx.AsyncClient(verify=False) # noqa: S501 # nosec B501 — same logger.info("SSL verification disabled for ChatOpenAI") - return ChatOpenAI(**llm_kwargs) + return _require_chat_openai()(**llm_kwargs) elif settings_config in ("settings.rits.toml", "settings.rits.proxy.toml"): api_base = os.getenv("RITS_BASE_URL") or os.getenv("LITE_LLM_URL") or os.getenv("OPENAI_BASE_URL") diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index 611deb5f..8dc4fe70 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -100,12 +100,13 @@ Optional repository variables: - `MODEL_NAME`: default model override used by the workflow environment. - `RITS_BASE_URL`: RITS endpoint override for `provider=rits`. -- `LITELLM_BASE_URL`: LiteLLM endpoint override for `provider=litellm`. +- `LITELLM_BASE_URL`: LiteLLM endpoint override for `provider=litellm`; the workflow injects it into `OPENAI_BASE_URL`. Provider defaults used by `run-pr-regression-eval.sh`: - `provider=rits` selects `settings.rits.toml`, requires `RITS_API_KEY`, and passes `RITS_BASE_URL` to `models.py`. - `provider=litellm` selects `settings.litellm.toml` for the CUGA agent or `settings.openai.toml` for non-CUGA agents, requires `OPENAI_API_KEY`, and passes `OPENAI_BASE_URL` to `models.py`. Unless `model_name` is explicitly set in the PR comment, it uses `aws/gpt-oss-120b`. +- For `provider=rits`, overriding `model_name` in the PR command requires `RITS_BASE_URL` to be configured so the model name and endpoint cannot silently diverge. ## Run PR Evaluations diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index 2db04697..1ad6a1d1 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -331,6 +331,12 @@ case "${PROVIDER}" in echo "######## REPORT END ########" exit 2 fi + if [[ "${MODEL_NAME_FROM_COMMENT}" == "true" && -z "${RITS_BASE_URL:-}" ]]; then + echo "ERROR: provider=rits model_name overrides require RITS_BASE_URL to avoid mismatched model endpoints." + close_details + echo "######## REPORT END ########" + exit 2 + fi RITS_BASE_URL="${RITS_BASE_URL:-${DEFAULT_RITS_BASE_URL}}" export RITS_API_KEY RITS_BASE_URL ;; From 0c71d7642669d9d40397552a24e02a18beaf08b2 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 5 Oct 2026 11:27:26 -0400 Subject: [PATCH 39/40] Use AppWorld SDK only for CUGA PR evals --- scripts/pr-regression-testing/README.md | 2 ++ scripts/pr-regression-testing/run-pr-regression-eval.sh | 3 +++ 2 files changed, 5 insertions(+) diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index 8dc4fe70..c214ff6e 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -125,6 +125,8 @@ The `sha` value must be the current 40-character PR head commit SHA. The workflo - `num_tasks=4` - default AppWorld task IDs: `9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1` +AppWorld PR evaluations with `agent=cuga` are run with the AppWorld SDK evaluator (`--sdk`). + For `provider=litellm`, the default model changes to `aws/gpt-oss-120b` unless `model_name` is explicitly provided. ## Command Parameters diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index 1ad6a1d1..0d1e9cbd 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -498,6 +498,9 @@ EVAL_ARGS=( ) if [[ "${BENCHMARK}" == "appworld" ]]; then + if [[ "${AGENT}" == "cuga" ]]; then + EVAL_ARGS+=(--sdk) + fi if [[ -n "${EVAL_KEY}" ]]; then EVAL_ARGS+=(--eval-key "${EVAL_KEY}") fi From a7ed94843ba4ca67acebdf2bef57f560004ab5b7 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 5 Oct 2026 11:59:16 -0400 Subject: [PATCH 40/40] Format react agent helper for ruff --- benchmarks/helpers/react_agent.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/helpers/react_agent.py b/benchmarks/helpers/react_agent.py index 2f46afce..8c61b51c 100644 --- a/benchmarks/helpers/react_agent.py +++ b/benchmarks/helpers/react_agent.py @@ -224,7 +224,9 @@ def _create_llm(self) -> Any: api_base = os.getenv("RITS_BASE_URL") or os.getenv("LITE_LLM_URL") or os.getenv("OPENAI_BASE_URL") api_key = os.getenv("RITS_API_KEY") or os.getenv("LITE_LLM_KEY") or os.getenv("OPENAI_API_KEY") model_name = os.getenv("MODEL_NAME") or self.model or "openai/gpt-oss-120b-a100" - logger.info(f"RITS config: model={model_name}, api_base={api_base}, api_key_present={bool(api_key)}") + logger.info( + f"RITS config: model={model_name}, api_base={api_base}, api_key_present={bool(api_key)}" + ) return ChatRits(config={"model_name": model_name, "end_point": api_base, "api_key": api_key}) else: