From bce4c635c18f5386b0a9c85e944e965e4ce7ef24 Mon Sep 17 00:00:00 2001 From: Ankita Naik Date: Mon, 17 Aug 2026 18:26:03 -0400 Subject: [PATCH 01/40] Add PR regression testing workflow setup script This script sets up a PR regression testing environment by cloning necessary repositories, installing dependencies, and configuring a GitHub Actions runner. --- scripts/setup-pr-regression-workflow.sh | 289 ++++++++++++++++++++++++ 1 file changed, 289 insertions(+) create mode 100644 scripts/setup-pr-regression-workflow.sh diff --git a/scripts/setup-pr-regression-workflow.sh b/scripts/setup-pr-regression-workflow.sh new file mode 100644 index 00000000..17ab9547 --- /dev/null +++ b/scripts/setup-pr-regression-workflow.sh @@ -0,0 +1,289 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# ============================================================================= +# PR Regression Testing VM Bootstrap +# +# Modes: +# REPO_MODE=fork -> use temporary AnkitaNaik forks (default for now) +# REPO_MODE=upstream -> use cuga-project upstream repositories +# +# Final layout: +# +# ~/pr-regression-testing/ +# ├── cuga-agent/ +# ├── cuga-eval/ +# │ └── scripts/ +# │ └── run-eval.sh +# └── github-runner/ +# └── _work/ +# +# Required VM prerequisites: +# git curl tar python3 git-lfs +# +# Required environment variables: +# GITHUB_REPO_URL +# GITHUB_RUNNER_TOKEN +# +# Optional: +# REPO_MODE=fork|upstream +# ROOT_DIR +# RUNNER_NAME +# RUNNER_LABELS +# CUGA_AGENT_BRANCH +# CUGA_EVAL_BRANCH +# RUNNER_VERSION +# ============================================================================= + +REPO_MODE="${REPO_MODE:-fork}" +ROOT_DIR="${ROOT_DIR:-$HOME/pr-regression-testing}" + +RUNNER_NAME="${RUNNER_NAME:-$(hostname)}" +RUNNER_LABELS="${RUNNER_LABELS:-run-eval}" + +CUGA_AGENT_BRANCH="${CUGA_AGENT_BRANCH:-main}" +CUGA_EVAL_BRANCH="${CUGA_EVAL_BRANCH:-main}" + +case "${REPO_MODE}" in + fork) + DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/AnkitaNaik/cuga-agent.git" + DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/AnkitaNaik/cuga-eval.git" + ;; + upstream) + DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/cuga-project/cuga-agent.git" + DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/cuga-project/cuga-eval.git" + ;; + *) + echo "ERROR: REPO_MODE must be 'fork' or 'upstream'." >&2 + exit 2 + ;; +esac + +CUGA_AGENT_REPO_URL="${CUGA_AGENT_REPO_URL:-$DEFAULT_CUGA_AGENT_REPO_URL}" +CUGA_EVAL_REPO_URL="${CUGA_EVAL_REPO_URL:-$DEFAULT_CUGA_EVAL_REPO_URL}" + +CUGA_AGENT_DIR="${ROOT_DIR}/cuga-agent" +CUGA_EVAL_DIR="${ROOT_DIR}/cuga-eval" +GH_RUNNER_DIR="${ROOT_DIR}/github-runner" + +RUN_EVAL_SCRIPT="${CUGA_EVAL_DIR}/scripts/run-pr-regression-eval.sh" +x +log() { + printf '\n[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || die "Required command '$1' is missing." +} + +clone_or_update() { + local repo_url="$1" + local branch="$2" + local dest="$3" + local name="$4" + + if [[ -d "${dest}/.git" ]]; then + log "Updating ${name}" + git -C "${dest}" fetch --all --prune + git -C "${dest}" checkout "${branch}" + git -C "${dest}" pull --ff-only origin "${branch}" + else + log "Cloning ${name}" + git clone --branch "${branch}" "${repo_url}" "${dest}" + fi +} + +: "${GITHUB_REPO_URL:?Set GITHUB_REPO_URL, e.g. https://github.com/OWNER/REPO}" +: "${GITHUB_RUNNER_TOKEN:?Set GITHUB_RUNNER_TOKEN to the temporary runner token}" + +log "Repository mode: ${REPO_MODE}" +echo "cuga-agent: ${CUGA_AGENT_REPO_URL}" +echo "cuga-eval: ${CUGA_EVAL_REPO_URL}" + +log "Checking VM prerequisites" +need_cmd git +need_cmd curl +need_cmd tar +need_cmd python3 +need_cmd git-lfs + +python3 - <<'PY' +import sys +if not ((3, 12) <= sys.version_info[:2] < (3, 14)): + raise SystemExit( + f"ERROR: Python 3.12 or 3.13 is required; found " + f"{sys.version_info.major}.{sys.version_info.minor}" + ) +PY + +if ! command -v uv >/dev/null 2>&1; then + log "Installing uv" + curl -LsSf https://astral.sh/uv/install.sh | sh + export PATH="${HOME}/.local/bin:${HOME}/.cargo/bin:${PATH}" +fi +need_cmd uv + +mkdir -p "${ROOT_DIR}" + +clone_or_update \ + "${CUGA_AGENT_REPO_URL}" \ + "${CUGA_AGENT_BRANCH}" \ + "${CUGA_AGENT_DIR}" \ + "cuga-agent" + +clone_or_update \ + "${CUGA_EVAL_REPO_URL}" \ + "${CUGA_EVAL_BRANCH}" \ + "${CUGA_EVAL_DIR}" \ + "cuga-eval" + +log "Running cuga-eval/setup_cuga.sh" +cd "${CUGA_EVAL_DIR}" +chmod u+x ./setup_cuga.sh +bash ./setup_cuga.sh + +log "Creating/syncing cuga-eval environment" +uv venv +uv sync + +log "Setting up AppWorld" +git lfs install +chmod u+x ./setup_appworld.sh +bash ./setup_appworld.sh +uv sync --group appworld + +log "Checking run-eval.sh" +if [[ ! -f "${RUN_EVAL_SCRIPT}" ]]; then + cat >&2 </dev/null 2>&1; then + log "Runner already running" +else + log "Starting runner in background" + + nohup ./run.sh > "${GH_RUNNER_DIR}/runner.log" 2>&1 & + echo $! > "${GH_RUNNER_DIR}/runner.pid" +fi + +sleep 3 + +if ! pgrep -f "${GH_RUNNER_DIR}/bin/Runner.Listener" >/dev/null 2>&1; then + echo "ERROR: Runner did not remain active." >&2 + tail -n 50 "${GH_RUNNER_DIR}/runner.log" 2>/dev/null || true + exit 1 +fi + +echo +echo "======================================================================" +echo "PR regression testing VM is ready" +echo "======================================================================" +echo +echo "Repository mode:" +echo " ${REPO_MODE}" +echo +echo "Layout:" +echo " ${ROOT_DIR}/" +echo " ├── cuga-agent/" +echo " ├── cuga-eval/" +echo " │ └── scripts/run-eval.sh" +echo " └── github-runner/" +echo " └── _work/" +echo +echo "Repos:" +echo " cuga-agent: ${CUGA_AGENT_REPO_URL}" +echo " cuga-eval: ${CUGA_EVAL_REPO_URL}" +echo +echo "Runner logs:" +echo " tail -f ${GH_RUNNER_DIR}/runner.log" +echo +echo "Workflow target:" +echo " runs-on: [self-hosted, linux, ${RUNNER_LABELS}]" +echo +echo "Workflow script:" +echo " bash \"${RUN_EVAL_SCRIPT}\" \"\${COMMENT_BODY}\"" From 6fa7bdc7738cf4c970c21c624f7f3cd52f0bf6fe Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 19:13:25 -0400 Subject: [PATCH 02/40] feat(pr-eval): Update pr-regression setup script and add eval.sh script. --- scripts/setup-pr-regression-workflow.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/setup-pr-regression-workflow.sh b/scripts/setup-pr-regression-workflow.sh index 17ab9547..6b5ff34b 100644 --- a/scripts/setup-pr-regression-workflow.sh +++ b/scripts/setup-pr-regression-workflow.sh @@ -14,7 +14,8 @@ set -Eeuo pipefail # ├── cuga-agent/ # ├── cuga-eval/ # │ └── scripts/ -# │ └── run-eval.sh +# │ ├── run-pr-regression-eval.sh +# │ └── run-pr-eval.sh -> run-pr-regression-eval.sh # └── github-runner/ # └── _work/ # @@ -32,6 +33,7 @@ set -Eeuo pipefail # RUNNER_LABELS # CUGA_AGENT_BRANCH # CUGA_EVAL_BRANCH +# RUN_EVAL_SCRIPT # RUNNER_VERSION # ============================================================================= @@ -39,7 +41,7 @@ REPO_MODE="${REPO_MODE:-fork}" ROOT_DIR="${ROOT_DIR:-$HOME/pr-regression-testing}" RUNNER_NAME="${RUNNER_NAME:-$(hostname)}" -RUNNER_LABELS="${RUNNER_LABELS:-run-eval}" +RUNNER_LABELS="${RUNNER_LABELS:-run-pr-eval}" CUGA_AGENT_BRANCH="${CUGA_AGENT_BRANCH:-main}" CUGA_EVAL_BRANCH="${CUGA_EVAL_BRANCH:-main}" @@ -66,8 +68,9 @@ CUGA_AGENT_DIR="${ROOT_DIR}/cuga-agent" CUGA_EVAL_DIR="${ROOT_DIR}/cuga-eval" GH_RUNNER_DIR="${ROOT_DIR}/github-runner" -RUN_EVAL_SCRIPT="${CUGA_EVAL_DIR}/scripts/run-pr-regression-eval.sh" -x +RUN_EVAL_SCRIPT="${RUN_EVAL_SCRIPT:-${CUGA_EVAL_DIR}/scripts/run-pr-regression-eval.sh}" +WORKFLOW_EVAL_SCRIPT="${CUGA_EVAL_DIR}/scripts/run-pr-eval.sh" + log() { printf '\n[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" } @@ -157,7 +160,7 @@ chmod u+x ./setup_appworld.sh bash ./setup_appworld.sh uv sync --group appworld -log "Checking run-eval.sh" +log "Checking PR regression evaluation entrypoint" if [[ ! -f "${RUN_EVAL_SCRIPT}" ]]; then cat >&2 < Date: Mon, 17 Aug 2026 19:15:42 -0400 Subject: [PATCH 03/40] Add main pr-regression runner. --- scripts/run-pr-regression-eval.sh | 184 ++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 scripts/run-pr-regression-eval.sh diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh new file mode 100644 index 00000000..536f4f78 --- /dev/null +++ b/scripts/run-pr-regression-eval.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +START_EPOCH="$(date +%s)" +START_TIME="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" + +log() { + printf '[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" +} + +on_exit() { + status=$? + end_epoch="$(date +%s)" + end_time="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" + duration=$((end_epoch - START_EPOCH + + minutes=$((duration / 60)) + seconds=$((duration % 60)))) + + + echo "######## REPORT START ########" + echo + echo "## Run summary" + echo "- Started: ${START_TIME}" + echo "- Finished: ${end_time}" + echo "- Duration: ${minutes}m ${seconds}s" + echo "- Exit code: ${status}" + + if [[ ${status} -eq 0 ]]; then + echo "- Result: SUCCESS" + else + echo "- Result: FAILED" + fi + echo "######## REPORT END ########" +} +trap on_exit EXIT + +: "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" + +DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" +DEFAULT_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" +DEFAULT_SPLIT_NAME="default" +DEFAULT_NUM_TASKS="4" + +MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" +TASK_IDS="${TASK_IDS:-$DEFAULT_TASK_IDS}" +SPLIT_NAME="${SPLIT_NAME:-$DEFAULT_SPLIT_NAME}" +NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" + +# Parse whitespace-separated key=value parameters from the PR comment. +# Supported aliases: +# model_name=... +# task_id=id1,id2 +# task_ids=id1,id2 +# split_name=... +# num_tasks=... +COMMENT_BODY="${1:-}" + +# Remove Windows carriage returns and newlines. +COMMENT_BODY="${COMMENT_BODY//$'\r'/ }" +COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" + +read -r -a TOKENS <<< "${COMMENT_BODY}" +echo "######## REPORT START ########" +for token in "${TOKENS[@]}"; do + case "${token}" in + /run-eval|/appworld-eval) + ;; + + model_name=*) + MODEL_NAME="${token#model_name=}" + ;; + + task_id=*|task_ids=*) + TASK_IDS="${token#*=}" + TASK_IDS="${TASK_IDS//,/ }" + ;; + + split_name=*) + SPLIT_NAME="${token#split_name=}" + ;; + + num_tasks=*) + NUM_TASKS="${token#num_tasks=}" + ;; + + "") + ;; + + *) + echo "ERROR: Unsupported parameter: ${token}" + echo "Supported parameters: model_name, task_id, task_ids, split_name, num_tasks" + exit 2 + ;; + esac +done + +if [[ -z "${MODEL_NAME}" ]]; then + echo "ERROR: model_name cannot be empty." + exit 2 +fi + +if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then + echo "ERROR: num_tasks must be a positive integer." + exit 2 +fi + +read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" + +if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then + echo "ERROR: At least one task_id is required." + exit 2 +fi +echo "######## REPORT END ########" + +# num_tasks is currently a dummy input, but limit the supplied task list to make +# the mock behavior predictable. +if [[ ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then + TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") +fi + +export DYNACONF_SERVER_PORTS__REGISTRY="${DYNACONF_SERVER_PORTS__REGISTRY:-8100}" +export AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-settings.rits.toml}" +export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" +export MODEL_NAME +export ENVIRONMENT_URL="${ENVIRONMENT_URL:-http://127.0.0.1:8000}" +export APIS_URL="${APIS_URL:-http://127.0.0.1:9000}" + +EVAL_REPO="/proj/cuga_models/cuga-eval" + +if [[ ! -d "${EVAL_REPO}" ]]; then + echo "ERROR: Evaluation repo not found: ${EVAL_REPO}" + exit 1 +fi + +cd "${EVAL_REPO}" + +echo "######## REPORT START ########" +echo "# PR Evaluation" +echo +echo "## Run metadata" +echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" +echo "- PR: ${PR_NUMBER:-unknown}" +echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" +echo "- Runner: ${RUNNER_NAME:-$(hostname)}" +echo +echo "## Evaluation configuration" +echo "- Model: ${MODEL_NAME}" +echo "- Split: ${SPLIT_NAME}" +echo "- Requested num_tasks: ${NUM_TASKS}" +echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" +echo "- Task IDs:" +for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" +done +echo +echo "######## REPORT END ########" + +log "Verifying required commands" +command -v uv >/dev/null 2>&1 || { + echo "ERROR: uv is not installed or is not in PATH." + exit 127 +} + +command -v curl >/dev/null 2>&1 || { + echo "ERROR: curl is not installed or is not in PATH." + exit 127 +} + +log "Verifying local benchmark services" +for endpoint in "${ENVIRONMENT_URL}" "${APIS_URL}"; do + if ! curl --silent --show-error --fail --max-time 5 "${endpoint}" >/dev/null; then + echo "ERROR: Required service is unavailable at ${endpoint}" + exit 1 + fi +done + +log "Starting AppWorld React evaluation" + +uv run --no-sync python -m benchmarks.appworld.appworld_eval_react \ + --agent react \ + --task-id "${TASK_ID_ARRAY[@]}" \ + --environment-url "${ENVIRONMENT_URL}" \ + --apis-url "${APIS_URL}" From 37b275ac6060ac1224031e1ffa219afed40b750f Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 20:00:23 -0400 Subject: [PATCH 04/40] Update hard-coded paths in regression runner for VM compatability. --- scripts/run-pr-regression-eval.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) mode change 100644 => 100755 scripts/run-pr-regression-eval.sh diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh old mode 100644 new mode 100755 index 536f4f78..71ad8f64 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -12,11 +12,9 @@ on_exit() { status=$? end_epoch="$(date +%s)" end_time="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" - duration=$((end_epoch - START_EPOCH - + duration=$((end_epoch - START_EPOCH)) minutes=$((duration / 60)) - seconds=$((duration % 60)))) - + seconds=$((duration % 60)) echo "######## REPORT START ########" echo @@ -62,9 +60,10 @@ COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" read -r -a TOKENS <<< "${COMMENT_BODY}" echo "######## REPORT START ########" +echo "## Requested parameters" for token in "${TOKENS[@]}"; do case "${token}" in - /run-eval|/appworld-eval) + /run-pr-eval) ;; model_name=*) @@ -90,6 +89,7 @@ for token in "${TOKENS[@]}"; do *) echo "ERROR: Unsupported parameter: ${token}" echo "Supported parameters: model_name, task_id, task_ids, split_name, num_tasks" + echo "######## REPORT END ########" exit 2 ;; esac @@ -97,11 +97,13 @@ done if [[ -z "${MODEL_NAME}" ]]; then echo "ERROR: model_name cannot be empty." + echo "######## REPORT END ########" exit 2 fi if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then echo "ERROR: num_tasks must be a positive integer." + echo "######## REPORT END ########" exit 2 fi @@ -109,8 +111,13 @@ read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then echo "ERROR: At least one task_id is required." + echo "######## REPORT END ########" exit 2 fi +echo "- Model: ${MODEL_NAME}" +echo "- Split: ${SPLIT_NAME}" +echo "- Num tasks: ${NUM_TASKS}" +echo "- Task IDs: ${TASK_IDS}" echo "######## REPORT END ########" # num_tasks is currently a dummy input, but limit the supplied task list to make @@ -126,7 +133,8 @@ export MODEL_NAME export ENVIRONMENT_URL="${ENVIRONMENT_URL:-http://127.0.0.1:8000}" export APIS_URL="${APIS_URL:-http://127.0.0.1:9000}" -EVAL_REPO="/proj/cuga_models/cuga-eval" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EVAL_REPO="${EVAL_REPO:-$(cd "${SCRIPT_DIR}/.." && pwd)}" if [[ ! -d "${EVAL_REPO}" ]]; then echo "ERROR: Evaluation repo not found: ${EVAL_REPO}" From 0d6e49abf065ce600e3b4d1c9cb4cfc5a8debce5 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 20:25:03 -0400 Subject: [PATCH 05/40] Add support for RITS in the react agent. --- benchmarks/appworld/appworld_eval_react.py | 7 ++-- benchmarks/helpers/react_agent.py | 43 +++++++++++++++++----- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/benchmarks/appworld/appworld_eval_react.py b/benchmarks/appworld/appworld_eval_react.py index fbbd5a5b..a8d269cc 100644 --- a/benchmarks/appworld/appworld_eval_react.py +++ b/benchmarks/appworld/appworld_eval_react.py @@ -33,6 +33,7 @@ from datetime import datetime from typing import Any, Optional +import httpx from appworld import AppWorld, load_task_ids # pyright: ignore[reportAttributeAccessIssue] from cuga.backend.activity_tracker.tracker import ActivityTracker, Step from cuga.backend.cuga_graph.state.agent_state import VariablesManager @@ -181,13 +182,13 @@ def _print_appworld_summary(report: dict): avg_steps = report.get("avg_steps", 0) avg_duration = report.get("avg_duration", 0) - print("\n" + "=" * 80) - print("EVALUATION COMPLETE") - print("=" * 80) + print("######## REPORT START ########") + print("## EVALUATION COMPLETE") print(f"Total Tasks: {total}") print(f"Completed: {completed}/{total} ({success_rate:.1%})") print(f"Avg Steps: {avg_steps:.2f}") print(f"Avg Duration: {avg_duration:.2f}s") + print("######## REPORT END ########") def _render_initial_prompt(world: AppWorld) -> str: diff --git a/benchmarks/helpers/react_agent.py b/benchmarks/helpers/react_agent.py index be1c941c..2b9f8526 100644 --- a/benchmarks/helpers/react_agent.py +++ b/benchmarks/helpers/react_agent.py @@ -19,6 +19,7 @@ from pathlib import Path from typing import Any, Optional +from langchain_openai import ChatOpenAI from cuga.backend.cuga_graph.nodes.cuga_lite.providers.combined import ( CombinedToolProvider, ) @@ -44,6 +45,30 @@ class ReactInvokeResult: react_steps: int = 0 +class ChatRits(ChatOpenAI): + """RITS chat model integration using langchain-openai.""" + + def __init__(self, config: dict[str, Any]): + model_name = config.get("model_name", "openai/gpt-oss-120b-a100") + end_point = config.get( + "end_point", + "https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100", + ) + rits_api_key = config.get("api_key") or os.getenv("RITS_API_KEY") + if rits_api_key is None: + raise ValueError("RITS_API_KEY is required for settings.rits.toml") + + params = config.get("params", {}) + rits_config: dict[str, Any] = { + "model_name": model_name, + "api_key": "/", + "default_headers": {"RITS_API_KEY": rits_api_key}, + "base_url": end_point.rstrip("/") + "/v1", + } + rits_config.update(params) + super().__init__(**rits_config) + + class GenericReactAgent: """Prompt-template driven generic ReAct agent over benchmark tools.""" @@ -149,14 +174,6 @@ def _create_llm(self) -> Any: ) elif settings_config == "settings.openai.toml": - try: - from langchain_openai import ChatOpenAI - except ImportError as exc: - raise RuntimeError( - "langchain-openai is required for React agent with OpenAI. " - "Install with: pip install langchain-openai" - ) from exc - disable_ssl = os.getenv("CUGA_DISABLE_SSL", "").lower() in ("true", "1", "yes") ssl_verify = ( os.getenv("OPENAI_SSL_VERIFY", "true").lower() not in ("false", "0", "no") and not disable_ssl @@ -183,10 +200,18 @@ def _create_llm(self) -> Any: return ChatOpenAI(**llm_kwargs) + elif settings_config in ("settings.rits.toml", "settings.rits.proxy.toml"): + api_base = os.getenv("LITE_LLM_URL") or os.getenv("OPENAI_BASE_URL") or os.getenv("RITS_BASE_URL") + api_key = os.getenv("LITE_LLM_KEY") or os.getenv("OPENAI_API_KEY") or os.getenv("RITS_API_KEY") + model_name = os.getenv("MODEL_NAME") or self.model or "openai/gpt-oss-120b-a100" + logger.info(f"RITS config: model={model_name}, api_base={api_base}, api_key_present={bool(api_key)}") + return ChatRits(config={"model_name": model_name, "end_point": api_base, "api_key": api_key}) + else: raise RuntimeError( "Unsupported AGENT_SETTING_CONFIG for React agent. " - "Expected 'settings.groq.toml' or 'settings.openai.toml'." + "Expected 'settings.groq.toml', 'settings.openai.toml', " + "'settings.rits.toml', or 'settings.rits.proxy.toml'." ) def _get_llm(self) -> Any: From 70db4cda548956d323ca9bd56e71962ad355a0e6 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 20:36:11 -0400 Subject: [PATCH 06/40] Remove hard-coded port for AppWorld. --- scripts/run-pr-regression-eval.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 71ad8f64..e0935fc5 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -126,7 +126,6 @@ if [[ ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") fi -export DYNACONF_SERVER_PORTS__REGISTRY="${DYNACONF_SERVER_PORTS__REGISTRY:-8100}" export AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-settings.rits.toml}" export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" export MODEL_NAME From bbeb57f4c2cc579d9fbccfd81615ca63aae9b261 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 20:53:25 -0400 Subject: [PATCH 07/40] Remove hard-coded ports. --- scripts/run-pr-regression-eval.sh | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index e0935fc5..8d6b1071 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -129,8 +129,6 @@ fi export AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-settings.rits.toml}" export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" export MODEL_NAME -export ENVIRONMENT_URL="${ENVIRONMENT_URL:-http://127.0.0.1:8000}" -export APIS_URL="${APIS_URL:-http://127.0.0.1:9000}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" EVAL_REPO="${EVAL_REPO:-$(cd "${SCRIPT_DIR}/.." && pwd)}" @@ -174,18 +172,11 @@ command -v curl >/dev/null 2>&1 || { exit 127 } -log "Verifying local benchmark services" -for endpoint in "${ENVIRONMENT_URL}" "${APIS_URL}"; do - if ! curl --silent --show-error --fail --max-time 5 "${endpoint}" >/dev/null; then - echo "ERROR: Required service is unavailable at ${endpoint}" - exit 1 - fi -done - log "Starting AppWorld React evaluation" -uv run --no-sync python -m benchmarks.appworld.appworld_eval_react \ - --agent react \ - --task-id "${TASK_ID_ARRAY[@]}" \ - --environment-url "${ENVIRONMENT_URL}" \ - --apis-url "${APIS_URL}" +EVAL_ARGS=( + --agent react + --task "${TASK_ID_ARRAY[@]}" +) + +bash benchmarks/appworld/eval.sh "${EVAL_ARGS[@]}" From 13629abb9007b82d03880201bc05725cc028936a Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:01:24 -0400 Subject: [PATCH 08/40] Default using Langfuse to false.: --- scripts/run-pr-regression-eval.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 8d6b1071..8aa2e425 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -127,6 +127,7 @@ if [[ ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then fi export AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-settings.rits.toml}" +export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" export MODEL_NAME From 802906b0a475f79686284b147089db185d7b5def Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:12:49 -0400 Subject: [PATCH 09/40] Add support for all the agents for AppWorld. --- scripts/run-pr-regression-eval.sh | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 8aa2e425..37d28419 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -39,11 +39,13 @@ DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" DEFAULT_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" DEFAULT_SPLIT_NAME="default" DEFAULT_NUM_TASKS="4" +DEFAULT_AGENT="react" MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" TASK_IDS="${TASK_IDS:-$DEFAULT_TASK_IDS}" SPLIT_NAME="${SPLIT_NAME:-$DEFAULT_SPLIT_NAME}" NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" +AGENT="${AGENT:-$DEFAULT_AGENT}" # Parse whitespace-separated key=value parameters from the PR comment. # Supported aliases: @@ -52,6 +54,7 @@ NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" # task_ids=id1,id2 # split_name=... # num_tasks=... +# agent=react|cuga|codeact COMMENT_BODY="${1:-}" # Remove Windows carriage returns and newlines. @@ -83,12 +86,16 @@ for token in "${TOKENS[@]}"; do NUM_TASKS="${token#num_tasks=}" ;; + agent=*|agent_type=*) + AGENT="${token#*=}" + ;; + "") ;; *) echo "ERROR: Unsupported parameter: ${token}" - echo "Supported parameters: model_name, task_id, task_ids, split_name, num_tasks" + echo "Supported parameters: model_name, task_id, task_ids, split_name, num_tasks, agent" echo "######## REPORT END ########" exit 2 ;; @@ -107,6 +114,16 @@ if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then exit 2 fi +case "${AGENT}" in + react|cuga|codeact) + ;; + *) + echo "ERROR: agent must be one of: react, cuga, codeact." + echo "######## REPORT END ########" + exit 2 + ;; +esac + read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then @@ -115,6 +132,7 @@ if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then exit 2 fi echo "- Model: ${MODEL_NAME}" +echo "- Agent: ${AGENT}" echo "- Split: ${SPLIT_NAME}" echo "- Num tasks: ${NUM_TASKS}" echo "- Task IDs: ${TASK_IDS}" @@ -151,6 +169,7 @@ echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown) echo "- Runner: ${RUNNER_NAME:-$(hostname)}" echo echo "## Evaluation configuration" +echo "- Agent: ${AGENT}" echo "- Model: ${MODEL_NAME}" echo "- Split: ${SPLIT_NAME}" echo "- Requested num_tasks: ${NUM_TASKS}" @@ -173,10 +192,10 @@ command -v curl >/dev/null 2>&1 || { exit 127 } -log "Starting AppWorld React evaluation" +log "Starting AppWorld ${AGENT} evaluation" EVAL_ARGS=( - --agent react + --agent "${AGENT}" --task "${TASK_ID_ARRAY[@]}" ) From e99de5c5f4978d2a4cc17890d5a349c12cdc4fc1 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:27:39 -0400 Subject: [PATCH 10/40] Log the complete report in the PR comment. --- benchmarks/helpers/compare_report.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/helpers/compare_report.py b/benchmarks/helpers/compare_report.py index fc8d0cc9..75bf0045 100644 --- a/benchmarks/helpers/compare_report.py +++ b/benchmarks/helpers/compare_report.py @@ -1349,7 +1349,8 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: fence_open = (lambda: "```text") if markdown else (lambda: "") fence_close = (lambda: "```") if markdown else (lambda: "") - lines = [h1("Evaluation Report"), ""] + lines = ["######## REPORT START ########"] + lines.extend([h1("Evaluation Report"), ""]) lines.append(h2("Summary")) lines.append("") if markdown: @@ -1589,6 +1590,7 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: markdown=markdown, ) ) + lines.extend("######## REPORT END ########") return "\n".join(lines) From dc8b4881183769c512e6cc2489784293b4cf6f42 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:28:01 -0400 Subject: [PATCH 11/40] Log the agent name. --- scripts/run-pr-regression-eval.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 37d28419..63b516ba 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -167,6 +167,7 @@ echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" echo "- PR: ${PR_NUMBER:-unknown}" echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" echo "- Runner: ${RUNNER_NAME:-$(hostname)}" +echo "- Agent: ${AGENT}" echo echo "## Evaluation configuration" echo "- Agent: ${AGENT}" From 724b62de27f5cc1be6f076bef04ccd52ea170836 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:31:15 -0400 Subject: [PATCH 12/40] fix(appworld): skip Langfuse fetch when tracing is disabled --- benchmarks/appworld/appworld_eval.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmarks/appworld/appworld_eval.py b/benchmarks/appworld/appworld_eval.py index 8c45a10f..7d34d6bd 100644 --- a/benchmarks/appworld/appworld_eval.py +++ b/benchmarks/appworld/appworld_eval.py @@ -335,8 +335,9 @@ async def run_agent_on_task( langfuse_data = None # Extract Langfuse data if trace_id is available - langfuse_handler = LangfuseTraceHandler(langfuse_trace_id) - langfuse_data = await langfuse_handler.get_langfuse_data() + if langfuse_trace_id: + langfuse_handler = LangfuseTraceHandler(langfuse_trace_id) + langfuse_data = await langfuse_handler.get_langfuse_data() if langfuse_trace_id: task_result.trace_id = langfuse_trace_id if langfuse_data: From b0e2af6620d7793637726b3bf145ad4176d1e3dc Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:38:17 -0400 Subject: [PATCH 13/40] fix(pr-eval): improve collapsible PR report formatting --- benchmarks/helpers/compare_report.py | 13 ++++++------ scripts/run-pr-regression-eval.sh | 30 +++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/benchmarks/helpers/compare_report.py b/benchmarks/helpers/compare_report.py index 75bf0045..9bf715f3 100644 --- a/benchmarks/helpers/compare_report.py +++ b/benchmarks/helpers/compare_report.py @@ -1590,7 +1590,7 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: markdown=markdown, ) ) - lines.extend("######## REPORT END ########") + lines.append("######## REPORT END ########") return "\n".join(lines) @@ -1628,14 +1628,15 @@ def main(): # the canonical bundle location at the end of the run, which is what the # user actually wants to navigate to. if args.output: - # Compare-mode and eval-mode both produce markdown for the saved file - # and re-render plain text for the terminal. + # Compare-mode and eval-mode both produce markdown for the saved file. + # For eval-mode, also print markdown to stdout because PR workflows + # capture stdout and post marked report blocks back to GitHub comments. if "command" in args and getattr(args, "command", None) == "eval": - plain = generate_eval_report(args.result_file, markdown=False) + stdout_report = report else: - plain = generate_report(config_results, markdown=False) + stdout_report = generate_report(config_results, markdown=False) Path(args.output).write_text(report) - print(plain) + print(stdout_report) else: # No file requested — just print whatever generate_* produced. print(report) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 63b516ba..57c150bc 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -8,6 +8,18 @@ log() { printf '[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" } +open_details() { + local title="$1" + echo "
" + echo "${title}" + echo +} + +close_details() { + echo + echo "
" +} + on_exit() { status=$? end_epoch="$(date +%s)" @@ -18,7 +30,7 @@ on_exit() { echo "######## REPORT START ########" echo - echo "## Run summary" + open_details "Run summary" echo "- Started: ${START_TIME}" echo "- Finished: ${end_time}" echo "- Duration: ${minutes}m ${seconds}s" @@ -29,6 +41,7 @@ on_exit() { else echo "- Result: FAILED" fi + close_details echo "######## REPORT END ########" } trap on_exit EXIT @@ -63,7 +76,7 @@ COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" read -r -a TOKENS <<< "${COMMENT_BODY}" echo "######## REPORT START ########" -echo "## Requested parameters" +open_details "Requested parameters" for token in "${TOKENS[@]}"; do case "${token}" in /run-pr-eval) @@ -96,6 +109,7 @@ for token in "${TOKENS[@]}"; do *) echo "ERROR: Unsupported parameter: ${token}" echo "Supported parameters: model_name, task_id, task_ids, split_name, num_tasks, agent" + close_details echo "######## REPORT END ########" exit 2 ;; @@ -104,12 +118,14 @@ done if [[ -z "${MODEL_NAME}" ]]; then echo "ERROR: model_name cannot be empty." + close_details echo "######## REPORT END ########" exit 2 fi if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then echo "ERROR: num_tasks must be a positive integer." + close_details echo "######## REPORT END ########" exit 2 fi @@ -119,6 +135,7 @@ case "${AGENT}" in ;; *) echo "ERROR: agent must be one of: react, cuga, codeact." + close_details echo "######## REPORT END ########" exit 2 ;; @@ -128,6 +145,7 @@ read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then echo "ERROR: At least one task_id is required." + close_details echo "######## REPORT END ########" exit 2 fi @@ -136,6 +154,7 @@ echo "- Agent: ${AGENT}" echo "- Split: ${SPLIT_NAME}" echo "- Num tasks: ${NUM_TASKS}" echo "- Task IDs: ${TASK_IDS}" +close_details echo "######## REPORT END ########" # num_tasks is currently a dummy input, but limit the supplied task list to make @@ -162,14 +181,15 @@ cd "${EVAL_REPO}" echo "######## REPORT START ########" echo "# PR Evaluation" echo -echo "## Run metadata" +open_details "Run metadata" echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" echo "- PR: ${PR_NUMBER:-unknown}" echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" echo "- Runner: ${RUNNER_NAME:-$(hostname)}" echo "- Agent: ${AGENT}" +close_details echo -echo "## Evaluation configuration" +open_details "Evaluation configuration" echo "- Agent: ${AGENT}" echo "- Model: ${MODEL_NAME}" echo "- Split: ${SPLIT_NAME}" @@ -179,7 +199,7 @@ echo "- Task IDs:" for task_id in "${TASK_ID_ARRAY[@]}"; do echo " - ${task_id}" done -echo +close_details echo "######## REPORT END ########" log "Verifying required commands" From e9b39914e5a46c93bed92937318386de6123c7cc Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:45:00 -0400 Subject: [PATCH 14/40] Format the summary report. --- benchmarks/helpers/compare_report.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/benchmarks/helpers/compare_report.py b/benchmarks/helpers/compare_report.py index 9bf715f3..f220beea 100644 --- a/benchmarks/helpers/compare_report.py +++ b/benchmarks/helpers/compare_report.py @@ -1350,9 +1350,16 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: fence_close = (lambda: "```") if markdown else (lambda: "") lines = ["######## REPORT START ########"] - lines.extend([h1("Evaluation Report"), ""]) - lines.append(h2("Summary")) - lines.append("") + if markdown: + lines.append("
") + lines.append("Evaluation Report") + lines.append("") + lines.append(h2("Summary")) + lines.append("") + else: + lines.extend([h1("Evaluation Report"), ""]) + lines.append(h2("Summary")) + lines.append("") if markdown: lines.append(f"- **Pass@1**: {parsed['passed']}/{parsed['total']} ({parsed['rate']:.1%})") lines.append(f"- **Total Tokens**: {_fmt(parsed['tokens'])}") @@ -1590,6 +1597,9 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: markdown=markdown, ) ) + if markdown: + lines.append("") + lines.append("
") lines.append("######## REPORT END ########") return "\n".join(lines) From a5a28dc1fe6bb3155cf15153dd0bd57ae37d140a Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 21:52:38 -0400 Subject: [PATCH 15/40] fix(pr-eval): support RITS credentials for CUGA agent runs --- scripts/run-pr-regression-eval.sh | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 57c150bc..57cade61 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -46,19 +46,19 @@ on_exit() { } trap on_exit EXIT -: "${OPENAI_API_KEY:?OPENAI_API_KEY is required}" - DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" DEFAULT_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" DEFAULT_SPLIT_NAME="default" DEFAULT_NUM_TASKS="4" DEFAULT_AGENT="react" +DEFAULT_AGENT_SETTING_CONFIG="settings.rits.toml" MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" TASK_IDS="${TASK_IDS:-$DEFAULT_TASK_IDS}" SPLIT_NAME="${SPLIT_NAME:-$DEFAULT_SPLIT_NAME}" NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" AGENT="${AGENT:-$DEFAULT_AGENT}" +AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-$DEFAULT_AGENT_SETTING_CONFIG}" # Parse whitespace-separated key=value parameters from the PR comment. # Supported aliases: @@ -141,6 +141,25 @@ case "${AGENT}" in ;; esac +case "${AGENT_SETTING_CONFIG}" in + settings.rits.toml|settings.rits.proxy.toml) + if [[ -z "${RITS_API_KEY:-}" ]]; then + echo "ERROR: RITS_API_KEY is required when AGENT_SETTING_CONFIG=${AGENT_SETTING_CONFIG}." + close_details + echo "######## REPORT END ########" + exit 2 + fi + ;; + *) + if [[ -z "${OPENAI_API_KEY:-}" ]]; then + echo "ERROR: OPENAI_API_KEY is required when AGENT_SETTING_CONFIG=${AGENT_SETTING_CONFIG}." + close_details + echo "######## REPORT END ########" + exit 2 + fi + ;; +esac + read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then @@ -163,7 +182,7 @@ if [[ ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") fi -export AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-settings.rits.toml}" +export AGENT_SETTING_CONFIG export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" export MODEL_NAME From 68352bd4fc07b1779bc37ccf593425b7918048d5 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 22:12:17 -0400 Subject: [PATCH 16/40] Fix logging. --- scripts/run-pr-regression-eval.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 57cade61..0b5dc93f 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -185,6 +185,11 @@ fi export AGENT_SETTING_CONFIG export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" +if [[ -z "${RITS_BASE_URL:-}" ]]; then + RITS_BASE_URL="${OPENAI_BASE_URL%/}" + [[ "${RITS_BASE_URL}" == */v1 ]] || RITS_BASE_URL="${RITS_BASE_URL}/v1" + export RITS_BASE_URL +fi export MODEL_NAME SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" From 3d4d44cb44db269e49010d29ebdeb87d829a55cb Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 17 Aug 2026 23:13:09 -0400 Subject: [PATCH 17/40] feat:Support benchmark selection in PR eval --- scripts/run-pr-regression-eval.sh | 92 +++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 24 deletions(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 0b5dc93f..4ba8558b 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -47,15 +47,15 @@ on_exit() { trap on_exit EXIT DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" -DEFAULT_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" -DEFAULT_SPLIT_NAME="default" +DEFAULT_APPWORLD_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" +DEFAULT_BENCHMARK="appworld" DEFAULT_NUM_TASKS="4" DEFAULT_AGENT="react" DEFAULT_AGENT_SETTING_CONFIG="settings.rits.toml" MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" -TASK_IDS="${TASK_IDS:-$DEFAULT_TASK_IDS}" -SPLIT_NAME="${SPLIT_NAME:-$DEFAULT_SPLIT_NAME}" +TASK_IDS="${TASK_IDS:-}" +BENCHMARK="${BENCHMARK:-$DEFAULT_BENCHMARK}" NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" AGENT="${AGENT:-$DEFAULT_AGENT}" AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-$DEFAULT_AGENT_SETTING_CONFIG}" @@ -65,7 +65,7 @@ AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-$DEFAULT_AGENT_SETTING_CONFIG}" # model_name=... # task_id=id1,id2 # task_ids=id1,id2 -# split_name=... +# benchmark=appworld|m3 # num_tasks=... # agent=react|cuga|codeact COMMENT_BODY="${1:-}" @@ -91,8 +91,8 @@ for token in "${TOKENS[@]}"; do TASK_IDS="${TASK_IDS//,/ }" ;; - split_name=*) - SPLIT_NAME="${token#split_name=}" + benchmark=*) + BENCHMARK="${token#benchmark=}" ;; num_tasks=*) @@ -108,7 +108,7 @@ for token in "${TOKENS[@]}"; do *) echo "ERROR: Unsupported parameter: ${token}" - echo "Supported parameters: model_name, task_id, task_ids, split_name, num_tasks, agent" + echo "Supported parameters: model_name, task_id, task_ids, benchmark, num_tasks, agent" close_details echo "######## REPORT END ########" exit 2 @@ -130,6 +130,9 @@ if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then exit 2 fi +AGENT="$(printf '%s' "${AGENT}" | tr '[:upper:]' '[:lower:]')" +BENCHMARK="$(printf '%s' "${BENCHMARK}" | tr '[:upper:]' '[:lower:]')" + case "${AGENT}" in react|cuga|codeact) ;; @@ -141,6 +144,17 @@ case "${AGENT}" in ;; esac +case "${BENCHMARK}" in + appworld|m3) + ;; + *) + echo "ERROR: benchmark must be one of: appworld, m3." + close_details + echo "######## REPORT END ########" + exit 2 + ;; +esac + case "${AGENT_SETTING_CONFIG}" in settings.rits.toml|settings.rits.proxy.toml) if [[ -z "${RITS_API_KEY:-}" ]]; then @@ -160,25 +174,33 @@ case "${AGENT_SETTING_CONFIG}" in ;; esac +if [[ -z "${TASK_IDS}" && "${BENCHMARK}" == "appworld" ]]; then + TASK_IDS="${DEFAULT_APPWORLD_TASK_IDS}" +fi + read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" -if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then - echo "ERROR: At least one task_id is required." +if [[ ${#TASK_ID_ARRAY[@]} -eq 0 && "${BENCHMARK}" == "appworld" ]]; then + echo "ERROR: At least one task_id is required for benchmark=appworld." close_details echo "######## REPORT END ########" exit 2 fi echo "- Model: ${MODEL_NAME}" echo "- Agent: ${AGENT}" -echo "- Split: ${SPLIT_NAME}" +echo "- Benchmark: ${BENCHMARK}" echo "- Num tasks: ${NUM_TASKS}" -echo "- Task IDs: ${TASK_IDS}" +if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + echo "- Task IDs: ${TASK_IDS}" +else + echo "- Task IDs: benchmark default" +fi close_details echo "######## REPORT END ########" -# num_tasks is currently a dummy input, but limit the supplied task list to make -# the mock behavior predictable. -if [[ ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then +# For AppWorld, num_tasks limits the explicit task list. M3 maps num_tasks to +# --max-samples-per-domain below. +if [[ "${BENCHMARK}" == "appworld" && ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") fi @@ -211,18 +233,31 @@ echo "- PR: ${PR_NUMBER:-unknown}" echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" echo "- Runner: ${RUNNER_NAME:-$(hostname)}" echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" close_details echo open_details "Evaluation configuration" echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" echo "- Model: ${MODEL_NAME}" -echo "- Split: ${SPLIT_NAME}" echo "- Requested num_tasks: ${NUM_TASKS}" -echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" -echo "- Task IDs:" -for task_id in "${TASK_ID_ARRAY[@]}"; do - echo " - ${task_id}" -done +if [[ "${BENCHMARK}" == "appworld" ]]; then + echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" + echo "- Task IDs:" + for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" + done +else + echo "- M3 max samples per domain: ${NUM_TASKS}" + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + echo "- M3 task filters:" + for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" + done + else + echo "- M3 task filters: benchmark default" + fi +fi close_details echo "######## REPORT END ########" @@ -237,11 +272,20 @@ command -v curl >/dev/null 2>&1 || { exit 127 } -log "Starting AppWorld ${AGENT} evaluation" +log "Starting ${BENCHMARK} ${AGENT} evaluation" EVAL_ARGS=( + --benchmark "${BENCHMARK}" --agent "${AGENT}" - --task "${TASK_ID_ARRAY[@]}" ) -bash benchmarks/appworld/eval.sh "${EVAL_ARGS[@]}" +if [[ "${BENCHMARK}" == "appworld" ]]; then + EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") +else + EVAL_ARGS+=(--m3-data --max-samples-per-domain "${NUM_TASKS}") + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") + fi +fi + +bash scripts/eval.sh "${EVAL_ARGS[@]}" From e66d37bc5c32c91bf50a4530e5a77dceb94c7c0f Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 07:18:49 -0400 Subject: [PATCH 18/40] Update evaluation report. --- scripts/run-pr-regression-eval.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index 4ba8558b..b149c9ce 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -225,15 +225,13 @@ fi cd "${EVAL_REPO}" echo "######## REPORT START ########" -echo "# PR Evaluation" +echo "## Evaluation Metrics" echo open_details "Run metadata" echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" echo "- PR: ${PR_NUMBER:-unknown}" echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" echo "- Runner: ${RUNNER_NAME:-$(hostname)}" -echo "- Agent: ${AGENT}" -echo "- Benchmark: ${BENCHMARK}" close_details echo open_details "Evaluation configuration" From c6b3325226357423e813e686425c2ecda336e6e1 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 07:30:44 -0400 Subject: [PATCH 19/40] Add litellm support. --- scripts/run-pr-regression-eval.sh | 66 ++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index b149c9ce..de117e5d 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -51,14 +51,14 @@ DEFAULT_APPWORLD_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" DEFAULT_BENCHMARK="appworld" DEFAULT_NUM_TASKS="4" DEFAULT_AGENT="react" -DEFAULT_AGENT_SETTING_CONFIG="settings.rits.toml" +DEFAULT_PROVIDER="rits" MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" TASK_IDS="${TASK_IDS:-}" BENCHMARK="${BENCHMARK:-$DEFAULT_BENCHMARK}" NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" AGENT="${AGENT:-$DEFAULT_AGENT}" -AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-$DEFAULT_AGENT_SETTING_CONFIG}" +PROVIDER="${PROVIDER:-$DEFAULT_PROVIDER}" # Parse whitespace-separated key=value parameters from the PR comment. # Supported aliases: @@ -68,6 +68,7 @@ AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-$DEFAULT_AGENT_SETTING_CONFIG}" # benchmark=appworld|m3 # num_tasks=... # agent=react|cuga|codeact +# provider=rits|litellm COMMENT_BODY="${1:-}" # Remove Windows carriage returns and newlines. @@ -103,12 +104,16 @@ for token in "${TOKENS[@]}"; do AGENT="${token#*=}" ;; + provider=*) + PROVIDER="${token#provider=}" + ;; + "") ;; *) echo "ERROR: Unsupported parameter: ${token}" - echo "Supported parameters: model_name, task_id, task_ids, benchmark, num_tasks, agent" + echo "Supported parameters: model_name, task_id, task_ids, benchmark, num_tasks, agent, provider" close_details echo "######## REPORT END ########" exit 2 @@ -132,6 +137,7 @@ fi AGENT="$(printf '%s' "${AGENT}" | tr '[:upper:]' '[:lower:]')" BENCHMARK="$(printf '%s' "${BENCHMARK}" | tr '[:upper:]' '[:lower:]')" +PROVIDER="$(printf '%s' "${PROVIDER}" | tr '[:upper:]' '[:lower:]')" case "${AGENT}" in react|cuga|codeact) @@ -155,25 +161,57 @@ case "${BENCHMARK}" in ;; esac -case "${AGENT_SETTING_CONFIG}" in - settings.rits.toml|settings.rits.proxy.toml) +case "${PROVIDER}" in + rits) + AGENT_SETTING_CONFIG="settings.rits.toml" if [[ -z "${RITS_API_KEY:-}" ]]; then - echo "ERROR: RITS_API_KEY is required when AGENT_SETTING_CONFIG=${AGENT_SETTING_CONFIG}." + echo "ERROR: RITS_API_KEY is required when provider=rits." close_details echo "######## REPORT END ########" exit 2 fi ;; - *) + litellm) + if [[ "${AGENT}" == "cuga" ]]; then + AGENT_SETTING_CONFIG="settings.litellm.toml" + else + AGENT_SETTING_CONFIG="settings.openai.toml" + fi + OPENAI_BASE_URL="${LITE_LLM_URL:-${OPENAI_BASE_URL:-}}" + OPENAI_API_KEY="${LITE_LLM_KEY:-${OPENAI_API_KEY:-}}" if [[ -z "${OPENAI_API_KEY:-}" ]]; then - echo "ERROR: OPENAI_API_KEY is required when AGENT_SETTING_CONFIG=${AGENT_SETTING_CONFIG}." + echo "ERROR: LITE_LLM_KEY or OPENAI_API_KEY is required when provider=litellm." + close_details + echo "######## REPORT END ########" + exit 2 + fi + if [[ -z "${OPENAI_BASE_URL:-}" ]]; then + echo "ERROR: LITE_LLM_URL or OPENAI_BASE_URL is required when provider=litellm." close_details echo "######## REPORT END ########" exit 2 fi + export OPENAI_API_KEY OPENAI_BASE_URL + ;; + *) + echo "ERROR: provider must be one of: rits, litellm." + close_details + echo "######## REPORT END ########" + exit 2 ;; esac +if [[ "${PROVIDER}" == "rits" ]]; then + export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" + if [[ -z "${RITS_BASE_URL:-}" ]]; then + RITS_BASE_URL="${OPENAI_BASE_URL%/}" + [[ "${RITS_BASE_URL}" == */v1 ]] || RITS_BASE_URL="${RITS_BASE_URL}/v1" + export RITS_BASE_URL + fi +else + : +fi + if [[ -z "${TASK_IDS}" && "${BENCHMARK}" == "appworld" ]]; then TASK_IDS="${DEFAULT_APPWORLD_TASK_IDS}" fi @@ -189,6 +227,7 @@ fi echo "- Model: ${MODEL_NAME}" echo "- Agent: ${AGENT}" echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" echo "- Num tasks: ${NUM_TASKS}" if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then echo "- Task IDs: ${TASK_IDS}" @@ -206,12 +245,6 @@ fi export AGENT_SETTING_CONFIG export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" -export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" -if [[ -z "${RITS_BASE_URL:-}" ]]; then - RITS_BASE_URL="${OPENAI_BASE_URL%/}" - [[ "${RITS_BASE_URL}" == */v1 ]] || RITS_BASE_URL="${RITS_BASE_URL}/v1" - export RITS_BASE_URL -fi export MODEL_NAME SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -232,11 +265,16 @@ echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" echo "- PR: ${PR_NUMBER:-unknown}" echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" echo "- Runner: ${RUNNER_NAME:-$(hostname)}" +echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" close_details echo open_details "Evaluation configuration" echo "- Agent: ${AGENT}" echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" +echo "- Agent setting config: ${AGENT_SETTING_CONFIG}" echo "- Model: ${MODEL_NAME}" echo "- Requested num_tasks: ${NUM_TASKS}" if [[ "${BENCHMARK}" == "appworld" ]]; then From 1857befbb584a1fc97ab7fd8d1ddace37951b49c Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 07:50:26 -0400 Subject: [PATCH 20/40] Add litellm URL. --- scripts/run-pr-regression-eval.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh index de117e5d..13caceb7 100755 --- a/scripts/run-pr-regression-eval.sh +++ b/scripts/run-pr-regression-eval.sh @@ -177,7 +177,7 @@ case "${PROVIDER}" in else AGENT_SETTING_CONFIG="settings.openai.toml" fi - OPENAI_BASE_URL="${LITE_LLM_URL:-${OPENAI_BASE_URL:-}}" + OPENAI_BASE_URL="${LITE_LLM_URL:-${OPENAI_BASE_URL:-https://ete-litellm.ai-models.vpc-int.res.ibm.com/}}" OPENAI_API_KEY="${LITE_LLM_KEY:-${OPENAI_API_KEY:-}}" if [[ -z "${OPENAI_API_KEY:-}" ]]; then echo "ERROR: LITE_LLM_KEY or OPENAI_API_KEY is required when provider=litellm." From 59939792cd3c28d3ed6856047dd5d0fa3075c0fe Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 08:15:56 -0400 Subject: [PATCH 21/40] Reorganize files and add README. --- scripts/pr-regression-testing/README.md | 199 +++++++++++ .../run-pr-regression-eval.sh | 333 ++++++++++++++++++ .../setup-pr-regression-workflow.sh | 308 ++++++++++++++++ 3 files changed, 840 insertions(+) create mode 100644 scripts/pr-regression-testing/README.md create mode 100755 scripts/pr-regression-testing/run-pr-regression-eval.sh create mode 100644 scripts/pr-regression-testing/setup-pr-regression-workflow.sh diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md new file mode 100644 index 00000000..c3e5daed --- /dev/null +++ b/scripts/pr-regression-testing/README.md @@ -0,0 +1,199 @@ +# PR Regression Testing + +This folder contains the scripts used to set up and run PR-triggered regression evaluations from GitHub Actions comments. + +## Files + +- `setup-pr-regression-workflow.sh`: bootstraps the self-hosted GitHub Actions runner VM. +- `run-pr-regression-eval.sh`: parses `/run-pr-eval` PR comments and runs the requested benchmark. + +## Setup The PR Regression Flow + +1. Add the workflow file to the `cuga-agent` repository: + + ```text + .github/workflows/run-pr-eval.yml + ``` + +2. Configure the workflow to use the self-hosted runner label: + + ```yaml + runs-on: [self-hosted, linux, run-pr-eval] + ``` + +3. In GitHub, create a self-hosted runner token for the `cuga-agent` repository. + + Go to: + + ```text + cuga-agent repository -> Settings -> Actions -> Runners -> New self-hosted runner + ``` + + Copy the repository URL and temporary runner token from the generated `config.sh` command. + +4. On the runner VM, export the required setup variables: + + ```bash + export GITHUB_REPO_URL="https://github.com/OWNER/cuga-agent" + export GITHUB_RUNNER_TOKEN="YOUR_TEMPORARY_RUNNER_TOKEN" + ``` + +5. Optionally choose repo mode and branches: + + ```bash + export REPO_MODE="fork" + export CUGA_AGENT_BRANCH="main" + export CUGA_EVAL_BRANCH="main" + ``` + + Supported `REPO_MODE` values: + + - `fork`: uses `https://github.com/AnkitaNaik/cuga-agent.git` and `https://github.com/AnkitaNaik/cuga-eval.git`. + - `upstream`: uses `https://github.com/cuga-project/cuga-agent.git` and `https://github.com/cuga-project/cuga-eval.git`. + +6. Run the setup script on the VM as the normal runner user, not with `sudo`: + + ```bash + bash /path/to/cuga-eval/scripts/pr-regression-testing/setup-pr-regression-workflow.sh + ``` + + The script will: + + - verify required commands: `git`, `curl`, `tar`, `python3`, `git-lfs`, and `uv`; + - clone or update `cuga-agent` and `cuga-eval` under `~/pr-regression-testing`; + - run `cuga-eval/setup_cuga.sh`; + - create and sync the `uv` environment; + - run AppWorld setup; + - create the workflow compatibility symlink; + - download and register the GitHub Actions runner; + - start the runner in the background. + +7. Confirm the runner is online in GitHub: + + ```text + cuga-agent repository -> Settings -> Actions -> Runners + ``` + +8. Check runner logs on the VM if needed: + + ```bash + tail -f ~/pr-regression-testing/github-runner/runner.log + ``` + +## GitHub Secrets And Variables + +Configure these in the `cuga-agent` repository: + +```text +Settings -> Secrets and variables -> Actions +``` + +Required secrets: + +- `RITS_API_KEY`: required for `provider=rits`. +- `LITE_LLM_KEY`: required for `provider=litellm`, unless `OPENAI_API_KEY` is set instead. +- `OPENAI_API_KEY`: optional fallback key for OpenAI-compatible/LiteLLM flows. + +Optional repository variables: + +- `MODEL_NAME`: default model override used by the workflow environment. +- `OPENAI_BASE_URL`: default OpenAI-compatible endpoint override. +- `LITE_LLM_URL`: LiteLLM endpoint override. If omitted, `provider=litellm` defaults to `https://ete-litellm.ai-models.vpc-int.res.ibm.com/`. + +Provider defaults used by `run-pr-regression-eval.sh`: + +- `provider=rits` uses `RITS_API_KEY` and defaults `OPENAI_BASE_URL` to `https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100`. +- `provider=litellm` uses `LITE_LLM_KEY` first, then `OPENAI_API_KEY`; it uses `LITE_LLM_URL` first, then `OPENAI_BASE_URL`, then `https://ete-litellm.ai-models.vpc-int.res.ibm.com/`. Unless `model_name` is explicitly set in the PR comment, it uses `aws/gpt-oss-120b`. + +## Run PR Evaluations + +Create a comment on a pull request in `cuga-agent`: + +```text +/run-pr-eval +``` + +By default, this runs: + +- `benchmark=appworld` +- `agent=react` +- `provider=rits` +- `model_name=openai/gpt-oss-120b-a100` +- `num_tasks=4` +- default AppWorld task IDs: `9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1` + +For `provider=litellm`, the default model changes to `aws/gpt-oss-120b` unless `model_name` is explicitly provided. + +## Command Parameters + +The command accepts whitespace-separated `key=value` parameters: + +- `model_name`: model name to evaluate. +- `task_id`: one task ID, or comma-separated task IDs. +- `task_ids`: alias for `task_id`. +- `benchmark`: `appworld` or `m3`; defaults to `appworld`. +- `num_tasks`: positive integer; defaults to `4`. +- `agent`: `react`, `cuga`, or `codeact`; defaults to `react`. +- `provider`: `rits` or `litellm`; defaults to `rits`. + +Unsupported parameters cause the workflow to fail early with a list of supported parameters. + +## Examples + +Run the default AppWorld React-agent evaluation on RITS: + +```text +/run-pr-eval +``` + +Run one AppWorld task with the React agent on RITS: + +```text +/run-pr-eval provider=rits agent=react benchmark=appworld task_id=9aae7da_1 num_tasks=1 +``` + +Run one AppWorld task with the CUGA agent on LiteLLM: + +```text +/run-pr-eval provider=litellm agent=cuga benchmark=appworld task_id=9aae7da_1 num_tasks=1 +``` + +Run AppWorld with two explicit tasks: + +```text +/run-pr-eval benchmark=appworld task_id=9aae7da_1,365e0a3_1 num_tasks=2 +``` + +Run M3 with the default React agent: + +```text +/run-pr-eval benchmark=m3 num_tasks=1 +``` + +Run M3 with the CUGA agent and LiteLLM: + +```text +/run-pr-eval benchmark=m3 agent=cuga provider=litellm num_tasks=1 +``` + +Override the model: + +```text +/run-pr-eval provider=litellm agent=cuga benchmark=appworld model_name=aws/glm-5 task_id=9aae7da_1 num_tasks=1 +``` + +## Output + +The workflow posts a PR comment containing collapsible sections for: + +- requested parameters; +- run metadata; +- evaluation configuration; +- evaluation report; +- run summary. + +The full output is also uploaded as a GitHub Actions artifact and saved on the VM under: + +```text +~/pr-regression-testing/logs/ +``` diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh new file mode 100755 index 00000000..c3d48fce --- /dev/null +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -0,0 +1,333 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +START_EPOCH="$(date +%s)" +START_TIME="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" + +log() { + printf '[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" +} + +open_details() { + local title="$1" + echo "
" + echo "${title}" + echo +} + +close_details() { + echo + echo "
" +} + +on_exit() { + status=$? + end_epoch="$(date +%s)" + end_time="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" + duration=$((end_epoch - START_EPOCH)) + minutes=$((duration / 60)) + seconds=$((duration % 60)) + + echo "######## REPORT START ########" + echo + open_details "Run summary" + echo "- Started: ${START_TIME}" + echo "- Finished: ${end_time}" + echo "- Duration: ${minutes}m ${seconds}s" + echo "- Exit code: ${status}" + + if [[ ${status} -eq 0 ]]; then + echo "- Result: SUCCESS" + else + echo "- Result: FAILED" + fi + close_details + echo "######## REPORT END ########" +} +trap on_exit EXIT + +DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" +DEFAULT_LITELLM_MODEL_NAME="aws/gpt-oss-120b" +DEFAULT_APPWORLD_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" +DEFAULT_BENCHMARK="appworld" +DEFAULT_NUM_TASKS="4" +DEFAULT_AGENT="react" +DEFAULT_PROVIDER="rits" + +MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" +TASK_IDS="${TASK_IDS:-}" +BENCHMARK="${BENCHMARK:-$DEFAULT_BENCHMARK}" +NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" +AGENT="${AGENT:-$DEFAULT_AGENT}" +PROVIDER="${PROVIDER:-$DEFAULT_PROVIDER}" +MODEL_NAME_FROM_COMMENT=false + +# Parse whitespace-separated key=value parameters from the PR comment. +# Supported aliases: +# model_name=... +# task_id=id1,id2 +# task_ids=id1,id2 +# benchmark=appworld|m3 +# num_tasks=... +# agent=react|cuga|codeact +# provider=rits|litellm +COMMENT_BODY="${1:-}" + +# Remove Windows carriage returns and newlines. +COMMENT_BODY="${COMMENT_BODY//$'\r'/ }" +COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" + +read -r -a TOKENS <<< "${COMMENT_BODY}" +echo "######## REPORT START ########" +open_details "Requested parameters" +for token in "${TOKENS[@]}"; do + case "${token}" in + /run-pr-eval) + ;; + + model_name=*) + MODEL_NAME="${token#model_name=}" + MODEL_NAME_FROM_COMMENT=true + ;; + + task_id=*|task_ids=*) + TASK_IDS="${token#*=}" + TASK_IDS="${TASK_IDS//,/ }" + ;; + + benchmark=*) + BENCHMARK="${token#benchmark=}" + ;; + + num_tasks=*) + NUM_TASKS="${token#num_tasks=}" + ;; + + agent=*|agent_type=*) + AGENT="${token#*=}" + ;; + + provider=*) + PROVIDER="${token#provider=}" + ;; + + "") + ;; + + *) + echo "ERROR: Unsupported parameter: ${token}" + echo "Supported parameters: model_name, task_id, task_ids, benchmark, num_tasks, agent, provider" + close_details + echo "######## REPORT END ########" + exit 2 + ;; + esac +done + +if [[ -z "${MODEL_NAME}" ]]; then + echo "ERROR: model_name cannot be empty." + close_details + echo "######## REPORT END ########" + exit 2 +fi + +if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then + echo "ERROR: num_tasks must be a positive integer." + close_details + echo "######## REPORT END ########" + exit 2 +fi + +AGENT="$(printf '%s' "${AGENT}" | tr '[:upper:]' '[:lower:]')" +BENCHMARK="$(printf '%s' "${BENCHMARK}" | tr '[:upper:]' '[:lower:]')" +PROVIDER="$(printf '%s' "${PROVIDER}" | tr '[:upper:]' '[:lower:]')" + +case "${AGENT}" in + react|cuga|codeact) + ;; + *) + echo "ERROR: agent must be one of: react, cuga, codeact." + close_details + echo "######## REPORT END ########" + exit 2 + ;; +esac + +case "${BENCHMARK}" in + appworld|m3) + ;; + *) + echo "ERROR: benchmark must be one of: appworld, m3." + close_details + echo "######## REPORT END ########" + exit 2 + ;; +esac + +case "${PROVIDER}" in + rits) + AGENT_SETTING_CONFIG="settings.rits.toml" + if [[ -z "${RITS_API_KEY:-}" ]]; then + echo "ERROR: RITS_API_KEY is required when provider=rits." + close_details + echo "######## REPORT END ########" + exit 2 + fi + ;; + litellm) + if [[ "${AGENT}" == "cuga" ]]; then + AGENT_SETTING_CONFIG="settings.litellm.toml" + else + AGENT_SETTING_CONFIG="settings.openai.toml" + fi + if [[ "${MODEL_NAME_FROM_COMMENT}" == "false" ]]; then + MODEL_NAME="${DEFAULT_LITELLM_MODEL_NAME}" + fi + OPENAI_BASE_URL="${LITE_LLM_URL:-${OPENAI_BASE_URL:-https://ete-litellm.ai-models.vpc-int.res.ibm.com/}}" + OPENAI_API_KEY="${LITE_LLM_KEY:-${OPENAI_API_KEY:-}}" + if [[ -z "${OPENAI_API_KEY:-}" ]]; then + echo "ERROR: LITE_LLM_KEY or OPENAI_API_KEY is required when provider=litellm." + close_details + echo "######## REPORT END ########" + exit 2 + fi + if [[ -z "${OPENAI_BASE_URL:-}" ]]; then + echo "ERROR: LITE_LLM_URL or OPENAI_BASE_URL is required when provider=litellm." + close_details + echo "######## REPORT END ########" + exit 2 + fi + export OPENAI_API_KEY OPENAI_BASE_URL + ;; + *) + echo "ERROR: provider must be one of: rits, litellm." + close_details + echo "######## REPORT END ########" + exit 2 + ;; +esac + +if [[ "${PROVIDER}" == "rits" ]]; then + export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" + if [[ -z "${RITS_BASE_URL:-}" ]]; then + RITS_BASE_URL="${OPENAI_BASE_URL%/}" + [[ "${RITS_BASE_URL}" == */v1 ]] || RITS_BASE_URL="${RITS_BASE_URL}/v1" + export RITS_BASE_URL + fi +else + : +fi + +if [[ -z "${TASK_IDS}" && "${BENCHMARK}" == "appworld" ]]; then + TASK_IDS="${DEFAULT_APPWORLD_TASK_IDS}" +fi + +read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" + +if [[ ${#TASK_ID_ARRAY[@]} -eq 0 && "${BENCHMARK}" == "appworld" ]]; then + echo "ERROR: At least one task_id is required for benchmark=appworld." + close_details + echo "######## REPORT END ########" + exit 2 +fi +echo "- Model: ${MODEL_NAME}" +echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" +echo "- Num tasks: ${NUM_TASKS}" +if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + echo "- Task IDs: ${TASK_IDS}" +else + echo "- Task IDs: benchmark default" +fi +close_details +echo "######## REPORT END ########" + +# For AppWorld, num_tasks limits the explicit task list. M3 maps num_tasks to +# --max-samples-per-domain below. +if [[ "${BENCHMARK}" == "appworld" && ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then + TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") +fi + +export AGENT_SETTING_CONFIG +export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" +export MODEL_NAME + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EVAL_REPO="${EVAL_REPO:-$(cd "${SCRIPT_DIR}/.." && pwd)}" + +if [[ ! -d "${EVAL_REPO}" ]]; then + echo "ERROR: Evaluation repo not found: ${EVAL_REPO}" + exit 1 +fi + +cd "${EVAL_REPO}" + +echo "######## REPORT START ########" +echo "## Evaluation Metrics" +echo +open_details "Run metadata" +echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" +echo "- PR: ${PR_NUMBER:-unknown}" +echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" +echo "- Runner: ${RUNNER_NAME:-$(hostname)}" +echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" +close_details +echo +open_details "Evaluation configuration" +echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" +echo "- Agent setting config: ${AGENT_SETTING_CONFIG}" +echo "- Model: ${MODEL_NAME}" +echo "- Requested num_tasks: ${NUM_TASKS}" +if [[ "${BENCHMARK}" == "appworld" ]]; then + echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" + echo "- Task IDs:" + for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" + done +else + echo "- M3 max samples per domain: ${NUM_TASKS}" + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + echo "- M3 task filters:" + for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" + done + else + echo "- M3 task filters: benchmark default" + fi +fi +close_details +echo "######## REPORT END ########" + +log "Verifying required commands" +command -v uv >/dev/null 2>&1 || { + echo "ERROR: uv is not installed or is not in PATH." + exit 127 +} + +command -v curl >/dev/null 2>&1 || { + echo "ERROR: curl is not installed or is not in PATH." + exit 127 +} + +log "Starting ${BENCHMARK} ${AGENT} evaluation" + +EVAL_ARGS=( + --benchmark "${BENCHMARK}" + --agent "${AGENT}" +) + +if [[ "${BENCHMARK}" == "appworld" ]]; then + EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") +else + EVAL_ARGS+=(--m3-data --max-samples-per-domain "${NUM_TASKS}") + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") + fi +fi + +bash scripts/eval.sh "${EVAL_ARGS[@]}" diff --git a/scripts/pr-regression-testing/setup-pr-regression-workflow.sh b/scripts/pr-regression-testing/setup-pr-regression-workflow.sh new file mode 100644 index 00000000..b3c6f730 --- /dev/null +++ b/scripts/pr-regression-testing/setup-pr-regression-workflow.sh @@ -0,0 +1,308 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# ============================================================================= +# PR Regression Testing VM Bootstrap +# +# Modes: +# REPO_MODE=fork -> use temporary AnkitaNaik forks (default for now) +# REPO_MODE=upstream -> use cuga-project upstream repositories +# +# Final layout: +# +# ~/pr-regression-testing/ +# ├── cuga-agent/ +# ├── cuga-eval/ +# │ └── scripts/ +# │ ├── pr-regression-testing/ +# │ │ ├── setup-pr-regression-workflow.sh +# │ │ └── run-pr-regression-eval.sh +# │ └── run-pr-eval.sh -> pr-regression-testing/run-pr-regression-eval.sh +# └── github-runner/ +# └── _work/ +# +# Required VM prerequisites: +# git curl tar python3 git-lfs +# +# Required environment variables: +# GITHUB_REPO_URL +# GITHUB_RUNNER_TOKEN +# +# Optional: +# REPO_MODE=fork|upstream +# ROOT_DIR +# RUNNER_NAME +# RUNNER_LABELS +# CUGA_AGENT_BRANCH +# CUGA_EVAL_BRANCH +# RUN_EVAL_SCRIPT +# RUNNER_VERSION +# ============================================================================= + +REPO_MODE="${REPO_MODE:-fork}" +ROOT_DIR="${ROOT_DIR:-$HOME/pr-regression-testing}" + +RUNNER_NAME="${RUNNER_NAME:-$(hostname)}" +RUNNER_LABELS="${RUNNER_LABELS:-run-pr-eval}" + +CUGA_AGENT_BRANCH="${CUGA_AGENT_BRANCH:-main}" +CUGA_EVAL_BRANCH="${CUGA_EVAL_BRANCH:-main}" + +case "${REPO_MODE}" in + fork) + DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/AnkitaNaik/cuga-agent.git" + DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/AnkitaNaik/cuga-eval.git" + ;; + upstream) + DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/cuga-project/cuga-agent.git" + DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/cuga-project/cuga-eval.git" + ;; + *) + echo "ERROR: REPO_MODE must be 'fork' or 'upstream'." >&2 + exit 2 + ;; +esac + +CUGA_AGENT_REPO_URL="${CUGA_AGENT_REPO_URL:-$DEFAULT_CUGA_AGENT_REPO_URL}" +CUGA_EVAL_REPO_URL="${CUGA_EVAL_REPO_URL:-$DEFAULT_CUGA_EVAL_REPO_URL}" + +CUGA_AGENT_DIR="${ROOT_DIR}/cuga-agent" +CUGA_EVAL_DIR="${ROOT_DIR}/cuga-eval" +GH_RUNNER_DIR="${ROOT_DIR}/github-runner" + +RUN_EVAL_SCRIPT="${RUN_EVAL_SCRIPT:-${CUGA_EVAL_DIR}/scripts/pr-regression-testing/run-pr-regression-eval.sh}" +WORKFLOW_EVAL_SCRIPT="${CUGA_EVAL_DIR}/scripts/run-pr-eval.sh" + +log() { + printf '\n[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || die "Required command '$1' is missing." +} + +clone_or_update() { + local repo_url="$1" + local branch="$2" + local dest="$3" + local name="$4" + + if [[ -d "${dest}/.git" ]]; then + log "Updating ${name}" + git -C "${dest}" fetch --all --prune + git -C "${dest}" checkout "${branch}" + git -C "${dest}" pull --ff-only origin "${branch}" + else + log "Cloning ${name}" + git clone --branch "${branch}" "${repo_url}" "${dest}" + fi +} + +: "${GITHUB_REPO_URL:?Set GITHUB_REPO_URL, e.g. https://github.com/OWNER/REPO}" +: "${GITHUB_RUNNER_TOKEN:?Set GITHUB_RUNNER_TOKEN to the temporary runner token}" + +log "Repository mode: ${REPO_MODE}" +echo "cuga-agent: ${CUGA_AGENT_REPO_URL}" +echo "cuga-eval: ${CUGA_EVAL_REPO_URL}" + +log "Checking VM prerequisites" +need_cmd git +need_cmd curl +need_cmd tar +need_cmd python3 +need_cmd git-lfs + +python3 - <<'PY' +import sys +if not ((3, 12) <= sys.version_info[:2] < (3, 14)): + raise SystemExit( + f"ERROR: Python 3.12 or 3.13 is required; found " + f"{sys.version_info.major}.{sys.version_info.minor}" + ) +PY + +if ! command -v uv >/dev/null 2>&1; then + log "Installing uv" + curl -LsSf https://astral.sh/uv/install.sh | sh + export PATH="${HOME}/.local/bin:${HOME}/.cargo/bin:${PATH}" +fi +need_cmd uv + +mkdir -p "${ROOT_DIR}" + +clone_or_update \ + "${CUGA_AGENT_REPO_URL}" \ + "${CUGA_AGENT_BRANCH}" \ + "${CUGA_AGENT_DIR}" \ + "cuga-agent" + +clone_or_update \ + "${CUGA_EVAL_REPO_URL}" \ + "${CUGA_EVAL_BRANCH}" \ + "${CUGA_EVAL_DIR}" \ + "cuga-eval" + +log "Running cuga-eval/setup_cuga.sh" +cd "${CUGA_EVAL_DIR}" +chmod u+x ./setup_cuga.sh +bash ./setup_cuga.sh + +log "Creating/syncing cuga-eval environment" +uv venv +uv sync + +log "Setting up AppWorld" +git lfs install +chmod u+x ./setup_appworld.sh +bash ./setup_appworld.sh +uv sync --group appworld + +log "Checking PR regression evaluation entrypoint" +if [[ ! -f "${RUN_EVAL_SCRIPT}" ]]; then + cat >&2 </dev/null 2>&1; then + log "Runner already running" +else + log "Starting runner in background" + + nohup ./run.sh > "${GH_RUNNER_DIR}/runner.log" 2>&1 & + echo $! > "${GH_RUNNER_DIR}/runner.pid" +fi + +sleep 3 + +if ! pgrep -f "${GH_RUNNER_DIR}/bin/Runner.Listener" >/dev/null 2>&1; then + echo "ERROR: Runner did not remain active." >&2 + tail -n 50 "${GH_RUNNER_DIR}/runner.log" 2>/dev/null || true + exit 1 +fi + +echo +echo "======================================================================" +echo "PR regression testing VM is ready" +echo "======================================================================" +echo +echo "Repository mode:" +echo " ${REPO_MODE}" +echo +echo "Layout:" +echo " ${ROOT_DIR}/" +echo " ├── cuga-agent/" +echo " ├── cuga-eval/" +echo " │ └── scripts/run-eval.sh" +echo " └── github-runner/" +echo " └── _work/" +echo +echo "Repos:" +echo " cuga-agent: ${CUGA_AGENT_REPO_URL}" +echo " cuga-eval: ${CUGA_EVAL_REPO_URL}" +echo +echo "Runner logs:" +echo " tail -f ${GH_RUNNER_DIR}/runner.log" +echo +echo "Workflow target:" +echo " runs-on: [self-hosted, linux, ${RUNNER_LABELS}]" +echo +echo "Workflow script:" +echo " bash \"${RUN_EVAL_SCRIPT}\" \"\${COMMENT_BODY}\"" From 14dd0554fe436bf1061e31498a4174929386d33c Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 08:27:28 -0400 Subject: [PATCH 22/40] Fix script path. --- .../pr-regression-testing/run-pr-regression-eval.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index c3d48fce..07b88bdd 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -254,13 +254,24 @@ export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURE export MODEL_NAME SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -EVAL_REPO="${EVAL_REPO:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +if [[ -z "${EVAL_REPO:-}" ]]; then + if [[ -f "${SCRIPT_DIR}/../../scripts/eval.sh" ]]; then + EVAL_REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" + else + EVAL_REPO="$(cd "${SCRIPT_DIR}/.." && pwd)" + fi +fi if [[ ! -d "${EVAL_REPO}" ]]; then echo "ERROR: Evaluation repo not found: ${EVAL_REPO}" exit 1 fi +if [[ ! -f "${EVAL_REPO}/scripts/eval.sh" ]]; then + echo "ERROR: Evaluation script not found: ${EVAL_REPO}/scripts/eval.sh" + exit 1 +fi + cd "${EVAL_REPO}" echo "######## REPORT START ########" From 48f32e1a01f135a533321ef6fc12d52246ced249 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 08:27:53 -0400 Subject: [PATCH 23/40] Reorganize files. --- scripts/run-pr-regression-eval.sh | 327 ------------------------ scripts/setup-pr-regression-workflow.sh | 297 --------------------- 2 files changed, 624 deletions(-) delete mode 100755 scripts/run-pr-regression-eval.sh delete mode 100644 scripts/setup-pr-regression-workflow.sh diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh deleted file mode 100755 index 13caceb7..00000000 --- a/scripts/run-pr-regression-eval.sh +++ /dev/null @@ -1,327 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -START_EPOCH="$(date +%s)" -START_TIME="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" - -log() { - printf '[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" -} - -open_details() { - local title="$1" - echo "
" - echo "${title}" - echo -} - -close_details() { - echo - echo "
" -} - -on_exit() { - status=$? - end_epoch="$(date +%s)" - end_time="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" - duration=$((end_epoch - START_EPOCH)) - minutes=$((duration / 60)) - seconds=$((duration % 60)) - - echo "######## REPORT START ########" - echo - open_details "Run summary" - echo "- Started: ${START_TIME}" - echo "- Finished: ${end_time}" - echo "- Duration: ${minutes}m ${seconds}s" - echo "- Exit code: ${status}" - - if [[ ${status} -eq 0 ]]; then - echo "- Result: SUCCESS" - else - echo "- Result: FAILED" - fi - close_details - echo "######## REPORT END ########" -} -trap on_exit EXIT - -DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" -DEFAULT_APPWORLD_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" -DEFAULT_BENCHMARK="appworld" -DEFAULT_NUM_TASKS="4" -DEFAULT_AGENT="react" -DEFAULT_PROVIDER="rits" - -MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" -TASK_IDS="${TASK_IDS:-}" -BENCHMARK="${BENCHMARK:-$DEFAULT_BENCHMARK}" -NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" -AGENT="${AGENT:-$DEFAULT_AGENT}" -PROVIDER="${PROVIDER:-$DEFAULT_PROVIDER}" - -# Parse whitespace-separated key=value parameters from the PR comment. -# Supported aliases: -# model_name=... -# task_id=id1,id2 -# task_ids=id1,id2 -# benchmark=appworld|m3 -# num_tasks=... -# agent=react|cuga|codeact -# provider=rits|litellm -COMMENT_BODY="${1:-}" - -# Remove Windows carriage returns and newlines. -COMMENT_BODY="${COMMENT_BODY//$'\r'/ }" -COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" - -read -r -a TOKENS <<< "${COMMENT_BODY}" -echo "######## REPORT START ########" -open_details "Requested parameters" -for token in "${TOKENS[@]}"; do - case "${token}" in - /run-pr-eval) - ;; - - model_name=*) - MODEL_NAME="${token#model_name=}" - ;; - - task_id=*|task_ids=*) - TASK_IDS="${token#*=}" - TASK_IDS="${TASK_IDS//,/ }" - ;; - - benchmark=*) - BENCHMARK="${token#benchmark=}" - ;; - - num_tasks=*) - NUM_TASKS="${token#num_tasks=}" - ;; - - agent=*|agent_type=*) - AGENT="${token#*=}" - ;; - - provider=*) - PROVIDER="${token#provider=}" - ;; - - "") - ;; - - *) - echo "ERROR: Unsupported parameter: ${token}" - echo "Supported parameters: model_name, task_id, task_ids, benchmark, num_tasks, agent, provider" - close_details - echo "######## REPORT END ########" - exit 2 - ;; - esac -done - -if [[ -z "${MODEL_NAME}" ]]; then - echo "ERROR: model_name cannot be empty." - close_details - echo "######## REPORT END ########" - exit 2 -fi - -if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then - echo "ERROR: num_tasks must be a positive integer." - close_details - echo "######## REPORT END ########" - exit 2 -fi - -AGENT="$(printf '%s' "${AGENT}" | tr '[:upper:]' '[:lower:]')" -BENCHMARK="$(printf '%s' "${BENCHMARK}" | tr '[:upper:]' '[:lower:]')" -PROVIDER="$(printf '%s' "${PROVIDER}" | tr '[:upper:]' '[:lower:]')" - -case "${AGENT}" in - react|cuga|codeact) - ;; - *) - echo "ERROR: agent must be one of: react, cuga, codeact." - close_details - echo "######## REPORT END ########" - exit 2 - ;; -esac - -case "${BENCHMARK}" in - appworld|m3) - ;; - *) - echo "ERROR: benchmark must be one of: appworld, m3." - close_details - echo "######## REPORT END ########" - exit 2 - ;; -esac - -case "${PROVIDER}" in - rits) - AGENT_SETTING_CONFIG="settings.rits.toml" - if [[ -z "${RITS_API_KEY:-}" ]]; then - echo "ERROR: RITS_API_KEY is required when provider=rits." - close_details - echo "######## REPORT END ########" - exit 2 - fi - ;; - litellm) - if [[ "${AGENT}" == "cuga" ]]; then - AGENT_SETTING_CONFIG="settings.litellm.toml" - else - AGENT_SETTING_CONFIG="settings.openai.toml" - fi - OPENAI_BASE_URL="${LITE_LLM_URL:-${OPENAI_BASE_URL:-https://ete-litellm.ai-models.vpc-int.res.ibm.com/}}" - OPENAI_API_KEY="${LITE_LLM_KEY:-${OPENAI_API_KEY:-}}" - if [[ -z "${OPENAI_API_KEY:-}" ]]; then - echo "ERROR: LITE_LLM_KEY or OPENAI_API_KEY is required when provider=litellm." - close_details - echo "######## REPORT END ########" - exit 2 - fi - if [[ -z "${OPENAI_BASE_URL:-}" ]]; then - echo "ERROR: LITE_LLM_URL or OPENAI_BASE_URL is required when provider=litellm." - close_details - echo "######## REPORT END ########" - exit 2 - fi - export OPENAI_API_KEY OPENAI_BASE_URL - ;; - *) - echo "ERROR: provider must be one of: rits, litellm." - close_details - echo "######## REPORT END ########" - exit 2 - ;; -esac - -if [[ "${PROVIDER}" == "rits" ]]; then - export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" - if [[ -z "${RITS_BASE_URL:-}" ]]; then - RITS_BASE_URL="${OPENAI_BASE_URL%/}" - [[ "${RITS_BASE_URL}" == */v1 ]] || RITS_BASE_URL="${RITS_BASE_URL}/v1" - export RITS_BASE_URL - fi -else - : -fi - -if [[ -z "${TASK_IDS}" && "${BENCHMARK}" == "appworld" ]]; then - TASK_IDS="${DEFAULT_APPWORLD_TASK_IDS}" -fi - -read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" - -if [[ ${#TASK_ID_ARRAY[@]} -eq 0 && "${BENCHMARK}" == "appworld" ]]; then - echo "ERROR: At least one task_id is required for benchmark=appworld." - close_details - echo "######## REPORT END ########" - exit 2 -fi -echo "- Model: ${MODEL_NAME}" -echo "- Agent: ${AGENT}" -echo "- Benchmark: ${BENCHMARK}" -echo "- Provider: ${PROVIDER}" -echo "- Num tasks: ${NUM_TASKS}" -if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then - echo "- Task IDs: ${TASK_IDS}" -else - echo "- Task IDs: benchmark default" -fi -close_details -echo "######## REPORT END ########" - -# For AppWorld, num_tasks limits the explicit task list. M3 maps num_tasks to -# --max-samples-per-domain below. -if [[ "${BENCHMARK}" == "appworld" && ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then - TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") -fi - -export AGENT_SETTING_CONFIG -export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" -export MODEL_NAME - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -EVAL_REPO="${EVAL_REPO:-$(cd "${SCRIPT_DIR}/.." && pwd)}" - -if [[ ! -d "${EVAL_REPO}" ]]; then - echo "ERROR: Evaluation repo not found: ${EVAL_REPO}" - exit 1 -fi - -cd "${EVAL_REPO}" - -echo "######## REPORT START ########" -echo "## Evaluation Metrics" -echo -open_details "Run metadata" -echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" -echo "- PR: ${PR_NUMBER:-unknown}" -echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" -echo "- Runner: ${RUNNER_NAME:-$(hostname)}" -echo "- Agent: ${AGENT}" -echo "- Benchmark: ${BENCHMARK}" -echo "- Provider: ${PROVIDER}" -close_details -echo -open_details "Evaluation configuration" -echo "- Agent: ${AGENT}" -echo "- Benchmark: ${BENCHMARK}" -echo "- Provider: ${PROVIDER}" -echo "- Agent setting config: ${AGENT_SETTING_CONFIG}" -echo "- Model: ${MODEL_NAME}" -echo "- Requested num_tasks: ${NUM_TASKS}" -if [[ "${BENCHMARK}" == "appworld" ]]; then - echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" - echo "- Task IDs:" - for task_id in "${TASK_ID_ARRAY[@]}"; do - echo " - ${task_id}" - done -else - echo "- M3 max samples per domain: ${NUM_TASKS}" - if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then - echo "- M3 task filters:" - for task_id in "${TASK_ID_ARRAY[@]}"; do - echo " - ${task_id}" - done - else - echo "- M3 task filters: benchmark default" - fi -fi -close_details -echo "######## REPORT END ########" - -log "Verifying required commands" -command -v uv >/dev/null 2>&1 || { - echo "ERROR: uv is not installed or is not in PATH." - exit 127 -} - -command -v curl >/dev/null 2>&1 || { - echo "ERROR: curl is not installed or is not in PATH." - exit 127 -} - -log "Starting ${BENCHMARK} ${AGENT} evaluation" - -EVAL_ARGS=( - --benchmark "${BENCHMARK}" - --agent "${AGENT}" -) - -if [[ "${BENCHMARK}" == "appworld" ]]; then - EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") -else - EVAL_ARGS+=(--m3-data --max-samples-per-domain "${NUM_TASKS}") - if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then - EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") - fi -fi - -bash scripts/eval.sh "${EVAL_ARGS[@]}" diff --git a/scripts/setup-pr-regression-workflow.sh b/scripts/setup-pr-regression-workflow.sh deleted file mode 100644 index 6b5ff34b..00000000 --- a/scripts/setup-pr-regression-workflow.sh +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -# ============================================================================= -# PR Regression Testing VM Bootstrap -# -# Modes: -# REPO_MODE=fork -> use temporary AnkitaNaik forks (default for now) -# REPO_MODE=upstream -> use cuga-project upstream repositories -# -# Final layout: -# -# ~/pr-regression-testing/ -# ├── cuga-agent/ -# ├── cuga-eval/ -# │ └── scripts/ -# │ ├── run-pr-regression-eval.sh -# │ └── run-pr-eval.sh -> run-pr-regression-eval.sh -# └── github-runner/ -# └── _work/ -# -# Required VM prerequisites: -# git curl tar python3 git-lfs -# -# Required environment variables: -# GITHUB_REPO_URL -# GITHUB_RUNNER_TOKEN -# -# Optional: -# REPO_MODE=fork|upstream -# ROOT_DIR -# RUNNER_NAME -# RUNNER_LABELS -# CUGA_AGENT_BRANCH -# CUGA_EVAL_BRANCH -# RUN_EVAL_SCRIPT -# RUNNER_VERSION -# ============================================================================= - -REPO_MODE="${REPO_MODE:-fork}" -ROOT_DIR="${ROOT_DIR:-$HOME/pr-regression-testing}" - -RUNNER_NAME="${RUNNER_NAME:-$(hostname)}" -RUNNER_LABELS="${RUNNER_LABELS:-run-pr-eval}" - -CUGA_AGENT_BRANCH="${CUGA_AGENT_BRANCH:-main}" -CUGA_EVAL_BRANCH="${CUGA_EVAL_BRANCH:-main}" - -case "${REPO_MODE}" in - fork) - DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/AnkitaNaik/cuga-agent.git" - DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/AnkitaNaik/cuga-eval.git" - ;; - upstream) - DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/cuga-project/cuga-agent.git" - DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/cuga-project/cuga-eval.git" - ;; - *) - echo "ERROR: REPO_MODE must be 'fork' or 'upstream'." >&2 - exit 2 - ;; -esac - -CUGA_AGENT_REPO_URL="${CUGA_AGENT_REPO_URL:-$DEFAULT_CUGA_AGENT_REPO_URL}" -CUGA_EVAL_REPO_URL="${CUGA_EVAL_REPO_URL:-$DEFAULT_CUGA_EVAL_REPO_URL}" - -CUGA_AGENT_DIR="${ROOT_DIR}/cuga-agent" -CUGA_EVAL_DIR="${ROOT_DIR}/cuga-eval" -GH_RUNNER_DIR="${ROOT_DIR}/github-runner" - -RUN_EVAL_SCRIPT="${RUN_EVAL_SCRIPT:-${CUGA_EVAL_DIR}/scripts/run-pr-regression-eval.sh}" -WORKFLOW_EVAL_SCRIPT="${CUGA_EVAL_DIR}/scripts/run-pr-eval.sh" - -log() { - printf '\n[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" -} - -die() { - echo "ERROR: $*" >&2 - exit 1 -} - -need_cmd() { - command -v "$1" >/dev/null 2>&1 || die "Required command '$1' is missing." -} - -clone_or_update() { - local repo_url="$1" - local branch="$2" - local dest="$3" - local name="$4" - - if [[ -d "${dest}/.git" ]]; then - log "Updating ${name}" - git -C "${dest}" fetch --all --prune - git -C "${dest}" checkout "${branch}" - git -C "${dest}" pull --ff-only origin "${branch}" - else - log "Cloning ${name}" - git clone --branch "${branch}" "${repo_url}" "${dest}" - fi -} - -: "${GITHUB_REPO_URL:?Set GITHUB_REPO_URL, e.g. https://github.com/OWNER/REPO}" -: "${GITHUB_RUNNER_TOKEN:?Set GITHUB_RUNNER_TOKEN to the temporary runner token}" - -log "Repository mode: ${REPO_MODE}" -echo "cuga-agent: ${CUGA_AGENT_REPO_URL}" -echo "cuga-eval: ${CUGA_EVAL_REPO_URL}" - -log "Checking VM prerequisites" -need_cmd git -need_cmd curl -need_cmd tar -need_cmd python3 -need_cmd git-lfs - -python3 - <<'PY' -import sys -if not ((3, 12) <= sys.version_info[:2] < (3, 14)): - raise SystemExit( - f"ERROR: Python 3.12 or 3.13 is required; found " - f"{sys.version_info.major}.{sys.version_info.minor}" - ) -PY - -if ! command -v uv >/dev/null 2>&1; then - log "Installing uv" - curl -LsSf https://astral.sh/uv/install.sh | sh - export PATH="${HOME}/.local/bin:${HOME}/.cargo/bin:${PATH}" -fi -need_cmd uv - -mkdir -p "${ROOT_DIR}" - -clone_or_update \ - "${CUGA_AGENT_REPO_URL}" \ - "${CUGA_AGENT_BRANCH}" \ - "${CUGA_AGENT_DIR}" \ - "cuga-agent" - -clone_or_update \ - "${CUGA_EVAL_REPO_URL}" \ - "${CUGA_EVAL_BRANCH}" \ - "${CUGA_EVAL_DIR}" \ - "cuga-eval" - -log "Running cuga-eval/setup_cuga.sh" -cd "${CUGA_EVAL_DIR}" -chmod u+x ./setup_cuga.sh -bash ./setup_cuga.sh - -log "Creating/syncing cuga-eval environment" -uv venv -uv sync - -log "Setting up AppWorld" -git lfs install -chmod u+x ./setup_appworld.sh -bash ./setup_appworld.sh -uv sync --group appworld - -log "Checking PR regression evaluation entrypoint" -if [[ ! -f "${RUN_EVAL_SCRIPT}" ]]; then - cat >&2 </dev/null 2>&1; then - log "Runner already running" -else - log "Starting runner in background" - - nohup ./run.sh > "${GH_RUNNER_DIR}/runner.log" 2>&1 & - echo $! > "${GH_RUNNER_DIR}/runner.pid" -fi - -sleep 3 - -if ! pgrep -f "${GH_RUNNER_DIR}/bin/Runner.Listener" >/dev/null 2>&1; then - echo "ERROR: Runner did not remain active." >&2 - tail -n 50 "${GH_RUNNER_DIR}/runner.log" 2>/dev/null || true - exit 1 -fi - -echo -echo "======================================================================" -echo "PR regression testing VM is ready" -echo "======================================================================" -echo -echo "Repository mode:" -echo " ${REPO_MODE}" -echo -echo "Layout:" -echo " ${ROOT_DIR}/" -echo " ├── cuga-agent/" -echo " ├── cuga-eval/" -echo " │ └── scripts/run-eval.sh" -echo " └── github-runner/" -echo " └── _work/" -echo -echo "Repos:" -echo " cuga-agent: ${CUGA_AGENT_REPO_URL}" -echo " cuga-eval: ${CUGA_EVAL_REPO_URL}" -echo -echo "Runner logs:" -echo " tail -f ${GH_RUNNER_DIR}/runner.log" -echo -echo "Workflow target:" -echo " runs-on: [self-hosted, linux, ${RUNNER_LABELS}]" -echo -echo "Workflow script:" -echo " bash \"${RUN_EVAL_SCRIPT}\" \"\${COMMENT_BODY}\"" From 741bc30f1379e516e9a286a13a55f48935ae54ca Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 08:40:38 -0400 Subject: [PATCH 24/40] Update the README. --- scripts/pr-regression-testing/README.md | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index c3e5daed..a62d245e 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -5,7 +5,7 @@ This folder contains the scripts used to set up and run PR-triggered regression ## Files - `setup-pr-regression-workflow.sh`: bootstraps the self-hosted GitHub Actions runner VM. -- `run-pr-regression-eval.sh`: parses `/run-pr-eval` PR comments and runs the requested benchmark. +- `run-pr-regression-eval.sh`: parses `/run-pr-eval` PR comments and runs the requested benchmark using the `cuga-eval/scripts/eval.sh` script. ## Setup The PR Regression Flow @@ -15,13 +15,7 @@ This folder contains the scripts used to set up and run PR-triggered regression .github/workflows/run-pr-eval.yml ``` -2. Configure the workflow to use the self-hosted runner label: - - ```yaml - runs-on: [self-hosted, linux, run-pr-eval] - ``` - -3. In GitHub, create a self-hosted runner token for the `cuga-agent` repository. +2. In GitHub, create a self-hosted runner token for the `cuga-agent` repository. Go to: @@ -29,7 +23,13 @@ This folder contains the scripts used to set up and run PR-triggered regression cuga-agent repository -> Settings -> Actions -> Runners -> New self-hosted runner ``` - Copy the repository URL and temporary runner token from the generated `config.sh` command. + Copy temporary runner token from the generated `config.sh` command. + +3. Ensure the workflow file uses the self-hosted runner labels: + + ```yaml + runs-on: [self-hosted, linux, run-pr-eval] + ``` 4. On the runner VM, export the required setup variables: @@ -64,7 +64,6 @@ This folder contains the scripts used to set up and run PR-triggered regression - run `cuga-eval/setup_cuga.sh`; - create and sync the `uv` environment; - run AppWorld setup; - - create the workflow compatibility symlink; - download and register the GitHub Actions runner; - start the runner in the background. @@ -74,6 +73,8 @@ This folder contains the scripts used to set up and run PR-triggered regression cuga-agent repository -> Settings -> Actions -> Runners ``` + Ensure the runner is running with the labels `[self-hosted, linux, run-pr-eval]` i.e. same as workflow file. + 8. Check runner logs on the VM if needed: ```bash From 86d0c69be1cbc199855feea9581c002b32668f0b Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 20 Aug 2026 08:46:53 -0400 Subject: [PATCH 25/40] Update README with parameter name. --- scripts/pr-regression-testing/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index a62d245e..5f7ed5c3 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -36,6 +36,7 @@ This folder contains the scripts used to set up and run PR-triggered regression ```bash export GITHUB_REPO_URL="https://github.com/OWNER/cuga-agent" export GITHUB_RUNNER_TOKEN="YOUR_TEMPORARY_RUNNER_TOKEN" + export RUNNER_LABELS="run-pr-eval" ``` 5. Optionally choose repo mode and branches: From cd7b75d678ba61b64cc26e2fac814018804cd83e Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Sat, 5 Sep 2026 08:52:40 -0400 Subject: [PATCH 26/40] Add support for AppWorld subsets. --- scripts/pr-regression-testing/README.md | 31 ++++++++++++ .../run-pr-regression-eval.sh | 49 +++++++++++++++---- 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index 5f7ed5c3..6f1e5de1 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -133,6 +133,7 @@ The command accepts whitespace-separated `key=value` parameters: - `model_name`: model name to evaluate. - `task_id`: one task ID, or comma-separated task IDs. - `task_ids`: alias for `task_id`. +- `eval_key`: AppWorld task group from `benchmarks/appworld/eval_config.toml`, such as `test_easy`, `test_med`, or `test_hard`. - `benchmark`: `appworld` or `m3`; defaults to `appworld`. - `num_tasks`: positive integer; defaults to `4`. - `agent`: `react`, `cuga`, or `codeact`; defaults to `react`. @@ -140,6 +141,18 @@ The command accepts whitespace-separated `key=value` parameters: Unsupported parameters cause the workflow to fail early with a list of supported parameters. +Useful AppWorld `eval_key` values: + +- `test_easy` +- `test_med` +- `test_hard` +- `test_challenge_easy` +- `test_challenge_med` +- `test_challenge_hard` +- `test_normal_easy` +- `test_normal_med` +- `test_normal_hard` + ## Examples Run the default AppWorld React-agent evaluation on RITS: @@ -166,6 +179,24 @@ Run AppWorld with two explicit tasks: /run-pr-eval benchmark=appworld task_id=9aae7da_1,365e0a3_1 num_tasks=2 ``` +Run AppWorld easy tasks from `eval_config.toml`: + +```text +/run-pr-eval benchmark=appworld eval_key=test_easy +``` + +Run AppWorld medium tasks from `eval_config.toml`: + +```text +/run-pr-eval benchmark=appworld eval_key=test_med +``` + +Run AppWorld hard tasks from `eval_config.toml`: + +```text +/run-pr-eval benchmark=appworld eval_key=test_hard +``` + Run M3 with the default React agent: ```text diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index 07b88bdd..9660552b 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -56,6 +56,7 @@ DEFAULT_PROVIDER="rits" MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" TASK_IDS="${TASK_IDS:-}" +EVAL_KEY="${EVAL_KEY:-}" BENCHMARK="${BENCHMARK:-$DEFAULT_BENCHMARK}" NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" AGENT="${AGENT:-$DEFAULT_AGENT}" @@ -67,6 +68,7 @@ MODEL_NAME_FROM_COMMENT=false # model_name=... # task_id=id1,id2 # task_ids=id1,id2 +# eval_key=test_easy|test_med|test_hard|... # benchmark=appworld|m3 # num_tasks=... # agent=react|cuga|codeact @@ -95,6 +97,10 @@ for token in "${TOKENS[@]}"; do TASK_IDS="${TASK_IDS//,/ }" ;; + eval_key=*|eval-key=*) + EVAL_KEY="${token#*=}" + ;; + benchmark=*) BENCHMARK="${token#benchmark=}" ;; @@ -116,7 +122,7 @@ for token in "${TOKENS[@]}"; do *) echo "ERROR: Unsupported parameter: ${token}" - echo "Supported parameters: model_name, task_id, task_ids, benchmark, num_tasks, agent, provider" + echo "Supported parameters: model_name, task_id, task_ids, eval_key, benchmark, num_tasks, agent, provider" close_details echo "######## REPORT END ########" exit 2 @@ -141,6 +147,7 @@ fi AGENT="$(printf '%s' "${AGENT}" | tr '[:upper:]' '[:lower:]')" BENCHMARK="$(printf '%s' "${BENCHMARK}" | tr '[:upper:]' '[:lower:]')" PROVIDER="$(printf '%s' "${PROVIDER}" | tr '[:upper:]' '[:lower:]')" +EVAL_KEY="$(printf '%s' "${EVAL_KEY}" | tr '[:upper:]' '[:lower:]')" case "${AGENT}" in react|cuga|codeact) @@ -164,6 +171,13 @@ case "${BENCHMARK}" in ;; esac +if [[ -n "${EVAL_KEY}" && "${BENCHMARK}" != "appworld" ]]; then + echo "ERROR: eval_key is only supported for benchmark=appworld." + close_details + echo "######## REPORT END ########" + exit 2 +fi + case "${PROVIDER}" in rits) AGENT_SETTING_CONFIG="settings.rits.toml" @@ -218,14 +232,14 @@ else : fi -if [[ -z "${TASK_IDS}" && "${BENCHMARK}" == "appworld" ]]; then +if [[ -z "${TASK_IDS}" && -z "${EVAL_KEY}" && "${BENCHMARK}" == "appworld" ]]; then TASK_IDS="${DEFAULT_APPWORLD_TASK_IDS}" fi read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" -if [[ ${#TASK_ID_ARRAY[@]} -eq 0 && "${BENCHMARK}" == "appworld" ]]; then - echo "ERROR: At least one task_id is required for benchmark=appworld." +if [[ ${#TASK_ID_ARRAY[@]} -eq 0 && -z "${EVAL_KEY}" && "${BENCHMARK}" == "appworld" ]]; then + echo "ERROR: At least one task_id or eval_key is required for benchmark=appworld." close_details echo "######## REPORT END ########" exit 2 @@ -234,6 +248,9 @@ echo "- Model: ${MODEL_NAME}" echo "- Agent: ${AGENT}" echo "- Benchmark: ${BENCHMARK}" echo "- Provider: ${PROVIDER}" +if [[ -n "${EVAL_KEY}" ]]; then + echo "- Eval key: ${EVAL_KEY}" +fi echo "- Num tasks: ${NUM_TASKS}" if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then echo "- Task IDs: ${TASK_IDS}" @@ -295,11 +312,18 @@ echo "- Agent setting config: ${AGENT_SETTING_CONFIG}" echo "- Model: ${MODEL_NAME}" echo "- Requested num_tasks: ${NUM_TASKS}" if [[ "${BENCHMARK}" == "appworld" ]]; then - echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" - echo "- Task IDs:" - for task_id in "${TASK_ID_ARRAY[@]}"; do - echo " - ${task_id}" - done + if [[ -n "${EVAL_KEY}" ]]; then + echo "- Eval key: ${EVAL_KEY}" + fi + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" + echo "- Task IDs:" + for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" + done + else + echo "- Task IDs: selected by eval_key" + fi else echo "- M3 max samples per domain: ${NUM_TASKS}" if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then @@ -333,7 +357,12 @@ EVAL_ARGS=( ) if [[ "${BENCHMARK}" == "appworld" ]]; then - EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") + if [[ -n "${EVAL_KEY}" ]]; then + EVAL_ARGS+=(--eval-key "${EVAL_KEY}") + fi + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") + fi else EVAL_ARGS+=(--m3-data --max-samples-per-domain "${NUM_TASKS}") if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then From 0c7feed859d93a2e5fa2bc2f882a6925ca3d7a0d Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Wed, 16 Sep 2026 22:25:22 -0400 Subject: [PATCH 27/40] fix(pr-eval): gate PR report formatting and fix RITS config --- benchmarks/appworld/eval.sh | 7 +- benchmarks/bpo/eval.sh | 7 +- benchmarks/helpers/compare_report.py | 27 ++++--- benchmarks/helpers/react_agent.py | 17 +++-- .../helpers/tests/test_compare_report.py | 74 +++++++++++++++++++ benchmarks/m3/eval.sh | 7 +- benchmarks/oak_health_insurance/eval.sh | 7 +- 7 files changed, 119 insertions(+), 27 deletions(-) diff --git a/benchmarks/appworld/eval.sh b/benchmarks/appworld/eval.sh index 9c68ee96..54e77d49 100755 --- a/benchmarks/appworld/eval.sh +++ b/benchmarks/appworld/eval.sh @@ -307,8 +307,11 @@ if [ -n "$LATEST_RESULT" ] && [ "${NO_BUNDLE:-false}" != "true" ]; then # Generate eval report REPORT_TMP=$(mktemp /tmp/appworld_eval_report_XXXXXX) - uv run --no-sync python -m benchmarks.helpers.compare_report eval \ - --result-file "$LATEST_RESULT" --output "$REPORT_TMP" + REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then + REPORT_ARGS+=(--for-pr-comment --stdout-markdown) + fi + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" BUNDLE_ARGS=(assemble --benchmark appworld --result-files "$LATEST_RESULT" diff --git a/benchmarks/bpo/eval.sh b/benchmarks/bpo/eval.sh index f4694dd9..746081c2 100755 --- a/benchmarks/bpo/eval.sh +++ b/benchmarks/bpo/eval.sh @@ -409,8 +409,11 @@ if [ $EVAL_EXIT_CODE -eq 0 ]; then # Generate eval report REPORT_TMP=$(mktemp /tmp/bpo_eval_report_XXXXXX) - uv run --no-sync python -m benchmarks.helpers.compare_report eval \ - --result-file "$LATEST_RESULT" --output "$REPORT_TMP" + REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then + REPORT_ARGS+=(--for-pr-comment --stdout-markdown) + fi + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" BUNDLE_ARGS=(assemble --benchmark bpo --result-files "$LATEST_RESULT" diff --git a/benchmarks/helpers/compare_report.py b/benchmarks/helpers/compare_report.py index f220beea..5b396b7a 100644 --- a/benchmarks/helpers/compare_report.py +++ b/benchmarks/helpers/compare_report.py @@ -1330,7 +1330,7 @@ def _bucket_m3_tasks(tasks: dict) -> tuple: return rows, True -def generate_eval_report(result_file: str, markdown: bool = True) -> str: +def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment: bool = False) -> str: """Generate a single-evaluation-run report. When the result file is M3-shaped (each task has ``m3_task_id`` + ``domain``), @@ -1338,7 +1338,8 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: rows are readable instead of an unattributed UUID. When ``markdown=False``, the same content is rendered as a plain-text monospace table for terminals (compare.sh's stdout path); ``markdown=True`` (default) is what gets saved - into the bundle's report.md. + into the bundle's report.md. ``for_pr_comment`` is an explicit opt-in for + the PR-regression workflow's collapsible, sentinel-wrapped markdown. """ parsed = parse_result_file(result_file) rows, grouped = _bucket_m3_tasks(parsed["tasks"]) @@ -1349,15 +1350,15 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: fence_open = (lambda: "```text") if markdown else (lambda: "") fence_close = (lambda: "```") if markdown else (lambda: "") - lines = ["######## REPORT START ########"] - if markdown: + if markdown and for_pr_comment: + lines = ["######## REPORT START ########"] lines.append("
") lines.append("Evaluation Report") lines.append("") lines.append(h2("Summary")) lines.append("") else: - lines.extend([h1("Evaluation Report"), ""]) + lines = [h1("Evaluation Report"), ""] lines.append(h2("Summary")) lines.append("") if markdown: @@ -1597,10 +1598,10 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: markdown=markdown, ) ) - if markdown: + if markdown and for_pr_comment: lines.append("") lines.append("
") - lines.append("######## REPORT END ########") + lines.append("######## REPORT END ########") return "\n".join(lines) @@ -1615,8 +1616,10 @@ def main(): parser.add_argument("command") # consume "eval" parser.add_argument("--result-file", required=True) parser.add_argument("--output", "-o", default=None) + parser.add_argument("--for-pr-comment", action="store_true") + parser.add_argument("--stdout-markdown", action="store_true") args = parser.parse_args() - report = generate_eval_report(args.result_file) + report = generate_eval_report(args.result_file, for_pr_comment=args.for_pr_comment) else: parser = argparse.ArgumentParser(description="Generate comparison report") parser.add_argument("command") # consume "compare" @@ -1638,11 +1641,11 @@ def main(): # the canonical bundle location at the end of the run, which is what the # user actually wants to navigate to. if args.output: - # Compare-mode and eval-mode both produce markdown for the saved file. - # For eval-mode, also print markdown to stdout because PR workflows - # capture stdout and post marked report blocks back to GitHub comments. if "command" in args and getattr(args, "command", None) == "eval": - stdout_report = report + if getattr(args, "for_pr_comment", False) or getattr(args, "stdout_markdown", False): + stdout_report = report + else: + stdout_report = generate_eval_report(args.result_file, markdown=False) else: stdout_report = generate_report(config_results, markdown=False) Path(args.output).write_text(report) diff --git a/benchmarks/helpers/react_agent.py b/benchmarks/helpers/react_agent.py index 2b9f8526..6a11157c 100644 --- a/benchmarks/helpers/react_agent.py +++ b/benchmarks/helpers/react_agent.py @@ -50,20 +50,23 @@ class ChatRits(ChatOpenAI): def __init__(self, config: dict[str, Any]): model_name = config.get("model_name", "openai/gpt-oss-120b-a100") - end_point = config.get( - "end_point", - "https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100", + end_point = ( + config.get("end_point") + or "https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100" ) rits_api_key = config.get("api_key") or os.getenv("RITS_API_KEY") if rits_api_key is None: raise ValueError("RITS_API_KEY is required for settings.rits.toml") params = config.get("params", {}) + base_url = end_point.rstrip("/") + if not base_url.endswith("/v1"): + base_url = f"{base_url}/v1" rits_config: dict[str, Any] = { - "model_name": model_name, + "model": model_name, "api_key": "/", "default_headers": {"RITS_API_KEY": rits_api_key}, - "base_url": end_point.rstrip("/") + "/v1", + "base_url": base_url, } rits_config.update(params) super().__init__(**rits_config) @@ -201,8 +204,8 @@ def _create_llm(self) -> Any: return ChatOpenAI(**llm_kwargs) elif settings_config in ("settings.rits.toml", "settings.rits.proxy.toml"): - api_base = os.getenv("LITE_LLM_URL") or os.getenv("OPENAI_BASE_URL") or os.getenv("RITS_BASE_URL") - api_key = os.getenv("LITE_LLM_KEY") or os.getenv("OPENAI_API_KEY") or os.getenv("RITS_API_KEY") + api_base = os.getenv("RITS_BASE_URL") or os.getenv("LITE_LLM_URL") or os.getenv("OPENAI_BASE_URL") + api_key = os.getenv("RITS_API_KEY") or os.getenv("LITE_LLM_KEY") or os.getenv("OPENAI_API_KEY") model_name = os.getenv("MODEL_NAME") or self.model or "openai/gpt-oss-120b-a100" logger.info(f"RITS config: model={model_name}, api_base={api_base}, api_key_present={bool(api_key)}") return ChatRits(config={"model_name": model_name, "end_point": api_base, "api_key": api_key}) diff --git a/benchmarks/helpers/tests/test_compare_report.py b/benchmarks/helpers/tests/test_compare_report.py index e5f6afe2..0a67e51a 100644 --- a/benchmarks/helpers/tests/test_compare_report.py +++ b/benchmarks/helpers/tests/test_compare_report.py @@ -21,10 +21,12 @@ """ import json +import sys from pathlib import Path import pytest +from benchmarks.helpers import compare_report as compare_report_module from benchmarks.helpers.compare_report import ( _last_turn_judge_scores, _m3_capability_group, @@ -218,6 +220,78 @@ def test_parse_sdk_results_includes_react_steps(): assert parsed["tasks"]["t1"]["steps"] == 3 +def test_eval_report_default_omits_pr_comment_wrapping(tmp_path): + result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False}) + + report = generate_eval_report(result_file) + + assert report.startswith("# Evaluation Report") + assert "######## REPORT START ########" not in report + assert "######## REPORT END ########" not in report + assert "Evaluation Report" not in report + + +def test_eval_report_pr_comment_wrapping_is_opt_in(tmp_path): + result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False}) + + report = generate_eval_report(result_file, for_pr_comment=True) + + assert report.startswith("######## REPORT START ########") + assert "Evaluation Report" in report + assert report.rstrip().endswith("######## REPORT END ########") + + +def test_eval_main_with_output_prints_plain_text_by_default(tmp_path, monkeypatch, capsys): + result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False}) + output_file = tmp_path / "report.md" + monkeypatch.setattr( + sys, + "argv", + [ + "compare_report.py", + "eval", + "--result-file", + result_file, + "--output", + str(output_file), + ], + ) + + compare_report_module.main() + + stdout = capsys.readouterr().out + saved_report = output_file.read_text() + assert stdout.startswith("\nEvaluation Report") + assert "######## REPORT START ########" not in stdout + assert saved_report.startswith("# Evaluation Report") + assert "######## REPORT START ########" not in saved_report + + +def test_eval_main_pr_comment_mode_prints_markdown_markers(tmp_path, monkeypatch, capsys): + result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False}) + output_file = tmp_path / "report.md" + monkeypatch.setattr( + sys, + "argv", + [ + "compare_report.py", + "eval", + "--result-file", + result_file, + "--output", + str(output_file), + "--for-pr-comment", + ], + ) + + compare_report_module.main() + + stdout = capsys.readouterr().out + saved_report = output_file.read_text() + assert stdout.startswith("######## REPORT START ########") + assert saved_report.startswith("######## REPORT START ########") + + def _m3_run(tmp_path: Path, name: str, tasks) -> str: """Write an M3-shape ``*_final_report.json`` and return its path. diff --git a/benchmarks/m3/eval.sh b/benchmarks/m3/eval.sh index 160dfd37..eef2bb31 100755 --- a/benchmarks/m3/eval.sh +++ b/benchmarks/m3/eval.sh @@ -263,8 +263,11 @@ create_bundle() { # want the bundle, so don't let `set -e` abort here). local report_tmp report_tmp=$(mktemp /tmp/m3_eval_report_XXXXXX) - uv run --no-sync python -m benchmarks.helpers.compare_report eval \ - --result-file "$latest_result" --output "$report_tmp" || \ + local report_args=(eval --result-file "$latest_result" --output "$report_tmp") + if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then + report_args+=(--for-pr-comment --stdout-markdown) + fi + uv run --no-sync python -m benchmarks.helpers.compare_report "${report_args[@]}" || \ echo -e "${YELLOW:-}Report generation failed — bundling without report.${NC:-}" local bundle_args=(assemble --benchmark m3 diff --git a/benchmarks/oak_health_insurance/eval.sh b/benchmarks/oak_health_insurance/eval.sh index 747c4b7d..de7c3288 100755 --- a/benchmarks/oak_health_insurance/eval.sh +++ b/benchmarks/oak_health_insurance/eval.sh @@ -242,8 +242,11 @@ if [ $EVAL_EXIT -eq 0 ]; then # Generate eval report REPORT_TMP=$(mktemp /tmp/oak_eval_report_XXXXXX) - uv run --no-sync python -m benchmarks.helpers.compare_report eval \ - --result-file "$LATEST_RESULT" --output "$REPORT_TMP" + REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then + REPORT_ARGS+=(--for-pr-comment --stdout-markdown) + fi + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" BUNDLE_ARGS=(assemble --benchmark oak_health_insurance --result-files "$LATEST_RESULT" From 8675ad9b2ad7d33f0f617c77c0e4a36348393a47 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Wed, 16 Sep 2026 22:26:09 -0400 Subject: [PATCH 28/40] fix(appworld): gate PR summary markers by environment Make AppWorld summary REPORT START/END markers opt-in via PR_EVAL_FOR_PR_COMMENT across cuga, codeact, and react evaluators so local runs keep clean terminal output while PR regression runs remain structured. --- benchmarks/appworld/appworld_eval.py | 13 ++++++++++--- benchmarks/appworld/appworld_eval_codeact.py | 13 ++++++++++--- benchmarks/appworld/appworld_eval_react.py | 13 ++++++++++--- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/benchmarks/appworld/appworld_eval.py b/benchmarks/appworld/appworld_eval.py index 7d34d6bd..8082c909 100644 --- a/benchmarks/appworld/appworld_eval.py +++ b/benchmarks/appworld/appworld_eval.py @@ -470,14 +470,21 @@ def _print_appworld_summary(report: dict): success_rate = report.get("success_rate", 0) avg_steps = report.get("avg_steps", 0) avg_duration = report.get("avg_duration", 0) + for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes") - print("\n" + "=" * 80) - print("EVALUATION COMPLETE") - print("=" * 80) + if for_pr_comment: + print("######## REPORT START ########") + print("## EVALUATION COMPLETE") + else: + print("\n" + "=" * 80) + print("EVALUATION COMPLETE") + print("=" * 80) print(f"Total Tasks: {total}") print(f"Completed: {completed}/{total} ({success_rate:.1%})") print(f"Avg Steps: {avg_steps:.2f}") print(f"Avg Duration: {avg_duration:.2f}s") + if for_pr_comment: + print("######## REPORT END ########") async def main(): diff --git a/benchmarks/appworld/appworld_eval_codeact.py b/benchmarks/appworld/appworld_eval_codeact.py index f92fa516..96f5a9e2 100644 --- a/benchmarks/appworld/appworld_eval_codeact.py +++ b/benchmarks/appworld/appworld_eval_codeact.py @@ -96,14 +96,21 @@ def _print_appworld_summary(report: dict): success_rate = report.get("success_rate", 0) avg_steps = report.get("avg_steps", 0) avg_duration = report.get("avg_duration", 0) + for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes") - print("\n" + "=" * 80) - print("EVALUATION COMPLETE") - print("=" * 80) + if for_pr_comment: + print("######## REPORT START ########") + print("## EVALUATION COMPLETE") + else: + print("\n" + "=" * 80) + print("EVALUATION COMPLETE") + print("=" * 80) print(f"Total Tasks: {total}") print(f"Completed: {completed}/{total} ({success_rate:.1%})") print(f"Avg Steps: {avg_steps:.2f}") print(f"Avg Duration: {avg_duration:.2f}s") + if for_pr_comment: + print("######## REPORT END ########") def _parse_prompt_to_messages(input_str: str) -> list[dict[str, Any]]: diff --git a/benchmarks/appworld/appworld_eval_react.py b/benchmarks/appworld/appworld_eval_react.py index a8d269cc..d79c357e 100644 --- a/benchmarks/appworld/appworld_eval_react.py +++ b/benchmarks/appworld/appworld_eval_react.py @@ -181,14 +181,21 @@ def _print_appworld_summary(report: dict): success_rate = report.get("success_rate", 0) avg_steps = report.get("avg_steps", 0) avg_duration = report.get("avg_duration", 0) + for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes") - print("######## REPORT START ########") - print("## EVALUATION COMPLETE") + if for_pr_comment: + print("######## REPORT START ########") + print("## EVALUATION COMPLETE") + else: + print("\n" + "=" * 80) + print("EVALUATION COMPLETE") + print("=" * 80) print(f"Total Tasks: {total}") print(f"Completed: {completed}/{total} ({success_rate:.1%})") print(f"Avg Steps: {avg_steps:.2f}") print(f"Avg Duration: {avg_duration:.2f}s") - print("######## REPORT END ########") + if for_pr_comment: + print("######## REPORT END ########") def _render_initial_prompt(world: AppWorld) -> str: From 0ba3c690a3ec0a731e3d8094f65c42317b52005c Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Wed, 16 Sep 2026 22:26:43 -0400 Subject: [PATCH 29/40] fix(pr-eval): align runner script with validated workflow input Accept sanitized PR eval parameters, normalize num_tasks as base-10, tolerate workflow SHA metadata, and update setup/README guidance for the SHA-pinned run-pr-eval flow. --- scripts/pr-regression-testing/README.md | 45 +++---- .../run-pr-regression-eval.sh | 123 +++++++++++++++--- .../setup-pr-regression-workflow.sh | 5 +- 3 files changed, 130 insertions(+), 43 deletions(-) diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index 6f1e5de1..b4115d1b 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -5,7 +5,7 @@ This folder contains the scripts used to set up and run PR-triggered regression ## Files - `setup-pr-regression-workflow.sh`: bootstraps the self-hosted GitHub Actions runner VM. -- `run-pr-regression-eval.sh`: parses `/run-pr-eval` PR comments and runs the requested benchmark using the `cuga-eval/scripts/eval.sh` script. +- `run-pr-regression-eval.sh`: consumes validated `/run-pr-eval` parameters and runs the requested benchmark using the `cuga-eval/scripts/eval.sh` script. ## Setup The PR Regression Flow @@ -112,10 +112,10 @@ Provider defaults used by `run-pr-regression-eval.sh`: Create a comment on a pull request in `cuga-agent`: ```text -/run-pr-eval +/run-pr-eval sha= ``` -By default, this runs: +The `sha` value must be the current 40-character PR head commit SHA. The workflow validates it before checkout and does not forward it to the evaluation process. By default, the forwarded evaluation runs: - `benchmark=appworld` - `agent=react` @@ -130,6 +130,7 @@ For `provider=litellm`, the default model changes to `aws/gpt-oss-120b` unless ` The command accepts whitespace-separated `key=value` parameters: +- `sha`: required by the GitHub workflow; must be the current 40-character PR head SHA. - `model_name`: model name to evaluate. - `task_id`: one task ID, or comma-separated task IDs. - `task_ids`: alias for `task_id`. @@ -139,7 +140,9 @@ The command accepts whitespace-separated `key=value` parameters: - `agent`: `react`, `cuga`, or `codeact`; defaults to `react`. - `provider`: `rits` or `litellm`; defaults to `rits`. -Unsupported parameters cause the workflow to fail early with a list of supported parameters. +Unsupported parameters cause the workflow or runner script to fail early with a list of supported parameters. + +The workflow parser tolerates spaces around `=`, markdown code ticks, mixed-case values for `agent`, `benchmark`, and `provider`, and spaces after commas in task lists. For example, `provider = litellm` and `task_id = 9aae7da_1, 365e0a3_1` are accepted. The runner script also accepts sanitized CLI-style arguments such as `--benchmark appworld --agent react --provider rits`. Useful AppWorld `eval_key` values: @@ -158,75 +161,65 @@ Useful AppWorld `eval_key` values: Run the default AppWorld React-agent evaluation on RITS: ```text -/run-pr-eval +/run-pr-eval sha= ``` Run one AppWorld task with the React agent on RITS: ```text -/run-pr-eval provider=rits agent=react benchmark=appworld task_id=9aae7da_1 num_tasks=1 +/run-pr-eval sha= provider=rits agent=react benchmark=appworld task_id=9aae7da_1 num_tasks=1 ``` Run one AppWorld task with the CUGA agent on LiteLLM: ```text -/run-pr-eval provider=litellm agent=cuga benchmark=appworld task_id=9aae7da_1 num_tasks=1 +/run-pr-eval sha= provider=litellm agent=cuga benchmark=appworld task_id=9aae7da_1 num_tasks=1 ``` Run AppWorld with two explicit tasks: ```text -/run-pr-eval benchmark=appworld task_id=9aae7da_1,365e0a3_1 num_tasks=2 +/run-pr-eval sha= benchmark=appworld task_id=9aae7da_1,365e0a3_1 num_tasks=2 ``` Run AppWorld easy tasks from `eval_config.toml`: ```text -/run-pr-eval benchmark=appworld eval_key=test_easy +/run-pr-eval sha= benchmark=appworld eval_key=test_easy ``` Run AppWorld medium tasks from `eval_config.toml`: ```text -/run-pr-eval benchmark=appworld eval_key=test_med +/run-pr-eval sha= benchmark=appworld eval_key=test_med ``` Run AppWorld hard tasks from `eval_config.toml`: ```text -/run-pr-eval benchmark=appworld eval_key=test_hard +/run-pr-eval sha= benchmark=appworld eval_key=test_hard ``` Run M3 with the default React agent: ```text -/run-pr-eval benchmark=m3 num_tasks=1 +/run-pr-eval sha= benchmark=m3 num_tasks=1 ``` Run M3 with the CUGA agent and LiteLLM: ```text -/run-pr-eval benchmark=m3 agent=cuga provider=litellm num_tasks=1 +/run-pr-eval sha= benchmark=m3 agent=cuga provider=litellm num_tasks=1 ``` Override the model: ```text -/run-pr-eval provider=litellm agent=cuga benchmark=appworld model_name=aws/glm-5 task_id=9aae7da_1 num_tasks=1 +/run-pr-eval sha= provider=litellm agent=cuga benchmark=appworld model_name=aws/glm-5 task_id=9aae7da_1 num_tasks=1 ``` ## Output -The workflow posts a PR comment containing collapsible sections for: - -- requested parameters; -- run metadata; -- evaluation configuration; -- evaluation report; -- run summary. - -The full output is also uploaded as a GitHub Actions artifact and saved on the VM under: +The workflow posts a PR comment containing trusted run metadata and a link to the uploaded artifact. It intentionally does not render PR-controlled evaluation markdown directly in the comment. -```text -~/pr-regression-testing/logs/ -``` +The full output is uploaded as a GitHub Actions artifact. During the workflow run, logs are written under the checked-out workspace in `pr-eval-logs/`. diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index 9660552b..c9db5d01 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -63,7 +63,21 @@ AGENT="${AGENT:-$DEFAULT_AGENT}" PROVIDER="${PROVIDER:-$DEFAULT_PROVIDER}" MODEL_NAME_FROM_COMMENT=false +usage_error() { + local message="$1" + echo "ERROR: ${message}" + echo "Supported parameters: model_name, task_id, task_ids, eval_key, benchmark, num_tasks, agent, provider, sha" + close_details + echo "######## REPORT END ########" + exit 2 +} + # Parse whitespace-separated key=value parameters from the PR comment. +# Common comment formatting is normalized first, so these are equivalent: +# provider=litellm +# provider = litellm +# provider= litellm +# provider =litellm # Supported aliases: # model_name=... # task_id=id1,id2 @@ -73,11 +87,90 @@ MODEL_NAME_FROM_COMMENT=false # num_tasks=... # agent=react|cuga|codeact # provider=rits|litellm -COMMENT_BODY="${1:-}" +COMMAND_TOKENS=() +if [[ $# -gt 0 ]]; then + while [[ $# -gt 0 ]]; do + case "$1" in + /run-pr-eval) + shift + ;; + --model-name|--model_name) + [[ -n "${2:-}" && "${2:-}" != --* ]] || { + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "$1 requires a value." + } + COMMAND_TOKENS+=("model_name=$2") + shift 2 + ;; + --task-id|--task-ids|--task|--tasks) + [[ -n "${2:-}" && "${2:-}" != --* ]] || { + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "$1 requires a value." + } + COMMAND_TOKENS+=("task_ids=$2") + shift 2 + ;; + --eval-key|--eval_key) + [[ -n "${2:-}" && "${2:-}" != --* ]] || { + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "$1 requires a value." + } + COMMAND_TOKENS+=("eval_key=$2") + shift 2 + ;; + --benchmark|--num-tasks|--num_tasks|--agent|--agent-type|--agent_type|--provider|--sha) + key="${1#--}" + key="${key//-/_}" + [[ -n "${2:-}" && "${2:-}" != --* ]] || { + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "$1 requires a value." + } + COMMAND_TOKENS+=("${key}=$2") + shift 2 + ;; + *=*) + COMMAND_TOKENS+=("$1") + shift + ;; + *) + if [[ $# -eq 1 && "$1" == *"/run-pr-eval"* ]]; then + COMMENT_BODY="$1" + shift + else + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "Unsupported argument: $1" + fi + ;; + esac + done +fi -# Remove Windows carriage returns and newlines. +if [[ ${#COMMAND_TOKENS[@]} -gt 0 ]]; then + COMMENT_BODY="/run-pr-eval ${COMMAND_TOKENS[*]}" +else + COMMENT_BODY="${COMMENT_BODY:-}" +fi + +# Remove Windows carriage returns, newlines, tabs, and markdown code ticks. COMMENT_BODY="${COMMENT_BODY//$'\r'/ }" COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" +COMMENT_BODY="${COMMENT_BODY//$'\t'/ }" +COMMENT_BODY="${COMMENT_BODY//\`/ }" +if [[ "${COMMENT_BODY}" == *"/run-pr-eval"* ]]; then + COMMENT_BODY="/run-pr-eval ${COMMENT_BODY#*/run-pr-eval}" +fi + +# Normalize spaces around "=" for supported keys, and tolerate spaces after +# commas in comma-separated values such as task_id=a, b. +COMMENT_BODY="$( + printf '%s' "${COMMENT_BODY}" | + sed -E 's/(^|[[:space:]])(model_name|task_id|task_ids|eval_key|eval-key|benchmark|num_tasks|agent|agent_type|provider|sha)[[:space:]]*=[[:space:]]*/\1\2=/g; s/,[[:space:]]+/,/g' +)" read -r -a TOKENS <<< "${COMMENT_BODY}" echo "######## REPORT START ########" @@ -117,31 +210,28 @@ for token in "${TOKENS[@]}"; do PROVIDER="${token#provider=}" ;; + sha=*) + ;; + "") ;; *) - echo "ERROR: Unsupported parameter: ${token}" - echo "Supported parameters: model_name, task_id, task_ids, eval_key, benchmark, num_tasks, agent, provider" - close_details - echo "######## REPORT END ########" - exit 2 + usage_error "Unsupported parameter: ${token}" ;; esac done if [[ -z "${MODEL_NAME}" ]]; then - echo "ERROR: model_name cannot be empty." - close_details - echo "######## REPORT END ########" - exit 2 + usage_error "model_name cannot be empty." fi -if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then - echo "ERROR: num_tasks must be a positive integer." - close_details - echo "######## REPORT END ########" - exit 2 +if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]]; then + usage_error "num_tasks must be a positive integer." +fi +NUM_TASKS=$((10#${NUM_TASKS})) +if (( NUM_TASKS < 1 )); then + usage_error "num_tasks must be a positive integer." fi AGENT="$(printf '%s' "${AGENT}" | tr '[:upper:]' '[:lower:]')" @@ -269,6 +359,7 @@ fi export AGENT_SETTING_CONFIG export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" export MODEL_NAME +export PR_EVAL_FOR_PR_COMMENT="${PR_EVAL_FOR_PR_COMMENT:-true}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [[ -z "${EVAL_REPO:-}" ]]; then diff --git a/scripts/pr-regression-testing/setup-pr-regression-workflow.sh b/scripts/pr-regression-testing/setup-pr-regression-workflow.sh index b3c6f730..c9bd9694 100644 --- a/scripts/pr-regression-testing/setup-pr-regression-workflow.sh +++ b/scripts/pr-regression-testing/setup-pr-regression-workflow.sh @@ -290,7 +290,7 @@ echo "Layout:" echo " ${ROOT_DIR}/" echo " ├── cuga-agent/" echo " ├── cuga-eval/" -echo " │ └── scripts/run-eval.sh" +echo " │ └── scripts/pr-regression-testing/run-pr-regression-eval.sh" echo " └── github-runner/" echo " └── _work/" echo @@ -306,3 +306,6 @@ echo " runs-on: [self-hosted, linux, ${RUNNER_LABELS}]" echo echo "Workflow script:" echo " bash \"${RUN_EVAL_SCRIPT}\" \"\${COMMENT_BODY}\"" +echo +echo "PR command format:" +echo " /run-pr-eval sha= benchmark=appworld provider=rits agent=react" From e84d3ed7b023bdeae2e973491b8e8a2a155d443c Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Wed, 16 Sep 2026 22:44:21 -0400 Subject: [PATCH 30/40] fix(pr-eval): restore inline evaluation report comments Post the extracted PR evaluation report markdown in the PR comment again while keeping the workflow artifact link for full logs and oversized output. --- scripts/pr-regression-testing/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index b4115d1b..8c633e13 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -220,6 +220,11 @@ Override the model: ## Output -The workflow posts a PR comment containing trusted run metadata and a link to the uploaded artifact. It intentionally does not render PR-controlled evaluation markdown directly in the comment. +The workflow posts a PR comment containing: -The full output is uploaded as a GitHub Actions artifact. During the workflow run, logs are written under the checked-out workspace in `pr-eval-logs/`. +- trusted run metadata; +- requested command and SHA; +- the evaluation report markdown emitted by the PR regression runner; +- a link to the uploaded workflow artifact. + +The full output is also uploaded as a GitHub Actions artifact. During the workflow run, logs are written under the checked-out workspace in `pr-eval-logs/`. From 8cf2a93124bb7739de6791adfc49a2a4a167f8bc Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Wed, 16 Sep 2026 23:22:26 -0400 Subject: [PATCH 31/40] Fork aligned with main. --- benchmarks/appworld/appworld_eval.py | 10 +- benchmarks/helpers/compare_report.py | 24 +++ scripts/run-pr-regression-eval.sh | 247 +++++++++++++++++++++++++++ 3 files changed, 280 insertions(+), 1 deletion(-) create mode 100644 scripts/run-pr-regression-eval.sh diff --git a/benchmarks/appworld/appworld_eval.py b/benchmarks/appworld/appworld_eval.py index 8082c909..8f892663 100644 --- a/benchmarks/appworld/appworld_eval.py +++ b/benchmarks/appworld/appworld_eval.py @@ -177,6 +177,14 @@ def _get_registry_base_url() -> str: return "http://localhost:8001" +def _is_langfuse_tracing_enabled() -> bool: + env_value = os.getenv("DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING") + if env_value is not None: + return env_value.strip().lower() in {"1", "true", "yes", "on"} + + return bool(getattr(getattr(settings, "advanced_features", None), "langfuse_tracing", False)) + + async def run_agent_on_task( task_id: str, experiment_name: str = "api_cuga_agent", @@ -217,7 +225,7 @@ async def run_agent_on_task( # Without this, LangfuseCallbackHandler creates a new root trace for every # agent_loop_obj.run() call in the controller while-loop. _langfuse = None - if _LANGFUSE_AVAILABLE and _get_langfuse_client is not None: + if _is_langfuse_tracing_enabled() and _LANGFUSE_AVAILABLE and _get_langfuse_client is not None: try: _langfuse = _get_langfuse_client() _run_uid = uuid.uuid4().hex[:8] diff --git a/benchmarks/helpers/compare_report.py b/benchmarks/helpers/compare_report.py index 1c1117e3..93e8b5f9 100644 --- a/benchmarks/helpers/compare_report.py +++ b/benchmarks/helpers/compare_report.py @@ -1537,15 +1537,24 @@ def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment fence_open = (lambda: "```text") if markdown else (lambda: "") fence_close = (lambda: "```") if markdown else (lambda: "") +<<<<<<< Updated upstream if markdown and for_pr_comment: lines = ["######## REPORT START ########"] +======= + lines = ["######## REPORT START ########"] + if markdown: +>>>>>>> Stashed changes lines.append("
") lines.append("Evaluation Report") lines.append("") lines.append(h2("Summary")) lines.append("") else: +<<<<<<< Updated upstream lines = [h1("Evaluation Report"), ""] +======= + lines.extend([h1("Evaluation Report"), ""]) +>>>>>>> Stashed changes lines.append(h2("Summary")) lines.append("") if markdown: @@ -1834,10 +1843,17 @@ def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment markdown=markdown, ) ) +<<<<<<< Updated upstream if markdown and for_pr_comment: lines.append("") lines.append("
") lines.append("######## REPORT END ########") +======= + if markdown: + lines.append("") + lines.append("") + lines.append("######## REPORT END ########") +>>>>>>> Stashed changes return "\n".join(lines) @@ -1877,11 +1893,19 @@ def main(): # the canonical bundle location at the end of the run, which is what the # user actually wants to navigate to. if args.output: +<<<<<<< Updated upstream if "command" in args and getattr(args, "command", None) == "eval": if getattr(args, "for_pr_comment", False) or getattr(args, "stdout_markdown", False): stdout_report = report else: stdout_report = generate_eval_report(args.result_file, markdown=False) +======= + # Compare-mode and eval-mode both produce markdown for the saved file. + # For eval-mode, also print markdown to stdout because PR workflows + # capture stdout and post marked report blocks back to GitHub comments. + if "command" in args and getattr(args, "command", None) == "eval": + stdout_report = report +>>>>>>> Stashed changes else: stdout_report = generate_report(config_results, markdown=False) Path(args.output).write_text(report) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh new file mode 100644 index 00000000..0b5dc93f --- /dev/null +++ b/scripts/run-pr-regression-eval.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +START_EPOCH="$(date +%s)" +START_TIME="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" + +log() { + printf '[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" +} + +open_details() { + local title="$1" + echo "
" + echo "${title}" + echo +} + +close_details() { + echo + echo "
" +} + +on_exit() { + status=$? + end_epoch="$(date +%s)" + end_time="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" + duration=$((end_epoch - START_EPOCH)) + minutes=$((duration / 60)) + seconds=$((duration % 60)) + + echo "######## REPORT START ########" + echo + open_details "Run summary" + echo "- Started: ${START_TIME}" + echo "- Finished: ${end_time}" + echo "- Duration: ${minutes}m ${seconds}s" + echo "- Exit code: ${status}" + + if [[ ${status} -eq 0 ]]; then + echo "- Result: SUCCESS" + else + echo "- Result: FAILED" + fi + close_details + echo "######## REPORT END ########" +} +trap on_exit EXIT + +DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" +DEFAULT_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" +DEFAULT_SPLIT_NAME="default" +DEFAULT_NUM_TASKS="4" +DEFAULT_AGENT="react" +DEFAULT_AGENT_SETTING_CONFIG="settings.rits.toml" + +MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" +TASK_IDS="${TASK_IDS:-$DEFAULT_TASK_IDS}" +SPLIT_NAME="${SPLIT_NAME:-$DEFAULT_SPLIT_NAME}" +NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" +AGENT="${AGENT:-$DEFAULT_AGENT}" +AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-$DEFAULT_AGENT_SETTING_CONFIG}" + +# Parse whitespace-separated key=value parameters from the PR comment. +# Supported aliases: +# model_name=... +# task_id=id1,id2 +# task_ids=id1,id2 +# split_name=... +# num_tasks=... +# agent=react|cuga|codeact +COMMENT_BODY="${1:-}" + +# Remove Windows carriage returns and newlines. +COMMENT_BODY="${COMMENT_BODY//$'\r'/ }" +COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" + +read -r -a TOKENS <<< "${COMMENT_BODY}" +echo "######## REPORT START ########" +open_details "Requested parameters" +for token in "${TOKENS[@]}"; do + case "${token}" in + /run-pr-eval) + ;; + + model_name=*) + MODEL_NAME="${token#model_name=}" + ;; + + task_id=*|task_ids=*) + TASK_IDS="${token#*=}" + TASK_IDS="${TASK_IDS//,/ }" + ;; + + split_name=*) + SPLIT_NAME="${token#split_name=}" + ;; + + num_tasks=*) + NUM_TASKS="${token#num_tasks=}" + ;; + + agent=*|agent_type=*) + AGENT="${token#*=}" + ;; + + "") + ;; + + *) + echo "ERROR: Unsupported parameter: ${token}" + echo "Supported parameters: model_name, task_id, task_ids, split_name, num_tasks, agent" + close_details + echo "######## REPORT END ########" + exit 2 + ;; + esac +done + +if [[ -z "${MODEL_NAME}" ]]; then + echo "ERROR: model_name cannot be empty." + close_details + echo "######## REPORT END ########" + exit 2 +fi + +if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then + echo "ERROR: num_tasks must be a positive integer." + close_details + echo "######## REPORT END ########" + exit 2 +fi + +case "${AGENT}" in + react|cuga|codeact) + ;; + *) + echo "ERROR: agent must be one of: react, cuga, codeact." + close_details + echo "######## REPORT END ########" + exit 2 + ;; +esac + +case "${AGENT_SETTING_CONFIG}" in + settings.rits.toml|settings.rits.proxy.toml) + if [[ -z "${RITS_API_KEY:-}" ]]; then + echo "ERROR: RITS_API_KEY is required when AGENT_SETTING_CONFIG=${AGENT_SETTING_CONFIG}." + close_details + echo "######## REPORT END ########" + exit 2 + fi + ;; + *) + if [[ -z "${OPENAI_API_KEY:-}" ]]; then + echo "ERROR: OPENAI_API_KEY is required when AGENT_SETTING_CONFIG=${AGENT_SETTING_CONFIG}." + close_details + echo "######## REPORT END ########" + exit 2 + fi + ;; +esac + +read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" + +if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then + echo "ERROR: At least one task_id is required." + close_details + echo "######## REPORT END ########" + exit 2 +fi +echo "- Model: ${MODEL_NAME}" +echo "- Agent: ${AGENT}" +echo "- Split: ${SPLIT_NAME}" +echo "- Num tasks: ${NUM_TASKS}" +echo "- Task IDs: ${TASK_IDS}" +close_details +echo "######## REPORT END ########" + +# num_tasks is currently a dummy input, but limit the supplied task list to make +# the mock behavior predictable. +if [[ ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then + TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") +fi + +export AGENT_SETTING_CONFIG +export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" +export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" +if [[ -z "${RITS_BASE_URL:-}" ]]; then + RITS_BASE_URL="${OPENAI_BASE_URL%/}" + [[ "${RITS_BASE_URL}" == */v1 ]] || RITS_BASE_URL="${RITS_BASE_URL}/v1" + export RITS_BASE_URL +fi +export MODEL_NAME + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EVAL_REPO="${EVAL_REPO:-$(cd "${SCRIPT_DIR}/.." && pwd)}" + +if [[ ! -d "${EVAL_REPO}" ]]; then + echo "ERROR: Evaluation repo not found: ${EVAL_REPO}" + exit 1 +fi + +cd "${EVAL_REPO}" + +echo "######## REPORT START ########" +echo "# PR Evaluation" +echo +open_details "Run metadata" +echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" +echo "- PR: ${PR_NUMBER:-unknown}" +echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" +echo "- Runner: ${RUNNER_NAME:-$(hostname)}" +echo "- Agent: ${AGENT}" +close_details +echo +open_details "Evaluation configuration" +echo "- Agent: ${AGENT}" +echo "- Model: ${MODEL_NAME}" +echo "- Split: ${SPLIT_NAME}" +echo "- Requested num_tasks: ${NUM_TASKS}" +echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" +echo "- Task IDs:" +for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" +done +close_details +echo "######## REPORT END ########" + +log "Verifying required commands" +command -v uv >/dev/null 2>&1 || { + echo "ERROR: uv is not installed or is not in PATH." + exit 127 +} + +command -v curl >/dev/null 2>&1 || { + echo "ERROR: curl is not installed or is not in PATH." + exit 127 +} + +log "Starting AppWorld ${AGENT} evaluation" + +EVAL_ARGS=( + --agent "${AGENT}" + --task "${TASK_ID_ARRAY[@]}" +) + +bash benchmarks/appworld/eval.sh "${EVAL_ARGS[@]}" From 342ed08f4f849c5130e2461809f4f6bfb6563137 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Fri, 18 Sep 2026 11:34:14 -0400 Subject: [PATCH 32/40] Update teh compare_report for merging. --- benchmarks/helpers/compare_report.py | 24 --- scripts/run-pr-regression-eval.sh | 247 --------------------------- 2 files changed, 271 deletions(-) delete mode 100644 scripts/run-pr-regression-eval.sh diff --git a/benchmarks/helpers/compare_report.py b/benchmarks/helpers/compare_report.py index 93e8b5f9..1c1117e3 100644 --- a/benchmarks/helpers/compare_report.py +++ b/benchmarks/helpers/compare_report.py @@ -1537,24 +1537,15 @@ def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment fence_open = (lambda: "```text") if markdown else (lambda: "") fence_close = (lambda: "```") if markdown else (lambda: "") -<<<<<<< Updated upstream if markdown and for_pr_comment: lines = ["######## REPORT START ########"] -======= - lines = ["######## REPORT START ########"] - if markdown: ->>>>>>> Stashed changes lines.append("
") lines.append("Evaluation Report") lines.append("") lines.append(h2("Summary")) lines.append("") else: -<<<<<<< Updated upstream lines = [h1("Evaluation Report"), ""] -======= - lines.extend([h1("Evaluation Report"), ""]) ->>>>>>> Stashed changes lines.append(h2("Summary")) lines.append("") if markdown: @@ -1843,17 +1834,10 @@ def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment markdown=markdown, ) ) -<<<<<<< Updated upstream if markdown and for_pr_comment: lines.append("") lines.append("
") lines.append("######## REPORT END ########") -======= - if markdown: - lines.append("") - lines.append("") - lines.append("######## REPORT END ########") ->>>>>>> Stashed changes return "\n".join(lines) @@ -1893,19 +1877,11 @@ def main(): # the canonical bundle location at the end of the run, which is what the # user actually wants to navigate to. if args.output: -<<<<<<< Updated upstream if "command" in args and getattr(args, "command", None) == "eval": if getattr(args, "for_pr_comment", False) or getattr(args, "stdout_markdown", False): stdout_report = report else: stdout_report = generate_eval_report(args.result_file, markdown=False) -======= - # Compare-mode and eval-mode both produce markdown for the saved file. - # For eval-mode, also print markdown to stdout because PR workflows - # capture stdout and post marked report blocks back to GitHub comments. - if "command" in args and getattr(args, "command", None) == "eval": - stdout_report = report ->>>>>>> Stashed changes else: stdout_report = generate_report(config_results, markdown=False) Path(args.output).write_text(report) diff --git a/scripts/run-pr-regression-eval.sh b/scripts/run-pr-regression-eval.sh deleted file mode 100644 index 0b5dc93f..00000000 --- a/scripts/run-pr-regression-eval.sh +++ /dev/null @@ -1,247 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -START_EPOCH="$(date +%s)" -START_TIME="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" - -log() { - printf '[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" -} - -open_details() { - local title="$1" - echo "
" - echo "${title}" - echo -} - -close_details() { - echo - echo "
" -} - -on_exit() { - status=$? - end_epoch="$(date +%s)" - end_time="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" - duration=$((end_epoch - START_EPOCH)) - minutes=$((duration / 60)) - seconds=$((duration % 60)) - - echo "######## REPORT START ########" - echo - open_details "Run summary" - echo "- Started: ${START_TIME}" - echo "- Finished: ${end_time}" - echo "- Duration: ${minutes}m ${seconds}s" - echo "- Exit code: ${status}" - - if [[ ${status} -eq 0 ]]; then - echo "- Result: SUCCESS" - else - echo "- Result: FAILED" - fi - close_details - echo "######## REPORT END ########" -} -trap on_exit EXIT - -DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" -DEFAULT_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" -DEFAULT_SPLIT_NAME="default" -DEFAULT_NUM_TASKS="4" -DEFAULT_AGENT="react" -DEFAULT_AGENT_SETTING_CONFIG="settings.rits.toml" - -MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" -TASK_IDS="${TASK_IDS:-$DEFAULT_TASK_IDS}" -SPLIT_NAME="${SPLIT_NAME:-$DEFAULT_SPLIT_NAME}" -NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" -AGENT="${AGENT:-$DEFAULT_AGENT}" -AGENT_SETTING_CONFIG="${AGENT_SETTING_CONFIG:-$DEFAULT_AGENT_SETTING_CONFIG}" - -# Parse whitespace-separated key=value parameters from the PR comment. -# Supported aliases: -# model_name=... -# task_id=id1,id2 -# task_ids=id1,id2 -# split_name=... -# num_tasks=... -# agent=react|cuga|codeact -COMMENT_BODY="${1:-}" - -# Remove Windows carriage returns and newlines. -COMMENT_BODY="${COMMENT_BODY//$'\r'/ }" -COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" - -read -r -a TOKENS <<< "${COMMENT_BODY}" -echo "######## REPORT START ########" -open_details "Requested parameters" -for token in "${TOKENS[@]}"; do - case "${token}" in - /run-pr-eval) - ;; - - model_name=*) - MODEL_NAME="${token#model_name=}" - ;; - - task_id=*|task_ids=*) - TASK_IDS="${token#*=}" - TASK_IDS="${TASK_IDS//,/ }" - ;; - - split_name=*) - SPLIT_NAME="${token#split_name=}" - ;; - - num_tasks=*) - NUM_TASKS="${token#num_tasks=}" - ;; - - agent=*|agent_type=*) - AGENT="${token#*=}" - ;; - - "") - ;; - - *) - echo "ERROR: Unsupported parameter: ${token}" - echo "Supported parameters: model_name, task_id, task_ids, split_name, num_tasks, agent" - close_details - echo "######## REPORT END ########" - exit 2 - ;; - esac -done - -if [[ -z "${MODEL_NAME}" ]]; then - echo "ERROR: model_name cannot be empty." - close_details - echo "######## REPORT END ########" - exit 2 -fi - -if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]] || [[ "${NUM_TASKS}" -lt 1 ]]; then - echo "ERROR: num_tasks must be a positive integer." - close_details - echo "######## REPORT END ########" - exit 2 -fi - -case "${AGENT}" in - react|cuga|codeact) - ;; - *) - echo "ERROR: agent must be one of: react, cuga, codeact." - close_details - echo "######## REPORT END ########" - exit 2 - ;; -esac - -case "${AGENT_SETTING_CONFIG}" in - settings.rits.toml|settings.rits.proxy.toml) - if [[ -z "${RITS_API_KEY:-}" ]]; then - echo "ERROR: RITS_API_KEY is required when AGENT_SETTING_CONFIG=${AGENT_SETTING_CONFIG}." - close_details - echo "######## REPORT END ########" - exit 2 - fi - ;; - *) - if [[ -z "${OPENAI_API_KEY:-}" ]]; then - echo "ERROR: OPENAI_API_KEY is required when AGENT_SETTING_CONFIG=${AGENT_SETTING_CONFIG}." - close_details - echo "######## REPORT END ########" - exit 2 - fi - ;; -esac - -read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" - -if [[ ${#TASK_ID_ARRAY[@]} -eq 0 ]]; then - echo "ERROR: At least one task_id is required." - close_details - echo "######## REPORT END ########" - exit 2 -fi -echo "- Model: ${MODEL_NAME}" -echo "- Agent: ${AGENT}" -echo "- Split: ${SPLIT_NAME}" -echo "- Num tasks: ${NUM_TASKS}" -echo "- Task IDs: ${TASK_IDS}" -close_details -echo "######## REPORT END ########" - -# num_tasks is currently a dummy input, but limit the supplied task list to make -# the mock behavior predictable. -if [[ ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then - TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") -fi - -export AGENT_SETTING_CONFIG -export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" -export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" -if [[ -z "${RITS_BASE_URL:-}" ]]; then - RITS_BASE_URL="${OPENAI_BASE_URL%/}" - [[ "${RITS_BASE_URL}" == */v1 ]] || RITS_BASE_URL="${RITS_BASE_URL}/v1" - export RITS_BASE_URL -fi -export MODEL_NAME - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -EVAL_REPO="${EVAL_REPO:-$(cd "${SCRIPT_DIR}/.." && pwd)}" - -if [[ ! -d "${EVAL_REPO}" ]]; then - echo "ERROR: Evaluation repo not found: ${EVAL_REPO}" - exit 1 -fi - -cd "${EVAL_REPO}" - -echo "######## REPORT START ########" -echo "# PR Evaluation" -echo -open_details "Run metadata" -echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" -echo "- PR: ${PR_NUMBER:-unknown}" -echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" -echo "- Runner: ${RUNNER_NAME:-$(hostname)}" -echo "- Agent: ${AGENT}" -close_details -echo -open_details "Evaluation configuration" -echo "- Agent: ${AGENT}" -echo "- Model: ${MODEL_NAME}" -echo "- Split: ${SPLIT_NAME}" -echo "- Requested num_tasks: ${NUM_TASKS}" -echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" -echo "- Task IDs:" -for task_id in "${TASK_ID_ARRAY[@]}"; do - echo " - ${task_id}" -done -close_details -echo "######## REPORT END ########" - -log "Verifying required commands" -command -v uv >/dev/null 2>&1 || { - echo "ERROR: uv is not installed or is not in PATH." - exit 127 -} - -command -v curl >/dev/null 2>&1 || { - echo "ERROR: curl is not installed or is not in PATH." - exit 127 -} - -log "Starting AppWorld ${AGENT} evaluation" - -EVAL_ARGS=( - --agent "${AGENT}" - --task "${TASK_ID_ARRAY[@]}" -) - -bash benchmarks/appworld/eval.sh "${EVAL_ARGS[@]}" From 60c59d669e93ea61e01d395dd5f35cb97fa75baa Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Fri, 18 Sep 2026 11:47:52 -0400 Subject: [PATCH 33/40] Move report parsing to the runner. --- .../run-pr-regression-eval.sh | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index c9db5d01..c9c5bd06 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -8,6 +8,58 @@ log() { printf '[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" } +extract_report_blocks() { + local input_path="$1" + local output_path="$2" + + if [[ ! -f "${input_path}" ]]; then + echo "No evaluation output artifact was available." > "${output_path}" + return 0 + fi + + awk ' + $0 == "######## REPORT START ########" { + in_report = 1 + block = "" + next + } + $0 == "######## REPORT END ########" { + if (in_report) { + sub(/^[[:space:]]+/, "", block) + sub(/[[:space:]]+$/, "", block) + if (length(block) > 0) { + if (printed) { + print "" + print "" + } + print block + printed = 1 + } + } + in_report = 0 + block = "" + next + } + in_report { + block = block $0 "\n" + } + END { + if (!printed) { + print "No report section found between REPORT START and REPORT END markers." + } + } + ' "${input_path}" > "${output_path}" +} + +if [[ "${1:-}" == "--extract-report" ]]; then + if [[ $# -ne 3 ]]; then + echo "Usage: $0 --extract-report " >&2 + exit 2 + fi + extract_report_blocks "$2" "$3" + exit 0 +fi + open_details() { local title="$1" echo "
" From 4e9ed0ff0a9e505cc38874d3b5acac5c39cc2526 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Fri, 18 Sep 2026 15:28:34 -0400 Subject: [PATCH 34/40] Clarify PR eval provider environment contract --- scripts/pr-regression-testing/README.md | 12 +++++----- .../run-pr-regression-eval.sh | 22 ++++++------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index 8c633e13..611deb5f 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -93,19 +93,19 @@ Settings -> Secrets and variables -> Actions Required secrets: - `RITS_API_KEY`: required for `provider=rits`. -- `LITE_LLM_KEY`: required for `provider=litellm`, unless `OPENAI_API_KEY` is set instead. -- `OPENAI_API_KEY`: optional fallback key for OpenAI-compatible/LiteLLM flows. +- `LITE_LLM_KEY`: preferred secret for `provider=litellm`; the workflow injects it into `OPENAI_API_KEY`. +- `OPENAI_API_KEY`: optional fallback secret for `provider=litellm` when `LITE_LLM_KEY` is not set. Optional repository variables: - `MODEL_NAME`: default model override used by the workflow environment. -- `OPENAI_BASE_URL`: default OpenAI-compatible endpoint override. -- `LITE_LLM_URL`: LiteLLM endpoint override. If omitted, `provider=litellm` defaults to `https://ete-litellm.ai-models.vpc-int.res.ibm.com/`. +- `RITS_BASE_URL`: RITS endpoint override for `provider=rits`. +- `LITELLM_BASE_URL`: LiteLLM endpoint override for `provider=litellm`. Provider defaults used by `run-pr-regression-eval.sh`: -- `provider=rits` uses `RITS_API_KEY` and defaults `OPENAI_BASE_URL` to `https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100`. -- `provider=litellm` uses `LITE_LLM_KEY` first, then `OPENAI_API_KEY`; it uses `LITE_LLM_URL` first, then `OPENAI_BASE_URL`, then `https://ete-litellm.ai-models.vpc-int.res.ibm.com/`. Unless `model_name` is explicitly set in the PR comment, it uses `aws/gpt-oss-120b`. +- `provider=rits` selects `settings.rits.toml`, requires `RITS_API_KEY`, and passes `RITS_BASE_URL` to `models.py`. +- `provider=litellm` selects `settings.litellm.toml` for the CUGA agent or `settings.openai.toml` for non-CUGA agents, requires `OPENAI_API_KEY`, and passes `OPENAI_BASE_URL` to `models.py`. Unless `model_name` is explicitly set in the PR comment, it uses `aws/gpt-oss-120b`. ## Run PR Evaluations diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index c9c5bd06..2db04697 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -100,6 +100,8 @@ trap on_exit EXIT DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" DEFAULT_LITELLM_MODEL_NAME="aws/gpt-oss-120b" +DEFAULT_RITS_BASE_URL="https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100/v1" +DEFAULT_LITELLM_BASE_URL="https://ete-litellm.ai-models.vpc-int.res.ibm.com/" DEFAULT_APPWORLD_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" DEFAULT_BENCHMARK="appworld" DEFAULT_NUM_TASKS="4" @@ -329,6 +331,8 @@ case "${PROVIDER}" in echo "######## REPORT END ########" exit 2 fi + RITS_BASE_URL="${RITS_BASE_URL:-${DEFAULT_RITS_BASE_URL}}" + export RITS_API_KEY RITS_BASE_URL ;; litellm) if [[ "${AGENT}" == "cuga" ]]; then @@ -339,16 +343,15 @@ case "${PROVIDER}" in if [[ "${MODEL_NAME_FROM_COMMENT}" == "false" ]]; then MODEL_NAME="${DEFAULT_LITELLM_MODEL_NAME}" fi - OPENAI_BASE_URL="${LITE_LLM_URL:-${OPENAI_BASE_URL:-https://ete-litellm.ai-models.vpc-int.res.ibm.com/}}" - OPENAI_API_KEY="${LITE_LLM_KEY:-${OPENAI_API_KEY:-}}" + OPENAI_BASE_URL="${OPENAI_BASE_URL:-${DEFAULT_LITELLM_BASE_URL}}" if [[ -z "${OPENAI_API_KEY:-}" ]]; then - echo "ERROR: LITE_LLM_KEY or OPENAI_API_KEY is required when provider=litellm." + echo "ERROR: OPENAI_API_KEY is required when provider=litellm." close_details echo "######## REPORT END ########" exit 2 fi if [[ -z "${OPENAI_BASE_URL:-}" ]]; then - echo "ERROR: LITE_LLM_URL or OPENAI_BASE_URL is required when provider=litellm." + echo "ERROR: OPENAI_BASE_URL is required when provider=litellm." close_details echo "######## REPORT END ########" exit 2 @@ -363,17 +366,6 @@ case "${PROVIDER}" in ;; esac -if [[ "${PROVIDER}" == "rits" ]]; then - export OPENAI_BASE_URL="${OPENAI_BASE_URL:-https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100}" - if [[ -z "${RITS_BASE_URL:-}" ]]; then - RITS_BASE_URL="${OPENAI_BASE_URL%/}" - [[ "${RITS_BASE_URL}" == */v1 ]] || RITS_BASE_URL="${RITS_BASE_URL}/v1" - export RITS_BASE_URL - fi -else - : -fi - if [[ -z "${TASK_IDS}" && -z "${EVAL_KEY}" && "${BENCHMARK}" == "appworld" ]]; then TASK_IDS="${DEFAULT_APPWORLD_TASK_IDS}" fi From 2269957a4e60cb50189bd9a3533a71b89c644971 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Fri, 18 Sep 2026 16:36:21 -0400 Subject: [PATCH 35/40] Fix ReAct eval Ruff import issues --- benchmarks/appworld/appworld_eval_react.py | 1 - benchmarks/helpers/react_agent.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmarks/appworld/appworld_eval_react.py b/benchmarks/appworld/appworld_eval_react.py index d79c357e..0346fdb3 100644 --- a/benchmarks/appworld/appworld_eval_react.py +++ b/benchmarks/appworld/appworld_eval_react.py @@ -33,7 +33,6 @@ from datetime import datetime from typing import Any, Optional -import httpx from appworld import AppWorld, load_task_ids # pyright: ignore[reportAttributeAccessIssue] from cuga.backend.activity_tracker.tracker import ActivityTracker, Step from cuga.backend.cuga_graph.state.agent_state import VariablesManager diff --git a/benchmarks/helpers/react_agent.py b/benchmarks/helpers/react_agent.py index 6a11157c..e3477f17 100644 --- a/benchmarks/helpers/react_agent.py +++ b/benchmarks/helpers/react_agent.py @@ -19,11 +19,11 @@ from pathlib import Path from typing import Any, Optional -from langchain_openai import ChatOpenAI from cuga.backend.cuga_graph.nodes.cuga_lite.providers.combined import ( CombinedToolProvider, ) from langchain_core.messages import HumanMessage +from langchain_openai import ChatOpenAI from loguru import logger TOOL_BLOCK_RE = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE) From ca01b44b2896d124907abac65af6bb27231b7d78 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 24 Sep 2026 09:19:14 -0400 Subject: [PATCH 36/40] Keep PR eval comment formatting out of bundles --- benchmarks/appworld/eval.sh | 5 +++-- benchmarks/bpo/eval.sh | 5 +++-- benchmarks/m3/eval.sh | 8 +++++--- benchmarks/oak_health_insurance/eval.sh | 5 +++-- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/benchmarks/appworld/eval.sh b/benchmarks/appworld/eval.sh index c0e27f70..0629545c 100755 --- a/benchmarks/appworld/eval.sh +++ b/benchmarks/appworld/eval.sh @@ -411,10 +411,11 @@ if [ -n "$LATEST_RESULT" ] && [ "${NO_BUNDLE:-false}" != "true" ]; then # Generate eval report REPORT_TMP=$(mktemp /tmp/appworld_eval_report_XXXXXX) REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then - REPORT_ARGS+=(--for-pr-comment --stdout-markdown) + uv run --no-sync python -m benchmarks.helpers.compare_report \ + eval --result-file "$LATEST_RESULT" --for-pr-comment --stdout-markdown fi - uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" BUNDLE_ARGS=(assemble --benchmark appworld --result-files "$LATEST_RESULT" diff --git a/benchmarks/bpo/eval.sh b/benchmarks/bpo/eval.sh index 746081c2..569c9d60 100755 --- a/benchmarks/bpo/eval.sh +++ b/benchmarks/bpo/eval.sh @@ -410,10 +410,11 @@ if [ $EVAL_EXIT_CODE -eq 0 ]; then # Generate eval report REPORT_TMP=$(mktemp /tmp/bpo_eval_report_XXXXXX) REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then - REPORT_ARGS+=(--for-pr-comment --stdout-markdown) + uv run --no-sync python -m benchmarks.helpers.compare_report \ + eval --result-file "$LATEST_RESULT" --for-pr-comment --stdout-markdown fi - uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" BUNDLE_ARGS=(assemble --benchmark bpo --result-files "$LATEST_RESULT" diff --git a/benchmarks/m3/eval.sh b/benchmarks/m3/eval.sh index bdc1f4e1..ea417aad 100755 --- a/benchmarks/m3/eval.sh +++ b/benchmarks/m3/eval.sh @@ -264,11 +264,13 @@ create_bundle() { local report_tmp report_tmp=$(mktemp /tmp/m3_eval_report_XXXXXX) local report_args=(eval --result-file "$latest_result" --output "$report_tmp") - if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then - report_args+=(--for-pr-comment --stdout-markdown) - fi uv run --no-sync python -m benchmarks.helpers.compare_report "${report_args[@]}" || \ echo -e "${YELLOW:-}Report generation failed — bundling without report.${NC:-}" + if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then + uv run --no-sync python -m benchmarks.helpers.compare_report \ + eval --result-file "$latest_result" --for-pr-comment --stdout-markdown || \ + echo -e "${YELLOW:-}PR report generation failed — continuing with bundle.${NC:-}" + fi local bundle_args=(assemble --benchmark m3 --result-files "$latest_result" diff --git a/benchmarks/oak_health_insurance/eval.sh b/benchmarks/oak_health_insurance/eval.sh index de7c3288..d937dd59 100755 --- a/benchmarks/oak_health_insurance/eval.sh +++ b/benchmarks/oak_health_insurance/eval.sh @@ -243,10 +243,11 @@ if [ $EVAL_EXIT -eq 0 ]; then # Generate eval report REPORT_TMP=$(mktemp /tmp/oak_eval_report_XXXXXX) REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then - REPORT_ARGS+=(--for-pr-comment --stdout-markdown) + uv run --no-sync python -m benchmarks.helpers.compare_report \ + eval --result-file "$LATEST_RESULT" --for-pr-comment --stdout-markdown fi - uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" BUNDLE_ARGS=(assemble --benchmark oak_health_insurance --result-files "$LATEST_RESULT" From 02dd85d1cdaf733b9aa54115ea08e0821f2656be Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 24 Sep 2026 09:34:39 -0400 Subject: [PATCH 37/40] Harden PR eval provider config handling --- benchmarks/helpers/react_agent.py | 23 ++++++++++++++++--- scripts/pr-regression-testing/README.md | 3 ++- .../run-pr-regression-eval.sh | 6 +++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/benchmarks/helpers/react_agent.py b/benchmarks/helpers/react_agent.py index e3477f17..2f46afce 100644 --- a/benchmarks/helpers/react_agent.py +++ b/benchmarks/helpers/react_agent.py @@ -23,9 +23,16 @@ CombinedToolProvider, ) from langchain_core.messages import HumanMessage -from langchain_openai import ChatOpenAI from loguru import logger +try: + from langchain_openai import ChatOpenAI as _ChatOpenAI +except ImportError as exc: + _ChatOpenAI = None + _CHAT_OPENAI_IMPORT_ERROR = exc +else: + _CHAT_OPENAI_IMPORT_ERROR = None + TOOL_BLOCK_RE = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE) PROMPT_ROLE_RE = re.compile(r"(USER|ASSISTANT|SYSTEM):\n", re.IGNORECASE) PROMPT_PATH = Path(__file__).with_name("react_agent_prompt.txt") @@ -45,10 +52,20 @@ class ReactInvokeResult: react_steps: int = 0 -class ChatRits(ChatOpenAI): +def _require_chat_openai(): + if _ChatOpenAI is None: + raise RuntimeError( + "langchain-openai is required for React agent with OpenAI-compatible or RITS providers. " + "Install with: pip install langchain-openai" + ) from _CHAT_OPENAI_IMPORT_ERROR + return _ChatOpenAI + + +class ChatRits(_ChatOpenAI if _ChatOpenAI is not None else object): """RITS chat model integration using langchain-openai.""" def __init__(self, config: dict[str, Any]): + _require_chat_openai() model_name = config.get("model_name", "openai/gpt-oss-120b-a100") end_point = ( config.get("end_point") @@ -201,7 +218,7 @@ def _create_llm(self) -> Any: llm_kwargs["http_async_client"] = httpx.AsyncClient(verify=False) # noqa: S501 # nosec B501 — same logger.info("SSL verification disabled for ChatOpenAI") - return ChatOpenAI(**llm_kwargs) + return _require_chat_openai()(**llm_kwargs) elif settings_config in ("settings.rits.toml", "settings.rits.proxy.toml"): api_base = os.getenv("RITS_BASE_URL") or os.getenv("LITE_LLM_URL") or os.getenv("OPENAI_BASE_URL") diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index 611deb5f..8dc4fe70 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -100,12 +100,13 @@ Optional repository variables: - `MODEL_NAME`: default model override used by the workflow environment. - `RITS_BASE_URL`: RITS endpoint override for `provider=rits`. -- `LITELLM_BASE_URL`: LiteLLM endpoint override for `provider=litellm`. +- `LITELLM_BASE_URL`: LiteLLM endpoint override for `provider=litellm`; the workflow injects it into `OPENAI_BASE_URL`. Provider defaults used by `run-pr-regression-eval.sh`: - `provider=rits` selects `settings.rits.toml`, requires `RITS_API_KEY`, and passes `RITS_BASE_URL` to `models.py`. - `provider=litellm` selects `settings.litellm.toml` for the CUGA agent or `settings.openai.toml` for non-CUGA agents, requires `OPENAI_API_KEY`, and passes `OPENAI_BASE_URL` to `models.py`. Unless `model_name` is explicitly set in the PR comment, it uses `aws/gpt-oss-120b`. +- For `provider=rits`, overriding `model_name` in the PR command requires `RITS_BASE_URL` to be configured so the model name and endpoint cannot silently diverge. ## Run PR Evaluations diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index 2db04697..1ad6a1d1 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -331,6 +331,12 @@ case "${PROVIDER}" in echo "######## REPORT END ########" exit 2 fi + if [[ "${MODEL_NAME_FROM_COMMENT}" == "true" && -z "${RITS_BASE_URL:-}" ]]; then + echo "ERROR: provider=rits model_name overrides require RITS_BASE_URL to avoid mismatched model endpoints." + close_details + echo "######## REPORT END ########" + exit 2 + fi RITS_BASE_URL="${RITS_BASE_URL:-${DEFAULT_RITS_BASE_URL}}" export RITS_API_KEY RITS_BASE_URL ;; From 9f6c7a1bbd215f5acb64462d64001d66e5c9dece Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Thu, 1 Oct 2026 14:43:40 -0400 Subject: [PATCH 38/40] Use AppWorld SDK evaluator for PR evals --- scripts/pr-regression-testing/README.md | 2 ++ scripts/pr-regression-testing/run-pr-regression-eval.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index 8dc4fe70..b28ca513 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -125,6 +125,8 @@ The `sha` value must be the current 40-character PR head commit SHA. The workflo - `num_tasks=4` - default AppWorld task IDs: `9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1` +AppWorld PR evaluations are run with the AppWorld SDK evaluator (`--sdk`). + For `provider=litellm`, the default model changes to `aws/gpt-oss-120b` unless `model_name` is explicitly provided. ## Command Parameters diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index 1ad6a1d1..c13a048f 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -498,6 +498,7 @@ EVAL_ARGS=( ) if [[ "${BENCHMARK}" == "appworld" ]]; then + EVAL_ARGS+=(--sdk) if [[ -n "${EVAL_KEY}" ]]; then EVAL_ARGS+=(--eval-key "${EVAL_KEY}") fi From a25b56480065abb2a9f327d83dd6e1a4b62d4cec Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 5 Oct 2026 11:29:20 -0400 Subject: [PATCH 39/40] Use AppWorld SDK only for CUGA PR evals --- scripts/pr-regression-testing/README.md | 2 +- scripts/pr-regression-testing/run-pr-regression-eval.sh | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md index b28ca513..c214ff6e 100644 --- a/scripts/pr-regression-testing/README.md +++ b/scripts/pr-regression-testing/README.md @@ -125,7 +125,7 @@ The `sha` value must be the current 40-character PR head commit SHA. The workflo - `num_tasks=4` - default AppWorld task IDs: `9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1` -AppWorld PR evaluations are run with the AppWorld SDK evaluator (`--sdk`). +AppWorld PR evaluations with `agent=cuga` are run with the AppWorld SDK evaluator (`--sdk`). For `provider=litellm`, the default model changes to `aws/gpt-oss-120b` unless `model_name` is explicitly provided. diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh index c13a048f..0d1e9cbd 100755 --- a/scripts/pr-regression-testing/run-pr-regression-eval.sh +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -498,7 +498,9 @@ EVAL_ARGS=( ) if [[ "${BENCHMARK}" == "appworld" ]]; then - EVAL_ARGS+=(--sdk) + if [[ "${AGENT}" == "cuga" ]]; then + EVAL_ARGS+=(--sdk) + fi if [[ -n "${EVAL_KEY}" ]]; then EVAL_ARGS+=(--eval-key "${EVAL_KEY}") fi From ab308914d789dd1371444c4a01d6763c82d8c185 Mon Sep 17 00:00:00 2001 From: AnkitaNaik Date: Mon, 5 Oct 2026 11:31:40 -0400 Subject: [PATCH 40/40] Format React agent helper with Ruff --- benchmarks/helpers/react_agent.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/helpers/react_agent.py b/benchmarks/helpers/react_agent.py index 2f46afce..8c61b51c 100644 --- a/benchmarks/helpers/react_agent.py +++ b/benchmarks/helpers/react_agent.py @@ -224,7 +224,9 @@ def _create_llm(self) -> Any: api_base = os.getenv("RITS_BASE_URL") or os.getenv("LITE_LLM_URL") or os.getenv("OPENAI_BASE_URL") api_key = os.getenv("RITS_API_KEY") or os.getenv("LITE_LLM_KEY") or os.getenv("OPENAI_API_KEY") model_name = os.getenv("MODEL_NAME") or self.model or "openai/gpt-oss-120b-a100" - logger.info(f"RITS config: model={model_name}, api_base={api_base}, api_key_present={bool(api_key)}") + logger.info( + f"RITS config: model={model_name}, api_base={api_base}, api_key_present={bool(api_key)}" + ) return ChatRits(config={"model_name": model_name, "end_point": api_base, "api_key": api_key}) else: