diff --git a/benchmarks/appworld/appworld_eval.py b/benchmarks/appworld/appworld_eval.py
index 8c45a10f..8f892663 100644
--- a/benchmarks/appworld/appworld_eval.py
+++ b/benchmarks/appworld/appworld_eval.py
@@ -177,6 +177,14 @@ def _get_registry_base_url() -> str:
return "http://localhost:8001"
+def _is_langfuse_tracing_enabled() -> bool:
+ env_value = os.getenv("DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING")
+ if env_value is not None:
+ return env_value.strip().lower() in {"1", "true", "yes", "on"}
+
+ return bool(getattr(getattr(settings, "advanced_features", None), "langfuse_tracing", False))
+
+
async def run_agent_on_task(
task_id: str,
experiment_name: str = "api_cuga_agent",
@@ -217,7 +225,7 @@ async def run_agent_on_task(
# Without this, LangfuseCallbackHandler creates a new root trace for every
# agent_loop_obj.run() call in the controller while-loop.
_langfuse = None
- if _LANGFUSE_AVAILABLE and _get_langfuse_client is not None:
+ if _is_langfuse_tracing_enabled() and _LANGFUSE_AVAILABLE and _get_langfuse_client is not None:
try:
_langfuse = _get_langfuse_client()
_run_uid = uuid.uuid4().hex[:8]
@@ -335,8 +343,9 @@ async def run_agent_on_task(
langfuse_data = None
# Extract Langfuse data if trace_id is available
- langfuse_handler = LangfuseTraceHandler(langfuse_trace_id)
- langfuse_data = await langfuse_handler.get_langfuse_data()
+ if langfuse_trace_id:
+ langfuse_handler = LangfuseTraceHandler(langfuse_trace_id)
+ langfuse_data = await langfuse_handler.get_langfuse_data()
if langfuse_trace_id:
task_result.trace_id = langfuse_trace_id
if langfuse_data:
@@ -469,14 +478,21 @@ def _print_appworld_summary(report: dict):
success_rate = report.get("success_rate", 0)
avg_steps = report.get("avg_steps", 0)
avg_duration = report.get("avg_duration", 0)
+ for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes")
- print("\n" + "=" * 80)
- print("EVALUATION COMPLETE")
- print("=" * 80)
+ if for_pr_comment:
+ print("######## REPORT START ########")
+ print("## EVALUATION COMPLETE")
+ else:
+ print("\n" + "=" * 80)
+ print("EVALUATION COMPLETE")
+ print("=" * 80)
print(f"Total Tasks: {total}")
print(f"Completed: {completed}/{total} ({success_rate:.1%})")
print(f"Avg Steps: {avg_steps:.2f}")
print(f"Avg Duration: {avg_duration:.2f}s")
+ if for_pr_comment:
+ print("######## REPORT END ########")
async def main():
diff --git a/benchmarks/appworld/appworld_eval_codeact.py b/benchmarks/appworld/appworld_eval_codeact.py
index f92fa516..96f5a9e2 100644
--- a/benchmarks/appworld/appworld_eval_codeact.py
+++ b/benchmarks/appworld/appworld_eval_codeact.py
@@ -96,14 +96,21 @@ def _print_appworld_summary(report: dict):
success_rate = report.get("success_rate", 0)
avg_steps = report.get("avg_steps", 0)
avg_duration = report.get("avg_duration", 0)
+ for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes")
- print("\n" + "=" * 80)
- print("EVALUATION COMPLETE")
- print("=" * 80)
+ if for_pr_comment:
+ print("######## REPORT START ########")
+ print("## EVALUATION COMPLETE")
+ else:
+ print("\n" + "=" * 80)
+ print("EVALUATION COMPLETE")
+ print("=" * 80)
print(f"Total Tasks: {total}")
print(f"Completed: {completed}/{total} ({success_rate:.1%})")
print(f"Avg Steps: {avg_steps:.2f}")
print(f"Avg Duration: {avg_duration:.2f}s")
+ if for_pr_comment:
+ print("######## REPORT END ########")
def _parse_prompt_to_messages(input_str: str) -> list[dict[str, Any]]:
diff --git a/benchmarks/appworld/appworld_eval_react.py b/benchmarks/appworld/appworld_eval_react.py
index fbbd5a5b..0346fdb3 100644
--- a/benchmarks/appworld/appworld_eval_react.py
+++ b/benchmarks/appworld/appworld_eval_react.py
@@ -180,14 +180,21 @@ def _print_appworld_summary(report: dict):
success_rate = report.get("success_rate", 0)
avg_steps = report.get("avg_steps", 0)
avg_duration = report.get("avg_duration", 0)
+ for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes")
- print("\n" + "=" * 80)
- print("EVALUATION COMPLETE")
- print("=" * 80)
+ if for_pr_comment:
+ print("######## REPORT START ########")
+ print("## EVALUATION COMPLETE")
+ else:
+ print("\n" + "=" * 80)
+ print("EVALUATION COMPLETE")
+ print("=" * 80)
print(f"Total Tasks: {total}")
print(f"Completed: {completed}/{total} ({success_rate:.1%})")
print(f"Avg Steps: {avg_steps:.2f}")
print(f"Avg Duration: {avg_duration:.2f}s")
+ if for_pr_comment:
+ print("######## REPORT END ########")
def _render_initial_prompt(world: AppWorld) -> str:
diff --git a/benchmarks/appworld/eval.sh b/benchmarks/appworld/eval.sh
index a7f7ec0f..0629545c 100755
--- a/benchmarks/appworld/eval.sh
+++ b/benchmarks/appworld/eval.sh
@@ -410,8 +410,12 @@ if [ -n "$LATEST_RESULT" ] && [ "${NO_BUNDLE:-false}" != "true" ]; then
# Generate eval report
REPORT_TMP=$(mktemp /tmp/appworld_eval_report_XXXXXX)
- uv run --no-sync python -m benchmarks.helpers.compare_report eval \
- --result-file "$LATEST_RESULT" --output "$REPORT_TMP"
+ REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP")
+ uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}"
+ if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then
+ uv run --no-sync python -m benchmarks.helpers.compare_report \
+ eval --result-file "$LATEST_RESULT" --for-pr-comment --stdout-markdown
+ fi
BUNDLE_ARGS=(assemble --benchmark appworld
--result-files "$LATEST_RESULT"
diff --git a/benchmarks/bpo/eval.sh b/benchmarks/bpo/eval.sh
index f4694dd9..569c9d60 100755
--- a/benchmarks/bpo/eval.sh
+++ b/benchmarks/bpo/eval.sh
@@ -409,8 +409,12 @@ if [ $EVAL_EXIT_CODE -eq 0 ]; then
# Generate eval report
REPORT_TMP=$(mktemp /tmp/bpo_eval_report_XXXXXX)
- uv run --no-sync python -m benchmarks.helpers.compare_report eval \
- --result-file "$LATEST_RESULT" --output "$REPORT_TMP"
+ REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP")
+ uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}"
+ if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then
+ uv run --no-sync python -m benchmarks.helpers.compare_report \
+ eval --result-file "$LATEST_RESULT" --for-pr-comment --stdout-markdown
+ fi
BUNDLE_ARGS=(assemble --benchmark bpo
--result-files "$LATEST_RESULT"
diff --git a/benchmarks/helpers/compare_report.py b/benchmarks/helpers/compare_report.py
index 58382cfd..1c1117e3 100644
--- a/benchmarks/helpers/compare_report.py
+++ b/benchmarks/helpers/compare_report.py
@@ -1516,7 +1516,7 @@ def _bucket_m3_tasks(tasks: dict) -> tuple:
return rows, True
-def generate_eval_report(result_file: str, markdown: bool = True) -> str:
+def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment: bool = False) -> str:
"""Generate a single-evaluation-run report.
When the result file is M3-shaped (each task has ``m3_task_id`` + ``domain``),
@@ -1524,7 +1524,8 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str:
rows are readable instead of an unattributed UUID. When ``markdown=False``,
the same content is rendered as a plain-text monospace table for terminals
(compare.sh's stdout path); ``markdown=True`` (default) is what gets saved
- into the bundle's report.md.
+ into the bundle's report.md. ``for_pr_comment`` is an explicit opt-in for
+ the PR-regression workflow's collapsible, sentinel-wrapped markdown.
"""
parsed = parse_result_file(result_file)
rows, grouped = _bucket_m3_tasks(parsed["tasks"])
@@ -1536,9 +1537,17 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str:
fence_open = (lambda: "```text") if markdown else (lambda: "")
fence_close = (lambda: "```") if markdown else (lambda: "")
- lines = [h1("Evaluation Report"), ""]
- lines.append(h2("Summary"))
- lines.append("")
+ if markdown and for_pr_comment:
+ lines = ["######## REPORT START ########"]
+ lines.append("")
+ lines.append("Evaluation Report
")
+ lines.append("")
+ lines.append(h2("Summary"))
+ lines.append("")
+ else:
+ lines = [h1("Evaluation Report"), ""]
+ lines.append(h2("Summary"))
+ lines.append("")
if markdown:
lines.append(f"- **Pass@1**: {parsed['passed']}/{parsed['total']} ({parsed['rate']:.1%})")
lines.append(f"- **Total Tokens**: {_fmt(parsed['tokens'])}")
@@ -1825,6 +1834,10 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str:
markdown=markdown,
)
)
+ if markdown and for_pr_comment:
+ lines.append("")
+ lines.append(" ")
+ lines.append("######## REPORT END ########")
return "\n".join(lines)
@@ -1839,8 +1852,10 @@ def main():
parser.add_argument("command") # consume "eval"
parser.add_argument("--result-file", required=True)
parser.add_argument("--output", "-o", default=None)
+ parser.add_argument("--for-pr-comment", action="store_true")
+ parser.add_argument("--stdout-markdown", action="store_true")
args = parser.parse_args()
- report = generate_eval_report(args.result_file)
+ report = generate_eval_report(args.result_file, for_pr_comment=args.for_pr_comment)
else:
parser = argparse.ArgumentParser(description="Generate comparison report")
parser.add_argument("command") # consume "compare"
@@ -1862,14 +1877,15 @@ def main():
# the canonical bundle location at the end of the run, which is what the
# user actually wants to navigate to.
if args.output:
- # Compare-mode and eval-mode both produce markdown for the saved file
- # and re-render plain text for the terminal.
if "command" in args and getattr(args, "command", None) == "eval":
- plain = generate_eval_report(args.result_file, markdown=False)
+ if getattr(args, "for_pr_comment", False) or getattr(args, "stdout_markdown", False):
+ stdout_report = report
+ else:
+ stdout_report = generate_eval_report(args.result_file, markdown=False)
else:
- plain = generate_report(config_results, markdown=False)
+ stdout_report = generate_report(config_results, markdown=False)
Path(args.output).write_text(report)
- print(plain)
+ print(stdout_report)
else:
# No file requested — just print whatever generate_* produced.
print(report)
diff --git a/benchmarks/helpers/react_agent.py b/benchmarks/helpers/react_agent.py
index be1c941c..8c61b51c 100644
--- a/benchmarks/helpers/react_agent.py
+++ b/benchmarks/helpers/react_agent.py
@@ -25,6 +25,14 @@
from langchain_core.messages import HumanMessage
from loguru import logger
+try:
+ from langchain_openai import ChatOpenAI as _ChatOpenAI
+except ImportError as exc:
+ _ChatOpenAI = None
+ _CHAT_OPENAI_IMPORT_ERROR = exc
+else:
+ _CHAT_OPENAI_IMPORT_ERROR = None
+
TOOL_BLOCK_RE = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE)
PROMPT_ROLE_RE = re.compile(r"(USER|ASSISTANT|SYSTEM):\n", re.IGNORECASE)
PROMPT_PATH = Path(__file__).with_name("react_agent_prompt.txt")
@@ -44,6 +52,43 @@ class ReactInvokeResult:
react_steps: int = 0
+def _require_chat_openai():
+ if _ChatOpenAI is None:
+ raise RuntimeError(
+ "langchain-openai is required for React agent with OpenAI-compatible or RITS providers. "
+ "Install with: pip install langchain-openai"
+ ) from _CHAT_OPENAI_IMPORT_ERROR
+ return _ChatOpenAI
+
+
+class ChatRits(_ChatOpenAI if _ChatOpenAI is not None else object):
+ """RITS chat model integration using langchain-openai."""
+
+ def __init__(self, config: dict[str, Any]):
+ _require_chat_openai()
+ model_name = config.get("model_name", "openai/gpt-oss-120b-a100")
+ end_point = (
+ config.get("end_point")
+ or "https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100"
+ )
+ rits_api_key = config.get("api_key") or os.getenv("RITS_API_KEY")
+ if rits_api_key is None:
+ raise ValueError("RITS_API_KEY is required for settings.rits.toml")
+
+ params = config.get("params", {})
+ base_url = end_point.rstrip("/")
+ if not base_url.endswith("/v1"):
+ base_url = f"{base_url}/v1"
+ rits_config: dict[str, Any] = {
+ "model": model_name,
+ "api_key": "/",
+ "default_headers": {"RITS_API_KEY": rits_api_key},
+ "base_url": base_url,
+ }
+ rits_config.update(params)
+ super().__init__(**rits_config)
+
+
class GenericReactAgent:
"""Prompt-template driven generic ReAct agent over benchmark tools."""
@@ -149,14 +194,6 @@ def _create_llm(self) -> Any:
)
elif settings_config == "settings.openai.toml":
- try:
- from langchain_openai import ChatOpenAI
- except ImportError as exc:
- raise RuntimeError(
- "langchain-openai is required for React agent with OpenAI. "
- "Install with: pip install langchain-openai"
- ) from exc
-
disable_ssl = os.getenv("CUGA_DISABLE_SSL", "").lower() in ("true", "1", "yes")
ssl_verify = (
os.getenv("OPENAI_SSL_VERIFY", "true").lower() not in ("false", "0", "no") and not disable_ssl
@@ -181,12 +218,22 @@ def _create_llm(self) -> Any:
llm_kwargs["http_async_client"] = httpx.AsyncClient(verify=False) # noqa: S501 # nosec B501 — same
logger.info("SSL verification disabled for ChatOpenAI")
- return ChatOpenAI(**llm_kwargs)
+ return _require_chat_openai()(**llm_kwargs)
+
+ elif settings_config in ("settings.rits.toml", "settings.rits.proxy.toml"):
+ api_base = os.getenv("RITS_BASE_URL") or os.getenv("LITE_LLM_URL") or os.getenv("OPENAI_BASE_URL")
+ api_key = os.getenv("RITS_API_KEY") or os.getenv("LITE_LLM_KEY") or os.getenv("OPENAI_API_KEY")
+ model_name = os.getenv("MODEL_NAME") or self.model or "openai/gpt-oss-120b-a100"
+ logger.info(
+ f"RITS config: model={model_name}, api_base={api_base}, api_key_present={bool(api_key)}"
+ )
+ return ChatRits(config={"model_name": model_name, "end_point": api_base, "api_key": api_key})
else:
raise RuntimeError(
"Unsupported AGENT_SETTING_CONFIG for React agent. "
- "Expected 'settings.groq.toml' or 'settings.openai.toml'."
+ "Expected 'settings.groq.toml', 'settings.openai.toml', "
+ "'settings.rits.toml', or 'settings.rits.proxy.toml'."
)
def _get_llm(self) -> Any:
diff --git a/benchmarks/helpers/tests/test_compare_report.py b/benchmarks/helpers/tests/test_compare_report.py
index 46f5a55d..93345b8d 100644
--- a/benchmarks/helpers/tests/test_compare_report.py
+++ b/benchmarks/helpers/tests/test_compare_report.py
@@ -21,10 +21,12 @@
"""
import json
+import sys
from pathlib import Path
import pytest
+from benchmarks.helpers import compare_report as compare_report_module
from benchmarks.helpers.compare_report import (
_aggregate_receipt_costs,
_last_turn_judge_scores,
@@ -221,6 +223,78 @@ def test_parse_sdk_results_includes_react_steps():
assert parsed["tasks"]["t1"]["steps"] == 3
+def test_eval_report_default_omits_pr_comment_wrapping(tmp_path):
+ result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False})
+
+ report = generate_eval_report(result_file)
+
+ assert report.startswith("# Evaluation Report")
+ assert "######## REPORT START ########" not in report
+ assert "######## REPORT END ########" not in report
+ assert "Evaluation Report" not in report
+
+
+def test_eval_report_pr_comment_wrapping_is_opt_in(tmp_path):
+ result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False})
+
+ report = generate_eval_report(result_file, for_pr_comment=True)
+
+ assert report.startswith("######## REPORT START ########")
+ assert "Evaluation Report" in report
+ assert report.rstrip().endswith("######## REPORT END ########")
+
+
+def test_eval_main_with_output_prints_plain_text_by_default(tmp_path, monkeypatch, capsys):
+ result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False})
+ output_file = tmp_path / "report.md"
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "compare_report.py",
+ "eval",
+ "--result-file",
+ result_file,
+ "--output",
+ str(output_file),
+ ],
+ )
+
+ compare_report_module.main()
+
+ stdout = capsys.readouterr().out
+ saved_report = output_file.read_text()
+ assert stdout.startswith("\nEvaluation Report")
+ assert "######## REPORT START ########" not in stdout
+ assert saved_report.startswith("# Evaluation Report")
+ assert "######## REPORT START ########" not in saved_report
+
+
+def test_eval_main_pr_comment_mode_prints_markdown_markers(tmp_path, monkeypatch, capsys):
+ result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False})
+ output_file = tmp_path / "report.md"
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "compare_report.py",
+ "eval",
+ "--result-file",
+ result_file,
+ "--output",
+ str(output_file),
+ "--for-pr-comment",
+ ],
+ )
+
+ compare_report_module.main()
+
+ stdout = capsys.readouterr().out
+ saved_report = output_file.read_text()
+ assert stdout.startswith("######## REPORT START ########")
+ assert saved_report.startswith("######## REPORT START ########")
+
+
def _m3_run(tmp_path: Path, name: str, tasks) -> str:
"""Write an M3-shape ``*_final_report.json`` and return its path.
diff --git a/benchmarks/m3/eval.sh b/benchmarks/m3/eval.sh
index 1c79bb7c..ea417aad 100755
--- a/benchmarks/m3/eval.sh
+++ b/benchmarks/m3/eval.sh
@@ -263,9 +263,14 @@ create_bundle() {
# want the bundle, so don't let `set -e` abort here).
local report_tmp
report_tmp=$(mktemp /tmp/m3_eval_report_XXXXXX)
- uv run --no-sync python -m benchmarks.helpers.compare_report eval \
- --result-file "$latest_result" --output "$report_tmp" || \
+ local report_args=(eval --result-file "$latest_result" --output "$report_tmp")
+ uv run --no-sync python -m benchmarks.helpers.compare_report "${report_args[@]}" || \
echo -e "${YELLOW:-}Report generation failed — bundling without report.${NC:-}"
+ if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then
+ uv run --no-sync python -m benchmarks.helpers.compare_report \
+ eval --result-file "$latest_result" --for-pr-comment --stdout-markdown || \
+ echo -e "${YELLOW:-}PR report generation failed — continuing with bundle.${NC:-}"
+ fi
local bundle_args=(assemble --benchmark m3
--result-files "$latest_result"
diff --git a/benchmarks/oak_health_insurance/eval.sh b/benchmarks/oak_health_insurance/eval.sh
index 747c4b7d..d937dd59 100755
--- a/benchmarks/oak_health_insurance/eval.sh
+++ b/benchmarks/oak_health_insurance/eval.sh
@@ -242,8 +242,12 @@ if [ $EVAL_EXIT -eq 0 ]; then
# Generate eval report
REPORT_TMP=$(mktemp /tmp/oak_eval_report_XXXXXX)
- uv run --no-sync python -m benchmarks.helpers.compare_report eval \
- --result-file "$LATEST_RESULT" --output "$REPORT_TMP"
+ REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP")
+ uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}"
+ if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then
+ uv run --no-sync python -m benchmarks.helpers.compare_report \
+ eval --result-file "$LATEST_RESULT" --for-pr-comment --stdout-markdown
+ fi
BUNDLE_ARGS=(assemble --benchmark oak_health_insurance
--result-files "$LATEST_RESULT"
diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md
new file mode 100644
index 00000000..c214ff6e
--- /dev/null
+++ b/scripts/pr-regression-testing/README.md
@@ -0,0 +1,233 @@
+# PR Regression Testing
+
+This folder contains the scripts used to set up and run PR-triggered regression evaluations from GitHub Actions comments.
+
+## Files
+
+- `setup-pr-regression-workflow.sh`: bootstraps the self-hosted GitHub Actions runner VM.
+- `run-pr-regression-eval.sh`: consumes validated `/run-pr-eval` parameters and runs the requested benchmark using the `cuga-eval/scripts/eval.sh` script.
+
+## Setup The PR Regression Flow
+
+1. Add the workflow file to the `cuga-agent` repository:
+
+ ```text
+ .github/workflows/run-pr-eval.yml
+ ```
+
+2. In GitHub, create a self-hosted runner token for the `cuga-agent` repository.
+
+ Go to:
+
+ ```text
+ cuga-agent repository -> Settings -> Actions -> Runners -> New self-hosted runner
+ ```
+
+ Copy temporary runner token from the generated `config.sh` command.
+
+3. Ensure the workflow file uses the self-hosted runner labels:
+
+ ```yaml
+ runs-on: [self-hosted, linux, run-pr-eval]
+ ```
+
+4. On the runner VM, export the required setup variables:
+
+ ```bash
+ export GITHUB_REPO_URL="https://github.com/OWNER/cuga-agent"
+ export GITHUB_RUNNER_TOKEN="YOUR_TEMPORARY_RUNNER_TOKEN"
+ export RUNNER_LABELS="run-pr-eval"
+ ```
+
+5. Optionally choose repo mode and branches:
+
+ ```bash
+ export REPO_MODE="fork"
+ export CUGA_AGENT_BRANCH="main"
+ export CUGA_EVAL_BRANCH="main"
+ ```
+
+ Supported `REPO_MODE` values:
+
+ - `fork`: uses `https://github.com/AnkitaNaik/cuga-agent.git` and `https://github.com/AnkitaNaik/cuga-eval.git`.
+ - `upstream`: uses `https://github.com/cuga-project/cuga-agent.git` and `https://github.com/cuga-project/cuga-eval.git`.
+
+6. Run the setup script on the VM as the normal runner user, not with `sudo`:
+
+ ```bash
+ bash /path/to/cuga-eval/scripts/pr-regression-testing/setup-pr-regression-workflow.sh
+ ```
+
+ The script will:
+
+ - verify required commands: `git`, `curl`, `tar`, `python3`, `git-lfs`, and `uv`;
+ - clone or update `cuga-agent` and `cuga-eval` under `~/pr-regression-testing`;
+ - run `cuga-eval/setup_cuga.sh`;
+ - create and sync the `uv` environment;
+ - run AppWorld setup;
+ - download and register the GitHub Actions runner;
+ - start the runner in the background.
+
+7. Confirm the runner is online in GitHub:
+
+ ```text
+ cuga-agent repository -> Settings -> Actions -> Runners
+ ```
+
+ Ensure the runner is running with the labels `[self-hosted, linux, run-pr-eval]` i.e. same as workflow file.
+
+8. Check runner logs on the VM if needed:
+
+ ```bash
+ tail -f ~/pr-regression-testing/github-runner/runner.log
+ ```
+
+## GitHub Secrets And Variables
+
+Configure these in the `cuga-agent` repository:
+
+```text
+Settings -> Secrets and variables -> Actions
+```
+
+Required secrets:
+
+- `RITS_API_KEY`: required for `provider=rits`.
+- `LITE_LLM_KEY`: preferred secret for `provider=litellm`; the workflow injects it into `OPENAI_API_KEY`.
+- `OPENAI_API_KEY`: optional fallback secret for `provider=litellm` when `LITE_LLM_KEY` is not set.
+
+Optional repository variables:
+
+- `MODEL_NAME`: default model override used by the workflow environment.
+- `RITS_BASE_URL`: RITS endpoint override for `provider=rits`.
+- `LITELLM_BASE_URL`: LiteLLM endpoint override for `provider=litellm`; the workflow injects it into `OPENAI_BASE_URL`.
+
+Provider defaults used by `run-pr-regression-eval.sh`:
+
+- `provider=rits` selects `settings.rits.toml`, requires `RITS_API_KEY`, and passes `RITS_BASE_URL` to `models.py`.
+- `provider=litellm` selects `settings.litellm.toml` for the CUGA agent or `settings.openai.toml` for non-CUGA agents, requires `OPENAI_API_KEY`, and passes `OPENAI_BASE_URL` to `models.py`. Unless `model_name` is explicitly set in the PR comment, it uses `aws/gpt-oss-120b`.
+- For `provider=rits`, overriding `model_name` in the PR command requires `RITS_BASE_URL` to be configured so the model name and endpoint cannot silently diverge.
+
+## Run PR Evaluations
+
+Create a comment on a pull request in `cuga-agent`:
+
+```text
+/run-pr-eval sha=
+```
+
+The `sha` value must be the current 40-character PR head commit SHA. The workflow validates it before checkout and does not forward it to the evaluation process. By default, the forwarded evaluation runs:
+
+- `benchmark=appworld`
+- `agent=react`
+- `provider=rits`
+- `model_name=openai/gpt-oss-120b-a100`
+- `num_tasks=4`
+- default AppWorld task IDs: `9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1`
+
+AppWorld PR evaluations with `agent=cuga` are run with the AppWorld SDK evaluator (`--sdk`).
+
+For `provider=litellm`, the default model changes to `aws/gpt-oss-120b` unless `model_name` is explicitly provided.
+
+## Command Parameters
+
+The command accepts whitespace-separated `key=value` parameters:
+
+- `sha`: required by the GitHub workflow; must be the current 40-character PR head SHA.
+- `model_name`: model name to evaluate.
+- `task_id`: one task ID, or comma-separated task IDs.
+- `task_ids`: alias for `task_id`.
+- `eval_key`: AppWorld task group from `benchmarks/appworld/eval_config.toml`, such as `test_easy`, `test_med`, or `test_hard`.
+- `benchmark`: `appworld` or `m3`; defaults to `appworld`.
+- `num_tasks`: positive integer; defaults to `4`.
+- `agent`: `react`, `cuga`, or `codeact`; defaults to `react`.
+- `provider`: `rits` or `litellm`; defaults to `rits`.
+
+Unsupported parameters cause the workflow or runner script to fail early with a list of supported parameters.
+
+The workflow parser tolerates spaces around `=`, markdown code ticks, mixed-case values for `agent`, `benchmark`, and `provider`, and spaces after commas in task lists. For example, `provider = litellm` and `task_id = 9aae7da_1, 365e0a3_1` are accepted. The runner script also accepts sanitized CLI-style arguments such as `--benchmark appworld --agent react --provider rits`.
+
+Useful AppWorld `eval_key` values:
+
+- `test_easy`
+- `test_med`
+- `test_hard`
+- `test_challenge_easy`
+- `test_challenge_med`
+- `test_challenge_hard`
+- `test_normal_easy`
+- `test_normal_med`
+- `test_normal_hard`
+
+## Examples
+
+Run the default AppWorld React-agent evaluation on RITS:
+
+```text
+/run-pr-eval sha=
+```
+
+Run one AppWorld task with the React agent on RITS:
+
+```text
+/run-pr-eval sha= provider=rits agent=react benchmark=appworld task_id=9aae7da_1 num_tasks=1
+```
+
+Run one AppWorld task with the CUGA agent on LiteLLM:
+
+```text
+/run-pr-eval sha= provider=litellm agent=cuga benchmark=appworld task_id=9aae7da_1 num_tasks=1
+```
+
+Run AppWorld with two explicit tasks:
+
+```text
+/run-pr-eval sha= benchmark=appworld task_id=9aae7da_1,365e0a3_1 num_tasks=2
+```
+
+Run AppWorld easy tasks from `eval_config.toml`:
+
+```text
+/run-pr-eval sha= benchmark=appworld eval_key=test_easy
+```
+
+Run AppWorld medium tasks from `eval_config.toml`:
+
+```text
+/run-pr-eval sha= benchmark=appworld eval_key=test_med
+```
+
+Run AppWorld hard tasks from `eval_config.toml`:
+
+```text
+/run-pr-eval sha= benchmark=appworld eval_key=test_hard
+```
+
+Run M3 with the default React agent:
+
+```text
+/run-pr-eval sha= benchmark=m3 num_tasks=1
+```
+
+Run M3 with the CUGA agent and LiteLLM:
+
+```text
+/run-pr-eval sha= benchmark=m3 agent=cuga provider=litellm num_tasks=1
+```
+
+Override the model:
+
+```text
+/run-pr-eval sha= provider=litellm agent=cuga benchmark=appworld model_name=aws/glm-5 task_id=9aae7da_1 num_tasks=1
+```
+
+## Output
+
+The workflow posts a PR comment containing:
+
+- trusted run metadata;
+- requested command and SHA;
+- the evaluation report markdown emitted by the PR regression runner;
+- a link to the uploaded workflow artifact.
+
+The full output is also uploaded as a GitHub Actions artifact. During the workflow run, logs are written under the checked-out workspace in `pr-eval-logs/`.
diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh
new file mode 100755
index 00000000..0d1e9cbd
--- /dev/null
+++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh
@@ -0,0 +1,517 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+START_EPOCH="$(date +%s)"
+START_TIME="$(date -u +"%Y-%m-%d %H:%M:%S UTC")"
+
+log() {
+ printf '[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*"
+}
+
+extract_report_blocks() {
+ local input_path="$1"
+ local output_path="$2"
+
+ if [[ ! -f "${input_path}" ]]; then
+ echo "No evaluation output artifact was available." > "${output_path}"
+ return 0
+ fi
+
+ awk '
+ $0 == "######## REPORT START ########" {
+ in_report = 1
+ block = ""
+ next
+ }
+ $0 == "######## REPORT END ########" {
+ if (in_report) {
+ sub(/^[[:space:]]+/, "", block)
+ sub(/[[:space:]]+$/, "", block)
+ if (length(block) > 0) {
+ if (printed) {
+ print ""
+ print ""
+ }
+ print block
+ printed = 1
+ }
+ }
+ in_report = 0
+ block = ""
+ next
+ }
+ in_report {
+ block = block $0 "\n"
+ }
+ END {
+ if (!printed) {
+ print "No report section found between REPORT START and REPORT END markers."
+ }
+ }
+ ' "${input_path}" > "${output_path}"
+}
+
+if [[ "${1:-}" == "--extract-report" ]]; then
+ if [[ $# -ne 3 ]]; then
+ echo "Usage: $0 --extract-report " >&2
+ exit 2
+ fi
+ extract_report_blocks "$2" "$3"
+ exit 0
+fi
+
+open_details() {
+ local title="$1"
+ echo ""
+ echo "${title}
"
+ echo
+}
+
+close_details() {
+ echo
+ echo " "
+}
+
+on_exit() {
+ status=$?
+ end_epoch="$(date +%s)"
+ end_time="$(date -u +"%Y-%m-%d %H:%M:%S UTC")"
+ duration=$((end_epoch - START_EPOCH))
+ minutes=$((duration / 60))
+ seconds=$((duration % 60))
+
+ echo "######## REPORT START ########"
+ echo
+ open_details "Run summary"
+ echo "- Started: ${START_TIME}"
+ echo "- Finished: ${end_time}"
+ echo "- Duration: ${minutes}m ${seconds}s"
+ echo "- Exit code: ${status}"
+
+ if [[ ${status} -eq 0 ]]; then
+ echo "- Result: SUCCESS"
+ else
+ echo "- Result: FAILED"
+ fi
+ close_details
+ echo "######## REPORT END ########"
+}
+trap on_exit EXIT
+
+DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100"
+DEFAULT_LITELLM_MODEL_NAME="aws/gpt-oss-120b"
+DEFAULT_RITS_BASE_URL="https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100/v1"
+DEFAULT_LITELLM_BASE_URL="https://ete-litellm.ai-models.vpc-int.res.ibm.com/"
+DEFAULT_APPWORLD_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1"
+DEFAULT_BENCHMARK="appworld"
+DEFAULT_NUM_TASKS="4"
+DEFAULT_AGENT="react"
+DEFAULT_PROVIDER="rits"
+
+MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}"
+TASK_IDS="${TASK_IDS:-}"
+EVAL_KEY="${EVAL_KEY:-}"
+BENCHMARK="${BENCHMARK:-$DEFAULT_BENCHMARK}"
+NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}"
+AGENT="${AGENT:-$DEFAULT_AGENT}"
+PROVIDER="${PROVIDER:-$DEFAULT_PROVIDER}"
+MODEL_NAME_FROM_COMMENT=false
+
+usage_error() {
+ local message="$1"
+ echo "ERROR: ${message}"
+ echo "Supported parameters: model_name, task_id, task_ids, eval_key, benchmark, num_tasks, agent, provider, sha"
+ close_details
+ echo "######## REPORT END ########"
+ exit 2
+}
+
+# Parse whitespace-separated key=value parameters from the PR comment.
+# Common comment formatting is normalized first, so these are equivalent:
+# provider=litellm
+# provider = litellm
+# provider= litellm
+# provider =litellm
+# Supported aliases:
+# model_name=...
+# task_id=id1,id2
+# task_ids=id1,id2
+# eval_key=test_easy|test_med|test_hard|...
+# benchmark=appworld|m3
+# num_tasks=...
+# agent=react|cuga|codeact
+# provider=rits|litellm
+COMMAND_TOKENS=()
+if [[ $# -gt 0 ]]; then
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ /run-pr-eval)
+ shift
+ ;;
+ --model-name|--model_name)
+ [[ -n "${2:-}" && "${2:-}" != --* ]] || {
+ echo "######## REPORT START ########"
+ open_details "Requested parameters"
+ usage_error "$1 requires a value."
+ }
+ COMMAND_TOKENS+=("model_name=$2")
+ shift 2
+ ;;
+ --task-id|--task-ids|--task|--tasks)
+ [[ -n "${2:-}" && "${2:-}" != --* ]] || {
+ echo "######## REPORT START ########"
+ open_details "Requested parameters"
+ usage_error "$1 requires a value."
+ }
+ COMMAND_TOKENS+=("task_ids=$2")
+ shift 2
+ ;;
+ --eval-key|--eval_key)
+ [[ -n "${2:-}" && "${2:-}" != --* ]] || {
+ echo "######## REPORT START ########"
+ open_details "Requested parameters"
+ usage_error "$1 requires a value."
+ }
+ COMMAND_TOKENS+=("eval_key=$2")
+ shift 2
+ ;;
+ --benchmark|--num-tasks|--num_tasks|--agent|--agent-type|--agent_type|--provider|--sha)
+ key="${1#--}"
+ key="${key//-/_}"
+ [[ -n "${2:-}" && "${2:-}" != --* ]] || {
+ echo "######## REPORT START ########"
+ open_details "Requested parameters"
+ usage_error "$1 requires a value."
+ }
+ COMMAND_TOKENS+=("${key}=$2")
+ shift 2
+ ;;
+ *=*)
+ COMMAND_TOKENS+=("$1")
+ shift
+ ;;
+ *)
+ if [[ $# -eq 1 && "$1" == *"/run-pr-eval"* ]]; then
+ COMMENT_BODY="$1"
+ shift
+ else
+ echo "######## REPORT START ########"
+ open_details "Requested parameters"
+ usage_error "Unsupported argument: $1"
+ fi
+ ;;
+ esac
+ done
+fi
+
+if [[ ${#COMMAND_TOKENS[@]} -gt 0 ]]; then
+ COMMENT_BODY="/run-pr-eval ${COMMAND_TOKENS[*]}"
+else
+ COMMENT_BODY="${COMMENT_BODY:-}"
+fi
+
+# Remove Windows carriage returns, newlines, tabs, and markdown code ticks.
+COMMENT_BODY="${COMMENT_BODY//$'\r'/ }"
+COMMENT_BODY="${COMMENT_BODY//$'\n'/ }"
+COMMENT_BODY="${COMMENT_BODY//$'\t'/ }"
+COMMENT_BODY="${COMMENT_BODY//\`/ }"
+if [[ "${COMMENT_BODY}" == *"/run-pr-eval"* ]]; then
+ COMMENT_BODY="/run-pr-eval ${COMMENT_BODY#*/run-pr-eval}"
+fi
+
+# Normalize spaces around "=" for supported keys, and tolerate spaces after
+# commas in comma-separated values such as task_id=a, b.
+COMMENT_BODY="$(
+ printf '%s' "${COMMENT_BODY}" |
+ sed -E 's/(^|[[:space:]])(model_name|task_id|task_ids|eval_key|eval-key|benchmark|num_tasks|agent|agent_type|provider|sha)[[:space:]]*=[[:space:]]*/\1\2=/g; s/,[[:space:]]+/,/g'
+)"
+
+read -r -a TOKENS <<< "${COMMENT_BODY}"
+echo "######## REPORT START ########"
+open_details "Requested parameters"
+for token in "${TOKENS[@]}"; do
+ case "${token}" in
+ /run-pr-eval)
+ ;;
+
+ model_name=*)
+ MODEL_NAME="${token#model_name=}"
+ MODEL_NAME_FROM_COMMENT=true
+ ;;
+
+ task_id=*|task_ids=*)
+ TASK_IDS="${token#*=}"
+ TASK_IDS="${TASK_IDS//,/ }"
+ ;;
+
+ eval_key=*|eval-key=*)
+ EVAL_KEY="${token#*=}"
+ ;;
+
+ benchmark=*)
+ BENCHMARK="${token#benchmark=}"
+ ;;
+
+ num_tasks=*)
+ NUM_TASKS="${token#num_tasks=}"
+ ;;
+
+ agent=*|agent_type=*)
+ AGENT="${token#*=}"
+ ;;
+
+ provider=*)
+ PROVIDER="${token#provider=}"
+ ;;
+
+ sha=*)
+ ;;
+
+ "")
+ ;;
+
+ *)
+ usage_error "Unsupported parameter: ${token}"
+ ;;
+ esac
+done
+
+if [[ -z "${MODEL_NAME}" ]]; then
+ usage_error "model_name cannot be empty."
+fi
+
+if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]]; then
+ usage_error "num_tasks must be a positive integer."
+fi
+NUM_TASKS=$((10#${NUM_TASKS}))
+if (( NUM_TASKS < 1 )); then
+ usage_error "num_tasks must be a positive integer."
+fi
+
+AGENT="$(printf '%s' "${AGENT}" | tr '[:upper:]' '[:lower:]')"
+BENCHMARK="$(printf '%s' "${BENCHMARK}" | tr '[:upper:]' '[:lower:]')"
+PROVIDER="$(printf '%s' "${PROVIDER}" | tr '[:upper:]' '[:lower:]')"
+EVAL_KEY="$(printf '%s' "${EVAL_KEY}" | tr '[:upper:]' '[:lower:]')"
+
+case "${AGENT}" in
+ react|cuga|codeact)
+ ;;
+ *)
+ echo "ERROR: agent must be one of: react, cuga, codeact."
+ close_details
+ echo "######## REPORT END ########"
+ exit 2
+ ;;
+esac
+
+case "${BENCHMARK}" in
+ appworld|m3)
+ ;;
+ *)
+ echo "ERROR: benchmark must be one of: appworld, m3."
+ close_details
+ echo "######## REPORT END ########"
+ exit 2
+ ;;
+esac
+
+if [[ -n "${EVAL_KEY}" && "${BENCHMARK}" != "appworld" ]]; then
+ echo "ERROR: eval_key is only supported for benchmark=appworld."
+ close_details
+ echo "######## REPORT END ########"
+ exit 2
+fi
+
+case "${PROVIDER}" in
+ rits)
+ AGENT_SETTING_CONFIG="settings.rits.toml"
+ if [[ -z "${RITS_API_KEY:-}" ]]; then
+ echo "ERROR: RITS_API_KEY is required when provider=rits."
+ close_details
+ echo "######## REPORT END ########"
+ exit 2
+ fi
+ if [[ "${MODEL_NAME_FROM_COMMENT}" == "true" && -z "${RITS_BASE_URL:-}" ]]; then
+ echo "ERROR: provider=rits model_name overrides require RITS_BASE_URL to avoid mismatched model endpoints."
+ close_details
+ echo "######## REPORT END ########"
+ exit 2
+ fi
+ RITS_BASE_URL="${RITS_BASE_URL:-${DEFAULT_RITS_BASE_URL}}"
+ export RITS_API_KEY RITS_BASE_URL
+ ;;
+ litellm)
+ if [[ "${AGENT}" == "cuga" ]]; then
+ AGENT_SETTING_CONFIG="settings.litellm.toml"
+ else
+ AGENT_SETTING_CONFIG="settings.openai.toml"
+ fi
+ if [[ "${MODEL_NAME_FROM_COMMENT}" == "false" ]]; then
+ MODEL_NAME="${DEFAULT_LITELLM_MODEL_NAME}"
+ fi
+ OPENAI_BASE_URL="${OPENAI_BASE_URL:-${DEFAULT_LITELLM_BASE_URL}}"
+ if [[ -z "${OPENAI_API_KEY:-}" ]]; then
+ echo "ERROR: OPENAI_API_KEY is required when provider=litellm."
+ close_details
+ echo "######## REPORT END ########"
+ exit 2
+ fi
+ if [[ -z "${OPENAI_BASE_URL:-}" ]]; then
+ echo "ERROR: OPENAI_BASE_URL is required when provider=litellm."
+ close_details
+ echo "######## REPORT END ########"
+ exit 2
+ fi
+ export OPENAI_API_KEY OPENAI_BASE_URL
+ ;;
+ *)
+ echo "ERROR: provider must be one of: rits, litellm."
+ close_details
+ echo "######## REPORT END ########"
+ exit 2
+ ;;
+esac
+
+if [[ -z "${TASK_IDS}" && -z "${EVAL_KEY}" && "${BENCHMARK}" == "appworld" ]]; then
+ TASK_IDS="${DEFAULT_APPWORLD_TASK_IDS}"
+fi
+
+read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}"
+
+if [[ ${#TASK_ID_ARRAY[@]} -eq 0 && -z "${EVAL_KEY}" && "${BENCHMARK}" == "appworld" ]]; then
+ echo "ERROR: At least one task_id or eval_key is required for benchmark=appworld."
+ close_details
+ echo "######## REPORT END ########"
+ exit 2
+fi
+echo "- Model: ${MODEL_NAME}"
+echo "- Agent: ${AGENT}"
+echo "- Benchmark: ${BENCHMARK}"
+echo "- Provider: ${PROVIDER}"
+if [[ -n "${EVAL_KEY}" ]]; then
+ echo "- Eval key: ${EVAL_KEY}"
+fi
+echo "- Num tasks: ${NUM_TASKS}"
+if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then
+ echo "- Task IDs: ${TASK_IDS}"
+else
+ echo "- Task IDs: benchmark default"
+fi
+close_details
+echo "######## REPORT END ########"
+
+# For AppWorld, num_tasks limits the explicit task list. M3 maps num_tasks to
+# --max-samples-per-domain below.
+if [[ "${BENCHMARK}" == "appworld" && ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then
+ TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}")
+fi
+
+export AGENT_SETTING_CONFIG
+export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}"
+export MODEL_NAME
+export PR_EVAL_FOR_PR_COMMENT="${PR_EVAL_FOR_PR_COMMENT:-true}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+if [[ -z "${EVAL_REPO:-}" ]]; then
+ if [[ -f "${SCRIPT_DIR}/../../scripts/eval.sh" ]]; then
+ EVAL_REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+ else
+ EVAL_REPO="$(cd "${SCRIPT_DIR}/.." && pwd)"
+ fi
+fi
+
+if [[ ! -d "${EVAL_REPO}" ]]; then
+ echo "ERROR: Evaluation repo not found: ${EVAL_REPO}"
+ exit 1
+fi
+
+if [[ ! -f "${EVAL_REPO}/scripts/eval.sh" ]]; then
+ echo "ERROR: Evaluation script not found: ${EVAL_REPO}/scripts/eval.sh"
+ exit 1
+fi
+
+cd "${EVAL_REPO}"
+
+echo "######## REPORT START ########"
+echo "## Evaluation Metrics"
+echo
+open_details "Run metadata"
+echo "- Repository: ${GITHUB_REPOSITORY:-unknown}"
+echo "- PR: ${PR_NUMBER:-unknown}"
+echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}"
+echo "- Runner: ${RUNNER_NAME:-$(hostname)}"
+echo "- Agent: ${AGENT}"
+echo "- Benchmark: ${BENCHMARK}"
+echo "- Provider: ${PROVIDER}"
+close_details
+echo
+open_details "Evaluation configuration"
+echo "- Agent: ${AGENT}"
+echo "- Benchmark: ${BENCHMARK}"
+echo "- Provider: ${PROVIDER}"
+echo "- Agent setting config: ${AGENT_SETTING_CONFIG}"
+echo "- Model: ${MODEL_NAME}"
+echo "- Requested num_tasks: ${NUM_TASKS}"
+if [[ "${BENCHMARK}" == "appworld" ]]; then
+ if [[ -n "${EVAL_KEY}" ]]; then
+ echo "- Eval key: ${EVAL_KEY}"
+ fi
+ if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then
+ echo "- Effective task count: ${#TASK_ID_ARRAY[@]}"
+ echo "- Task IDs:"
+ for task_id in "${TASK_ID_ARRAY[@]}"; do
+ echo " - ${task_id}"
+ done
+ else
+ echo "- Task IDs: selected by eval_key"
+ fi
+else
+ echo "- M3 max samples per domain: ${NUM_TASKS}"
+ if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then
+ echo "- M3 task filters:"
+ for task_id in "${TASK_ID_ARRAY[@]}"; do
+ echo " - ${task_id}"
+ done
+ else
+ echo "- M3 task filters: benchmark default"
+ fi
+fi
+close_details
+echo "######## REPORT END ########"
+
+log "Verifying required commands"
+command -v uv >/dev/null 2>&1 || {
+ echo "ERROR: uv is not installed or is not in PATH."
+ exit 127
+}
+
+command -v curl >/dev/null 2>&1 || {
+ echo "ERROR: curl is not installed or is not in PATH."
+ exit 127
+}
+
+log "Starting ${BENCHMARK} ${AGENT} evaluation"
+
+EVAL_ARGS=(
+ --benchmark "${BENCHMARK}"
+ --agent "${AGENT}"
+)
+
+if [[ "${BENCHMARK}" == "appworld" ]]; then
+ if [[ "${AGENT}" == "cuga" ]]; then
+ EVAL_ARGS+=(--sdk)
+ fi
+ if [[ -n "${EVAL_KEY}" ]]; then
+ EVAL_ARGS+=(--eval-key "${EVAL_KEY}")
+ fi
+ if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then
+ EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}")
+ fi
+else
+ EVAL_ARGS+=(--m3-data --max-samples-per-domain "${NUM_TASKS}")
+ if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then
+ EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}")
+ fi
+fi
+
+bash scripts/eval.sh "${EVAL_ARGS[@]}"
diff --git a/scripts/pr-regression-testing/setup-pr-regression-workflow.sh b/scripts/pr-regression-testing/setup-pr-regression-workflow.sh
new file mode 100644
index 00000000..c9bd9694
--- /dev/null
+++ b/scripts/pr-regression-testing/setup-pr-regression-workflow.sh
@@ -0,0 +1,311 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# =============================================================================
+# PR Regression Testing VM Bootstrap
+#
+# Modes:
+# REPO_MODE=fork -> use temporary AnkitaNaik forks (default for now)
+# REPO_MODE=upstream -> use cuga-project upstream repositories
+#
+# Final layout:
+#
+# ~/pr-regression-testing/
+# ├── cuga-agent/
+# ├── cuga-eval/
+# │ └── scripts/
+# │ ├── pr-regression-testing/
+# │ │ ├── setup-pr-regression-workflow.sh
+# │ │ └── run-pr-regression-eval.sh
+# │ └── run-pr-eval.sh -> pr-regression-testing/run-pr-regression-eval.sh
+# └── github-runner/
+# └── _work/
+#
+# Required VM prerequisites:
+# git curl tar python3 git-lfs
+#
+# Required environment variables:
+# GITHUB_REPO_URL
+# GITHUB_RUNNER_TOKEN
+#
+# Optional:
+# REPO_MODE=fork|upstream
+# ROOT_DIR
+# RUNNER_NAME
+# RUNNER_LABELS
+# CUGA_AGENT_BRANCH
+# CUGA_EVAL_BRANCH
+# RUN_EVAL_SCRIPT
+# RUNNER_VERSION
+# =============================================================================
+
+REPO_MODE="${REPO_MODE:-fork}"
+ROOT_DIR="${ROOT_DIR:-$HOME/pr-regression-testing}"
+
+RUNNER_NAME="${RUNNER_NAME:-$(hostname)}"
+RUNNER_LABELS="${RUNNER_LABELS:-run-pr-eval}"
+
+CUGA_AGENT_BRANCH="${CUGA_AGENT_BRANCH:-main}"
+CUGA_EVAL_BRANCH="${CUGA_EVAL_BRANCH:-main}"
+
+case "${REPO_MODE}" in
+ fork)
+ DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/AnkitaNaik/cuga-agent.git"
+ DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/AnkitaNaik/cuga-eval.git"
+ ;;
+ upstream)
+ DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/cuga-project/cuga-agent.git"
+ DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/cuga-project/cuga-eval.git"
+ ;;
+ *)
+ echo "ERROR: REPO_MODE must be 'fork' or 'upstream'." >&2
+ exit 2
+ ;;
+esac
+
+CUGA_AGENT_REPO_URL="${CUGA_AGENT_REPO_URL:-$DEFAULT_CUGA_AGENT_REPO_URL}"
+CUGA_EVAL_REPO_URL="${CUGA_EVAL_REPO_URL:-$DEFAULT_CUGA_EVAL_REPO_URL}"
+
+CUGA_AGENT_DIR="${ROOT_DIR}/cuga-agent"
+CUGA_EVAL_DIR="${ROOT_DIR}/cuga-eval"
+GH_RUNNER_DIR="${ROOT_DIR}/github-runner"
+
+RUN_EVAL_SCRIPT="${RUN_EVAL_SCRIPT:-${CUGA_EVAL_DIR}/scripts/pr-regression-testing/run-pr-regression-eval.sh}"
+WORKFLOW_EVAL_SCRIPT="${CUGA_EVAL_DIR}/scripts/run-pr-eval.sh"
+
+log() {
+ printf '\n[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*"
+}
+
+die() {
+ echo "ERROR: $*" >&2
+ exit 1
+}
+
+need_cmd() {
+ command -v "$1" >/dev/null 2>&1 || die "Required command '$1' is missing."
+}
+
+clone_or_update() {
+ local repo_url="$1"
+ local branch="$2"
+ local dest="$3"
+ local name="$4"
+
+ if [[ -d "${dest}/.git" ]]; then
+ log "Updating ${name}"
+ git -C "${dest}" fetch --all --prune
+ git -C "${dest}" checkout "${branch}"
+ git -C "${dest}" pull --ff-only origin "${branch}"
+ else
+ log "Cloning ${name}"
+ git clone --branch "${branch}" "${repo_url}" "${dest}"
+ fi
+}
+
+: "${GITHUB_REPO_URL:?Set GITHUB_REPO_URL, e.g. https://github.com/OWNER/REPO}"
+: "${GITHUB_RUNNER_TOKEN:?Set GITHUB_RUNNER_TOKEN to the temporary runner token}"
+
+log "Repository mode: ${REPO_MODE}"
+echo "cuga-agent: ${CUGA_AGENT_REPO_URL}"
+echo "cuga-eval: ${CUGA_EVAL_REPO_URL}"
+
+log "Checking VM prerequisites"
+need_cmd git
+need_cmd curl
+need_cmd tar
+need_cmd python3
+need_cmd git-lfs
+
+python3 - <<'PY'
+import sys
+if not ((3, 12) <= sys.version_info[:2] < (3, 14)):
+ raise SystemExit(
+ f"ERROR: Python 3.12 or 3.13 is required; found "
+ f"{sys.version_info.major}.{sys.version_info.minor}"
+ )
+PY
+
+if ! command -v uv >/dev/null 2>&1; then
+ log "Installing uv"
+ curl -LsSf https://astral.sh/uv/install.sh | sh
+ export PATH="${HOME}/.local/bin:${HOME}/.cargo/bin:${PATH}"
+fi
+need_cmd uv
+
+mkdir -p "${ROOT_DIR}"
+
+clone_or_update \
+ "${CUGA_AGENT_REPO_URL}" \
+ "${CUGA_AGENT_BRANCH}" \
+ "${CUGA_AGENT_DIR}" \
+ "cuga-agent"
+
+clone_or_update \
+ "${CUGA_EVAL_REPO_URL}" \
+ "${CUGA_EVAL_BRANCH}" \
+ "${CUGA_EVAL_DIR}" \
+ "cuga-eval"
+
+log "Running cuga-eval/setup_cuga.sh"
+cd "${CUGA_EVAL_DIR}"
+chmod u+x ./setup_cuga.sh
+bash ./setup_cuga.sh
+
+log "Creating/syncing cuga-eval environment"
+uv venv
+uv sync
+
+log "Setting up AppWorld"
+git lfs install
+chmod u+x ./setup_appworld.sh
+bash ./setup_appworld.sh
+uv sync --group appworld
+
+log "Checking PR regression evaluation entrypoint"
+if [[ ! -f "${RUN_EVAL_SCRIPT}" ]]; then
+ cat >&2 </dev/null 2>&1; then
+ log "Runner already running"
+else
+ log "Starting runner in background"
+
+ nohup ./run.sh > "${GH_RUNNER_DIR}/runner.log" 2>&1 &
+ echo $! > "${GH_RUNNER_DIR}/runner.pid"
+fi
+
+sleep 3
+
+if ! pgrep -f "${GH_RUNNER_DIR}/bin/Runner.Listener" >/dev/null 2>&1; then
+ echo "ERROR: Runner did not remain active." >&2
+ tail -n 50 "${GH_RUNNER_DIR}/runner.log" 2>/dev/null || true
+ exit 1
+fi
+
+echo
+echo "======================================================================"
+echo "PR regression testing VM is ready"
+echo "======================================================================"
+echo
+echo "Repository mode:"
+echo " ${REPO_MODE}"
+echo
+echo "Layout:"
+echo " ${ROOT_DIR}/"
+echo " ├── cuga-agent/"
+echo " ├── cuga-eval/"
+echo " │ └── scripts/pr-regression-testing/run-pr-regression-eval.sh"
+echo " └── github-runner/"
+echo " └── _work/"
+echo
+echo "Repos:"
+echo " cuga-agent: ${CUGA_AGENT_REPO_URL}"
+echo " cuga-eval: ${CUGA_EVAL_REPO_URL}"
+echo
+echo "Runner logs:"
+echo " tail -f ${GH_RUNNER_DIR}/runner.log"
+echo
+echo "Workflow target:"
+echo " runs-on: [self-hosted, linux, ${RUNNER_LABELS}]"
+echo
+echo "Workflow script:"
+echo " bash \"${RUN_EVAL_SCRIPT}\" \"\${COMMENT_BODY}\""
+echo
+echo "PR command format:"
+echo " /run-pr-eval sha= benchmark=appworld provider=rits agent=react"