diff --git a/benchmarks/appworld/appworld_eval.py b/benchmarks/appworld/appworld_eval.py index 8c45a10f..8f892663 100644 --- a/benchmarks/appworld/appworld_eval.py +++ b/benchmarks/appworld/appworld_eval.py @@ -177,6 +177,14 @@ def _get_registry_base_url() -> str: return "http://localhost:8001" +def _is_langfuse_tracing_enabled() -> bool: + env_value = os.getenv("DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING") + if env_value is not None: + return env_value.strip().lower() in {"1", "true", "yes", "on"} + + return bool(getattr(getattr(settings, "advanced_features", None), "langfuse_tracing", False)) + + async def run_agent_on_task( task_id: str, experiment_name: str = "api_cuga_agent", @@ -217,7 +225,7 @@ async def run_agent_on_task( # Without this, LangfuseCallbackHandler creates a new root trace for every # agent_loop_obj.run() call in the controller while-loop. _langfuse = None - if _LANGFUSE_AVAILABLE and _get_langfuse_client is not None: + if _is_langfuse_tracing_enabled() and _LANGFUSE_AVAILABLE and _get_langfuse_client is not None: try: _langfuse = _get_langfuse_client() _run_uid = uuid.uuid4().hex[:8] @@ -335,8 +343,9 @@ async def run_agent_on_task( langfuse_data = None # Extract Langfuse data if trace_id is available - langfuse_handler = LangfuseTraceHandler(langfuse_trace_id) - langfuse_data = await langfuse_handler.get_langfuse_data() + if langfuse_trace_id: + langfuse_handler = LangfuseTraceHandler(langfuse_trace_id) + langfuse_data = await langfuse_handler.get_langfuse_data() if langfuse_trace_id: task_result.trace_id = langfuse_trace_id if langfuse_data: @@ -469,14 +478,21 @@ def _print_appworld_summary(report: dict): success_rate = report.get("success_rate", 0) avg_steps = report.get("avg_steps", 0) avg_duration = report.get("avg_duration", 0) + for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes") - print("\n" + "=" * 80) - print("EVALUATION COMPLETE") - print("=" * 80) + if for_pr_comment: + print("######## REPORT START ########") + print("## EVALUATION COMPLETE") + else: + print("\n" + "=" * 80) + print("EVALUATION COMPLETE") + print("=" * 80) print(f"Total Tasks: {total}") print(f"Completed: {completed}/{total} ({success_rate:.1%})") print(f"Avg Steps: {avg_steps:.2f}") print(f"Avg Duration: {avg_duration:.2f}s") + if for_pr_comment: + print("######## REPORT END ########") async def main(): diff --git a/benchmarks/appworld/appworld_eval_codeact.py b/benchmarks/appworld/appworld_eval_codeact.py index f92fa516..96f5a9e2 100644 --- a/benchmarks/appworld/appworld_eval_codeact.py +++ b/benchmarks/appworld/appworld_eval_codeact.py @@ -96,14 +96,21 @@ def _print_appworld_summary(report: dict): success_rate = report.get("success_rate", 0) avg_steps = report.get("avg_steps", 0) avg_duration = report.get("avg_duration", 0) + for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes") - print("\n" + "=" * 80) - print("EVALUATION COMPLETE") - print("=" * 80) + if for_pr_comment: + print("######## REPORT START ########") + print("## EVALUATION COMPLETE") + else: + print("\n" + "=" * 80) + print("EVALUATION COMPLETE") + print("=" * 80) print(f"Total Tasks: {total}") print(f"Completed: {completed}/{total} ({success_rate:.1%})") print(f"Avg Steps: {avg_steps:.2f}") print(f"Avg Duration: {avg_duration:.2f}s") + if for_pr_comment: + print("######## REPORT END ########") def _parse_prompt_to_messages(input_str: str) -> list[dict[str, Any]]: diff --git a/benchmarks/appworld/appworld_eval_react.py b/benchmarks/appworld/appworld_eval_react.py index fbbd5a5b..0346fdb3 100644 --- a/benchmarks/appworld/appworld_eval_react.py +++ b/benchmarks/appworld/appworld_eval_react.py @@ -180,14 +180,21 @@ def _print_appworld_summary(report: dict): success_rate = report.get("success_rate", 0) avg_steps = report.get("avg_steps", 0) avg_duration = report.get("avg_duration", 0) + for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes") - print("\n" + "=" * 80) - print("EVALUATION COMPLETE") - print("=" * 80) + if for_pr_comment: + print("######## REPORT START ########") + print("## EVALUATION COMPLETE") + else: + print("\n" + "=" * 80) + print("EVALUATION COMPLETE") + print("=" * 80) print(f"Total Tasks: {total}") print(f"Completed: {completed}/{total} ({success_rate:.1%})") print(f"Avg Steps: {avg_steps:.2f}") print(f"Avg Duration: {avg_duration:.2f}s") + if for_pr_comment: + print("######## REPORT END ########") def _render_initial_prompt(world: AppWorld) -> str: diff --git a/benchmarks/appworld/eval.sh b/benchmarks/appworld/eval.sh index a7f7ec0f..0629545c 100755 --- a/benchmarks/appworld/eval.sh +++ b/benchmarks/appworld/eval.sh @@ -410,8 +410,12 @@ if [ -n "$LATEST_RESULT" ] && [ "${NO_BUNDLE:-false}" != "true" ]; then # Generate eval report REPORT_TMP=$(mktemp /tmp/appworld_eval_report_XXXXXX) - uv run --no-sync python -m benchmarks.helpers.compare_report eval \ - --result-file "$LATEST_RESULT" --output "$REPORT_TMP" + REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" + if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then + uv run --no-sync python -m benchmarks.helpers.compare_report \ + eval --result-file "$LATEST_RESULT" --for-pr-comment --stdout-markdown + fi BUNDLE_ARGS=(assemble --benchmark appworld --result-files "$LATEST_RESULT" diff --git a/benchmarks/bpo/eval.sh b/benchmarks/bpo/eval.sh index f4694dd9..569c9d60 100755 --- a/benchmarks/bpo/eval.sh +++ b/benchmarks/bpo/eval.sh @@ -409,8 +409,12 @@ if [ $EVAL_EXIT_CODE -eq 0 ]; then # Generate eval report REPORT_TMP=$(mktemp /tmp/bpo_eval_report_XXXXXX) - uv run --no-sync python -m benchmarks.helpers.compare_report eval \ - --result-file "$LATEST_RESULT" --output "$REPORT_TMP" + REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" + if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then + uv run --no-sync python -m benchmarks.helpers.compare_report \ + eval --result-file "$LATEST_RESULT" --for-pr-comment --stdout-markdown + fi BUNDLE_ARGS=(assemble --benchmark bpo --result-files "$LATEST_RESULT" diff --git a/benchmarks/helpers/compare_report.py b/benchmarks/helpers/compare_report.py index 58382cfd..1c1117e3 100644 --- a/benchmarks/helpers/compare_report.py +++ b/benchmarks/helpers/compare_report.py @@ -1516,7 +1516,7 @@ def _bucket_m3_tasks(tasks: dict) -> tuple: return rows, True -def generate_eval_report(result_file: str, markdown: bool = True) -> str: +def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment: bool = False) -> str: """Generate a single-evaluation-run report. When the result file is M3-shaped (each task has ``m3_task_id`` + ``domain``), @@ -1524,7 +1524,8 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: rows are readable instead of an unattributed UUID. When ``markdown=False``, the same content is rendered as a plain-text monospace table for terminals (compare.sh's stdout path); ``markdown=True`` (default) is what gets saved - into the bundle's report.md. + into the bundle's report.md. ``for_pr_comment`` is an explicit opt-in for + the PR-regression workflow's collapsible, sentinel-wrapped markdown. """ parsed = parse_result_file(result_file) rows, grouped = _bucket_m3_tasks(parsed["tasks"]) @@ -1536,9 +1537,17 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: fence_open = (lambda: "```text") if markdown else (lambda: "") fence_close = (lambda: "```") if markdown else (lambda: "") - lines = [h1("Evaluation Report"), ""] - lines.append(h2("Summary")) - lines.append("") + if markdown and for_pr_comment: + lines = ["######## REPORT START ########"] + lines.append("
") + lines.append("Evaluation Report") + lines.append("") + lines.append(h2("Summary")) + lines.append("") + else: + lines = [h1("Evaluation Report"), ""] + lines.append(h2("Summary")) + lines.append("") if markdown: lines.append(f"- **Pass@1**: {parsed['passed']}/{parsed['total']} ({parsed['rate']:.1%})") lines.append(f"- **Total Tokens**: {_fmt(parsed['tokens'])}") @@ -1825,6 +1834,10 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str: markdown=markdown, ) ) + if markdown and for_pr_comment: + lines.append("") + lines.append("
") + lines.append("######## REPORT END ########") return "\n".join(lines) @@ -1839,8 +1852,10 @@ def main(): parser.add_argument("command") # consume "eval" parser.add_argument("--result-file", required=True) parser.add_argument("--output", "-o", default=None) + parser.add_argument("--for-pr-comment", action="store_true") + parser.add_argument("--stdout-markdown", action="store_true") args = parser.parse_args() - report = generate_eval_report(args.result_file) + report = generate_eval_report(args.result_file, for_pr_comment=args.for_pr_comment) else: parser = argparse.ArgumentParser(description="Generate comparison report") parser.add_argument("command") # consume "compare" @@ -1862,14 +1877,15 @@ def main(): # the canonical bundle location at the end of the run, which is what the # user actually wants to navigate to. if args.output: - # Compare-mode and eval-mode both produce markdown for the saved file - # and re-render plain text for the terminal. if "command" in args and getattr(args, "command", None) == "eval": - plain = generate_eval_report(args.result_file, markdown=False) + if getattr(args, "for_pr_comment", False) or getattr(args, "stdout_markdown", False): + stdout_report = report + else: + stdout_report = generate_eval_report(args.result_file, markdown=False) else: - plain = generate_report(config_results, markdown=False) + stdout_report = generate_report(config_results, markdown=False) Path(args.output).write_text(report) - print(plain) + print(stdout_report) else: # No file requested — just print whatever generate_* produced. print(report) diff --git a/benchmarks/helpers/react_agent.py b/benchmarks/helpers/react_agent.py index be1c941c..8c61b51c 100644 --- a/benchmarks/helpers/react_agent.py +++ b/benchmarks/helpers/react_agent.py @@ -25,6 +25,14 @@ from langchain_core.messages import HumanMessage from loguru import logger +try: + from langchain_openai import ChatOpenAI as _ChatOpenAI +except ImportError as exc: + _ChatOpenAI = None + _CHAT_OPENAI_IMPORT_ERROR = exc +else: + _CHAT_OPENAI_IMPORT_ERROR = None + TOOL_BLOCK_RE = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE) PROMPT_ROLE_RE = re.compile(r"(USER|ASSISTANT|SYSTEM):\n", re.IGNORECASE) PROMPT_PATH = Path(__file__).with_name("react_agent_prompt.txt") @@ -44,6 +52,43 @@ class ReactInvokeResult: react_steps: int = 0 +def _require_chat_openai(): + if _ChatOpenAI is None: + raise RuntimeError( + "langchain-openai is required for React agent with OpenAI-compatible or RITS providers. " + "Install with: pip install langchain-openai" + ) from _CHAT_OPENAI_IMPORT_ERROR + return _ChatOpenAI + + +class ChatRits(_ChatOpenAI if _ChatOpenAI is not None else object): + """RITS chat model integration using langchain-openai.""" + + def __init__(self, config: dict[str, Any]): + _require_chat_openai() + model_name = config.get("model_name", "openai/gpt-oss-120b-a100") + end_point = ( + config.get("end_point") + or "https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100" + ) + rits_api_key = config.get("api_key") or os.getenv("RITS_API_KEY") + if rits_api_key is None: + raise ValueError("RITS_API_KEY is required for settings.rits.toml") + + params = config.get("params", {}) + base_url = end_point.rstrip("/") + if not base_url.endswith("/v1"): + base_url = f"{base_url}/v1" + rits_config: dict[str, Any] = { + "model": model_name, + "api_key": "/", + "default_headers": {"RITS_API_KEY": rits_api_key}, + "base_url": base_url, + } + rits_config.update(params) + super().__init__(**rits_config) + + class GenericReactAgent: """Prompt-template driven generic ReAct agent over benchmark tools.""" @@ -149,14 +194,6 @@ def _create_llm(self) -> Any: ) elif settings_config == "settings.openai.toml": - try: - from langchain_openai import ChatOpenAI - except ImportError as exc: - raise RuntimeError( - "langchain-openai is required for React agent with OpenAI. " - "Install with: pip install langchain-openai" - ) from exc - disable_ssl = os.getenv("CUGA_DISABLE_SSL", "").lower() in ("true", "1", "yes") ssl_verify = ( os.getenv("OPENAI_SSL_VERIFY", "true").lower() not in ("false", "0", "no") and not disable_ssl @@ -181,12 +218,22 @@ def _create_llm(self) -> Any: llm_kwargs["http_async_client"] = httpx.AsyncClient(verify=False) # noqa: S501 # nosec B501 — same logger.info("SSL verification disabled for ChatOpenAI") - return ChatOpenAI(**llm_kwargs) + return _require_chat_openai()(**llm_kwargs) + + elif settings_config in ("settings.rits.toml", "settings.rits.proxy.toml"): + api_base = os.getenv("RITS_BASE_URL") or os.getenv("LITE_LLM_URL") or os.getenv("OPENAI_BASE_URL") + api_key = os.getenv("RITS_API_KEY") or os.getenv("LITE_LLM_KEY") or os.getenv("OPENAI_API_KEY") + model_name = os.getenv("MODEL_NAME") or self.model or "openai/gpt-oss-120b-a100" + logger.info( + f"RITS config: model={model_name}, api_base={api_base}, api_key_present={bool(api_key)}" + ) + return ChatRits(config={"model_name": model_name, "end_point": api_base, "api_key": api_key}) else: raise RuntimeError( "Unsupported AGENT_SETTING_CONFIG for React agent. " - "Expected 'settings.groq.toml' or 'settings.openai.toml'." + "Expected 'settings.groq.toml', 'settings.openai.toml', " + "'settings.rits.toml', or 'settings.rits.proxy.toml'." ) def _get_llm(self) -> Any: diff --git a/benchmarks/helpers/tests/test_compare_report.py b/benchmarks/helpers/tests/test_compare_report.py index 46f5a55d..93345b8d 100644 --- a/benchmarks/helpers/tests/test_compare_report.py +++ b/benchmarks/helpers/tests/test_compare_report.py @@ -21,10 +21,12 @@ """ import json +import sys from pathlib import Path import pytest +from benchmarks.helpers import compare_report as compare_report_module from benchmarks.helpers.compare_report import ( _aggregate_receipt_costs, _last_turn_judge_scores, @@ -221,6 +223,78 @@ def test_parse_sdk_results_includes_react_steps(): assert parsed["tasks"]["t1"]["steps"] == 3 +def test_eval_report_default_omits_pr_comment_wrapping(tmp_path): + result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False}) + + report = generate_eval_report(result_file) + + assert report.startswith("# Evaluation Report") + assert "######## REPORT START ########" not in report + assert "######## REPORT END ########" not in report + assert "Evaluation Report" not in report + + +def test_eval_report_pr_comment_wrapping_is_opt_in(tmp_path): + result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False}) + + report = generate_eval_report(result_file, for_pr_comment=True) + + assert report.startswith("######## REPORT START ########") + assert "Evaluation Report" in report + assert report.rstrip().endswith("######## REPORT END ########") + + +def test_eval_main_with_output_prints_plain_text_by_default(tmp_path, monkeypatch, capsys): + result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False}) + output_file = tmp_path / "report.md" + monkeypatch.setattr( + sys, + "argv", + [ + "compare_report.py", + "eval", + "--result-file", + result_file, + "--output", + str(output_file), + ], + ) + + compare_report_module.main() + + stdout = capsys.readouterr().out + saved_report = output_file.read_text() + assert stdout.startswith("\nEvaluation Report") + assert "######## REPORT START ########" not in stdout + assert saved_report.startswith("# Evaluation Report") + assert "######## REPORT START ########" not in saved_report + + +def test_eval_main_pr_comment_mode_prints_markdown_markers(tmp_path, monkeypatch, capsys): + result_file = _appworld_run(tmp_path, "run.json", {"A": True, "B": False}) + output_file = tmp_path / "report.md" + monkeypatch.setattr( + sys, + "argv", + [ + "compare_report.py", + "eval", + "--result-file", + result_file, + "--output", + str(output_file), + "--for-pr-comment", + ], + ) + + compare_report_module.main() + + stdout = capsys.readouterr().out + saved_report = output_file.read_text() + assert stdout.startswith("######## REPORT START ########") + assert saved_report.startswith("######## REPORT START ########") + + def _m3_run(tmp_path: Path, name: str, tasks) -> str: """Write an M3-shape ``*_final_report.json`` and return its path. diff --git a/benchmarks/m3/eval.sh b/benchmarks/m3/eval.sh index 1c79bb7c..ea417aad 100755 --- a/benchmarks/m3/eval.sh +++ b/benchmarks/m3/eval.sh @@ -263,9 +263,14 @@ create_bundle() { # want the bundle, so don't let `set -e` abort here). local report_tmp report_tmp=$(mktemp /tmp/m3_eval_report_XXXXXX) - uv run --no-sync python -m benchmarks.helpers.compare_report eval \ - --result-file "$latest_result" --output "$report_tmp" || \ + local report_args=(eval --result-file "$latest_result" --output "$report_tmp") + uv run --no-sync python -m benchmarks.helpers.compare_report "${report_args[@]}" || \ echo -e "${YELLOW:-}Report generation failed — bundling without report.${NC:-}" + if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then + uv run --no-sync python -m benchmarks.helpers.compare_report \ + eval --result-file "$latest_result" --for-pr-comment --stdout-markdown || \ + echo -e "${YELLOW:-}PR report generation failed — continuing with bundle.${NC:-}" + fi local bundle_args=(assemble --benchmark m3 --result-files "$latest_result" diff --git a/benchmarks/oak_health_insurance/eval.sh b/benchmarks/oak_health_insurance/eval.sh index 747c4b7d..d937dd59 100755 --- a/benchmarks/oak_health_insurance/eval.sh +++ b/benchmarks/oak_health_insurance/eval.sh @@ -242,8 +242,12 @@ if [ $EVAL_EXIT -eq 0 ]; then # Generate eval report REPORT_TMP=$(mktemp /tmp/oak_eval_report_XXXXXX) - uv run --no-sync python -m benchmarks.helpers.compare_report eval \ - --result-file "$LATEST_RESULT" --output "$REPORT_TMP" + REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP") + uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}" + if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then + uv run --no-sync python -m benchmarks.helpers.compare_report \ + eval --result-file "$LATEST_RESULT" --for-pr-comment --stdout-markdown + fi BUNDLE_ARGS=(assemble --benchmark oak_health_insurance --result-files "$LATEST_RESULT" diff --git a/scripts/pr-regression-testing/README.md b/scripts/pr-regression-testing/README.md new file mode 100644 index 00000000..c214ff6e --- /dev/null +++ b/scripts/pr-regression-testing/README.md @@ -0,0 +1,233 @@ +# PR Regression Testing + +This folder contains the scripts used to set up and run PR-triggered regression evaluations from GitHub Actions comments. + +## Files + +- `setup-pr-regression-workflow.sh`: bootstraps the self-hosted GitHub Actions runner VM. +- `run-pr-regression-eval.sh`: consumes validated `/run-pr-eval` parameters and runs the requested benchmark using the `cuga-eval/scripts/eval.sh` script. + +## Setup The PR Regression Flow + +1. Add the workflow file to the `cuga-agent` repository: + + ```text + .github/workflows/run-pr-eval.yml + ``` + +2. In GitHub, create a self-hosted runner token for the `cuga-agent` repository. + + Go to: + + ```text + cuga-agent repository -> Settings -> Actions -> Runners -> New self-hosted runner + ``` + + Copy temporary runner token from the generated `config.sh` command. + +3. Ensure the workflow file uses the self-hosted runner labels: + + ```yaml + runs-on: [self-hosted, linux, run-pr-eval] + ``` + +4. On the runner VM, export the required setup variables: + + ```bash + export GITHUB_REPO_URL="https://github.com/OWNER/cuga-agent" + export GITHUB_RUNNER_TOKEN="YOUR_TEMPORARY_RUNNER_TOKEN" + export RUNNER_LABELS="run-pr-eval" + ``` + +5. Optionally choose repo mode and branches: + + ```bash + export REPO_MODE="fork" + export CUGA_AGENT_BRANCH="main" + export CUGA_EVAL_BRANCH="main" + ``` + + Supported `REPO_MODE` values: + + - `fork`: uses `https://github.com/AnkitaNaik/cuga-agent.git` and `https://github.com/AnkitaNaik/cuga-eval.git`. + - `upstream`: uses `https://github.com/cuga-project/cuga-agent.git` and `https://github.com/cuga-project/cuga-eval.git`. + +6. Run the setup script on the VM as the normal runner user, not with `sudo`: + + ```bash + bash /path/to/cuga-eval/scripts/pr-regression-testing/setup-pr-regression-workflow.sh + ``` + + The script will: + + - verify required commands: `git`, `curl`, `tar`, `python3`, `git-lfs`, and `uv`; + - clone or update `cuga-agent` and `cuga-eval` under `~/pr-regression-testing`; + - run `cuga-eval/setup_cuga.sh`; + - create and sync the `uv` environment; + - run AppWorld setup; + - download and register the GitHub Actions runner; + - start the runner in the background. + +7. Confirm the runner is online in GitHub: + + ```text + cuga-agent repository -> Settings -> Actions -> Runners + ``` + + Ensure the runner is running with the labels `[self-hosted, linux, run-pr-eval]` i.e. same as workflow file. + +8. Check runner logs on the VM if needed: + + ```bash + tail -f ~/pr-regression-testing/github-runner/runner.log + ``` + +## GitHub Secrets And Variables + +Configure these in the `cuga-agent` repository: + +```text +Settings -> Secrets and variables -> Actions +``` + +Required secrets: + +- `RITS_API_KEY`: required for `provider=rits`. +- `LITE_LLM_KEY`: preferred secret for `provider=litellm`; the workflow injects it into `OPENAI_API_KEY`. +- `OPENAI_API_KEY`: optional fallback secret for `provider=litellm` when `LITE_LLM_KEY` is not set. + +Optional repository variables: + +- `MODEL_NAME`: default model override used by the workflow environment. +- `RITS_BASE_URL`: RITS endpoint override for `provider=rits`. +- `LITELLM_BASE_URL`: LiteLLM endpoint override for `provider=litellm`; the workflow injects it into `OPENAI_BASE_URL`. + +Provider defaults used by `run-pr-regression-eval.sh`: + +- `provider=rits` selects `settings.rits.toml`, requires `RITS_API_KEY`, and passes `RITS_BASE_URL` to `models.py`. +- `provider=litellm` selects `settings.litellm.toml` for the CUGA agent or `settings.openai.toml` for non-CUGA agents, requires `OPENAI_API_KEY`, and passes `OPENAI_BASE_URL` to `models.py`. Unless `model_name` is explicitly set in the PR comment, it uses `aws/gpt-oss-120b`. +- For `provider=rits`, overriding `model_name` in the PR command requires `RITS_BASE_URL` to be configured so the model name and endpoint cannot silently diverge. + +## Run PR Evaluations + +Create a comment on a pull request in `cuga-agent`: + +```text +/run-pr-eval sha= +``` + +The `sha` value must be the current 40-character PR head commit SHA. The workflow validates it before checkout and does not forward it to the evaluation process. By default, the forwarded evaluation runs: + +- `benchmark=appworld` +- `agent=react` +- `provider=rits` +- `model_name=openai/gpt-oss-120b-a100` +- `num_tasks=4` +- default AppWorld task IDs: `9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1` + +AppWorld PR evaluations with `agent=cuga` are run with the AppWorld SDK evaluator (`--sdk`). + +For `provider=litellm`, the default model changes to `aws/gpt-oss-120b` unless `model_name` is explicitly provided. + +## Command Parameters + +The command accepts whitespace-separated `key=value` parameters: + +- `sha`: required by the GitHub workflow; must be the current 40-character PR head SHA. +- `model_name`: model name to evaluate. +- `task_id`: one task ID, or comma-separated task IDs. +- `task_ids`: alias for `task_id`. +- `eval_key`: AppWorld task group from `benchmarks/appworld/eval_config.toml`, such as `test_easy`, `test_med`, or `test_hard`. +- `benchmark`: `appworld` or `m3`; defaults to `appworld`. +- `num_tasks`: positive integer; defaults to `4`. +- `agent`: `react`, `cuga`, or `codeact`; defaults to `react`. +- `provider`: `rits` or `litellm`; defaults to `rits`. + +Unsupported parameters cause the workflow or runner script to fail early with a list of supported parameters. + +The workflow parser tolerates spaces around `=`, markdown code ticks, mixed-case values for `agent`, `benchmark`, and `provider`, and spaces after commas in task lists. For example, `provider = litellm` and `task_id = 9aae7da_1, 365e0a3_1` are accepted. The runner script also accepts sanitized CLI-style arguments such as `--benchmark appworld --agent react --provider rits`. + +Useful AppWorld `eval_key` values: + +- `test_easy` +- `test_med` +- `test_hard` +- `test_challenge_easy` +- `test_challenge_med` +- `test_challenge_hard` +- `test_normal_easy` +- `test_normal_med` +- `test_normal_hard` + +## Examples + +Run the default AppWorld React-agent evaluation on RITS: + +```text +/run-pr-eval sha= +``` + +Run one AppWorld task with the React agent on RITS: + +```text +/run-pr-eval sha= provider=rits agent=react benchmark=appworld task_id=9aae7da_1 num_tasks=1 +``` + +Run one AppWorld task with the CUGA agent on LiteLLM: + +```text +/run-pr-eval sha= provider=litellm agent=cuga benchmark=appworld task_id=9aae7da_1 num_tasks=1 +``` + +Run AppWorld with two explicit tasks: + +```text +/run-pr-eval sha= benchmark=appworld task_id=9aae7da_1,365e0a3_1 num_tasks=2 +``` + +Run AppWorld easy tasks from `eval_config.toml`: + +```text +/run-pr-eval sha= benchmark=appworld eval_key=test_easy +``` + +Run AppWorld medium tasks from `eval_config.toml`: + +```text +/run-pr-eval sha= benchmark=appworld eval_key=test_med +``` + +Run AppWorld hard tasks from `eval_config.toml`: + +```text +/run-pr-eval sha= benchmark=appworld eval_key=test_hard +``` + +Run M3 with the default React agent: + +```text +/run-pr-eval sha= benchmark=m3 num_tasks=1 +``` + +Run M3 with the CUGA agent and LiteLLM: + +```text +/run-pr-eval sha= benchmark=m3 agent=cuga provider=litellm num_tasks=1 +``` + +Override the model: + +```text +/run-pr-eval sha= provider=litellm agent=cuga benchmark=appworld model_name=aws/glm-5 task_id=9aae7da_1 num_tasks=1 +``` + +## Output + +The workflow posts a PR comment containing: + +- trusted run metadata; +- requested command and SHA; +- the evaluation report markdown emitted by the PR regression runner; +- a link to the uploaded workflow artifact. + +The full output is also uploaded as a GitHub Actions artifact. During the workflow run, logs are written under the checked-out workspace in `pr-eval-logs/`. diff --git a/scripts/pr-regression-testing/run-pr-regression-eval.sh b/scripts/pr-regression-testing/run-pr-regression-eval.sh new file mode 100755 index 00000000..0d1e9cbd --- /dev/null +++ b/scripts/pr-regression-testing/run-pr-regression-eval.sh @@ -0,0 +1,517 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +START_EPOCH="$(date +%s)" +START_TIME="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" + +log() { + printf '[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" +} + +extract_report_blocks() { + local input_path="$1" + local output_path="$2" + + if [[ ! -f "${input_path}" ]]; then + echo "No evaluation output artifact was available." > "${output_path}" + return 0 + fi + + awk ' + $0 == "######## REPORT START ########" { + in_report = 1 + block = "" + next + } + $0 == "######## REPORT END ########" { + if (in_report) { + sub(/^[[:space:]]+/, "", block) + sub(/[[:space:]]+$/, "", block) + if (length(block) > 0) { + if (printed) { + print "" + print "" + } + print block + printed = 1 + } + } + in_report = 0 + block = "" + next + } + in_report { + block = block $0 "\n" + } + END { + if (!printed) { + print "No report section found between REPORT START and REPORT END markers." + } + } + ' "${input_path}" > "${output_path}" +} + +if [[ "${1:-}" == "--extract-report" ]]; then + if [[ $# -ne 3 ]]; then + echo "Usage: $0 --extract-report " >&2 + exit 2 + fi + extract_report_blocks "$2" "$3" + exit 0 +fi + +open_details() { + local title="$1" + echo "
" + echo "${title}" + echo +} + +close_details() { + echo + echo "
" +} + +on_exit() { + status=$? + end_epoch="$(date +%s)" + end_time="$(date -u +"%Y-%m-%d %H:%M:%S UTC")" + duration=$((end_epoch - START_EPOCH)) + minutes=$((duration / 60)) + seconds=$((duration % 60)) + + echo "######## REPORT START ########" + echo + open_details "Run summary" + echo "- Started: ${START_TIME}" + echo "- Finished: ${end_time}" + echo "- Duration: ${minutes}m ${seconds}s" + echo "- Exit code: ${status}" + + if [[ ${status} -eq 0 ]]; then + echo "- Result: SUCCESS" + else + echo "- Result: FAILED" + fi + close_details + echo "######## REPORT END ########" +} +trap on_exit EXIT + +DEFAULT_MODEL_NAME="openai/gpt-oss-120b-a100" +DEFAULT_LITELLM_MODEL_NAME="aws/gpt-oss-120b" +DEFAULT_RITS_BASE_URL="https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100/v1" +DEFAULT_LITELLM_BASE_URL="https://ete-litellm.ai-models.vpc-int.res.ibm.com/" +DEFAULT_APPWORLD_TASK_IDS="9aae7da_1 365e0a3_1 eb5ad85_1 5e27cd7_1" +DEFAULT_BENCHMARK="appworld" +DEFAULT_NUM_TASKS="4" +DEFAULT_AGENT="react" +DEFAULT_PROVIDER="rits" + +MODEL_NAME="${MODEL_NAME:-$DEFAULT_MODEL_NAME}" +TASK_IDS="${TASK_IDS:-}" +EVAL_KEY="${EVAL_KEY:-}" +BENCHMARK="${BENCHMARK:-$DEFAULT_BENCHMARK}" +NUM_TASKS="${NUM_TASKS:-$DEFAULT_NUM_TASKS}" +AGENT="${AGENT:-$DEFAULT_AGENT}" +PROVIDER="${PROVIDER:-$DEFAULT_PROVIDER}" +MODEL_NAME_FROM_COMMENT=false + +usage_error() { + local message="$1" + echo "ERROR: ${message}" + echo "Supported parameters: model_name, task_id, task_ids, eval_key, benchmark, num_tasks, agent, provider, sha" + close_details + echo "######## REPORT END ########" + exit 2 +} + +# Parse whitespace-separated key=value parameters from the PR comment. +# Common comment formatting is normalized first, so these are equivalent: +# provider=litellm +# provider = litellm +# provider= litellm +# provider =litellm +# Supported aliases: +# model_name=... +# task_id=id1,id2 +# task_ids=id1,id2 +# eval_key=test_easy|test_med|test_hard|... +# benchmark=appworld|m3 +# num_tasks=... +# agent=react|cuga|codeact +# provider=rits|litellm +COMMAND_TOKENS=() +if [[ $# -gt 0 ]]; then + while [[ $# -gt 0 ]]; do + case "$1" in + /run-pr-eval) + shift + ;; + --model-name|--model_name) + [[ -n "${2:-}" && "${2:-}" != --* ]] || { + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "$1 requires a value." + } + COMMAND_TOKENS+=("model_name=$2") + shift 2 + ;; + --task-id|--task-ids|--task|--tasks) + [[ -n "${2:-}" && "${2:-}" != --* ]] || { + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "$1 requires a value." + } + COMMAND_TOKENS+=("task_ids=$2") + shift 2 + ;; + --eval-key|--eval_key) + [[ -n "${2:-}" && "${2:-}" != --* ]] || { + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "$1 requires a value." + } + COMMAND_TOKENS+=("eval_key=$2") + shift 2 + ;; + --benchmark|--num-tasks|--num_tasks|--agent|--agent-type|--agent_type|--provider|--sha) + key="${1#--}" + key="${key//-/_}" + [[ -n "${2:-}" && "${2:-}" != --* ]] || { + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "$1 requires a value." + } + COMMAND_TOKENS+=("${key}=$2") + shift 2 + ;; + *=*) + COMMAND_TOKENS+=("$1") + shift + ;; + *) + if [[ $# -eq 1 && "$1" == *"/run-pr-eval"* ]]; then + COMMENT_BODY="$1" + shift + else + echo "######## REPORT START ########" + open_details "Requested parameters" + usage_error "Unsupported argument: $1" + fi + ;; + esac + done +fi + +if [[ ${#COMMAND_TOKENS[@]} -gt 0 ]]; then + COMMENT_BODY="/run-pr-eval ${COMMAND_TOKENS[*]}" +else + COMMENT_BODY="${COMMENT_BODY:-}" +fi + +# Remove Windows carriage returns, newlines, tabs, and markdown code ticks. +COMMENT_BODY="${COMMENT_BODY//$'\r'/ }" +COMMENT_BODY="${COMMENT_BODY//$'\n'/ }" +COMMENT_BODY="${COMMENT_BODY//$'\t'/ }" +COMMENT_BODY="${COMMENT_BODY//\`/ }" +if [[ "${COMMENT_BODY}" == *"/run-pr-eval"* ]]; then + COMMENT_BODY="/run-pr-eval ${COMMENT_BODY#*/run-pr-eval}" +fi + +# Normalize spaces around "=" for supported keys, and tolerate spaces after +# commas in comma-separated values such as task_id=a, b. +COMMENT_BODY="$( + printf '%s' "${COMMENT_BODY}" | + sed -E 's/(^|[[:space:]])(model_name|task_id|task_ids|eval_key|eval-key|benchmark|num_tasks|agent|agent_type|provider|sha)[[:space:]]*=[[:space:]]*/\1\2=/g; s/,[[:space:]]+/,/g' +)" + +read -r -a TOKENS <<< "${COMMENT_BODY}" +echo "######## REPORT START ########" +open_details "Requested parameters" +for token in "${TOKENS[@]}"; do + case "${token}" in + /run-pr-eval) + ;; + + model_name=*) + MODEL_NAME="${token#model_name=}" + MODEL_NAME_FROM_COMMENT=true + ;; + + task_id=*|task_ids=*) + TASK_IDS="${token#*=}" + TASK_IDS="${TASK_IDS//,/ }" + ;; + + eval_key=*|eval-key=*) + EVAL_KEY="${token#*=}" + ;; + + benchmark=*) + BENCHMARK="${token#benchmark=}" + ;; + + num_tasks=*) + NUM_TASKS="${token#num_tasks=}" + ;; + + agent=*|agent_type=*) + AGENT="${token#*=}" + ;; + + provider=*) + PROVIDER="${token#provider=}" + ;; + + sha=*) + ;; + + "") + ;; + + *) + usage_error "Unsupported parameter: ${token}" + ;; + esac +done + +if [[ -z "${MODEL_NAME}" ]]; then + usage_error "model_name cannot be empty." +fi + +if [[ ! "${NUM_TASKS}" =~ ^[0-9]+$ ]]; then + usage_error "num_tasks must be a positive integer." +fi +NUM_TASKS=$((10#${NUM_TASKS})) +if (( NUM_TASKS < 1 )); then + usage_error "num_tasks must be a positive integer." +fi + +AGENT="$(printf '%s' "${AGENT}" | tr '[:upper:]' '[:lower:]')" +BENCHMARK="$(printf '%s' "${BENCHMARK}" | tr '[:upper:]' '[:lower:]')" +PROVIDER="$(printf '%s' "${PROVIDER}" | tr '[:upper:]' '[:lower:]')" +EVAL_KEY="$(printf '%s' "${EVAL_KEY}" | tr '[:upper:]' '[:lower:]')" + +case "${AGENT}" in + react|cuga|codeact) + ;; + *) + echo "ERROR: agent must be one of: react, cuga, codeact." + close_details + echo "######## REPORT END ########" + exit 2 + ;; +esac + +case "${BENCHMARK}" in + appworld|m3) + ;; + *) + echo "ERROR: benchmark must be one of: appworld, m3." + close_details + echo "######## REPORT END ########" + exit 2 + ;; +esac + +if [[ -n "${EVAL_KEY}" && "${BENCHMARK}" != "appworld" ]]; then + echo "ERROR: eval_key is only supported for benchmark=appworld." + close_details + echo "######## REPORT END ########" + exit 2 +fi + +case "${PROVIDER}" in + rits) + AGENT_SETTING_CONFIG="settings.rits.toml" + if [[ -z "${RITS_API_KEY:-}" ]]; then + echo "ERROR: RITS_API_KEY is required when provider=rits." + close_details + echo "######## REPORT END ########" + exit 2 + fi + if [[ "${MODEL_NAME_FROM_COMMENT}" == "true" && -z "${RITS_BASE_URL:-}" ]]; then + echo "ERROR: provider=rits model_name overrides require RITS_BASE_URL to avoid mismatched model endpoints." + close_details + echo "######## REPORT END ########" + exit 2 + fi + RITS_BASE_URL="${RITS_BASE_URL:-${DEFAULT_RITS_BASE_URL}}" + export RITS_API_KEY RITS_BASE_URL + ;; + litellm) + if [[ "${AGENT}" == "cuga" ]]; then + AGENT_SETTING_CONFIG="settings.litellm.toml" + else + AGENT_SETTING_CONFIG="settings.openai.toml" + fi + if [[ "${MODEL_NAME_FROM_COMMENT}" == "false" ]]; then + MODEL_NAME="${DEFAULT_LITELLM_MODEL_NAME}" + fi + OPENAI_BASE_URL="${OPENAI_BASE_URL:-${DEFAULT_LITELLM_BASE_URL}}" + if [[ -z "${OPENAI_API_KEY:-}" ]]; then + echo "ERROR: OPENAI_API_KEY is required when provider=litellm." + close_details + echo "######## REPORT END ########" + exit 2 + fi + if [[ -z "${OPENAI_BASE_URL:-}" ]]; then + echo "ERROR: OPENAI_BASE_URL is required when provider=litellm." + close_details + echo "######## REPORT END ########" + exit 2 + fi + export OPENAI_API_KEY OPENAI_BASE_URL + ;; + *) + echo "ERROR: provider must be one of: rits, litellm." + close_details + echo "######## REPORT END ########" + exit 2 + ;; +esac + +if [[ -z "${TASK_IDS}" && -z "${EVAL_KEY}" && "${BENCHMARK}" == "appworld" ]]; then + TASK_IDS="${DEFAULT_APPWORLD_TASK_IDS}" +fi + +read -r -a TASK_ID_ARRAY <<< "${TASK_IDS}" + +if [[ ${#TASK_ID_ARRAY[@]} -eq 0 && -z "${EVAL_KEY}" && "${BENCHMARK}" == "appworld" ]]; then + echo "ERROR: At least one task_id or eval_key is required for benchmark=appworld." + close_details + echo "######## REPORT END ########" + exit 2 +fi +echo "- Model: ${MODEL_NAME}" +echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" +if [[ -n "${EVAL_KEY}" ]]; then + echo "- Eval key: ${EVAL_KEY}" +fi +echo "- Num tasks: ${NUM_TASKS}" +if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + echo "- Task IDs: ${TASK_IDS}" +else + echo "- Task IDs: benchmark default" +fi +close_details +echo "######## REPORT END ########" + +# For AppWorld, num_tasks limits the explicit task list. M3 maps num_tasks to +# --max-samples-per-domain below. +if [[ "${BENCHMARK}" == "appworld" && ${NUM_TASKS} -lt ${#TASK_ID_ARRAY[@]} ]]; then + TASK_ID_ARRAY=("${TASK_ID_ARRAY[@]:0:${NUM_TASKS}}") +fi + +export AGENT_SETTING_CONFIG +export DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING="${DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING:-false}" +export MODEL_NAME +export PR_EVAL_FOR_PR_COMMENT="${PR_EVAL_FOR_PR_COMMENT:-true}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -z "${EVAL_REPO:-}" ]]; then + if [[ -f "${SCRIPT_DIR}/../../scripts/eval.sh" ]]; then + EVAL_REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" + else + EVAL_REPO="$(cd "${SCRIPT_DIR}/.." && pwd)" + fi +fi + +if [[ ! -d "${EVAL_REPO}" ]]; then + echo "ERROR: Evaluation repo not found: ${EVAL_REPO}" + exit 1 +fi + +if [[ ! -f "${EVAL_REPO}/scripts/eval.sh" ]]; then + echo "ERROR: Evaluation script not found: ${EVAL_REPO}/scripts/eval.sh" + exit 1 +fi + +cd "${EVAL_REPO}" + +echo "######## REPORT START ########" +echo "## Evaluation Metrics" +echo +open_details "Run metadata" +echo "- Repository: ${GITHUB_REPOSITORY:-unknown}" +echo "- PR: ${PR_NUMBER:-unknown}" +echo "- Commit: ${PR_HEAD_SHA:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" +echo "- Runner: ${RUNNER_NAME:-$(hostname)}" +echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" +close_details +echo +open_details "Evaluation configuration" +echo "- Agent: ${AGENT}" +echo "- Benchmark: ${BENCHMARK}" +echo "- Provider: ${PROVIDER}" +echo "- Agent setting config: ${AGENT_SETTING_CONFIG}" +echo "- Model: ${MODEL_NAME}" +echo "- Requested num_tasks: ${NUM_TASKS}" +if [[ "${BENCHMARK}" == "appworld" ]]; then + if [[ -n "${EVAL_KEY}" ]]; then + echo "- Eval key: ${EVAL_KEY}" + fi + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + echo "- Effective task count: ${#TASK_ID_ARRAY[@]}" + echo "- Task IDs:" + for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" + done + else + echo "- Task IDs: selected by eval_key" + fi +else + echo "- M3 max samples per domain: ${NUM_TASKS}" + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + echo "- M3 task filters:" + for task_id in "${TASK_ID_ARRAY[@]}"; do + echo " - ${task_id}" + done + else + echo "- M3 task filters: benchmark default" + fi +fi +close_details +echo "######## REPORT END ########" + +log "Verifying required commands" +command -v uv >/dev/null 2>&1 || { + echo "ERROR: uv is not installed or is not in PATH." + exit 127 +} + +command -v curl >/dev/null 2>&1 || { + echo "ERROR: curl is not installed or is not in PATH." + exit 127 +} + +log "Starting ${BENCHMARK} ${AGENT} evaluation" + +EVAL_ARGS=( + --benchmark "${BENCHMARK}" + --agent "${AGENT}" +) + +if [[ "${BENCHMARK}" == "appworld" ]]; then + if [[ "${AGENT}" == "cuga" ]]; then + EVAL_ARGS+=(--sdk) + fi + if [[ -n "${EVAL_KEY}" ]]; then + EVAL_ARGS+=(--eval-key "${EVAL_KEY}") + fi + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") + fi +else + EVAL_ARGS+=(--m3-data --max-samples-per-domain "${NUM_TASKS}") + if [[ ${#TASK_ID_ARRAY[@]} -gt 0 ]]; then + EVAL_ARGS+=(--task "${TASK_ID_ARRAY[@]}") + fi +fi + +bash scripts/eval.sh "${EVAL_ARGS[@]}" diff --git a/scripts/pr-regression-testing/setup-pr-regression-workflow.sh b/scripts/pr-regression-testing/setup-pr-regression-workflow.sh new file mode 100644 index 00000000..c9bd9694 --- /dev/null +++ b/scripts/pr-regression-testing/setup-pr-regression-workflow.sh @@ -0,0 +1,311 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# ============================================================================= +# PR Regression Testing VM Bootstrap +# +# Modes: +# REPO_MODE=fork -> use temporary AnkitaNaik forks (default for now) +# REPO_MODE=upstream -> use cuga-project upstream repositories +# +# Final layout: +# +# ~/pr-regression-testing/ +# ├── cuga-agent/ +# ├── cuga-eval/ +# │ └── scripts/ +# │ ├── pr-regression-testing/ +# │ │ ├── setup-pr-regression-workflow.sh +# │ │ └── run-pr-regression-eval.sh +# │ └── run-pr-eval.sh -> pr-regression-testing/run-pr-regression-eval.sh +# └── github-runner/ +# └── _work/ +# +# Required VM prerequisites: +# git curl tar python3 git-lfs +# +# Required environment variables: +# GITHUB_REPO_URL +# GITHUB_RUNNER_TOKEN +# +# Optional: +# REPO_MODE=fork|upstream +# ROOT_DIR +# RUNNER_NAME +# RUNNER_LABELS +# CUGA_AGENT_BRANCH +# CUGA_EVAL_BRANCH +# RUN_EVAL_SCRIPT +# RUNNER_VERSION +# ============================================================================= + +REPO_MODE="${REPO_MODE:-fork}" +ROOT_DIR="${ROOT_DIR:-$HOME/pr-regression-testing}" + +RUNNER_NAME="${RUNNER_NAME:-$(hostname)}" +RUNNER_LABELS="${RUNNER_LABELS:-run-pr-eval}" + +CUGA_AGENT_BRANCH="${CUGA_AGENT_BRANCH:-main}" +CUGA_EVAL_BRANCH="${CUGA_EVAL_BRANCH:-main}" + +case "${REPO_MODE}" in + fork) + DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/AnkitaNaik/cuga-agent.git" + DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/AnkitaNaik/cuga-eval.git" + ;; + upstream) + DEFAULT_CUGA_AGENT_REPO_URL="https://github.com/cuga-project/cuga-agent.git" + DEFAULT_CUGA_EVAL_REPO_URL="https://github.com/cuga-project/cuga-eval.git" + ;; + *) + echo "ERROR: REPO_MODE must be 'fork' or 'upstream'." >&2 + exit 2 + ;; +esac + +CUGA_AGENT_REPO_URL="${CUGA_AGENT_REPO_URL:-$DEFAULT_CUGA_AGENT_REPO_URL}" +CUGA_EVAL_REPO_URL="${CUGA_EVAL_REPO_URL:-$DEFAULT_CUGA_EVAL_REPO_URL}" + +CUGA_AGENT_DIR="${ROOT_DIR}/cuga-agent" +CUGA_EVAL_DIR="${ROOT_DIR}/cuga-eval" +GH_RUNNER_DIR="${ROOT_DIR}/github-runner" + +RUN_EVAL_SCRIPT="${RUN_EVAL_SCRIPT:-${CUGA_EVAL_DIR}/scripts/pr-regression-testing/run-pr-regression-eval.sh}" +WORKFLOW_EVAL_SCRIPT="${CUGA_EVAL_DIR}/scripts/run-pr-eval.sh" + +log() { + printf '\n[%s] %s\n' "$(date -u +"%Y-%m-%d %H:%M:%S UTC")" "$*" +} + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || die "Required command '$1' is missing." +} + +clone_or_update() { + local repo_url="$1" + local branch="$2" + local dest="$3" + local name="$4" + + if [[ -d "${dest}/.git" ]]; then + log "Updating ${name}" + git -C "${dest}" fetch --all --prune + git -C "${dest}" checkout "${branch}" + git -C "${dest}" pull --ff-only origin "${branch}" + else + log "Cloning ${name}" + git clone --branch "${branch}" "${repo_url}" "${dest}" + fi +} + +: "${GITHUB_REPO_URL:?Set GITHUB_REPO_URL, e.g. https://github.com/OWNER/REPO}" +: "${GITHUB_RUNNER_TOKEN:?Set GITHUB_RUNNER_TOKEN to the temporary runner token}" + +log "Repository mode: ${REPO_MODE}" +echo "cuga-agent: ${CUGA_AGENT_REPO_URL}" +echo "cuga-eval: ${CUGA_EVAL_REPO_URL}" + +log "Checking VM prerequisites" +need_cmd git +need_cmd curl +need_cmd tar +need_cmd python3 +need_cmd git-lfs + +python3 - <<'PY' +import sys +if not ((3, 12) <= sys.version_info[:2] < (3, 14)): + raise SystemExit( + f"ERROR: Python 3.12 or 3.13 is required; found " + f"{sys.version_info.major}.{sys.version_info.minor}" + ) +PY + +if ! command -v uv >/dev/null 2>&1; then + log "Installing uv" + curl -LsSf https://astral.sh/uv/install.sh | sh + export PATH="${HOME}/.local/bin:${HOME}/.cargo/bin:${PATH}" +fi +need_cmd uv + +mkdir -p "${ROOT_DIR}" + +clone_or_update \ + "${CUGA_AGENT_REPO_URL}" \ + "${CUGA_AGENT_BRANCH}" \ + "${CUGA_AGENT_DIR}" \ + "cuga-agent" + +clone_or_update \ + "${CUGA_EVAL_REPO_URL}" \ + "${CUGA_EVAL_BRANCH}" \ + "${CUGA_EVAL_DIR}" \ + "cuga-eval" + +log "Running cuga-eval/setup_cuga.sh" +cd "${CUGA_EVAL_DIR}" +chmod u+x ./setup_cuga.sh +bash ./setup_cuga.sh + +log "Creating/syncing cuga-eval environment" +uv venv +uv sync + +log "Setting up AppWorld" +git lfs install +chmod u+x ./setup_appworld.sh +bash ./setup_appworld.sh +uv sync --group appworld + +log "Checking PR regression evaluation entrypoint" +if [[ ! -f "${RUN_EVAL_SCRIPT}" ]]; then + cat >&2 </dev/null 2>&1; then + log "Runner already running" +else + log "Starting runner in background" + + nohup ./run.sh > "${GH_RUNNER_DIR}/runner.log" 2>&1 & + echo $! > "${GH_RUNNER_DIR}/runner.pid" +fi + +sleep 3 + +if ! pgrep -f "${GH_RUNNER_DIR}/bin/Runner.Listener" >/dev/null 2>&1; then + echo "ERROR: Runner did not remain active." >&2 + tail -n 50 "${GH_RUNNER_DIR}/runner.log" 2>/dev/null || true + exit 1 +fi + +echo +echo "======================================================================" +echo "PR regression testing VM is ready" +echo "======================================================================" +echo +echo "Repository mode:" +echo " ${REPO_MODE}" +echo +echo "Layout:" +echo " ${ROOT_DIR}/" +echo " ├── cuga-agent/" +echo " ├── cuga-eval/" +echo " │ └── scripts/pr-regression-testing/run-pr-regression-eval.sh" +echo " └── github-runner/" +echo " └── _work/" +echo +echo "Repos:" +echo " cuga-agent: ${CUGA_AGENT_REPO_URL}" +echo " cuga-eval: ${CUGA_EVAL_REPO_URL}" +echo +echo "Runner logs:" +echo " tail -f ${GH_RUNNER_DIR}/runner.log" +echo +echo "Workflow target:" +echo " runs-on: [self-hosted, linux, ${RUNNER_LABELS}]" +echo +echo "Workflow script:" +echo " bash \"${RUN_EVAL_SCRIPT}\" \"\${COMMENT_BODY}\"" +echo +echo "PR command format:" +echo " /run-pr-eval sha= benchmark=appworld provider=rits agent=react"