feat: PR Regression Testing Workflow

benchmarks/appworld/appworld_eval.py

@@ -177,6 +177,14 @@ def _get_registry_base_url() -> str:
     return "http://localhost:8001"
 
 
+def _is_langfuse_tracing_enabled() -> bool:
+    env_value = os.getenv("DYNACONF_ADVANCED_FEATURES__LANGFUSE_TRACING")
+    if env_value is not None:
+        return env_value.strip().lower() in {"1", "true", "yes", "on"}
+
+    return bool(getattr(getattr(settings, "advanced_features", None), "langfuse_tracing", False))
+
+
 async def run_agent_on_task(
     task_id: str,
     experiment_name: str = "api_cuga_agent",
@@ -217,7 +225,7 @@ async def run_agent_on_task(
     # Without this, LangfuseCallbackHandler creates a new root trace for every
     # agent_loop_obj.run() call in the controller while-loop.
     _langfuse = None
-    if _LANGFUSE_AVAILABLE and _get_langfuse_client is not None:
+    if _is_langfuse_tracing_enabled() and _LANGFUSE_AVAILABLE and _get_langfuse_client is not None:
         try:
             _langfuse = _get_langfuse_client()
             _run_uid = uuid.uuid4().hex[:8]
@@ -335,8 +343,9 @@ async def run_agent_on_task(
 
         langfuse_data = None
         # Extract Langfuse data if trace_id is available
-        langfuse_handler = LangfuseTraceHandler(langfuse_trace_id)
-        langfuse_data = await langfuse_handler.get_langfuse_data()
+        if langfuse_trace_id:
+            langfuse_handler = LangfuseTraceHandler(langfuse_trace_id)
+            langfuse_data = await langfuse_handler.get_langfuse_data()
         if langfuse_trace_id:
             task_result.trace_id = langfuse_trace_id
         if langfuse_data:
@@ -469,14 +478,21 @@ def _print_appworld_summary(report: dict):
     success_rate = report.get("success_rate", 0)
     avg_steps = report.get("avg_steps", 0)
     avg_duration = report.get("avg_duration", 0)
+    for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes")
 
-    print("\n" + "=" * 80)
-    print("EVALUATION COMPLETE")
-    print("=" * 80)
+    if for_pr_comment:
+        print("######## REPORT START ########")
+        print("## EVALUATION COMPLETE")
+    else:
+        print("\n" + "=" * 80)
+        print("EVALUATION COMPLETE")
+        print("=" * 80)
     print(f"Total Tasks: {total}")
     print(f"Completed: {completed}/{total} ({success_rate:.1%})")
     print(f"Avg Steps: {avg_steps:.2f}")
     print(f"Avg Duration: {avg_duration:.2f}s")
+    if for_pr_comment:
+        print("######## REPORT END ########")
 
 
 async def main():

benchmarks/appworld/appworld_eval_codeact.py

@@ -96,14 +96,21 @@ def _print_appworld_summary(report: dict):
     success_rate = report.get("success_rate", 0)
     avg_steps = report.get("avg_steps", 0)
     avg_duration = report.get("avg_duration", 0)
+    for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes")
 
-    print("\n" + "=" * 80)
-    print("EVALUATION COMPLETE")
-    print("=" * 80)
+    if for_pr_comment:
+        print("######## REPORT START ########")
+        print("## EVALUATION COMPLETE")
+    else:
+        print("\n" + "=" * 80)
+        print("EVALUATION COMPLETE")
+        print("=" * 80)
     print(f"Total Tasks: {total}")
     print(f"Completed: {completed}/{total} ({success_rate:.1%})")
     print(f"Avg Steps: {avg_steps:.2f}")
     print(f"Avg Duration: {avg_duration:.2f}s")
+    if for_pr_comment:
+        print("######## REPORT END ########")
 
 
 def _parse_prompt_to_messages(input_str: str) -> list[dict[str, Any]]:

benchmarks/appworld/appworld_eval_react.py

@@ -180,14 +180,21 @@ def _print_appworld_summary(report: dict):
     success_rate = report.get("success_rate", 0)
     avg_steps = report.get("avg_steps", 0)
     avg_duration = report.get("avg_duration", 0)
+    for_pr_comment = os.getenv("PR_EVAL_FOR_PR_COMMENT", "").lower() in ("true", "1", "yes")
 
-    print("\n" + "=" * 80)
-    print("EVALUATION COMPLETE")
-    print("=" * 80)
+    if for_pr_comment:
+        print("######## REPORT START ########")
+        print("## EVALUATION COMPLETE")
+    else:
+        print("\n" + "=" * 80)
+        print("EVALUATION COMPLETE")
+        print("=" * 80)
     print(f"Total Tasks: {total}")
     print(f"Completed: {completed}/{total} ({success_rate:.1%})")
     print(f"Avg Steps: {avg_steps:.2f}")
     print(f"Avg Duration: {avg_duration:.2f}s")
+    if for_pr_comment:
+        print("######## REPORT END ########")
 
 
 def _render_initial_prompt(world: AppWorld) -> str:

benchmarks/appworld/eval.sh

@@ -410,8 +410,12 @@ if [ -n "$LATEST_RESULT" ] && [ "${NO_BUNDLE:-false}" != "true" ]; then
 
         # Generate eval report
         REPORT_TMP=$(mktemp /tmp/appworld_eval_report_XXXXXX)
-        uv run --no-sync python -m benchmarks.helpers.compare_report eval \
-            --result-file "$LATEST_RESULT" --output "$REPORT_TMP"
+        REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP")
+        uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}"
+        if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then
+            uv run --no-sync python -m benchmarks.helpers.compare_report \
+                eval --result-file "$LATEST_RESULT" --for-pr-comment --stdout-markdown
+        fi
 
         BUNDLE_ARGS=(assemble --benchmark appworld
             --result-files "$LATEST_RESULT"

benchmarks/bpo/eval.sh

@@ -409,8 +409,12 @@ if [ $EVAL_EXIT_CODE -eq 0 ]; then
 
             # Generate eval report
             REPORT_TMP=$(mktemp /tmp/bpo_eval_report_XXXXXX)
-            uv run --no-sync python -m benchmarks.helpers.compare_report eval \
-                --result-file "$LATEST_RESULT" --output "$REPORT_TMP"
+            REPORT_ARGS=(eval --result-file "$LATEST_RESULT" --output "$REPORT_TMP")
+            uv run --no-sync python -m benchmarks.helpers.compare_report "${REPORT_ARGS[@]}"
+            if [[ "${PR_EVAL_FOR_PR_COMMENT:-false}" == "true" ]]; then
+                uv run --no-sync python -m benchmarks.helpers.compare_report \
+                    eval --result-file "$LATEST_RESULT" --for-pr-comment --stdout-markdown
+            fi
 
             BUNDLE_ARGS=(assemble --benchmark bpo
                 --result-files "$LATEST_RESULT"

benchmarks/helpers/compare_report.py

@@ -1516,15 +1516,16 @@ def _bucket_m3_tasks(tasks: dict) -> tuple:
     return rows, True
 
 
-def generate_eval_report(result_file: str, markdown: bool = True) -> str:
+def generate_eval_report(result_file: str, markdown: bool = True, for_pr_comment: bool = False) -> str:
     """Generate a single-evaluation-run report.
 
     When the result file is M3-shaped (each task has ``m3_task_id`` + ``domain``),
     tasks are grouped per (capability, domain) with a 1-N ordinal so the table
     rows are readable instead of an unattributed UUID. When ``markdown=False``,
     the same content is rendered as a plain-text monospace table for terminals
     (compare.sh's stdout path); ``markdown=True`` (default) is what gets saved
-    into the bundle's report.md.
+    into the bundle's report.md. ``for_pr_comment`` is an explicit opt-in for
+    the PR-regression workflow's collapsible, sentinel-wrapped markdown.
     """
     parsed = parse_result_file(result_file)
     rows, grouped = _bucket_m3_tasks(parsed["tasks"])
@@ -1536,9 +1537,17 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str:
     fence_open = (lambda: "```text") if markdown else (lambda: "")
     fence_close = (lambda: "```") if markdown else (lambda: "")
 
-    lines = [h1("Evaluation Report"), ""]
-    lines.append(h2("Summary"))
-    lines.append("")
+    if markdown and for_pr_comment:
+        lines = ["######## REPORT START ########"]
+        lines.append("<details>")
+        lines.append("<summary>Evaluation Report</summary>")
+        lines.append("")
+        lines.append(h2("Summary"))
+        lines.append("")
+    else:
+        lines = [h1("Evaluation Report"), ""]
+        lines.append(h2("Summary"))
+        lines.append("")
     if markdown:
         lines.append(f"- **Pass@1**: {parsed['passed']}/{parsed['total']} ({parsed['rate']:.1%})")
         lines.append(f"- **Total Tokens**: {_fmt(parsed['tokens'])}")
@@ -1825,6 +1834,10 @@ def generate_eval_report(result_file: str, markdown: bool = True) -> str:
             markdown=markdown,
         )
     )
+    if markdown and for_pr_comment:
+        lines.append("")
+        lines.append("</details>")
+        lines.append("######## REPORT END ########")
 
     return "\n".join(lines)
 
@@ -1839,8 +1852,10 @@ def main():
             parser.add_argument("command")  # consume "eval"
             parser.add_argument("--result-file", required=True)
             parser.add_argument("--output", "-o", default=None)
+            parser.add_argument("--for-pr-comment", action="store_true")
+            parser.add_argument("--stdout-markdown", action="store_true")
             args = parser.parse_args()
-            report = generate_eval_report(args.result_file)
+            report = generate_eval_report(args.result_file, for_pr_comment=args.for_pr_comment)
         else:
             parser = argparse.ArgumentParser(description="Generate comparison report")
             parser.add_argument("command")  # consume "compare"
@@ -1862,14 +1877,15 @@ def main():
     # the canonical bundle location at the end of the run, which is what the
     # user actually wants to navigate to.
     if args.output:
-        # Compare-mode and eval-mode both produce markdown for the saved file
-        # and re-render plain text for the terminal.
         if "command" in args and getattr(args, "command", None) == "eval":
-            plain = generate_eval_report(args.result_file, markdown=False)
+            if getattr(args, "for_pr_comment", False) or getattr(args, "stdout_markdown", False):
+                stdout_report = report
+            else:
+                stdout_report = generate_eval_report(args.result_file, markdown=False)
         else:
-            plain = generate_report(config_results, markdown=False)
+            stdout_report = generate_report(config_results, markdown=False)
         Path(args.output).write_text(report)
-        print(plain)
+        print(stdout_report)
     else:
         # No file requested — just print whatever generate_* produced.
         print(report)

benchmarks/helpers/react_agent.py

@@ -25,6 +25,14 @@
 from langchain_core.messages import HumanMessage
 from loguru import logger
 
+try:
+    from langchain_openai import ChatOpenAI as _ChatOpenAI
+except ImportError as exc:
+    _ChatOpenAI = None
+    _CHAT_OPENAI_IMPORT_ERROR = exc
+else:
+    _CHAT_OPENAI_IMPORT_ERROR = None
+
 TOOL_BLOCK_RE = re.compile(r"```json\s*(.*?)\s*```", re.DOTALL | re.IGNORECASE)
 PROMPT_ROLE_RE = re.compile(r"(USER|ASSISTANT|SYSTEM):\n", re.IGNORECASE)
 PROMPT_PATH = Path(__file__).with_name("react_agent_prompt.txt")
@@ -44,6 +52,43 @@ class ReactInvokeResult:
     react_steps: int = 0
 
 
+def _require_chat_openai():
+    if _ChatOpenAI is None:
+        raise RuntimeError(
+            "langchain-openai is required for React agent with OpenAI-compatible or RITS providers. "
+            "Install with: pip install langchain-openai"
+        ) from _CHAT_OPENAI_IMPORT_ERROR
+    return _ChatOpenAI
+
+
+class ChatRits(_ChatOpenAI if _ChatOpenAI is not None else object):
+    """RITS chat model integration using langchain-openai."""
+
+    def __init__(self, config: dict[str, Any]):
+        _require_chat_openai()
+        model_name = config.get("model_name", "openai/gpt-oss-120b-a100")
+        end_point = (
+            config.get("end_point")
+            or "https://inference-3scale-apicast-production.apps.rits.fmaas.res.ibm.com/gpt-oss-120b-a100"
+        )
+        rits_api_key = config.get("api_key") or os.getenv("RITS_API_KEY")
+        if rits_api_key is None:
+            raise ValueError("RITS_API_KEY is required for settings.rits.toml")
+
+        params = config.get("params", {})
+        base_url = end_point.rstrip("/")
+        if not base_url.endswith("/v1"):
+            base_url = f"{base_url}/v1"
+        rits_config: dict[str, Any] = {
+            "model": model_name,
+            "api_key": "/",
+            "default_headers": {"RITS_API_KEY": rits_api_key},
+            "base_url": base_url,
+        }
+        rits_config.update(params)
+        super().__init__(**rits_config)
+
+
 class GenericReactAgent:
     """Prompt-template driven generic ReAct agent over benchmark tools."""
 
@@ -149,14 +194,6 @@ def _create_llm(self) -> Any:
             )
 
         elif settings_config == "settings.openai.toml":
-            try:
-                from langchain_openai import ChatOpenAI
-            except ImportError as exc:
-                raise RuntimeError(
-                    "langchain-openai is required for React agent with OpenAI. "
-                    "Install with: pip install langchain-openai"
-                ) from exc
-
             disable_ssl = os.getenv("CUGA_DISABLE_SSL", "").lower() in ("true", "1", "yes")
             ssl_verify = (
                 os.getenv("OPENAI_SSL_VERIFY", "true").lower() not in ("false", "0", "no") and not disable_ssl
@@ -181,12 +218,22 @@ def _create_llm(self) -> Any:
                 llm_kwargs["http_async_client"] = httpx.AsyncClient(verify=False)  # noqa: S501  # nosec B501 — same
                 logger.info("SSL verification disabled for ChatOpenAI")
 
-            return ChatOpenAI(**llm_kwargs)
+            return _require_chat_openai()(**llm_kwargs)
+
+        elif settings_config in ("settings.rits.toml", "settings.rits.proxy.toml"):
+            api_base = os.getenv("RITS_BASE_URL") or os.getenv("LITE_LLM_URL") or os.getenv("OPENAI_BASE_URL")
+            api_key = os.getenv("RITS_API_KEY") or os.getenv("LITE_LLM_KEY") or os.getenv("OPENAI_API_KEY")
+            model_name = os.getenv("MODEL_NAME") or self.model or "openai/gpt-oss-120b-a100"
+            logger.info(
+                f"RITS config: model={model_name}, api_base={api_base}, api_key_present={bool(api_key)}"
+            )
+            return ChatRits(config={"model_name": model_name, "end_point": api_base, "api_key": api_key})
 
         else:
             raise RuntimeError(
                 "Unsupported AGENT_SETTING_CONFIG for React agent. "
-                "Expected 'settings.groq.toml' or 'settings.openai.toml'."
+                "Expected 'settings.groq.toml', 'settings.openai.toml', "
+                "'settings.rits.toml', or 'settings.rits.proxy.toml'."
             )
 
     def _get_llm(self) -> Any: