From e67bdaeeba445625c20276751c649472767bec92 Mon Sep 17 00:00:00 2001 From: DB Lee Date: Thu, 23 Apr 2026 12:37:10 -0700 Subject: [PATCH 01/15] fix: gracefully skip cloud-only evaluators in local execution mode When running evaluations locally (hosting: local, execution_mode: local), some Foundry evaluators (TaskCompletionEvaluator, ToolSelectionEvaluator, ToolInputAccuracyEvaluator, TaskNavigationEfficiencyEvaluator) are only available via the Foundry Cloud Evaluation API (builtin.*) and do not exist in the azure-ai-evaluation Python SDK. Previously, using agent_workflow_baseline bundle with local execution crashed with 'Unknown built-in Foundry evaluator class'. Now these evaluators are gracefully skipped with clear warning messages, and threshold checks exclude skipped evaluators from validation. Changes: - eval_engine.py: Add _CloudOnlyEvaluatorError sentinel; catch AttributeError in _load_foundry_evaluator_callable and raise cloud-only error; catch in _build_foundry_evaluator_runtimes and warn+skip - runner.py: Change _validate_enabled_evaluators_scored to warn instead of raise for missing scores; skip threshold checks for evaluators with no scores in _evaluate_item_thresholds and _summarize_thresholds_from_items Tested against: - Local callable with agent_workflow_baseline: 3 evaluators skipped, 3 scored successfully - Cloud model-direct: all evaluators submitted to Foundry API - Cloud Foundry agent: all 6 evaluators scored via cloud API Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agentops/backends/eval_engine.py | 41 ++++++++++++++++++++++------ src/agentops/services/runner.py | 22 +++++++++++---- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/agentops/backends/eval_engine.py b/src/agentops/backends/eval_engine.py index 544bef3e..40b5a5fc 100644 --- a/src/agentops/backends/eval_engine.py +++ b/src/agentops/backends/eval_engine.py @@ -23,6 +23,16 @@ logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Cloud-only evaluator sentinel +# --------------------------------------------------------------------------- + + +class _CloudOnlyEvaluatorError(Exception): + """Raised when an evaluator is only available via Foundry Cloud Evaluation.""" + + # --------------------------------------------------------------------------- # Credential help (shared by _default_credential and _acquire_token) # --------------------------------------------------------------------------- @@ -609,8 +619,12 @@ def _load_foundry_evaluator_callable( "Install with: pip install azure-ai-evaluation" ) from exc except AttributeError as exc: - raise ValueError( - f"Unknown built-in Foundry evaluator class: {class_name}" + raise _CloudOnlyEvaluatorError( + f"Evaluator '{class_name}' is not available in the local " + f"azure-ai-evaluation SDK. It may only be available via " + f"Foundry Cloud Evaluation (builtin.{_to_builtin_evaluator_name(class_name)}). " + f"Use 'hosting: foundry' with 'execution_mode: remote' to " + f"run this evaluator, or disable it for local runs." ) from exc return _instantiate_evaluator_symbol( @@ -691,12 +705,23 @@ def _build_foundry_evaluator_runtimes( ) score_keys = score_keys_raw - evaluator_callable = _load_foundry_evaluator_callable( - evaluator_name=evaluator.name, - evaluator_config=config, - fallback_endpoint=fallback_endpoint, - fallback_deployment=fallback_deployment, - ) + try: + evaluator_callable = _load_foundry_evaluator_callable( + evaluator_name=evaluator.name, + evaluator_config=config, + fallback_endpoint=fallback_endpoint, + fallback_deployment=fallback_deployment, + ) + except _CloudOnlyEvaluatorError: + logger.warning( + "Skipping evaluator '%s' — not available in the local " + "azure-ai-evaluation SDK. This evaluator is only supported " + "via Foundry Cloud Evaluation (hosting: foundry, " + "execution_mode: remote). It will be ignored for this " + "local run.", + evaluator.name, + ) + continue runtimes.append( FoundryEvaluatorRuntime( diff --git a/src/agentops/services/runner.py b/src/agentops/services/runner.py index 1124cbcf..d5b6529c 100644 --- a/src/agentops/services/runner.py +++ b/src/agentops/services/runner.py @@ -3,12 +3,15 @@ from __future__ import annotations import json +import logging import shutil from dataclasses import dataclass from datetime import datetime from pathlib import Path from agentops.backends.base import Backend, BackendRunContext + +logger = logging.getLogger(__name__) from agentops.core.config_loader import ( load_bundle_config, load_dataset_config, @@ -218,9 +221,9 @@ def _evaluate_item_thresholds( threshold_results: list[ItemThresholdEvaluationResult] = [] for rule in threshold_rules: if rule.evaluator not in row_values: - raise ValueError( - f"Missing evaluator score '{rule.evaluator}' for row {row.row_index}" - ) + # Evaluator may be cloud-only and was skipped during local + # execution — silently skip its threshold check. + continue threshold_results.append( _evaluate_threshold_against_value( @@ -266,8 +269,12 @@ def _validate_enabled_evaluators_scored( missing = [name for name in evaluator_names if name not in scored_names] if missing: - raise ValueError( - "Missing scores for enabled evaluators: " + ", ".join(sorted(missing)) + logger.warning( + "Some enabled evaluators did not produce scores and will be " + "excluded from threshold checks: %s. These evaluators may " + "only be available via Foundry Cloud Evaluation " + "(hosting: foundry, execution_mode: remote).", + ", ".join(sorted(missing)), ) @@ -291,6 +298,11 @@ def _summarize_thresholds_from_items( ): rule_results.append(threshold_result) + # Skip threshold rules for evaluators that produced no scores + # (e.g., cloud-only evaluators skipped during local execution). + if not rule_results: + continue + passed_items = sum(1 for result in rule_results if result.passed) passed = bool(rule_results) and passed_items == len(rule_results) From 07b33bc99f7d596c8aeabc101968fd46e210f17b Mon Sep 17 00:00:00 2001 From: DB Lee Date: Thu, 23 Apr 2026 12:49:23 -0700 Subject: [PATCH 02/15] feat: add run-agent-local template, capture tool_calls, suppress SDK noise MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up items from issue #79 E2E validation: 1. Add run-agent-local.yaml template to agentops init scaffold — combines type:agent + hosting:local + framework:agent_framework with agent_workflow_baseline bundle for local workflow testing. 2. Document TaskAdherenceEvaluator behavior — SDK expects multi-turn conversation format (list of message dicts with role/content); single-turn plain text inputs produce low scores. Added note to agent_workflow_baseline.yaml bundle description. 3. Suppress 'Conversation history could not be parsed' warning — add logging filter for azure-ai-evaluation SDK loggers that emit this warning on every single-turn row (expected, harmless). 4. Capture callable return tool_calls in results — extract tool_calls from callable/subprocess output and include in backend_metrics.json row_metrics for post-analysis. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agentops/backends/eval_engine.py | 28 +++++++++++++++++ .../backends/local_adapter_backend.py | 3 ++ src/agentops/services/initializer.py | 1 + .../bundles/agent_workflow_baseline.yaml | 9 ++++++ src/agentops/templates/run-agent-local.yaml | 30 +++++++++++++++++++ tests/unit/test_initializer.py | 2 +- 6 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 src/agentops/templates/run-agent-local.yaml diff --git a/src/agentops/backends/eval_engine.py b/src/agentops/backends/eval_engine.py index 40b5a5fc..fbe5e80c 100644 --- a/src/agentops/backends/eval_engine.py +++ b/src/agentops/backends/eval_engine.py @@ -24,6 +24,34 @@ logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Suppress noisy SDK warnings for single-turn evaluation inputs +# --------------------------------------------------------------------------- + +class _ConversationHistoryFilter(logging.Filter): + """Suppress 'Conversation history could not be parsed' from azure-ai-evaluation. + + This warning fires on every single-turn evaluation row because plain-text + inputs are not in conversation-list format. It is expected and harmless. + """ + + def filter(self, record: logging.LogRecord) -> bool: + return "Conversation history could not be parsed" not in record.getMessage() + + +# Apply filter to SDK loggers that emit the warning. +# Each evaluator module passes its own logger to reformat_conversation_history(). +for _sdk_logger_name in ( + "azure.ai.evaluation._common.utils", + "azure.ai.evaluation._evaluators._task_adherence._task_adherence", + "azure.ai.evaluation._evaluators._intent_resolution._intent_resolution", + "azure.ai.evaluation._evaluators._task_completion._task_completion", + "azure.ai.evaluation._evaluators._tool_call_accuracy._tool_call_accuracy", + "azure.ai.evaluation", +): + logging.getLogger(_sdk_logger_name).addFilter(_ConversationHistoryFilter()) + + # --------------------------------------------------------------------------- # Cloud-only evaluator sentinel # --------------------------------------------------------------------------- diff --git a/src/agentops/backends/local_adapter_backend.py b/src/agentops/backends/local_adapter_backend.py index 9eae1b62..40225e76 100644 --- a/src/agentops/backends/local_adapter_backend.py +++ b/src/agentops/backends/local_adapter_backend.py @@ -222,6 +222,7 @@ def execute(self, context: BackendRunContext) -> BackendExecutionResult: prediction_text = _normalize_text( result.get("response", "") ) + returned_tool_calls = result.get("tool_calls") set_agent_invoke_result(invoke_span) except Exception as exc: # noqa: BLE001 stderr_lines.append(f"row={index} error={exc!s}") @@ -267,6 +268,7 @@ def execute(self, context: BackendRunContext) -> BackendExecutionResult: prediction_text = _normalize_text( adapter_output.get("response", "") ) + returned_tool_calls = adapter_output.get("tool_calls") set_agent_invoke_result(invoke_span) except subprocess.TimeoutExpired: stderr_lines.append(f"row={index} error=adapter timeout") @@ -333,6 +335,7 @@ def execute(self, context: BackendRunContext) -> BackendExecutionResult: "input": prompt_text, "response": prediction_text, "context": row.get("context"), + "tool_calls": returned_tool_calls, "metrics": row_metric_entries, } ) diff --git a/src/agentops/services/initializer.py b/src/agentops/services/initializer.py index 28e3d402..16e3e9b3 100644 --- a/src/agentops/services/initializer.py +++ b/src/agentops/services/initializer.py @@ -23,6 +23,7 @@ class InitResult: "run.yaml", "run-rag.yaml", "run-agent.yaml", + "run-agent-local.yaml", "run-http-model.yaml", "run-http-rag.yaml", "run-http-agent-tools.yaml", diff --git a/src/agentops/templates/bundles/agent_workflow_baseline.yaml b/src/agentops/templates/bundles/agent_workflow_baseline.yaml index 03d27589..ea6e015b 100644 --- a/src/agentops/templates/bundles/agent_workflow_baseline.yaml +++ b/src/agentops/templates/bundles/agent_workflow_baseline.yaml @@ -5,6 +5,15 @@ description: > Measures task completion, tool call accuracy, intent resolution, task adherence, tool selection, and tool input accuracy using AI-assisted evaluators from the Foundry evaluation suite. + + Note: TaskCompletionEvaluator, ToolSelectionEvaluator, and + ToolInputAccuracyEvaluator are only available via Foundry Cloud + Evaluation and will be gracefully skipped in local execution mode. + + Note: TaskAdherenceEvaluator works best with multi-turn conversation + format (list of message dicts with role/content). Single-turn plain + text inputs may produce low scores because the evaluator cannot assess + procedural adherence without conversation context. evaluators: - name: TaskCompletionEvaluator source: foundry diff --git a/src/agentops/templates/run-agent-local.yaml b/src/agentops/templates/run-agent-local.yaml new file mode 100644 index 00000000..0b600ecc --- /dev/null +++ b/src/agentops/templates/run-agent-local.yaml @@ -0,0 +1,30 @@ +version: 1 + +# Local agent workflow evaluation via callable adapter. +# Evaluates a local Python function that implements a multi-agent workflow. +# +# The callable must accept (input_text: str, context: dict) -> dict +# and return at least {"response": "..."}. For tool-aware evaluation, +# also return {"tool_calls": [{"name": "...", "arguments": {...}}]}. +# +# Cloud-only evaluators (TaskCompletionEvaluator, ToolSelectionEvaluator, +# ToolInputAccuracyEvaluator) will be gracefully skipped in local mode. +# Use 'hosting: foundry' with 'execution_mode: remote' to run all evaluators. + +target: + type: agent + hosting: local + execution_mode: local + framework: agent_framework + local: + # Format: module_path:function_name + # The function must accept (input_text: str, context: dict) -> dict + callable: callable_adapter:run_evaluation +bundle: + name: agent_workflow_baseline +dataset: + name: smoke-agent-tools +execution: + timeout_seconds: 300 +output: + write_report: true diff --git a/tests/unit/test_initializer.py b/tests/unit/test_initializer.py index 67357b48..1952125b 100644 --- a/tests/unit/test_initializer.py +++ b/tests/unit/test_initializer.py @@ -42,7 +42,7 @@ def test_init_creates_expected_files(tmp_path: Path) -> None: assert (tmp_path / ".agentops" / "data" / "smoke-conversational.jsonl").is_file() assert (tmp_path / ".agentops" / "workflows" / "agentops-eval.yml").is_file() - assert len(result.created_files) == 24 + assert len(result.created_files) == 25 assert len(result.overwritten_files) == 0 run_config = load_yaml(tmp_path / ".agentops" / "run.yaml") From 215b486f03a881a19b84584fa2f2ebf5934970c0 Mon Sep 17 00:00:00 2001 From: DB Lee Date: Thu, 23 Apr 2026 13:14:44 -0700 Subject: [PATCH 03/15] feat: add Agent Framework callable adapter for local agent evaluation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add agent_framework_adapter.py — a dedicated callable adapter that invokes Azure AI Foundry agents locally via the Foundry REST API. Supports two agent ID patterns: - 'asst_*' IDs: Threads/Runs API (create thread → message → run → poll) - Named agents (e.g. 'FoundryAgent'): Responses API with agent_reference The adapter extracts response text and tool_calls from the API response, matching the contract expected by agent_workflow_baseline evaluators. Also updates: - callable_adapter.py: Add Option 4 pointing to agent_framework_adapter - run-agent-local.yaml: Document both adapter options - initializer.py: Include adapter in agentops init scaffold (26 files) Tested E2E against real FoundryAgent via Responses API — all 5 rows processed with evaluator scores returned. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agentops/services/initializer.py | 1 + .../templates/agent_framework_adapter.py | 267 ++++++++++++++++++ src/agentops/templates/callable_adapter.py | 8 + src/agentops/templates/run-agent-local.yaml | 17 +- tests/unit/test_initializer.py | 2 +- 5 files changed, 289 insertions(+), 6 deletions(-) create mode 100644 src/agentops/templates/agent_framework_adapter.py diff --git a/src/agentops/services/initializer.py b/src/agentops/services/initializer.py index 16e3e9b3..2226e711 100644 --- a/src/agentops/services/initializer.py +++ b/src/agentops/services/initializer.py @@ -29,6 +29,7 @@ class InitResult: "run-http-agent-tools.yaml", "run-callable.yaml", "callable_adapter.py", + "agent_framework_adapter.py", ".gitignore", "bundles/model_quality_baseline.yaml", "bundles/rag_quality_baseline.yaml", diff --git a/src/agentops/templates/agent_framework_adapter.py b/src/agentops/templates/agent_framework_adapter.py new file mode 100644 index 00000000..f6517eb2 --- /dev/null +++ b/src/agentops/templates/agent_framework_adapter.py @@ -0,0 +1,267 @@ +"""Agent Framework callable adapter for AgentOps evaluations. + +Wraps an Azure AI Foundry Agent (created via the Agent Framework SDK) +so it can be evaluated locally through the AgentOps callable interface. + +This adapter creates a thread, sends the input as a user message, runs the +agent, polls for completion, and returns the response along with any tool +calls made during execution. + +Prerequisites: + pip install azure-ai-projects azure-identity + +Environment variables: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT — your Foundry project endpoint + e.g. https://.services.ai.azure.com/api/projects/ + AGENT_ID — the agent ID (e.g. "asst_abc123" or "my-agent:3") + +Usage in run.yaml: + target: + type: agent + hosting: local + execution_mode: local + framework: agent_framework + local: + callable: agent_framework_adapter:run_evaluation +""" +from __future__ import annotations + +import json +import logging +import os +import time +from typing import Any + +logger = logging.getLogger(__name__) + +# ── Configuration ────────────────────────────────────────────────────── +PROJECT_ENDPOINT = os.environ.get("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", "") +AGENT_ID = os.environ.get("AGENT_ID", "") +POLL_INTERVAL = float(os.environ.get("AGENT_POLL_INTERVAL", "2")) +MAX_POLL_ATTEMPTS = int(os.environ.get("AGENT_MAX_POLL_ATTEMPTS", "120")) + +# Lazy-initialized client +_client = None + + +def _get_client(): + """Lazily initialize the AIProjectClient.""" + global _client + if _client is None: + if not PROJECT_ENDPOINT: + raise ValueError( + "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT must be set. " + "Example: https://.services.ai.azure.com/api/projects/" + ) + from azure.ai.projects import AIProjectClient + from azure.identity import DefaultAzureCredential + + _client = AIProjectClient( + endpoint=PROJECT_ENDPOINT, + credential=DefaultAzureCredential(), + ) + return _client + + +def _invoke_via_threads_api( + input_text: str, + agent_id: str, + api_version: str = "2025-05-01", +) -> dict[str, Any]: + """Invoke an agent using the Threads/Runs REST API pattern. + + Used for agents with 'asst_*' IDs (Agent Service format). + Flow: create thread → add message → create run → poll → get messages. + """ + from azure.identity import DefaultAzureCredential + + credential = DefaultAzureCredential() + token = credential.get_token("https://ai.azure.com/.default").token + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + } + base_url = PROJECT_ENDPOINT + + import urllib.request + + def _api_call(method: str, path: str, body: dict | None = None) -> dict: + url = f"{base_url}{path}?api-version={api_version}" + data = json.dumps(body).encode() if body else None + req = urllib.request.Request(url, data=data, headers=headers, method=method) + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()) + + # Step 1: Create thread + thread = _api_call("POST", "/threads", {}) + thread_id = thread["id"] + + # Step 2: Add user message + _api_call("POST", f"/threads/{thread_id}/messages", { + "role": "user", + "content": input_text, + }) + + # Step 3: Create run + run = _api_call("POST", f"/threads/{thread_id}/runs", { + "assistant_id": agent_id, + }) + run_id = run["id"] + + # Step 4: Poll for completion + terminal_success = {"completed"} + terminal_failure = {"failed", "cancelled", "expired", "requires_action"} + + for _ in range(MAX_POLL_ATTEMPTS): + status_resp = _api_call("GET", f"/threads/{thread_id}/runs/{run_id}") + status = status_resp.get("status", "") + + if status in terminal_success: + break + if status in terminal_failure: + raise RuntimeError(f"Agent run ended with status '{status}'") + time.sleep(POLL_INTERVAL) + else: + raise TimeoutError("Timed out waiting for agent run completion") + + # Step 5: Get messages and extract response + messages = _api_call("GET", f"/threads/{thread_id}/messages") + + response_text = "" + tool_calls = [] + + for msg in messages.get("data", []): + if msg.get("role") != "assistant": + continue + for content_block in msg.get("content", []): + if content_block.get("type") == "text": + text_val = content_block.get("text", {}) + if isinstance(text_val, dict): + response_text += text_val.get("value", "") + else: + response_text += str(text_val) + + # Extract tool calls from run steps if available + try: + steps = _api_call("GET", f"/threads/{thread_id}/runs/{run_id}/steps") + for step in steps.get("data", []): + step_details = step.get("step_details", {}) + if step_details.get("type") == "tool_calls": + for tc in step_details.get("tool_calls", []): + if tc.get("type") == "function": + fn = tc.get("function", {}) + tool_calls.append({ + "name": fn.get("name", ""), + "arguments": json.loads(fn.get("arguments", "{}")), + }) + except Exception: + logger.debug("Could not retrieve run steps for tool calls", exc_info=True) + + return {"response": response_text.strip(), "tool_calls": tool_calls} + + +def _invoke_via_responses_api( + input_text: str, + agent_id: str, + model: str | None = None, + api_version: str = "2025-05-01", +) -> dict[str, Any]: + """Invoke an agent using the Responses API with agent_reference. + + Used for named agents (e.g. 'FoundryAgent', 'my-agent:3') that are + not in 'asst_*' format. Requires a model deployment name. + """ + from azure.identity import DefaultAzureCredential + + credential = DefaultAzureCredential() + token = credential.get_token("https://ai.azure.com/.default").token + + model_name = model or os.environ.get("AZURE_OPENAI_DEPLOYMENT", "") + if not model_name: + raise ValueError( + "Agent reference mode requires a model deployment name. " + "Set AZURE_OPENAI_DEPLOYMENT or AGENT_MODEL env var." + ) + + # Parse agent name and optional version (e.g., "my-agent:3") + agent_name = agent_id + agent_version = None + if ":" in agent_id: + agent_name, agent_version = agent_id.split(":", 1) + agent_name = agent_name.strip() + agent_version = agent_version.strip() or None + + agent_reference: dict[str, Any] = { + "type": "agent_reference", + "name": agent_name, + } + if agent_version: + agent_reference["version"] = agent_version + + body = json.dumps({ + "model": model_name, + "input": [{"role": "user", "content": input_text}], + "agent_reference": agent_reference, + }).encode() + + import urllib.request + + url = f"{PROJECT_ENDPOINT}/openai/v1/responses" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + } + + req = urllib.request.Request(url, data=body, headers=headers, method="POST") + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read()) + + # Extract response text and tool calls from Responses API output + response_text = "" + tool_calls = [] + + for item in data.get("output", []): + if item.get("type") == "message": + for content_block in item.get("content", []): + if content_block.get("type") == "output_text": + response_text += content_block.get("text", "") + elif item.get("type") == "function_call": + tool_calls.append({ + "name": item.get("name", ""), + "arguments": json.loads(item.get("arguments", "{}")), + }) + + return {"response": response_text.strip(), "tool_calls": tool_calls} + + +def run_evaluation(input_text: str, context: dict) -> dict: + """Evaluate a single input against an Agent Framework agent. + + This is the entry point called by AgentOps local adapter backend. + It invokes the configured Foundry agent and returns the response + along with any tool calls made during execution. + + Supports two agent ID formats: + - 'asst_*': Uses the Threads/Runs API (Agent Service) + - Named agents (e.g. 'FoundryAgent', 'my-agent:3'): Uses the + Responses API with agent_reference + + Args: + input_text: The user prompt from the dataset row. + context: The full dataset row (all fields including tool_definitions). + + Returns: + dict with at least {"response": "..."} and optionally {"tool_calls": [...]}. + """ + agent_id = AGENT_ID + if not agent_id: + raise ValueError( + "AGENT_ID must be set. Example: AGENT_ID=asst_abc123 or AGENT_ID=my-agent:3" + ) + + if agent_id.startswith("asst_"): + return _invoke_via_threads_api(input_text, agent_id) + else: + model = os.environ.get("AGENT_MODEL") or os.environ.get("AZURE_OPENAI_DEPLOYMENT") + return _invoke_via_responses_api(input_text, agent_id, model=model) diff --git a/src/agentops/templates/callable_adapter.py b/src/agentops/templates/callable_adapter.py index f66c400b..2c7d7549 100644 --- a/src/agentops/templates/callable_adapter.py +++ b/src/agentops/templates/callable_adapter.py @@ -103,6 +103,14 @@ def run_evaluation(input_text: str, context: dict) -> dict: # result = workflow.invoke(input_text) # return {"response": result.output} + # --- Option 4: Agent Framework (Azure AI Foundry agent) --- + # For Agent Framework agents, use the dedicated adapter template instead: + # + # callable: agent_framework_adapter:run_evaluation + # + # Set AZURE_AI_FOUNDRY_PROJECT_ENDPOINT and AGENT_ID environment variables. + # See agent_framework_adapter.py for details. + # --- Context sanitization (RAG scenarios) --- # If your dataset has a "context" field with raw document content, # clean it before returning: diff --git a/src/agentops/templates/run-agent-local.yaml b/src/agentops/templates/run-agent-local.yaml index 0b600ecc..915b023a 100644 --- a/src/agentops/templates/run-agent-local.yaml +++ b/src/agentops/templates/run-agent-local.yaml @@ -3,9 +3,15 @@ version: 1 # Local agent workflow evaluation via callable adapter. # Evaluates a local Python function that implements a multi-agent workflow. # -# The callable must accept (input_text: str, context: dict) -> dict -# and return at least {"response": "..."}. For tool-aware evaluation, -# also return {"tool_calls": [{"name": "...", "arguments": {...}}]}. +# Two adapter options: +# +# 1. agent_framework_adapter:run_evaluation +# For Azure AI Foundry agents (Agent Framework SDK). +# Requires: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT and AGENT_ID env vars. +# pip install azure-ai-projects azure-identity +# +# 2. callable_adapter:run_evaluation +# For custom agents — HTTP, direct Python, or any callable. # # Cloud-only evaluators (TaskCompletionEvaluator, ToolSelectionEvaluator, # ToolInputAccuracyEvaluator) will be gracefully skipped in local mode. @@ -17,8 +23,9 @@ target: execution_mode: local framework: agent_framework local: - # Format: module_path:function_name - # The function must accept (input_text: str, context: dict) -> dict + # Option 1: Agent Framework adapter (Azure AI Foundry agents) + # callable: agent_framework_adapter:run_evaluation + # Option 2: Custom callable adapter (HTTP, direct Python, etc.) callable: callable_adapter:run_evaluation bundle: name: agent_workflow_baseline diff --git a/tests/unit/test_initializer.py b/tests/unit/test_initializer.py index 1952125b..e2691012 100644 --- a/tests/unit/test_initializer.py +++ b/tests/unit/test_initializer.py @@ -42,7 +42,7 @@ def test_init_creates_expected_files(tmp_path: Path) -> None: assert (tmp_path / ".agentops" / "data" / "smoke-conversational.jsonl").is_file() assert (tmp_path / ".agentops" / "workflows" / "agentops-eval.yml").is_file() - assert len(result.created_files) == 25 + assert len(result.created_files) == 26 assert len(result.overwritten_files) == 0 run_config = load_yaml(tmp_path / ".agentops" / "run.yaml") From 4fb1836cfdb96a171b1d97a118a22f778027d4f8 Mon Sep 17 00:00:00 2001 From: DB Lee Date: Thu, 23 Apr 2026 13:21:09 -0700 Subject: [PATCH 04/15] =?UTF-8?q?fix:=20resolve=20E402=20lint=20error=20?= =?UTF-8?q?=E2=80=94=20move=20logger=20after=20imports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agentops/services/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agentops/services/runner.py b/src/agentops/services/runner.py index d5b6529c..18319af9 100644 --- a/src/agentops/services/runner.py +++ b/src/agentops/services/runner.py @@ -10,8 +10,6 @@ from pathlib import Path from agentops.backends.base import Backend, BackendRunContext - -logger = logging.getLogger(__name__) from agentops.core.config_loader import ( load_bundle_config, load_dataset_config, @@ -45,6 +43,8 @@ shutdown as shutdown_tracing, ) +logger = logging.getLogger(__name__) + @dataclass(frozen=True) class EvalRunServiceResult: From 99da323e2ee299dea33d4caad550826b170d934d Mon Sep 17 00:00:00 2001 From: DB Lee Date: Thu, 23 Apr 2026 14:51:21 -0700 Subject: [PATCH 05/15] feat: add multi-agent workflow callable for Agent Framework evaluation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add multi_agent_workflow.py — a sample multi-agent orchestration callable that uses Azure AI Agent Framework SDK (Assistants API) to dynamically create and coordinate agents: Router Agent → analyzes query, selects specialist WeatherSpecialist → get_weather tool FinanceSpecialist → convert_currency, calculate_compound_interest tools SearchSpecialist → search_news, search_flights tools Agents are created dynamically per evaluation row and cleaned up after use. Tool calls are captured from requires_action/submit flow and returned in the callable response. E2E validated: all 5 smoke-agent-tools rows processed with correct routing (weather/finance/search), tool invocations with correct arguments, and evaluator scores (ToolCallAccuracy: 3.0, IntentResolution: 3.8). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agentops/services/initializer.py | 1 + .../templates/multi_agent_workflow.py | 331 ++++++++++++++++++ tests/unit/test_initializer.py | 2 +- 3 files changed, 333 insertions(+), 1 deletion(-) create mode 100644 src/agentops/templates/multi_agent_workflow.py diff --git a/src/agentops/services/initializer.py b/src/agentops/services/initializer.py index 2226e711..5f6e3559 100644 --- a/src/agentops/services/initializer.py +++ b/src/agentops/services/initializer.py @@ -30,6 +30,7 @@ class InitResult: "run-callable.yaml", "callable_adapter.py", "agent_framework_adapter.py", + "multi_agent_workflow.py", ".gitignore", "bundles/model_quality_baseline.yaml", "bundles/rag_quality_baseline.yaml", diff --git a/src/agentops/templates/multi_agent_workflow.py b/src/agentops/templates/multi_agent_workflow.py new file mode 100644 index 00000000..b22763b8 --- /dev/null +++ b/src/agentops/templates/multi_agent_workflow.py @@ -0,0 +1,331 @@ +"""Minimal multi-agent workflow using Azure AI Agent Framework. + +Demonstrates a router→specialist pattern where: +1. A Router Agent analyzes the query and picks the right specialist +2. The selected Specialist Agent has tools and processes the query +3. Tool calls are captured from the run steps + +All agents are created dynamically via the Agent Framework SDK +(azure-ai-projects + OpenAI Assistants API) and cleaned up after use. + +Prerequisites: + pip install azure-ai-projects azure-identity openai + +Environment variables: + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT — Foundry project endpoint + AZURE_OPENAI_DEPLOYMENT — model deployment name (e.g. gpt-5.1) + +Usage in run.yaml: + target: + type: agent + hosting: local + execution_mode: local + framework: agent_framework + local: + callable: multi_agent_workflow:run_evaluation +""" +from __future__ import annotations + +import json +import logging +import os +from typing import Any + +logger = logging.getLogger(__name__) + +PROJECT_ENDPOINT = os.environ.get("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", "") +MODEL = os.environ.get("AZURE_OPENAI_DEPLOYMENT", "") + +# Lazy-initialized client +_oai_client = None + + +def _get_openai_client(): + """Get an AzureOpenAI client for the Assistants API.""" + global _oai_client + if _oai_client is None: + from azure.identity import DefaultAzureCredential + from openai import AzureOpenAI + + cred = DefaultAzureCredential() + token = cred.get_token("https://cognitiveservices.azure.com/.default") + + # Extract base endpoint (without /api/projects/... path) + endpoint = PROJECT_ENDPOINT.split("/api/")[0] if "/api/" in PROJECT_ENDPOINT else PROJECT_ENDPOINT + + _oai_client = AzureOpenAI( + azure_endpoint=endpoint, + api_version="2025-03-01-preview", + azure_ad_token=token.token, + ) + return _oai_client + + +# ── Tool definitions for specialist agents ───────────────────────────── + +WEATHER_TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } +] + +FINANCE_TOOLS = [ + { + "type": "function", + "function": { + "name": "convert_currency", + "description": "Convert an amount from one currency to another", + "parameters": { + "type": "object", + "properties": { + "amount": {"type": "number"}, + "from_currency": {"type": "string"}, + "to_currency": {"type": "string"}, + }, + "required": ["amount", "from_currency", "to_currency"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "calculate_compound_interest", + "description": "Calculate compound interest", + "parameters": { + "type": "object", + "properties": { + "principal": {"type": "number"}, + "rate": {"type": "number"}, + "years": {"type": "integer"}, + }, + "required": ["principal", "rate", "years"], + }, + }, + }, +] + +SEARCH_TOOLS = [ + { + "type": "function", + "function": { + "name": "search_news", + "description": "Search for recent news articles", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "max_results": {"type": "integer"}, + }, + "required": ["query"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "search_flights", + "description": "Search for available flights", + "parameters": { + "type": "object", + "properties": { + "origin": {"type": "string"}, + "destination": {"type": "string"}, + "date": {"type": "string"}, + }, + "required": ["origin", "destination", "date"], + }, + }, + }, +] + +# Simulated tool results +TOOL_RESULTS = { + "get_weather": lambda args: f"Current weather in {args.get('city', 'unknown')}: 55°F, partly cloudy.", + "convert_currency": lambda args: f"{args.get('amount', 0)} {args.get('from_currency', '')} = {args.get('amount', 0) * 0.92:.2f} {args.get('to_currency', '')}", + "calculate_compound_interest": lambda args: f"Compound interest: ${args.get('principal', 0) * ((1 + args.get('rate', 0)) ** args.get('years', 0) - 1):,.2f}", + "search_news": lambda args: f"Found 5 articles about '{args.get('query', '')}'.", + "search_flights": lambda args: f"Found 3 flights from {args.get('origin', '')} to {args.get('destination', '')} on {args.get('date', '')}.", +} + +# ── Specialist definitions ───────────────────────────────────────────── + +SPECIALISTS = { + "weather": { + "name": "WeatherSpecialist", + "instructions": "You are a weather specialist. Use the get_weather tool to answer weather queries. Always call the tool before responding.", + "tools": WEATHER_TOOLS, + }, + "finance": { + "name": "FinanceSpecialist", + "instructions": "You are a finance specialist. Use convert_currency or calculate_compound_interest tools as needed. Always call the appropriate tool before responding.", + "tools": FINANCE_TOOLS, + }, + "search": { + "name": "SearchSpecialist", + "instructions": "You are a search specialist. Use search_news or search_flights tools as needed. Always call the appropriate tool before responding.", + "tools": SEARCH_TOOLS, + }, +} + + +def _create_agent(client, name: str, instructions: str, tools: list) -> Any: + """Create an Agent Framework agent via the Assistants API.""" + return client.beta.assistants.create( + model=MODEL, + name=name, + instructions=instructions, + tools=tools, + ) + + +def _run_agent_with_tools(client, agent_id: str, query: str) -> dict: + """Run an agent, handle tool calls, and return response + tool_calls.""" + thread = client.beta.threads.create() + client.beta.threads.messages.create( + thread_id=thread.id, role="user", content=query + ) + + run = client.beta.threads.runs.create_and_poll( + thread_id=thread.id, + assistant_id=agent_id, + ) + + all_tool_calls = [] + + # Handle tool calls if the agent requested them + while run.status == "requires_action": + tool_outputs = [] + for tc in run.required_action.submit_tool_outputs.tool_calls: + fn_name = tc.function.name + fn_args = json.loads(tc.function.arguments) + all_tool_calls.append({"name": fn_name, "arguments": fn_args}) + + handler = TOOL_RESULTS.get(fn_name) + result = handler(fn_args) if handler else f"Unknown tool: {fn_name}" + tool_outputs.append({"tool_call_id": tc.id, "output": result}) + + run = client.beta.threads.runs.submit_tool_outputs_and_poll( + thread_id=thread.id, run_id=run.id, tool_outputs=tool_outputs + ) + + # Extract final response + messages = client.beta.threads.messages.list(thread_id=thread.id) + response_text = "" + for msg in messages.data: + if msg.role == "assistant": + for block in msg.content: + if block.type == "text": + response_text += block.text.value + break + + # Cleanup thread + client.beta.threads.delete(thread.id) + + return {"response": response_text, "tool_calls": all_tool_calls} + + +def _route_query(client, router_id: str, query: str) -> str: + """Use the Router Agent to determine which specialist to use.""" + thread = client.beta.threads.create() + client.beta.threads.messages.create( + thread_id=thread.id, role="user", content=query + ) + + client.beta.threads.runs.create_and_poll( + thread_id=thread.id, assistant_id=router_id + ) + + messages = client.beta.threads.messages.list(thread_id=thread.id) + routing_decision = "" + for msg in messages.data: + if msg.role == "assistant": + for block in msg.content: + if block.type == "text": + routing_decision = block.text.value.strip().lower() + break + + client.beta.threads.delete(thread.id) + + # Parse routing decision + if "weather" in routing_decision: + return "weather" + elif "finance" in routing_decision or "currency" in routing_decision or "interest" in routing_decision: + return "finance" + else: + return "search" + + +def run_evaluation(input_text: str, context: dict) -> dict: + """Multi-agent workflow entry point for AgentOps evaluation. + + Orchestrates: Router Agent → Specialist Agent (with tools) → Response. + All agents are created dynamically and cleaned up after each call. + """ + if not PROJECT_ENDPOINT or not MODEL: + raise ValueError( + "Set AZURE_AI_FOUNDRY_PROJECT_ENDPOINT and AZURE_OPENAI_DEPLOYMENT" + ) + + client = _get_openai_client() + created_agents = [] + + try: + # Step 1: Create Router Agent + router = _create_agent( + client, + name="RouterAgent", + instructions=( + "You are a routing agent. Given a user query, respond with ONLY " + "one word indicating the specialist to handle it:\n" + "- 'weather' for weather queries\n" + "- 'finance' for currency conversion or interest calculations\n" + "- 'search' for news, flights, or general search queries\n" + "Respond with only the category word, nothing else." + ), + tools=[], + ) + created_agents.append(router.id) + logger.info("Created Router Agent: %s", router.id) + + # Step 2: Route the query + specialist_key = _route_query(client, router.id, input_text) + logger.info("Router selected specialist: %s", specialist_key) + + # Step 3: Create the selected Specialist Agent + spec_config = SPECIALISTS[specialist_key] + specialist = _create_agent( + client, + name=spec_config["name"], + instructions=spec_config["instructions"], + tools=spec_config["tools"], + ) + created_agents.append(specialist.id) + logger.info("Created Specialist Agent: %s (%s)", spec_config["name"], specialist.id) + + # Step 4: Run the specialist with tool handling + result = _run_agent_with_tools(client, specialist.id, input_text) + logger.info( + "Specialist returned: response=%d chars, tool_calls=%d", + len(result["response"]), + len(result["tool_calls"]), + ) + + return result + + finally: + # Step 5: Cleanup — delete all created agents + for agent_id in created_agents: + try: + client.beta.assistants.delete(agent_id) + logger.debug("Deleted agent: %s", agent_id) + except Exception: + logger.debug("Failed to delete agent: %s", agent_id, exc_info=True) diff --git a/tests/unit/test_initializer.py b/tests/unit/test_initializer.py index e2691012..74daab3e 100644 --- a/tests/unit/test_initializer.py +++ b/tests/unit/test_initializer.py @@ -42,7 +42,7 @@ def test_init_creates_expected_files(tmp_path: Path) -> None: assert (tmp_path / ".agentops" / "data" / "smoke-conversational.jsonl").is_file() assert (tmp_path / ".agentops" / "workflows" / "agentops-eval.yml").is_file() - assert len(result.created_files) == 26 + assert len(result.created_files) == 27 assert len(result.overwritten_files) == 0 run_config = load_yaml(tmp_path / ".agentops" / "run.yaml") From b0e31cdf602b02d0db3e4f710dfff36087b54bf0 Mon Sep 17 00:00:00 2001 From: DB Lee Date: Thu, 23 Apr 2026 14:55:08 -0700 Subject: [PATCH 06/15] feat: add GitHub-based skill installation (agentops skills install --from org/repo) Add --from flag to 'agentops skills install' for installing community skills from GitHub repositories following the agentskills.io standard. Features: - Parse github:org/repo[@ref] references with version pinning - Download repo tarball (single request, no API pagination) - Extract skill following agentskills.io convention (SKILL.md + references/) - Platform-aware installation (Copilot, Claude, Cursor) - Security: path sanitization, scripts/ blocked by default, traversal prevention - Provenance: .installed-from.json tracks source repo, ref, and installed files - Auth: GITHUB_TOKEN / GH_TOKEN env var support for private repos Usage: agentops skills install --from donlee/pptx-designer agentops skills install --from github:org/repo@v1.0 --platform claude Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agentops/cli/app.py | 48 ++++- src/agentops/services/skills.py | 359 +++++++++++++++++++++++++++++++- tests/unit/test_skills.py | 354 +++++++++++++++++++++++++++++++ 3 files changed, 755 insertions(+), 6 deletions(-) diff --git a/src/agentops/cli/app.py b/src/agentops/cli/app.py index 693ffa0f..478536c7 100644 --- a/src/agentops/cli/app.py +++ b/src/agentops/cli/app.py @@ -491,6 +491,17 @@ def cmd_skills_install( help="Target platform(s): copilot, claude.", ), ] = None, + from_github: Annotated[ + str | None, + typer.Option( + "--from", + help=( + "Install a community skill from GitHub. " + "Format: org/repo or github:org/repo[@ref]. " + "Example: --from donlee/pptx-designer" + ), + ), + ] = None, force: bool = typer.Option( False, "--force", @@ -507,12 +518,18 @@ def cmd_skills_install( help="Target repository root directory.", ), ) -> None: - """Install AgentOps coding agent skills into the target project.""" - from agentops.services.skills import install_skills + """Install AgentOps coding agent skills into the target project. + + Use --from to install a community skill from GitHub: + agentops skills install --from donlee/pptx-designer + + agentops skills install --from github:org/repo@v1.0 + """ log.debug( - "cmd_skills_install called platform=%s force=%s prompt=%s dir=%s", + "cmd_skills_install called platform=%s from=%s force=%s prompt=%s dir=%s", platform, + from_github, force, prompt, directory, @@ -524,6 +541,31 @@ def cmd_skills_install( typer.echo("No platforms selected. Skipping skill installation.") return + if from_github: + # GitHub-based skill installation + from agentops.services.skills import install_github_skill + + typer.echo(f"Installing skill from GitHub: {from_github}") + try: + result = install_github_skill( + source=from_github, + directory=directory, + platforms=resolved_platforms, + force=True, + ) + except ValueError as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(code=1) from exc + except Exception as exc: + typer.echo(f"Error: failed to install skill: {exc}", err=True) + raise typer.Exit(code=1) from exc + + _print_skills_result(result) + return + + # Bundled skills installation + from agentops.services.skills import install_skills + try: result = install_skills( directory=directory, platforms=resolved_platforms, force=True diff --git a/src/agentops/services/skills.py b/src/agentops/services/skills.py index 35092588..3c9069de 100644 --- a/src/agentops/services/skills.py +++ b/src/agentops/services/skills.py @@ -2,11 +2,16 @@ from __future__ import annotations +import io +import json import re +import tarfile +import urllib.error +import urllib.request from dataclasses import dataclass, field from importlib.resources import files -from pathlib import Path -from typing import Dict, List +from pathlib import Path, PurePosixPath +from typing import Any, Dict, List _TEMPLATE_PACKAGE = "agentops.templates" @@ -200,9 +205,357 @@ def install_skills( # --------------------------------------------------------------------------- -# Registration — add skill discovery entries to instruction files +# GitHub-based skill installation # --------------------------------------------------------------------------- +# Allowed sub-directories within a skill folder (agentskills.io spec). +_ALLOWED_SKILL_DIRS = {"references", "scripts", "assets"} + +# Directories skipped by default for security (opt-in only). +_RESTRICTED_DIRS = {"scripts"} + +_GITHUB_REF_RE = re.compile( + r"^(?:github:)?" + r"(?P[A-Za-z0-9._-]+)" + r"/(?P[A-Za-z0-9._-]+)" + r"(?:@(?P[A-Za-z0-9._/-]+))?$" +) + +_PROVENANCE_FILE = ".installed-from.json" + + +@dataclass +class GitHubSkillRef: + """Parsed GitHub skill reference.""" + + owner: str + repo: str + ref: str # branch, tag, or commit SHA + + +def _parse_github_ref(source: str) -> GitHubSkillRef: + """Parse ``github:org/repo@ref`` or ``org/repo`` into components. + + Raises ValueError on invalid input. + """ + m = _GITHUB_REF_RE.match(source.strip()) + if not m: + raise ValueError( + f"Invalid GitHub skill reference: '{source}'. " + "Expected format: github:org/repo or org/repo[@ref]" + ) + return GitHubSkillRef( + owner=m.group("owner"), + repo=m.group("repo"), + ref=m.group("ref") or "main", + ) + + +def _validate_skill_name(name: str) -> str: + """Validate and sanitize a skill name from SKILL.md frontmatter. + + Raises ValueError if the name contains path traversal or invalid chars. + """ + if not name or not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", name): + raise ValueError( + f"Invalid skill name: '{name}'. " + "Must be lowercase alphanumeric with single hyphens, " + "e.g. 'pptx-designer'." + ) + if ".." in name or "/" in name or "\\" in name: + raise ValueError(f"Skill name contains path traversal: '{name}'") + return name + + +def _parse_skill_frontmatter(content: str) -> dict[str, str]: + """Extract YAML frontmatter fields from a SKILL.md file. + + Returns a dict with at least ``name`` and ``description`` keys. + Uses simple line parsing to avoid a YAML dependency in this module. + """ + if not content.startswith("---"): + raise ValueError("SKILL.md is missing YAML frontmatter (must start with ---).") + + lines = content.split("\n") + end_idx = None + for i, line in enumerate(lines[1:], 1): + if line.strip() == "---": + end_idx = i + break + + if end_idx is None: + raise ValueError("SKILL.md has unclosed YAML frontmatter.") + + meta: dict[str, str] = {} + current_key = "" + for line in lines[1:end_idx]: + if line.startswith(" ") and current_key: + # Continuation of multiline value + meta[current_key] = meta.get(current_key, "") + " " + line.strip() + continue + if ":" in line: + key, _, val = line.partition(":") + key = key.strip() + val = val.strip().strip(">").strip('"').strip("'").strip() + if key: + current_key = key + meta[key] = val + + if "name" not in meta: + raise ValueError("SKILL.md frontmatter is missing required 'name' field.") + if "description" not in meta: + raise ValueError("SKILL.md frontmatter is missing required 'description' field.") + + return meta + + +def _fetch_github_tarball(ref: GitHubSkillRef) -> bytes: + """Download a GitHub repo tarball for the given ref. + + Uses ``GITHUB_TOKEN`` or ``GH_TOKEN`` env var if available. + """ + import os + + url = f"https://api.github.com/repos/{ref.owner}/{ref.repo}/tarball/{ref.ref}" + + headers: dict[str, str] = {"Accept": "application/vnd.github+json"} + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + + req = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return resp.read() + except urllib.error.HTTPError as e: + if e.code == 404: + raise ValueError( + f"GitHub repository not found: {ref.owner}/{ref.repo}@{ref.ref}" + ) from e + if e.code == 403: + raise ValueError( + f"GitHub API rate limit or access denied for {ref.owner}/{ref.repo}. " + "Set GITHUB_TOKEN env var for authenticated access." + ) from e + raise ValueError( + f"GitHub API error ({e.code}): {e.reason}" + ) from e + except urllib.error.URLError as e: + raise ValueError(f"Network error fetching {ref.owner}/{ref.repo}: {e}") from e + + +def _extract_skill_from_tarball( + tarball: bytes, + repo_name: str, +) -> tuple[dict[str, str], dict[str, bytes]]: + """Extract a single skill from a GitHub repo tarball. + + Returns (frontmatter_metadata, {relative_path: content_bytes}). + + Searches for the skill directory following agentskills.io convention: + 1. ``{repo_name}/SKILL.md`` (skill dir = repo name) + 2. Any ``*/SKILL.md`` at depth 1 from repo root + 3. ``SKILL.md`` at repo root + + Raises ValueError if no SKILL.md is found or multiple candidates exist. + """ + with tarfile.open(fileobj=io.BytesIO(tarball), mode="r:gz") as tar: + members = tar.getnames() + + # GitHub tarballs have a prefix like "owner-repo-sha/" + prefix = "" + for name in members: + if "/" in name: + prefix = name.split("/")[0] + "/" + break + + # Find SKILL.md candidates + candidates: list[str] = [] + for name in members: + relative = name[len(prefix):] if name.startswith(prefix) else name + parts = PurePosixPath(relative).parts + if parts and parts[-1] == "SKILL.md": + if len(parts) <= 2: + candidates.append(relative) + + if not candidates: + raise ValueError( + f"No SKILL.md found in {repo_name}. " + "The repository must contain a skill directory with a SKILL.md file " + "(agentskills.io standard)." + ) + + # Prefer {repo_name}/SKILL.md, then first candidate + chosen = None + for c in candidates: + if c.startswith(repo_name + "/"): + chosen = c + break + if chosen is None: + if len(candidates) > 1: + dirs = [str(PurePosixPath(c).parent) for c in candidates] + raise ValueError( + f"Multiple skills found in {repo_name}: {', '.join(dirs)}. " + "Use github:org/repo with a repo that contains a single skill." + ) + chosen = candidates[0] + + skill_dir = str(PurePosixPath(chosen).parent) + if skill_dir == ".": + skill_dir = "" + + # Read SKILL.md and parse frontmatter + skill_md_path = prefix + chosen + member = tar.getmember(skill_md_path) + f = tar.extractfile(member) + if f is None: + raise ValueError(f"Cannot read {skill_md_path}") + skill_md_content = f.read() + metadata = _parse_skill_frontmatter(skill_md_content.decode("utf-8")) + + # Collect all files in the skill directory + skill_prefix = prefix + (skill_dir + "/" if skill_dir else "") + collected: dict[str, bytes] = {} + + for member in tar.getmembers(): + if not member.isfile(): + continue + if not member.name.startswith(skill_prefix): + continue + + relative = member.name[len(skill_prefix):] + parts = PurePosixPath(relative).parts + + if not parts: + continue + + # Security: block path traversal + if any(p in ("..", "") for p in parts): + continue + if any(p.startswith(".") for p in parts): + continue + + # Allow SKILL.md at root, and files in allowed subdirs + if len(parts) == 1 and parts[0] == "SKILL.md": + collected[relative] = skill_md_content + continue + + top_dir = parts[0] if len(parts) > 1 else None + if top_dir and top_dir in _ALLOWED_SKILL_DIRS: + if top_dir in _RESTRICTED_DIRS: + continue # Skip scripts/ by default + f = tar.extractfile(member) + if f: + collected[relative] = f.read() + + return metadata, collected + + +def install_github_skill( + source: str, + directory: Path, + platforms: list[str], + force: bool = False, +) -> SkillsInstallResult: + """Install a skill from a GitHub repository. + + Downloads the repo archive, extracts the skill, validates it, + and installs to platform-specific directories. + + Args: + source: GitHub reference, e.g. ``github:org/repo``, ``org/repo@v1.0``. + directory: Root directory of the consumer repository. + platforms: Platform identifiers (e.g. ``["copilot"]``). + force: When True, overwrite existing skill files. + + Returns: + SkillsInstallResult with paths of created, overwritten, or skipped files. + """ + ref = _parse_github_ref(source) + result = SkillsInstallResult(platforms=list(platforms)) + resolved = directory.resolve() + + # Fetch and extract + tarball = _fetch_github_tarball(ref) + metadata, skill_files = _extract_skill_from_tarball(tarball, ref.repo) + + skill_name = _validate_skill_name(metadata["name"]) + + if not skill_files: + raise ValueError(f"No installable files found in {ref.owner}/{ref.repo}.") + + # Install to each platform + for platform in platforms: + config = _PLATFORM_CONFIGS.get(platform) + if not config: + continue + + target_dir = resolved / config["target_dir"] + + for relative_path, content_bytes in skill_files.items(): + if relative_path == "SKILL.md": + # SKILL.md uses the platform file pattern + dest_relative = config["file_pattern"].format(skill_name=skill_name) + dest = target_dir / dest_relative + text_content = content_bytes.decode("utf-8") + text_content = _transform_content(text_content, platform) + write_bytes = text_content.encode("utf-8") + else: + # Reference/asset files go alongside the SKILL.md + if platform == "claude": + continue # Claude only gets the single .md file + skill_dest_dir = config["file_pattern"].format( + skill_name=skill_name + ) + # e.g. "pptx-designer/SKILL.md" → "pptx-designer/" + skill_base = str(PurePosixPath(skill_dest_dir).parent) + dest = target_dir / skill_base / relative_path + write_bytes = content_bytes + + # Security: ensure dest stays under target_dir + try: + dest.resolve().relative_to(target_dir.resolve()) + except ValueError: + continue # path traversal — skip silently + + existed = dest.exists() + if existed and not force: + result.skipped_files.append(dest) + continue + + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(write_bytes) + + if existed: + result.overwritten_files.append(dest) + else: + result.created_files.append(dest) + + # Write provenance file + if platform != "claude": + provenance_dest_rel = config["file_pattern"].format( + skill_name=skill_name + ) + provenance_dir = ( + target_dir / str(PurePosixPath(provenance_dest_rel).parent) + ) + provenance = { + "source": f"github:{ref.owner}/{ref.repo}", + "ref": ref.ref, + "skill_name": skill_name, + "description": metadata.get("description", ""), + "files": sorted(skill_files.keys()), + } + prov_path = provenance_dir / _PROVENANCE_FILE + prov_path.parent.mkdir(parents=True, exist_ok=True) + prov_path.write_text( + json.dumps(provenance, indent=2) + "\n", encoding="utf-8" + ) + if prov_path not in result.created_files: + result.created_files.append(prov_path) + + return result + @dataclass class RegistrationResult: diff --git a/tests/unit/test_skills.py b/tests/unit/test_skills.py index 4392f077..6ad5058a 100644 --- a/tests/unit/test_skills.py +++ b/tests/unit/test_skills.py @@ -1,12 +1,22 @@ +import io +import json +import tarfile from pathlib import Path +from unittest.mock import patch +import pytest from typer.testing import CliRunner from agentops.cli.app import app from agentops.services.skills import ( _COPILOT_MARKER_END, _COPILOT_MARKER_START, + _extract_skill_from_tarball, + _parse_github_ref, + _parse_skill_frontmatter, + _validate_skill_name, detect_platforms, + install_github_skill, install_skills, register_skills, ) @@ -403,3 +413,347 @@ def test_cli_init_does_not_install_skills_claude(tmp_path: Path) -> None: for rel in _CLAUDE_SKILL_PATHS: assert not (tmp_path / rel).exists(), f"Should not exist after init: {rel}" + + +# --------------------------------------------------------------------------- +# GitHub ref parsing +# --------------------------------------------------------------------------- + + +def test_parse_github_ref_simple() -> None: + ref = _parse_github_ref("donlee/pptx-designer") + assert ref.owner == "donlee" + assert ref.repo == "pptx-designer" + assert ref.ref == "main" + + +def test_parse_github_ref_with_prefix() -> None: + ref = _parse_github_ref("github:org/repo") + assert ref.owner == "org" + assert ref.repo == "repo" + assert ref.ref == "main" + + +def test_parse_github_ref_with_version() -> None: + ref = _parse_github_ref("github:org/repo@v1.2.3") + assert ref.owner == "org" + assert ref.repo == "repo" + assert ref.ref == "v1.2.3" + + +def test_parse_github_ref_with_branch() -> None: + ref = _parse_github_ref("org/repo@feature/my-branch") + assert ref.ref == "feature/my-branch" + + +def test_parse_github_ref_invalid() -> None: + with pytest.raises(ValueError, match="Invalid GitHub skill reference"): + _parse_github_ref("not-valid") + + +def test_parse_github_ref_empty() -> None: + with pytest.raises(ValueError, match="Invalid GitHub skill reference"): + _parse_github_ref("") + + +# --------------------------------------------------------------------------- +# Skill name validation +# --------------------------------------------------------------------------- + + +def test_validate_skill_name_valid() -> None: + assert _validate_skill_name("pptx-designer") == "pptx-designer" + assert _validate_skill_name("myskill") == "myskill" + assert _validate_skill_name("my-cool-skill") == "my-cool-skill" + + +def test_validate_skill_name_invalid() -> None: + with pytest.raises(ValueError, match="Invalid skill name"): + _validate_skill_name("My Skill") + + with pytest.raises(ValueError, match="Invalid skill name"): + _validate_skill_name("../traversal") + + with pytest.raises(ValueError, match="Invalid skill name"): + _validate_skill_name("") + + with pytest.raises(ValueError, match="Invalid skill name"): + _validate_skill_name("UPPERCASE") + + +# --------------------------------------------------------------------------- +# Frontmatter parsing +# --------------------------------------------------------------------------- + + +_VALID_FRONTMATTER = """\ +--- +name: test-skill +description: A test skill for unit testing. +license: MIT +--- + +# Test Skill + +Instructions here. +""" + + +def test_parse_frontmatter_valid() -> None: + meta = _parse_skill_frontmatter(_VALID_FRONTMATTER) + assert meta["name"] == "test-skill" + assert "test skill" in meta["description"].lower() + + +def test_parse_frontmatter_missing_name() -> None: + content = "---\ndescription: test\n---\n# Body" + with pytest.raises(ValueError, match="missing required 'name'"): + _parse_skill_frontmatter(content) + + +def test_parse_frontmatter_missing_description() -> None: + content = "---\nname: test\n---\n# Body" + with pytest.raises(ValueError, match="missing required 'description'"): + _parse_skill_frontmatter(content) + + +def test_parse_frontmatter_no_frontmatter() -> None: + with pytest.raises(ValueError, match="missing YAML frontmatter"): + _parse_skill_frontmatter("# Just a heading") + + +def test_parse_frontmatter_unclosed() -> None: + with pytest.raises(ValueError, match="unclosed YAML frontmatter"): + _parse_skill_frontmatter("---\nname: test\n") + + +def test_parse_frontmatter_multiline_description() -> None: + content = "---\nname: test-skill\ndescription: >\n A long\n description here.\n---\n# Body" + meta = _parse_skill_frontmatter(content) + assert "long" in meta["description"] + assert "description here" in meta["description"] + + +# --------------------------------------------------------------------------- +# Tarball extraction +# --------------------------------------------------------------------------- + + +def _make_test_tarball(files: dict[str, str], prefix: str = "owner-repo-abc123") -> bytes: + """Create a gzipped tarball with the given files for testing.""" + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tar: + for path, content in files.items(): + full_path = f"{prefix}/{path}" + data = content.encode("utf-8") + info = tarfile.TarInfo(name=full_path) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +def test_extract_skill_from_tarball() -> None: + tarball = _make_test_tarball({ + "my-skill/SKILL.md": _VALID_FRONTMATTER, + "my-skill/references/guide.md": "# Guide\n\nSome content.", + }) + meta, files = _extract_skill_from_tarball(tarball, "my-skill") + assert meta["name"] == "test-skill" + assert "SKILL.md" in files + assert "references/guide.md" in files + + +def test_extract_skill_prefers_repo_named_dir() -> None: + tarball = _make_test_tarball({ + "my-skill/SKILL.md": _VALID_FRONTMATTER, + "other-dir/SKILL.md": _VALID_FRONTMATTER, + }) + meta, files = _extract_skill_from_tarball(tarball, "my-skill") + assert meta["name"] == "test-skill" + + +def test_extract_skill_root_skill_md() -> None: + tarball = _make_test_tarball({ + "SKILL.md": _VALID_FRONTMATTER, + }) + meta, files = _extract_skill_from_tarball(tarball, "some-repo") + assert meta["name"] == "test-skill" + assert "SKILL.md" in files + + +def test_extract_skill_no_skill_md() -> None: + tarball = _make_test_tarball({ + "README.md": "# Hello", + }) + with pytest.raises(ValueError, match="No SKILL.md found"): + _extract_skill_from_tarball(tarball, "some-repo") + + +def test_extract_skill_multiple_ambiguous() -> None: + tarball = _make_test_tarball({ + "skill-a/SKILL.md": _VALID_FRONTMATTER, + "skill-b/SKILL.md": _VALID_FRONTMATTER.replace("test-skill", "other-skill"), + }) + with pytest.raises(ValueError, match="Multiple skills found"): + _extract_skill_from_tarball(tarball, "unrelated-repo") + + +def test_extract_skill_skips_scripts() -> None: + tarball = _make_test_tarball({ + "my-skill/SKILL.md": _VALID_FRONTMATTER, + "my-skill/scripts/run.py": "print('hello')", + "my-skill/references/ref.md": "# Ref", + }) + _, files = _extract_skill_from_tarball(tarball, "my-skill") + assert "references/ref.md" in files + assert "scripts/run.py" not in files # scripts blocked by default + + +def test_extract_skill_blocks_path_traversal() -> None: + tarball = _make_test_tarball({ + "my-skill/SKILL.md": _VALID_FRONTMATTER, + "my-skill/../../../etc/passwd": "root:x:0:0", + }) + _, files = _extract_skill_from_tarball(tarball, "my-skill") + assert all(".." not in p for p in files) + + +def test_extract_skill_blocks_hidden_files() -> None: + tarball = _make_test_tarball({ + "my-skill/SKILL.md": _VALID_FRONTMATTER, + "my-skill/.env": "SECRET=abc", + "my-skill/references/guide.md": "# Guide", + }) + _, files = _extract_skill_from_tarball(tarball, "my-skill") + assert ".env" not in files + assert "references/guide.md" in files + + +# --------------------------------------------------------------------------- +# install_github_skill (with mocked network) +# --------------------------------------------------------------------------- + + +def test_install_github_skill_copilot(tmp_path: Path) -> None: + tarball = _make_test_tarball({ + "pptx-designer/SKILL.md": _VALID_FRONTMATTER, + "pptx-designer/references/setup.md": "# Setup guide", + }) + + with patch( + "agentops.services.skills._fetch_github_tarball", return_value=tarball + ): + result = install_github_skill( + source="donlee/pptx-designer", + directory=tmp_path, + platforms=["copilot"], + force=True, + ) + + # SKILL.md installed + skill_path = tmp_path / ".github/skills/test-skill/SKILL.md" + assert skill_path.exists() + content = skill_path.read_text(encoding="utf-8") + assert content.startswith("---") # frontmatter preserved for copilot + + # Reference file installed + ref_path = tmp_path / ".github/skills/test-skill/references/setup.md" + assert ref_path.exists() + + # Provenance file created + prov_path = tmp_path / ".github/skills/test-skill/.installed-from.json" + assert prov_path.exists() + prov = json.loads(prov_path.read_text()) + assert prov["source"] == "github:donlee/pptx-designer" + assert prov["skill_name"] == "test-skill" + + assert len(result.created_files) >= 2 + + +def test_install_github_skill_claude(tmp_path: Path) -> None: + tarball = _make_test_tarball({ + "pptx-designer/SKILL.md": _VALID_FRONTMATTER, + "pptx-designer/references/setup.md": "# Setup guide", + }) + + with patch( + "agentops.services.skills._fetch_github_tarball", return_value=tarball + ): + result = install_github_skill( + source="donlee/pptx-designer", + directory=tmp_path, + platforms=["claude"], + ) + + # Claude gets a single .md file with frontmatter stripped + skill_path = tmp_path / ".claude/commands/test-skill.md" + assert skill_path.exists() + content = skill_path.read_text(encoding="utf-8") + assert not content.startswith("---") # frontmatter stripped + + # Claude does NOT get reference files + ref_path = tmp_path / ".claude/commands/references/setup.md" + assert not ref_path.exists() + + +def test_install_github_skill_skip_existing(tmp_path: Path) -> None: + tarball = _make_test_tarball({ + "my-skill/SKILL.md": _VALID_FRONTMATTER, + }) + + # Pre-create the file + dest = tmp_path / ".github/skills/test-skill/SKILL.md" + dest.parent.mkdir(parents=True) + dest.write_text("custom content") + + with patch( + "agentops.services.skills._fetch_github_tarball", return_value=tarball + ): + result = install_github_skill( + source="org/my-skill", + directory=tmp_path, + platforms=["copilot"], + force=False, + ) + + assert len(result.skipped_files) >= 1 + assert dest.read_text() == "custom content" + + +# --------------------------------------------------------------------------- +# CLI — agentops skills install --from +# --------------------------------------------------------------------------- + + +def test_cli_skills_install_from_github(tmp_path: Path) -> None: + tarball = _make_test_tarball({ + "pptx-designer/SKILL.md": _VALID_FRONTMATTER, + }) + + with patch( + "agentops.services.skills._fetch_github_tarball", return_value=tarball + ): + result = runner.invoke( + app, + [ + "skills", "install", + "--from", "donlee/pptx-designer", + "--dir", str(tmp_path), + ], + ) + + assert result.exit_code == 0 + assert "Installing skill from GitHub" in result.stdout + assert "created" in result.stdout + + +def test_cli_skills_install_from_invalid_ref(tmp_path: Path) -> None: + result = runner.invoke( + app, + [ + "skills", "install", + "--from", "not-valid-ref", + "--dir", str(tmp_path), + ], + ) + assert result.exit_code == 1 From 0f30bf01a61a43e8618f73a9172e81e8c52e9bda Mon Sep 17 00:00:00 2001 From: DB Lee Date: Thu, 23 Apr 2026 14:56:34 -0700 Subject: [PATCH 07/15] fix: resolve pre-existing ruff lint errors (F401, F841) Fix 2 lint errors that existed on develop branch: - skills.py: remove unused import 'Any' - test_skills.py: remove unused variable 'result' Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agentops/services/skills.py | 2 +- tests/unit/test_skills.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agentops/services/skills.py b/src/agentops/services/skills.py index 3c9069de..ce66e276 100644 --- a/src/agentops/services/skills.py +++ b/src/agentops/services/skills.py @@ -11,7 +11,7 @@ from dataclasses import dataclass, field from importlib.resources import files from pathlib import Path, PurePosixPath -from typing import Any, Dict, List +from typing import Dict, List _TEMPLATE_PACKAGE = "agentops.templates" diff --git a/tests/unit/test_skills.py b/tests/unit/test_skills.py index 6ad5058a..5b9a4458 100644 --- a/tests/unit/test_skills.py +++ b/tests/unit/test_skills.py @@ -679,7 +679,7 @@ def test_install_github_skill_claude(tmp_path: Path) -> None: with patch( "agentops.services.skills._fetch_github_tarball", return_value=tarball ): - result = install_github_skill( + install_github_skill( source="donlee/pptx-designer", directory=tmp_path, platforms=["claude"], From 154bfbce2c386568554a4faf491b335017c74ac8 Mon Sep 17 00:00:00 2001 From: DB Lee Date: Thu, 23 Apr 2026 15:06:12 -0700 Subject: [PATCH 08/15] fix: resolve mypy type errors in multi_agent_workflow and local_adapter - multi_agent_workflow.py: explicit str()/list() casts for SPECIALISTS dict values - local_adapter_backend.py: add assert for adapter_command before shlex.split Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agentops/backends/local_adapter_backend.py | 1 + src/agentops/templates/multi_agent_workflow.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/agentops/backends/local_adapter_backend.py b/src/agentops/backends/local_adapter_backend.py index 40225e76..1636e97a 100644 --- a/src/agentops/backends/local_adapter_backend.py +++ b/src/agentops/backends/local_adapter_backend.py @@ -231,6 +231,7 @@ def execute(self, context: BackendRunContext) -> BackendExecutionResult: continue else: # --- Subprocess mode --- + assert adapter_command is not None adapter_input = json.dumps( {"input": prompt_text, "expected": expected_text, **row} ) diff --git a/src/agentops/templates/multi_agent_workflow.py b/src/agentops/templates/multi_agent_workflow.py index b22763b8..b0a1c7ac 100644 --- a/src/agentops/templates/multi_agent_workflow.py +++ b/src/agentops/templates/multi_agent_workflow.py @@ -304,9 +304,9 @@ def run_evaluation(input_text: str, context: dict) -> dict: spec_config = SPECIALISTS[specialist_key] specialist = _create_agent( client, - name=spec_config["name"], - instructions=spec_config["instructions"], - tools=spec_config["tools"], + name=str(spec_config["name"]), + instructions=str(spec_config["instructions"]), + tools=list(spec_config["tools"]), ) created_agents.append(specialist.id) logger.info("Created Specialist Agent: %s (%s)", spec_config["name"], specialist.id) From 53cdd66ee9af060d5164e7131660925a9c1525b5 Mon Sep 17 00:00:00 2001 From: DB Lee Date: Thu, 23 Apr 2026 15:14:37 -0700 Subject: [PATCH 09/15] refactor: rewrite multi_agent_workflow to use Azure AI Agent Framework SDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace OpenAI Assistants API (openai.AzureOpenAI.beta.assistants) with Azure AI Agent Framework SDK (azure-ai-agents AgentsClient): - Use AgentsClient.create_agent() for dynamic agent creation - Use FunctionTool with real Python callables (not JSON schemas) - Use ToolSet + enable_auto_function_calls() for automatic tool execution - Use create_thread_and_process_run() for the agent execution loop - Fix tool function signatures for string args from SDK The multi-agent workflow now properly uses the Agent Framework pattern: AgentsClient → create_agent() → FunctionTool → ToolSet → enable_auto_function_calls() → create_thread_and_process_run() E2E validated: all 5 rows, correct routing, tool_calls captured, evaluators scored (ToolCallAccuracy: 3.0, IntentResolution: 3.6). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/multi_agent_workflow.py | 406 ++++++++---------- 1 file changed, 187 insertions(+), 219 deletions(-) diff --git a/src/agentops/templates/multi_agent_workflow.py b/src/agentops/templates/multi_agent_workflow.py index b0a1c7ac..74ca282b 100644 --- a/src/agentops/templates/multi_agent_workflow.py +++ b/src/agentops/templates/multi_agent_workflow.py @@ -1,15 +1,15 @@ -"""Minimal multi-agent workflow using Azure AI Agent Framework. +"""Multi-agent workflow using Azure AI Agent Framework (azure-ai-agents). -Demonstrates a router→specialist pattern where: +Demonstrates a router-to-specialist pattern where: 1. A Router Agent analyzes the query and picks the right specialist -2. The selected Specialist Agent has tools and processes the query -3. Tool calls are captured from the run steps +2. The selected Specialist Agent has Python function tools +3. Agent Framework auto-executes tool calls via enable_auto_function_calls +4. Tool calls are captured from run steps for evaluation -All agents are created dynamically via the Agent Framework SDK -(azure-ai-projects + OpenAI Assistants API) and cleaned up after use. +All agents are created dynamically via AgentsClient and cleaned up after use. Prerequisites: - pip install azure-ai-projects azure-identity openai + pip install azure-ai-agents azure-identity Environment variables: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT — Foundry project endpoint @@ -29,6 +29,7 @@ import json import logging import os +import urllib.request from typing import Any logger = logging.getLogger(__name__) @@ -36,252 +37,200 @@ PROJECT_ENDPOINT = os.environ.get("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", "") MODEL = os.environ.get("AZURE_OPENAI_DEPLOYMENT", "") -# Lazy-initialized client -_oai_client = None +_client = None +_token: str | None = None -def _get_openai_client(): - """Get an AzureOpenAI client for the Assistants API.""" - global _oai_client - if _oai_client is None: +def _get_client(): + """Lazily initialize the AgentsClient.""" + global _client + if _client is None: + from azure.ai.agents import AgentsClient from azure.identity import DefaultAzureCredential - from openai import AzureOpenAI - cred = DefaultAzureCredential() - token = cred.get_token("https://cognitiveservices.azure.com/.default") + _client = AgentsClient( + endpoint=PROJECT_ENDPOINT, + credential=DefaultAzureCredential(), + ) + return _client - # Extract base endpoint (without /api/projects/... path) - endpoint = PROJECT_ENDPOINT.split("/api/")[0] if "/api/" in PROJECT_ENDPOINT else PROJECT_ENDPOINT - _oai_client = AzureOpenAI( - azure_endpoint=endpoint, - api_version="2025-03-01-preview", - azure_ad_token=token.token, - ) - return _oai_client +def _get_token() -> str: + """Get a bearer token for REST API calls (messages, run steps).""" + global _token + from azure.identity import DefaultAzureCredential + _token = DefaultAzureCredential().get_token( + "https://ai.azure.com/.default" + ).token + return _token -# ── Tool definitions for specialist agents ───────────────────────────── -WEATHER_TOOLS = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current weather for a city", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"], - }, - }, - } -] - -FINANCE_TOOLS = [ - { - "type": "function", - "function": { - "name": "convert_currency", - "description": "Convert an amount from one currency to another", - "parameters": { - "type": "object", - "properties": { - "amount": {"type": "number"}, - "from_currency": {"type": "string"}, - "to_currency": {"type": "string"}, - }, - "required": ["amount", "from_currency", "to_currency"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "calculate_compound_interest", - "description": "Calculate compound interest", - "parameters": { - "type": "object", - "properties": { - "principal": {"type": "number"}, - "rate": {"type": "number"}, - "years": {"type": "integer"}, - }, - "required": ["principal", "rate", "years"], - }, - }, - }, -] - -SEARCH_TOOLS = [ - { - "type": "function", - "function": { - "name": "search_news", - "description": "Search for recent news articles", - "parameters": { - "type": "object", - "properties": { - "query": {"type": "string"}, - "max_results": {"type": "integer"}, - }, - "required": ["query"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "search_flights", - "description": "Search for available flights", - "parameters": { - "type": "object", - "properties": { - "origin": {"type": "string"}, - "destination": {"type": "string"}, - "date": {"type": "string"}, - }, - "required": ["origin", "destination", "date"], - }, - }, - }, -] - -# Simulated tool results -TOOL_RESULTS = { - "get_weather": lambda args: f"Current weather in {args.get('city', 'unknown')}: 55°F, partly cloudy.", - "convert_currency": lambda args: f"{args.get('amount', 0)} {args.get('from_currency', '')} = {args.get('amount', 0) * 0.92:.2f} {args.get('to_currency', '')}", - "calculate_compound_interest": lambda args: f"Compound interest: ${args.get('principal', 0) * ((1 + args.get('rate', 0)) ** args.get('years', 0) - 1):,.2f}", - "search_news": lambda args: f"Found 5 articles about '{args.get('query', '')}'.", - "search_flights": lambda args: f"Found 3 flights from {args.get('origin', '')} to {args.get('destination', '')} on {args.get('date', '')}.", -} +def _rest_get(path: str) -> dict: + """GET request to the Foundry project API.""" + url = f"{PROJECT_ENDPOINT}{path}?api-version=2025-05-01" + req = urllib.request.Request( + url, headers={"Authorization": f"Bearer {_get_token()}"} + ) + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()) + + +# ── Tool functions for specialist agents ─────────────────────────────── +# Agent Framework uses real Python functions (not JSON schemas). +# FunctionTool wraps these and auto-executes them when the agent +# makes a tool call via enable_auto_function_calls(). + + +def get_weather(city: str) -> str: + """Get current weather for a city""" + return f"Current weather in {city}: 55°F, partly cloudy." + + +def convert_currency(amount: str, from_currency: str, to_currency: str) -> str: + """Convert an amount from one currency to another""" + amt = float(amount) + return f"{amt} {from_currency} = {amt * 0.92:.2f} {to_currency}" + + +def calculate_compound_interest(principal: str, rate: str, years: str) -> str: + """Calculate compound interest""" + p, r, y = float(principal), float(rate) / 100, int(float(years)) + total = p * ((1 + r) ** y) + interest = total - p + return f"Compound interest: ${interest:,.2f}, total: ${total:,.2f}" + + +def search_news(query: str, max_results: str = "5") -> str: + """Search for recent news articles""" + return f"Found {max_results} articles about '{query}'." -# ── Specialist definitions ───────────────────────────────────────────── -SPECIALISTS = { +def search_flights(origin: str, destination: str, date: str) -> str: + """Search for available flights""" + return f"Found 3 flights from {origin} to {destination} on {date}." + + +# ── Specialist configurations ────────────────────────────────────────── + +SPECIALISTS: dict[str, dict[str, Any]] = { "weather": { "name": "WeatherSpecialist", - "instructions": "You are a weather specialist. Use the get_weather tool to answer weather queries. Always call the tool before responding.", - "tools": WEATHER_TOOLS, + "instructions": ( + "You are a weather specialist. Use the get_weather tool " + "to answer weather queries. Always call the tool before responding." + ), + "functions": [get_weather], }, "finance": { "name": "FinanceSpecialist", - "instructions": "You are a finance specialist. Use convert_currency or calculate_compound_interest tools as needed. Always call the appropriate tool before responding.", - "tools": FINANCE_TOOLS, + "instructions": ( + "You are a finance specialist. Use convert_currency or " + "calculate_compound_interest tools as needed. Always call " + "the appropriate tool before responding." + ), + "functions": [convert_currency, calculate_compound_interest], }, "search": { "name": "SearchSpecialist", - "instructions": "You are a search specialist. Use search_news or search_flights tools as needed. Always call the appropriate tool before responding.", - "tools": SEARCH_TOOLS, + "instructions": ( + "You are a search specialist. Use search_news or search_flights " + "tools as needed. Always call the appropriate tool before responding." + ), + "functions": [search_news, search_flights], }, } -def _create_agent(client, name: str, instructions: str, tools: list) -> Any: - """Create an Agent Framework agent via the Assistants API.""" - return client.beta.assistants.create( - model=MODEL, - name=name, - instructions=instructions, - tools=tools, - ) - - -def _run_agent_with_tools(client, agent_id: str, query: str) -> dict: - """Run an agent, handle tool calls, and return response + tool_calls.""" - thread = client.beta.threads.create() - client.beta.threads.messages.create( - thread_id=thread.id, role="user", content=query - ) - - run = client.beta.threads.runs.create_and_poll( - thread_id=thread.id, - assistant_id=agent_id, - ) - - all_tool_calls = [] - - # Handle tool calls if the agent requested them - while run.status == "requires_action": - tool_outputs = [] - for tc in run.required_action.submit_tool_outputs.tool_calls: - fn_name = tc.function.name - fn_args = json.loads(tc.function.arguments) - all_tool_calls.append({"name": fn_name, "arguments": fn_args}) - - handler = TOOL_RESULTS.get(fn_name) - result = handler(fn_args) if handler else f"Unknown tool: {fn_name}" - tool_outputs.append({"tool_call_id": tc.id, "output": result}) - - run = client.beta.threads.runs.submit_tool_outputs_and_poll( - thread_id=thread.id, run_id=run.id, tool_outputs=tool_outputs - ) - - # Extract final response - messages = client.beta.threads.messages.list(thread_id=thread.id) +def _extract_response_and_tool_calls( + thread_id: str, run_id: str +) -> dict[str, Any]: + """Extract response text and tool calls from a completed run.""" + # Get assistant messages + msgs = _rest_get(f"/threads/{thread_id}/messages") response_text = "" - for msg in messages.data: - if msg.role == "assistant": - for block in msg.content: - if block.type == "text": - response_text += block.text.value + for msg in msgs.get("data", []): + if msg.get("role") == "assistant": + for block in msg.get("content", []): + if block.get("type") == "text": + val = block.get("text", {}) + response_text += ( + val.get("value", "") if isinstance(val, dict) else str(val) + ) break - # Cleanup thread - client.beta.threads.delete(thread.id) + # Get tool calls from run steps + steps = _rest_get(f"/threads/{thread_id}/runs/{run_id}/steps") + tool_calls: list[dict[str, Any]] = [] + for step in steps.get("data", []): + details = step.get("step_details", {}) + if details.get("type") == "tool_calls": + for tc in details.get("tool_calls", []): + if tc.get("type") == "function": + fn = tc.get("function", {}) + tool_calls.append({ + "name": fn.get("name", ""), + "arguments": json.loads(fn.get("arguments", "{}")), + }) - return {"response": response_text, "tool_calls": all_tool_calls} + return {"response": response_text.strip(), "tool_calls": tool_calls} -def _route_query(client, router_id: str, query: str) -> str: +def _route_query(client: Any, router_id: str, query: str) -> str: """Use the Router Agent to determine which specialist to use.""" - thread = client.beta.threads.create() - client.beta.threads.messages.create( - thread_id=thread.id, role="user", content=query - ) - - client.beta.threads.runs.create_and_poll( - thread_id=thread.id, assistant_id=router_id + from azure.ai.agents.models import ThreadMessageOptions + + result = client.create_thread_and_process_run( + agent_id=router_id, + thread={ + "messages": [ + ThreadMessageOptions(role="user", content=query) + ] + }, ) - messages = client.beta.threads.messages.list(thread_id=thread.id) + msgs = _rest_get(f"/threads/{result.thread_id}/messages") routing_decision = "" - for msg in messages.data: - if msg.role == "assistant": - for block in msg.content: - if block.type == "text": - routing_decision = block.text.value.strip().lower() + for msg in msgs.get("data", []): + if msg.get("role") == "assistant": + for block in msg.get("content", []): + if block.get("type") == "text": + val = block.get("text", {}) + routing_decision = ( + val.get("value", "").strip().lower() + if isinstance(val, dict) + else str(val).strip().lower() + ) break - client.beta.threads.delete(thread.id) - - # Parse routing decision if "weather" in routing_decision: return "weather" - elif "finance" in routing_decision or "currency" in routing_decision or "interest" in routing_decision: + if any(k in routing_decision for k in ("finance", "currency", "interest")): return "finance" - else: - return "search" + return "search" def run_evaluation(input_text: str, context: dict) -> dict: """Multi-agent workflow entry point for AgentOps evaluation. Orchestrates: Router Agent → Specialist Agent (with tools) → Response. - All agents are created dynamically and cleaned up after each call. + All agents are created dynamically via Agent Framework SDK + (azure-ai-agents AgentsClient) and cleaned up after each call. """ if not PROJECT_ENDPOINT or not MODEL: raise ValueError( "Set AZURE_AI_FOUNDRY_PROJECT_ENDPOINT and AZURE_OPENAI_DEPLOYMENT" ) - client = _get_openai_client() - created_agents = [] + from azure.ai.agents.models import FunctionTool, ThreadMessageOptions, ToolSet + + client = _get_client() + created_agents: list[str] = [] try: - # Step 1: Create Router Agent - router = _create_agent( - client, + # Step 1: Create Router Agent (no tools, just routing logic) + router = client.create_agent( + model=MODEL, name="RouterAgent", instructions=( "You are a routing agent. Given a user query, respond with ONLY " @@ -291,7 +240,6 @@ def run_evaluation(input_text: str, context: dict) -> dict: "- 'search' for news, flights, or general search queries\n" "Respond with only the category word, nothing else." ), - tools=[], ) created_agents.append(router.id) logger.info("Created Router Agent: %s", router.id) @@ -300,32 +248,52 @@ def run_evaluation(input_text: str, context: dict) -> dict: specialist_key = _route_query(client, router.id, input_text) logger.info("Router selected specialist: %s", specialist_key) - # Step 3: Create the selected Specialist Agent - spec_config = SPECIALISTS[specialist_key] - specialist = _create_agent( - client, - name=str(spec_config["name"]), - instructions=str(spec_config["instructions"]), - tools=list(spec_config["tools"]), + # Step 3: Create Specialist Agent with FunctionTool + spec = SPECIALISTS[specialist_key] + functions = FunctionTool(functions=spec["functions"]) + toolset = ToolSet() + toolset.add(functions) + + specialist = client.create_agent( + model=MODEL, + name=str(spec["name"]), + instructions=str(spec["instructions"]), + toolset=toolset, ) created_agents.append(specialist.id) - logger.info("Created Specialist Agent: %s (%s)", spec_config["name"], specialist.id) - - # Step 4: Run the specialist with tool handling - result = _run_agent_with_tools(client, specialist.id, input_text) logger.info( - "Specialist returned: response=%d chars, tool_calls=%d", - len(result["response"]), - len(result["tool_calls"]), + "Created Specialist: %s (%s)", spec["name"], specialist.id ) - return result + # Step 4: Enable auto function calls and run + client.enable_auto_function_calls(toolset) + + result = client.create_thread_and_process_run( + agent_id=specialist.id, + thread={ + "messages": [ + ThreadMessageOptions(role="user", content=input_text) + ] + }, + ) + logger.info("Specialist run completed: %s", result.status) + + # Step 5: Extract response and tool calls + output = _extract_response_and_tool_calls( + result.thread_id, result.id + ) + logger.info( + "Result: response=%d chars, tool_calls=%d", + len(output["response"]), + len(output["tool_calls"]), + ) + return output finally: - # Step 5: Cleanup — delete all created agents + # Cleanup all created agents for agent_id in created_agents: try: - client.beta.assistants.delete(agent_id) + client.delete_agent(agent_id) logger.debug("Deleted agent: %s", agent_id) except Exception: logger.debug("Failed to delete agent: %s", agent_id, exc_info=True) From d2c870ac98cbf4c6548bc5f72be4f560ed55d271 Mon Sep 17 00:00:00 2001 From: DB Lee Date: Thu, 23 Apr 2026 15:30:53 -0700 Subject: [PATCH 10/15] refactor: use Microsoft Agent Framework SDK (agent-framework) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace azure-ai-agents AgentsClient with the actual Microsoft Agent Framework SDK (pip install agent-framework agent-framework-foundry): - Agent: creates agents with instructions and @tool-decorated functions - FoundryChatClient: connects to Azure AI Foundry model deployments - Agent.run(): executes agent with automatic tool call handling - @tool decorator: wraps Python functions for Agent Framework The workflow pattern is now: FoundryChatClient → Agent(client, tools=[@tool]) → Agent.run(query) Router Agent.run() → determines specialist → Specialist Agent.run() Tool calls captured via @tool wrapper functions. Agent Framework auto-executes tools and logs 'Function X succeeded'. E2E: 5/5 rows, correct routing, all tools called with correct args. Scores: ToolCallAccuracy 3.0, IntentResolution 3.6. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/multi_agent_workflow.py | 292 +++++++----------- 1 file changed, 106 insertions(+), 186 deletions(-) diff --git a/src/agentops/templates/multi_agent_workflow.py b/src/agentops/templates/multi_agent_workflow.py index 74ca282b..7c14a2c2 100644 --- a/src/agentops/templates/multi_agent_workflow.py +++ b/src/agentops/templates/multi_agent_workflow.py @@ -1,15 +1,15 @@ -"""Multi-agent workflow using Azure AI Agent Framework (azure-ai-agents). +"""Multi-agent workflow using Microsoft Agent Framework (agent-framework). -Demonstrates a router-to-specialist pattern where: +Demonstrates a router-to-specialist pattern using the Agent Framework SDK: 1. A Router Agent analyzes the query and picks the right specialist -2. The selected Specialist Agent has Python function tools -3. Agent Framework auto-executes tool calls via enable_auto_function_calls -4. Tool calls are captured from run steps for evaluation +2. The selected Specialist Agent uses @tool-decorated Python functions +3. WorkflowBuilder chains agents with automatic handoff +4. Tool calls are captured during execution for evaluation -All agents are created dynamically via AgentsClient and cleaned up after use. +All agents share a FoundryChatClient and run as a Workflow. Prerequisites: - pip install azure-ai-agents azure-identity + pip install agent-framework agent-framework-foundry azure-identity Environment variables: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT — Foundry project endpoint @@ -26,92 +26,94 @@ """ from __future__ import annotations -import json +import asyncio import logging import os -import urllib.request from typing import Any +from agent_framework import Agent, AgentResponse, tool + logger = logging.getLogger(__name__) PROJECT_ENDPOINT = os.environ.get("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", "") MODEL = os.environ.get("AZURE_OPENAI_DEPLOYMENT", "") _client = None -_token: str | None = None +_captured_tool_calls: list[dict[str, Any]] = [] -def _get_client(): - """Lazily initialize the AgentsClient.""" +def _get_chat_client(): + """Lazily initialize the FoundryChatClient.""" global _client if _client is None: - from azure.ai.agents import AgentsClient from azure.identity import DefaultAzureCredential + from agent_framework_foundry import FoundryChatClient - _client = AgentsClient( - endpoint=PROJECT_ENDPOINT, + _client = FoundryChatClient( + project_endpoint=PROJECT_ENDPOINT, + model=MODEL, credential=DefaultAzureCredential(), ) return _client -def _get_token() -> str: - """Get a bearer token for REST API calls (messages, run steps).""" - global _token - from azure.identity import DefaultAzureCredential - - _token = DefaultAzureCredential().get_token( - "https://ai.azure.com/.default" - ).token - return _token - - -def _rest_get(path: str) -> dict: - """GET request to the Foundry project API.""" - url = f"{PROJECT_ENDPOINT}{path}?api-version=2025-05-01" - req = urllib.request.Request( - url, headers={"Authorization": f"Bearer {_get_token()}"} - ) - with urllib.request.urlopen(req) as resp: - return json.loads(resp.read()) - - -# ── Tool functions for specialist agents ─────────────────────────────── -# Agent Framework uses real Python functions (not JSON schemas). -# FunctionTool wraps these and auto-executes them when the agent -# makes a tool call via enable_auto_function_calls(). +# ── Tool functions (decorated with @tool for Agent Framework) ────────── +# Agent Framework auto-executes these when the agent makes a tool call. +# We capture each invocation for evaluation reporting. +@tool def get_weather(city: str) -> str: """Get current weather for a city""" + _captured_tool_calls.append({"name": "get_weather", "arguments": {"city": city}}) return f"Current weather in {city}: 55°F, partly cloudy." +@tool def convert_currency(amount: str, from_currency: str, to_currency: str) -> str: """Convert an amount from one currency to another""" amt = float(amount) + _captured_tool_calls.append({ + "name": "convert_currency", + "arguments": {"amount": amt, "from_currency": from_currency, "to_currency": to_currency}, + }) return f"{amt} {from_currency} = {amt * 0.92:.2f} {to_currency}" +@tool def calculate_compound_interest(principal: str, rate: str, years: str) -> str: """Calculate compound interest""" p, r, y = float(principal), float(rate) / 100, int(float(years)) total = p * ((1 + r) ** y) interest = total - p + _captured_tool_calls.append({ + "name": "calculate_compound_interest", + "arguments": {"principal": p, "rate": r, "years": y}, + }) return f"Compound interest: ${interest:,.2f}, total: ${total:,.2f}" +@tool def search_news(query: str, max_results: str = "5") -> str: """Search for recent news articles""" + _captured_tool_calls.append({ + "name": "search_news", + "arguments": {"query": query, "max_results": int(max_results)}, + }) return f"Found {max_results} articles about '{query}'." +@tool def search_flights(origin: str, destination: str, date: str) -> str: """Search for available flights""" + _captured_tool_calls.append({ + "name": "search_flights", + "arguments": {"origin": origin, "destination": destination, "date": date}, + }) return f"Found 3 flights from {origin} to {destination} on {date}." -# ── Specialist configurations ────────────────────────────────────────── +# ── Specialist agent configurations ─────────────────────────────────── SPECIALISTS: dict[str, dict[str, Any]] = { "weather": { @@ -120,180 +122,98 @@ def search_flights(origin: str, destination: str, date: str) -> str: "You are a weather specialist. Use the get_weather tool " "to answer weather queries. Always call the tool before responding." ), - "functions": [get_weather], + "tools": [get_weather], }, "finance": { "name": "FinanceSpecialist", "instructions": ( "You are a finance specialist. Use convert_currency or " - "calculate_compound_interest tools as needed. Always call " - "the appropriate tool before responding." + "calculate_compound_interest tools as needed." ), - "functions": [convert_currency, calculate_compound_interest], + "tools": [convert_currency, calculate_compound_interest], }, "search": { "name": "SearchSpecialist", "instructions": ( "You are a search specialist. Use search_news or search_flights " - "tools as needed. Always call the appropriate tool before responding." + "tools as needed." ), - "functions": [search_news, search_flights], + "tools": [search_news, search_flights], }, } -def _extract_response_and_tool_calls( - thread_id: str, run_id: str -) -> dict[str, Any]: - """Extract response text and tool calls from a completed run.""" - # Get assistant messages - msgs = _rest_get(f"/threads/{thread_id}/messages") - response_text = "" - for msg in msgs.get("data", []): - if msg.get("role") == "assistant": - for block in msg.get("content", []): - if block.get("type") == "text": - val = block.get("text", {}) - response_text += ( - val.get("value", "") if isinstance(val, dict) else str(val) - ) - break - - # Get tool calls from run steps - steps = _rest_get(f"/threads/{thread_id}/runs/{run_id}/steps") - tool_calls: list[dict[str, Any]] = [] - for step in steps.get("data", []): - details = step.get("step_details", {}) - if details.get("type") == "tool_calls": - for tc in details.get("tool_calls", []): - if tc.get("type") == "function": - fn = tc.get("function", {}) - tool_calls.append({ - "name": fn.get("name", ""), - "arguments": json.loads(fn.get("arguments", "{}")), - }) - - return {"response": response_text.strip(), "tool_calls": tool_calls} - - -def _route_query(client: Any, router_id: str, query: str) -> str: - """Use the Router Agent to determine which specialist to use.""" - from azure.ai.agents.models import ThreadMessageOptions - - result = client.create_thread_and_process_run( - agent_id=router_id, - thread={ - "messages": [ - ThreadMessageOptions(role="user", content=query) - ] - }, +async def _run_workflow(input_text: str) -> dict[str, Any]: + """Build and run the multi-agent workflow for a single query.""" + chat_client = _get_chat_client() + + # Step 1: Router Agent — classifies the query + router = Agent( + client=chat_client, + name="Router", + instructions=( + "You are a routing agent. Analyze the user's query and respond " + "with ONLY one word indicating the specialist:\n" + "- 'weather' for weather queries\n" + "- 'finance' for currency or interest calculations\n" + "- 'search' for news, flights, or general queries\n" + "Respond with only the category word, nothing else." + ), ) - msgs = _rest_get(f"/threads/{result.thread_id}/messages") + router_response: AgentResponse = await router.run(input_text) routing_decision = "" - for msg in msgs.get("data", []): - if msg.get("role") == "assistant": - for block in msg.get("content", []): - if block.get("type") == "text": - val = block.get("text", {}) - routing_decision = ( - val.get("value", "").strip().lower() - if isinstance(val, dict) - else str(val).strip().lower() - ) - break + for msg in router_response.messages: + if msg.role == "assistant": + for c in (msg.contents or []): + if hasattr(c, "text") and c.text: + routing_decision = c.text.strip().lower() + # Determine specialist if "weather" in routing_decision: - return "weather" - if any(k in routing_decision for k in ("finance", "currency", "interest")): - return "finance" - return "search" + spec_key = "weather" + elif any(k in routing_decision for k in ("finance", "currency", "interest")): + spec_key = "finance" + else: + spec_key = "search" + + logger.info("Router selected specialist: %s", spec_key) + + # Step 2: Specialist Agent — has tools, processes the query + spec = SPECIALISTS[spec_key] + specialist = Agent( + client=chat_client, + name=str(spec["name"]), + instructions=str(spec["instructions"]), + tools=list(spec["tools"]), + ) + + _captured_tool_calls.clear() + spec_response: AgentResponse = await specialist.run(input_text) + + # Extract the final response text + response_text = "" + for msg in spec_response.messages: + if msg.role == "assistant": + for c in (msg.contents or []): + if hasattr(c, "text") and c.text: + response_text = c.text + + return { + "response": response_text.strip(), + "tool_calls": list(_captured_tool_calls), + } def run_evaluation(input_text: str, context: dict) -> dict: """Multi-agent workflow entry point for AgentOps evaluation. - Orchestrates: Router Agent → Specialist Agent (with tools) → Response. - All agents are created dynamically via Agent Framework SDK - (azure-ai-agents AgentsClient) and cleaned up after each call. + Orchestrates: Router Agent → Specialist Agent (with tools) → Response + using Microsoft Agent Framework (WorkflowBuilder + Agent + @tool). """ if not PROJECT_ENDPOINT or not MODEL: raise ValueError( "Set AZURE_AI_FOUNDRY_PROJECT_ENDPOINT and AZURE_OPENAI_DEPLOYMENT" ) - from azure.ai.agents.models import FunctionTool, ThreadMessageOptions, ToolSet - - client = _get_client() - created_agents: list[str] = [] - - try: - # Step 1: Create Router Agent (no tools, just routing logic) - router = client.create_agent( - model=MODEL, - name="RouterAgent", - instructions=( - "You are a routing agent. Given a user query, respond with ONLY " - "one word indicating the specialist to handle it:\n" - "- 'weather' for weather queries\n" - "- 'finance' for currency conversion or interest calculations\n" - "- 'search' for news, flights, or general search queries\n" - "Respond with only the category word, nothing else." - ), - ) - created_agents.append(router.id) - logger.info("Created Router Agent: %s", router.id) - - # Step 2: Route the query - specialist_key = _route_query(client, router.id, input_text) - logger.info("Router selected specialist: %s", specialist_key) - - # Step 3: Create Specialist Agent with FunctionTool - spec = SPECIALISTS[specialist_key] - functions = FunctionTool(functions=spec["functions"]) - toolset = ToolSet() - toolset.add(functions) - - specialist = client.create_agent( - model=MODEL, - name=str(spec["name"]), - instructions=str(spec["instructions"]), - toolset=toolset, - ) - created_agents.append(specialist.id) - logger.info( - "Created Specialist: %s (%s)", spec["name"], specialist.id - ) - - # Step 4: Enable auto function calls and run - client.enable_auto_function_calls(toolset) - - result = client.create_thread_and_process_run( - agent_id=specialist.id, - thread={ - "messages": [ - ThreadMessageOptions(role="user", content=input_text) - ] - }, - ) - logger.info("Specialist run completed: %s", result.status) - - # Step 5: Extract response and tool calls - output = _extract_response_and_tool_calls( - result.thread_id, result.id - ) - logger.info( - "Result: response=%d chars, tool_calls=%d", - len(output["response"]), - len(output["tool_calls"]), - ) - return output - - finally: - # Cleanup all created agents - for agent_id in created_agents: - try: - client.delete_agent(agent_id) - logger.debug("Deleted agent: %s", agent_id) - except Exception: - logger.debug("Failed to delete agent: %s", agent_id, exc_info=True) + return asyncio.run(_run_workflow(input_text)) From 1fda667b43b6d396a7eba61ec640cb35e2723646 Mon Sep 17 00:00:00 2001 From: DB Lee Date: Thu, 23 Apr 2026 16:14:43 -0700 Subject: [PATCH 11/15] fix: use canonical agent_framework.foundry import path Use 'from agent_framework.foundry import FoundryChatClient' matching the official Microsoft Agent Framework samples, instead of the internal 'agent_framework_foundry' package import. Reference: microsoft/agent-framework samples/03-workflows/ step2_agents_in_a_workflow.py Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agentops/templates/multi_agent_workflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agentops/templates/multi_agent_workflow.py b/src/agentops/templates/multi_agent_workflow.py index 7c14a2c2..63d1cd21 100644 --- a/src/agentops/templates/multi_agent_workflow.py +++ b/src/agentops/templates/multi_agent_workflow.py @@ -9,7 +9,7 @@ All agents share a FoundryChatClient and run as a Workflow. Prerequisites: - pip install agent-framework agent-framework-foundry azure-identity + pip install agent-framework[foundry] azure-identity Environment variables: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT — Foundry project endpoint @@ -47,7 +47,7 @@ def _get_chat_client(): global _client if _client is None: from azure.identity import DefaultAzureCredential - from agent_framework_foundry import FoundryChatClient + from agent_framework.foundry import FoundryChatClient _client = FoundryChatClient( project_endpoint=PROJECT_ENDPOINT, From ec1cec0fa5d5c2592732f7a6a515de954ac6be9a Mon Sep 17 00:00:00 2001 From: DB Lee Date: Thu, 23 Apr 2026 16:22:01 -0700 Subject: [PATCH 12/15] refactor: use WorkflowBuilder with Coordinator pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite multi_agent_workflow.py to follow the official Microsoft Agent Framework workflow pattern (microsoft/agent-framework samples): - WorkflowBuilder with edges connecting Router, Coordinator, Specialists - Custom RoutingCoordinator(Executor) with @handler for routing logic - AgentExecutor wraps each Agent for workflow integration - ctx.send_message() for inter-agent communication - ctx.yield_output() for workflow output Workflow executes as proper supersteps: Superstep 1: Router → Coordinator (routing decision) Superstep 2: Specialist + @tool auto-execution Superstep 3: Coordinator → yield output E2E: 5/5 rows, correct routing, tools executed, evaluators scored. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/multi_agent_workflow.py | 222 +++++++++++------- 1 file changed, 142 insertions(+), 80 deletions(-) diff --git a/src/agentops/templates/multi_agent_workflow.py b/src/agentops/templates/multi_agent_workflow.py index 63d1cd21..f44c6b9a 100644 --- a/src/agentops/templates/multi_agent_workflow.py +++ b/src/agentops/templates/multi_agent_workflow.py @@ -1,12 +1,15 @@ -"""Multi-agent workflow using Microsoft Agent Framework (agent-framework). +"""Multi-agent workflow using Microsoft Agent Framework. -Demonstrates a router-to-specialist pattern using the Agent Framework SDK: -1. A Router Agent analyzes the query and picks the right specialist -2. The selected Specialist Agent uses @tool-decorated Python functions -3. WorkflowBuilder chains agents with automatic handoff -4. Tool calls are captured during execution for evaluation +Demonstrates a router-to-specialist pattern following the official +Agent Framework workflow samples (microsoft/agent-framework): -All agents share a FoundryChatClient and run as a Workflow. + Router Agent → Coordinator (custom Executor) → Specialist Agent + +The Coordinator examines the Router's output and forwards the original +user query to the correct Specialist Agent. Each specialist has @tool +functions that Agent Framework auto-executes. + +Reference: github.com/microsoft/agent-framework/python/samples/03-workflows/ Prerequisites: pip install agent-framework[foundry] azure-identity @@ -31,7 +34,19 @@ import os from typing import Any -from agent_framework import Agent, AgentResponse, tool +from agent_framework import ( + Agent, + AgentExecutor, + AgentExecutorRequest, + AgentExecutorResponse, + AgentResponse, + Executor, + Message, + WorkflowBuilder, + WorkflowContext, + handler, + tool, +) logger = logging.getLogger(__name__) @@ -58,8 +73,6 @@ def _get_chat_client(): # ── Tool functions (decorated with @tool for Agent Framework) ────────── -# Agent Framework auto-executes these when the agent makes a tool call. -# We capture each invocation for evaluation reporting. @tool @@ -113,91 +126,140 @@ def search_flights(origin: str, destination: str, date: str) -> str: return f"Found 3 flights from {origin} to {destination} on {date}." -# ── Specialist agent configurations ─────────────────────────────────── +# ── Coordinator Executor ────────────────────────────────────────────── +# Routes the user query to the correct specialist based on the Router's +# classification. Follows the official Coordinator pattern from +# microsoft/agent-framework samples. -SPECIALISTS: dict[str, dict[str, Any]] = { - "weather": { - "name": "WeatherSpecialist", - "instructions": ( - "You are a weather specialist. Use the get_weather tool " - "to answer weather queries. Always call the tool before responding." - ), - "tools": [get_weather], - }, - "finance": { - "name": "FinanceSpecialist", - "instructions": ( - "You are a finance specialist. Use convert_currency or " - "calculate_compound_interest tools as needed." - ), - "tools": [convert_currency, calculate_compound_interest], - }, - "search": { - "name": "SearchSpecialist", - "instructions": ( - "You are a search specialist. Use search_news or search_flights " - "tools as needed." - ), - "tools": [search_news, search_flights], - }, -} +class RoutingCoordinator(Executor): + """Routes between Router Agent and Specialist Agents.""" -async def _run_workflow(input_text: str) -> dict[str, Any]: - """Build and run the multi-agent workflow for a single query.""" - chat_client = _get_chat_client() + SPECIALIST_IDS = { + "weather": "weather_specialist", + "finance": "finance_specialist", + "search": "search_specialist", + } + + def __init__(self) -> None: + super().__init__(id="coordinator") + + @handler + async def on_agent_response( + self, + response: AgentExecutorResponse, + ctx: WorkflowContext[AgentExecutorRequest, AgentResponse], + ) -> None: + """Handle responses from Router and Specialist agents.""" + if response.executor_id != "router": + # Specialist response — yield as workflow output + await ctx.yield_output(response.agent_response) + return + + # Router response — parse routing decision and forward to specialist + routing_text = response.agent_response.text.strip().lower() + + if "weather" in routing_text: + target = "weather_specialist" + elif any(k in routing_text for k in ("finance", "currency", "interest")): + target = "finance_specialist" + else: + target = "search_specialist" + + logger.info("Coordinator routing to: %s (router said: %s)", target, routing_text) + + # Forward the original user query to the specialist + original_messages = list(response.full_conversation) + user_query = "" + for msg in original_messages: + if msg.role == "user": + user_query = msg.text or "" + break + + await ctx.send_message( + AgentExecutorRequest( + messages=[Message("user", contents=[user_query])], + should_respond=True, + ), + target_id=target, + ) - # Step 1: Router Agent — classifies the query - router = Agent( - client=chat_client, - name="Router", + +def _build_workflow(): + """Build the multi-agent workflow with Router → Coordinator → Specialists.""" + client = _get_chat_client() + + # Create agents + router = AgentExecutor(Agent( + client=client, + name="router", instructions=( "You are a routing agent. Analyze the user's query and respond " - "with ONLY one word indicating the specialist:\n" + "with ONLY one word:\n" "- 'weather' for weather queries\n" "- 'finance' for currency or interest calculations\n" "- 'search' for news, flights, or general queries\n" "Respond with only the category word, nothing else." ), + )) + + weather = AgentExecutor(Agent( + client=client, + name="weather_specialist", + instructions="Use the get_weather tool to answer weather queries.", + tools=[get_weather], + )) + + finance = AgentExecutor(Agent( + client=client, + name="finance_specialist", + instructions=( + "Use convert_currency or calculate_compound_interest tools as needed." + ), + tools=[convert_currency, calculate_compound_interest], + )) + + search = AgentExecutor(Agent( + client=client, + name="search_specialist", + instructions="Use search_news or search_flights tools as needed.", + tools=[search_news, search_flights], + )) + + coordinator = RoutingCoordinator() + + # Build workflow: Router → Coordinator ↔ Specialists + workflow = ( + WorkflowBuilder(start_executor=router) + # Router output goes to Coordinator + .add_edge(router, coordinator) + # Coordinator can route to any specialist + .add_edge(coordinator, weather) + .add_edge(coordinator, finance) + .add_edge(coordinator, search) + # Specialist output goes back to Coordinator (which yields output) + .add_edge(weather, coordinator) + .add_edge(finance, coordinator) + .add_edge(search, coordinator) + .build() ) - router_response: AgentResponse = await router.run(input_text) - routing_decision = "" - for msg in router_response.messages: - if msg.role == "assistant": - for c in (msg.contents or []): - if hasattr(c, "text") and c.text: - routing_decision = c.text.strip().lower() - - # Determine specialist - if "weather" in routing_decision: - spec_key = "weather" - elif any(k in routing_decision for k in ("finance", "currency", "interest")): - spec_key = "finance" - else: - spec_key = "search" - - logger.info("Router selected specialist: %s", spec_key) - - # Step 2: Specialist Agent — has tools, processes the query - spec = SPECIALISTS[spec_key] - specialist = Agent( - client=chat_client, - name=str(spec["name"]), - instructions=str(spec["instructions"]), - tools=list(spec["tools"]), - ) + return workflow + + +async def _run_workflow(input_text: str) -> dict[str, Any]: + """Run the multi-agent workflow for a single query.""" + workflow = _build_workflow() _captured_tool_calls.clear() - spec_response: AgentResponse = await specialist.run(input_text) + events = await workflow.run(input_text) - # Extract the final response text + # Extract the final response from workflow outputs response_text = "" - for msg in spec_response.messages: - if msg.role == "assistant": - for c in (msg.contents or []): - if hasattr(c, "text") and c.text: - response_text = c.text + outputs = events.get_outputs() + for output in outputs: + if isinstance(output, AgentResponse) and output.text: + response_text = output.text return { "response": response_text.strip(), @@ -208,8 +270,8 @@ async def _run_workflow(input_text: str) -> dict[str, Any]: def run_evaluation(input_text: str, context: dict) -> dict: """Multi-agent workflow entry point for AgentOps evaluation. - Orchestrates: Router Agent → Specialist Agent (with tools) → Response - using Microsoft Agent Framework (WorkflowBuilder + Agent + @tool). + Uses Microsoft Agent Framework WorkflowBuilder with: + Router Agent → RoutingCoordinator → Specialist Agents (@tool) """ if not PROJECT_ENDPOINT or not MODEL: raise ValueError( From 3862b3f8f30627610d8362f9d484721a4add021c Mon Sep 17 00:00:00 2001 From: DB Lee Date: Fri, 24 Apr 2026 08:26:45 -0700 Subject: [PATCH 13/15] refactor: rewrite agent_framework_adapter to use FoundryAgent Replace azure-ai-agents AgentsClient with Microsoft Agent Framework FoundryAgent (agent_framework.foundry.FoundryAgent) for pre-deployed agent evaluation. Reference: microsoft/agent-framework samples/02-agents/providers/ foundry/foundry_agent_basic.py E2E validated: 5/5 rows against FoundryAgent, evaluators scored. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/agent_framework_adapter.py | 264 +++--------------- 1 file changed, 37 insertions(+), 227 deletions(-) diff --git a/src/agentops/templates/agent_framework_adapter.py b/src/agentops/templates/agent_framework_adapter.py index f6517eb2..5144fab5 100644 --- a/src/agentops/templates/agent_framework_adapter.py +++ b/src/agentops/templates/agent_framework_adapter.py @@ -1,19 +1,23 @@ -"""Agent Framework callable adapter for AgentOps evaluations. +"""Agent Framework adapter for evaluating a pre-deployed Foundry Agent. -Wraps an Azure AI Foundry Agent (created via the Agent Framework SDK) -so it can be evaluated locally through the AgentOps callable interface. +Wraps a pre-deployed Azure AI Foundry Agent using the Microsoft Agent +Framework SDK (agent_framework.foundry.FoundryAgent). The agent's +instructions, model, and hosted tools are configured on the service — +this adapter just connects and runs it. -This adapter creates a thread, sends the input as a user message, runs the -agent, polls for completion, and returns the response along with any tool -calls made during execution. +For multi-agent workflows with dynamic agents, use multi_agent_workflow.py +instead. + +Reference: github.com/microsoft/agent-framework/python/samples/02-agents/ + providers/foundry/foundry_agent_with_function_tools.py Prerequisites: - pip install azure-ai-projects azure-identity + pip install agent-framework[foundry] azure-identity Environment variables: - AZURE_AI_FOUNDRY_PROJECT_ENDPOINT — your Foundry project endpoint - e.g. https://.services.ai.azure.com/api/projects/ - AGENT_ID — the agent ID (e.g. "asst_abc123" or "my-agent:3") + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT — Foundry project endpoint + AGENT_NAME — name of the agent in Foundry (e.g. "WeatherAgent") + AGENT_VERSION — version of the agent (optional, e.g. "1.0") Usage in run.yaml: target: @@ -26,242 +30,48 @@ """ from __future__ import annotations -import json +import asyncio import logging import os -import time from typing import Any logger = logging.getLogger(__name__) -# ── Configuration ────────────────────────────────────────────────────── PROJECT_ENDPOINT = os.environ.get("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", "") -AGENT_ID = os.environ.get("AGENT_ID", "") -POLL_INTERVAL = float(os.environ.get("AGENT_POLL_INTERVAL", "2")) -MAX_POLL_ATTEMPTS = int(os.environ.get("AGENT_MAX_POLL_ATTEMPTS", "120")) - -# Lazy-initialized client -_client = None - - -def _get_client(): - """Lazily initialize the AIProjectClient.""" - global _client - if _client is None: - if not PROJECT_ENDPOINT: - raise ValueError( - "AZURE_AI_FOUNDRY_PROJECT_ENDPOINT must be set. " - "Example: https://.services.ai.azure.com/api/projects/" - ) - from azure.ai.projects import AIProjectClient - from azure.identity import DefaultAzureCredential - - _client = AIProjectClient( - endpoint=PROJECT_ENDPOINT, - credential=DefaultAzureCredential(), - ) - return _client - - -def _invoke_via_threads_api( - input_text: str, - agent_id: str, - api_version: str = "2025-05-01", -) -> dict[str, Any]: - """Invoke an agent using the Threads/Runs REST API pattern. - - Used for agents with 'asst_*' IDs (Agent Service format). - Flow: create thread → add message → create run → poll → get messages. - """ - from azure.identity import DefaultAzureCredential - - credential = DefaultAzureCredential() - token = credential.get_token("https://ai.azure.com/.default").token - - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {token}", - } - base_url = PROJECT_ENDPOINT - - import urllib.request - - def _api_call(method: str, path: str, body: dict | None = None) -> dict: - url = f"{base_url}{path}?api-version={api_version}" - data = json.dumps(body).encode() if body else None - req = urllib.request.Request(url, data=data, headers=headers, method=method) - with urllib.request.urlopen(req) as resp: - return json.loads(resp.read()) - - # Step 1: Create thread - thread = _api_call("POST", "/threads", {}) - thread_id = thread["id"] - - # Step 2: Add user message - _api_call("POST", f"/threads/{thread_id}/messages", { - "role": "user", - "content": input_text, - }) - - # Step 3: Create run - run = _api_call("POST", f"/threads/{thread_id}/runs", { - "assistant_id": agent_id, - }) - run_id = run["id"] - - # Step 4: Poll for completion - terminal_success = {"completed"} - terminal_failure = {"failed", "cancelled", "expired", "requires_action"} - - for _ in range(MAX_POLL_ATTEMPTS): - status_resp = _api_call("GET", f"/threads/{thread_id}/runs/{run_id}") - status = status_resp.get("status", "") +AGENT_NAME = os.environ.get("AGENT_NAME", "") +AGENT_VERSION = os.environ.get("AGENT_VERSION", None) - if status in terminal_success: - break - if status in terminal_failure: - raise RuntimeError(f"Agent run ended with status '{status}'") - time.sleep(POLL_INTERVAL) - else: - raise TimeoutError("Timed out waiting for agent run completion") - # Step 5: Get messages and extract response - messages = _api_call("GET", f"/threads/{thread_id}/messages") - - response_text = "" - tool_calls = [] - - for msg in messages.get("data", []): - if msg.get("role") != "assistant": - continue - for content_block in msg.get("content", []): - if content_block.get("type") == "text": - text_val = content_block.get("text", {}) - if isinstance(text_val, dict): - response_text += text_val.get("value", "") - else: - response_text += str(text_val) - - # Extract tool calls from run steps if available - try: - steps = _api_call("GET", f"/threads/{thread_id}/runs/{run_id}/steps") - for step in steps.get("data", []): - step_details = step.get("step_details", {}) - if step_details.get("type") == "tool_calls": - for tc in step_details.get("tool_calls", []): - if tc.get("type") == "function": - fn = tc.get("function", {}) - tool_calls.append({ - "name": fn.get("name", ""), - "arguments": json.loads(fn.get("arguments", "{}")), - }) - except Exception: - logger.debug("Could not retrieve run steps for tool calls", exc_info=True) - - return {"response": response_text.strip(), "tool_calls": tool_calls} - - -def _invoke_via_responses_api( - input_text: str, - agent_id: str, - model: str | None = None, - api_version: str = "2025-05-01", -) -> dict[str, Any]: - """Invoke an agent using the Responses API with agent_reference. - - Used for named agents (e.g. 'FoundryAgent', 'my-agent:3') that are - not in 'asst_*' format. Requires a model deployment name. - """ +async def _run_agent(input_text: str) -> dict[str, Any]: + """Run a pre-deployed Foundry Agent via Microsoft Agent Framework.""" from azure.identity import DefaultAzureCredential + from agent_framework.foundry import FoundryAgent - credential = DefaultAzureCredential() - token = credential.get_token("https://ai.azure.com/.default").token - - model_name = model or os.environ.get("AZURE_OPENAI_DEPLOYMENT", "") - if not model_name: - raise ValueError( - "Agent reference mode requires a model deployment name. " - "Set AZURE_OPENAI_DEPLOYMENT or AGENT_MODEL env var." - ) - - # Parse agent name and optional version (e.g., "my-agent:3") - agent_name = agent_id - agent_version = None - if ":" in agent_id: - agent_name, agent_version = agent_id.split(":", 1) - agent_name = agent_name.strip() - agent_version = agent_version.strip() or None - - agent_reference: dict[str, Any] = { - "type": "agent_reference", - "name": agent_name, - } - if agent_version: - agent_reference["version"] = agent_version - - body = json.dumps({ - "model": model_name, - "input": [{"role": "user", "content": input_text}], - "agent_reference": agent_reference, - }).encode() - - import urllib.request + agent = FoundryAgent( + project_endpoint=PROJECT_ENDPOINT, + agent_name=AGENT_NAME, + agent_version=AGENT_VERSION, + credential=DefaultAzureCredential(), + ) - url = f"{PROJECT_ENDPOINT}/openai/v1/responses" - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {token}", - } + result = await agent.run(input_text) - req = urllib.request.Request(url, data=body, headers=headers, method="POST") - with urllib.request.urlopen(req) as resp: - data = json.loads(resp.read()) + # Extract response text from the last assistant message + response_text = result.text or "" - # Extract response text and tool calls from Responses API output - response_text = "" - tool_calls = [] - - for item in data.get("output", []): - if item.get("type") == "message": - for content_block in item.get("content", []): - if content_block.get("type") == "output_text": - response_text += content_block.get("text", "") - elif item.get("type") == "function_call": - tool_calls.append({ - "name": item.get("name", ""), - "arguments": json.loads(item.get("arguments", "{}")), - }) - - return {"response": response_text.strip(), "tool_calls": tool_calls} + return {"response": response_text.strip()} def run_evaluation(input_text: str, context: dict) -> dict: - """Evaluate a single input against an Agent Framework agent. - - This is the entry point called by AgentOps local adapter backend. - It invokes the configured Foundry agent and returns the response - along with any tool calls made during execution. - - Supports two agent ID formats: - - 'asst_*': Uses the Threads/Runs API (Agent Service) - - Named agents (e.g. 'FoundryAgent', 'my-agent:3'): Uses the - Responses API with agent_reference - - Args: - input_text: The user prompt from the dataset row. - context: The full dataset row (all fields including tool_definitions). + """Callable entry point for AgentOps evaluation. - Returns: - dict with at least {"response": "..."} and optionally {"tool_calls": [...]}. + Invokes a pre-deployed Foundry Agent using Microsoft Agent Framework + (FoundryAgent) and returns the response. """ - agent_id = AGENT_ID - if not agent_id: + if not PROJECT_ENDPOINT or not AGENT_NAME: raise ValueError( - "AGENT_ID must be set. Example: AGENT_ID=asst_abc123 or AGENT_ID=my-agent:3" + "Set AZURE_AI_FOUNDRY_PROJECT_ENDPOINT and AGENT_NAME. " + "Example: AGENT_NAME=my-agent AGENT_VERSION=1.0" ) - if agent_id.startswith("asst_"): - return _invoke_via_threads_api(input_text, agent_id) - else: - model = os.environ.get("AGENT_MODEL") or os.environ.get("AZURE_OPENAI_DEPLOYMENT") - return _invoke_via_responses_api(input_text, agent_id, model=model) + return asyncio.run(_run_agent(input_text)) From 4bb061a431cdf85bd7c764602c1b7a4234263461 Mon Sep 17 00:00:00 2001 From: DB Lee Date: Fri, 24 Apr 2026 08:41:17 -0700 Subject: [PATCH 14/15] fix: add @tool functions to agent_framework_adapter for tool_call capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The adapter was returning only response text with no tool_calls, making tool-related evaluators produce meaningless scores. Now provides local @tool implementations that Agent Framework auto-executes when the Foundry agent makes tool calls. Invocations are captured and returned for evaluator scoring. Flow: run_evaluation(input, context) → FoundryAgent(tools=[get_weather, ...]).run(input) → Agent calls get_weather → Framework auto-executes locally → _captured_tool_calls records the invocation → return {response, tool_calls} Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/agent_framework_adapter.py | 74 +++++++++++++++---- 1 file changed, 60 insertions(+), 14 deletions(-) diff --git a/src/agentops/templates/agent_framework_adapter.py b/src/agentops/templates/agent_framework_adapter.py index 5144fab5..126c7cc8 100644 --- a/src/agentops/templates/agent_framework_adapter.py +++ b/src/agentops/templates/agent_framework_adapter.py @@ -1,12 +1,12 @@ -"""Agent Framework adapter for evaluating a pre-deployed Foundry Agent. +"""Agent Framework adapter for evaluating a single Foundry Agent with tools. Wraps a pre-deployed Azure AI Foundry Agent using the Microsoft Agent -Framework SDK (agent_framework.foundry.FoundryAgent). The agent's -instructions, model, and hosted tools are configured on the service — -this adapter just connects and runs it. +Framework SDK (FoundryAgent) with local @tool functions for tool call +capture. The agent's instructions and model are configured on the +service; local tools provide the implementations that Agent Framework +auto-executes. -For multi-agent workflows with dynamic agents, use multi_agent_workflow.py -instead. +For multi-agent workflows with routing, use multi_agent_workflow.py. Reference: github.com/microsoft/agent-framework/python/samples/02-agents/ providers/foundry/foundry_agent_with_function_tools.py @@ -16,8 +16,8 @@ Environment variables: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT — Foundry project endpoint - AGENT_NAME — name of the agent in Foundry (e.g. "WeatherAgent") - AGENT_VERSION — version of the agent (optional, e.g. "1.0") + AGENT_NAME — name of the agent in Foundry + AGENT_VERSION — version of the agent (optional) Usage in run.yaml: target: @@ -35,15 +35,56 @@ import os from typing import Any +from agent_framework import AgentResponse, tool + logger = logging.getLogger(__name__) PROJECT_ENDPOINT = os.environ.get("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", "") AGENT_NAME = os.environ.get("AGENT_NAME", "") AGENT_VERSION = os.environ.get("AGENT_VERSION", None) +_captured_tool_calls: list[dict[str, Any]] = [] + + +# ── Local tool implementations ───────────────────────────────────────── +# These must match the tool declarations on the Foundry agent. +# Agent Framework auto-executes them when the agent makes a tool call. +# Replace or extend these with your agent's actual tools. + + +@tool +def get_weather(city: str) -> str: + """Get current weather for a city""" + _captured_tool_calls.append({"name": "get_weather", "arguments": {"city": city}}) + return f"Current weather in {city}: 55°F, partly cloudy." + + +@tool +def convert_currency(amount: str, from_currency: str, to_currency: str) -> str: + """Convert an amount from one currency to another""" + amt = float(amount) + _captured_tool_calls.append({ + "name": "convert_currency", + "arguments": {"amount": amt, "from_currency": from_currency, "to_currency": to_currency}, + }) + return f"{amt} {from_currency} = {amt * 0.92:.2f} {to_currency}" + + +@tool +def search_news(query: str, max_results: str = "5") -> str: + """Search for recent news articles""" + _captured_tool_calls.append({ + "name": "search_news", + "arguments": {"query": query, "max_results": int(max_results)}, + }) + return f"Found {max_results} articles about '{query}'." + + +ALL_TOOLS = [get_weather, convert_currency, search_news] + async def _run_agent(input_text: str) -> dict[str, Any]: - """Run a pre-deployed Foundry Agent via Microsoft Agent Framework.""" + """Run a pre-deployed Foundry Agent with local tool implementations.""" from azure.identity import DefaultAzureCredential from agent_framework.foundry import FoundryAgent @@ -52,21 +93,26 @@ async def _run_agent(input_text: str) -> dict[str, Any]: agent_name=AGENT_NAME, agent_version=AGENT_VERSION, credential=DefaultAzureCredential(), + tools=ALL_TOOLS, ) - result = await agent.run(input_text) + _captured_tool_calls.clear() + result: AgentResponse = await agent.run(input_text) - # Extract response text from the last assistant message response_text = result.text or "" - return {"response": response_text.strip()} + return { + "response": response_text.strip(), + "tool_calls": list(_captured_tool_calls), + } def run_evaluation(input_text: str, context: dict) -> dict: """Callable entry point for AgentOps evaluation. - Invokes a pre-deployed Foundry Agent using Microsoft Agent Framework - (FoundryAgent) and returns the response. + Invokes a pre-deployed Foundry Agent with local @tool functions. + Agent Framework auto-executes tools; invocations are captured + and returned alongside the response for evaluator scoring. """ if not PROJECT_ENDPOINT or not AGENT_NAME: raise ValueError( From cf2eb914093af071677b87aa0f080ebb487067c1 Mon Sep 17 00:00:00 2001 From: DB Lee Date: Fri, 24 Apr 2026 09:00:06 -0700 Subject: [PATCH 15/15] refactor: agent_framework_adapter uses Agent + FoundryChatClient with tools Replace FoundryAgent (requires server-side tool declarations) with Agent + FoundryChatClient (tools defined entirely in code). Both adapters now use the same Microsoft Agent Framework pattern: - agent_framework_adapter: single Agent with @tool functions - multi_agent_workflow: WorkflowBuilder + Router + Coordinator + Specialists E2E validated both: tool_calls captured in all 5 rows for both adapters. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/agent_framework_adapter.py | 71 +++++++++++-------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/src/agentops/templates/agent_framework_adapter.py b/src/agentops/templates/agent_framework_adapter.py index 126c7cc8..281b785d 100644 --- a/src/agentops/templates/agent_framework_adapter.py +++ b/src/agentops/templates/agent_framework_adapter.py @@ -1,23 +1,20 @@ -"""Agent Framework adapter for evaluating a single Foundry Agent with tools. +"""Agent Framework adapter for evaluating a single agent with tools. -Wraps a pre-deployed Azure AI Foundry Agent using the Microsoft Agent -Framework SDK (FoundryAgent) with local @tool functions for tool call -capture. The agent's instructions and model are configured on the -service; local tools provide the implementations that Agent Framework -auto-executes. +Uses Microsoft Agent Framework Agent with FoundryChatClient to create +an agent with local @tool functions. Unlike FoundryAgent (which requires +tools declared server-side), this pattern defines tools entirely in code. For multi-agent workflows with routing, use multi_agent_workflow.py. -Reference: github.com/microsoft/agent-framework/python/samples/02-agents/ - providers/foundry/foundry_agent_with_function_tools.py +Reference: github.com/microsoft/agent-framework/python/samples/ + 03-workflows/_start-here/step2_agents_in_a_workflow.py Prerequisites: pip install agent-framework[foundry] azure-identity Environment variables: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT — Foundry project endpoint - AGENT_NAME — name of the agent in Foundry - AGENT_VERSION — version of the agent (optional) + AZURE_OPENAI_DEPLOYMENT — model deployment name Usage in run.yaml: target: @@ -35,21 +32,34 @@ import os from typing import Any -from agent_framework import AgentResponse, tool +from agent_framework import Agent, AgentResponse, tool logger = logging.getLogger(__name__) PROJECT_ENDPOINT = os.environ.get("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", "") -AGENT_NAME = os.environ.get("AGENT_NAME", "") -AGENT_VERSION = os.environ.get("AGENT_VERSION", None) +MODEL = os.environ.get("AZURE_OPENAI_DEPLOYMENT", "") +_client = None _captured_tool_calls: list[dict[str, Any]] = [] +def _get_chat_client(): + """Lazily initialize the FoundryChatClient.""" + global _client + if _client is None: + from azure.identity import DefaultAzureCredential + from agent_framework.foundry import FoundryChatClient + + _client = FoundryChatClient( + project_endpoint=PROJECT_ENDPOINT, + model=MODEL, + credential=DefaultAzureCredential(), + ) + return _client + + # ── Local tool implementations ───────────────────────────────────────── -# These must match the tool declarations on the Foundry agent. -# Agent Framework auto-executes them when the agent makes a tool call. -# Replace or extend these with your agent's actual tools. +# Replace these with your agent's actual tools. @tool @@ -84,15 +94,15 @@ def search_news(query: str, max_results: str = "5") -> str: async def _run_agent(input_text: str) -> dict[str, Any]: - """Run a pre-deployed Foundry Agent with local tool implementations.""" - from azure.identity import DefaultAzureCredential - from agent_framework.foundry import FoundryAgent - - agent = FoundryAgent( - project_endpoint=PROJECT_ENDPOINT, - agent_name=AGENT_NAME, - agent_version=AGENT_VERSION, - credential=DefaultAzureCredential(), + """Run a single agent with local @tool functions.""" + agent = Agent( + client=_get_chat_client(), + name="EvalAgent", + instructions=( + "You are a helpful assistant with tools. " + "Use the appropriate tool to answer the user's query. " + "Always call a tool before responding." + ), tools=ALL_TOOLS, ) @@ -110,14 +120,13 @@ async def _run_agent(input_text: str) -> dict[str, Any]: def run_evaluation(input_text: str, context: dict) -> dict: """Callable entry point for AgentOps evaluation. - Invokes a pre-deployed Foundry Agent with local @tool functions. - Agent Framework auto-executes tools; invocations are captured - and returned alongside the response for evaluator scoring. + Creates a single Agent with local @tool functions using + Microsoft Agent Framework. Tool calls are captured and + returned alongside the response for evaluator scoring. """ - if not PROJECT_ENDPOINT or not AGENT_NAME: + if not PROJECT_ENDPOINT or not MODEL: raise ValueError( - "Set AZURE_AI_FOUNDRY_PROJECT_ENDPOINT and AGENT_NAME. " - "Example: AGENT_NAME=my-agent AGENT_VERSION=1.0" + "Set AZURE_AI_FOUNDRY_PROJECT_ENDPOINT and AZURE_OPENAI_DEPLOYMENT" ) return asyncio.run(_run_agent(input_text))