From 1294e82500be55306ee03b7e83dd0eb358ee3d04 Mon Sep 17 00:00:00 2001 From: Gen TANG Date: Sun, 2 Aug 2026 22:50:12 +0800 Subject: [PATCH 1/4] add trace detail module --- README.md | 30 ++++- README.zh-CN.md | 25 +++- docs/architecture.md | 26 ++-- docs/evaluation.md | 3 + src/yada/agents/default.py | 52 +++++++- src/yada/evals/agents/yada.py | 9 +- src/yada/evals/cli.py | 6 + src/yada/models/deepseek.py | 50 ++++++-- src/yada/run/cli.py | 13 +- src/yada/traces/__init__.py | 11 +- src/yada/traces/cli.py | 26 +++- src/yada/traces/jsonl.py | 54 +++++++- src/yada/traces/provenance.py | 61 +++++++++ src/yada/traces/report.py | 69 +++++++++- tests/agents/test_default.py | 59 ++++++++- tests/evals/test_yada_agent.py | 69 ++++++++++ tests/models/test_deepseek.py | 5 + tests/traces/test_trace_report.py | 207 +++++++++++++++++++++++++++++- 18 files changed, 727 insertions(+), 48 deletions(-) create mode 100644 src/yada/traces/provenance.py create mode 100644 tests/evals/test_yada_agent.py diff --git a/README.md b/README.md index f39de63..98cc9fc 100644 --- a/README.md +++ b/README.md @@ -138,21 +138,37 @@ Tools: - `finish`: rejected until a test or build succeeds after the latest patch. DeepSeek thinking-mode `reasoning_content` is retained in memory and passed back -after tool calls, as required by the API. JSONL traces redact the reasoning text -by default while retaining its length and hash. Each event includes a schema -version, run ID, sequence, and elapsed time; model context growth and tool-call -latency are recorded as explicit events. Use `--trace-reasoning` only if you -intentionally want to store raw reasoning. +after tool calls, as required by the API. The default `--trace-level summary` +records compact context metrics. `--trace-level debug` additionally records the +complete sanitized provider payload for every model turn. JSONL traces redact +reasoning text by default while retaining its length and hash, and also redact +common API keys, authorization values, tokens, passwords, and secrets. Use +`--trace-reasoning` only if you intentionally want to store raw reasoning. + +Capture a replayable debug trace during an evaluation: + +```bash +uv run yada eval \ + --case benchmarks/swebench_verified/pytest-10051 \ + --agent yada \ + --yes \ + --trace-level debug +``` Inspect a completed or interrupted run without manually scanning JSONL: ```bash uv run yada-trace .yada/runs/20260801T120000.000000Z.jsonl +uv run yada-trace eval-results/.artifacts/yada-trace.jsonl --step 8 +uv run yada-trace eval-results/.artifacts/yada-trace.jsonl --verbose ``` The report correlates model requests, tool-call IDs, errors, reminders, and the -final verification state into a compact timeline. The source JSONL remains the -durable, streaming-friendly record. +final verification state into a compact timeline. `--step` and `--verbose` +expand sanitized model messages, tool arguments, patches, stdout, and stderr. +The source JSONL remains the durable, streaming-friendly record. Debug traces can +contain source code and test output even after secret redaction, so handle them as +sensitive artifacts. ## Safety model diff --git a/README.zh-CN.md b/README.zh-CN.md index 0f80ef5..868b395 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -97,18 +97,33 @@ DeepSeek 工具调用 - `finish`：最新修改后没有成功测试或构建时直接拒绝。 DeepSeek 思考模式要求工具轮次继续回传 `reasoning_content`。Yada 会在内存中 -保留并正确回传，但 JSONL 轨迹默认只记录长度和 Hash；每个事件还带有 -`schema_version`、`run_id`、严格递增序号与累计耗时，并记录上下文增长、 -模型错误和工具耗时。只有显式使用 `--trace-reasoning` 才会落盘完整推理。 +保留并正确回传。默认的 `--trace-level summary` 只记录紧凑的上下文指标； +`--trace-level debug` 还会保存每轮模型请求的完整脱敏 provider payload。 +JSONL 默认只保留 reasoning 的长度和 Hash，并脱敏常见 API key、 +Authorization、token、password 和 secret。只有显式使用 +`--trace-reasoning` 才会落盘完整推理。 + +在评测中生成可还原的 debug trace： + +```bash +uv run yada eval \ + --case benchmarks/swebench_verified/pytest-10051 \ + --agent yada \ + --yes \ + --trace-level debug +``` 无需手工翻阅 JSONL，可以直接生成关联后的诊断时间线： ```bash uv run yada-trace .yada/runs/20260801T120000.000000Z.jsonl +uv run yada-trace eval-results/.artifacts/yada-trace.jsonl --step 8 +uv run yada-trace eval-results/.artifacts/yada-trace.jsonl --verbose ``` -报告会汇总模型轮次、工具调用 ID、失败、协议提醒和最终验证状态；JSONL 仍然是 -可流式写入、崩溃后可恢复检查的原始记录。 +报告会汇总模型轮次、工具调用 ID、失败、协议提醒和最终验证状态。 +`--step` 和 `--verbose` 会展开脱敏后的模型消息、工具参数、Patch、stdout 和 +stderr。Debug trace 脱敏后仍可能包含源码和测试输出，应当作敏感 artifact 处理。 ## 安全边界 diff --git a/docs/architecture.md b/docs/architecture.md index 673cabc..ec56400 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -89,15 +89,23 @@ verification gate; only `test` and `build` do. ## Trace diagnostics JSONL is the crash-safe source of truth, but it is not the debugging interface. -Each event includes `schema_version`, `run_id`, `sequence`, UTC time, and elapsed -milliseconds. Model requests capture message count and serialized context size; -responses capture provider metadata and latency; tool calls and results share a -`tool_call_id`; `plan_decision` records the side-effect-free policy outcome. -`model_error` leaves an explicit failed request in interrupted runs. - -`yada-trace PATH` validates this stream and renders a bounded timeline with run -outcome, tool failures, protocol reminders, and per-step durations. It never -echoes full prompts, arguments, reasoning, or command output into the summary. +Schema v2 has two capture levels. `summary` records context size and event timing; +`debug` also stores the sanitized provider payload built by the same client method +used for the HTTP request. Responses, planner decisions, tool calls, and tool +results remain correlated by step, request ID, and tool-call ID. `run_start` +records Yada version/commit, workspace base commit, case ID when available, and +the model configuration. + +`yada-trace PATH` renders the compact summary. `--step N` expands one request → +response → tools slice, while `--verbose` expands the full timeline. Reasoning +is length/hash-redacted unless `--trace-reasoning` is explicit. Common secret keys +and bearer/API-key-like text are redacted in both modes. A debug trace can still +contain source code and test output and must be handled as a sensitive artifact. + +The MVP stores a full sanitized request snapshot per turn. This deliberately +favors deterministic inspection over delta complexity; content-addressed prompts +or message deltas can replace it later if measured trace size justifies the added +reader and compatibility cost. ## Security boundary diff --git a/docs/evaluation.md b/docs/evaluation.md index df8a6e1..4eba07b 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -107,6 +107,9 @@ benchmark uses uv. ## Agent adapters The native adapter constructs Yada in process and records a normal JSONL trace. +Use `--trace-level debug` to capture sanitized provider payloads for replayable +step inspection, then use `yada-trace PATH --step N` or `--verbose` to expand it. +The default `summary` level keeps request storage compact. It respects step and output-token budgets. A deployment-level container or process supervisor should enforce the hard wall-time limit. diff --git a/src/yada/agents/default.py b/src/yada/agents/default.py index f37787f..98fc11b 100644 --- a/src/yada/agents/default.py +++ b/src/yada/agents/default.py @@ -17,6 +17,7 @@ from yada.models import CompletionClient from yada.tools import ToolRunner from yada.traces import TraceWriter +from yada.traces.provenance import client_trace_config, collect_provenance @dataclass(frozen=True) @@ -49,6 +50,7 @@ class Agent: emit: Function used for concise interactive output. planner: Optional policy replacement for tests or experiments. executor: Optional execution replacement for tests or experiments. + trace_metadata: Optional benchmark or caller metadata added to provenance. """ def __init__( @@ -61,6 +63,7 @@ def __init__( emit: Callable[[str], None] = print, planner: Planner | None = None, executor: Executor | None = None, + trace_metadata: dict[str, Any] | None = None, ) -> None: if max_steps < 1: raise ValueError("max_steps must be positive") @@ -71,6 +74,7 @@ def __init__( self.emit = emit self.planner = planner or Planner() self.executor = executor or Executor(tools=tools, trace=trace, emit=emit) + self.trace_metadata = dict(trace_metadata or {}) def run(self, task: str) -> AgentResult: """Run a coding task until verified completion or the step limit. @@ -95,6 +99,12 @@ def run(self, task: str) -> AgentResult: "task": task, "workspace": str(self.tools.workspace.root), "max_steps": self.max_steps, + "trace_level": self.trace.level, + "model_config": client_trace_config(self.client), + "provenance": collect_provenance( + self.tools.workspace.root, + extra=self.trace_metadata, + ), }, ) @@ -102,9 +112,27 @@ def run(self, task: str) -> AgentResult: for step in range(1, self.max_steps + 1): self.emit(f"\n[{step}/{self.max_steps}] Asking DeepSeek...") context_metrics = _message_metrics(messages) + request_id = f"{self.trace.run_id}:model:{step}" + request_record: dict[str, Any] = { + "step": step, + "request_id": request_id, + "context": context_metrics, + } + if self.trace.level == "debug": + request_record["payload"] = _model_request_payload( + self.client, + messages, + self.tools.schemas, + ) + request_record["capture"] = { + "reasoning": ( + "included" if self.trace.include_reasoning else "redacted" + ), + "secrets": "redacted", + } self.trace.write( "model_request", - {"step": step, "context": context_metrics}, + request_record, ) started = time.monotonic() try: @@ -118,6 +146,7 @@ def run(self, task: str) -> AgentResult: "model_error", { "step": step, + "request_id": request_id, "duration_ms": round((time.monotonic() - started) * 1000), "error_type": type(exc).__name__, "error": str(exc), @@ -132,6 +161,7 @@ def run(self, task: str) -> AgentResult: "assistant", { "step": step, + "request_id": request_id, "duration_ms": duration_ms, "request_context": context_metrics, "message": assistant_message, @@ -218,6 +248,26 @@ def _message_metrics(messages: list[dict[str, Any]]) -> dict[str, int]: } +def _model_request_payload( + client: CompletionClient, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], +) -> dict[str, Any]: + """Return the provider payload, with a model-neutral fallback for test clients.""" + + builder = getattr(client, "request_payload", None) + if callable(builder): + payload = builder(messages=messages, tools=tools) + if not isinstance(payload, dict): + raise TypeError("completion client request_payload must return an object") + return payload + return { + "model": getattr(client, "model", "unknown"), + "messages": messages, + "tools": tools, + } + + def _merge_usage(total: dict[str, int], current: dict[str, Any]) -> None: """Flatten and add integer usage counters reported by the provider.""" diff --git a/src/yada/evals/agents/yada.py b/src/yada/evals/agents/yada.py index d7757ad..4c12872 100644 --- a/src/yada/evals/agents/yada.py +++ b/src/yada/evals/agents/yada.py @@ -32,6 +32,7 @@ def __init__( command_timeout_seconds: int = 120, command_policy: str = "ask", include_reasoning: bool = False, + trace_level: str = "summary", client_factory: Callable[[RunBudget], CompletionClient] | None = None, emit: Callable[[str], None] = print, ) -> None: @@ -46,6 +47,7 @@ def __init__( self.command_timeout_seconds = command_timeout_seconds self.command_policy = command_policy self.include_reasoning = include_reasoning + self.trace_level = trace_level self.client_factory = client_factory self.emit = emit @@ -78,9 +80,14 @@ def run( agent = Agent( client=client, tools=tools, - trace=TraceWriter(trace_path, include_reasoning=self.include_reasoning), + trace=TraceWriter( + trace_path, + include_reasoning=self.include_reasoning, + level=self.trace_level, + ), max_steps=budget.max_steps, emit=self.emit, + trace_metadata={"case_id": prepared.task.instance_id}, ) started = time.monotonic() diff --git a/src/yada/evals/cli.py b/src/yada/evals/cli.py index 746a479..c6c1af2 100644 --- a/src/yada/evals/cli.py +++ b/src/yada/evals/cli.py @@ -83,6 +83,11 @@ def build_parser() -> argparse.ArgumentParser: ) model.add_argument("--yes", action="store_true") model.add_argument("--trace-reasoning", action="store_true") + model.add_argument( + "--trace-level", + choices=["summary", "debug"], + default="summary", + ) local = parser.add_argument_group("local benchmark") local.add_argument("--manifest", type=Path) @@ -187,6 +192,7 @@ def run_cli(argv: list[str] | None = None) -> int: command_timeout_seconds=args.command_timeout, command_policy="allow" if args.yes else args.command_policy, include_reasoning=args.trace_reasoning, + trace_level=args.trace_level, ) else: if not args.agent_command: diff --git a/src/yada/models/deepseek.py b/src/yada/models/deepseek.py index f7e3d55..81dea1c 100644 --- a/src/yada/models/deepseek.py +++ b/src/yada/models/deepseek.py @@ -69,19 +69,7 @@ def complete( DeepSeekAPIError: If transport retries fail or the response is malformed. """ - payload: dict[str, Any] = { - "model": self.model, - "messages": messages, - "tools": tools, - "max_tokens": self.max_output_tokens, - "thinking": {"type": "enabled" if self.thinking else "disabled"}, - } - if self.thinking: - payload["reasoning_effort"] = self.reasoning_effort - else: - # DeepSeek V4 thinking mode rejects tool_choice. Non-thinking mode - # accepts the ordinary OpenAI-compatible automatic selection value. - payload["tool_choice"] = "auto" + payload = self.request_payload(messages=messages, tools=tools) body = json.dumps(payload, ensure_ascii=False).encode("utf-8") request = urllib.request.Request( @@ -121,6 +109,42 @@ def complete( finish_reason=choice.get("finish_reason"), ) + def request_payload( + self, + *, + messages: list[dict[str, Any]], + tools: list[dict[str, Any]], + ) -> dict[str, Any]: + """Build the exact JSON body used by ``complete`` for tracing parity.""" + + payload: dict[str, Any] = { + "model": self.model, + "messages": messages, + "tools": tools, + "max_tokens": self.max_output_tokens, + "thinking": {"type": "enabled" if self.thinking else "disabled"}, + } + if self.thinking: + payload["reasoning_effort"] = self.reasoning_effort + else: + # DeepSeek V4 thinking mode rejects tool_choice. Non-thinking mode + # accepts the ordinary OpenAI-compatible automatic selection value. + payload["tool_choice"] = "auto" + return payload + + def trace_config(self) -> dict[str, Any]: + """Return model settings safe to persist in ``run_start``.""" + + return { + "provider": "deepseek", + "model": self.model, + "thinking": self.thinking, + "reasoning_effort": self.reasoning_effort if self.thinking else None, + "max_output_tokens": self.max_output_tokens, + "timeout_seconds": self.timeout_seconds, + "max_retries": self.max_retries, + } + def _send_with_retries(self, request: urllib.request.Request) -> dict[str, Any]: last_error: Exception | None = None for attempt in range(self.max_retries + 1): diff --git a/src/yada/run/cli.py b/src/yada/run/cli.py index 7193eaf..daafd4f 100644 --- a/src/yada/run/cli.py +++ b/src/yada/run/cli.py @@ -72,6 +72,12 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Store full reasoning_content in the trace; redacted by default.", ) + parser.add_argument( + "--trace-level", + choices=["summary", "debug"], + default="summary", + help="summary stores compact metrics; debug stores sanitized model payloads.", + ) parser.add_argument("--version", action="version", version=f"yada {__version__}") return parser @@ -131,7 +137,11 @@ def run_cli(argv: list[str] | None = None) -> int: command_policy=command_policy, command_timeout_seconds=args.command_timeout, ), - trace=TraceWriter(trace_path, include_reasoning=args.trace_reasoning), + trace=TraceWriter( + trace_path, + include_reasoning=args.trace_reasoning, + level=args.trace_level, + ), max_steps=args.max_steps, ) @@ -142,6 +152,7 @@ def run_cli(argv: list[str] | None = None) -> int: f"(thinking={args.thinking}, effort={args.reasoning_effort})" ) print(f"Trace: {trace_path}") + print(f"Trace level: {args.trace_level}") if command_policy == "allow": print( "WARNING: command execution is autonomous; use a container for untrusted repos." diff --git a/src/yada/traces/__init__.py b/src/yada/traces/__init__.py index 0e71c65..5d8e455 100644 --- a/src/yada/traces/__init__.py +++ b/src/yada/traces/__init__.py @@ -1,12 +1,19 @@ """Trace persistence and human-readable run diagnostics.""" -from yada.traces.jsonl import TRACE_SCHEMA_VERSION, TraceWriter -from yada.traces.report import TraceFormatError, read_trace, render_trace_report +from yada.traces.jsonl import TRACE_LEVELS, TRACE_SCHEMA_VERSION, TraceWriter +from yada.traces.report import ( + TraceFormatError, + read_trace, + reconstruct_model_request, + render_trace_report, +) __all__ = [ + "TRACE_LEVELS", "TRACE_SCHEMA_VERSION", "TraceFormatError", "TraceWriter", "read_trace", + "reconstruct_model_request", "render_trace_report", ] diff --git a/src/yada/traces/cli.py b/src/yada/traces/cli.py index 7e27a82..9a89424 100644 --- a/src/yada/traces/cli.py +++ b/src/yada/traces/cli.py @@ -17,6 +17,16 @@ def build_parser() -> argparse.ArgumentParser: description="Summarize a Yada JSONL run as a correlated timeline.", ) parser.add_argument("trace", type=Path, help="Path to a Yada JSONL trace.") + parser.add_argument( + "--step", + type=_positive_step, + help="Show full model and tool details for one agent step.", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Expand event payloads for the complete timeline.", + ) return parser @@ -25,7 +35,11 @@ def run_cli(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: - report = render_trace_report(args.trace.expanduser().resolve()) + report = render_trace_report( + args.trace.expanduser().resolve(), + step=args.step, + verbose=args.verbose, + ) except (OSError, TraceFormatError) as exc: print(f"yada-trace: {exc}", file=sys.stderr) return 2 @@ -33,6 +47,16 @@ def run_cli(argv: list[str] | None = None) -> int: return 0 +def _positive_step(value: str) -> int: + try: + step = int(value) + except ValueError as exc: + raise argparse.ArgumentTypeError("step must be a positive integer") from exc + if step < 1: + raise argparse.ArgumentTypeError("step must be a positive integer") + return step + + def main() -> None: """Console-script entry point.""" diff --git a/src/yada/traces/jsonl.py b/src/yada/traces/jsonl.py index 3f0738c..97342d8 100644 --- a/src/yada/traces/jsonl.py +++ b/src/yada/traces/jsonl.py @@ -9,13 +9,44 @@ import hashlib import json +import re import time import uuid from datetime import datetime, timezone from pathlib import Path from typing import Any -TRACE_SCHEMA_VERSION = 1 +TRACE_SCHEMA_VERSION = 2 +TRACE_LEVELS = {"summary", "debug"} +_REDACTED = "[REDACTED]" +_SECRET_KEYS = { + "authorization", + "credential", + "credentials", + "password", + "proxy_authorization", + "secret", + "token", +} +_SECRET_SUFFIXES = ( + "_api_key", + "_access_token", + "_auth_token", + "_credential", + "_credentials", + "_password", + "_refresh_token", + "_secret", +) +_SECRET_TEXT_PATTERNS = ( + re.compile(r"(?i)\b(bearer\s+)[A-Za-z0-9._~+/=-]+"), + re.compile( + r"(?i)\b((?:api[_-]?key|access[_-]?token|auth[_-]?token|" + r"refresh[_-]?token|password|secret|credential)s?\s*[:=]\s*)" + r"(?:[\"'][^\"']*[\"']|[^\s,;]+)" + ), + re.compile(r"\bsk-[A-Za-z0-9_-]{8,}\b"), +) class TraceWriter: @@ -25,6 +56,8 @@ class TraceWriter: path: Destination file, or ``None`` to disable tracing. include_reasoning: Persist raw ``reasoning_content`` when true. The default stores only its length and SHA-256 digest. + level: ``summary`` stores compact request metrics; ``debug`` allows the + agent to persist sanitized provider payload snapshots. run_id: Optional stable identifier, primarily useful for deterministic tests or importing events from an external orchestrator. """ @@ -34,10 +67,14 @@ def __init__( path: Path | None, *, include_reasoning: bool = False, + level: str = "summary", run_id: str | None = None, ) -> None: + if level not in TRACE_LEVELS: + raise ValueError(f"trace level must be one of {sorted(TRACE_LEVELS)}") self.path = path self.include_reasoning = include_reasoning + self.level = level self.run_id = run_id or uuid.uuid4().hex self._sequence = 0 self._started = time.monotonic() @@ -83,6 +120,8 @@ def _sanitize(self, value: Any) -> Any: "chars": len(text), "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(), } + elif _is_secret_key(key): + sanitized[key] = _REDACTED else: sanitized[key] = self._sanitize(item) return sanitized @@ -90,4 +129,17 @@ def _sanitize(self, value: Any) -> Any: return [self._sanitize(item) for item in value] if isinstance(value, tuple): return [self._sanitize(item) for item in value] + if isinstance(value, str): + return _sanitize_text(value) return value + + +def _is_secret_key(key: Any) -> bool: + normalized = str(key).casefold().replace("-", "_") + return normalized in _SECRET_KEYS or normalized.endswith(_SECRET_SUFFIXES) + + +def _sanitize_text(value: str) -> str: + sanitized = _SECRET_TEXT_PATTERNS[0].sub(r"\1[REDACTED]", value) + sanitized = _SECRET_TEXT_PATTERNS[1].sub(r"\1[REDACTED]", sanitized) + return _SECRET_TEXT_PATTERNS[2].sub(_REDACTED, sanitized) diff --git a/src/yada/traces/provenance.py b/src/yada/traces/provenance.py new file mode 100644 index 0000000..bf1f6a9 --- /dev/null +++ b/src/yada/traces/provenance.py @@ -0,0 +1,61 @@ +"""Small, failure-tolerant helpers for trace reproducibility metadata.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Any + +from yada import __version__ + + +def collect_provenance( + workspace: Path, + *, + extra: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Return stable local versions without making tracing a run dependency.""" + + values: dict[str, Any] = { + "yada_version": __version__, + "yada_commit": _yada_commit(), + "workspace_base_commit": git_head(workspace), + } + values.update(extra or {}) + return values + + +def client_trace_config(client: object) -> dict[str, Any]: + """Return a provider-safe model configuration when the client exposes one.""" + + config = getattr(client, "trace_config", None) + if callable(config): + value = config() + if isinstance(value, dict): + return value + return {"model": str(getattr(client, "model", "unknown"))} + + +def git_head(path: Path) -> str | None: + """Read a Git HEAD if available, returning ``None`` for non-repositories.""" + + try: + result = subprocess.run( + ["git", "-C", str(path), "rev-parse", "HEAD"], + text=True, + capture_output=True, + timeout=5, + check=False, + ) + except (OSError, subprocess.SubprocessError): + return None + value = result.stdout.strip() + return value if result.returncode == 0 and value else None + + +def _yada_commit() -> str | None: + source = Path(__file__).resolve() + for parent in source.parents: + if (parent / "pyproject.toml").is_file() and (parent / "src/yada").is_dir(): + return git_head(parent) + return None diff --git a/src/yada/traces/report.py b/src/yada/traces/report.py index ae83912..3a547ac 100644 --- a/src/yada/traces/report.py +++ b/src/yada/traces/report.py @@ -6,6 +6,8 @@ from pathlib import Path from typing import Any +from yada.traces.jsonl import TRACE_SCHEMA_VERSION + class TraceFormatError(ValueError): """Raised when a JSONL trace contains an invalid event record.""" @@ -43,13 +45,52 @@ def read_trace(path: Path) -> list[dict[str, Any]]: ) if not isinstance(record.get("data"), dict): raise TraceFormatError(f"line {line_number}: data must be an object") + schema_version = record.get("schema_version") + if isinstance(schema_version, int) and schema_version > TRACE_SCHEMA_VERSION: + raise TraceFormatError( + f"line {line_number}: trace schema {schema_version} is newer than " + f"supported schema {TRACE_SCHEMA_VERSION}" + ) events.append(record) if not events: raise TraceFormatError("trace contains no events") return events -def render_trace_report(path: Path) -> str: +def reconstruct_model_request( + events: list[dict[str, Any]], + step: int, +) -> dict[str, Any]: + """Return the sanitized provider payload captured for one model turn. + + Summary and legacy traces intentionally lack this payload. Callers receive a + clear error instead of a partial reconstruction assembled from source code. + """ + + request = next( + ( + event + for event in events + if event["event"] == "model_request" and event["data"].get("step") == step + ), + None, + ) + if request is None: + raise TraceFormatError(f"trace contains no model request for step {step}") + payload = request["data"].get("payload") + if not isinstance(payload, dict): + raise TraceFormatError( + f"step {step} has no captured model payload; rerun with --trace-level debug" + ) + return payload + + +def render_trace_report( + path: Path, + *, + step: int | None = None, + verbose: bool = False, +) -> str: """Render a compact run summary and chronological event timeline. The report intentionally summarizes large prompts, arguments, and command @@ -64,6 +105,13 @@ def render_trace_report(path: Path) -> str: """ events = read_trace(path) + timeline_events = events + if step is not None: + timeline_events = [ + event for event in events if event["data"].get("step") == step + ] + if not timeline_events: + raise TraceFormatError(f"trace contains no events for step {step}") start = _first_event(events, "run_start") end = _first_event(reversed(events), "run_end") model_turns = sum(event["event"] == "assistant" for event in events) @@ -99,6 +147,7 @@ def render_trace_report(path: Path) -> str: f"Path: {path}", f"Run: {run_id}", f"Model: {start_data.get('model', 'unknown')}", + f"Trace level: {start_data.get('trace_level', 'legacy')}", f"Task: {_one_line(start_data.get('task', 'unknown'), 160)}", f"Outcome: {outcome}", ( @@ -106,12 +155,26 @@ def render_trace_report(path: Path) -> str: f"{model_turns} model turns, {len(tool_results)} tool results, " f"{tool_failures} tool failures, {reminders} reminders, {last_elapsed} ms" ), - "Timeline:", + f"Timeline{f' (step {step})' if step is not None else ''}:", ] - for fallback_sequence, event in enumerate(events, 1): + detailed = verbose or step is not None + for fallback_sequence, event in enumerate(timeline_events, 1): sequence = event.get("sequence", fallback_sequence) elapsed = event.get("elapsed_ms", "?") lines.append(f" [{sequence!s:>3} +{elapsed!s:>6}ms] {_describe_event(event)}") + if detailed: + if event["event"] == "model_request" and "payload" not in event["data"]: + lines.append( + " payload unavailable: rerun with --trace-level debug" + ) + rendered = json.dumps( + event["data"], + ensure_ascii=False, + indent=2, + sort_keys=True, + default=str, + ) + lines.extend(f" {line}" for line in rendered.splitlines()) return "\n".join(lines) + "\n" diff --git a/tests/agents/test_default.py b/tests/agents/test_default.py index c1f07ec..15898d3 100644 --- a/tests/agents/test_default.py +++ b/tests/agents/test_default.py @@ -9,7 +9,7 @@ from yada.environments import CommandApprover from yada.models import Completion from yada.tools import ToolRunner -from yada.traces import TraceWriter +from yada.traces import TraceWriter, read_trace, reconstruct_model_request def tool_call(call_id: str, name: str, arguments: dict[str, Any]) -> Completion: @@ -41,9 +41,21 @@ class FakeClient: def __init__(self, completions: list[Completion]) -> None: self.completions = completions self.seen_messages: list[list[dict[str, Any]]] = [] + self.seen_payloads: list[dict[str, Any]] = [] + + def request_payload(self, *, messages, tools): + return { + "model": self.model, + "messages": messages, + "tools": tools, + "thinking": {"type": "enabled"}, + } def complete(self, *, messages, tools): self.seen_messages.append(list(messages)) + self.seen_payloads.append( + json.loads(json.dumps(self.request_payload(messages=messages, tools=tools))) + ) return self.completions.pop(0) @@ -110,6 +122,51 @@ def add(a, b): assert client.seen_messages[1][-2]["reasoning_content"] == "reasoning for read_file" +def test_debug_trace_reconstructs_exact_client_payload(tmp_path: Path) -> None: + path = tmp_path / "value.py" + path.write_text("VALUE = 1\n", encoding="utf-8") + runner = ToolRunner(tmp_path, approver=CommandApprover("allow")) + client = FakeClient( + [ + tool_call("call-read", "read_file", {"path": "value.py"}), + Completion( + message={"role": "assistant", "content": "still working"}, + usage={"prompt_tokens": 2, "completion_tokens": 1}, + model="fake-deepseek-v4-pro", + finish_reason="stop", + ), + ] + ) + trace_path = tmp_path / ".yada" / "debug.jsonl" + agent = Agent( + client=client, + tools=runner, + trace=TraceWriter( + trace_path, + level="debug", + include_reasoning=True, + run_id="debug-run", + ), + max_steps=2, + emit=lambda _: None, + trace_metadata={"case_id": "fixture-1"}, + ) + + result = agent.run("Inspect VALUE") + + assert not result.finished + events = read_trace(trace_path) + assert reconstruct_model_request(events, 1) == client.seen_payloads[0] + assert reconstruct_model_request(events, 2) == client.seen_payloads[1] + second_messages = reconstruct_model_request(events, 2)["messages"] + assert second_messages[-2]["reasoning_content"] == "reasoning for read_file" + run_start = events[0]["data"] + assert run_start["trace_level"] == "debug" + assert run_start["provenance"]["case_id"] == "fixture-1" + assert "yada_version" in run_start["provenance"] + assert "workspace_base_commit" in run_start["provenance"] + + def test_planner_rejects_finish_mixed_with_other_calls() -> None: planner = Planner() assistant_message = { diff --git a/tests/evals/test_yada_agent.py b/tests/evals/test_yada_agent.py new file mode 100644 index 0000000..8fc5f0d --- /dev/null +++ b/tests/evals/test_yada_agent.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +from yada.evals import EvalTask, PreparedTask, RunBudget +from yada.evals.agents import YadaAgentAdapter +from yada.models import Completion +from yada.traces import read_trace, reconstruct_model_request + + +class OneTurnClient: + model = "fake-deepseek" + + def request_payload(self, *, messages, tools): + return {"model": self.model, "messages": messages, "tools": tools} + + def complete(self, *, messages, tools): + return Completion( + message={"role": "assistant", "content": "inspect next"}, + usage={"prompt_tokens": 2, "completion_tokens": 1}, + model=self.model, + finish_reason="stop", + ) + + +def test_yada_eval_trace_includes_case_and_workspace_provenance( + tmp_path: Path, +) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "value.py").write_text("VALUE = 1\n", encoding="utf-8") + subprocess.run(["git", "init", "-q"], cwd=workspace, check=True) + subprocess.run( + ["git", "config", "user.email", "test@example.com"], + cwd=workspace, + check=True, + ) + subprocess.run(["git", "config", "user.name", "Test"], cwd=workspace, check=True) + subprocess.run(["git", "add", "value.py"], cwd=workspace, check=True) + subprocess.run(["git", "commit", "-qm", "base"], cwd=workspace, check=True) + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=workspace, + check=True, + text=True, + capture_output=True, + ).stdout.strip() + run_dir = tmp_path / "artifacts" + run_dir.mkdir() + client = OneTurnClient() + adapter = YadaAgentAdapter( + client_factory=lambda _: client, + trace_level="debug", + emit=lambda _: None, + ) + + result = adapter.run( + PreparedTask(EvalTask("case-123", "Inspect VALUE"), workspace), + RunBudget(max_steps=1), + run_dir, + ) + + assert result.status == "unfinished" + events = read_trace(run_dir / "yada-trace.jsonl") + start = events[0]["data"] + assert start["provenance"]["case_id"] == "case-123" + assert start["provenance"]["workspace_base_commit"] == head + assert reconstruct_model_request(events, 1)["model"] == "fake-deepseek" diff --git a/tests/models/test_deepseek.py b/tests/models/test_deepseek.py index a3c7ccc..2185711 100644 --- a/tests/models/test_deepseek.py +++ b/tests/models/test_deepseek.py @@ -47,11 +47,16 @@ def fake_send(request): ) payload = json.loads(sent[0].data.decode("utf-8")) + assert payload == client.request_payload( + messages=[{"role": "user", "content": "hello"}], tools=[] + ) assert payload["thinking"] == {"type": "enabled"} assert payload["reasoning_effort"] == "max" assert "tool_choice" not in payload assert completion.message["reasoning_content"] == "must be passed back" assert completion.system_fingerprint == "fingerprint-1" + assert client.trace_config()["provider"] == "deepseek" + assert "api_key" not in client.trace_config() def test_non_thinking_mode_uses_automatic_tool_choice(monkeypatch) -> None: diff --git a/tests/traces/test_trace_report.py b/tests/traces/test_trace_report.py index 8d3e66c..67c7810 100644 --- a/tests/traces/test_trace_report.py +++ b/tests/traces/test_trace_report.py @@ -5,13 +5,31 @@ import pytest -from yada.traces import TraceFormatError, TraceWriter, render_trace_report +from yada.evals.cli import build_parser as build_eval_parser +from yada.run.cli import build_parser as build_run_parser +from yada.traces import ( + TRACE_SCHEMA_VERSION, + TraceFormatError, + TraceWriter, + read_trace, + reconstruct_model_request, + render_trace_report, +) +from yada.traces.cli import run_cli def test_trace_events_have_correlation_metadata_and_redaction(tmp_path: Path) -> None: path = tmp_path / "run.jsonl" trace = TraceWriter(path, run_id="run-test") - trace.write("run_start", {"model": "fake", "task": "fix", "workspace": "."}) + trace.write( + "run_start", + { + "model": "fake", + "task": "fix", + "workspace": ".", + "trace_level": "summary", + }, + ) trace.write( "assistant", { @@ -27,7 +45,7 @@ def test_trace_events_have_correlation_metadata_and_redaction(tmp_path: Path) -> events = [json.loads(line) for line in path.read_text().splitlines()] assert [event["sequence"] for event in events] == [1, 2, 3] assert {event["run_id"] for event in events} == {"run-test"} - assert {event["schema_version"] for event in events} == {1} + assert {event["schema_version"] for event in events} == {TRACE_SCHEMA_VERSION} redacted = events[1]["data"]["message"]["reasoning_content"] assert redacted["redacted"] is True assert redacted["chars"] == len("private chain") @@ -35,10 +53,183 @@ def test_trace_events_have_correlation_metadata_and_redaction(tmp_path: Path) -> report = render_trace_report(path) assert "Run: run-test" in report + assert "Trace level: summary" in report assert "Outcome: unfinished" in report assert "step=1 assistant duration=7ms" in report +def test_debug_trace_redacts_reasoning_and_common_secrets(tmp_path: Path) -> None: + path = tmp_path / "debug.jsonl" + trace = TraceWriter(path, level="debug", run_id="debug") + trace.write( + "model_request", + { + "step": 1, + "payload": { + "model": "fake", + "api_key": "sk-secret-value", + "messages": [ + { + "role": "assistant", + "reasoning_content": "private reasoning", + "content": "Authorization: Bearer top-secret-token", + } + ], + "tools": [], + }, + }, + ) + + payload = reconstruct_model_request(read_trace(path), 1) + + assert payload["api_key"] == "[REDACTED]" + assert payload["messages"][0]["reasoning_content"]["redacted"] is True + assert payload["messages"][0]["content"] == "Authorization: Bearer [REDACTED]" + stored = path.read_text(encoding="utf-8") + assert "sk-secret-value" not in stored + assert "top-secret-token" not in stored + assert "private reasoning" not in stored + + +def test_trace_reasoning_can_be_included_in_debug_payload(tmp_path: Path) -> None: + path = tmp_path / "reasoning.jsonl" + trace = TraceWriter(path, level="debug", include_reasoning=True) + trace.write( + "model_request", + { + "step": 2, + "payload": { + "messages": [ + {"role": "assistant", "reasoning_content": "retained reasoning"} + ] + }, + }, + ) + + payload = reconstruct_model_request(read_trace(path), 2) + + assert payload["messages"][0]["reasoning_content"] == "retained reasoning" + + +def test_step_and_verbose_reports_show_full_tool_details( + tmp_path: Path, capsys +) -> None: + path = tmp_path / "details.jsonl" + trace = TraceWriter(path, level="debug", run_id="details") + trace.write( + "run_start", + { + "model": "fake", + "task": "debug", + "workspace": ".", + "trace_level": "debug", + }, + ) + trace.write( + "model_request", + { + "step": 1, + "context": {"message_count": 2, "serialized_chars": 20}, + "payload": { + "model": "fake", + "messages": [{"role": "user", "content": "debug"}], + "tools": [], + }, + }, + ) + trace.write( + "assistant", + { + "step": 1, + "message": {"role": "assistant", "tool_calls": []}, + "duration_ms": 1, + "finish_reason": "tool_calls", + }, + ) + trace.write( + "tool_call", + { + "step": 1, + "tool_call_id": "call-1", + "tool": "run_command", + "arguments": {"argv": ["pytest", "-q"], "purpose": "test"}, + }, + ) + trace.write( + "tool_result", + { + "step": 1, + "tool_call_id": "call-1", + "tool": "run_command", + "duration_ms": 4, + "result": { + "ok": True, + "argv": ["pytest", "-q"], + "stdout": "1 failed", + "stderr": "failure detail", + "exit_code": 1, + }, + }, + ) + trace.write("run_end", {"finished": False, "steps": 1, "summary": "failed"}) + + summary = render_trace_report(path) + detail = render_trace_report(path, step=1) + verbose = render_trace_report(path, verbose=True) + + assert '"pytest"' not in summary + assert '"pytest"' in detail + assert '"stdout": "1 failed"' in detail + assert '"stderr": "failure detail"' in detail + assert '"messages"' in detail + assert '"pytest"' in verbose + assert run_cli([str(path), "--step", "1"]) == 0 + assert '"stdout": "1 failed"' in capsys.readouterr().out + + +def test_summary_trace_explains_missing_request_payload(tmp_path: Path) -> None: + path = tmp_path / "summary.jsonl" + trace = TraceWriter(path) + trace.write("model_request", {"step": 1, "context": {"message_count": 2}}) + + with pytest.raises(TraceFormatError, match="--trace-level debug"): + reconstruct_model_request(read_trace(path), 1) + + detail = render_trace_report(path, step=1) + assert "payload unavailable: rerun with --trace-level debug" in detail + + +def test_report_reads_legacy_schema_and_rejects_future_schema(tmp_path: Path) -> None: + legacy = tmp_path / "legacy.jsonl" + legacy.write_text( + json.dumps( + { + "schema_version": 1, + "event": "run_start", + "data": {"model": "old", "task": "fix", "workspace": "."}, + } + ) + + "\n", + encoding="utf-8", + ) + assert "Trace level: legacy" in render_trace_report(legacy) + + future = tmp_path / "future.jsonl" + future.write_text( + json.dumps( + { + "schema_version": TRACE_SCHEMA_VERSION + 1, + "event": "run_start", + "data": {}, + } + ) + + "\n", + encoding="utf-8", + ) + with pytest.raises(TraceFormatError, match="newer than supported"): + read_trace(future) + + def test_report_marks_trace_without_run_end_as_interrupted(tmp_path: Path) -> None: path = tmp_path / "crashed.jsonl" trace = TraceWriter(path, run_id="crashed") @@ -60,3 +251,13 @@ def test_report_rejects_malformed_jsonl(tmp_path: Path) -> None: with pytest.raises(TraceFormatError, match="line 2"): render_trace_report(path) + + +def test_agent_clis_expose_trace_level() -> None: + run_args = build_run_parser().parse_args(["fix", "--trace-level", "debug"]) + eval_args = build_eval_parser().parse_args( + ["--case", "case-dir", "--trace-level", "debug"] + ) + + assert run_args.trace_level == "debug" + assert eval_args.trace_level == "debug" From d21da90d32c3151a9f181aea3936765d689b0915 Mon Sep 17 00:00:00 2001 From: Gen TANG Date: Sun, 2 Aug 2026 23:00:12 +0800 Subject: [PATCH 2/4] add some docs for yada trace --- README.md | 7 +- docs/architecture.md | 1 + docs/tracing.md | 168 ++++++++++++++++++++++++++++++++++ src/yada/agents/default.py | 29 +++--- src/yada/models/base.py | 1 + src/yada/models/deepseek.py | 5 + tests/agents/test_default.py | 10 ++ tests/models/test_deepseek.py | 34 +++++++ 8 files changed, 240 insertions(+), 15 deletions(-) create mode 100644 docs/tracing.md diff --git a/README.md b/README.md index 98cc9fc..266f1f5 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,8 @@ final verification state into a compact timeline. `--step` and `--verbose` expand sanitized model messages, tool arguments, patches, stdout, and stderr. The source JSONL remains the durable, streaming-friendly record. Debug traces can contain source code and test output even after secret redaction, so handle them as -sensitive artifacts. +sensitive artifacts. See [docs/tracing.md](docs/tracing.md) for the event +reference, field-presence semantics, lifecycle, and `jq` recipes. ## Safety model @@ -246,8 +247,8 @@ and DeepSeek's official [thinking-mode](https://api-docs.deepseek.com/guides/thi and [tool-call](https://api-docs.deepseek.com/guides/tool_calls) contracts. The implementation is original and intentionally smaller than those systems. -See [docs/architecture.md](docs/architecture.md) for the detailed contracts and planned -ablations. +See [docs/architecture.md](docs/architecture.md) for the detailed contracts and +planned ablations, and [docs/tracing.md](docs/tracing.md) for the trace schema. ## Current non-goals diff --git a/docs/architecture.md b/docs/architecture.md index ec56400..a84ac4d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -101,6 +101,7 @@ response → tools slice, while `--verbose` expands the full timeline. Reasoning is length/hash-redacted unless `--trace-reasoning` is explicit. Common secret keys and bearer/API-key-like text are redacted in both modes. A debug trace can still contain source code and test output and must be handled as a sensitive artifact. +The complete event and field reference lives in [tracing.md](tracing.md). The MVP stores a full sanitized request snapshot per turn. This deliberately favors deterministic inspection over delta complexity; content-addressed prompts diff --git a/docs/tracing.md b/docs/tracing.md new file mode 100644 index 0000000..a6e5a06 --- /dev/null +++ b/docs/tracing.md @@ -0,0 +1,168 @@ +# Yada trace reference + +Yada records one append-only JSON object per line. JSONL is the durable source of +truth: a completed write remains inspectable if the agent later crashes, and the +file can be streamed with ordinary tools. `yada-trace` is the terminal view over +that source data. + +## Record envelope + +Schema v2 records have these top-level fields: + +| Field | Meaning | +| --- | --- | +| `schema_version` | Integer format version. Current value: `2`. | +| `run_id` | Correlates every record produced by one agent run. | +| `sequence` | One-based event order within the run. | +| `timestamp` | UTC ISO 8601 wall-clock timestamp. | +| `elapsed_ms` | Milliseconds since the `TraceWriter` was created. | +| `event` | Event type described below. | +| `data` | Event-specific object. | + +Readers must ignore unknown `data` fields. Adding an optional field does not +require a schema-version increase; changing or removing an existing field does. +`TraceWriter` accepts extension event names, but the ten names below are Yada's +core event vocabulary. + +## Lifecycle and correlation + +```text +run_start + step 1..N + model_request ──→ assistant ──→ plan_decision + │ ├─→ protocol_reminder + └─→ model_error ├─→ protocol_violation + └─→ tool_call ──→ tool_result +run_end +``` + +- `step` groups one model request, its response, planning decision, and tools. +- `request_id` joins `model_request` to `assistant` or `model_error`. +- `tool_call_id` joins each `tool_call` to its `tool_result` and to the next + request's `role=tool` message. +- Multiple tool calls can occur in one step and retain model-provided order. +- A transport exception or process interruption can leave a trace without + `run_end`. Readers should report this as interrupted, not successful. + +## Core events + +| Event | Cardinality and meaning | Important `data` fields | +| --- | --- | --- | +| `run_start` | Once, before the loop. | `model`, `task`, `workspace`, `max_steps`, `trace_level`, `model_config`, `provenance` | +| `model_request` | Once per attempted model turn. | `step`, `request_id`, `context`; debug traces also contain `payload` and `capture` | +| `assistant` | Once after a successful model request. | `step`, `request_id`, `duration_ms`, `message`, `message_field_presence`, `usage`, response metadata, `finish_reason` | +| `model_error` | Instead of `assistant` when a request raises. | `step`, `request_id`, `duration_ms`, `error_type`, `error` | +| `plan_decision` | Once after each `assistant`. | `step`, `action`, ordered `tools`, `rejection_error` | +| `protocol_reminder` | When the assistant returns no tool call. | `step`, `text` | +| `protocol_violation` | When a tool batch is rejected before side effects. | `step`, `error`, `call_count` | +| `tool_call` | Before every attempted tool execution. | `step`, `tool_call_id`, `tool`, `arguments`; invalid calls use `raw_arguments`, rejected calls use `rejected` | +| `tool_result` | After every `tool_call`, including rejected calls. | `step`, `tool_call_id`, `tool`, `duration_ms`, `result` | +| `run_end` | Once after a graceful terminal outcome. | `finished`, `steps`, `summary`, accumulated `usage`, `final_state` | + +`tool_result.data.result.ok` means that the Yada tool protocol completed. For +`run_command`, a process can still have a non-zero `exit_code`; check both fields +when looking for failures. + +## Assistant message fields + +An assistant response is a compound message: + +- `content` is optional user-facing narration. +- `reasoning_content` is DeepSeek thinking state. +- `tool_calls` contains requested actions. + +A tool-calling response commonly has non-empty `reasoning_content`, one or more +`tool_calls`, and `content=""`. That is a valid model response: the tool call is +the action for the turn. Yada appends the whole assistant message to the +conversation so DeepSeek reasoning is available on subsequent tool-call turns. + +The DeepSeek adapter normalizes an omitted `role` to `"assistant"` and an omitted +`content` to `""`. The `assistant.data.message_field_presence` object preserves +what the provider actually sent before normalization: + +```json +{ + "message": {"role": "assistant", "content": "", "tool_calls": []}, + "message_field_presence": { + "role_present": true, + "content_present": false, + "reasoning_content_present": false, + "tool_calls_present": true + } +} +``` + +Interpret `content` with its presence flag: + +| Normalized value | `content_present` | Meaning | +| --- | --- | --- | +| `""` | `true` | The provider explicitly returned an empty string. | +| `""` | `false` | The provider omitted the field; Yada supplied its default. | +| non-empty string | `true` | The provider returned narration. | + +The metadata is present for adapters that expose upstream field presence. Older +traces and custom completion clients can omit it. + +## Capture levels and sensitive data + +`--trace-level summary` records event timing, response data, tool activity, and +compact request-context metrics. `--trace-level debug` additionally records the +sanitized provider request payload built by the same client method used for the +HTTP request. + +Reasoning is replaced by its character count and SHA-256 digest unless +`--trace-reasoning` is explicit. API-key-like fields and bearer, token, password, +credential, and secret text are redacted in both capture levels. Redaction does +not make a trace public: prompts, source code, patches, paths, and test output can +remain sensitive. + +## Inspection recipes + +Render a run summary or expand one step: + +```bash +uv run yada-trace TRACE.jsonl +uv run yada-trace TRACE.jsonl --step 12 +uv run yada-trace TRACE.jsonl --verbose +``` + +List event counts: + +```bash +jq -r '.event' TRACE.jsonl | sort | uniq -c +``` + +Show the exact sanitized request payload captured for step 12: + +```bash +jq 'select(.event == "model_request" and .data.step == 12) | .data.payload' \ + TRACE.jsonl +``` + +Show the response channels and upstream field presence for every model turn: + +```bash +jq 'select(.event == "assistant") | + {step: .data.step, + finish_reason: .data.finish_reason, + message_field_presence: .data.message_field_presence, + content: .data.message.content, + reasoning_content: .data.message.reasoning_content, + tool_calls: .data.message.tool_calls}' TRACE.jsonl +``` + +Find rejected tools and commands with non-zero exit codes: + +```bash +jq 'select(.event == "tool_result" and + (.data.result.ok == false or ((.data.result.exit_code // 0) != 0))) | + {step: .data.step, tool: .data.tool, result: .data.result}' TRACE.jsonl +``` + +## Compatibility + +The reader accepts legacy records and rejects schema versions newer than it +understands. Event order and correlation identifiers are the replay contract; +timestamps are diagnostic and should not be used to reconstruct missing events. +Yada currently records enough data for deterministic inspection, not automatic +re-execution of model or shell side effects. diff --git a/src/yada/agents/default.py b/src/yada/agents/default.py index 98fc11b..341a5f7 100644 --- a/src/yada/agents/default.py +++ b/src/yada/agents/default.py @@ -157,20 +157,25 @@ def run(self, task: str) -> AgentResult: _merge_usage(total_usage, completion.usage) assistant_message = completion.message messages.append(assistant_message) + assistant_record = { + "step": step, + "request_id": request_id, + "duration_ms": duration_ms, + "request_context": context_metrics, + "message": assistant_message, + "usage": completion.usage, + "response_id": completion.response_id, + "model": completion.model, + "system_fingerprint": completion.system_fingerprint, + "finish_reason": completion.finish_reason, + } + if completion.message_field_presence is not None: + assistant_record["message_field_presence"] = ( + completion.message_field_presence + ) self.trace.write( "assistant", - { - "step": step, - "request_id": request_id, - "duration_ms": duration_ms, - "request_context": context_metrics, - "message": assistant_message, - "usage": completion.usage, - "response_id": completion.response_id, - "model": completion.model, - "system_fingerprint": completion.system_fingerprint, - "finish_reason": completion.finish_reason, - }, + assistant_record, ) plan = self.planner.plan( diff --git a/src/yada/models/base.py b/src/yada/models/base.py index 8218d79..eeb3498 100644 --- a/src/yada/models/base.py +++ b/src/yada/models/base.py @@ -16,6 +16,7 @@ class Completion: model: str | None = None system_fingerprint: str | None = None finish_reason: str | None = None + message_field_presence: dict[str, bool] | None = None class CompletionClient(Protocol): diff --git a/src/yada/models/deepseek.py b/src/yada/models/deepseek.py index 81dea1c..0ff71a8 100644 --- a/src/yada/models/deepseek.py +++ b/src/yada/models/deepseek.py @@ -98,6 +98,10 @@ def complete( for key in ("role", "content", "reasoning_content", "tool_calls") if key in raw_message } + message_field_presence = { + f"{key}_present": key in raw_message + for key in ("role", "content", "reasoning_content", "tool_calls") + } message.setdefault("role", "assistant") message.setdefault("content", "") return Completion( @@ -107,6 +111,7 @@ def complete( model=response_data.get("model"), system_fingerprint=response_data.get("system_fingerprint"), finish_reason=choice.get("finish_reason"), + message_field_presence=message_field_presence, ) def request_payload( diff --git a/tests/agents/test_default.py b/tests/agents/test_default.py index 15898d3..122bb16 100644 --- a/tests/agents/test_default.py +++ b/tests/agents/test_default.py @@ -32,6 +32,12 @@ def tool_call(call_id: str, name: str, arguments: dict[str, Any]) -> Completion: usage={"prompt_tokens": 10, "completion_tokens": 5}, model="fake-deepseek-v4-pro", finish_reason="tool_calls", + message_field_presence={ + "role_present": True, + "content_present": True, + "reasoning_content_present": True, + "tool_calls_present": True, + }, ) @@ -118,6 +124,10 @@ def add(a, b): trace = trace_path.read_text(encoding="utf-8") assert '"redacted": true' in trace assert "reasoning for read_file" not in trace + assistant = next( + event for event in read_trace(trace_path) if event["event"] == "assistant" + ) + assert assistant["data"]["message_field_presence"]["content_present"] is True # DeepSeek requires the prior assistant reasoning_content on the next request. assert client.seen_messages[1][-2]["reasoning_content"] == "reasoning for read_file" diff --git a/tests/models/test_deepseek.py b/tests/models/test_deepseek.py index 2185711..d3bde4f 100644 --- a/tests/models/test_deepseek.py +++ b/tests/models/test_deepseek.py @@ -53,12 +53,46 @@ def fake_send(request): assert payload["thinking"] == {"type": "enabled"} assert payload["reasoning_effort"] == "max" assert "tool_choice" not in payload + assert completion.message["content"] == "" assert completion.message["reasoning_content"] == "must be passed back" + assert completion.message_field_presence == { + "role_present": True, + "content_present": True, + "reasoning_content_present": True, + "tool_calls_present": True, + } assert completion.system_fingerprint == "fingerprint-1" assert client.trace_config()["provider"] == "deepseek" assert "api_key" not in client.trace_config() +def test_completion_distinguishes_missing_content_from_empty_content( + monkeypatch, +) -> None: + client = DeepSeekClient(api_key="test-key") + response = { + "choices": [ + { + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "reasoning_content": "call a tool", + "tool_calls": [], + }, + } + ] + } + + monkeypatch.setattr(client, "_send_with_retries", lambda request: response) + + completion = client.complete(messages=[], tools=[]) + + assert completion.message["content"] == "" + assert completion.message_field_presence is not None + assert completion.message_field_presence["content_present"] is False + assert completion.message_field_presence["reasoning_content_present"] is True + + def test_non_thinking_mode_uses_automatic_tool_choice(monkeypatch) -> None: client = DeepSeekClient(api_key="test-key", thinking=False) response = { From d102582e12f9aadb65ef421dc1b7c12610e9ad13 Mon Sep 17 00:00:00 2001 From: Gen TANG Date: Sun, 2 Aug 2026 23:14:04 +0800 Subject: [PATCH 3/4] update the trace folder name --- README.md | 13 ++++++++---- docs/tracing.md | 7 +++++++ src/yada/evals/cli.py | 42 ++++++++++++++++++++++++------------- src/yada/run/cli.py | 16 +++++++------- src/yada/utils/naming.py | 43 ++++++++++++++++++++++++++++++++++++++ tests/utils/test_naming.py | 40 +++++++++++++++++++++++++++++++++++ 6 files changed, 134 insertions(+), 27 deletions(-) create mode 100644 src/yada/utils/naming.py create mode 100644 tests/utils/test_naming.py diff --git a/README.md b/README.md index 266f1f5..9264d17 100644 --- a/README.md +++ b/README.md @@ -158,9 +158,13 @@ uv run yada eval \ Inspect a completed or interrupted run without manually scanning JSONL: ```bash -uv run yada-trace .yada/runs/20260801T120000.000000Z.jsonl -uv run yada-trace eval-results/.artifacts/yada-trace.jsonl --step 8 -uv run yada-trace eval-results/.artifacts/yada-trace.jsonl --verbose +uv run yada-trace \ + .yada/runs/fix-parser-edge-case__2026-08-02_12-26-26.123456Z.jsonl +uv run yada-trace \ + eval-results/pytest-dev__pytest-10051__2026-08-02_12-26-26.123456Z.artifacts/yada-trace.jsonl \ + --step 8 +uv run yada-trace eval-results/__.artifacts/yada-trace.jsonl \ + --verbose ``` The report correlates model requests, tool-call IDs, errors, reminders, and the @@ -225,7 +229,8 @@ tests/ ├── evals/ ├── models/ ├── tools/ -└── traces/ +├── traces/ +└── utils/ ``` `Planner` owns conversation policy and validates the next action without I/O. diff --git a/docs/tracing.md b/docs/tracing.md index a6e5a06..35ca921 100644 --- a/docs/tracing.md +++ b/docs/tracing.md @@ -118,6 +118,13 @@ remain sensitive. ## Inspection recipes +Default paths include a sanitized task name and a readable UTC timestamp. A +direct run resembles +`.yada/runs/fix-parser-boundary-issue__2026-08-02_12-26-26.123456Z.jsonl`. +An evaluation stores its trace under a directory such as +`eval-results/pytest-dev__pytest-10051__2026-08-02_12-26-26.123456Z.artifacts/`. +Explicit `--trace`, `--output`, and `--artifact-dir` values are never renamed. + Render a run summary or expand one step: ```bash diff --git a/src/yada/evals/cli.py b/src/yada/evals/cli.py index c6c1af2..30799ee 100644 --- a/src/yada/evals/cli.py +++ b/src/yada/evals/cli.py @@ -6,13 +6,13 @@ import os import shlex import sys -from datetime import datetime, timezone from pathlib import Path from yada.evals.agents import CommandAgentAdapter, YadaAgentAdapter from yada.evals.base import RunBudget from yada.evals.benchmarks import LocalBenchmark, SWEbenchBenchmark from yada.evals.runner import EvalRunner +from yada.utils.naming import readable_run_name def build_parser() -> argparse.ArgumentParser: @@ -44,8 +44,9 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--output", type=Path, - default=_default_output_path(), - help="Result JSON path.", + help=( + "Result JSON path (default: eval-results/__.json)." + ), ) parser.add_argument( "--artifact-dir", @@ -132,13 +133,6 @@ def run_cli(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) - output_path = args.output.expanduser().resolve() - artifact_dir = ( - args.artifact_dir.expanduser().resolve() - if args.artifact_dir - else output_path.with_suffix("").with_name(output_path.stem + ".artifacts") - ) - if args.case is not None: if args.benchmark not in {None, "local"}: parser.error("--case can only be used with --benchmark local") @@ -152,12 +146,14 @@ def run_cli(argv: list[str] | None = None) -> int: manifest_path, cache_root=args.cache_dir, ) - instance_id = args.instance or "" + instance_id = args.instance or _manifest_instance_id(benchmark) + task_name = instance_id or case_path.stem elif args.benchmark == "local": if args.manifest is None: parser.error("--manifest is required for --benchmark local") benchmark = LocalBenchmark(args.manifest, cache_root=args.cache_dir) - instance_id = args.instance or "" + instance_id = args.instance or _manifest_instance_id(benchmark) + task_name = instance_id or args.manifest.stem elif args.benchmark == "swebench": if not args.instance: parser.error("--instance is required for --benchmark swebench") @@ -175,9 +171,21 @@ def run_cli(argv: list[str] | None = None) -> int: grade_timeout_seconds=args.grade_timeout, ) instance_id = args.instance + task_name = instance_id else: parser.error("provide --case or --benchmark") + output_path = ( + args.output.expanduser().resolve() + if args.output is not None + else _default_output_path(task_name).resolve() + ) + artifact_dir = ( + args.artifact_dir.expanduser().resolve() + if args.artifact_dir + else output_path.with_suffix("").with_name(output_path.stem + ".artifacts") + ) + if args.agent == "yada": api_key = os.environ.get("DEEPSEEK_API_KEY", "") if not api_key: @@ -235,6 +243,10 @@ def run_cli(argv: list[str] | None = None) -> int: return 2 -def _default_output_path() -> Path: - timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") - return Path("eval-results") / f"{timestamp}.json" +def _default_output_path(task_name: str) -> Path: + return Path("eval-results") / f"{readable_run_name(task_name)}.json" + + +def _manifest_instance_id(benchmark: LocalBenchmark) -> str: + value = benchmark.manifest.get("instance_id") + return value if isinstance(value, str) else "" diff --git a/src/yada/run/cli.py b/src/yada/run/cli.py index daafd4f..68fde0f 100644 --- a/src/yada/run/cli.py +++ b/src/yada/run/cli.py @@ -5,7 +5,6 @@ import argparse import os import sys -from datetime import datetime, timezone from pathlib import Path from yada import __version__ @@ -13,6 +12,7 @@ from yada.models import DeepSeekAPIError, DeepSeekClient from yada.tools import ToolRunner from yada.traces import TraceWriter +from yada.utils.naming import readable_run_name def build_parser() -> argparse.ArgumentParser: @@ -65,7 +65,10 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--trace", type=Path, - help="JSONL trace path (default: WORKSPACE/.yada/runs/.jsonl).", + help=( + "JSONL trace path (default: " + "WORKSPACE/.yada/runs/__.jsonl)." + ), ) parser.add_argument( "--trace-reasoning", @@ -120,7 +123,7 @@ def run_cli(argv: list[str] | None = None) -> int: if not api_key: parser.error("DEEPSEEK_API_KEY is not set") - trace_path = args.trace or _default_trace_path(workspace) + trace_path = args.trace or _default_trace_path(workspace, task) command_policy = "allow" if args.yes else args.command_policy agent = Agent( client=DeepSeekClient( @@ -182,11 +185,8 @@ def run_cli(argv: list[str] | None = None) -> int: return 0 if result.finished else 2 -def _default_trace_path(workspace: Path) -> Path: - # Microseconds prevent two short runs launched in the same second from - # appending unrelated events to one file. - timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") - return workspace / ".yada" / "runs" / f"{timestamp}.jsonl" +def _default_trace_path(workspace: Path, task: str) -> Path: + return workspace / ".yada" / "runs" / f"{readable_run_name(task)}.jsonl" def main() -> None: diff --git a/src/yada/utils/naming.py b/src/yada/utils/naming.py new file mode 100644 index 0000000..4e24bcf --- /dev/null +++ b/src/yada/utils/naming.py @@ -0,0 +1,43 @@ +"""Readable, portable names for local run artifacts.""" + +from __future__ import annotations + +import re +import unicodedata +from datetime import datetime, timezone + +_WINDOWS_RESERVED_NAMES = { + "aux", + "con", + "nul", + "prn", + *(f"com{number}" for number in range(1, 10)), + *(f"lpt{number}" for number in range(1, 10)), +} + + +def task_slug(task: str, *, max_length: int = 48) -> str: + """Return a short cross-platform filename component derived from a task.""" + + if max_length < 1: + raise ValueError("max_length must be positive") + normalized = unicodedata.normalize("NFKC", task).casefold().strip() + slug = "".join( + character if character.isalnum() or character in {"-", "_"} else "-" + for character in normalized + ) + slug = re.sub(r"-+", "-", slug).strip("-_.") + slug = slug[:max_length].rstrip("-_.") or "task" + if slug in _WINDOWS_RESERVED_NAMES: + slug = f"task-{slug}" + return slug + + +def readable_run_name(task: str, *, now: datetime | None = None) -> str: + """Combine a task slug with a sortable, readable UTC timestamp.""" + + instant = now or datetime.now(timezone.utc) + if instant.tzinfo is None: + instant = instant.replace(tzinfo=timezone.utc) + timestamp = instant.astimezone(timezone.utc).strftime("%Y-%m-%d_%H-%M-%S.%fZ") + return f"{task_slug(task)}__{timestamp}" diff --git a/tests/utils/test_naming.py b/tests/utils/test_naming.py new file mode 100644 index 0000000..175ceed --- /dev/null +++ b/tests/utils/test_naming.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest + +from yada.utils.naming import readable_run_name, task_slug + + +def test_task_slug_keeps_meaningful_unicode_and_removes_unsafe_characters() -> None: + assert task_slug("修复 parser 的边界问题，并运行测试") == ( + "修复-parser-的边界问题-并运行测试" + ) + assert task_slug(" Fix path/to: parser? ") == "fix-path-to-parser" + assert task_slug("pytest-dev__pytest-10051") == "pytest-dev__pytest-10051" + assert task_slug("CON") == "task-con" + assert task_slug("***") == "task" + + +def test_task_slug_is_bounded() -> None: + assert task_slug("one two three", max_length=7) == "one-two" + with pytest.raises(ValueError, match="positive"): + task_slug("task", max_length=0) + + +def test_readable_run_name_uses_utc_and_microseconds() -> None: + china_time = datetime( + 2026, + 8, + 2, + 20, + 26, + 26, + 123456, + tzinfo=timezone(timedelta(hours=8)), + ) + + assert readable_run_name("Pytest 10051", now=china_time) == ( + "pytest-10051__2026-08-02_12-26-26.123456Z" + ) From bf19499c0af4aa3e3e6791304779db40fd75c2cf Mon Sep 17 00:00:00 2001 From: Gen TANG Date: Sun, 2 Aug 2026 23:27:26 +0800 Subject: [PATCH 4/4] keep reasoning content when --trace-level debug --- README.md | 8 +++--- docs/architecture.md | 7 ++--- docs/tracing.md | 44 ++++++++----------------------- src/yada/agents/default.py | 8 +----- src/yada/evals/agents/yada.py | 3 --- src/yada/evals/cli.py | 2 -- src/yada/models/base.py | 1 - src/yada/models/deepseek.py | 5 ---- src/yada/run/cli.py | 11 +++----- src/yada/traces/jsonl.py | 10 +++---- tests/agents/test_default.py | 11 -------- tests/evals/test_yada_agent.py | 8 +++++- tests/models/test_deepseek.py | 14 ++-------- tests/traces/test_trace_report.py | 21 ++++++++++----- 14 files changed, 51 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index 9264d17..31b4eb0 100644 --- a/README.md +++ b/README.md @@ -140,10 +140,10 @@ Tools: DeepSeek thinking-mode `reasoning_content` is retained in memory and passed back after tool calls, as required by the API. The default `--trace-level summary` records compact context metrics. `--trace-level debug` additionally records the -complete sanitized provider payload for every model turn. JSONL traces redact -reasoning text by default while retaining its length and hash, and also redact -common API keys, authorization values, tokens, passwords, and secrets. Use -`--trace-reasoning` only if you intentionally want to store raw reasoning. +complete sanitized provider payload and reasoning text for every model turn. +Summary traces replace reasoning with its length and hash. Both levels redact +common API keys, authorization values, tokens, passwords, and secrets. Debug +traces contain sensitive model context and must be handled accordingly. Capture a replayable debug trace during an evaluation: diff --git a/docs/architecture.md b/docs/architecture.md index a84ac4d..31bcf9a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -98,9 +98,10 @@ the model configuration. `yada-trace PATH` renders the compact summary. `--step N` expands one request → response → tools slice, while `--verbose` expands the full timeline. Reasoning -is length/hash-redacted unless `--trace-reasoning` is explicit. Common secret keys -and bearer/API-key-like text are redacted in both modes. A debug trace can still -contain source code and test output and must be handled as a sensitive artifact. +is length/hash-redacted in summary traces and automatically retained in debug +traces. Common secret keys and bearer/API-key-like text are redacted in both +modes. A debug trace can still contain reasoning, source code, and test output +and must be handled as a sensitive artifact. The complete event and field reference lives in [tracing.md](tracing.md). The MVP stores a full sanitized request snapshot per turn. This deliberately diff --git a/docs/tracing.md b/docs/tracing.md index 35ca921..736c938 100644 --- a/docs/tracing.md +++ b/docs/tracing.md @@ -50,7 +50,7 @@ run_end | --- | --- | --- | | `run_start` | Once, before the loop. | `model`, `task`, `workspace`, `max_steps`, `trace_level`, `model_config`, `provenance` | | `model_request` | Once per attempted model turn. | `step`, `request_id`, `context`; debug traces also contain `payload` and `capture` | -| `assistant` | Once after a successful model request. | `step`, `request_id`, `duration_ms`, `message`, `message_field_presence`, `usage`, response metadata, `finish_reason` | +| `assistant` | Once after a successful model request. | `step`, `request_id`, `duration_ms`, `message`, `usage`, response metadata, `finish_reason` | | `model_error` | Instead of `assistant` when a request raises. | `step`, `request_id`, `duration_ms`, `error_type`, `error` | | `plan_decision` | Once after each `assistant`. | `step`, `action`, ordered `tools`, `rejection_error` | | `protocol_reminder` | When the assistant returns no tool call. | `step`, `text` | @@ -77,31 +77,8 @@ the action for the turn. Yada appends the whole assistant message to the conversation so DeepSeek reasoning is available on subsequent tool-call turns. The DeepSeek adapter normalizes an omitted `role` to `"assistant"` and an omitted -`content` to `""`. The `assistant.data.message_field_presence` object preserves -what the provider actually sent before normalization: - -```json -{ - "message": {"role": "assistant", "content": "", "tool_calls": []}, - "message_field_presence": { - "role_present": true, - "content_present": false, - "reasoning_content_present": false, - "tool_calls_present": true - } -} -``` - -Interpret `content` with its presence flag: - -| Normalized value | `content_present` | Meaning | -| --- | --- | --- | -| `""` | `true` | The provider explicitly returned an empty string. | -| `""` | `false` | The provider omitted the field; Yada supplied its default. | -| non-empty string | `true` | The provider returned narration. | - -The metadata is present for adapters that expose upstream field presence. Older -traces and custom completion clients can omit it. +`content` to `""`. Use `finish_reason` and `tool_calls` to interpret an empty +content value: on a tool-calling turn, the tool call is the model's action. ## Capture levels and sensitive data @@ -110,11 +87,13 @@ compact request-context metrics. `--trace-level debug` additionally records the sanitized provider request payload built by the same client method used for the HTTP request. -Reasoning is replaced by its character count and SHA-256 digest unless -`--trace-reasoning` is explicit. API-key-like fields and bearer, token, password, -credential, and secret text are redacted in both capture levels. Redaction does -not make a trace public: prompts, source code, patches, paths, and test output can -remain sensitive. +Summary traces replace reasoning with its character count and SHA-256 digest. +Debug traces automatically retain reasoning because it is essential for +understanding intermediate tool-calling turns where `content` may be empty. +API-key-like fields and bearer, token, password, credential, and secret text are +redacted in both capture levels. Redaction does not make a trace public: debug +traces can contain reasoning, prompts, source code, patches, paths, and test +output. ## Inspection recipes @@ -146,13 +125,12 @@ jq 'select(.event == "model_request" and .data.step == 12) | .data.payload' \ TRACE.jsonl ``` -Show the response channels and upstream field presence for every model turn: +Show the response channels for every model turn: ```bash jq 'select(.event == "assistant") | {step: .data.step, finish_reason: .data.finish_reason, - message_field_presence: .data.message_field_presence, content: .data.message.content, reasoning_content: .data.message.reasoning_content, tool_calls: .data.message.tool_calls}' TRACE.jsonl diff --git a/src/yada/agents/default.py b/src/yada/agents/default.py index 341a5f7..f5bb196 100644 --- a/src/yada/agents/default.py +++ b/src/yada/agents/default.py @@ -125,9 +125,7 @@ def run(self, task: str) -> AgentResult: self.tools.schemas, ) request_record["capture"] = { - "reasoning": ( - "included" if self.trace.include_reasoning else "redacted" - ), + "reasoning": "included", "secrets": "redacted", } self.trace.write( @@ -169,10 +167,6 @@ def run(self, task: str) -> AgentResult: "system_fingerprint": completion.system_fingerprint, "finish_reason": completion.finish_reason, } - if completion.message_field_presence is not None: - assistant_record["message_field_presence"] = ( - completion.message_field_presence - ) self.trace.write( "assistant", assistant_record, diff --git a/src/yada/evals/agents/yada.py b/src/yada/evals/agents/yada.py index 4c12872..c4857a4 100644 --- a/src/yada/evals/agents/yada.py +++ b/src/yada/evals/agents/yada.py @@ -31,7 +31,6 @@ def __init__( api_timeout_seconds: int = 300, command_timeout_seconds: int = 120, command_policy: str = "ask", - include_reasoning: bool = False, trace_level: str = "summary", client_factory: Callable[[RunBudget], CompletionClient] | None = None, emit: Callable[[str], None] = print, @@ -46,7 +45,6 @@ def __init__( self.api_timeout_seconds = api_timeout_seconds self.command_timeout_seconds = command_timeout_seconds self.command_policy = command_policy - self.include_reasoning = include_reasoning self.trace_level = trace_level self.client_factory = client_factory self.emit = emit @@ -82,7 +80,6 @@ def run( tools=tools, trace=TraceWriter( trace_path, - include_reasoning=self.include_reasoning, level=self.trace_level, ), max_steps=budget.max_steps, diff --git a/src/yada/evals/cli.py b/src/yada/evals/cli.py index 30799ee..3b565f1 100644 --- a/src/yada/evals/cli.py +++ b/src/yada/evals/cli.py @@ -83,7 +83,6 @@ def build_parser() -> argparse.ArgumentParser: default="ask", ) model.add_argument("--yes", action="store_true") - model.add_argument("--trace-reasoning", action="store_true") model.add_argument( "--trace-level", choices=["summary", "debug"], @@ -199,7 +198,6 @@ def run_cli(argv: list[str] | None = None) -> int: api_timeout_seconds=args.api_timeout, command_timeout_seconds=args.command_timeout, command_policy="allow" if args.yes else args.command_policy, - include_reasoning=args.trace_reasoning, trace_level=args.trace_level, ) else: diff --git a/src/yada/models/base.py b/src/yada/models/base.py index eeb3498..8218d79 100644 --- a/src/yada/models/base.py +++ b/src/yada/models/base.py @@ -16,7 +16,6 @@ class Completion: model: str | None = None system_fingerprint: str | None = None finish_reason: str | None = None - message_field_presence: dict[str, bool] | None = None class CompletionClient(Protocol): diff --git a/src/yada/models/deepseek.py b/src/yada/models/deepseek.py index 0ff71a8..81dea1c 100644 --- a/src/yada/models/deepseek.py +++ b/src/yada/models/deepseek.py @@ -98,10 +98,6 @@ def complete( for key in ("role", "content", "reasoning_content", "tool_calls") if key in raw_message } - message_field_presence = { - f"{key}_present": key in raw_message - for key in ("role", "content", "reasoning_content", "tool_calls") - } message.setdefault("role", "assistant") message.setdefault("content", "") return Completion( @@ -111,7 +107,6 @@ def complete( model=response_data.get("model"), system_fingerprint=response_data.get("system_fingerprint"), finish_reason=choice.get("finish_reason"), - message_field_presence=message_field_presence, ) def request_payload( diff --git a/src/yada/run/cli.py b/src/yada/run/cli.py index 68fde0f..66ab2f3 100644 --- a/src/yada/run/cli.py +++ b/src/yada/run/cli.py @@ -70,16 +70,14 @@ def build_parser() -> argparse.ArgumentParser: "WORKSPACE/.yada/runs/__.jsonl)." ), ) - parser.add_argument( - "--trace-reasoning", - action="store_true", - help="Store full reasoning_content in the trace; redacted by default.", - ) parser.add_argument( "--trace-level", choices=["summary", "debug"], default="summary", - help="summary stores compact metrics; debug stores sanitized model payloads.", + help=( + "summary redacts reasoning; debug stores sanitized model payloads " + "and reasoning." + ), ) parser.add_argument("--version", action="version", version=f"yada {__version__}") return parser @@ -142,7 +140,6 @@ def run_cli(argv: list[str] | None = None) -> int: ), trace=TraceWriter( trace_path, - include_reasoning=args.trace_reasoning, level=args.trace_level, ), max_steps=args.max_steps, diff --git a/src/yada/traces/jsonl.py b/src/yada/traces/jsonl.py index 97342d8..cf348ee 100644 --- a/src/yada/traces/jsonl.py +++ b/src/yada/traces/jsonl.py @@ -54,10 +54,8 @@ class TraceWriter: Args: path: Destination file, or ``None`` to disable tracing. - include_reasoning: Persist raw ``reasoning_content`` when true. The - default stores only its length and SHA-256 digest. - level: ``summary`` stores compact request metrics; ``debug`` allows the - agent to persist sanitized provider payload snapshots. + level: ``summary`` stores compact request metrics and redacts reasoning; + ``debug`` stores sanitized provider payloads and reasoning text. run_id: Optional stable identifier, primarily useful for deterministic tests or importing events from an external orchestrator. """ @@ -66,14 +64,12 @@ def __init__( self, path: Path | None, *, - include_reasoning: bool = False, level: str = "summary", run_id: str | None = None, ) -> None: if level not in TRACE_LEVELS: raise ValueError(f"trace level must be one of {sorted(TRACE_LEVELS)}") self.path = path - self.include_reasoning = include_reasoning self.level = level self.run_id = run_id or uuid.uuid4().hex self._sequence = 0 @@ -113,7 +109,7 @@ def _sanitize(self, value: Any) -> Any: if isinstance(value, dict): sanitized: dict[str, Any] = {} for key, item in value.items(): - if key == "reasoning_content" and not self.include_reasoning: + if key == "reasoning_content" and self.level != "debug": text = str(item or "") sanitized[key] = { "redacted": True, diff --git a/tests/agents/test_default.py b/tests/agents/test_default.py index 122bb16..2a9d479 100644 --- a/tests/agents/test_default.py +++ b/tests/agents/test_default.py @@ -32,12 +32,6 @@ def tool_call(call_id: str, name: str, arguments: dict[str, Any]) -> Completion: usage={"prompt_tokens": 10, "completion_tokens": 5}, model="fake-deepseek-v4-pro", finish_reason="tool_calls", - message_field_presence={ - "role_present": True, - "content_present": True, - "reasoning_content_present": True, - "tool_calls_present": True, - }, ) @@ -124,10 +118,6 @@ def add(a, b): trace = trace_path.read_text(encoding="utf-8") assert '"redacted": true' in trace assert "reasoning for read_file" not in trace - assistant = next( - event for event in read_trace(trace_path) if event["event"] == "assistant" - ) - assert assistant["data"]["message_field_presence"]["content_present"] is True # DeepSeek requires the prior assistant reasoning_content on the next request. assert client.seen_messages[1][-2]["reasoning_content"] == "reasoning for read_file" @@ -154,7 +144,6 @@ def test_debug_trace_reconstructs_exact_client_payload(tmp_path: Path) -> None: trace=TraceWriter( trace_path, level="debug", - include_reasoning=True, run_id="debug-run", ), max_steps=2, diff --git a/tests/evals/test_yada_agent.py b/tests/evals/test_yada_agent.py index 8fc5f0d..eea4e00 100644 --- a/tests/evals/test_yada_agent.py +++ b/tests/evals/test_yada_agent.py @@ -17,7 +17,11 @@ def request_payload(self, *, messages, tools): def complete(self, *, messages, tools): return Completion( - message={"role": "assistant", "content": "inspect next"}, + message={ + "role": "assistant", + "content": "inspect next", + "reasoning_content": "debug reasoning", + }, usage={"prompt_tokens": 2, "completion_tokens": 1}, model=self.model, finish_reason="stop", @@ -67,3 +71,5 @@ def test_yada_eval_trace_includes_case_and_workspace_provenance( assert start["provenance"]["case_id"] == "case-123" assert start["provenance"]["workspace_base_commit"] == head assert reconstruct_model_request(events, 1)["model"] == "fake-deepseek" + assistant = next(event for event in events if event["event"] == "assistant") + assert assistant["data"]["message"]["reasoning_content"] == "debug reasoning" diff --git a/tests/models/test_deepseek.py b/tests/models/test_deepseek.py index d3bde4f..2b91fee 100644 --- a/tests/models/test_deepseek.py +++ b/tests/models/test_deepseek.py @@ -55,20 +55,12 @@ def fake_send(request): assert "tool_choice" not in payload assert completion.message["content"] == "" assert completion.message["reasoning_content"] == "must be passed back" - assert completion.message_field_presence == { - "role_present": True, - "content_present": True, - "reasoning_content_present": True, - "tool_calls_present": True, - } assert completion.system_fingerprint == "fingerprint-1" assert client.trace_config()["provider"] == "deepseek" assert "api_key" not in client.trace_config() -def test_completion_distinguishes_missing_content_from_empty_content( - monkeypatch, -) -> None: +def test_completion_normalizes_missing_content(monkeypatch) -> None: client = DeepSeekClient(api_key="test-key") response = { "choices": [ @@ -88,9 +80,7 @@ def test_completion_distinguishes_missing_content_from_empty_content( completion = client.complete(messages=[], tools=[]) assert completion.message["content"] == "" - assert completion.message_field_presence is not None - assert completion.message_field_presence["content_present"] is False - assert completion.message_field_presence["reasoning_content_present"] is True + assert completion.message["reasoning_content"] == "call a tool" def test_non_thinking_mode_uses_automatic_tool_choice(monkeypatch) -> None: diff --git a/tests/traces/test_trace_report.py b/tests/traces/test_trace_report.py index 67c7810..f313e70 100644 --- a/tests/traces/test_trace_report.py +++ b/tests/traces/test_trace_report.py @@ -58,7 +58,9 @@ def test_trace_events_have_correlation_metadata_and_redaction(tmp_path: Path) -> assert "step=1 assistant duration=7ms" in report -def test_debug_trace_redacts_reasoning_and_common_secrets(tmp_path: Path) -> None: +def test_debug_trace_includes_reasoning_and_redacts_common_secrets( + tmp_path: Path, +) -> None: path = tmp_path / "debug.jsonl" trace = TraceWriter(path, level="debug", run_id="debug") trace.write( @@ -71,7 +73,9 @@ def test_debug_trace_redacts_reasoning_and_common_secrets(tmp_path: Path) -> Non "messages": [ { "role": "assistant", - "reasoning_content": "private reasoning", + "reasoning_content": ( + "private reasoning; Authorization: Bearer reasoning-token" + ), "content": "Authorization: Bearer top-secret-token", } ], @@ -83,17 +87,20 @@ def test_debug_trace_redacts_reasoning_and_common_secrets(tmp_path: Path) -> Non payload = reconstruct_model_request(read_trace(path), 1) assert payload["api_key"] == "[REDACTED]" - assert payload["messages"][0]["reasoning_content"]["redacted"] is True + assert payload["messages"][0]["reasoning_content"] == ( + "private reasoning; Authorization: Bearer [REDACTED]" + ) assert payload["messages"][0]["content"] == "Authorization: Bearer [REDACTED]" stored = path.read_text(encoding="utf-8") assert "sk-secret-value" not in stored assert "top-secret-token" not in stored - assert "private reasoning" not in stored + assert "reasoning-token" not in stored + assert "private reasoning" in stored -def test_trace_reasoning_can_be_included_in_debug_payload(tmp_path: Path) -> None: +def test_debug_trace_always_includes_reasoning(tmp_path: Path) -> None: path = tmp_path / "reasoning.jsonl" - trace = TraceWriter(path, level="debug", include_reasoning=True) + trace = TraceWriter(path, level="debug") trace.write( "model_request", { @@ -261,3 +268,5 @@ def test_agent_clis_expose_trace_level() -> None: assert run_args.trace_level == "debug" assert eval_args.trace_level == "debug" + assert "--trace-reasoning" not in build_run_parser().format_help() + assert "--trace-reasoning" not in build_eval_parser().format_help()