From d606fab5b3bb9d8b24e46d8df440c33832d18e13 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Thu, 8 Oct 2026 18:09:28 +0000 Subject: [PATCH 1/2] Seal attempt work for #181 (succeeded) Attempt-Id: 2026-10-08T17:59:10.133579Z --- .env.example | 32 ++-- CONTEXT.md | 4 + docs/agents/operator-onboarding.md | 22 ++- simple_coding_agent/config.py | 70 +++++++- simple_coding_agent/model_execution.py | 69 +++++++- tests/test_config.py | 57 +++++++ tests/test_model_execution.py | 213 ++++++++++++++++++++++++- 7 files changed, 445 insertions(+), 22 deletions(-) diff --git a/.env.example b/.env.example index 0450d9f..cf3df9b 100644 --- a/.env.example +++ b/.env.example @@ -58,17 +58,31 @@ TARGET_REPO=owner/repository # PR_MERGE_METHOD selects the merge method for both paths. # PR_MERGE_METHOD=merge -# Provider-side spending caps are the source of truth. MAX_BUDGET_USD=5 is an -# SDK circuit breaker, not authoritative Meta billing. MAX_TURNS bounds turns -# per attempt. Before either hard limit, the agent asks the model to hand off -# its progress cooperatively: SOFT_THRESHOLD_PERCENTAGE (0-1, exclusive) sets -# how far below MAX_BUDGET_USD that cost soft threshold sits -# (soft = max_budget_usd - max_budget_usd * percentage). Turns/MODEL_TIMEOUT -# have no soft threshold; reaching either gets one best-effort handoff -# follow-up instead. See docs/agents/operator-onboarding.md. +# Token budget and handoff reserve. Each attempt counts every processed and +# generated token (unweighted, subagents included) against MAX_BUDGET_TOKENS +# (default 4000000); reaching it stops the attempt unconditionally (hard +# ceiling). Before that, the agent asks the model to hand off cooperatively +# once the soft threshold is crossed; the crossing latches for the attempt +# and is evaluated on every counted response and in the PreToolUse hook. +# The effective soft threshold is the earlier of two rules, computed from +# the main thread's latest context size: +# soft = min(MAX_BUDGET_TOKENS * (1 - SOFT_THRESHOLD_PERCENTAGE), +# MAX_BUDGET_TOKENS - (HANDOFF_RESERVE_TURNS + 1) * main_context_tokens) +# SOFT_THRESHOLD_PERCENTAGE (0-1, exclusive; default 0.2) sets the fixed +# share of the budget, and HANDOFF_RESERVE_TURNS (integer 1-20; default 6) +# guarantees that many turns after the crossing turn at the current context +# size. A huge context whose reserve exceeds the remaining budget trips the +# threshold on the next check. MAX_TURNS bounds turns per attempt. +# MAX_BUDGET_USD (default 20) is only a looser SDK circuit breaker, not +# authoritative Meta billing, with a startup floor check against the token +# budget at the $5/MTok reference rate. Turns/MODEL_TIMEOUT have no soft +# threshold; reaching either gets one best-effort handoff follow-up instead. +# See docs/agents/operator-onboarding.md. # MAX_TURNS=60 -# MAX_BUDGET_USD=5 +# MAX_BUDGET_TOKENS=4000000 # SOFT_THRESHOLD_PERCENTAGE=0.2 +# HANDOFF_RESERVE_TURNS=6 +# MAX_BUDGET_USD=20 # # MODEL_NAME pins the Meta model with no fallback. CLAUDE_AGENT_SDK_VERSION # pins the runtime that provenance verification checks against at startup diff --git a/CONTEXT.md b/CONTEXT.md index eb5cf6d..dd68e0e 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -64,6 +64,10 @@ _Avoid_: Cost budget, spend limit The point in the token budget past which the model must end the attempt with a handoff, leaving the rest of the budget for writing the Handoff note. _Avoid_: Cost threshold, warning limit +**Handoff reserve**: +The number of turns guaranteed after the turn that crosses the soft threshold, sized at the main thread's latest context size (`HANDOFF_RESERVE_TURNS`, default 6). It pulls the effective soft threshold earlier when the context is large. +_Avoid_: Turn cap, handoff note + **Hard ceiling**: The point at which the attempt is stopped regardless of the model, whether or not it has handed off. _Avoid_: Budget cap, kill limit diff --git a/docs/agents/operator-onboarding.md b/docs/agents/operator-onboarding.md index 813c222..0c83988 100644 --- a/docs/agents/operator-onboarding.md +++ b/docs/agents/operator-onboarding.md @@ -166,10 +166,24 @@ default-model rates. They are still estimates for observability, not a measured bill: rely on Meta's billing dashboard for accurate cost data. Before the hard ceiling, the agent tries a cooperative `handoff`: once the -token budget crosses `soft = MAX_BUDGET_TOKENS * (1 - -SOFT_THRESHOLD_PERCENTAGE)` tokens (default `MAX_BUDGET_TOKENS=4000000` and -`SOFT_THRESHOLD_PERCENTAGE=0.2`, i.e. 3,200,000 tokens), it asks the model to -commit its progress and stop. The token budget comes from a per-attempt +token budget crosses the effective soft threshold, it asks the model to +commit its progress and stop. The threshold is the earlier of two rules, +computed from the main thread's latest context size (defaults +`MAX_BUDGET_TOKENS=4000000`, `SOFT_THRESHOLD_PERCENTAGE=0.2`, +`HANDOFF_RESERVE_TURNS=6`): + +`soft = min(MAX_BUDGET_TOKENS * (1 - SOFT_THRESHOLD_PERCENTAGE), MAX_BUDGET_TOKENS - (HANDOFF_RESERVE_TURNS + 1) * main_context_tokens)` + +`SOFT_THRESHOLD_PERCENTAGE` (0-1, exclusive) sets the fixed share of the +budget (3,200,000 tokens by default); `HANDOFF_RESERVE_TURNS` (integer 1-20) +guarantees that many turns after the crossing turn at the current context +size (with a 150K context the reserve rule sets the threshold to 2,950,000 +tokens, leaving at least six turns before the hard ceiling). The threshold +is evaluated on every counted response and in the `PreToolUse` hook, and +once crossed it stays crossed for the attempt, even if a compaction later +shrinks the context. A huge context whose reserve exceeds the remaining +budget trips the threshold on the next check. The token budget comes from +a per-attempt ledger: each streamed response is first counted as a local estimate and is replaced by the usage the backend reports (`message_delta` events) when that arrives, so the threshold still works when the backend reports no usage; the diff --git a/simple_coding_agent/config.py b/simple_coding_agent/config.py index c5a053d..e744c31 100644 --- a/simple_coding_agent/config.py +++ b/simple_coding_agent/config.py @@ -34,6 +34,38 @@ class ProfileError(ValueError): _MERGE_METHODS = frozenset({"merge", "squash", "rebase"}) _DEFAULT_MODEL_STREAM_IDLE_TIMEOUT_MS = 60000 _DEFAULT_CLAUDE_AGENT_SDK_VERSION = "0.2.163" +_DEFAULT_HANDOFF_RESERVE_TURNS = 6 +_MAX_HANDOFF_RESERVE_TURNS = 20 + + +def effective_soft_threshold_tokens( + max_budget_tokens: int, + soft_threshold_percentage: float, + handoff_reserve_turns: int, + main_context_tokens: float, +) -> tuple[int, str, int]: + """Effective soft threshold, the rule that set it, and the reserve size. + + The threshold is the earlier of two rules, computed from the main + thread's latest context size (not its peak): a fixed share of the + budget (``percentage`` rule) and a handoff reserve of + ``handoff_reserve_turns`` turns after the crossing turn at the current + context size (``reserve`` rule):: + + soft = min(max_budget_tokens * (1 - pct), + max_budget_tokens - (reserve_turns + 1) * main_context_tokens) + + When the reserve exceeds the remaining budget (a huge context) the + reserve rule yields a threshold at or below zero, so the next limit + check fires immediately. That is correct behaviour, not an error. + """ + + percentage_threshold = int(max_budget_tokens * (1 - soft_threshold_percentage)) + reserve_tokens = int((handoff_reserve_turns + 1) * main_context_tokens) + reserve_threshold = max_budget_tokens - reserve_tokens + if reserve_threshold <= percentage_threshold: + return reserve_threshold, "reserve", reserve_tokens + return percentage_threshold, "percentage", reserve_tokens @dataclass(frozen=True) @@ -61,6 +93,7 @@ class RuntimeConfig: max_budget_usd: int = 20 max_budget_tokens: int = 4_000_000 soft_threshold_percentage: float = 0.2 + handoff_reserve_turns: int = _DEFAULT_HANDOFF_RESERVE_TURNS model_base_url: str = _DEFAULT_MODEL_BASE_URL model_api_key: str = "" model_auth_mode: str = _DEFAULT_MODEL_AUTH_MODE @@ -70,10 +103,27 @@ class RuntimeConfig: @property def soft_threshold_tokens(self) -> int: - """Token budget reading past which the attempt must hand off.""" + """Percentage-rule token reading past which the attempt must hand off. + + This is the fixed-share rule only + (``max_budget_tokens * (1 - soft_threshold_percentage)``). The + effective threshold is the earlier of this rule and the handoff + reserve; use :meth:`effective_soft_threshold` with the main thread's + latest context size for that. + """ return int(self.max_budget_tokens * (1 - self.soft_threshold_percentage)) + def effective_soft_threshold(self, main_context_tokens: float) -> tuple[int, str, int]: + """Effective threshold, rule, and reserve size for a context size.""" + + return effective_soft_threshold_tokens( + self.max_budget_tokens, + self.soft_threshold_percentage, + self.handoff_reserve_turns, + main_context_tokens, + ) + @property def credential_redactions(self) -> tuple[str, ...]: """Secrets that must never appear in logs or profile-command output.""" @@ -158,6 +208,9 @@ def load_runtime_config(environ: Mapping[str, str] | None = None) -> RuntimeConf max_budget_usd=_positive_integer(values, "MAX_BUDGET_USD", 20), max_budget_tokens=_positive_integer(values, "MAX_BUDGET_TOKENS", 4_000_000), soft_threshold_percentage=_fraction(values, "SOFT_THRESHOLD_PERCENTAGE", 0.2), + handoff_reserve_turns=_reserve_turns( + values, "HANDOFF_RESERVE_TURNS", _DEFAULT_HANDOFF_RESERVE_TURNS + ), pr_auto_merge=_auto_merge_mode(values), pr_merge_method=_merge_method(values), ) @@ -290,6 +343,21 @@ def _fraction(values: Mapping[str, str], name: str, default: float) -> float: return number +def _reserve_turns(values: Mapping[str, str], name: str, default: int) -> int: + value = values.get(name, str(default)) + try: + number = int(value) + except ValueError as error: + raise ConfigurationError( + f"{name} must be an integer between 1 and {_MAX_HANDOFF_RESERVE_TURNS}" + ) from error + if not 1 <= number <= _MAX_HANDOFF_RESERVE_TURNS: + raise ConfigurationError( + f"{name} must be an integer between 1 and {_MAX_HANDOFF_RESERVE_TURNS}" + ) + return number + + def _boolean(values: Mapping[str, str], name: str, default: bool) -> bool: value = values.get(name, str(default).lower()).lower() if value == "true": diff --git a/simple_coding_agent/model_execution.py b/simple_coding_agent/model_execution.py index ec40f8d..82096dc 100644 --- a/simple_coding_agent/model_execution.py +++ b/simple_coding_agent/model_execution.py @@ -46,8 +46,10 @@ # an estimate on first sight, ``StreamEvent`` ``message_delta`` usage and # non-zero ``AssistantMessage.usage`` replace the estimate with the measured # categories, and tool results / user messages grow the thread context. The -# soft threshold is evaluated against ``ledger.budget_tokens`` on every ledger -# update; only ``limits_checked`` logging waits for a response to settle. +# effective soft threshold (the percentage rule vs the handoff reserve at +# the main thread's latest context size) is evaluated against +# ``ledger.budget_tokens`` on every ledger update and in the PreToolUse +# hook; only ``limits_checked`` logging waits for a response to settle. _HANDOFF_FOLLOWUP_TIMEOUT = 300 # One-time ``prompt_cache_ineffective`` diagnostic: evaluated when the main # thread's K-th measured response settles. Observational only. @@ -506,6 +508,8 @@ def __init__( self._started_at = self._clock() self._soft_threshold_crossed = False self._soft_threshold_crossed_at_tokens: float | None = None + self._soft_threshold_rule: str | None = None + self._soft_threshold_effective_tokens: int | None = None self._total_cost_usd: float | None = None self._handoff_context_delivered = False self._working_directory: Path | None = None @@ -955,6 +959,8 @@ async def execute( self._started_at = self._clock() self._soft_threshold_crossed = False self._soft_threshold_crossed_at_tokens = None + self._soft_threshold_rule = None + self._soft_threshold_effective_tokens = None self._total_cost_usd = None self._token_hard_ceiling_reached = False self._handoff_context_delivered = False @@ -981,12 +987,18 @@ async def execute( self._tool_pending = {} self._bash_history = [] self._bash_pending = {} + threshold, rule, reserve_tokens = self._effective_threshold() self._log( "model_execution_started", f"issue_body_length={len(issue_body)}; limits: " f"max_budget_usd={self._config.max_budget_usd:.4f}; " f"max_budget_tokens={self._config.max_budget_tokens}; " - f"soft_threshold_tokens={self._config.soft_threshold_tokens}; " + f"soft_threshold_tokens={threshold}; " + f"soft_threshold_percentage={self._config.soft_threshold_percentage}; " + f"reserve_turns={self._config.handoff_reserve_turns}; " + f"rule={rule}; " + f"main_context_tokens={self._latest_main_context_tokens():.0f}; " + f"reserve_tokens={reserve_tokens}; " f"max_turns={self._config.max_turns}; " f"timeout_seconds={self._config.model_timeout}", ) @@ -1700,12 +1712,16 @@ def _token_limits_detail(self, thread_id: str, response_id: str) -> str: else: main_context = 0.0 turns = 0 + effective, rule, reserve_tokens = self._config.effective_soft_threshold(main_context) detail = ( f"thread={thread_id}; " f"budget_tokens={budget:.0f}; " f"measured_tokens={measured:.0f}; " f"estimated_tokens={estimated:.0f}; " - f"soft_threshold_tokens={self._config.soft_threshold_tokens}; " + f"soft_threshold_tokens={effective}; " + f"rule={rule}; " + f"reserve_turns={self._config.handoff_reserve_turns}; " + f"reserve_tokens={reserve_tokens}; " f"max_budget_tokens={self._config.max_budget_tokens}; " f"main_context_tokens={main_context:.0f}; " f"turns={turns}; " @@ -1750,17 +1766,49 @@ def _elapsed_seconds(self) -> int: delta = (self._clock() - self._started_at).total_seconds() return max(0, int(delta)) + def _latest_main_context_tokens(self) -> float: + """The main thread's latest context size (0 before it starts). + + The effective soft threshold is computed from this latest reading, + not the peak: a compaction that shrinks the context moves the + reserve rule later again, but a latched crossing stays crossed. + """ + + if self._ledger.has_thread("main"): + return self._ledger.context_tokens("main") + return 0.0 + + def _effective_threshold(self) -> tuple[int, str, int]: + """Effective threshold, rule, and reserve size at the live context.""" + + return self._config.effective_soft_threshold(self._latest_main_context_tokens()) + def _check_limits(self, thread_id: str = "main") -> None: - """Latch one-time soft-threshold and hard-ceiling crossings from the ledger.""" + """Latch one-time soft-threshold and hard-ceiling crossings from the ledger. + + The soft threshold is the effective one: the earlier of the + percentage rule and the handoff reserve at the main thread's + latest context size. It is evaluated on every counted response + (via ``_after_ledger_update``) and in the PreToolUse hook; once + crossed it stays crossed, even if a compaction later shrinks the + context. + """ budget = self._ledger.budget_tokens - threshold = self._config.soft_threshold_tokens + main_context = self._latest_main_context_tokens() + threshold, rule, reserve_tokens = self._config.effective_soft_threshold(main_context) if not self._soft_threshold_crossed and budget >= threshold: self._soft_threshold_crossed = True self._soft_threshold_crossed_at_tokens = budget + self._soft_threshold_rule = rule + self._soft_threshold_effective_tokens = threshold self._log( "token_soft_threshold_crossed", f"budget_tokens={budget:.0f}; soft_threshold_tokens={threshold}; " + f"rule={rule}; " + f"reserve_turns={self._config.handoff_reserve_turns}; " + f"main_context_tokens={main_context:.0f}; " + f"reserve_tokens={reserve_tokens}; " f"thread={thread_id}", ) self._stop_running_subagents() @@ -2007,6 +2055,10 @@ def _reconcile_token_budget( "mode": reconciliation.mode, "max_budget_tokens": self._config.max_budget_tokens, "soft_threshold_tokens": self._config.soft_threshold_tokens, + "soft_threshold_percentage": self._config.soft_threshold_percentage, + "handoff_reserve_turns": self._config.handoff_reserve_turns, + "soft_threshold_rule": self._soft_threshold_rule, + "effective_soft_threshold_tokens": self._soft_threshold_effective_tokens, "estimated_tokens": round(reconciliation.estimated_tokens), "actual_tokens": reconciliation.actual_tokens, "error_ratio": _round_ratio(reconciliation.error_ratio), @@ -2052,6 +2104,11 @@ def _reconcile_token_budget( async def _guard_and_record_pre_tool_use( self, hook_input: Any, tool_use_id: str | None, context: Any ) -> dict[str, Any]: + # The soft threshold is evaluated here as well as on every counted + # response: the ledger reading is unchanged since the last update, + # but the effective threshold moves with the main context, and a + # crossing here latches exactly like a response-time crossing. + self._check_limits(_hook_agent_id(hook_input) or "main") if self._token_hard_ceiling_reached: return _deny( "Token budget reached the hard ceiling " diff --git a/tests/test_config.py b/tests/test_config.py index 47d4e5d..dffe258 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -6,6 +6,7 @@ from simple_coding_agent.config import ( ConfigurationError, ProfileError, + effective_soft_threshold_tokens, load_repository_profile, load_runtime_config, ) @@ -39,6 +40,7 @@ def test_loads_required_operator_settings_and_defaults(tmp_path: Path) -> None: assert config.soft_threshold_tokens == 3_200_000 assert config.pr_auto_merge == "false" assert config.pr_merge_method == "merge" + assert config.handoff_reserve_turns == 6 def test_default_runtime_versions_match_dockerfile_pins(tmp_path: Path) -> None: @@ -518,3 +520,58 @@ def test_empty_override_path_falls_back_to_the_default_in_repo_location(tmp_path profile = load_repository_profile(tmp_path, "") assert profile.setup == ("pytest",) + + +def test_handoff_reserve_turns_accepts_the_full_range(tmp_path: Path) -> None: + for value in ("1", "6", "20"): + config = load_runtime_config(_operator_env(tmp_path, HANDOFF_RESERVE_TURNS=value)) + + assert config.handoff_reserve_turns == int(value) + + +@pytest.mark.parametrize("value", ["0", "21", "-1", "many", "6.5", ""]) +def test_handoff_reserve_turns_rejects_out_of_range_or_non_integer( + tmp_path: Path, value: str +) -> None: + with pytest.raises(ConfigurationError) as error: + load_runtime_config(_operator_env(tmp_path, HANDOFF_RESERVE_TURNS=value)) + + assert "HANDOFF_RESERVE_TURNS" in str(error.value) + assert "github-secret" not in str(error.value) + assert "meta-secret" not in str(error.value) + + +def test_effective_soft_threshold_uses_the_reserve_rule_for_a_large_context() -> None: + threshold, rule, reserve_tokens = effective_soft_threshold_tokens( + 4_000_000, 0.2, 6, 150_000 + ) + + assert threshold == 2_950_000 + assert rule == "reserve" + assert reserve_tokens == 7 * 150_000 + + +def test_effective_soft_threshold_uses_the_percentage_rule_for_a_small_context() -> None: + threshold, rule, reserve_tokens = effective_soft_threshold_tokens( + 4_000_000, 0.2, 6, 50_000 + ) + + assert threshold == 3_200_000 + assert rule == "percentage" + assert reserve_tokens == 7 * 50_000 + + +def test_effective_soft_threshold_fires_immediately_when_the_reserve_exceeds_the_budget() -> ( + None +): + threshold, rule, _ = effective_soft_threshold_tokens(4_000_000, 0.2, 6, 600_000) + + assert rule == "reserve" + assert threshold <= 0 + + +def test_runtime_config_effective_soft_threshold_matches_the_helper(tmp_path: Path) -> None: + config = load_runtime_config(_operator_env(tmp_path)) + + assert config.effective_soft_threshold(150_000) == (2_950_000, "reserve", 1_050_000) + assert config.effective_soft_threshold(50_000) == (3_200_000, "percentage", 350_000) diff --git a/tests/test_model_execution.py b/tests/test_model_execution.py index 0e92df4..534360e 100644 --- a/tests/test_model_execution.py +++ b/tests/test_model_execution.py @@ -1168,7 +1168,14 @@ def client_factory(options: object) -> FakeClient: crossed = [d for name, d, _ in events if name == "token_soft_threshold_crossed"] assert len(crossed) == 1 crossed_budget = int(crossed[0].split("budget_tokens=")[1].split(";")[0]) - assert 1_120_000 <= crossed_budget < config.max_budget_tokens + # The ~10K context per turn pulls the reserve rule below the 1,120,000 + # percentage rule, so the crossing lands earlier with room to spare. + assert crossed_budget < config.soft_threshold_tokens + assert crossed_budget < config.max_budget_tokens + assert "rule=reserve" in crossed[0] + assert "reserve_turns=6" in crossed[0] + assert "main_context_tokens=" in crossed[0] + assert "reserve_tokens=" in crossed[0] assert "thread=main" in crossed[0] assert executor._ledger.measured_tokens == 0 assert any(name == "token_soft_threshold_handoff_followup" for name, _, _ in events) @@ -1186,7 +1193,7 @@ def client_factory(options: object) -> FakeClient: AssistantMessage( content=[], model="muse-spark-1.3-contributor", - usage={"cache_read_input_tokens": 2_500_000, "input_tokens": 10_000}, + usage={"cache_read_input_tokens": 250_000, "input_tokens": 10_000}, ), result(), ], @@ -2999,3 +3006,205 @@ def test_token_snapshot_degrades_outside_a_git_repo(tmp_path: Path) -> None: assert "(unknown)" in snapshot assert "(unavailable)" in snapshot assert ".agent/handoff/.md" in snapshot + + +# --- Handoff reserve in turns (soft = earlier of percentage vs reserve) ------- + + +def _crossed_value(detail: str, name: str) -> str: + for part in detail.split("; "): + key, _, value = part.partition("=") + if key == name: + return value + raise AssertionError(f"{name!r} missing from {detail!r}") + + +def test_reserve_rule_leaves_handoff_turns_before_the_hard_ceiling( + tmp_path: Path, +) -> None: + events: list[tuple[str, str, str]] = [] + captured: list[FakeClient] = [] + + def client_factory(options: object) -> FakeClient: + messages: list[object] = [ + AssistantMessage( + content=[], + model="muse-spark-1.3-contributor", + message_id=f"r{index}", + usage={"input_tokens": 150_000, "output_tokens": 1_000}, + ) + for index in range(25) + ] + messages.append(result()) + client = FakeClient( + options, messages, followup_messages=[result(stop_reason="end_turn")] + ) + captured.append(client) + return client + + executor = ModelExecutor( + runtime_config(tmp_path), + client_factory=client_factory, + event_log=lambda event, detail="", level="INFO", issue_number=None: events.append( + (event, detail, level) + ), + ) + execution = asyncio.run(executor.execute(issue_body="Fix it.", working_directory=tmp_path)) + + # No handoff was invoked, so the crossing ends the attempt incomplete + # after the single follow-up. + assert execution.status is ModelExecutionStatus.MODEL_LIMIT_REACHED + assert len(captured[0].queried_prompts) == 1 + crossed = [d for name, d, _ in events if name == "token_soft_threshold_crossed"] + assert len(crossed) == 1 + assert _crossed_value(crossed[0], "rule") == "reserve" + assert _crossed_value(crossed[0], "reserve_turns") == "6" + assert _crossed_value(crossed[0], "soft_threshold_tokens") == "2943000" + assert _crossed_value(crossed[0], "main_context_tokens") == "151000" + assert _crossed_value(crossed[0], "reserve_tokens") == "1057000" + # At least reserve_turns turns of budget remain after the crossing + # turn, counting that turn's overshoot. + crossed_budget = int(_crossed_value(crossed[0], "budget_tokens")) + main_context = int(_crossed_value(crossed[0], "main_context_tokens")) + assert 4_000_000 - crossed_budget >= 6 * main_context + budget = execution.token_budget + assert budget["soft_threshold_crossed"] is True + assert budget["soft_threshold_rule"] == "reserve" + assert budget["effective_soft_threshold_tokens"] == 2_943_000 + + +def test_pre_tool_use_check_latches_the_soft_threshold(tmp_path: Path) -> None: + events: list[tuple[str, str]] = [] + captured: list[FakeClient] = [] + + def client_factory(options: object) -> FakeClient: + client = FakeClient(options, [result()]) + captured.append(client) + return client + + config = replace(runtime_config(tmp_path), max_budget_tokens=1_000_000) + executor = ModelExecutor( + config, + client_factory=client_factory, + event_log=lambda event, detail="", level="INFO", issue_number=None: events.append((event, detail)), + ) + asyncio.run(executor.execute(issue_body="Fix it.", working_directory=tmp_path)) + pre_hook = captured[0].options.hooks["PreToolUse"][0].hooks[0] + + # Grow the ledger past the effective threshold without a limit check: + # a 900K main context leaves no room for the 7-turn reserve. + executor._ledger.observe_response("main", "r-big", 0) + executor._ledger.observe_usage("main", "r-big", {"input_tokens": 900_000}) + assert executor._soft_threshold_crossed is False + + decision = asyncio.run( + pre_hook({"tool_name": "Bash", "tool_input": {"command": "pytest"}}, None, {}) + ) + + # The PreToolUse crossing latches and logs like a response-time one, + # and the same call already enforces the post-crossing guard. + assert executor._soft_threshold_crossed is True + assert executor._soft_threshold_rule == "reserve" + crossed = [detail for name, detail in events if name == "token_soft_threshold_crossed"] + assert len(crossed) == 1 + assert _crossed_value(crossed[0], "rule") == "reserve" + assert _crossed_value(crossed[0], "reserve_turns") == "6" + assert _crossed_value(crossed[0], "main_context_tokens") == "900000" + assert _crossed_value(crossed[0], "reserve_tokens") == "6300000" + assert "thread=main" in crossed[0] + assert decision["hookSpecificOutput"]["permissionDecision"] == "deny" + denied = asyncio.run( + pre_hook( + {"tool_name": "Edit", "tool_input": {"file_path": "src/a.py"}, "agent_id": "sub-1"}, + None, + {}, + ) + ) + assert denied["hookSpecificOutput"]["permissionDecision"] == "deny" + assert ( + asyncio.run( + pre_hook({"tool_name": "Skill", "tool_input": {"skill": "handoff"}}, None, {}) + ) + == {} + ) + + +def test_huge_context_trips_the_threshold_on_the_next_check(tmp_path: Path) -> None: + events: list[tuple[str, str]] = [] + captured: list[FakeClient] = [] + + def client_factory(options: object) -> FakeClient: + client = FakeClient(options, [result()]) + captured.append(client) + return client + + executor = ModelExecutor( + runtime_config(tmp_path), + client_factory=client_factory, + event_log=lambda event, detail="", level="INFO", issue_number=None: events.append((event, detail)), + ) + asyncio.run(executor.execute(issue_body="Fix it.", working_directory=tmp_path)) + + # A 600K context needs a 4.2M reserve against a 4M budget, so the + # reserve rule sits below zero and the next check fires immediately. + executor._ledger.observe_response("main", "r-huge", 0) + executor._ledger.observe_usage("main", "r-huge", {"input_tokens": 600_000}) + assert executor._soft_threshold_crossed is False + executor._check_limits() + + assert executor._soft_threshold_crossed is True + assert executor._soft_threshold_rule == "reserve" + assert executor._soft_threshold_effective_tokens == -200_000 + crossed = [detail for name, detail in events if name == "token_soft_threshold_crossed"] + assert len(crossed) == 1 + assert _crossed_value(crossed[0], "soft_threshold_tokens") == "-200000" + assert _crossed_value(crossed[0], "rule") == "reserve" + + +def test_threshold_events_carry_the_reserve_fields(tmp_path: Path) -> None: + events: list[tuple[str, str, int | None]] = [] + + def client_factory(options: object) -> FakeClient: + return FakeClient( + options, + [ + AssistantMessage( + content=[], + model="muse-spark-1.3-contributor", + message_id="r1", + usage={"input_tokens": 1000, "output_tokens": 100}, + ), + result(), + ], + ) + + executor = ModelExecutor( + runtime_config(tmp_path), + client_factory=client_factory, + event_log=lambda event, detail="", level="INFO", issue_number=None: events.append((event, detail, issue_number)), + ) + execution = asyncio.run( + executor.execute(issue_body="Fix it.", working_directory=tmp_path, issue_number=7) + ) + + assert execution.status is ModelExecutionStatus.SUCCEEDED + started = next(detail for name, detail, _ in events if name == "model_execution_started") + assert "max_budget_tokens=4000000" in started + assert "soft_threshold_percentage=0.2" in started + assert "reserve_turns=6" in started + assert "soft_threshold_tokens=3200000" in started + assert "rule=percentage" in started + limits = [detail for name, detail, _ in events if name == "limits_checked"] + assert len(limits) == 1 + assert "soft_threshold_tokens=3200000" in limits[0] + assert "rule=percentage" in limits[0] + assert "reserve_turns=6" in limits[0] + assert "reserve_tokens=" in limits[0] + assert "max_budget_tokens=4000000" in limits[0] + budget = execution.token_budget + assert budget["max_budget_tokens"] == 4_000_000 + assert budget["soft_threshold_tokens"] == 3_200_000 + assert budget["soft_threshold_percentage"] == 0.2 + assert budget["handoff_reserve_turns"] == 6 + assert budget["soft_threshold_rule"] is None + assert budget["effective_soft_threshold_tokens"] is None From 267821d6fe792c0e844ce49965a9ae34d0995d63 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 11 Oct 2026 15:44:01 +0000 Subject: [PATCH 2/2] Explain small cache-read fixture in pricing test The handoff-reserve rule trips on multi-million-token contexts, so the fixture stays small to keep this pricing test out of a handoff. Co-Authored-By: Claude Code --- tests/test_model_execution.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_model_execution.py b/tests/test_model_execution.py index 534360e..adb35a7 100644 --- a/tests/test_model_execution.py +++ b/tests/test_model_execution.py @@ -1185,7 +1185,8 @@ def client_factory(options: object) -> FakeClient: def test_per_category_token_pricing_discounts_cache_read_tokens(tmp_path: Path) -> None: captured: list[FakeClient] = [] - + # Kept small on purpose: a multi-million-token context would trip the + # handoff-reserve rule and steer this pricing test into a handoff. def client_factory(options: object) -> FakeClient: client = FakeClient( options,