diff --git a/changes/532.fixed.md b/changes/532.fixed.md new file mode 100644 index 00000000..8e2853bd --- /dev/null +++ b/changes/532.fixed.md @@ -0,0 +1 @@ +**A small local model that writes a tool call out as text no longer shows you the raw call.** Models like qwen3:8b sometimes answered with the call itself, such as a note to remember, as text, so nothing happened. Sentient now makes that call for real, with the same checks and approvals as any other, or asks the model once to make it properly. diff --git a/sentient/agent/loop.py b/sentient/agent/loop.py index a79d5bf5..885be4db 100644 --- a/sentient/agent/loop.py +++ b/sentient/agent/loop.py @@ -49,6 +49,7 @@ reversible, rule_levels, second_look, + text_calls, ) from sentient.agent import grants as grant_rules from sentient.agent import incognito as incognito_outputs @@ -208,6 +209,11 @@ # ---------------------------------------------------------------------- transcript helpers +def _call_key(name: str, arguments: Any) -> tuple[str, str]: + """A call's tool and arguments, comparable across rounds.""" + return name, json.dumps(arguments, sort_keys=True, ensure_ascii=False, default=str) + + def _normalize_tool_calls(calls: Any) -> list[dict]: out: list[dict] = [] seen: set[str] = set() @@ -877,6 +883,9 @@ async def run_loop( text_acc = "" tools_disabled = False empty_nudged = False + call_nudge: dict | None = None # the one nudge about a tool call written out as text (#532) + known = {t.name for t in self.registry.tools(include_hidden=True)} + made: set[tuple[str, str]] = set() # (tool, arguments) of every call this run made round_no = 0 def interrupted() -> bool: if interrupt is None or not interrupt(): @@ -906,6 +915,7 @@ def interrupted() -> bool: ctx.user_words.append(text) # the user's own words count for "Only if I asked" (#140) text_acc = "" think_acc = "" + shown = 0 # characters of text_acc streamed so far; a possible written-out tool call is held back (#532) tool_calls: list[ToolCall] = [] try: async for chunk in self._model_stream(role, messages, tools, model, source == "chat" and not unprompted, @@ -915,7 +925,11 @@ def interrupted() -> bool: yield ThinkingDelta(text=chunk.thinking, **ev) if chunk.text: text_acc += chunk.text - yield TextDelta(text=chunk.text, **ev) + upto = (text_calls.held_from(text_acc, known, text_calls.line_start(text_acc, shown)) + if tools else len(text_acc)) + if upto > shown: + yield TextDelta(text=text_acc[shown:upto], **ev) + shown = upto if chunk.done: tool_calls = chunk.tool_calls if chunk.model and chunk.model != (model or self.llm.model_for(role)): @@ -966,6 +980,43 @@ def interrupted() -> bool: return result.thinking += think_acc ctx.result_limit = result.tool_result_limit # a tool returning a part sizes it to fit (#336) + if call_nudge is not None and messages and messages[-1] is call_nudge: + messages.pop() # the model has answered the nudge: it is scaffolding, not part of the conversation + only = {tc.name for tc in tool_calls} if tool_calls else None + written = text_calls.find(text_acc, self._tool_schema, only) if tools and shown < len(text_acc) else None + streamed = text_acc[:shown] + if written is not None and tool_calls: + text_acc = written.cleaned # it made the real calls too: the written copy is never shown + elif written is not None and written.runnable and all(_call_key(n, a) in made for n, a in written.calls): + # real qwen3:8b wrote the same call out again after its result: it already ran, so this is the answer + log.info("%s: the model wrote out %s again after it ran; not running it twice", source, + ", ".join(written.names)) + text_acc = written.before + elif written is not None and written.runnable: + # a small model wrote its call out as text: it runs as the real call, through every usual check + log.info("%s: the model wrote %s out as text; running it as a call", source, ", ".join(written.names)) + tool_calls = [ToolCall(id=f"text_call_{new_id()[:12]}", name=name, arguments=args) + for name, args in written.calls] + text_acc = written.before + elif written is not None and written.is_action: + text_acc = written.before + if call_nudge is None and round_no < rounds: + log.info("%s: the model wrote %s out as text and it can't run; asking once to call it", + source, written.names[0]) + if shown < len(text_acc) and text_acc.startswith(streamed): + yield TextDelta(text=text_acc[shown:], **ev) + if text_acc.strip(): + messages.append({"role": "assistant", "content": text_acc}) + if persist: + await persist("assistant", text_acc, thinking=think_acc or None) + call_nudge = {"role": "user", "content": text_calls.NUDGE.format(name=written.names[0])} + messages.append(call_nudge) + continue + if not text_acc.strip() and not result.findings: + text_acc = text_calls.NOT_RUN_REPLY + if shown < len(text_acc) and text_acc.startswith(streamed): # the rest of what was held back, if any + yield TextDelta(text=text_acc[shown:], **ev) + shown = len(text_acc) if not tool_calls: pending = steer.take() if steer is not None else [] @@ -1014,6 +1065,7 @@ def interrupted() -> bool: return # a delete for good becomes its app's trash version unless the user asked for it to be permanent (#142); # swapped before the call is saved, so the transcript, approvals and second look all see the trash call + made.update(_call_key(tc.name, tc.arguments) for tc in tool_calls) swapped = self._prefer_trash(tool_calls, messages, ctx) if swapped and tool_names is not None: tool_names = [*tool_names, *(tc.name for tc in tool_calls if tc.id in swapped and tc.name not in tool_names)] @@ -1330,6 +1382,11 @@ async def _plan_call( plan.wants_ok, plan.ask_rule = True, False return plan + def _tool_schema(self, name: str) -> dict | None: + """The JSON schema of a registered tool's arguments, None for a name no tool has (#532).""" + tool = self.registry.get(name) + return tool.openai_schema()["function"]["parameters"] if tool is not None else None + def _prefer_trash(self, tool_calls: list[ToolCall], messages: list[dict], ctx: ToolContext) -> dict[str, str]: """Swap each delete-for-good call for its app's trash version, in place (#142, ``sentient.agent.reversible``). Returns ``{call id: the tool the model called}`` for the calls swapped. Skipped when ``prefer_trash`` is off, diff --git a/sentient/agent/text_calls.py b/sentient/agent/text_calls.py new file mode 100644 index 00000000..85019556 --- /dev/null +++ b/sentient/agent/text_calls.py @@ -0,0 +1,401 @@ +"""Tool calls a small model wrote out as text instead of making them (#532). + +Real qwen3:8b sometimes answers with the call itself as text: ``{"name": "memory_remember", "arguments": {...}}``, +the same inside ```` tags or a code fence, or ``memory_remember({...})``, now and then after a short +sentence ("I'll note that.") and once wrapped in a made-up ``[inside content CODE ... ]`` marker. Nothing ran, and +the user saw the raw call. + +Two parts, both deterministic: + +- ``held_from`` tells the loop which tail of a streaming reply could still turn out to be such a call, so that part + is held back instead of shown. Anything that clearly is not a call (prose, other code) is shown at once. +- ``find`` reads a finished reply. When it is (essentially) only written-out calls of known tools, with arguments + that fit each tool, the loop makes them real calls through the usual path, so risk, rules and approvals all apply. + A written call that can't be run (broken JSON, wrong arguments, an unknown tool in ```` tags) gets one + nudge instead. A code example inside a normal prose answer is left alone: only a reply that is just the call, or + a short sentence around it, counts. +""" + +from __future__ import annotations + +import ast +import json +import re +from collections.abc import Callable, Collection +from dataclasses import dataclass, field +from typing import Any + +from sentient.llm.provider import parse_json_loose, tool_arguments + +# keys that name the tool, in order; "type" only counts when its value is a tool's name (qwen wrote that once) +NAME_KEYS = ("name", "tool", "tool_name", "function", "type") +ARG_KEYS = ("arguments", "parameters", "args", "input", "params") +TAGS = ("", "") +TAG_RE = re.compile(r"<(tool_calls?)>(.*?)(?:|$)", re.DOTALL) # the closing tag matches the opening one +# prose around written calls, in characters: up to this the call is the reply's action ("I'll note that." + call) +MAX_AROUND = 400 +NUDGE = ( + "You wrote the {name} call out as text, so it did not run and the user can't see it. " + "Make the tool call itself now, or answer the user in plain words." +) +NOT_RUN_REPLY = "I tried to use one of my tools, but the step didn't go through. Please ask me again." +SchemaFn = Callable[[str], "dict | None"] # a known tool's JSON schema for its arguments, None for unknown names + +_WRAPPER_LINE = re.compile(r"^\s*(?:```[\w-]*||\[[^\n]*)\s*$") +_TRAILING_WRAPPERS = re.compile(r"(?:\s*(?:```[\w-]*||\[[^\]\n]*|`))*\s*$") +_IDENT = re.compile(r"(?:functions\.)?([A-Za-z_]\w*)") +_NAME_FIELD = re.compile(r'"(name|tool|tool_name|function|type)"\s*:\s*"((?:[^"\\]|\\.)*)("?)') +_CALL_START = re.compile(r"(?") + cleaned: str = "" # the reply without the written calls + runnable: bool = False # the reply is the call(s), and every one fits its tool: run them + is_action: bool = False # the reply is the call(s) (alone or with a short sentence), not prose with an example + + +def _strip_prefix(name: str) -> str: + return name.removeprefix("functions.").strip() + + +# ---------------------------------------------------------------------- while streaming +def held_from(text: str, names: Collection[str], start: int = 0) -> int: + """Index in ``text`` from which the reply must be held back because it could still be a written-out call of one + of ``names``; ``len(text)`` when all of it can be shown. Only lines from ``start`` on are looked at.""" + tag = _tag_start(text, start) + ls = start + while ls < tag: + if _could_be_call(text[ls:], names): + return ls + nl = text.find("\n", ls) + end = tag if nl == -1 else min(nl, tag) + brace = text.find("{", ls, end) # "I'll remember this: {"name": ...}": a call after prose on the same line + while brace != -1: + if text[brace + 1 :].lstrip()[:1] in ('"', "") and _inner_could_be_call(text[brace:], names): + return brace + brace = text.find("{", brace + 1, end) + if nl == -1: + break + ls = nl + 1 + return tag + + +def line_start(text: str, pos: int) -> int: + """Start of the line that holds ``pos``.""" + return text.rfind("\n", 0, pos) + 1 + + +def _tag_start(text: str, start: int) -> int: + """Where a ```` tag (or the start of one, at the very end) begins; ``len(text)`` when none.""" + idx = text.find("".startswith(text[lt:]): + return lt + return len(text) + + +def _could_be_call(s: str, names: Collection[str]) -> bool: + """True while the text from a line start can still be a written call (undecided counts as yes).""" + body = s.lstrip(" \t") + if not body: + return True + if body[0] == "\n": + return False # a blank line; the next line is looked at on its own + if body.startswith("```"): + nl = body.find("\n") + if nl == -1: + return True # the fence line is still coming + return _inner_could_be_call(body[nl + 1 :], names) + if body[0] == "[" and len(body) > 1 and body[1].isalpha(): + nl = body.find("\n") # a marker line like "[inside content CODE": it depends on the next line + if nl == -1: + return True + return _inner_could_be_call(body[nl + 1 :], names) + if any(t.startswith(body[: len(t)]) for t in TAGS): + return True + return _inner_could_be_call(body, names) + + +def _inner_could_be_call(s: str, names: Collection[str]) -> bool: + inner = s.lstrip() + if not inner: + return True + if inner[0] == "[": + inner = inner[1:].lstrip() + if not inner: + return True + if inner[0] == "{": + for m in _NAME_FIELD.finditer(inner): + key, value, closed = m.group(1), _strip_prefix(m.group(2)), m.group(3) + if not closed: + return key == "type" or any(n.startswith(value) for n in names) or ( + key == "function" and "function".startswith(value)) + if value in names: + return True + if key == "type" or (key == "function" and value == "function"): + continue # {"type": "function"|"tool_call", ...}: the tool's name follows + return False + return _balanced_end(inner, 0) == -1 # no tool name yet: held until the object closes + if TAGS[0].startswith(inner[: len(TAGS[0])]): + return True + m = _IDENT.match(inner) + if not m: + return False + ident = m.group(1) + if m.end() == len(inner): # the word is still coming + whole = inner[: m.end()] + return any(n.startswith(ident) for n in names) or "functions.".startswith(whole) + rest = inner[m.end() :].lstrip(" ") + return ident in names and (not rest or rest[0] == "(") + + +def _balanced_end(s: str, i: int) -> int: + """Index just past the bracket that closes the one at ``s[i]``, minding JSON and Python strings; -1 if open.""" + pairs = {"{": "}", "[": "]", "(": ")"} + stack: list[str] = [] + quote = "" + j = i + while j < len(s): + c = s[j] + if quote: + if c == "\\": + j += 2 + continue + if c == quote: + quote = "" + elif c in "\"'": + quote = c + elif c in pairs: + stack.append(pairs[c]) + elif stack and c == stack[-1]: + stack.pop() + if not stack: + return j + 1 + j += 1 + return -1 + + +# ---------------------------------------------------------------------- a finished reply +def find(text: str, schema_for: SchemaFn, only: Collection[str] | None = None) -> Found | None: + """The written-out calls in a finished reply, or None when it has none (or a tool name is only mentioned). + With ``only`` (the tools the model really called), just the written copies of those calls count.""" + if not text or not text.strip(): + return None + spans: list[tuple[int, int, list[tuple[str, Any]]]] = [] # (start, end, [(tool, arguments or None)]) + for m in TAG_RE.finditer(text): + # a tag that never closes may end inside the arguments: only a complete leading object counts, never a + # repaired one + body = m.group(2) + if m.group(0).endswith(f""): + calls = _calls_in(body, schema_for, loose=True) + else: + calls = _leading_calls(body, schema_for) + spans.append((m.start(), m.end(), calls or [(_named(body), None)])) + for start, end, calls in _json_spans(text, schema_for) + _paren_spans(text, schema_for): + if not any(start < e and s < end for s, e, _ in spans): + spans.append((start, end, calls)) + if not spans: + broken = _broken(text, schema_for) + if broken is None: + return None + spans.append(broken) + if only is not None: + spans = [sp for sp in spans if all(not name or name in only for name, _ in sp[2])] + if not spans: + return None + spans.sort(key=lambda sp: sp[0]) + names: list[str] = [] + calls: list[tuple[str, dict]] = [] + fits = True + for _s, _e, written in spans: + for name, args in written: + names.append(name or "tool") + schema = schema_for(name) if name else None + if schema is None or not isinstance(args, dict) or not _fits(args, schema): + fits = False + else: + calls.append((name, args)) + pieces, last = [], 0 + for s, e, _ in spans: + pieces.append(text[last:s]) + last = e + pieces.append(text[last:]) + before = _TRAILING_WRAPPERS.sub("", pieces[0]).rstrip() + after = "\n".join(_prose(p) for p in pieces[1:]).strip() + around = len(_prose(pieces[0])) + len(after) + is_action = around <= MAX_AROUND + cleaned = "\n\n".join(p for p in (before, after) if p) + return Found(calls=calls if fits else [], names=names, before=before, cleaned=cleaned, + runnable=is_action and fits and bool(calls), is_action=is_action) + + +def _prose(piece: str) -> str: + """A piece of the reply without wrapper lines (fences, tags, ``[...]`` markers).""" + return "\n".join(line for line in piece.splitlines() if not _WRAPPER_LINE.match(line)).strip() + + +def _named(body: str) -> str: + """The tool a broken written call names, for the nudge; "" when it names none.""" + m = _NAME_FIELD.search(body) + name = _strip_prefix(m.group(2)) if m else "" + return "" if name == "function" else name + + +def _json_spans(text: str, schema_for: SchemaFn) -> list[tuple[int, int, list[tuple[str, Any]]]]: + """Every complete JSON object or array in ``text`` that is a written call of a known tool.""" + decoder = json.JSONDecoder() + out: list[tuple[int, int, list[tuple[str, Any]]]] = [] + i = 0 + while i < len(text): + if text[i] in "{[": + try: + obj, end = decoder.raw_decode(text, i) + except json.JSONDecodeError: + end = -1 + if end != -1: + calls = _as_calls(obj, schema_for) + if calls: + out.append((i, end, calls)) + i = end + continue + closing = _balanced_end(text, i) # loosely written JSON (single quotes, trailing commas) + if closing != -1: + calls = _calls_in(text[i:closing], schema_for, loose=True) + if calls: + out.append((i, closing, calls)) + i = closing + continue + i += 1 + return out + + +def _paren_spans(text: str, schema_for: SchemaFn) -> list[tuple[int, int, list[tuple[str, Any]]]]: + """``memory_remember({...})`` or ``memory_remember(fact="...")`` for a known tool.""" + out: list[tuple[int, int, list[tuple[str, Any]]]] = [] + for m in _CALL_START.finditer(text): + name = m.group(1) + schema = schema_for(name) + if schema is None: + continue + close = _balanced_end(text, m.end() - 1) + if close == -1: + continue + args = _paren_arguments(text[m.end() : close - 1], schema) + if args is not None: # "memory_remember (the memory tool)" in a sentence is a mention, not a call + out.append((m.start(), close, [(name, args)])) + return out + + +def _broken(text: str, schema_for: SchemaFn) -> tuple[int, int, list[tuple[str, Any]]] | None: + """A written call of a known tool that doesn't parse (cut off, broken JSON): its span, to nudge about.""" + for m in _NAME_FIELD.finditer(text): + name = _strip_prefix(m.group(2)) + if not m.group(3) or schema_for(name) is None: + continue + brace = text.rfind("{", 0, m.start()) + if brace == -1: + continue + end = _balanced_end(text, brace) + return brace, (len(text) if end == -1 else end), [(name, None)] + return None + + +def _paren_arguments(inner: str, schema: dict) -> dict | None: + inner = inner.strip() + if not inner: + return {} + if inner.startswith("{"): + try: + args = parse_json_loose(inner) + except ValueError: + return None + return args if isinstance(args, dict) else None + try: + node = ast.parse(f"f({inner})", mode="eval").body + assert isinstance(node, ast.Call) + args = {kw.arg: ast.literal_eval(kw.value) for kw in node.keywords if kw.arg} + if node.args: # one plain value for a tool with one argument: memory_remember("Maya ...") + props = list((schema.get("properties") or {}).keys()) + if len(node.args) != 1 or len(props) != 1 or len(node.keywords): + return None + args = {props[0]: ast.literal_eval(node.args[0])} + return args + except (SyntaxError, ValueError, AssertionError, TypeError): + return None + + +def _leading_calls(body: str, schema_for: SchemaFn) -> list[tuple[str, Any]]: + """The calls in a complete JSON object or array at the start of ``body`` (whatever follows it); [] otherwise.""" + body = re.sub(r"^\s*```[\w-]*\s*", "", body).lstrip() + try: + obj, _end = json.JSONDecoder().raw_decode(body) + except json.JSONDecodeError: + return [] + return _as_calls(obj, schema_for) + + +def _calls_in(body: str, schema_for: SchemaFn, *, loose: bool = False) -> list[tuple[str, Any]]: + """The calls written in one block (a tag's or a loosely written object's text).""" + body = re.sub(r"^\s*```[\w-]*\s*|\s*```\s*$", "", body.strip()) + try: + obj = json.loads(body) + except json.JSONDecodeError: + if not loose: + return [] + try: + obj = parse_json_loose(body) + except ValueError: + spans = _paren_spans(body, schema_for) + return spans[0][2] if spans else [] + return _as_calls(obj, schema_for) + + +def _as_calls(obj: Any, schema_for: SchemaFn) -> list[tuple[str, Any]]: + """``obj`` as written calls of known tools; [] when it isn't one (or names a tool that doesn't exist).""" + if isinstance(obj, list): + calls = [_as_call(o, schema_for) for o in obj] + return calls if calls and all(c is not None for c in calls) else [] # type: ignore[misc] + call = _as_call(obj, schema_for) + return [call] if call else [] + + +def _as_call(obj: Any, schema_for: SchemaFn) -> tuple[str, Any] | None: + if not isinstance(obj, dict): + return None + if isinstance(obj.get("function"), dict): # {"type": "function", "function": {"name": ..., "arguments": ...}} + return _as_call(obj["function"], schema_for) + name_key = next( + (k for k in NAME_KEYS if isinstance(obj.get(k), str) and schema_for(_strip_prefix(obj[k])) is not None), None + ) + if name_key is None: + return None + name = _strip_prefix(obj[name_key]) + arg_key = next((k for k in ARG_KEYS if k in obj), None) + if arg_key is None: # {"name": "memory_remember", "fact": "..."}: the other keys are the arguments + props = (schema_for(name) or {}).get("properties") or {} + drop = {name_key} | ({"id", "type"} - set(props)) # call bookkeeping, unless the tool takes them + args: Any = {k: v for k, v in obj.items() if k not in drop} + else: + args = obj[arg_key] + if isinstance(args, str): + args = tool_arguments(args) + if "_raw" in args: + args = None + return name, args + + +def _fits(args: dict, schema: dict) -> bool: + """The arguments name only the tool's own parameters and every required one.""" + props = schema.get("properties") + closed = isinstance(props, dict) and props and not schema.get("additionalProperties") + if closed and any(k not in props for k in args): + return False + return all(k in args for k in schema.get("required") or ()) diff --git a/tests/test_text_tool_calls.py b/tests/test_text_tool_calls.py new file mode 100644 index 00000000..e4b14717 --- /dev/null +++ b/tests/test_text_tool_calls.py @@ -0,0 +1,341 @@ +"""Real qwen3:8b sometimes writes a tool call out as text instead of making it (#532): the call never ran and the +user saw the raw call. The loop now holds such text back while it streams, runs a written-out call of a known tool +through the usual path (so risk rules and approvals apply), nudges once when it can't run, and leaves code in a +normal prose answer alone.""" + +import pytest + +from sentient.agent import text_calls +from sentient.app import SentientApp +from sentient.llm.events import ApprovalRequest, Done, TextDelta, ToolCallEvent, ToolResultEvent +from sentient.tools.base import Risk, ToolContext, ToolPlugin, tool +from tests.conftest import FakeProvider, tool_call + +FACT = "Maya runs every Sunday at 6 a.m." +CALL = '{"name": "memory_remember", "arguments": {"fact": "Maya runs every Sunday at 6 a.m."}}' +RAW_BITS = ('"arguments"', "", "memory_remember", '"fact"', "inside content") + +SHAPES = { + "bare json": CALL, + "tool_call tags": f"\n{CALL}\n", + "code fence": f"```json\n{CALL}\n```", + "call with a json object": f'memory_remember({{"fact": "{FACT}"}})', + "call with keywords": f'memory_remember(fact="{FACT}")', + "openai shape": '{"type": "function", "function": {"name": "memory_remember", "arguments": "{\\"fact\\": ' + f'\\"{FACT}\\"}}"}}}}', + # the exact reply real qwen3:8b gave while #531 was tested: a sentence, then the call in a made-up marker + "qwen marker after a sentence": "I'll note your Sunday run. \n\n[inside content CODE\n" + '{"type": "memory_remember", "arguments": {"fact": "Maya runs every Sunday at 6 a.m."}}' + "\n[end of inside content CODE]", + "a claim after the call": f"{CALL}\nDone, I saved that!", +} + + +class RecordingProvider(FakeProvider): + """Keeps a copy of each call's message contents (the loop removes its nudge from the list afterwards).""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.seen: list[list[str]] = [] + + async def stream(self, role, messages, tools=None, *, model=None): + self.seen.append([str(m.get("content")) for m in messages]) + async for chunk in super().stream(role, messages, tools, model=model): + yield chunk + + +async def _turn(config, replies, text="Keep in mind that I run every Sunday at 6 a.m.", setup=None, on_event=None): + llm = RecordingProvider(replies=replies) + app = await SentientApp(config, llm=llm, enable_background=False).start() + try: + if setup is not None: + setup(app) + sid = await app.store.create_session(channel="desktop") + events = [] + async for ev in app.agent.run_turn(sid, text): + events.append(ev) + if on_event is not None: + await on_event(app, ev) + rows = await app.store.recent_messages(sid, 50) + memories = [m["content"] for m in await app.memory.list_facts()] + return llm, events, rows, memories + finally: + await app.stop() + + +def _shown(events) -> str: + return "".join(e.text for e in events if isinstance(e, TextDelta)) + + +def _done(events) -> Done: + return next(e for e in events if isinstance(e, Done)) + + +def _nudged(llm: RecordingProvider) -> int: + """How many nudges about a written-out call the model was sent.""" + return max((sum(1 for c in contents if c.startswith("You wrote the ")) for contents in llm.seen), default=0) + + +@pytest.mark.parametrize("reply", SHAPES.values(), ids=SHAPES.keys()) +async def test_a_written_out_call_runs_as_a_real_call(config, reply): + llm, events, rows, memories = await _turn(config, [reply, "Noted, I'll keep Sunday mornings free."]) + calls = [e for e in events if isinstance(e, ToolCallEvent)] + assert [(c.name, c.arguments) for c in calls] == [("memory_remember", {"fact": FACT})] + result = next(e for e in events if isinstance(e, ToolResultEvent)) + assert not result.is_error, result.result + assert any(FACT in m for m in memories) + shown = _shown(events) + assert not any(bit in shown for bit in RAW_BITS), shown # the raw call never reaches the user + assert "Done, I saved that!" not in shown # a claim written before the call ran is dropped with it + assert _done(events).content == "Noted, I'll keep Sunday mornings free." + # the transcript keeps it as a real call, so the model reads it back as one + call_rows = [r for r in rows if r["role"] == "assistant" and r.get("tool_calls")] + assert len(call_rows) == 1 and not any(bit in (call_rows[0]["content"] or "") for bit in RAW_BITS) + assert _nudged(llm) == 0 + + +async def test_the_sentence_before_a_written_call_stays(config): + _, events, rows, _ = await _turn(config, [SHAPES["qwen marker after a sentence"], "Got it."]) + assert _shown(events).startswith("I'll note your Sunday run.") + call_row = next(r for r in rows if r["role"] == "assistant" and r.get("tool_calls")) + assert call_row["content"] == "I'll note your Sunday run." + + +PROSE = { + "json config": 'Here is a config you can use:\n```json\n{"name": "my-app", "version": 2}\n```\nSave it as app.json.', + "python code": "Try this:\n```python\nprint(sorted(items))\n```\nIt prints the list in order.", + "a mention": "I used memory_remember (the memory tool) to keep that.", + "a long answer with an example call": "Sentient's memory works like this. " * 15 + + f"\n```json\n{CALL}\n```\nThat is what a call looks like.", + "an unknown tool": '{"name": "book_flight", "arguments": {"to": "Pune"}}', +} + + +@pytest.mark.parametrize("reply", PROSE.values(), ids=PROSE.keys()) +async def test_code_in_a_normal_answer_is_not_a_call(config, reply): + llm, events, _, _ = await _turn(config, [reply, "unused"], text="Show me an example") + assert not [e for e in events if isinstance(e, ToolCallEvent)] + assert _shown(events) == reply and _done(events).content == reply + assert len(llm.seen) == 1 and _nudged(llm) == 0 + + +async def test_an_unknown_tool_in_tool_call_tags_is_nudged_not_shown(config): + reply = '{"name": "book_flight", "arguments": {"to": "Pune"}}' + llm, events, rows, _ = await _turn(config, [reply, "I can't book flights, sorry."]) + assert not [e for e in events if isinstance(e, ToolCallEvent)] + assert _nudged(llm) == 1 + assert "book_flight" in llm.seen[1][-1] + assert _shown(events) == "I can't book flights, sorry." + assert all("You wrote the " not in (r.get("content") or "") for r in rows) # the nudge is never saved + + +async def test_a_call_that_does_not_fit_its_tool_gets_one_nudge(config): + bad = '{"name": "memory_remember", "arguments": {"note": "Maya runs on Sundays"}}' # no such argument + llm, events, _, memories = await _turn(config, [bad, [tool_call("memory_remember", fact=FACT)], "Saved."]) + assert _nudged(llm) == 1 + assert [e.name for e in events if isinstance(e, ToolCallEvent)] == ["memory_remember"] + assert any(FACT in m for m in memories) + assert not any(bit in _shown(events) for bit in RAW_BITS) + assert _done(events).content == "Saved." + + +async def test_a_cut_off_call_gets_one_nudge_and_never_a_second(config): + cut = '{"name": "memory_remember", "arguments": {"fact": "Maya runs' + llm, events, _, _ = await _turn(config, [cut, cut, "unused"]) + assert _nudged(llm) == 1 and len(llm.seen) == 2 + assert not [e for e in events if isinstance(e, ToolCallEvent)] + assert _done(events).content == text_calls.NOT_RUN_REPLY + assert not any(bit in _shown(events) for bit in RAW_BITS) + + +async def test_a_written_out_risky_call_still_asks(config): + sent: list[str] = [] + + @tool("send_postcard", risk=Risk.send) + async def send_postcard(ctx: ToolContext, to: str, text: str) -> dict: + """Post a postcard to someone outside Sentient.""" + sent.append(text) + return {"posted": text} + + class Postcards(ToolPlugin): + id = "postcards" + display_name = "Postcards" + tools = [send_postcard] + + asked: list[ApprovalRequest] = [] + + async def deny(app, ev): + if isinstance(ev, ApprovalRequest): + asked.append(ev) + app.approvals.resolve(ev.approval_id, "deny") + + config.tools.approvals.mode = "ask" + reply = '\n{"name": "send_postcard", "arguments": {"to": "Arjun", "text": "Hi"}}\n' + _, events, _, _ = await _turn( + config, [reply, "I didn't send it."], text="Send Arjun a postcard saying hi", + setup=lambda app: app.registry.register(Postcards()), on_event=deny, + ) + assert len(asked) == 1 and asked[0].name == "send_postcard" and asked[0].risk == "send" + assert sent == [] + result = next(e for e in events if isinstance(e, ToolResultEvent)) + assert result.is_error + + +async def test_a_real_call_with_a_written_copy_shows_neither_copy(config): + reply_calls = [tool_call("memory_remember", fact=FACT)] + + class Both(RecordingProvider): + """One answer with text that repeats the call and the real call itself.""" + + async def stream(self, role, messages, tools=None, *, model=None): + from sentient.llm.provider import StreamChunk, ToolCall + + if not self.seen: + self.seen.append([]) + yield StreamChunk(text=f"Saving it.\n{CALL}", model="fake") + yield StreamChunk(done=True, tool_calls=[ToolCall(**tc) for tc in reply_calls], model="fake") + return + async for chunk in super().stream(role, messages, tools, model=model): + yield chunk + + llm = Both(replies=["Saved."]) + app = await SentientApp(config, llm=llm, enable_background=False).start() + try: + sid = await app.store.create_session(channel="desktop") + events = [ev async for ev in app.agent.run_turn(sid, "remember my Sunday run")] + finally: + await app.stop() + assert [e.name for e in events if isinstance(e, ToolCallEvent)] == ["memory_remember"] + shown = _shown(events) + assert shown.startswith("Saving it.") and shown.endswith("Saved.") + assert not any(bit in shown for bit in RAW_BITS), shown + + +async def test_a_model_without_tools_streams_text_untouched(config): + from sentient.agent.loop import LoopResult + + llm = FakeProvider(replies=[CALL]) + app = await SentientApp(config, llm=llm, enable_background=False).start() + try: + result = LoopResult() + ctx = ToolContext(store=app.store, config=config, llm=llm) + events = [e async for e in app.agent.run_loop([{"role": "user", "content": "hi"}], ctx, result=result, + tool_names=[])] + finally: + await app.stop() + assert _shown(events) == CALL and result.text == CALL + + +# ---------------------------------------------------------------------- the helpers on their own +SCHEMAS = {"memory_remember": {"type": "object", "properties": {"fact": {"type": "string"}}, "required": ["fact"]}} + + +def _stream(reply: str, step: int = 3) -> str: + """What the loop would have shown of ``reply`` while it streamed in pieces of ``step`` characters.""" + acc, shown = "", 0 + for i in range(0, len(reply), step): + acc += reply[i : i + step] + shown = max(shown, text_calls.held_from(acc, SCHEMAS, text_calls.line_start(acc, shown))) + return acc[:shown] + + +@pytest.mark.parametrize("reply", SHAPES.values(), ids=SHAPES.keys()) +def test_a_written_call_is_held_back_while_it_streams(reply): + shown = _stream(reply) + assert not any(bit in shown for bit in RAW_BITS), shown + + +@pytest.mark.parametrize("reply", [p for k, p in PROSE.items() if "example" not in k]) +def test_ordinary_text_streams_at_once(reply): + assert _stream(reply) == reply + + +def test_find_reads_the_call_and_the_sentence_before_it(): + found = text_calls.find(SHAPES["qwen marker after a sentence"], SCHEMAS.get) + assert found is not None and found.runnable + assert found.calls == [("memory_remember", {"fact": FACT})] + assert found.before == "I'll note your Sunday run." + + +def test_find_ignores_unknown_tools_and_mentions(): + assert text_calls.find(PROSE["an unknown tool"], SCHEMAS.get) is None + assert text_calls.find(PROSE["a mention"], SCHEMAS.get) is None + example = text_calls.find(PROSE["a long answer with an example call"], SCHEMAS.get) + assert example is not None and not example.is_action and not example.runnable + + +# ---------------------------------------------------------------------- review follow-ups +async def test_a_call_after_prose_on_the_same_line_runs(config): + reply = f"I will remember this: {CALL}" + _, events, _, memories = await _turn(config, [reply, "Saved."]) + assert [e.name for e in events if isinstance(e, ToolCallEvent)] == ["memory_remember"] + assert any(FACT in m for m in memories) + shown = _shown(events) + assert shown.startswith("I will remember this:") and not any(bit in shown for bit in RAW_BITS), shown + + +def test_a_name_last_call_stays_held_however_long(): + reply = '{"arguments": {"fact": "' + "Maya likes long walks. " * 30 + '"}, "name": "memory_remember"}' + assert _stream(reply) == "" + found = text_calls.find(reply, SCHEMAS.get) + assert found is not None and found.runnable + + +def test_tool_call_tags_in_a_long_answer_are_not_run(): + reply = "Here is how tool calls look in Sentient. " * 15 + f"\n{CALL}\nThat's the format." + found = text_calls.find(reply, SCHEMAS.get) + assert found is not None and not found.is_action and not found.runnable + + +def test_an_unclosed_tag_runs_only_a_complete_call(): + complete = text_calls.find(f"\n{CALL}", SCHEMAS.get) + assert complete is not None and complete.runnable + cut = text_calls.find('\n{"name": "memory_remember", "arguments": {"fact": "Maya ru', SCHEMAS.get) + assert cut is not None and cut.is_action and not cut.runnable # nudged, never run with a cut-off value + + +def test_a_real_call_keeps_an_unrelated_example_in_the_text(): + reply = 'Saving it. For reference: {"name": "web_search", "arguments": {"query": "x"}}\n' + CALL + schemas = {**SCHEMAS, "web_search": {"type": "object", "properties": {"query": {"type": "string"}}}} + found = text_calls.find(reply, schemas.get, only={"memory_remember"}) + assert found is not None and "web_search" in found.cleaned and "memory_remember" not in found.cleaned + + +def test_id_and_type_stay_when_the_tool_takes_them(): + schemas = {"open_ticket": {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}} + found = text_calls.find('{"name": "open_ticket", "id": "T-7"}', schemas.get) + assert found is not None and found.calls == [("open_ticket", {"id": "T-7"})] + + +async def test_the_nudge_is_gone_once_the_model_calls_the_tool(config): + bad = '{"name": "memory_remember", "arguments": {"note": "Maya runs on Sundays"}}' + llm, _, _, _ = await _turn(config, [bad, [tool_call("memory_remember", fact=FACT)], "Saved."]) + assert any(c.startswith("You wrote the ") for c in llm.seen[1]) + assert not any(c.startswith("You wrote the ") for c in llm.seen[2]) # not sent again after the real call + + +def test_a_closing_tag_must_match_its_opening_tag(): + cut = '\n{"name": "memory_remember", "arguments": {"fact": "Maya ru' + found = text_calls.find(cut, SCHEMAS.get) + assert found is not None and not found.runnable # unclosed: never repaired into a cut-off fact + + +def test_an_unknown_type_does_not_hide_the_name_after_it(): + reply = '{"type": "tool_call", "name": "memory_remember", "arguments": {"fact": "Maya runs every Sunday at 6 a.m."}}' + assert _stream(reply) == "" + found = text_calls.find(reply, SCHEMAS.get) + assert found is not None and found.runnable and found.calls == [("memory_remember", {"fact": FACT})] + + +def test_an_unclosed_tag_keeps_a_complete_call_followed_by_prose(): + found = text_calls.find(f"\n{CALL}\nI saved it.", SCHEMAS.get) + assert found is not None and found.runnable and found.calls == [("memory_remember", {"fact": FACT})] + + +async def test_the_same_call_written_again_after_it_ran_is_not_run_twice(config): + # real qwen3:8b, in a chat that already held written-out calls, wrote the call again after its result + llm, events, _, _ = await _turn(config, [f"I'll note it.\n{CALL}", f"I'll note your Sunday run.\n{CALL}", "unused"]) + assert [e.name for e in events if isinstance(e, ToolCallEvent)] == ["memory_remember"] + assert len(llm.seen) == 2 + assert _done(events).content == "I'll note your Sunday run." + assert not any(bit in _shown(events) for bit in RAW_BITS)