From 76b5cb7e87ac0d4bdc33ffe204db46e74388a466 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 21 Sep 2026 16:02:24 -0700 Subject: [PATCH 1/8] Report the tagged version in built images Two things made the GUI's version badge lie. `src/asaree/_version.py` is gitignored but was not in .dockerignore, so a host-side `uv sync` left one behind that `COPY src/` baked into the image -- and since _app_version() prefers it over the distribution metadata, the badge froze at whatever commit that checkout last built. Separately, /app is a partial checkout (no frontend/, tests/, docs/, or root files), so everything missing reads as deleted against the mounted index and `git describe --dirty` returns -dirty. setuptools_scm treats dirty like distance, so building the v0.6.0 tag produced 0.6.1.dev0: every release announced itself as a prerelease of the next one. Marking the absent paths assume-unchanged before `uv sync` restores the clean describe the tag names. Verified: a build of the v0.6.0 commit now installs asaree==0.6.0 and _app_version() returns 0.6.0, so the badge reads v0.6.0. --- .dockerignore | 7 +++++++ Dockerfile | 28 ++++++++++++++++++++++++---- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.dockerignore b/.dockerignore index ce2491d..79e3b3b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,6 +18,13 @@ data/ # stray `COPY . .` can never bake the full history, and whatever was ever # committed and later removed from it, into an image layer. .git/ +# hatch-vcs writes this at build time and it's gitignored, so a host-side `uv +# sync` leaves one behind in the working tree -- stamped with whatever commit +# that checkout last built. Copying it in would freeze the image's reported +# version at that commit forever: _app_version() reads _version.py first and +# only falls back to the distribution metadata, so the file the build itself +# generates never gets a chance to win. +src/asaree/_version.py sdk/ *.log *_out.ipynb diff --git a/Dockerfile b/Dockerfile index 55a6fa7..d72ede7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -96,9 +96,24 @@ COPY scripts/ ./scripts/ # outside Compose therefore needs the context passed explicitly: # # docker build --build-context gitdir=./.git --secret id=gh_token,env=GH_TOKEN . -RUN --mount=type=bind,from=gitdir,target=/app/.git \ +# +# The `update-index` line is what keeps the derived version honest. /app is a +# deliberately partial checkout -- frontend/, tests/, docs/ and the repo root's +# own files are never COPYed -- so against the mounted index every one of them +# reads as deleted and `git describe --dirty` (which is exactly what hatch-vcs +# runs) appends `-dirty`. setuptools_scm treats dirty like distance: a build of +# the tagged commit v0.6.0 came out as 0.6.1.dev0, i.e. every release build +# announced itself as a prerelease of the *next* one. Marking the absent paths +# assume-unchanged makes describe see the clean tree the tag actually names. +# The cost, accepted: uncommitted edits to the files that ARE copied no longer +# mark the build dirty either -- "which release is this server" is what the +# badge is for, and a dev build is identified by its commit distance anyway. +# `readwrite` is required to write the index; the mount is a throwaway copy of +# the named context, so the host's .git is not touched. +RUN --mount=type=bind,from=gitdir,target=/app/.git,readwrite \ --mount=type=cache,target=/root/.cache/uv,sharing=locked \ - uv sync --frozen --no-dev + git -C /app ls-files -d -z | xargs -0 -r git -C /app update-index --assume-unchanged \ + && uv sync --frozen --no-dev # Absent on purpose: the repo's .env. AsareeSettings reads host-side URLs # (localhost:5432) that are wrong inside a compose network; real values @@ -123,9 +138,14 @@ COPY tests/ ./tests/ # Python source of truth. Keep the test image narrow while making that source # available at the same repository-relative path used outside containers. COPY frontend/src/lib/metricCatalog.ts ./frontend/src/lib/metricCatalog.ts -RUN --mount=type=bind,from=gitdir,target=/app/.git \ +# Same assume-unchanged guard as the application stage: this reinstalls the +# project (now with tests/ present, so a different set of paths is missing), +# which regenerates _version.py -- without it the test image would overwrite +# the correct version with a dirty one. +RUN --mount=type=bind,from=gitdir,target=/app/.git,readwrite \ --mount=type=cache,target=/root/.cache/uv,sharing=locked \ - uv sync --frozen --group dev + git -C /app ls-files -d -z | xargs -0 -r git -C /app update-index --assume-unchanged \ + && uv sync --frozen --group dev CMD ["pytest", "tests/", "-q", "--tb=short"] From bd1f53bd3d7425f947fd254a630d6572d95bff81 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 21 Sep 2026 16:50:06 -0700 Subject: [PATCH 2/8] Size Reason+Act iteration caps to the wiring A Reason+Act run that hits max_iterations does not fail: Motoro's loop falls through its for...else, keeps the last tool result as the output, and still marks the run completed. Downstream that is indistinguishable from a finished run -- the Output Parser reads a tool dump instead of a report, and every contracted field the dump does not happen to state comes back null. That is what the "Extracted fields: null" reports were. The cap is a safety stop, not a budget: the agent exits the moment it answers, so a cap above what a run needs costs nothing while one below it silently truncates. Raise the new-node default from 15 to 30 and, in the inspector, offer the number the driven agent's wiring implies -- measured from the run that failed, where one tool call costs one iteration and a Script costs several (run it, then write throwaway Python to read the JSON it left on disk). --- .../components/protocol/ProtocolCanvas.tsx | 2 + .../ReasonActPatternNodeInspector.tsx | 24 ++++++ frontend/src/lib/reasonActIterations.test.ts | 85 +++++++++++++++++++ frontend/src/lib/reasonActIterations.ts | 84 ++++++++++++++++++ frontend/src/types/protocols.ts | 11 ++- 5 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 frontend/src/lib/reasonActIterations.test.ts create mode 100644 frontend/src/lib/reasonActIterations.ts diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index 0cc5150..a1db894 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -24,6 +24,7 @@ import { newNodeId } from '@/lib/nodeId' import { handoffPeers, promptReferenceScope } from '@/lib/promptReferences' import { mergeProtocolSaveIntoCache, protocolForExperimentQueryKey, protocolGraphQueryKey, toPersistedGraph } from '@/lib/protocolGraph' import { TERMINAL_RUN_STATUSES } from '@/lib/protocolRun' +import { suggestedMaxIterations } from '@/lib/reasonActIterations' import { defaultAgentNodeData, defaultAnthropicLlmNodeData, @@ -2140,6 +2141,7 @@ export const ProtocolCanvas = forwardRef setSelectedNodeId(null)} /> diff --git a/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx b/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx index a9eb79f..fd99788 100644 --- a/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx +++ b/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx @@ -31,6 +31,7 @@ export function ReasonActPatternNodeInspector({ node, experimentId, factorNodeLabel, + suggestedIterations, onChange, onClose, }: { @@ -40,6 +41,9 @@ export function ReasonActPatternNodeInspector({ // -- distinct from data.label, which is this node's own plain label shown // in the header title. factorNodeLabel: string + // What the driven agent's wiring implies (lib/reasonActIterations.ts), or + // null when this pattern drives no agent yet. + suggestedIterations: number | null onChange: (nodeId: string, data: ReasonActPatternNodeData) => void onClose: () => void }) { @@ -55,6 +59,13 @@ export function ReasonActPatternNodeInspector({ const config = data.config const bindings = data.factor_bindings ?? {} + // Only ever offered as a raise. Going below what the wiring needs truncates + // the run into a payload of nulls that still reports as completed, while + // going above it costs nothing -- the loop stops when the agent answers -- + // so there is no symmetric "you set this too high" to warn about. + const underIterated = + suggestedIterations != null && (config.max_iterations == null || config.max_iterations < suggestedIterations) + const missingFields: string[] = [] if (config.max_iterations == null) missingFields.push('Max iterations') if (config.include_scratchpad && config.scratchpad_window == null) missingFields.push('Scratchpad window') @@ -122,6 +133,19 @@ export function ReasonActPatternNodeInspector({ value={config.max_iterations ?? ''} onChange={(e) => patchConfig({ max_iterations: e.target.value === '' ? null : Number(e.target.value) })} /> + {underIterated && ( +

+ This agent's wiring suggests at least {suggestedIterations} — each tool call costs an iteration, + and a run that hits the cap stops mid-work with its answer unwritten.{' '} + +

+ )} )} diff --git a/frontend/src/lib/reasonActIterations.test.ts b/frontend/src/lib/reasonActIterations.test.ts new file mode 100644 index 0000000..084ec4a --- /dev/null +++ b/frontend/src/lib/reasonActIterations.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from 'vitest' +import { suggestedMaxIterations } from './reasonActIterations' +import type { ProtocolGraph, ProtocolNode } from '@/types/protocols' + +function node(id: string, type: string, config: Record = {}): ProtocolNode { + return { id, type, position: { x: 0, y: 0 }, data: { label: id, config } } as unknown as ProtocolNode +} + +function graph(nodes: ProtocolNode[], edges: ProtocolGraph['edges']): ProtocolGraph { + return { nodes, edges } +} + +const patternEdge = { + id: 'e-pattern', + source: 'pattern-1', + target: 'agent-1', + sourceHandle: null, + targetHandle: 'architectural_pattern', +} + +function wire(id: string, source: string, targetHandle: string) { + return { id, source, target: 'agent-1', sourceHandle: null, targetHandle } +} + +describe('suggestedMaxIterations', () => { + it('is null when the pattern drives no agent', () => { + const g = graph([node('pattern-1', 'pattern_reason_act'), node('agent-1', 'agent')], []) + expect(suggestedMaxIterations(g, 'pattern-1')).toBeNull() + }) + + it('sizes the run that exposed the problem: 4 scripts plus a dataset and a skill', () => { + const g = graph( + [ + node('pattern-1', 'pattern_reason_act'), + node('agent-1', 'agent'), + node('s1', 'script'), + node('s2', 'script'), + node('s3', 'script'), + node('s4', 'script'), + node('d1', 'dataset'), + node('k1', 'skill'), + ], + [ + patternEdge, + wire('e1', 's1', 'tool'), + wire('e2', 's2', 'tool'), + wire('e3', 's3', 'tool'), + wire('e4', 's4', 'tool'), + wire('e5', 'd1', 'dataset'), + wire('e6', 'k1', 'skill'), + ], + ) + // 4 + 4*5 + 2*2 = 28, rounded up to 30 -- the value that run needed and + // its configured 15 did not give it. + expect(suggestedMaxIterations(g, 'pattern-1')).toBe(30) + }) + + it('floors a bare agent at 10 rather than suggesting a number too small to matter', () => { + const g = graph([node('pattern-1', 'pattern_reason_act'), node('agent-1', 'agent')], [patternEdge]) + expect(suggestedMaxIterations(g, 'pattern-1')).toBe(10) + }) + + it('ignores a disabled connector, which costs the loop nothing', () => { + const g = graph( + [ + node('pattern-1', 'pattern_reason_act'), + node('agent-1', 'agent'), + node('s1', 'script'), + node('s2', 'script', { enabled: false }), + ], + [patternEdge, wire('e1', 's1', 'tool'), wire('e2', 's2', 'tool')], + ) + // 4 + 5 = 9, floored to 10; the disabled second script would have made it 15. + expect(suggestedMaxIterations(g, 'pattern-1')).toBe(10) + }) + + it('caps at the catalog schema maximum', () => { + const scripts = Array.from({ length: 30 }, (_, i) => node(`s${i}`, 'script')) + const g = graph( + [node('pattern-1', 'pattern_reason_act'), node('agent-1', 'agent'), ...scripts], + [patternEdge, ...scripts.map((s, i) => wire(`e${i}`, s.id, 'tool'))], + ) + expect(suggestedMaxIterations(g, 'pattern-1')).toBe(100) + }) +}) diff --git a/frontend/src/lib/reasonActIterations.ts b/frontend/src/lib/reasonActIterations.ts new file mode 100644 index 0000000..6aad75f --- /dev/null +++ b/frontend/src/lib/reasonActIterations.ts @@ -0,0 +1,84 @@ +import type { ProtocolGraph, ProtocolNode } from '@/types/protocols' +import { isMcpToolNodeType } from '@/lib/mcpToolMetrics' + +// What one wired thing costs the loop, in iterations. Measured from a real +// run rather than guessed (protocol "ML Reasoning Experiment", 4 Script nodes +// + Dataset + Skill, max_iterations 15): the agent spends ONE tool call per +// iteration, and a script is never one call -- it runs the script, then +// spends further iterations writing throwaway Python to read the JSON the +// script left on disk. Those 4 scripts consumed 13 iterations and the run was +// still mid-work when the cap stopped it, so the report was never written and +// its Output Parser had only a tool dump to read (every field came back +// null). Everything else -- an MCP tool, a skill, a dataset, a knowledge +// bundle -- is closer to a single load/describe call. +const SCRIPT_ITERATIONS = 5 +const CONNECTOR_ITERATIONS = 2 +// Session setup (open_workspace/load_skill) plus the iteration in which the +// agent finally writes its answer -- the one that matters most, because an +// answer that never gets written is exactly the failure this is sized to +// avoid. +const BASE_ITERATIONS = 4 +// The catalog schema's own bounds (Motoro's reason_act configuration_schema: +// minimum 1, maximum 100). The floor is higher than the schema's because a +// suggestion of "3" is noise; below this the cap is not what limits the run. +const MIN_SUGGESTION = 10 +const MAX_SUGGESTION = 100 + +function isEnabled(node: ProtocolNode): boolean { + const config = node.data.config as unknown as Record | undefined + return config?.enabled !== false +} + +/** A wired node that will cost the agent at least one tool call. + * + * Deliberately looser than `contextualMetricSuggestions`'s `sourceIsCallable`: + * that one asks "is this configured well enough to produce a metric", and a + * half-configured Script still has to be *budgeted* for, because the user will + * finish configuring it long before they revisit this number. + */ +function connectorCost(node: ProtocolNode): number { + if (!isEnabled(node)) return 0 + if (node.type === 'script') return SCRIPT_ITERATIONS + if (isMcpToolNodeType(node.type)) return CONNECTOR_ITERATIONS + if (node.type === 'skill' || node.type === 'dataset' || node.type === 'okf_bundle' || node.type === 'okf_document') { + return CONNECTOR_ITERATIONS + } + return 0 +} + +/** The `max_iterations` this pattern node's wiring implies, or `null` when it + * drives no agent yet. + * + * `max_iterations` is a **safety stop, not a budget**: a Reason+Act agent exits + * the moment it answers, so a cap above what a run needs costs nothing, while + * one below it silently truncates the run -- Motoro's loop keeps the last tool + * result as the output and still marks the run `completed` + * (`engine/runtime.py`'s `for...else`), so a cut-off run and a finished one + * look identical downstream. That asymmetry is why this rounds up and why the + * inspector only ever offers to *raise* the number. + * + * Rounded to the nearest 5 because the precision is fake -- it is a budget for + * a loop whose length depends on what the model decides to do -- and a round + * number reads as the estimate it is. + */ +export function suggestedMaxIterations(graph: ProtocolGraph, patternNodeId: string): number | null { + const nodes = new Map(graph.nodes.map((node) => [node.id, node])) + const agentIds = graph.edges + .filter((edge) => edge.source === patternNodeId && edge.targetHandle === 'architectural_pattern') + .map((edge) => edge.target) + .filter((id) => nodes.get(id)?.type === 'agent') + if (agentIds.length === 0) return null + + // The max across agents, not the sum: each agent runs its own loop, and the + // cap applies to each of them separately. + const costs = agentIds.map((agentId) => { + const wired = graph.edges + .filter((edge) => edge.target === agentId) + .map((edge) => nodes.get(edge.source)) + .filter((node): node is ProtocolNode => node !== undefined) + return wired.reduce((total, node) => total + connectorCost(node), BASE_ITERATIONS) + }) + + const suggestion = Math.ceil(Math.max(...costs) / 5) * 5 + return Math.min(MAX_SUGGESTION, Math.max(MIN_SUGGESTION, suggestion)) +} diff --git a/frontend/src/types/protocols.ts b/frontend/src/types/protocols.ts index 9d1c5cb..a1f05bc 100644 --- a/frontend/src/types/protocols.ts +++ b/frontend/src/types/protocols.ts @@ -764,10 +764,19 @@ export interface ReasonActPatternNodeData { [key: string]: unknown } +// `max_iterations: 30` departs from the catalog schema's own default of 15 on +// purpose. The cap is a safety stop, not a budget -- the loop exits as soon as +// the agent answers, so a generous cap costs a simple agent nothing, while a +// tight one truncates a tool-using agent mid-work: Motoro keeps the last tool +// result as the run output and still reports `completed`, so the run looks +// finished and its Output Parser silently yields a payload of nulls. 15 was +// measured as too low for even a modest ASAREE canvas (4 Script nodes spent 13 +// iterations before the report was started) -- see lib/reasonActIterations.ts, +// which sizes the same estimate against the actual wiring once there is any. export function defaultReasonActPatternNodeData(label = 'Reason + Act'): ReasonActPatternNodeData { return { label, - config: { max_iterations: 15, include_scratchpad: true, scratchpad_window: 10, observation_format: 'raw' }, + config: { max_iterations: 30, include_scratchpad: true, scratchpad_window: 10, observation_format: 'raw' }, } } From 58e36e5a5a8f7f71ca6c23c31aa2bf3ae0a5a5d2 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 21 Sep 2026 16:57:01 -0700 Subject: [PATCH 3/8] Make a cap-truncated agent run say so Motoro reports a Reason+Act run that exhausts max_iterations as `completed`, keeping whatever the last Act returned as its output. Two things downstream then lie: the node run shows a green "Done", and a wired Output Parser handed a tool dump instead of a written answer produces a payload whose every field is null -- rendered as a tidy list of results whose answer happens to be "null". Read the truncation off agent_runs.pattern_overrides, which the ReasonAct pattern already persists (reason_act_state.max_iterations_hit); the step rows cannot answer it, since the stored Act step carries the pre-hook should_continue and is false on every run. Carry it as a `truncation` flag on the completed node run rather than a new status: ~20 sites read that vocabulary to mean "did this node produce usable output", and for a truncated run the answer is still yes. The badge goes amber, and the output panel says what happened and which number to raise. Separately, flag an all-null payload with a caveat. The extractor types every contracted field `T | None` and may not infer, so "found nothing" and "the answer genuinely had none of these" are the same object; one null among real values is a real partial reading and stays silent. --- .../protocol/NodeRunOutputPanel.tsx | 23 +++++- .../components/protocol/ProtocolCanvas.tsx | 1 + .../components/protocol/TestRunResults.tsx | 2 +- .../components/protocol/nodes/AgentNode.tsx | 5 +- .../protocol/nodes/CriticGateNode.tsx | 8 +- frontend/src/lib/protocolRun.ts | 17 +++- frontend/src/types/protocols.ts | 8 ++ src/asaree/services/protocol_execution.py | 82 +++++++++++++++++-- tests/test_protocol_execution.py | 64 ++++++++++++++- 9 files changed, 195 insertions(+), 15 deletions(-) diff --git a/frontend/src/components/protocol/NodeRunOutputPanel.tsx b/frontend/src/components/protocol/NodeRunOutputPanel.tsx index 0b6562b..2537394 100644 --- a/frontend/src/components/protocol/NodeRunOutputPanel.tsx +++ b/frontend/src/components/protocol/NodeRunOutputPanel.tsx @@ -188,6 +188,25 @@ export function UnresolvedReferencesNote({ names }: { names: string[] }) { ) } +// The loop ran out of iterations before the agent answered. +// +// Not an error, and not a failure the run itself reports: Motoro keeps +// whatever the last Act produced and still marks the run `completed`, so +// without this the output below looks like a considered answer when it is +// really a tool result the agent never got to write up. Says what to do about +// it, because the fix is one number in the Reason + Act inspector. +export function TruncationNote({ truncation }: { truncation: NodeRunState['truncation'] }) { + if (!truncation) return null + const cap = truncation.max_iterations + return ( +

+ This agent was stopped by its iteration limit{cap ? ` of ${cap}` : ''} rather than finishing — the output below is + whatever its last step returned, not an answer it wrote. Raise Max iterations on + the Reason + Act node and run it again. +

+ ) +} + // What the Output Parser pulled out of the answer above, if one was wired. // // Below the output, never in place of it: the free text is what the agent @@ -259,7 +278,7 @@ export function NodeRunOutputPanel({ // both columns would say the split means less than it does. showReceivedPrompt?: boolean }) { - const badge = nodeRunBadge(nodeRun?.status) + const badge = nodeRunBadge(nodeRun?.status, Boolean(nodeRun?.truncation)) const unresolved = nodeRun?.unresolved_references ?? [] if (!nodeRun) { @@ -311,6 +330,8 @@ export function NodeRunOutputPanel({ referenceLabel(ref, referenceNames))} /> )} + + {nodeRun.error ? (

Error

diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index a1db894..ee9812c 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -916,6 +916,7 @@ export const ProtocolCanvas = forwardRef([ // = actively running, emerald = done, dim = not started/skipped, red = // failed. A different status domain, but the same color meanings everywhere // reads as one consistent system rather than two competing ones. -export function nodeRunBadge(status: NodeRunStatus | undefined): { label: string; className: string } | null { +// +// *truncated* is the one thing that overrides the status it is paired with: a +// run cut off by its iteration ceiling is recorded as `completed` on purpose +// (its work is real and downstream nodes consumed it), but a green "Done" is +// the wrong thing to tell someone whose agent never got to write its answer. +// Amber, the same "finished, with a caveat" color the app already uses. +export function nodeRunBadge( + status: NodeRunStatus | undefined, + truncated = false, +): { label: string; className: string } | null { + if (status === 'completed' && truncated) { + return { + label: 'Hit iteration limit', + className: 'border-transparent bg-[color-mix(in_oklch,var(--chart-4),transparent_80%)] text-[color:var(--chart-4)]', + } + } switch (status) { case 'pending': return { label: 'Queued', className: 'border-transparent bg-[color-mix(in_oklch,var(--chart-4),transparent_80%)] text-[color:var(--chart-4)]' } diff --git a/frontend/src/types/protocols.ts b/frontend/src/types/protocols.ts index a1f05bc..5465743 100644 --- a/frontend/src/types/protocols.ts +++ b/frontend/src/types/protocols.ts @@ -67,6 +67,14 @@ export interface NodeRunState { // to fail without taking the prose down with it. payload?: Record | null caveats?: string[] + // Present only when the agent's loop was cut off by its iteration ceiling + // instead of by the agent deciding it was done. The run still reports + // `completed`, and deliberately so -- everything it did up to the ceiling is + // real work that downstream nodes consumed (see `_truncation_fields`'s note + // on why this is not a status). But the *answer* was never written: what the + // run hands on is whatever the last tool happened to return, which is how a + // wired Output Parser ends up with a payload of nulls. + truncation?: { reason: string; iterations?: number | null; max_iterations?: number | null } | null // Critic Gate only -- absent on a plain agent's NodeRunState. `run_id` // above doubles as the CRITIC's own run (not the upstream worker's) for a // gate, so its own Sense/Reason/Plan/Act steps are inspectable the same diff --git a/src/asaree/services/protocol_execution.py b/src/asaree/services/protocol_execution.py index e5d4e80..4cfb699 100644 --- a/src/asaree/services/protocol_execution.py +++ b/src/asaree/services/protocol_execution.py @@ -24,7 +24,7 @@ import logging import re import uuid -from collections.abc import Collection, Iterable +from collections.abc import Collection, Iterable, Mapping from dataclasses import dataclass from pathlib import Path from typing import Any @@ -3442,6 +3442,29 @@ async def _sync_durable_agent(*, name: str, owner_id: uuid.UUID, fields: dict[st return await update_agent(existing.id, **fields) +# What an all-null payload means, said once. Motoro's extractor builds its +# model with every contracted field typed ``T | None`` and is forbidden from +# inferring a value the text does not state, so "the parser ran and found +# nothing" and "the parser ran and the answer genuinely had none of these" +# produce the identical object: every key present, every value null. Without +# this the UI renders that as a result -- a tidy list of fields whose answer is +# "null" -- which reads as a finding rather than as a failed read. +_EMPTY_PAYLOAD_CAVEAT = ( + "every field the Output Parser declared came back empty: the text it was given " + "stated none of them, so this is a failed read rather than a result" +) + + +def _payload_is_empty(payload: Any) -> bool: + """True when a payload came back shaped but entirely unfilled. + + Non-empty on purpose: a contract declaring no fields at all would otherwise + trip this vacuously, and there is nothing to warn about when nothing was + asked for. + """ + return isinstance(payload, Mapping) and len(payload) > 0 and all(value is None for value in payload.values()) + + def _extraction_fields(envelope: OutputEnvelope | None) -> dict[str, Any] | None: """What an Output Parser contributed to one node run, or ``None``. @@ -3456,13 +3479,50 @@ def _extraction_fields(envelope: OutputEnvelope | None) -> dict[str, Any] | None if envelope is None: return None fields: dict[str, Any] = {} + caveats = list(envelope.caveats or []) if envelope.payload is not None: fields["payload"] = envelope.payload - if envelope.caveats: - fields["caveats"] = list(envelope.caveats) + if _payload_is_empty(envelope.payload): + caveats.append(_EMPTY_PAYLOAD_CAVEAT) + if caveats: + fields["caveats"] = caveats return fields or None +def _truncation_fields(run: Any) -> dict[str, Any] | None: + """Whether the agent's loop was cut off by its iteration ceiling, or ``None``. + + A Reason+Act run that exhausts ``max_iterations`` does NOT fail: Motoro's + loop falls through its ``for...else``, keeps whatever the last Act produced + as the run output, and still reports ``completed`` + (``motoro/engine/runtime.py``). Downstream that is indistinguishable from an + agent that finished -- which is how a run whose report was never written + ends up handed to an Output Parser that can only find nulls in a tool dump. + + Read from ``agent_runs.pattern_overrides`` rather than rederived from the + step list: the ReasonAct pattern already records exactly this + (``reason_act_state``, see its ``_state``/``_persist_state``), and the step + rows cannot answer it -- the persisted Act step carries the pre-hook + ``should_continue`` and is ``false`` on every run, truncated or not. + + Deliberately NOT expressed as a node-run *status*: the status vocabulary is + read in ~20 places (metric collection, result-node gating, the conversation + and supervisor flows) that all mean "did this node produce usable output", + and the answer for a truncated run is still yes -- its work up to the + ceiling is real. This is a flag on a completed run, the way ``caveats`` is. + """ + state = (getattr(run, "pattern_overrides", None) or {}).get("reason_act_state") + if not isinstance(state, Mapping) or not state.get("max_iterations_hit"): + return None + return { + "truncation": { + "reason": str(state.get("terminated_by") or "max_iterations"), + "iterations": state.get("iterations"), + "max_iterations": state.get("max_iterations"), + } + } + + async def _run_agent_node( node: dict[str, Any], *, @@ -3486,10 +3546,12 @@ async def _run_agent_node( this node's own step trace (``GET /runs/{run_id}/steps``); only ``None`` if agent creation/sync itself failed before a run could even be created. - ``extraction`` is what an Output Parser contributed, ready to merge into the - node run: ``payload`` (the envelope's typed object) when the extraction - succeeded, ``caveats`` when it had something to say about why it did not. - ``None`` when there was no parser, or nothing to report. It is returned + ``extraction`` is the annotation fragment, ready to merge into the node run: + ``payload`` (the envelope's typed object) when the extraction succeeded, + ``caveats`` when it had something to say about why it did not, and + ``truncation`` when the agent's loop was cut off by its iteration ceiling + (:func:`_truncation_fields`) rather than by the agent deciding it was done. + ``None`` when there was no parser and nothing to report. It is returned *alongside* ``output_text``, never instead of it: extraction is post-hoc and best-effort (``extract_payload`` returns ``(None, caveats)`` rather than raising), so the prose handoff must never depend on it having worked -- and @@ -3632,7 +3694,11 @@ async def _run_agent_node( return None, finished.error, run.id, None envelope = parse_envelope(finished.output) output_text = envelope.result if envelope is not None else (finished.output or "") - return output_text, None, run.id, _extraction_fields(envelope) + # Merged into one fragment because both are node-run annotations on a run + # that completed, and they are usually seen together: hitting the ceiling + # is the single most common reason the parser has nothing to read. + node_fields = {**(_extraction_fields(envelope) or {}), **(_truncation_fields(finished) or {})} + return output_text, None, run.id, node_fields or None async def _run_critic( diff --git a/tests/test_protocol_execution.py b/tests/test_protocol_execution.py index 2c80da0..dc4a428 100644 --- a/tests/test_protocol_execution.py +++ b/tests/test_protocol_execution.py @@ -863,7 +863,7 @@ def test_build_user_input_lists_multiple_bound_scripts() -> None: assert "2 scripts are wired" in result assert "first-report" in result assert "second-report" in result - assert 'run_wired_script(script=...)' in result + assert "run_wired_script(script=...)" in result assert "print('first')" not in result assert "print('second')" not in result @@ -1934,6 +1934,68 @@ def test_the_extraction_fragment_carries_only_what_there_is() -> None: } +def test_an_all_null_payload_is_flagged_as_a_failed_read() -> None: + """The extractor's model types every contracted field `T | None` and may not + infer, so "found nothing" and "the answer had none of these" produce the + same all-null object -- which the UI would otherwise show as a tidy list of + results whose answer is null. One null among real values is a real partial + reading and says nothing.""" + from motoro.schemas.output import OutputEnvelope + + fields = pe._extraction_fields(OutputEnvelope(result="x", payload={"a": None, "b": None})) + assert fields is not None + assert fields["payload"] == {"a": None, "b": None} + assert fields["caveats"] == [pe._EMPTY_PAYLOAD_CAVEAT] + + # Kept alongside whatever the extractor already said, not instead of it. + both = pe._extraction_fields(OutputEnvelope(result="x", payload={"a": None}, caveats=["guessed"])) + assert both is not None + assert both["caveats"] == ["guessed", pe._EMPTY_PAYLOAD_CAVEAT] + + # A contract that declared nothing has nothing to warn about, and a payload + # with any real value in it was a successful read. + assert pe._extraction_fields(OutputEnvelope(result="x", payload={})) == {"payload": {}} + assert pe._extraction_fields(OutputEnvelope(result="x", payload={"a": None, "b": 2})) == { + "payload": {"a": None, "b": 2} + } + + +def test_a_ceiling_truncated_run_is_flagged_even_though_it_completed() -> None: + """Motoro reports a cap-exhausted Reason+Act run as `completed` with the + last tool result as its output, so the only thing that distinguishes it + from a finished run is the loop summary the pattern persists.""" + + class _Run: + def __init__(self, overrides: dict[str, object] | None) -> None: + self.pattern_overrides = overrides + + hit = { + "reason_act_state": { + "iterations": 15, + "max_iterations": 15, + "max_iterations_hit": True, + "terminated_by": "max_iterations", + } + } + assert pe._truncation_fields(_Run(hit)) == { + "truncation": {"reason": "max_iterations", "iterations": 15, "max_iterations": 15} + } + + # An agent that decided it was done, a non-ReasonAct pattern, and a run + # whose telemetry write lost its race all read as "nothing to say". + done = { + "reason_act_state": { + "iterations": 2, + "max_iterations": 15, + "max_iterations_hit": False, + "terminated_by": "final_answer", + } + } + assert pe._truncation_fields(_Run(done)) is None + assert pe._truncation_fields(_Run({"other_pattern_state": {}})) is None + assert pe._truncation_fields(_Run(None)) is None + + async def test_run_protocol_stores_the_extraction_beside_the_output_text( owner_id: uuid.UUID, monkeypatch: pytest.MonkeyPatch ) -> None: From 8af7a7ff68d75c4c67b812c3aacf7d41e00d5daf Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Mon, 21 Sep 2026 17:20:38 -0700 Subject: [PATCH 4/8] Warn on an undersized iteration cap, and stop scoring truncated runs The wiring-derived `max_iterations` suggestion only existed inside the Reason + Act inspector, where you had to already be looking at the field to see it. It now drives the node's own warning triangle and the pre-run "run anyway?" scan, so an undersized cap is visible on the canvas and interrupts a Run before it burns tokens on a loop that will be cut off. The suggestion moves to a single `suggestedIterationsByPattern` memo on the canvas so the node card and the inspector can't disagree about the number. A replicate whose agent hit the ceiling no longer counts as scored: the metrics it produced are real measurements of an unfinished run, and letting them project would make a cell read "3/3 scored" when all three agents stopped mid-work. "Scored" is `metric_values` being set, so withholding the projection excludes it from the cell accents, the design-history counts and the factorial analysis at once. The numbers survive on the attempt and in `artifacts["measurement"]`, and `artifacts["truncation"]` is what lets a "completed but unscored" replicate explain itself in the Runs tab. --- .../components/protocol/ProtocolCanvas.tsx | 16 +++- .../ReasonActPatternNodeInspector.tsx | 4 +- frontend/src/components/protocol/RunsTab.tsx | 43 +++++++++- .../components/protocol/nodeConfigIssues.ts | 15 ++++ .../protocol/nodes/ReasonActPatternNode.tsx | 14 ++- frontend/src/lib/reasonActIterations.test.ts | 19 +++- frontend/src/lib/reasonActIterations.ts | 11 +++ frontend/src/types/experiments.ts | 6 ++ src/asaree/api/experiments.py | 4 + src/asaree/services/protocol_execution.py | 14 +++ src/asaree/services/protocol_runs.py | 40 ++++++++- tests/test_protocol_runs.py | 86 +++++++++++++++++++ 12 files changed, 264 insertions(+), 8 deletions(-) diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index ee9812c..01dabbd 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -781,6 +781,18 @@ export const ProtocolCanvas = forwardRef { + const patternIds = nodes.filter((n) => n.type === 'pattern_reason_act').map((n) => n.id) + if (patternIds.length === 0) return new Map() + const graph = toPersistedGraph(nodes, edges) + return new Map(patternIds.map((id) => [id, suggestedMaxIterations(graph, id)])) + }, [nodes, edges]) + // Who each agent may consult under the Peer Collaboration coordination // strategy, mirroring services/protocol_execution.py's _connected_agent_ids: // a plain (non-connector) edge joining two Agent nodes, read undirected. The @@ -966,6 +978,7 @@ export const ProtocolCanvas = forwardRef 0), + suggestedIterations: suggestedIterationsByPattern.get(n.id) ?? null, }, } }) @@ -979,6 +992,7 @@ export const ProtocolCanvas = forwardRef setSelectedNodeId(null)} /> diff --git a/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx b/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx index fd99788..3c547f3 100644 --- a/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx +++ b/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx @@ -1,6 +1,7 @@ import { Repeat2 } from 'lucide-react' import { useState } from 'react' import { nodeAccent } from '@/lib/nodeAccent' +import { isUnderIterated } from '@/lib/reasonActIterations' import { Button } from '@/components/ui/button' import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog' import { Input } from '@/components/ui/input' @@ -63,8 +64,7 @@ export function ReasonActPatternNodeInspector({ // the run into a payload of nulls that still reports as completed, while // going above it costs nothing -- the loop stops when the agent answers -- // so there is no symmetric "you set this too high" to warn about. - const underIterated = - suggestedIterations != null && (config.max_iterations == null || config.max_iterations < suggestedIterations) + const underIterated = isUnderIterated(config.max_iterations, suggestedIterations) const missingFields: string[] = [] if (config.max_iterations == null) missingFields.push('Max iterations') diff --git a/frontend/src/components/protocol/RunsTab.tsx b/frontend/src/components/protocol/RunsTab.tsx index b247816..e80187d 100644 --- a/frontend/src/components/protocol/RunsTab.tsx +++ b/frontend/src/components/protocol/RunsTab.tsx @@ -83,6 +83,17 @@ const OBSOLETE_TRIAL_BADGE = { className: 'border-transparent bg-[color:var(--chart-2)]/10 text-[color:var(--chart-2)]', } +// Ranked above the plain status badge for the same reason Obsolete is: the +// row's status really is "completed", and that is exactly the misreading +// worth preventing. Amber, the app's "finished, with a caveat" color. +const TRUNCATED_TRIAL_BADGE = { + label: 'Hit iteration limit', + className: 'border-transparent bg-[color:var(--chart-4)]/10 text-[color:var(--chart-4)]', +} + +const TRUNCATED_REPLICATE_HELP = + 'An agent in this replicate was stopped by its iteration limit, so it finished without finishing its work. It is not counted as scored. Raise Max iterations on the Reason + Act node and run it again.' + function formatCurrency(value: number | null): string | null { if (value === null || !Number.isFinite(value)) return null return new Intl.NumberFormat(undefined, { style: 'currency', currency: 'USD', maximumFractionDigits: 2 }).format(value) @@ -749,6 +760,11 @@ export function RunsTab({ (count, cell) => count + cell.replicates.filter((replicate) => trialsByLabel.get(replicate.replicate_label)?.obsolete).length, 0, ) + const truncatedCells = cells.filter((cell) => cell.replicates.some((replicate) => trialsByLabel.get(replicate.replicate_label)?.truncated)) + const truncatedReplicateCount = cells.reduce( + (count, cell) => count + cell.replicates.filter((replicate) => trialsByLabel.get(replicate.replicate_label)?.truncated).length, + 0, + ) // Runs stays operational rather than becoming a second Results dashboard: // this one compact line answers whether there is work in flight, while // comparison metrics, spend, and outputs stay in the Results rail item. @@ -818,6 +834,12 @@ export function RunsTab({ className="flex size-4 shrink-0 items-center justify-center rounded-full bg-card ring-1 ring-[color:var(--chart-4)]/40" /> )} + {truncatedReplicateCount > 0 && ( + + )} trialsByLabel.get(replicate.replicate_label)?.obsolete).length + const truncatedCount = cell.replicates.filter((replicate) => trialsByLabel.get(replicate.replicate_label)?.truncated).length const cellResult = cellResultsByLabel.get(cell.label) const cellUsage = cellResult ? usageSummary(cellResult) : [] const activeCellRunIds = cell.replicates @@ -887,6 +910,12 @@ export function RunsTab({ className="flex size-4 shrink-0 items-center justify-center rounded-full bg-card ring-1 ring-[color:var(--chart-4)]/40" /> )} + {truncatedCount > 0 && ( + + )}
{remaining.length > 0 && (
@@ -935,7 +964,13 @@ export function RunsTab({ const trial = trialsByLabel.get(replicate.replicate_label) const replicateResult = replicateResultsByLabel.get(replicate.replicate_label) const replicateUsage = replicateResult ? usageSummary(replicateResult) : [] - const badge = trial ? (trial.obsolete ? OBSOLETE_TRIAL_BADGE : trialStatusBadge(trial.status)) : null + const badge = trial + ? trial.obsolete + ? OBSOLETE_TRIAL_BADGE + : trial.truncated + ? TRUNCATED_TRIAL_BADGE + : trialStatusBadge(trial.status) + : null return (
  • @@ -948,6 +983,12 @@ export function RunsTab({ className="flex size-4 shrink-0 items-center justify-center rounded-full bg-card ring-1 ring-[color:var(--chart-4)]/40" /> )} + {trial?.truncated && !trial.obsolete && ( + + )}
    {replicateUsage.length > 0 &&

    {replicateUsage.map((value) => {value})}

    }
    • diff --git a/frontend/src/components/protocol/nodeConfigIssues.ts b/frontend/src/components/protocol/nodeConfigIssues.ts index aec4aac..ddc476f 100644 --- a/frontend/src/components/protocol/nodeConfigIssues.ts +++ b/frontend/src/components/protocol/nodeConfigIssues.ts @@ -1,5 +1,7 @@ import type { QueryClient } from '@tanstack/react-query' import type { Edge, Node } from '@xyflow/react' +import { toPersistedGraph } from '@/lib/protocolGraph' +import { isUnderIterated, suggestedMaxIterations } from '@/lib/reasonActIterations' import type { LLMSettingModelsResponse } from '@/types/llmSettings' import type { OkfBundle, OkfDocument } from '@/types/okf' import type { Skill } from '@/types/skills' @@ -44,6 +46,9 @@ export interface NodeConfigIssue { // same as LlmNode.tsx's own empty-list case -- not treated as an issue. export function findNodeConfigIssues(nodes: Node[], edges: Edge[], queryClient: QueryClient): NodeConfigIssue[] { const agentIdsWithLlm = new Set(edges.filter((e) => e.targetHandle === 'ai').map((e) => e.target)) + // suggestedMaxIterations walks the persisted shape (it also runs against a + // graph loaded from the server), so convert once rather than per node. + const graph = toPersistedGraph(nodes, edges) const result: NodeConfigIssue[] = [] for (const node of nodes) { @@ -190,6 +195,16 @@ export function findNodeConfigIssues(nodes: Node[], edges: Edge[], queryClient: const config = (node.data as ReasonActPatternNodeData).config if (config.max_iterations == null) issues.push('Max iterations is required') if (config.include_scratchpad && config.scratchpad_window == null) issues.push('Scratchpad window is required') + // A cap below what the driven agent's wiring needs (see + // lib/reasonActIterations.ts) IS worth interrupting a Run for, unlike + // the no-tools case below: the run burns real tokens and still reports + // `completed`, but Motoro cuts the loop off before the agent writes its + // answer, so what comes back is a tool dump and an Output Parser full + // of nulls. Cheaper to raise the number than to pay for the run twice. + const suggested = suggestedMaxIterations(graph, node.id) + if (config.max_iterations != null && isUnderIterated(config.max_iterations, suggested)) { + issues.push(`Max iterations (${config.max_iterations}) is below what this agent's wiring needs (about ${suggested})`) + } // The "no tools wired, so this loop won't loop" warning deliberately // ISN'T repeated here -- it lives only where the canvas warning icon // is computed (ProtocolCanvas.tsx's agentIdsWithCallableTools). Unlike diff --git a/frontend/src/components/protocol/nodes/ReasonActPatternNode.tsx b/frontend/src/components/protocol/nodes/ReasonActPatternNode.tsx index 184157d..fff1c72 100644 --- a/frontend/src/components/protocol/nodes/ReasonActPatternNode.tsx +++ b/frontend/src/components/protocol/nodes/ReasonActPatternNode.tsx @@ -1,4 +1,5 @@ import { nodeAccent } from '@/lib/nodeAccent' +import { isUnderIterated } from '@/lib/reasonActIterations' import { useNodeConnections, type NodeProps } from '@xyflow/react' import { Repeat2 } from 'lucide-react' import type { ReasonActPatternNodeData } from '@/types/protocols' @@ -22,7 +23,7 @@ export function ReasonActPatternNode({ id, data, selected, -}: NodeProps & { data: ReasonActPatternNodeData & { hostHasNoTools?: boolean } }) { +}: NodeProps & { data: ReasonActPatternNodeData & { hostHasNoTools?: boolean; suggestedIterations?: number | null } }) { // An agent's execution pattern must never go to zero (see // ProtocolCanvas.tsx's nonDeletablePatternNodeIds), so once this is // actually wired into an agent, its hover toolbar offers Swap instead of @@ -39,6 +40,17 @@ export function ReasonActPatternNode({ const warnings: string[] = [] if (data.config.max_iterations == null) warnings.push('Max iterations is required') if (data.config.include_scratchpad && data.config.scratchpad_window == null) warnings.push('Scratchpad window is required') + // The cap is set, but lower than the driven agent's own wiring needs + // (lib/reasonActIterations.ts; computed in ProtocolCanvas.tsx because it + // depends on the AGENT's connectors, not this node's config). A warning + // rather than a silent default, because exhausting the cap does not fail the + // run: Motoro keeps the last tool result and reports `completed`, so the + // symptom the user actually sees is an Output Parser full of nulls, several + // steps removed from the number that caused it. + if (data.config.max_iterations != null && isUnderIterated(data.config.max_iterations, data.suggestedIterations ?? null)) + warnings.push( + `Max iterations (${data.config.max_iterations}) is below what this agent's wiring needs (about ${data.suggestedIterations}) -- the loop will be cut off before the agent writes its answer`, + ) // Not a misconfiguration -- the run succeeds. It just doesn't LOOP: with // nothing callable bound, motoro's reason_act ends on turn one (its own // `implicit_final_answer` path), so the arm is a single LLM call wearing a diff --git a/frontend/src/lib/reasonActIterations.test.ts b/frontend/src/lib/reasonActIterations.test.ts index 084ec4a..2c17baa 100644 --- a/frontend/src/lib/reasonActIterations.test.ts +++ b/frontend/src/lib/reasonActIterations.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { suggestedMaxIterations } from './reasonActIterations' +import { isUnderIterated, suggestedMaxIterations } from './reasonActIterations' import type { ProtocolGraph, ProtocolNode } from '@/types/protocols' function node(id: string, type: string, config: Record = {}): ProtocolNode { @@ -83,3 +83,20 @@ describe('suggestedMaxIterations', () => { expect(suggestedMaxIterations(g, 'pattern-1')).toBe(100) }) }) + +describe('isUnderIterated', () => { + it('says nothing when the wiring implies no suggestion', () => { + expect(isUnderIterated(5, null)).toBe(false) + expect(isUnderIterated(null, null)).toBe(false) + }) + + it('flags a cap below the suggestion, and an unset one', () => { + expect(isUnderIterated(15, 30)).toBe(true) + expect(isUnderIterated(null, 30)).toBe(true) + }) + + it('leaves a cap at or above the suggestion alone -- too high costs nothing', () => { + expect(isUnderIterated(30, 30)).toBe(false) + expect(isUnderIterated(100, 30)).toBe(false) + }) +}) diff --git a/frontend/src/lib/reasonActIterations.ts b/frontend/src/lib/reasonActIterations.ts index 6aad75f..2b7f8d8 100644 --- a/frontend/src/lib/reasonActIterations.ts +++ b/frontend/src/lib/reasonActIterations.ts @@ -82,3 +82,14 @@ export function suggestedMaxIterations(graph: ProtocolGraph, patternNodeId: stri const suggestion = Math.ceil(Math.max(...costs) / 5) * 5 return Math.min(MAX_SUGGESTION, Math.max(MIN_SUGGESTION, suggestion)) } + +/** Whether a configured cap is below what the wiring needs. + * + * An unset cap counts as under-iterated: an empty field is exactly the case + * where the suggestion helps most. The two surfaces that already report "Max + * iterations is required" separately (the node's warning triangle and the + * pre-run scan) gate on the field being set, so neither says it twice. + */ +export function isUnderIterated(maxIterations: number | null | undefined, suggested: number | null): boolean { + return suggested != null && (maxIterations == null || maxIterations < suggested) +} diff --git a/frontend/src/types/experiments.ts b/frontend/src/types/experiments.ts index e2f52af..2632113 100644 --- a/frontend/src/types/experiments.ts +++ b/frontend/src/types/experiments.ts @@ -213,6 +213,12 @@ export interface Trial { // True when this run used an older published canvas version than the // protocol's current published version. obsolete: boolean + // An agent in this run was stopped by its iteration ceiling, so the run + // finished without finishing its work. Such a replicate is `completed` and + // deliberately UNSCORED -- the backend withholds the measurement projection + // rather than let an unfinished run count toward a cell's scored tally -- + // so this is the only thing that explains the empty metric_values. + truncated: boolean error: string | null updated_at: string } diff --git a/src/asaree/api/experiments.py b/src/asaree/api/experiments.py index 8451b4f..bad4794 100644 --- a/src/asaree/api/experiments.py +++ b/src/asaree/api/experiments.py @@ -1168,6 +1168,9 @@ class TrialResponse(BaseModel): status: str run_id: uuid.UUID | None obsolete: bool + # Finished, but an agent hit its iteration ceiling on the way -- so this + # row is `completed` with no metric_values on purpose (see ExperimentTrial). + truncated: bool error: str | None updated_at: datetime @@ -1190,6 +1193,7 @@ async def list_experiment_trials_endpoint( status=_RUN_STATUS_TO_TRIAL_STATUS.get(t.status, t.status), run_id=t.run_id, obsolete=t.obsolete, + truncated=t.truncated, error=t.error, updated_at=t.updated_at, ) diff --git a/src/asaree/services/protocol_execution.py b/src/asaree/services/protocol_execution.py index 4cfb699..36a7c5d 100644 --- a/src/asaree/services/protocol_execution.py +++ b/src/asaree/services/protocol_execution.py @@ -82,6 +82,7 @@ get_cancel_requested_at, get_protocol_run, is_current_replicate_attempt, + node_run_truncation, set_status, touch_protocol_run_heartbeat, update_node_run, @@ -4763,6 +4764,19 @@ async def run_protocol(protocol_run_id: uuid.UUID) -> None: "artifacts": { "output_text": node_runs[result_node_id].get("output_text"), "protocol_run_id": str(protocol_run_id), + # Why this replicate will come back unscored (see + # record_measurement_evaluation). Written here, on + # the same pass as output_text, so the marker + # exists even for a replicate whose measurement + # never runs -- otherwise "completed but unscored" + # would have nothing to explain itself with. Safe + # against a rerun: create_protocol_run clears + # `artifacts` when it claims the slot. + **( + {"truncation": truncation} + if (truncation := node_run_truncation(node_runs)) is not None + else {} + ), } }, revision_id=design_revision_id, diff --git a/src/asaree/services/protocol_runs.py b/src/asaree/services/protocol_runs.py index 46ed0c9..58c29de 100644 --- a/src/asaree/services/protocol_runs.py +++ b/src/asaree/services/protocol_runs.py @@ -10,7 +10,7 @@ import logging import uuid -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import UTC, datetime from typing import Any @@ -36,6 +36,22 @@ TERMINAL_PROTOCOL_RUN_STATUSES = frozenset({"completed", "failed", "cancelled", "limit_reached"}) +def node_run_truncation(node_runs: Mapping[str, Any] | None) -> dict[str, Any] | None: + """The first agent in this run whose loop was cut off by its iteration + ceiling (see ``protocol_execution._truncation_fields``), or ``None``. + + First rather than all: the consequence is the same whichever agent it was + -- the replicate's numbers describe an unfinished run -- and naming one + node is what makes the message actionable. The rest of the chain is in the + node timeline for anyone who wants it. + """ + for node_id, node_run in (node_runs or {}).items(): + truncation = node_run.get("truncation") if isinstance(node_run, Mapping) else None + if isinstance(truncation, Mapping): + return {"node_id": node_id, **dict(truncation)} + return None + + def _apply_status(run: ProtocolRun, *, status: str, error: str | None, now: datetime) -> None: run.status = status if error is not None: @@ -422,7 +438,18 @@ async def record_measurement_evaluation( artifacts = dict(replicate.artifacts or {}) artifacts["measurement"] = evaluation.to_document() replicate.artifacts = artifacts - if measured_values: + # A replicate whose agent was cut off by its iteration ceiling is NOT + # scored: the metrics are real measurements of an unfinished run, and + # projecting them would let a cell read "3/3 scored" when all three + # agents stopped mid-work. "Scored" is `metric_values` being set -- + # one predicate, read by the cell accents, the design-history counts, + # and the factorial analysis alike -- so withholding the projection is + # what excludes it from every one of them at once. + # + # The numbers are not lost: `attempt_result["metric_values"]` above and + # `artifacts["measurement"]` here both keep the full document, so the + # run stays inspectable and a re-run at a workable cap scores normally. + if measured_values and node_run_truncation(run.node_runs) is None: replicate.metric_values = {**(replicate.metric_values or {}), **measured_values} await db.flush() await db.refresh(run) @@ -489,6 +516,14 @@ class ExperimentTrial: # protocol's current one. This is derived on read, preserving the run's # actual lifecycle status and history rather than mutating either. obsolete: bool + # An agent in this replicate's run was cut off by its iteration ceiling, so + # the run finished without finishing its work and its measurements were + # deliberately not projected onto the replicate (see + # ``record_measurement_evaluation``). Derived on read from the marker the + # run left in ``artifacts``, the same way ``obsolete`` is derived rather + # than stored: a row that is `completed` and unscored is otherwise + # inexplicable. + truncated: bool error: str | None updated_at: datetime @@ -567,6 +602,7 @@ async def list_experiment_trials( status=status, run_id=replicate.run_id, obsolete=obsolete, + truncated=bool((replicate.artifacts or {}).get("truncation")), error=error, updated_at=updated_at, ) diff --git a/tests/test_protocol_runs.py b/tests/test_protocol_runs.py index 61699ad..1b02769 100644 --- a/tests/test_protocol_runs.py +++ b/tests/test_protocol_runs.py @@ -764,6 +764,92 @@ async def test_measurement_evaluations_are_immutable_per_attempt_with_one_curren await delete_experiment(db, experiment_id) +async def test_a_truncated_run_keeps_its_numbers_but_does_not_score_its_replicate( + owner_id: uuid.UUID, +) -> None: + """An agent cut off by its iteration ceiling measured an unfinished run.""" + async with get_session() as db: + experiment = await create_experiment(db, name=f"truncated-{uuid.uuid4().hex}", owner_id=owner_id) + protocol = await create_protocol( + db, + name=f"truncated-protocol-{uuid.uuid4().hex}", + owner_id=owner_id, + experiment_id=experiment.id, + ) + replicate = await upsert_replicate( + db, + experiment_id=experiment.id, + replicate_label="cell-1", + fields={"factor_values": {"tier": "small"}}, + ) + run = await create_protocol_run( + db, + protocol_id=protocol.id, + owner_id=owner_id, + replicate_label=replicate.replicate_label, + factor_values=replicate.factor_values, + replicate_result_id=replicate.id, + design_revision_id=replicate.design_revision_id, + ) + run.node_runs = { + "agent-1": { + "status": "completed", + "truncation": {"reason": "max_iterations", "iterations": 15, "max_iterations": 15}, + } + } + await db.flush() + await record_measurement_evaluation( + db, + run.id, + MeasurementEvaluation( + replicate_id=str(replicate.id), + attempt_id=str(run.id), + observations=( + MetricObservation( + metric_id="accuracy", + metric_name="Accuracy", + value_type="number", + status="measured", + value=0.9, + error=None, + attempt_id=str(run.id), + producer=ProducerProvenance( + binding_id="reported", + producer_id="asaree.reported", + kind="reported", + version="1", + ), + input_provenance={"facts": {"protocol_run_id": str(run.id)}}, + ), + ), + artifacts=(), + ), + ) + experiment_id = experiment.id + protocol_id = protocol.id + run_id = run.id + + async with get_session() as db: + stored_run = await get_protocol_run(db, run_id) + assert stored_run is not None + assert stored_run.attempt_result is not None + # The measurement itself is real and stays inspectable. + assert stored_run.attempt_result["metric_values"] == {"Accuracy": 0.9} + stored_replicate = await get_replicate( + db, + experiment_id=experiment_id, + replicate_label="cell-1", + ) + assert stored_replicate is not None + assert not stored_replicate.metric_values + assert stored_replicate.artifacts is not None + assert stored_replicate.artifacts["measurement"]["attempt_id"] == str(run_id) + + async with get_session() as db: + await delete_protocol(db, protocol_id) + await delete_experiment(db, experiment_id) + + async def test_runtime_measurement_is_snapshotted_on_attempt_and_current_replicate( owner_id: uuid.UUID, monkeypatch ) -> None: From 1426da1c23ba1c84fbe5c57b712d66d75e25ad4a Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 22 Sep 2026 11:45:37 -0700 Subject: [PATCH 5/8] Name nodes, not uuids, in the run results panel Node progress listed one row per node in the graph keyed by node id, and only the agents had a name to fall back on -- so a dataset, an output parser or a script showed a bare uuid that matches nothing the user can point at on the canvas. lib/nodeNames.ts is the shared answer: a node's own label when the user set one, otherwise the placeholder its card already shows for that type, with same-named nodes numbered in canvas order so three unlabelled Scripts don't read as three identical rows. --- .../components/protocol/ProtocolCanvas.tsx | 17 +++-- .../protocol/TestRunResults.test.tsx | 4 +- .../components/protocol/TestRunResults.tsx | 9 ++- frontend/src/lib/nodeNames.test.ts | 39 ++++++++++ frontend/src/lib/nodeNames.ts | 76 +++++++++++++++++++ 5 files changed, 133 insertions(+), 12 deletions(-) create mode 100644 frontend/src/lib/nodeNames.test.ts create mode 100644 frontend/src/lib/nodeNames.ts diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index 01dabbd..0890fc5 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -24,6 +24,7 @@ import { newNodeId } from '@/lib/nodeId' import { handoffPeers, promptReferenceScope } from '@/lib/promptReferences' import { mergeProtocolSaveIntoCache, protocolForExperimentQueryKey, protocolGraphQueryKey, toPersistedGraph } from '@/lib/protocolGraph' import { TERMINAL_RUN_STATUSES } from '@/lib/protocolRun' +import { nodeDisplayNames } from '@/lib/nodeNames' import { suggestedMaxIterations } from '@/lib/reasonActIterations' import { defaultAgentNodeData, @@ -888,10 +889,12 @@ export const ProtocolCanvas = forwardRef new Map(nodes.filter((n) => n.type === 'agent').map((n) => [n.id, (n.data as AgentNodeData).label || 'Agent'])), - [nodes], - ) + // Every node, not just the agents: the run panels list a node_run per node + // in the graph (datasets and output parsers included -- see + // run_protocol's own loop), and a raw uuid there names nothing the user can + // find on the canvas. A superset is harmless for the transcript, whose + // speaker ids are always agents. + const nodeNames = useMemo(() => nodeDisplayNames(nodes), [nodes]) // The experiment's declared coordination strategy, which decides what the // main handles MEAN -- whether a lead marker is in force, and whether the @@ -1980,10 +1983,10 @@ export const ProtocolCanvas = forwardRef {testResultsOpen && testRunQuery.data && ( - setTestResultsOpen(false)} /> + setTestResultsOpen(false)} /> )} {playResultsOpen && playResult && ( - setPlayResultsOpen(false)} /> + setPlayResultsOpen(false)} /> )} {/* One top-left column rather than two independently-positioned overlays: the lock badge and the transcript are both anchored @@ -2002,7 +2005,7 @@ export const ProtocolCanvas = forwardRef )} {showStandaloneConversation && runQuery.data?.conversation && ( - + )} )} diff --git a/frontend/src/components/protocol/TestRunResults.test.tsx b/frontend/src/components/protocol/TestRunResults.test.tsx index b23f558..0e51c95 100644 --- a/frontend/src/components/protocol/TestRunResults.test.tsx +++ b/frontend/src/components/protocol/TestRunResults.test.tsx @@ -53,7 +53,7 @@ describe('TestRunResults', () => { state: 'working', entry_agent_id: 'agent-1', messages: [{ message_id: 'message-1', sequence: 1, from_agent_id: 'user', to_agent_id: 'agent-1', parts: [{ kind: 'text', text: 'Analyze this.' }], created_at: '2026-09-16T12:00:00Z' }], }, - })} agentNames={new Map([['agent-1', 'Analyst']])} onClose={vi.fn()} />) + })} nodeNames={new Map([['agent-1', 'Analyst']])} onClose={vi.fn()} />) expect(screen.getByText('Task execution in progress…')).toBeInTheDocument() expect(screen.getByText('Analyst')).toBeInTheDocument() @@ -116,7 +116,7 @@ describe('TestRunResults', () => { evaluation: { duration_seconds: 2, cost_usd: null }, total: { duration_seconds: 10, cost_usd: null }, }, - })} agentNames={new Map([['agent-1', 'Analyst']])} onClose={vi.fn()} />) + })} nodeNames={new Map([['agent-1', 'Analyst']])} onClose={vi.fn()} />) expect(screen.getByText('Partially evaluated')).toBeInTheDocument() expect(screen.getByText('Task')).toBeInTheDocument() diff --git a/frontend/src/components/protocol/TestRunResults.tsx b/frontend/src/components/protocol/TestRunResults.tsx index ae72d72..1987588 100644 --- a/frontend/src/components/protocol/TestRunResults.tsx +++ b/frontend/src/components/protocol/TestRunResults.tsx @@ -73,7 +73,10 @@ function ObservationCard({ observation, artifacts }: { observation: MetricObserv ) } -export function TestRunResults({ run, onClose, agentNames = new Map(), title = 'Test Run Results' }: { run: TestRun; onClose: () => void; agentNames?: Map; title?: string }) { +// `nodeNames` is every canvas node's friendly name by id (lib/nodeNames.ts), +// not just the agents': Node progress lists a row per node in the graph, and a +// bare uuid there matches nothing the user can point at on the canvas. +export function TestRunResults({ run, onClose, nodeNames = new Map(), title = 'Test Run Results' }: { run: TestRun; onClose: () => void; nodeNames?: Map; title?: string }) { const running = !TERMINAL_RUN_STATUSES.has(run.status) const nodeRuns = Object.entries(run.execution_summary.node_runs) const timestamp = new Date(run.created_at) @@ -106,14 +109,14 @@ export function TestRunResults({ run, onClose, agentNames = new Map(), title = ' {nodeRuns.map(([nodeId, nodeRun]) => { const badge = nodeRunBadge(nodeRun.status, Boolean(nodeRun.truncation)) return
    - {agentNames.get(nodeId) ?? nodeId}{badge && {badge.label}} + {nodeNames.get(nodeId) ?? nodeId}{badge && {badge.label}}
    {nodeRun.output_text ? 
    {nodeRun.output_text}
    :

    No output recorded yet.

    }{nodeRun.error &&

    {nodeRun.error}

    }
    })} )} - {run.conversation && } + {run.conversation && } {run.observations.length > 0 ? (

    Metric observations

    {run.observations.map((item) => )}
    ) : TERMINAL_RUN_STATUSES.has(run.status) ?

    No runtime metrics were declared.

    : null} diff --git a/frontend/src/lib/nodeNames.test.ts b/frontend/src/lib/nodeNames.test.ts new file mode 100644 index 0000000..cac2a14 --- /dev/null +++ b/frontend/src/lib/nodeNames.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest' +import { nodeDisplayNames } from './nodeNames' + +describe('nodeDisplayNames', () => { + it('prefers the label the user set', () => { + const names = nodeDisplayNames([{ id: 'n1', type: 'agent', data: { label: 'Analyst' } }]) + expect(names.get('n1')).toBe('Analyst') + }) + + it('falls back to the placeholder the unlabelled node shows on its card', () => { + const names = nodeDisplayNames([ + { id: 'n1', type: 'output_parser', data: { label: '' } }, + { id: 'n2', type: 'llm_anthropic', data: {} }, + ]) + expect(names.get('n1')).toBe('Output Parser') + expect(names.get('n2')).toBe('Anthropic') + }) + + it('numbers nodes that would otherwise share a name', () => { + const names = nodeDisplayNames([ + { id: 'a', type: 'script', data: {} }, + { id: 'b', type: 'script', data: {} }, + { id: 'c', type: 'script', data: { label: 'Scorer' } }, + ]) + expect(names.get('a')).toBe('Script 1') + expect(names.get('b')).toBe('Script 2') + expect(names.get('c')).toBe('Scorer') + }) + + it('words an unknown node type rather than leaving a uuid', () => { + const names = nodeDisplayNames([{ id: 'n1', type: 'web_search', data: {} }]) + expect(names.get('n1')).toBe('Web Search') + }) + + it('omits a node it cannot name, so callers keep their id fallback', () => { + const names = nodeDisplayNames([{ id: 'n1', data: {} }]) + expect(names.has('n1')).toBe(false) + }) +}) diff --git a/frontend/src/lib/nodeNames.ts b/frontend/src/lib/nodeNames.ts new file mode 100644 index 0000000..f0fd38e --- /dev/null +++ b/frontend/src/lib/nodeNames.ts @@ -0,0 +1,76 @@ +// What to call a canvas node in prose -- a run panel, a transcript, a warning +// -- when a raw node id ("f3a91c2e-...") tells the reader nothing about which +// box on the canvas it means. +// +// A node's own `data.label` is the answer whenever the user set one, so this is +// really about the ones they didn't: an unlabelled node still renders a +// placeholder on its card ("Dataset", "Output Parser" -- see each node +// component's `placeholder` prop), and that placeholder is what the user is +// looking at, so it's the name they'll recognise. The table below is those +// placeholders; keep it in step with the node components if one is renamed. +const NODE_TYPE_NAMES: Record = { + agent: 'Agent', + critic_gate: 'Critic Gate', + dataset: 'Dataset', + llm_anthropic: 'Anthropic', + llm_openai: 'OpenAI', + llm_azure_foundry: 'Azure AI Foundry', + llm_openrouter: 'OpenRouter', + llm_local: 'Local', + mcp_tool: 'MCP Tool', + mcp_client_tool: 'MCP Client Tool', + memory: 'Memory', + okf_bundle: 'OKF Bundle', + okf_document: 'OKF Document', + output_parser: 'Output Parser', + pattern_reason_act: 'Reason + Act', + pattern_single_agent_baseline: 'Single-Agent Baseline', + script: 'Script', + skill: 'Skill', +} + +export interface NamedNode { + id: string + type?: string + data?: { label?: string } | Record +} + +function fallbackName(type: string | undefined): string | null { + if (!type) return null + if (NODE_TYPE_NAMES[type]) return NODE_TYPE_NAMES[type] + // An unknown type is still better read as words than as an id -- a node kind + // added without a line above shows up as "Web Search", not as a uuid. + return type.replace(/_/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase()) +} + +/** Friendly names for canvas nodes, by node id. + * + * Nodes that end up sharing a name (three unlabelled Scripts) are numbered in + * canvas order -- "Script 1", "Script 2" -- because a run panel listing three + * identical rows is no more useful than one listing three ids. A node the + * caller can't name at all is simply absent from the map, so callers keep their + * own `?? nodeId` fallback. + */ +export function nodeDisplayNames(nodes: readonly NamedNode[]): Map { + const named = nodes.map((node) => { + const label = (node.data as { label?: string } | undefined)?.label?.trim() + return { id: node.id, name: label || fallbackName(node.type) } + }) + const counts = new Map() + for (const { name } of named) { + if (name) counts.set(name, (counts.get(name) ?? 0) + 1) + } + const seen = new Map() + const result = new Map() + for (const { id, name } of named) { + if (!name) continue + if ((counts.get(name) ?? 0) < 2) { + result.set(id, name) + continue + } + const index = (seen.get(name) ?? 0) + 1 + seen.set(name, index) + result.set(id, `${name} ${index}`) + } + return result +} From c00617abe8cee2fd553e171bd52111539fb62238 Mon Sep 17 00:00:00 2001 From: Jay Moran Date: Tue, 22 Sep 2026 11:50:12 -0700 Subject: [PATCH 6/8] Flag a Reason + Act node whose last run hit its own ceiling The agent node already badges "Hit iteration limit" after a truncated run, but the number that caused it is configured on the Reason + Act node driving that agent, which showed nothing -- so the finding pointed at a node with no fix on it. The truncation now crosses that edge and raises the pattern node's own warning triangle. Deliberately not a findNodeConfigIssues entry: that scan is static pre-flight state, and a past run's outcome would keep blocking the pre-run dialog after the cap was already raised, until the next run happened to replace it. The card warning clears as soon as the cap is above what died. An observed truncation also outranks the wiring heuristic when suggesting a number -- a run that died at 40 is the loop itself saying 40 was short, where suggestedMaxIterations is only guessing from what's connected. --- .../components/protocol/ProtocolCanvas.tsx | 36 +++++++++++++++++-- .../ReasonActPatternNodeInspector.tsx | 10 ++++-- .../protocol/nodes/ReasonActPatternNode.tsx | 24 +++++++++++-- frontend/src/lib/reasonActIterations.test.ts | 27 +++++++++++++- frontend/src/lib/reasonActIterations.ts | 20 +++++++++++ 5 files changed, 110 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/protocol/ProtocolCanvas.tsx b/frontend/src/components/protocol/ProtocolCanvas.tsx index 0890fc5..90fac8f 100644 --- a/frontend/src/components/protocol/ProtocolCanvas.tsx +++ b/frontend/src/components/protocol/ProtocolCanvas.tsx @@ -25,7 +25,7 @@ import { handoffPeers, promptReferenceScope } from '@/lib/promptReferences' import { mergeProtocolSaveIntoCache, protocolForExperimentQueryKey, protocolGraphQueryKey, toPersistedGraph } from '@/lib/protocolGraph' import { TERMINAL_RUN_STATUSES } from '@/lib/protocolRun' import { nodeDisplayNames } from '@/lib/nodeNames' -import { suggestedMaxIterations } from '@/lib/reasonActIterations' +import { raiseForTruncation, suggestedMaxIterations } from '@/lib/reasonActIterations' import { defaultAgentNodeData, defaultAnthropicLlmNodeData, @@ -49,6 +49,7 @@ import type { LlmNodeData, McpToolNodeData, MemoryNodeData, + NodeRunState, OkfBundleNodeData, OkfDocumentNodeData, OutputParserNodeData, @@ -787,13 +788,41 @@ export const ProtocolCanvas = forwardRef { + const wiringIterationsByPattern = useMemo(() => { const patternIds = nodes.filter((n) => n.type === 'pattern_reason_act').map((n) => n.id) if (patternIds.length === 0) return new Map() const graph = toPersistedGraph(nodes, edges) return new Map(patternIds.map((id) => [id, suggestedMaxIterations(graph, id)])) }, [nodes, edges]) + // What the last run's own loop reported, by PATTERN node id: a Reason+Act + // agent that exhausted its ceiling (services/protocol_execution.py's + // _truncation_fields) leaves the marker on the AGENT's node_run, but the cap + // that caused it is configured on the pattern node driving that agent, so the + // finding has to be carried across the edge to be actionable. + const truncationByPattern = useMemo(() => { + const map = new Map() + for (const n of nodes) { + if (n.type !== 'pattern_reason_act') continue + const hostId = patternHostIds.get(n.id) + const truncation = hostId ? runQuery.data?.node_runs[hostId]?.truncation : null + if (truncation) map.set(n.id, truncation) + } + return map + }, [nodes, patternHostIds, runQuery.data]) + + // A truncated run outranks the wiring estimate -- see raiseForTruncation. + const suggestedIterationsByPattern = useMemo( + () => + new Map( + [...wiringIterationsByPattern].map(([id, wiring]) => [ + id, + raiseForTruncation(wiring, truncationByPattern.get(id)?.max_iterations), + ]), + ), + [wiringIterationsByPattern, truncationByPattern], + ) + // Who each agent may consult under the Peer Collaboration coordination // strategy, mirroring services/protocol_execution.py's _connected_agent_ids: // a plain (non-connector) edge joining two Agent nodes, read undirected. The @@ -982,6 +1011,7 @@ export const ProtocolCanvas = forwardRef 0), suggestedIterations: suggestedIterationsByPattern.get(n.id) ?? null, + hostTruncation: truncationByPattern.get(n.id) ?? null, }, } }) @@ -996,6 +1026,7 @@ export const ProtocolCanvas = forwardRef setSelectedNodeId(null)} /> diff --git a/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx b/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx index 3c547f3..c237952 100644 --- a/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx +++ b/frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx @@ -33,6 +33,7 @@ export function ReasonActPatternNodeInspector({ experimentId, factorNodeLabel, suggestedIterations, + truncatedAt, onChange, onClose, }: { @@ -45,6 +46,9 @@ export function ReasonActPatternNodeInspector({ // What the driven agent's wiring implies (lib/reasonActIterations.ts), or // null when this pattern drives no agent yet. suggestedIterations: number | null + // The cap the last run actually died at, when it did -- evidence rather than + // estimate, so the hint below cites it instead of the wiring. + truncatedAt: number | null onChange: (nodeId: string, data: ReasonActPatternNodeData) => void onClose: () => void }) { @@ -135,8 +139,10 @@ export function ReasonActPatternNodeInspector({ /> {underIterated && (

    - This agent's wiring suggests at least {suggestedIterations} — each tool call costs an iteration, - and a run that hits the cap stops mid-work with its answer unwritten.{' '} + {truncatedAt != null + ? `The last run stopped at ${truncatedAt} with its answer unwritten, so at least ${suggestedIterations} — ` + : `This agent's wiring suggests at least ${suggestedIterations} — `} + each tool call costs an iteration, and a run that hits the cap stops mid-work.{' '}