Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,13 @@ data/
# stray `COPY . .` can never bake the full history, and whatever was ever
# committed and later removed from it, into an image layer.
.git/
# hatch-vcs writes this at build time and it's gitignored, so a host-side `uv
# sync` leaves one behind in the working tree -- stamped with whatever commit
# that checkout last built. Copying it in would freeze the image's reported
# version at that commit forever: _app_version() reads _version.py first and
# only falls back to the distribution metadata, so the file the build itself
# generates never gets a chance to win.
src/asaree/_version.py
sdk/
*.log
*_out.ipynb
28 changes: 24 additions & 4 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,24 @@ COPY scripts/ ./scripts/
# outside Compose therefore needs the context passed explicitly:
#
# docker build --build-context gitdir=./.git --secret id=gh_token,env=GH_TOKEN .
RUN --mount=type=bind,from=gitdir,target=/app/.git \
#
# The `update-index` line is what keeps the derived version honest. /app is a
# deliberately partial checkout -- frontend/, tests/, docs/ and the repo root's
# own files are never COPYed -- so against the mounted index every one of them
# reads as deleted and `git describe --dirty` (which is exactly what hatch-vcs
# runs) appends `-dirty`. setuptools_scm treats dirty like distance: a build of
# the tagged commit v0.6.0 came out as 0.6.1.dev0, i.e. every release build
# announced itself as a prerelease of the *next* one. Marking the absent paths
# assume-unchanged makes describe see the clean tree the tag actually names.
# The cost, accepted: uncommitted edits to the files that ARE copied no longer
# mark the build dirty either -- "which release is this server" is what the
# badge is for, and a dev build is identified by its commit distance anyway.
# `readwrite` is required to write the index; the mount is a throwaway copy of
# the named context, so the host's .git is not touched.
RUN --mount=type=bind,from=gitdir,target=/app/.git,readwrite \
--mount=type=cache,target=/root/.cache/uv,sharing=locked \
uv sync --frozen --no-dev
git -C /app ls-files -d -z | xargs -0 -r git -C /app update-index --assume-unchanged \
&& uv sync --frozen --no-dev

# Absent on purpose: the repo's .env. AsareeSettings reads host-side URLs
# (localhost:5432) that are wrong inside a compose network; real values
Expand All @@ -123,9 +138,14 @@ COPY tests/ ./tests/
# Python source of truth. Keep the test image narrow while making that source
# available at the same repository-relative path used outside containers.
COPY frontend/src/lib/metricCatalog.ts ./frontend/src/lib/metricCatalog.ts
RUN --mount=type=bind,from=gitdir,target=/app/.git \
# Same assume-unchanged guard as the application stage: this reinstalls the
# project (now with tests/ present, so a different set of paths is missing),
# which regenerates _version.py -- without it the test image would overwrite
# the correct version with a dirty one.
RUN --mount=type=bind,from=gitdir,target=/app/.git,readwrite \
--mount=type=cache,target=/root/.cache/uv,sharing=locked \
uv sync --frozen --group dev
git -C /app ls-files -d -z | xargs -0 -r git -C /app update-index --assume-unchanged \
&& uv sync --frozen --group dev

CMD ["pytest", "tests/", "-q", "--tb=short"]

Expand Down
23 changes: 22 additions & 1 deletion frontend/src/components/protocol/NodeRunOutputPanel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,25 @@ export function UnresolvedReferencesNote({ names }: { names: string[] }) {
)
}

// The loop ran out of iterations before the agent answered.
//
// Not an error, and not a failure the run itself reports: Motoro keeps
// whatever the last Act produced and still marks the run `completed`, so
// without this the output below looks like a considered answer when it is
// really a tool result the agent never got to write up. Says what to do about
// it, because the fix is one number in the Reason + Act inspector.
export function TruncationNote({ truncation }: { truncation: NodeRunState['truncation'] }) {
if (!truncation) return null
const cap = truncation.max_iterations
return (
<p className="rounded-lg border border-[color:var(--chart-4)]/40 bg-[color:var(--chart-4)]/5 p-3 text-xs text-[color:var(--chart-4)]">
This agent was stopped by its iteration limit{cap ? ` of ${cap}` : ''} rather than finishing — the output below is
whatever its last step returned, not an answer it wrote. Raise <span className="font-mono">Max iterations</span> on
the Reason + Act node and run it again.
</p>
)
}

// What the Output Parser pulled out of the answer above, if one was wired.
//
// Below the output, never in place of it: the free text is what the agent
Expand Down Expand Up @@ -259,7 +278,7 @@ export function NodeRunOutputPanel({
// both columns would say the split means less than it does.
showReceivedPrompt?: boolean
}) {
const badge = nodeRunBadge(nodeRun?.status)
const badge = nodeRunBadge(nodeRun?.status, Boolean(nodeRun?.truncation))
const unresolved = nodeRun?.unresolved_references ?? []

if (!nodeRun) {
Expand Down Expand Up @@ -311,6 +330,8 @@ export function NodeRunOutputPanel({
<UnresolvedReferencesNote names={unresolved.map((ref) => referenceLabel(ref, referenceNames))} />
)}

<TruncationNote truncation={nodeRun.truncation} />

{nodeRun.error ? (
<div className="space-y-1.5">
<p className="text-sm font-medium">Error</p>
Expand Down
93 changes: 86 additions & 7 deletions frontend/src/components/protocol/ProtocolCanvas.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import { newNodeId } from '@/lib/nodeId'
import { handoffPeers, promptReferenceScope } from '@/lib/promptReferences'
import { mergeProtocolSaveIntoCache, protocolForExperimentQueryKey, protocolGraphQueryKey, toPersistedGraph } from '@/lib/protocolGraph'
import { TERMINAL_RUN_STATUSES } from '@/lib/protocolRun'
import { nodeDisplayNames } from '@/lib/nodeNames'
import { raiseForTruncation, suggestedMaxIterations } from '@/lib/reasonActIterations'
import {
defaultAgentNodeData,
defaultAnthropicLlmNodeData,
Expand All @@ -47,6 +49,7 @@ import type {
LlmNodeData,
McpToolNodeData,
MemoryNodeData,
NodeRunState,
OkfBundleNodeData,
OkfDocumentNodeData,
OutputParserNodeData,
Expand Down Expand Up @@ -780,6 +783,72 @@ export const ProtocolCanvas = forwardRef<ProtocolCanvasHandle, {
return map
}, [edges])

// The iteration cap each Reason+Act node's driven agent actually needs
// (lib/reasonActIterations.ts), by pattern node id. Computed here, once for
// the whole canvas, because it depends on the AGENT's wiring rather than the
// pattern node's own data -- and because the node card's warning triangle
// and the inspector's "Use N" hint have to agree on the number.
const wiringIterationsByPattern = useMemo(() => {
const patternIds = nodes.filter((n) => n.type === 'pattern_reason_act').map((n) => n.id)
if (patternIds.length === 0) return new Map<string, number | null>()
const graph = toPersistedGraph(nodes, edges)
return new Map(patternIds.map((id) => [id, suggestedMaxIterations(graph, id)]))
}, [nodes, edges])

// The most recent thing this canvas actually did, whichever kind it was.
// The node badges read `runQuery` alone, because that's the run the canvas
// is *watching*; a Test Run reports itself in its own results panel instead
// (list_protocol_runs excludes test runs, so it can never seed runQuery).
// Config findings can't follow that split: a Test Run is how you iterate on
// the canvas, so "your cap is too low" learned from one has to reach the
// node you'd fix. Newest wins, so raising the cap and running for real
// clears a finding the earlier Test Run left behind.
const latestNodeRuns = useMemo(() => {
const run = runQuery.data
const test = testRunQuery.data
if (!run) return test?.execution_summary.node_runs
if (!test) return run.node_runs
return test.created_at > run.created_at ? test.execution_summary.node_runs : run.node_runs
}, [runQuery.data, testRunQuery.data])

// What that run's own loop reported, by PATTERN node id: a Reason+Act agent
// that exhausted its ceiling (services/protocol_execution.py's
// _truncation_fields) leaves the marker on the AGENT's node_run, but the cap
// that caused it is configured on the pattern node driving that agent, so the
// finding has to be carried across the edge to be actionable.
const truncationByPattern = useMemo(() => {
const map = new Map<string, NodeRunState['truncation']>()
for (const n of nodes) {
if (n.type !== 'pattern_reason_act') continue
const hostId = patternHostIds.get(n.id)
const truncation = hostId ? latestNodeRuns?.[hostId]?.truncation : null
if (truncation) map.set(n.id, truncation)
}
return map
}, [nodes, patternHostIds, latestNodeRuns])

// Just the caps, for the pre-run scan (findNodeConfigIssues), which names
// the node but has no run of its own to read.
const truncatedCaps = useMemo(() => {
const map = new Map<string, number>()
for (const [patternId, truncation] of truncationByPattern) {
if (typeof truncation?.max_iterations === 'number') map.set(patternId, truncation.max_iterations)
}
return map
}, [truncationByPattern])

// A truncated run outranks the wiring estimate -- see raiseForTruncation.
const suggestedIterationsByPattern = useMemo(
() =>
new Map(
[...wiringIterationsByPattern].map(([id, wiring]) => [
id,
raiseForTruncation(wiring, truncationByPattern.get(id)?.max_iterations),
]),
),
[wiringIterationsByPattern, truncationByPattern],
)

// Who each agent may consult under the Peer Collaboration coordination
// strategy, mirroring services/protocol_execution.py's _connected_agent_ids:
// a plain (non-connector) edge joining two Agent nodes, read undirected. The
Expand Down Expand Up @@ -875,10 +944,12 @@ export const ProtocolCanvas = forwardRef<ProtocolCanvasHandle, {
return map
}, [nodes, edges])

const agentNames = useMemo(
() => new Map(nodes.filter((n) => n.type === 'agent').map((n) => [n.id, (n.data as AgentNodeData).label || 'Agent'])),
[nodes],
)
// Every node, not just the agents: the run panels list a node_run per node
// in the graph (datasets and output parsers included -- see
// run_protocol's own loop), and a raw uuid there names nothing the user can
// find on the canvas. A superset is harmless for the transcript, whose
// speaker ids are always agents.
const nodeNames = useMemo(() => nodeDisplayNames(nodes), [nodes])

// The experiment's declared coordination strategy, which decides what the
// main handles MEAN -- whether a lead marker is in force, and whether the
Expand Down Expand Up @@ -915,6 +986,7 @@ export const ProtocolCanvas = forwardRef<ProtocolCanvasHandle, {
data: {
...n.data,
runStatus: runQuery.data?.node_runs[n.id]?.status,
runTruncated: Boolean(runQuery.data?.node_runs[n.id]?.truncation),
missingLlm: n.type === 'agent' && !agentIdsWithLlm.has(n.id),
// "Require specific output format" is on, but nothing says what the
// format is. Unlike missingLlm this doesn't stop the run -- the agent
Expand Down Expand Up @@ -964,6 +1036,8 @@ export const ProtocolCanvas = forwardRef<ProtocolCanvasHandle, {
!!patternHostId &&
!agentIdsWithCallableTools.has(patternHostId) &&
!(isPeerCollaboration && (peerIdsByAgent.get(patternHostId)?.length ?? 0) > 0),
suggestedIterations: suggestedIterationsByPattern.get(n.id) ?? null,
hostTruncation: truncationByPattern.get(n.id) ?? null,
},
}
})
Expand All @@ -977,6 +1051,8 @@ export const ProtocolCanvas = forwardRef<ProtocolCanvasHandle, {
agentIdsWithCallableTools,
patternHostIds,
peerIdsByAgent,
suggestedIterationsByPattern,
truncationByPattern,
llmConfigByAgent,
isPeerCollaboration,
isSupervisor,
Expand Down Expand Up @@ -1964,10 +2040,10 @@ export const ProtocolCanvas = forwardRef<ProtocolCanvasHandle, {
/>
</div>
{testResultsOpen && testRunQuery.data && (
<TestRunResults run={testRunQuery.data} agentNames={agentNames} onClose={() => setTestResultsOpen(false)} />
<TestRunResults run={testRunQuery.data} nodeNames={nodeNames} onClose={() => setTestResultsOpen(false)} />
)}
{playResultsOpen && playResult && (
<TestRunResults title="Play Results" run={playResult} agentNames={agentNames} onClose={() => setPlayResultsOpen(false)} />
<TestRunResults title="Play Results" run={playResult} nodeNames={nodeNames} onClose={() => setPlayResultsOpen(false)} />
)}
{/* One top-left column rather than two independently-positioned
overlays: the lock badge and the transcript are both anchored
Expand All @@ -1986,7 +2062,7 @@ export const ProtocolCanvas = forwardRef<ProtocolCanvasHandle, {
</div>
)}
{showStandaloneConversation && runQuery.data?.conversation && (
<ConversationTranscript conversation={runQuery.data.conversation} agentNames={agentNames} />
<ConversationTranscript conversation={runQuery.data.conversation} agentNames={nodeNames} />
)}
</div>
)}
Expand Down Expand Up @@ -2140,6 +2216,8 @@ export const ProtocolCanvas = forwardRef<ProtocolCanvasHandle, {
}}
experimentId={experimentId}
factorNodeLabel={factorNodeLabel}
suggestedIterations={suggestedIterationsByPattern.get(selectedNode.id) ?? null}
truncatedAt={truncationByPattern.get(selectedNode.id)?.max_iterations ?? null}
onChange={updateNodeData}
onClose={() => setSelectedNodeId(null)}
/>
Expand Down Expand Up @@ -2204,6 +2282,7 @@ export const ProtocolCanvas = forwardRef<ProtocolCanvasHandle, {
nodes={nodes}
edges={edges}
queryClient={queryClient}
truncatedCaps={truncatedCaps}
onCancel={() => setPendingRunConfirm(null)}
onConfirm={confirmPendingRun}
hasUnpublishedChanges={hasUnpublishedChanges}
Expand Down
30 changes: 30 additions & 0 deletions frontend/src/components/protocol/ReasonActPatternNodeInspector.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { Repeat2 } from 'lucide-react'
import { useState } from 'react'
import { nodeAccent } from '@/lib/nodeAccent'
import { isUnderIterated } from '@/lib/reasonActIterations'
import { Button } from '@/components/ui/button'
import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog'
import { Input } from '@/components/ui/input'
Expand Down Expand Up @@ -31,6 +32,8 @@ export function ReasonActPatternNodeInspector({
node,
experimentId,
factorNodeLabel,
suggestedIterations,
truncatedAt,
onChange,
onClose,
}: {
Expand All @@ -40,6 +43,12 @@ export function ReasonActPatternNodeInspector({
// -- distinct from data.label, which is this node's own plain label shown
// in the header title.
factorNodeLabel: string
// What the driven agent's wiring implies (lib/reasonActIterations.ts), or
// null when this pattern drives no agent yet.
suggestedIterations: number | null
// The cap the last run actually died at, when it did -- evidence rather than
// estimate, so the hint below cites it instead of the wiring.
truncatedAt: number | null
onChange: (nodeId: string, data: ReasonActPatternNodeData) => void
onClose: () => void
}) {
Expand All @@ -55,6 +64,12 @@ export function ReasonActPatternNodeInspector({
const config = data.config
const bindings = data.factor_bindings ?? {}

// Only ever offered as a raise. Going below what the wiring needs truncates
// the run into a payload of nulls that still reports as completed, while
// going above it costs nothing -- the loop stops when the agent answers --
// so there is no symmetric "you set this too high" to warn about.
const underIterated = isUnderIterated(config.max_iterations, suggestedIterations)

const missingFields: string[] = []
if (config.max_iterations == null) missingFields.push('Max iterations')
if (config.include_scratchpad && config.scratchpad_window == null) missingFields.push('Scratchpad window')
Expand Down Expand Up @@ -122,6 +137,21 @@ export function ReasonActPatternNodeInspector({
value={config.max_iterations ?? ''}
onChange={(e) => patchConfig({ max_iterations: e.target.value === '' ? null : Number(e.target.value) })}
/>
{underIterated && (
<p className="text-xs text-[color:var(--chart-4)]">
{truncatedAt != null
? `The last run stopped at ${truncatedAt} with its answer unwritten, so at least ${suggestedIterations} — `
: `This agent's wiring suggests at least ${suggestedIterations} — `}
each tool call costs an iteration, and a run that hits the cap stops mid-work.{' '}
<button
type="button"
className="underline underline-offset-2 hover:no-underline"
onClick={() => patchConfig({ max_iterations: suggestedIterations })}
>
Use {suggestedIterations}
</button>
</p>
)}
</div>
)}
</FactorBindableField>
Expand Down
Loading
Loading