diff --git a/apps/daemon/internal/dispatch/cancellation_budget_test.go b/apps/daemon/internal/dispatch/cancellation_budget_test.go new file mode 100644 index 000000000..41e08833d --- /dev/null +++ b/apps/daemon/internal/dispatch/cancellation_budget_test.go @@ -0,0 +1,81 @@ +package dispatch_test + +import ( + "context" + "testing" + "testing/synctest" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/agent" + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/dispatch" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" +) + +type hangingCancelExecutor struct { + turn *hangingCancelTurn + closeBudget time.Duration +} + +func (e *hangingCancelExecutor) StartTurn(_ context.Context, id string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Turn, error) { + e.turn = &hangingCancelTurn{reusableTurn: &reusableTurn{id: id, out: out, settled: make(chan struct{})}} + return e.turn, nil +} +func (e *hangingCancelExecutor) Close(ctx context.Context) error { + deadline, _ := ctx.Deadline() + e.closeBudget = time.Until(deadline) + e.turn.finish() + return nil +} + +type hangingCancelTurn struct { + *reusableTurn + cancelBudget time.Duration +} + +func (turn *hangingCancelTurn) Cancel(ctx context.Context) error { + turn.cancels.Add(1) + deadline, _ := ctx.Deadline() + turn.cancelBudget = time.Until(deadline) + <-ctx.Done() + return ctx.Err() +} + +func TestNativeCancellationTimeoutCannotBecomeAppliedAfterClose(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + owner := &hangingCancelExecutor{} + reg := agent.NewRegistry() + registerExecutorKind(reg, proto.SupportedAgentKind{Kind: "reusable", Available: true, Capabilities: prototest.Capabilities(proto.AgentKindCapabilities{EnvironmentNone: proto.CapabilitySupported})}, func(context.Context, agent.PrepareRequest) (agent.Executor, error) { return owner, nil }) + sender := &recSender{} + r, err := dispatch.New(dispatch.Config{Registry: reg, Sender: sender, IdleTimeout: time.Hour}) + if err != nil { + t.Fatal(err) + } + defer r.Shutdown(t.Context()) + assign(t, r, preparationSessionID, "") + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "request", executorRequest())); err != nil { + t.Fatal(err) + } + ready := waitPreparationStatus(t, sender, "request", "ready", "") + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, ExecutorID: ready.ExecutorID, RunID: "run", Input: proto.TextInput("input")})); err != nil { + t.Fatal(err) + } + synctest.Wait() + if err := r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "cancel"})); err != nil { + t.Fatal(err) + } + synctest.Wait() + time.Sleep(proto.CancellationConfirmationTimeout) + synctest.Wait() + acks := cancellationAcks(sender) + if len(acks) != 1 || acks[0].Applied || acks[0].Outcome != nil || (acks[0].ErrorCode != "cancel_timeout" && acks[0].ErrorCode != "cancel_failed") { + t.Fatalf("timeout lost unknown outcome: %+v", acks) + } + if owner.turn.cancels.Load() != 1 || owner.turn.cancelBudget != 30*time.Second || owner.closeBudget != 10*time.Second { + t.Fatalf("calls=%d native budget=%v close budget=%v", owner.turn.cancels.Load(), owner.turn.cancelBudget, owner.closeBudget) + } + if r.ActiveRuns() != 0 { + t.Fatal("confirmed close did not release retired owner") + } + }) +} diff --git a/apps/daemon/internal/dispatch/executor.go b/apps/daemon/internal/dispatch/executor.go index 7dfea527b..4883f913d 100644 --- a/apps/daemon/internal/dispatch/executor.go +++ b/apps/daemon/internal/dispatch/executor.go @@ -350,7 +350,7 @@ func (r *Router) closeExecutor(owner *executorState) error { r.mu.Unlock() var err error if native != nil { - ctx, cancel := context.WithTimeout(context.Background(), preparedCancelTimeout) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) err = native.Close(ctx) cancel() } diff --git a/apps/daemon/internal/dispatch/preparation_cancel.go b/apps/daemon/internal/dispatch/preparation_cancel.go index c0da3fcaf..c030168a2 100644 --- a/apps/daemon/internal/dispatch/preparation_cancel.go +++ b/apps/daemon/internal/dispatch/preparation_cancel.go @@ -7,11 +7,9 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" ) -const preparedCancelTimeout = 10 * time.Second - func (r *Router) sendPreparedCancellation(state *sessionState, handoff *preparedHandoff, release *preparedRelease, attempt *preparedReleaseAttempt, env proto.Envelope, deliveryID string) { defer r.shutdownWG.Done() - timer := time.NewTimer(preparedCancelTimeout) + timer := time.NewTimer(proto.CancellationConfirmationTimeout) defer timer.Stop() ack := proto.InteractionDecisionAckPayload{ErrorCode: "cancel_timeout"} select { diff --git a/apps/daemon/internal/dispatch/preparation_cancel_test.go b/apps/daemon/internal/dispatch/preparation_cancel_test.go index a5fede803..0ded850b7 100644 --- a/apps/daemon/internal/dispatch/preparation_cancel_test.go +++ b/apps/daemon/internal/dispatch/preparation_cancel_test.go @@ -7,6 +7,7 @@ import ( "sync" "sync/atomic" "testing" + "testing/synctest" "time" "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/dispatch" @@ -73,69 +74,77 @@ func (s cancellationOutputSender) Send(ctx context.Context, env proto.Envelope) } func TestPreparedCancellationWaitsForOutputAndCleanup(t *testing.T) { - sender := cancellationOutputSender{recSender: &recSender{}, entered: make(chan struct{}), release: make(chan struct{})} - startEntered, startReturn := make(chan struct{}), make(chan struct{}) - cancelEntered := make(chan struct{}) - cleanupEntered, cleanupReturn := make(chan struct{}), make(chan struct{}) - p := &cancellationPreparation{controlledPreparation: &controlledPreparation{closed: make(chan struct{})}, outcome: proto.DonePayload{ - Usage: proto.Usage{Tokens: &proto.TokenUsage{InputTokens: 7, OutputTokens: 3, TotalTokens: 10}}, Metadata: map[string]any{proto.DoneMetaAgentSessionID: "observed-native"}, - }} - var session *fakeSession - p.start = func(_ context.Context, id string, _ proto.MessageInput, out chan<- proto.Envelope) (fixtureSession, error) { - session = &fakeSession{out: out, closeOutOnCancel: true, - postCancelEnvelopes: []proto.Envelope{mustEnv(t, proto.TypeDone, id, p.outcome)}} - out <- mustEnv(t, proto.TypeDelta, id, proto.DeltaPayload{Delta: "observed"}) - out <- mustEnv(t, proto.TypeUsage, id, proto.UsagePayload{Usage: p.outcome.Usage}) - close(startEntered) - <-startReturn - return session, nil - } - p.cancel = func(ctx context.Context) error { - close(cancelEntered) - if err := session.Cancel(ctx); err != nil { - return err + synctest.Test(t, func(t *testing.T) { + sender := cancellationOutputSender{recSender: &recSender{}, entered: make(chan struct{}), release: make(chan struct{})} + startEntered, startReturn := make(chan struct{}), make(chan struct{}) + cancelEntered := make(chan struct{}) + cleanupEntered, cleanupReturn := make(chan struct{}), make(chan struct{}) + p := &cancellationPreparation{controlledPreparation: &controlledPreparation{closed: make(chan struct{})}, outcome: proto.DonePayload{ + Usage: proto.Usage{Tokens: &proto.TokenUsage{InputTokens: 7, OutputTokens: 3, TotalTokens: 10}}, Metadata: map[string]any{proto.DoneMetaAgentSessionID: "observed-native"}, + }} + var session *fakeSession + p.start = func(_ context.Context, id string, _ proto.MessageInput, out chan<- proto.Envelope) (fixtureSession, error) { + session = &fakeSession{out: out, closeOutOnCancel: true, + postCancelEnvelopes: []proto.Envelope{mustEnv(t, proto.TypeDone, id, p.outcome)}} + out <- mustEnv(t, proto.TypeDelta, id, proto.DeltaPayload{Delta: "observed"}) + out <- mustEnv(t, proto.TypeUsage, id, proto.UsagePayload{Usage: p.outcome.Usage}) + close(startEntered) + <-startReturn + return session, nil } - close(cleanupEntered) - <-cleanupReturn - return nil - } - r := preparationRouter(t, sender, time.Minute, func(context.Context, proto.PromptRequestPayload) (preparedFixture, error) { return p, nil }) - startCancellationPreparation(t, r, sender.recSender) - <-startEntered - if err := r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "cancel"})); err != nil { - t.Fatal(err) - } - <-cancelEntered - if len(cancellationAcks(sender.recSender)) != 0 { - t.Fatal("receipt preceded Start handoff") - } - close(startReturn) - <-sender.entered - if len(cancellationAcks(sender.recSender)) != 0 { - t.Fatal("receipt preceded output delivery") - } - // Another receipt may wait, but it must not repeat native cancellation. - _ = r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "second-delivery"})) - close(sender.release) - <-cleanupEntered - if len(cancellationAcks(sender.recSender)) != 0 || r.ActiveRuns() != 1 { - t.Fatal("cleanup released ownership early") - } - close(cleanupReturn) - waitFor(t, func() bool { return len(cancellationAcks(sender.recSender)) == 2 && r.ActiveRuns() == 0 }, "prepared cancellation receipts") - if p.calls.Load() != 1 || session.cancels() != 1 { - t.Fatal("native cancellation was repeated") - } - for _, ack := range cancellationAcks(sender.recSender) { - if !ack.Applied || ack.ErrorCode != "" || ack.Outcome == nil || !reflect.DeepEqual(*ack.Outcome, p.outcome) { - t.Fatalf("observed outcome lost: %+v", ack) + p.cancel = func(ctx context.Context) error { + close(cancelEntered) + if err := session.Cancel(ctx); err != nil { + return err + } + close(cleanupEntered) + <-cleanupReturn + return nil } - } - got := sender.typesFor("run") - want := []string{proto.TypeDelta, proto.TypeUsage, proto.TypeDone, proto.TypeInteractionDecisionAck, proto.TypeInteractionDecisionAck} - if !reflect.DeepEqual(got, want) { - t.Fatalf("output/receipt order = %v", got) - } + r := preparationRouter(t, sender, time.Minute, func(context.Context, proto.PromptRequestPayload) (preparedFixture, error) { return p, nil }) + startCancellationPreparation(t, r, sender.recSender) + <-startEntered + if err := r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "cancel"})); err != nil { + t.Fatal(err) + } + <-cancelEntered + if len(cancellationAcks(sender.recSender)) != 0 { + t.Fatal("receipt preceded Start handoff") + } + close(startReturn) + <-sender.entered + if len(cancellationAcks(sender.recSender)) != 0 { + t.Fatal("receipt preceded output delivery") + } + // Another receipt may wait, but it must not repeat native cancellation. + _ = r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "second-delivery"})) + // Confirmation remains pending past both former native and Core bounds. + time.Sleep(17 * time.Second) + synctest.Wait() + if len(cancellationAcks(sender.recSender)) != 0 { + t.Fatal("receipt expired before the shared cancellation budget") + } + close(sender.release) + <-cleanupEntered + if len(cancellationAcks(sender.recSender)) != 0 || r.ActiveRuns() != 1 { + t.Fatal("cleanup released ownership early") + } + close(cleanupReturn) + waitFor(t, func() bool { return len(cancellationAcks(sender.recSender)) == 2 && r.ActiveRuns() == 0 }, "prepared cancellation receipts") + if p.calls.Load() != 1 || session.cancels() != 1 { + t.Fatal("native cancellation was repeated") + } + for _, ack := range cancellationAcks(sender.recSender) { + if !ack.Applied || ack.ErrorCode != "" || ack.Outcome == nil || !reflect.DeepEqual(*ack.Outcome, p.outcome) { + t.Fatalf("observed outcome lost: %+v", ack) + } + } + got := sender.typesFor("run") + want := []string{proto.TypeDelta, proto.TypeUsage, proto.TypeDone, proto.TypeInteractionDecisionAck, proto.TypeInteractionDecisionAck} + if !reflect.DeepEqual(got, want) { + t.Fatalf("output/receipt order = %v", got) + } + }) } func TestPreparedCancellationBeforeTransferPreservesUnknownOutcome(t *testing.T) { @@ -250,44 +259,46 @@ func TestPreparedCancellationFailuresRemainConservative(t *testing.T) { } func TestPreparedCancellationTimeoutKeepsCapacityUntilStartReturns(t *testing.T) { - sender := &recSender{} - entered, allowReturn := make(chan struct{}), make(chan struct{}) - p := &cancellationPreparation{controlledPreparation: &controlledPreparation{closed: make(chan struct{})}} - p.cancel = func(context.Context) error { return p.Close() } - p.start = func(ctx context.Context, _ string, _ proto.MessageInput, _ chan<- proto.Envelope) (fixtureSession, error) { - close(entered) - <-allowReturn - return nil, ctx.Err() - } - var count atomic.Int32 - r := preparationRouter(t, sender, time.Minute, func(context.Context, proto.PromptRequestPayload) (preparedFixture, error) { - if count.Add(1) == 1 { - return p, nil + synctest.Test(t, func(t *testing.T) { + sender := &recSender{} + entered, allowReturn := make(chan struct{}), make(chan struct{}) + p := &cancellationPreparation{controlledPreparation: &controlledPreparation{closed: make(chan struct{})}} + p.cancel = func(context.Context) error { return p.Close() } + p.start = func(ctx context.Context, _ string, _ proto.MessageInput, _ chan<- proto.Envelope) (fixtureSession, error) { + close(entered) + <-allowReturn + return nil, ctx.Err() } - return &controlledPreparation{closed: make(chan struct{})}, nil - }) - startCancellationPreparation(t, r, sender) - <-entered + var count atomic.Int32 + r := preparationRouter(t, sender, time.Minute, func(context.Context, proto.PromptRequestPayload) (preparedFixture, error) { + if count.Add(1) == 1 { + return p, nil + } + return &controlledPreparation{closed: make(chan struct{})}, nil + }) + startCancellationPreparation(t, r, sender) + <-entered - _ = r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "cancel"})) - deadline := time.Now().Add(12 * time.Second) - for len(cancellationAcks(sender)) == 0 && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } - acks := cancellationAcks(sender) - if len(acks) != 1 || acks[0].Applied || acks[0].Outcome != nil || acks[0].ErrorCode != "cancel_timeout" || r.ActiveRuns() != 1 { - t.Fatal("timeout claimed settlement or lost ownership", acks) - } - if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "overflow", preparationRequest())); err == nil { - t.Fatal("timed-out cancellation returned capacity early") - } - close(allowReturn) - waitFor(t, func() bool { return r.ActiveRuns() == 0 }, "late cancelled Start cleanup") - if len(cancellationAcks(sender)) != 1 { - t.Fatal("late settlement emitted a second receipt") - } - if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "replacement", preparationRequest())); err != nil { - t.Fatal("completed cleanup retained capacity", err) - } - waitPreparationStatus(t, sender, "replacement", "ready", "") + _ = r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "cancel"})) + deadline := time.Now().Add(proto.CancellationConfirmationTimeout + time.Second) + for len(cancellationAcks(sender)) == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + acks := cancellationAcks(sender) + if len(acks) != 1 || acks[0].Applied || acks[0].Outcome != nil || acks[0].ErrorCode != "cancel_timeout" || r.ActiveRuns() != 1 { + t.Fatal("timeout claimed settlement or lost ownership", acks) + } + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "overflow", preparationRequest())); err == nil { + t.Fatal("timed-out cancellation returned capacity early") + } + close(allowReturn) + waitFor(t, func() bool { return r.ActiveRuns() == 0 }, "late cancelled Start cleanup") + if len(cancellationAcks(sender)) != 1 { + t.Fatal("late settlement emitted a second receipt") + } + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "replacement", preparationRequest())); err != nil { + t.Fatal("completed cleanup retained capacity", err) + } + waitPreparationStatus(t, sender, "replacement", "ready", "") + }) } diff --git a/apps/daemon/internal/dispatch/prepared_handoff.go b/apps/daemon/internal/dispatch/prepared_handoff.go index 3b2b28253..54a6953a2 100644 --- a/apps/daemon/internal/dispatch/prepared_handoff.go +++ b/apps/daemon/internal/dispatch/prepared_handoff.go @@ -160,7 +160,7 @@ func (r *Router) runPreparedRelease(state *sessionState, handoff *preparedHandof outcome = &proto.DonePayload{} } if turn != nil { - ctx, cancel := context.WithTimeout(context.Background(), preparedCancelTimeout) + ctx, cancel := context.WithTimeout(context.Background(), proto.CancellationConfirmationTimeout) cancelDone := make(chan error, 1) settled := make(chan struct{}) go func() { diff --git a/contracts/agents-api/harness-onboarding.md b/contracts/agents-api/harness-onboarding.md index 5978104bd..f9c5bdc2a 100644 --- a/contracts/agents-api/harness-onboarding.md +++ b/contracts/agents-api/harness-onboarding.md @@ -118,7 +118,7 @@ A Session owns one reusable Executor in its connected Runtime; a Turn owns one i - For a cancelled stdio MCP call, capture its declared service before native abort can finish the local request, retain starts admitted during that Turn's cancellation, and drain the native Turn before calling `ViewSession.StopMCP` with those service labels. The host stops the selected services and all their descendants, including their earlier background work, and confirms each old Process scope with `ScopeClosed`. It fences new alias starts during this operation; an unconfirmed stop retains ownership and blocks new starts for those aliases. Other MCP services and the workspace remain. HTTP MCP is outside this operation. Stdio MCP admission requires the Process service's delegated cgroup v2 scope; a node or container without that capability returns typed unsupported. - After confirmed scope termination, confirm that the selected old native MCP clients cannot be reused and that subsequent calls create fresh connections through the maintained native owner before declaring the Executor reusable. A local MCP cancellation error, notification, process leader exit or attachment-close acknowledgment does not prove remote scope termination. Repeated or late cancellation joins the original Turn's recorded targets and result; it never selects a successor's services. -**What the Runtime does around a Turn.** One output consumer starts before native Start, drains the bounded 64-frame channel and keeps the terminal observation until Start publication, Turn settlement and admitted operation receipts finish. Natural completion never calls Cancel. Input and function admission close before settlement; operations already admitted hold their barrier through native receipts and outbound acknowledgement. The Runtime sends cancellation to the Turn before waiting on that barrier, because a written input may need a native interrupt to produce its receipt. It joins native settlement, any required confirmed Executor close, output drain and all admitted operations before an applied acknowledgement or reuse, and only then forwards Done or an applied cancellation receipt. A failed Close can report failure while keeping the same Run and outstanding operations for retry; a closed caller wait cannot manufacture an applied input receipt. The Runtime commits native continuity and releases the old Run's admission before publishing Done, since the receiver may start another Turn at once; a late terminal-send failure belongs to the old Run and cannot invalidate a successor that already owns the Executor. Connection shutdown owns transport-loss cleanup. The settlement wait is ten seconds and the receipt send budget five seconds; a timeout is not proof of quiescence. +**What the Runtime does around a Turn.** One output consumer starts before native Start, drains the bounded 64-frame channel and keeps the terminal observation until Start publication, Turn settlement and admitted operation receipts finish. Natural completion never calls Cancel. Input and function admission close before settlement; operations already admitted hold their barrier through native receipts and outbound acknowledgement. The Runtime sends cancellation to the Turn before waiting on that barrier, because a written input may need a native interrupt to produce its receipt. It joins native settlement, any required confirmed Executor close, output drain and all admitted operations before an applied acknowledgement or reuse, and only then forwards Done or an applied cancellation receipt. A failed Close can report failure while keeping the same Run and outstanding operations for retry; a closed caller wait cannot manufacture an applied input receipt. The Runtime commits native continuity and releases the old Run's admission before publishing Done, since the receiver may start another Turn at once; a late terminal-send failure belongs to the old Run and cannot invalidate a successor that already owns the Executor. Connection shutdown owns transport-loss cleanup. Settlement and cancellation receipt waits follow the [Core–Runtime cancellation budget](../../docs/runtime-protocol.md#active-input-receipts); the receipt send budget is five seconds; a timeout is not proof of quiescence. ## Events, inputs and optional capabilities diff --git a/contracts/agents-api/zh/harness-onboarding.md b/contracts/agents-api/zh/harness-onboarding.md index a94b5faa6..01fb5ae92 100644 --- a/contracts/agents-api/zh/harness-onboarding.md +++ b/contracts/agents-api/zh/harness-onboarding.md @@ -1,7 +1,7 @@ --- title: "添加 Harness" source: contracts/agents-api/harness-onboarding.md -source_hash: f98c9ba9ff366f46dc2cb567c4b5e6a4f3538f59b769c7554bac40a2fa1160e3 +source_hash: d3a764e466f1a85db7218a40ba621d1ee1f47751e7ef4e4e0a1ed507cf7c4b2e --- **Harness** 是一种运行模型和工具循环的原生代理引擎(Codex、Claude Code、MiniMax Code)。**Harness 适配器**将 Runtime 的 Executor 和 Turn 契约转换到该引擎的 SDK 或协议。本文档定义 Runtime–Harness 协议:适配器接口及其生命周期义务、注册、支持声明和验收。 @@ -120,7 +120,7 @@ Session 在其已连接的 Runtime 中拥有一个可复用的 Executor;Turn - 取消 stdio MCP 调用时,应在原生 abort 结束本地请求之前记录其已声明服务,保留该 Turn 取消期间准入的调用,在原生 Turn 排空后以这些服务标签调用 `ViewSession.StopMCP`。Host 终止选中的服务及其全部后代,包括这些服务先前产生的后台工作,并通过 `ScopeClosed` 确认各个旧 Process 作用域已结束。该操作期间阻止对应别名的新启动;未确认的终止保留所有权,并继续阻止这些别名的新启动。其他 MCP 服务与 workspace 保留。HTTP MCP 不属于此操作。Stdio MCP 准入要求 Process 服务具备已委派的 cgroup v2 作用域;缺少该能力的节点或容器返回 typed unsupported。 - 作用域终止得到确认后,应确认选中的旧 MCP client 已无法复用,且后续调用会通过维护中的原生 owner 创建新连接,之后才能声明 Executor 可复用。本地 MCP 取消错误、通知、进程 leader 退出或 attachment 关闭确认都不能证明远端作用域已终止。重复或迟到的取消只等待原 Turn 已记录的目标和结果,不能重新选择后继 Turn 的服务。 -**Runtime 在 Turn 前后执行的工作。** 一个输出消费者会在原生 Start 之前启动,耗尽有界的 64 帧通道,并将终态观察保留到 Start 发布、Turn 结算和已准入操作回执完成为止。正常完成绝不调用 Cancel。输入和函数准入会在结算前关闭;已准入的操作会持有其屏障,直至原生回执和出站确认完成。Runtime 会在等待该屏障之前向 Turn 发送取消,因为已写入的输入可能需要原生中断才能生成回执。Runtime 会汇合原生结算、所需的已确认 Executor 关闭、输出耗尽和所有已准入操作,然后应用确认或执行复用,之后才会转发 Done 或已应用的取消回执。Close 失败可以报告失败,同时保留同一 Run 和未完成操作以供重试;已关闭的调用方等待无法凭空生成已应用输入回执。Runtime 会在发布 Done 前提交原生连续性状态并释放旧 Run 的准入,因为接收方可能立即启动另一个 Turn;迟到的终态发送失败属于旧 Run,不能使已拥有 Executor 的后继对象失效。连接关闭负责传输丢失清理。结算等待时间为十秒,回执发送预算为五秒;超时不能证明已达到静默状态。 +**Runtime 在 Turn 前后执行的工作。** 一个输出消费者会在原生 Start 之前启动,耗尽有界的 64 帧通道,并将终态观察保留到 Start 发布、Turn 结算和已准入操作回执完成为止。正常完成绝不调用 Cancel。输入和函数准入会在结算前关闭;已准入的操作会持有其屏障,直至原生回执和出站确认完成。Runtime 会在等待该屏障之前向 Turn 发送取消,因为已写入的输入可能需要原生中断才能生成回执。Runtime 会汇合原生结算、所需的已确认 Executor 关闭、输出耗尽和所有已准入操作,然后应用确认或执行复用,之后才会转发 Done 或已应用的取消回执。Close 失败可以报告失败,同时保留同一 Run 和未完成操作以供重试;已关闭的调用方等待无法凭空生成已应用输入回执。Runtime 会在发布 Done 前提交原生连续性状态并释放旧 Run 的准入,因为接收方可能立即启动另一个 Turn;迟到的终态发送失败属于旧 Run,不能使已拥有 Executor 的后继对象失效。连接关闭负责传输丢失清理。结算与取消回执等待遵循 [Core–Runtime 取消预算](../../../docs/zh/runtime-protocol.md#active-input-receipts);回执发送预算为五秒;超时不能证明已达到静默状态。 ## 事件、输入和可选能力 {#events-inputs-and-optional-capabilities} diff --git a/docs/runtime-protocol.md b/docs/runtime-protocol.md index d0d40270d..73fe3fb4f 100644 --- a/docs/runtime-protocol.md +++ b/docs/runtime-protocol.md @@ -148,7 +148,7 @@ Core delivers active input as `prompt_steer`, one input at a time per Run, and w | Native acceptance | `accepted` arrives under the Turn lifetime, with no automatic redelivery | | Done | The Runtime sends `done` after native Turn settlement and after the receipt send of any in-flight input; the native write and receipt send budgets bound that input | -Neither `written` nor a send failure advances Core's input cursor. Once cancellation is sent, its receipt owns the terminal outcome even if an input becomes unknown first; Core records `cancel_unconfirmed` when no cancellation confirmation arrives within 15 seconds. A cancellation receipt carries the stopped Turn's confirmed continuity snapshot when no `done` is emitted. +Neither `written` nor a send failure advances Core's input cursor. Once cancellation is sent, its receipt owns the terminal outcome even if an input becomes unknown first; Core records `cancel_unconfirmed` when no cancellation confirmation arrives within the bounded wait. `CancellationConfirmationTimeout` defines one 30-second limit for native cancellation and settlement, the Runtime receipt wait, and Core's cancellation exchange and watchdog. Each wait starts when its operation begins; earlier caller cancellation or transport loss still wins, and confirmed results or failures return immediately. A result at the exact deadline may remain unconfirmed across transport. Function-result acknowledgement remains bounded at 15 seconds and Executor resource close at 10 seconds; neither resource retirement nor expiry proves Turn settlement. A cancellation receipt carries the stopped Turn's confirmed continuity snapshot when no `done` is emitted. ## What each acknowledgement proves diff --git a/docs/zh/runtime-protocol.md b/docs/zh/runtime-protocol.md index e3d5965e5..a79217cc7 100644 --- a/docs/zh/runtime-protocol.md +++ b/docs/zh/runtime-protocol.md @@ -1,7 +1,7 @@ --- title: "Core–Runtime 协议" source: docs/runtime-protocol.md -source_hash: 5003cac2a7dd5d1926c8004a400a12cb947070581351dc45b9f2f7b7323dd726 +source_hash: 9ca34714c8bdfc64a484f9f81b92eb5ca05a08c63da55ac5ddb85f1c0b4e85b9 --- 此协议在 Runtime daemon 获取机器凭据后连接 Core 与 daemon,定义 daemon 连接上消息的含义和顺序。wire 类型、限制和验证器仅在 [`internal/agentdaemon/proto`](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/internal/agentdaemon/proto) 中定义一次;Core 的 [gateway](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/services/core/internal/runtimegateway) 与参考 Runtime 的 [dispatcher](https://github.com/MiniMax-AI/OpenAgentCore/tree/main/apps/daemon/internal/dispatch) 都使用它们,因此无需同步第二套 payload schema。签发凭据和打开连接的 HTTP 路由见[机器连接 API](../../contracts/agents-api/zh/machine-api.md)。 @@ -150,7 +150,7 @@ Core 通过 `prompt_steer` 交付活动输入,每个 Run 一次交付一个输 | 原生接受 | `accepted` 在 Turn 生命周期内到达,不自动重新交付 | | Done | Runtime 在原生 Turn 结算完成、且正在处理的输入的回执发送结束后才发送 `done`;该输入受原生写入与回执发送预算限制 | -`written` 和发送失败都不会推进 Core 的 input cursor。取消发出后,即使输入先变为未知,终结结果也由取消回执负责;15 秒内没有取消确认时,Core 记录 `cancel_unconfirmed`。未发出 `done` 时,取消回执携带已停止 Turn 的已确认 continuity snapshot。 +`written` 和发送失败都不会推进 Core 的 input cursor。取消发出后,即使输入先变为未知,终结结果也由取消回执负责;有界等待内没有取消确认时,Core 记录 `cancel_unconfirmed`。`CancellationConfirmationTimeout` 为原生取消与结算、Runtime 回执等待以及 Core 的取消交换与监控统一规定 30 秒上限。各项等待从对应操作开始时计时;更早的调用方取消或传输丢失仍优先生效,已确认的结果或失败立即返回。恰好在期限到达时产生的结果经过传输后仍可能无法确认。函数结果确认仍限制为 15 秒,Executor 资源关闭仍限制为 10 秒;资源回收或超时均不能证明 Turn 已结算。未发出 `done` 时,取消回执携带已停止 Turn 的已确认 continuity snapshot。 ## 每类确认所证明的事实 {#what-each-acknowledgement-proves} diff --git a/internal/agentdaemon/proto/outbound.go b/internal/agentdaemon/proto/outbound.go index a8708eaff..c396a07f9 100644 --- a/internal/agentdaemon/proto/outbound.go +++ b/internal/agentdaemon/proto/outbound.go @@ -2,6 +2,7 @@ package proto import ( "encoding/json" + "time" "github.com/MiniMax-AI/OpenAgentCore/internal/modelprovider" ) @@ -51,6 +52,10 @@ type PromptRequestPayload struct { DisableSubagents bool `json:"disable_subagents,omitempty"` } +// CancellationConfirmationTimeout bounds native settlement and cancellation receipt waits. +// An earlier caller or transport cancellation still wins; expiry is not confirmation. +const CancellationConfirmationTimeout = 30 * time.Second + // PromptCancelPayload optionally requests an application receipt; identity is on Envelope.ID. type PromptCancelPayload struct { DeliveryID string `json:"delivery_id,omitempty"` diff --git a/services/core/internal/execution/delivery.go b/services/core/internal/execution/delivery.go index bd6a63684..4b9939038 100644 --- a/services/core/internal/execution/delivery.go +++ b/services/core/internal/execution/delivery.go @@ -266,7 +266,7 @@ func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, pe } case <-workReady: if !cancelSent.IsZero() { - if time.Since(cancelSent) > 15*time.Second { + if time.Since(cancelSent) > proto.CancellationConfirmationTimeout { result.ErrorCode = "cancel_unconfirmed" return } diff --git a/services/core/internal/execution/runtime_protocol_test.go b/services/core/internal/execution/runtime_protocol_test.go index 51e926496..f70dd49b9 100644 --- a/services/core/internal/execution/runtime_protocol_test.go +++ b/services/core/internal/execution/runtime_protocol_test.go @@ -4,8 +4,11 @@ import ( "context" "encoding/json" "testing" + "testing/synctest" + "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) // Public Turn failure can describe lost orchestration, independently of native @@ -52,3 +55,22 @@ func TestRuntimeProtocolUnknownEffectRetainsObservationFailure(t *testing.T) { }) } } + +func TestRuntimeProtocolDoneWaitsForDelayedCancellationReceipt(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + result := Result{Done: proto.DonePayload{Usage: proto.Usage{InputTokens: 7}}} + j := journal{writer: &recoveringWriter{}, next: 1} + replies := make(chan cancellationResult, 1) + go func() { + time.Sleep(17 * time.Second) + replies <- cancellationResult{ack: proto.InteractionDecisionAckPayload{Applied: true, Outcome: &proto.DonePayload{Usage: proto.Usage{InputTokens: 9}, Metadata: map[string]any{proto.DoneMetaAgentSessionID: "native"}}}} + }() + started := time.Now() + if status := finishDelivery(t.Context(), &j, &result, replies, false, nil); status != sessions.TurnCancelled { + t.Fatal(status, result) + } + if time.Since(started) != 17*time.Second || result.Done.Usage.InputTokens != 9 || result.Done.Metadata[proto.DoneMetaAgentSessionID] != "native" { + t.Fatalf("late receipt lost authority: %+v", result) + } + }) +} diff --git a/services/core/internal/runtimegateway/exchange_test.go b/services/core/internal/runtimegateway/exchange_test.go index 81a3fdd5d..0f2e9ba63 100644 --- a/services/core/internal/runtimegateway/exchange_test.go +++ b/services/core/internal/runtimegateway/exchange_test.go @@ -1,8 +1,11 @@ package runtimegateway import ( + "context" "errors" "testing" + "testing/synctest" + "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" ) @@ -53,3 +56,75 @@ func TestFramesMustNameTheRequestAssignment(t *testing.T) { }) } } + +func TestInteractionConfirmationBudgets(t *testing.T) { + for _, tc := range []struct { + name, kind string + delay time.Duration + timeout time.Duration + applied bool + code string + wantErr bool + }{ + {name: "cancel delayed confirmation", kind: proto.TypePromptCancel, delay: 17 * time.Second, applied: true}, + {name: "cancel early failure", kind: proto.TypePromptCancel, delay: time.Second, code: "cancel_failed"}, + {name: "cancel hangs", kind: proto.TypePromptCancel, delay: 31 * time.Second, wantErr: true}, + {name: "function unchanged", kind: proto.TypeFunctionResult, delay: 16 * time.Second, wantErr: true}, + {name: "caller deadline wins", kind: proto.TypePromptCancel, delay: 4 * time.Second, timeout: 3 * time.Second, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + s := NewSession(newFakeConn(), "device", "tenant", "test", NewRegistry(), nil) + defer s.Close("test") + frame, _ := proto.NewEnvelope(tc.kind, "run", proto.PromptCancelPayload{DeliveryID: "delivery"}) + frame.Assignment = testAssignment + ctx := t.Context() + if tc.timeout != 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, tc.timeout) + defer cancel() + } + result := make(chan error, 1) + started := time.Now() + go func() { + ack, err := s.SendAndWaitInteractionAck(ctx, frame, "delivery") + if err == nil && (ack.Applied != tc.applied || ack.ErrorCode != tc.code) { + t.Error("receipt changed", ack) + } + result <- err + }() + request := <-s.sendCh + stopReply := make(chan struct{}) + defer close(stopReply) + go func() { + select { + case <-time.After(tc.delay): + case <-stopReply: + return + } + reply, _ := request.Reply(proto.TypeInteractionDecisionAck, proto.InteractionDecisionAckPayload{DeliveryID: "delivery", Applied: tc.applied, ErrorCode: tc.code}) + s.dispatch(reply) + }() + err := <-result + wantDuration := tc.delay + if tc.wantErr { + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatal("expected deadline", err) + } + wantDuration = proto.CancellationConfirmationTimeout + if tc.kind == proto.TypeFunctionResult { + wantDuration = 15 * time.Second + } + if tc.timeout != 0 { + wantDuration = tc.timeout + } + } else if err != nil { + t.Fatal(err) + } + if elapsed := time.Since(started); elapsed != wantDuration { + t.Fatalf("elapsed %v, want %v", elapsed, wantDuration) + } + }) + }) + } +} diff --git a/services/core/internal/runtimegateway/session.go b/services/core/internal/runtimegateway/session.go index c41d7ad5e..d9120a862 100644 --- a/services/core/internal/runtimegateway/session.go +++ b/services/core/internal/runtimegateway/session.go @@ -35,7 +35,7 @@ var ( WriteTimeout = 10 * time.Second // InteractionAckTimeout bounds the application-level round trip for a - // function result or cancellation after it is written to the daemon. + // function result after it is written to the daemon. InteractionAckTimeout = 15 * time.Second // ReadLimit caps a single inbound frame at 4 MiB. tool_call @@ -325,7 +325,11 @@ func (s *Session) SendAndWaitInteractionAck(ctx context.Context, env proto.Envel // The exchange prefers an ack that raced the deadline; treating an // already-applied decision as retryable can trigger a contradictory second // response. - waitCtx, cancel := context.WithTimeout(ctx, InteractionAckTimeout) + timeout := InteractionAckTimeout + if env.Type == proto.TypePromptCancel { + timeout = proto.CancellationConfirmationTimeout + } + waitCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() reply, err := s.exchangeFrame(waitCtx, env, waiter) if err != nil {