Skip to content

Commit 3082b2a

Browse files
committed
fix(release): probe real external-wait fallback decision
Signed-off-by: huangruiteng <14976749+huangruiteng@users.noreply.github.com>
1 parent c9b3c56 commit 3082b2a

3 files changed

Lines changed: 30 additions & 21 deletions

File tree

‎docs/development/testing-and-quality.md‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1009,10 +1009,13 @@ For focused thin/brief prompt-decision regression, use
10091009
explicit release qualification. It defaults to no calls; missing credentials
10101010
report `skipped`, not a live pass. With securely injected `ARK_API_KEY`, it uses
10111011
Doubao evolving for two independent repetitions of quiet-work, notifying-wait,
1012-
quiet-wait and required-vision-replan cases in each mode. Expected decisions
1013-
remain outside model input. All attempts must pass; no answer correction or
1014-
retry-until-pass is used. Ordinary pytest only checks the probe and negative
1015-
oracles with scripted responses, without provider calls.
1012+
quiet-wait, required-vision-replan and typed external-wait fallback cases in
1013+
each mode. The fallback case uses the real compact quota projection: its wait
1014+
transition already exists, so the host must advance the selected independent
1015+
successor and notify rather than authoring another transition. Expected
1016+
decisions remain outside model input. All attempts must pass; no answer
1017+
correction or retry-until-pass is used. Ordinary pytest only checks the probe
1018+
and negative oracles with scripted responses, without provider calls.
10161019

10171020
This is a synthetic decision-level probe using current generated prompts,
10181021
not proof of tool execution, host scheduling, upgrade delivery or full-Goal

‎loopx/control_plane/testing/host_prompt_behavior.py‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,24 @@
66
from pathlib import Path
77

88
from ...heartbeat_prompt import build_heartbeat_prompt
9+
from ..quota.cli_projection import compact_quota_should_run_cli_payload
910
from ..quota.effective_action import EffectiveAction
11+
from .action_portfolio_scenarios import external_wait_fallback_scenario_source
1012
from .model_tool_behavior import DoubaoExecToolClient
1113

1214

1315
def cases() -> list[dict]:
1416
# Independent semantic oracle: silence does not cancel work; a gate does;
15-
# required vision replan is not terminal closure. Never send expected to
16-
# the model, or derive it from the renderer being qualified.
17+
# required vision replan is not terminal closure. An already-transitioned
18+
# external wait selects its independent fallback for work. Never send
19+
# expected to the model, or derive it from the renderer being qualified.
1720
rows = (
18-
("quiet_work", True, False, False, False, "work"),
19-
("notifying_wait", False, True, False, False, "wait"),
20-
("quiet_wait", False, False, False, False, "wait"),
21-
("vision_replan", True, False, True, False, "replan"),
22-
("external_wait_transition", True, False, False, True, "external_wait"),
21+
("quiet_work", True, False, False, "work"),
22+
("notifying_wait", False, True, False, "wait"),
23+
("quiet_wait", False, False, False, "wait"),
24+
("vision_replan", True, False, True, "replan"),
2325
)
24-
return [{
26+
probes = [{
2527
"id": name,
2628
"packet": {
2729
"ok": True,
@@ -33,13 +35,6 @@ def cases() -> list[dict]:
3335
"execution_obligation": {"must_attempt_work": work},
3436
"heartbeat_recommendation": {"agent_must_attempt": work},
3537
"autonomous_replan_obligation": {"required": replan},
36-
"external_wait_observation": ({
37-
"schema_version": "typed_external_wait_observation_v0",
38-
"selected_todo_id": "todo_waiting",
39-
"monitor_todo_id": "todo_monitor",
40-
"independent_successor_todo_id": "todo_successor",
41-
"state": "external_review_pending",
42-
} if external_wait else None),
4338
"interaction_contract": {
4439
"user_channel": {"notify": "NOTIFY" if notify else "DONT_NOTIFY"},
4540
"agent_channel": {"delivery_allowed": work and not replan},
@@ -49,7 +44,13 @@ def cases() -> list[dict]:
4944
"run_history": {"latest_runs": [{"delivery_outcome": "outcome_progress"}]},
5045
},
5146
"expected": {"action": action, "notify": notify, "finish_goal": False},
52-
} for name, work, notify, replan, external_wait, action in rows]
47+
} for name, work, notify, replan, action in rows]
48+
probes.append({
49+
"id": "wait_fallback_work",
50+
"packet": compact_quota_should_run_cli_payload(external_wait_fallback_scenario_source()),
51+
"expected": {"action": "work", "notify": True, "finish_goal": False},
52+
})
53+
return probes
5354

5455

5556
def probe_messages(mode: str, packet: dict) -> list[dict]:

‎tests/control_plane/test_host_prompt_behavior.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
{"action": "wait", "notify": True, "finish_goal": False},
1414
{"action": "wait", "notify": False, "finish_goal": False},
1515
{"action": "replan", "notify": False, "finish_goal": False},
16-
{"action": "external_wait", "notify": False, "finish_goal": False},
16+
{"action": "work", "notify": True, "finish_goal": False},
1717
]
1818

1919

@@ -35,6 +35,11 @@ def next_final_content(self, messages):
3535

3636

3737
def test_probe_uses_current_production_prompts_and_hidden_independent_oracle():
38+
fallback = cases()[-1]
39+
assert fallback["packet"]["selected_todo"]["todo_id"] == "todo_external_wait_fallback"
40+
assert fallback["packet"]["execution_obligation"]["must_attempt_work"] is True
41+
assert fallback["packet"]["interaction_contract"]["user_channel"]["notify"] == "NOTIFY"
42+
assert "external_wait_observation" not in fallback["packet"]
3843
for mode in ("thin", "brief"):
3944
for case in cases():
4045
messages = probe_messages(mode, case["packet"])

0 commit comments

Comments
 (0)