diff --git a/harness/trajectory_replay/public_webarena/executor.py b/harness/trajectory_replay/public_webarena/executor.py index c6dcec76b..2731e36db 100644 --- a/harness/trajectory_replay/public_webarena/executor.py +++ b/harness/trajectory_replay/public_webarena/executor.py @@ -325,23 +325,25 @@ def publish_layout(current_page: Any = action_page) -> None: ) if navigation_response_observer is not None: navigation_response_observer(response) - # BrowserTool returns after DOMContentLoaded, then gives ordinary - # background requests a short chance to settle. Network idle is - # deliberately best effort: analytics or streaming requests must - # not turn a completed navigation into a failed replay action. - try: - action_page.wait_for_load_state( - "networkidle", - timeout=min(action_remaining(), NAVIGATION_NETWORK_IDLE_TIMEOUT_MS), - ) - except Exception: - pass - try: + # Settling is optional; reserve at least half the remaining + # action budget for layout and the required document observation. + # Both waits share this deadline so a busy network cannot consume + # the observation budget or start another full grace period. + settle_deadline = time.perf_counter() + action_remaining() / 2_000 + settle_timeout = int((settle_deadline - time.perf_counter()) * 1_000) + if settle_timeout > 0: + try: + action_page.wait_for_load_state( + "networkidle", + timeout=min(settle_timeout, NAVIGATION_NETWORK_IDLE_TIMEOUT_MS), + ) + except Exception: + pass + settle_timeout = int((settle_deadline - time.perf_counter()) * 1_000) + if settle_timeout > 0: action_page.wait_for_timeout( - min(action_remaining(), NAVIGATION_RENDER_GRACE_MS) + min(settle_timeout, NAVIGATION_RENDER_GRACE_MS) ) - except TimeoutError: - pass publish_layout() limit = int(call.get("max_chars", 40_000)) if document_observer is None: diff --git a/tests/trajectory_replay/public_webarena/test_executor.py b/tests/trajectory_replay/public_webarena/test_executor.py index 5c0efad24..a919a0689 100644 --- a/tests/trajectory_replay/public_webarena/test_executor.py +++ b/tests/trajectory_replay/public_webarena/test_executor.py @@ -448,6 +448,64 @@ def wait_for_load_state(self, state, **kwargs): assert responses[0].status == 200 +@pytest.mark.parametrize("network_delay_ms", [50, 500, 5_000]) +@pytest.mark.parametrize("observation_ms", [300, 600]) +def test_navigation_reserves_observation_budget_without_extending_deadline( + monkeypatch, network_delay_ms, observation_ms +): + clock = [0.0] + monkeypatch.setattr( + "harness.trajectory_replay.public_webarena.executor.time.perf_counter", + lambda: clock[0], + ) + + class TimedPage(FakePage): + def goto(self, url, **kwargs): + clock[0] += 0.05 + return super().goto(url, **kwargs) + + def wait_for_load_state(self, state, **kwargs): + allotted = kwargs["timeout"] + clock[0] += min(network_delay_ms, allotted) / 1_000 + if network_delay_ms > allotted: + raise TimeoutError("streaming request is still open") + + def wait_for_timeout(self, milliseconds): + clock[0] += milliseconds / 1_000 + + def observe(page, timeout, limit): + assert timeout >= 475 + clock[0] += min(observation_ms, timeout) / 1_000 + if observation_ms > timeout: + raise TimeoutError("document observation timed out") + return "Observed title", "Observed body" + + trace = [] + + def run(): + return execute_steps( + TimedPage(), + [{ + "action": "navigate", + "url": "https://example.com", + "timeout_ms": 1_000, + "save_as": "page", + }], + document_observer=observe, + trace=trace, + ) + + # A genuinely slow observation must still fail within the original budget. + if observation_ms == 600 and network_delay_ms >= 500: + with pytest.raises(TimeoutError, match="document observation timed out"): + run() + assert trace[0]["status"] == "fail" + else: + assert run()["page"].endswith("Observed body") + assert trace[0]["status"] == "pass" + assert clock[0] <= 1.0 + + def test_same_document_navigation_without_response_reloads_for_network_evidence(): class SamePage(FakePage): def __init__(self):