From ed84dda18975f2dc1acc05a0b34403b2fc115acb Mon Sep 17 00:00:00 2001 From: suchintan <3853670+suchintan@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:38:26 +0000 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=94=84=20synced=20local=20'benchmarks?= =?UTF-8?q?/'=20with=20remote=20'benchmarks/'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/Skyvern-AI/rustwright-cloud/pull/239 --- benchmarks/process_tree_memory.py | 212 ++++++++++++++++++++++++++++-- 1 file changed, 200 insertions(+), 12 deletions(-) diff --git a/benchmarks/process_tree_memory.py b/benchmarks/process_tree_memory.py index 2866d04..9438cef 100644 --- a/benchmarks/process_tree_memory.py +++ b/benchmarks/process_tree_memory.py @@ -3,11 +3,23 @@ import functools import os import subprocess +import sys import threading from dataclasses import dataclass from pathlib import Path from typing import Any, Callable, TypeVar, cast +PROCESS_ROLES = ( + "browser", + "renderer", + "gpu", + "utility", + "network", + "python-host", + "node-driver", + "other", +) + SAMPLE_INTERVAL_SECONDS = 0.05 PS_TIMEOUT_SECONDS = 2.0 @@ -18,10 +30,12 @@ class ProcessTreeRssSample: rss_self_kb: int | None rss_tree_kb: int | None + pss_tree_kb: int | None = None + pss_by_role_kb: dict[str, int] | None = None class ProcessTreeRssSampler: - """Sample peak RSS for a process and its descendants outside the timing path.""" + """Sample peak RSS and PSS attribution for a process and its descendants.""" def __init__(self, root_pid: int, sample_interval: float | None = None) -> None: self.root_pid = root_pid @@ -75,15 +89,48 @@ def summary(self) -> dict[str, Any]: samples = list(self.samples) rss_self_kb = max_optional(sample.rss_self_kb for sample in samples) rss_tree_kb = max_optional(sample.rss_tree_kb for sample in samples) + peak_sample = max( + (sample for sample in samples if sample.rss_tree_kb is not None), + key=lambda sample: sample.rss_tree_kb, + default=None, + ) + pss_tree_kb = peak_sample.pss_tree_kb if peak_sample is not None else None + pss_by_role_kb = ( + dict(peak_sample.pss_by_role_kb) + if peak_sample is not None and peak_sample.pss_by_role_kb is not None + else None + ) + pss_required = sys.platform.startswith("linux") + pss_available = ( + type(pss_tree_kb) is int + and pss_tree_kb > 0 + and pss_by_role_kb is not None + and set(pss_by_role_kb) == set(PROCESS_ROLES) + and all(type(value) is int and value >= 0 for value in pss_by_role_kb.values()) + and sum(pss_by_role_kb.values()) == pss_tree_kb + ) + rss_available = ( + type(rss_self_kb) is int + and rss_self_kb > 0 + and type(rss_tree_kb) is int + and rss_tree_kb > 0 + and rss_tree_kb >= rss_self_kb + ) return { "rss_self_kb": rss_self_kb, "rss_tree_kb": rss_tree_kb, + "pss_tree_kb": pss_tree_kb, + "pss_by_role_kb": pss_by_role_kb, + "pss_required": pss_required, + "pss_available": pss_available, + "pss_statistic": "at_tree_rss_peak", + "pss_scope": "benchmark_process_and_descendants", "samples_collected": len(samples), "sampling_interval_ms": self.sample_interval * 1000, "statistic": "peak", "scope": "benchmark_process_and_descendants", "sampling_mode": "background_thread_procfs_or_ps", - "available": rss_self_kb is not None and rss_tree_kb is not None, + "available": rss_available and (not pss_required or pss_available), } @@ -130,15 +177,10 @@ def sample_process_tree_rss(root_pid: int) -> ProcessTreeRssSample: ) -def procfs_process_tree_rss( +def _process_tree_from_table( root_pid: int, - proc_root: Path | None = None, -) -> ProcessTreeRssSample | None: - process_table, complete = procfs_process_table( - PROC_ROOT if proc_root is None else proc_root - ) - if not complete or root_pid not in process_table: - return None + process_table: dict[int, tuple[int, int | None]], +) -> list[int]: children: dict[int, list[int]] = {} for pid, (parent_pid, _) in process_table.items(): children.setdefault(parent_pid, []).append(pid) @@ -150,12 +192,158 @@ def procfs_process_tree_rss( continue tree.append(pid) stack.extend(children.get(pid, [])) + return tree + + +def procfs_process_tree_rss( + root_pid: int, + proc_root: Path | None = None, +) -> ProcessTreeRssSample | None: + resolved_root = PROC_ROOT if proc_root is None else proc_root + process_table, complete = procfs_process_table(resolved_root) + if not complete or root_pid not in process_table: + return None + tree = _process_tree_from_table(root_pid, process_table) + rss_self_kb = process_table[root_pid][1] + rss_tree_kb = sum_required(process_table[pid][1] for pid in tree) + if not sys.platform.startswith("linux") or rss_tree_kb is None: + return ProcessTreeRssSample( + rss_self_kb=rss_self_kb, + rss_tree_kb=rss_tree_kb, + ) + pss = procfs_process_tree_pss(tree, resolved_root) + if pss is None: + # A process can exit between the status walk and smaps_rollup reads. + # Re-walk once to distinguish that race from a genuinely partial read. + process_table, complete = procfs_process_table(resolved_root) + if complete and root_pid in process_table: + tree = _process_tree_from_table(root_pid, process_table) + rss_self_kb = process_table[root_pid][1] + rss_tree_kb = sum_required(process_table[pid][1] for pid in tree) + pss = ( + procfs_process_tree_pss(tree, resolved_root) + if rss_tree_kb is not None + else None + ) + if pss is None: + return ProcessTreeRssSample( + rss_self_kb=rss_self_kb, + rss_tree_kb=rss_tree_kb, + ) + pss_tree_kb, pss_by_role_kb = pss return ProcessTreeRssSample( - rss_self_kb=process_table[root_pid][1], - rss_tree_kb=sum_required(process_table[pid][1] for pid in tree), + rss_self_kb=rss_self_kb, + rss_tree_kb=rss_tree_kb, + pss_tree_kb=pss_tree_kb, + pss_by_role_kb=pss_by_role_kb, ) +def _argv_tokens(cmdline: str | bytes | list[str] | tuple[str, ...]) -> list[str]: + if isinstance(cmdline, bytes): + text = cmdline.replace(b"\x00", b" ").decode("utf-8", errors="replace") + return text.split() + if isinstance(cmdline, str): + return cmdline.replace("\x00", " ").split() + return [str(value) for value in cmdline] + + +def _argv_option(args: list[str], option: str) -> str | None: + for index, token in enumerate(args): + if token == option: + return args[index + 1] if index + 1 < len(args) else None + prefix = f"{option}=" + if token.startswith(prefix): + return token[len(prefix) :] + return None + + +def _is_chrome_executable(executable: str) -> bool: + return executable in { + "chrome", + "chromium", + "chromium-browser", + "google-chrome", + "google-chrome-stable", + "headless_shell", + "chrome-headless-shell", + } + + +def classify_process_role(cmdline: str | bytes | list[str] | tuple[str, ...]) -> str: + """Classify a process from its executable and exact Chromium argv tokens.""" + + args = _argv_tokens(cmdline) + if not args: + return "other" + executable = Path(args[0]).name.lower() + if "python" in executable: + return "python-host" + if executable in {"node", "nodejs"} or executable.startswith("node-") or executable.endswith("-node"): + return "node-driver" + + process_type = (_argv_option(args, "--type") or "").lower() + utility_subtype = (_argv_option(args, "--utility-sub-type") or "").lower() + if utility_subtype == "network.mojom.networkservice" or process_type == "network": + return "network" + if process_type == "renderer": + return "renderer" + if process_type in {"gpu", "gpu-process"}: + return "gpu" + if process_type == "utility": + return "utility" + if _is_chrome_executable(executable) and not process_type: + return "browser" + return "other" + + +def _procfs_pss_kb(process_dir: Path) -> int | None: + try: + smaps_rollup = (process_dir / "smaps_rollup").read_text(encoding="utf-8") + except (OSError, UnicodeError): + return None + pss_kb: int | None = None + for line in smaps_rollup.splitlines(): + if not line.startswith("Pss:"): + continue + parts = line.split() + if len(parts) < 2: + return None + try: + parsed = int(parts[1]) + except ValueError: + return None + if parsed < 0: + return None + pss_kb = parsed + break + return pss_kb + + +def procfs_process_tree_pss( + tree: list[int], + proc_root: Path | None = None, +) -> tuple[int, dict[str, int]] | None: + """Read PSS for every tree process and aggregate it by process role.""" + + resolved_root = PROC_ROOT if proc_root is None else proc_root + by_role = {role: 0 for role in PROCESS_ROLES} + total = 0 + for pid in tree: + process_dir = resolved_root / str(pid) + pss_kb = _procfs_pss_kb(process_dir) + if pss_kb is None: + return None + try: + cmdline = (process_dir / "cmdline").read_bytes() + except (OSError, UnicodeError): + cmdline = b"" + role = classify_process_role(cmdline) + by_role[role] += pss_kb + total += pss_kb + return total, by_role + + def procfs_process_table( proc_root: Path, ) -> tuple[dict[int, tuple[int, int | None]], bool]: From 7da82757df1125b8a8277a44f4a74974ce9f71ba Mon Sep 17 00:00:00 2001 From: suchintan <3853670+suchintan@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:38:26 +0000 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=94=84=20synced=20local=20'python/'?= =?UTF-8?q?=20with=20remote=20'python/'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/Skyvern-AI/rustwright-cloud/pull/239 --- python/rustwright/_async_generated.py | 2 +- python/rustwright/async_api.py | 2 + python/rustwright/sync_api.py | 651 ++++++++++++++++++++------ 3 files changed, 512 insertions(+), 143 deletions(-) diff --git a/python/rustwright/_async_generated.py b/python/rustwright/_async_generated.py index 0997493..54dcfb9 100644 --- a/python/rustwright/_async_generated.py +++ b/python/rustwright/_async_generated.py @@ -1,5 +1,5 @@ # This file is generated by tools/generate_async_api.py. Do not edit. -# sync_api.py sha256: 96400fc6ad22ef6c66648b17602e2b8c6edc1afefda0d437409b7e9641c50657 +# sync_api.py sha256: 42a5e130e3fa8f0641c1c97f83cea4f088fa5b29716a38cf81998f97eb4c06c8 from __future__ import annotations from pathlib import Path diff --git a/python/rustwright/async_api.py b/python/rustwright/async_api.py index 013b6f1..aa20866 100644 --- a/python/rustwright/async_api.py +++ b/python/rustwright/async_api.py @@ -2348,6 +2348,7 @@ async def _close_native_impl(self, *, reason: Optional[str], for_browser_close: if self._sync._browser is not None and self._sync in self._sync._browser._contexts: self._sync._browser._contexts.remove(self._sync) _emit_event(self._sync._event_handlers, "close", self._sync) + self._sync._release_memory_buffers() def is_closed(self) -> bool: return self._sync.is_closed() @@ -3259,6 +3260,7 @@ async def _close_native(self, normalized_reason: Optional[str], unload: bool) -> if self._sync._context is not None and self._sync in self._sync._context._pages: self._sync._context._pages.remove(self._sync) _emit_event(self._sync._event_handlers, "close", self._sync) + self._sync._release_memory_buffers() if ( self._sync._owns_context and self._sync._context is not None diff --git a/python/rustwright/sync_api.py b/python/rustwright/sync_api.py index d34d35e..25812e4 100644 --- a/python/rustwright/sync_api.py +++ b/python/rustwright/sync_api.py @@ -51,6 +51,8 @@ _CONSOLE_HISTORY_SETTLE_TIMEOUT_SECONDS = 0.001 _CONSOLE_HISTORY_BUFFER = "__console_history__" _PAGE_ERROR_HISTORY_BUFFER = "__page_error_history__" +_NAVIGATION_RESPONSE_RETENTION_COUNT = 20 +_NAVIGATION_RESPONSE_RETENTION_BYTES = 8 * 1024 * 1024 _MULTIPART_BOUNDARY_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789AB" _CHECKED_STATE_JS = """(el) => { const tagName = String(el && el.tagName || '').toUpperCase(); @@ -6555,35 +6557,59 @@ def header_values(self, name: str) -> list[str]: def headers_array(self) -> list[dict[str, str]]: return list(self._headers_array_cache) + def _body_cache_was_populated(self) -> None: + if self._page is not None: + self._page._prune_navigation_responses() + def _set_body_from_payload(self, payload: Any) -> None: if not isinstance(payload, dict): return body = payload.get("body", "") if payload.get("base64Encoded"): - self._body_cache = base64.b64decode(body) + decoded_body = base64.b64decode(body) else: - self._body_cache = str(body).encode("utf-8") - if self.request is not None: - self.request._sizes = { - **(self.request._sizes or {}), - "responseBodySize": len(self._body_cache), - } + decoded_body = str(body).encode("utf-8") + page = self._page + if page is None: + self._body_cache = decoded_body + if self.request is not None: + self.request._sizes = { + **(self.request._sizes or {}), + "responseBodySize": len(decoded_body), + } + else: + with page._network_history_lock_for(): + self._body_cache = decoded_body + if self.request is not None: + self.request._sizes = { + **(self.request._sizes or {}), + "responseBodySize": len(decoded_body), + } + self._body_cache_was_populated() def _cache_body(self, timeout_ms: Optional[float] = None) -> bytes: - if self._body_cache is not None: - return self._body_cache - if self._page is None or not self._request_id: - self._body_cache = b"" + page = self._page + if page is None: + if self._body_cache is None: + self._body_cache = b"" return self._body_cache - fulfilled_body = self._page._fulfilled_route_bodies.get(str(self._request_id)) - if fulfilled_body is not None: - self._body_cache = fulfilled_body + with page._network_history_lock_for(): + if self._body_cache is not None: + return self._body_cache + if not self._request_id: + self._body_cache = b"" + else: + fulfilled_body = page._fulfilled_route_bodies.pop(str(self._request_id), None) + if fulfilled_body is not None: + self._body_cache = fulfilled_body + if self._body_cache is not None: + self._body_cache_was_populated() return self._body_cache payload = json.loads( _call( - self._page._core.response_body, + page._core.response_body, self._request_id, - self._page._default_timeout if timeout_ms is None else timeout_ms, + page._default_timeout if timeout_ms is None else timeout_ms, ) ) self._set_body_from_payload(payload) @@ -8365,22 +8391,16 @@ def _network_event_matches_descriptor(page: "Page", matcher: Any, event: Request def _network_log_state(page: "Page", kind: str) -> dict[str, Any]: - if kind == "request": - return {"log_offset": len(page._request_log)} - if kind == "response": - return {"log_offset": len(page._response_log)} + if kind in {"request", "response"}: + return {"log_sequence": page._network_log_cursor(kind)} return {} def _network_log_values(page: "Page", state: dict[str, Any], kind: str) -> Iterable[Request | Response]: - offset = state.get("log_offset") - if offset is None: + sequence = state.get("log_sequence") + if not isinstance(sequence, int): return () - if kind == "request": - return page._request_log[offset:] - if kind == "response": - return page._response_log[offset:] - return () + return page._network_log_values_since(kind, sequence) def _page_console_waiter(page: "Page", timeout_ms: Optional[float] = None) -> Any: @@ -9532,7 +9552,7 @@ def __init__(self, target: Any, event: str, predicate: Any, timeout: Optional[fl self._error: Optional[BaseException] = None self._handler: Optional[Callable[..., Any]] = None self._reject_close_handler: Optional[Callable[..., Any]] = None - self._page_log_offsets: dict[Page, int] = {} + self._page_log_sequences: dict[Page, int] = {} self._predicate_results: dict[int, list[tuple[Any, bool, str]]] = {} self._predicate_lock = threading.Lock() @@ -9558,10 +9578,7 @@ def _predicate_accepts(self, value: Any, *, source: str) -> bool: def __enter__(self) -> "_LocalEventContextManager": if self._event in {"request", "response"} and hasattr(self._target, "_pages"): for page in list(getattr(self._target, "_pages", [])): - if self._event == "request": - self._page_log_offsets[page] = len(page._request_log) - else: - self._page_log_offsets[page] = len(page._response_log) + self._page_log_sequences[page] = page._network_log_cursor(self._event) def handler(*args: Any) -> None: value: Any @@ -9603,12 +9620,12 @@ def __exit__(self, exc_type, exc, tb) -> None: self._ready.set() break if deadline is None: - step = 0.05 if self._page_log_offsets else None + step = 0.05 if self._page_log_sequences else None else: remaining = deadline - time.monotonic() if remaining <= 0: break - step = min(0.05, remaining) if self._page_log_offsets else remaining + step = min(0.05, remaining) if self._page_log_sequences else remaining self._ready.wait(step) if not self._ready.is_set(): fallback = self._fallback_context_network_event() @@ -9630,10 +9647,10 @@ def __exit__(self, exc_type, exc, tb) -> None: def _fallback_context_network_event(self) -> Any: if self._event not in {"request", "response"}: return None - for page, offset in list(self._page_log_offsets.items()): - events = page._request_log if self._event == "request" else page._response_log - self._page_log_offsets[page] = len(events) - for event in events[offset:]: + for page, sequence in list(self._page_log_sequences.items()): + events, next_sequence = page._network_log_snapshot_since(self._event, sequence) + self._page_log_sequences[page] = next_sequence + for event in events: if self._predicate_accepts(event, source="fallback"): return event return None @@ -12526,7 +12543,7 @@ def __init__(self, context: "BrowserContext"): self._artifact_resources: dict[str, bytes] = {} self._action_pages: dict[str, "Page"] = {} self._group_stack: list[str] = [] - self._response_log_offsets: dict["Page", int] = {} + self._response_log_sequences: dict["Page", int] = {} self._call_counter = 0 self._group_counter = 0 self._event_handlers: dict[str, list[Callable[..., Any]]] = {} @@ -12574,7 +12591,9 @@ def _start_chunk_recording(self, *, title: Optional[str] = None, name: Optional[ self._artifact_resources.clear() self._action_pages.clear() self._group_stack.clear() - self._response_log_offsets = {page: len(page._response_log) for page in self._context.pages} + self._response_log_sequences = { + page: page._network_log_cursor("response") for page in self._context.pages + } self._call_counter = 0 self._group_counter = 0 self._source_file_indexes.clear() @@ -12679,7 +12698,7 @@ def _close_groups(self) -> None: def _attach_page(self, page: "Page") -> None: if not self._recording or page in self._network_handlers: return - self._response_log_offsets.setdefault(page, len(page._response_log)) + self._response_log_sequences.setdefault(page, page._network_log_cursor("response")) def response_handler(_: Response) -> None: return None @@ -12913,8 +12932,8 @@ def _write_trace_zip(self, path: str | Path, data: bytes) -> None: seen: set[tuple[Optional[str], str, Optional[int]]] = set() monotonic_time = self._start_monotonic_ms for page in list(self._context.pages): - offset = self._response_log_offsets.get(page, 0) - for response in page._response_log[offset:]: + sequence = self._response_log_sequences.get(page, 0) + for response in page._network_log_values_since("response", sequence): if url_parse.urlparse(response.url).path == "/favicon.ico": continue key = (response._request_id, response.url, response.status) @@ -13384,6 +13403,22 @@ def _remove_page_event_bridges(self, event: str) -> None: page.remove_listener(page_event, handler) handlers.clear() + def _release_memory_buffers(self) -> None: + self._pages.clear() + self._background_pages.clear() + self._service_workers.clear() + self._emitted_service_worker_ids.clear() + self._popup_bridge_handlers.clear() + self._web_error_bridge_handlers.clear() + self._page_load_bridge_handlers.clear() + self._page_close_bridge_handlers.clear() + self._init_scripts.clear() + self._routes.clear() + self._har_routes.clear() + self._websocket_routes.clear() + self._bindings.clear() + self._storage_state_origins.clear() + def close(self, *, reason: Optional[str] = None) -> None: if self._closed: return @@ -13420,6 +13455,7 @@ def close(self, *, reason: Optional[str] = None) -> None: if self._browser is not None and self in self._browser._contexts: self._browser._contexts.remove(self) _emit_event(self._event_handlers, "close", self) + self._release_memory_buffers() def _cleanup_default_context_state(self) -> None: if self._core is not None or self._browser is None: @@ -15726,14 +15762,19 @@ def __init__( self._set_content_html_document_known: Optional[bool] = True self._frame_object_cache: dict[str, Frame] = {} self._frame_event_cache: dict[str, Frame] = {} + self._network_history_lock = threading.RLock() self._request_log: list[Request] = [] + self._request_log_sequences: list[int] = [] + self._next_request_log_sequence = 0 self._request_log_keys: set[tuple[str, str]] = set() - self._request_log_condition = threading.Condition() + self._request_log_condition = threading.Condition(self._network_history_lock) self._request_log_generation = 0 self._request_history_wait_until = 0.0 self._request_history_pending_keys: set[tuple[str, str]] = set() self._requests_by_key: dict[tuple[str, str], Request] = {} self._response_log: list[Response] = [] + self._response_log_sequences: list[int] = [] + self._next_response_log_sequence = 0 self._response_log_keys: set[tuple[Any, str, Any]] = set() self._navigation_responses: list[Response] = [] self._fulfilled_route_bodies: dict[str, bytes] = {} @@ -16314,66 +16355,357 @@ def _settle_console_dispatch_after_action(self, marker: Optional[int]) -> None: break self._console_dispatch_condition.wait(timeout=remaining) + def _network_history_lock_for(self) -> Any: + lock = getattr(self, "_network_history_lock", None) + if lock is None: + lock = self.__dict__.setdefault("_network_history_lock", threading.RLock()) + return lock + + def _ensure_request_log_sequences(self) -> None: + sequences = getattr(self, "_request_log_sequences", None) + if sequences is None: + sequences = [] + self._request_log_sequences = sequences + if len(sequences) > len(self._request_log): + del sequences[len(self._request_log):] + next_sequence = max(getattr(self, "_next_request_log_sequence", 0), 0) + if sequences: + next_sequence = max(next_sequence, sequences[-1] + 1) + while len(sequences) < len(self._request_log): + sequences.append(next_sequence) + next_sequence += 1 + self._next_request_log_sequence = next_sequence + + def _ensure_response_log_sequences(self) -> None: + sequences = getattr(self, "_response_log_sequences", None) + if sequences is None: + sequences = [] + self._response_log_sequences = sequences + if len(sequences) > len(self._response_log): + del sequences[len(self._response_log):] + next_sequence = max(getattr(self, "_next_response_log_sequence", 0), 0) + if sequences: + next_sequence = max(next_sequence, sequences[-1] + 1) + while len(sequences) < len(self._response_log): + sequences.append(next_sequence) + next_sequence += 1 + self._next_response_log_sequence = next_sequence + + def _network_log_cursor(self, kind: str) -> int: + with self._network_history_lock_for(): + if kind == "request": + self._ensure_request_log_sequences() + return self._next_request_log_sequence + if kind == "response": + self._ensure_response_log_sequences() + return self._next_response_log_sequence + return 0 + + def _network_log_snapshot_since( + self, + kind: str, + sequence: int, + ) -> tuple[list[Request | Response], int]: + with self._network_history_lock_for(): + if kind == "request": + self._ensure_request_log_sequences() + events = [ + event + for event_sequence, event in zip(self._request_log_sequences, self._request_log) + if event_sequence >= sequence + ] + return events, self._next_request_log_sequence + if kind == "response": + self._ensure_response_log_sequences() + events = [ + event + for event_sequence, event in zip(self._response_log_sequences, self._response_log) + if event_sequence >= sequence + ] + return events, self._next_response_log_sequence + return [], 0 + + def _network_log_values_since( + self, + kind: str, + sequence: int, + ) -> list[Request | Response]: + return self._network_log_snapshot_since(kind, sequence)[0] + def _remember_navigation_response(self, response: Optional[Response]) -> Optional[Response]: - if response is not None: + if response is None: + return None + with self._network_history_lock_for(): if response.request is not None: response.request._response = response self._navigation_responses.append(response) - self._navigation_responses = self._navigation_responses[-20:] if response._request_id: self._record_response(response) + self._prune_navigation_responses_locked() return response + def _drop_page_owned_response(self, response: Response) -> None: + with self._network_history_lock_for(): + self._drop_page_owned_response_locked(response) + + def _drop_page_owned_response_locked(self, response: Response) -> None: + if not hasattr(self, "_request_log"): + self._request_log = [] + if not hasattr(self, "_request_log_keys"): + self._request_log_keys = set() + if not hasattr(self, "_request_history_pending_keys"): + self._request_history_pending_keys = set() + if not hasattr(self, "_requests_by_key"): + self._requests_by_key = {} + if not hasattr(self, "_response_log"): + self._response_log = [] + if not hasattr(self, "_response_log_keys"): + self._response_log_keys = set() + if not hasattr(self, "_navigation_responses"): + self._navigation_responses = [] + if not hasattr(self, "_fulfilled_route_bodies"): + self._fulfilled_route_bodies = {} + self._navigation_responses = [ + candidate for candidate in self._navigation_responses if candidate is not response + ] + self._ensure_response_log_sequences() + retained_response_entries = [ + (event_sequence, candidate) + for event_sequence, candidate in zip(self._response_log_sequences, self._response_log) + if candidate is not response + ] + self._response_log_sequences = [event_sequence for event_sequence, _ in retained_response_entries] + self._response_log = [candidate for _, candidate in retained_response_entries] + + associated_request_ids: set[int] = set() + for request in [*self._request_log, *self._requests_by_key.values()]: + if request is response.request or request._response is response: + associated_request_ids.add(id(request)) + self._ensure_request_log_sequences() + retained_request_entries = [ + (event_sequence, request) + for event_sequence, request in zip(self._request_log_sequences, self._request_log) + if id(request) not in associated_request_ids + ] + self._request_log_sequences = [event_sequence for event_sequence, _ in retained_request_entries] + self._request_log = [request for _, request in retained_request_entries] + self._requests_by_key = { + key: request + for key, request in self._requests_by_key.items() + if id(request) not in associated_request_ids + } + self._request_log_keys = { + key + for request in self._request_log + if (key := self._request_key(request)) is not None + } + retained_request_keys = self._request_log_keys | set(self._requests_by_key) + self._request_history_pending_keys.intersection_update(retained_request_keys) + + self._response_log_keys = { + (candidate._request_id, candidate.url, candidate.status) + for candidate in self._response_log + } + request_id = None if response._request_id is None else str(response._request_id) + if request_id is not None and not any( + candidate._request_id is not None + and str(candidate._request_id) == request_id + for candidate in [*self._navigation_responses, *self._response_log] + ): + self._fulfilled_route_bodies.pop(request_id, None) + + def _page_owned_bodies_locked(self) -> dict[int, bytes]: + page_owned: dict[int, bytes] = {} + for response in self._navigation_responses: + if response._body_cache is not None: + page_owned.setdefault(id(response._body_cache), response._body_cache) + for response in self._response_log: + if response._body_cache is not None: + page_owned.setdefault(id(response._body_cache), response._body_cache) + for body in self._fulfilled_route_bodies.values(): + page_owned.setdefault(id(body), body) + return page_owned + + def _prune_navigation_responses(self) -> None: + with self._network_history_lock_for(): + self._prune_navigation_responses_locked() + + def _prune_navigation_responses_locked(self) -> None: + if not hasattr(self, "_navigation_responses"): + self._navigation_responses = [] + if not hasattr(self, "_response_log"): + self._response_log = [] + if not hasattr(self, "_response_log_keys"): + self._response_log_keys = set() + if not hasattr(self, "_fulfilled_route_bodies"): + self._fulfilled_route_bodies = {} + self._ensure_response_log_sequences() + if len(self._navigation_responses) > _NAVIGATION_RESPONSE_RETENTION_COUNT: + overflow = len(self._navigation_responses) - _NAVIGATION_RESPONSE_RETENTION_COUNT + evicted = self._navigation_responses[:overflow] + self._navigation_responses = self._navigation_responses[overflow:] + for response in evicted: + if not any(candidate is response for candidate in self._navigation_responses): + self._drop_page_owned_response_locked(response) + while True: + page_owned = self._page_owned_bodies_locked() + total_bytes = sum(len(body) for body in page_owned.values()) + if total_bytes <= _NAVIGATION_RESPONSE_RETENTION_BYTES: + break + response = next( + ( + candidate + for candidate in self._navigation_responses + if candidate._body_cache is not None + ), + None, + ) + if response is None: + response = next( + ( + candidate + for candidate in self._response_log + if candidate._body_cache is not None + ), + None, + ) + if response is not None: + self._drop_page_owned_response_locked(response) + continue + if self._fulfilled_route_bodies: + old_request_id = next(iter(self._fulfilled_route_bodies)) + self._fulfilled_route_bodies.pop(old_request_id, None) + continue + break + def _cached_navigation_response_for_current_url(self) -> Optional[Response]: try: current_url = self.url except Exception: return None current_key = current_url.rstrip("/") if current_url.startswith(("http://", "https://")) else current_url - for response in reversed(self._navigation_responses): + with self._network_history_lock_for(): + navigation_responses = list(self._navigation_responses) + for response in reversed(navigation_responses): response_key = response.url.rstrip("/") if response.url.startswith(("http://", "https://")) else response.url if response_key == current_key: return self._record_response(response) return None + def _transfer_fulfilled_route_body_locked(self, response: Response) -> None: + request_id = None if response._request_id is None else str(response._request_id) + if request_id is None: + return + body = self._fulfilled_route_bodies.pop(request_id, None) + if body is None: + return + matching_response = response + for candidate in [*self._navigation_responses, *self._response_log]: + if ( + candidate is not response + and candidate._request_id is not None + and str(candidate._request_id) == request_id + ): + matching_response = candidate + break + if matching_response._body_cache is None: + matching_response._body_cache = body + def _record_response(self, response: Response) -> Response: - if response.request is not None: - response.request._response = response - response.request = self._record_request(response.request) - key = (response._request_id, response.url, response.status) - if key in self._response_log_keys: + with self._network_history_lock_for(): + if response.request is not None: + response.request._response = response + response.request = self._record_request(response.request) + if not hasattr(self, "_response_log_keys"): + self._response_log_keys = set() + if not hasattr(self, "_response_log"): + self._response_log = [] + if not hasattr(self, "_fulfilled_route_bodies"): + self._fulfilled_route_bodies = {} + self._ensure_response_log_sequences() + self._transfer_fulfilled_route_body_locked(response) + key = (response._request_id, response.url, response.status) + if key in self._response_log_keys: + return response + self._response_log.append(response) + self._response_log_sequences.append(self._next_response_log_sequence) + self._next_response_log_sequence += 1 + self._response_log_keys.add(key) + overflow = len(self._response_log) - 100 + if overflow > 0: + evicted = self._response_log[:overflow] + self._response_log = self._response_log[overflow:] + self._response_log_sequences = self._response_log_sequences[overflow:] + for old_response in evicted: + if not any(candidate is old_response for candidate in self._navigation_responses): + self._drop_page_owned_response_locked(old_response) + self._response_log_keys = { + (candidate._request_id, candidate.url, candidate.status) + for candidate in self._response_log + } return response - self._response_log.append(response) - self._response_log_keys.add(key) - overflow = len(self._response_log) - 100 - if overflow > 0: - for old_response in self._response_log[:overflow]: - self._response_log_keys.discard((old_response._request_id, old_response.url, old_response.status)) - self._response_log = self._response_log[overflow:] - return response def _retain_navigation_response_bodies(self) -> None: - retained: list[Response] = [] - for response in self._navigation_responses: - if response._body_cache is not None: - retained.append(response) - continue + with self._network_history_lock_for(): + responses = list(self._navigation_responses) + for response in responses: + with self._network_history_lock_for(): + if response._body_cache is not None: + continue try: response._cache_body(timeout_ms=250.0) except Exception: pass - retained.append(response) - self._navigation_responses = retained[-20:] + self._prune_navigation_responses() + + def _release_memory_buffers(self) -> None: + release = getattr(self._core, "release_memory_buffers", None) + if callable(release): + try: + release() + except Exception: + pass + with self._network_history_lock_for(): + self._request_log.clear() + self._request_log_sequences.clear() + self._request_log_keys.clear() + self._requests_by_key.clear() + self._response_log.clear() + self._response_log_sequences.clear() + self._response_log_keys.clear() + self._navigation_responses.clear() + self._fulfilled_route_bodies.clear() + self._request_history_pending_keys.clear() + self._request_history_wait_until = 0.0 + self._request_log_generation += 1 + with self._request_log_condition: + self._request_log_condition.notify_all() + self._console_messages.clear() + self._page_errors.clear() + self._frame_object_cache.clear() + self._frame_event_cache.clear() + self._workers.clear() + self._worker_console_forwards.clear() + self._owned_cdp_sessions.clear() + self._routes.clear() + self._websocket_routes.clear() + self._bindings.clear() + self._har_recordings.clear() + self._locator_handlers.clear() + self._clock_script_names.clear() + self._clock_initialized.clear() + self._video = None def _navigation_response_request_ids(self) -> list[str]: request_ids: list[str] = [] - for response in self._navigation_responses: - if response._body_cache is not None: - continue - if response._request_id: - request_ids.append(str(response._request_id)) - else: - response._body_cache = b"" + with self._network_history_lock_for(): + for response in self._navigation_responses: + if response._body_cache is not None: + continue + if response._request_id: + request_ids.append(str(response._request_id)) + else: + response._body_cache = b"" return request_ids def _apply_retained_navigation_response_bodies(self, payload: Any) -> None: @@ -16382,11 +16714,12 @@ def _apply_retained_navigation_response_bodies(self, payload: Any) -> None: retained = payload.get("response_bodies") if not isinstance(retained, dict): return - responses = { - str(response._request_id): response - for response in self._navigation_responses - if response._request_id and response._body_cache is None - } + with self._network_history_lock_for(): + responses = { + str(response._request_id): response + for response in self._navigation_responses + if response._request_id and response._body_cache is None + } for request_id, body_payload in retained.items(): response = responses.get(str(request_id)) if response is None: @@ -16533,7 +16866,9 @@ def goto( if message.startswith("Page.goto: net::ERR_HTTP_RESPONSE_CODE_FAILURE"): deadline = time.monotonic() + 1.0 while time.monotonic() < deadline: - for response in reversed(self._response_log): + with self._network_history_lock_for(): + response_log = list(self._response_log) + for response in reversed(response_log): if response.url == target_url and int(response.status or 0) >= 400: self._trace_end_action(call_id, result={"response": {"url": response.url, "status": response.status}}) return self._remember_navigation_response(response) @@ -16829,7 +17164,7 @@ def _wait_for_network_event( waiter: Any = None, method: str = "Page.wait_for_event", reject_on_close: bool = True, - log_offset: Optional[int] = None, + log_sequence: Optional[int] = None, ) -> Request | Response: if kind not in {"request", "response", "requestfinished", "requestfailed"}: raise Error(f"unsupported network event kind: {kind}") @@ -16841,7 +17176,7 @@ def _wait_for_network_event( waiter=waiter, reject_on_close=reject_on_close, method=method, - state={} if log_offset is None else {"log_offset": log_offset}, + state={} if log_sequence is None else {"log_sequence": log_sequence}, ) def _wait_for_navigation_response( @@ -19976,16 +20311,22 @@ def opener(self) -> Optional["Page"]: def requests(self) -> list[Request]: - if not self._closed and time.monotonic() < self._request_history_wait_until: - deadline = self._request_history_wait_until - generation = self._request_log_generation + while True: + with self._network_history_lock_for(): + if self._closed: + break + deadline = self._request_history_wait_until + generation = self._request_log_generation + remaining = deadline - time.monotonic() + if remaining <= 0: + break with self._request_log_condition: - while self._request_log_generation == generation: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - self._request_log_condition.wait(timeout=remaining) - return list(self._request_log) + self._request_log_condition.wait(timeout=remaining) + with self._network_history_lock_for(): + if self._request_log_generation != generation: + break + with self._network_history_lock_for(): + return list(self._request_log) def _console_messages_snapshot(self, filter: Optional[str]) -> list[ConsoleMessage]: if filter == "all": @@ -20396,6 +20737,7 @@ def close(self, *, run_before_unload: Optional[bool] = None, reason: Optional[st if self._context is not None and self in self._context._pages: self._context._pages.remove(self) _emit_event(self._event_handlers, "close", self) + self._release_memory_buffers() if self._owns_context and self._context is not None: self._context.close() @@ -21052,33 +21394,39 @@ def _handle_frame_lifecycle_event(self, event: str, payload: dict[str, Any]) -> def _dispatch_frame_lifecycle_event(self, event: str, frame: Frame) -> None: self._dispatch_event_handlers(event, frame) - def _record_request(self, request: Request) -> Request: - request = self._adopt_request(request) - key = self._request_key(request) - if key is not None and request.method and key in self._request_history_pending_keys: - self._request_history_pending_keys.discard(key) - if not self._request_history_pending_keys: - self._request_history_wait_until = 0.0 - if key is not None and key in self._request_log_keys: - with self._request_log_condition: - self._request_log_generation += 1 - self._request_log_condition.notify_all() - return request - self._request_log.append(request) - if key is not None: - self._request_log_keys.add(key) - overflow = len(self._request_log) - 100 - if overflow > 0: - for old_request in self._request_log[:overflow]: - old_key = self._request_key(old_request) - if old_key is not None: - self._request_log_keys.discard(old_key) - self._request_history_pending_keys.discard(old_key) - self._request_log = self._request_log[overflow:] + def _notify_request_log_change_locked(self) -> None: + self._request_log_generation += 1 with self._request_log_condition: - self._request_log_generation += 1 self._request_log_condition.notify_all() - return request + + def _record_request(self, request: Request) -> Request: + with self._network_history_lock_for(): + request = self._adopt_request(request) + key = self._request_key(request) + if key is not None and request.method and key in self._request_history_pending_keys: + self._request_history_pending_keys.discard(key) + if not self._request_history_pending_keys: + self._request_history_wait_until = 0.0 + if key is not None and key in self._request_log_keys: + self._notify_request_log_change_locked() + return request + self._ensure_request_log_sequences() + self._request_log.append(request) + self._request_log_sequences.append(self._next_request_log_sequence) + self._next_request_log_sequence += 1 + if key is not None: + self._request_log_keys.add(key) + overflow = len(self._request_log) - 100 + if overflow > 0: + for old_request in self._request_log[:overflow]: + old_key = self._request_key(old_request) + if old_key is not None: + self._request_log_keys.discard(old_key) + self._request_history_pending_keys.discard(old_key) + self._request_log = self._request_log[overflow:] + self._request_log_sequences = self._request_log_sequences[overflow:] + self._notify_request_log_change_locked() + return request def _request_key(self, request: Request) -> Optional[tuple[str, str]]: if not request._request_id or not request.url: @@ -21122,36 +21470,55 @@ def _merge_request_state(self, target: Request, source: Request) -> Request: return target def _adopt_request(self, request: Request) -> Request: - key = self._request_key(request) - if key is None: - return request - existing = self._requests_by_key.get(key) - if existing is None: - self._requests_by_key[key] = request - if len(self._requests_by_key) > 300: - for old_key in list(self._requests_by_key)[:-300]: - self._requests_by_key.pop(old_key, None) - return request - return self._merge_request_state(existing, request) + with self._network_history_lock_for(): + key = self._request_key(request) + if key is None: + return request + existing = self._requests_by_key.get(key) + if existing is None: + self._requests_by_key[key] = request + if len(self._requests_by_key) > 300: + for old_key in list(self._requests_by_key)[:-300]: + self._requests_by_key.pop(old_key, None) + return request + return self._merge_request_state(existing, request) def _link_response_request(self, response: Response) -> None: - if response.request is None: - return - request = self._adopt_request(response.request) - response.request = request - request._response = response - self._record_request(request) - if not request.method and request._request_id: - key = self._request_key(request) - if key is not None: - self._request_history_pending_keys.add(key) - self._request_history_wait_until = time.monotonic() + _HISTORY_EVENT_DRAIN_TIMEOUT_SECONDS + with self._network_history_lock_for(): + if response.request is None: + return + request = self._adopt_request(response.request) + response.request = request + request._response = response + self._record_request(request) + if not request.method and request._request_id: + key = self._request_key(request) + if key is not None: + self._request_history_pending_keys.add(key) + self._request_history_wait_until = time.monotonic() + _HISTORY_EVENT_DRAIN_TIMEOUT_SECONDS def _remember_fulfilled_route_body(self, request_id: str, body: bytes) -> None: - self._fulfilled_route_bodies[str(request_id)] = bytes(body) - if len(self._fulfilled_route_bodies) > 200: - for old_request_id in list(self._fulfilled_route_bodies)[:-200]: - self._fulfilled_route_bodies.pop(old_request_id, None) + request_id = str(request_id) + stored_body = bytes(body) + with self._network_history_lock_for(): + if not hasattr(self, "_fulfilled_route_bodies"): + self._fulfilled_route_bodies = {} + matching_response = next( + ( + response + for response in [*self._navigation_responses, *self._response_log] + if response._request_id is not None + and str(response._request_id) == request_id + ), + None, + ) + if matching_response is None: + self._fulfilled_route_bodies[request_id] = stored_body + while len(self._fulfilled_route_bodies) > 200: + self._fulfilled_route_bodies.pop(next(iter(self._fulfilled_route_bodies))) + elif matching_response._body_cache is None: + matching_response._body_cache = stored_body + self._prune_navigation_responses_locked() def _ensure_console_thread(self) -> None: if self._runtime_observation_enabled: From 82ef7d5bf4188c9281027e07dfe777c734df8182 Mon Sep 17 00:00:00 2001 From: suchintan <3853670+suchintan@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:38:26 +0000 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=94=84=20synced=20local=20'src/'=20wi?= =?UTF-8?q?th=20remote=20'src/'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/Skyvern-AI/rustwright-cloud/pull/239 --- src/lib.rs | 2094 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 1817 insertions(+), 277 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b50662d..9eddf5d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,7 +18,7 @@ use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering}; use std::sync::LazyLock; #[cfg(feature = "python")] use std::sync::OnceLock; -use std::sync::{Arc, Condvar, Mutex, Weak}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, Weak}; use std::time::{Duration, Instant}; use base64::Engine; @@ -366,6 +366,7 @@ struct PendingCommandGuard { pending: CdpPendingMap, outstanding: CdpOutstandingMap, traffic_log: Arc>, + retention_gate: Arc, } struct QueuedCdpCommand { @@ -412,18 +413,25 @@ impl PendingCommandGuard { pending: &CdpPendingMap, outstanding: &CdpOutstandingMap, traffic_log: &Arc>, + retention_gate: &Arc, ) -> Self { Self { id, pending: Arc::clone(pending), outstanding: Arc::clone(outstanding), traffic_log: Arc::clone(traffic_log), + retention_gate: Arc::clone(retention_gate), } } } impl Drop for PendingCommandGuard { fn drop(&mut self) { + let Some(_retention_guard) = self.retention_gate.lock_for_write() else { + self.pending.lock().unwrap().remove(&self.id); + self.outstanding.lock().unwrap().remove(&self.id); + return; + }; let was_pending = self.pending.lock().unwrap().remove(&self.id).is_some(); let outstanding = self.outstanding.lock().unwrap().remove(&self.id); if was_pending { @@ -444,7 +452,12 @@ impl Drop for SpawnedTaskAbortGuard { } const CDP_EVENT_LOG_LIMIT: usize = 8192; +const CDP_EVENT_LOG_MAX_BYTES: usize = 8 * 1024 * 1024; +const CDP_EVENT_LOG_MAX_ENTRY_BYTES: usize = 64 * 1024; +const CDP_RETAINED_STRING_MAX_BYTES: usize = 8 * 1024; +const CDP_RETAINED_ARRAY_MAX_ITEMS: usize = 128; const CDP_DIAGNOSTIC_TRAFFIC_LIMIT: usize = 32; +const CDP_EVENT_UNREPLAYABLE_MARKER: &str = "__rustwright_cdp_event_unreplayable"; const CDP_DIAGNOSTIC_LIST_LIMIT: usize = 32; const CDP_DIAGNOSTIC_QUERY_TIMEOUT: Duration = Duration::from_millis(250); const TIMEOUT_DIAGNOSTIC_BANNER: &str = "RUSTWRIGHT TIMEOUT DIAGNOSTIC"; @@ -9946,6 +9959,7 @@ multiline-compatible = """4.5.6""" cursor_slot: Arc::clone(&cursor_slot), requests: Arc::clone(&shared_requests), working_requests: Arc::clone(&working_requests), + page: None, }; drop(lease); @@ -9972,6 +9986,7 @@ multiline-compatible = """4.5.6""" cursor_slot: Arc::clone(&cursor_slot), requests: Arc::clone(&shared_requests), working_requests, + page: None, }; { let mut live = shared_requests.lock().unwrap(); @@ -10031,6 +10046,117 @@ multiline-compatible = """4.5.6""" ); } + #[cfg(feature = "python")] + #[test] + fn delivered_page_event_lease_does_not_merge_after_page_release() { + let harness = navigation_test_harness(4); + let page = Arc::clone(&harness.page); + let shared_requests = Arc::clone(&page.network_requests); + let mut working_store = NetworkRequestStore::new(2); + let snapshot = NetworkRequestSnapshot { + seq: 1, + request: json!({ "url": "https://example.test/late" }), + redirect_ancestry: Vec::new(), + }; + working_store.requests.insert( + "request-1".to_string(), + NetworkRequestEntry { + current: snapshot.clone(), + applied_by_seq: BTreeMap::from([(1, snapshot)]), + }, + ); + working_store + .applied_order + .push_back((1, "request-1".to_string())); + let working_requests = Arc::new(Mutex::new(working_store)); + let (_sender, receiver) = broadcast::channel(4); + let receiver_slot = Arc::new(Mutex::new(None)); + let state_slot = Arc::new(Mutex::new(None)); + let cursor_slot = Arc::new(Mutex::new(0)); + let lease = PageEventStreamLease { + receiver: Some(receiver), + state: Some(PageEventStreamState::new()), + rollback_state: Some(PageEventStreamState::new()), + cursor: 2, + rollback_cursor: 0, + delivered: true, + receiver_slot, + state_slot, + cursor_slot, + requests: Arc::clone(&shared_requests), + working_requests, + page: Some(page.clone()), + }; + + page.release_memory_buffers(); + drop(lease); + + assert!(shared_requests.lock().unwrap().requests.is_empty()); + } + + #[cfg(feature = "python")] + #[test] + fn network_backfill_tombstone_resets_store_and_reports_overflow() { + fn request_event(url: &str) -> Value { + json!({ + "sessionId": "page-session", + "method": "Network.requestWillBeSent", + "params": { + "requestId": url, + "request": { + "url": url, + "method": "GET", + "headers": {}, + }, + }, + }) + } + + let mut event_log = CdpEventLog::new(); + event_log.push(request_event("https://example.test/before")); + let mut tombstone = json!({ + "sessionId": "page-session", + "method": "Network.requestWillBeSent", + "params": { + "requestId": "tombstone", + "request": { + "url": "https://example.test/tombstone", + "method": "POST", + "headers": {"x-tombstone": "must-not-apply"}, + }, + }, + }); + tombstone + .as_object_mut() + .unwrap() + .insert(CDP_EVENT_UNREPLAYABLE_MARKER.to_string(), Value::Bool(true)); + event_log.push(tombstone); + event_log.push(request_event("https://example.test/current")); + let current_event = event_log.entries_since(0)[2].1.clone(); + let event_log = Arc::new(Mutex::new(event_log)); + let requests = Arc::new(Mutex::new(NetworkRequestStore::new(0))); + let mut state = NetworkObservationState::new(); + + let result = process_network_observation_event( + 2, + ¤t_event, + &event_log, + "page-session", + &requests, + true, + &mut state, + ); + + assert_eq!(result, Err(1)); + let requests = requests.lock().unwrap(); + assert_eq!(requests.next_applied_seq, 3); + assert!(requests.requests.is_empty()); + assert!(requests.applied_order.is_empty()); + drop(requests); + assert!(state.pending_responses.is_empty()); + assert!(state.response_extra_infos.is_empty()); + } + #[cfg(feature = "python")] #[test] fn stale_page_event_ack_does_not_restore_requests_cleared_by_overflow() { @@ -10074,6 +10200,7 @@ multiline-compatible = """4.5.6""" cursor_slot, requests: Arc::clone(&shared_requests), working_requests, + page: None, }; shared_requests.lock().unwrap().reset_after_overflow(20); @@ -10240,6 +10367,7 @@ multiline-compatible = """4.5.6""" cursor_slot: Arc::new(Mutex::new(7_999)), requests: Arc::clone(&shared_requests), working_requests, + page: None, }; shared_requests.lock().unwrap().reset_after_overflow(8_000); @@ -10336,6 +10464,7 @@ multiline-compatible = """4.5.6""" cursor_slot: Arc::new(Mutex::new(7_999)), requests: Arc::clone(&shared_requests), working_requests, + page: None, }; shared_requests.lock().unwrap().reset_after_overflow(8_000); @@ -10632,6 +10761,33 @@ multiline-compatible = """4.5.6""" assert_eq!(batch[0]["payload"]["text"], "after failed capture"); } + #[tokio::test] + async fn release_memory_buffers_clears_console_capture_after_setup_settles() { + let mut harness = navigation_test_harness(16); + let page = Arc::clone(&harness.page); + page.console_capture.requested.store(true, Ordering::SeqCst); + let capture_page = Arc::clone(&page); + let capture = tokio::spawn(async move { + enable_console_capture_for_session( + &capture_page, + "page-session", + Duration::from_secs(1), + ) + .await + }); + let enable = harness.next_command("Runtime.enable").await; + page.release_memory_buffers(); + harness.reply(&enable, json!({})); + + assert!(matches!( + capture.await.unwrap(), + Err(RwError::TargetClosed(TargetClosedKind::Page)) + )); + assert!(!page.console_capture.requested.load(Ordering::SeqCst)); + let enabled_sessions = page.console_capture.enabled_sessions.lock().await; + assert!(enabled_sessions.is_empty()); + } + #[tokio::test] async fn worker_resume_claim_and_failure_use_production_watchdog_path() { let mut harness = navigation_test_harness(16); @@ -16489,6 +16645,128 @@ return this.dataset.mainWorldOverride === "observed"; harness.reply_error(&file_chooser, "setup deliberately withheld"); } + #[tokio::test] + async fn manual_oopif_attachment_does_not_repopulate_released_page() { + let mut harness = navigation_test_harness(8); + let page = Arc::clone(&harness.page); + let target_info = json!({ + "type": "iframe", + "targetId": "released-child-frame", + "parentFrameId": "root-frame", + }); + let resolver_page = Arc::clone(&page); + let resolver = tokio::spawn(async move { + attach_iframe_target_for_frame( + resolver_page, + "released-child-frame", + &target_info, + "page-session", + OperationDeadline::new(Duration::from_secs(2)), + ) + .await + }); + + let attach = harness.next_command("Target.attachToTarget").await; + page.release_memory_buffers(); + harness.reply(&attach, json!({ "sessionId": "released-child-session" })); + + let detach = harness.next_command("Target.detachFromTarget").await; + assert_eq!( + detach["params"], + json!({ "sessionId": "released-child-session" }) + ); + let result = resolver.await.unwrap(); + assert!(matches!( + result, + Err(RwError::TargetClosed(TargetClosedKind::Page)) + )); + let state = harness.page.frame_state.lock().unwrap(); + assert!(state.frames.is_empty()); + assert!(state.frame_sessions.is_empty()); + assert!(state.session_frames.is_empty()); + assert!(state.frame_session_waiters.is_empty()); + } + + #[test] + fn page_event_tombstone_rebuilds_child_session_ownership() { + let harness = navigation_test_harness(8); + { + let mut frame_state = harness.page.frame_state.lock().unwrap(); + frame_state.record_frame( + "main-frame".to_string(), + None, + None, + Some("https://example.test/".to_string()), + "page-session".to_string(), + ); + frame_state.record_session_for_frame("child-frame", "child-session"); + frame_state.record_frame( + "child-frame".to_string(), + Some("main-frame".to_string()), + None, + Some("https://child.example.test/".to_string()), + "child-session".to_string(), + ); + } + let oversized_attachment = json!({ + "sessionId": "page-session", + "method": "Target.attachedToTarget", + "params": { + "sessionId": "child-session", + "waitingForDebugger": vec!["x".repeat(CDP_EVENT_LOG_MAX_ENTRY_BYTES); 16], + "targetInfo": { + "type": "iframe", + "targetId": "x".repeat(CDP_EVENT_LOG_MAX_ENTRY_BYTES), + "subtype": "x".repeat(CDP_EVENT_LOG_MAX_ENTRY_BYTES), + "title": "x".repeat(CDP_EVENT_LOG_MAX_ENTRY_BYTES), + "url": "x".repeat(CDP_EVENT_LOG_MAX_ENTRY_BYTES), + "openerId": "x".repeat(CDP_EVENT_LOG_MAX_ENTRY_BYTES), + "parentFrameId": "x".repeat(CDP_EVENT_LOG_MAX_ENTRY_BYTES), + }, + }, + }); + let (tombstone, _) = compact_retained_cdp_event(oversized_attachment); + assert!(is_unreplayable_cdp_event(&tombstone)); + let mut stream_state = PageEventStreamState::for_session("page-session".to_string()); + stream_state + .owned_sessions + .insert("stale-session".to_string()); + let console = json!({ + "sessionId": "child-session", + "method": "Runtime.consoleAPICalled", + "params": { + "type": "log", + "args": [{"type": "string", "value": "after overflow"}], + }, + }); + + process_page_observation_event_with_page( + 0, + &tombstone, + &harness.event_log, + "page-session", + &harness.page.network_requests, + &mut stream_state, + Some(&harness.page), + ); + assert!(stream_state.owned_sessions.contains("child-session")); + assert!(!stream_state.owned_sessions.contains("stale-session")); + process_page_observation_event_with_page( + 1, + &console, + &harness.event_log, + "page-session", + &harness.page.network_requests, + &mut stream_state, + Some(&harness.page), + ); + let mut batch = Vec::new(); + append_ready_page_events(&mut batch, &mut stream_state, 8); + assert_eq!(batch[0]["kind"], "_overflow"); + assert_eq!(batch[1]["kind"], "console"); + assert_eq!(batch[1]["payload"]["text"], "after overflow"); + } + #[tokio::test] async fn auto_attached_oopif_resolution_returns_at_routable_before_setup_replies() { let mut harness = navigation_test_harness(16); @@ -22644,6 +22922,295 @@ return this.dataset.mainWorldOverride === "observed"; .action_dispatch_receipts .is_empty()); } + #[test] + fn cdp_event_log_bounds_retained_payloads_without_changing_live_events() { + let mut log = CdpEventLog::new(); + let event = json!({ + "sessionId": "page-session", + "method": "Runtime.consoleAPICalled", + "params": { + "type": "log", + "args": [{ "value": "x".repeat(CDP_EVENT_LOG_MAX_ENTRY_BYTES * 2) }] + } + }); + let live = log.push(event.clone()); + assert_eq!( + live.pointer("/params/args/0/value") + .and_then(Value::as_str) + .map(str::len), + Some(CDP_EVENT_LOG_MAX_ENTRY_BYTES * 2) + ); + let retained = log.entries_since(0); + assert_eq!(retained.len(), 1); + assert_eq!(retained[0].0, 0); + assert!(retained[0] + .1 + .pointer("/params/args/0/value") + .and_then(Value::as_str) + .is_some_and(|value| value.len() <= CDP_RETAINED_STRING_MAX_BYTES)); + assert!(log.retained_bytes <= CDP_EVENT_LOG_MAX_BYTES); + assert_eq!(log.cursor_after_event(0, &live), 1); + } + #[test] + fn cdp_event_log_tombstones_payloads_that_remain_oversized_after_compaction() { + let mut log = CdpEventLog::new(); + let event = json!({ + "sessionId": "page-session", + "method": "Runtime.consoleAPICalled", + "params": { + "type": "log", + "args": (0..9) + .map(|index| json!({ "type": "string", "value": format!("{index}:{}", "x".repeat(8_193)) })) + .collect::>() + } + }); + + let live = log.push(event); + let retained = log.entries_since(0); + assert_eq!(retained.len(), 1); + assert_eq!(retained[0].0, 0); + assert_eq!(live["method"], "Runtime.consoleAPICalled"); + assert_eq!(retained[0].1["method"], "Runtime.consoleAPICalled"); + assert_eq!(retained[0].1["sessionId"], "page-session"); + assert_eq!(retained[0].1["__rustwright_cdp_event_seq"], 0); + assert_eq!( + retained[0].1[CDP_EVENT_UNREPLAYABLE_MARKER], + Value::Bool(true) + ); + assert!(retained[0].1.pointer("/params/args").is_none()); + assert!(retained_cdp_event_size(&retained[0].1) <= CDP_EVENT_LOG_MAX_ENTRY_BYTES); + } + + #[test] + fn page_event_stream_turns_compacted_tombstone_into_explicit_overflow() { + let mut log = CdpEventLog::new(); + let event = json!({ + "sessionId": "page-session", + "method": "Network.requestWillBeSent", + "params": { + "requestId": "request-1", + "request": { + "url": "https://example.test/large", + "method": "GET", + "headers": (0..9) + .map(|index| format!("{index}:{}", "x".repeat(8_193))) + .collect::>() + } + } + }); + log.push(event); + let retained = log.entries_since(0).remove(0).1; + let event_log = Arc::new(Mutex::new(log)); + let requests = Arc::new(Mutex::new(NetworkRequestStore::new(0))); + let mut state = PageEventStreamState::new(); + + process_page_observation_event( + 0, + &retained, + &event_log, + "page-session", + &requests, + &mut state, + ); + + assert_eq!(state.ready.get(&0).unwrap()["kind"], "_overflow"); + assert_eq!( + state.ready.get(&0).unwrap()["payload"]["reason"], + "unreplayable_event" + ); + assert_eq!(requests.lock().unwrap().next_applied_seq, 1); + } + + #[test] + fn cdp_event_log_byte_horizon_uses_stored_entry_sizes() { + let mut log = CdpEventLog::new(); + for index in 0..CDP_EVENT_LOG_LIMIT { + log.push(json!({ + "method": "Test.byteHorizon", + "params": { "index": index, "payload": "x".repeat(4_096) } + })); + } + + assert!(log.events.len() < CDP_EVENT_LOG_LIMIT); + assert!(log.oldest_seq() > 0); + assert!(log.retained_bytes <= CDP_EVENT_LOG_MAX_BYTES); + assert_eq!( + log.retained_bytes, + log.events + .iter() + .map(|entry| entry.retained_bytes) + .sum::() + ); + } + + #[test] + fn terminal_event_log_release_shrinks_backing_storage() { + let mut log = CdpEventLog::new(); + log.push(json!({ "method": "Test.retained", "params": { "value": "x" } })); + assert!(log.events.capacity() > 0); + log.clear_retained(); + assert!(log.events.is_empty()); + assert_eq!(log.events.capacity(), 0); + assert_eq!(log.retained_bytes, 0); + } + + #[test] + fn late_dispatch_after_terminal_release_cannot_resurrect_retained_state() { + let harness = navigation_test_harness(4); + let client = Arc::clone(&harness.page.browser.client); + client.release_memory_buffers(); + + harness.emit(json!({ + "sessionId": "page-session", + "method": "Runtime.consoleAPICalled", + "params": { "type": "log", "args": [{ "value": "late" }] } + })); + + assert!(harness.event_log.lock().unwrap().events.is_empty()); + assert!(harness + .page + .browser + .client + .traffic_log + .lock() + .unwrap() + .entries + .is_empty()); + assert!(harness + .page + .browser + .client + .runtime_state + .lock() + .unwrap() + .serializers + .is_empty()); + } + + #[tokio::test] + async fn in_flight_page_event_after_terminal_release_cannot_resurrect_page_state() { + let harness = navigation_test_harness(4); + let page = Arc::clone(&harness.page); + harness.listen_for_oopif_events(); + let (mut entered, mut completed, released) = + install_page_lifecycle_test_barrier(&page, "child-session"); + + harness.emit(json!({ + "sessionId": "page-session", + "method": "Target.attachedToTarget", + "params": { + "sessionId": "child-session", + "targetInfo": { + "type": "iframe", + "targetId": "child-frame", + "parentFrameId": "main-frame", + }, + }, + })); + tokio::time::timeout(Duration::from_secs(1), entered.changed()) + .await + .expect("page event handler must reach the lifecycle barrier") + .expect("lifecycle barrier sender must remain connected"); + + page.release_memory_buffers(); + page.browser.client.release_memory_buffers(); + released.notify_one(); + tokio::time::timeout(Duration::from_secs(1), completed.changed()) + .await + .expect("page event handler must leave the lifecycle barrier") + .expect("lifecycle completion sender must remain connected"); + + assert!(!page + .frame_state + .lock() + .unwrap() + .owns_session("child-session")); + assert!(page + .browser + .client + .event_log + .lock() + .unwrap() + .events + .is_empty()); + } + + #[tokio::test] + async fn console_worker_waiter_drops_attachment_after_terminal_release() { + let harness = navigation_test_harness(4); + let page = Arc::clone(&harness.page); + let event_log = Arc::clone(&harness.event_log); + let (mut entered, mut completed, released) = + install_page_lifecycle_test_barrier(&page, "worker-session"); + + harness.emit(json!({ + "sessionId": "page-session", + "method": "Target.attachedToTarget", + "params": { + "sessionId": "worker-session", + "targetInfo": { + "type": "worker", + "targetId": "worker-target", + }, + }, + })); + let event = event_log + .lock() + .unwrap() + .entries_since(0) + .last() + .expect("worker attachment event") + .1 + .clone(); + let waiter_page = Arc::clone(&page); + let waiter_log = Arc::clone(&event_log); + let waiter = tokio::spawn(async move { + let mut cursor = 0; + let replay_until = HashMap::new(); + process_console_wait_event( + event, + &mut cursor, + 0, + &waiter_log, + "page-session", + &replay_until, + "console", + Some(waiter_page), + None, + None, + None, + false, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(1), entered.changed()) + .await + .expect("console waiter must reach the lifecycle barrier") + .expect("lifecycle barrier sender must remain connected"); + + page.release_memory_buffers(); + released.notify_one(); + tokio::time::timeout(Duration::from_secs(1), completed.changed()) + .await + .expect("console waiter must leave the lifecycle barrier") + .expect("lifecycle barrier sender must remain connected"); + assert_eq!(waiter.await.unwrap().unwrap(), None); + assert!(!page + .frame_state + .lock() + .unwrap() + .owns_session("worker-session")); + assert_eq!( + remove_page_worker_session_from_detachment( + &page, + &json!({ + "method": "Target.detachedFromTarget", + "params": {"sessionId": "worker-session"}, + }), + ), + None + ); + } #[test] fn commencing_receipt_is_terminal_only_for_single_evaluation_dispatch() { @@ -23372,6 +23939,8 @@ return this.dataset.mainWorldOverride === "observed"; let mut log = CdpEventLog { next_seq: 10, events: VecDeque::with_capacity(CDP_EVENT_LOG_LIMIT), + retained_bytes: 0, + retention_gate: Arc::new(CdpRetentionGate::new()), action_dispatch_receipts: HashMap::new(), console_replay_cutoffs: HashMap::new(), console_replay_cutoff_generations: HashMap::new(), @@ -24309,6 +24878,10 @@ impl CdpTrafficLog { fn snapshot(&self) -> Vec { self.entries.iter().cloned().collect() } + fn clear_for_close(&mut self) { + self.entries = VecDeque::new(); + self.last_page_navigate = HashMap::new(); + } fn page_navigate_waiting_for( &self, @@ -24670,6 +25243,14 @@ impl CdpRuntimeState { self.frame_loaders .retain(|(loader_session_id, _), _| loader_session_id != session_id); } + fn clear_for_close(&mut self) { + self.serializers = HashMap::new(); + self.serializer_install_locks = HashMap::new(); + self.serializer_generations = HashMap::new(); + self.execution_realms = HashMap::new(); + self.session_realms = HashMap::new(); + self.frame_loaders = HashMap::new(); + } } fn spawn_serializer_release_pump( @@ -24913,9 +25494,273 @@ fn merge_console_replay_cutoff(stored: &mut u64, cutoff: u64) { } } +fn compact_retained_cdp_value(value: &mut Value) { + match value { + Value::String(text) => { + if text.len() > CDP_RETAINED_STRING_MAX_BYTES { + let mut end = CDP_RETAINED_STRING_MAX_BYTES; + while !text.is_char_boundary(end) { + end -= 1; + } + text.truncate(end); + } + } + Value::Array(values) => { + values.truncate(CDP_RETAINED_ARRAY_MAX_ITEMS); + for value in values { + compact_retained_cdp_value(value); + } + } + Value::Object(values) => { + for value in values.values_mut() { + compact_retained_cdp_value(value); + } + } + Value::Null | Value::Bool(_) | Value::Number(_) => {} + } +} + +struct CountingWriter { + bytes: usize, +} + +impl Write for CountingWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.bytes = self.bytes.saturating_add(bytes.len()); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +fn retained_cdp_event_size(event: &Value) -> usize { + let mut writer = CountingWriter { bytes: 0 }; + serde_json::to_writer(&mut writer, event).map_or(usize::MAX, |_| writer.bytes) +} + +fn retained_object_fields(value: &Value, fields: &[&str]) -> Value { + let mut retained = serde_json::Map::new(); + if let Some(object) = value.as_object() { + for field in fields { + if let Some(value) = object.get(*field) { + retained.insert((*field).to_string(), value.clone()); + } + } + } + Value::Object(retained) +} + +fn retained_cdp_event_params(method: &str, params: &Value) -> Option { + let mut retained = serde_json::Map::new(); + let fields: &[&str] = match method { + "Page.frameAttached" => &["frameId", "parentFrameId"], + "Page.frameDetached" => &["frameId", "reason"], + "Page.navigatedWithinDocument" => &["frameId", "url", "navigationType"], + "Page.frameNavigated" => &["frame"], + "Target.attachedToTarget" => &["sessionId", "targetInfo", "waitingForDebugger"], + "Target.detachedFromTarget" => &["sessionId", "targetId"], + "Runtime.executionContextCreated" => &["context"], + "Runtime.executionContextDestroyed" => &["executionContextId", "executionContextUniqueId"], + "Network.requestWillBeSent" => &[ + "requestId", + "loaderId", + "documentURL", + "type", + "frameId", + "hasUserGesture", + "redirectHasExtraInfo", + ], + "Network.requestWillBeSentExtraInfo" => &["requestId"], + "Network.responseReceived" => &["requestId", "loaderId", "type", "frameId", "hasExtraInfo"], + "Network.responseReceivedExtraInfo" => { + &["requestId", "statusCode", "resourceIPAddressSpace"] + } + "Network.loadingFinished" | "Network.loadingFailed" => &[ + "requestId", + "encodedDataLength", + "errorText", + "blockedReason", + "canceled", + ], + "Runtime.consoleAPICalled" => &[ + "type", + "executionContextId", + "timestamp", + "exceptionId", + "stackTrace", + ], + "Runtime.exceptionThrown" => &["timestamp", "exceptionDetails"], + _ => &[], + }; + if let Some(object) = params.as_object() { + for field in fields { + if let Some(value) = object.get(*field) { + let value = match (*field, method) { + ("frame", "Page.frameNavigated") => retained_object_fields( + value, + &[ + "id", + "parentId", + "loaderId", + "url", + "name", + "securityOrigin", + "mimeType", + "unreachableUrl", + "adFrameType", + ], + ), + ("targetInfo", "Target.attachedToTarget") => retained_object_fields( + value, + &[ + "targetId", + "type", + "subtype", + "title", + "url", + "openerId", + "parentFrameId", + ], + ), + ("context", "Runtime.executionContextCreated") => { + let mut context = retained_object_fields( + value, + &["id", "uniqueId", "name", "origin", "auxData"], + ); + if let Some(aux_data) = value.get("auxData") { + if let Some(object) = context.as_object_mut() { + object.insert( + "auxData".to_string(), + retained_object_fields( + aux_data, + &["frameId", "isDefault", "type"], + ), + ); + } + } + context + } + _ => value.clone(), + }; + retained.insert((*field).to_string(), value); + } + } + if method == "Network.requestWillBeSent" { + if let Some(request) = object.get("request") { + retained.insert( + "request".to_string(), + retained_object_fields( + request, + &["url", "method", "initialPriority", "referrerPolicy"], + ), + ); + } + } else if method == "Network.responseReceived" { + if let Some(response) = object.get("response") { + retained.insert( + "response".to_string(), + retained_object_fields( + response, + &[ + "url", + "status", + "statusText", + "mimeType", + "protocol", + "securityState", + "encodedDataLength", + ], + ), + ); + } + } + } + (!retained.is_empty()).then_some(Value::Object(retained)) +} + +fn compact_retained_cdp_event(mut event: Value) -> (Value, usize) { + let original_size = retained_cdp_event_size(&event); + if original_size <= CDP_EVENT_LOG_MAX_ENTRY_BYTES { + return (event, original_size); + } + compact_retained_cdp_value(&mut event); + if retained_cdp_event_size(&event) > CDP_EVENT_LOG_MAX_ENTRY_BYTES { + let mut retained = serde_json::Map::new(); + for key in ["sessionId", "method", "__rustwright_cdp_event_seq"] { + if let Some(value) = event.get(key) { + retained.insert(key.to_string(), value.clone()); + } + } + let method = event.get("method").and_then(Value::as_str).unwrap_or(""); + if let Some(mut params) = event + .get("params") + .and_then(|params| retained_cdp_event_params(method, params)) + { + compact_retained_cdp_value(&mut params); + let mut candidate = retained.clone(); + candidate.insert("params".to_string(), params.clone()); + if retained_cdp_event_size(&Value::Object(candidate)) <= CDP_EVENT_LOG_MAX_ENTRY_BYTES { + retained.insert("params".to_string(), params); + } + } + retained.insert(CDP_EVENT_UNREPLAYABLE_MARKER.to_string(), Value::Bool(true)); + event = Value::Object(retained); + } + let retained_size = retained_cdp_event_size(&event); + (event, retained_size) +} + +fn is_unreplayable_cdp_event(event: &Value) -> bool { + event + .get(CDP_EVENT_UNREPLAYABLE_MARKER) + .and_then(Value::as_bool) + .unwrap_or(false) +} + +struct CdpRetentionGate { + write_lock: Mutex<()>, + enabled: AtomicBool, +} + +impl CdpRetentionGate { + fn new() -> Self { + Self { + write_lock: Mutex::new(()), + enabled: AtomicBool::new(true), + } + } + + fn lock_for_write(&self) -> Option> { + let guard = self.write_lock.lock().unwrap(); + self.enabled.load(Ordering::SeqCst).then_some(guard) + } + fn lock_for_release(&self) -> MutexGuard<'_, ()> { + self.write_lock.lock().unwrap() + } + + fn disable(&self) { + let _guard = self.write_lock.lock().unwrap(); + self.enabled.store(false, Ordering::SeqCst); + } + + fn is_enabled(&self) -> bool { + self.enabled.load(Ordering::SeqCst) + } +} + +struct CdpEventLogEntry { + sequence: u64, + event: Value, + retained_bytes: usize, +} + struct CdpEventLog { next_seq: u64, - events: VecDeque<(u64, Value)>, + events: VecDeque, + retained_bytes: usize, + retention_gate: Arc, action_dispatch_receipts: HashMap, console_replay_cutoffs: HashMap, console_replay_cutoff_generations: HashMap, @@ -24928,6 +25773,8 @@ impl CdpEventLog { Self { next_seq: 0, events: VecDeque::with_capacity(CDP_EVENT_LOG_LIMIT), + retained_bytes: 0, + retention_gate: Arc::new(CdpRetentionGate::new()), action_dispatch_receipts: HashMap::new(), console_replay_cutoff_cancellations: HashMap::new(), console_replay_cutoffs: HashMap::new(), @@ -25023,9 +25870,20 @@ impl CdpEventLog { Value::Number(seq.into()), ); } - self.events.push_back((seq, event.clone())); - while self.events.len() > CDP_EVENT_LOG_LIMIT { - self.events.pop_front(); + let (retained_event, retained_bytes) = compact_retained_cdp_event(event.clone()); + self.retained_bytes = self.retained_bytes.saturating_add(retained_bytes); + self.events.push_back(CdpEventLogEntry { + sequence: seq, + event: retained_event, + retained_bytes, + }); + while self.events.len() > CDP_EVENT_LOG_LIMIT + || self.retained_bytes > CDP_EVENT_LOG_MAX_BYTES + { + let Some(evicted) = self.events.pop_front() else { + break; + }; + self.retained_bytes = self.retained_bytes.saturating_sub(evicted.retained_bytes); } // Cutoff pruning is low-water driven, not part of the hot event path. A bounded batch // keeps the map inexpensive under high-volume Runtime/Network traffic. @@ -25079,6 +25937,15 @@ impl CdpEventLog { result } + fn clear_retained(&mut self) { + self.events = VecDeque::new(); + self.retained_bytes = 0; + self.action_dispatch_receipts = HashMap::new(); + self.console_replay_cutoffs = HashMap::new(); + self.console_replay_cutoff_generations = HashMap::new(); + self.console_replay_cutoff_cancellations = HashMap::new(); + } + fn entries_since(&self, cursor: u64) -> Vec<(u64, Value)> { self.entries_since_limited(cursor, usize::MAX) } @@ -25090,7 +25957,7 @@ impl CdpEventLog { .iter() .skip(start) .take(limit) - .map(|(seq, event)| (*seq, event.clone())) + .map(|entry| (entry.sequence, entry.event.clone())) .collect() } @@ -25101,22 +25968,27 @@ impl CdpEventLog { .iter() .skip(start_index) .take(end.saturating_sub(start) as usize) - .map(|(seq, event)| (*seq, event.clone())) + .map(|entry| (entry.sequence, entry.event.clone())) .collect() } - fn cursor_after_event(&self, cursor: u64, event: &Value) -> u64 { + if let Some(sequence) = cdp_event_sequence(event) { + return sequence + .checked_add(1) + .filter(|next| *next > cursor) + .unwrap_or(cursor); + } self.events .iter() - .find(|(seq, candidate)| *seq >= cursor && candidate == event) - .map(|(seq, _)| seq.wrapping_add(1)) + .find(|entry| entry.sequence >= cursor && entry.event == *event) + .map(|entry| entry.sequence.wrapping_add(1)) .unwrap_or(cursor) } fn oldest_seq(&self) -> u64 { self.events .front() - .map(|(seq, _)| *seq) + .map(|entry| entry.sequence) .unwrap_or(self.next_seq) } } @@ -25131,6 +26003,10 @@ fn dispatch_cdp_payload_with_diagnostics( runtime_state: Arc>, playwright_like_first_reply: Option<&AtomicBool>, ) { + let retention_gate = event_log.lock().unwrap().retention_gate.clone(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + return; + }; if let Some(id) = payload.get("id").and_then(Value::as_u64) { let sender = pending.lock().unwrap().remove(&id); let command = outstanding.lock().unwrap().remove(&id); @@ -25219,7 +26095,12 @@ fn close_pending_cdp_commands( pending: CdpPendingMap, outstanding: CdpOutstandingMap, traffic_log: Arc>, + event_log: Arc>, ) { + let retention_gate = event_log.lock().unwrap().retention_gate.clone(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + return; + }; let pending_commands = { let mut pending = pending.lock().unwrap(); pending.drain().collect::>() @@ -25547,6 +26428,7 @@ impl CdpClient { let (events, _) = broadcast::channel(4096); let events_reader = events.clone(); let event_log = Arc::new(Mutex::new(CdpEventLog::new())); + let event_log_writer = Arc::clone(&event_log); let event_log_reader = Arc::clone(&event_log); let traffic_log = Arc::new(Mutex::new(CdpTrafficLog::new())); let traffic_log_writer = Arc::clone(&traffic_log); @@ -25594,10 +26476,14 @@ impl CdpClient { } if written { if let Some(diagnostic) = diagnostic { - traffic_log_writer - .lock() - .unwrap() - .push(diagnostic.traffic_entry("sent")); + let retention_gate = + event_log_writer.lock().unwrap().retention_gate.clone(); + if let Some(_retention_guard) = retention_gate.lock_for_write() { + traffic_log_writer + .lock() + .unwrap() + .push(diagnostic.traffic_entry("sent")); + }; } } if !written { @@ -25639,7 +26525,12 @@ impl CdpClient { } alive_reader.store(false, Ordering::SeqCst); alive_tx_reader.send_replace(false); - close_pending_cdp_commands(pending_reader, outstanding_reader, traffic_log_reader); + close_pending_cdp_commands( + pending_reader, + outstanding_reader, + traffic_log_reader, + event_log_reader, + ); }); Ok(Arc::new(Self { @@ -25675,6 +26566,7 @@ impl CdpClient { let (events, _) = broadcast::channel(4096); let events_dispatcher = events.clone(); let event_log = Arc::new(Mutex::new(CdpEventLog::new())); + let event_log_writer = Arc::clone(&event_log); let event_log_dispatcher = Arc::clone(&event_log); let traffic_log = Arc::new(Mutex::new(CdpTrafficLog::new())); let traffic_log_writer = Arc::clone(&traffic_log); @@ -25720,10 +26612,14 @@ impl CdpClient { } if written { if let Some(diagnostic) = diagnostic { - traffic_log_writer - .lock() - .unwrap() - .push(diagnostic.traffic_entry("sent")); + let retention_gate = + event_log_writer.lock().unwrap().retention_gate.clone(); + if let Some(_retention_guard) = retention_gate.lock_for_write() { + traffic_log_writer + .lock() + .unwrap() + .push(diagnostic.traffic_entry("sent")); + }; } } if !written { @@ -25782,6 +26678,7 @@ impl CdpClient { pending_dispatcher, outstanding_dispatcher, traffic_log_dispatcher, + event_log_dispatcher, ); }); @@ -25822,6 +26719,13 @@ impl CdpClient { dispatch_id: &str, token: &str, ) -> RwResult> { + let retention_gate = self.retention_gate(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + return Err(RwError::Disconnected); + }; + if !self.is_connected() { + return Err(RwError::Disconnected); + } let registered = self.event_log.lock().unwrap().register_action_dispatch( session_id, dispatch_id.to_string(), @@ -25862,6 +26766,14 @@ impl CdpClient { self.alive.load(Ordering::SeqCst) } + fn retention_gate(&self) -> Arc { + self.event_log.lock().unwrap().retention_gate.clone() + } + + fn retention_enabled(&self) -> bool { + self.retention_gate().is_enabled() + } + fn close(&self) { self.alive.store(false, Ordering::SeqCst); self.alive_tx.send_replace(false); @@ -25873,6 +26785,17 @@ impl CdpClient { self.alive_tx.send_replace(false); } + fn release_memory_buffers(&self) { + // Serialize terminal release with every dispatcher and page-event write. Once disabled, + // no reader or in-flight handler can repopulate the stores after this clear. + self.retention_gate().disable(); + self.event_log.lock().unwrap().clear_retained(); + self.traffic_log.lock().unwrap().clear_for_close(); + self.runtime_state.lock().unwrap().clear_for_close(); + self.pending.lock().unwrap().clear(); + self.outstanding.lock().unwrap().clear(); + } + fn record_sent_command(&self, method: &str) { if method == "Page.getFrameTree" { self.sent_get_frame_tree_count @@ -25992,6 +26915,10 @@ impl CdpClient { session_id: Option<&str>, timeout: Duration, ) -> RwResult { + let retention_gate = self.retention_gate(); + let retention_guard = retention_gate + .lock_for_write() + .ok_or(RwError::Disconnected)?; if !self.is_connected() { return Err(RwError::Disconnected); } @@ -26008,8 +26935,13 @@ impl CdpClient { write_state: Arc::new(CdpWriteState::new()), }, ); - let pending_guard = - PendingCommandGuard::new(id, &self.pending, &self.outstanding, &self.traffic_log); + let pending_guard = PendingCommandGuard::new( + id, + &self.pending, + &self.outstanding, + &self.traffic_log, + &retention_gate, + ); let mut payload = json!({ "id": id, @@ -26033,10 +26965,12 @@ impl CdpClient { }) }; if send_result.is_err() { + drop(retention_guard); self.mark_closed(); return Err(RwError::Disconnected); } self.record_sent_command(method); + drop(retention_guard); Ok(QueuedCdpCommand { method: method.to_string(), timeout, @@ -26115,6 +27049,10 @@ impl CdpClient { tracker: Option, event_cursor: Option<&mut u64>, ) -> RwResult { + let retention_gate = self.retention_gate(); + let retention_guard = retention_gate + .lock_for_write() + .ok_or(RwError::Disconnected)?; if !self.is_connected() { return Err(RwError::Disconnected); } @@ -26135,8 +27073,13 @@ impl CdpClient { write_state, }, ); - let _pending_guard = - PendingCommandGuard::new(id, &self.pending, &self.outstanding, &self.traffic_log); + let _pending_guard = PendingCommandGuard::new( + id, + &self.pending, + &self.outstanding, + &self.traffic_log, + &retention_gate, + ); let mut payload = json!({ "id": id, @@ -26166,10 +27109,12 @@ impl CdpClient { None => self.write_tx.send(outgoing), }; if send_result.is_err() { + drop(retention_guard); self.mark_closed(); return Err(RwError::Disconnected); } self.record_sent_command(method); + drop(retention_guard); match tokio::time::timeout(timeout, rx).await { Ok(Ok(result)) => result.map_err(|error| wrap_cdp_command_error(method, error)), @@ -26185,9 +27130,18 @@ impl CdpClient { session_id: Option<&str>, timeout: Duration, ) -> RwResult { + let retention_gate = self.retention_gate(); + let retention_guard = retention_gate + .lock_for_write() + .ok_or(RwError::Disconnected)?; if !self.is_connected() { return Err(RwError::Disconnected); } + let method_json = serde_json::to_string(method)?; + let session_id_json = match session_id { + Some(session_id) => Some(serde_json::to_string(session_id)?), + None => None, + }; let id = self.next_id.fetch_add(1, Ordering::SeqCst); let (tx, rx) = oneshot::channel(); self.pending.lock().unwrap().insert(id, tx); @@ -26201,12 +27155,15 @@ impl CdpClient { write_state: Arc::new(CdpWriteState::new()), }, ); - let _pending_guard = - PendingCommandGuard::new(id, &self.pending, &self.outstanding, &self.traffic_log); + let _pending_guard = PendingCommandGuard::new( + id, + &self.pending, + &self.outstanding, + &self.traffic_log, + &retention_gate, + ); - let method_json = serde_json::to_string(method)?; - let payload = if let Some(session_id) = session_id { - let session_id_json = serde_json::to_string(session_id)?; + let payload = if let Some(session_id_json) = &session_id_json { format!( "{{\"id\":{id},\"method\":{method_json},\"params\":{params_json},\"sessionId\":{session_id_json}}}" ) @@ -26223,10 +27180,12 @@ impl CdpClient { }) .is_err() { + drop(retention_guard); self.mark_closed(); return Err(RwError::Disconnected); } self.record_sent_command(method); + drop(retention_guard); match tokio::time::timeout(timeout, rx).await { Ok(Ok(result)) => result.map_err(|error| match error { @@ -26248,14 +27207,18 @@ impl CdpClient { session_id: Option<&str>, timeout: Duration, ) -> RwResult> { - if !self.is_connected() { - return Err(RwError::Disconnected); - } let method_json = serde_json::to_string(method)?; let session_id_json = match session_id { Some(session_id) => Some(serde_json::to_string(session_id)?), None => None, }; + let retention_gate = self.retention_gate(); + let retention_guard = retention_gate + .lock_for_write() + .ok_or(RwError::Disconnected)?; + if !self.is_connected() { + return Err(RwError::Disconnected); + } let mut receivers = Vec::with_capacity(params_json_list.len()); for params_json in params_json_list { let id = self.next_id.fetch_add(1, Ordering::SeqCst); @@ -26271,8 +27234,13 @@ impl CdpClient { write_state: Arc::new(CdpWriteState::new()), }, ); - let pending_guard = - PendingCommandGuard::new(id, &self.pending, &self.outstanding, &self.traffic_log); + let pending_guard = PendingCommandGuard::new( + id, + &self.pending, + &self.outstanding, + &self.traffic_log, + &retention_gate, + ); let payload = if let Some(session_id_json) = &session_id_json { format!( @@ -26291,12 +27259,14 @@ impl CdpClient { }) .is_err() { + drop(retention_guard); self.mark_closed(); return Err(RwError::Disconnected); } self.record_sent_command(method); receivers.push((id, rx, pending_guard)); } + drop(retention_guard); let mut results = Vec::with_capacity(receivers.len()); for (_id, rx, _pending_guard) in receivers { @@ -27071,6 +28041,7 @@ async fn close_browser_cleanup(browser: Arc) -> RwResult<()> { .await; } browser.client.close(); + browser.client.release_memory_buffers(); browser.attached_pages.clear(); tokio::task::spawn_blocking(move || -> RwResult<()> { @@ -27411,7 +28382,7 @@ struct PageInner { #[derive(Default)] struct ConsoleCaptureState { requested: AtomicBool, - enabled_sessions: tokio::sync::Mutex>, + enabled_sessions: Arc>>, } struct CreatedTargetGuard { @@ -27584,12 +28555,57 @@ impl PageInner { self.iframe_setup_tasks.abort_all(); } fn clear_worker_resume_handoffs(&self) { + let retention_gate = self.browser.client.retention_gate(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + return; + }; + self.clear_worker_resume_handoffs_locked(); + } + + fn clear_worker_resume_handoffs_locked(&self) { self.frame_state .lock() .unwrap() .clear_worker_resume_handoffs(); } + fn release_memory_buffers(&self) { + self.target_closed.store(true, Ordering::SeqCst); + let retention_gate = self.browser.client.retention_gate(); + let _retention_guard = retention_gate.lock_for_release(); + self.network_requests.lock().unwrap().clear_for_close(); + self.native_network_records + .lock() + .unwrap() + .clear_for_close(); + self.console_records.lock().unwrap().clear_for_close(); + self.console_replay_until_event_cursor + .lock() + .unwrap() + .clear(); + self.frame_state.lock().unwrap().clear_for_close(); + self.console_capture + .requested + .store(false, Ordering::SeqCst); + let enabled_sessions = Arc::clone(&self.console_capture.enabled_sessions); + let sessions_cleared = match enabled_sessions.try_lock() { + Ok(mut enabled_sessions_guard) => { + enabled_sessions_guard.clear(); + true + } + Err(_) => false, + }; + if !sessions_cleared { + if let Some(runtime) = self.browser.runtime.0.as_ref() { + runtime.spawn(async move { + enabled_sessions.lock().await.clear(); + }); + } + } + self.background_override_active + .store(false, Ordering::SeqCst); + } + fn close_in_background(&self) { self.clear_worker_resume_handoffs(); self.abort_iframe_setup_tasks(); @@ -27693,6 +28709,9 @@ impl NetworkRequestStore { self.reset_generation = self.reset_clock.fetch_add(1, Ordering::SeqCst) + 1; self.reset_cursor = next_applied_seq; } + fn clear_for_close(&mut self) { + self.reset_after_overflow(self.next_applied_seq); + } fn record_applied_request(&mut self, seq: u64, request_id: String) { self.applied_order.push_back((seq, request_id)); @@ -27864,6 +28883,21 @@ struct ConsoleRecordStore { } impl ConsoleRecordStore { + fn clear_for_close(&mut self) { + self.accepted_navigation_epochs = AcceptedNavigationEpochs::default(); + self.navigation_epoch = 0; + self.records = VecDeque::new(); + self.evictions_by_epoch = HashMap::new(); + self.evictions_total = 0; + } + fn reset_after_unreplayable(&mut self) { + let epoch = self.navigation_epoch; + self.accepted_navigation_epochs = AcceptedNavigationEpochs::default(); + self.records.clear(); + self.evictions_by_epoch.clear(); + self.evictions_by_epoch.insert(epoch, 1); + self.evictions_total = self.evictions_total.saturating_add(1); + } fn record_navigation(&mut self, sequence: Option, epoch: u64) -> bool { if !self.accepted_navigation_epochs.accepts(sequence) { return false; @@ -27995,6 +29029,23 @@ impl NativeNetworkRecordStore { evictions_by_epoch: HashMap::new(), } } + fn clear_for_close(&mut self) { + self.accepted_navigation_epochs = AcceptedNavigationEpochs::default(); + self.navigation_epoch = 0; + self.navigation_start_index = 0; + self.current_loader_id = None; + self.records = VecDeque::new(); + self.active_by_request = HashMap::new(); + self.evictions_by_epoch = HashMap::new(); + } + fn reset_after_unreplayable(&mut self) { + let epoch = self.navigation_epoch; + self.accepted_navigation_epochs = AcceptedNavigationEpochs::default(); + self.records.clear(); + self.active_by_request.clear(); + self.evictions_by_epoch.clear(); + self.evictions_by_epoch.insert(epoch, 1); + } fn begin_document_navigation( &mut self, @@ -28431,6 +29482,42 @@ impl PageFrameState { session_updates, } } + fn clear_for_close(&mut self) { + self.frame_sessions = HashMap::new(); + self.worker_sessions = HashSet::new(); + self.worker_target_sessions = HashMap::new(); + self.worker_session_event_sequences = HashMap::new(); + self.worker_resume_handoffs = HashMap::new(); + self.frame_attachment_generations = HashMap::new(); + self.frame_swaps = HashMap::new(); + self.frame_session_waiters = HashMap::new(); + self.frame_attachment_locks = HashMap::new(); + self.session_frames = HashMap::new(); + self.frames = HashMap::new(); + self.child_order = HashMap::new(); + self.frame_loader_ids = HashMap::new(); + self.execution_contexts = HashMap::new(); + self.frame_session_errors = HashMap::new(); + self.iframe_sessions_armed = HashSet::new(); + self.iframe_sessions_routable = HashSet::new(); + self.iframe_sessions_ready = HashSet::new(); + self.iframe_setup_started = HashSet::new(); + self.page_domain_enabled_sessions = HashSet::new(); + self.frame_tree_populated_sessions = HashSet::new(); + self.frame_tree_dirty_sessions = HashSet::new(); + self.frame_tree_generations = HashMap::new(); + self.frame_invalidation_sequences = HashMap::new(); + self.worker_forwarding_interest = false; + self.worker_event_interest = false; + self.worker_auto_attach_requested = false; + self.frame_event_listener_registered = false; + self.frame_ownership_generation = 0; + self.frame_event_cursor = 0; + #[cfg(test)] + { + self.locator_resolution_reresolve_count = 0; + } + } fn set_frame_event_cursor(&mut self, cursor: u64) { self.frame_event_cursor = cursor; @@ -30077,6 +31164,7 @@ struct PyWorker { #[pyclass(name = "_NetworkEventWaiter")] struct PyNetworkEventWaiter { browser: Arc, + page: Arc, receiver: Mutex>>, event_log: Arc>, cursor: Mutex, @@ -30125,6 +31213,7 @@ struct PageEventStreamLease { cursor_slot: Arc>, requests: Arc>, working_requests: Arc>, + page: Option>, } #[cfg(feature = "python")] @@ -30154,6 +31243,18 @@ impl PageEventStreamLease { fn deliver(mut self) { self.delivered = true; } + + fn commit_delivered(&mut self) { + *self.state_slot.lock().unwrap() = self.state.take(); + *self.cursor_slot.lock().unwrap() = self.cursor; + let committed = self.working_requests.lock().unwrap().clone(); + self.requests.lock().unwrap().merge_committed(&committed); + } + + fn restore_rollback(&mut self) { + *self.state_slot.lock().unwrap() = self.rollback_state.take(); + *self.cursor_slot.lock().unwrap() = self.rollback_cursor; + } } #[cfg(feature = "python")] @@ -30161,13 +31262,22 @@ impl Drop for PageEventStreamLease { fn drop(&mut self) { *self.receiver_slot.lock().unwrap() = self.receiver.take(); if self.delivered { - *self.state_slot.lock().unwrap() = self.state.take(); - *self.cursor_slot.lock().unwrap() = self.cursor; - let committed = self.working_requests.lock().unwrap().clone(); - self.requests.lock().unwrap().merge_committed(&committed); + if let Some(page) = self.page.as_ref().map(Arc::clone) { + let retention_gate = page.browser.client.retention_gate(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + self.restore_rollback(); + return; + }; + if page.lifecycle.is_closing_or_closed() + || page.target_closed.load(Ordering::SeqCst) + { + self.restore_rollback(); + return; + } + } + self.commit_delivered(); } else { - *self.state_slot.lock().unwrap() = self.rollback_state.take(); - *self.cursor_slot.lock().unwrap() = self.rollback_cursor; + self.restore_rollback(); } } } @@ -30240,11 +31350,37 @@ impl PageEventStreamState { let replay_generations = event_log.console_replay_cutoff_generations.clone(); let replay_cancellations = event_log.console_replay_cutoff_cancellations.clone(); drop(event_log); + let retention_gate = client.retention_gate(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + return (receiver, event_cursor, PageEventStreamState::for_page(page)); + }; + if page.lifecycle.is_closing_or_closed() || page.target_closed.load(Ordering::SeqCst) { + return (receiver, event_cursor, PageEventStreamState::for_page(page)); + } let mut page_state = PageEventStreamState::for_page(page); for (sequence, event) in entries { if sequence >= event_cursor { break; } + if is_unreplayable_cdp_event(&event) { + page_state.reset_after_overflow(); + page.network_requests + .lock() + .unwrap() + .reset_after_overflow(sequence.saturating_add(1)); + page_state.ready.insert( + sequence, + page_event_envelope( + sequence, + "_overflow", + json!({ + "dropped": 1, + "reason": "unreplayable_event", + }), + ), + ); + continue; + } page_state.apply_target_transition(&event, &page.session_id); } page_state.seed_replay_cutoffs(&replay_cutoffs); @@ -34904,10 +36040,12 @@ impl Drop for FrameAttachmentLockLease { let Some(page) = self.page.upgrade() else { return; }; - page.frame_state - .lock() - .unwrap() - .reclaim_attachment_lock(&self.frame_id, &self.attachment_lock); + let _ = with_live_page_retention(&page, || { + page.frame_state + .lock() + .unwrap() + .reclaim_attachment_lock(&self.frame_id, &self.attachment_lock); + }); } } @@ -34968,14 +36106,18 @@ async fn attach_iframe_target_for_frame( let Some(session_id) = attached.get("sessionId").and_then(Value::as_str) else { return Ok(None); }; - let expected_session = match claim_manual_attached_iframe_session( + let Some(claim) = claim_manual_attached_iframe_session( &page, frame_id, parent_frame_id, session_id, target_name, target_url, - ) { + ) else { + detach_session_best_effort(Arc::clone(&page.browser), session_id.to_string()); + return Err(RwError::TargetClosed(TargetClosedKind::Page)); + }; + let expected_session = match claim { FrameSessionOwnershipClaim::Claimed(pin) => { spawn_attached_iframe_session_initialization( &page, @@ -35232,7 +36374,7 @@ async fn wait_for_frame_session( expected_session: Option, deadline: OperationDeadline, ) -> RwResult> { - let (mut updates, mut expected_session) = { + let Some((mut updates, mut expected_session)) = with_live_page_retention(page, || { let mut state = page.frame_state.lock().unwrap(); let expected_session = expected_session.or_else(|| { state @@ -35243,6 +36385,8 @@ async fn wait_for_frame_session( state.register_frame_session_waiter(frame_id, pin.generation); } (state.subscribe_session_updates(), expected_session) + }) else { + return Err(RwError::TargetClosed(TargetClosedKind::Page)); }; let mut _waiter_registration = expected_session @@ -35253,7 +36397,7 @@ async fn wait_for_frame_session( generation: pin.generation, }); loop { - let outcome = { + let Some(outcome) = with_live_page_retention(page, || { let mut state = page.frame_state.lock().unwrap(); if expected_session.is_none() { if let Some(pin) = state @@ -35283,6 +36427,8 @@ async fn wait_for_frame_session( .routable_session_pin_for_frame(frame_id, different_from_session_id) .map(|pin| Ok(Some(pin.session_id))) } + }) else { + return Err(RwError::TargetClosed(TargetClosedKind::Page)); }; if let Some(outcome) = outcome { return outcome; @@ -35308,10 +36454,12 @@ struct FrameSessionWaiterRegistration { impl Drop for FrameSessionWaiterRegistration { fn drop(&mut self) { if let Some(page) = self.page.upgrade() { - page.frame_state - .lock() - .unwrap() - .unregister_frame_session_waiter(&self.frame_id, self.generation); + let _ = with_live_page_retention(&page, || { + page.frame_state + .lock() + .unwrap() + .unregister_frame_session_waiter(&self.frame_id, self.generation); + }); } } } @@ -37378,6 +38526,7 @@ async fn page_close_cleanup( Arc::as_ptr(&page), ); page.close_target_on_drop.store(false, Ordering::SeqCst); + page.release_memory_buffers(); Ok(()) } @@ -40229,6 +41378,7 @@ return win.__rustwrightCleanupDrag ? win.__rustwrightCleanupDrag() : false; let client = Arc::clone(&self.inner.browser.client); Ok(PyNetworkEventWaiter { browser: Arc::clone(&self.inner.browser), + page: Arc::clone(&self.inner), receiver: Mutex::new(Some(client.subscribe())), event_log: Arc::clone(&client.event_log), cursor: Mutex::new(client.event_cursor()), @@ -40615,7 +41765,11 @@ return win.__rustwrightCleanupDrag ? win.__rustwrightCleanupDrag() : false; let timeout = BrowserInner::command_timeout(timeout_ms); browser .block_on(async move { - register_page_owned_workers_from_event_log(&page_for_task).await; + if register_page_owned_workers_from_event_log(&page_for_task).await { + return Err(RwError::Message( + "CDP event log contains unreplayable worker history".to_string(), + )); + } let result = client .send("Target.getTargets", json!({}), None, timeout) .await?; @@ -41488,6 +42642,7 @@ impl PyNetworkEventWaiter { .take() .ok_or_else(|| PyRuntimeError::new_err("network waiter is already waiting"))?; let browser = Arc::clone(&self.browser); + let page = Arc::clone(&self.page); let session_id = self.session_id.clone(); let kind = self.kind.clone(); let requests = Arc::clone(&self.requests); @@ -41502,6 +42657,7 @@ impl PyNetworkEventWaiter { &session_id, &kind, requests, + page, timeout, )); (result, receiver, cursor) @@ -41686,6 +42842,7 @@ impl PyPageEventStream { cursor_slot: Arc::clone(&self.cursor), requests: Arc::clone(&self.requests), working_requests: Arc::clone(&working_requests), + page: Some(Arc::clone(&self.page)), }; let closed = Arc::clone(&self.closed); let pending_batch = Arc::clone(&self.pending_batch); @@ -43876,6 +45033,11 @@ impl RustwrightPage { } for (sequence, event) in entries { cursor = sequence.saturating_add(1); + if is_unreplayable_cdp_event(&event) { + reset_page_history_after_unreplayable(&page, sequence); + task_queue.record_upstream_drop(1); + continue; + } if let Some(detail) = native_navigation_detail_from_cdp(&page, &event) { // `sequence + 1` uses the same convention as the event // log cursor: all events ingressed before cursor C have @@ -47217,22 +48379,24 @@ fn claim_manual_attached_iframe_session( child_session_id: &str, target_name: Option, target_url: Option, -) -> FrameSessionOwnershipClaim { - let mut state = page.frame_state.lock().unwrap(); - match state.claim_manual_session_for_frame(frame_id, child_session_id) { - FrameSessionOwnershipClaim::Existing(pin) => FrameSessionOwnershipClaim::Existing(pin), - FrameSessionOwnershipClaim::Claimed(pin) => { - state.record_frame( - frame_id.to_string(), - parent_frame_id, - None, - None, - child_session_id.to_string(), - ); - state.seed_attached_frame_metadata(frame_id, &pin, target_name, target_url); - FrameSessionOwnershipClaim::Claimed(pin) +) -> Option { + with_live_page_retention(page, || { + let mut state = page.frame_state.lock().unwrap(); + match state.claim_manual_session_for_frame(frame_id, child_session_id) { + FrameSessionOwnershipClaim::Existing(pin) => FrameSessionOwnershipClaim::Existing(pin), + FrameSessionOwnershipClaim::Claimed(pin) => { + state.record_frame( + frame_id.to_string(), + parent_frame_id, + None, + None, + child_session_id.to_string(), + ); + state.seed_attached_frame_metadata(frame_id, &pin, target_name, target_url); + FrameSessionOwnershipClaim::Claimed(pin) + } } - } + }) } #[cfg(test)] @@ -47267,12 +48431,12 @@ fn register_attached_iframe_session_at_sequence( event_sequence: Option, timeout: Duration, ) -> RwResult> { - let session_pin = { + let Some((session_pin, previous_generation)) = with_live_page_retention(&page, || { let mut state = page.frame_state.lock().unwrap(); if !state.owns_session(&parent_session_id) || !state.mark_frame_cache_dirty_at_sequence(&parent_session_id, event_sequence) { - return Ok(None); + return None; } let previous_generation = state .session_pin_for_frame(&frame_id) @@ -47289,13 +48453,15 @@ fn register_attached_iframe_session_at_sequence( None, child_session_id, ); - drop(state); - if previous_generation.is_some_and(|generation| generation != session_pin.generation) { - page.iframe_setup_tasks - .abort_generations(previous_generation); - } - session_pin + Some((session_pin, previous_generation)) + }) + .flatten() else { + return Ok(None); }; + if previous_generation.is_some_and(|generation| generation != session_pin.generation) { + page.iframe_setup_tasks + .abort_generations(previous_generation); + } spawn_attached_iframe_session_initialization(&page, frame_id, session_pin.clone(), timeout); Ok(Some(session_pin)) } @@ -47371,6 +48537,13 @@ fn enqueue_focus_emulation( async fn clear_console_capture_for_session(page: &PageInner, session_id: &str) { let mut enabled_sessions = page.console_capture.enabled_sessions.lock().await; + let retention_gate = page.browser.client.retention_gate(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + return; + }; + if page.lifecycle.is_closing_or_closed() || page.target_closed.load(Ordering::SeqCst) { + return; + } enabled_sessions.remove(session_id); page.console_replay_until_event_cursor .lock() @@ -47378,7 +48551,7 @@ async fn clear_console_capture_for_session(page: &PageInner, session_id: &str) { .remove(session_id); } -async fn register_page_owned_iframes_from_event_log(page: &PageInner) { +async fn register_page_owned_iframes_from_event_log(page: &PageInner) -> bool { let events = page .browser .client @@ -47386,10 +48559,15 @@ async fn register_page_owned_iframes_from_event_log(page: &PageInner) { .lock() .unwrap() .entries_since(0); - let detached_sessions = { + let Some((detached_sessions, unreplayable_sequence)) = with_live_page_retention(page, || { let mut state = page.frame_state.lock().unwrap(); let mut detached_sessions = Vec::new(); + let mut unreplayable_sequence = None; for (sequence, event) in events { + if is_unreplayable_cdp_event(&event) { + unreplayable_sequence = Some(sequence); + break; + } match event.get("method").and_then(Value::as_str) { Some("Target.attachedToTarget") => { let target_info = event.pointer("/params/targetInfo").unwrap_or(&Value::Null); @@ -47450,8 +48628,14 @@ async fn register_page_owned_iframes_from_event_log(page: &PageInner) { _ => {} } } - detached_sessions + (detached_sessions, unreplayable_sequence) + }) else { + return true; }; + if let Some(sequence) = unreplayable_sequence { + reset_page_history_after_unreplayable(page, sequence); + return true; + } for (session_id, cutoff) in detached_sessions { page.browser .client @@ -47461,9 +48645,10 @@ async fn register_page_owned_iframes_from_event_log(page: &PageInner) { .finalize_console_replay_cutoff_on_detach(&session_id, cutoff); clear_console_capture_for_session(page, &session_id).await; } + false } -async fn register_page_owned_workers_from_event_log(page: &PageInner) { +async fn register_page_owned_workers_from_event_log(page: &PageInner) -> bool { let events = page .browser .client @@ -47471,10 +48656,15 @@ async fn register_page_owned_workers_from_event_log(page: &PageInner) { .lock() .unwrap() .entries_since(0); - let detached_sessions = { + let Some((detached_sessions, unreplayable_sequence)) = with_live_page_retention(page, || { let mut state = page.frame_state.lock().unwrap(); let mut detached_sessions = Vec::new(); + let mut unreplayable_sequence = None; for (sequence, event) in events { + if is_unreplayable_cdp_event(&event) { + unreplayable_sequence = Some(sequence); + break; + } match event.get("method").and_then(Value::as_str) { Some("Target.attachedToTarget") => { let target_info = event.pointer("/params/targetInfo").unwrap_or(&Value::Null); @@ -47520,8 +48710,14 @@ async fn register_page_owned_workers_from_event_log(page: &PageInner) { _ => {} } } - detached_sessions + (detached_sessions, unreplayable_sequence) + }) else { + return true; }; + if let Some(sequence) = unreplayable_sequence { + reset_page_history_after_unreplayable(page, sequence); + return true; + } for (session_id, cutoff) in detached_sessions { page.browser .client @@ -47531,6 +48727,7 @@ async fn register_page_owned_workers_from_event_log(page: &PageInner) { .finalize_console_replay_cutoff_on_detach(&session_id, cutoff); clear_console_capture_for_session(page, &session_id).await; } + false } async fn enable_console_capture( @@ -47538,9 +48735,20 @@ async fn enable_console_capture( timeout: Duration, ) -> RwResult> { let deadline = Instant::now() + timeout; - page.console_capture.requested.store(true, Ordering::SeqCst); - register_page_owned_iframes_from_event_log(page).await; - register_page_owned_workers_from_event_log(page).await; + if with_live_page_retention(page, || { + page.console_capture.requested.store(true, Ordering::SeqCst); + }) + .is_none() + { + return Err(RwError::TargetClosed(TargetClosedKind::Page)); + } + if register_page_owned_iframes_from_event_log(page).await + || register_page_owned_workers_from_event_log(page).await + { + return Err(RwError::Message( + "CDP event log contains unreplayable console history".to_string(), + )); + } let sessions = { let frame_state = page.frame_state.lock().unwrap(); let mut sessions = frame_state.all_owned_session_ids(); @@ -47592,6 +48800,13 @@ async fn enable_console_capture_for_session( // single-flight and records the session before any later console_records() // call can send the detectable command a second time. let mut enabled_sessions = page.console_capture.enabled_sessions.lock().await; + let retention_gate = page.browser.client.retention_gate(); + let Some(retention_guard) = retention_gate.lock_for_write() else { + return Err(RwError::TargetClosed(TargetClosedKind::Page)); + }; + if page.lifecycle.is_closing_or_closed() || page.target_closed.load(Ordering::SeqCst) { + return Err(RwError::TargetClosed(TargetClosedKind::Page)); + } if enabled_sessions.contains(session_id) { return Ok(page.browser.client.event_cursor()); } @@ -47609,6 +48824,7 @@ async fn enable_console_capture_for_session( .lock() .unwrap() .insert(session_id.to_owned(), u64::MAX); + drop(retention_guard); let replay_until = match page .browser .client @@ -47617,32 +48833,45 @@ async fn enable_console_capture_for_session( { Ok(_) => { let cutoff = page.browser.client.event_cursor(); - page.browser - .client - .event_log - .lock() - .unwrap() - .finalize_console_replay_cutoff(session_id, cutoff_generation, cutoff); + let Some(cutoff) = with_live_page_retention(page, || { + page.browser + .client + .event_log + .lock() + .unwrap() + .finalize_console_replay_cutoff(session_id, cutoff_generation, cutoff); + cutoff + }) else { + return Err(RwError::TargetClosed(TargetClosedKind::Page)); + }; cutoff } Err(error) => { - page.console_replay_until_event_cursor - .lock() - .unwrap() - .remove(session_id); - page.browser - .client - .event_log - .lock() - .unwrap() - .cancel_console_replay_cutoff(session_id, cutoff_generation); + let _ = with_live_page_retention(page, || { + page.console_replay_until_event_cursor + .lock() + .unwrap() + .remove(session_id); + page.browser + .client + .event_log + .lock() + .unwrap() + .cancel_console_replay_cutoff(session_id, cutoff_generation); + }); return Err(error); } }; - page.console_replay_until_event_cursor - .lock() - .unwrap() - .insert(session_id.to_owned(), replay_until); + if with_live_page_retention(page, || { + page.console_replay_until_event_cursor + .lock() + .unwrap() + .insert(session_id.to_owned(), replay_until); + }) + .is_none() + { + return Err(RwError::TargetClosed(TargetClosedKind::Page)); + } let remaining = timeout.saturating_sub(started.elapsed()); if page.observation_event_cursor.load(Ordering::SeqCst) < replay_until { let waited = tokio::time::timeout(remaining, async { @@ -47652,14 +48881,22 @@ async fn enable_console_capture_for_session( }) .await; if waited.is_err() { - let mut replay_windows = page.console_replay_until_event_cursor.lock().unwrap(); - if replay_windows.get(session_id) == Some(&replay_until) { - replay_windows.remove(session_id); - } + let _ = with_live_page_retention(page, || { + let mut replay_windows = page.console_replay_until_event_cursor.lock().unwrap(); + if replay_windows.get(session_id) == Some(&replay_until) { + replay_windows.remove(session_id); + } + }); return Err(RwError::Timeout(timeout.as_millis() as u64)); } } - enabled_sessions.insert(session_id.to_string()); + if with_live_page_retention(page, || { + enabled_sessions.insert(session_id.to_string()); + }) + .is_none() + { + return Err(RwError::TargetClosed(TargetClosedKind::Page)); + } Ok(replay_until) } @@ -47695,6 +48932,13 @@ fn spawn_attached_iframe_session_initialization( session_pin: FrameSessionPin, timeout: Duration, ) { + let retention_gate = page.browser.client.retention_gate(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + return; + }; + if page.lifecycle.is_closing_or_closed() || page.target_closed.load(Ordering::SeqCst) { + return; + } if !iframe_session_is_live(page, &frame_id, &session_pin) { return; } @@ -47837,15 +49081,16 @@ fn spawn_attached_iframe_session_initialization( let Some(page) = weak_page.upgrade() else { return Ok(()); }; - if !iframe_session_is_live(&page, &frame_id, &session_pin) { - return Ok(()); - } - let Some(should_start) = page - .frame_state - .lock() - .unwrap() - .claim_iframe_session_setup(&frame_id, &session_pin) - else { + let Some(should_start) = with_live_page_retention(&page, || { + if !iframe_session_is_live(&page, &frame_id, &session_pin) { + return None; + } + page.frame_state + .lock() + .unwrap() + .claim_iframe_session_setup(&frame_id, &session_pin) + }) + .flatten() else { return Ok(()); }; drop(page); @@ -47857,12 +49102,14 @@ fn spawn_attached_iframe_session_initialization( let Some(page) = weak_page.upgrade() else { return Ok(()); }; - if iframe_session_is_live(&page, &frame_id, &session_pin) { - page.frame_state - .lock() - .unwrap() - .mark_iframe_session_ready(&frame_id, &session_pin); - } + let _ = with_live_page_retention(&page, || { + if iframe_session_is_live(&page, &frame_id, &session_pin) { + page.frame_state + .lock() + .unwrap() + .mark_iframe_session_ready(&frame_id, &session_pin); + } + }); Ok(()) } .await; @@ -47877,16 +49124,18 @@ fn spawn_attached_iframe_session_initialization( } drop(handles); if let Err(error) = result { - if iframe_session_is_live(&page, &frame_id, &session_pin) { - page.frame_state - .lock() - .unwrap() - .record_frame_session_error_if_current( - &frame_id, - &session_pin, - error.to_string(), - ); - } + let _ = with_live_page_retention(&page, || { + if iframe_session_is_live(&page, &frame_id, &session_pin) { + page.frame_state + .lock() + .unwrap() + .record_frame_session_error_if_current( + &frame_id, + &session_pin, + error.to_string(), + ); + } + }); } } }); @@ -47914,27 +49163,41 @@ async fn setup_attached_iframe_session( let child_session_id = &session_pin.session_id; enable_attached_session_domains(&client, child_session_id, Duration::from_secs(5)).await?; let active_page = ensure_live()?; - active_page - .frame_state - .lock() - .unwrap() - .mark_page_domain_enabled(child_session_id); + let Some(true) = with_live_page_retention(&active_page, || { + if !iframe_session_is_live(&active_page, &frame_id, &session_pin) { + return false; + } + active_page + .frame_state + .lock() + .unwrap() + .mark_page_domain_enabled(child_session_id); + true + }) else { + return Err(RwError::Message( + "iframe session detached during attachment".to_string(), + )); + }; drop(active_page); enable_action_dispatch_binding_for_session(&client, child_session_id, Duration::from_secs(5)) .await?; let active_page = ensure_live()?; let focus_enqueued = enqueue_focus_emulation(&client, child_session_id, Duration::from_secs(5))?; - if !active_page - .frame_state - .lock() - .unwrap() - .mark_iframe_session_routable(&frame_id, &session_pin, focus_enqueued) - { + let Some(true) = with_live_page_retention(&active_page, || { + if !iframe_session_is_live(&active_page, &frame_id, &session_pin) { + return false; + } + active_page + .frame_state + .lock() + .unwrap() + .mark_iframe_session_routable(&frame_id, &session_pin, focus_enqueued) + }) else { return Err(RwError::Message( "iframe session detached during attachment".to_string(), )); - } + }; drop(active_page); enable_file_chooser_intercept_for_session(&client, child_session_id, Duration::from_secs(5)) .await?; @@ -48567,6 +49830,13 @@ fn record_accepted_navigation_observation( observation: AcceptedNavigationObservation, event_sequence: Option, ) { + let retention_gate = page.browser.client.retention_gate(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + return; + }; + if page.lifecycle.is_closing_or_closed() || page.target_closed.load(Ordering::SeqCst) { + return; + } let next_index = page .browser .next_native_network_index @@ -48598,6 +49868,13 @@ fn prune_page_navigation_epochs_to_contiguous_progress( page: &PageInner, navigation_transition: &NavigationTransitionState, ) { + let retention_gate = page.browser.client.retention_gate(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + return; + }; + if page.lifecycle.is_closing_or_closed() || page.target_closed.load(Ordering::SeqCst) { + return; + } let Some(low_water_mark) = navigation_transition.contiguous_event_watermark else { return; }; @@ -48658,17 +49935,22 @@ async fn resume_worker_after_claim( ) .await .map(|_| ()); - let mut state = page.frame_state.lock().unwrap(); - match result { - Ok(()) => { - state.complete_worker_resume_handoff(session_id); - Ok(()) - } - Err(error) => { - state.mark_worker_resume_handoff_failed(session_id); - Err(error) + let Some(result) = with_live_page_retention(page, || { + let mut state = page.frame_state.lock().unwrap(); + match result { + Ok(()) => { + state.complete_worker_resume_handoff(session_id); + Ok(()) + } + Err(error) => { + state.mark_worker_resume_handoff_failed(session_id); + Err(error) + } } - } + }) else { + return Err(RwError::TargetClosed(TargetClosedKind::Page)); + }; + result } fn worker_resume_page_is_live(page: &PageInner) -> bool { @@ -48708,11 +49990,14 @@ async fn worker_resume_watchdog_step( if !worker_resume_page_is_live(page) { return false; } - let should_resume = page - .frame_state - .lock() - .unwrap() - .take_worker_resume_fallback(session_id); + let Some(should_resume) = with_live_page_retention(page, || { + page.frame_state + .lock() + .unwrap() + .take_worker_resume_fallback(session_id) + }) else { + return false; + }; if should_resume { let _ = resume_worker_after_claim(page, session_id, Duration::from_secs(1)).await; } @@ -48741,7 +50026,64 @@ fn spawn_worker_resume_watchdog(page: Arc, session_id: String) { }); } +fn with_live_page_retention(page: &PageInner, write: impl FnOnce() -> T) -> Option { + let retention_gate = page.browser.client.retention_gate(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + return None; + }; + if page.lifecycle.is_closing_or_closed() || page.target_closed.load(Ordering::SeqCst) { + return None; + } + Some(write()) +} + +fn reset_page_history_after_unreplayable(page: &PageInner, sequence: u64) { + let retention_gate = page.browser.client.retention_gate(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + return; + }; + if page.lifecycle.is_closing_or_closed() || page.target_closed.load(Ordering::SeqCst) { + return; + } + reset_page_history_after_unreplayable_locked(page, sequence); +} + +fn reset_page_history_after_unreplayable_locked(page: &PageInner, sequence: u64) { + page.network_requests + .lock() + .unwrap() + .reset_after_overflow(sequence.saturating_add(1)); + page.native_network_records + .lock() + .unwrap() + .reset_after_unreplayable(); + page.console_records + .lock() + .unwrap() + .reset_after_unreplayable(); + page.console_replay_until_event_cursor + .lock() + .unwrap() + .clear(); + page.frame_state + .lock() + .unwrap() + .mark_all_frame_caches_dirty(); +} async fn handle_page_oopif_event(page: Arc, event: Value) { + if page.lifecycle.is_closing_or_closed() + || page.target_closed.load(Ordering::SeqCst) + || !page.browser.client.retention_enabled() + { + return; + } + if is_unreplayable_cdp_event(&event) { + reset_page_history_after_unreplayable( + &page, + cdp_event_sequence(&event).unwrap_or_default(), + ); + return; + } let method = event.get("method").and_then(Value::as_str).unwrap_or(""); let event_sequence = cdp_event_sequence(&event); let is_page_navigation = matches!( @@ -48751,6 +50093,12 @@ async fn handle_page_oopif_event(page: Arc, event: Value) { if !is_page_navigation { record_page_observation_event(&page, &event); } + if page.lifecycle.is_closing_or_closed() + || page.target_closed.load(Ordering::SeqCst) + || !page.browser.client.retention_enabled() + { + return; + } #[cfg(test)] wait_for_page_lifecycle_test_barrier(&page, &event).await; if method == "Target.targetCrashed" @@ -48808,7 +50156,7 @@ async fn handle_page_oopif_event(page: Arc, event: Value) { .and_then(Value::as_str) .unwrap_or(""); if target_type == "worker" { - let accepted = { + let Some(accepted) = with_live_page_retention(&page, || { let mut state = page.frame_state.lock().unwrap(); if let Some(target_id) = target_info.get("targetId").and_then(Value::as_str) { state.record_worker_target_session_at_sequence( @@ -48825,16 +50173,20 @@ async fn handle_page_oopif_event(page: Arc, event: Value) { event_sequence, )) } + }) else { + return; }; if !accepted { return; } let child_session_id = child_session_id.to_string(); - let resume_without_consumer = { + let Some(resume_without_consumer) = with_live_page_retention(&page, || { let mut state = page.frame_state.lock().unwrap(); state.arm_worker_resume_handoff(&child_session_id, event_sequence); !state.worker_event_interest() && state.take_worker_resume_fallback(&child_session_id) + }) else { + return; }; if resume_without_consumer { let page_for_resume = Arc::clone(&page); @@ -48891,12 +50243,14 @@ async fn handle_page_oopif_event(page: Arc, event: Value) { event_sequence, Duration::from_secs(5), ) { - let mut state = page.frame_state.lock().unwrap(); - if pin.session_id == child_session_id - && state.session_pin_still_owns_frame(frame_id, &pin) - { - state.seed_attached_frame_metadata(frame_id, &pin, target_name, target_url); - } + let _ = with_live_page_retention(&page, || { + let mut state = page.frame_state.lock().unwrap(); + if pin.session_id == child_session_id + && state.session_pin_still_owns_frame(frame_id, &pin) + { + state.seed_attached_frame_metadata(frame_id, &pin, target_name, target_url); + } + }); } if page .frame_state @@ -48925,13 +50279,15 @@ async fn handle_page_oopif_event(page: Arc, event: Value) { else { return; }; - let (worker_detached, generations) = { + let Some((worker_detached, generations)) = with_live_page_retention(&page, || { let mut state = page.frame_state.lock().unwrap(); let worker_detached = state.remove_worker_session_at_sequence(detached_session_id, event_sequence); let generations = state.setup_generations_for_session(detached_session_id); state.detach_session_at_sequence(detached_session_id, event_sequence); (worker_detached, generations) + }) else { + return; }; page.browser .client @@ -48959,10 +50315,10 @@ async fn handle_page_oopif_event(page: Arc, event: Value) { return; } let params = event.get("params").unwrap_or(&Value::Null); - match method { + let Some(observation) = with_live_page_retention(&page, || match method { "Page.frameAttached" => { let Some(frame_id) = params.get("frameId").and_then(Value::as_str) else { - return; + return None; }; let parent_id = params .get("parentFrameId") @@ -48978,64 +50334,54 @@ async fn handle_page_oopif_event(page: Arc, event: Value) { event_session_id.to_string(), ); } + None } "Page.frameNavigated" => { let _navigation_transition = page.navigation_transition_lock.lock().unwrap(); - let observation = { - let mut main_frame_id = page.main_frame_id.lock().unwrap(); - let mut state = page.frame_state.lock().unwrap(); - apply_accepted_frame_navigation_transition( - &mut state, - &mut main_frame_id, - &page.crashed, - page.session_id.as_str(), - event_session_id, - event_sequence, - params.get("frame").unwrap_or(&Value::Null), - ) - }; - if let Some(observation) = observation { - record_accepted_navigation_observation(&page, observation, event_sequence); - } + let mut main_frame_id = page.main_frame_id.lock().unwrap(); + let mut state = page.frame_state.lock().unwrap(); + apply_accepted_frame_navigation_transition( + &mut state, + &mut main_frame_id, + &page.crashed, + page.session_id.as_str(), + event_session_id, + event_sequence, + params.get("frame").unwrap_or(&Value::Null), + ) } "Page.navigatedWithinDocument" => { let _navigation_transition = page.navigation_transition_lock.lock().unwrap(); - let observation = { - let main_frame_id = page.main_frame_id.lock().unwrap(); - let mut state = page.frame_state.lock().unwrap(); - apply_accepted_same_document_navigation_transition( - &mut state, - &main_frame_id, - page.session_id.as_str(), - event_session_id, - event_sequence, - params, - ) - }; - if let Some(observation) = observation { - record_accepted_navigation_observation(&page, observation, event_sequence); - } + let main_frame_id = page.main_frame_id.lock().unwrap(); + let mut state = page.frame_state.lock().unwrap(); + apply_accepted_same_document_navigation_transition( + &mut state, + &main_frame_id, + page.session_id.as_str(), + event_session_id, + event_sequence, + params, + ) } "Page.frameDetached" => { let reason = params.get("reason").and_then(Value::as_str).unwrap_or(""); let mut state = page.frame_state.lock().unwrap(); if state.mark_frame_cache_dirty_at_sequence(event_session_id, event_sequence) { if let Some(frame_id) = params.get("frameId").and_then(Value::as_str) { - if reason == "swap" { + let removed_generations = if reason == "swap" { let removed_generation = state.mark_frame_swap_pending(frame_id, event_session_id); state.record_frame_swap(frame_id); - drop(state); - page.iframe_setup_tasks - .abort_generations(removed_generation); + removed_generation.into_iter().collect() } else { - let removed_generations = state.remove_frame(frame_id); - drop(state); - page.iframe_setup_tasks - .abort_generations(removed_generations); - } + state.remove_frame(frame_id) + }; + drop(state); + page.iframe_setup_tasks + .abort_generations(removed_generations); } } + None } "Runtime.executionContextDestroyed" => { if let Some(context_id) = params.get("executionContextId") { @@ -49044,14 +50390,21 @@ async fn handle_page_oopif_event(page: Arc, event: Value) { .unwrap() .invalidate_execution_context(event_session_id, context_id); } + None } "Runtime.executionContextsCleared" => { page.frame_state .lock() .unwrap() .invalidate_execution_contexts_for_session(event_session_id); + None } - _ => {} + _ => None, + }) else { + return; + }; + if let Some(observation) = observation { + record_accepted_navigation_observation(&page, observation, event_sequence); } } @@ -49064,6 +50417,20 @@ fn record_page_observation_event_with_ownership( event: &Value, replay_owned_session: Option, ) { + let retention_gate = page.browser.client.retention_gate(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + return; + }; + if page.lifecycle.is_closing_or_closed() || page.target_closed.load(Ordering::SeqCst) { + return; + } + if is_unreplayable_cdp_event(event) { + reset_page_history_after_unreplayable_locked( + page, + cdp_event_sequence(event).unwrap_or_default(), + ); + return; + } let event_sequence = cdp_event_sequence(event); let event_cursor = event_sequence.map(|sequence| sequence.saturating_add(1)); let event_session_id = event.get("sessionId").and_then(Value::as_str); @@ -49882,6 +51249,7 @@ struct ReconciledFrameCacheInvalidations { navigation_observations: Vec, replayed_page_observations: Vec, removed_setup_generations: Vec, + unreplayable_sequences: Vec, } fn reconcile_frame_cache_invalidations_locked( @@ -49906,6 +51274,11 @@ fn reconcile_frame_cache_invalidations_locked( .unwrap_or(frame_replay_cursor); for (sequence, event) in log.entries_since(replay_start) { let method = event.get("method").and_then(Value::as_str).unwrap_or(""); + if is_unreplayable_cdp_event(&event) { + reconciliation.unreplayable_sequences.push(sequence); + state.mark_all_frame_caches_dirty(); + continue; + } if observation_replay_cursor.is_some_and(|cursor| sequence >= cursor) && !matches!( method, @@ -50166,6 +51539,13 @@ fn reconcile_frame_cache_invalidations_impl_with_hook( // Serializing gate commit through post-lock observation prevents a newer // live navigation from overtaking a replayed observation. The same guard // owns the one contiguous progress watermark. + let retention_gate = page.browser.client.retention_gate(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + return; + }; + if page.lifecycle.is_closing_or_closed() || page.target_closed.load(Ordering::SeqCst) { + return; + } let mut navigation_transition = page.navigation_transition_lock.lock().unwrap(); // Global page cache lock order: event_log -> main_frame_id -> frame_state. let (reconciliation, unknown_gap_boundary) = { @@ -50174,9 +51554,9 @@ fn reconcile_frame_cache_invalidations_impl_with_hook( let mut main_frame_id = page.main_frame_id.lock().unwrap(); let mut state = page.frame_state.lock().unwrap(); let oldest_sequence = log.oldest_seq(); - let unknown_gap_boundary = navigation_transition + let mut unknown_gap_boundary = navigation_transition .has_unreplayable_gap(oldest_sequence) - .then(|| oldest_sequence - 1); + .then(|| oldest_sequence.saturating_sub(1)); if listener_gap { state.mark_all_frame_caches_dirty(); } @@ -50189,8 +51569,19 @@ fn reconcile_frame_cache_invalidations_impl_with_hook( None, listener_gap.then(|| page.observation_event_cursor.load(Ordering::SeqCst)), ); + if let Some(sequence) = reconciliation.unreplayable_sequences.iter().copied().min() { + let marker_boundary = sequence.saturating_sub(1); + unknown_gap_boundary = Some( + unknown_gap_boundary + .map_or(marker_boundary, |current| current.min(marker_boundary)), + ); + } (reconciliation, unknown_gap_boundary) }; + drop(_retention_guard); + for sequence in reconciliation.unreplayable_sequences.iter().copied() { + reset_page_history_after_unreplayable(page, sequence); + } if let Some(sequence) = unknown_gap_boundary { // The bounded log can no longer replay every uncovered listener event. // Place the conservative boundary immediately before the oldest @@ -50358,6 +51749,13 @@ async fn refresh_frame_tree_session_locked( // Commit lock order is event_log -> main_frame_id -> frame_state. Holding // event_log prevents a transport event from being inserted between the // cursor barrier and the generation/registration recheck. + let retention_gate = page.browser.client.retention_gate(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + return Ok(()); + }; + if page.lifecycle.is_closing_or_closed() || page.target_closed.load(Ordering::SeqCst) { + return Ok(()); + } let root_frame_id = tree.root_frame_id.clone(); let navigation_transition = page.navigation_transition_lock.lock().unwrap(); let (reconciliation, authoritative_effects) = { @@ -50393,6 +51791,7 @@ async fn refresh_frame_tree_session_locked( }; (reconciliation, authoritative_effects) }; + drop(_retention_guard); record_reconciled_navigation_observations(page, reconciliation.navigation_observations); drop(navigation_transition); page.iframe_setup_tasks.abort_generations( @@ -52577,6 +53976,14 @@ fn process_network_observation_event( state: &mut NetworkObservationState, ) -> Result, u64> { let method = event.get("method").and_then(Value::as_str).unwrap_or(""); + if is_unreplayable_cdp_event(event) { + requests + .lock() + .unwrap() + .reset_after_overflow(seq.saturating_add(1)); + *state = NetworkObservationState::new(); + return Err(1); + } let request_payload = { let mut requests = requests.lock().unwrap(); if seq > requests.next_applied_seq { @@ -52602,6 +54009,11 @@ fn process_network_observation_event( requests.reset_after_overflow(seq.saturating_add(1)); return Err(dropped); } + if is_unreplayable_cdp_event(&missed_event) { + requests.reset_after_overflow(seq.saturating_add(1)); + *state = NetworkObservationState::new(); + return Err(1); + } apply_network_request_mutation( missed_seq, &missed_event, @@ -52762,8 +54174,39 @@ fn process_page_observation_event_with_page( session_id: &str, requests: &Arc>, state: &mut PageEventStreamState, - _page: Option<&PageInner>, + page: Option<&PageInner>, ) { + if page.is_some_and(|page| { + page.lifecycle.is_closing_or_closed() + || page.target_closed.load(Ordering::SeqCst) + || !page.browser.client.retention_enabled() + }) { + return; + } + if is_unreplayable_cdp_event(event) { + if let Some(page) = page { + reset_page_history_after_unreplayable_locked(page, seq); + *state = PageEventStreamState::for_page(page); + } else { + requests + .lock() + .unwrap() + .reset_after_overflow(seq.saturating_add(1)); + state.reset_after_overflow(); + } + state.ready.insert( + seq, + page_event_envelope( + seq, + "_overflow", + json!({ + "dropped": 1, + "reason": "unreplayable_event", + }), + ), + ); + return; + } state.ensure_main_session(session_id); state.sync_replay_cutoffs_for_event(event, event_log); state.apply_target_transition(event, session_id); @@ -52898,6 +54341,13 @@ async fn wait_for_page_event_batch( let deadline = tokio::time::Instant::now() + timeout; let mut batch = Vec::with_capacity(max_events); loop { + if page.lifecycle.is_closing_or_closed() + || page.target_closed.load(Ordering::SeqCst) + || !page.browser.client.retention_enabled() + { + batch.push(page_event_envelope(*cursor, "_closed", Value::Null)); + return (batch, true); + } if *close_rx.borrow() { batch.push(page_event_envelope(*cursor, "_closed", Value::Null)); return (batch, true); @@ -52925,9 +54375,14 @@ async fn wait_for_page_event_batch( let scanned_full_chunk = entries.len() >= raw_limit; if *cursor < oldest_seq { let dropped = oldest_seq - *cursor; - *cursor = oldest_seq; - *state = PageEventStreamState::for_page(page); - requests.lock().unwrap().reset_after_overflow(oldest_seq); + let Some(()) = with_live_page_retention(page, || { + *cursor = oldest_seq; + *state = PageEventStreamState::for_page(page); + requests.lock().unwrap().reset_after_overflow(oldest_seq); + }) else { + batch.push(page_event_envelope(*cursor, "_closed", Value::Null)); + return (batch, true); + }; batch.push(page_event_envelope( *cursor, "_overflow", @@ -52942,6 +54397,11 @@ async fn wait_for_page_event_batch( continue; } *cursor = seq.saturating_add(1); + let retention_gate = page.browser.client.retention_gate(); + let Some(_retention_guard) = retention_gate.lock_for_write() else { + batch.push(page_event_envelope(*cursor, "_closed", Value::Null)); + return (batch, true); + }; process_page_observation_event_with_page( seq, &event, @@ -53014,6 +54474,7 @@ async fn wait_for_network_event( session_id: &str, kind: &str, requests: Arc>, + page: Arc, timeout: Duration, ) -> (RwResult, u64) { let deadline = tokio::time::Instant::now() + timeout; @@ -53021,6 +54482,12 @@ async fn wait_for_network_event( let mut state = NetworkObservationState::new(); let mut ready_responses = BTreeMap::::new(); loop { + if page.lifecycle.is_closing_or_closed() + || page.target_closed.load(Ordering::SeqCst) + || !page.browser.client.retention_enabled() + { + return (Err(RwError::TargetClosed(TargetClosedKind::Page)), cursor); + } let now = tokio::time::Instant::now(); if now >= deadline { return (Err(RwError::Timeout(timeout.as_millis() as u64)), cursor); @@ -53047,7 +54514,13 @@ async fn wait_for_network_event( }; if cursor < oldest_seq { let dropped = oldest_seq - cursor; - requests.lock().unwrap().reset_after_overflow(oldest_seq); + if with_live_page_retention(&page, || { + requests.lock().unwrap().reset_after_overflow(oldest_seq); + }) + .is_none() + { + return (Err(RwError::TargetClosed(TargetClosedKind::Page)), cursor); + } return ( Err(RwError::Message(format!( "CDP event log overflow: dropped {dropped} event(s)" @@ -53057,9 +54530,14 @@ async fn wait_for_network_event( } for (seq, event) in entries { cursor = seq.saturating_add(1); - let matched = match process_network_wait_event( - seq, &event, &event_log, session_id, kind, &requests, &mut state, - ) { + let Some(processed) = with_live_page_retention(&page, || { + process_network_wait_event( + seq, &event, &event_log, session_id, kind, &requests, &mut state, + ) + }) else { + return (Err(RwError::TargetClosed(TargetClosedKind::Page)), cursor); + }; + let matched = match processed { Ok(matched) => matched, Err(dropped) => { return ( @@ -53381,25 +54859,32 @@ fn page_worker_session_from_attachment(page: &PageInner, event: &Value) -> Optio return None; } let child_session_id = event.pointer("/params/sessionId").and_then(Value::as_str)?; - let mut state = page.frame_state.lock().unwrap(); - if !state.owns_session(parent_session_id) { - return None; - } - let accepted = if let Some(target_id) = target_info.get("targetId").and_then(Value::as_str) { - state.record_worker_target_session_at_sequence(target_id, child_session_id, event_sequence) - } else { - state.worker_session_attachment_is_current(child_session_id, event_sequence) - && (state.worker_session_recorded_at_sequence(child_session_id, event_sequence) - || state.record_worker_session_at_sequence(child_session_id, event_sequence)) - }; - if !accepted { - return None; - } - state.arm_worker_resume_handoff(child_session_id, event_sequence); - if !state.claim_worker_resume_handoff(child_session_id) { - return None; - } - Some(child_session_id.to_string()) + with_live_page_retention(page, || { + let mut state = page.frame_state.lock().unwrap(); + if !state.owns_session(parent_session_id) { + return None; + } + let accepted = if let Some(target_id) = target_info.get("targetId").and_then(Value::as_str) + { + state.record_worker_target_session_at_sequence( + target_id, + child_session_id, + event_sequence, + ) + } else { + state.worker_session_attachment_is_current(child_session_id, event_sequence) + && (state.worker_session_recorded_at_sequence(child_session_id, event_sequence) + || state.record_worker_session_at_sequence(child_session_id, event_sequence)) + }; + if !accepted { + return None; + } + state.arm_worker_resume_handoff(child_session_id, event_sequence); + state + .claim_worker_resume_handoff(child_session_id) + .then(|| child_session_id.to_string()) + }) + .flatten() } fn remove_page_worker_session_from_detachment(page: &PageInner, event: &Value) -> Option { @@ -53407,11 +54892,12 @@ fn remove_page_worker_session_from_detachment(page: &PageInner, event: &Value) - return None; } let session_id = event.pointer("/params/sessionId").and_then(Value::as_str)?; - let accepted = page - .frame_state - .lock() - .unwrap() - .remove_worker_session_at_sequence(session_id, cdp_event_sequence(event)); + let accepted = with_live_page_retention(page, || { + page.frame_state + .lock() + .unwrap() + .remove_worker_session_at_sequence(session_id, cdp_event_sequence(event)) + })?; accepted.then(|| session_id.to_string()) } @@ -53428,12 +54914,24 @@ async fn process_console_wait_event( event_type: Option<&str>, text: Option<&str>, exact_text: bool, -) -> Option { +) -> RwResult> { + if is_unreplayable_cdp_event(&event) { + let sequence = cdp_event_sequence(&event).unwrap_or_default(); + if let Some(page) = page.as_deref() { + reset_page_history_after_unreplayable(page, sequence); + } + if let Some(state) = page_state.as_mut() { + state.reset_after_overflow(); + } + return Err(RwError::Message(format!( + "CDP event log contains unreplayable event at sequence {sequence}" + ))); + } let event_sequence = cdp_event_sequence(&event); let event_session_id = event.get("sessionId").and_then(Value::as_str); let method = event.get("method").and_then(Value::as_str); if event_sequence.is_some_and(|sequence| sequence < *event_cursor) { - return None; + return Ok(None); } if let Some(sequence) = event_sequence { *event_cursor = sequence.saturating_add(1); @@ -53454,11 +54952,13 @@ async fn process_console_wait_event( ) && replay_cutoff.is_some_and(|cutoff| { event_sequence.is_some_and(|sequence| sequence < cutoff && sequence < replay_start_cursor) }) { - return None; + return Ok(None); } if let Some(page) = page.as_ref() { if method == Some("Target.attachedToTarget") { + #[cfg(test)] + wait_for_page_lifecycle_test_barrier(page, &event).await; if let Some(child_session_id) = page_worker_session_from_attachment(page, &event) { let page = Arc::clone(page); tokio::spawn(async move { @@ -53476,10 +54976,12 @@ async fn process_console_wait_event( ) .await { - page.frame_state - .lock() - .unwrap() - .mark_worker_resume_handoff_failed(&child_session_id); + let _ = with_live_page_retention(&page, || { + page.frame_state + .lock() + .unwrap() + .mark_worker_resume_handoff_failed(&child_session_id); + }); return; } let _ = @@ -53487,7 +54989,7 @@ async fn process_console_wait_event( .await; }); } - return None; + return Ok(None); } if method == Some("Target.detachedFromTarget") { if let Some(detached_session_id) = @@ -53501,7 +55003,7 @@ async fn process_console_wait_event( .finalize_console_replay_cutoff_on_detach(&detached_session_id, event_sequence); clear_console_capture_for_session(page, &detached_session_id).await; } - return None; + return Ok(None); } } @@ -53517,34 +55019,36 @@ async fn process_console_wait_event( }) }); if !matches_session { - return None; + return Ok(None); } if event_kind == "console" { if method != Some("Runtime.consoleAPICalled") || !console_event_matches(&event, event_type, text, exact_text) { - return None; + return Ok(None); } - let mut message = console_from_event(&event)?; + let Some(mut message) = console_from_event(&event) else { + return Ok(None); + }; if let Some(sequence) = event_sequence { if let Some(object) = message.as_object_mut() { object.insert("__rustwright_cdp_event_seq".to_string(), json!(sequence)); } } - return Some(message.to_string()); + return Ok(Some(message.to_string())); } if method != Some("Runtime.exceptionThrown") || !page_error_event_matches(&event, event_type, text, exact_text) { - return None; + return Ok(None); } - Some( + Ok(Some( event .get("params") .cloned() .unwrap_or(Value::Null) .to_string(), - ) + )) } async fn wait_for_console_event( events: &mut broadcast::Receiver, @@ -53598,7 +55102,7 @@ async fn wait_for_console_event( }; match received { Ok(event) => { - if let Some(result) = process_console_wait_event( + match process_console_wait_event( event, &mut event_cursor, replay_start_cursor, @@ -53614,7 +55118,9 @@ async fn wait_for_console_event( ) .await { - return (Ok(result), event_cursor); + Ok(Some(result)) => return (Ok(result), event_cursor), + Ok(None) => {} + Err(error) => return (Err(error), event_cursor), } } Err(broadcast::error::RecvError::Lagged(_)) => { @@ -53636,7 +55142,7 @@ async fn wait_for_console_event( ); } for (_, event) in replay { - if let Some(result) = process_console_wait_event( + match process_console_wait_event( event, &mut event_cursor, replay_start_cursor, @@ -53652,7 +55158,9 @@ async fn wait_for_console_event( ) .await { - return (Ok(result), event_cursor); + Ok(Some(result)) => return (Ok(result), event_cursor), + Ok(None) => {} + Err(error) => return (Err(error), event_cursor), } } } @@ -54269,6 +55777,12 @@ fn worker_from_attachment_event( browser: &Arc, event: &Value, ) -> RwResult> { + if is_unreplayable_cdp_event(event) { + let sequence = cdp_event_sequence(event).unwrap_or_default(); + return Err(RwError::Message(format!( + "CDP event log contains unreplayable worker event at sequence {sequence}" + ))); + } if event.get("method").and_then(Value::as_str) != Some("Target.attachedToTarget") { return Ok(None); } @@ -54290,20 +55804,23 @@ fn worker_from_attachment_event( .ok_or_else(|| RwError::Message("worker target did not include sessionId".to_string()))? .to_string(); let event_sequence = cdp_event_sequence(event); - let accepted = { + let Some(()) = with_live_page_retention(page, || { let mut state = page.frame_state.lock().unwrap(); - state.owns_session(parent_session_id) - && state.record_worker_target_session_at_sequence( + if !state.owns_session(parent_session_id) + || !state.record_worker_target_session_at_sequence( &target_id, &session_id, event_sequence, ) - }; - if !accepted { + { + return None; + } + state.arm_worker_resume_handoff(&session_id, event_sequence); + Some(()) + }) + .flatten() else { return Ok(None); - } - let mut state = page.frame_state.lock().unwrap(); - state.arm_worker_resume_handoff(&session_id, event_sequence); + }; let url = info .get("url") .and_then(Value::as_str) @@ -54557,6 +56074,14 @@ async fn wait_for_worker_close( continue; } event_cursor = sequence.saturating_add(1); + if is_unreplayable_cdp_event(event) { + return ( + Err(RwError::Message(format!( + "CDP event log contains unreplayable worker-close event at sequence {sequence}" + ))), + event_cursor, + ); + } let method = event.get("method").and_then(Value::as_str).unwrap_or(""); if method == "Target.detachedFromTarget" && event.pointer("/params/sessionId").and_then(Value::as_str) == Some(session_id) @@ -54581,6 +56106,15 @@ async fn wait_for_worker_close( } event_cursor = sequence.saturating_add(1); } + if is_unreplayable_cdp_event(&event) { + let sequence = cdp_event_sequence(&event).unwrap_or_default(); + return ( + Err(RwError::Message(format!( + "CDP event log contains unreplayable worker-close event at sequence {sequence}" + ))), + event_cursor, + ); + } let method = event.get("method").and_then(Value::as_str).unwrap_or(""); if method == "Target.detachedFromTarget" && event.pointer("/params/sessionId").and_then(Value::as_str) @@ -55232,6 +56766,12 @@ fn process_navigation_event( restore_initiation_cursor: Option, matcher: &mut NavigationEventState, ) -> RwResult> { + if is_unreplayable_cdp_event(event) { + let sequence = cdp_event_sequence(event).unwrap_or_default(); + return Err(RwError::Message(format!( + "CDP event log contains unreplayable navigation event at sequence {sequence}" + ))); + } if event.get("sessionId").and_then(Value::as_str) != Some(session_id) { return Ok(None); } From bfd11e2010ee27259f0f73d8cb3aadeb64307559 Mon Sep 17 00:00:00 2001 From: suchintan <3853670+suchintan@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:38:26 +0000 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=94=84=20synced=20local=20'tests/'=20?= =?UTF-8?q?with=20remote=20'tests/'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/Skyvern-AI/rustwright-cloud/pull/239 --- tests/test_async_generated_fresh.py | 2 +- tests/test_benchmark_memory.py | 491 +++++++++++++++++++++++++++- tests/test_rustwright_sync_api.py | 73 ++++- 3 files changed, 557 insertions(+), 9 deletions(-) diff --git a/tests/test_async_generated_fresh.py b/tests/test_async_generated_fresh.py index e2d5553..987b7c1 100644 --- a/tests/test_async_generated_fresh.py +++ b/tests/test_async_generated_fresh.py @@ -53,7 +53,7 @@ def test_every_class_async_method_has_one_owner() -> None: assert actual_hand == expected_hand assert actual_hand.isdisjoint(expected_generated) - assert len(actual_hand | expected_generated) == 389 + assert len(actual_hand | expected_generated) == 391 def test_non_class_async_helpers_stay_hand_written() -> None: diff --git a/tests/test_benchmark_memory.py b/tests/test_benchmark_memory.py index 0356c07..504a4c2 100644 --- a/tests/test_benchmark_memory.py +++ b/tests/test_benchmark_memory.py @@ -1,11 +1,23 @@ from __future__ import annotations import argparse +import base64 import json +import sys +import threading import time +from types import SimpleNamespace + +import pytest from benchmarks import process_tree_memory -from tools.run_benchmark_matrix import aggregate_repetitions, run_impl +from rustwright.sync_api import Page, Request, Response, _LocalEventContextManager +from tools.run_benchmark_matrix import ( + aggregate_repetitions, + main as benchmark_matrix_main, + markdown_table, + run_impl, +) def test_process_tree_walk_and_rss_sum(monkeypatch) -> None: @@ -154,6 +166,9 @@ def test_decorator_attaches_background_peak_memory(monkeypatch) -> None: process_tree_memory.ProcessTreeRssSample(110, 600), ] ) + # The synthetic samples intentionally omit PSS. Exercise the portable + # RSS-only contract instead of making this fixture Linux-specific. + monkeypatch.setattr(process_tree_memory.sys, "platform", "darwin") def fake_sample(_root_pid: int) -> process_tree_memory.ProcessTreeRssSample: return next(samples, process_tree_memory.ProcessTreeRssSample(110, 600)) @@ -193,8 +208,482 @@ def test_matrix_aggregates_memory_and_preserves_unavailable_values() -> None: memory = aggregate["rustwright"]["memory"] assert memory["rss_self_kb"]["median"] == 110 assert memory["rss_tree_kb"]["median"] == 500 + assert memory["pss_by_role_statistic"] == "independent_marginal_medians" unavailable = aggregate_repetitions( [{**base, "memory": {"rss_self_kb": None, "rss_tree_kb": None}}] ) assert unavailable["rustwright"]["memory"]["rss_tree_kb"]["median"] is None + + +def test_matrix_does_not_publish_partial_repetition_aggregates() -> None: + passed = { + "implementation": "rustwright", + "status": "passed", + "total_mean_ms": 10.0, + "memory": {"rss_self_kb": 100, "rss_tree_kb": 500}, + "cases": {"case": {"mean_ms": 10.0}}, + } + failed = { + "implementation": "rustwright", + "status": "failed", + "failure_kind": "memory_unavailable", + "repetition": 2, + } + results = [ + {**passed, "repetition": 1}, + failed, + {**passed, "repetition": 3, "total_mean_ms": 12.0}, + ] + assert aggregate_repetitions(results, planned_repetitions=3) == {} + diagnostic = aggregate_repetitions(results) + assert diagnostic["rustwright"]["runs"] == 2 + + rendered = markdown_table( + { + "implementations": ["rustwright"], + "implementation_status": {"rustwright": "failed"}, + "aggregate": {}, + "diagnostic_aggregate": diagnostic, + "speedups": {}, + "results": results, + } + ) + assert "| rustwright | failed |" in rendered + assert "Noncanonical diagnostic aggregates" in rendered + + +def test_process_role_classifier_handles_exact_browser_and_non_browser_tokens() -> None: + assert process_tree_memory.classify_process_role( + b"/usr/bin/python3\x00-m\x00benchmarks.run_benchmarks" + ) == "python-host" + assert process_tree_memory.classify_process_role( + b"/usr/bin/python3\x00--type=renderer" + ) == "python-host" + assert process_tree_memory.classify_process_role( + b"/usr/bin/chrome\x00--type=renderer\x00--renderer-client-id=4" + ) == "renderer" + assert process_tree_memory.classify_process_role( + b"/usr/bin/chrome\x00--type=utility\x00--utility-sub-type=network.mojom.NetworkService" + ) == "network" + assert process_tree_memory.classify_process_role( + "/usr/bin/chrome --enable-features=NetworkServiceInProcess" + ) == "browser" + assert process_tree_memory.classify_process_role( + "/usr/bin/chrome --type=utility --utility-sub-type=storage.mojom.StorageService" + ) == "utility" + assert process_tree_memory.classify_process_role( + "/usr/bin/chrome --type=utility --utility-sub-type=fooNetworkService" + ) == "utility" + assert process_tree_memory.classify_process_role("/usr/bin/chrome --type=gpu-process") == "gpu" + assert process_tree_memory.classify_process_role("/usr/bin/chrome --type=gpu") == "gpu" + assert process_tree_memory.classify_process_role( + "/usr/bin/chrome --type=renderer" + ) == "renderer" + assert process_tree_memory.classify_process_role("/usr/bin/chrome") == "browser" + assert process_tree_memory.classify_process_role("/usr/bin/headless_shell") == "browser" + assert process_tree_memory.classify_process_role("/usr/bin/node driver.js") == "node-driver" + assert process_tree_memory.classify_process_role("/usr/bin/node --type=renderer") == "node-driver" + assert process_tree_memory.classify_process_role("/usr/bin/nodejs driver.js") == "node-driver" + assert process_tree_memory.classify_process_role("/usr/bin/helper --type=network") == "network" + assert process_tree_memory.classify_process_role("/usr/bin/sleep 1") == "other" + + +def test_procfs_process_tree_pss_aggregates_roles_and_rejects_partial_reads(tmp_path) -> None: + processes = { + 10: ("/usr/bin/python3", 100), + 11: ("/usr/bin/chrome", 200), + 12: ("/usr/bin/chrome", 300), + } + for pid, (executable, pss_kb) in processes.items(): + process_dir = tmp_path / str(pid) + process_dir.mkdir() + (process_dir / "cmdline").write_bytes( + f"{executable}\x00--type=renderer\x00".encode() + if pid == 12 + else executable.encode() + ) + (process_dir / "smaps_rollup").write_text(f"Pss:\t{pss_kb} kB\n", encoding="utf-8") + + total, by_role = process_tree_memory.procfs_process_tree_pss([10, 11, 12], tmp_path) + assert total == 600 + assert by_role["python-host"] == 100 + assert by_role["browser"] == 200 + assert by_role["renderer"] == 300 + assert sum(by_role.values()) == total + + (tmp_path / "12" / "smaps_rollup").write_text("Pss:\t-1 kB\n", encoding="utf-8") + assert process_tree_memory.procfs_process_tree_pss([10, 11, 12], tmp_path) is None + + +def test_pss_validity_is_fail_closed_at_tree_rss_peak(monkeypatch) -> None: + monkeypatch.setattr(process_tree_memory.sys, "platform", "linux") + sampler = process_tree_memory.ProcessTreeRssSampler(1) + sampler._append( + process_tree_memory.ProcessTreeRssSample( + 100, + 500, + 40, + {"browser": 40}, + ) + ) + sampler._append( + process_tree_memory.ProcessTreeRssSample( + 110, + 600, + None, + None, + ) + ) + summary = sampler.summary() + assert summary["pss_statistic"] == "at_tree_rss_peak" + assert summary["pss_tree_kb"] is None + assert summary["pss_available"] is False + assert summary["available"] is False + + +def test_non_linux_summary_accepts_rss_with_empty_pss_schema(monkeypatch) -> None: + monkeypatch.setattr(process_tree_memory.sys, "platform", "darwin") + sampler = process_tree_memory.ProcessTreeRssSampler(1, sample_interval=0.05) + sampler._append(process_tree_memory.ProcessTreeRssSample(100, 500)) + + assert sampler.summary() == { + "rss_self_kb": 100, + "rss_tree_kb": 500, + "pss_tree_kb": None, + "pss_by_role_kb": None, + "pss_required": False, + "pss_available": False, + "pss_statistic": "at_tree_rss_peak", + "pss_scope": "benchmark_process_and_descendants", + "samples_collected": 1, + "sampling_interval_ms": 50.0, + "statistic": "peak", + "scope": "benchmark_process_and_descendants", + "sampling_mode": "background_thread_procfs_or_ps", + "available": True, + } + + +def _memory_test_page() -> Page: + page = Page.__new__(Page) + page._navigation_responses = [] + page._network_history_lock = threading.RLock() + page._request_log = [] + page._request_log_sequences = [] + page._next_request_log_sequence = 0 + page._request_log_keys = set() + page._request_log_generation = 0 + page._request_history_pending_keys = set() + page._requests_by_key = {} + page._response_log = [] + page._response_log_sequences = [] + page._next_response_log_sequence = 0 + page._response_log_keys = set() + page._request_log_condition = threading.Condition(page._network_history_lock) + page._fulfilled_route_bodies = {} + return page + +def test_navigation_response_ring_caps_count_and_bytes() -> None: + page = Page.__new__(Page) + page._navigation_responses = [] + responses = [ + Response(url=f"https://example.test/{index}", _body_cache=b"x" * (512 * 1024)) + for index in range(20) + ] + for response in responses: + page._remember_navigation_response(response) + + assert len(page._navigation_responses) == 16 + assert page._navigation_responses[0] is responses[4] + assert responses[0] not in page._navigation_responses + assert responses[0]._body_cache == b"x" * (512 * 1024) + assert sum( + len(response._body_cache) + for response in page._navigation_responses + if response._body_cache is not None + ) <= 8 * 1024 * 1024 + + +def test_navigation_response_eviction_releases_page_owned_graph_without_clearing_body() -> None: + page = _memory_test_page() + body_size = 512 * 1024 + held_response = None + held_request = None + for index in range(25): + request_id = f"request-{index}" + url = f"https://example.test/{index}" + body = (f"body-{index}".encode() * body_size)[:body_size] + request = Request( + url=url, + method="GET", + resource_type="document", + request_id=request_id, + _page=page, + ) + response = Response( + url=url, + status=200, + request_id=request_id, + request=request, + _page=page, + _body_cache=body, + ) + page._fulfilled_route_bodies[request_id] = body + page._remember_navigation_response(response) + if index == 0: + held_response = response + held_request = request + + page_owned_responses = { + id(response): response + for response in [*page._navigation_responses, *page._response_log] + } + page_owned_bodies = { + id(response._body_cache): response._body_cache + for response in page_owned_responses.values() + if response._body_cache is not None + } + page_owned_bodies.update( + {id(body): body for body in page._fulfilled_route_bodies.values()} + ) + + assert len(page._navigation_responses) == 16 + assert sum(len(body) for body in page_owned_bodies.values()) <= 8 * 1024 * 1024 + assert held_response is not None + assert held_request is not None + assert held_response not in page._navigation_responses + assert held_response not in page._response_log + assert held_request not in page._request_log + assert all(key[0] != "request-0" for key in page._requests_by_key) + assert "request-0" not in page._fulfilled_route_bodies + assert held_response.body() == (b"body-0" * body_size)[:body_size] + +def test_fulfilled_route_body_staging_is_bounded_without_response_caches() -> None: + page = _memory_test_page() + body_size = 1024 * 1024 + for index in range(100): + body = bytes([index]) * body_size + page._remember_fulfilled_route_body(f"fulfilled-{index}", body) + + page_owned_bodies = page._page_owned_bodies_locked() + assert sum(len(body) for body in page_owned_bodies.values()) <= 8 * 1024 * 1024 + assert len(page._fulfilled_route_bodies) <= 8 + assert "fulfilled-0" not in page._fulfilled_route_bodies + assert "fulfilled-99" in page._fulfilled_route_bodies + + +def test_fulfilled_route_body_moves_into_recorded_response() -> None: + page = _memory_test_page() + body = b"fulfilled-body" + response = Response( + url="https://example.test/fulfilled", + status=200, + request_id="fulfilled-request", + _page=page, + ) + page._remember_fulfilled_route_body("fulfilled-request", body) + page._remember_navigation_response(response) + + assert response._body_cache is body + assert "fulfilled-request" not in page._fulfilled_route_bodies + + +def test_network_history_pruning_serializes_event_pump_append() -> None: + page = _memory_test_page() + body_size = 512 * 1024 + for index in range(16): + page._remember_navigation_response( + Response( + url=f"https://example.test/{index}", + status=200, + request_id=f"request-{index}", + _page=page, + _body_cache=b"x" * body_size, + ) + ) + + snapshot_started = threading.Event() + release_snapshot = threading.Event() + append_started = threading.Event() + append_done = threading.Event() + original_snapshot = page._page_owned_bodies_locked + + def blocked_snapshot() -> dict[int, bytes]: + if not snapshot_started.is_set(): + snapshot_started.set() + assert release_snapshot.wait(timeout=1.0) + return original_snapshot() + + page._page_owned_bodies_locked = blocked_snapshot + + prune_thread = threading.Thread(target=page._prune_navigation_responses) + prune_thread.start() + assert snapshot_started.wait(timeout=1.0) + + appended = Response( + url="https://example.test/concurrent", + status=200, + request_id="request-concurrent", + _page=page, + _body_cache=b"concurrent", + ) + + def append_response() -> None: + append_started.set() + page._remember_navigation_response(appended) + append_done.set() + + append_thread = threading.Thread(target=append_response) + append_thread.start() + assert append_started.wait(timeout=1.0) + assert not append_done.wait(timeout=0.05) + release_snapshot.set() + prune_thread.join(timeout=1.0) + append_thread.join(timeout=1.0) + + assert not prune_thread.is_alive() + assert not append_thread.is_alive() + assert appended in page._navigation_responses + assert appended in page._response_log + + +def test_navigation_response_body_cache_prunes_post_hoc() -> None: + page = _memory_test_page() + body_size = 512 * 1024 + responses = [] + for index in range(16): + response = Response( + url=f"https://example.test/{index}", + status=200, + request_id=f"request-{index}", + _page=page, + _body_cache=b"x" * body_size, + ) + responses.append(response) + page._remember_navigation_response(response) + + late = Response( + url="https://example.test/late", + status=200, + request_id="request-late", + _page=page, + ) + page._remember_navigation_response(late) + late_body = b"late" * (body_size // 4) + page._default_timeout = 1_000.0 + page._core = SimpleNamespace( + response_body=lambda _request_id, _timeout: json.dumps( + { + "body": base64.b64encode(late_body).decode("ascii"), + "base64Encoded": True, + } + ) + ) + + assert late.body() == late_body + assert responses[0] not in page._navigation_responses + assert responses[0] not in page._response_log + assert responses[0].body() == b"x" * body_size + assert sum( + len(response._body_cache) + for response in page._navigation_responses + if response._body_cache is not None + ) <= 8 * 1024 * 1024 + + +def test_response_waiter_cursor_survives_pruning_during_handler_registration() -> None: + page = _memory_test_page() + body_size = 512 * 1024 + for index in range(16): + page._remember_navigation_response( + Response( + url=f"https://example.test/{index}", + status=200, + request_id=f"request-{index}", + _page=page, + _body_cache=b"x" * body_size, + ) + ) + + class Context: + _default_timeout = 1_000.0 + + def __init__(self) -> None: + self._pages = [page] + self._handlers = {} + + def on(self, event: str, handler) -> None: + if event == "response": + page._remember_navigation_response( + Response( + url="https://example.test/matching", + status=200, + request_id="request-matching", + _page=page, + _body_cache=b"matching", + ) + ) + self._handlers[event] = handler + + def remove_listener(self, event: str, handler) -> None: + if self._handlers.get(event) is handler: + self._handlers.pop(event, None) + + target = Context() + with _LocalEventContextManager( + target, + "response", + lambda response: response.url.endswith("/matching"), + timeout=1_000.0, + ) as waiter: + pass + + assert waiter.value.url == "https://example.test/matching" + + +@pytest.mark.parametrize("repetitions", ["0", "-1"]) +def test_benchmark_matrix_rejects_non_positive_repetitions_at_parse_time( + monkeypatch, repetitions: str +) -> None: + monkeypatch.setattr( + sys, + "argv", + ["run_benchmark_matrix.py", "--repetitions", repetitions], + ) + with pytest.raises(SystemExit) as error: + benchmark_matrix_main() + assert error.value.code == 2 + +def test_matrix_rejects_invalid_or_partial_pss(monkeypatch) -> None: + completed = argparse.Namespace( + returncode=0, + stdout=json.dumps( + { + "implementation": "rustwright", + "memory": { + "available": True, + "rss_self_kb": 100, + "rss_tree_kb": 500, + "pss_required": True, + "pss_available": True, + "pss_tree_kb": 10, + "pss_by_role_kb": {"browser": 10}, + }, + } + ), + stderr="", + ) + monkeypatch.setattr("tools.run_benchmark_matrix.subprocess.run", lambda *_args, **_kwargs: completed) + args = argparse.Namespace( + lifecycle="warm-browser", + rebuild_rustwright=False, + skip_rustwright_rebuild=False, + iterations=1, + suite="strict", + case_filters=[], + timeout=10, + ) + result = run_impl(args, "rustwright") + assert result["status"] == "failed" + assert result["failure_kind"] == "memory_unavailable" diff --git a/tests/test_rustwright_sync_api.py b/tests/test_rustwright_sync_api.py index dbb2591..024a8ab 100644 --- a/tests/test_rustwright_sync_api.py +++ b/tests/test_rustwright_sync_api.py @@ -40,7 +40,7 @@ import pytest import rustwright -from rustwright.sync_api import Error, TimeoutError, expect, sync_playwright +from rustwright.sync_api import Error, Response, TimeoutError, expect, sync_playwright rustwright.enable_playwright_compat() @@ -1259,6 +1259,15 @@ def do_GET(self): self.end_headers() self.wfile.write(body) return + if self.path.startswith("/large"): + body = b"retained-response-" + (b"x" * (512 * 1024 - 18)) + self.send_response(200, "OK") + self.send_header("Cache-Control", "no-store") + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return if self.path == "/headers": body = json.dumps({"x-test": self.headers.get("X-Test-Header")}).encode("utf-8") self.send_response(200, "OK") @@ -5415,6 +5424,44 @@ def test_navigation_response_body_remains_readable_after_later_navigation(page, assert first.json() == {"x-test": None} +def test_navigation_response_body_survives_count_byte_eviction_and_renderer_swap( + page, http_server, cross_origin_frame_server +): + first = page.goto(f"{http_server}/large?first=1") + assert first is not None + expected = first.body() + + for index in range(24): + page.goto(f"{http_server}/large?index={index}") + page.goto(f"{cross_origin_frame_server}/json") + + assert first.body() == expected + + +def test_async_navigation_response_body_survives_count_byte_eviction_and_renderer_swap( + http_server, cross_origin_frame_server +): + async def run() -> None: + from rustwright.async_api import async_playwright + + async with async_playwright() as playwright: + browser = await playwright.chromium.launch(headless=True) + context = await browser.new_context() + page = await context.new_page() + try: + first = await page.goto(f"{http_server}/large?first=1") + assert first is not None + expected = await first.body() + for index in range(24): + await page.goto(f"{http_server}/large?index={index}") + await page.goto(f"{cross_origin_frame_server}/json") + assert await first.body() == expected + finally: + await browser.close() + + asyncio.run(run()) + + def test_subframe_request_and_response_frame_attribution(page, http_server): page.goto(f"{http_server}/frame-fetch-page") frame = page.frame(name="child") @@ -32286,6 +32333,7 @@ def __init__(self): self._closed = False self._closing = False self.dispatch_task = None + self.release_calls = 0 self._core = FakeCore(self) def _stop_event_pump(self): @@ -32295,6 +32343,10 @@ def _stop_event_pump(self): def _mark_owned_cdp_sessions_closed(self): pass + def _release_memory_buffers(self): + self.release_calls += 1 + order.append("release") + sync_page = FakePage() page = object.__new__(async_api.AsyncPage) page._sync = sync_page @@ -32304,9 +32356,10 @@ def _mark_owned_cdp_sessions_closed(self): order.append("close-complete") assert sync_page.dispatch_task is not None await sync_page.dispatch_task - assert seen == ["beforeunload"] - assert order == ["close-command", "dialog", "stop-pump", "close-complete"] + + assert sync_page.release_calls == 1 + assert order == ["close-command", "dialog", "stop-pump", "release", "close-complete"] asyncio.run(run()) @@ -32321,6 +32374,10 @@ async def run() -> None: context = await browser.new_context() page = await context.new_page() + page._sync._response_log.append(Response(url="https://example.test/retained")) + page._sync._navigation_responses.append( + Response(url="https://example.test/navigation", _body_cache=b"body") + ) page_close_events = [] context_close_events = [] page.on("close", page_close_events.append) @@ -32329,6 +32386,8 @@ async def run() -> None: await asyncio.gather(page.close(), page.close()) assert core.sent_target_close_count() - target_closes == 1 assert len(page_close_events) == 1 + assert page._sync._response_log == [] + assert page._sync._navigation_responses == [] context_disposals = core.sent_context_dispose_count() await asyncio.gather(context.close(), context.close()) @@ -32922,11 +32981,11 @@ async def run() -> None: handler_entered = threading.Event() release_handler = threading.Event() - def block_one_pump_delivery(kind, payload): + def block_one_pump_delivery(kind, payload, **kwargs): if not handler_entered.is_set(): handler_entered.set() release_handler.wait(5) - return original_handler(kind, payload) + return original_handler(kind, payload, **kwargs) page._sync._handle_observation_event = block_one_pump_delivery try: @@ -32997,12 +33056,12 @@ async def run() -> None: release = threading.Event() handled = [] - def slow_first_handler(kind, payload): + def slow_first_handler(kind, payload, **kwargs): handled.append(kind) if len(handled) == 1: entered.set() release.wait(5) - return original_handler(kind, payload) + return original_handler(kind, payload, **kwargs) page._sync._handle_observation_event = slow_first_handler consumer = asyncio.create_task( From 91799b58479a8ef9ea861921906832a1d7c93c95 Mon Sep 17 00:00:00 2001 From: suchintan <3853670+suchintan@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:38:26 +0000 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=94=84=20synced=20local=20'tools/'=20?= =?UTF-8?q?with=20remote=20'tools/'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/Skyvern-AI/rustwright-cloud/pull/239 --- tools/generate_async_api.py | 3 +- tools/run_benchmark_matrix.py | 219 ++++++++++++++++++++++++++++------ 2 files changed, 184 insertions(+), 38 deletions(-) diff --git a/tools/generate_async_api.py b/tools/generate_async_api.py index 4533a09..5c7cd21 100644 --- a/tools/generate_async_api.py +++ b/tools/generate_async_api.py @@ -107,7 +107,7 @@ "AsyncExpectation": ("_run_sync_assertion", "to_have_text", "to_contain_text", "to_be_visible", "to_be_hidden", "to_be_enabled", "to_be_disabled", "to_be_editable", "to_be_checked", "to_be_attached", "to_be_empty", "to_be_focused", "to_have_count", "to_have_value", "to_have_values", "to_have_attribute", "to_have_id", "to_have_class", "to_contain_class", "to_have_role", "to_have_accessible_name", "to_have_accessible_description", "to_have_accessible_error_message", "to_match_aria_snapshot", "to_have_css", "to_have_js_property", "to_be_in_viewport", "to_be_ok", "to_have_title", "to_have_url"), "AsyncBrowser": ("__aenter__", "__aexit__", "new_page", "new_context", "close", "_close_native"), "AsyncBrowserContext": ("__aenter__", "__aexit__", "new_page", "close", "_close_for_browser_close", "_close_native", "_close_native_impl", "expose_function", "expose_binding", "route", "unroute", "unroute_all", "wait_for_event", "route_web_socket"), - "AsyncPage": ("_event_pump", "_consume_event_batch", "goto", "wait_for_load_state", "evaluate", "add_script_tag", "add_style_tag", "expose_function", "expose_binding", "wait_for_selector", "click", "fill", "inner_text", "wait_for_event", "route", "unroute", "unroute_all", "route_web_socket", "screenshot", "add_locator_handler", "remove_locator_handler", "close", "_close_native"), + "AsyncPage": ("__aenter__", "__aexit__", "_event_pump", "_consume_event_batch", "goto", "wait_for_load_state", "evaluate", "add_script_tag", "add_style_tag", "expose_function", "expose_binding", "wait_for_selector", "click", "fill", "inner_text", "wait_for_event", "route", "unroute", "unroute_all", "route_web_socket", "screenshot", "add_locator_handler", "remove_locator_handler", "close", "_close_native"), "AsyncFrame": ("wait_for_load_state",), "AsyncLocator": ("drag_to",), "AsyncAPIRequestContext": ("_run_request", "fetch", "get", "post", "put", "patch", "delete", "head"), @@ -133,6 +133,7 @@ "_install_async_expectation_negated_aliases.make_alias.alias", "_call_async_assertion_impl_method", "_make_async_assertion_impl_method.method", + "_wrap_async_event_handler.wrapper.run_awaitable_with_state.await_result", "_make_async_assertion_public_method.method", ) diff --git a/tools/run_benchmark_matrix.py b/tools/run_benchmark_matrix.py index 83e25b0..51d2252 100644 --- a/tools/run_benchmark_matrix.py +++ b/tools/run_benchmark_matrix.py @@ -16,6 +16,16 @@ ROOT = Path(__file__).resolve().parents[1] DEFAULT_IMPLS = ["rustwright", "playwright", "typescript-playwright"] STRICT_IMPLS = ["rustwright", "playwright"] +REQUIRED_PSS_ROLES = { + "browser", + "renderer", + "gpu", + "utility", + "network", + "python-host", + "node-driver", + "other", +} def output_to_text(value: Any) -> str: @@ -30,6 +40,16 @@ def default_iterations() -> int: return int(os.environ.get("BENCHMARK_FULL_ITERATIONS") or os.environ.get("BENCHMARK_ITERATIONS") or "10") +def positive_repetitions(value: str) -> int: + try: + repetitions = int(value) + except ValueError: + raise argparse.ArgumentTypeError("repetitions must be an integer") from None + if repetitions < 1: + raise argparse.ArgumentTypeError("repetitions must be at least 1") + return repetitions + + def extract_json(output: str) -> dict[str, Any]: decoder = json.JSONDecoder() for index, char in enumerate(output): @@ -86,7 +106,7 @@ def run_impl(args: argparse.Namespace, implementation: str) -> dict[str, Any]: return result result = extract_json(combined) memory = result.get("memory") - memory_available = ( + rss_available = ( isinstance(memory, dict) and memory.get("available") is True and all( @@ -95,14 +115,38 @@ def run_impl(args: argparse.Namespace, implementation: str) -> dict[str, Any]: ) and memory["rss_tree_kb"] >= memory["rss_self_kb"] ) + pss_by_role = memory.get("pss_by_role_kb") if isinstance(memory, dict) else None + pss_required = isinstance(memory, dict) and memory.get("pss_required") is True + pss_values_supplied = isinstance(memory, dict) and ( + memory.get("pss_tree_kb") is not None + or pss_by_role is not None + or memory.get("pss_available") is True + ) + pss_values_valid = ( + isinstance(memory, dict) + and type(memory.get("pss_tree_kb")) is int + and memory["pss_tree_kb"] > 0 + and isinstance(pss_by_role, dict) + and set(pss_by_role) == REQUIRED_PSS_ROLES + and all(type(value) is int and value >= 0 for value in pss_by_role.values()) + and sum(pss_by_role.values()) == memory["pss_tree_kb"] + and memory.get("pss_available") is True + ) + pss_valid = pss_values_valid if pss_values_supplied else not pss_required + memory_available = rss_available and ( + pss_valid if pss_required or pss_values_supplied else True + ) if memory_available: result["status"] = "passed" else: result["status"] = "failed" result["failure_kind"] = "memory_unavailable" + missing = "process-tree RSS" + if pss_required or pss_values_supplied: + missing += " or PSS" result["output_tail"] = ( - "benchmark completed, but process-tree RSS was unavailable; " - "canonical matrix runs require paired latency and memory results" + f"benchmark completed, but {missing} was unavailable or invalid; " + "canonical matrix runs require paired latency and complete memory results" ) result["container_isolation"] = "separate_container" result["command"] = " ".join(command) @@ -113,6 +157,31 @@ def run_impl(args: argparse.Namespace, implementation: str) -> dict[str, Any]: result["matrix_lifecycle"] = args.lifecycle return result +def complete_repetition_group(items: list[dict[str, Any]], planned_repetitions: int) -> bool: + if len(items) != planned_repetitions or any(item.get("status") != "passed" for item in items): + return False + repetitions = [item.get("repetition") for item in items] + if any(repetition is not None for repetition in repetitions): + return set(repetitions) == set(range(1, planned_repetitions + 1)) + return True + + +def implementation_repetition_status( + results: list[dict[str, Any]], + implementations: list[str], + planned_repetitions: int, +) -> dict[str, str]: + grouped: dict[str, list[dict[str, Any]]] = {} + for item in results: + grouped.setdefault(item.get("implementation", ""), []).append(item) + return { + implementation: ( + "passed" + if complete_repetition_group(grouped.get(implementation, []), planned_repetitions) + else "failed" + ) + for implementation in implementations + } def passed_results(results: list[dict[str, Any]]) -> list[dict[str, Any]]: return [item for item in results if item.get("status") == "passed"] @@ -292,15 +361,29 @@ def case_winners_from_aggregate(aggregate: dict[str, Any]) -> dict[str, Any]: return {"win_counts": win_counts, "cases": rows} -def aggregate_repetitions(results: list[dict[str, Any]]) -> dict[str, Any]: +def aggregate_repetitions( + results: list[dict[str, Any]], planned_repetitions: int | None = None +) -> dict[str, Any]: grouped: dict[str, list[dict[str, Any]]] = {} - for item in passed_results(results): + for item in results: grouped.setdefault(item["implementation"], []).append(item) + if planned_repetitions is not None: + grouped = { + implementation: items + for implementation, items in grouped.items() + if complete_repetition_group(items, planned_repetitions) + } + else: + grouped = { + implementation: passed_results(items) + for implementation, items in grouped.items() + if passed_results(items) + } aggregate = {} for implementation, items in grouped.items(): totals = [float(item["total_mean_ms"]) for item in items] case_names = sorted({name for item in items for name in item.get("cases", {})}) - memory_fields = ("rss_self_kb", "rss_tree_kb") + memory_fields = ("rss_self_kb", "rss_tree_kb", "pss_tree_kb") memory = { field: aggregate_optional_values( [ @@ -311,12 +394,40 @@ def aggregate_repetitions(results: list[dict[str, Any]]) -> dict[str, Any]: ) for field in memory_fields } + roles = sorted( + { + role + for item in items + for role in (item.get("memory", {}).get("pss_by_role_kb") or {}) + } + ) + memory["pss_by_role_kb"] = { + role: aggregate_optional_values( + [ + float(item["memory"]["pss_by_role_kb"][role]) + for item in items + if role in (item.get("memory", {}).get("pss_by_role_kb") or {}) + ] + ) + for role in roles + } + memory["pss_by_role_statistic"] = "independent_marginal_medians" + memory["pss_required"] = any( + item.get("memory", {}).get("pss_required") is True for item in items + ) + memory["pss_statistic"] = "at_tree_rss_peak" aggregate[implementation] = { "runs": len(items), "total_mean_ms": aggregate_values(totals), "memory": memory, "cases": { - name: aggregate_values([float(item["cases"][name]["mean_ms"]) for item in items if name in item.get("cases", {})]) + name: aggregate_values( + [ + float(item["cases"][name]["mean_ms"]) + for item in items + if name in item.get("cases", {}) + ] + ) for name in case_names }, } @@ -427,34 +538,44 @@ def markdown_table(result: dict[str, Any]) -> str: "| Implementation | Status | Total median ms | Comparison |", "| --- | --- | ---: | --- |", ] - speedup = result["speedups"] - rows_source = result.get("aggregate", {}).items() if result.get("aggregate") else [] - if rows_source: - for name, item in rows_source: - total = item["total_mean_ms"]["median"] - p25 = item["total_mean_ms"]["p25"] - p75 = item["total_mean_ms"]["p75"] - comparison = "baseline" - rustwright = result["aggregate"].get("rustwright") - if name != "rustwright" and rustwright: - baseline = float(total) - rust = float(rustwright["total_mean_ms"]["median"]) - if baseline > 0: - reduction = (baseline - rust) / baseline * 100 - comparison = f"Rustwright lower by {reduction:.1f}%" if reduction >= 0 else f"Rustwright higher by {abs(reduction):.1f}%" - rows.append(f"| {name} | passed | {float(total):.2f} (p25 {float(p25):.2f}, p75 {float(p75):.2f}) | {comparison} |") - return "\n".join(rows) - for item in result["results"]: - name = item["implementation"] - if item.get("status") != "passed": + aggregate = result.get("aggregate") or {} + statuses = result.get("implementation_status") or {} + implementations = result.get("implementations") or list( + dict.fromkeys(item["implementation"] for item in result.get("results", [])) + ) + rustwright = aggregate.get("rustwright") + for name in implementations: + item = aggregate.get(name) + status = statuses.get(name, "passed" if item is not None else "failed") + if status != "passed" or item is None: rows.append(f"| {name} | failed | | see JSON output |") continue + total = item["total_mean_ms"]["median"] + p25 = item["total_mean_ms"]["p25"] + p75 = item["total_mean_ms"]["p75"] comparison = "baseline" - if name != "rustwright": - value = speedup.get(f"vs_{name}_reduction_pct") - if value is not None: - comparison = f"Rustwright lower by {value:.1f}%" if value >= 0 else f"Rustwright higher by {abs(value):.1f}%" - rows.append(f"| {name} | passed | {float(item['total_mean_ms']):.2f} | {comparison} |") + if name != "rustwright" and rustwright: + baseline = float(total) + rust = float(rustwright["total_mean_ms"]["median"]) + if baseline > 0: + reduction = (baseline - rust) / baseline * 100 + comparison = ( + f"Rustwright lower by {reduction:.1f}%" + if reduction >= 0 + else f"Rustwright higher by {abs(reduction):.1f}%" + ) + rows.append( + f"| {name} | passed | {float(total):.2f} " + f"(p25 {float(p25):.2f}, p75 {float(p75):.2f}) | {comparison} |" + ) + if result.get("diagnostic_aggregate"): + rows.extend( + [ + "", + "Noncanonical diagnostic aggregates are available under " + "`diagnostic_aggregate`; they are excluded from canonical metrics.", + ] + ) return "\n".join(rows) @@ -480,7 +601,12 @@ def main() -> int: description="Run benchmark implementations in separate Docker containers through tools/docker_test.sh." ) parser.add_argument("--iterations", type=int, default=default_iterations()) - parser.add_argument("--repetitions", type=int, default=1, help="Repeat the full implementation matrix this many times.") + parser.add_argument( + "--repetitions", + type=positive_repetitions, + default=1, + help="Repeat the full implementation matrix this many times.", + ) parser.add_argument("--impl", action="append", help="Implementation to run. Defaults to the standard matrix.") parser.add_argument("--include-puppeteer", action="store_true", help="Add typescript-puppeteer to the default matrix.") parser.add_argument("--suite", choices=["equivalent", "strict"], default="equivalent") @@ -528,7 +654,11 @@ def main() -> int: unsupported = [implementation for implementation in implementations if implementation not in STRICT_IMPLS] if unsupported: raise SystemExit(f"--suite strict only supports: {', '.join(STRICT_IMPLS)}; got {', '.join(unsupported)}") - planned_runs = [(repetition, implementation) for repetition in range(args.repetitions) for implementation in implementations] + planned_runs = [ + (repetition, implementation) + for repetition in range(args.repetitions) + for implementation in implementations + ] docker_preflight = docker_health_check(args.docker_preflight_timeout) if docker_preflight.get("status") != "healthy": results = skipped_results_after_docker_preflight(planned_runs) @@ -552,21 +682,36 @@ def main() -> int: results.append(result) if result.get("failure_kind") == "docker_daemon_error": stop_remaining_reason = "skipped_after_docker_daemon_error" - aggregate = aggregate_repetitions(results) + implementation_status = implementation_repetition_status( + results, + implementations, + args.repetitions, + ) + canonical_complete = all(status == "passed" for status in implementation_status.values()) + diagnostic_aggregate = aggregate_repetitions(results) + aggregate = ( + aggregate_repetitions(results, planned_repetitions=args.repetitions) + if canonical_complete + else {} + ) result = { "iterations": args.iterations, "repetitions": args.repetitions, "suite": args.suite, "lifecycle": args.lifecycle, "case_filters": args.case_filters or [], + "implementations": implementations, + "implementation_status": implementation_status, + "canonical_aggregate_complete": canonical_complete, "container_isolation": "one_container_per_implementation_per_repetition", "metadata": matrix_metadata(args, implementations, docker_preflight), "results": results, "rustwright_rebuild_target_cache": os.environ.get("RUSTWRIGHT_DOCKER_REBUILD_TARGET_CACHE") == "1", "aggregate": aggregate, + "diagnostic_aggregate": diagnostic_aggregate if not canonical_complete else {}, "common_case_comparison": common_case_comparison(aggregate), - "case_winners": case_winners_from_aggregate(aggregate) if aggregate else case_winners_from_results(results), - "speedups": speedups_from_aggregate(aggregate) if aggregate else speedups_from_results(results), + "case_winners": case_winners_from_aggregate(aggregate) if aggregate else {}, + "speedups": speedups_from_aggregate(aggregate) if aggregate else {}, } output_path = Path(args.output) if args.output else default_result_path(args) if not output_path.is_absolute(): From 44a6ccb1e5ee3d240d78a7bd084b844dad5399ad Mon Sep 17 00:00:00 2001 From: suchintan <3853670+suchintan@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:38:27 +0000 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=94=84=20synced=20local=20'CHANGELOG.?= =?UTF-8?q?md'=20with=20remote=20'CHANGELOG.md'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/Skyvern-AI/rustwright-cloud/pull/239 --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 898c987..64696a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,16 @@ All notable user-facing changes to Rustwright are documented in this file. page. Closing the page disposes its context, matching Playwright isolation. - Navigation internals now retain response bodies and drain console and page-error history before each navigation. +- Navigation response retention now bounds page-owned navigation-response + entries to 20 and all page-owned response-body storage, including staged + fulfilled-route bodies, to 8 MiB. Evicted responses release page-owned + request, response, and fulfilled-body links, while a + caller-retained `Response` keeps its cached body readable after eviction + and later renderer-changing navigation. +- Process-tree benchmark memory summaries report RSS on every platform and + include Linux PSS fields (`pss_tree_kb`, `pss_by_role_kb`, `pss_available`, + and `pss_required`) when PSS is available. Role medians are independent + marginal medians. - Chromium launches now include a broader resource-safe default switch set. - User-provided feature-list launch arguments now merge with defaults into one switch per feature flag.