diff --git a/moli-core/src/runtime/lifecycle_fetch.rs b/moli-core/src/runtime/lifecycle_fetch.rs index 9f652c247ca..5be46e656e4 100644 --- a/moli-core/src/runtime/lifecycle_fetch.rs +++ b/moli-core/src/runtime/lifecycle_fetch.rs @@ -20,7 +20,8 @@ impl Browser { /// The decision runs in the renderer owner turn that observes DCL/load; /// it does not expose an intermediate Page or require a second owner /// command. The original `timeout` covers the request, the first lifecycle - /// target, any successor-navigation grace period, and the successor target. + /// target, any successor-navigation grace period, the successor target, + /// and the post-Load stability observation requested by Done. pub async fn fetch_document_with_lifecycle_decider( &self, request: Request, @@ -43,7 +44,8 @@ impl Browser { } /// Applies a lifecycle decision using a caller-owned absolute deadline. - /// The same deadline can then gate response, selector, and script waits; + /// Done consumes this deadline while observing post-Load DOM stability; + /// the caller can reuse any remaining budget for later waits. /// `raw_document_policy` decides whether a non-Page response is read or /// rejected from its headers. pub async fn fetch_document_with_lifecycle_decider_and_deadline( @@ -67,14 +69,24 @@ impl Browser { "a lifecycle decider requires DCL, load, or done" ); let decider = RendererLifecycleDecider::new(decider); - self.fetch_document_to_base_stage( - request, - wait_until, - deadline, - RendererReplyBoundary::Stage, - Some(decider), - raw_document_policy, - ) - .await + let fetched = self + .fetch_document_to_base_stage( + request, + wait_until, + deadline, + RendererReplyBoundary::Stage, + Some(decider), + raw_document_policy, + ) + .await?; + + match fetched { + FetchedDocument::Page(mut page) => { + self.wait_for_page_readiness_with_deadline(&mut page, wait_until, deadline) + .await?; + Ok(FetchedDocument::Page(page)) + } + FetchedDocument::Raw(raw) => Ok(FetchedDocument::Raw(raw)), + } } } diff --git a/moli-core/src/runtime/mod.rs b/moli-core/src/runtime/mod.rs index cf35a9878a9..03a24cc9965 100644 --- a/moli-core/src/runtime/mod.rs +++ b/moli-core/src/runtime/mod.rs @@ -94,13 +94,10 @@ impl RawDocumentFetchPolicy { } fn is_fetch_readiness_timeout(error: &anyhow::Error, wait_until: RenderedDomWaitUntil) -> bool { - let expected = match wait_until { - RenderedDomWaitUntil::NetworkIdle => "timed out waiting for networkidle", - RenderedDomWaitUntil::DomStable => "timed out waiting for domstable", - RenderedDomWaitUntil::DomContentLoaded - | RenderedDomWaitUntil::Load - | RenderedDomWaitUntil::Done => return false, + let Some(wait) = wait_until.best_effort_page_wait() else { + return false; }; + let expected = wait.timeout_message(); error.chain().any(|cause| cause.to_string() == expected) } @@ -178,13 +175,29 @@ pub enum RenderedDomWaitUntil { Load, NetworkIdle, DomStable, + /// Reach Load, then observe best-effort DOM stability. Done, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BestEffortPageWait { + NetworkIdle, + DomStable, +} + +impl BestEffortPageWait { + fn timeout_message(self) -> &'static str { + match self { + Self::NetworkIdle => "timed out waiting for networkidle", + Self::DomStable => "timed out waiting for domstable", + } + } +} + impl RenderedDomWaitUntil { /// The concrete lifecycle boundary at which a live `Page` can first be - /// handed back to the host. Network-idle and DOM-stable are observations - /// made on that live Page, after Load and DCL respectively. + /// handed back to the host. Network-idle, DOM-stable, and Done add an + /// observation on that live Page after Load, DCL, and Load respectively. fn base_stage(self) -> PageVmInitStage { match self { Self::DomContentLoaded | Self::DomStable => PageVmInitStage::DomContentLoaded, @@ -192,8 +205,12 @@ impl RenderedDomWaitUntil { } } - fn has_best_effort_page_wait(self) -> bool { - matches!(self, Self::NetworkIdle | Self::DomStable) + fn best_effort_page_wait(self) -> Option { + match self { + Self::NetworkIdle => Some(BestEffortPageWait::NetworkIdle), + Self::DomStable | Self::Done => Some(BestEffortPageWait::DomStable), + Self::DomContentLoaded | Self::Load => None, + } } } @@ -498,8 +515,9 @@ impl Browser { } /// Reaches the concrete lifecycle boundary that makes a live Page - /// available. For NetworkIdle and DomStable this is only the Load or DCL - /// base stage; their best-effort observation runs outside materialization. + /// available. For NetworkIdle, DomStable, and Done this is only the Load, + /// DCL, or Load base stage; their best-effort observation runs outside + /// materialization. async fn fetch_document_to_base_stage( &self, request: Request, @@ -523,8 +541,8 @@ impl Browser { ); // This deadline is strict: if the response or base DCL/Load stage does // not arrive in time, there is no Page on which best-effort readiness - // can operate. Only the later NetworkIdle/DomStable observation may - // soften an expired deadline. + // can operate. Only the later NetworkIdle/DOM-stability observation + // may soften an expired deadline. let outer_deadline = deadline.at(); let requested_url = request.url.clone(); if is_about_blank_url(&requested_url) { @@ -807,7 +825,10 @@ impl Browser { .await } - /// Observes NetworkIdle or DomStable with the unspent part of `deadline`. + /// Observes NetworkIdle or DOM stability with the unspent part of + /// `deadline`. Done uses the DOM-stability observation after first reaching + /// Load, so post-load fragments are included by the default completion + /// strategy without changing the explicit Load boundary. /// /// A timeout in this observation is deliberately best-effort: the Page /// has already reached its required Load/DCL base stage, so the current @@ -819,29 +840,24 @@ impl Browser { wait_until: RenderedDomWaitUntil, deadline: FetchDeadline, ) -> Result<()> { - if !wait_until.has_best_effort_page_wait() { + let Some(page_wait) = wait_until.best_effort_page_wait() else { return Ok(()); - } + }; let loader = self.resource_request_client(); let remaining = deadline.remaining(); - let result = match wait_until { - RenderedDomWaitUntil::NetworkIdle => { + let result = match page_wait { + BestEffortPageWait::NetworkIdle => { tokio::time::timeout_at( deadline.at(), page.wait_for_network_idle(&loader, remaining), ) .await } - RenderedDomWaitUntil::DomStable => { + BestEffortPageWait::DomStable => { tokio::time::timeout_at(deadline.at(), page.wait_for_dom_stable(&loader, remaining)) .await } - RenderedDomWaitUntil::DomContentLoaded - | RenderedDomWaitUntil::Load - | RenderedDomWaitUntil::Done => unreachable!( - "concrete lifecycle modes returned before starting a best-effort Page wait" - ), }; let timeout_error = match result { diff --git a/moli-core/src/runtime/tests.rs b/moli-core/src/runtime/tests.rs index fcdb9baaece..8f671ded57a 100644 --- a/moli-core/src/runtime/tests.rs +++ b/moli-core/src/runtime/tests.rs @@ -2147,6 +2147,11 @@ async fn best_effort_page_readiness_consumes_only_the_remaining_fetch_deadline() "/wait-until-slow-interval-dom-mutation", "id=\"mutation-count\"", ), + ( + RenderedDomWaitUntil::Done, + "/wait-until-slow-interval-dom-mutation", + "id=\"mutation-count\"", + ), ] { // The main response costs 500 ms. Readiness must receive only the // roughly 500 ms left in this one-second plan, not a fresh second. @@ -2216,6 +2221,10 @@ async fn page_fetch_wait_until_apis_do_not_restart_best_effort_timeout() -> Resu RenderedDomWaitUntil::DomStable, "/wait-until-slow-interval-dom-mutation", ), + ( + RenderedDomWaitUntil::Done, + "/wait-until-slow-interval-dom-mutation", + ), ] { let started = std::time::Instant::now(); let page = browser @@ -2241,6 +2250,7 @@ async fn best_effort_readiness_does_not_soften_a_base_lifecycle_timeout() -> Res for wait_until in [ RenderedDomWaitUntil::NetworkIdle, RenderedDomWaitUntil::DomStable, + RenderedDomWaitUntil::Done, ] { // No Page exists at 200 ms because the main response itself takes // 500 ms. Best effort applies only after the Load/DCL base boundary. diff --git a/moli-core/tests/fetch_behaviors.rs b/moli-core/tests/fetch_behaviors.rs index cc18e39b340..9d42f90c7b4 100644 --- a/moli-core/tests/fetch_behaviors.rs +++ b/moli-core/tests/fetch_behaviors.rs @@ -1566,6 +1566,47 @@ async fn fetch_with_domstable_waits_for_delayed_fetch_dom_mutation() -> Result<( Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn fetch_with_done_waits_for_post_load_fetch_dom_mutation() -> Result<()> { + let server = FixtureServer::spawn().await?; + let browser = Browser::new(AppConfig::default())?; + + let page = browser + .fetch_with_wait_until( + &server.url("/wait-until-delayed-fetch"), + RenderedDomWaitUntil::Done, + Duration::from_secs(5), + ) + .await?; + let html = page.serialize_html_async().await?; + assert!(html.contains("data-state=\"settled\""), "html={html}"); + assert!(html.contains("id=\"late\""), "html={html}"); + assert!(html.contains(">settled<"), "html={html}"); + + server.shutdown().await; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn fetch_with_done_waits_for_external_script_inserted_after_load() -> Result<()> { + let server = FixtureServer::spawn().await?; + let browser = Browser::new(AppConfig::default())?; + + let page = browser + .fetch_with_wait_until( + &server.url("/wait-until-post-load-external-script"), + RenderedDomWaitUntil::Done, + Duration::from_secs(5), + ) + .await?; + let html = page.serialize_html_async().await?; + assert!(html.contains("id=\"post-load-results\""), "html={html}"); + assert!(html.contains(">rendered result<"), "html={html}"); + + server.shutdown().await; + Ok(()) +} + #[tokio::test(flavor = "multi_thread")] async fn fetch_with_domstable_waits_for_late_complete_dom_mutation() -> Result<()> { let server = FixtureServer::spawn().await?; diff --git a/moli-test-support/src/routes_wait.rs b/moli-test-support/src/routes_wait.rs index f0bffdf247c..4000163d502 100644 --- a/moli-test-support/src/routes_wait.rs +++ b/moli-test-support/src/routes_wait.rs @@ -19,6 +19,23 @@ const WAIT_UNTIL_DOMCONTENTLOADED_RUNTIME_SCRIPT_VERY_SLOW_HTML: &str = ""; +const WAIT_UNTIL_POST_LOAD_EXTERNAL_SCRIPT_HTML: &str = r#" +post-load external script +
initial
+ +"#; const WAIT_UNTIL_COMPLETE_DELAYED_DOM_MUTATION_HTML: &str = ""; const WAIT_UNTIL_COMPLETE_SLOW_FETCH_HTML: &str = ""; const WAIT_UNTIL_COMPLETE_SLOW_XHR_HTML: &str = ""; @@ -276,6 +293,10 @@ pub(super) fn add_wait_routes(router: Router) -> Router { "/wait-until-delayed-fetch", get(wait_until_delayed_fetch_page), ) + .route( + "/wait-until-post-load-external-script", + get(wait_until_post_load_external_script_page), + ) .route( "/wait-until-complete-delayed-dom-mutation", get(wait_until_complete_delayed_dom_mutation_page), @@ -664,6 +685,10 @@ async fn wait_until_delayed_fetch_page() -> Html<&'static str> { Html(WAIT_UNTIL_DELAYED_FETCH_HTML) } +async fn wait_until_post_load_external_script_page() -> Html<&'static str> { + Html(WAIT_UNTIL_POST_LOAD_EXTERNAL_SCRIPT_HTML) +} + async fn wait_until_complete_delayed_dom_mutation_page() -> Html<&'static str> { Html(WAIT_UNTIL_COMPLETE_DELAYED_DOM_MUTATION_HTML) } diff --git a/moli/src/cli.rs b/moli/src/cli.rs index 4dc5b4e5efd..618622a3713 100644 --- a/moli/src/cli.rs +++ b/moli/src/cli.rs @@ -175,6 +175,9 @@ pub struct FetchArgs { #[arg(long, value_enum, value_delimiter = ',')] pub strip_mode: Vec, + /// Choose the page completion boundary. `done` reaches Load and then + /// observes best-effort DOM stability; use `load` for the exact browser + /// lifecycle milestone. #[arg( long = "wait", alias = "wait-until", @@ -270,8 +273,9 @@ pub struct FetchArgs { /// Maximum total readiness time in milliseconds. Initial and HTTP-error /// replacement navigations, the selected lifecycle stage, response match, - /// selector, and script waits share one absolute deadline. Network-idle and - /// DOM-stable return the current page with a warning when it expires. + /// selector, and script waits share one absolute deadline. Done, + /// network-idle, and DOM-stable return the current page with a warning when + /// their best-effort observation expires. #[arg( short, long, diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rs index 4e166243410..b9da8f382c3 100644 --- a/moli/tests/fetch_cli.rs +++ b/moli/tests/fetch_cli.rs @@ -2672,6 +2672,52 @@ fn cli_domstable_waits_for_late_content() -> Result<()> { Ok(()) } +#[test] +fn cli_default_done_waits_for_post_load_content() -> Result<()> { + let runtime = tokio::runtime::Runtime::new()?; + let server = runtime.block_on(FixtureServer::spawn())?; + let url = server.url("/wait-until-delayed-fetch"); + let output = run_fetch_cli_with_default_wait_and_dump(&url, "html")?; + runtime.block_on(server.shutdown()); + + assert!( + output.status.success(), + "moli fetch failed: stdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let stdout = clean_output(&output.stdout); + assert!(stdout.contains("data-state=\"settled\""), "stdout={stdout}"); + assert!(stdout.contains("id=\"late\""), "stdout={stdout}"); + assert!(stdout.contains(">settled<"), "stdout={stdout}"); + Ok(()) +} + +#[test] +fn cli_default_done_waits_for_external_script_inserted_after_load() -> Result<()> { + let runtime = tokio::runtime::Runtime::new()?; + let server = runtime.block_on(FixtureServer::spawn())?; + let url = server.url("/wait-until-post-load-external-script"); + let output = run_fetch_cli_with_default_wait_and_dump(&url, "html")?; + runtime.block_on(server.shutdown()); + + assert!( + output.status.success(), + "moli fetch failed: stdout={}\nstderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let stdout = clean_output(&output.stdout); + assert!( + stdout.contains("id=\"post-load-results\""), + "stdout={stdout}" + ); + assert!(stdout.contains(">rendered result<"), "stdout={stdout}"); + Ok(()) +} + #[test] fn cli_networkidle_timeout_logs_warning_and_returns_best_effort_output() -> Result<()> { let runtime = tokio::runtime::Runtime::new()?; @@ -2783,6 +2829,15 @@ fn cli_domstable_uses_only_the_deadline_left_after_a_slow_main_response() -> Res ) } +#[test] +fn cli_done_uses_only_the_deadline_left_after_a_slow_main_response() -> Result<()> { + assert_cli_best_effort_stage_uses_remaining_deadline( + "done", + "/wait-until-slow-interval-dom-mutation", + "id=\"mutation-count\"", + ) +} + #[test] fn cli_best_effort_stage_does_not_soften_a_slow_base_lifecycle_timeout() -> Result<()> { let runtime = tokio::runtime::Runtime::new()?; @@ -2923,6 +2978,11 @@ fn cli_quiet_page_domstable_uses_remaining_best_effort_budget() -> Result<()> { assert_cli_quiet_page_uses_remaining_best_effort_budget("domstable") } +#[test] +fn cli_quiet_page_done_uses_remaining_best_effort_budget() -> Result<()> { + assert_cli_quiet_page_uses_remaining_best_effort_budget("done") +} + #[test] fn cli_post_readiness_wait_cannot_restart_an_exhausted_best_effort_deadline() -> Result<()> { let runtime = tokio::runtime::Runtime::new()?; @@ -3128,12 +3188,12 @@ fn cli_zero_redirect_wait_returns_terminal_302_before_delayed_page_work() -> Res } #[test] -fn cli_slow_streaming_terminal_302_does_not_add_a_post_load_wait() -> Result<()> { +fn cli_slow_streaming_terminal_302_exact_stage_does_not_add_a_post_load_wait() -> Result<()> { let runtime = tokio::runtime::Runtime::new()?; let server = runtime.block_on(FixtureServer::spawn())?; let url = server.url("/wait-until-redirect-status-slow-streaming-302"); - for wait_until in ["domcontentloaded", "load", "done"] { + for wait_until in ["domcontentloaded", "load"] { let output = run_fetch_cli_with_wait_until_and_dump(&url, wait_until, "json")?; assert!( output.status.success(), @@ -3746,12 +3806,12 @@ fn cli_default_done_returns_raw_http_error_status_and_body() -> Result<()> { } #[test] -fn cli_slow_streaming_500_returns_complete_body_without_an_additive_wait() -> Result<()> { +fn cli_slow_streaming_500_exact_stage_has_no_additive_wait() -> Result<()> { let runtime = tokio::runtime::Runtime::new()?; let server = runtime.block_on(FixtureServer::spawn())?; let url = server.url("/wait-until-http-error-slow-streaming-500"); - for wait_until in ["domcontentloaded", "load", "done"] { + for wait_until in ["domcontentloaded", "load"] { let output = run_fetch_cli_with_wait_until_and_dump(&url, wait_until, "json")?; assert!( output.status.success(),