From 01ae4b7c66cb809d1cb9d13075504ce27b1b7672 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:09:25 +0800 Subject: [PATCH 1/6] fix(parser): preserve text document MIME and encoding semantics --- moli-encoding/src/document.rs | 25 +++ moli-encoding/src/tests.rs | 96 +++++++++++ moli-parser/src/html.rs | 93 +++++++++- moli-parser/src/session.rs | 14 +- moli-parser/src/stream.rs | 16 +- .../tests/classic/extracted/navigation.rs | 93 ++++++++++ moli-protocol/src/domains/emulation/tests.rs | 18 +- .../src/domains/page/tests/lifecycle.rs | 16 +- .../src/domains/page/tests/runtime.rs | 10 +- .../tests_background_staging/environment.rs | 8 +- moli-renderer-v8/src/dom_parser.rs | 6 +- moli-renderer-v8/src/live_document_parser.rs | 23 +++ .../context_host/child_documents/commit.rs | 1 + .../child_documents/live_parser.rs | 8 + .../context_host/child_documents/loads.rs | 23 ++- .../src/runtime/phase_one/bootstrap.rs | 2 + .../src/runtime/phase_one/state.rs | 19 ++ .../src/runtime/phase_one/streaming.rs | 162 ++++++++++++++++-- moli-web-mime/src/classification.rs | 8 + moli-web-mime/src/lib.rs | 6 +- moli-web-mime/src/tests.rs | 22 +++ moli/src/fetch_dump/tests.rs | 4 +- 22 files changed, 624 insertions(+), 49 deletions(-) diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rs index 1ceb79b9f9f..8b3a70398bc 100644 --- a/moli-encoding/src/document.rs +++ b/moli-encoding/src/document.rs @@ -47,9 +47,26 @@ pub struct HtmlDocumentStreamingDecoder { url_hint: Option, decoder: Option, selected_encoding: Option<&'static Encoding>, + sniff_html_declarations: bool, } impl HtmlDocumentStreamingDecoder { + /// Decode a text document without interpreting literal HTML/XML declarations. + /// JSON defaults to UTF-8; other text retains the configured legacy detector. + pub fn new_text_document( + headers: &[(String, Vec)], + url_hint: &str, + detector: LegacyEncodingDetector, + json: bool, + ) -> Self { + let mut decoder = Self::new_with_options(headers, None, Some(url_hint), Some(detector)); + decoder.sniff_html_declarations = false; + if json && decoder.transport_encoding.is_none() { + decoder.transport_encoding = Some(encoding_rs::UTF_8); + } + decoder + } + pub fn new(headers: &[(String, Vec)]) -> Self { Self::new_with_options(headers, None, None, None) } @@ -88,6 +105,7 @@ impl HtmlDocumentStreamingDecoder { url_hint: url_hint.map(str::to_owned), decoder: None, selected_encoding: None, + sniff_html_declarations: true, } } @@ -97,6 +115,7 @@ impl HtmlDocumentStreamingDecoder { pub fn document_encoding_name(&self) -> &'static str { self.selected_encoding_name() + .or_else(|| self.transport_encoding.map(Encoding::name)) .unwrap_or(self.fallback_encoding.name()) } @@ -149,6 +168,12 @@ impl HtmlDocumentStreamingDecoder { if let Some(encoding) = self.transport_encoding { return Some(encoding); } + if !self.sniff_html_declarations { + return (finishing || self.sniff_buffer.len() >= 1024).then(|| { + self.detected_legacy_content_encoding() + .unwrap_or(self.fallback_encoding) + }); + } if let Some(encoding) = encoding_for_document_utf16_xml_prefix(&self.sniff_buffer) { return Some(encoding); } diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs index 6693d5d5452..2217e0521a1 100644 --- a/moli-encoding/src/tests.rs +++ b/moli-encoding/src/tests.rs @@ -24,6 +24,102 @@ fn unexpected_legacy_encoding_detector( panic!("declared encoding must take precedence over heuristic detection") } +#[test] +fn json_text_document_defaults_to_utf8_across_every_chunk_split() { + let input = "{\"name\":\"Gülçek\"}"; + for split in 0..=input.len() { + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://example.test/", + unexpected_legacy_encoding_detector, + true, + ); + let mut output = decoder.push(&input.as_bytes()[..split]).join(""); + output.push_str(&decoder.push(&input.as_bytes()[split..]).join("")); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, input, "split {split}"); + assert_eq!(decoder.selected_encoding_name(), Some("UTF-8")); + } +} + +#[test] +fn plain_text_document_honors_transport_charset() { + let headers = [( + "Content-Type".to_owned(), + b"text/plain; charset=gbk".to_vec(), + )]; + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &headers, + "https://example.test/", + unexpected_legacy_encoding_detector, + false, + ); + let mut output = decoder.push(&gbk_bytes("家居")).join(""); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, "家居"); + assert_eq!(decoder.selected_encoding_name(), Some("GBK")); +} + +#[test] +fn text_document_bom_precedes_transport_charset() { + let headers = [( + "Content-Type".to_owned(), + b"text/plain; charset=gbk".to_vec(), + )]; + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &headers, + "https://example.test/", + unexpected_legacy_encoding_detector, + false, + ); + assert_eq!(decoder.document_encoding_name(), "GBK"); + assert_eq!(decoder.selected_encoding_name(), None); + assert!(decoder.push(&[0xef]).is_empty()); + assert!(decoder.push(&[0xbb]).is_empty()); + let mut tail = vec![0xbf]; + tail.extend_from_slice("Gülçek".as_bytes()); + let mut output = decoder.push(&tail).join(""); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, "Gülçek"); + assert_eq!(decoder.selected_encoding_name(), Some("UTF-8")); + assert_eq!(decoder.document_encoding_name(), "UTF-8"); +} + +#[test] +fn plain_text_document_retains_legacy_detection_without_a_charset() { + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://legacy.example/", + test_legacy_encoding_detector, + false, + ); + let mut output = decoder.push(&[0x80]).join(""); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, "А"); + assert_eq!(decoder.selected_encoding_name(), Some("IBM866")); +} + +#[test] +fn plain_text_document_ignores_literal_encoding_declarations() { + for prefix in [ + "", + "", + ] { + let mut input = prefix.as_bytes().to_vec(); + input.push(0xe9); + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://example.test/", + test_legacy_encoding_detector, + false, + ); + let mut output = decoder.push(&input).join(""); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, format!("{prefix}é")); + assert_eq!(decoder.selected_encoding_name(), Some("windows-1252")); + } +} + #[test] fn content_type_charset_is_selected() { let headers: Vec<(String, Vec)> = vec![( diff --git a/moli-parser/src/html.rs b/moli-parser/src/html.rs index 11c8a72f507..e66b18367bd 100644 --- a/moli-parser/src/html.rs +++ b/moli-parser/src/html.rs @@ -27,7 +27,7 @@ use super::live_target::{ ParserDomMutationConsumer, ParserDomReadConsumer, ParserElementCreationConsumer, ParserMutationEffectConsumer, ParserMutationEffectDelivery, ParserRuntimeDomSinks, ParserStreamHtmlTreeSinkTarget, new_live_document_root_html_tree_sink_stream, - new_parser_stream_html_tree_sink_stream, + new_parser_stream_html_tree_sink_stream, new_parser_stream_html_tree_sink_target, }; use super::{ ParserSourcePosition, html_chunks, @@ -438,6 +438,47 @@ impl HtmlParser { DocumentStream::new_parser_stream(final_url, self.scripting_enabled) } + /// Start a text document using the HTML tokenizer's plaintext state. + /// Subsequent chunks remain literal text inside the browser-owned `pre`. + pub fn start_text_document(&self, final_url: Url, content_type: &str) -> DocumentStream { + let stream = + DocumentStream::new_text_document(new_parser_stream_html_tree_sink_target(final_url)); + stream.inner.initialize_text_document(); + let mut host = stream.inner.take_parser_stream_dom_host(); + let document_handle = host.document_handle(); + host.set_document_content_type_for_handle(document_handle, content_type); + stream.inner.restore_parser_stream_dom_host(host); + stream + } + + /// Initialize the same literal-text shell in an existing document. The + /// embedder owns that document's MIME metadata and live DOM callbacks. + pub fn start_live_text_document_root( + &self, + final_url: Url, + document_handle: NativeNodeId, + consumer: &mut T, + ) -> DocumentStream + where + T: ParserDomReadConsumer + + ParserDomMutationConsumer + + ParserMutationEffectConsumer + + ParserElementCreationConsumer, + { + let stream = DocumentStream::new_text_document( + ParserStreamHtmlTreeSinkTarget::new_live_document_root(final_url, document_handle), + ); + // SAFETY: the consumer is exclusively borrowed until the step guard + // removes the erased callbacks, including when initialization unwinds. + let sinks = unsafe { ParserRuntimeDomSinks::from_consumer(consumer) }; + stream.inner.enter_runtime_dom_sinks_parse_step(sinks); + { + let step = RuntimeDomSinksParserStep { stream: &stream }; + step.stream.inner.initialize_text_document(); + } + stream + } + pub fn start_live_document_root( &self, final_url: Url, @@ -551,6 +592,17 @@ impl Default for ParserInputQueue { } impl DocumentStream { + fn new_text_document(target: ParserStreamHtmlTreeSinkTarget) -> Self { + let mut options = html_parse_opts_with_scripting(false); + // The byte decoder has already consumed the transport BOM. In plaintext + // every remaining U+FEFF is content, including at a chunk boundary. + options.tokenizer.discard_bom = false; + Self { + inner: HtmlTreeSinkStream::from_target_with_options(target, options), + input: RefCell::default(), + } + } + fn new_parser_stream(final_url: Url, scripting_enabled: bool) -> Self { Self { inner: new_parser_stream_html_tree_sink_stream(final_url, scripting_enabled), @@ -1843,6 +1895,45 @@ mod tests { const HTML_NS: &str = "http://www.w3.org/1999/xhtml"; const MATHML_NS: &str = "http://www.w3.org/1998/Math/MathML"; + #[test] + fn text_document_stream_preserves_literal_markup_and_leading_newline() { + let payload = + "\u{feff}\nGülçek&\u{feff}"; + for content_type in ["text/plain", "application/json", "application/problem+json"] { + let stream = HtmlParser::SCRIPTING_ENABLED.start_text_document( + Url::parse("https://example.test/data").unwrap(), + content_type, + ); + for character in payload.chars() { + stream.feed(&character.to_string()); + } + let document = stream.finish(); + let pre = first_element_by_ns(&document, HTML_NS, "pre"); + assert_eq!(document.text_content(pre).as_deref(), Some(payload)); + assert_eq!(document.document().unwrap().content_type(), content_type); + for tag in ["b", "script"] { + assert!( + document + .elements_by_tag_name_ns( + document.document_node_id(), + Some(HTML_NS), + tag, + true + ) + .is_empty() + ); + } + } + } + + #[test] + fn html_document_still_parses_markup_and_entities() { + let document = parse_test_document("Gülçek&"); + let bold = first_element_by_ns(&document, HTML_NS, "b"); + assert_eq!(document.text_content(bold).as_deref(), Some("Gülçek&")); + assert_eq!(document.document().unwrap().content_type(), "text/html"); + } + fn parse_test_document(html: &str) -> NativeDom { HtmlParser::SCRIPTING_ENABLED.parse( Url::parse("https://example.test/").expect("test url"), diff --git a/moli-parser/src/session.rs b/moli-parser/src/session.rs index 9fa20921c37..545ad39c625 100644 --- a/moli-parser/src/session.rs +++ b/moli-parser/src/session.rs @@ -203,6 +203,16 @@ impl TokenSink for EmbedderPausingTreeBuilder { } impl HtmlParserSession { + pub(super) fn initialize_text_document(&self) { + // Parse only the browser-owned shell. Response bytes enter the tokenizer + // after it has switched to plaintext, so tags and entities stay literal. + self.process(StrTendril::from(concat!( + "", + "
\n"
+        )));
+        self.tokenizer.set_plaintext_state();
+    }
+
     fn new(sink: DocumentSink, opts: ParseOpts) -> Self {
         let tree_builder = EmbedderPausingTreeBuilder::new(sink, opts.tree_builder);
         Self {
@@ -389,10 +399,10 @@ fn feed_with_definitive_encoding(
 
 pub(super) fn new_html_tree_sink_session(
     target: ParserStreamHtmlTreeSinkTarget,
-    scripting_enabled: bool,
+    options: ParseOpts,
 ) -> HtmlTreeSinkSession {
     let sink = DocumentSink::new(target);
-    let parser = HtmlParserSession::new(sink, html_parse_opts_with_scripting(scripting_enabled));
+    let parser = HtmlParserSession::new(sink, options);
     let script_input = ParserInputQueue::default();
 
     HtmlTreeSinkSession {
diff --git a/moli-parser/src/stream.rs b/moli-parser/src/stream.rs
index 6faf1e1fee8..1b19b2dbb45 100644
--- a/moli-parser/src/stream.rs
+++ b/moli-parser/src/stream.rs
@@ -429,11 +429,25 @@ pub(crate) fn prepare_parser_script_handoff_for_static_document(
 }
 
 impl HtmlTreeSinkStream {
+    pub(super) fn initialize_text_document(&self) {
+        self.parser.initialize_text_document();
+    }
+
     pub(super) fn from_target_with_scripting(
         target: ParserStreamHtmlTreeSinkTarget,
         scripting_enabled: bool,
     ) -> Self {
-        let session = new_html_tree_sink_session(target, scripting_enabled);
+        Self::from_target_with_options(
+            target,
+            super::session::html_parse_opts_with_scripting(scripting_enabled),
+        )
+    }
+
+    pub(super) fn from_target_with_options(
+        target: ParserStreamHtmlTreeSinkTarget,
+        options: html5ever::ParseOpts,
+    ) -> Self {
+        let session = new_html_tree_sink_session(target, options);
         Self {
             parser: session.parser,
             script_input: session.script_input,
diff --git a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
index 9f891a382bb..57c4abc15e4 100644
--- a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
+++ b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
@@ -1,5 +1,98 @@
 use super::*;
 
+#[tokio::test]
+async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() {
+    const PAYLOAD: &str =
+        "literal&Gülçek";
+    let cases = [
+        (
+            "plain",
+            "text/plain; charset=utf-8",
+            "text/plain",
+            true,
+            PAYLOAD,
+        ),
+        (
+            "plain-bom",
+            "text/plain; charset=utf-8",
+            "text/plain",
+            true,
+            "\u{feff}\u{feff}",
+        ),
+        (
+            "json",
+            "application/json",
+            "application/json",
+            true,
+            PAYLOAD,
+        ),
+        (
+            "javascript",
+            "text/javascript; charset=utf-8",
+            "text/javascript",
+            true,
+            PAYLOAD,
+        ),
+        (
+            "html",
+            "text/html; charset=utf-8",
+            "text/html",
+            false,
+            PAYLOAD,
+        ),
+    ];
+    let mut fixture = axum::Router::new();
+    for (name, mime, _, _, payload) in cases {
+        fixture = fixture.route(
+            &format!("/{name}"),
+            axum::routing::get(move || async move { ([(header::CONTENT_TYPE, mime)], payload) }),
+        );
+    }
+    fixture = fixture.route(
+        "/frame/{kind}",
+        axum::routing::get(
+            |axum::extract::Path(kind): axum::extract::Path| async move {
+                (
+                    [(header::CONTENT_TYPE, "text/html")],
+                    format!(""),
+                )
+            },
+        ),
+    );
+    let (addr, _server) = spawn_dedicated_fixture_server(fixture, "document-mime");
+    let app = build_router(test_state());
+    let session = classic_request_json(app.clone(), Method::POST, "/session").await;
+    let session_id = session["value"]["sessionId"].as_str().unwrap();
+    for (name, _, content_type, literal, payload) in cases {
+        for prefix in ["", "frame/"] {
+            let navigated = classic_request_json_with_body(
+                app.clone(),
+                Method::POST,
+                &format!("/session/{session_id}/url"),
+                json!({"url":format!("http://{addr}/{prefix}{name}")}),
+            )
+            .await;
+            assert_eq!(navigated, json!({"value":null}), "{prefix}{name}");
+            let observed = classic_request_json_with_body(
+                app.clone(), Method::POST, &format!("/session/{session_id}/execute/sync"),
+                json!({"script": "const w=document.querySelector('iframe')?.contentWindow ?? window; return [w.document.body.textContent,w.executed??null,w.document.querySelectorAll('script').length,w.document.contentType];", "args": []}),
+            ).await;
+            let expected = if literal {
+                json!([
+                    payload.strip_prefix('\u{feff}').unwrap_or(payload),
+                    null,
+                    0,
+                    content_type
+                ])
+            } else {
+                json!(["literal&Gülçek", 42, 1, content_type])
+            };
+            assert_eq!(observed["value"], expected, "{prefix}{name}");
+        }
+    }
+    classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;
+}
+
 #[tokio::test]
 async fn webdriver_classic_page_load_strategy_maps_navigation_wait_policy() {
     let app = build_router(test_state());
diff --git a/moli-protocol/src/domains/emulation/tests.rs b/moli-protocol/src/domains/emulation/tests.rs
index dd3ad74ca27..2b677b6fe7e 100644
--- a/moli-protocol/src/domains/emulation/tests.rs
+++ b/moli-protocol/src/domains/emulation/tests.rs
@@ -573,7 +573,9 @@ async fn admitted_idle_override_is_visible_to_concurrent_same_site_navigation()
             listener,
             Router::new().route(
                 "/",
-                get(|| async { "idle navigation" }),
+                get(|| async {
+                    axum::response::Html("idle navigation")
+                }),
             ),
         )
         .await
@@ -644,7 +646,9 @@ async fn idle_override_updates_idle_detector_and_clear_restores_actual_state() {
             listener,
             Router::new().route(
                 "/",
-                get(|| async { "idle detector" }),
+                get(|| async {
+                    axum::response::Html("idle detector")
+                }),
             ),
         )
         .await
@@ -2219,7 +2223,7 @@ async fn emulation_async_dispatch_updates_live_page_user_agent_and_xhr_header()
 #[tokio::test(flavor = "multi_thread")]
 async fn emulation_async_dispatch_updates_live_page_surface_without_mutating_accept_language() {
     async fn page_handler() -> impl IntoResponse {
-        "ok"
+        axum::response::Html("ok")
     }
 
     async fn xhr_handler(
@@ -2714,9 +2718,9 @@ async fn locale_override_updates_intl_without_mutating_language_surfaces() {
             .get(axum::http::header::ACCEPT_LANGUAGE)
             .and_then(|value| value.to_str().ok())
             .unwrap_or("");
-        format!(
+        axum::response::Html(format!(
             ""
-        )
+        ))
     }
 
     let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -2770,9 +2774,9 @@ async fn bidi_user_context_locale_composes_with_user_agent_on_all_identity_surfa
             .get(axum::http::header::ACCEPT_LANGUAGE)
             .and_then(|value| value.to_str().ok())
             .unwrap_or("");
-        format!(
+        axum::response::Html(format!(
             ""
-        )
+        ))
     }
 
     let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
diff --git a/moli-protocol/src/domains/page/tests/lifecycle.rs b/moli-protocol/src/domains/page/tests/lifecycle.rs
index 92e5947f8df..afa29aece92 100644
--- a/moli-protocol/src/domains/page/tests/lifecycle.rs
+++ b/moli-protocol/src/domains/page/tests/lifecycle.rs
@@ -2384,11 +2384,15 @@ async fn network_navigations_use_unique_document_loader_ids() {
             axum::Router::new()
                 .route(
                     "/first",
-                    axum::routing::get(|| async { "
first
" }), + axum::routing::get(|| async { + axum::response::Html("
first
") + }), ) .route( "/second", - axum::routing::get(|| async { "
second
" }), + axum::routing::get(|| async { + axum::response::Html("
second
") + }), ), ) .await @@ -2467,11 +2471,15 @@ async fn navigations_without_network_domain_still_use_unique_loader_ids() { axum::Router::new() .route( "/first", - axum::routing::get(|| async { "
first
" }), + axum::routing::get(|| async { + axum::response::Html("
first
") + }), ) .route( "/second", - axum::routing::get(|| async { "
second
" }), + axum::routing::get(|| async { + axum::response::Html("
second
") + }), ), ) .await diff --git a/moli-protocol/src/domains/page/tests/runtime.rs b/moli-protocol/src/domains/page/tests/runtime.rs index ab651a20952..1a1e48497d4 100644 --- a/moli-protocol/src/domains/page/tests/runtime.rs +++ b/moli-protocol/src/domains/page/tests/runtime.rs @@ -1269,20 +1269,22 @@ async fn runtime_evaluate_child_history_back_emits_child_frame_navigation_and_li .route( "/main", axum::routing::get(|| async { - "" + axum::response::Html( + "", + ) }), ) .route( "/initial", - axum::routing::get(|| async { "initial" }), + axum::routing::get(|| async { axum::response::Html("initial") }), ) .route( "/history-a", - axum::routing::get(|| async { "history-a" }), + axum::routing::get(|| async { axum::response::Html("history-a") }), ) .route( "/history-b", - axum::routing::get(|| async { "history-b" }), + axum::routing::get(|| async { axum::response::Html("history-b") }), ), ) .await diff --git a/moli-protocol/src/domains/target/tests/tests_background_staging/environment.rs b/moli-protocol/src/domains/target/tests/tests_background_staging/environment.rs index e0670cf8a5f..d5306df9cbc 100644 --- a/moli-protocol/src/domains/target/tests/tests_background_staging/environment.rs +++ b/moli-protocol/src/domains/target/tests/tests_background_staging/environment.rs @@ -284,9 +284,9 @@ async fn same_context_background_session_can_clear_its_own_locale_before_activat .get(axum::http::header::ACCEPT_LANGUAGE) .and_then(|value| value.to_str().ok()) .unwrap_or(""); - format!( + axum::response::Html(format!( "" - ) + )) } let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -451,7 +451,9 @@ async fn same_context_background_session_can_clear_its_own_locale_before_activat #[tokio::test(flavor = "multi_thread")] async fn same_context_background_session_can_clear_its_own_timezone_before_activation() { async fn handler() -> impl IntoResponse { - "" + axum::response::Html( + "", + ) } let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/moli-renderer-v8/src/dom_parser.rs b/moli-renderer-v8/src/dom_parser.rs index cff5ea1cc26..e72deebc20c 100644 --- a/moli-renderer-v8/src/dom_parser.rs +++ b/moli-renderer-v8/src/dom_parser.rs @@ -378,7 +378,11 @@ pub(crate) fn preserve_decoded_bom_only_browsing_context_body<'a>( source: &'a str, content_type: Option<&str>, ) -> std::borrow::Cow<'a, str> { - if source == "\u{feff}" && !content_type.is_some_and(is_dom_parser_xml_mime) { + if source == "\u{feff}" + && !content_type.is_some_and(|mime| { + is_dom_parser_xml_mime(mime) || moli_web_mime::is_text_document_mime(mime) + }) + { std::borrow::Cow::Borrowed("\u{feff}") } else { std::borrow::Cow::Borrowed(source) diff --git a/moli-renderer-v8/src/live_document_parser.rs b/moli-renderer-v8/src/live_document_parser.rs index 1c513162cc5..e483965c862 100644 --- a/moli-renderer-v8/src/live_document_parser.rs +++ b/moli-renderer-v8/src/live_document_parser.rs @@ -757,6 +757,14 @@ impl DocumentParserSession { ) } + pub(crate) fn start_main_text_document(document_url: Url, content_type: &str) -> Self { + Self::new_html( + HtmlParser::with_scripting_enabled(false) + .start_text_document(document_url, content_type), + DocumentParserLifetime::Finite, + ) + } + pub(crate) fn start_finite_live_document( document_url: Url, document_handle: NativeNodeId, @@ -769,6 +777,21 @@ impl DocumentParserSession { ) } + pub(crate) fn start_finite_live_text_document( + document_url: Url, + document_handle: NativeNodeId, + owner: &mut impl LiveDocumentParserOwner, + ) -> Self { + Self::new_html( + HtmlParser::with_scripting_enabled(false).start_live_text_document_root( + document_url, + document_handle, + owner, + ), + DocumentParserLifetime::Finite, + ) + } + pub(crate) fn start_finite_live_xml_document( document_url: Url, document_handle: NativeNodeId, diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs index ee356deebef..a1d27afdb6b 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs @@ -239,6 +239,7 @@ impl JsContextHost { parser_base_url, source.as_ref(), is_xml_document, + snapshot.content_type.as_deref(), ); Some(ChildDocumentInstallResult { initial_classic_ready_work: parser_start.initial_classic_ready_work, diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs index 631bd47aad5..fc8f2f3a420 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs @@ -1705,6 +1705,7 @@ impl JsContextHost { document_base_url: Url, markup: &str, is_xml_document: bool, + content_type: Option<&str>, ) -> ChildLiveDocumentParserStartResult { let owner = FrameDocumentOwner::new(owner_local_window_id, owner_document_id); self.child_document_parsers.clear(owner); @@ -1713,6 +1714,13 @@ impl JsContextHost { document_base_url, document_handle, ) + } else if content_type.is_some_and(moli_web_mime::is_text_document_mime) { + let mut parser_owner = ChildFrameLiveParserOwner::new(self, scope, document_handle); + DocumentParserSession::start_finite_live_text_document( + document_base_url, + document_handle, + &mut parser_owner, + ) } else { DocumentParserSession::start_finite_live_document( document_base_url, diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs index ebcbbeb135f..d429de74250 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs @@ -21,7 +21,8 @@ use crate::{ LoadedChildDocument, SubresourceResponseBody, }, }; -use moli_encoding::decode_html_document_with_fallback; +use moli_encoding::{HtmlDocumentStreamingDecoder, decode_html_document_with_fallback}; +use moli_encoding_detector::detect_legacy_html_encoding; pub(crate) struct AppliedChildDocumentLoadCompletion { /// Initial parser-classic work produced by the committed child document. @@ -791,8 +792,24 @@ fn child_document_load_outcome_from_response( let body_bytes = response_body .try_bytes() .map_err(|error| format!("failed to read child document response body: {error}"))?; - let (markup, character_set) = - decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback)); + let (markup, character_set) = if let Some(mime) = content_type + .as_deref() + .filter(|mime| moli_web_mime::is_text_document_mime(mime)) + { + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &head.headers, + head.final_url.as_str(), + detect_legacy_html_encoding, + moli_web_mime::is_json_module_mime(mime) || mime == "text/json", + ); + let mut markup = decoder.push(&body_bytes).concat(); + if let Some(tail) = decoder.finish() { + markup.push_str(&tail); + } + (markup, decoder.document_encoding_name()) + } else { + decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback)) + }; (markup, character_set.to_owned()) }; let policy_container = diff --git a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs index 5af62b82194..454433e6c3a 100644 --- a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs +++ b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs @@ -25,6 +25,7 @@ impl ConcurrentParseTimeRuntime { let (page_vm, triggered_navigation) = { let buffered_document_preloads = &mut state.buffered_document_preloads; let service_worker_preload_context = state.service_worker_preload_context.as_ref(); + let document_character_set = &state.document_character_set; PageVm::new_from_parser_stream_and_run_document_start( page_id, local_executor, @@ -34,6 +35,7 @@ impl ConcurrentParseTimeRuntime { &mut state.parser_session, started, |page_vm| { + page_vm.set_document_character_set(document_character_set.clone()); admit_pending_preloads( page_vm, buffered_document_preloads, diff --git a/moli-renderer-v8/src/runtime/phase_one/state.rs b/moli-renderer-v8/src/runtime/phase_one/state.rs index 3e9302c0dc3..01ae4154837 100644 --- a/moli-renderer-v8/src/runtime/phase_one/state.rs +++ b/moli-renderer-v8/src/runtime/phase_one/state.rs @@ -9,6 +9,7 @@ pub(super) struct ParseTimeDriverState { pub(super) buffered_document_preloads: Box, pub(super) service_worker_preload_context: Option, pub(super) input_closed: bool, + pub(super) is_text_document: bool, } impl ParseTimeDriverState { @@ -29,6 +30,23 @@ impl ParseTimeDriverState { buffered_document_preloads: Box::default(), service_worker_preload_context: None, input_closed: false, + is_text_document: false, + } + } + + pub(super) fn new_text(final_url: Url, mime: &str) -> Self { + Self { + parser_session: DocumentParserSession::start_main_text_document( + final_url.clone(), + mime, + ), + final_url, + document_character_set: "UTF-8".to_owned(), + scheduler: DocumentScriptScheduler::new(), + buffered_document_preloads: Box::default(), + service_worker_preload_context: None, + input_closed: false, + is_text_document: true, } } @@ -41,6 +59,7 @@ impl ParseTimeDriverState { buffered_document_preloads: Box::default(), service_worker_preload_context: None, input_closed: false, + is_text_document: false, } } diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rs index f3bfeed0729..fa8df43b89a 100644 --- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs +++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs @@ -94,10 +94,8 @@ impl ConcurrentParseTimeRuntime { env.apply_navigation_response_headers(&response_final_url, &response_headers); let mut body_source = RawDocumentBodySource::fetch_response(response); - let mut state = ParseTimeDriverState::new_with_scripting_enabled( - response_final_url, - main_document_parser_scripting_enabled(&env), - ); + let (mut state, mut decoder) = + response_document_parser(response_final_url, &response_headers, &env); state .buffered_document_preloads .set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission( @@ -124,11 +122,6 @@ impl ConcurrentParseTimeRuntime { ) }); state.service_worker_preload_context = service_worker_preload_context.clone(); - let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector( - &response_headers, - state.final_url.as_str(), - detect_legacy_html_encoding, - ); // Raw navigation bodies are decoded during prebootstrap scan. The decoder // is carried forward so split multibyte sequences are not decoded twice or // lost between the scan and parser handoff. @@ -332,10 +325,7 @@ impl ConcurrentParseTimeRuntime { }, ))); } - let mut state = ParseTimeDriverState::new_with_scripting_enabled( - final_url, - main_document_parser_scripting_enabled(&env), - ); + let (mut state, mut decoder) = response_document_parser(final_url, &response_headers, &env); state .buffered_document_preloads .set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission( @@ -362,11 +352,6 @@ impl ConcurrentParseTimeRuntime { ) }); state.service_worker_preload_context = service_worker_preload_context.clone(); - let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector( - &response_headers, - state.final_url.as_str(), - detect_legacy_html_encoding, - ); // External raw bodies replay captured chunks. Pre-scan only what is already // buffered before bootstrap so the producer's backpressure boundary still // controls how far ahead the parser can get. @@ -451,6 +436,10 @@ impl ConcurrentParseTimeRuntime { chunk: String, service_worker_context: Option<&ServiceWorkerScriptPreloadContext>, ) { + if self.state.is_text_document { + self.state.parser_session.queue_arrived_chunk(chunk); + return; + } self.state .buffered_document_preloads .append_to_main_document_scan_with_service_worker_context( @@ -607,6 +596,9 @@ fn scan_prebootstrap_html_chunk_into_state( chunk: &str, service_worker_context: Option<&ServiceWorkerScriptPreloadContext>, ) { + if state.is_text_document { + return; + } state .buffered_document_preloads .append_to_main_document_prebootstrap_scan_with_service_worker_context( @@ -617,6 +609,41 @@ fn scan_prebootstrap_html_chunk_into_state( ); } +fn response_document_parser( + final_url: Url, + headers: &[(String, Vec)], + env: &PageVmEnvConfig, +) -> (ParseTimeDriverState, HtmlDocumentStreamingDecoder) { + let content_type = moli_web_mime::response_document_content_type(headers); + let text_type = content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime)); + let mut state = if let Some(mime) = &text_type { + ParseTimeDriverState::new_text(final_url, mime) + } else { + ParseTimeDriverState::new_with_scripting_enabled( + final_url, + main_document_parser_scripting_enabled(env), + ) + }; + let decoder = if let Some(mime) = text_type { + HtmlDocumentStreamingDecoder::new_text_document( + headers, + state.final_url.as_str(), + detect_legacy_html_encoding, + moli_web_mime::is_json_module_mime(&mime) || mime == "text/json", + ) + } else { + HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector( + headers, + state.final_url.as_str(), + detect_legacy_html_encoding, + ) + }; + // Headers/defaults are observable even when no body chunk is ready yet. + // A later BOM or decoder decision can still replace this tentative value. + sync_state_document_character_set_from_decoder(&mut state, &decoder); + (state, decoder) +} + pub(super) fn enqueue_streaming_raw_chunk( runtime: &mut ConcurrentParseTimeRuntime, decoder: &mut HtmlDocumentStreamingDecoder, @@ -910,6 +937,105 @@ mod tests { use crate::parser::ScriptSource; use tokio::io::{AsyncReadExt, AsyncWriteExt}; + #[tokio::test] + async fn text_document_charset_is_visible_to_document_start_before_body_arrives() { + let _js_runtime_owner = crate::JsRuntime::initialize(); + tokio::task::LocalSet::new().run_until(async { + for (mime, chunk, expected) in [ + ("text/plain; charset=gbk", None, "GBK"), + ("application/json", None, "UTF-8"), + ("text/plain; charset=gbk", Some(b"\xef\xbb\xbfready".as_slice()), "UTF-8"), + ] { + let loader_owner = ResourceRequestClient::new(&FetchConfig::default()).unwrap(); + let loader = loader_owner.handle(); + let mut env = default_test_page_vm_env_config(); + env.document_start_scripts.push(crate::DocumentStartScript { + registry_key: None, + devtools_session: None, + source: "document.title = document.characterSet".to_owned(), + world_name: None, + has_bidi_channel_argument: false, + bidi_channel_handoffs: Vec::new(), + }); + let (mut state, mut decoder) = response_document_parser( + Url::parse("https://example.test/data").unwrap(), + &[("Content-Type".to_owned(), mime.as_bytes().to_vec())], &env, + ); + if let Some(chunk) = chunk { + let chunks = decoder.push(chunk); + sync_state_document_character_set_from_decoder(&mut state, &decoder); + for chunk in chunks { + state.parser_session.queue_arrived_chunk(chunk); + } + } + assert_eq!(state.document_character_set, expected); + let (_, page_vm, navigated) = ConcurrentParseTimeRuntime::bootstrap_page_vm_from_state_on_fresh_local_task( + PageId::new_for_testing(1), JsLocalExecutor::new(), loader, env, + PageVmRuntimeHooks::standalone_without_owner_reservation_for_test(), state, + Instant::now(), "charset test bootstrap closed", + ).await.unwrap(); + assert!(!navigated); + assert_eq!(page_vm.vm().snapshot_live_document().document_title(), expected, "{mime}"); + } + }).await; + } + + #[test] + fn response_document_dispatch_distinguishes_text_from_html_and_xml() { + for (mime, text) in [ + ("application/json", true), + ("application/problem+json", true), + ("text/plain; charset=gbk", true), + ("text/javascript", true), + ("text/html", false), + ("application/xhtml+xml", false), + ("application/xml", false), + ("text/xml", false), + ] { + let (state, _) = response_document_parser( + Url::parse("https://example.test/data").unwrap(), + &[("Content-Type".to_owned(), mime.as_bytes().to_vec())], + &default_test_page_vm_env_config(), + ); + assert_eq!(state.is_text_document, text, "{mime}"); + } + } + + #[tokio::test] + async fn text_document_does_not_preload_literal_script_tags() { + let loader = ResourceRequestClient::new(&moli_fetch::FetchConfig::default()) + .expect("default loader"); + let runtime_hooks = PageVmRuntimeHooks::standalone_without_owner_reservation_for_test(); + for (mime, expected_preload) in [ + ("text/plain", false), + ("application/json", false), + ("text/html", true), + ] { + let (mut state, _) = response_document_parser( + Url::parse("https://example.test/data").unwrap(), + &[("Content-Type".to_owned(), mime.as_bytes().to_vec())], + &default_test_page_vm_env_config(), + ); + state.buffered_document_preloads.bind_resource_runtime( + runtime_hooks.owner_wake(), + runtime_hooks.resource_task_runner(), + ); + scan_prebootstrap_html_chunk_into_state( + &mut state, + &loader, + "", + None, + ); + assert_eq!( + state.buffered_document_preloads.entries.contains_key( + &classic_preload_key_for_streaming_test("https://example.test/literal.js") + ), + expected_preload, + "{mime}", + ); + } + } + #[tokio::test] async fn external_raw_document_body_source_collects_chunks_and_completion() { let (completion_tx, completion_rx) = oneshot::channel(); diff --git a/moli-web-mime/src/classification.rs b/moli-web-mime/src/classification.rs index 3ff3a0dd50f..35fca10b589 100644 --- a/moli-web-mime/src/classification.rs +++ b/moli-web-mime/src/classification.rs @@ -51,6 +51,14 @@ pub fn is_text_mime_essence(input: &str) -> bool { input.starts_with("text/") } +/// Navigation responses rendered as literal text rather than HTML or XML. +/// This also includes JSON and JavaScript resources opened as documents. +pub fn is_text_document_mime(input: &str) -> bool { + !is_html_document_mime(input) + && !is_dom_parser_xml_mime(input) + && (is_text_mime(input) || is_json_module_mime(input) || is_javascript_mime(input)) +} + /// Whether a `style` element's raw `type` attribute selects classic CSS. /// /// HTML's update-a-style-block algorithm compares the untrimmed attribute diff --git a/moli-web-mime/src/lib.rs b/moli-web-mime/src/lib.rs index e631473eaff..806eb245521 100644 --- a/moli-web-mime/src/lib.rs +++ b/moli-web-mime/src/lib.rs @@ -14,9 +14,9 @@ pub use classification::{ is_font_mime, is_font_mime_essence, is_form_urlencoded_mime, is_html_document_mime, is_image_mime, is_image_mime_essence, is_javascript_mime, is_javascript_mime_essence, is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime, is_png_image_mime_essence, - is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence, is_text_mime, - is_text_mime_essence, is_video_mime, is_video_mime_essence, is_webassembly_mime, - multipart_form_data_boundary, + is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence, + is_text_document_mime, is_text_mime, is_text_mime_essence, is_video_mime, + is_video_mime_essence, is_webassembly_mime, multipart_form_data_boundary, }; pub use data_url::{ data_url_body_and_computed_mime_type, data_url_body_and_mime_type, data_url_mime_type, diff --git a/moli-web-mime/src/tests.rs b/moli-web-mime/src/tests.rs index 010209cc787..d29adb69b7a 100644 --- a/moli-web-mime/src/tests.rs +++ b/moli-web-mime/src/tests.rs @@ -870,3 +870,25 @@ fn script_like_mime_type_block_matches_fetch_response_rule() { )); assert!(!should_script_like_response_be_blocked_due_to_mime_type(&[])); } +#[test] +fn document_text_classification_excludes_html_and_xml() { + for mime in [ + "text/plain", + "Text/Css; charset=utf-8", + "application/json", + "application/ld+json", + "application/javascript", + ] { + assert!(crate::is_text_document_mime(mime), "{mime}"); + } + for mime in [ + "text/html", + "text/xml", + "application/xhtml+xml", + "image/svg+xml", + "image/png", + "application/octet-stream", + ] { + assert!(!crate::is_text_document_mime(mime), "{mime}"); + } +} diff --git a/moli/src/fetch_dump/tests.rs b/moli/src/fetch_dump/tests.rs index 73240abaf6e..b2f93f81c2c 100644 --- a/moli/src/fetch_dump/tests.rs +++ b/moli/src/fetch_dump/tests.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use anyhow::Result; -use axum::{Router, routing::get}; +use axum::{Router, response::Html, routing::get}; use moli_core::{ LayoutPolicy, page::Page, @@ -36,7 +36,7 @@ async fn load_page_with_config( "/", get(move || { let body = Arc::clone(&server_body); - async move { (*body).clone() } + async move { Html((*body).clone()) } }), ); axum::serve(listener, app).await.unwrap(); From e50a06501e7081857a9c0483350bad5036633306 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:09:25 +0800 Subject: [PATCH 2/6] fix(document): share literal text parsing across child document paths --- moli-parser/src/html.rs | 38 ++++++++++ moli-parser/src/session.rs | 5 +- .../tests/classic/extracted/navigation.rs | 32 +++++++- moli-renderer-v8/src/dom_parser.rs | 76 +++++++++++++------ .../context_host/child_documents/commit.rs | 8 -- .../child_documents/live_parser.rs | 8 -- 6 files changed, 124 insertions(+), 43 deletions(-) diff --git a/moli-parser/src/html.rs b/moli-parser/src/html.rs index e66b18367bd..532cdddf6fa 100644 --- a/moli-parser/src/html.rs +++ b/moli-parser/src/html.rs @@ -1926,6 +1926,44 @@ mod tests { } } + #[test] + fn text_document_shell_has_no_doctype_and_preserves_normalized_literal_text() { + for mime in [ + "text/plain", + "application/json", + "application/problem+json", + "text/javascript", + ] { + let stream = HtmlParser::SCRIPTING_ENABLED + .start_text_document(Url::parse("https://example.test/data").unwrap(), mime); + for chunk in [ + "\n<&", + "\r", + "\nbeta\rgamma\0", + "
", + ] { + stream.feed(chunk); + } + let document = stream.finish_dom_host(); + let root = document.document_handle(); + let children = document.child_handles(root).collect::>(); + assert_eq!(children.len(), 1, "no synthetic doctype: {mime}"); + assert!(document.is_html_element_named(children[0], "html")); + let expected_mode = HtmlParser::SCRIPTING_ENABLED.parse_dom_host( + Url::parse("https://example.test/control").unwrap(), + "

control".to_owned(), + ); + assert_eq!( + document.document_quirks_mode_for_handle(root), + expected_mode.document_quirks_mode_for_handle(expected_mode.document_handle()) + ); + assert_eq!( + document.text_content(root).as_deref(), + Some("\n<&\nbeta\ngamma\u{fffd}") + ); + } + } + #[test] fn html_document_still_parses_markup_and_entities() { let document = parse_test_document("Gülçek&"); diff --git a/moli-parser/src/session.rs b/moli-parser/src/session.rs index 545ad39c625..b17c6097bb5 100644 --- a/moli-parser/src/session.rs +++ b/moli-parser/src/session.rs @@ -207,9 +207,12 @@ impl HtmlParserSession { // Parse only the browser-owned shell. Response bytes enter the tokenizer // after it has switched to plaintext, so tags and entities stay literal. self.process(StrTendril::from(concat!( - "", + "", "

\n"
         )));
+        // Text documents have no doctype but always use no-quirks mode.
+        self.sink()
+            .set_quirks_mode(html5ever::tree_builder::QuirksMode::NoQuirks);
         self.tokenizer.set_plaintext_state();
     }
 
diff --git a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
index 57c4abc15e4..abf196d998b 100644
--- a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
+++ b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
@@ -12,6 +12,20 @@ async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents()
             true,
             PAYLOAD,
         ),
+        (
+            "plain.xml",
+            "text/plain; charset=utf-8",
+            "text/plain",
+            true,
+            "\n<&\r\nbeta\rgamma\0
", + ), + ( + "json.xml", + "application/problem+json", + "application/problem+json", + true, + PAYLOAD, + ), ( "plain-bom", "text/plain; charset=utf-8", @@ -79,7 +93,12 @@ async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() ).await; let expected = if literal { json!([ - payload.strip_prefix('\u{feff}').unwrap_or(payload), + payload + .strip_prefix('\u{feff}') + .unwrap_or(payload) + .replace("\r\n", "\n") + .replace('\r', "\n") + .replace('\0', "\u{fffd}"), null, 0, content_type @@ -88,6 +107,17 @@ async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() json!(["literal&Gülçek", 42, 1, content_type]) }; assert_eq!(observed["value"], expected, "{prefix}{name}"); + if literal { + let structure = classic_request_json_with_body( + app.clone(), Method::POST, &format!("/session/{session_id}/execute/sync"), + json!({"script": "const d=(document.querySelector('iframe')?.contentWindow ?? window).document; return [d.doctype===null,d.compatMode,d.body.children.length,d.body.firstElementChild.localName,d.querySelectorAll('pre').length];", "args": []}), + ).await; + assert_eq!( + structure["value"], + json!([true, "CSS1Compat", 1, "pre", 1]), + "{prefix}{name}" + ); + } } } classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await; diff --git a/moli-renderer-v8/src/dom_parser.rs b/moli-renderer-v8/src/dom_parser.rs index e72deebc20c..43f1e5b2105 100644 --- a/moli-renderer-v8/src/dom_parser.rs +++ b/moli-renderer-v8/src/dom_parser.rs @@ -1,5 +1,4 @@ use crate::web_api_interfaces; -use html5ever::tree_builder::QuirksMode; use moli_web_mime::{is_dom_parser_xml_mime, is_html_document_mime}; use moli_webapi_declare::WebApiFunctionTemplate; use url::Url; @@ -396,7 +395,7 @@ fn parse_browsing_context_document_snapshot( html_parser: HtmlParser, ) -> (DomHost, DetachedDocumentKind) { if content_type.is_some_and(is_dom_parser_xml_mime) - || child_document_url_is_xml_like(&document_url) + || (content_type.is_none() && child_document_url_is_xml_like(&document_url)) { let parser = XmlParser; return ( @@ -404,13 +403,12 @@ fn parse_browsing_context_document_snapshot( DetachedDocumentKind::Xml, ); } - if content_type.is_some_and(|mime| mime.eq_ignore_ascii_case("text/plain")) { - let mut document = - html_parser.parse_dom_host(document_url, plain_text_document_parser_input(source)); - // Text documents are HTML Documents whose mode is explicitly no-quirks, - // despite having no doctype that would select that mode through parsing. - document.set_html_quirks_mode_for_parser(QuirksMode::NoQuirks); - return (document, DetachedDocumentKind::Html); + if let Some(content_type) = + content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime)) + { + let stream = HtmlParser::SCRIPTING_DISABLED.start_text_document(document_url, content_type); + stream.feed(source); + return (stream.finish_dom_host(), DetachedDocumentKind::Html); } ( html_parser.parse_dom_host(document_url, source.to_owned()), @@ -418,22 +416,6 @@ fn parse_browsing_context_document_snapshot( ) } -pub(crate) fn plain_text_document_parser_input(source: &str) -> String { - let mut input = String::with_capacity(source.len().saturating_add(64)); - // The HTML parser discards the first LF after
; preserve any source LF.
-    input.push_str("
\n");
-    for character in source.chars() {
-        match character {
-            '&' => input.push_str("&"),
-            '<' => input.push_str("<"),
-            '\0' => input.push('\u{fffd}'),
-            _ => input.push(character),
-        }
-    }
-    input.push_str("
"); - input -} - fn child_document_url_is_xml_like(url: &Url) -> bool { let path = url.path().to_ascii_lowercase(); path.ends_with(".xml") || path.ends_with(".xhtml") || path.ends_with(".svg") @@ -602,6 +584,50 @@ mod tests { assert!(disabled.element_handle_by_id("fallback").is_some()); } + #[test] + fn child_projection_explicit_mime_overrides_xml_extension() { + let source = "\n<&
"; + for mime in [ + "text/plain", + "application/json", + "application/problem+json", + "text/javascript", + ] { + let (document, kind) = parse_browsing_context_document_snapshot( + Url::parse("https://example.test/data.xml").unwrap(), + source, + Some(mime), + HtmlParser::SCRIPTING_ENABLED, + ); + let root = document.document_handle(); + assert_eq!(kind, DetachedDocumentKind::Html); + assert_eq!( + document.text_content(root).as_deref(), + Some(source), + "{mime}" + ); + let children = document.child_handles(root).collect::>(); + assert_eq!(children.len(), 1); + assert!(document.is_html_element_named(children[0], "html")); + assert_eq!( + document.document_quirks_mode_for_handle(root), + Some(selectors::matching::QuirksMode::NoQuirks) + ); + } + let (document, kind) = parse_browsing_context_document_snapshot( + Url::parse("https://example.test/page.svg").unwrap(), + "&", + Some("text/html"), + HtmlParser::SCRIPTING_ENABLED, + ); + assert_eq!(kind, DetachedDocumentKind::Html); + assert!(document.element_handle_by_id("parsed").is_some()); + assert_eq!( + document.text_content(document.document_handle()).as_deref(), + Some("&") + ); + } + #[test] fn child_plain_text_document_uses_pre_and_no_quirks_mode() { let (document, kind) = parse_browsing_context_document_snapshot( diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs index a1d27afdb6b..1d4e2d9bcc3 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs @@ -170,16 +170,8 @@ impl JsContextHost { navigation_loader: Option, is_xml_document: bool, ) -> Option { - let is_plain_text_document = snapshot - .content_type - .as_deref() - .is_some_and(|mime| mime.eq_ignore_ascii_case("text/plain")); let source = if is_xml_document { std::borrow::Cow::Borrowed(snapshot.markup.as_str()) - } else if is_plain_text_document { - std::borrow::Cow::Owned(crate::dom_parser::plain_text_document_parser_input( - &snapshot.markup, - )) } else { crate::dom_parser::preserve_decoded_bom_only_browsing_context_body( &snapshot.markup, diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs index fc8f2f3a420..c644e32ed30 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs @@ -1012,14 +1012,6 @@ impl JsContextHost { let mut owner = ChildFrameLiveParserOwner::new(self, scope, document_handle); parser.finish(&mut owner) }; - if self - .dom_host() - .document_content_type_for_handle(document_handle) - .is_some_and(|mime| mime.eq_ignore_ascii_case("text/plain")) - { - self.dom_host_mut() - .set_html_quirks_mode_for_parser_document(document_handle, QuirksMode::NoQuirks); - } self.queue_live_child_parser_discovery_signals( child_handle, document_handle, From 43b242cf94e4fc1463b78de33e049ada9cff5c0c Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:09:25 +0800 Subject: [PATCH 3/6] fix(document): construct projection parser with the public API --- moli-renderer-v8/src/dom_parser.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moli-renderer-v8/src/dom_parser.rs b/moli-renderer-v8/src/dom_parser.rs index 43f1e5b2105..0dfa341eb56 100644 --- a/moli-renderer-v8/src/dom_parser.rs +++ b/moli-renderer-v8/src/dom_parser.rs @@ -406,7 +406,8 @@ fn parse_browsing_context_document_snapshot( if let Some(content_type) = content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime)) { - let stream = HtmlParser::SCRIPTING_DISABLED.start_text_document(document_url, content_type); + let stream = HtmlParser::with_scripting_enabled(false) + .start_text_document(document_url, content_type); stream.feed(source); return (stream.finish_dom_host(), DetachedDocumentKind::Html); } From 56e65ba712ffe6015666156d3428f490bd991b89 Mon Sep 17 00:00:00 2001 From: ldm0 Date: Tue, 29 Sep 2026 09:53:35 +0800 Subject: [PATCH 4/6] fix(document): preserve inherited encoding and XML MIME routing Keep valid inherited text encodings ahead of heuristic detection while preserving BOM, transport charset, and JSON default precedence. Share the response decoder between streaming main documents, asynchronous child loads, data URLs, and blocking child snapshots. Classify navigation XML MIME types independently of DOMParser's allowlist, including +xml subtypes in parser routing and text exclusions. Add regressions for inherited UTF-8, literal encoding declarations, XML namespaces, and DOMParser rejection of navigation-only XML types. Validation: - cargo fmt --all - cargo clippy --workspace --all-targets --all-features -- -D warnings - NEXTEST_TEST_THREADS=8 cargo nextest run --no-fail-fast 18,952 passed, 16 skipped. Two existing timing tests failed under the default concurrency, then passed individually and in the full run above. --- moli-encoding/src/document.rs | 16 +- moli-encoding/src/tests.rs | 46 +++++ .../tests/classic/extracted/navigation.rs | 158 ++++++++++++++++++ moli-protocol/src/conn/runtime_load.rs | 2 +- .../src/document_response_decoder.rs | 77 +++++++++ moli-renderer-v8/src/dom_parser.rs | 34 +++- moli-renderer-v8/src/lib.rs | 1 + .../context_host/child_documents/commit.rs | 4 +- .../context_host/child_documents/loads.rs | 41 +---- .../context_host/child_documents/snapshots.rs | 32 ++-- .../context_host/child_frames/discovery.rs | 2 +- .../src/native_bridge/context_host/popups.rs | 2 +- .../src/runtime/phase_one/streaming.rs | 26 +-- moli-web-mime/src/classification.rs | 16 +- moli-web-mime/src/lib.rs | 9 +- moli-web-mime/src/sniffing.rs | 14 +- moli-web-mime/src/tests.rs | 44 +++++ 17 files changed, 428 insertions(+), 96 deletions(-) create mode 100644 moli-renderer-v8/src/document_response_decoder.rs diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rs index 8b3a70398bc..db8e77016fa 100644 --- a/moli-encoding/src/document.rs +++ b/moli-encoding/src/document.rs @@ -52,14 +52,23 @@ pub struct HtmlDocumentStreamingDecoder { impl HtmlDocumentStreamingDecoder { /// Decode a text document without interpreting literal HTML/XML declarations. - /// JSON defaults to UTF-8; other text retains the configured legacy detector. + /// BOMs and transport charsets take precedence. JSON then defaults to UTF-8; + /// other text uses a valid inherited encoding before heuristic detection. pub fn new_text_document( headers: &[(String, Vec)], url_hint: &str, detector: LegacyEncodingDetector, json: bool, + inherited_encoding: Option<&str>, ) -> Self { - let mut decoder = Self::new_with_options(headers, None, Some(url_hint), Some(detector)); + let inherited_encoding = + inherited_encoding.filter(|label| encoding_for_label(label).is_some()); + let mut decoder = Self::new_with_options( + headers, + inherited_encoding, + Some(url_hint), + inherited_encoding.is_none().then_some(detector), + ); decoder.sniff_html_declarations = false; if json && decoder.transport_encoding.is_none() { decoder.transport_encoding = Some(encoding_rs::UTF_8); @@ -169,6 +178,9 @@ impl HtmlDocumentStreamingDecoder { return Some(encoding); } if !self.sniff_html_declarations { + if self.legacy_encoding_detector.is_none() { + return Some(self.fallback_encoding); + } return (finishing || self.sniff_buffer.len() >= 1024).then(|| { self.detected_legacy_content_encoding() .unwrap_or(self.fallback_encoding) diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs index 2217e0521a1..1ce5be37b05 100644 --- a/moli-encoding/src/tests.rs +++ b/moli-encoding/src/tests.rs @@ -33,6 +33,7 @@ fn json_text_document_defaults_to_utf8_across_every_chunk_split() { "https://example.test/", unexpected_legacy_encoding_detector, true, + Some("GBK"), ); let mut output = decoder.push(&input.as_bytes()[..split]).join(""); output.push_str(&decoder.push(&input.as_bytes()[split..]).join("")); @@ -53,6 +54,7 @@ fn plain_text_document_honors_transport_charset() { "https://example.test/", unexpected_legacy_encoding_detector, false, + Some("UTF-8"), ); let mut output = decoder.push(&gbk_bytes("家居")).join(""); output.push_str(&decoder.finish().unwrap_or_default()); @@ -71,6 +73,7 @@ fn text_document_bom_precedes_transport_charset() { "https://example.test/", unexpected_legacy_encoding_detector, false, + Some("windows-1251"), ); assert_eq!(decoder.document_encoding_name(), "GBK"); assert_eq!(decoder.selected_encoding_name(), None); @@ -92,6 +95,7 @@ fn plain_text_document_retains_legacy_detection_without_a_charset() { "https://legacy.example/", test_legacy_encoding_detector, false, + None, ); let mut output = decoder.push(&[0x80]).join(""); output.push_str(&decoder.finish().unwrap_or_default()); @@ -99,6 +103,47 @@ fn plain_text_document_retains_legacy_detection_without_a_charset() { assert_eq!(decoder.selected_encoding_name(), Some("IBM866")); } +#[test] +fn plain_text_document_inherits_encoding_before_detection_across_chunk_splits() { + for (input, encoding) in [ + ("plain ASCII", encoding_rs::UTF_8), + ("吴姓-姓氏渊源", encoding_rs::UTF_8), + ("家居", encoding_rs::GBK), + ] { + let bytes = encoding.encode(input).0; + for split in 0..=bytes.len() { + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://example.test/", + unexpected_legacy_encoding_detector, + false, + Some(encoding.name()), + ); + assert_eq!(decoder.document_encoding_name(), encoding.name()); + let mut output = decoder.push(&bytes[..split]).concat(); + output.push_str(&decoder.push(&bytes[split..]).concat()); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, input, "{} split {split}", encoding.name()); + assert_eq!(decoder.selected_encoding_name(), Some(encoding.name())); + } + } +} + +#[test] +fn invalid_inherited_text_encoding_does_not_disable_detection() { + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://legacy.example/", + test_legacy_encoding_detector, + false, + Some("not-an-encoding"), + ); + let mut output = decoder.push(&[0x80]).concat(); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, "А"); + assert_eq!(decoder.selected_encoding_name(), Some("IBM866")); +} + #[test] fn plain_text_document_ignores_literal_encoding_declarations() { for prefix in [ @@ -112,6 +157,7 @@ fn plain_text_document_ignores_literal_encoding_declarations() { "https://example.test/", test_legacy_encoding_detector, false, + None, ); let mut output = decoder.push(&input).join(""); output.push_str(&decoder.finish().unwrap_or_default()); diff --git a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs index abf196d998b..9649ec823e4 100644 --- a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs +++ b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs @@ -1,5 +1,163 @@ use super::*; +#[tokio::test] +async fn webdriver_classic_text_children_preserve_parent_encoding_without_a_charset() { + const LITERAL: &str = "吴姓-姓氏渊源"; + let cases = [ + ("ascii", "plain ASCII"), + ("utf8", "吴姓-姓氏渊源"), + ("literal", LITERAL), + ("data", LITERAL), + ]; + let mut fixture = axum::Router::new(); + for (name, payload) in cases { + fixture = fixture.route( + &format!("/{name}"), + axum::routing::get( + move || async move { ([(header::CONTENT_TYPE, "text/plain")], payload) }, + ), + ); + } + fixture = fixture.route( + "/parent/{kind}", + axum::routing::get( + |axum::extract::Path(kind): axum::extract::Path| async move { + let target = if kind == "data" { + format!( + "data:text/plain;base64,{}", + base64::Engine::encode(&BASE64_STANDARD, LITERAL) + ) + } else { + format!("/{kind}") + }; + ( + [(header::CONTENT_TYPE, "text/html; charset=utf-8")], + format!(""), + ) + }, + ), + ); + let (addr, _server) = spawn_dedicated_fixture_server(fixture, "inherited-text-encoding"); + let app = build_router(test_state()); + let session = classic_request_json(app.clone(), Method::POST, "/session").await; + let session_id = session["value"]["sessionId"].as_str().unwrap(); + for (name, payload) in cases { + classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/url"), + json!({"url":format!("http://{addr}/parent/{name}")}), + ) + .await; + let parent = classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/execute/sync"), + json!({"script": "return document.characterSet;", "args": []}), + ) + .await; + assert_eq!(parent["value"], json!("UTF-8"), "{name}"); + classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/frame"), + json!({"id": 0}), + ) + .await; + let observed = classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/execute/sync"), + json!({ + "script": "return [document.body.textContent,document.characterSet,document.contentType];", + "args": [], + }), + ).await; + assert_eq!( + observed["value"], + json!([payload, "UTF-8", "text/plain"]), + "{name}" + ); + classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/frame"), + json!({"id": null}), + ) + .await; + } + classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await; +} + +#[tokio::test] +async fn webdriver_classic_xml_suffix_navigation_preserves_namespaces_and_dom_parser_types() { + let cases = [ + ("atom.xml", "application/atom+xml"), + ("rss.xml", "application/rss+xml"), + ("custom.xml", "text/example+xml"), + ]; + let mut fixture = axum::Router::new(); + for (name, mime) in cases { + fixture = fixture.route( + &format!("/{name}"), + axum::routing::get(move || async move { + ( + [(header::CONTENT_TYPE, mime)], + "吴姓", + ) + }), + ); + } + fixture = fixture.route( + "/parent/{kind}", + axum::routing::get( + |axum::extract::Path(kind): axum::extract::Path| async move { + ( + [(header::CONTENT_TYPE, "text/html; charset=windows-1252")], + format!(""), + ) + }, + ), + ); + let (addr, _server) = spawn_dedicated_fixture_server(fixture, "xml-suffix-navigation"); + let app = build_router(test_state()); + let session = classic_request_json(app.clone(), Method::POST, "/session").await; + let session_id = session["value"]["sessionId"].as_str().unwrap(); + for (name, mime) in cases { + for prefix in ["", "parent/"] { + classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/url"), + json!({"url":format!("http://{addr}/{prefix}{name}")}), + ) + .await; + let observed = classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/execute/sync"), + json!({ + "script": "const d=document.querySelector('iframe')?.contentDocument ?? document; let rejected=false; try { new DOMParser().parseFromString('',d.contentType); } catch(error) { rejected=error instanceof TypeError; } const root=d.getElementsByTagNameNS('http://www.w3.org/2005/Atom','feed')[0]; return [root.localName,root.namespaceURI,root.textContent,d.contentType,d.characterSet,rejected];", + "args": [], + }), + ).await; + assert_eq!( + observed["value"], + json!([ + "feed", + "http://www.w3.org/2005/Atom", + "吴姓", + mime, + "UTF-8", + true + ]), + "{prefix}{name}" + ); + } + } + classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await; +} + #[tokio::test] async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() { const PAYLOAD: &str = diff --git a/moli-protocol/src/conn/runtime_load.rs b/moli-protocol/src/conn/runtime_load.rs index 0f9674b2531..d44b8a737e1 100644 --- a/moli-protocol/src/conn/runtime_load.rs +++ b/moli-protocol/src/conn/runtime_load.rs @@ -158,7 +158,7 @@ async fn prepare_network_error_page_navigation_with_engine_async( fn response_headers_indicate_xml_document(headers: &[(String, Vec)]) -> bool { moli_web_mime::response_document_content_type(headers) - .is_some_and(|mime| moli_web_mime::is_dom_parser_xml_mime(&mime)) + .is_some_and(|mime| moli_web_mime::is_xml_document_mime(&mime)) } #[derive(Clone, Debug)] diff --git a/moli-renderer-v8/src/document_response_decoder.rs b/moli-renderer-v8/src/document_response_decoder.rs new file mode 100644 index 00000000000..5d291b90ef1 --- /dev/null +++ b/moli-renderer-v8/src/document_response_decoder.rs @@ -0,0 +1,77 @@ +use moli_encoding::HtmlDocumentStreamingDecoder; +use moli_encoding_detector::detect_legacy_html_encoding; +use moli_web_mime::{is_json_document_mime, is_text_document_mime, is_xml_document_mime}; +use url::Url; + +pub(crate) fn new_document_response_decoder( + headers: &[(String, Vec)], + content_type: Option<&str>, + final_url: &Url, + inherited_encoding: Option<&str>, +) -> HtmlDocumentStreamingDecoder { + if let Some(mime) = content_type.filter(|mime| is_text_document_mime(mime)) { + HtmlDocumentStreamingDecoder::new_text_document( + headers, + final_url.as_str(), + detect_legacy_html_encoding, + is_json_document_mime(mime), + inherited_encoding, + ) + } else if content_type.is_some_and(is_xml_document_mime) { + // XML declarations still apply; neither parent encoding nor heuristics + // can replace XML's UTF-8 default. + HtmlDocumentStreamingDecoder::new_with_fallback(headers, Some("UTF-8")) + } else if inherited_encoding.is_some() { + HtmlDocumentStreamingDecoder::new_with_fallback(headers, inherited_encoding) + } else { + HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector( + headers, + final_url.as_str(), + detect_legacy_html_encoding, + ) + } +} + +pub(crate) fn decode_document_response( + bytes: &[u8], + headers: &[(String, Vec)], + content_type: Option<&str>, + final_url: &Url, + inherited_encoding: Option<&str>, +) -> (String, &'static str) { + let mut decoder = + new_document_response_decoder(headers, content_type, final_url, inherited_encoding); + let mut text = decoder.push(bytes).concat(); + if let Some(tail) = decoder.finish() { + text.push_str(&tail); + } + (text, decoder.document_encoding_name()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn json_and_xml_responses_keep_utf8_defaults_with_a_legacy_parent() { + for (mime, source) in [ + ("application/json", "{\"text\":\"吴姓\"}"), + ("Text/JSON", "{\"text\":\"吴姓\"}"), + ("application/problem+json", "{\"text\":\"吴姓\"}"), + ("application/atom+xml", "吴姓"), + ("application/rss+xml", "吴姓"), + ("text/example+xml", "吴姓"), + ] { + let headers = [("Content-Type".to_owned(), mime.as_bytes().to_vec())]; + let (decoded, encoding) = decode_document_response( + source.as_bytes(), + &headers, + Some(mime), + &Url::parse("https://example.test/document").unwrap(), + Some("GBK"), + ); + assert_eq!(decoded, source, "{mime}"); + assert_eq!(encoding, "UTF-8", "{mime}"); + } + } +} diff --git a/moli-renderer-v8/src/dom_parser.rs b/moli-renderer-v8/src/dom_parser.rs index 0dfa341eb56..d764b479d09 100644 --- a/moli-renderer-v8/src/dom_parser.rs +++ b/moli-renderer-v8/src/dom_parser.rs @@ -1,5 +1,5 @@ use crate::web_api_interfaces; -use moli_web_mime::{is_dom_parser_xml_mime, is_html_document_mime}; +use moli_web_mime::{is_dom_parser_xml_mime, is_html_document_mime, is_xml_document_mime}; use moli_webapi_declare::WebApiFunctionTemplate; use url::Url; @@ -379,7 +379,7 @@ pub(crate) fn preserve_decoded_bom_only_browsing_context_body<'a>( ) -> std::borrow::Cow<'a, str> { if source == "\u{feff}" && !content_type.is_some_and(|mime| { - is_dom_parser_xml_mime(mime) || moli_web_mime::is_text_document_mime(mime) + is_xml_document_mime(mime) || moli_web_mime::is_text_document_mime(mime) }) { std::borrow::Cow::Borrowed("\u{feff}") @@ -394,7 +394,7 @@ fn parse_browsing_context_document_snapshot( content_type: Option<&str>, html_parser: HtmlParser, ) -> (DomHost, DetachedDocumentKind) { - if content_type.is_some_and(is_dom_parser_xml_mime) + if content_type.is_some_and(is_xml_document_mime) || (content_type.is_none() && child_document_url_is_xml_like(&document_url)) { let parser = XmlParser; @@ -585,6 +585,34 @@ mod tests { assert!(disabled.element_handle_by_id("fallback").is_some()); } + #[test] + fn child_projection_preserves_xml_suffix_document_structure_and_namespace() { + let source = "Example"; + for mime in [ + "application/atom+xml", + "application/rss+xml", + "text/example+xml", + ] { + for path in ["feed.xml", "feed.html", "feed"] { + let (document, kind) = parse_browsing_context_document_snapshot( + Url::parse(&format!("https://example.test/{path}")).unwrap(), + source, + Some(mime), + HtmlParser::SCRIPTING_ENABLED, + ); + let root = first_document_element(&document); + let element = document + .node(root) + .and_then(|node| node.as_element()) + .unwrap(); + assert_eq!(kind, DetachedDocumentKind::Xml, "{mime} at {path}"); + assert_eq!(element.local_name(), "feed", "{mime} at {path}"); + assert_eq!(element.namespace(), "http://www.w3.org/2005/Atom"); + assert_eq!(document.text_content(root).as_deref(), Some("Example")); + } + } + } + #[test] fn child_projection_explicit_mime_overrides_xml_extension() { let source = "\n<&"; diff --git a/moli-renderer-v8/src/lib.rs b/moli-renderer-v8/src/lib.rs index 4f6db92afb1..40a2339213a 100644 --- a/moli-renderer-v8/src/lib.rs +++ b/moli-renderer-v8/src/lib.rs @@ -48,6 +48,7 @@ mod document_cookie_owner; mod document_language; mod document_last_modified; mod document_module_graph; +mod document_response_decoder; mod document_runtime; mod document_script_scheduler; mod document_task_lane; diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs index 1d4e2d9bcc3..f8016efb814 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs @@ -6,7 +6,7 @@ use crate::frame_owner_model::{ FrameDocumentLocalWindowTransition, FrameLocalWindowOwnerTransition, }; use crate::{document_runtime::DomHandle, frame_owner_model::FrameDocumentOwnerTransition}; -use moli_web_mime::is_dom_parser_xml_mime; +use moli_web_mime::is_xml_document_mime; use url::Url; pub(crate) struct ChildDocumentInstallResult { @@ -135,7 +135,7 @@ impl JsContextHost { let is_xml_document = snapshot .content_type .as_deref() - .is_some_and(is_dom_parser_xml_mime); + .is_some_and(is_xml_document_mime); self.install_live_child_document_from_snapshot( scope, handle, diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs index d429de74250..90fac5d675b 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs @@ -10,6 +10,7 @@ use crate::{ content_security_policy::{ ContentSecurityPolicyViolationEventFields, send_content_security_policy_reports, }, + document_response_decoder::decode_document_response, document_runtime::{DocumentNavigationEmbeddingContext, DocumentPolicyContainer, DomHandle}, document_script_scheduler::FrameDocumentClassicScriptSchedulerWork, frame_owner_model::{ @@ -21,8 +22,6 @@ use crate::{ LoadedChildDocument, SubresourceResponseBody, }, }; -use moli_encoding::{HtmlDocumentStreamingDecoder, decode_html_document_with_fallback}; -use moli_encoding_detector::detect_legacy_html_encoding; pub(crate) struct AppliedChildDocumentLoadCompletion { /// Initial parser-classic work produced by the committed child document. @@ -757,17 +756,6 @@ impl JsContextHost { } } -fn child_document_fallback_character_set( - parent_character_set: &str, - content_type: Option<&str>, -) -> String { - if content_type.is_some_and(moli_web_mime::is_dom_parser_xml_mime) { - "UTF-8".to_owned() - } else { - parent_character_set.to_owned() - } -} - fn child_document_load_outcome_from_response( request_url: String, request_method: String, @@ -786,30 +774,17 @@ fn child_document_load_outcome_from_response( let encoded_data_length = response_body.len(); let content_type = child_document_content_type_from_headers(&head.headers) .or_else(|| child_document_content_type_for_url(&head.final_url)); - let fallback = - child_document_fallback_character_set(parent_character_set, content_type.as_deref()); let (markup, character_set) = { let body_bytes = response_body .try_bytes() .map_err(|error| format!("failed to read child document response body: {error}"))?; - let (markup, character_set) = if let Some(mime) = content_type - .as_deref() - .filter(|mime| moli_web_mime::is_text_document_mime(mime)) - { - let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( - &head.headers, - head.final_url.as_str(), - detect_legacy_html_encoding, - moli_web_mime::is_json_module_mime(mime) || mime == "text/json", - ); - let mut markup = decoder.push(&body_bytes).concat(); - if let Some(tail) = decoder.finish() { - markup.push_str(&tail); - } - (markup, decoder.document_encoding_name()) - } else { - decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback)) - }; + let (markup, character_set) = decode_document_response( + &body_bytes, + &head.headers, + content_type.as_deref(), + &head.final_url, + Some(parent_character_set), + ); (markup, character_set.to_owned()) }; let policy_container = diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs index 71d318c3536..20e3c344bda 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs @@ -1,11 +1,10 @@ use super::super::{ChildBrowsingContextBootstrap, ChildBrowsingContextSnapshot, JsContextHost}; use super::configure_child_document_navigation_request; +use crate::document_response_decoder::decode_document_response; use crate::document_runtime::{DocumentPolicyContainer, DomHandle}; -use moli_encoding::decode_html_document_with_fallback; use moli_fetch::Request; use moli_web_mime::{ - is_dom_parser_xml_mime, is_html_document_mime, resource_mime_essence_for_url, - response_document_content_type, + is_html_document_mime, resource_mime_essence_for_url, response_document_content_type, }; use url::Url; @@ -52,7 +51,7 @@ impl JsContextHost { if !mime_type.is_empty() && !is_html_document_mime(&mime_type) { return None; } - let character_set = self.child_document_fallback_character_set(Some("text/html")); + let character_set = self.document_character_set().to_owned(); Some(ChildBrowsingContextSnapshot::with_character_set( url.clone(), body, @@ -64,11 +63,12 @@ impl JsContextHost { "data" => crate::network_host::local_url_response(url).map(|response| { let head = response.head(); let content_type = child_document_content_type_from_headers(&head.headers); - let fallback = self.child_document_fallback_character_set(content_type.as_deref()); - let (body, character_set) = decode_html_document_with_fallback( + let (body, character_set) = decode_document_response( response.body_bytes(), &head.headers, - Some(&fallback), + content_type.as_deref(), + &head.final_url, + Some(self.document_character_set()), ); let policy_container = DocumentPolicyContainer::from_navigation_response_headers( &head.headers, @@ -116,11 +116,12 @@ impl JsContextHost { let head = response.head(); let content_type = child_document_content_type_from_headers(&head.headers) .or_else(|| child_document_content_type_for_url(&head.final_url)); - let fallback = self.child_document_fallback_character_set(content_type.as_deref()); - let (markup, character_set) = decode_html_document_with_fallback( + let (markup, character_set) = decode_document_response( response.body_bytes(), &head.headers, - Some(&fallback), + content_type.as_deref(), + &head.final_url, + Some(self.document_character_set()), ); let policy_container = DocumentPolicyContainer::from_navigation_response_headers( &head.headers, @@ -170,17 +171,6 @@ impl JsContextHost { ChildBrowsingContextBootstrap::Request(_) => None, } } - - pub(in crate::native_bridge::context_host) fn child_document_fallback_character_set( - &self, - content_type: Option<&str>, - ) -> String { - if content_type.is_some_and(is_dom_parser_xml_mime) { - "UTF-8".to_owned() - } else { - self.document_character_set().to_owned() - } - } } pub(in crate::native_bridge::context_host) fn child_document_content_type_from_headers( diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs b/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs index 10c829ac1c6..1a84f1a04fa 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs @@ -238,7 +238,7 @@ impl JsContextHost { return false; } moli_web_mime::is_html_document_mime(&mime) - || moli_web_mime::is_dom_parser_xml_mime(&mime) + || moli_web_mime::is_xml_document_mime(&mime) || moli_web_mime::is_text_mime(&mime) || moli_web_mime::is_json_module_mime(&mime) || moli_web_mime::is_javascript_mime(&mime) diff --git a/moli-renderer-v8/src/native_bridge/context_host/popups.rs b/moli-renderer-v8/src/native_bridge/context_host/popups.rs index 5204a4d9214..4f3e187a247 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/popups.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/popups.rs @@ -2310,7 +2310,7 @@ impl JsContextHost { super::child_documents::child_document_content_type_from_headers(&head.headers); let fallback = if content_type .as_deref() - .is_some_and(moli_web_mime::is_dom_parser_xml_mime) + .is_some_and(moli_web_mime::is_xml_document_mime) { "UTF-8".to_owned() } else { diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rs index fa8df43b89a..f8c9351b983 100644 --- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs +++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs @@ -3,8 +3,8 @@ use super::super::script_preloads::{ServiceWorkerScriptPreloadContext, admit_pen use super::parser_blocking_pending::main_parser_blocking_classic_script_item; use super::scaffold::continue_phase_one_until_streaming_boundary_on_execution_context; use super::*; +use crate::document_response_decoder::new_document_response_decoder; use moli_encoding::HtmlDocumentStreamingDecoder; -use moli_encoding_detector::detect_legacy_html_encoding; use moli_web_mime::response_headers_indicate_attachment_download; use tokio::sync::{mpsc, oneshot}; @@ -471,7 +471,7 @@ impl ConcurrentParseTimeRuntime { fn response_headers_indicate_xml_document(headers: &[(String, Vec)]) -> bool { moli_web_mime::response_document_content_type(headers) - .is_some_and(|mime| moli_web_mime::is_dom_parser_xml_mime(&mime)) + .is_some_and(|mime| moli_web_mime::is_xml_document_mime(&mime)) } const EXTERNAL_RAW_DOCUMENT_BODY_BUFFERED_CHUNKS: usize = 8; @@ -615,8 +615,10 @@ fn response_document_parser( env: &PageVmEnvConfig, ) -> (ParseTimeDriverState, HtmlDocumentStreamingDecoder) { let content_type = moli_web_mime::response_document_content_type(headers); - let text_type = content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime)); - let mut state = if let Some(mime) = &text_type { + let text_type = content_type + .as_deref() + .filter(|mime| moli_web_mime::is_text_document_mime(mime)); + let mut state = if let Some(mime) = text_type { ParseTimeDriverState::new_text(final_url, mime) } else { ParseTimeDriverState::new_with_scripting_enabled( @@ -624,20 +626,8 @@ fn response_document_parser( main_document_parser_scripting_enabled(env), ) }; - let decoder = if let Some(mime) = text_type { - HtmlDocumentStreamingDecoder::new_text_document( - headers, - state.final_url.as_str(), - detect_legacy_html_encoding, - moli_web_mime::is_json_module_mime(&mime) || mime == "text/json", - ) - } else { - HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector( - headers, - state.final_url.as_str(), - detect_legacy_html_encoding, - ) - }; + let decoder = + new_document_response_decoder(headers, content_type.as_deref(), &state.final_url, None); // Headers/defaults are observable even when no body chunk is ready yet. // A later BOM or decoder decision can still replace this tentative value. sync_state_document_character_set_from_decoder(&mut state, &decoder); diff --git a/moli-web-mime/src/classification.rs b/moli-web-mime/src/classification.rs index 35fca10b589..9574cdde24e 100644 --- a/moli-web-mime/src/classification.rs +++ b/moli-web-mime/src/classification.rs @@ -13,6 +13,14 @@ pub fn is_dom_parser_xml_mime(input: &str) -> bool { }) } +/// XML navigation responses, including registered and custom `+xml` subtypes. +/// DOMParser's Web API accepts a narrower, separately classified set of types. +pub fn is_xml_document_mime(input: &str) -> bool { + mime_essence(input).is_some_and(|mime| { + matches!(mime.as_str(), "text/xml" | "application/xml") || mime.ends_with("+xml") + }) +} + pub fn is_javascript_mime(input: &str) -> bool { mime_essence(input).is_some_and(|mime| is_javascript_mime_essence(&mime)) } @@ -55,7 +63,7 @@ pub fn is_text_mime_essence(input: &str) -> bool { /// This also includes JSON and JavaScript resources opened as documents. pub fn is_text_document_mime(input: &str) -> bool { !is_html_document_mime(input) - && !is_dom_parser_xml_mime(input) + && !is_xml_document_mime(input) && (is_text_mime(input) || is_json_module_mime(input) || is_javascript_mime(input)) } @@ -143,6 +151,12 @@ pub fn is_json_module_mime(input: &str) -> bool { mime_essence(input).is_some_and(|mime| mime == "application/json" || mime.ends_with("+json")) } +pub fn is_json_document_mime(input: &str) -> bool { + mime_essence(input).is_some_and(|mime| { + matches!(mime.as_str(), "application/json" | "text/json") || mime.ends_with("+json") + }) +} + pub fn is_webassembly_mime(input: &str) -> bool { mime_essence(input).is_some_and(|mime| mime == "application/wasm") } diff --git a/moli-web-mime/src/lib.rs b/moli-web-mime/src/lib.rs index 806eb245521..bcae3a9eb85 100644 --- a/moli-web-mime/src/lib.rs +++ b/moli-web-mime/src/lib.rs @@ -13,10 +13,11 @@ pub use classification::{ is_css_style_element_type_attribute, is_css_stylesheet_type_hint, is_dom_parser_xml_mime, is_font_mime, is_font_mime_essence, is_form_urlencoded_mime, is_html_document_mime, is_image_mime, is_image_mime_essence, is_javascript_mime, is_javascript_mime_essence, - is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime, is_png_image_mime_essence, - is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence, - is_text_document_mime, is_text_mime, is_text_mime_essence, is_video_mime, - is_video_mime_essence, is_webassembly_mime, multipart_form_data_boundary, + is_json_document_mime, is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime, + is_png_image_mime_essence, is_supported_document_mime_type, is_svg_image_mime, + is_svg_image_mime_essence, is_text_document_mime, is_text_mime, is_text_mime_essence, + is_video_mime, is_video_mime_essence, is_webassembly_mime, is_xml_document_mime, + multipart_form_data_boundary, }; pub use data_url::{ data_url_body_and_computed_mime_type, data_url_body_and_mime_type, data_url_mime_type, diff --git a/moli-web-mime/src/sniffing.rs b/moli-web-mime/src/sniffing.rs index 5b21532b6c5..28f0d6342ab 100644 --- a/moli-web-mime/src/sniffing.rs +++ b/moli-web-mime/src/sniffing.rs @@ -1,4 +1,4 @@ -use crate::classification::{is_dom_parser_xml_mime, is_html_document_mime}; +use crate::classification::{is_html_document_mime, is_xml_document_mime}; use crate::parse::mime_essence; pub const RESOURCE_HEADER_BYTE_LIMIT: usize = 1445; @@ -212,7 +212,7 @@ fn computed_browsing_mime_type( } fn computed_image_mime_type(supplied: Option<&str>, header: &[u8]) -> String { - if supplied.is_some_and(is_xml_mime) { + if supplied.is_some_and(is_xml_document_mime) { return supplied.expect("checked supplied MIME").into(); } sniff_image_mime_type(header) @@ -222,7 +222,7 @@ fn computed_image_mime_type(supplied: Option<&str>, header: &[u8]) -> String { } fn computed_audio_video_mime_type(supplied: Option<&str>, header: &[u8]) -> String { - if supplied.is_some_and(is_xml_mime) { + if supplied.is_some_and(is_xml_document_mime) { return supplied.expect("checked supplied MIME").into(); } sniff_audio_video_mime_type(header) @@ -232,7 +232,7 @@ fn computed_audio_video_mime_type(supplied: Option<&str>, header: &[u8]) -> Stri } fn computed_font_mime_type(supplied: Option<&str>, header: &[u8]) -> String { - if supplied.is_some_and(is_xml_mime) { + if supplied.is_some_and(is_xml_document_mime) { return supplied.expect("checked supplied MIME").into(); } sniff_font_mime_type(header) @@ -424,11 +424,7 @@ fn is_binary_data_byte(byte: u8) -> bool { } fn is_xml_or_html_mime(mime: &str) -> bool { - is_xml_mime(mime) || is_html_mime(mime) -} - -fn is_xml_mime(mime: &str) -> bool { - is_dom_parser_xml_mime(mime) || mime.ends_with("+xml") + is_xml_document_mime(mime) || is_html_mime(mime) } fn is_html_mime(mime: &str) -> bool { diff --git a/moli-web-mime/src/tests.rs b/moli-web-mime/src/tests.rs index d29adb69b7a..d361b05e035 100644 --- a/moli-web-mime/src/tests.rs +++ b/moli-web-mime/src/tests.rs @@ -886,9 +886,53 @@ fn document_text_classification_excludes_html_and_xml() { "text/xml", "application/xhtml+xml", "image/svg+xml", + "application/atom+xml", + "application/rss+xml", + "text/example+xml; charset=utf-8", "image/png", "application/octet-stream", ] { assert!(!crate::is_text_document_mime(mime), "{mime}"); } } + +#[test] +fn navigation_xml_mime_classification_keeps_dom_parser_allowlist_separate() { + for mime in [ + "text/xml", + "application/xml", + "application/xhtml+xml", + "image/svg+xml", + ] { + assert!(is_xml_document_mime(mime), "{mime}"); + assert!(is_dom_parser_xml_mime(mime), "{mime}"); + } + for mime in [ + "Application/Atom+XML; charset=utf-8", + "application/rss+xml", + "text/example+xml", + ] { + assert!(is_xml_document_mime(mime), "{mime}"); + assert!(!is_dom_parser_xml_mime(mime), "{mime}"); + assert!(!is_text_document_mime(mime), "{mime}"); + } + for mime in [ + "text/html", + "text/plain", + "application/xml-dtd", + "invalid+xml", + ] { + assert!(!is_xml_document_mime(mime), "{mime}"); + } +} + +#[test] +fn navigation_json_mime_includes_text_json_without_changing_module_mime() { + for mime in ["application/json", "application/problem+json"] { + assert!(is_json_document_mime(mime), "{mime}"); + assert!(is_json_module_mime(mime), "{mime}"); + } + assert!(is_json_document_mime("Text/JSON; charset=utf-8")); + assert!(!is_json_module_mime("Text/JSON; charset=utf-8")); + assert!(!is_json_document_mime("text/plain")); +} From 9b4450a9b272f39e322b0603d843643cfce23700 Mon Sep 17 00:00:00 2001 From: ldm0 Date: Tue, 29 Sep 2026 11:58:55 +0800 Subject: [PATCH 5/6] fix(document): decode main XML responses before parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep network XML response bytes intact through the existing raw-body renderer path. Decode both network and external XML bodies with the shared response decoder, then install the selected encoding before document bootstrap. Remove the obsolete ready-body capture variant used by the old XML path. Cover matching main-document and iframe text, namespaces, and characterSet for Atom, RSS, and text/*+xml responses with Windows-1252 headers, XML declarations, and UTF-16LE/BE BOMs. Exercise the external input as single-byte chunks and verify that BOMs override a conflicting transport charset. Validation: - Before the fix, the network regression returned caf� / UTF-8 instead of café / windows-1252. - cargo fmt --all - cargo clippy --workspace --all-targets --all-features -- -D warnings - Six focused XML tests passed. - NEXTEST_TEST_THREADS=8 cargo nextest run --no-fail-fast: 18,954 passed, 16 skipped. --- .../tests/classic/extracted/navigation.rs | 115 ++++++++++++++++ moli-protocol/src/conn/runtime_load.rs | 126 ++---------------- .../src/runtime/phase_one/bootstrap.rs | 5 + .../src/runtime/phase_one/streaming.rs | 11 +- .../tests/extracted/navigation_streaming.rs | 97 ++++++++++++++ 5 files changed, 238 insertions(+), 116 deletions(-) diff --git a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs index 9649ec823e4..ab41e02230f 100644 --- a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs +++ b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs @@ -158,6 +158,121 @@ async fn webdriver_classic_xml_suffix_navigation_preserves_namespaces_and_dom_pa classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await; } +#[tokio::test] +async fn webdriver_classic_xml_main_and_child_documents_decode_response_bytes() { + const SOURCE: &str = "café"; + let legacy_body = b"caf\xe9"; + let cases = [ + ( + "charset", + "; charset=windows-1252", + legacy_body.to_vec(), + "windows-1252", + ), + ( + "declaration", + "", + [ + b"".as_slice(), + legacy_body, + ] + .concat(), + "windows-1252", + ), + ( + "utf16le", + "; charset=windows-1252", + [ + vec![0xff, 0xfe], + SOURCE.encode_utf16().flat_map(u16::to_le_bytes).collect(), + ] + .concat(), + "UTF-16LE", + ), + ( + "utf16be", + "", + [ + vec![0xfe, 0xff], + SOURCE.encode_utf16().flat_map(u16::to_be_bytes).collect(), + ] + .concat(), + "UTF-16BE", + ), + ]; + let mimes = [ + "application/atom+xml", + "application/rss+xml", + "text/example+xml", + ]; + let mut fixture = axum::Router::new(); + for (index, mime) in mimes.iter().enumerate() { + for (name, charset, body, _) in &cases { + let content_type = format!("{mime}{charset}"); + let body = body.clone(); + fixture = fixture.route( + &format!("/{index}-{name}"), + axum::routing::get(move || { + let content_type = content_type.clone(); + let body = body.clone(); + async move { ([(header::CONTENT_TYPE, content_type)], body) } + }), + ); + } + } + fixture = fixture.route( + "/parent/{kind}", + axum::routing::get( + |axum::extract::Path(kind): axum::extract::Path| async move { + ( + [(header::CONTENT_TYPE, "text/html; charset=utf-8")], + format!(""), + ) + }, + ), + ); + let (addr, _server) = spawn_dedicated_fixture_server(fixture, "xml-response-encoding"); + let app = build_router(test_state()); + let session = classic_request_json(app.clone(), Method::POST, "/session").await; + let session_id = session["value"]["sessionId"].as_str().unwrap(); + for (index, mime) in mimes.iter().enumerate() { + for (name, _, _, encoding) in &cases { + for prefix in ["", "parent/"] { + let path = format!("{prefix}{index}-{name}"); + let navigated = classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/url"), + json!({"url":format!("http://{addr}/{path}")}), + ) + .await; + assert_eq!(navigated, json!({"value":null}), "{path}"); + let observed = classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/execute/sync"), + json!({ + "script": "const d=document.querySelector('iframe')?.contentDocument ?? document; const root=d.getElementsByTagNameNS('http://www.w3.org/2005/Atom','feed')[0]; return [root.localName,root.namespaceURI,root.textContent,d.contentType,d.characterSet];", + "args": [], + }), + ).await; + assert_eq!( + observed["value"], + json!([ + "feed", + "http://www.w3.org/2005/Atom", + "café", + mime, + encoding + ]), + "{path}" + ); + } + } + } + classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await; +} + #[tokio::test] async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() { const PAYLOAD: &str = diff --git a/moli-protocol/src/conn/runtime_load.rs b/moli-protocol/src/conn/runtime_load.rs index d44b8a737e1..6b5109e783e 100644 --- a/moli-protocol/src/conn/runtime_load.rs +++ b/moli-protocol/src/conn/runtime_load.rs @@ -156,11 +156,6 @@ async fn prepare_network_error_page_navigation_with_engine_async( .map(NavigationLoadOutcome::response_commit_ready) } -fn response_headers_indicate_xml_document(headers: &[(String, Vec)]) -> bool { - moli_web_mime::response_document_content_type(headers) - .is_some_and(|mime| moli_web_mime::is_xml_document_mime(&mime)) -} - #[derive(Clone, Debug)] pub(crate) struct InitialDocumentPageOwner { pub(crate) browser_context_id: String, @@ -306,25 +301,17 @@ pub struct ResponseCommitReady { network_error_page: Option, } -enum ResponseCommitBodyCapture { - Pending(tokio::task::JoinHandle>), - Ready(CapturedBody), -} +struct ResponseCommitBodyCapture(tokio::task::JoinHandle>); impl ResponseCommitBodyCapture { fn abort(self) { - if let Self::Pending(task) = self { - task.abort(); - } + self.0.abort(); } async fn resolve(self) -> anyhow::Result { - match self { - Self::Pending(task) => task - .await - .context("main document body capture task failed")?, - Self::Ready(body) => Ok(body), - } + self.0 + .await + .context("main document body capture task failed")? } } @@ -1283,7 +1270,7 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async( let body_network_progress_state = body_progress_source.body_network_progress_for_completed_events(network_events); let body_progress_source_for_body_finish = body_progress_source.clone(); - let mut initial_body_chunk = if response_status_may_use_http_error_page(response_status) { + let initial_body_chunk = if response_status_may_use_http_error_page(response_status) { match first_nonempty_response_body_chunk(&mut response).await? { Some(chunk) => Some(chunk), None => { @@ -1308,97 +1295,8 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async( None }; - if response_headers_indicate_xml_document(&response_headers) { - let redirected = response.redirected; - let mut body_writer = CapturedBodyWriter::default(); - if let Some(chunk) = initial_body_chunk.take() { - body_writer - .append(&chunk) - .context("failed to capture XML page body")?; - } - while let Some(chunk) = response.next_chunk().await { - body_writer - .append(&chunk) - .context("failed to capture XML page body")?; - } - response - .finish() - .await - .context("failed to read XML page body from stream")?; - let captured_body = body_writer - .finish() - .context("failed to finish captured XML page body")?; - let response_text = captured_body - .materialize_bytes() - .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) - .context("failed to materialize XML page body")?; - let page_storage = load_inputs.page_storage_handles(); - let main_document_commit = load_inputs - .main_document_commit_for_final_url(&final_url, None) - .map(Arc::new); - let prepared_page = engine - .prepare_document_page_from_response_with_storage_and_inspector_session_restores_async( - page_reservation, - load_inputs.document_replacement.clone(), - page_storage.into_navigation_storage(), - requested_url.clone(), - final_url.clone(), - load_inputs.navigation_initiator_url.clone(), - redirected, - redirect_chain.len(), - response_status, - response_headers.clone(), - response_text, - load_inputs.document_start_scripts.clone(), - load_inputs.runtime_bindings.clone(), - load_inputs - .runtime_inspector_session_restore_snapshots - .clone(), - load_inputs.extra_http_headers.clone(), - load_inputs.script_execution_disabled, - load_inputs.bypass_content_security_policy, - load_inputs.emulated_media.clone(), - load_inputs.viewport_surface, - load_inputs.network_offline, - load_inputs.blocked_url_patterns.clone(), - fetch_subresource_interception_enabled, - fetch_subresource_interception_resource_type, - load_inputs.root_frame_id.clone(), - resource_source, - main_document_commit.as_deref().cloned(), - ) - .await - .with_context(|| format!("failed to prepare XML page `{}`", requested_url))?; - if timing_enabled { - tracing::info!( - target: "moli_cdp_nav_timing", - url = %requested_url, - stage = "response_commit_ready", - elapsed_ms = timing_started.elapsed().as_millis(), - ); - } - return Ok(NavigationLoadOutcome::response_commit_ready( - ResponseCommitReady { - prepared_page: Some(prepared_page), - body_capture: Some(ResponseCommitBodyCapture::Ready(captured_body)), - body_completion_sink, - body_progress_source: body_progress_source_for_body_finish, - body_network_progress_state: Some(body_network_progress_state), - synthetic_body: false, - requested_url, - final_url, - request_method, - request_headers, - response_status, - response_headers, - response_from_cache, - timing_started: timing_enabled.then_some(timing_started), - main_document_commit, - network_error_page: None, - }, - )); - } - + // Preserve response bytes until the renderer selects the document encoding, + // including XML charsets and BOMs. let (body_tx, body_rx) = mpsc::channel(EXTERNAL_RAW_BODY_CHANNEL_CAPACITY); let (completion_tx, completion_rx) = oneshot::channel(); let raw_body = moli_core::runtime::ExternalRawDocumentBodyStream::new(body_rx, completion_rx); @@ -1467,7 +1365,7 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async( Ok(NavigationLoadOutcome::response_commit_ready( ResponseCommitReady { prepared_page: Some(prepared_page), - body_capture: Some(ResponseCommitBodyCapture::Pending(body_capture_task)), + body_capture: Some(ResponseCommitBodyCapture(body_capture_task)), body_completion_sink, body_progress_source: body_progress_source_for_body_finish, body_network_progress_state: Some(body_network_progress_state), @@ -3804,7 +3702,7 @@ async fn prepare_captured_document_response_with_engine_async( Ok(ResponseCommitReady { prepared_page: Some(prepared_page), - body_capture: Some(ResponseCommitBodyCapture::Pending(body_capture_task)), + body_capture: Some(ResponseCommitBodyCapture(body_capture_task)), body_completion_sink: None, body_progress_source, body_network_progress_state: Some(body_network_progress_state), @@ -3950,7 +3848,7 @@ mod tests { let task = super::spawn_streaming_body_capture(response, None, body_tx, completion_tx); assert_eq!(body_rx.recv().await.unwrap(), b"partial body"); assert!(body_rx.recv().await.is_none()); - let navigation_error = super::ResponseCommitBodyCapture::Pending(task) + let navigation_error = super::ResponseCommitBodyCapture(task) .resolve() .await .expect_err("partial response must fail navigation capture"); @@ -3977,7 +3875,7 @@ mod tests { async fn cancelled_body_capture_retains_join_error() { let task = tokio::spawn(std::future::pending::>()); task.abort(); - let error = super::ResponseCommitBodyCapture::Pending(task) + let error = super::ResponseCommitBodyCapture(task) .resolve() .await .expect_err("cancelled capture must fail"); diff --git a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs index 454433e6c3a..1f5ec22b1fb 100644 --- a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs +++ b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs @@ -140,11 +140,16 @@ impl ConcurrentParseTimeRuntime { runtime_hooks: PageVmRuntimeHooks, final_url: Url, document_content_type: String, + document_character_set: &str, stage: PageVmInitStage, source: String, started: Instant, ) -> Result { let mut state = ParseTimeDriverState::new_xml(final_url.clone()); + state.document_character_set = document_character_set.to_owned(); + state + .buffered_document_preloads + .set_document_character_set(document_character_set); state .parser_session .set_xml_document_content_type(document_content_type); diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rs index f8c9351b983..395da9c8c2e 100644 --- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs +++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs @@ -3,7 +3,7 @@ use super::super::script_preloads::{ServiceWorkerScriptPreloadContext, admit_pen use super::parser_blocking_pending::main_parser_blocking_classic_script_item; use super::scaffold::continue_phase_one_until_streaming_boundary_on_execution_context; use super::*; -use crate::document_response_decoder::new_document_response_decoder; +use crate::document_response_decoder::{decode_document_response, new_document_response_decoder}; use moli_encoding::HtmlDocumentStreamingDecoder; use moli_web_mime::response_headers_indicate_attachment_download; use tokio::sync::{mpsc, oneshot}; @@ -301,9 +301,15 @@ impl ConcurrentParseTimeRuntime { let mut body_source = RawDocumentBodySource::External(raw_body); if response_headers_indicate_xml_document(&response_headers) { let body = collect_streaming_raw_body(&mut body_source).await?; - let source = String::from_utf8_lossy(&body).into_owned(); let content_type = moli_web_mime::response_document_content_type(&response_headers) .unwrap_or_else(|| "application/xml".to_owned()); + let (source, character_set) = decode_document_response( + &body, + &response_headers, + Some(&content_type), + &final_url, + None, + ); let outcome = Self::finish_creation_from_xml_bootstrap( page_id, local_executor, @@ -312,6 +318,7 @@ impl ConcurrentParseTimeRuntime { runtime_hooks, final_url, content_type, + character_set, stage, source, started, diff --git a/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs b/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs index bb3e6a40e4f..51ad6972021 100644 --- a/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs +++ b/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs @@ -1,5 +1,102 @@ use super::*; +#[tokio::test(flavor = "multi_thread")] +async fn external_raw_xml_document_decodes_response_bytes() { + const SOURCE: &str = "café"; + let legacy_body = b"caf\xe9"; + let cases = [ + ( + "application/atom+xml; charset=windows-1252", + legacy_body.to_vec(), + "windows-1252", + ), + ( + "application/atom+xml", + [ + b"".as_slice(), + legacy_body, + ] + .concat(), + "windows-1252", + ), + ( + "application/atom+xml; charset=windows-1252", + [ + vec![0xff, 0xfe], + SOURCE.encode_utf16().flat_map(u16::to_le_bytes).collect(), + ] + .concat(), + "UTF-16LE", + ), + ( + "application/atom+xml", + [ + vec![0xfe, 0xff], + SOURCE.encode_utf16().flat_map(u16::to_be_bytes).collect(), + ] + .concat(), + "UTF-16BE", + ), + ]; + let runtime = JsRuntime::initialize(); + let loader = + ResourceRequestClient::new(&moli_fetch::FetchConfig::default()).expect("default loader"); + for (content_type, body, encoding) in cases { + let (completion_tx, completion_rx) = oneshot::channel(); + let (body_tx, raw_body) = ExternalRawDocumentBodyStream::channel(completion_rx); + let producer = tokio::spawn(async move { + for byte in body { + body_tx + .send(vec![byte]) + .await + .expect("body byte should send"); + } + completion_tx.send(Ok(())).expect("body should complete"); + }); + let prepared = prepare_test_external_raw_document_with_content_type( + &runtime, + &loader, + url::Url::parse("https://example.test/feed").unwrap(), + content_type, + raw_body, + ) + .await; + let permit = prepared.issue_commit_permit(); + let (mut page, _, _, _, pending_download) = + prepared.commit(permit).await.expect("XML should commit"); + assert!(pending_download.is_none()); + producer.await.expect("body producer should finish"); + let (reply, _) = page + .run_async_command(RendererPageCommand::EvaluateExpression { + expression: concat!( + "JSON.stringify((() => { const root=document.getElementsByTagNameNS(", + "'http://www.w3.org/2005/Atom','feed')[0]; ", + "return [root.localName,root.namespaceURI,root.textContent,", + "document.contentType,document.characterSet]; })())", + ) + .to_owned(), + await_promise: false, + }) + .await + .expect("XML response metadata and text should evaluate"); + assert_eq!( + renderer_json_value(reply), + Some(serde_json::Value::String( + serde_json::json!([ + "feed", + "http://www.w3.org/2005/Atom", + "café", + "application/atom+xml", + encoding + ]) + .to_string() + )), + "{content_type}: {encoding}" + ); + page.close_async().await.expect("XML page should close"); + } +} + #[tokio::test(flavor = "multi_thread")] async fn streaming_xml_document_executes_parser_blocking_xhtml_script() { let runtime = JsRuntime::initialize(); From 50393080f266afb383fab210ebeb6a9a5ed2b863 Mon Sep 17 00:00:00 2001 From: ldm0 Date: Tue, 29 Sep 2026 16:12:08 +0800 Subject: [PATCH 6/6] fix(encoding): exclude HTML meta from XML encoding selection Give document decoding explicit HTML, XML, and text declaration policies. XML ignores HTML meta elements, retains BOM and transport charset precedence, and selects an initial XML declaration or UTF-16 signature before its UTF-8 default. Keep partial declarations and signatures buffered across chunks. Cover conflicting meta declarations with and without an XML declaration, including main documents, iframes, and external raw input. Adapt seven Chromium XML/XHTML navigation scenarios and three decoder boundary tests from Chromium a03603fe9af6230a12f1b2fb2c18a7d003a0d937; source paths are recorded next to the tests. These include KOI8-R script inheritance, CP1251 XML, BOM-less UTF-16, supplementary Unicode characters, and BOM boundaries. Validation: - cargo fmt --all - cargo clippy --workspace --all-targets --all-features -- -D warnings - XML and Chromium regression tests passed. - NEXTEST_TEST_THREADS=8 cargo nextest run --no-fail-fast: 18,980 passed, 16 skipped. --- moli-encoding/src/document.rs | 92 +++++++-- moli-encoding/src/tests.rs | 181 ++++++++++++++++++ .../tests/classic/extracted/navigation.rs | 167 +++++++++++++++- .../src/document_response_decoder.rs | 32 +++- .../tests/extracted/navigation_streaming.rs | 6 +- 5 files changed, 458 insertions(+), 20 deletions(-) diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rs index db8e77016fa..188351fc6d0 100644 --- a/moli-encoding/src/document.rs +++ b/moli-encoding/src/document.rs @@ -36,6 +36,13 @@ pub fn decode_html_document_with_fallback( (output, encoding) } +#[derive(Clone, Copy)] +enum DocumentDeclarationPolicy { + Html, + Xml, + Text, +} + pub struct HtmlDocumentStreamingDecoder { transport_encoding: Option<&'static Encoding>, fallback_encoding: &'static Encoding, @@ -47,7 +54,7 @@ pub struct HtmlDocumentStreamingDecoder { url_hint: Option, decoder: Option, selected_encoding: Option<&'static Encoding>, - sniff_html_declarations: bool, + declaration_policy: DocumentDeclarationPolicy, } impl HtmlDocumentStreamingDecoder { @@ -68,16 +75,28 @@ impl HtmlDocumentStreamingDecoder { inherited_encoding, Some(url_hint), inherited_encoding.is_none().then_some(detector), + DocumentDeclarationPolicy::Text, ); - decoder.sniff_html_declarations = false; if json && decoder.transport_encoding.is_none() { decoder.transport_encoding = Some(encoding_rs::UTF_8); } decoder } + /// Decode XML using BOMs, transport charsets, and its initial declaration. + /// HTML meta elements never select an encoding; undeclared XML uses UTF-8. + pub fn new_xml_document(headers: &[(String, Vec)]) -> Self { + Self::new_with_options( + headers, + Some("UTF-8"), + None, + None, + DocumentDeclarationPolicy::Xml, + ) + } + pub fn new(headers: &[(String, Vec)]) -> Self { - Self::new_with_options(headers, None, None, None) + Self::new_with_options(headers, None, None, None, DocumentDeclarationPolicy::Html) } pub fn new_with_legacy_encoding_detector( @@ -85,14 +104,26 @@ impl HtmlDocumentStreamingDecoder { url_hint: &str, detector: LegacyEncodingDetector, ) -> Self { - Self::new_with_options(headers, None, Some(url_hint), Some(detector)) + Self::new_with_options( + headers, + None, + Some(url_hint), + Some(detector), + DocumentDeclarationPolicy::Html, + ) } pub fn new_with_fallback( headers: &[(String, Vec)], fallback_encoding: Option<&str>, ) -> Self { - Self::new_with_options(headers, fallback_encoding, None, None) + Self::new_with_options( + headers, + fallback_encoding, + None, + None, + DocumentDeclarationPolicy::Html, + ) } fn new_with_options( @@ -100,6 +131,7 @@ impl HtmlDocumentStreamingDecoder { fallback_encoding: Option<&str>, url_hint: Option<&str>, legacy_encoding_detector: Option, + declaration_policy: DocumentDeclarationPolicy, ) -> Self { Self { transport_encoding: encoding_from_response_headers(headers), @@ -114,7 +146,7 @@ impl HtmlDocumentStreamingDecoder { url_hint: url_hint.map(str::to_owned), decoder: None, selected_encoding: None, - sniff_html_declarations: true, + declaration_policy, } } @@ -177,15 +209,49 @@ impl HtmlDocumentStreamingDecoder { if let Some(encoding) = self.transport_encoding { return Some(encoding); } - if !self.sniff_html_declarations { - if self.legacy_encoding_detector.is_none() { - return Some(self.fallback_encoding); + match self.declaration_policy { + DocumentDeclarationPolicy::Html => self.html_encoding_ready(finishing), + DocumentDeclarationPolicy::Xml => self.xml_encoding_ready(finishing), + DocumentDeclarationPolicy::Text => { + if self.legacy_encoding_detector.is_none() { + return Some(self.fallback_encoding); + } + (finishing || self.sniff_buffer.len() >= 1024).then(|| { + self.detected_legacy_content_encoding() + .unwrap_or(self.fallback_encoding) + }) + } + } + } + + fn xml_encoding_ready(&self, finishing: bool) -> Option<&'static Encoding> { + let bytes = &self.sniff_buffer; + if let Some(encoding) = encoding_for_document_utf16_xml_prefix(bytes) { + return Some(encoding); + } + if !finishing + && (bytes_could_still_be_utf16_xml_prefix(bytes) || b" is an ordinary processing instruction. + if bytes.starts_with(b"') else { + return finishing.then_some(self.fallback_encoding); + }; + if bytes[end - 1] == b'?' { + return Some( + encoding_for_document_xml_declaration(&bytes[..=end]) + .unwrap_or(self.fallback_encoding), + ); } - return (finishing || self.sniff_buffer.len() >= 1024).then(|| { - self.detected_legacy_content_encoding() - .unwrap_or(self.fallback_encoding) - }); } + Some(self.fallback_encoding) + } + + fn html_encoding_ready(&mut self, finishing: bool) -> Option<&'static Encoding> { if let Some(encoding) = encoding_for_document_utf16_xml_prefix(&self.sniff_buffer) { return Some(encoding); } diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs index 1ce5be37b05..b4c0104e7a5 100644 --- a/moli-encoding/src/tests.rs +++ b/moli-encoding/src/tests.rs @@ -24,6 +24,187 @@ fn unexpected_legacy_encoding_detector( panic!("declared encoding must take precedence over heuristic detection") } +#[test] +fn xml_document_ignores_html_meta_across_chunk_splits() { + for declaration in ["", ""] { + for meta in [ + "", + "", + ] { + let source = format!("{declaration}{meta}café"); + assert_xml_document_decodes_across_chunk_splits( + &[], + source.as_bytes(), + &source, + "UTF-8", + ); + } + } +} + +fn assert_xml_document_decodes_across_chunk_splits( + headers: &[(String, Vec)], + bytes: &[u8], + source: &str, + encoding: &str, +) { + for split in 0..=bytes.len() { + let mut decoder = HtmlDocumentStreamingDecoder::new_xml_document(headers); + let mut decoded = decoder.push(&bytes[..split]).concat(); + decoded.push_str(&decoder.push(&bytes[split..]).concat()); + decoded.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(decoded, source, "{encoding}: split {split}"); + assert_eq!(decoder.document_encoding_name(), encoding, "split {split}"); + } + let mut decoder = HtmlDocumentStreamingDecoder::new_xml_document(headers); + let mut decoded = String::new(); + for byte in bytes { + decoded.push_str(&decoder.push(std::slice::from_ref(byte)).concat()); + } + decoded.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(decoded, source, "{encoding}: single-byte chunks"); + assert_eq!(decoder.document_encoding_name(), encoding); +} + +#[test] +fn xml_document_preserves_encoding_precedence_across_chunk_splits() { + let source = "\ + café"; + let legacy = encoding_rs::WINDOWS_1252.encode(source).0.into_owned(); + assert_xml_document_decodes_across_chunk_splits(&[], &legacy, source, "windows-1252"); + + let headers = [( + "Content-Type".to_owned(), + b"application/xml; charset=utf-8".to_vec(), + )]; + assert_xml_document_decodes_across_chunk_splits(&headers, source.as_bytes(), source, "UTF-8"); + + let headers = [( + "Content-Type".to_owned(), + b"application/xml; charset=windows-1252".to_vec(), + )]; + let bom_body = [b"\xef\xbb\xbf".as_slice(), source.as_bytes()].concat(); + assert_xml_document_decodes_across_chunk_splits(&headers, &bom_body, source, "UTF-8"); + + let source = "\ + café"; + for (bytes, bom, encoding) in [ + (utf16le_bytes(source), [0xff, 0xfe], "UTF-16LE"), + (utf16be_bytes(source), [0xfe, 0xff], "UTF-16BE"), + ] { + assert_xml_document_decodes_across_chunk_splits(&[], &bytes, source, encoding); + let bom_body = [bom.as_slice(), &bytes].concat(); + assert_xml_document_decodes_across_chunk_splits(&headers, &bom_body, source, encoding); + } +} + +#[test] +fn xml_document_ignores_encoding_attributes_in_processing_instructions() { + let source = "\ + café"; + assert_xml_document_decodes_across_chunk_splits(&[], source.as_bytes(), source, "UTF-8"); +} + +#[test] +fn xml_document_selects_encoding_at_the_end_of_its_declaration() { + let mut decoder = HtmlDocumentStreamingDecoder::new_xml_document(&[]); + let mut decoded = decoder + .push(b"").concat()); + assert_eq!(decoder.selected_encoding_name(), Some("windows-1252")); + decoded.push_str(&decoder.push(b"caf\xe9").concat()); + decoded.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!( + decoded, + "café" + ); +} + +#[test] +fn xml_document_finishes_empty_or_incomplete_declarations_with_utf8() { + for source in [ + "", + "<", + "foo"; + for (mut decoder, expected_encoding) in [ + (HtmlDocumentStreamingDecoder::new(&[]), "UTF-8"), + (HtmlDocumentStreamingDecoder::new_xml_document(&[]), "UTF-8"), + ( + HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://example.test/", + test_legacy_encoding_detector, + false, + None, + ), + "windows-1252", + ), + ] { + let mut decoded = String::new(); + for byte in source { + decoded.push_str(&decoder.push(std::slice::from_ref(byte)).concat()); + } + decoded.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(decoded.as_bytes(), source); + assert_eq!(decoder.document_encoding_name(), expected_encoding); + } +} + +// Chromium text_resource_decoder_test.cc, BrokenBOMs (revision above). +#[test] +fn chromium_plain_text_broken_boms_are_flushed_with_the_default_encoding() { + for (bytes, expected) in [ + (b"\xef\xbb".as_slice(), "ï»"), + (b"\xff", "ÿ"), + (b"\xfe", "þ"), + ] { + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://example.test/", + test_legacy_encoding_detector, + false, + None, + ); + for byte in bytes { + assert!(decoder.push(std::slice::from_ref(byte)).is_empty()); + } + assert_eq!(decoder.finish().as_deref(), Some(expected)); + assert_eq!(decoder.document_encoding_name(), "windows-1252"); + } +} + +// Adapt Chromium fast/encoding/bom-in-content{,-utf16}.html to the XML policy. +#[test] +fn chromium_bom_in_content_is_preserved_by_xml_decoding() { + let source = "\u{feff}"; + for (bytes, bom, encoding) in [ + ( + source.as_bytes().to_vec(), + b"\xef\xbb\xbf".as_slice(), + "UTF-8", + ), + (utf16le_bytes(source), b"\xff\xfe", "UTF-16LE"), + (utf16be_bytes(source), b"\xfe\xff", "UTF-16BE"), + ] { + let bytes = [bom, &bytes].concat(); + assert_xml_document_decodes_across_chunk_splits(&[], &bytes, source, encoding); + } +} + #[test] fn json_text_document_defaults_to_utf8_across_every_chunk_split() { let input = "{\"name\":\"Gülçek\"}"; diff --git a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs index ab41e02230f..3b712d0e0ee 100644 --- a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs +++ b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs @@ -160,9 +160,22 @@ async fn webdriver_classic_xml_suffix_navigation_preserves_namespaces_and_dom_pa #[tokio::test] async fn webdriver_classic_xml_main_and_child_documents_decode_response_bytes() { - const SOURCE: &str = "café"; - let legacy_body = b"caf\xe9"; + const SOURCE: &str = "\ + café"; + let legacy_body = b"\ + caf\xe9"; let cases = [ + ("meta", "", SOURCE.as_bytes().to_vec(), "UTF-8"), + ( + "declared-utf8-meta", + "", + [ + b"".as_slice(), + SOURCE.as_bytes(), + ] + .concat(), + "UTF-8", + ), ( "charset", "; charset=windows-1252", @@ -273,6 +286,156 @@ async fn webdriver_classic_xml_main_and_child_documents_decode_response_bytes() classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await; } +#[tokio::test] +async fn webdriver_classic_chromium_xml_encoding_scenarios() { + // Adapted from Chromium a03603fe9af6230a12f1b2fb2c18a7d003a0d937, + // third_party/blink/web_tests/fast/encoding/{meta-in-xhtml.xhtml, + // default-xhtml-encoding.xhtml, xml-utf-8-default.xml, + // external-script-charset.xhtml, dumpAsText/utf-16-no-bom.xml} and + // http/tests/xmlviewer/dumpAsText/xmlviewer-charset-{cp1251,utf8}.xml. + const XHTML_NAMESPACE: &str = "http://www.w3.org/1999/xhtml"; + let report_charset = "

"; + let unicode = "—∆∏π𝓐𝖚"; + let cases = [ + ( + "meta-in-xhtml", + "application/xhtml+xml", + format!( + "\ + \ + {report_charset}" + ).into_bytes(), + "UTF-8", + "UTF-8", + Some(XHTML_NAMESPACE), + ), + ( + "default-xhtml-encoding", + "application/xhtml+xml", + format!("{report_charset}").into_bytes(), + "UTF-8", + "UTF-8", + Some(XHTML_NAMESPACE), + ), + ( + "xml-utf-8-default", + "application/xml", + format!( + "\ +

{unicode}

" + ).into_bytes(), + "UTF-8", + unicode, + Some(XHTML_NAMESPACE), + ), + ( + "external-script-charset", + "application/xhtml+xml", + format!( + "\ +

" + ).into_bytes(), + "KOI8-R", + "PASS", + Some(XHTML_NAMESPACE), + ), + ( + "xmlviewer-charset-cp1251", + "application/xml", + b"\ + SU\xd1\xd1\xc5SS".to_vec(), + "windows-1251", + "SUССЕSS", + None, + ), + ( + "xmlviewer-charset-utf8", + "application/xml", + "\ + SUССЕSS".as_bytes().to_vec(), + "UTF-8", + "SUССЕSS", + None, + ), + ( + "utf-16-no-bom", + "application/xml", + "ё" + .encode_utf16().flat_map(u16::to_be_bytes).collect(), + "UTF-16BE", + "ё", + None, + ), + ]; + let mut fixture = axum::Router::new(); + for (name, mime, body, _, _, _) in &cases { + let mime = *mime; + let body = body.clone(); + fixture = fixture.route( + &format!("/{name}"), + axum::routing::get(move || { + let body = body.clone(); + async move { ([(header::CONTENT_TYPE, mime)], body) } + }), + ); + } + fixture = fixture + .route( + "/external-script-charset-koi-8.js", + axum::routing::get(|| async { + ( + [(header::CONTENT_TYPE, "text/javascript")], + b"document.getElementById('result').textContent = ('\xf3' == '\\u0421') ? 'PASS' : 'FAIL';".as_slice(), + ) + }), + ) + .route( + "/parent/{kind}", + axum::routing::get( + |axum::extract::Path(kind): axum::extract::Path| async move { + ( + [(header::CONTENT_TYPE, "text/html; charset=windows-1252")], + format!(""), + ) + }, + ), + ); + let (addr, _server) = spawn_dedicated_fixture_server(fixture, "chromium-xml-encoding"); + let app = build_router(test_state()); + let session = classic_request_json(app.clone(), Method::POST, "/session").await; + let session_id = session["value"]["sessionId"].as_str().unwrap(); + for (name, mime, _, encoding, expected, namespace) in cases { + for prefix in ["", "parent/"] { + let navigated = classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/url"), + json!({"url":format!("http://{addr}/{prefix}{name}")}), + ) + .await; + assert_eq!(navigated, json!({"value":null}), "{prefix}{name}"); + let observed = classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/execute/sync"), + json!({ + "script": "const d=document.querySelector('iframe')?.contentDocument ?? document; const result=d.getElementById('result'); return [result.textContent,d.characterSet,d.contentType,result.namespaceURI];", + "args": [], + }), + ).await; + assert_eq!( + observed["value"], + json!([expected, encoding, mime, namespace]), + "{prefix}{name}" + ); + } + } + classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await; +} + #[tokio::test] async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() { const PAYLOAD: &str = diff --git a/moli-renderer-v8/src/document_response_decoder.rs b/moli-renderer-v8/src/document_response_decoder.rs index 5d291b90ef1..4acc309b926 100644 --- a/moli-renderer-v8/src/document_response_decoder.rs +++ b/moli-renderer-v8/src/document_response_decoder.rs @@ -18,9 +18,7 @@ pub(crate) fn new_document_response_decoder( inherited_encoding, ) } else if content_type.is_some_and(is_xml_document_mime) { - // XML declarations still apply; neither parent encoding nor heuristics - // can replace XML's UTF-8 default. - HtmlDocumentStreamingDecoder::new_with_fallback(headers, Some("UTF-8")) + HtmlDocumentStreamingDecoder::new_xml_document(headers) } else if inherited_encoding.is_some() { HtmlDocumentStreamingDecoder::new_with_fallback(headers, inherited_encoding) } else { @@ -52,6 +50,34 @@ pub(crate) fn decode_document_response( mod tests { use super::*; + #[test] + fn xml_responses_ignore_html_meta_encoding_declarations() { + for mime in [ + "application/xml", + "text/xml", + "application/atom+xml", + "text/example+xml", + "application/xhtml+xml", + ] { + let headers = [("Content-Type".to_owned(), mime.as_bytes().to_vec())]; + for declaration in ["", ""] { + let source = format!( + "{declaration}\ + café" + ); + let (decoded, encoding) = decode_document_response( + source.as_bytes(), + &headers, + Some(mime), + &Url::parse("https://example.test/document").unwrap(), + Some("GBK"), + ); + assert_eq!(decoded, source, "{mime}: {declaration}"); + assert_eq!(encoding, "UTF-8", "{mime}: {declaration}"); + } + } + } + #[test] fn json_and_xml_responses_keep_utf8_defaults_with_a_legacy_parent() { for (mime, source) in [ diff --git a/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs b/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs index 51ad6972021..cffd375545e 100644 --- a/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs +++ b/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs @@ -2,8 +2,10 @@ use super::*; #[tokio::test(flavor = "multi_thread")] async fn external_raw_xml_document_decodes_response_bytes() { - const SOURCE: &str = "café"; - let legacy_body = b"caf\xe9"; + const SOURCE: &str = "\ + café"; + let legacy_body = b"\ + caf\xe9"; let cases = [ ( "application/atom+xml; charset=windows-1252",