diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rs index 1ceb79b9f9f..188351fc6d0 100644 --- a/moli-encoding/src/document.rs +++ b/moli-encoding/src/document.rs @@ -36,6 +36,13 @@ pub fn decode_html_document_with_fallback( (output, encoding) } +#[derive(Clone, Copy)] +enum DocumentDeclarationPolicy { + Html, + Xml, + Text, +} + pub struct HtmlDocumentStreamingDecoder { transport_encoding: Option<&'static Encoding>, fallback_encoding: &'static Encoding, @@ -47,11 +54,49 @@ pub struct HtmlDocumentStreamingDecoder { url_hint: Option, decoder: Option, selected_encoding: Option<&'static Encoding>, + declaration_policy: DocumentDeclarationPolicy, } impl HtmlDocumentStreamingDecoder { + /// Decode a text document without interpreting literal HTML/XML declarations. + /// BOMs and transport charsets take precedence. JSON then defaults to UTF-8; + /// other text uses a valid inherited encoding before heuristic detection. + pub fn new_text_document( + headers: &[(String, Vec)], + url_hint: &str, + detector: LegacyEncodingDetector, + json: bool, + inherited_encoding: Option<&str>, + ) -> Self { + let inherited_encoding = + inherited_encoding.filter(|label| encoding_for_label(label).is_some()); + let mut decoder = Self::new_with_options( + headers, + inherited_encoding, + Some(url_hint), + inherited_encoding.is_none().then_some(detector), + DocumentDeclarationPolicy::Text, + ); + if json && decoder.transport_encoding.is_none() { + decoder.transport_encoding = Some(encoding_rs::UTF_8); + } + decoder + } + + /// Decode XML using BOMs, transport charsets, and its initial declaration. + /// HTML meta elements never select an encoding; undeclared XML uses UTF-8. + pub fn new_xml_document(headers: &[(String, Vec)]) -> Self { + Self::new_with_options( + headers, + Some("UTF-8"), + None, + None, + DocumentDeclarationPolicy::Xml, + ) + } + pub fn new(headers: &[(String, Vec)]) -> Self { - Self::new_with_options(headers, None, None, None) + Self::new_with_options(headers, None, None, None, DocumentDeclarationPolicy::Html) } pub fn new_with_legacy_encoding_detector( @@ -59,14 +104,26 @@ impl HtmlDocumentStreamingDecoder { url_hint: &str, detector: LegacyEncodingDetector, ) -> Self { - Self::new_with_options(headers, None, Some(url_hint), Some(detector)) + Self::new_with_options( + headers, + None, + Some(url_hint), + Some(detector), + DocumentDeclarationPolicy::Html, + ) } pub fn new_with_fallback( headers: &[(String, Vec)], fallback_encoding: Option<&str>, ) -> Self { - Self::new_with_options(headers, fallback_encoding, None, None) + Self::new_with_options( + headers, + fallback_encoding, + None, + None, + DocumentDeclarationPolicy::Html, + ) } fn new_with_options( @@ -74,6 +131,7 @@ impl HtmlDocumentStreamingDecoder { fallback_encoding: Option<&str>, url_hint: Option<&str>, legacy_encoding_detector: Option, + declaration_policy: DocumentDeclarationPolicy, ) -> Self { Self { transport_encoding: encoding_from_response_headers(headers), @@ -88,6 +146,7 @@ impl HtmlDocumentStreamingDecoder { url_hint: url_hint.map(str::to_owned), decoder: None, selected_encoding: None, + declaration_policy, } } @@ -97,6 +156,7 @@ impl HtmlDocumentStreamingDecoder { pub fn document_encoding_name(&self) -> &'static str { self.selected_encoding_name() + .or_else(|| self.transport_encoding.map(Encoding::name)) .unwrap_or(self.fallback_encoding.name()) } @@ -149,6 +209,49 @@ impl HtmlDocumentStreamingDecoder { if let Some(encoding) = self.transport_encoding { return Some(encoding); } + match self.declaration_policy { + DocumentDeclarationPolicy::Html => self.html_encoding_ready(finishing), + DocumentDeclarationPolicy::Xml => self.xml_encoding_ready(finishing), + DocumentDeclarationPolicy::Text => { + if self.legacy_encoding_detector.is_none() { + return Some(self.fallback_encoding); + } + (finishing || self.sniff_buffer.len() >= 1024).then(|| { + self.detected_legacy_content_encoding() + .unwrap_or(self.fallback_encoding) + }) + } + } + } + + fn xml_encoding_ready(&self, finishing: bool) -> Option<&'static Encoding> { + let bytes = &self.sniff_buffer; + if let Some(encoding) = encoding_for_document_utf16_xml_prefix(bytes) { + return Some(encoding); + } + if !finishing + && (bytes_could_still_be_utf16_xml_prefix(bytes) || b" is an ordinary processing instruction. + if bytes.starts_with(b"') else { + return finishing.then_some(self.fallback_encoding); + }; + if bytes[end - 1] == b'?' { + return Some( + encoding_for_document_xml_declaration(&bytes[..=end]) + .unwrap_or(self.fallback_encoding), + ); + } + } + Some(self.fallback_encoding) + } + + fn html_encoding_ready(&mut self, finishing: bool) -> Option<&'static Encoding> { if let Some(encoding) = encoding_for_document_utf16_xml_prefix(&self.sniff_buffer) { return Some(encoding); } diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs index 6693d5d5452..b4c0104e7a5 100644 --- a/moli-encoding/src/tests.rs +++ b/moli-encoding/src/tests.rs @@ -24,6 +24,329 @@ fn unexpected_legacy_encoding_detector( panic!("declared encoding must take precedence over heuristic detection") } +#[test] +fn xml_document_ignores_html_meta_across_chunk_splits() { + for declaration in ["", ""] { + for meta in [ + "", + "", + ] { + let source = format!("{declaration}{meta}café"); + assert_xml_document_decodes_across_chunk_splits( + &[], + source.as_bytes(), + &source, + "UTF-8", + ); + } + } +} + +fn assert_xml_document_decodes_across_chunk_splits( + headers: &[(String, Vec)], + bytes: &[u8], + source: &str, + encoding: &str, +) { + for split in 0..=bytes.len() { + let mut decoder = HtmlDocumentStreamingDecoder::new_xml_document(headers); + let mut decoded = decoder.push(&bytes[..split]).concat(); + decoded.push_str(&decoder.push(&bytes[split..]).concat()); + decoded.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(decoded, source, "{encoding}: split {split}"); + assert_eq!(decoder.document_encoding_name(), encoding, "split {split}"); + } + let mut decoder = HtmlDocumentStreamingDecoder::new_xml_document(headers); + let mut decoded = String::new(); + for byte in bytes { + decoded.push_str(&decoder.push(std::slice::from_ref(byte)).concat()); + } + decoded.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(decoded, source, "{encoding}: single-byte chunks"); + assert_eq!(decoder.document_encoding_name(), encoding); +} + +#[test] +fn xml_document_preserves_encoding_precedence_across_chunk_splits() { + let source = "\ + café"; + let legacy = encoding_rs::WINDOWS_1252.encode(source).0.into_owned(); + assert_xml_document_decodes_across_chunk_splits(&[], &legacy, source, "windows-1252"); + + let headers = [( + "Content-Type".to_owned(), + b"application/xml; charset=utf-8".to_vec(), + )]; + assert_xml_document_decodes_across_chunk_splits(&headers, source.as_bytes(), source, "UTF-8"); + + let headers = [( + "Content-Type".to_owned(), + b"application/xml; charset=windows-1252".to_vec(), + )]; + let bom_body = [b"\xef\xbb\xbf".as_slice(), source.as_bytes()].concat(); + assert_xml_document_decodes_across_chunk_splits(&headers, &bom_body, source, "UTF-8"); + + let source = "\ + café"; + for (bytes, bom, encoding) in [ + (utf16le_bytes(source), [0xff, 0xfe], "UTF-16LE"), + (utf16be_bytes(source), [0xfe, 0xff], "UTF-16BE"), + ] { + assert_xml_document_decodes_across_chunk_splits(&[], &bytes, source, encoding); + let bom_body = [bom.as_slice(), &bytes].concat(); + assert_xml_document_decodes_across_chunk_splits(&headers, &bom_body, source, encoding); + } +} + +#[test] +fn xml_document_ignores_encoding_attributes_in_processing_instructions() { + let source = "\ + café"; + assert_xml_document_decodes_across_chunk_splits(&[], source.as_bytes(), source, "UTF-8"); +} + +#[test] +fn xml_document_selects_encoding_at_the_end_of_its_declaration() { + let mut decoder = HtmlDocumentStreamingDecoder::new_xml_document(&[]); + let mut decoded = decoder + .push(b"").concat()); + assert_eq!(decoder.selected_encoding_name(), Some("windows-1252")); + decoded.push_str(&decoder.push(b"caf\xe9").concat()); + decoded.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!( + decoded, + "café" + ); +} + +#[test] +fn xml_document_finishes_empty_or_incomplete_declarations_with_utf8() { + for source in [ + "", + "<", + "foo"; + for (mut decoder, expected_encoding) in [ + (HtmlDocumentStreamingDecoder::new(&[]), "UTF-8"), + (HtmlDocumentStreamingDecoder::new_xml_document(&[]), "UTF-8"), + ( + HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://example.test/", + test_legacy_encoding_detector, + false, + None, + ), + "windows-1252", + ), + ] { + let mut decoded = String::new(); + for byte in source { + decoded.push_str(&decoder.push(std::slice::from_ref(byte)).concat()); + } + decoded.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(decoded.as_bytes(), source); + assert_eq!(decoder.document_encoding_name(), expected_encoding); + } +} + +// Chromium text_resource_decoder_test.cc, BrokenBOMs (revision above). +#[test] +fn chromium_plain_text_broken_boms_are_flushed_with_the_default_encoding() { + for (bytes, expected) in [ + (b"\xef\xbb".as_slice(), "ï»"), + (b"\xff", "ÿ"), + (b"\xfe", "þ"), + ] { + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://example.test/", + test_legacy_encoding_detector, + false, + None, + ); + for byte in bytes { + assert!(decoder.push(std::slice::from_ref(byte)).is_empty()); + } + assert_eq!(decoder.finish().as_deref(), Some(expected)); + assert_eq!(decoder.document_encoding_name(), "windows-1252"); + } +} + +// Adapt Chromium fast/encoding/bom-in-content{,-utf16}.html to the XML policy. +#[test] +fn chromium_bom_in_content_is_preserved_by_xml_decoding() { + let source = "\u{feff}"; + for (bytes, bom, encoding) in [ + ( + source.as_bytes().to_vec(), + b"\xef\xbb\xbf".as_slice(), + "UTF-8", + ), + (utf16le_bytes(source), b"\xff\xfe", "UTF-16LE"), + (utf16be_bytes(source), b"\xfe\xff", "UTF-16BE"), + ] { + let bytes = [bom, &bytes].concat(); + assert_xml_document_decodes_across_chunk_splits(&[], &bytes, source, encoding); + } +} + +#[test] +fn json_text_document_defaults_to_utf8_across_every_chunk_split() { + let input = "{\"name\":\"Gülçek\"}"; + for split in 0..=input.len() { + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://example.test/", + unexpected_legacy_encoding_detector, + true, + Some("GBK"), + ); + let mut output = decoder.push(&input.as_bytes()[..split]).join(""); + output.push_str(&decoder.push(&input.as_bytes()[split..]).join("")); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, input, "split {split}"); + assert_eq!(decoder.selected_encoding_name(), Some("UTF-8")); + } +} + +#[test] +fn plain_text_document_honors_transport_charset() { + let headers = [( + "Content-Type".to_owned(), + b"text/plain; charset=gbk".to_vec(), + )]; + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &headers, + "https://example.test/", + unexpected_legacy_encoding_detector, + false, + Some("UTF-8"), + ); + let mut output = decoder.push(&gbk_bytes("家居")).join(""); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, "家居"); + assert_eq!(decoder.selected_encoding_name(), Some("GBK")); +} + +#[test] +fn text_document_bom_precedes_transport_charset() { + let headers = [( + "Content-Type".to_owned(), + b"text/plain; charset=gbk".to_vec(), + )]; + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &headers, + "https://example.test/", + unexpected_legacy_encoding_detector, + false, + Some("windows-1251"), + ); + assert_eq!(decoder.document_encoding_name(), "GBK"); + assert_eq!(decoder.selected_encoding_name(), None); + assert!(decoder.push(&[0xef]).is_empty()); + assert!(decoder.push(&[0xbb]).is_empty()); + let mut tail = vec![0xbf]; + tail.extend_from_slice("Gülçek".as_bytes()); + let mut output = decoder.push(&tail).join(""); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, "Gülçek"); + assert_eq!(decoder.selected_encoding_name(), Some("UTF-8")); + assert_eq!(decoder.document_encoding_name(), "UTF-8"); +} + +#[test] +fn plain_text_document_retains_legacy_detection_without_a_charset() { + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://legacy.example/", + test_legacy_encoding_detector, + false, + None, + ); + let mut output = decoder.push(&[0x80]).join(""); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, "А"); + assert_eq!(decoder.selected_encoding_name(), Some("IBM866")); +} + +#[test] +fn plain_text_document_inherits_encoding_before_detection_across_chunk_splits() { + for (input, encoding) in [ + ("plain ASCII", encoding_rs::UTF_8), + ("吴姓-姓氏渊源", encoding_rs::UTF_8), + ("家居", encoding_rs::GBK), + ] { + let bytes = encoding.encode(input).0; + for split in 0..=bytes.len() { + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://example.test/", + unexpected_legacy_encoding_detector, + false, + Some(encoding.name()), + ); + assert_eq!(decoder.document_encoding_name(), encoding.name()); + let mut output = decoder.push(&bytes[..split]).concat(); + output.push_str(&decoder.push(&bytes[split..]).concat()); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, input, "{} split {split}", encoding.name()); + assert_eq!(decoder.selected_encoding_name(), Some(encoding.name())); + } + } +} + +#[test] +fn invalid_inherited_text_encoding_does_not_disable_detection() { + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://legacy.example/", + test_legacy_encoding_detector, + false, + Some("not-an-encoding"), + ); + let mut output = decoder.push(&[0x80]).concat(); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, "А"); + assert_eq!(decoder.selected_encoding_name(), Some("IBM866")); +} + +#[test] +fn plain_text_document_ignores_literal_encoding_declarations() { + for prefix in [ + "", + "", + ] { + let mut input = prefix.as_bytes().to_vec(); + input.push(0xe9); + let mut decoder = HtmlDocumentStreamingDecoder::new_text_document( + &[], + "https://example.test/", + test_legacy_encoding_detector, + false, + None, + ); + let mut output = decoder.push(&input).join(""); + output.push_str(&decoder.finish().unwrap_or_default()); + assert_eq!(output, format!("{prefix}é")); + assert_eq!(decoder.selected_encoding_name(), Some("windows-1252")); + } +} + #[test] fn content_type_charset_is_selected() { let headers: Vec<(String, Vec)> = vec![( diff --git a/moli-parser/src/html.rs b/moli-parser/src/html.rs index 11c8a72f507..532cdddf6fa 100644 --- a/moli-parser/src/html.rs +++ b/moli-parser/src/html.rs @@ -27,7 +27,7 @@ use super::live_target::{ ParserDomMutationConsumer, ParserDomReadConsumer, ParserElementCreationConsumer, ParserMutationEffectConsumer, ParserMutationEffectDelivery, ParserRuntimeDomSinks, ParserStreamHtmlTreeSinkTarget, new_live_document_root_html_tree_sink_stream, - new_parser_stream_html_tree_sink_stream, + new_parser_stream_html_tree_sink_stream, new_parser_stream_html_tree_sink_target, }; use super::{ ParserSourcePosition, html_chunks, @@ -438,6 +438,47 @@ impl HtmlParser { DocumentStream::new_parser_stream(final_url, self.scripting_enabled) } + /// Start a text document using the HTML tokenizer's plaintext state. + /// Subsequent chunks remain literal text inside the browser-owned `pre`. + pub fn start_text_document(&self, final_url: Url, content_type: &str) -> DocumentStream { + let stream = + DocumentStream::new_text_document(new_parser_stream_html_tree_sink_target(final_url)); + stream.inner.initialize_text_document(); + let mut host = stream.inner.take_parser_stream_dom_host(); + let document_handle = host.document_handle(); + host.set_document_content_type_for_handle(document_handle, content_type); + stream.inner.restore_parser_stream_dom_host(host); + stream + } + + /// Initialize the same literal-text shell in an existing document. The + /// embedder owns that document's MIME metadata and live DOM callbacks. + pub fn start_live_text_document_root( + &self, + final_url: Url, + document_handle: NativeNodeId, + consumer: &mut T, + ) -> DocumentStream + where + T: ParserDomReadConsumer + + ParserDomMutationConsumer + + ParserMutationEffectConsumer + + ParserElementCreationConsumer, + { + let stream = DocumentStream::new_text_document( + ParserStreamHtmlTreeSinkTarget::new_live_document_root(final_url, document_handle), + ); + // SAFETY: the consumer is exclusively borrowed until the step guard + // removes the erased callbacks, including when initialization unwinds. + let sinks = unsafe { ParserRuntimeDomSinks::from_consumer(consumer) }; + stream.inner.enter_runtime_dom_sinks_parse_step(sinks); + { + let step = RuntimeDomSinksParserStep { stream: &stream }; + step.stream.inner.initialize_text_document(); + } + stream + } + pub fn start_live_document_root( &self, final_url: Url, @@ -551,6 +592,17 @@ impl Default for ParserInputQueue { } impl DocumentStream { + fn new_text_document(target: ParserStreamHtmlTreeSinkTarget) -> Self { + let mut options = html_parse_opts_with_scripting(false); + // The byte decoder has already consumed the transport BOM. In plaintext + // every remaining U+FEFF is content, including at a chunk boundary. + options.tokenizer.discard_bom = false; + Self { + inner: HtmlTreeSinkStream::from_target_with_options(target, options), + input: RefCell::default(), + } + } + fn new_parser_stream(final_url: Url, scripting_enabled: bool) -> Self { Self { inner: new_parser_stream_html_tree_sink_stream(final_url, scripting_enabled), @@ -1843,6 +1895,83 @@ mod tests { const HTML_NS: &str = "http://www.w3.org/1999/xhtml"; const MATHML_NS: &str = "http://www.w3.org/1998/Math/MathML"; + #[test] + fn text_document_stream_preserves_literal_markup_and_leading_newline() { + let payload = + "\u{feff}\nGülçek&\u{feff}"; + for content_type in ["text/plain", "application/json", "application/problem+json"] { + let stream = HtmlParser::SCRIPTING_ENABLED.start_text_document( + Url::parse("https://example.test/data").unwrap(), + content_type, + ); + for character in payload.chars() { + stream.feed(&character.to_string()); + } + let document = stream.finish(); + let pre = first_element_by_ns(&document, HTML_NS, "pre"); + assert_eq!(document.text_content(pre).as_deref(), Some(payload)); + assert_eq!(document.document().unwrap().content_type(), content_type); + for tag in ["b", "script"] { + assert!( + document + .elements_by_tag_name_ns( + document.document_node_id(), + Some(HTML_NS), + tag, + true + ) + .is_empty() + ); + } + } + } + + #[test] + fn text_document_shell_has_no_doctype_and_preserves_normalized_literal_text() { + for mime in [ + "text/plain", + "application/json", + "application/problem+json", + "text/javascript", + ] { + let stream = HtmlParser::SCRIPTING_ENABLED + .start_text_document(Url::parse("https://example.test/data").unwrap(), mime); + for chunk in [ + "\n<&", + "\r", + "\nbeta\rgamma\0", + "", + ] { + stream.feed(chunk); + } + let document = stream.finish_dom_host(); + let root = document.document_handle(); + let children = document.child_handles(root).collect::>(); + assert_eq!(children.len(), 1, "no synthetic doctype: {mime}"); + assert!(document.is_html_element_named(children[0], "html")); + let expected_mode = HtmlParser::SCRIPTING_ENABLED.parse_dom_host( + Url::parse("https://example.test/control").unwrap(), + "

control".to_owned(), + ); + assert_eq!( + document.document_quirks_mode_for_handle(root), + expected_mode.document_quirks_mode_for_handle(expected_mode.document_handle()) + ); + assert_eq!( + document.text_content(root).as_deref(), + Some("\n<&\nbeta\ngamma\u{fffd}") + ); + } + } + + #[test] + fn html_document_still_parses_markup_and_entities() { + let document = parse_test_document("Gülçek&"); + let bold = first_element_by_ns(&document, HTML_NS, "b"); + assert_eq!(document.text_content(bold).as_deref(), Some("Gülçek&")); + assert_eq!(document.document().unwrap().content_type(), "text/html"); + } + fn parse_test_document(html: &str) -> NativeDom { HtmlParser::SCRIPTING_ENABLED.parse( Url::parse("https://example.test/").expect("test url"), diff --git a/moli-parser/src/session.rs b/moli-parser/src/session.rs index 9fa20921c37..b17c6097bb5 100644 --- a/moli-parser/src/session.rs +++ b/moli-parser/src/session.rs @@ -203,6 +203,19 @@ impl TokenSink for EmbedderPausingTreeBuilder { } impl HtmlParserSession { + pub(super) fn initialize_text_document(&self) { + // Parse only the browser-owned shell. Response bytes enter the tokenizer + // after it has switched to plaintext, so tags and entities stay literal. + self.process(StrTendril::from(concat!( + "", + "

\n"
+        )));
+        // Text documents have no doctype but always use no-quirks mode.
+        self.sink()
+            .set_quirks_mode(html5ever::tree_builder::QuirksMode::NoQuirks);
+        self.tokenizer.set_plaintext_state();
+    }
+
     fn new(sink: DocumentSink, opts: ParseOpts) -> Self {
         let tree_builder = EmbedderPausingTreeBuilder::new(sink, opts.tree_builder);
         Self {
@@ -389,10 +402,10 @@ fn feed_with_definitive_encoding(
 
 pub(super) fn new_html_tree_sink_session(
     target: ParserStreamHtmlTreeSinkTarget,
-    scripting_enabled: bool,
+    options: ParseOpts,
 ) -> HtmlTreeSinkSession {
     let sink = DocumentSink::new(target);
-    let parser = HtmlParserSession::new(sink, html_parse_opts_with_scripting(scripting_enabled));
+    let parser = HtmlParserSession::new(sink, options);
     let script_input = ParserInputQueue::default();
 
     HtmlTreeSinkSession {
diff --git a/moli-parser/src/stream.rs b/moli-parser/src/stream.rs
index 6faf1e1fee8..1b19b2dbb45 100644
--- a/moli-parser/src/stream.rs
+++ b/moli-parser/src/stream.rs
@@ -429,11 +429,25 @@ pub(crate) fn prepare_parser_script_handoff_for_static_document(
 }
 
 impl HtmlTreeSinkStream {
+    pub(super) fn initialize_text_document(&self) {
+        self.parser.initialize_text_document();
+    }
+
     pub(super) fn from_target_with_scripting(
         target: ParserStreamHtmlTreeSinkTarget,
         scripting_enabled: bool,
     ) -> Self {
-        let session = new_html_tree_sink_session(target, scripting_enabled);
+        Self::from_target_with_options(
+            target,
+            super::session::html_parse_opts_with_scripting(scripting_enabled),
+        )
+    }
+
+    pub(super) fn from_target_with_options(
+        target: ParserStreamHtmlTreeSinkTarget,
+        options: html5ever::ParseOpts,
+    ) -> Self {
+        let session = new_html_tree_sink_session(target, options);
         Self {
             parser: session.parser,
             script_input: session.script_input,
diff --git a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
index 9f891a382bb..3b712d0e0ee 100644
--- a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
+++ b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
@@ -1,5 +1,564 @@
 use super::*;
 
+#[tokio::test]
+async fn webdriver_classic_text_children_preserve_parent_encoding_without_a_charset() {
+    const LITERAL: &str = "吴姓-姓氏渊源";
+    let cases = [
+        ("ascii", "plain ASCII"),
+        ("utf8", "吴姓-姓氏渊源"),
+        ("literal", LITERAL),
+        ("data", LITERAL),
+    ];
+    let mut fixture = axum::Router::new();
+    for (name, payload) in cases {
+        fixture = fixture.route(
+            &format!("/{name}"),
+            axum::routing::get(
+                move || async move { ([(header::CONTENT_TYPE, "text/plain")], payload) },
+            ),
+        );
+    }
+    fixture = fixture.route(
+        "/parent/{kind}",
+        axum::routing::get(
+            |axum::extract::Path(kind): axum::extract::Path| async move {
+                let target = if kind == "data" {
+                    format!(
+                        "data:text/plain;base64,{}",
+                        base64::Engine::encode(&BASE64_STANDARD, LITERAL)
+                    )
+                } else {
+                    format!("/{kind}")
+                };
+                (
+                    [(header::CONTENT_TYPE, "text/html; charset=utf-8")],
+                    format!(""),
+                )
+            },
+        ),
+    );
+    let (addr, _server) = spawn_dedicated_fixture_server(fixture, "inherited-text-encoding");
+    let app = build_router(test_state());
+    let session = classic_request_json(app.clone(), Method::POST, "/session").await;
+    let session_id = session["value"]["sessionId"].as_str().unwrap();
+    for (name, payload) in cases {
+        classic_request_json_with_body(
+            app.clone(),
+            Method::POST,
+            &format!("/session/{session_id}/url"),
+            json!({"url":format!("http://{addr}/parent/{name}")}),
+        )
+        .await;
+        let parent = classic_request_json_with_body(
+            app.clone(),
+            Method::POST,
+            &format!("/session/{session_id}/execute/sync"),
+            json!({"script": "return document.characterSet;", "args": []}),
+        )
+        .await;
+        assert_eq!(parent["value"], json!("UTF-8"), "{name}");
+        classic_request_json_with_body(
+            app.clone(),
+            Method::POST,
+            &format!("/session/{session_id}/frame"),
+            json!({"id": 0}),
+        )
+        .await;
+        let observed = classic_request_json_with_body(
+            app.clone(),
+            Method::POST,
+            &format!("/session/{session_id}/execute/sync"),
+            json!({
+                "script": "return [document.body.textContent,document.characterSet,document.contentType];",
+                "args": [],
+            }),
+        ).await;
+        assert_eq!(
+            observed["value"],
+            json!([payload, "UTF-8", "text/plain"]),
+            "{name}"
+        );
+        classic_request_json_with_body(
+            app.clone(),
+            Method::POST,
+            &format!("/session/{session_id}/frame"),
+            json!({"id": null}),
+        )
+        .await;
+    }
+    classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;
+}
+
+#[tokio::test]
+async fn webdriver_classic_xml_suffix_navigation_preserves_namespaces_and_dom_parser_types() {
+    let cases = [
+        ("atom.xml", "application/atom+xml"),
+        ("rss.xml", "application/rss+xml"),
+        ("custom.xml", "text/example+xml"),
+    ];
+    let mut fixture = axum::Router::new();
+    for (name, mime) in cases {
+        fixture = fixture.route(
+            &format!("/{name}"),
+            axum::routing::get(move || async move {
+                (
+                    [(header::CONTENT_TYPE, mime)],
+                    "吴姓",
+                )
+            }),
+        );
+    }
+    fixture = fixture.route(
+        "/parent/{kind}",
+        axum::routing::get(
+            |axum::extract::Path(kind): axum::extract::Path| async move {
+                (
+                    [(header::CONTENT_TYPE, "text/html; charset=windows-1252")],
+                    format!(""),
+                )
+            },
+        ),
+    );
+    let (addr, _server) = spawn_dedicated_fixture_server(fixture, "xml-suffix-navigation");
+    let app = build_router(test_state());
+    let session = classic_request_json(app.clone(), Method::POST, "/session").await;
+    let session_id = session["value"]["sessionId"].as_str().unwrap();
+    for (name, mime) in cases {
+        for prefix in ["", "parent/"] {
+            classic_request_json_with_body(
+                app.clone(),
+                Method::POST,
+                &format!("/session/{session_id}/url"),
+                json!({"url":format!("http://{addr}/{prefix}{name}")}),
+            )
+            .await;
+            let observed = classic_request_json_with_body(
+                app.clone(),
+                Method::POST,
+                &format!("/session/{session_id}/execute/sync"),
+                json!({
+                    "script": "const d=document.querySelector('iframe')?.contentDocument ?? document; let rejected=false; try { new DOMParser().parseFromString('',d.contentType); } catch(error) { rejected=error instanceof TypeError; } const root=d.getElementsByTagNameNS('http://www.w3.org/2005/Atom','feed')[0]; return [root.localName,root.namespaceURI,root.textContent,d.contentType,d.characterSet,rejected];",
+                    "args": [],
+                }),
+            ).await;
+            assert_eq!(
+                observed["value"],
+                json!([
+                    "feed",
+                    "http://www.w3.org/2005/Atom",
+                    "吴姓",
+                    mime,
+                    "UTF-8",
+                    true
+                ]),
+                "{prefix}{name}"
+            );
+        }
+    }
+    classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;
+}
+
+#[tokio::test]
+async fn webdriver_classic_xml_main_and_child_documents_decode_response_bytes() {
+    const SOURCE: &str = "\
+                         café";
+    let legacy_body = b"\
+                        caf\xe9";
+    let cases = [
+        ("meta", "", SOURCE.as_bytes().to_vec(), "UTF-8"),
+        (
+            "declared-utf8-meta",
+            "",
+            [
+                b"".as_slice(),
+                SOURCE.as_bytes(),
+            ]
+            .concat(),
+            "UTF-8",
+        ),
+        (
+            "charset",
+            "; charset=windows-1252",
+            legacy_body.to_vec(),
+            "windows-1252",
+        ),
+        (
+            "declaration",
+            "",
+            [
+                b"".as_slice(),
+                legacy_body,
+            ]
+            .concat(),
+            "windows-1252",
+        ),
+        (
+            "utf16le",
+            "; charset=windows-1252",
+            [
+                vec![0xff, 0xfe],
+                SOURCE.encode_utf16().flat_map(u16::to_le_bytes).collect(),
+            ]
+            .concat(),
+            "UTF-16LE",
+        ),
+        (
+            "utf16be",
+            "",
+            [
+                vec![0xfe, 0xff],
+                SOURCE.encode_utf16().flat_map(u16::to_be_bytes).collect(),
+            ]
+            .concat(),
+            "UTF-16BE",
+        ),
+    ];
+    let mimes = [
+        "application/atom+xml",
+        "application/rss+xml",
+        "text/example+xml",
+    ];
+    let mut fixture = axum::Router::new();
+    for (index, mime) in mimes.iter().enumerate() {
+        for (name, charset, body, _) in &cases {
+            let content_type = format!("{mime}{charset}");
+            let body = body.clone();
+            fixture = fixture.route(
+                &format!("/{index}-{name}"),
+                axum::routing::get(move || {
+                    let content_type = content_type.clone();
+                    let body = body.clone();
+                    async move { ([(header::CONTENT_TYPE, content_type)], body) }
+                }),
+            );
+        }
+    }
+    fixture = fixture.route(
+        "/parent/{kind}",
+        axum::routing::get(
+            |axum::extract::Path(kind): axum::extract::Path| async move {
+                (
+                    [(header::CONTENT_TYPE, "text/html; charset=utf-8")],
+                    format!(""),
+                )
+            },
+        ),
+    );
+    let (addr, _server) = spawn_dedicated_fixture_server(fixture, "xml-response-encoding");
+    let app = build_router(test_state());
+    let session = classic_request_json(app.clone(), Method::POST, "/session").await;
+    let session_id = session["value"]["sessionId"].as_str().unwrap();
+    for (index, mime) in mimes.iter().enumerate() {
+        for (name, _, _, encoding) in &cases {
+            for prefix in ["", "parent/"] {
+                let path = format!("{prefix}{index}-{name}");
+                let navigated = classic_request_json_with_body(
+                    app.clone(),
+                    Method::POST,
+                    &format!("/session/{session_id}/url"),
+                    json!({"url":format!("http://{addr}/{path}")}),
+                )
+                .await;
+                assert_eq!(navigated, json!({"value":null}), "{path}");
+                let observed = classic_request_json_with_body(
+                    app.clone(),
+                    Method::POST,
+                    &format!("/session/{session_id}/execute/sync"),
+                    json!({
+                        "script": "const d=document.querySelector('iframe')?.contentDocument ?? document; const root=d.getElementsByTagNameNS('http://www.w3.org/2005/Atom','feed')[0]; return [root.localName,root.namespaceURI,root.textContent,d.contentType,d.characterSet];",
+                        "args": [],
+                    }),
+                ).await;
+                assert_eq!(
+                    observed["value"],
+                    json!([
+                        "feed",
+                        "http://www.w3.org/2005/Atom",
+                        "café",
+                        mime,
+                        encoding
+                    ]),
+                    "{path}"
+                );
+            }
+        }
+    }
+    classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;
+}
+
+#[tokio::test]
+async fn webdriver_classic_chromium_xml_encoding_scenarios() {
+    // Adapted from Chromium a03603fe9af6230a12f1b2fb2c18a7d003a0d937,
+    // third_party/blink/web_tests/fast/encoding/{meta-in-xhtml.xhtml,
+    // default-xhtml-encoding.xhtml, xml-utf-8-default.xml,
+    // external-script-charset.xhtml, dumpAsText/utf-16-no-bom.xml} and
+    // http/tests/xmlviewer/dumpAsText/xmlviewer-charset-{cp1251,utf8}.xml.
+    const XHTML_NAMESPACE: &str = "http://www.w3.org/1999/xhtml";
+    let report_charset = "

"; + let unicode = "—∆∏π𝓐𝖚"; + let cases = [ + ( + "meta-in-xhtml", + "application/xhtml+xml", + format!( + "\ + \ + {report_charset}" + ).into_bytes(), + "UTF-8", + "UTF-8", + Some(XHTML_NAMESPACE), + ), + ( + "default-xhtml-encoding", + "application/xhtml+xml", + format!("{report_charset}").into_bytes(), + "UTF-8", + "UTF-8", + Some(XHTML_NAMESPACE), + ), + ( + "xml-utf-8-default", + "application/xml", + format!( + "\ +

{unicode}

" + ).into_bytes(), + "UTF-8", + unicode, + Some(XHTML_NAMESPACE), + ), + ( + "external-script-charset", + "application/xhtml+xml", + format!( + "\ +

" + ).into_bytes(), + "KOI8-R", + "PASS", + Some(XHTML_NAMESPACE), + ), + ( + "xmlviewer-charset-cp1251", + "application/xml", + b"\ + SU\xd1\xd1\xc5SS".to_vec(), + "windows-1251", + "SUССЕSS", + None, + ), + ( + "xmlviewer-charset-utf8", + "application/xml", + "\ + SUССЕSS".as_bytes().to_vec(), + "UTF-8", + "SUССЕSS", + None, + ), + ( + "utf-16-no-bom", + "application/xml", + "ё" + .encode_utf16().flat_map(u16::to_be_bytes).collect(), + "UTF-16BE", + "ё", + None, + ), + ]; + let mut fixture = axum::Router::new(); + for (name, mime, body, _, _, _) in &cases { + let mime = *mime; + let body = body.clone(); + fixture = fixture.route( + &format!("/{name}"), + axum::routing::get(move || { + let body = body.clone(); + async move { ([(header::CONTENT_TYPE, mime)], body) } + }), + ); + } + fixture = fixture + .route( + "/external-script-charset-koi-8.js", + axum::routing::get(|| async { + ( + [(header::CONTENT_TYPE, "text/javascript")], + b"document.getElementById('result').textContent = ('\xf3' == '\\u0421') ? 'PASS' : 'FAIL';".as_slice(), + ) + }), + ) + .route( + "/parent/{kind}", + axum::routing::get( + |axum::extract::Path(kind): axum::extract::Path| async move { + ( + [(header::CONTENT_TYPE, "text/html; charset=windows-1252")], + format!(""), + ) + }, + ), + ); + let (addr, _server) = spawn_dedicated_fixture_server(fixture, "chromium-xml-encoding"); + let app = build_router(test_state()); + let session = classic_request_json(app.clone(), Method::POST, "/session").await; + let session_id = session["value"]["sessionId"].as_str().unwrap(); + for (name, mime, _, encoding, expected, namespace) in cases { + for prefix in ["", "parent/"] { + let navigated = classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/url"), + json!({"url":format!("http://{addr}/{prefix}{name}")}), + ) + .await; + assert_eq!(navigated, json!({"value":null}), "{prefix}{name}"); + let observed = classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/execute/sync"), + json!({ + "script": "const d=document.querySelector('iframe')?.contentDocument ?? document; const result=d.getElementById('result'); return [result.textContent,d.characterSet,d.contentType,result.namespaceURI];", + "args": [], + }), + ).await; + assert_eq!( + observed["value"], + json!([expected, encoding, mime, namespace]), + "{prefix}{name}" + ); + } + } + classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await; +} + +#[tokio::test] +async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() { + const PAYLOAD: &str = + "literal&Gülçek"; + let cases = [ + ( + "plain", + "text/plain; charset=utf-8", + "text/plain", + true, + PAYLOAD, + ), + ( + "plain.xml", + "text/plain; charset=utf-8", + "text/plain", + true, + "\n<&\r\nbeta\rgamma\0

", + ), + ( + "json.xml", + "application/problem+json", + "application/problem+json", + true, + PAYLOAD, + ), + ( + "plain-bom", + "text/plain; charset=utf-8", + "text/plain", + true, + "\u{feff}\u{feff}", + ), + ( + "json", + "application/json", + "application/json", + true, + PAYLOAD, + ), + ( + "javascript", + "text/javascript; charset=utf-8", + "text/javascript", + true, + PAYLOAD, + ), + ( + "html", + "text/html; charset=utf-8", + "text/html", + false, + PAYLOAD, + ), + ]; + let mut fixture = axum::Router::new(); + for (name, mime, _, _, payload) in cases { + fixture = fixture.route( + &format!("/{name}"), + axum::routing::get(move || async move { ([(header::CONTENT_TYPE, mime)], payload) }), + ); + } + fixture = fixture.route( + "/frame/{kind}", + axum::routing::get( + |axum::extract::Path(kind): axum::extract::Path| async move { + ( + [(header::CONTENT_TYPE, "text/html")], + format!(""), + ) + }, + ), + ); + let (addr, _server) = spawn_dedicated_fixture_server(fixture, "document-mime"); + let app = build_router(test_state()); + let session = classic_request_json(app.clone(), Method::POST, "/session").await; + let session_id = session["value"]["sessionId"].as_str().unwrap(); + for (name, _, content_type, literal, payload) in cases { + for prefix in ["", "frame/"] { + let navigated = classic_request_json_with_body( + app.clone(), + Method::POST, + &format!("/session/{session_id}/url"), + json!({"url":format!("http://{addr}/{prefix}{name}")}), + ) + .await; + assert_eq!(navigated, json!({"value":null}), "{prefix}{name}"); + let observed = classic_request_json_with_body( + app.clone(), Method::POST, &format!("/session/{session_id}/execute/sync"), + json!({"script": "const w=document.querySelector('iframe')?.contentWindow ?? window; return [w.document.body.textContent,w.executed??null,w.document.querySelectorAll('script').length,w.document.contentType];", "args": []}), + ).await; + let expected = if literal { + json!([ + payload + .strip_prefix('\u{feff}') + .unwrap_or(payload) + .replace("\r\n", "\n") + .replace('\r', "\n") + .replace('\0', "\u{fffd}"), + null, + 0, + content_type + ]) + } else { + json!(["literal&Gülçek", 42, 1, content_type]) + }; + assert_eq!(observed["value"], expected, "{prefix}{name}"); + if literal { + let structure = classic_request_json_with_body( + app.clone(), Method::POST, &format!("/session/{session_id}/execute/sync"), + json!({"script": "const d=(document.querySelector('iframe')?.contentWindow ?? window).document; return [d.doctype===null,d.compatMode,d.body.children.length,d.body.firstElementChild.localName,d.querySelectorAll('pre').length];", "args": []}), + ).await; + assert_eq!( + structure["value"], + json!([true, "CSS1Compat", 1, "pre", 1]), + "{prefix}{name}" + ); + } + } + } + classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await; +} + #[tokio::test] async fn webdriver_classic_page_load_strategy_maps_navigation_wait_policy() { let app = build_router(test_state()); diff --git a/moli-protocol/src/conn/runtime_load.rs b/moli-protocol/src/conn/runtime_load.rs index 0f9674b2531..6b5109e783e 100644 --- a/moli-protocol/src/conn/runtime_load.rs +++ b/moli-protocol/src/conn/runtime_load.rs @@ -156,11 +156,6 @@ async fn prepare_network_error_page_navigation_with_engine_async( .map(NavigationLoadOutcome::response_commit_ready) } -fn response_headers_indicate_xml_document(headers: &[(String, Vec)]) -> bool { - moli_web_mime::response_document_content_type(headers) - .is_some_and(|mime| moli_web_mime::is_dom_parser_xml_mime(&mime)) -} - #[derive(Clone, Debug)] pub(crate) struct InitialDocumentPageOwner { pub(crate) browser_context_id: String, @@ -306,25 +301,17 @@ pub struct ResponseCommitReady { network_error_page: Option, } -enum ResponseCommitBodyCapture { - Pending(tokio::task::JoinHandle>), - Ready(CapturedBody), -} +struct ResponseCommitBodyCapture(tokio::task::JoinHandle>); impl ResponseCommitBodyCapture { fn abort(self) { - if let Self::Pending(task) = self { - task.abort(); - } + self.0.abort(); } async fn resolve(self) -> anyhow::Result { - match self { - Self::Pending(task) => task - .await - .context("main document body capture task failed")?, - Self::Ready(body) => Ok(body), - } + self.0 + .await + .context("main document body capture task failed")? } } @@ -1283,7 +1270,7 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async( let body_network_progress_state = body_progress_source.body_network_progress_for_completed_events(network_events); let body_progress_source_for_body_finish = body_progress_source.clone(); - let mut initial_body_chunk = if response_status_may_use_http_error_page(response_status) { + let initial_body_chunk = if response_status_may_use_http_error_page(response_status) { match first_nonempty_response_body_chunk(&mut response).await? { Some(chunk) => Some(chunk), None => { @@ -1308,97 +1295,8 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async( None }; - if response_headers_indicate_xml_document(&response_headers) { - let redirected = response.redirected; - let mut body_writer = CapturedBodyWriter::default(); - if let Some(chunk) = initial_body_chunk.take() { - body_writer - .append(&chunk) - .context("failed to capture XML page body")?; - } - while let Some(chunk) = response.next_chunk().await { - body_writer - .append(&chunk) - .context("failed to capture XML page body")?; - } - response - .finish() - .await - .context("failed to read XML page body from stream")?; - let captured_body = body_writer - .finish() - .context("failed to finish captured XML page body")?; - let response_text = captured_body - .materialize_bytes() - .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) - .context("failed to materialize XML page body")?; - let page_storage = load_inputs.page_storage_handles(); - let main_document_commit = load_inputs - .main_document_commit_for_final_url(&final_url, None) - .map(Arc::new); - let prepared_page = engine - .prepare_document_page_from_response_with_storage_and_inspector_session_restores_async( - page_reservation, - load_inputs.document_replacement.clone(), - page_storage.into_navigation_storage(), - requested_url.clone(), - final_url.clone(), - load_inputs.navigation_initiator_url.clone(), - redirected, - redirect_chain.len(), - response_status, - response_headers.clone(), - response_text, - load_inputs.document_start_scripts.clone(), - load_inputs.runtime_bindings.clone(), - load_inputs - .runtime_inspector_session_restore_snapshots - .clone(), - load_inputs.extra_http_headers.clone(), - load_inputs.script_execution_disabled, - load_inputs.bypass_content_security_policy, - load_inputs.emulated_media.clone(), - load_inputs.viewport_surface, - load_inputs.network_offline, - load_inputs.blocked_url_patterns.clone(), - fetch_subresource_interception_enabled, - fetch_subresource_interception_resource_type, - load_inputs.root_frame_id.clone(), - resource_source, - main_document_commit.as_deref().cloned(), - ) - .await - .with_context(|| format!("failed to prepare XML page `{}`", requested_url))?; - if timing_enabled { - tracing::info!( - target: "moli_cdp_nav_timing", - url = %requested_url, - stage = "response_commit_ready", - elapsed_ms = timing_started.elapsed().as_millis(), - ); - } - return Ok(NavigationLoadOutcome::response_commit_ready( - ResponseCommitReady { - prepared_page: Some(prepared_page), - body_capture: Some(ResponseCommitBodyCapture::Ready(captured_body)), - body_completion_sink, - body_progress_source: body_progress_source_for_body_finish, - body_network_progress_state: Some(body_network_progress_state), - synthetic_body: false, - requested_url, - final_url, - request_method, - request_headers, - response_status, - response_headers, - response_from_cache, - timing_started: timing_enabled.then_some(timing_started), - main_document_commit, - network_error_page: None, - }, - )); - } - + // Preserve response bytes until the renderer selects the document encoding, + // including XML charsets and BOMs. let (body_tx, body_rx) = mpsc::channel(EXTERNAL_RAW_BODY_CHANNEL_CAPACITY); let (completion_tx, completion_rx) = oneshot::channel(); let raw_body = moli_core::runtime::ExternalRawDocumentBodyStream::new(body_rx, completion_rx); @@ -1467,7 +1365,7 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async( Ok(NavigationLoadOutcome::response_commit_ready( ResponseCommitReady { prepared_page: Some(prepared_page), - body_capture: Some(ResponseCommitBodyCapture::Pending(body_capture_task)), + body_capture: Some(ResponseCommitBodyCapture(body_capture_task)), body_completion_sink, body_progress_source: body_progress_source_for_body_finish, body_network_progress_state: Some(body_network_progress_state), @@ -3804,7 +3702,7 @@ async fn prepare_captured_document_response_with_engine_async( Ok(ResponseCommitReady { prepared_page: Some(prepared_page), - body_capture: Some(ResponseCommitBodyCapture::Pending(body_capture_task)), + body_capture: Some(ResponseCommitBodyCapture(body_capture_task)), body_completion_sink: None, body_progress_source, body_network_progress_state: Some(body_network_progress_state), @@ -3950,7 +3848,7 @@ mod tests { let task = super::spawn_streaming_body_capture(response, None, body_tx, completion_tx); assert_eq!(body_rx.recv().await.unwrap(), b"partial body"); assert!(body_rx.recv().await.is_none()); - let navigation_error = super::ResponseCommitBodyCapture::Pending(task) + let navigation_error = super::ResponseCommitBodyCapture(task) .resolve() .await .expect_err("partial response must fail navigation capture"); @@ -3977,7 +3875,7 @@ mod tests { async fn cancelled_body_capture_retains_join_error() { let task = tokio::spawn(std::future::pending::>()); task.abort(); - let error = super::ResponseCommitBodyCapture::Pending(task) + let error = super::ResponseCommitBodyCapture(task) .resolve() .await .expect_err("cancelled capture must fail"); diff --git a/moli-protocol/src/domains/emulation/tests.rs b/moli-protocol/src/domains/emulation/tests.rs index dd3ad74ca27..2b677b6fe7e 100644 --- a/moli-protocol/src/domains/emulation/tests.rs +++ b/moli-protocol/src/domains/emulation/tests.rs @@ -573,7 +573,9 @@ async fn admitted_idle_override_is_visible_to_concurrent_same_site_navigation() listener, Router::new().route( "/", - get(|| async { "idle navigation" }), + get(|| async { + axum::response::Html("idle navigation") + }), ), ) .await @@ -644,7 +646,9 @@ async fn idle_override_updates_idle_detector_and_clear_restores_actual_state() { listener, Router::new().route( "/", - get(|| async { "idle detector" }), + get(|| async { + axum::response::Html("idle detector") + }), ), ) .await @@ -2219,7 +2223,7 @@ async fn emulation_async_dispatch_updates_live_page_user_agent_and_xhr_header() #[tokio::test(flavor = "multi_thread")] async fn emulation_async_dispatch_updates_live_page_surface_without_mutating_accept_language() { async fn page_handler() -> impl IntoResponse { - "ok" + axum::response::Html("ok") } async fn xhr_handler( @@ -2714,9 +2718,9 @@ async fn locale_override_updates_intl_without_mutating_language_surfaces() { .get(axum::http::header::ACCEPT_LANGUAGE) .and_then(|value| value.to_str().ok()) .unwrap_or(""); - format!( + axum::response::Html(format!( "" - ) + )) } let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -2770,9 +2774,9 @@ async fn bidi_user_context_locale_composes_with_user_agent_on_all_identity_surfa .get(axum::http::header::ACCEPT_LANGUAGE) .and_then(|value| value.to_str().ok()) .unwrap_or(""); - format!( + axum::response::Html(format!( "" - ) + )) } let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/moli-protocol/src/domains/page/tests/lifecycle.rs b/moli-protocol/src/domains/page/tests/lifecycle.rs index 92e5947f8df..afa29aece92 100644 --- a/moli-protocol/src/domains/page/tests/lifecycle.rs +++ b/moli-protocol/src/domains/page/tests/lifecycle.rs @@ -2384,11 +2384,15 @@ async fn network_navigations_use_unique_document_loader_ids() { axum::Router::new() .route( "/first", - axum::routing::get(|| async { "
first
" }), + axum::routing::get(|| async { + axum::response::Html("
first
") + }), ) .route( "/second", - axum::routing::get(|| async { "
second
" }), + axum::routing::get(|| async { + axum::response::Html("
second
") + }), ), ) .await @@ -2467,11 +2471,15 @@ async fn navigations_without_network_domain_still_use_unique_loader_ids() { axum::Router::new() .route( "/first", - axum::routing::get(|| async { "
first
" }), + axum::routing::get(|| async { + axum::response::Html("
first
") + }), ) .route( "/second", - axum::routing::get(|| async { "
second
" }), + axum::routing::get(|| async { + axum::response::Html("
second
") + }), ), ) .await diff --git a/moli-protocol/src/domains/page/tests/runtime.rs b/moli-protocol/src/domains/page/tests/runtime.rs index ab651a20952..1a1e48497d4 100644 --- a/moli-protocol/src/domains/page/tests/runtime.rs +++ b/moli-protocol/src/domains/page/tests/runtime.rs @@ -1269,20 +1269,22 @@ async fn runtime_evaluate_child_history_back_emits_child_frame_navigation_and_li .route( "/main", axum::routing::get(|| async { - "" + axum::response::Html( + "", + ) }), ) .route( "/initial", - axum::routing::get(|| async { "initial" }), + axum::routing::get(|| async { axum::response::Html("initial") }), ) .route( "/history-a", - axum::routing::get(|| async { "history-a" }), + axum::routing::get(|| async { axum::response::Html("history-a") }), ) .route( "/history-b", - axum::routing::get(|| async { "history-b" }), + axum::routing::get(|| async { axum::response::Html("history-b") }), ), ) .await diff --git a/moli-protocol/src/domains/target/tests/tests_background_staging/environment.rs b/moli-protocol/src/domains/target/tests/tests_background_staging/environment.rs index e0670cf8a5f..d5306df9cbc 100644 --- a/moli-protocol/src/domains/target/tests/tests_background_staging/environment.rs +++ b/moli-protocol/src/domains/target/tests/tests_background_staging/environment.rs @@ -284,9 +284,9 @@ async fn same_context_background_session_can_clear_its_own_locale_before_activat .get(axum::http::header::ACCEPT_LANGUAGE) .and_then(|value| value.to_str().ok()) .unwrap_or(""); - format!( + axum::response::Html(format!( "" - ) + )) } let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -451,7 +451,9 @@ async fn same_context_background_session_can_clear_its_own_locale_before_activat #[tokio::test(flavor = "multi_thread")] async fn same_context_background_session_can_clear_its_own_timezone_before_activation() { async fn handler() -> impl IntoResponse { - "" + axum::response::Html( + "", + ) } let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/moli-renderer-v8/src/document_response_decoder.rs b/moli-renderer-v8/src/document_response_decoder.rs new file mode 100644 index 00000000000..4acc309b926 --- /dev/null +++ b/moli-renderer-v8/src/document_response_decoder.rs @@ -0,0 +1,103 @@ +use moli_encoding::HtmlDocumentStreamingDecoder; +use moli_encoding_detector::detect_legacy_html_encoding; +use moli_web_mime::{is_json_document_mime, is_text_document_mime, is_xml_document_mime}; +use url::Url; + +pub(crate) fn new_document_response_decoder( + headers: &[(String, Vec)], + content_type: Option<&str>, + final_url: &Url, + inherited_encoding: Option<&str>, +) -> HtmlDocumentStreamingDecoder { + if let Some(mime) = content_type.filter(|mime| is_text_document_mime(mime)) { + HtmlDocumentStreamingDecoder::new_text_document( + headers, + final_url.as_str(), + detect_legacy_html_encoding, + is_json_document_mime(mime), + inherited_encoding, + ) + } else if content_type.is_some_and(is_xml_document_mime) { + HtmlDocumentStreamingDecoder::new_xml_document(headers) + } else if inherited_encoding.is_some() { + HtmlDocumentStreamingDecoder::new_with_fallback(headers, inherited_encoding) + } else { + HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector( + headers, + final_url.as_str(), + detect_legacy_html_encoding, + ) + } +} + +pub(crate) fn decode_document_response( + bytes: &[u8], + headers: &[(String, Vec)], + content_type: Option<&str>, + final_url: &Url, + inherited_encoding: Option<&str>, +) -> (String, &'static str) { + let mut decoder = + new_document_response_decoder(headers, content_type, final_url, inherited_encoding); + let mut text = decoder.push(bytes).concat(); + if let Some(tail) = decoder.finish() { + text.push_str(&tail); + } + (text, decoder.document_encoding_name()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn xml_responses_ignore_html_meta_encoding_declarations() { + for mime in [ + "application/xml", + "text/xml", + "application/atom+xml", + "text/example+xml", + "application/xhtml+xml", + ] { + let headers = [("Content-Type".to_owned(), mime.as_bytes().to_vec())]; + for declaration in ["", ""] { + let source = format!( + "{declaration}\ + café" + ); + let (decoded, encoding) = decode_document_response( + source.as_bytes(), + &headers, + Some(mime), + &Url::parse("https://example.test/document").unwrap(), + Some("GBK"), + ); + assert_eq!(decoded, source, "{mime}: {declaration}"); + assert_eq!(encoding, "UTF-8", "{mime}: {declaration}"); + } + } + } + + #[test] + fn json_and_xml_responses_keep_utf8_defaults_with_a_legacy_parent() { + for (mime, source) in [ + ("application/json", "{\"text\":\"吴姓\"}"), + ("Text/JSON", "{\"text\":\"吴姓\"}"), + ("application/problem+json", "{\"text\":\"吴姓\"}"), + ("application/atom+xml", "吴姓"), + ("application/rss+xml", "吴姓"), + ("text/example+xml", "吴姓"), + ] { + let headers = [("Content-Type".to_owned(), mime.as_bytes().to_vec())]; + let (decoded, encoding) = decode_document_response( + source.as_bytes(), + &headers, + Some(mime), + &Url::parse("https://example.test/document").unwrap(), + Some("GBK"), + ); + assert_eq!(decoded, source, "{mime}"); + assert_eq!(encoding, "UTF-8", "{mime}"); + } + } +} diff --git a/moli-renderer-v8/src/dom_parser.rs b/moli-renderer-v8/src/dom_parser.rs index cff5ea1cc26..d764b479d09 100644 --- a/moli-renderer-v8/src/dom_parser.rs +++ b/moli-renderer-v8/src/dom_parser.rs @@ -1,6 +1,5 @@ use crate::web_api_interfaces; -use html5ever::tree_builder::QuirksMode; -use moli_web_mime::{is_dom_parser_xml_mime, is_html_document_mime}; +use moli_web_mime::{is_dom_parser_xml_mime, is_html_document_mime, is_xml_document_mime}; use moli_webapi_declare::WebApiFunctionTemplate; use url::Url; @@ -378,7 +377,11 @@ pub(crate) fn preserve_decoded_bom_only_browsing_context_body<'a>( source: &'a str, content_type: Option<&str>, ) -> std::borrow::Cow<'a, str> { - if source == "\u{feff}" && !content_type.is_some_and(is_dom_parser_xml_mime) { + if source == "\u{feff}" + && !content_type.is_some_and(|mime| { + is_xml_document_mime(mime) || moli_web_mime::is_text_document_mime(mime) + }) + { std::borrow::Cow::Borrowed("\u{feff}") } else { std::borrow::Cow::Borrowed(source) @@ -391,8 +394,8 @@ fn parse_browsing_context_document_snapshot( content_type: Option<&str>, html_parser: HtmlParser, ) -> (DomHost, DetachedDocumentKind) { - if content_type.is_some_and(is_dom_parser_xml_mime) - || child_document_url_is_xml_like(&document_url) + if content_type.is_some_and(is_xml_document_mime) + || (content_type.is_none() && child_document_url_is_xml_like(&document_url)) { let parser = XmlParser; return ( @@ -400,13 +403,13 @@ fn parse_browsing_context_document_snapshot( DetachedDocumentKind::Xml, ); } - if content_type.is_some_and(|mime| mime.eq_ignore_ascii_case("text/plain")) { - let mut document = - html_parser.parse_dom_host(document_url, plain_text_document_parser_input(source)); - // Text documents are HTML Documents whose mode is explicitly no-quirks, - // despite having no doctype that would select that mode through parsing. - document.set_html_quirks_mode_for_parser(QuirksMode::NoQuirks); - return (document, DetachedDocumentKind::Html); + if let Some(content_type) = + content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime)) + { + let stream = HtmlParser::with_scripting_enabled(false) + .start_text_document(document_url, content_type); + stream.feed(source); + return (stream.finish_dom_host(), DetachedDocumentKind::Html); } ( html_parser.parse_dom_host(document_url, source.to_owned()), @@ -414,22 +417,6 @@ fn parse_browsing_context_document_snapshot( ) } -pub(crate) fn plain_text_document_parser_input(source: &str) -> String { - let mut input = String::with_capacity(source.len().saturating_add(64)); - // The HTML parser discards the first LF after
; preserve any source LF.
-    input.push_str("
\n");
-    for character in source.chars() {
-        match character {
-            '&' => input.push_str("&"),
-            '<' => input.push_str("<"),
-            '\0' => input.push('\u{fffd}'),
-            _ => input.push(character),
-        }
-    }
-    input.push_str("
"); - input -} - fn child_document_url_is_xml_like(url: &Url) -> bool { let path = url.path().to_ascii_lowercase(); path.ends_with(".xml") || path.ends_with(".xhtml") || path.ends_with(".svg") @@ -598,6 +585,78 @@ mod tests { assert!(disabled.element_handle_by_id("fallback").is_some()); } + #[test] + fn child_projection_preserves_xml_suffix_document_structure_and_namespace() { + let source = "Example"; + for mime in [ + "application/atom+xml", + "application/rss+xml", + "text/example+xml", + ] { + for path in ["feed.xml", "feed.html", "feed"] { + let (document, kind) = parse_browsing_context_document_snapshot( + Url::parse(&format!("https://example.test/{path}")).unwrap(), + source, + Some(mime), + HtmlParser::SCRIPTING_ENABLED, + ); + let root = first_document_element(&document); + let element = document + .node(root) + .and_then(|node| node.as_element()) + .unwrap(); + assert_eq!(kind, DetachedDocumentKind::Xml, "{mime} at {path}"); + assert_eq!(element.local_name(), "feed", "{mime} at {path}"); + assert_eq!(element.namespace(), "http://www.w3.org/2005/Atom"); + assert_eq!(document.text_content(root).as_deref(), Some("Example")); + } + } + } + + #[test] + fn child_projection_explicit_mime_overrides_xml_extension() { + let source = "\n<&
"; + for mime in [ + "text/plain", + "application/json", + "application/problem+json", + "text/javascript", + ] { + let (document, kind) = parse_browsing_context_document_snapshot( + Url::parse("https://example.test/data.xml").unwrap(), + source, + Some(mime), + HtmlParser::SCRIPTING_ENABLED, + ); + let root = document.document_handle(); + assert_eq!(kind, DetachedDocumentKind::Html); + assert_eq!( + document.text_content(root).as_deref(), + Some(source), + "{mime}" + ); + let children = document.child_handles(root).collect::>(); + assert_eq!(children.len(), 1); + assert!(document.is_html_element_named(children[0], "html")); + assert_eq!( + document.document_quirks_mode_for_handle(root), + Some(selectors::matching::QuirksMode::NoQuirks) + ); + } + let (document, kind) = parse_browsing_context_document_snapshot( + Url::parse("https://example.test/page.svg").unwrap(), + "&", + Some("text/html"), + HtmlParser::SCRIPTING_ENABLED, + ); + assert_eq!(kind, DetachedDocumentKind::Html); + assert!(document.element_handle_by_id("parsed").is_some()); + assert_eq!( + document.text_content(document.document_handle()).as_deref(), + Some("&") + ); + } + #[test] fn child_plain_text_document_uses_pre_and_no_quirks_mode() { let (document, kind) = parse_browsing_context_document_snapshot( diff --git a/moli-renderer-v8/src/lib.rs b/moli-renderer-v8/src/lib.rs index 4f6db92afb1..40a2339213a 100644 --- a/moli-renderer-v8/src/lib.rs +++ b/moli-renderer-v8/src/lib.rs @@ -48,6 +48,7 @@ mod document_cookie_owner; mod document_language; mod document_last_modified; mod document_module_graph; +mod document_response_decoder; mod document_runtime; mod document_script_scheduler; mod document_task_lane; diff --git a/moli-renderer-v8/src/live_document_parser.rs b/moli-renderer-v8/src/live_document_parser.rs index 1c513162cc5..e483965c862 100644 --- a/moli-renderer-v8/src/live_document_parser.rs +++ b/moli-renderer-v8/src/live_document_parser.rs @@ -757,6 +757,14 @@ impl DocumentParserSession { ) } + pub(crate) fn start_main_text_document(document_url: Url, content_type: &str) -> Self { + Self::new_html( + HtmlParser::with_scripting_enabled(false) + .start_text_document(document_url, content_type), + DocumentParserLifetime::Finite, + ) + } + pub(crate) fn start_finite_live_document( document_url: Url, document_handle: NativeNodeId, @@ -769,6 +777,21 @@ impl DocumentParserSession { ) } + pub(crate) fn start_finite_live_text_document( + document_url: Url, + document_handle: NativeNodeId, + owner: &mut impl LiveDocumentParserOwner, + ) -> Self { + Self::new_html( + HtmlParser::with_scripting_enabled(false).start_live_text_document_root( + document_url, + document_handle, + owner, + ), + DocumentParserLifetime::Finite, + ) + } + pub(crate) fn start_finite_live_xml_document( document_url: Url, document_handle: NativeNodeId, diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs index ee356deebef..f8016efb814 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs @@ -6,7 +6,7 @@ use crate::frame_owner_model::{ FrameDocumentLocalWindowTransition, FrameLocalWindowOwnerTransition, }; use crate::{document_runtime::DomHandle, frame_owner_model::FrameDocumentOwnerTransition}; -use moli_web_mime::is_dom_parser_xml_mime; +use moli_web_mime::is_xml_document_mime; use url::Url; pub(crate) struct ChildDocumentInstallResult { @@ -135,7 +135,7 @@ impl JsContextHost { let is_xml_document = snapshot .content_type .as_deref() - .is_some_and(is_dom_parser_xml_mime); + .is_some_and(is_xml_document_mime); self.install_live_child_document_from_snapshot( scope, handle, @@ -170,16 +170,8 @@ impl JsContextHost { navigation_loader: Option, is_xml_document: bool, ) -> Option { - let is_plain_text_document = snapshot - .content_type - .as_deref() - .is_some_and(|mime| mime.eq_ignore_ascii_case("text/plain")); let source = if is_xml_document { std::borrow::Cow::Borrowed(snapshot.markup.as_str()) - } else if is_plain_text_document { - std::borrow::Cow::Owned(crate::dom_parser::plain_text_document_parser_input( - &snapshot.markup, - )) } else { crate::dom_parser::preserve_decoded_bom_only_browsing_context_body( &snapshot.markup, @@ -239,6 +231,7 @@ impl JsContextHost { parser_base_url, source.as_ref(), is_xml_document, + snapshot.content_type.as_deref(), ); Some(ChildDocumentInstallResult { initial_classic_ready_work: parser_start.initial_classic_ready_work, diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs index 631bd47aad5..c644e32ed30 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs @@ -1012,14 +1012,6 @@ impl JsContextHost { let mut owner = ChildFrameLiveParserOwner::new(self, scope, document_handle); parser.finish(&mut owner) }; - if self - .dom_host() - .document_content_type_for_handle(document_handle) - .is_some_and(|mime| mime.eq_ignore_ascii_case("text/plain")) - { - self.dom_host_mut() - .set_html_quirks_mode_for_parser_document(document_handle, QuirksMode::NoQuirks); - } self.queue_live_child_parser_discovery_signals( child_handle, document_handle, @@ -1705,6 +1697,7 @@ impl JsContextHost { document_base_url: Url, markup: &str, is_xml_document: bool, + content_type: Option<&str>, ) -> ChildLiveDocumentParserStartResult { let owner = FrameDocumentOwner::new(owner_local_window_id, owner_document_id); self.child_document_parsers.clear(owner); @@ -1713,6 +1706,13 @@ impl JsContextHost { document_base_url, document_handle, ) + } else if content_type.is_some_and(moli_web_mime::is_text_document_mime) { + let mut parser_owner = ChildFrameLiveParserOwner::new(self, scope, document_handle); + DocumentParserSession::start_finite_live_text_document( + document_base_url, + document_handle, + &mut parser_owner, + ) } else { DocumentParserSession::start_finite_live_document( document_base_url, diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs index ebcbbeb135f..90fac5d675b 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs @@ -10,6 +10,7 @@ use crate::{ content_security_policy::{ ContentSecurityPolicyViolationEventFields, send_content_security_policy_reports, }, + document_response_decoder::decode_document_response, document_runtime::{DocumentNavigationEmbeddingContext, DocumentPolicyContainer, DomHandle}, document_script_scheduler::FrameDocumentClassicScriptSchedulerWork, frame_owner_model::{ @@ -21,7 +22,6 @@ use crate::{ LoadedChildDocument, SubresourceResponseBody, }, }; -use moli_encoding::decode_html_document_with_fallback; pub(crate) struct AppliedChildDocumentLoadCompletion { /// Initial parser-classic work produced by the committed child document. @@ -756,17 +756,6 @@ impl JsContextHost { } } -fn child_document_fallback_character_set( - parent_character_set: &str, - content_type: Option<&str>, -) -> String { - if content_type.is_some_and(moli_web_mime::is_dom_parser_xml_mime) { - "UTF-8".to_owned() - } else { - parent_character_set.to_owned() - } -} - fn child_document_load_outcome_from_response( request_url: String, request_method: String, @@ -785,14 +774,17 @@ fn child_document_load_outcome_from_response( let encoded_data_length = response_body.len(); let content_type = child_document_content_type_from_headers(&head.headers) .or_else(|| child_document_content_type_for_url(&head.final_url)); - let fallback = - child_document_fallback_character_set(parent_character_set, content_type.as_deref()); let (markup, character_set) = { let body_bytes = response_body .try_bytes() .map_err(|error| format!("failed to read child document response body: {error}"))?; - let (markup, character_set) = - decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback)); + let (markup, character_set) = decode_document_response( + &body_bytes, + &head.headers, + content_type.as_deref(), + &head.final_url, + Some(parent_character_set), + ); (markup, character_set.to_owned()) }; let policy_container = diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs index 71d318c3536..20e3c344bda 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs @@ -1,11 +1,10 @@ use super::super::{ChildBrowsingContextBootstrap, ChildBrowsingContextSnapshot, JsContextHost}; use super::configure_child_document_navigation_request; +use crate::document_response_decoder::decode_document_response; use crate::document_runtime::{DocumentPolicyContainer, DomHandle}; -use moli_encoding::decode_html_document_with_fallback; use moli_fetch::Request; use moli_web_mime::{ - is_dom_parser_xml_mime, is_html_document_mime, resource_mime_essence_for_url, - response_document_content_type, + is_html_document_mime, resource_mime_essence_for_url, response_document_content_type, }; use url::Url; @@ -52,7 +51,7 @@ impl JsContextHost { if !mime_type.is_empty() && !is_html_document_mime(&mime_type) { return None; } - let character_set = self.child_document_fallback_character_set(Some("text/html")); + let character_set = self.document_character_set().to_owned(); Some(ChildBrowsingContextSnapshot::with_character_set( url.clone(), body, @@ -64,11 +63,12 @@ impl JsContextHost { "data" => crate::network_host::local_url_response(url).map(|response| { let head = response.head(); let content_type = child_document_content_type_from_headers(&head.headers); - let fallback = self.child_document_fallback_character_set(content_type.as_deref()); - let (body, character_set) = decode_html_document_with_fallback( + let (body, character_set) = decode_document_response( response.body_bytes(), &head.headers, - Some(&fallback), + content_type.as_deref(), + &head.final_url, + Some(self.document_character_set()), ); let policy_container = DocumentPolicyContainer::from_navigation_response_headers( &head.headers, @@ -116,11 +116,12 @@ impl JsContextHost { let head = response.head(); let content_type = child_document_content_type_from_headers(&head.headers) .or_else(|| child_document_content_type_for_url(&head.final_url)); - let fallback = self.child_document_fallback_character_set(content_type.as_deref()); - let (markup, character_set) = decode_html_document_with_fallback( + let (markup, character_set) = decode_document_response( response.body_bytes(), &head.headers, - Some(&fallback), + content_type.as_deref(), + &head.final_url, + Some(self.document_character_set()), ); let policy_container = DocumentPolicyContainer::from_navigation_response_headers( &head.headers, @@ -170,17 +171,6 @@ impl JsContextHost { ChildBrowsingContextBootstrap::Request(_) => None, } } - - pub(in crate::native_bridge::context_host) fn child_document_fallback_character_set( - &self, - content_type: Option<&str>, - ) -> String { - if content_type.is_some_and(is_dom_parser_xml_mime) { - "UTF-8".to_owned() - } else { - self.document_character_set().to_owned() - } - } } pub(in crate::native_bridge::context_host) fn child_document_content_type_from_headers( diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs b/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs index 10c829ac1c6..1a84f1a04fa 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs @@ -238,7 +238,7 @@ impl JsContextHost { return false; } moli_web_mime::is_html_document_mime(&mime) - || moli_web_mime::is_dom_parser_xml_mime(&mime) + || moli_web_mime::is_xml_document_mime(&mime) || moli_web_mime::is_text_mime(&mime) || moli_web_mime::is_json_module_mime(&mime) || moli_web_mime::is_javascript_mime(&mime) diff --git a/moli-renderer-v8/src/native_bridge/context_host/popups.rs b/moli-renderer-v8/src/native_bridge/context_host/popups.rs index 5204a4d9214..4f3e187a247 100644 --- a/moli-renderer-v8/src/native_bridge/context_host/popups.rs +++ b/moli-renderer-v8/src/native_bridge/context_host/popups.rs @@ -2310,7 +2310,7 @@ impl JsContextHost { super::child_documents::child_document_content_type_from_headers(&head.headers); let fallback = if content_type .as_deref() - .is_some_and(moli_web_mime::is_dom_parser_xml_mime) + .is_some_and(moli_web_mime::is_xml_document_mime) { "UTF-8".to_owned() } else { diff --git a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs index 5af62b82194..1f5ec22b1fb 100644 --- a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs +++ b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs @@ -25,6 +25,7 @@ impl ConcurrentParseTimeRuntime { let (page_vm, triggered_navigation) = { let buffered_document_preloads = &mut state.buffered_document_preloads; let service_worker_preload_context = state.service_worker_preload_context.as_ref(); + let document_character_set = &state.document_character_set; PageVm::new_from_parser_stream_and_run_document_start( page_id, local_executor, @@ -34,6 +35,7 @@ impl ConcurrentParseTimeRuntime { &mut state.parser_session, started, |page_vm| { + page_vm.set_document_character_set(document_character_set.clone()); admit_pending_preloads( page_vm, buffered_document_preloads, @@ -138,11 +140,16 @@ impl ConcurrentParseTimeRuntime { runtime_hooks: PageVmRuntimeHooks, final_url: Url, document_content_type: String, + document_character_set: &str, stage: PageVmInitStage, source: String, started: Instant, ) -> Result { let mut state = ParseTimeDriverState::new_xml(final_url.clone()); + state.document_character_set = document_character_set.to_owned(); + state + .buffered_document_preloads + .set_document_character_set(document_character_set); state .parser_session .set_xml_document_content_type(document_content_type); diff --git a/moli-renderer-v8/src/runtime/phase_one/state.rs b/moli-renderer-v8/src/runtime/phase_one/state.rs index 3e9302c0dc3..01ae4154837 100644 --- a/moli-renderer-v8/src/runtime/phase_one/state.rs +++ b/moli-renderer-v8/src/runtime/phase_one/state.rs @@ -9,6 +9,7 @@ pub(super) struct ParseTimeDriverState { pub(super) buffered_document_preloads: Box, pub(super) service_worker_preload_context: Option, pub(super) input_closed: bool, + pub(super) is_text_document: bool, } impl ParseTimeDriverState { @@ -29,6 +30,23 @@ impl ParseTimeDriverState { buffered_document_preloads: Box::default(), service_worker_preload_context: None, input_closed: false, + is_text_document: false, + } + } + + pub(super) fn new_text(final_url: Url, mime: &str) -> Self { + Self { + parser_session: DocumentParserSession::start_main_text_document( + final_url.clone(), + mime, + ), + final_url, + document_character_set: "UTF-8".to_owned(), + scheduler: DocumentScriptScheduler::new(), + buffered_document_preloads: Box::default(), + service_worker_preload_context: None, + input_closed: false, + is_text_document: true, } } @@ -41,6 +59,7 @@ impl ParseTimeDriverState { buffered_document_preloads: Box::default(), service_worker_preload_context: None, input_closed: false, + is_text_document: false, } } diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rs index f3bfeed0729..395da9c8c2e 100644 --- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs +++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs @@ -3,8 +3,8 @@ use super::super::script_preloads::{ServiceWorkerScriptPreloadContext, admit_pen use super::parser_blocking_pending::main_parser_blocking_classic_script_item; use super::scaffold::continue_phase_one_until_streaming_boundary_on_execution_context; use super::*; +use crate::document_response_decoder::{decode_document_response, new_document_response_decoder}; use moli_encoding::HtmlDocumentStreamingDecoder; -use moli_encoding_detector::detect_legacy_html_encoding; use moli_web_mime::response_headers_indicate_attachment_download; use tokio::sync::{mpsc, oneshot}; @@ -94,10 +94,8 @@ impl ConcurrentParseTimeRuntime { env.apply_navigation_response_headers(&response_final_url, &response_headers); let mut body_source = RawDocumentBodySource::fetch_response(response); - let mut state = ParseTimeDriverState::new_with_scripting_enabled( - response_final_url, - main_document_parser_scripting_enabled(&env), - ); + let (mut state, mut decoder) = + response_document_parser(response_final_url, &response_headers, &env); state .buffered_document_preloads .set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission( @@ -124,11 +122,6 @@ impl ConcurrentParseTimeRuntime { ) }); state.service_worker_preload_context = service_worker_preload_context.clone(); - let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector( - &response_headers, - state.final_url.as_str(), - detect_legacy_html_encoding, - ); // Raw navigation bodies are decoded during prebootstrap scan. The decoder // is carried forward so split multibyte sequences are not decoded twice or // lost between the scan and parser handoff. @@ -308,9 +301,15 @@ impl ConcurrentParseTimeRuntime { let mut body_source = RawDocumentBodySource::External(raw_body); if response_headers_indicate_xml_document(&response_headers) { let body = collect_streaming_raw_body(&mut body_source).await?; - let source = String::from_utf8_lossy(&body).into_owned(); let content_type = moli_web_mime::response_document_content_type(&response_headers) .unwrap_or_else(|| "application/xml".to_owned()); + let (source, character_set) = decode_document_response( + &body, + &response_headers, + Some(&content_type), + &final_url, + None, + ); let outcome = Self::finish_creation_from_xml_bootstrap( page_id, local_executor, @@ -319,6 +318,7 @@ impl ConcurrentParseTimeRuntime { runtime_hooks, final_url, content_type, + character_set, stage, source, started, @@ -332,10 +332,7 @@ impl ConcurrentParseTimeRuntime { }, ))); } - let mut state = ParseTimeDriverState::new_with_scripting_enabled( - final_url, - main_document_parser_scripting_enabled(&env), - ); + let (mut state, mut decoder) = response_document_parser(final_url, &response_headers, &env); state .buffered_document_preloads .set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission( @@ -362,11 +359,6 @@ impl ConcurrentParseTimeRuntime { ) }); state.service_worker_preload_context = service_worker_preload_context.clone(); - let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector( - &response_headers, - state.final_url.as_str(), - detect_legacy_html_encoding, - ); // External raw bodies replay captured chunks. Pre-scan only what is already // buffered before bootstrap so the producer's backpressure boundary still // controls how far ahead the parser can get. @@ -451,6 +443,10 @@ impl ConcurrentParseTimeRuntime { chunk: String, service_worker_context: Option<&ServiceWorkerScriptPreloadContext>, ) { + if self.state.is_text_document { + self.state.parser_session.queue_arrived_chunk(chunk); + return; + } self.state .buffered_document_preloads .append_to_main_document_scan_with_service_worker_context( @@ -482,7 +478,7 @@ impl ConcurrentParseTimeRuntime { fn response_headers_indicate_xml_document(headers: &[(String, Vec)]) -> bool { moli_web_mime::response_document_content_type(headers) - .is_some_and(|mime| moli_web_mime::is_dom_parser_xml_mime(&mime)) + .is_some_and(|mime| moli_web_mime::is_xml_document_mime(&mime)) } const EXTERNAL_RAW_DOCUMENT_BODY_BUFFERED_CHUNKS: usize = 8; @@ -607,6 +603,9 @@ fn scan_prebootstrap_html_chunk_into_state( chunk: &str, service_worker_context: Option<&ServiceWorkerScriptPreloadContext>, ) { + if state.is_text_document { + return; + } state .buffered_document_preloads .append_to_main_document_prebootstrap_scan_with_service_worker_context( @@ -617,6 +616,31 @@ fn scan_prebootstrap_html_chunk_into_state( ); } +fn response_document_parser( + final_url: Url, + headers: &[(String, Vec)], + env: &PageVmEnvConfig, +) -> (ParseTimeDriverState, HtmlDocumentStreamingDecoder) { + let content_type = moli_web_mime::response_document_content_type(headers); + let text_type = content_type + .as_deref() + .filter(|mime| moli_web_mime::is_text_document_mime(mime)); + let mut state = if let Some(mime) = text_type { + ParseTimeDriverState::new_text(final_url, mime) + } else { + ParseTimeDriverState::new_with_scripting_enabled( + final_url, + main_document_parser_scripting_enabled(env), + ) + }; + let decoder = + new_document_response_decoder(headers, content_type.as_deref(), &state.final_url, None); + // Headers/defaults are observable even when no body chunk is ready yet. + // A later BOM or decoder decision can still replace this tentative value. + sync_state_document_character_set_from_decoder(&mut state, &decoder); + (state, decoder) +} + pub(super) fn enqueue_streaming_raw_chunk( runtime: &mut ConcurrentParseTimeRuntime, decoder: &mut HtmlDocumentStreamingDecoder, @@ -910,6 +934,105 @@ mod tests { use crate::parser::ScriptSource; use tokio::io::{AsyncReadExt, AsyncWriteExt}; + #[tokio::test] + async fn text_document_charset_is_visible_to_document_start_before_body_arrives() { + let _js_runtime_owner = crate::JsRuntime::initialize(); + tokio::task::LocalSet::new().run_until(async { + for (mime, chunk, expected) in [ + ("text/plain; charset=gbk", None, "GBK"), + ("application/json", None, "UTF-8"), + ("text/plain; charset=gbk", Some(b"\xef\xbb\xbfready".as_slice()), "UTF-8"), + ] { + let loader_owner = ResourceRequestClient::new(&FetchConfig::default()).unwrap(); + let loader = loader_owner.handle(); + let mut env = default_test_page_vm_env_config(); + env.document_start_scripts.push(crate::DocumentStartScript { + registry_key: None, + devtools_session: None, + source: "document.title = document.characterSet".to_owned(), + world_name: None, + has_bidi_channel_argument: false, + bidi_channel_handoffs: Vec::new(), + }); + let (mut state, mut decoder) = response_document_parser( + Url::parse("https://example.test/data").unwrap(), + &[("Content-Type".to_owned(), mime.as_bytes().to_vec())], &env, + ); + if let Some(chunk) = chunk { + let chunks = decoder.push(chunk); + sync_state_document_character_set_from_decoder(&mut state, &decoder); + for chunk in chunks { + state.parser_session.queue_arrived_chunk(chunk); + } + } + assert_eq!(state.document_character_set, expected); + let (_, page_vm, navigated) = ConcurrentParseTimeRuntime::bootstrap_page_vm_from_state_on_fresh_local_task( + PageId::new_for_testing(1), JsLocalExecutor::new(), loader, env, + PageVmRuntimeHooks::standalone_without_owner_reservation_for_test(), state, + Instant::now(), "charset test bootstrap closed", + ).await.unwrap(); + assert!(!navigated); + assert_eq!(page_vm.vm().snapshot_live_document().document_title(), expected, "{mime}"); + } + }).await; + } + + #[test] + fn response_document_dispatch_distinguishes_text_from_html_and_xml() { + for (mime, text) in [ + ("application/json", true), + ("application/problem+json", true), + ("text/plain; charset=gbk", true), + ("text/javascript", true), + ("text/html", false), + ("application/xhtml+xml", false), + ("application/xml", false), + ("text/xml", false), + ] { + let (state, _) = response_document_parser( + Url::parse("https://example.test/data").unwrap(), + &[("Content-Type".to_owned(), mime.as_bytes().to_vec())], + &default_test_page_vm_env_config(), + ); + assert_eq!(state.is_text_document, text, "{mime}"); + } + } + + #[tokio::test] + async fn text_document_does_not_preload_literal_script_tags() { + let loader = ResourceRequestClient::new(&moli_fetch::FetchConfig::default()) + .expect("default loader"); + let runtime_hooks = PageVmRuntimeHooks::standalone_without_owner_reservation_for_test(); + for (mime, expected_preload) in [ + ("text/plain", false), + ("application/json", false), + ("text/html", true), + ] { + let (mut state, _) = response_document_parser( + Url::parse("https://example.test/data").unwrap(), + &[("Content-Type".to_owned(), mime.as_bytes().to_vec())], + &default_test_page_vm_env_config(), + ); + state.buffered_document_preloads.bind_resource_runtime( + runtime_hooks.owner_wake(), + runtime_hooks.resource_task_runner(), + ); + scan_prebootstrap_html_chunk_into_state( + &mut state, + &loader, + "", + None, + ); + assert_eq!( + state.buffered_document_preloads.entries.contains_key( + &classic_preload_key_for_streaming_test("https://example.test/literal.js") + ), + expected_preload, + "{mime}", + ); + } + } + #[tokio::test] async fn external_raw_document_body_source_collects_chunks_and_completion() { let (completion_tx, completion_rx) = oneshot::channel(); diff --git a/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs b/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs index bb3e6a40e4f..cffd375545e 100644 --- a/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs +++ b/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs @@ -1,5 +1,104 @@ use super::*; +#[tokio::test(flavor = "multi_thread")] +async fn external_raw_xml_document_decodes_response_bytes() { + const SOURCE: &str = "\ + café"; + let legacy_body = b"\ + caf\xe9"; + let cases = [ + ( + "application/atom+xml; charset=windows-1252", + legacy_body.to_vec(), + "windows-1252", + ), + ( + "application/atom+xml", + [ + b"".as_slice(), + legacy_body, + ] + .concat(), + "windows-1252", + ), + ( + "application/atom+xml; charset=windows-1252", + [ + vec![0xff, 0xfe], + SOURCE.encode_utf16().flat_map(u16::to_le_bytes).collect(), + ] + .concat(), + "UTF-16LE", + ), + ( + "application/atom+xml", + [ + vec![0xfe, 0xff], + SOURCE.encode_utf16().flat_map(u16::to_be_bytes).collect(), + ] + .concat(), + "UTF-16BE", + ), + ]; + let runtime = JsRuntime::initialize(); + let loader = + ResourceRequestClient::new(&moli_fetch::FetchConfig::default()).expect("default loader"); + for (content_type, body, encoding) in cases { + let (completion_tx, completion_rx) = oneshot::channel(); + let (body_tx, raw_body) = ExternalRawDocumentBodyStream::channel(completion_rx); + let producer = tokio::spawn(async move { + for byte in body { + body_tx + .send(vec![byte]) + .await + .expect("body byte should send"); + } + completion_tx.send(Ok(())).expect("body should complete"); + }); + let prepared = prepare_test_external_raw_document_with_content_type( + &runtime, + &loader, + url::Url::parse("https://example.test/feed").unwrap(), + content_type, + raw_body, + ) + .await; + let permit = prepared.issue_commit_permit(); + let (mut page, _, _, _, pending_download) = + prepared.commit(permit).await.expect("XML should commit"); + assert!(pending_download.is_none()); + producer.await.expect("body producer should finish"); + let (reply, _) = page + .run_async_command(RendererPageCommand::EvaluateExpression { + expression: concat!( + "JSON.stringify((() => { const root=document.getElementsByTagNameNS(", + "'http://www.w3.org/2005/Atom','feed')[0]; ", + "return [root.localName,root.namespaceURI,root.textContent,", + "document.contentType,document.characterSet]; })())", + ) + .to_owned(), + await_promise: false, + }) + .await + .expect("XML response metadata and text should evaluate"); + assert_eq!( + renderer_json_value(reply), + Some(serde_json::Value::String( + serde_json::json!([ + "feed", + "http://www.w3.org/2005/Atom", + "café", + "application/atom+xml", + encoding + ]) + .to_string() + )), + "{content_type}: {encoding}" + ); + page.close_async().await.expect("XML page should close"); + } +} + #[tokio::test(flavor = "multi_thread")] async fn streaming_xml_document_executes_parser_blocking_xhtml_script() { let runtime = JsRuntime::initialize(); diff --git a/moli-web-mime/src/classification.rs b/moli-web-mime/src/classification.rs index 3ff3a0dd50f..9574cdde24e 100644 --- a/moli-web-mime/src/classification.rs +++ b/moli-web-mime/src/classification.rs @@ -13,6 +13,14 @@ pub fn is_dom_parser_xml_mime(input: &str) -> bool { }) } +/// XML navigation responses, including registered and custom `+xml` subtypes. +/// DOMParser's Web API accepts a narrower, separately classified set of types. +pub fn is_xml_document_mime(input: &str) -> bool { + mime_essence(input).is_some_and(|mime| { + matches!(mime.as_str(), "text/xml" | "application/xml") || mime.ends_with("+xml") + }) +} + pub fn is_javascript_mime(input: &str) -> bool { mime_essence(input).is_some_and(|mime| is_javascript_mime_essence(&mime)) } @@ -51,6 +59,14 @@ pub fn is_text_mime_essence(input: &str) -> bool { input.starts_with("text/") } +/// Navigation responses rendered as literal text rather than HTML or XML. +/// This also includes JSON and JavaScript resources opened as documents. +pub fn is_text_document_mime(input: &str) -> bool { + !is_html_document_mime(input) + && !is_xml_document_mime(input) + && (is_text_mime(input) || is_json_module_mime(input) || is_javascript_mime(input)) +} + /// Whether a `style` element's raw `type` attribute selects classic CSS. /// /// HTML's update-a-style-block algorithm compares the untrimmed attribute @@ -135,6 +151,12 @@ pub fn is_json_module_mime(input: &str) -> bool { mime_essence(input).is_some_and(|mime| mime == "application/json" || mime.ends_with("+json")) } +pub fn is_json_document_mime(input: &str) -> bool { + mime_essence(input).is_some_and(|mime| { + matches!(mime.as_str(), "application/json" | "text/json") || mime.ends_with("+json") + }) +} + pub fn is_webassembly_mime(input: &str) -> bool { mime_essence(input).is_some_and(|mime| mime == "application/wasm") } diff --git a/moli-web-mime/src/lib.rs b/moli-web-mime/src/lib.rs index e631473eaff..bcae3a9eb85 100644 --- a/moli-web-mime/src/lib.rs +++ b/moli-web-mime/src/lib.rs @@ -13,9 +13,10 @@ pub use classification::{ is_css_style_element_type_attribute, is_css_stylesheet_type_hint, is_dom_parser_xml_mime, is_font_mime, is_font_mime_essence, is_form_urlencoded_mime, is_html_document_mime, is_image_mime, is_image_mime_essence, is_javascript_mime, is_javascript_mime_essence, - is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime, is_png_image_mime_essence, - is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence, is_text_mime, - is_text_mime_essence, is_video_mime, is_video_mime_essence, is_webassembly_mime, + is_json_document_mime, is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime, + is_png_image_mime_essence, is_supported_document_mime_type, is_svg_image_mime, + is_svg_image_mime_essence, is_text_document_mime, is_text_mime, is_text_mime_essence, + is_video_mime, is_video_mime_essence, is_webassembly_mime, is_xml_document_mime, multipart_form_data_boundary, }; pub use data_url::{ diff --git a/moli-web-mime/src/sniffing.rs b/moli-web-mime/src/sniffing.rs index 5b21532b6c5..28f0d6342ab 100644 --- a/moli-web-mime/src/sniffing.rs +++ b/moli-web-mime/src/sniffing.rs @@ -1,4 +1,4 @@ -use crate::classification::{is_dom_parser_xml_mime, is_html_document_mime}; +use crate::classification::{is_html_document_mime, is_xml_document_mime}; use crate::parse::mime_essence; pub const RESOURCE_HEADER_BYTE_LIMIT: usize = 1445; @@ -212,7 +212,7 @@ fn computed_browsing_mime_type( } fn computed_image_mime_type(supplied: Option<&str>, header: &[u8]) -> String { - if supplied.is_some_and(is_xml_mime) { + if supplied.is_some_and(is_xml_document_mime) { return supplied.expect("checked supplied MIME").into(); } sniff_image_mime_type(header) @@ -222,7 +222,7 @@ fn computed_image_mime_type(supplied: Option<&str>, header: &[u8]) -> String { } fn computed_audio_video_mime_type(supplied: Option<&str>, header: &[u8]) -> String { - if supplied.is_some_and(is_xml_mime) { + if supplied.is_some_and(is_xml_document_mime) { return supplied.expect("checked supplied MIME").into(); } sniff_audio_video_mime_type(header) @@ -232,7 +232,7 @@ fn computed_audio_video_mime_type(supplied: Option<&str>, header: &[u8]) -> Stri } fn computed_font_mime_type(supplied: Option<&str>, header: &[u8]) -> String { - if supplied.is_some_and(is_xml_mime) { + if supplied.is_some_and(is_xml_document_mime) { return supplied.expect("checked supplied MIME").into(); } sniff_font_mime_type(header) @@ -424,11 +424,7 @@ fn is_binary_data_byte(byte: u8) -> bool { } fn is_xml_or_html_mime(mime: &str) -> bool { - is_xml_mime(mime) || is_html_mime(mime) -} - -fn is_xml_mime(mime: &str) -> bool { - is_dom_parser_xml_mime(mime) || mime.ends_with("+xml") + is_xml_document_mime(mime) || is_html_mime(mime) } fn is_html_mime(mime: &str) -> bool { diff --git a/moli-web-mime/src/tests.rs b/moli-web-mime/src/tests.rs index 010209cc787..d361b05e035 100644 --- a/moli-web-mime/src/tests.rs +++ b/moli-web-mime/src/tests.rs @@ -870,3 +870,69 @@ fn script_like_mime_type_block_matches_fetch_response_rule() { )); assert!(!should_script_like_response_be_blocked_due_to_mime_type(&[])); } +#[test] +fn document_text_classification_excludes_html_and_xml() { + for mime in [ + "text/plain", + "Text/Css; charset=utf-8", + "application/json", + "application/ld+json", + "application/javascript", + ] { + assert!(crate::is_text_document_mime(mime), "{mime}"); + } + for mime in [ + "text/html", + "text/xml", + "application/xhtml+xml", + "image/svg+xml", + "application/atom+xml", + "application/rss+xml", + "text/example+xml; charset=utf-8", + "image/png", + "application/octet-stream", + ] { + assert!(!crate::is_text_document_mime(mime), "{mime}"); + } +} + +#[test] +fn navigation_xml_mime_classification_keeps_dom_parser_allowlist_separate() { + for mime in [ + "text/xml", + "application/xml", + "application/xhtml+xml", + "image/svg+xml", + ] { + assert!(is_xml_document_mime(mime), "{mime}"); + assert!(is_dom_parser_xml_mime(mime), "{mime}"); + } + for mime in [ + "Application/Atom+XML; charset=utf-8", + "application/rss+xml", + "text/example+xml", + ] { + assert!(is_xml_document_mime(mime), "{mime}"); + assert!(!is_dom_parser_xml_mime(mime), "{mime}"); + assert!(!is_text_document_mime(mime), "{mime}"); + } + for mime in [ + "text/html", + "text/plain", + "application/xml-dtd", + "invalid+xml", + ] { + assert!(!is_xml_document_mime(mime), "{mime}"); + } +} + +#[test] +fn navigation_json_mime_includes_text_json_without_changing_module_mime() { + for mime in ["application/json", "application/problem+json"] { + assert!(is_json_document_mime(mime), "{mime}"); + assert!(is_json_module_mime(mime), "{mime}"); + } + assert!(is_json_document_mime("Text/JSON; charset=utf-8")); + assert!(!is_json_module_mime("Text/JSON; charset=utf-8")); + assert!(!is_json_document_mime("text/plain")); +} diff --git a/moli/src/fetch_dump/tests.rs b/moli/src/fetch_dump/tests.rs index 73240abaf6e..b2f93f81c2c 100644 --- a/moli/src/fetch_dump/tests.rs +++ b/moli/src/fetch_dump/tests.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use anyhow::Result; -use axum::{Router, routing::get}; +use axum::{Router, response::Html, routing::get}; use moli_core::{ LayoutPolicy, page::Page, @@ -36,7 +36,7 @@ async fn load_page_with_config( "/", get(move || { let body = Arc::clone(&server_body); - async move { (*body).clone() } + async move { Html((*body).clone()) } }), ); axum::serve(listener, app).await.unwrap();