fix(parser): render text and JSON documents without HTML side effects

moli-encoding/src/document.rs

@@ -47,9 +47,26 @@ pub struct HtmlDocumentStreamingDecoder {
     url_hint: Option<String>,
     decoder: Option<Decoder>,
     selected_encoding: Option<&'static Encoding>,
+    sniff_html_declarations: bool,
 }
 
 impl HtmlDocumentStreamingDecoder {
+    /// Decode a text document without interpreting literal HTML/XML declarations.
+    /// JSON defaults to UTF-8; other text retains the configured legacy detector.
+    pub fn new_text_document(
+        headers: &[(String, String)],
+        url_hint: &str,
+        detector: LegacyEncodingDetector,
+        json: bool,
+    ) -> Self {
+        let mut decoder = Self::new_with_options(headers, None, Some(url_hint), Some(detector));
+        decoder.sniff_html_declarations = false;
+        if json && decoder.transport_encoding.is_none() {
+            decoder.transport_encoding = Some(encoding_rs::UTF_8);
+        }
+        decoder
+    }
+
     pub fn new(headers: &[(String, String)]) -> Self {
         Self::new_with_options(headers, None, None, None)
     }
@@ -88,6 +105,7 @@ impl HtmlDocumentStreamingDecoder {
             url_hint: url_hint.map(str::to_owned),
             decoder: None,
             selected_encoding: None,
+            sniff_html_declarations: true,
         }
     }
 
@@ -97,6 +115,7 @@ impl HtmlDocumentStreamingDecoder {
 
     pub fn document_encoding_name(&self) -> &'static str {
         self.selected_encoding_name()
+            .or_else(|| self.transport_encoding.map(Encoding::name))
             .unwrap_or(self.fallback_encoding.name())
     }
 
@@ -149,6 +168,12 @@ impl HtmlDocumentStreamingDecoder {
         if let Some(encoding) = self.transport_encoding {
             return Some(encoding);
         }
+        if !self.sniff_html_declarations {
+            return (finishing || self.sniff_buffer.len() >= 1024).then(|| {
+                self.detected_legacy_content_encoding()
+                    .unwrap_or(self.fallback_encoding)
+            });
+        }
         if let Some(encoding) = encoding_for_document_utf16_xml_prefix(&self.sniff_buffer) {
             return Some(encoding);
         }

moli-encoding/src/tests.rs

@@ -24,6 +24,102 @@ fn unexpected_legacy_encoding_detector(
     panic!("declared encoding must take precedence over heuristic detection")
 }
 
+#[test]
+fn json_text_document_defaults_to_utf8_across_every_chunk_split() {
+    let input = "{\"name\":\"Gülçek\"}";
+    for split in 0..=input.len() {
+        let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+            &[],
+            "https://example.test/",
+            unexpected_legacy_encoding_detector,
+            true,
+        );
+        let mut output = decoder.push(&input.as_bytes()[..split]).join("");
+        output.push_str(&decoder.push(&input.as_bytes()[split..]).join(""));
+        output.push_str(&decoder.finish().unwrap_or_default());
+        assert_eq!(output, input, "split {split}");
+        assert_eq!(decoder.selected_encoding_name(), Some("UTF-8"));
+    }
+}
+
+#[test]
+fn plain_text_document_honors_transport_charset() {
+    let headers = [(
+        "Content-Type".to_owned(),
+        "text/plain; charset=gbk".to_owned(),
+    )];
+    let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+        &headers,
+        "https://example.test/",
+        unexpected_legacy_encoding_detector,
+        false,
+    );
+    let mut output = decoder.push(&gbk_bytes("<b>家居</b>")).join("");
+    output.push_str(&decoder.finish().unwrap_or_default());
+    assert_eq!(output, "<b>家居</b>");
+    assert_eq!(decoder.selected_encoding_name(), Some("GBK"));
+}
+
+#[test]
+fn text_document_bom_precedes_transport_charset() {
+    let headers = [(
+        "Content-Type".to_owned(),
+        "text/plain; charset=gbk".to_owned(),
+    )];
+    let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+        &headers,
+        "https://example.test/",
+        unexpected_legacy_encoding_detector,
+        false,
+    );
+    assert_eq!(decoder.document_encoding_name(), "GBK");
+    assert_eq!(decoder.selected_encoding_name(), None);
+    assert!(decoder.push(&[0xef]).is_empty());
+    assert!(decoder.push(&[0xbb]).is_empty());
+    let mut tail = vec![0xbf];
+    tail.extend_from_slice("Gülçek".as_bytes());
+    let mut output = decoder.push(&tail).join("");
+    output.push_str(&decoder.finish().unwrap_or_default());
+    assert_eq!(output, "Gülçek");
+    assert_eq!(decoder.selected_encoding_name(), Some("UTF-8"));
+    assert_eq!(decoder.document_encoding_name(), "UTF-8");
+}
+
+#[test]
+fn plain_text_document_retains_legacy_detection_without_a_charset() {
+    let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+        &[],
+        "https://legacy.example/",
+        test_legacy_encoding_detector,
+        false,
+    );
+    let mut output = decoder.push(&[0x80]).join("");
+    output.push_str(&decoder.finish().unwrap_or_default());
+    assert_eq!(output, "А");
+    assert_eq!(decoder.selected_encoding_name(), Some("IBM866"));
+}
+
+#[test]
+fn plain_text_document_ignores_literal_encoding_declarations() {
+    for prefix in [
+        "<meta charset=utf-8>",
+        "<?xml version='1.0' encoding='utf-8'?>",
+    ] {
+        let mut input = prefix.as_bytes().to_vec();
+        input.push(0xe9);
+        let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+            &[],
+            "https://example.test/",
+            test_legacy_encoding_detector,
+            false,
+        );
+        let mut output = decoder.push(&input).join("");
+        output.push_str(&decoder.finish().unwrap_or_default());
+        assert_eq!(output, format!("{prefix}é"));
+        assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));
+    }
+}
+
 #[test]
 fn content_type_charset_is_selected() {
     let headers = vec![(

moli-parser/src/html.rs

@@ -27,7 +27,7 @@ use super::live_target::{
     ParserDomMutationConsumer, ParserDomReadConsumer, ParserElementCreationConsumer,
     ParserMutationEffectConsumer, ParserMutationEffectDelivery, ParserRuntimeDomSinks,
     ParserStreamHtmlTreeSinkTarget, new_live_document_root_html_tree_sink_stream,
-    new_parser_stream_html_tree_sink_stream,
+    new_parser_stream_html_tree_sink_stream, new_parser_stream_html_tree_sink_target,
 };
 use super::{
     ParserSourcePosition, html_chunks,
@@ -438,6 +438,47 @@ impl HtmlParser {
         DocumentStream::new_parser_stream(final_url, self.scripting_enabled)
     }
 
+    /// Start a text document using the HTML tokenizer's plaintext state.
+    /// Subsequent chunks remain literal text inside the browser-owned `pre`.
+    pub fn start_text_document(&self, final_url: Url, content_type: &str) -> DocumentStream {
+        let stream =
+            DocumentStream::new_text_document(new_parser_stream_html_tree_sink_target(final_url));
+        stream.inner.initialize_text_document();
+        let mut host = stream.inner.take_parser_stream_dom_host();
+        let document_handle = host.document_handle();
+        host.set_document_content_type_for_handle(document_handle, content_type);
+        stream.inner.restore_parser_stream_dom_host(host);
+        stream
+    }
+
+    /// Initialize the same literal-text shell in an existing document. The
+    /// embedder owns that document's MIME metadata and live DOM callbacks.
+    pub fn start_live_text_document_root<T>(
+        &self,
+        final_url: Url,
+        document_handle: NativeNodeId,
+        consumer: &mut T,
+    ) -> DocumentStream
+    where
+        T: ParserDomReadConsumer
+            + ParserDomMutationConsumer
+            + ParserMutationEffectConsumer
+            + ParserElementCreationConsumer,
+    {
+        let stream = DocumentStream::new_text_document(
+            ParserStreamHtmlTreeSinkTarget::new_live_document_root(final_url, document_handle),
+        );
+        // SAFETY: the consumer is exclusively borrowed until the step guard
+        // removes the erased callbacks, including when initialization unwinds.
+        let sinks = unsafe { ParserRuntimeDomSinks::from_consumer(consumer) };
+        stream.inner.enter_runtime_dom_sinks_parse_step(sinks);
+        {
+            let step = RuntimeDomSinksParserStep { stream: &stream };
+            step.stream.inner.initialize_text_document();
+        }
+        stream
+    }
+
     pub fn start_live_document_root(
         &self,
         final_url: Url,
@@ -551,6 +592,17 @@ impl Default for ParserInputQueue {
 }
 
 impl DocumentStream {
+    fn new_text_document(target: ParserStreamHtmlTreeSinkTarget) -> Self {
+        let mut options = html_parse_opts_with_scripting(false);
+        // The byte decoder has already consumed the transport BOM. In plaintext
+        // every remaining U+FEFF is content, including at a chunk boundary.
+        options.tokenizer.discard_bom = false;
+        Self {
+            inner: HtmlTreeSinkStream::from_target_with_options(target, options),
+            input: RefCell::default(),
+        }
+    }
+
     fn new_parser_stream(final_url: Url, scripting_enabled: bool) -> Self {
         Self {
             inner: new_parser_stream_html_tree_sink_stream(final_url, scripting_enabled),
@@ -1843,6 +1895,45 @@ mod tests {
     const HTML_NS: &str = "http://www.w3.org/1999/xhtml";
     const MATHML_NS: &str = "http://www.w3.org/1998/Math/MathML";
 
+    #[test]
+    fn text_document_stream_preserves_literal_markup_and_leading_newline() {
+        let payload =
+            "\u{feff}\n<b>Gülçek</b>&amp;<script>window.executed=1</script></pre>\u{feff}";
+        for content_type in ["text/plain", "application/json", "application/problem+json"] {
+            let stream = HtmlParser::SCRIPTING_ENABLED.start_text_document(
+                Url::parse("https://example.test/data").unwrap(),
+                content_type,
+            );
+            for character in payload.chars() {
+                stream.feed(&character.to_string());
+            }
+            let document = stream.finish();
+            let pre = first_element_by_ns(&document, HTML_NS, "pre");
+            assert_eq!(document.text_content(pre).as_deref(), Some(payload));
+            assert_eq!(document.document().unwrap().content_type(), content_type);
+            for tag in ["b", "script"] {
+                assert!(
+                    document
+                        .elements_by_tag_name_ns(
+                            document.document_node_id(),
+                            Some(HTML_NS),
+                            tag,
+                            true
+                        )
+                        .is_empty()
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn html_document_still_parses_markup_and_entities() {
+        let document = parse_test_document("<b>Gülçek&amp;</b>");
+        let bold = first_element_by_ns(&document, HTML_NS, "b");
+        assert_eq!(document.text_content(bold).as_deref(), Some("Gülçek&"));
+        assert_eq!(document.document().unwrap().content_type(), "text/html");
+    }
+
     fn parse_test_document(html: &str) -> NativeDom {
         HtmlParser::SCRIPTING_ENABLED.parse(
             Url::parse("https://example.test/").expect("test url"),

moli-parser/src/session.rs

@@ -167,6 +167,16 @@ impl TokenSink for EmbedderPausingTreeBuilder {
 }
 
 impl HtmlParserSession {
+    pub(super) fn initialize_text_document(&self) {
+        // Parse only the browser-owned shell. Response bytes enter the tokenizer
+        // after it has switched to plaintext, so tags and entities stay literal.
+        self.process(StrTendril::from(concat!(
+            "<!doctype html><html><head></head><body>",
+            "<pre style=\"word-wrap: break-word; white-space: pre-wrap;\">\n"
+        )));
+        self.tokenizer.set_plaintext_state();
+    }
+
     fn new(sink: DocumentSink, opts: ParseOpts) -> Self {
         let tree_builder = EmbedderPausingTreeBuilder::new(sink, opts.tree_builder);
         Self {
@@ -352,10 +362,10 @@ fn feed_with_definitive_encoding(
 
 pub(super) fn new_html_tree_sink_session(
     target: ParserStreamHtmlTreeSinkTarget,
-    scripting_enabled: bool,
+    options: ParseOpts,
 ) -> HtmlTreeSinkSession {
     let sink = DocumentSink::new(target);
-    let parser = HtmlParserSession::new(sink, html_parse_opts_with_scripting(scripting_enabled));
+    let parser = HtmlParserSession::new(sink, options);
     let script_input = ParserInputQueue::default();
 
     HtmlTreeSinkSession {

moli-parser/src/stream.rs

@@ -429,11 +429,25 @@ pub(crate) fn prepare_parser_script_handoff_for_static_document(
 }
 
 impl HtmlTreeSinkStream {
+    pub(super) fn initialize_text_document(&self) {
+        self.parser.initialize_text_document();
+    }
+
     pub(super) fn from_target_with_scripting(
         target: ParserStreamHtmlTreeSinkTarget,
         scripting_enabled: bool,
     ) -> Self {
-        let session = new_html_tree_sink_session(target, scripting_enabled);
+        Self::from_target_with_options(
+            target,
+            super::session::html_parse_opts_with_scripting(scripting_enabled),
+        )
+    }
+
+    pub(super) fn from_target_with_options(
+        target: ParserStreamHtmlTreeSinkTarget,
+        options: html5ever::ParseOpts,
+    ) -> Self {
+        let session = new_html_tree_sink_session(target, options);
         Self {
             parser: session.parser,
             script_input: session.script_input,