From 7f0b9fa66f547568e1f222c3508dd7b31f47b92f Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:06:53 +0800 Subject: [PATCH 01/12] fix(html2md): preserve media resources and readable controls --- moli-html2md/README.md | 7 +- moli-html2md/src/content.rs | 86 ++++ moli-html2md/src/form.rs | 50 +++ moli-html2md/src/lib.rs | 6 + moli-html2md/src/machine.rs | 153 +++++++- moli-html2md/src/media.rs | 301 ++++++++++++++ moli-html2md/tests/regressions.rs | 368 +++++++++++++++++- moli-html2md/tests/turndown/README.md | 4 +- moli-html2md/tests/turndown/expectations.json | 4 + 9 files changed, 963 insertions(+), 16 deletions(-) create mode 100644 moli-html2md/src/content.rs create mode 100644 moli-html2md/src/form.rs create mode 100644 moli-html2md/src/media.rs diff --git a/moli-html2md/README.md b/moli-html2md/README.md index 133f4be7577..f5228366f8a 100644 --- a/moli-html2md/README.md +++ b/moli-html2md/README.md @@ -75,7 +75,8 @@ another conversion path. Complex layouts remain readable blocks instead of raw HTML tables. Supported output includes headings, paragraphs, emphasis, strikethrough, links, -images, lists, blockquotes, hard breaks, and fenced code. Inline HTML is used when +images, lists, blockquotes, hard breaks, media source links, readable form values, +separated form choices, and fenced code. Inline HTML is used when Markdown delimiters cannot express an emphasis boundary. Script/style/head/noscript/template subtrees are omitted. @@ -116,3 +117,7 @@ The HTML parser, reference JSON reader, and `pulldown-cmark` renderer are all **test-only** dependencies. Tests use checked-in reference outputs and require neither Node nor network access. NativeDom integration and real HTML fixtures are tested in `moli-renderer-v8`. + +Unavailable images retain their text alternatives. Media references use standard +source declarations before lazy-loading fallbacks. Empty accessible controls retain +their labels; password values and hidden or file input values are omitted. diff --git a/moli-html2md/src/content.rs b/moli-html2md/src/content.rs new file mode 100644 index 00000000000..4ebce0590f9 --- /dev/null +++ b/moli-html2md/src/content.rs @@ -0,0 +1,86 @@ +use crate::{Dom, NodeKind}; + +pub(crate) fn has_readable_content( + dom: &D, + root: D::NodeId, + limit: usize, +) -> bool { + let mut pending = vec![(root, 0)]; + while let Some((node, depth)) = pending.pop() { + if matches!( + dom.node_kind(node), + NodeKind::Element("head" | "title" | "script" | "style" | "noscript" | "template") + ) { + continue; + } + match dom.node_kind(node) { + NodeKind::Text(text) if !text.trim_matches(char::is_whitespace).is_empty() => { + return true; + } + NodeKind::Element("img") + if crate::media::source(dom, node).is_some() + || dom + .attribute(node, "alt") + .is_some_and(|alt| !alt.trim().is_empty()) => + { + return true; + } + NodeKind::Element("input") if crate::form::input_text(dom, node).is_some() => { + return true; + } + NodeKind::Element("textarea") => { + if let Some(value) = crate::form::textarea_text(dom, node) { + if !value.trim().is_empty() { + return true; + } + // A present live value is authoritative even when the user + // cleared it. Do not inspect serialized default children. + continue; + } + } + NodeKind::Element(_) if depth > 0 && fallback_text(dom, node).is_some() => return true, + _ => {} + } + if depth + 1 >= limit { + continue; + } + let mut child = dom.first_child(node); + while let Some(id) = child { + pending.push((id, depth + 1)); + child = dom.next_sibling(id); + } + } + false +} + +pub(crate) fn fallback_text(dom: &D, node: D::NodeId) -> Option { + for name in ["aria-label", "data-original-title", "title"] { + if let Some(value) = dom + .attribute(node, name) + .map(str::trim) + .filter(|value| !value.is_empty()) + { + return Some(value.to_owned()); + } + } + if matches!(dom.node_kind(node), NodeKind::Element("svg")) { + let mut child = dom.first_child(node); + while let Some(id) = child { + if dom.node_kind(id) == NodeKind::Element("title") { + let mut text = String::new(); + let mut title_child = dom.first_child(id); + while let Some(part) = title_child { + if let NodeKind::Text(value) = dom.node_kind(part) { + text.push_str(value); + } + title_child = dom.next_sibling(part); + } + if !text.trim().is_empty() { + return Some(text); + } + } + child = dom.next_sibling(id); + } + } + None +} diff --git a/moli-html2md/src/form.rs b/moli-html2md/src/form.rs new file mode 100644 index 00000000000..6beb52aa2fe --- /dev/null +++ b/moli-html2md/src/form.rs @@ -0,0 +1,50 @@ +use std::borrow::Cow; + +use crate::Dom; + +pub(crate) fn input_text<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Option> { + let kind = dom.attribute(node, "type").unwrap_or("text"); + if kind.eq_ignore_ascii_case("checkbox") || kind.eq_ignore_ascii_case("radio") { + return Some(Cow::Borrowed(if dom.attribute(node, "checked").is_some() { + "☑" + } else { + "☐" + })); + } + if ["hidden", "file"] + .iter() + .any(|excluded| kind.eq_ignore_ascii_case(excluded)) + { + return None; + } + if kind.eq_ignore_ascii_case("image") { + return dom + .attribute(node, "alt") + .filter(|value| !value.is_empty()) + .map(Cow::Borrowed); + } + let value = (!kind.eq_ignore_ascii_case("password")) + .then(|| dom.attribute(node, "value")) + .flatten() + .filter(|value| !value.is_empty()) + .or_else(|| { + dom.attribute(node, "placeholder") + .filter(|value| !value.is_empty()) + }) + .or_else(|| { + dom.attribute(node, "aria-label") + .filter(|value| !value.is_empty()) + }) + .or_else(|| { + dom.attribute(node, "title") + .filter(|value| !value.is_empty()) + }); + value.map(Cow::Borrowed) +} + +pub(crate) fn textarea_text<'a, D: Dom + ?Sized>( + dom: &'a D, + node: D::NodeId, +) -> Option> { + dom.attribute(node, "value").map(Cow::Borrowed) +} diff --git a/moli-html2md/src/lib.rs b/moli-html2md/src/lib.rs index 2562a5125da..cb1e3eb5547 100644 --- a/moli-html2md/src/lib.rs +++ b/moli-html2md/src/lib.rs @@ -1,6 +1,9 @@ +mod content; mod converter; mod dom; +mod form; mod machine; +mod media; mod options; mod output; mod table; @@ -16,3 +19,6 @@ extern crate self as moli_html2md; #[cfg(test)] mod copy_tests; + +#[doc(hidden)] +pub use media::{SrcsetCandidate, parse_srcset}; diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs index 62ff7cd9ee6..e28ace78b47 100644 --- a/moli-html2md/src/machine.rs +++ b/moli-html2md/src/machine.rs @@ -8,6 +8,8 @@ enum Task<'a, Id> { Visit(Id, usize), Children(Option, usize), Boundary, + ChoiceBoundary, + RestoreSvg(bool), PopStyle, EndLink(usize), EndQuote, @@ -38,6 +40,7 @@ struct Machine<'a, D: Dom + ?Sized> { tables: Vec>, raw: String, serial: usize, + in_svg: bool, } pub(crate) fn convert(dom: &D, root: D::NodeId, options: &Options) -> String { @@ -50,6 +53,7 @@ pub(crate) fn convert(dom: &D, root: D::NodeId, options: &Optio tables: Vec::new(), raw: String::new(), serial: 0, + in_svg: false, }; machine.run(); machine.take_writer().into_string() @@ -69,6 +73,8 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } Task::Children(None, _) | Task::RawChildren(None, _, _) => {} Task::Boundary => self.writer().boundary(2), + Task::ChoiceBoundary => self.writer().boundary(1), + Task::RestoreSvg(previous) => self.in_svg = previous, Task::PopStyle => self.writer().pop_style(), Task::EndLink(serial) => self.writer().end_link(serial), Task::EndQuote => { @@ -205,8 +211,45 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { if let Some(table) = self.tables.last_mut() { table.visit_element(tag); } + let tooltip = self + .dom + .attribute(node, "data-toggle") + .is_some_and(|value| value.eq_ignore_ascii_case("tooltip")); + let tooltip_fallback = + if tooltip && !subtree_has_content(self.dom, node, self.options.max_depth - depth) { + self.dom + .attribute(node, "data-original-title") + .filter(|value| !value.trim().is_empty()) + } else { + None + }; + if tooltip && self.dom.attribute(node, "title").is_some() { + self.writer().boundary(1); + self.tasks.push(Task::ChoiceBoundary); + } + if tag == "i" + && self.dom.first_child(node).is_none() + && self.dom.attribute(node, "class").is_some_and(|class| { + class.split_ascii_whitespace().any(|token| { + let token = token.to_ascii_lowercase(); + matches!( + token.as_str(), + "check" | "fa-check" | "icon-check" | "bi-check" + ) + }) + }) + { + self.writer().text("✓"); + return; + } match tag { "head" | "script" | "style" | "noscript" | "template" => return, + "title" if !self.in_svg => return, + "svg" | "foreignObject" | "foreignobject" => { + self.tasks.push(Task::RestoreSvg(self.in_svg)); + self.in_svg = tag == "svg"; + } + "br" => { self.writer().hard_break(); return; @@ -215,6 +258,19 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { self.writer().block("---".into(), 2, 2); return; } + "select" | "optgroup" | "option" | "button" | "time" => { + self.writer().boundary(1); + if matches!(tag, "optgroup" | "option") + && let Some(label) = self.dom.attribute(node, "label") + { + self.writer().text(label); + self.writer().boundary(1); + if tag == "option" { + return; + } + } + self.tasks.push(Task::ChoiceBoundary); + } "strong" | "b" => self.style(Style::Strong), "em" | "i" => self.style(Style::Emphasis), "del" | "s" | "strike" => self.style(Style::Strike), @@ -222,7 +278,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { if let Some(href) = self .dom .attribute(node, "href") - .filter(|href| !href.is_empty()) + .filter(|href| !href.is_empty() && crate::media::safe_url(href, false)) { self.serial += 1; let serial = self.serial; @@ -239,15 +295,36 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { "img" => { let alt = self.dom.attribute(node, "alt").unwrap_or_default(); let title = self.dom.attribute(node, "title"); - if let Some(src) = self - .dom - .attribute(node, "src") - .filter(|src| !src.is_empty()) - { - self.writer().image(alt, src, title); + if let Some(src) = crate::media::source(self.dom, node) { + self.writer().image(alt, &src, title); + } else { + // A failed or absent resource must not erase its textual + // alternative, especially when it is a link's only label. + self.writer().text(alt); + } + return; + } + "input" => { + if let Some(value) = crate::form::input_text(self.dom, node) { + self.writer().text(&value); } return; } + "textarea" => { + if let Some(value) = crate::form::textarea_text(self.dom, node) { + self.writer().text(&value); + return; + } + } + "iframe" | "video" | "audio" => { + if self.media(node, tag, depth) { + return; + } + if is_block(tag) { + self.writer().boundary(2); + self.tasks.push(Task::Boundary); + } + } "code" | "pre" => { self.raw.clear(); let end = if tag == "pre" { @@ -292,15 +369,61 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { self.writer().boundary(2); self.tasks.push(Task::Boundary); } + _ if quote_container(tag, self.dom.attribute(node, "class")) => { + self.capture(); + self.tasks.push(Task::EndQuote); + } _ if is_block(tag) => { self.writer().boundary(2); self.tasks.push(Task::Boundary); } _ => {} } + if let Some(value) = tooltip_fallback { + self.writer().text(value); + } else if let Some(label) = crate::content::fallback_text(self.dom, node) + && !subtree_has_content(self.dom, node, self.options.max_depth - depth) + { + self.writer().text(&label); + return; + } self.children(node, depth + 1); } + fn media(&mut self, node: D::NodeId, tag: &str, depth: usize) -> bool { + let urls = crate::media::sources(self.dom, node, tag, depth, self.options.max_depth); + if urls.is_empty() { + return false; + } + self.writer().boundary(2); + if tag == "video" + && let Some(poster) = self + .dom + .attribute(node, "poster") + .filter(|src| !src.is_empty() && crate::media::safe_url(src, true)) + { + self.writer().image("", poster, None); + self.writer().boundary(1); + } + let label = crate::media::label(self.dom, node, tag); + for href in urls { + self.serial += 1; + let serial = self.serial; + let opened = self.writer().push_style(Style::Link { + serial, + href, + title: None, + }); + self.writer().text(label); + if opened { + self.writer().end_link(serial); + } + self.writer().boundary(1); + } + self.writer().boundary(2); + true + } + fn table_cell(&mut self) { let table = self.tables.last_mut().expect("table conversion is active"); if let Some(cell) = table.next_cell() { @@ -449,6 +572,22 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } } +fn quote_container(tag: &str, class: Option<&str>) -> bool { + matches!(tag, "div" | "section" | "article" | "aside") + && class.is_some_and(|class| { + class.split_ascii_whitespace().any(|token| { + matches!( + token.to_ascii_lowercase().as_str(), + "quote" | "quoteblock" | "quote-block" | "quoted-text" + ) + }) + }) +} + +fn subtree_has_content(dom: &D, root: D::NodeId, limit: usize) -> bool { + crate::content::has_readable_content(dom, root, limit) +} + fn class_language(class: Option<&str>) -> Option<&str> { class? .split_ascii_whitespace() diff --git a/moli-html2md/src/media.rs b/moli-html2md/src/media.rs new file mode 100644 index 00000000000..84f9e7fdcd0 --- /dev/null +++ b/moli-html2md/src/media.rs @@ -0,0 +1,301 @@ +use std::borrow::Cow; + +use crate::{Dom, NodeKind}; + +/// Frozen DOMs can still carry lazy-loading placeholders. These conventional +/// attributes declare the resource the page intends to load, not the spacer. +pub(crate) fn source<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Option> { + // A responsive declaration determines the rendered resource even when a + // fallback `src` is present. Private lazy-load attributes remain fallback + // inputs and never override a valid standard declaration. + if let Some(value) = dom + .attribute(node, "srcset") + .and_then(|value| largest_srcset_candidate(value, true)) + { + return Some(value); + } + let src = direct_attribute(dom, node, "src", true); + if src.is_some_and(|value| !is_explicit_placeholder(value)) { + return src.map(Cow::Borrowed); + } + if let Some(value) = dom + .attribute(node, "data-srcset") + .and_then(|value| largest_srcset_candidate(value, true)) + { + return Some(value); + } + ["data-src", "data-lazy-src", "data-original"] + .into_iter() + .find_map(|name| direct_attribute(dom, node, name, true)) + .or(src) + .map(Cow::Borrowed) +} + +fn direct_source(dom: &D, node: D::NodeId, image: bool) -> Option<&str> { + let src = direct_attribute(dom, node, "src", image); + if src.is_some_and(is_explicit_placeholder) { + return direct_attribute(dom, node, "data-src", image) + .or_else(|| direct_attribute(dom, node, "data-lazy-src", image)) + .or_else(|| direct_attribute(dom, node, "data-original", image)) + .or(src); + } + src.or_else(|| direct_attribute(dom, node, "data-src", image)) + .or_else(|| direct_attribute(dom, node, "data-lazy-src", image)) + .or_else(|| direct_attribute(dom, node, "data-original", image)) +} + +fn direct_attribute<'a, D: Dom + ?Sized>( + dom: &'a D, + node: D::NodeId, + name: &str, + image: bool, +) -> Option<&'a str> { + dom.attribute(node, name).map(str::trim).filter(|value| { + !value.is_empty() + && !value.starts_with('#') + && *value != "about:blank" + && safe_url(value, image) + }) +} + +fn is_explicit_placeholder(value: &str) -> bool { + let lower = value.to_ascii_lowercase(); + let path = lower + .split(['?', '#']) + .next() + .unwrap_or(&lower) + .trim_end_matches('/'); + matches!( + path.rsplit('/').next().unwrap_or_default(), + "placeholder.gif" + | "placeholder.png" + | "spacer.gif" + | "spacer.png" + | "transparent.gif" + | "transparent.png" + | "blank.gif" + | "blank.png" + ) +} + +#[doc(hidden)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct SrcsetCandidate<'a> { + pub url: &'a str, + pub descriptor: &'a str, + pub score: f64, +} + +#[doc(hidden)] +pub fn parse_srcset(srcset: &str) -> Vec> { + let bytes = srcset.as_bytes(); + let mut offset = 0; + let mut candidates = Vec::new(); + while offset < bytes.len() { + while offset < bytes.len() && (bytes[offset].is_ascii_whitespace() || bytes[offset] == b',') + { + offset += 1; + } + let url_start = offset; + while offset < bytes.len() && !bytes[offset].is_ascii_whitespace() { + offset += 1; + } + if url_start == offset { + break; + } + let mut url_end = offset; + while url_end > url_start && bytes[url_end - 1] == b',' { + url_end -= 1; + } + let url = &srcset[url_start..url_end]; + let ended_with_comma = url_end < offset; + if ended_with_comma { + if !url.is_empty() { + candidates.push(SrcsetCandidate { + url, + descriptor: "", + score: 1.0, + }); + } + continue; + } + + let descriptor_start = offset; + let mut parentheses = 0usize; + while offset < bytes.len() { + match bytes[offset] { + b'(' => parentheses += 1, + b')' => parentheses = parentheses.saturating_sub(1), + b',' if parentheses == 0 => break, + _ => {} + } + offset += 1; + } + let descriptor = srcset[descriptor_start..offset].trim(); + let score = if descriptor.is_empty() { + Some(1.0) + } else { + descriptor_score(descriptor) + }; + if let Some(score) = score + && !url.is_empty() + { + candidates.push(SrcsetCandidate { + url, + descriptor, + score, + }); + } + } + candidates +} + +fn descriptor_score(descriptor: &str) -> Option { + let mut width = None; + let mut height = None; + let mut density = None; + for token in descriptor.split_ascii_whitespace() { + if let Some(value) = token.strip_suffix('w') { + if width.is_some() { + return None; + } + width = Some(valid_positive_integer(value)?); + } else if let Some(value) = token.strip_suffix('h') { + if height.is_some() { + return None; + } + height = Some(valid_positive_integer(value)?); + } else if let Some(value) = token.strip_suffix('x') { + if density.is_some() { + return None; + } + density = Some(valid_positive_float(value)?); + } else { + return None; + } + } + match (width, height, density) { + (Some(width), _, None) => Some(width as f64), + (None, None, Some(density)) => Some(density), + _ => None, + } +} + +fn valid_positive_integer(value: &str) -> Option { + (!value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| value.parse::().ok()) + .flatten() + .filter(|value| *value > 0) +} + +fn valid_positive_float(value: &str) -> Option { + let bytes = value.as_bytes(); + if bytes.is_empty() || bytes[0] == b'+' { + return None; + } + let mut offset = usize::from(bytes[0] == b'-'); + if offset == bytes.len() { + return None; + } + let integer_start = offset; + while offset < bytes.len() && bytes[offset].is_ascii_digit() { + offset += 1; + } + let has_integer = offset > integer_start; + let mut has_fraction = false; + if bytes.get(offset) == Some(&b'.') { + offset += 1; + let fraction = offset; + while offset < bytes.len() && bytes[offset].is_ascii_digit() { + offset += 1; + } + if fraction == offset { + return None; + } + has_fraction = true; + } + if !has_integer && !has_fraction { + return None; + } + if matches!(bytes.get(offset), Some(b'e' | b'E')) { + offset += 1; + if matches!(bytes.get(offset), Some(b'+' | b'-')) { + offset += 1; + } + let exponent = offset; + while offset < bytes.len() && bytes[offset].is_ascii_digit() { + offset += 1; + } + if exponent == offset { + return None; + } + } + (offset == bytes.len()) + .then(|| value.parse::().ok()) + .flatten() + .filter(|value| value.is_finite() && *value >= 0.0) +} + +fn largest_srcset_candidate(srcset: &str, image: bool) -> Option> { + parse_srcset(srcset) + .into_iter() + .filter(|candidate| safe_url(candidate.url, image)) + .max_by(|left, right| left.score.total_cmp(&right.score)) + .map(|candidate| Cow::Owned(candidate.url.to_owned())) +} + +pub(crate) fn safe_url(value: &str, image: bool) -> bool { + let normalized: String = value + .trim() + .chars() + .filter(|ch| !ch.is_ascii_control()) + .collect(); + let Some((scheme, _)) = normalized.split_once(':') else { + return true; + }; + if scheme.contains(['/', '#', '?']) { + return true; + } + matches!( + scheme.to_ascii_lowercase().as_str(), + "http" | "https" | "mailto" | "tel" | "ftp" + ) || (image && normalized.to_ascii_lowercase().starts_with("data:image/")) +} + +pub(crate) fn sources<'a, D: Dom + ?Sized>( + dom: &'a D, + node: D::NodeId, + tag: &str, + depth: usize, + limit: usize, +) -> Vec<&'a str> { + let mut urls = Vec::new(); + let mut seen = std::collections::HashSet::new(); + if let Some(src) = direct_source(dom, node, tag == "img") { + seen.insert(src); + urls.push(src); + } + if tag != "iframe" && depth + 1 < limit { + let mut child = dom.first_child(node); + while let Some(id) = child { + child = dom.next_sibling(id); + if dom.node_kind(id) == NodeKind::Element("source") + && let Some(src) = direct_source(dom, id, false) + && seen.insert(src) + { + urls.push(src); + } + } + } + urls +} + +pub(crate) fn label<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId, tag: &str) -> &'a str { + dom.attribute(node, "title") + .filter(|label| !label.trim().is_empty()) + .unwrap_or(match tag { + "video" => "Video", + "audio" => "Audio", + _ => "Embedded content", + }) +} diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rs index f5e12486a8b..b69ad3dbadd 100644 --- a/moli-html2md/tests/regressions.rs +++ b/moli-html2md/tests/regressions.rs @@ -1,6 +1,6 @@ mod support; -use moli_html2md::{Converter, Options}; +use moli_html2md::{Converter, Options, parse_srcset}; use support::{Tree, rendered_html}; fn markdown(html: &str, preformatted_code: bool) -> String { @@ -420,12 +420,17 @@ fn heading_attributes_keep_the_same_meaning_as_attributes_outside_headings() { #[test] fn images_without_sources_do_not_emit_placeholder_markdown() { - for html in [ - "ab", - "alabelb", - "alabelb", + assert_eq!(markdown("ab", false), "ab"); + for image in [ + "Email", + "Email", + "Email", ] { - assert_eq!(markdown(html, false), "ab"); + let source = format!("{image}"); + assert_eq!( + rendered_html(&markdown(&source, false)), + "

Email

\n" + ); } } @@ -488,3 +493,354 @@ fn fallback_block_elements_separate_neighboring_words() { ); } } + +#[test] +fn body_metadata_does_not_become_article_text() { + let result = markdown( + "

News

Actual story

Untitled templatePublisher
", + false, + ); + assert!(result.contains("Actual story")); + assert!(result.contains("Publisher")); + assert!(!result.contains("Untitled template"), "{result}"); + let graphic = markdown( + "Chart descriptionRevenueTemplate metadata", + false, + ); + assert!(graphic.contains("Chart description"), "{graphic}"); + assert!(graphic.contains("Revenue"), "{graphic}"); + assert!(!graphic.contains("Template metadata"), "{graphic}"); +} + +#[test] +fn form_choices_keep_option_and_group_boundaries() { + let result = markdown( + "", + false, + ); + assert!(result.contains("All posts\n"), "{result}"); + assert!(result.contains("Recent\n1 day\n1 week"), "{result}"); + assert!(!result.contains("Internal long label"), "{result}"); + assert_eq!(markdown("

website

", false), "website"); +} + +#[test] +fn embedded_media_retains_usable_sources() { + let result = markdown( + "

Watch the interview:

End.

", + false, + ); + for link in [ + "[Interview](https://video.example/embed/42)", + "(/movie.webm)", + "(/movie.mp4)", + "[Episode](/episode.ogg)", + "(/cover.jpg)", + ] { + assert!(result.contains(link), "{result}"); + } + assert_eq!(result.matches("(/movie.mp4)").count(), 1, "{result}"); + assert!(result.ends_with("End."), "{result}"); +} + +#[test] +fn responsive_image_without_src_retains_largest_candidate() { + assert_eq!( + markdown( + "Course cover", + false, + ), + "![Course cover](//cdn.example/cover-500.jpg)" + ); +} + +#[test] +fn concrete_loaded_image_wins_over_an_unexpanded_lazy_template() { + let result = markdown( + "Basin", + false, + ); + assert_eq!(result, "![Basin](https://cdn.example/basin_300x.jpg)"); +} + +#[test] +fn adjacent_controls_and_dates_remain_separate_text_items() { + let result = rendered_html(&markdown( + "

", + false, + )); + assert_eq!( + result, + "

Cancel\nSubscribe

\n

2024-01-02\n2025-03-04

\n" + ); +} + +#[test] +fn empty_accessible_element_retains_its_declared_text() { + let result = markdown( + "

By Reporter

", + false, + ); + assert!( + result.contains("Published: 1:16 p.m. Updated: 10:28 a.m."), + "{result}" + ); + assert!(result.contains("[Search](/search)"), "{result}"); +} + +#[test] +fn conventional_quote_container_retains_quote_semantics() { + let result = markdown( + "
Earlier statement.

Current reply.

", + false, + ); + assert!(result.contains("> Earlier statement."), "{result}"); + assert!(result.contains("Current reply."), "{result}"); +} + +#[test] +fn quote_classes_do_not_replace_list_or_table_semantics() { + let list = markdown( + "
  • First
  • Second
", + false, + ); + assert_eq!(list, "- First\n- Second"); + let table = rendered_html(&markdown( + "
Name
Ada
", + false, + )); + assert!(table.contains(""), "{table}"); + assert!(table.contains(""), "{table}"); +} + +#[test] +fn tooltip_trigger_retains_its_reader_facing_detail() { + let result = markdown( + "

Paid Orientation Details

", + false, + ); + assert_eq!(result, "Paid Orientation Details"); + let link = markdown( + "Download", + false, + ); + assert_eq!(link, "[Download](/download)"); + let fallback = markdown( + "", + false, + ); + assert_eq!(fallback, "$100/day"); + let fallback_link = markdown( + "", + false, + ); + assert_eq!(fallback_link, "[Download](/download)"); + let metadata = markdown( + "January 24, 20183 mins read", + false, + ); + assert_eq!(metadata, "January 24, 2018\n3 mins read"); +} + +#[test] +fn resource_only_headings_are_not_dropped() { + let result = markdown( + "

Article

", + false, + ); + assert!(result.contains("# [![](/logo.svg)](/home)"), "{result}"); + assert!(result.contains("## Accessible title"), "{result}"); + assert!(result.contains("Article"), "{result}"); + let alternative = markdown( + "

Product manual

Safe title

", + false, + ); + assert!(alternative.contains("# Product manual"), "{alternative}"); + assert!(alternative.contains("## Safe title"), "{alternative}"); +} + +#[test] +fn fallback_labels_preserve_all_children_and_remain_literal_text() { + let result = markdown( + "

", + false, + ); + assert!(result.contains("Save"), "{result}"); + assert!(result.contains("Delete"), "{result}"); + assert!(result.contains("# [One](/one)[Two](/two)"), "{result}"); + let rendered = rendered_html(&result); + assert!(!rendered.contains("href=\"/unintended\""), "{rendered}"); + assert!(rendered.contains("[Open](/unintended)"), "{rendered}"); +} + +#[test] +fn responsive_image_falls_back_from_unusable_lazy_candidates() { + for lazy in ["", "javascript:bad() 1x"] { + let result = markdown( + &format!( + "Chart" + ), + false, + ); + assert_eq!(result, "![Chart](/chart.png)"); + } +} + +#[test] +fn empty_check_icon_retains_positive_state() { + let result = markdown( + "
Image recognition
Speech recognition
", + false, + ); + assert!(result.contains("Image recognition ✓"), "{result}"); + assert!(result.contains("Speech recognition"), "{result}"); + assert_eq!(result.matches('✓').count(), 1, "{result}"); +} + +#[test] +fn form_values_preserve_current_readable_state_without_secrets() { + let source = "
Ada
PeriodMonSun
Morning

"; + let result = rendered_html(&markdown(source, false)); + for value in [ + "Morning", + "☑", + "☐", + "Search term", + "Your Email", + "Search", + "Map button", + "Password", + "Notes", + ] { + assert!(result.contains(value), "missing {value}: {result}"); + } + for secret in ["secret", "token", "private.pdf"] { + assert!(!result.contains(secret), "leaked {secret}: {result}"); + } +} + +#[test] +fn unsafe_resource_urls_keep_labels_without_emitting_active_links() { + let source = "

Open panelPosterEmail

"; + let result = markdown(source, false); + assert!(result.contains("Open panel"), "{result}"); + assert!(result.contains("Poster"), "{result}"); + assert!( + result.contains("[Email](mailto:a@example.test)"), + "{result}" + ); + assert!(!result.contains("javascript:"), "{result}"); +} + +#[test] +fn responsive_images_preserve_url_commas_and_use_declared_candidates() { + let comma_url = "https://cdn.example/c_fill,w_640,h_480/photo.jpg"; + for content in [ + format!("Photo"), + format!("Photo"), + format!( + "
Photo
" + ), + ] { + let result = markdown(&content, false); + assert!(result.contains(comma_url), "{result}"); + } + let result = markdown( + "Photo", + false, + ); + assert!(result.contains("/current.jpg"), "{result}"); + assert!(!result.contains("/old.jpg"), "{result}"); + let result = markdown( + "Photo", + false, + ); + assert!(result.contains("/large.jpg"), "{result}"); + for source in [ + "Photo", + "Photo", + ] { + let result = markdown(source, false); + assert!(!result.contains("/old.jpg"), "{result}"); + } + let defaults = markdown( + "PhotoSafe", + false, + ); + assert!(defaults.contains("![Photo](/large.jpg)"), "{defaults}"); + assert!(defaults.contains("![Safe](/safe.jpg)"), "{defaults}"); +} + +#[test] +fn srcset_rejects_invalid_numbers_and_commas_inside_parentheses() { + let result = markdown( + "DensityParentheses", + false, + ); + assert!(result.contains("![Density](/safe.jpg)"), "{result}"); + assert!( + result.contains("![Parentheses](/safe-parentheses.jpg)"), + "{result}" + ); + assert!(!result.contains("phantom.jpg"), "{result}"); +} + +#[test] +fn srcset_density_uses_the_html_floating_point_grammar() { + let result = markdown( + "FractionExponentZeroNegative zeroInvalid plusInvalid trailing dotNegative", + false, + ); + for expected in [ + "![Fraction](/fraction.jpg)", + "![Exponent](/exponent.jpg)", + "![Zero](/zero.jpg)", + "![Negative zero](/negative-zero.jpg)", + ] { + assert!(result.contains(expected), "missing {expected}: {result}"); + } + for rejected in ["invalid-plus.jpg", "invalid-dot.jpg", "negative.jpg"] { + assert!(!result.contains(rejected), "accepted {rejected}: {result}"); + } +} + +#[test] +fn srcset_recovers_only_the_future_compatible_width_height_pair() { + let paired = parse_srcset("/photo.jpg 640w 480h"); + assert_eq!(paired.len(), 1); + assert_eq!(paired[0].url, "/photo.jpg"); + assert_eq!(paired[0].score, 640.0); + + for invalid in [ + "/height-only.jpg 480h", + "/duplicate-width.jpg 640w 800w", + "/duplicate-height.jpg 640w 480h 720h", + "/density-height.jpg 2x 480h", + ] { + assert!(parse_srcset(invalid).is_empty(), "accepted {invalid}"); + } +} + +#[test] +fn lazy_media_uses_declared_resources_instead_of_spacers() { + let source = "OneTwo"; + let result = markdown(source, false); + for target in [ + "(/photo-one.jpg)", + "(/photo-two.jpg)", + "(//video.example/lecture)", + "(/ordinary.jpg)", + ] { + assert!(result.contains(target), "{result}"); + } + assert!(!result.contains("placeholder"), "{result}"); + let table = markdown( + "
Photo
", + false, + ); + assert!( + rendered_html(&table).contains("src=\"/actual.jpg\""), + "{table}" + ); + assert!(!table.contains("spacer.gif"), "{table}"); +} diff --git a/moli-html2md/tests/turndown/README.md b/moli-html2md/tests/turndown/README.md index 703252d8d2e..b986e0662e7 100644 --- a/moli-html2md/tests/turndown/README.md +++ b/moli-html2md/tests/turndown/README.md @@ -25,7 +25,7 @@ accepts equivalent delimiter choices and list indentation while detecting lost text, incorrect links, changed paragraph/list structure, and visible delimiters. There are also exact Markdown assertions in the original and regression suites. -`expectations.json` documents 12 deliberate differences with exact Moli outputs. +`expectations.json` documents 13 deliberate differences with exact Moli outputs. These preserve bare pre blocks and avoid upstream losses of literal characters, emphasis, whitespace, and list boundaries. They remain active assertions; no case is skipped. A reference change that makes a difference obsolete also fails @@ -36,7 +36,7 @@ rather than the plugin's raw HTML fallback. ## Findings tracked outside the reference corpus -The 12 exceptions above describe this corpus, not every design difference or +The 13 exceptions above describe this corpus, not every design difference or every correctness issue. [tracking.rs](../tracking.rs) records these open review findings with active assertions of both Markdown and rendered HTML: diff --git a/moli-html2md/tests/turndown/expectations.json b/moli-html2md/tests/turndown/expectations.json index 9a9593aad97..cc813ee462f 100644 --- a/moli-html2md/tests/turndown/expectations.json +++ b/moli-html2md/tests/turndown/expectations.json @@ -43,6 +43,10 @@ "markdown": "before` `after", "reason": "Explicit preformatted-code mode preserves the spaces inside code. Turndown removes a whitespace-only code element entirely." }, + "extra_missing_image_src": { + "markdown": "beforemissingafter", + "reason": "An unavailable image retains its existing text alternative. Dropping alt also removes the only accessible label from image-only navigation links; Turndown loses that text." + }, "extra_split_list_marker": { "markdown": "12\\. text", "reason": "Escape list markers across text-node boundaries. Turndown escapes nodes individually and accidentally creates an ordered list." From 597a0197af885f426471c95cac47e21264af760f Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:41:41 +0800 Subject: [PATCH 02/12] test(fetch): bind media semantics and freeze response metadata --- moli-html2md/README.md | 2 +- moli-renderer-v8/src/runtime/page_dump.rs | 22 +++++++++++++++++++++- moli/tests/fetch_cli/http_fail.rs | 8 +++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/moli-html2md/README.md b/moli-html2md/README.md index f5228366f8a..9fd30269b5d 100644 --- a/moli-html2md/README.md +++ b/moli-html2md/README.md @@ -100,7 +100,7 @@ strip/base/frame transformations remain the responsibility of the caller. whitespace, escaping, code, tables, depth limits, and 20,000 nested nodes on a 64 KiB thread stack. The [Turndown differential suite](tests/turndown/README.md) pins 147 upstream fixtures and 50 additional cases. It compares rendered meaning, -checks DOM immutability, and explicitly asserts 12 documented differences that +checks DOM immutability, and explicitly asserts 13 documented differences that preserve Moli's handling of literal text, code, emphasis, and list boundaries. Targeted regressions also cover empty elements, links, attributes in headings and tables, code whitespace and language names, list numbering, and loose lists. diff --git a/moli-renderer-v8/src/runtime/page_dump.rs b/moli-renderer-v8/src/runtime/page_dump.rs index ee2f639de6a..98e11792af5 100644 --- a/moli-renderer-v8/src/runtime/page_dump.rs +++ b/moli-renderer-v8/src/runtime/page_dump.rs @@ -500,7 +500,27 @@ mod tests { "

link

", "## [link](/a \"one two\")", ), - ("alabelblabelc", "abc"), + ] { + assert_eq!(markdown_from_html(html), expected, "{html}"); + } + } + + #[test] + fn markdown_renderer_retains_labels_for_unavailable_images() { + for (html, expected) in [ + ( + "alabelblabelc", + "alabelblabelc", + ), + ( + "Email", + "[Email](mailto:contact@example.test)", + ), + ( + "[label](javascript:alert(1))", + r"\[label\](javascript:alert(1))", + ), + ("abc", "abc"), ] { assert_eq!(markdown_from_html(html), expected, "{html}"); } diff --git a/moli/tests/fetch_cli/http_fail.rs b/moli/tests/fetch_cli/http_fail.rs index c01bb76179d..89b53b1043e 100644 --- a/moli/tests/fetch_cli/http_fail.rs +++ b/moli/tests/fetch_cli/http_fail.rs @@ -25,6 +25,7 @@ use super::subprocess::run_command_with_process_timeout; const HTML: &str = "HTTP fixture

HTTP fixture

response body

"; +const FIXTURE_DATE: &str = "Thu, 01 Jan 1970 00:00:00 GMT"; const RAW_BODY: &[u8] = b"\0\xffraw HTTP error body"; struct HttpFailureFixture { @@ -41,7 +42,7 @@ impl HttpFailureFixture { ( StatusCode::from_u16(status).unwrap(), // Output comparisons must not depend on request timing. - [("date", "Thu, 01 Jan 1970 00:00:00 GMT")], + [("date", FIXTURE_DATE)], Html(HTML), ) }), @@ -270,6 +271,11 @@ async fn fail_with_body_preserves_selected_output_format() -> Result<()> { let expected = server.run("/status/500", &args)?; assert_eq!(expected.status.code(), Some(0), "{expected:?}"); assert!(!expected.stdout.is_empty(), "{args:?}"); + if args == ["--dump", "json"] { + // Requests can cross a wall-clock second. All response facts, + // including Date, must stay controlled for this equality check. + std::thread::sleep(Duration::from_millis(1100)); + } let mut fail_args = args.clone(); fail_args.push("--fail-with-body"); let output = server.run("/status/500", &fail_args)?; From 65bf8e5c34ba5c6d7e59e8e28f01812a16fdfcf7 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:43:39 +0800 Subject: [PATCH 03/12] fix(html2md): avoid inferring content from private names --- moli-html2md/src/content.rs | 2 +- moli-html2md/src/machine.rs | 51 +--------------- moli-html2md/src/media.rs | 28 +-------- moli-html2md/tests/regressions.rs | 58 ++---------------- moli-html2md/tests/review_semantics.rs | 85 ++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 132 deletions(-) create mode 100644 moli-html2md/tests/review_semantics.rs diff --git a/moli-html2md/src/content.rs b/moli-html2md/src/content.rs index 4ebce0590f9..e99e7ee155f 100644 --- a/moli-html2md/src/content.rs +++ b/moli-html2md/src/content.rs @@ -54,7 +54,7 @@ pub(crate) fn has_readable_content( } pub(crate) fn fallback_text(dom: &D, node: D::NodeId) -> Option { - for name in ["aria-label", "data-original-title", "title"] { + for name in ["aria-label", "title"] { if let Some(value) = dom .attribute(node, name) .map(str::trim) diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs index e28ace78b47..ef0d12da3ad 100644 --- a/moli-html2md/src/machine.rs +++ b/moli-html2md/src/machine.rs @@ -211,37 +211,6 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { if let Some(table) = self.tables.last_mut() { table.visit_element(tag); } - let tooltip = self - .dom - .attribute(node, "data-toggle") - .is_some_and(|value| value.eq_ignore_ascii_case("tooltip")); - let tooltip_fallback = - if tooltip && !subtree_has_content(self.dom, node, self.options.max_depth - depth) { - self.dom - .attribute(node, "data-original-title") - .filter(|value| !value.trim().is_empty()) - } else { - None - }; - if tooltip && self.dom.attribute(node, "title").is_some() { - self.writer().boundary(1); - self.tasks.push(Task::ChoiceBoundary); - } - if tag == "i" - && self.dom.first_child(node).is_none() - && self.dom.attribute(node, "class").is_some_and(|class| { - class.split_ascii_whitespace().any(|token| { - let token = token.to_ascii_lowercase(); - matches!( - token.as_str(), - "check" | "fa-check" | "icon-check" | "bi-check" - ) - }) - }) - { - self.writer().text("✓"); - return; - } match tag { "head" | "script" | "style" | "noscript" | "template" => return, "title" if !self.in_svg => return, @@ -369,19 +338,13 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { self.writer().boundary(2); self.tasks.push(Task::Boundary); } - _ if quote_container(tag, self.dom.attribute(node, "class")) => { - self.capture(); - self.tasks.push(Task::EndQuote); - } _ if is_block(tag) => { self.writer().boundary(2); self.tasks.push(Task::Boundary); } _ => {} } - if let Some(value) = tooltip_fallback { - self.writer().text(value); - } else if let Some(label) = crate::content::fallback_text(self.dom, node) + if let Some(label) = crate::content::fallback_text(self.dom, node) && !subtree_has_content(self.dom, node, self.options.max_depth - depth) { self.writer().text(&label); @@ -572,18 +535,6 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } } -fn quote_container(tag: &str, class: Option<&str>) -> bool { - matches!(tag, "div" | "section" | "article" | "aside") - && class.is_some_and(|class| { - class.split_ascii_whitespace().any(|token| { - matches!( - token.to_ascii_lowercase().as_str(), - "quote" | "quoteblock" | "quote-block" | "quoted-text" - ) - }) - }) -} - fn subtree_has_content(dom: &D, root: D::NodeId, limit: usize) -> bool { crate::content::has_readable_content(dom, root, limit) } diff --git a/moli-html2md/src/media.rs b/moli-html2md/src/media.rs index 84f9e7fdcd0..33728bb2346 100644 --- a/moli-html2md/src/media.rs +++ b/moli-html2md/src/media.rs @@ -15,7 +15,7 @@ pub(crate) fn source<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Option return Some(value); } let src = direct_attribute(dom, node, "src", true); - if src.is_some_and(|value| !is_explicit_placeholder(value)) { + if src.is_some() { return src.map(Cow::Borrowed); } if let Some(value) = dom @@ -33,12 +33,6 @@ pub(crate) fn source<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Option fn direct_source(dom: &D, node: D::NodeId, image: bool) -> Option<&str> { let src = direct_attribute(dom, node, "src", image); - if src.is_some_and(is_explicit_placeholder) { - return direct_attribute(dom, node, "data-src", image) - .or_else(|| direct_attribute(dom, node, "data-lazy-src", image)) - .or_else(|| direct_attribute(dom, node, "data-original", image)) - .or(src); - } src.or_else(|| direct_attribute(dom, node, "data-src", image)) .or_else(|| direct_attribute(dom, node, "data-lazy-src", image)) .or_else(|| direct_attribute(dom, node, "data-original", image)) @@ -58,26 +52,6 @@ fn direct_attribute<'a, D: Dom + ?Sized>( }) } -fn is_explicit_placeholder(value: &str) -> bool { - let lower = value.to_ascii_lowercase(); - let path = lower - .split(['?', '#']) - .next() - .unwrap_or(&lower) - .trim_end_matches('/'); - matches!( - path.rsplit('/').next().unwrap_or_default(), - "placeholder.gif" - | "placeholder.png" - | "spacer.gif" - | "spacer.png" - | "transparent.gif" - | "transparent.png" - | "blank.gif" - | "blank.png" - ) -} - #[doc(hidden)] #[derive(Clone, Copy, Debug, PartialEq)] pub struct SrcsetCandidate<'a> { diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rs index b69ad3dbadd..ae881c2df2b 100644 --- a/moli-html2md/tests/regressions.rs +++ b/moli-html2md/tests/regressions.rs @@ -588,16 +588,6 @@ fn empty_accessible_element_retains_its_declared_text() { assert!(result.contains("[Search](/search)"), "{result}"); } -#[test] -fn conventional_quote_container_retains_quote_semantics() { - let result = markdown( - "
Earlier statement.

Current reply.

", - false, - ); - assert!(result.contains("> Earlier statement."), "{result}"); - assert!(result.contains("Current reply."), "{result}"); -} - #[test] fn quote_classes_do_not_replace_list_or_table_semantics() { let list = markdown( @@ -613,35 +603,6 @@ fn quote_classes_do_not_replace_list_or_table_semantics() { assert!(table.contains("Ada"), "{table}"); } -#[test] -fn tooltip_trigger_retains_its_reader_facing_detail() { - let result = markdown( - "

Paid Orientation Details

", - false, - ); - assert_eq!(result, "Paid Orientation Details"); - let link = markdown( - "Download", - false, - ); - assert_eq!(link, "[Download](/download)"); - let fallback = markdown( - "", - false, - ); - assert_eq!(fallback, "$100/day"); - let fallback_link = markdown( - "", - false, - ); - assert_eq!(fallback_link, "[Download](/download)"); - let metadata = markdown( - "January 24, 20183 mins read", - false, - ); - assert_eq!(metadata, "January 24, 2018\n3 mins read"); -} - #[test] fn resource_only_headings_are_not_dropped() { let result = markdown( @@ -686,17 +647,6 @@ fn responsive_image_falls_back_from_unusable_lazy_candidates() { } } -#[test] -fn empty_check_icon_retains_positive_state() { - let result = markdown( - "
Image recognition
Speech recognition
", - false, - ); - assert!(result.contains("Image recognition ✓"), "{result}"); - assert!(result.contains("Speech recognition"), "{result}"); - assert_eq!(result.matches('✓').count(), 1, "{result}"); -} - #[test] fn form_values_preserve_current_readable_state_without_secrets() { let source = "
PeriodMonSun
Morning

"; @@ -752,7 +702,7 @@ fn responsive_images_preserve_url_commas_and_use_declared_candidates() { assert!(result.contains("/current.jpg"), "{result}"); assert!(!result.contains("/old.jpg"), "{result}"); let result = markdown( - "Photo", + "Photo", false, ); assert!(result.contains("/large.jpg"), "{result}"); @@ -822,8 +772,8 @@ fn srcset_recovers_only_the_future_compatible_width_height_pair() { } #[test] -fn lazy_media_uses_declared_resources_instead_of_spacers() { - let source = "OneTwo"; +fn lazy_media_uses_declared_resources_when_standard_sources_are_unavailable() { + let source = "OneTwo"; let result = markdown(source, false); for target in [ "(/photo-one.jpg)", @@ -835,7 +785,7 @@ fn lazy_media_uses_declared_resources_instead_of_spacers() { } assert!(!result.contains("placeholder"), "{result}"); let table = markdown( - "
Photo
", + "
Photo
", false, ); assert!( diff --git a/moli-html2md/tests/review_semantics.rs b/moli-html2md/tests/review_semantics.rs new file mode 100644 index 00000000000..e5c2fa94e02 --- /dev/null +++ b/moli-html2md/tests/review_semantics.rs @@ -0,0 +1,85 @@ +mod support; + +use moli_html2md::convert; +use std::cell::Cell; +use support::{Tree, rendered_html}; + +fn markdown(html: &str) -> String { + let tree = Tree::parse(html); + convert(&tree, tree.root) +} + +#[test] +fn private_class_names_do_not_invent_document_semantics() { + let result = markdown( + "
Request a price quote
\ +
Actual quotation
\ + \ + ", + ); + let html = rendered_html(&result); + assert!(!result.contains('✓'), "{result}"); + assert!(!result.contains("Private plugin state"), "{result}"); + assert!(html.contains("

Request a price quote

"), "{html}"); + assert!(html.contains("
"), "{html}"); + assert!(result.contains("Actual quotation"), "{result}"); + assert!(result.contains("Save"), "{result}"); +} + +#[test] +fn standard_image_sources_are_not_overridden_by_filename_guesses() { + for source in [ + "/photos/placeholder.gif", + "/art/blank.png", + "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==", + ] { + let result = markdown(&format!( + "Original" + )); + assert!(result.contains(source), "{result}"); + assert!(!result.contains("private-alternative"), "{result}"); + } + let result = + markdown("Fallback"); + assert!(result.contains("declared-photo.jpg"), "{result}"); +} + +#[test] +fn nested_accessible_labels_do_not_rescan_the_whole_tree() { + struct Counted { + tree: Tree, + visits: Cell, + } + impl moli_html2md::Dom for Counted { + type NodeId = usize; + fn node_kind(&self, node: usize) -> moli_html2md::NodeKind<'_> { + self.visits.set(self.visits.get() + 1); + self.tree.node_kind(node) + } + fn first_child(&self, node: usize) -> Option { + self.tree.first_child(node) + } + fn next_sibling(&self, node: usize) -> Option { + self.tree.next_sibling(node) + } + fn attribute(&self, node: usize, name: &str) -> Option<&str> { + self.tree.attribute(node, name) + } + } + let count = |depth| { + let html = format!( + "{}Readable terminal{}", + "".repeat(depth), + "".repeat(depth), + ); + let dom = Counted { + tree: Tree::parse(&html), + visits: Cell::new(0), + }; + assert_eq!(convert(&dom, dom.tree.root), "Readable terminal"); + dom.visits.get() + }; + let small = count(100); + let large = count(400); + assert!(large <= small * 5, "{small} -> {large}"); +} From b7f7eeb9f7cdf0e8a42d4824b2ea5a311eafb5f3 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:02:07 +0800 Subject: [PATCH 04/12] fix(html2md): distinguish live textarea values from source attributes --- moli-html2md/src/dom.rs | 5 ++ moli-html2md/src/form.rs | 2 +- moli-html2md/tests/review_semantics.rs | 75 +++++++++++++++++++------- 3 files changed, 61 insertions(+), 21 deletions(-) diff --git a/moli-html2md/src/dom.rs b/moli-html2md/src/dom.rs index 583e68467fb..5ab891099c0 100644 --- a/moli-html2md/src/dom.rs +++ b/moli-html2md/src/dom.rs @@ -26,4 +26,9 @@ pub trait Dom { fn first_child(&self, node: Self::NodeId) -> Option; fn next_sibling(&self, node: Self::NodeId) -> Option; fn attribute(&self, node: Self::NodeId, name: &str) -> Option<&str>; + /// A live control value supplied by a browser adapter. None leaves source + /// textarea content authoritative; Some("") is an explicitly cleared value. + fn control_value(&self, _node: Self::NodeId) -> Option<&str> { + None + } } diff --git a/moli-html2md/src/form.rs b/moli-html2md/src/form.rs index 6beb52aa2fe..e62b3eb41d3 100644 --- a/moli-html2md/src/form.rs +++ b/moli-html2md/src/form.rs @@ -46,5 +46,5 @@ pub(crate) fn textarea_text<'a, D: Dom + ?Sized>( dom: &'a D, node: D::NodeId, ) -> Option> { - dom.attribute(node, "value").map(Cow::Borrowed) + dom.control_value(node).map(Cow::Borrowed) } diff --git a/moli-html2md/tests/review_semantics.rs b/moli-html2md/tests/review_semantics.rs index e5c2fa94e02..3191ca76424 100644 --- a/moli-html2md/tests/review_semantics.rs +++ b/moli-html2md/tests/review_semantics.rs @@ -4,6 +4,27 @@ use moli_html2md::convert; use std::cell::Cell; use support::{Tree, rendered_html}; +struct Counted { + tree: Tree, + visits: Cell, +} +impl moli_html2md::Dom for Counted { + type NodeId = usize; + fn node_kind(&self, node: usize) -> moli_html2md::NodeKind<'_> { + self.visits.set(self.visits.get() + 1); + self.tree.node_kind(node) + } + fn first_child(&self, node: usize) -> Option { + self.tree.first_child(node) + } + fn next_sibling(&self, node: usize) -> Option { + self.tree.next_sibling(node) + } + fn attribute(&self, node: usize, name: &str) -> Option<&str> { + self.tree.attribute(node, name) + } +} + fn markdown(html: &str) -> String { let tree = Tree::parse(html); convert(&tree, tree.root) @@ -46,26 +67,6 @@ fn standard_image_sources_are_not_overridden_by_filename_guesses() { #[test] fn nested_accessible_labels_do_not_rescan_the_whole_tree() { - struct Counted { - tree: Tree, - visits: Cell, - } - impl moli_html2md::Dom for Counted { - type NodeId = usize; - fn node_kind(&self, node: usize) -> moli_html2md::NodeKind<'_> { - self.visits.set(self.visits.get() + 1); - self.tree.node_kind(node) - } - fn first_child(&self, node: usize) -> Option { - self.tree.first_child(node) - } - fn next_sibling(&self, node: usize) -> Option { - self.tree.next_sibling(node) - } - fn attribute(&self, node: usize, name: &str) -> Option<&str> { - self.tree.attribute(node, name) - } - } let count = |depth| { let html = format!( "{}Readable terminal{}", @@ -83,3 +84,37 @@ fn nested_accessible_labels_do_not_rescan_the_whole_tree() { let large = count(400); assert!(large <= small * 5, "{small} -> {large}"); } + +#[test] +fn textarea_uses_its_content_not_a_nonstandard_value_attribute() { + for value in ["unrelated", ""] { + let result = markdown(&format!( + "
" + )); + assert!(result.contains("Authored text"), "{result}"); + assert!(!result.contains("unrelated"), "{result}"); + } +} + +#[test] +fn accessible_label_queries_remain_linear_with_large_unreadable_branches() { + let count = |depth| { + let branch = "".repeat(20); + let html = format!( + "{}{}{}", + format!("{branch}").repeat(depth), + branch, + "".repeat(depth) + ); + let dom = Counted { + tree: Tree::parse(&html), + visits: Cell::new(0), + }; + let result = convert(&dom, dom.tree.root); + assert_eq!(result.matches("Fallback").count(), 1, "{result}"); + dom.visits.get() + }; + let small = count(30); + let large = count(120); + assert!(large <= small * 5, "{small} -> {large}"); +} From 8266428a0baa22336e7eb3a9cca3e784e7dc0dac Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:41:36 +0800 Subject: [PATCH 05/12] fix(html2md): base accessible fallback on emitted child content --- moli-html2md/README.md | 8 ++ moli-html2md/src/content.rs | 111 +++++++++++++------------ moli-html2md/src/form.rs | 9 +- moli-html2md/src/machine.rs | 33 +++++--- moli-html2md/src/media.rs | 36 ++++++-- moli-html2md/src/writer.rs | 11 +++ moli-html2md/tests/regressions.rs | 19 ++--- moli-html2md/tests/review_semantics.rs | 97 ++++++++++++++++++++- moli/tests/fetch_cli/http_fail.rs | 8 +- 9 files changed, 235 insertions(+), 97 deletions(-) diff --git a/moli-html2md/README.md b/moli-html2md/README.md index 9fd30269b5d..334ba059f09 100644 --- a/moli-html2md/README.md +++ b/moli-html2md/README.md @@ -121,3 +121,11 @@ are tested in `moli-renderer-v8`. Unavailable images retain their text alternatives. Media references use standard source declarations before lazy-loading fallbacks. Empty accessible controls retain their labels; password values and hidden or file input values are omitted. + +Plain DOM attributes describe the supplied snapshot, including default form +values and checked state. Browser adapters can expose live attributes and +`control_value` instead. Placeholder hints are not field values. Empty named +controls may use accessible labels; generic containers do not acquire text +from tooltip titles or prohibited names. Child output takes precedence over +any fallback label. Links retain application protocols while executable +JavaScript and inline HTML document URLs are excluded. diff --git a/moli-html2md/src/content.rs b/moli-html2md/src/content.rs index e99e7ee155f..d0105117c26 100644 --- a/moli-html2md/src/content.rs +++ b/moli-html2md/src/content.rs @@ -1,59 +1,64 @@ use crate::{Dom, NodeKind}; -pub(crate) fn has_readable_content( - dom: &D, - root: D::NodeId, - limit: usize, -) -> bool { - let mut pending = vec![(root, 0)]; - while let Some((node, depth)) = pending.pop() { - if matches!( - dom.node_kind(node), - NodeKind::Element("head" | "title" | "script" | "style" | "noscript" | "template") - ) { - continue; - } - match dom.node_kind(node) { - NodeKind::Text(text) if !text.trim_matches(char::is_whitespace).is_empty() => { - return true; - } - NodeKind::Element("img") - if crate::media::source(dom, node).is_some() - || dom - .attribute(node, "alt") - .is_some_and(|alt| !alt.trim().is_empty()) => - { - return true; - } - NodeKind::Element("input") if crate::form::input_text(dom, node).is_some() => { - return true; - } - NodeKind::Element("textarea") => { - if let Some(value) = crate::form::textarea_text(dom, node) { - if !value.trim().is_empty() { - return true; - } - // A present live value is authoritative even when the user - // cleared it. Do not inspect serialized default children. - continue; - } - } - NodeKind::Element(_) if depth > 0 && fallback_text(dom, node).is_some() => return true, - _ => {} - } - if depth + 1 >= limit { - continue; - } - let mut child = dom.first_child(node); - while let Some(id) = child { - pending.push((id, depth + 1)); - child = dom.next_sibling(id); - } - } - false -} - pub(crate) fn fallback_text(dom: &D, node: D::NodeId) -> Option { + if dom + .attribute(node, "aria-hidden") + .is_some_and(|v| v.trim().eq_ignore_ascii_case("true")) + || dom.attribute(node, "hidden").is_some() + || dom.attribute(node, "role").is_some_and(|role| { + matches!( + role.split_ascii_whitespace() + .next() + .unwrap_or_default() + .to_ascii_lowercase() + .as_str(), + "none" | "presentation" + ) + }) + { + return None; + } + let named_tag = matches!( + dom.node_kind(node), + NodeKind::Element( + "a" | "button" + | "img" + | "svg" + | "input" + | "textarea" + | "select" + | "option" + | "optgroup" + | "h1" + | "h2" + | "h3" + | "h4" + | "h5" + | "h6" + ) + ); + let named_role = dom.attribute(node, "role").is_some_and(|role| { + matches!( + role.split_ascii_whitespace() + .next() + .unwrap_or_default() + .to_ascii_lowercase() + .as_str(), + "button" + | "link" + | "img" + | "heading" + | "checkbox" + | "radio" + | "textbox" + | "combobox" + | "switch" + | "tab" + ) + }); + if !named_tag && !named_role { + return None; + } for name in ["aria-label", "title"] { if let Some(value) = dom .attribute(node, name) diff --git a/moli-html2md/src/form.rs b/moli-html2md/src/form.rs index e62b3eb41d3..2907c3aee3c 100644 --- a/moli-html2md/src/form.rs +++ b/moli-html2md/src/form.rs @@ -24,13 +24,12 @@ pub(crate) fn input_text<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Op .map(Cow::Borrowed); } let value = (!kind.eq_ignore_ascii_case("password")) - .then(|| dom.attribute(node, "value")) + .then(|| { + dom.control_value(node) + .or_else(|| dom.attribute(node, "value")) + }) .flatten() .filter(|value| !value.is_empty()) - .or_else(|| { - dom.attribute(node, "placeholder") - .filter(|value| !value.is_empty()) - }) .or_else(|| { dom.attribute(node, "aria-label") .filter(|value| !value.is_empty()) diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs index ef0d12da3ad..575c87e24aa 100644 --- a/moli-html2md/src/machine.rs +++ b/moli-html2md/src/machine.rs @@ -9,6 +9,7 @@ enum Task<'a, Id> { Children(Option, usize), Boundary, ChoiceBoundary, + Fallback(String, usize), RestoreSvg(bool), PopStyle, EndLink(usize), @@ -74,6 +75,13 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { Task::Children(None, _) | Task::RawChildren(None, _, _) => {} Task::Boundary => self.writer().boundary(2), Task::ChoiceBoundary => self.writer().boundary(1), + Task::Fallback(label, before) => { + if self.writer().content_mark() == before { + self.writer().text(" "); + self.writer().text(&label); + self.writer().text(" "); + } + } Task::RestoreSvg(previous) => self.in_svg = previous, Task::PopStyle => self.writer().pop_style(), Task::EndLink(serial) => self.writer().end_link(serial), @@ -227,7 +235,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { self.writer().block("---".into(), 2, 2); return; } - "select" | "optgroup" | "option" | "button" | "time" => { + "select" | "optgroup" | "option" => { self.writer().boundary(1); if matches!(tag, "optgroup" | "option") && let Some(label) = self.dom.attribute(node, "label") @@ -247,7 +255,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { if let Some(href) = self .dom .attribute(node, "href") - .filter(|href| !href.is_empty() && crate::media::safe_url(href, false)) + .filter(|href| !href.is_empty() && crate::media::safe_link_url(href)) { self.serial += 1; let serial = self.serial; @@ -344,11 +352,9 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } _ => {} } - if let Some(label) = crate::content::fallback_text(self.dom, node) - && !subtree_has_content(self.dom, node, self.options.max_depth - depth) - { - self.writer().text(&label); - return; + if let Some(label) = crate::content::fallback_text(self.dom, node) { + let before = self.writer().content_mark(); + self.tasks.push(Task::Fallback(label, before)); } self.children(node, depth + 1); } @@ -365,7 +371,8 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { .attribute(node, "poster") .filter(|src| !src.is_empty() && crate::media::safe_url(src, true)) { - self.writer().image("", poster, None); + let poster_label = crate::media::poster_label(self.dom, node); + self.writer().image(&poster_label, poster, None); self.writer().boundary(1); } let label = crate::media::label(self.dom, node, tag); @@ -380,6 +387,12 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { self.writer().text(label); if opened { self.writer().end_link(serial); + } else { + // Markdown cannot nest a media link inside a surrounding link. + // Retain its destination as text rather than silently losing it. + self.writer().text(" ("); + self.writer().text(href); + self.writer().text(")"); } self.writer().boundary(1); } @@ -535,10 +548,6 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } } -fn subtree_has_content(dom: &D, root: D::NodeId, limit: usize) -> bool { - crate::content::has_readable_content(dom, root, limit) -} - fn class_language(class: Option<&str>) -> Option<&str> { class? .split_ascii_whitespace() diff --git a/moli-html2md/src/media.rs b/moli-html2md/src/media.rs index 33728bb2346..6a1b43ddfd5 100644 --- a/moli-html2md/src/media.rs +++ b/moli-html2md/src/media.rs @@ -2,6 +2,9 @@ use std::borrow::Cow; use crate::{Dom, NodeKind}; +/// Private lazy-loading conventions, used only when standard sources are absent. +const LAZY_SOURCE_ATTRIBUTES: [&str; 3] = ["data-src", "data-lazy-src", "data-original"]; + /// Frozen DOMs can still carry lazy-loading placeholders. These conventional /// attributes declare the resource the page intends to load, not the spacer. pub(crate) fn source<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Option> { @@ -24,18 +27,19 @@ pub(crate) fn source<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Option { return Some(value); } - ["data-src", "data-lazy-src", "data-original"] + LAZY_SOURCE_ATTRIBUTES .into_iter() .find_map(|name| direct_attribute(dom, node, name, true)) - .or(src) .map(Cow::Borrowed) } fn direct_source(dom: &D, node: D::NodeId, image: bool) -> Option<&str> { let src = direct_attribute(dom, node, "src", image); - src.or_else(|| direct_attribute(dom, node, "data-src", image)) - .or_else(|| direct_attribute(dom, node, "data-lazy-src", image)) - .or_else(|| direct_attribute(dom, node, "data-original", image)) + src.or_else(|| { + LAZY_SOURCE_ATTRIBUTES + .into_iter() + .find_map(|name| direct_attribute(dom, node, name, image)) + }) } fn direct_attribute<'a, D: Dom + ?Sized>( @@ -215,7 +219,7 @@ fn largest_srcset_candidate(srcset: &str, image: bool) -> Option> { .into_iter() .filter(|candidate| safe_url(candidate.url, image)) .max_by(|left, right| left.score.total_cmp(&right.score)) - .map(|candidate| Cow::Owned(candidate.url.to_owned())) + .map(|candidate| Cow::Borrowed(candidate.url)) } pub(crate) fn safe_url(value: &str, image: bool) -> bool { @@ -245,7 +249,7 @@ pub(crate) fn sources<'a, D: Dom + ?Sized>( ) -> Vec<&'a str> { let mut urls = Vec::new(); let mut seen = std::collections::HashSet::new(); - if let Some(src) = direct_source(dom, node, tag == "img") { + if let Some(src) = direct_source(dom, node, false) { seen.insert(src); urls.push(src); } @@ -273,3 +277,21 @@ pub(crate) fn label<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId, tag: &str) _ => "Embedded content", }) } + +/// Links preserve application protocols; executable inline-document schemes do not. +pub(crate) fn safe_link_url(value: &str) -> bool { + let normalized: String = value + .trim() + .chars() + .filter(|c| !c.is_ascii_control()) + .collect(); + let normalized = normalized.to_ascii_lowercase(); + !normalized.starts_with("javascript:") + && !normalized.starts_with("vbscript:") + && !normalized.starts_with("data:text/html") + && !normalized.starts_with("data:application/xhtml+xml") +} + +pub(crate) fn poster_label(dom: &D, node: D::NodeId) -> String { + format!("{} poster", label(dom, node, "video")) +} diff --git a/moli-html2md/src/writer.rs b/moli-html2md/src/writer.rs index 7417fdf72c5..18277faa810 100644 --- a/moli-html2md/src/writer.rs +++ b/moli-html2md/src/writer.rs @@ -26,6 +26,7 @@ struct OpenStyle<'a> { pub(crate) struct Writer<'a> { // Blocks are immutable fragments; inline edits only touch the local tail. prefix: Output, + content_revision: usize, output: String, desired: Vec>, emitted: Vec>, @@ -97,6 +98,7 @@ impl<'a> Writer<'a> { } pub(crate) fn text(&mut self, text: &str) { + self.content_revision += usize::from(text.chars().any(|c| !c.is_whitespace())); self.flush_code(); for ch in text.chars() { if is_space(ch) { @@ -153,6 +155,7 @@ impl<'a> Writer<'a> { } let text = text.trim_matches(is_space); if !text.is_empty() { + self.content_revision += 1; if self.code.is_none() { self.prepare_inline('`'); self.code = Some(String::new()); @@ -181,6 +184,7 @@ impl<'a> Writer<'a> { } self.flush_code(); if preformatted { + self.content_revision += 1; self.prepare_inline('`'); self.code = Some(text.replace("\r\n", " ").replace(['\n', '\r'], " ")); return; @@ -203,6 +207,7 @@ impl<'a> Writer<'a> { } pub(crate) fn image(&mut self, alt: &str, src: &str, title: Option<&str>) { + self.content_revision += 1; self.flush_code(); self.prepare_inline('!'); self.output.push_str("!["); @@ -236,6 +241,7 @@ impl<'a> Writer<'a> { } pub(crate) fn hard_break(&mut self) { + self.content_revision += 1; self.flush_code(); self.close_to(0, None); self.flush_spaces(false); @@ -250,6 +256,7 @@ impl<'a> Writer<'a> { self.boundary(before.max(after)); return; } + self.content_revision += 1; // Closing styles before detaching the tail keeps all inline edit // offsets local to the current String, never inside a finished block. self.boundary(before); @@ -272,6 +279,10 @@ impl<'a> Writer<'a> { self.prefix } + pub(crate) fn content_mark(&self) -> usize { + self.content_revision + } + fn is_empty(&self) -> bool { self.output.is_empty() && self.prefix.is_empty() } diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rs index ae881c2df2b..4c14c2ac043 100644 --- a/moli-html2md/tests/regressions.rs +++ b/moli-html2md/tests/regressions.rs @@ -564,14 +564,14 @@ fn concrete_loaded_image_wins_over_an_unexpanded_lazy_template() { } #[test] -fn adjacent_controls_and_dates_remain_separate_text_items() { +fn authored_spaces_between_controls_and_dates_remain_inline() { let result = rendered_html(&markdown( - "

", + "

", false, )); assert_eq!( result, - "

Cancel\nSubscribe

\n

2024-01-02\n2025-03-04

\n" + "

Cancel Subscribe

\n

2024-01-02 2025-03-04

\n" ); } @@ -581,10 +581,7 @@ fn empty_accessible_element_retains_its_declared_text() { "

By Reporter

", false, ); - assert!( - result.contains("Published: 1:16 p.m. Updated: 10:28 a.m."), - "{result}" - ); + assert!(!result.contains("Published:"), "{result}"); assert!(result.contains("[Search](/search)"), "{result}"); } @@ -606,7 +603,7 @@ fn quote_classes_do_not_replace_list_or_table_semantics() { #[test] fn resource_only_headings_are_not_dropped() { let result = markdown( - "

Article

", + "

Article

", false, ); assert!(result.contains("# [![](/logo.svg)](/home)"), "{result}"); @@ -628,7 +625,7 @@ fn fallback_labels_preserve_all_children_and_remain_literal_text() { ); assert!(result.contains("Save"), "{result}"); assert!(result.contains("Delete"), "{result}"); - assert!(result.contains("# [One](/one)[Two](/two)"), "{result}"); + assert!(result.contains("# [One](/one) [Two](/two)"), "{result}"); let rendered = rendered_html(&result); assert!(!rendered.contains("href=\"/unintended\""), "{rendered}"); assert!(rendered.contains("[Open](/unintended)"), "{rendered}"); @@ -648,7 +645,7 @@ fn responsive_image_falls_back_from_unusable_lazy_candidates() { } #[test] -fn form_values_preserve_current_readable_state_without_secrets() { +fn source_form_values_preserve_readable_state_without_secrets() { let source = "
PeriodMonSun
Morning

"; let result = rendered_html(&markdown(source, false)); for value in [ @@ -656,7 +653,6 @@ fn form_values_preserve_current_readable_state_without_secrets() { "☑", "☐", "Search term", - "Your Email", "Search", "Map button", "Password", @@ -664,6 +660,7 @@ fn form_values_preserve_current_readable_state_without_secrets() { ] { assert!(result.contains(value), "missing {value}: {result}"); } + assert!(!result.contains("Your Email"), "{result}"); for secret in ["secret", "token", "private.pdf"] { assert!(!result.contains(secret), "leaked {secret}: {result}"); } diff --git a/moli-html2md/tests/review_semantics.rs b/moli-html2md/tests/review_semantics.rs index 3191ca76424..377627ca676 100644 --- a/moli-html2md/tests/review_semantics.rs +++ b/moli-html2md/tests/review_semantics.rs @@ -70,7 +70,7 @@ fn nested_accessible_labels_do_not_rescan_the_whole_tree() { let count = |depth| { let html = format!( "{}Readable terminal{}", - "".repeat(depth), + "".repeat(depth), "".repeat(depth), ); let dom = Counted { @@ -102,7 +102,7 @@ fn accessible_label_queries_remain_linear_with_large_unreadable_branches() { let branch = "".repeat(20); let html = format!( "{}{}{}", - format!("{branch}").repeat(depth), + format!("{branch}").repeat(depth), branch, "".repeat(depth) ); @@ -118,3 +118,96 @@ fn accessible_label_queries_remain_linear_with_large_unreadable_branches() { let large = count(120); assert!(large <= small * 5, "{small} -> {large}"); } + +#[test] +fn labels_cannot_replace_media_choices_or_rules() { + for html in [ + "
", + "", + "", + ] { + let result = markdown(html); + assert!( + result.contains("demo.mp4") + || result.contains("example.test/movie") + || result.contains("demo.ogg"), + "{html}: {result}" + ); + } + let result = markdown( + "", + ); + assert!( + result.contains("Group") && result.contains("First"), + "{result}" + ); + let result = markdown(""); + assert!(result.contains("---"), "{result}"); +} + +#[test] +fn accessible_fallback_is_named_content_and_has_word_boundaries() { + let result = markdown( + "

xy

", + ); + assert!(result.contains("x Save y"), "{result}"); + assert!(result.contains("Download"), "{result}"); + for excluded in [ + "Pagination", + "Advertisement", + "tooltip", + "Hidden", + "Loading", + "private state", + ] { + assert!(!result.contains(excluded), "{result}"); + } +} + +#[test] +fn phrasing_controls_and_dates_keep_inline_styles_and_table_cells() { + let result = markdown( + "Posted Posted
DateAction
x ya b
", + ); + let html = rendered_html(&result); + assert!(html.contains("Posted today"), "{html}"); + assert!(html.contains("Posted Oct 1"), "{html}"); + assert!(!result.contains("
"), "{result}"); +} + +#[test] +fn input_hints_do_not_masquerade_as_filled_values() { + let result = markdown( + "", + ); + assert!(!result.contains("Your Email"), "{result}"); + assert!( + result.contains("Actual value") && result.contains("Password"), + "{result}" + ); + assert!(!result.contains("secret"), "{result}"); +} + +#[test] +fn links_retain_legitimate_non_web_destinations() { + for href in [ + "magnet:?xt=urn:btih:123", + "sms:+12025550123", + "geo:1,2", + "file:///tmp/manual.pdf", + "myapp:open/123", + "blob:https://example.test/id", + "data:application/pdf;base64,JVBERi0=", + ] { + let result = markdown(&format!("Open")); + assert!(result.contains(href), "{href}: {result}"); + } + for href in [ + "javascript:alert(1)", + "vbscript:run()", + "data:text/html,unsafe", + ] { + let result = markdown(&format!("Safe label")); + assert!(!result.contains(href), "{result}"); + } +} diff --git a/moli/tests/fetch_cli/http_fail.rs b/moli/tests/fetch_cli/http_fail.rs index 89b53b1043e..c01bb76179d 100644 --- a/moli/tests/fetch_cli/http_fail.rs +++ b/moli/tests/fetch_cli/http_fail.rs @@ -25,7 +25,6 @@ use super::subprocess::run_command_with_process_timeout; const HTML: &str = "HTTP fixture

HTTP fixture

response body

"; -const FIXTURE_DATE: &str = "Thu, 01 Jan 1970 00:00:00 GMT"; const RAW_BODY: &[u8] = b"\0\xffraw HTTP error body"; struct HttpFailureFixture { @@ -42,7 +41,7 @@ impl HttpFailureFixture { ( StatusCode::from_u16(status).unwrap(), // Output comparisons must not depend on request timing. - [("date", FIXTURE_DATE)], + [("date", "Thu, 01 Jan 1970 00:00:00 GMT")], Html(HTML), ) }), @@ -271,11 +270,6 @@ async fn fail_with_body_preserves_selected_output_format() -> Result<()> { let expected = server.run("/status/500", &args)?; assert_eq!(expected.status.code(), Some(0), "{expected:?}"); assert!(!expected.stdout.is_empty(), "{args:?}"); - if args == ["--dump", "json"] { - // Requests can cross a wall-clock second. All response facts, - // including Date, must stay controlled for this equality check. - std::thread::sleep(Duration::from_millis(1100)); - } let mut fail_args = args.clone(); fail_args.push("--fail-with-body"); let output = server.run("/status/500", &fail_args)?; From 064c0276ad8255711a5f44973c1c51f95bed6aa5 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Thu, 22 Oct 2026 13:59:35 +0800 Subject: [PATCH 06/12] fix(html2md): keep inline media flow and accessible alternatives --- moli-html2md/README.md | 3 + moli-html2md/src/content.rs | 69 +++--- moli-html2md/src/dom.rs | 5 +- moli-html2md/src/machine.rs | 47 ++-- moli-html2md/tests/regressions.rs | 4 +- moli-html2md/tests/review_semantics.rs | 93 ++++++++ moli-html2md/tests/turndown/expectations.json | 8 + .../src/native_bridge/element/images/src.rs | 220 ++---------------- 8 files changed, 187 insertions(+), 262 deletions(-) diff --git a/moli-html2md/README.md b/moli-html2md/README.md index 334ba059f09..9ea85936bd0 100644 --- a/moli-html2md/README.md +++ b/moli-html2md/README.md @@ -129,3 +129,6 @@ controls may use accessible labels; generic containers do not acquire text from tooltip titles or prohibited names. Child output takes precedence over any fallback label. Links retain application protocols while executable JavaScript and inline HTML document URLs are excluded. + +SVG title children are tooltip metadata, not body text. They supply a name only +when an SVG has no emitted content; title elements outside SVG are omitted. diff --git a/moli-html2md/src/content.rs b/moli-html2md/src/content.rs index d0105117c26..f28e30c6a7c 100644 --- a/moli-html2md/src/content.rs +++ b/moli-html2md/src/content.rs @@ -1,20 +1,12 @@ use crate::{Dom, NodeKind}; +use std::borrow::Cow; pub(crate) fn fallback_text(dom: &D, node: D::NodeId) -> Option { if dom .attribute(node, "aria-hidden") .is_some_and(|v| v.trim().eq_ignore_ascii_case("true")) || dom.attribute(node, "hidden").is_some() - || dom.attribute(node, "role").is_some_and(|role| { - matches!( - role.split_ascii_whitespace() - .next() - .unwrap_or_default() - .to_ascii_lowercase() - .as_str(), - "none" | "presentation" - ) - }) + || primary_role(dom, node).is_some_and(|role| role_matches(role, PRESENTATIONAL_ROLES)) { return None; } @@ -22,9 +14,7 @@ pub(crate) fn fallback_text(dom: &D, node: D::NodeId) -> Option dom.node_kind(node), NodeKind::Element( "a" | "button" - | "img" | "svg" - | "input" | "textarea" | "select" | "option" @@ -37,25 +27,8 @@ pub(crate) fn fallback_text(dom: &D, node: D::NodeId) -> Option | "h6" ) ); - let named_role = dom.attribute(node, "role").is_some_and(|role| { - matches!( - role.split_ascii_whitespace() - .next() - .unwrap_or_default() - .to_ascii_lowercase() - .as_str(), - "button" - | "link" - | "img" - | "heading" - | "checkbox" - | "radio" - | "textbox" - | "combobox" - | "switch" - | "tab" - ) - }); + let named_role = + primary_role(dom, node).is_some_and(|role| role_matches(role, NAMED_WIDGET_ROLES)); if !named_tag && !named_role { return None; } @@ -89,3 +62,37 @@ pub(crate) fn fallback_text(dom: &D, node: D::NodeId) -> Option } None } + +const PRESENTATIONAL_ROLES: &[&str] = &["none", "presentation"]; +const NAMED_WIDGET_ROLES: &[&str] = &[ + "button", "link", "img", "heading", "checkbox", "radio", "textbox", "combobox", "switch", "tab", +]; + +// This structural converter uses the first authored token as a conservative +// naming hint; it does not implement an accessibility-tree role resolver. +fn primary_role(dom: &D, node: D::NodeId) -> Option<&str> { + dom.attribute(node, "role")?.split_ascii_whitespace().next() +} +fn role_matches(role: &str, candidates: &[&str]) -> bool { + candidates + .iter() + .any(|candidate| role.eq_ignore_ascii_case(candidate)) +} + +pub(crate) fn image_label(dom: &D, node: D::NodeId) -> Cow<'_, str> { + if let Some(alt) = dom.attribute(node, "alt").filter(|alt| !alt.is_empty()) { + return Cow::Borrowed(alt); + } + if primary_role(dom, node).is_some_and(|role| role_matches(role, PRESENTATIONAL_ROLES)) { + return Cow::Borrowed(""); + } + ["aria-label", "title"] + .into_iter() + .find_map(|name| { + dom.attribute(node, name) + .map(str::trim) + .filter(|value| !value.is_empty()) + }) + .map(Cow::Borrowed) + .unwrap_or(Cow::Borrowed("")) +} diff --git a/moli-html2md/src/dom.rs b/moli-html2md/src/dom.rs index 5ab891099c0..43311c6f7fc 100644 --- a/moli-html2md/src/dom.rs +++ b/moli-html2md/src/dom.rs @@ -26,8 +26,9 @@ pub trait Dom { fn first_child(&self, node: Self::NodeId) -> Option; fn next_sibling(&self, node: Self::NodeId) -> Option; fn attribute(&self, node: Self::NodeId, name: &str) -> Option<&str>; - /// A live control value supplied by a browser adapter. None leaves source - /// textarea content authoritative; Some("") is an explicitly cleared value. + /// Current input/textarea value supplied by an adapter. None uses the + /// source default; Some("") suppresses that default after a clear. An input + /// may still retain its accessible name when its current value is empty. fn control_value(&self, _node: Self::NodeId) -> Option<&str> { None } diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs index 575c87e24aa..947b91c9924 100644 --- a/moli-html2md/src/machine.rs +++ b/moli-html2md/src/machine.rs @@ -10,7 +10,6 @@ enum Task<'a, Id> { Boundary, ChoiceBoundary, Fallback(String, usize), - RestoreSvg(bool), PopStyle, EndLink(usize), EndQuote, @@ -41,7 +40,6 @@ struct Machine<'a, D: Dom + ?Sized> { tables: Vec>, raw: String, serial: usize, - in_svg: bool, } pub(crate) fn convert(dom: &D, root: D::NodeId, options: &Options) -> String { @@ -54,7 +52,6 @@ pub(crate) fn convert(dom: &D, root: D::NodeId, options: &Optio tables: Vec::new(), raw: String::new(), serial: 0, - in_svg: false, }; machine.run(); machine.take_writer().into_string() @@ -82,7 +79,6 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { self.writer().text(" "); } } - Task::RestoreSvg(previous) => self.in_svg = previous, Task::PopStyle => self.writer().pop_style(), Task::EndLink(serial) => self.writer().end_link(serial), Task::EndQuote => { @@ -221,12 +217,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } match tag { "head" | "script" | "style" | "noscript" | "template" => return, - "title" if !self.in_svg => return, - "svg" | "foreignObject" | "foreignobject" => { - self.tasks.push(Task::RestoreSvg(self.in_svg)); - self.in_svg = tag == "svg"; - } - + "title" => return, "br" => { self.writer().hard_break(); return; @@ -270,14 +261,14 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } } "img" => { - let alt = self.dom.attribute(node, "alt").unwrap_or_default(); + let alt = crate::content::image_label(self.dom, node); let title = self.dom.attribute(node, "title"); if let Some(src) = crate::media::source(self.dom, node) { - self.writer().image(alt, &src, title); + self.writer().image(&alt, &src, title); } else { // A failed or absent resource must not erase its textual // alternative, especially when it is a link's only label. - self.writer().text(alt); + self.writer().text(&alt); } return; } @@ -361,19 +352,23 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { fn media(&mut self, node: D::NodeId, tag: &str, depth: usize) -> bool { let urls = crate::media::sources(self.dom, node, tag, depth, self.options.max_depth); - if urls.is_empty() { + let poster = (tag == "video") + .then(|| self.dom.attribute(node, "poster")) + .flatten() + .filter(|src| !src.is_empty() && crate::media::safe_url(src, true)); + if urls.is_empty() && poster.is_none() { return false; } - self.writer().boundary(2); - if tag == "video" - && let Some(poster) = self - .dom - .attribute(node, "poster") - .filter(|src| !src.is_empty() && crate::media::safe_url(src, true)) - { + let block = is_block(tag); + if block { + self.writer().boundary(2); + } else { + self.writer().text(" "); + } + if let Some(poster) = poster { let poster_label = crate::media::poster_label(self.dom, node); self.writer().image(&poster_label, poster, None); - self.writer().boundary(1); + self.writer().text(" "); } let label = crate::media::label(self.dom, node, tag); for href in urls { @@ -394,9 +389,13 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { self.writer().text(href); self.writer().text(")"); } - self.writer().boundary(1); + self.writer().text(" "); + } + if block { + self.writer().boundary(2); + } else { + self.writer().text(" "); } - self.writer().boundary(2); true } diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rs index 4c14c2ac043..29eaffe89bb 100644 --- a/moli-html2md/tests/regressions.rs +++ b/moli-html2md/tests/regressions.rs @@ -507,7 +507,7 @@ fn body_metadata_does_not_become_article_text() { "Chart descriptionRevenueTemplate metadata", false, ); - assert!(graphic.contains("Chart description"), "{graphic}"); + assert!(!graphic.contains("Chart description"), "{graphic}"); assert!(graphic.contains("Revenue"), "{graphic}"); assert!(!graphic.contains("Template metadata"), "{graphic}"); } @@ -576,7 +576,7 @@ fn authored_spaces_between_controls_and_dates_remain_inline() { } #[test] -fn empty_accessible_element_retains_its_declared_text() { +fn generic_container_labels_are_not_article_text_but_link_labels_are() { let result = markdown( "

By Reporter

", false, diff --git a/moli-html2md/tests/review_semantics.rs b/moli-html2md/tests/review_semantics.rs index 377627ca676..5bf0ab5408f 100644 --- a/moli-html2md/tests/review_semantics.rs +++ b/moli-html2md/tests/review_semantics.rs @@ -211,3 +211,96 @@ fn links_retain_legitimate_non_web_destinations() { assert!(!result.contains(href), "{result}"); } } + +#[test] +fn inline_media_keeps_links_emphasis_and_table_flow() { + for (source, expected) in [ + ( + "Watch ", + "[Watch Video (/v.mp4)](/w)", + ), + ( + "

See below

", + "See [Embedded content](/frame) below", + ), + ( + "Clip now", + "*Clip [Video](/a.mp4) now*", + ), + ] { + assert_eq!(markdown(source), expected); + } + let result = markdown("
Clip now
"); + assert!(result.contains("Clip [Video](/a.mp4) now"), "{result}"); + assert!(!result.contains("
"), "{result}"); +} + +#[test] +fn named_images_and_poster_only_video_keep_their_alternatives() { + for source in [ + "", + "", + ] { + assert!(markdown(source).contains("![Video poster](/cover.jpg)")); + } + assert_eq!( + markdown(""), + "[Home](/home)" + ); + assert_eq!( + markdown(""), + "![Logo](/logo.png)" + ); + assert_eq!( + markdown("Authored"), + "![Authored](/logo.png)" + ); + assert_eq!( + markdown("TwitterFollow us"), + "[Twitter Follow us](/t)" + ); + assert_eq!( + markdown("Chart descriptionRevenue"), + "Revenue" + ); +} + +#[test] +fn live_controls_replace_defaults_including_explicit_clears() { + struct Live(Tree); + impl moli_html2md::Dom for Live { + type NodeId = usize; + fn node_kind(&self, n: usize) -> moli_html2md::NodeKind<'_> { + self.0.node_kind(n) + } + fn first_child(&self, n: usize) -> Option { + self.0.first_child(n) + } + fn next_sibling(&self, n: usize) -> Option { + self.0.next_sibling(n) + } + fn attribute(&self, n: usize, k: &str) -> Option<&str> { + self.0.attribute(n, k) + } + fn control_value(&self, n: usize) -> Option<&str> { + self.0.attribute(n, "data-live") + } + } + for tag in ["input", "textarea"] { + for value in ["Current", ""] { + let html = if tag == "input" { + format!("") + } else { + format!("") + }; + let dom = Live(Tree::parse(&html)); + let result = convert(&dom, dom.0.root); + assert!(!result.contains("Stale"), "{result}"); + if !value.is_empty() { + assert_eq!(result, "Current"); + } else { + assert_eq!(result, if tag == "input" { "Field" } else { "" }); + } + } + } +} diff --git a/moli-html2md/tests/turndown/expectations.json b/moli-html2md/tests/turndown/expectations.json index cc813ee462f..2944b9671bf 100644 --- a/moli-html2md/tests/turndown/expectations.json +++ b/moli-html2md/tests/turndown/expectations.json @@ -50,5 +50,13 @@ "extra_split_list_marker": { "markdown": "12\\. text", "reason": "Escape list markers across text-node boundaries. Turndown escapes nodes individually and accidentally creates an ordered list." + }, + "upstream_036": { + "markdown": "![the\ntitle](logo.png \"the\ntitle\")", + "reason": "An image without alt uses its authored title as a text alternative. Turndown leaves that label empty." + }, + "upstream_037": { + "markdown": "![\"hello\"](logo.png \"\\\"hello\\\"\")", + "reason": "An image without alt uses its authored title as a text alternative while retaining the resource title." } } diff --git a/moli-renderer-v8/src/native_bridge/element/images/src.rs b/moli-renderer-v8/src/native_bridge/element/images/src.rs index 45353fd831e..a26a2c08fa6 100644 --- a/moli-renderer-v8/src/native_bridge/element/images/src.rs +++ b/moli-renderer-v8/src/native_bridge/element/images/src.rs @@ -673,13 +673,6 @@ fn connected_empty_src_without_srcset(runtime: &JsContextHost, handle: DomHandle }) } -#[derive(Clone, Copy, Debug)] -enum SrcsetDescriptorState { - InDescriptor, - InParens, - AfterDescriptor, -} - #[derive(Clone, Copy, Debug, Default, PartialEq)] struct SrcsetDescriptor { width: Option, @@ -781,145 +774,25 @@ fn image_selection_viewport(runtime: &JsContextHost, handle: DomHandle) -> Image } fn parse_srcset_candidates(input: &str) -> Vec { - let mut index = 0usize; - let mut candidates = Vec::new(); - while index < input.len() { - index += input[index..] - .find(|ch| !is_html_space_or_comma(ch)) - .unwrap_or(input.len() - index); - if index >= input.len() { - break; - } - - let url_start = index; - index += input[index..] - .find(is_html_space) - .unwrap_or(input.len() - index); - let mut url = &input[url_start..index]; - if url.ends_with(',') { - url = url.trim_end_matches(','); - if !url.is_empty() { - candidates.push(SrcsetCandidate { - url: url.to_owned(), - descriptor: SrcsetDescriptor::default(), - }); - } - continue; - } - - index += input[index..] - .find(|ch| !is_html_space(ch)) - .unwrap_or(input.len() - index); - let (descriptors, next_index) = tokenize_srcset_descriptors(input, index); - index = next_index; - if let Some(descriptor) = parse_srcset_descriptors(&descriptors) { - candidates.push(SrcsetCandidate { - url: url.to_owned(), - descriptor, - }); - } - } - candidates -} - -fn tokenize_srcset_descriptors(input: &str, mut index: usize) -> (Vec, usize) { - let mut descriptors = Vec::new(); - let mut current = String::new(); - let mut state = SrcsetDescriptorState::InDescriptor; - while index < input.len() { - let ch = input[index..].chars().next().expect("valid char boundary"); - match state { - SrcsetDescriptorState::InDescriptor if is_html_space(ch) => { - if !current.is_empty() { - descriptors.push(std::mem::take(&mut current)); - state = SrcsetDescriptorState::AfterDescriptor; - } - index += ch.len_utf8(); - } - SrcsetDescriptorState::InDescriptor if ch == ',' => { - if !current.is_empty() { - descriptors.push(current); - } - index += ch.len_utf8(); - return (descriptors, index); - } - SrcsetDescriptorState::InDescriptor if ch == '(' => { - current.push(ch); - state = SrcsetDescriptorState::InParens; - index += ch.len_utf8(); - } - SrcsetDescriptorState::InDescriptor => { - current.push(ch); - index += ch.len_utf8(); - } - SrcsetDescriptorState::InParens if ch == ')' => { - current.push(ch); - state = SrcsetDescriptorState::InDescriptor; - index += ch.len_utf8(); - } - SrcsetDescriptorState::InParens => { - current.push(ch); - index += ch.len_utf8(); - } - SrcsetDescriptorState::AfterDescriptor if is_html_space(ch) => { - index += ch.len_utf8(); - } - SrcsetDescriptorState::AfterDescriptor => { - state = SrcsetDescriptorState::InDescriptor; - } - } - } - if !current.is_empty() { - descriptors.push(current); - } - (descriptors, index) -} - -fn parse_srcset_descriptors(descriptors: &[String]) -> Option { - let mut width = None; - let mut density = None; - let mut future_compat_h = None; - - for descriptor in descriptors { - let last = descriptor.chars().last()?; - let value = &descriptor[..descriptor.len() - last.len_utf8()]; - match last { - 'w' if is_valid_non_negative_integer(value) && width.is_none() && density.is_none() => { - let parsed = value.parse::().ok()?; - if parsed == 0 { - return None; - } - width = Some(parsed); - } - 'x' if is_valid_floating_point_number(value) - && width.is_none() - && density.is_none() - && future_compat_h.is_none() => - { - let parsed = value.parse::().ok()?; - if !parsed.is_finite() || parsed < 0.0 { - return None; - } - density = Some(parsed); - } - 'h' if is_valid_non_negative_integer(value) - && future_compat_h.is_none() - && density.is_none() => - { - let parsed = value.parse::().ok()?; - if parsed == 0 { - return None; + // Tokenization and descriptor validation are shared with document export. + // Selection retains its existing typed representation and viewport policy. + moli_html2md::parse_srcset(input) + .into_iter() + .filter_map(|candidate| { + let mut descriptor = SrcsetDescriptor::default(); + for token in candidate.descriptor.split_ascii_whitespace() { + if let Some(value) = token.strip_suffix('w') { + descriptor.width = Some(value.parse().ok()?); + } else if let Some(value) = token.strip_suffix('x') { + descriptor.density = Some(value.parse().ok()?); } - future_compat_h = Some(parsed); } - _ => return None, - } - } - - if future_compat_h.is_some() && width.is_none() { - return None; - } - Some(SrcsetDescriptor { width, density }) + Some(SrcsetCandidate { + url: candidate.url.to_owned(), + descriptor, + }) + }) + .collect() } fn source_size_for_sizes_attribute( @@ -1147,65 +1020,6 @@ fn inline_style_value<'a>(style: &'a str, property: &str) -> Option<&'a str> { }) } -fn is_html_space_or_comma(ch: char) -> bool { - ch == ',' || is_html_space(ch) -} - -fn is_html_space(ch: char) -> bool { - matches!( - ch, - '\u{0009}' | '\u{000a}' | '\u{000c}' | '\u{000d}' | '\u{0020}' - ) -} - -fn is_valid_non_negative_integer(value: &str) -> bool { - !value.is_empty() && value.chars().all(|ch| ch.is_ascii_digit()) -} - -fn is_valid_floating_point_number(value: &str) -> bool { - let bytes = value.as_bytes(); - let mut index = 0usize; - if bytes.get(index) == Some(&b'-') { - index += 1; - } - let integer_start = index; - while bytes.get(index).is_some_and(u8::is_ascii_digit) { - index += 1; - } - let integer_digits = index - integer_start; - let mut fraction_digits = 0usize; - let mut saw_decimal_point = false; - if bytes.get(index) == Some(&b'.') { - saw_decimal_point = true; - index += 1; - let fraction_start = index; - while bytes.get(index).is_some_and(u8::is_ascii_digit) { - index += 1; - } - fraction_digits = index - fraction_start; - } - if integer_digits == 0 && fraction_digits == 0 { - return false; - } - if saw_decimal_point && fraction_digits == 0 { - return false; - } - if matches!(bytes.get(index), Some(b'e' | b'E')) { - index += 1; - if matches!(bytes.get(index), Some(b'+' | b'-')) { - index += 1; - } - let exponent_start = index; - while bytes.get(index).is_some_and(u8::is_ascii_digit) { - index += 1; - } - if exponent_start == index { - return false; - } - } - index == bytes.len() -} - pub(in crate::native_bridge) fn image_current_src_getter_function<'s>( scope: &mut v8::PinScope<'s, '_>, args: v8::FunctionCallbackArguments<'s>, From e541e6115d7ec14494e4a37f2266fc55e0308b5e Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Thu, 22 Oct 2026 14:59:34 +0800 Subject: [PATCH 07/12] fix(html2md): preserve resource fallbacks and distinguish empty fields --- moli-html2md/README.md | 2 +- moli-html2md/src/content.rs | 32 ++++++++----- moli-html2md/src/dom.rs | 4 +- moli-html2md/src/form.rs | 11 ++++- moli-html2md/src/lib.rs | 1 - moli-html2md/src/machine.rs | 3 +- moli-html2md/src/media.rs | 47 ++++++++++++------- moli-html2md/tests/regressions.rs | 7 +-- moli-html2md/tests/review_semantics.rs | 25 +++++++++- .../src/native_bridge/element/images/src.rs | 22 +++++---- 10 files changed, 105 insertions(+), 49 deletions(-) diff --git a/moli-html2md/README.md b/moli-html2md/README.md index 9ea85936bd0..ca620073f13 100644 --- a/moli-html2md/README.md +++ b/moli-html2md/README.md @@ -124,7 +124,7 @@ their labels; password values and hidden or file input values are omitted. Plain DOM attributes describe the supplied snapshot, including default form values and checked state. Browser adapters can expose live attributes and -`control_value` instead. Placeholder hints are not field values. Empty named +`control_value` instead. Placeholder hints are not field values. Empty text-entry controls emit no value; button-like controls may use accessible labels; generic containers do not acquire text from tooltip titles or prohibited names. Child output takes precedence over any fallback label. Links retain application protocols while executable diff --git a/moli-html2md/src/content.rs b/moli-html2md/src/content.rs index f28e30c6a7c..e5e5241dd2c 100644 --- a/moli-html2md/src/content.rs +++ b/moli-html2md/src/content.rs @@ -80,19 +80,25 @@ fn role_matches(role: &str, candidates: &[&str]) -> bool { } pub(crate) fn image_label(dom: &D, node: D::NodeId) -> Cow<'_, str> { - if let Some(alt) = dom.attribute(node, "alt").filter(|alt| !alt.is_empty()) { - return Cow::Borrowed(alt); - } - if primary_role(dom, node).is_some_and(|role| role_matches(role, PRESENTATIONAL_ROLES)) { + if dom.attribute(node, "hidden").is_some() + || dom + .attribute(node, "aria-hidden") + .is_some_and(|v| v.eq_ignore_ascii_case("true")) + || primary_role(dom, node).is_some_and(|role| role_matches(role, PRESENTATIONAL_ROLES)) + { return Cow::Borrowed(""); } - ["aria-label", "title"] - .into_iter() - .find_map(|name| { - dom.attribute(node, name) - .map(str::trim) - .filter(|value| !value.is_empty()) - }) - .map(Cow::Borrowed) - .unwrap_or(Cow::Borrowed("")) + match dom.attribute(node, "alt") { + Some(alt) if !alt.is_empty() => Cow::Borrowed(alt), + Some(_) => Cow::Borrowed(dom.attribute(node, "aria-label").unwrap_or("")), + None => ["aria-label", "title"] + .into_iter() + .find_map(|name| { + dom.attribute(node, name) + .map(str::trim) + .filter(|v| !v.is_empty()) + }) + .map(Cow::Borrowed) + .unwrap_or(Cow::Borrowed("")), + } } diff --git a/moli-html2md/src/dom.rs b/moli-html2md/src/dom.rs index 43311c6f7fc..15a86903a37 100644 --- a/moli-html2md/src/dom.rs +++ b/moli-html2md/src/dom.rs @@ -27,8 +27,8 @@ pub trait Dom { fn next_sibling(&self, node: Self::NodeId) -> Option; fn attribute(&self, node: Self::NodeId, name: &str) -> Option<&str>; /// Current input/textarea value supplied by an adapter. None uses the - /// source default; Some("") suppresses that default after a clear. An input - /// may still retain its accessible name when its current value is empty. + /// source default; Some("") suppresses that default after a clear. Empty + /// text-entry controls do not emit their name as a field value. fn control_value(&self, _node: Self::NodeId) -> Option<&str> { None } diff --git a/moli-html2md/src/form.rs b/moli-html2md/src/form.rs index 2907c3aee3c..0553504c489 100644 --- a/moli-html2md/src/form.rs +++ b/moli-html2md/src/form.rs @@ -23,6 +23,9 @@ pub(crate) fn input_text<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Op .filter(|value| !value.is_empty()) .map(Cow::Borrowed); } + let button = ["submit", "button", "reset"] + .iter() + .any(|v| kind.eq_ignore_ascii_case(v)); let value = (!kind.eq_ignore_ascii_case("password")) .then(|| { dom.control_value(node) @@ -31,11 +34,15 @@ pub(crate) fn input_text<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Op .flatten() .filter(|value| !value.is_empty()) .or_else(|| { - dom.attribute(node, "aria-label") + button + .then(|| dom.attribute(node, "aria-label")) + .flatten() .filter(|value| !value.is_empty()) }) .or_else(|| { - dom.attribute(node, "title") + button + .then(|| dom.attribute(node, "title")) + .flatten() .filter(|value| !value.is_empty()) }); value.map(Cow::Borrowed) diff --git a/moli-html2md/src/lib.rs b/moli-html2md/src/lib.rs index cb1e3eb5547..7c835cd837b 100644 --- a/moli-html2md/src/lib.rs +++ b/moli-html2md/src/lib.rs @@ -20,5 +20,4 @@ extern crate self as moli_html2md; #[cfg(test)] mod copy_tests; -#[doc(hidden)] pub use media::{SrcsetCandidate, parse_srcset}; diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs index 947b91c9924..7ebfddf8f49 100644 --- a/moli-html2md/src/machine.rs +++ b/moli-html2md/src/machine.rs @@ -371,6 +371,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { self.writer().text(" "); } let label = crate::media::label(self.dom, node, tag); + let has_source = !urls.is_empty(); for href in urls { self.serial += 1; let serial = self.serial; @@ -396,7 +397,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } else { self.writer().text(" "); } - true + has_source } fn table_cell(&mut self) { diff --git a/moli-html2md/src/media.rs b/moli-html2md/src/media.rs index 6a1b43ddfd5..5955c9db95c 100644 --- a/moli-html2md/src/media.rs +++ b/moli-html2md/src/media.rs @@ -56,15 +56,19 @@ fn direct_attribute<'a, D: Dom + ?Sized>( }) } -#[doc(hidden)] +/// A validated HTML responsive-image candidate. Dimensions are descriptor +/// values, not an image-selection policy; consumers choose using their viewport. #[derive(Clone, Copy, Debug, PartialEq)] pub struct SrcsetCandidate<'a> { pub url: &'a str, pub descriptor: &'a str, - pub score: f64, + pub width: Option, + pub height: Option, + pub density: Option, } -#[doc(hidden)] +/// Parse HTML srcset syntax, discarding malformed candidates. +/// Non-ASCII spaces are resource/descriptor characters, not separators. pub fn parse_srcset(srcset: &str) -> Vec> { let bytes = srcset.as_bytes(); let mut offset = 0; @@ -92,7 +96,9 @@ pub fn parse_srcset(srcset: &str) -> Vec> { candidates.push(SrcsetCandidate { url, descriptor: "", - score: 1.0, + width: None, + height: None, + density: None, }); } continue; @@ -109,26 +115,27 @@ pub fn parse_srcset(srcset: &str) -> Vec> { } offset += 1; } - let descriptor = srcset[descriptor_start..offset].trim(); - let score = if descriptor.is_empty() { - Some(1.0) - } else { - descriptor_score(descriptor) - }; - if let Some(score) = score + let descriptor = srcset[descriptor_start..offset].trim_ascii(); + let parsed = descriptor_values(descriptor); + if let Some((width, height, density)) = parsed && !url.is_empty() { candidates.push(SrcsetCandidate { url, descriptor, - score, + width, + height, + density, }); } } candidates } -fn descriptor_score(descriptor: &str) -> Option { +fn descriptor_values(descriptor: &str) -> Option<(Option, Option, Option)> { + if descriptor.is_empty() { + return Some((None, None, None)); + } let mut width = None; let mut height = None; let mut density = None; @@ -153,8 +160,7 @@ fn descriptor_score(descriptor: &str) -> Option { } } match (width, height, density) { - (Some(width), _, None) => Some(width as f64), - (None, None, Some(density)) => Some(density), + (Some(_), _, None) | (None, None, Some(_)) => Some((width, height, density)), _ => None, } } @@ -218,7 +224,16 @@ fn largest_srcset_candidate(srcset: &str, image: bool) -> Option> { parse_srcset(srcset) .into_iter() .filter(|candidate| safe_url(candidate.url, image)) - .max_by(|left, right| left.score.total_cmp(&right.score)) + .max_by(|left, right| { + let score = |candidate: &SrcsetCandidate<'_>| { + candidate + .width + .map(|v| v as f64) + .or(candidate.density) + .unwrap_or(1.0) + }; + score(left).total_cmp(&score(right)) + }) .map(|candidate| Cow::Borrowed(candidate.url)) } diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rs index 29eaffe89bb..734eca2d6a3 100644 --- a/moli-html2md/tests/regressions.rs +++ b/moli-html2md/tests/regressions.rs @@ -655,13 +655,12 @@ fn source_form_values_preserve_readable_state_without_secrets() { "Search term", "Search", "Map button", - "Password", "Notes", ] { assert!(result.contains(value), "missing {value}: {result}"); } assert!(!result.contains("Your Email"), "{result}"); - for secret in ["secret", "token", "private.pdf"] { + for secret in ["secret", "Password", "token", "private.pdf"] { assert!(!result.contains(secret), "leaked {secret}: {result}"); } } @@ -756,10 +755,12 @@ fn srcset_recovers_only_the_future_compatible_width_height_pair() { let paired = parse_srcset("/photo.jpg 640w 480h"); assert_eq!(paired.len(), 1); assert_eq!(paired[0].url, "/photo.jpg"); - assert_eq!(paired[0].score, 640.0); + assert_eq!(paired[0].width, Some(640)); for invalid in [ "/height-only.jpg 480h", + "/non-ascii.jpg \u{3000}2x", + "/nbsp.jpg 2x\u{a0}", "/duplicate-width.jpg 640w 800w", "/duplicate-height.jpg 640w 480h 720h", "/density-height.jpg 2x 480h", diff --git a/moli-html2md/tests/review_semantics.rs b/moli-html2md/tests/review_semantics.rs index 5bf0ab5408f..30a50986410 100644 --- a/moli-html2md/tests/review_semantics.rs +++ b/moli-html2md/tests/review_semantics.rs @@ -182,7 +182,7 @@ fn input_hints_do_not_masquerade_as_filled_values() { ); assert!(!result.contains("Your Email"), "{result}"); assert!( - result.contains("Actual value") && result.contains("Password"), + result.contains("Actual value") && !result.contains("Password"), "{result}" ); assert!(!result.contains("secret"), "{result}"); @@ -299,8 +299,29 @@ fn live_controls_replace_defaults_including_explicit_clears() { if !value.is_empty() { assert_eq!(result, "Current"); } else { - assert_eq!(result, if tag == "input" { "Field" } else { "" }); + assert_eq!(result, ""); } } } } + +#[test] +fn media_fallback_and_decorative_images_keep_author_intent() { + for src in ["", "blob:https://site/id"] { + let output = markdown(&format!( + "" + )); + assert!(output.contains("![Video poster](/cover.jpg)"), "{output}"); + assert!(output.contains("[Download](/clip.mp4)"), "{output}"); + } + assert_eq!(markdown(""), ""); + assert_eq!( + markdown(""), + "![](/thumb.jpg \"Gallery\")" + ); + assert_eq!(markdown(""), ""); + assert_eq!( + markdown(""), + "Search" + ); +} diff --git a/moli-renderer-v8/src/native_bridge/element/images/src.rs b/moli-renderer-v8/src/native_bridge/element/images/src.rs index a26a2c08fa6..d78f77ec280 100644 --- a/moli-renderer-v8/src/native_bridge/element/images/src.rs +++ b/moli-renderer-v8/src/native_bridge/element/images/src.rs @@ -779,14 +779,10 @@ fn parse_srcset_candidates(input: &str) -> Vec { moli_html2md::parse_srcset(input) .into_iter() .filter_map(|candidate| { - let mut descriptor = SrcsetDescriptor::default(); - for token in candidate.descriptor.split_ascii_whitespace() { - if let Some(value) = token.strip_suffix('w') { - descriptor.width = Some(value.parse().ok()?); - } else if let Some(value) = token.strip_suffix('x') { - descriptor.density = Some(value.parse().ok()?); - } - } + let descriptor = SrcsetDescriptor { + width: candidate.width.map(u32::try_from).transpose().ok()?, + density: candidate.density, + }; Some(SrcsetCandidate { url: candidate.url.to_owned(), descriptor, @@ -1079,6 +1075,16 @@ mod tests { .map(|candidate| candidate.url) } + #[test] + fn srcset_parser_rejects_unicode_descriptor_spaces() { + for input in ["a.png \u{3000}2x, b.png 1x", "a.png 2x\u{a0}, b.png 1x"] { + let candidates = parse_srcset_candidates(input); + assert_eq!(candidates.len(), 1, "{input}"); + assert_eq!(candidates[0].url, "b.png"); + assert_eq!(candidates[0].descriptor.density, Some(1.0)); + } + } + #[test] fn srcset_parser_uses_html_ascii_whitespace() { assert_eq!( From 0fcaace7ba49372d880aa2dc5175278d1719e323 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:14:46 +0800 Subject: [PATCH 08/12] fix(html2md): ignore whitespace-only video posters --- moli-html2md/src/machine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs index 7ebfddf8f49..5dc020e11da 100644 --- a/moli-html2md/src/machine.rs +++ b/moli-html2md/src/machine.rs @@ -355,7 +355,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { let poster = (tag == "video") .then(|| self.dom.attribute(node, "poster")) .flatten() - .filter(|src| !src.is_empty() && crate::media::safe_url(src, true)); + .filter(|src| !src.trim().is_empty() && crate::media::safe_url(src, true)); if urls.is_empty() && poster.is_none() { return false; } From 04406976d40d4b5fdc71bc8f7255e7e5f23bebe2 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Thu, 22 Oct 2026 16:27:45 +0800 Subject: [PATCH 09/12] fix(renderer): preserve valid media alternatives and srcset candidates --- moli-html2md/README.md | 7 +-- moli-html2md/src/content.rs | 51 ++++++++++--------- moli-html2md/src/machine.rs | 5 +- moli-html2md/src/media.rs | 10 ++-- moli-html2md/tests/regressions.rs | 4 +- moli-html2md/tests/review_semantics.rs | 36 +++++++++++++ .../src/native_bridge/element/images/src.rs | 7 +++ 7 files changed, 83 insertions(+), 37 deletions(-) diff --git a/moli-html2md/README.md b/moli-html2md/README.md index ca620073f13..cf7ee6fe394 100644 --- a/moli-html2md/README.md +++ b/moli-html2md/README.md @@ -119,12 +119,13 @@ neither Node nor network access. NativeDom integration and real HTML fixtures are tested in `moli-renderer-v8`. Unavailable images retain their text alternatives. Media references use standard -source declarations before lazy-loading fallbacks. Empty accessible controls retain -their labels; password values and hidden or file input values are omitted. +source declarations before lazy-loading fallbacks. Password values and hidden or +file input values are omitted. Plain DOM attributes describe the supplied snapshot, including default form values and checked state. Browser adapters can expose live attributes and -`control_value` instead. Placeholder hints are not field values. Empty text-entry controls emit no value; button-like +`control_value` instead. Placeholder hints are not field values. +Empty text-entry controls emit no value; button-like controls may use accessible labels; generic containers do not acquire text from tooltip titles or prohibited names. Child output takes precedence over any fallback label. Links retain application protocols while executable diff --git a/moli-html2md/src/content.rs b/moli-html2md/src/content.rs index e5e5241dd2c..b37cbece8ee 100644 --- a/moli-html2md/src/content.rs +++ b/moli-html2md/src/content.rs @@ -2,12 +2,7 @@ use crate::{Dom, NodeKind}; use std::borrow::Cow; pub(crate) fn fallback_text(dom: &D, node: D::NodeId) -> Option { - if dom - .attribute(node, "aria-hidden") - .is_some_and(|v| v.trim().eq_ignore_ascii_case("true")) - || dom.attribute(node, "hidden").is_some() - || primary_role(dom, node).is_some_and(|role| role_matches(role, PRESENTATIONAL_ROLES)) - { + if excluded_from_naming(dom, node) { return None; } let named_tag = matches!( @@ -79,26 +74,34 @@ fn role_matches(role: &str, candidates: &[&str]) -> bool { .any(|candidate| role.eq_ignore_ascii_case(candidate)) } -pub(crate) fn image_label(dom: &D, node: D::NodeId) -> Cow<'_, str> { - if dom.attribute(node, "hidden").is_some() - || dom - .attribute(node, "aria-hidden") - .is_some_and(|v| v.eq_ignore_ascii_case("true")) +pub(crate) fn excluded_from_naming(dom: &D, node: D::NodeId) -> bool { + if dom + .attribute(node, "aria-hidden") + .is_some_and(|v| v.trim().eq_ignore_ascii_case("true")) + || dom.attribute(node, "hidden").is_some() || primary_role(dom, node).is_some_and(|role| role_matches(role, PRESENTATIONAL_ROLES)) { - return Cow::Borrowed(""); + return true; + } + false +} + +pub(crate) fn image_label(dom: &D, node: D::NodeId) -> Cow<'_, str> { + let alt = dom.attribute(node, "alt"); + if let Some(alt) = alt.filter(|alt| !alt.is_empty()) { + return Cow::Borrowed(alt); } - match dom.attribute(node, "alt") { - Some(alt) if !alt.is_empty() => Cow::Borrowed(alt), - Some(_) => Cow::Borrowed(dom.attribute(node, "aria-label").unwrap_or("")), - None => ["aria-label", "title"] - .into_iter() - .find_map(|name| { - dom.attribute(node, name) - .map(str::trim) - .filter(|v| !v.is_empty()) - }) - .map(Cow::Borrowed) - .unwrap_or(Cow::Borrowed("")), + if excluded_from_naming(dom, node) { + return Cow::Borrowed(""); } + ["aria-label", "title"] + .into_iter() + .filter(|name| *name != "title" || alt.is_none()) + .find_map(|name| { + dom.attribute(node, name) + .map(str::trim) + .filter(|v| !v.is_empty()) + }) + .map(Cow::Borrowed) + .unwrap_or(Cow::Borrowed("")) } diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs index 5dc020e11da..0abe1c24b3d 100644 --- a/moli-html2md/src/machine.rs +++ b/moli-html2md/src/machine.rs @@ -353,9 +353,8 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { fn media(&mut self, node: D::NodeId, tag: &str, depth: usize) -> bool { let urls = crate::media::sources(self.dom, node, tag, depth, self.options.max_depth); let poster = (tag == "video") - .then(|| self.dom.attribute(node, "poster")) - .flatten() - .filter(|src| !src.trim().is_empty() && crate::media::safe_url(src, true)); + .then(|| crate::media::direct_attribute(self.dom, node, "poster", true)) + .flatten(); if urls.is_empty() && poster.is_none() { return false; } diff --git a/moli-html2md/src/media.rs b/moli-html2md/src/media.rs index 5955c9db95c..a16003863fc 100644 --- a/moli-html2md/src/media.rs +++ b/moli-html2md/src/media.rs @@ -42,7 +42,7 @@ fn direct_source(dom: &D, node: D::NodeId, image: bool) -> Opti }) } -fn direct_attribute<'a, D: Dom + ?Sized>( +pub(crate) fn direct_attribute<'a, D: Dom + ?Sized>( dom: &'a D, node: D::NodeId, name: &str, @@ -105,12 +105,12 @@ pub fn parse_srcset(srcset: &str) -> Vec> { } let descriptor_start = offset; - let mut parentheses = 0usize; + let mut in_parentheses = false; while offset < bytes.len() { match bytes[offset] { - b'(' => parentheses += 1, - b')' => parentheses = parentheses.saturating_sub(1), - b',' if parentheses == 0 => break, + b'(' => in_parentheses = true, + b')' => in_parentheses = false, + b',' if !in_parentheses => break, _ => {} } offset += 1; diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rs index 734eca2d6a3..cb64f749ae8 100644 --- a/moli-html2md/tests/regressions.rs +++ b/moli-html2md/tests/regressions.rs @@ -646,14 +646,14 @@ fn responsive_image_falls_back_from_unusable_lazy_candidates() { #[test] fn source_form_values_preserve_readable_state_without_secrets() { - let source = "
PeriodMonSun
Morning

"; + let source = "
PeriodMonSun
Morning

"; let result = rendered_html(&markdown(source, false)); for value in [ "Morning", "☑", "☐", "Search term", - "Search", + "Go", "Map button", "Notes", ] { diff --git a/moli-html2md/tests/review_semantics.rs b/moli-html2md/tests/review_semantics.rs index 30a50986410..b882b2025ba 100644 --- a/moli-html2md/tests/review_semantics.rs +++ b/moli-html2md/tests/review_semantics.rs @@ -325,3 +325,39 @@ fn media_fallback_and_decorative_images_keep_author_intent() { "Search" ); } + +#[test] +fn authored_image_alternatives_survive_naming_exclusions() { + for attribute in ["hidden", "aria-hidden=' true '", "role='presentation'"] { + assert_eq!( + markdown(&format!( + "Revenue chart" + )), + "![Revenue chart](/c.png)" + ); + assert_eq!( + markdown(&format!( + "" + )), + "![](/c.png)" + ); + } + assert_eq!( + markdown(""), + "![](/c.png)" + ); + assert_eq!( + markdown(""), + "![Video poster](/cover.jpg)" + ); + for poster in ["#cover", "about:blank"] { + assert_eq!( + markdown(&format!("")), + "Fallback" + ); + } + assert_eq!( + markdown(""), + "![](b.png)" + ); +} diff --git a/moli-renderer-v8/src/native_bridge/element/images/src.rs b/moli-renderer-v8/src/native_bridge/element/images/src.rs index d78f77ec280..13cdf162d74 100644 --- a/moli-renderer-v8/src/native_bridge/element/images/src.rs +++ b/moli-renderer-v8/src/native_bridge/element/images/src.rs @@ -1075,6 +1075,13 @@ mod tests { .map(|candidate| candidate.url) } + #[test] + fn srcset_parentheses_do_not_nest() { + let candidates = parse_srcset_candidates("a.png ((x), b.png 2x"); + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].url, "b.png"); + } + #[test] fn srcset_parser_rejects_unicode_descriptor_spaces() { for input in ["a.png \u{3000}2x, b.png 1x", "a.png 2x\u{a0}, b.png 1x"] { From 7ae57a1cc8a0e4436038facfcc499e2301ffa00b Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:23:11 +0800 Subject: [PATCH 10/12] refactor(html2md): simplify private naming exclusion predicate --- moli-html2md/src/content.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/moli-html2md/src/content.rs b/moli-html2md/src/content.rs index b37cbece8ee..2b52c378098 100644 --- a/moli-html2md/src/content.rs +++ b/moli-html2md/src/content.rs @@ -74,16 +74,11 @@ fn role_matches(role: &str, candidates: &[&str]) -> bool { .any(|candidate| role.eq_ignore_ascii_case(candidate)) } -pub(crate) fn excluded_from_naming(dom: &D, node: D::NodeId) -> bool { - if dom - .attribute(node, "aria-hidden") +fn excluded_from_naming(dom: &D, node: D::NodeId) -> bool { + dom.attribute(node, "aria-hidden") .is_some_and(|v| v.trim().eq_ignore_ascii_case("true")) || dom.attribute(node, "hidden").is_some() || primary_role(dom, node).is_some_and(|role| role_matches(role, PRESENTATIONAL_ROLES)) - { - return true; - } - false } pub(crate) fn image_label(dom: &D, node: D::NodeId) -> Cow<'_, str> { From f5b8c708a936dea21d50649eb576818a533647b0 Mon Sep 17 00:00:00 2001 From: ldm0 Date: Fri, 23 Oct 2026 18:54:39 +0800 Subject: [PATCH 11/12] fix(html2md): retain option text and constrain name fallbacks --- moli-html2md/src/content.rs | 10 ++-- moli-html2md/src/machine.rs | 13 +++-- moli-html2md/tests/review_semantics.rs | 74 +++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 8 deletions(-) diff --git a/moli-html2md/src/content.rs b/moli-html2md/src/content.rs index 2b52c378098..5ec7a13c09b 100644 --- a/moli-html2md/src/content.rs +++ b/moli-html2md/src/content.rs @@ -1,7 +1,11 @@ use crate::{Dom, NodeKind}; use std::borrow::Cow; -pub(crate) fn fallback_text(dom: &D, node: D::NodeId) -> Option { +pub(crate) fn fallback_text( + dom: &D, + node: D::NodeId, + remaining_depth: usize, +) -> Option { if excluded_from_naming(dom, node) { return None; } @@ -10,7 +14,6 @@ pub(crate) fn fallback_text(dom: &D, node: D::NodeId) -> Option NodeKind::Element( "a" | "button" | "svg" - | "textarea" | "select" | "option" | "optgroup" @@ -36,7 +39,8 @@ pub(crate) fn fallback_text(dom: &D, node: D::NodeId) -> Option return Some(value.to_owned()); } } - if matches!(dom.node_kind(node), NodeKind::Element("svg")) { + // Title text is two levels below SVG and obeys the same depth limit as body text. + if matches!(dom.node_kind(node), NodeKind::Element("svg")) && remaining_depth > 2 { let mut child = dom.first_child(node); while let Some(id) = child { if dom.node_kind(id) == NodeKind::Element("title") { diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs index 0abe1c24b3d..2d24eb5c836 100644 --- a/moli-html2md/src/machine.rs +++ b/moli-html2md/src/machine.rs @@ -229,7 +229,10 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { "select" | "optgroup" | "option" => { self.writer().boundary(1); if matches!(tag, "optgroup" | "option") - && let Some(label) = self.dom.attribute(node, "label") + && let Some(label) = self + .dom + .attribute(node, "label") + .filter(|label| !label.is_empty()) { self.writer().text(label); self.writer().boundary(1); @@ -281,8 +284,10 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { "textarea" => { if let Some(value) = crate::form::textarea_text(self.dom, node) { self.writer().text(&value); - return; + } else { + self.children(node, depth + 1); } + return; } "iframe" | "video" | "audio" => { if self.media(node, tag, depth) { @@ -343,7 +348,9 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } _ => {} } - if let Some(label) = crate::content::fallback_text(self.dom, node) { + if let Some(label) = + crate::content::fallback_text(self.dom, node, self.options.max_depth - depth) + { let before = self.writer().content_mark(); self.tasks.push(Task::Fallback(label, before)); } diff --git a/moli-html2md/tests/review_semantics.rs b/moli-html2md/tests/review_semantics.rs index b882b2025ba..b4c0a29a8dd 100644 --- a/moli-html2md/tests/review_semantics.rs +++ b/moli-html2md/tests/review_semantics.rs @@ -1,6 +1,6 @@ mod support; -use moli_html2md::convert; +use moli_html2md::{Converter, Dom, Options, convert}; use std::cell::Cell; use support::{Tree, rendered_html}; @@ -96,6 +96,36 @@ fn textarea_uses_its_content_not_a_nonstandard_value_attribute() { } } +#[test] +fn empty_option_labels_keep_their_text_and_choice_boundaries() { + assert_eq!( + markdown( + "" + ), + "First choice\nPublic label\nThird choice" + ); +} + +#[test] +fn empty_default_textareas_do_not_emit_accessible_names() { + for attributes in [ + "aria-label='Comment'", + "title='Comment'", + "aria-label='Comment' role='textbox'", + ] { + for content in ["", " \n\t"] { + let html = format!(""); + assert_eq!(markdown(&html), "", "{html}"); + } + let html = format!(""); + assert_eq!(markdown(&html), "Authored text", "{html}"); + } + assert_eq!( + markdown(""), + "Comment" + ); +} + #[test] fn accessible_label_queries_remain_linear_with_large_unreadable_branches() { let count = |depth| { @@ -265,6 +295,44 @@ fn named_images_and_poster_only_video_keep_their_alternatives() { ); } +#[test] +fn svg_title_fallback_obeys_the_conversion_depth_limit() { + let dom = Tree::parse("Chart title"); + let svg = dom.first_child(dom.root).unwrap(); + for (root, svg_depth) in [(svg, 0), (dom.root, 1)] { + for max_depth in [0, 1, 2, 3, 4, usize::MAX] { + let converter = Converter::new(Options { + max_depth, + ..Default::default() + }); + let expected = if max_depth > svg_depth + 2 { + "Chart title" + } else { + "" + }; + assert_eq!( + converter.convert(&dom, root), + expected, + "svg_depth={svg_depth}, max_depth={max_depth}" + ); + } + } + for attribute in ["aria-label", "title"] { + let dom = Tree::parse(&format!( + "Child title" + )); + let svg = dom.first_child(dom.root).unwrap(); + assert_eq!( + Converter::new(Options { + max_depth: 1, + ..Default::default() + }) + .convert(&dom, svg), + "Authored name" + ); + } +} + #[test] fn live_controls_replace_defaults_including_explicit_clears() { struct Live(Tree); @@ -291,7 +359,9 @@ fn live_controls_replace_defaults_including_explicit_clears() { let html = if tag == "input" { format!("") } else { - format!("") + format!( + "" + ) }; let dom = Live(Tree::parse(&html)); let result = convert(&dom, dom.0.root); From d7ff25c4de6bf6d0527941e702e7da0c99c7c9e2 Mon Sep 17 00:00:00 2001 From: ldm0 Date: Fri, 23 Oct 2026 19:22:30 +0800 Subject: [PATCH 12/12] refactor(html2md): simplify media and form conversion --- moli-html2md/src/content.rs | 10 +++---- moli-html2md/src/form.rs | 59 +++++++++++++------------------------ moli-html2md/src/machine.rs | 26 ++++++---------- moli-html2md/src/media.rs | 57 +++++++++++++---------------------- 4 files changed, 54 insertions(+), 98 deletions(-) diff --git a/moli-html2md/src/content.rs b/moli-html2md/src/content.rs index 5ec7a13c09b..0be97431f72 100644 --- a/moli-html2md/src/content.rs +++ b/moli-html2md/src/content.rs @@ -1,5 +1,4 @@ use crate::{Dom, NodeKind}; -use std::borrow::Cow; pub(crate) fn fallback_text( dom: &D, @@ -85,13 +84,13 @@ fn excluded_from_naming(dom: &D, node: D::NodeId) -> bool { || primary_role(dom, node).is_some_and(|role| role_matches(role, PRESENTATIONAL_ROLES)) } -pub(crate) fn image_label(dom: &D, node: D::NodeId) -> Cow<'_, str> { +pub(crate) fn image_label(dom: &D, node: D::NodeId) -> &str { let alt = dom.attribute(node, "alt"); if let Some(alt) = alt.filter(|alt| !alt.is_empty()) { - return Cow::Borrowed(alt); + return alt; } if excluded_from_naming(dom, node) { - return Cow::Borrowed(""); + return ""; } ["aria-label", "title"] .into_iter() @@ -101,6 +100,5 @@ pub(crate) fn image_label(dom: &D, node: D::NodeId) -> Cow<'_, .map(str::trim) .filter(|v| !v.is_empty()) }) - .map(Cow::Borrowed) - .unwrap_or(Cow::Borrowed("")) + .unwrap_or_default() } diff --git a/moli-html2md/src/form.rs b/moli-html2md/src/form.rs index 0553504c489..f8b1e9ff4c1 100644 --- a/moli-html2md/src/form.rs +++ b/moli-html2md/src/form.rs @@ -1,56 +1,37 @@ -use std::borrow::Cow; - use crate::Dom; -pub(crate) fn input_text<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Option> { +pub(crate) fn input_text(dom: &D, node: D::NodeId) -> Option<&str> { let kind = dom.attribute(node, "type").unwrap_or("text"); if kind.eq_ignore_ascii_case("checkbox") || kind.eq_ignore_ascii_case("radio") { - return Some(Cow::Borrowed(if dom.attribute(node, "checked").is_some() { + return Some(if dom.attribute(node, "checked").is_some() { "☑" } else { "☐" - })); + }); } - if ["hidden", "file"] + if ["hidden", "file", "password"] .iter() .any(|excluded| kind.eq_ignore_ascii_case(excluded)) { return None; } if kind.eq_ignore_ascii_case("image") { - return dom - .attribute(node, "alt") - .filter(|value| !value.is_empty()) - .map(Cow::Borrowed); + return dom.attribute(node, "alt").filter(|value| !value.is_empty()); } - let button = ["submit", "button", "reset"] - .iter() - .any(|v| kind.eq_ignore_ascii_case(v)); - let value = (!kind.eq_ignore_ascii_case("password")) - .then(|| { - dom.control_value(node) - .or_else(|| dom.attribute(node, "value")) - }) - .flatten() + if let Some(value) = dom + .control_value(node) + .or_else(|| dom.attribute(node, "value")) .filter(|value| !value.is_empty()) - .or_else(|| { - button - .then(|| dom.attribute(node, "aria-label")) - .flatten() - .filter(|value| !value.is_empty()) - }) - .or_else(|| { - button - .then(|| dom.attribute(node, "title")) - .flatten() - .filter(|value| !value.is_empty()) - }); - value.map(Cow::Borrowed) -} - -pub(crate) fn textarea_text<'a, D: Dom + ?Sized>( - dom: &'a D, - node: D::NodeId, -) -> Option> { - dom.control_value(node).map(Cow::Borrowed) + { + return Some(value); + } + if !["submit", "button", "reset"] + .iter() + .any(|button| kind.eq_ignore_ascii_case(button)) + { + return None; + } + ["aria-label", "title"] + .into_iter() + .find_map(|name| dom.attribute(node, name).filter(|value| !value.is_empty())) } diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs index 2d24eb5c836..4fb5e6da8a8 100644 --- a/moli-html2md/src/machine.rs +++ b/moli-html2md/src/machine.rs @@ -267,37 +267,29 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { let alt = crate::content::image_label(self.dom, node); let title = self.dom.attribute(node, "title"); if let Some(src) = crate::media::source(self.dom, node) { - self.writer().image(&alt, &src, title); + self.writer().image(alt, src, title); } else { // A failed or absent resource must not erase its textual // alternative, especially when it is a link's only label. - self.writer().text(&alt); + self.writer().text(alt); } return; } "input" => { if let Some(value) = crate::form::input_text(self.dom, node) { - self.writer().text(&value); + self.writer().text(value); } return; } "textarea" => { - if let Some(value) = crate::form::textarea_text(self.dom, node) { - self.writer().text(&value); + if let Some(value) = self.dom.control_value(node) { + self.writer().text(value); } else { self.children(node, depth + 1); } return; } - "iframe" | "video" | "audio" => { - if self.media(node, tag, depth) { - return; - } - if is_block(tag) { - self.writer().boundary(2); - self.tasks.push(Task::Boundary); - } - } + "iframe" | "video" | "audio" if self.media(node, tag, depth) => return, "code" | "pre" => { self.raw.clear(); let end = if tag == "pre" { @@ -371,12 +363,12 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } else { self.writer().text(" "); } + let label = crate::media::label(self.dom, node, tag); if let Some(poster) = poster { - let poster_label = crate::media::poster_label(self.dom, node); - self.writer().image(&poster_label, poster, None); + self.writer() + .image(&format!("{label} poster"), poster, None); self.writer().text(" "); } - let label = crate::media::label(self.dom, node, tag); let has_source = !urls.is_empty(); for href in urls { self.serial += 1; diff --git a/moli-html2md/src/media.rs b/moli-html2md/src/media.rs index a16003863fc..b7ddc2f5084 100644 --- a/moli-html2md/src/media.rs +++ b/moli-html2md/src/media.rs @@ -1,5 +1,3 @@ -use std::borrow::Cow; - use crate::{Dom, NodeKind}; /// Private lazy-loading conventions, used only when standard sources are absent. @@ -7,38 +5,29 @@ const LAZY_SOURCE_ATTRIBUTES: [&str; 3] = ["data-src", "data-lazy-src", "data-or /// Frozen DOMs can still carry lazy-loading placeholders. These conventional /// attributes declare the resource the page intends to load, not the spacer. -pub(crate) fn source<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Option> { +pub(crate) fn source(dom: &D, node: D::NodeId) -> Option<&str> { // A responsive declaration determines the rendered resource even when a // fallback `src` is present. Private lazy-load attributes remain fallback // inputs and never override a valid standard declaration. - if let Some(value) = dom - .attribute(node, "srcset") - .and_then(|value| largest_srcset_candidate(value, true)) - { - return Some(value); - } - let src = direct_attribute(dom, node, "src", true); - if src.is_some() { - return src.map(Cow::Borrowed); - } - if let Some(value) = dom - .attribute(node, "data-srcset") - .and_then(|value| largest_srcset_candidate(value, true)) - { - return Some(value); - } - LAZY_SOURCE_ATTRIBUTES - .into_iter() - .find_map(|name| direct_attribute(dom, node, name, true)) - .map(Cow::Borrowed) + dom.attribute(node, "srcset") + .and_then(largest_srcset_candidate) + .or_else(|| direct_attribute(dom, node, "src", true)) + .or_else(|| { + dom.attribute(node, "data-srcset") + .and_then(largest_srcset_candidate) + }) + .or_else(|| { + LAZY_SOURCE_ATTRIBUTES + .into_iter() + .find_map(|name| direct_attribute(dom, node, name, true)) + }) } -fn direct_source(dom: &D, node: D::NodeId, image: bool) -> Option<&str> { - let src = direct_attribute(dom, node, "src", image); - src.or_else(|| { +fn direct_source(dom: &D, node: D::NodeId) -> Option<&str> { + direct_attribute(dom, node, "src", false).or_else(|| { LAZY_SOURCE_ATTRIBUTES .into_iter() - .find_map(|name| direct_attribute(dom, node, name, image)) + .find_map(|name| direct_attribute(dom, node, name, false)) }) } @@ -220,10 +209,10 @@ fn valid_positive_float(value: &str) -> Option { .filter(|value| value.is_finite() && *value >= 0.0) } -fn largest_srcset_candidate(srcset: &str, image: bool) -> Option> { +fn largest_srcset_candidate(srcset: &str) -> Option<&str> { parse_srcset(srcset) .into_iter() - .filter(|candidate| safe_url(candidate.url, image)) + .filter(|candidate| safe_url(candidate.url, true)) .max_by(|left, right| { let score = |candidate: &SrcsetCandidate<'_>| { candidate @@ -234,7 +223,7 @@ fn largest_srcset_candidate(srcset: &str, image: bool) -> Option> { }; score(left).total_cmp(&score(right)) }) - .map(|candidate| Cow::Borrowed(candidate.url)) + .map(|candidate| candidate.url) } pub(crate) fn safe_url(value: &str, image: bool) -> bool { @@ -264,7 +253,7 @@ pub(crate) fn sources<'a, D: Dom + ?Sized>( ) -> Vec<&'a str> { let mut urls = Vec::new(); let mut seen = std::collections::HashSet::new(); - if let Some(src) = direct_source(dom, node, false) { + if let Some(src) = direct_source(dom, node) { seen.insert(src); urls.push(src); } @@ -273,7 +262,7 @@ pub(crate) fn sources<'a, D: Dom + ?Sized>( while let Some(id) = child { child = dom.next_sibling(id); if dom.node_kind(id) == NodeKind::Element("source") - && let Some(src) = direct_source(dom, id, false) + && let Some(src) = direct_source(dom, id) && seen.insert(src) { urls.push(src); @@ -306,7 +295,3 @@ pub(crate) fn safe_link_url(value: &str) -> bool { && !normalized.starts_with("data:text/html") && !normalized.starts_with("data:application/xhtml+xml") } - -pub(crate) fn poster_label(dom: &D, node: D::NodeId) -> String { - format!("{} poster", label(dom, node, "video")) -}