diff --git a/moli-html2md/README.md b/moli-html2md/README.md index 133f4be7577..cf7ee6fe394 100644 --- a/moli-html2md/README.md +++ b/moli-html2md/README.md @@ -75,7 +75,8 @@ another conversion path. Complex layouts remain readable blocks instead of raw HTML tables. Supported output includes headings, paragraphs, emphasis, strikethrough, links, -images, lists, blockquotes, hard breaks, and fenced code. Inline HTML is used when +images, lists, blockquotes, hard breaks, media source links, readable form values, +separated form choices, and fenced code. Inline HTML is used when Markdown delimiters cannot express an emphasis boundary. Script/style/head/noscript/template subtrees are omitted. @@ -99,7 +100,7 @@ strip/base/frame transformations remain the responsibility of the caller. whitespace, escaping, code, tables, depth limits, and 20,000 nested nodes on a 64 KiB thread stack. The [Turndown differential suite](tests/turndown/README.md) pins 147 upstream fixtures and 50 additional cases. It compares rendered meaning, -checks DOM immutability, and explicitly asserts 12 documented differences that +checks DOM immutability, and explicitly asserts 13 documented differences that preserve Moli's handling of literal text, code, emphasis, and list boundaries. Targeted regressions also cover empty elements, links, attributes in headings and tables, code whitespace and language names, list numbering, and loose lists. @@ -116,3 +117,19 @@ The HTML parser, reference JSON reader, and `pulldown-cmark` renderer are all **test-only** dependencies. Tests use checked-in reference outputs and require neither Node nor network access. NativeDom integration and real HTML fixtures are tested in `moli-renderer-v8`. + +Unavailable images retain their text alternatives. Media references use standard +source declarations before lazy-loading fallbacks. Password values and hidden or +file input values are omitted. + +Plain DOM attributes describe the supplied snapshot, including default form +values and checked state. Browser adapters can expose live attributes and +`control_value` instead. Placeholder hints are not field values. +Empty text-entry controls emit no value; button-like +controls may use accessible labels; generic containers do not acquire text +from tooltip titles or prohibited names. Child output takes precedence over +any fallback label. Links retain application protocols while executable +JavaScript and inline HTML document URLs are excluded. + +SVG title children are tooltip metadata, not body text. They supply a name only +when an SVG has no emitted content; title elements outside SVG are omitted. diff --git a/moli-html2md/src/content.rs b/moli-html2md/src/content.rs new file mode 100644 index 00000000000..2b52c378098 --- /dev/null +++ b/moli-html2md/src/content.rs @@ -0,0 +1,102 @@ +use crate::{Dom, NodeKind}; +use std::borrow::Cow; + +pub(crate) fn fallback_text(dom: &D, node: D::NodeId) -> Option { + if excluded_from_naming(dom, node) { + return None; + } + let named_tag = matches!( + dom.node_kind(node), + NodeKind::Element( + "a" | "button" + | "svg" + | "textarea" + | "select" + | "option" + | "optgroup" + | "h1" + | "h2" + | "h3" + | "h4" + | "h5" + | "h6" + ) + ); + let named_role = + primary_role(dom, node).is_some_and(|role| role_matches(role, NAMED_WIDGET_ROLES)); + if !named_tag && !named_role { + return None; + } + for name in ["aria-label", "title"] { + if let Some(value) = dom + .attribute(node, name) + .map(str::trim) + .filter(|value| !value.is_empty()) + { + return Some(value.to_owned()); + } + } + if matches!(dom.node_kind(node), NodeKind::Element("svg")) { + let mut child = dom.first_child(node); + while let Some(id) = child { + if dom.node_kind(id) == NodeKind::Element("title") { + let mut text = String::new(); + let mut title_child = dom.first_child(id); + while let Some(part) = title_child { + if let NodeKind::Text(value) = dom.node_kind(part) { + text.push_str(value); + } + title_child = dom.next_sibling(part); + } + if !text.trim().is_empty() { + return Some(text); + } + } + child = dom.next_sibling(id); + } + } + None +} + +const PRESENTATIONAL_ROLES: &[&str] = &["none", "presentation"]; +const NAMED_WIDGET_ROLES: &[&str] = &[ + "button", "link", "img", "heading", "checkbox", "radio", "textbox", "combobox", "switch", "tab", +]; + +// This structural converter uses the first authored token as a conservative +// naming hint; it does not implement an accessibility-tree role resolver. +fn primary_role(dom: &D, node: D::NodeId) -> Option<&str> { + dom.attribute(node, "role")?.split_ascii_whitespace().next() +} +fn role_matches(role: &str, candidates: &[&str]) -> bool { + candidates + .iter() + .any(|candidate| role.eq_ignore_ascii_case(candidate)) +} + +fn excluded_from_naming(dom: &D, node: D::NodeId) -> bool { + dom.attribute(node, "aria-hidden") + .is_some_and(|v| v.trim().eq_ignore_ascii_case("true")) + || dom.attribute(node, "hidden").is_some() + || primary_role(dom, node).is_some_and(|role| role_matches(role, PRESENTATIONAL_ROLES)) +} + +pub(crate) fn image_label(dom: &D, node: D::NodeId) -> Cow<'_, str> { + let alt = dom.attribute(node, "alt"); + if let Some(alt) = alt.filter(|alt| !alt.is_empty()) { + return Cow::Borrowed(alt); + } + if excluded_from_naming(dom, node) { + return Cow::Borrowed(""); + } + ["aria-label", "title"] + .into_iter() + .filter(|name| *name != "title" || alt.is_none()) + .find_map(|name| { + dom.attribute(node, name) + .map(str::trim) + .filter(|v| !v.is_empty()) + }) + .map(Cow::Borrowed) + .unwrap_or(Cow::Borrowed("")) +} diff --git a/moli-html2md/src/dom.rs b/moli-html2md/src/dom.rs index 583e68467fb..15a86903a37 100644 --- a/moli-html2md/src/dom.rs +++ b/moli-html2md/src/dom.rs @@ -26,4 +26,10 @@ pub trait Dom { fn first_child(&self, node: Self::NodeId) -> Option; fn next_sibling(&self, node: Self::NodeId) -> Option; fn attribute(&self, node: Self::NodeId, name: &str) -> Option<&str>; + /// Current input/textarea value supplied by an adapter. None uses the + /// source default; Some("") suppresses that default after a clear. Empty + /// text-entry controls do not emit their name as a field value. + fn control_value(&self, _node: Self::NodeId) -> Option<&str> { + None + } } diff --git a/moli-html2md/src/form.rs b/moli-html2md/src/form.rs new file mode 100644 index 00000000000..0553504c489 --- /dev/null +++ b/moli-html2md/src/form.rs @@ -0,0 +1,56 @@ +use std::borrow::Cow; + +use crate::Dom; + +pub(crate) fn input_text<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Option> { + let kind = dom.attribute(node, "type").unwrap_or("text"); + if kind.eq_ignore_ascii_case("checkbox") || kind.eq_ignore_ascii_case("radio") { + return Some(Cow::Borrowed(if dom.attribute(node, "checked").is_some() { + "☑" + } else { + "☐" + })); + } + if ["hidden", "file"] + .iter() + .any(|excluded| kind.eq_ignore_ascii_case(excluded)) + { + return None; + } + if kind.eq_ignore_ascii_case("image") { + return dom + .attribute(node, "alt") + .filter(|value| !value.is_empty()) + .map(Cow::Borrowed); + } + let button = ["submit", "button", "reset"] + .iter() + .any(|v| kind.eq_ignore_ascii_case(v)); + let value = (!kind.eq_ignore_ascii_case("password")) + .then(|| { + dom.control_value(node) + .or_else(|| dom.attribute(node, "value")) + }) + .flatten() + .filter(|value| !value.is_empty()) + .or_else(|| { + button + .then(|| dom.attribute(node, "aria-label")) + .flatten() + .filter(|value| !value.is_empty()) + }) + .or_else(|| { + button + .then(|| dom.attribute(node, "title")) + .flatten() + .filter(|value| !value.is_empty()) + }); + value.map(Cow::Borrowed) +} + +pub(crate) fn textarea_text<'a, D: Dom + ?Sized>( + dom: &'a D, + node: D::NodeId, +) -> Option> { + dom.control_value(node).map(Cow::Borrowed) +} diff --git a/moli-html2md/src/lib.rs b/moli-html2md/src/lib.rs index 2562a5125da..7c835cd837b 100644 --- a/moli-html2md/src/lib.rs +++ b/moli-html2md/src/lib.rs @@ -1,6 +1,9 @@ +mod content; mod converter; mod dom; +mod form; mod machine; +mod media; mod options; mod output; mod table; @@ -16,3 +19,5 @@ extern crate self as moli_html2md; #[cfg(test)] mod copy_tests; + +pub use media::{SrcsetCandidate, parse_srcset}; diff --git a/moli-html2md/src/machine.rs b/moli-html2md/src/machine.rs index 62ff7cd9ee6..0abe1c24b3d 100644 --- a/moli-html2md/src/machine.rs +++ b/moli-html2md/src/machine.rs @@ -8,6 +8,8 @@ enum Task<'a, Id> { Visit(Id, usize), Children(Option, usize), Boundary, + ChoiceBoundary, + Fallback(String, usize), PopStyle, EndLink(usize), EndQuote, @@ -69,6 +71,14 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } Task::Children(None, _) | Task::RawChildren(None, _, _) => {} Task::Boundary => self.writer().boundary(2), + Task::ChoiceBoundary => self.writer().boundary(1), + Task::Fallback(label, before) => { + if self.writer().content_mark() == before { + self.writer().text(" "); + self.writer().text(&label); + self.writer().text(" "); + } + } Task::PopStyle => self.writer().pop_style(), Task::EndLink(serial) => self.writer().end_link(serial), Task::EndQuote => { @@ -207,6 +217,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } match tag { "head" | "script" | "style" | "noscript" | "template" => return, + "title" => return, "br" => { self.writer().hard_break(); return; @@ -215,6 +226,19 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { self.writer().block("---".into(), 2, 2); return; } + "select" | "optgroup" | "option" => { + self.writer().boundary(1); + if matches!(tag, "optgroup" | "option") + && let Some(label) = self.dom.attribute(node, "label") + { + self.writer().text(label); + self.writer().boundary(1); + if tag == "option" { + return; + } + } + self.tasks.push(Task::ChoiceBoundary); + } "strong" | "b" => self.style(Style::Strong), "em" | "i" => self.style(Style::Emphasis), "del" | "s" | "strike" => self.style(Style::Strike), @@ -222,7 +246,7 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { if let Some(href) = self .dom .attribute(node, "href") - .filter(|href| !href.is_empty()) + .filter(|href| !href.is_empty() && crate::media::safe_link_url(href)) { self.serial += 1; let serial = self.serial; @@ -237,17 +261,38 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } } "img" => { - let alt = self.dom.attribute(node, "alt").unwrap_or_default(); + let alt = crate::content::image_label(self.dom, node); let title = self.dom.attribute(node, "title"); - if let Some(src) = self - .dom - .attribute(node, "src") - .filter(|src| !src.is_empty()) - { - self.writer().image(alt, src, title); + if let Some(src) = crate::media::source(self.dom, node) { + self.writer().image(&alt, &src, title); + } else { + // A failed or absent resource must not erase its textual + // alternative, especially when it is a link's only label. + self.writer().text(&alt); + } + return; + } + "input" => { + if let Some(value) = crate::form::input_text(self.dom, node) { + self.writer().text(&value); } return; } + "textarea" => { + if let Some(value) = crate::form::textarea_text(self.dom, node) { + self.writer().text(&value); + return; + } + } + "iframe" | "video" | "audio" => { + if self.media(node, tag, depth) { + return; + } + if is_block(tag) { + self.writer().boundary(2); + self.tasks.push(Task::Boundary); + } + } "code" | "pre" => { self.raw.clear(); let end = if tag == "pre" { @@ -298,9 +343,62 @@ impl<'a, D: Dom + ?Sized> Machine<'a, D> { } _ => {} } + if let Some(label) = crate::content::fallback_text(self.dom, node) { + let before = self.writer().content_mark(); + self.tasks.push(Task::Fallback(label, before)); + } self.children(node, depth + 1); } + fn media(&mut self, node: D::NodeId, tag: &str, depth: usize) -> bool { + let urls = crate::media::sources(self.dom, node, tag, depth, self.options.max_depth); + let poster = (tag == "video") + .then(|| crate::media::direct_attribute(self.dom, node, "poster", true)) + .flatten(); + if urls.is_empty() && poster.is_none() { + return false; + } + let block = is_block(tag); + if block { + self.writer().boundary(2); + } else { + self.writer().text(" "); + } + if let Some(poster) = poster { + let poster_label = crate::media::poster_label(self.dom, node); + self.writer().image(&poster_label, poster, None); + self.writer().text(" "); + } + let label = crate::media::label(self.dom, node, tag); + let has_source = !urls.is_empty(); + for href in urls { + self.serial += 1; + let serial = self.serial; + let opened = self.writer().push_style(Style::Link { + serial, + href, + title: None, + }); + self.writer().text(label); + if opened { + self.writer().end_link(serial); + } else { + // Markdown cannot nest a media link inside a surrounding link. + // Retain its destination as text rather than silently losing it. + self.writer().text(" ("); + self.writer().text(href); + self.writer().text(")"); + } + self.writer().text(" "); + } + if block { + self.writer().boundary(2); + } else { + self.writer().text(" "); + } + has_source + } + fn table_cell(&mut self) { let table = self.tables.last_mut().expect("table conversion is active"); if let Some(cell) = table.next_cell() { diff --git a/moli-html2md/src/media.rs b/moli-html2md/src/media.rs new file mode 100644 index 00000000000..a16003863fc --- /dev/null +++ b/moli-html2md/src/media.rs @@ -0,0 +1,312 @@ +use std::borrow::Cow; + +use crate::{Dom, NodeKind}; + +/// Private lazy-loading conventions, used only when standard sources are absent. +const LAZY_SOURCE_ATTRIBUTES: [&str; 3] = ["data-src", "data-lazy-src", "data-original"]; + +/// Frozen DOMs can still carry lazy-loading placeholders. These conventional +/// attributes declare the resource the page intends to load, not the spacer. +pub(crate) fn source<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId) -> Option> { + // A responsive declaration determines the rendered resource even when a + // fallback `src` is present. Private lazy-load attributes remain fallback + // inputs and never override a valid standard declaration. + if let Some(value) = dom + .attribute(node, "srcset") + .and_then(|value| largest_srcset_candidate(value, true)) + { + return Some(value); + } + let src = direct_attribute(dom, node, "src", true); + if src.is_some() { + return src.map(Cow::Borrowed); + } + if let Some(value) = dom + .attribute(node, "data-srcset") + .and_then(|value| largest_srcset_candidate(value, true)) + { + return Some(value); + } + LAZY_SOURCE_ATTRIBUTES + .into_iter() + .find_map(|name| direct_attribute(dom, node, name, true)) + .map(Cow::Borrowed) +} + +fn direct_source(dom: &D, node: D::NodeId, image: bool) -> Option<&str> { + let src = direct_attribute(dom, node, "src", image); + src.or_else(|| { + LAZY_SOURCE_ATTRIBUTES + .into_iter() + .find_map(|name| direct_attribute(dom, node, name, image)) + }) +} + +pub(crate) fn direct_attribute<'a, D: Dom + ?Sized>( + dom: &'a D, + node: D::NodeId, + name: &str, + image: bool, +) -> Option<&'a str> { + dom.attribute(node, name).map(str::trim).filter(|value| { + !value.is_empty() + && !value.starts_with('#') + && *value != "about:blank" + && safe_url(value, image) + }) +} + +/// A validated HTML responsive-image candidate. Dimensions are descriptor +/// values, not an image-selection policy; consumers choose using their viewport. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct SrcsetCandidate<'a> { + pub url: &'a str, + pub descriptor: &'a str, + pub width: Option, + pub height: Option, + pub density: Option, +} + +/// Parse HTML srcset syntax, discarding malformed candidates. +/// Non-ASCII spaces are resource/descriptor characters, not separators. +pub fn parse_srcset(srcset: &str) -> Vec> { + let bytes = srcset.as_bytes(); + let mut offset = 0; + let mut candidates = Vec::new(); + while offset < bytes.len() { + while offset < bytes.len() && (bytes[offset].is_ascii_whitespace() || bytes[offset] == b',') + { + offset += 1; + } + let url_start = offset; + while offset < bytes.len() && !bytes[offset].is_ascii_whitespace() { + offset += 1; + } + if url_start == offset { + break; + } + let mut url_end = offset; + while url_end > url_start && bytes[url_end - 1] == b',' { + url_end -= 1; + } + let url = &srcset[url_start..url_end]; + let ended_with_comma = url_end < offset; + if ended_with_comma { + if !url.is_empty() { + candidates.push(SrcsetCandidate { + url, + descriptor: "", + width: None, + height: None, + density: None, + }); + } + continue; + } + + let descriptor_start = offset; + let mut in_parentheses = false; + while offset < bytes.len() { + match bytes[offset] { + b'(' => in_parentheses = true, + b')' => in_parentheses = false, + b',' if !in_parentheses => break, + _ => {} + } + offset += 1; + } + let descriptor = srcset[descriptor_start..offset].trim_ascii(); + let parsed = descriptor_values(descriptor); + if let Some((width, height, density)) = parsed + && !url.is_empty() + { + candidates.push(SrcsetCandidate { + url, + descriptor, + width, + height, + density, + }); + } + } + candidates +} + +fn descriptor_values(descriptor: &str) -> Option<(Option, Option, Option)> { + if descriptor.is_empty() { + return Some((None, None, None)); + } + let mut width = None; + let mut height = None; + let mut density = None; + for token in descriptor.split_ascii_whitespace() { + if let Some(value) = token.strip_suffix('w') { + if width.is_some() { + return None; + } + width = Some(valid_positive_integer(value)?); + } else if let Some(value) = token.strip_suffix('h') { + if height.is_some() { + return None; + } + height = Some(valid_positive_integer(value)?); + } else if let Some(value) = token.strip_suffix('x') { + if density.is_some() { + return None; + } + density = Some(valid_positive_float(value)?); + } else { + return None; + } + } + match (width, height, density) { + (Some(_), _, None) | (None, None, Some(_)) => Some((width, height, density)), + _ => None, + } +} + +fn valid_positive_integer(value: &str) -> Option { + (!value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit())) + .then(|| value.parse::().ok()) + .flatten() + .filter(|value| *value > 0) +} + +fn valid_positive_float(value: &str) -> Option { + let bytes = value.as_bytes(); + if bytes.is_empty() || bytes[0] == b'+' { + return None; + } + let mut offset = usize::from(bytes[0] == b'-'); + if offset == bytes.len() { + return None; + } + let integer_start = offset; + while offset < bytes.len() && bytes[offset].is_ascii_digit() { + offset += 1; + } + let has_integer = offset > integer_start; + let mut has_fraction = false; + if bytes.get(offset) == Some(&b'.') { + offset += 1; + let fraction = offset; + while offset < bytes.len() && bytes[offset].is_ascii_digit() { + offset += 1; + } + if fraction == offset { + return None; + } + has_fraction = true; + } + if !has_integer && !has_fraction { + return None; + } + if matches!(bytes.get(offset), Some(b'e' | b'E')) { + offset += 1; + if matches!(bytes.get(offset), Some(b'+' | b'-')) { + offset += 1; + } + let exponent = offset; + while offset < bytes.len() && bytes[offset].is_ascii_digit() { + offset += 1; + } + if exponent == offset { + return None; + } + } + (offset == bytes.len()) + .then(|| value.parse::().ok()) + .flatten() + .filter(|value| value.is_finite() && *value >= 0.0) +} + +fn largest_srcset_candidate(srcset: &str, image: bool) -> Option> { + parse_srcset(srcset) + .into_iter() + .filter(|candidate| safe_url(candidate.url, image)) + .max_by(|left, right| { + let score = |candidate: &SrcsetCandidate<'_>| { + candidate + .width + .map(|v| v as f64) + .or(candidate.density) + .unwrap_or(1.0) + }; + score(left).total_cmp(&score(right)) + }) + .map(|candidate| Cow::Borrowed(candidate.url)) +} + +pub(crate) fn safe_url(value: &str, image: bool) -> bool { + let normalized: String = value + .trim() + .chars() + .filter(|ch| !ch.is_ascii_control()) + .collect(); + let Some((scheme, _)) = normalized.split_once(':') else { + return true; + }; + if scheme.contains(['/', '#', '?']) { + return true; + } + matches!( + scheme.to_ascii_lowercase().as_str(), + "http" | "https" | "mailto" | "tel" | "ftp" + ) || (image && normalized.to_ascii_lowercase().starts_with("data:image/")) +} + +pub(crate) fn sources<'a, D: Dom + ?Sized>( + dom: &'a D, + node: D::NodeId, + tag: &str, + depth: usize, + limit: usize, +) -> Vec<&'a str> { + let mut urls = Vec::new(); + let mut seen = std::collections::HashSet::new(); + if let Some(src) = direct_source(dom, node, false) { + seen.insert(src); + urls.push(src); + } + if tag != "iframe" && depth + 1 < limit { + let mut child = dom.first_child(node); + while let Some(id) = child { + child = dom.next_sibling(id); + if dom.node_kind(id) == NodeKind::Element("source") + && let Some(src) = direct_source(dom, id, false) + && seen.insert(src) + { + urls.push(src); + } + } + } + urls +} + +pub(crate) fn label<'a, D: Dom + ?Sized>(dom: &'a D, node: D::NodeId, tag: &str) -> &'a str { + dom.attribute(node, "title") + .filter(|label| !label.trim().is_empty()) + .unwrap_or(match tag { + "video" => "Video", + "audio" => "Audio", + _ => "Embedded content", + }) +} + +/// Links preserve application protocols; executable inline-document schemes do not. +pub(crate) fn safe_link_url(value: &str) -> bool { + let normalized: String = value + .trim() + .chars() + .filter(|c| !c.is_ascii_control()) + .collect(); + let normalized = normalized.to_ascii_lowercase(); + !normalized.starts_with("javascript:") + && !normalized.starts_with("vbscript:") + && !normalized.starts_with("data:text/html") + && !normalized.starts_with("data:application/xhtml+xml") +} + +pub(crate) fn poster_label(dom: &D, node: D::NodeId) -> String { + format!("{} poster", label(dom, node, "video")) +} diff --git a/moli-html2md/src/writer.rs b/moli-html2md/src/writer.rs index 7417fdf72c5..18277faa810 100644 --- a/moli-html2md/src/writer.rs +++ b/moli-html2md/src/writer.rs @@ -26,6 +26,7 @@ struct OpenStyle<'a> { pub(crate) struct Writer<'a> { // Blocks are immutable fragments; inline edits only touch the local tail. prefix: Output, + content_revision: usize, output: String, desired: Vec>, emitted: Vec>, @@ -97,6 +98,7 @@ impl<'a> Writer<'a> { } pub(crate) fn text(&mut self, text: &str) { + self.content_revision += usize::from(text.chars().any(|c| !c.is_whitespace())); self.flush_code(); for ch in text.chars() { if is_space(ch) { @@ -153,6 +155,7 @@ impl<'a> Writer<'a> { } let text = text.trim_matches(is_space); if !text.is_empty() { + self.content_revision += 1; if self.code.is_none() { self.prepare_inline('`'); self.code = Some(String::new()); @@ -181,6 +184,7 @@ impl<'a> Writer<'a> { } self.flush_code(); if preformatted { + self.content_revision += 1; self.prepare_inline('`'); self.code = Some(text.replace("\r\n", " ").replace(['\n', '\r'], " ")); return; @@ -203,6 +207,7 @@ impl<'a> Writer<'a> { } pub(crate) fn image(&mut self, alt: &str, src: &str, title: Option<&str>) { + self.content_revision += 1; self.flush_code(); self.prepare_inline('!'); self.output.push_str("!["); @@ -236,6 +241,7 @@ impl<'a> Writer<'a> { } pub(crate) fn hard_break(&mut self) { + self.content_revision += 1; self.flush_code(); self.close_to(0, None); self.flush_spaces(false); @@ -250,6 +256,7 @@ impl<'a> Writer<'a> { self.boundary(before.max(after)); return; } + self.content_revision += 1; // Closing styles before detaching the tail keeps all inline edit // offsets local to the current String, never inside a finished block. self.boundary(before); @@ -272,6 +279,10 @@ impl<'a> Writer<'a> { self.prefix } + pub(crate) fn content_mark(&self) -> usize { + self.content_revision + } + fn is_empty(&self) -> bool { self.output.is_empty() && self.prefix.is_empty() } diff --git a/moli-html2md/tests/regressions.rs b/moli-html2md/tests/regressions.rs index f5e12486a8b..cb64f749ae8 100644 --- a/moli-html2md/tests/regressions.rs +++ b/moli-html2md/tests/regressions.rs @@ -1,6 +1,6 @@ mod support; -use moli_html2md::{Converter, Options}; +use moli_html2md::{Converter, Options, parse_srcset}; use support::{Tree, rendered_html}; fn markdown(html: &str, preformatted_code: bool) -> String { @@ -420,12 +420,17 @@ fn heading_attributes_keep_the_same_meaning_as_attributes_outside_headings() { #[test] fn images_without_sources_do_not_emit_placeholder_markdown() { - for html in [ - "ab", - "alabelb", - "alabelb", + assert_eq!(markdown("ab", false), "ab"); + for image in [ + "Email", + "Email", + "Email", ] { - assert_eq!(markdown(html, false), "ab"); + let source = format!("{image}"); + assert_eq!( + rendered_html(&markdown(&source, false)), + "

Email

\n" + ); } } @@ -488,3 +493,302 @@ fn fallback_block_elements_separate_neighboring_words() { ); } } + +#[test] +fn body_metadata_does_not_become_article_text() { + let result = markdown( + "

News

Actual story

Untitled templatePublisher
", + false, + ); + assert!(result.contains("Actual story")); + assert!(result.contains("Publisher")); + assert!(!result.contains("Untitled template"), "{result}"); + let graphic = markdown( + "Chart descriptionRevenueTemplate metadata", + false, + ); + assert!(!graphic.contains("Chart description"), "{graphic}"); + assert!(graphic.contains("Revenue"), "{graphic}"); + assert!(!graphic.contains("Template metadata"), "{graphic}"); +} + +#[test] +fn form_choices_keep_option_and_group_boundaries() { + let result = markdown( + "", + false, + ); + assert!(result.contains("All posts\n"), "{result}"); + assert!(result.contains("Recent\n1 day\n1 week"), "{result}"); + assert!(!result.contains("Internal long label"), "{result}"); + assert_eq!(markdown("

website

", false), "website"); +} + +#[test] +fn embedded_media_retains_usable_sources() { + let result = markdown( + "

Watch the interview:

End.

", + false, + ); + for link in [ + "[Interview](https://video.example/embed/42)", + "(/movie.webm)", + "(/movie.mp4)", + "[Episode](/episode.ogg)", + "(/cover.jpg)", + ] { + assert!(result.contains(link), "{result}"); + } + assert_eq!(result.matches("(/movie.mp4)").count(), 1, "{result}"); + assert!(result.ends_with("End."), "{result}"); +} + +#[test] +fn responsive_image_without_src_retains_largest_candidate() { + assert_eq!( + markdown( + "Course cover", + false, + ), + "![Course cover](//cdn.example/cover-500.jpg)" + ); +} + +#[test] +fn concrete_loaded_image_wins_over_an_unexpanded_lazy_template() { + let result = markdown( + "Basin", + false, + ); + assert_eq!(result, "![Basin](https://cdn.example/basin_300x.jpg)"); +} + +#[test] +fn authored_spaces_between_controls_and_dates_remain_inline() { + let result = rendered_html(&markdown( + "

", + false, + )); + assert_eq!( + result, + "

Cancel Subscribe

\n

2024-01-02 2025-03-04

\n" + ); +} + +#[test] +fn generic_container_labels_are_not_article_text_but_link_labels_are() { + let result = markdown( + "

By Reporter

", + false, + ); + assert!(!result.contains("Published:"), "{result}"); + assert!(result.contains("[Search](/search)"), "{result}"); +} + +#[test] +fn quote_classes_do_not_replace_list_or_table_semantics() { + let list = markdown( + "
  • First
  • Second
", + false, + ); + assert_eq!(list, "- First\n- Second"); + let table = rendered_html(&markdown( + "
Name
Ada
", + false, + )); + assert!(table.contains(""), "{table}"); + assert!(table.contains(""), "{table}"); +} + +#[test] +fn resource_only_headings_are_not_dropped() { + let result = markdown( + "

Article

", + false, + ); + assert!(result.contains("# [![](/logo.svg)](/home)"), "{result}"); + assert!(result.contains("## Accessible title"), "{result}"); + assert!(result.contains("Article"), "{result}"); + let alternative = markdown( + "

Product manual

Safe title

", + false, + ); + assert!(alternative.contains("# Product manual"), "{alternative}"); + assert!(alternative.contains("## Safe title"), "{alternative}"); +} + +#[test] +fn fallback_labels_preserve_all_children_and_remain_literal_text() { + let result = markdown( + "

", + false, + ); + assert!(result.contains("Save"), "{result}"); + assert!(result.contains("Delete"), "{result}"); + assert!(result.contains("# [One](/one) [Two](/two)"), "{result}"); + let rendered = rendered_html(&result); + assert!(!rendered.contains("href=\"/unintended\""), "{rendered}"); + assert!(rendered.contains("[Open](/unintended)"), "{rendered}"); +} + +#[test] +fn responsive_image_falls_back_from_unusable_lazy_candidates() { + for lazy in ["", "javascript:bad() 1x"] { + let result = markdown( + &format!( + "Chart" + ), + false, + ); + assert_eq!(result, "![Chart](/chart.png)"); + } +} + +#[test] +fn source_form_values_preserve_readable_state_without_secrets() { + let source = "
Ada
PeriodMonSun
Morning

"; + let result = rendered_html(&markdown(source, false)); + for value in [ + "Morning", + "☑", + "☐", + "Search term", + "Go", + "Map button", + "Notes", + ] { + assert!(result.contains(value), "missing {value}: {result}"); + } + assert!(!result.contains("Your Email"), "{result}"); + for secret in ["secret", "Password", "token", "private.pdf"] { + assert!(!result.contains(secret), "leaked {secret}: {result}"); + } +} + +#[test] +fn unsafe_resource_urls_keep_labels_without_emitting_active_links() { + let source = "

Open panelPosterEmail

"; + let result = markdown(source, false); + assert!(result.contains("Open panel"), "{result}"); + assert!(result.contains("Poster"), "{result}"); + assert!( + result.contains("[Email](mailto:a@example.test)"), + "{result}" + ); + assert!(!result.contains("javascript:"), "{result}"); +} + +#[test] +fn responsive_images_preserve_url_commas_and_use_declared_candidates() { + let comma_url = "https://cdn.example/c_fill,w_640,h_480/photo.jpg"; + for content in [ + format!("Photo"), + format!("Photo"), + format!( + "
Photo
" + ), + ] { + let result = markdown(&content, false); + assert!(result.contains(comma_url), "{result}"); + } + let result = markdown( + "Photo", + false, + ); + assert!(result.contains("/current.jpg"), "{result}"); + assert!(!result.contains("/old.jpg"), "{result}"); + let result = markdown( + "Photo", + false, + ); + assert!(result.contains("/large.jpg"), "{result}"); + for source in [ + "Photo", + "Photo", + ] { + let result = markdown(source, false); + assert!(!result.contains("/old.jpg"), "{result}"); + } + let defaults = markdown( + "PhotoSafe", + false, + ); + assert!(defaults.contains("![Photo](/large.jpg)"), "{defaults}"); + assert!(defaults.contains("![Safe](/safe.jpg)"), "{defaults}"); +} + +#[test] +fn srcset_rejects_invalid_numbers_and_commas_inside_parentheses() { + let result = markdown( + "DensityParentheses", + false, + ); + assert!(result.contains("![Density](/safe.jpg)"), "{result}"); + assert!( + result.contains("![Parentheses](/safe-parentheses.jpg)"), + "{result}" + ); + assert!(!result.contains("phantom.jpg"), "{result}"); +} + +#[test] +fn srcset_density_uses_the_html_floating_point_grammar() { + let result = markdown( + "FractionExponentZeroNegative zeroInvalid plusInvalid trailing dotNegative", + false, + ); + for expected in [ + "![Fraction](/fraction.jpg)", + "![Exponent](/exponent.jpg)", + "![Zero](/zero.jpg)", + "![Negative zero](/negative-zero.jpg)", + ] { + assert!(result.contains(expected), "missing {expected}: {result}"); + } + for rejected in ["invalid-plus.jpg", "invalid-dot.jpg", "negative.jpg"] { + assert!(!result.contains(rejected), "accepted {rejected}: {result}"); + } +} + +#[test] +fn srcset_recovers_only_the_future_compatible_width_height_pair() { + let paired = parse_srcset("/photo.jpg 640w 480h"); + assert_eq!(paired.len(), 1); + assert_eq!(paired[0].url, "/photo.jpg"); + assert_eq!(paired[0].width, Some(640)); + + for invalid in [ + "/height-only.jpg 480h", + "/non-ascii.jpg \u{3000}2x", + "/nbsp.jpg 2x\u{a0}", + "/duplicate-width.jpg 640w 800w", + "/duplicate-height.jpg 640w 480h 720h", + "/density-height.jpg 2x 480h", + ] { + assert!(parse_srcset(invalid).is_empty(), "accepted {invalid}"); + } +} + +#[test] +fn lazy_media_uses_declared_resources_when_standard_sources_are_unavailable() { + let source = "OneTwo"; + let result = markdown(source, false); + for target in [ + "(/photo-one.jpg)", + "(/photo-two.jpg)", + "(//video.example/lecture)", + "(/ordinary.jpg)", + ] { + assert!(result.contains(target), "{result}"); + } + assert!(!result.contains("placeholder"), "{result}"); + let table = markdown( + "
Photo
", + false, + ); + assert!( + rendered_html(&table).contains("src=\"/actual.jpg\""), + "{table}" + ); + assert!(!table.contains("spacer.gif"), "{table}"); +} diff --git a/moli-html2md/tests/review_semantics.rs b/moli-html2md/tests/review_semantics.rs new file mode 100644 index 00000000000..b882b2025ba --- /dev/null +++ b/moli-html2md/tests/review_semantics.rs @@ -0,0 +1,363 @@ +mod support; + +use moli_html2md::convert; +use std::cell::Cell; +use support::{Tree, rendered_html}; + +struct Counted { + tree: Tree, + visits: Cell, +} +impl moli_html2md::Dom for Counted { + type NodeId = usize; + fn node_kind(&self, node: usize) -> moli_html2md::NodeKind<'_> { + self.visits.set(self.visits.get() + 1); + self.tree.node_kind(node) + } + fn first_child(&self, node: usize) -> Option { + self.tree.first_child(node) + } + fn next_sibling(&self, node: usize) -> Option { + self.tree.next_sibling(node) + } + fn attribute(&self, node: usize, name: &str) -> Option<&str> { + self.tree.attribute(node, name) + } +} + +fn markdown(html: &str) -> String { + let tree = Tree::parse(html); + convert(&tree, tree.root) +} + +#[test] +fn private_class_names_do_not_invent_document_semantics() { + let result = markdown( + "
Request a price quote
\ +
Actual quotation
\ + \ + ", + ); + let html = rendered_html(&result); + assert!(!result.contains('✓'), "{result}"); + assert!(!result.contains("Private plugin state"), "{result}"); + assert!(html.contains("

Request a price quote

"), "{html}"); + assert!(html.contains("
"), "{html}"); + assert!(result.contains("Actual quotation"), "{result}"); + assert!(result.contains("Save"), "{result}"); +} + +#[test] +fn standard_image_sources_are_not_overridden_by_filename_guesses() { + for source in [ + "/photos/placeholder.gif", + "/art/blank.png", + "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==", + ] { + let result = markdown(&format!( + "Original" + )); + assert!(result.contains(source), "{result}"); + assert!(!result.contains("private-alternative"), "{result}"); + } + let result = + markdown("Fallback"); + assert!(result.contains("declared-photo.jpg"), "{result}"); +} + +#[test] +fn nested_accessible_labels_do_not_rescan_the_whole_tree() { + let count = |depth| { + let html = format!( + "{}Readable terminal{}", + "".repeat(depth), + "".repeat(depth), + ); + let dom = Counted { + tree: Tree::parse(&html), + visits: Cell::new(0), + }; + assert_eq!(convert(&dom, dom.tree.root), "Readable terminal"); + dom.visits.get() + }; + let small = count(100); + let large = count(400); + assert!(large <= small * 5, "{small} -> {large}"); +} + +#[test] +fn textarea_uses_its_content_not_a_nonstandard_value_attribute() { + for value in ["unrelated", ""] { + let result = markdown(&format!( + "
" + )); + assert!(result.contains("Authored text"), "{result}"); + assert!(!result.contains("unrelated"), "{result}"); + } +} + +#[test] +fn accessible_label_queries_remain_linear_with_large_unreadable_branches() { + let count = |depth| { + let branch = "".repeat(20); + let html = format!( + "{}{}{}", + format!("{branch}").repeat(depth), + branch, + "".repeat(depth) + ); + let dom = Counted { + tree: Tree::parse(&html), + visits: Cell::new(0), + }; + let result = convert(&dom, dom.tree.root); + assert_eq!(result.matches("Fallback").count(), 1, "{result}"); + dom.visits.get() + }; + let small = count(30); + let large = count(120); + assert!(large <= small * 5, "{small} -> {large}"); +} + +#[test] +fn labels_cannot_replace_media_choices_or_rules() { + for html in [ + "
", + "", + "", + ] { + let result = markdown(html); + assert!( + result.contains("demo.mp4") + || result.contains("example.test/movie") + || result.contains("demo.ogg"), + "{html}: {result}" + ); + } + let result = markdown( + "", + ); + assert!( + result.contains("Group") && result.contains("First"), + "{result}" + ); + let result = markdown(""); + assert!(result.contains("---"), "{result}"); +} + +#[test] +fn accessible_fallback_is_named_content_and_has_word_boundaries() { + let result = markdown( + "

xy

", + ); + assert!(result.contains("x Save y"), "{result}"); + assert!(result.contains("Download"), "{result}"); + for excluded in [ + "Pagination", + "Advertisement", + "tooltip", + "Hidden", + "Loading", + "private state", + ] { + assert!(!result.contains(excluded), "{result}"); + } +} + +#[test] +fn phrasing_controls_and_dates_keep_inline_styles_and_table_cells() { + let result = markdown( + "Posted Posted
DateAction
x ya b
", + ); + let html = rendered_html(&result); + assert!(html.contains("Posted today"), "{html}"); + assert!(html.contains("Posted Oct 1"), "{html}"); + assert!(!result.contains("
"), "{result}"); +} + +#[test] +fn input_hints_do_not_masquerade_as_filled_values() { + let result = markdown( + "", + ); + assert!(!result.contains("Your Email"), "{result}"); + assert!( + result.contains("Actual value") && !result.contains("Password"), + "{result}" + ); + assert!(!result.contains("secret"), "{result}"); +} + +#[test] +fn links_retain_legitimate_non_web_destinations() { + for href in [ + "magnet:?xt=urn:btih:123", + "sms:+12025550123", + "geo:1,2", + "file:///tmp/manual.pdf", + "myapp:open/123", + "blob:https://example.test/id", + "data:application/pdf;base64,JVBERi0=", + ] { + let result = markdown(&format!("Open")); + assert!(result.contains(href), "{href}: {result}"); + } + for href in [ + "javascript:alert(1)", + "vbscript:run()", + "data:text/html,unsafe", + ] { + let result = markdown(&format!("Safe label")); + assert!(!result.contains(href), "{result}"); + } +} + +#[test] +fn inline_media_keeps_links_emphasis_and_table_flow() { + for (source, expected) in [ + ( + "Watch ", + "[Watch Video (/v.mp4)](/w)", + ), + ( + "

See below

", + "See [Embedded content](/frame) below", + ), + ( + "Clip now", + "*Clip [Video](/a.mp4) now*", + ), + ] { + assert_eq!(markdown(source), expected); + } + let result = markdown("
Clip now
"); + assert!(result.contains("Clip [Video](/a.mp4) now"), "{result}"); + assert!(!result.contains("
"), "{result}"); +} + +#[test] +fn named_images_and_poster_only_video_keep_their_alternatives() { + for source in [ + "", + "", + ] { + assert!(markdown(source).contains("![Video poster](/cover.jpg)")); + } + assert_eq!( + markdown(""), + "[Home](/home)" + ); + assert_eq!( + markdown(""), + "![Logo](/logo.png)" + ); + assert_eq!( + markdown("Authored"), + "![Authored](/logo.png)" + ); + assert_eq!( + markdown("TwitterFollow us"), + "[Twitter Follow us](/t)" + ); + assert_eq!( + markdown("Chart descriptionRevenue"), + "Revenue" + ); +} + +#[test] +fn live_controls_replace_defaults_including_explicit_clears() { + struct Live(Tree); + impl moli_html2md::Dom for Live { + type NodeId = usize; + fn node_kind(&self, n: usize) -> moli_html2md::NodeKind<'_> { + self.0.node_kind(n) + } + fn first_child(&self, n: usize) -> Option { + self.0.first_child(n) + } + fn next_sibling(&self, n: usize) -> Option { + self.0.next_sibling(n) + } + fn attribute(&self, n: usize, k: &str) -> Option<&str> { + self.0.attribute(n, k) + } + fn control_value(&self, n: usize) -> Option<&str> { + self.0.attribute(n, "data-live") + } + } + for tag in ["input", "textarea"] { + for value in ["Current", ""] { + let html = if tag == "input" { + format!("") + } else { + format!("") + }; + let dom = Live(Tree::parse(&html)); + let result = convert(&dom, dom.0.root); + assert!(!result.contains("Stale"), "{result}"); + if !value.is_empty() { + assert_eq!(result, "Current"); + } else { + assert_eq!(result, ""); + } + } + } +} + +#[test] +fn media_fallback_and_decorative_images_keep_author_intent() { + for src in ["", "blob:https://site/id"] { + let output = markdown(&format!( + "" + )); + assert!(output.contains("![Video poster](/cover.jpg)"), "{output}"); + assert!(output.contains("[Download](/clip.mp4)"), "{output}"); + } + assert_eq!(markdown(""), ""); + assert_eq!( + markdown(""), + "![](/thumb.jpg \"Gallery\")" + ); + assert_eq!(markdown(""), ""); + assert_eq!( + markdown(""), + "Search" + ); +} + +#[test] +fn authored_image_alternatives_survive_naming_exclusions() { + for attribute in ["hidden", "aria-hidden=' true '", "role='presentation'"] { + assert_eq!( + markdown(&format!( + "Revenue chart" + )), + "![Revenue chart](/c.png)" + ); + assert_eq!( + markdown(&format!( + "" + )), + "![](/c.png)" + ); + } + assert_eq!( + markdown(""), + "![](/c.png)" + ); + assert_eq!( + markdown(""), + "![Video poster](/cover.jpg)" + ); + for poster in ["#cover", "about:blank"] { + assert_eq!( + markdown(&format!("")), + "Fallback" + ); + } + assert_eq!( + markdown(""), + "![](b.png)" + ); +} diff --git a/moli-html2md/tests/turndown/README.md b/moli-html2md/tests/turndown/README.md index 703252d8d2e..b986e0662e7 100644 --- a/moli-html2md/tests/turndown/README.md +++ b/moli-html2md/tests/turndown/README.md @@ -25,7 +25,7 @@ accepts equivalent delimiter choices and list indentation while detecting lost text, incorrect links, changed paragraph/list structure, and visible delimiters. There are also exact Markdown assertions in the original and regression suites. -`expectations.json` documents 12 deliberate differences with exact Moli outputs. +`expectations.json` documents 13 deliberate differences with exact Moli outputs. These preserve bare pre blocks and avoid upstream losses of literal characters, emphasis, whitespace, and list boundaries. They remain active assertions; no case is skipped. A reference change that makes a difference obsolete also fails @@ -36,7 +36,7 @@ rather than the plugin's raw HTML fallback. ## Findings tracked outside the reference corpus -The 12 exceptions above describe this corpus, not every design difference or +The 13 exceptions above describe this corpus, not every design difference or every correctness issue. [tracking.rs](../tracking.rs) records these open review findings with active assertions of both Markdown and rendered HTML: diff --git a/moli-html2md/tests/turndown/expectations.json b/moli-html2md/tests/turndown/expectations.json index 9a9593aad97..2944b9671bf 100644 --- a/moli-html2md/tests/turndown/expectations.json +++ b/moli-html2md/tests/turndown/expectations.json @@ -43,8 +43,20 @@ "markdown": "before` `after", "reason": "Explicit preformatted-code mode preserves the spaces inside code. Turndown removes a whitespace-only code element entirely." }, + "extra_missing_image_src": { + "markdown": "beforemissingafter", + "reason": "An unavailable image retains its existing text alternative. Dropping alt also removes the only accessible label from image-only navigation links; Turndown loses that text." + }, "extra_split_list_marker": { "markdown": "12\\. text", "reason": "Escape list markers across text-node boundaries. Turndown escapes nodes individually and accidentally creates an ordered list." + }, + "upstream_036": { + "markdown": "![the\ntitle](logo.png \"the\ntitle\")", + "reason": "An image without alt uses its authored title as a text alternative. Turndown leaves that label empty." + }, + "upstream_037": { + "markdown": "![\"hello\"](logo.png \"\\\"hello\\\"\")", + "reason": "An image without alt uses its authored title as a text alternative while retaining the resource title." } } diff --git a/moli-renderer-v8/src/native_bridge/element/images/src.rs b/moli-renderer-v8/src/native_bridge/element/images/src.rs index 45353fd831e..13cdf162d74 100644 --- a/moli-renderer-v8/src/native_bridge/element/images/src.rs +++ b/moli-renderer-v8/src/native_bridge/element/images/src.rs @@ -673,13 +673,6 @@ fn connected_empty_src_without_srcset(runtime: &JsContextHost, handle: DomHandle }) } -#[derive(Clone, Copy, Debug)] -enum SrcsetDescriptorState { - InDescriptor, - InParens, - AfterDescriptor, -} - #[derive(Clone, Copy, Debug, Default, PartialEq)] struct SrcsetDescriptor { width: Option, @@ -781,145 +774,21 @@ fn image_selection_viewport(runtime: &JsContextHost, handle: DomHandle) -> Image } fn parse_srcset_candidates(input: &str) -> Vec { - let mut index = 0usize; - let mut candidates = Vec::new(); - while index < input.len() { - index += input[index..] - .find(|ch| !is_html_space_or_comma(ch)) - .unwrap_or(input.len() - index); - if index >= input.len() { - break; - } - - let url_start = index; - index += input[index..] - .find(is_html_space) - .unwrap_or(input.len() - index); - let mut url = &input[url_start..index]; - if url.ends_with(',') { - url = url.trim_end_matches(','); - if !url.is_empty() { - candidates.push(SrcsetCandidate { - url: url.to_owned(), - descriptor: SrcsetDescriptor::default(), - }); - } - continue; - } - - index += input[index..] - .find(|ch| !is_html_space(ch)) - .unwrap_or(input.len() - index); - let (descriptors, next_index) = tokenize_srcset_descriptors(input, index); - index = next_index; - if let Some(descriptor) = parse_srcset_descriptors(&descriptors) { - candidates.push(SrcsetCandidate { - url: url.to_owned(), + // Tokenization and descriptor validation are shared with document export. + // Selection retains its existing typed representation and viewport policy. + moli_html2md::parse_srcset(input) + .into_iter() + .filter_map(|candidate| { + let descriptor = SrcsetDescriptor { + width: candidate.width.map(u32::try_from).transpose().ok()?, + density: candidate.density, + }; + Some(SrcsetCandidate { + url: candidate.url.to_owned(), descriptor, - }); - } - } - candidates -} - -fn tokenize_srcset_descriptors(input: &str, mut index: usize) -> (Vec, usize) { - let mut descriptors = Vec::new(); - let mut current = String::new(); - let mut state = SrcsetDescriptorState::InDescriptor; - while index < input.len() { - let ch = input[index..].chars().next().expect("valid char boundary"); - match state { - SrcsetDescriptorState::InDescriptor if is_html_space(ch) => { - if !current.is_empty() { - descriptors.push(std::mem::take(&mut current)); - state = SrcsetDescriptorState::AfterDescriptor; - } - index += ch.len_utf8(); - } - SrcsetDescriptorState::InDescriptor if ch == ',' => { - if !current.is_empty() { - descriptors.push(current); - } - index += ch.len_utf8(); - return (descriptors, index); - } - SrcsetDescriptorState::InDescriptor if ch == '(' => { - current.push(ch); - state = SrcsetDescriptorState::InParens; - index += ch.len_utf8(); - } - SrcsetDescriptorState::InDescriptor => { - current.push(ch); - index += ch.len_utf8(); - } - SrcsetDescriptorState::InParens if ch == ')' => { - current.push(ch); - state = SrcsetDescriptorState::InDescriptor; - index += ch.len_utf8(); - } - SrcsetDescriptorState::InParens => { - current.push(ch); - index += ch.len_utf8(); - } - SrcsetDescriptorState::AfterDescriptor if is_html_space(ch) => { - index += ch.len_utf8(); - } - SrcsetDescriptorState::AfterDescriptor => { - state = SrcsetDescriptorState::InDescriptor; - } - } - } - if !current.is_empty() { - descriptors.push(current); - } - (descriptors, index) -} - -fn parse_srcset_descriptors(descriptors: &[String]) -> Option { - let mut width = None; - let mut density = None; - let mut future_compat_h = None; - - for descriptor in descriptors { - let last = descriptor.chars().last()?; - let value = &descriptor[..descriptor.len() - last.len_utf8()]; - match last { - 'w' if is_valid_non_negative_integer(value) && width.is_none() && density.is_none() => { - let parsed = value.parse::().ok()?; - if parsed == 0 { - return None; - } - width = Some(parsed); - } - 'x' if is_valid_floating_point_number(value) - && width.is_none() - && density.is_none() - && future_compat_h.is_none() => - { - let parsed = value.parse::().ok()?; - if !parsed.is_finite() || parsed < 0.0 { - return None; - } - density = Some(parsed); - } - 'h' if is_valid_non_negative_integer(value) - && future_compat_h.is_none() - && density.is_none() => - { - let parsed = value.parse::().ok()?; - if parsed == 0 { - return None; - } - future_compat_h = Some(parsed); - } - _ => return None, - } - } - - if future_compat_h.is_some() && width.is_none() { - return None; - } - Some(SrcsetDescriptor { width, density }) + }) + }) + .collect() } fn source_size_for_sizes_attribute( @@ -1147,65 +1016,6 @@ fn inline_style_value<'a>(style: &'a str, property: &str) -> Option<&'a str> { }) } -fn is_html_space_or_comma(ch: char) -> bool { - ch == ',' || is_html_space(ch) -} - -fn is_html_space(ch: char) -> bool { - matches!( - ch, - '\u{0009}' | '\u{000a}' | '\u{000c}' | '\u{000d}' | '\u{0020}' - ) -} - -fn is_valid_non_negative_integer(value: &str) -> bool { - !value.is_empty() && value.chars().all(|ch| ch.is_ascii_digit()) -} - -fn is_valid_floating_point_number(value: &str) -> bool { - let bytes = value.as_bytes(); - let mut index = 0usize; - if bytes.get(index) == Some(&b'-') { - index += 1; - } - let integer_start = index; - while bytes.get(index).is_some_and(u8::is_ascii_digit) { - index += 1; - } - let integer_digits = index - integer_start; - let mut fraction_digits = 0usize; - let mut saw_decimal_point = false; - if bytes.get(index) == Some(&b'.') { - saw_decimal_point = true; - index += 1; - let fraction_start = index; - while bytes.get(index).is_some_and(u8::is_ascii_digit) { - index += 1; - } - fraction_digits = index - fraction_start; - } - if integer_digits == 0 && fraction_digits == 0 { - return false; - } - if saw_decimal_point && fraction_digits == 0 { - return false; - } - if matches!(bytes.get(index), Some(b'e' | b'E')) { - index += 1; - if matches!(bytes.get(index), Some(b'+' | b'-')) { - index += 1; - } - let exponent_start = index; - while bytes.get(index).is_some_and(u8::is_ascii_digit) { - index += 1; - } - if exponent_start == index { - return false; - } - } - index == bytes.len() -} - pub(in crate::native_bridge) fn image_current_src_getter_function<'s>( scope: &mut v8::PinScope<'s, '_>, args: v8::FunctionCallbackArguments<'s>, @@ -1265,6 +1075,23 @@ mod tests { .map(|candidate| candidate.url) } + #[test] + fn srcset_parentheses_do_not_nest() { + let candidates = parse_srcset_candidates("a.png ((x), b.png 2x"); + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].url, "b.png"); + } + + #[test] + fn srcset_parser_rejects_unicode_descriptor_spaces() { + for input in ["a.png \u{3000}2x, b.png 1x", "a.png 2x\u{a0}, b.png 1x"] { + let candidates = parse_srcset_candidates(input); + assert_eq!(candidates.len(), 1, "{input}"); + assert_eq!(candidates[0].url, "b.png"); + assert_eq!(candidates[0].descriptor.density, Some(1.0)); + } + } + #[test] fn srcset_parser_uses_html_ascii_whitespace() { assert_eq!( diff --git a/moli-renderer-v8/src/runtime/page_dump.rs b/moli-renderer-v8/src/runtime/page_dump.rs index ee2f639de6a..98e11792af5 100644 --- a/moli-renderer-v8/src/runtime/page_dump.rs +++ b/moli-renderer-v8/src/runtime/page_dump.rs @@ -500,7 +500,27 @@ mod tests { "

link

", "## [link](/a \"one two\")", ), - ("alabelblabelc", "abc"), + ] { + assert_eq!(markdown_from_html(html), expected, "{html}"); + } + } + + #[test] + fn markdown_renderer_retains_labels_for_unavailable_images() { + for (html, expected) in [ + ( + "alabelblabelc", + "alabelblabelc", + ), + ( + "Email", + "[Email](mailto:contact@example.test)", + ), + ( + "[label](javascript:alert(1))", + r"\[label\](javascript:alert(1))", + ), + ("abc", "abc"), ] { assert_eq!(markdown_from_html(html), expected, "{html}"); }