diff --git a/.gitignore b/.gitignore index c3cb32df..2cdf865c 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,6 @@ __pycache__/ /fuzz/corpus/ /fuzz/artifacts/ /fuzz/coverage/ + +# sznb: maturin 本地构建产物 +dist/ diff --git a/Cargo.toml b/Cargo.toml index 2a334b72..56bec806 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,6 @@ include = ["/src", "/examples/convert.rs", "/README.md", "/LICENSE"] [dev-dependencies] insta = "1" -sha2 = "0.11" [dependencies] cfb = "0.14.0" @@ -29,6 +28,8 @@ encoding_rs = "0.8.35" log = "0.4" pdf-inspector = "1.14.2" quick-xml = "0.41.0" +# Fork (sznb): content-addressed names for extracted images. +sha2 = "0.11" zip = { version = "8.6.0", default-features = false, features = ["deflate"] } [profile.release] diff --git a/README.md b/README.md index 2a920ec8..c37372f4 100644 --- a/README.md +++ b/README.md @@ -160,9 +160,9 @@ Only documents that need OCR leave the machine, and the whole document goes, sin | Format | Extensions | | ---------------- | ---------------------------------------------------------- | -| Word | `.doc`, `.docx`, `.docm` | +| Word | `.doc`, `.docx`, `.docm`, `.wps` | | PowerPoint | `.ppt`, `.pps`, `.pot`, `.pptx`, `.pptm`, `.ppsx`, `.ppsm` | -| Excel | `.xls`, `.xlsx`, `.xlsm`, `.xlsb` | +| Excel | `.xls`, `.xlsx`, `.xlsm`, `.xlsb`, `.et` | | OpenDocument | `.odt`, `.ods`, `.odp` | | Rich Text Format | `.rtf` | | EPUB | `.epub` | diff --git a/python/anydoc/__init__.py b/python/anydoc_sznb/__init__.py similarity index 86% rename from python/anydoc/__init__.py rename to python/anydoc_sznb/__init__.py index 1cd64e73..d861f631 100644 --- a/python/anydoc/__init__.py +++ b/python/anydoc_sznb/__init__.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Literal, get_args -from anydoc._anydoc import ( +from anydoc_sznb._anydoc import ( Asset, Block, Cell, @@ -17,7 +17,9 @@ ConvertError, Document, EncryptedError, + ExtractedImage, ImageSource, + ImagesResult, Inline, LinkTarget, List, @@ -34,9 +36,12 @@ format_from_extension, format_from_path, to_document, + to_markdown_with_images, + to_text_with_images, ) -from anydoc._anydoc import to_markdown as _to_markdown -from anydoc._anydoc import to_markdown_bytes as _to_markdown_bytes +from anydoc_sznb._anydoc import to_markdown as _to_markdown +from anydoc_sznb._anydoc import to_markdown_bytes as _to_markdown_bytes +from anydoc_sznb._anydoc import to_text_bytes Format = Literal[ "doc", "docx", "odt", "pdf", "ppt", "pptx", "rtf", "epub", "xlsx", "ods", "odp", "csv" @@ -91,6 +96,9 @@ def to_markdown_bytes( data: "bytes | bytearray", format: "Format | None" = None, *, + html_tables: bool = True, + dedup_cells: bool = True, + include_toc: bool = True, ocr: Ocr = "reject", parse_mode: ParseMode = "auto", api_key: "str | None" = None, @@ -98,11 +106,19 @@ def to_markdown_bytes( ) -> str: """Convert an in-memory document to Markdown. Without a format, it is detected from the content, which signature-less formats (CSV) have to - name explicitly. `ocr`, `parse_mode`, `api_key` and `api_url` are as - for `to_markdown`.""" + name explicitly. + + Fork (sznb): `html_tables` selects the table form — `True` (default) + renders HTML `` carrying merged-cell colspan/rowspan; `False` + renders GFM pipe tables (no span syntax). `dedup_cells` shows merged + content once on the origin cell (default) or repeats it in every covered + position; `include_toc=False` drops the Word table of contents. `ocr`, `parse_mode`, + `api_key` and `api_url` are as for `to_markdown`.""" _check_parse_mode(parse_mode) try: - return _to_markdown_bytes(data, format) + return _to_markdown_bytes( + data, format, html_tables=html_tables, dedup_cells=dedup_cells, include_toc=include_toc + ) except NeedsOcrError: if ocr != "hosted": raise @@ -206,9 +222,11 @@ def _version() -> str: "ConvertError", "Document", "EncryptedError", + "ExtractedImage", "Format", "HostedError", "ImageSource", + "ImagesResult", "Inline", "LinkTarget", "List", @@ -229,4 +247,7 @@ def _version() -> str: "to_document", "to_markdown", "to_markdown_bytes", + "to_markdown_with_images", + "to_text_bytes", + "to_text_with_images", ] diff --git a/python/anydoc/_anydoc.pyi b/python/anydoc_sznb/_anydoc.pyi similarity index 68% rename from python/anydoc/_anydoc.pyi rename to python/anydoc_sznb/_anydoc.pyi index ca45a408..81c3e692 100644 --- a/python/anydoc/_anydoc.pyi +++ b/python/anydoc_sznb/_anydoc.pyi @@ -66,10 +66,93 @@ def to_markdown(path: str | os.PathLike[str]) -> str: file content; the extension is the fallback for signature-less formats (CSV) and unrecognizable containers.""" -def to_markdown_bytes(data: bytes | bytearray, format: Format | None = None) -> str: +def to_markdown_bytes( + data: bytes | bytearray, + format: Format | None = None, + *, + html_tables: bool = True, + dedup_cells: bool = True, + include_toc: bool = True, +) -> str: """Convert an in-memory document to Markdown. Without a format, it is detected from the content, which signature-less formats (CSV) have to - name explicitly.""" + name explicitly. + + Fork (sznb): `html_tables` selects the table form — HTML `
` with + merged-cell colspan/rowspan (default) or GFM pipe tables. `dedup_cells` + shows merged content once on the origin cell (default) or repeats it in + every covered position (spans collapse). `include_toc=False` drops the + Word table of contents (`Table of Contents` SDTs and bare `TOC` field + results, with the 目录 title before them; docx / doc / wps only).""" + +def to_text_bytes( + data: bytes | bytearray, + format: Format | None = None, + *, + dedup_cells: bool = True, + include_toc: bool = True, +) -> str: + """Convert an in-memory document to plain text (no Markdown syntax; + tables render as tab-joined rows carrying merged content once, or in + every covered cell with `dedup_cells=False`). `include_toc` as for + `to_markdown_bytes`. Format handling as for `to_markdown_bytes`. + + Unsupported for `pdf`: PDF conversion produces Markdown directly and has + no document-model form.""" + +def to_markdown_with_images( + data: bytes | bytearray, + format: Format | None = None, + *, + html_tables: bool = True, + dedup_cells: bool = True, + include_toc: bool = True, + min_width: int = 50, + min_height: int = 50, + placeholder: str = "[IMG::{name}]", + name_prefix: str = "img_", +) -> ImagesResult: + """Fork (sznb): `to_markdown_bytes` plus image extraction. Embedded + images become placeholder tokens in place (`{name}` = the file name); + images under `min_width` x `min_height` pixels are dropped (unknown size + is kept); names are `{name_prefix}{sha256[:16]}{ext}`, content-addressed + so duplicates collapse. PDF yields its Markdown and no images.""" + +def to_text_with_images( + data: bytes | bytearray, + format: Format | None = None, + *, + dedup_cells: bool = True, + include_toc: bool = True, + min_width: int = 50, + min_height: int = 50, + placeholder: str = "[IMG::{name}]", + name_prefix: str = "img_", +) -> ImagesResult: + """Fork (sznb): `to_text_bytes` plus image extraction (as for + `to_markdown_with_images`). Unsupported for `pdf`.""" + +@final +class ExtractedImage: + name: str + """Content-addressed file name, e.g. `img_3f2a9c0d1b2e4f56.png`.""" + placeholder: str + """The token standing in for the image in the rendered text.""" + media_type: str + width: int | None + height: int | None + data: bytes + occurrences: int + """How many places in the document reference it.""" + origin_part: str + +@final +class ImagesResult: + text: str + images: list[ExtractedImage] + """In order of first appearance.""" + skipped: int + """Image references dropped for being under the size threshold.""" def to_document(data: bytes | bytearray, format: Format | None = None) -> Document: """Parse an in-memory document into the document model, which also @@ -148,18 +231,18 @@ class LinkTarget: @final class ImageSource: - kind: Literal["external", "asset", "unavailable"] + kind: Literal["external", "asset", "unavailable", "placeholder"] """external: absolute URL with a scheme. asset: embedded image, carried in `Document.assets`. unavailable: no usable source, only the alt text - remains.""" + remains. placeholder (fork): extracted, `url` holds the token.""" url: str | None - """external.""" + """external; placeholder: the token.""" asset_id: int | None """asset: index into `Document.assets`.""" @final class List: - marker: Literal["bullet", "decimal", "lower_alpha", "upper_alpha", "lower_roman", "upper_roman"] + marker: Literal["bullet", "decimal", "lower_alpha", "upper_alpha", "lower_roman", "upper_roman", "chinese_counting", "chinese_legal"] """The marker family the list uses in the source document.""" start: int """Ordinal the first item counts from.""" diff --git a/python/anydoc/py.typed b/python/anydoc_sznb/py.typed similarity index 100% rename from python/anydoc/py.typed rename to python/anydoc_sznb/py.typed diff --git a/python/pyproject.toml b/python/pyproject.toml index 81885146..c36cc73f 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "maturin" # The distribution installs as `firecrawl-anydoc` (the bare `anydoc` name on # PyPI is held by an unrelated package) and imports as `anydoc`. [project] -name = "firecrawl-anydoc" +name = "firecrawl-anydoc-sznb" description = "Convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown" readme = "README.md" license = "MIT" @@ -27,4 +27,4 @@ Repository = "https://github.com/firecrawl/anydoc" Issues = "https://github.com/firecrawl/anydoc/issues" [tool.maturin] -module-name = "anydoc._anydoc" +module-name = "anydoc_sznb._anydoc" diff --git a/python/src/document.rs b/python/src/document.rs index f0e2e844..1cc1809e 100644 --- a/python/src/document.rs +++ b/python/src/document.rs @@ -10,7 +10,7 @@ use pyo3::types::{PyBytes, PyList}; use anydoc::model; -#[pyclass(frozen, get_all, module = "anydoc")] +#[pyclass(frozen, get_all, module = "anydoc_sznb")] pub struct Document { /// list[Block] blocks: Py, @@ -21,7 +21,7 @@ pub struct Document { assets: Py, } -#[pyclass(frozen, get_all, module = "anydoc")] +#[pyclass(frozen, get_all, module = "anydoc_sznb")] pub struct Block { /// heading, paragraph, list, table, block_quote, code_block, rule, or /// math. @@ -86,7 +86,7 @@ fn block(py: Python<'_>, block: model::Block) -> PyResult { }) } -#[pyclass(frozen, get_all, module = "anydoc")] +#[pyclass(frozen, get_all, module = "anydoc_sznb")] pub struct Inline { /// text, link, image, anchor (a zero-width marker for an internal link /// target at this position), note_ref, line_break, or math. @@ -156,7 +156,7 @@ fn inline(py: Python<'_>, inline: model::Inline) -> PyResult { } /// Fully resolved character style. -#[pyclass(frozen, get_all, module = "anydoc")] +#[pyclass(frozen, get_all, module = "anydoc_sznb")] pub struct Style { bold: bool, italic: bool, @@ -170,7 +170,7 @@ impl From for Style { } } -#[pyclass(frozen, get_all, module = "anydoc")] +#[pyclass(frozen, get_all, module = "anydoc_sznb")] pub struct LinkTarget { /// external (absolute URL with a scheme), relative (scheme-less relative /// reference, preserved as written), or anchor (internal target: a @@ -191,14 +191,15 @@ impl From for LinkTarget { } } -#[pyclass(frozen, get_all, module = "anydoc")] +#[pyclass(frozen, get_all, module = "anydoc_sznb")] pub struct ImageSource { /// external (absolute URL with a scheme), asset (embedded image, carried /// in `Document.assets`), or unavailable (no usable source: the image's /// part is missing or unreadable and it has no URL; only the alt text - /// remains). + /// remains), or placeholder (fork: extracted by `to_*_with_images`, + /// `url` holds the literal token). kind: &'static str, - /// external. + /// external; placeholder: the token. url: Option, /// asset: index into `Document.assets`. asset_id: Option, @@ -216,11 +217,15 @@ impl From for ImageSource { model::ImageSource::Unavailable => { ImageSource { kind: "unavailable", url: None, asset_id: None } } + // Fork (sznb): the placeholder token travels in `url`. + model::ImageSource::Placeholder(token) => { + ImageSource { kind: "placeholder", url: Some(token), asset_id: None } + } } } } -#[pyclass(frozen, get_all, module = "anydoc")] +#[pyclass(frozen, get_all, module = "anydoc_sznb")] pub struct List { /// The marker family the list uses in the source document: bullet, /// decimal, lower_alpha, upper_alpha, lower_roman, or upper_roman. @@ -239,12 +244,14 @@ fn list(py: Python<'_>, list: model::List) -> PyResult { model::MarkerKind::UpperAlpha => "upper_alpha", model::MarkerKind::LowerRoman => "lower_roman", model::MarkerKind::UpperRoman => "upper_roman", + model::MarkerKind::ChineseCounting => "chinese_counting", + model::MarkerKind::ChineseLegal => "chinese_legal", }; let items = list.items.into_iter().map(|item| list_item(py, item)); Ok(List { marker, start: list.start, items: pylist(py, items)? }) } -#[pyclass(frozen, get_all, module = "anydoc")] +#[pyclass(frozen, get_all, module = "anydoc_sznb")] pub struct ListItem { /// list[Block] blocks: Py, @@ -261,7 +268,7 @@ fn list_item(py: Python<'_>, item: model::ListItem) -> PyResult { /// Canonical table grid: every logical grid position appears exactly once. /// Content and spans live on the origin slot, and each position a span covers /// holds a `covered` slot pointing back at that origin. -#[pyclass(frozen, get_all, module = "anydoc")] +#[pyclass(frozen, get_all, module = "anydoc_sznb")] pub struct Table { /// list[list[CellSlot]] grid: Py, @@ -287,7 +294,7 @@ fn table(py: Python<'_>, table: model::Table) -> PyResult
{ }) } -#[pyclass(frozen, get_all, module = "anydoc")] +#[pyclass(frozen, get_all, module = "anydoc_sznb")] pub struct CellSlot { /// origin or covered. kind: &'static str, @@ -316,7 +323,7 @@ fn cell_slot(py: Python<'_>, slot: model::CellSlot) -> PyResult { }) } -#[pyclass(frozen, get_all, module = "anydoc")] +#[pyclass(frozen, get_all, module = "anydoc_sznb")] pub struct Cell { /// list[Block] blocks: Py, @@ -328,7 +335,7 @@ fn cell(py: Python<'_>, cell: model::Cell) -> PyResult { Ok(Cell { blocks: blocks(py, cell.blocks)?, col_span: cell.col_span, row_span: cell.row_span }) } -#[pyclass(frozen, get_all, module = "anydoc")] +#[pyclass(frozen, get_all, module = "anydoc_sznb")] pub struct Note { id: String, /// footnote or endnote. @@ -350,7 +357,7 @@ fn note(py: Python<'_>, note: model::Note) -> PyResult { /// An embedded binary asset (image, object payload). Bytes are always /// retained, so a document stays self-contained. -#[pyclass(frozen, get_all, module = "anydoc")] +#[pyclass(frozen, get_all, module = "anydoc_sznb")] pub struct Asset { /// Index into `Document.assets`, as referenced by an image source. id: usize, diff --git a/python/src/lib.rs b/python/src/lib.rs index a38a09de..ff7f333c 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -5,11 +5,12 @@ use std::path::PathBuf; use pyo3::create_exception; use pyo3::exceptions::{PyException, PyValueError}; use pyo3::prelude::*; +use pyo3::types::{PyBytes, PyList}; mod document; create_exception!( - anydoc, + anydoc_sznb, ConvertError, PyException, "A complete conversion was impossible. Catch this to handle every kind of \ @@ -18,14 +19,14 @@ create_exception!( ); create_exception!( - anydoc, + anydoc_sznb, UnsupportedError, ConvertError, "The format is unknown, or cannot be converted at all." ); create_exception!( - anydoc, + anydoc_sznb, NeedsOcrError, ConvertError, "Pages of a PDF are scanned or image-only and need OCR, which anydoc does \ @@ -34,7 +35,7 @@ create_exception!( ); create_exception!( - anydoc, + anydoc_sznb, MalformedError, ConvertError, "The document is structurally unusable: no meaningful content could be \ @@ -43,14 +44,14 @@ create_exception!( ); create_exception!( - anydoc, + anydoc_sznb, EncryptedError, ConvertError, "The document is encrypted or password-protected." ); create_exception!( - anydoc, + anydoc_sznb, ResourceLimitError, ConvertError, "A fixed safety limit was crossed: decompression, nesting depth, node \ @@ -58,7 +59,7 @@ create_exception!( ); create_exception!( - anydoc, + anydoc_sznb, MissingPartError, ConvertError, "A part required for any meaningful output is absent. `part` names it." @@ -170,11 +171,161 @@ fn to_markdown(py: Python<'_>, path: PathBuf) -> PyResult { /// Convert an in-memory document to Markdown. Without a format, it is /// detected from the content, which signature-less formats (CSV) have to name /// explicitly. +/// +/// Fork (sznb) keywords: `html_tables` (HTML `
` with spans vs GFM), +/// `dedup_cells` (merged content once vs repeated in every covered cell), +/// `include_toc` (keep vs drop the Word table of contents). #[pyfunction] -#[pyo3(signature = (data, format=None))] -fn to_markdown_bytes(py: Python<'_>, data: Vec, format: Option<&str>) -> PyResult { +#[pyo3(signature = (data, format=None, *, html_tables=true, dedup_cells=true, include_toc=true))] +fn to_markdown_bytes( + py: Python<'_>, + data: Vec, + format: Option<&str>, + html_tables: bool, + dedup_cells: bool, + include_toc: bool, +) -> PyResult { + let format = format.map(parse_format).transpose()?; + let opts = anydoc::ConvertOptions { html_tables, dedup_cells, include_toc }; + py.detach(|| anydoc::to_markdown_bytes_with(&data, format, opts)) + .map_err(|e| convert_error(py, e)) +} + +/// Fork (sznb): convert an in-memory document to plain text (no Markdown +/// syntax; tables render as tab-joined rows). Format handling as for +/// `to_markdown_bytes`; PDF has no document-model form and is unsupported. +/// Fork (sznb) keywords: `dedup_cells`, `include_toc` (as for +/// `to_markdown_bytes`). +#[pyfunction] +#[pyo3(signature = (data, format=None, *, dedup_cells=true, include_toc=true))] +fn to_text_bytes( + py: Python<'_>, + data: Vec, + format: Option<&str>, + dedup_cells: bool, + include_toc: bool, +) -> PyResult { + let format = format.map(parse_format).transpose()?; + let opts = anydoc::ConvertOptions { dedup_cells, include_toc, ..Default::default() }; + py.detach(|| anydoc::to_text_bytes_with(&data, format, opts)).map_err(|e| convert_error(py, e)) +} + +/// Fork (sznb): one image extracted by `to_markdown_with_images` / +/// `to_text_with_images`. +#[pyclass(frozen, get_all, module = "anydoc_sznb")] +pub struct ExtractedImage { + /// Content-addressed file name, e.g. `img_3f2a9c0d1b2e4f56.png`. + name: String, + /// The token standing in for the image in the rendered text. + placeholder: String, + /// MIME type (sniffed from the bytes when recognizable). + media_type: String, + /// Pixel width, `None` when the header gives none. + width: Option, + /// Pixel height, `None` when the header gives none. + height: Option, + /// The payload, exactly as embedded. + data: Py, + /// How many places in the document reference it. + occurrences: usize, + /// Package part or stream the first copy came from. + origin_part: String, +} + +/// Fork (sznb): rendered text plus the images its placeholders name. +#[pyclass(frozen, get_all, module = "anydoc_sznb")] +pub struct ImagesResult { + /// The Markdown or plain text, placeholders in place. + text: String, + /// Extracted images, in order of first appearance. + images: Py, + /// Image references dropped for being under the size threshold. + skipped: usize, +} + +fn images_result( + py: Python<'_>, + (text, report): (String, anydoc::ImageReport), +) -> PyResult { + let images = report + .images + .into_iter() + .map(|i| { + Py::new( + py, + ExtractedImage { + data: PyBytes::new(py, &i.bytes).unbind(), + name: i.name, + placeholder: i.placeholder, + media_type: i.media_type, + width: i.width, + height: i.height, + occurrences: i.occurrences, + origin_part: i.origin_part, + }, + ) + }) + .collect::>>()?; + let images = PyList::new(py, images)?.unbind(); + Ok(ImagesResult { text, images, skipped: report.skipped }) +} + +/// Fork (sznb): `to_markdown_bytes` plus image extraction. Embedded images +/// become placeholder tokens (`placeholder`, `{name}` = file name) in place; +/// images under `min_width` x `min_height` pixels are dropped (size unknown +/// = kept); names are `{name_prefix}{sha256[:16]}{ext}` (content-addressed, +/// so duplicates collapse). PDF yields its Markdown and no images. +#[pyfunction] +#[pyo3(signature = (data, format=None, *, html_tables=true, dedup_cells=true, include_toc=true, + min_width=50, min_height=50, placeholder="[IMG::{name}]".to_string(), + name_prefix="img_".to_string()))] +#[allow(clippy::too_many_arguments)] +fn to_markdown_with_images( + py: Python<'_>, + data: Vec, + format: Option<&str>, + html_tables: bool, + dedup_cells: bool, + include_toc: bool, + min_width: u32, + min_height: u32, + placeholder: String, + name_prefix: String, +) -> PyResult { + let format = format.map(parse_format).transpose()?; + let opts = anydoc::ConvertOptions { html_tables, dedup_cells, include_toc }; + let images = anydoc::ImageOptions { min_width, min_height, placeholder, name_prefix }; + let out = py + .detach(|| anydoc::to_markdown_with_images(&data, format, opts, &images)) + .map_err(|e| convert_error(py, e))?; + images_result(py, out) +} + +/// Fork (sznb): `to_text_bytes` plus image extraction (as for +/// `to_markdown_with_images`). Unsupported for `pdf`. +#[pyfunction] +#[pyo3(signature = (data, format=None, *, dedup_cells=true, include_toc=true, + min_width=50, min_height=50, placeholder="[IMG::{name}]".to_string(), + name_prefix="img_".to_string()))] +#[allow(clippy::too_many_arguments)] +fn to_text_with_images( + py: Python<'_>, + data: Vec, + format: Option<&str>, + dedup_cells: bool, + include_toc: bool, + min_width: u32, + min_height: u32, + placeholder: String, + name_prefix: String, +) -> PyResult { let format = format.map(parse_format).transpose()?; - py.detach(|| anydoc::to_markdown_bytes(&data, format)).map_err(|e| convert_error(py, e)) + let opts = anydoc::ConvertOptions { dedup_cells, include_toc, ..Default::default() }; + let images = anydoc::ImageOptions { min_width, min_height, placeholder, name_prefix }; + let out = py + .detach(|| anydoc::to_text_with_images(&data, format, opts, &images)) + .map_err(|e| convert_error(py, e))?; + images_result(py, out) } /// Parse an in-memory document into the document model, which also carries @@ -203,7 +354,12 @@ fn _anydoc(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(format_from_path, m)?)?; m.add_function(wrap_pyfunction!(to_markdown, m)?)?; m.add_function(wrap_pyfunction!(to_markdown_bytes, m)?)?; + m.add_function(wrap_pyfunction!(to_text_bytes, m)?)?; m.add_function(wrap_pyfunction!(to_document, m)?)?; + m.add_function(wrap_pyfunction!(to_markdown_with_images, m)?)?; + m.add_function(wrap_pyfunction!(to_text_with_images, m)?)?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/formats/doc/lists.rs b/src/formats/doc/lists.rs index d0648216..1495d9cb 100644 --- a/src/formats/doc/lists.rs +++ b/src/formats/doc/lists.rs @@ -265,6 +265,14 @@ fn marker_for_nfc(nfc: u8) -> Option { 3 => Some(MarkerKind::UpperAlpha), 4 => Some(MarkerKind::LowerAlpha), 23 => Some(MarkerKind::Bullet), + // Fork (sznb): CJK counting codes ([MS-OSHARED] MSONFC) render + // 一、二、三 / 壹、贰、叁 instead of degrading to 1、2、3 — WPS-era + // government documents use them heavily (0x27 chineseCountingThousand + // verified in the wild on real .wps files). + 0x0A | 0x0B | 0x11 | 0x21 | 0x23 | 0x24 | 0x25 | 0x27 => { + Some(MarkerKind::ChineseCounting) + } + 0x10 | 0x22 | 0x26 => Some(MarkerKind::ChineseLegal), 0xFF => None, _ => Some(MarkerKind::Decimal), } diff --git a/src/formats/doc/mod.rs b/src/formats/doc/mod.rs index 085a2c5f..d600aab6 100644 --- a/src/formats/doc/mod.rs +++ b/src/formats/doc/mod.rs @@ -18,7 +18,7 @@ use crate::shared::assets::AssetSink; use crate::shared::binary::{get_u16, get_u32, read_ole_stream, utf16le_units}; use crate::shared::blockstyle::StyledRun; use crate::shared::delta::rebase_emphasis; -use crate::shared::fields::{FieldFrame, field_result}; +use crate::shared::fields::{FieldFrame, TocTracker, drop_toc_title, field_result}; use crate::shared::grid::{CellProp, GridRow, build_edge_table}; use crate::shared::list::MarkerKind; use crate::shared::list::{ListEntry, ListKey, flush_list}; @@ -28,7 +28,14 @@ use std::collections::HashMap; use std::io::Cursor; use stsh::Stylesheet; +#[cfg(test)] pub fn parse(bytes: &[u8]) -> Result { + parse_with(bytes, true) +} + +/// Fork (sznb): `include_toc = false` drops the result paragraphs of a `TOC` +/// field (and the 目录 / Contents title right before it). +pub fn parse_with(bytes: &[u8], include_toc: bool) -> Result { let cursor = Cursor::new(bytes); let mut ole = cfb::CompoundFile::open(cursor) .map_err(|e| ConvertError::malformed(format!("not an OLE2 compound file: {e}")))?; @@ -113,6 +120,7 @@ pub fn parse(bytes: &[u8]) -> Result { counters: std::cell::RefCell::new(Counters::default()), data, assets: std::cell::RefCell::new(AssetSink::new()), + skip_toc: !include_toc, }; let blocks = assembler.build_blocks(0, main_end)?; let mut notes = Vec::new(); @@ -599,6 +607,8 @@ struct Assembler { /// The Data stream, for `sprmCPicLocation` picture payloads. data: Vec, assets: std::cell::RefCell, + /// Fork (sznb): drop the table of contents (the service's `content=false`). + skip_toc: bool, } /// A paragraph's resolved properties: the style chain's contribution merged @@ -700,6 +710,7 @@ impl Assembler { let mut row: Vec> = Vec::new(); let mut table_rows: Vec = Vec::new(); let mut para = ParaBuilder::new(); + let mut toc = TocTracker::default(); let mut i = lo; while i < hi.min(self.text.chars.len()) { @@ -715,6 +726,26 @@ impl Assembler { let pap = self.effective_pap(fc, i); let inlines = std::mem::replace(&mut para, ParaBuilder::new()).finish(); let is_cell_mark = c == '\u{7}'; + // Fork (sznb): content=false — a TOC field's result + // paragraphs (top level) go, with the title right before. + if self.skip_toc { + let (in_toc, started) = toc.finish_paragraph(); + if in_toc && !(pap.effective.in_table.unwrap_or(false) || is_cell_mark) { + Self::flush_table( + &mut blocks, + &mut table_rows, + &mut row, + &mut cell_blocks, + )?; + if started { + styled.flush(&mut blocks); + flush_list(&mut blocks, &mut list_run); + drop_toc_title(&mut blocks); + } + i += 1; + continue; + } + } if pap.effective.in_table.unwrap_or(false) || is_cell_mark { // A table is a hard boundary for top-level list and // styled runs, even while its rows are accumulated. @@ -771,9 +802,18 @@ impl Assembler { } } '\u{b}' => para.push_inline(Inline::LineBreak), - '\u{13}' => para.field_begin(), - '\u{14}' => para.field_separate(), - '\u{15}' => para.field_end(), + '\u{13}' => { + toc.begin(); + para.field_begin(); + } + '\u{14}' => { + toc.separate(); + para.field_separate(); + } + '\u{15}' => { + toc.end(); + para.field_end(); + } '\t' => { let style = self.char_style(fc, i); para.push_char(' ', style); @@ -792,6 +832,10 @@ impl Assembler { '\u{2}' | '\u{5}' | '\u{8}' | '\u{1f}' => {} c if c.is_control() => {} c => { + if toc.reading_instr() { + let mut buf = [0u8; 4]; + toc.instr_text(c.encode_utf8(&mut buf)); + } let style = self.char_style(fc, i); para.push_char(c, style); } @@ -1007,7 +1051,7 @@ impl Assembler { return Ok(()); } let rows = std::mem::take(table_rows).into_iter().map(DocRow::into_grid_row).collect(); - if let Some(block) = build_edge_table(rows)? { + if let Some(block) = build_edge_table(rows, true)? { blocks.push(block); } Ok(()) @@ -1025,6 +1069,10 @@ impl DocRow { fn into_grid_row(self) -> GridRow { let header = self.tap.as_ref().is_some_and(|t| t.header); let tap = self.tap.unwrap_or_default(); + // Fork (sznb): rgdxaCenter[0] is the row's left edge - indented rows + // keep their column offset instead of collapsing to column 0. + // Rows without a TAP have no boundaries -> None (column 0). + let left = tap.boundaries.first().map(|&b| b as i64); let cells = self .cells .into_iter() @@ -1045,7 +1093,7 @@ impl DocRow { (blocks, prop) }) .collect(); - GridRow { cells, header } + GridRow { cells, header, left } } } diff --git a/src/formats/docx/content.rs b/src/formats/docx/content.rs index c5a7c18b..fcd764ed 100644 --- a/src/formats/docx/content.rs +++ b/src/formats/docx/content.rs @@ -11,7 +11,7 @@ use crate::package::relationships::{RelTarget, Relationships, TargetMode, rel_ta use crate::package::xml::{Element, ns}; use crate::shared::blockstyle::{BlockStyle, StyledRun}; use crate::shared::delta::rebase_emphasis; -use crate::shared::fields::{FieldFrame, field_result}; +use crate::shared::fields::{FieldFrame, TocTracker, drop_toc_title, field_result}; use crate::shared::header::resolve_header_rows; use crate::shared::list::{ListEntry, ListKey, flush_list}; use crate::shared::math::{omath_para_to_tex, omath_to_tex}; @@ -45,6 +45,8 @@ pub(super) struct Ctx<'a, 'b> { pub numbering: &'b Numbering, pub counters: &'b RefCell, pub assets: &'b RefCell, + /// Fork (sznb): drop the table of contents (the service's `content=false`). + pub skip_toc: bool, } impl<'a, 'b> Ctx<'a, 'b> { @@ -59,6 +61,7 @@ impl<'a, 'b> Ctx<'a, 'b> { numbering: self.numbering, counters: self.counters, assets: self.assets, + skip_toc: self.skip_toc, } } @@ -112,8 +115,57 @@ pub(super) enum ParaKind { pub(super) struct Runs { list: Vec, styled: StyledRun, + /// Fork (sznb): complex-field state across sibling paragraphs, tracked + /// only when the TOC is skipped (a `TOC` field spans many paragraphs). + toc: TocTracker, } +/// Fork (sznb): feed one paragraph's field characters (document order) to +/// the tracker; returns `(in_toc, toc_started_here)`. +fn paragraph_in_toc(toc: &mut TocTracker, p: &Element) -> (bool, bool) { + let mut stack: Vec<&Element> = vec![p]; + while let Some(e) = stack.pop() { + if e.ns.as_deref() == Some(ns::W) { + match e.local.as_str() { + "fldChar" => { + match e.attr(ns::W, "fldCharType") { + Some("begin") => toc.begin(), + Some("separate") => toc.separate(), + Some("end") => toc.end(), + _ => {} + } + continue; + } + "instrText" => { + toc.instr_text(&e.text()); + continue; + } + "fldSimple" => { + if let Some(instr) = e.attr(ns::W, "instr") { + toc.simple(instr); + } + } + // Text boxes carry their own paragraphs and field chains. + "txbxContent" => continue, + _ => {} + } + } + let kids: Vec<&Element> = e.child_elems().collect(); + stack.extend(kids.into_iter().rev()); + } + toc.finish_paragraph() +} + +/// A `Table of Contents` building-block SDT (`w:docPartGallery`). +fn is_toc_sdt(sdt: &Element) -> bool { + sdt.find(ns::W, "sdtPr") + .and_then(|pr| pr.find(ns::W, "docPartObj")) + .and_then(|o| o.find(ns::W, "docPartGallery")) + .and_then(|g| g.attr(ns::W, "val")) + .is_some_and(|v| v.trim().to_ascii_lowercase().starts_with("table of contents")) +} + + impl Runs { fn flush(&mut self, blocks: &mut Vec) { self.styled.flush(blocks); @@ -159,6 +211,18 @@ fn collect_blocks( } match child.local.as_str() { "p" => { + if ctx.skip_toc { + let (in_toc, started) = paragraph_in_toc(&mut runs.toc, child); + if in_toc { + // Fork (sznb): content=false — the TOC's result + // paragraphs go, and so does the title right before. + if started { + runs.flush(blocks); + drop_toc_title(blocks); + } + continue; + } + } let (kind, pieces) = parse_paragraph(child, ctx)?; emit_paragraph(kind, pieces, blocks, runs); } @@ -167,6 +231,13 @@ fn collect_blocks( blocks.extend(parse_table(child, ctx)?); } "sdt" => { + // Fork (sznb): content=false drops a TOC building block whole, + // with the title right before it (outside the SDT). + if ctx.skip_toc && is_toc_sdt(child) { + runs.flush(blocks); + drop_toc_title(blocks); + continue; + } if let Some(content) = child.find(ns::W, "sdtContent") { collect_blocks(content, ctx, blocks, runs)?; } @@ -225,7 +296,16 @@ fn emit_paragraph(kind: ParaKind, pieces: Vec, blocks: &mut Vec, r } ParaKind::Plain => { runs.flush(blocks); - blocks.extend(pieces_into_blocks(pieces)); + let mut out = pieces_into_blocks(pieces); + // Fork (sznb): a visually empty paragraph still emits an empty + // block — the source uses blank paragraphs as vertical spacing + // and the original pipeline renders each as a blank line; + // dropping them here lost those line breaks (the renderer folds + // consecutive empties and trims trailing ones). + if out.is_empty() { + out.push(Block::Paragraph(Vec::new())); + } + blocks.extend(out); } } } @@ -253,7 +333,15 @@ fn parse_paragraph(p: &Element, ctx: &Ctx) -> Result<(ParaKind, Vec), Con // Numbering resolves independently of heading semantics: a numbered // heading advances its sequence and keeps its number visible. - let numbering = resolve_numbering(ppr, pstyle_id, ctx)?; + // Fork (sznb): an empty paragraph that only carries a section break + // (`w:pPr/w:sectPr`) takes no number. Word and LibreOffice show no + // label on it and do not advance the list, so a numbered style on that + // carrier (typically Heading 1 right before the first chapter) must not + // consume `第1章` and shift every chapter by one. + let section_carrier = + ppr.is_some_and(|pr| pr.find(ns::W, "sectPr").is_some()) && !has_visible_content(p); + let numbering = + if section_carrier { None } else { resolve_numbering(ppr, pstyle_id, ctx)? }; // Toggle properties: the paragraph style chain's true-parity flips the // docDefaults base. Headings use the same resolution as body text. @@ -287,6 +375,23 @@ fn parse_paragraph(p: &Element, ctx: &Ctx) -> Result<(ParaKind, Vec), Con Ok((kind, walker.finish())) } +/// Fork (sznb): whether a paragraph renders anything besides its mark — +/// non-empty text, a tab, a symbol, or a drawing/picture/OLE object. The +/// `pPr` subtree is skipped (its `w:tabs/w:tab` are tab stops, not content). +fn has_visible_content(p: &Element) -> bool { + let visible = |e: &Element| { + (e.is(ns::W, "t") && !e.text().trim().is_empty()) + || e.is(ns::W, "tab") + || e.is(ns::W, "sym") + || e.is(ns::W, "drawing") + || e.is(ns::W, "pict") + || e.is(ns::W, "object") + }; + p.child_elems() + .filter(|c| !c.is(ns::W, "pPr")) + .any(|c| visible(c) || c.descendant_elems().any(visible)) +} + /// Resolve a paragraph's effective numbering per ECMA-376: the direct /// `numPr` children are tri-state and merge property-by-property with the /// style-inherited `numPr` (a missing `numId`/`ilvl` inherits; an explicit @@ -585,6 +690,21 @@ impl<'a, 'b, 'e> InlineWalker<'a, 'b, 'e> { // object's identity and payload must win over its preview. if let Some(ole) = elem.first_descendant(ns::O_VML, "OLEObject") { let prog_id = ole.attr(ns::O_VML, "ProgID").unwrap_or("object").to_string(); + // Fork (sznb): an embedded excel.sheet.8 workbook carries its + // chart's data as a BIFF8 stream — extract the worksheets as + // tables instead of an "Embedded object" marker line (the data + // is the whole point of the chart in these documents). + if prog_id.to_ascii_lowercase().starts_with("excel.") + && let Some(rel_id) = ole.attr_qualified(ns::R, "id") + && let Some((_, bytes)) = self.ctx.rel_part(rel_id)? + && let Some(tables) = super::ole::workbook_tables(&bytes) + { + let blocks: Vec = tables.into_iter().map(Block::Table).collect(); + if !blocks.is_empty() { + self.push_blocks(blocks); + return Ok(()); + } + } let alt = if descr.trim().is_empty() { format!("Embedded object: {prog_id}") } else { @@ -625,20 +745,19 @@ impl<'a, 'b, 'e> InlineWalker<'a, 'b, 'e> { self.ctx.assets, rel_id, )?; - match source { - Some(source) => self.push(Inline::Image { alt: descr, source }), - None => { - if !descr.trim().is_empty() { - self.push(Inline::Image { alt: descr, source: ImageSource::Unavailable }); - } - } + // Fork (sznb): a bitmap's `descr` is not body text. Word/WPS fill + // it with the inserted file's name (「图片1」「捕获」 from + // 捕获.PNG), and rendering it as the image's alt text spliced + // those words into paragraphs. The original pipeline emits + // nothing for images; keep the asset, drop the alt text. + if let Some(source) = source { + self.push(Inline::Image { alt: String::new(), source }); } return Ok(()); } - if !descr.trim().is_empty() { - self.push(Inline::Image { alt: descr, source: ImageSource::Unavailable }); - } + // Fork (sznb): a drawing with neither text, chart, OLE nor bitmap — + // likewise no `descr` text (see above). Ok(()) } @@ -807,6 +926,7 @@ pub(super) fn parse_table(tbl: &Element, ctx: &Ctx) -> Result, Conver .count(); let mut builder = GridBuilder::new(); + builder.keep_empty_rows(); for row in &matrix { builder.next_row(); for tc in row { diff --git a/src/formats/docx/mod.rs b/src/formats/docx/mod.rs index eee5317e..549e70bd 100644 --- a/src/formats/docx/mod.rs +++ b/src/formats/docx/mod.rs @@ -5,6 +5,7 @@ mod content; mod numbering; +mod ole; mod styles; use crate::error::ConvertError; @@ -17,7 +18,15 @@ use content::Ctx; use numbering::Counters; use std::cell::RefCell; +#[cfg(test)] pub fn parse(bytes: &[u8]) -> Result { + parse_with(bytes, true) +} + +/// Fork (sznb): `include_toc = false` drops the table of contents — a +/// `Table of Contents` SDT and the result paragraphs of a bare `TOC` field +/// (plus the 目录 / Contents title right before it). +pub fn parse_with(bytes: &[u8], include_toc: bool) -> Result { let pkg = match Package::open(bytes) { Ok(p) => p, Err(e) => return Err(crate::package::archive::probe_ole(bytes).unwrap_or(e)), @@ -70,6 +79,7 @@ pub fn parse(bytes: &[u8]) -> Result { numbering: &numbering, counters: &counters, assets: &assets, + skip_toc: !include_toc, }; let blocks = content::parse_blocks(body, &ctx)?; @@ -330,4 +340,224 @@ mod tests { .collect(); assert_eq!(headings, vec!["1. Intro", "2. Details"]); } + + /// Fork (sznb): `(numId, text)` paragraphs over a single-level abstract + /// with label `第%1项`; `nums` is the `` list. Returns the text + /// output lines. + fn shared_abstract_lines(paras: &[(u32, &str)], nums: &str) -> Vec { + let body: String = paras + .iter() + .map(|(num_id, text)| { + format!( + r#"{text}"# + ) + }) + .collect(); + let document = format!(r#"{body}"#); + let numbering = format!( + r#" + + + + {nums} + "# + ); + let bytes = docx_parts(&[("word/document.xml", &document), ("word/numbering.xml", &numbering)]); + let text = crate::to_text_bytes(&bytes, crate::Format::Docx).unwrap(); + text.lines().map(str::trim).filter(|l| !l.is_empty()).map(str::to_string).collect() + } + + #[test] + fn num_ids_sharing_an_abstract_share_one_counter() { + // Word/LibreOffice: numIds over the same abstractNum continue one + // sequence (a numId is only an instance handle, not a new list). + let lines = shared_abstract_lines( + &[(1, "丁一"), (2, "丁二"), (1, "丁三")], + r#" + "#, + ); + let joined = lines.join("\n"); + for want in ["第1项", "第2项", "第3项"] { + assert!(joined.contains(want), "missing {want} in:\n{joined}"); + } + assert!(joined.find("第1项") < joined.find("丁一")); + assert!(joined.find("丁一") < joined.find("第2项") && joined.find("第2项") < joined.find("丁二")); + assert!(joined.find("丁二") < joined.find("第3项") && joined.find("第3项") < joined.find("丁三")); + } + + #[test] + fn start_override_restarts_the_shared_counter_once() { + // startOverride restarts the shared counter at its value on that + // numId's first use; later paragraphs (either numId) continue: 1, 5, 6. + let lines = shared_abstract_lines( + &[(1, "甲"), (2, "乙"), (1, "丙"), (2, "丁")], + r#" + + "#, + ); + let joined = lines.join("\n"); + let order: Vec = ["第1项", "甲", "第5项", "乙", "第6项", "丙", "第7项", "丁"] + .iter() + .map(|w| joined.find(w).unwrap_or_else(|| panic!("missing {w} in:\n{joined}"))) + .collect(); + assert!(order.windows(2).all(|w| w[0] < w[1]), "wrong order in:\n{joined}"); + } + + #[test] + fn orphan_sub_level_consumes_the_upper_start() { + // Fork (sznb): Word/LibreOffice — sub-level items ahead of any + // top-level item use up the top level's start, so the first real + // top-level item is 2 (技术使用手册: 1.1–1.4 then 「2. 网关接口调用」). + let p = |ilvl: u8, text: &str| { + format!( + r#"{text}"# + ) + }; + let body = [p(1, "甲"), p(1, "乙"), p(0, "丙"), p(1, "丁")].concat(); + let document = format!(r#"{body}"#); + let numbering = format!( + r#" + + + + + + "# + ); + let bytes = docx_parts(&[("word/document.xml", &document), ("word/numbering.xml", &numbering)]); + let text = crate::to_text_bytes(&bytes, crate::Format::Docx).unwrap(); + let order: Vec = ["1.1", "甲", "1.2", "乙", "第2项", "丙", "2.1", "丁"] + .iter() + .map(|w| text.find(w).unwrap_or_else(|| panic!("missing {w} in:\n{text}"))) + .collect(); + assert!(order.windows(2).all(|w| w[0] < w[1]), "wrong order in:\n{text}"); + assert!(!text.contains("第1项"), "top level must not restart at 1:\n{text}"); + } + + #[test] + fn empty_section_break_carrier_takes_no_number() { + // Fork (sznb): an empty numbered paragraph that only holds a section + // break (144pages: Heading 1 + sectPr before 第1章) shows no label + // and does not advance the list; a plain empty numbered paragraph + // still counts, as in Word/LibreOffice. + let num_p = |extra: &str, run: &str| { + format!( + r#"{extra}{run}"# + ) + }; + let body = [ + num_p("", ""), + num_p("", "甲"), + num_p("", ""), + num_p("", "乙"), + ] + .concat(); + let document = format!(r#"{body}"#); + let numbering = format!( + r#" + + + + + "# + ); + let bytes = docx_parts(&[("word/document.xml", &document), ("word/numbering.xml", &numbering)]); + let text = crate::to_text_bytes(&bytes, crate::Format::Docx).unwrap(); + assert!(text.contains("第1章 甲") || text.contains("第1章甲"), "got:\n{text}"); + assert!(text.contains("第3章 乙") || text.contains("第3章乙"), "got:\n{text}"); + } + + // ---------- Fork (sznb): include_toc / dedup_cells ---------- + + fn run_p(text: &str) -> String { + format!(r#"{text}"#) + } + + fn opts(dedup_cells: bool, include_toc: bool, html_tables: bool) -> crate::ConvertOptions { + crate::ConvertOptions { html_tables, dedup_cells, include_toc } + } + + #[test] + fn toc_sdt_is_dropped_only_without_include_toc() { + let sdt = format!( + r#" + {}{}"#, + run_p("目录"), + run_p("第一章 总则 1") + ); + // Custom title outside the SDT (技术使用手册: 「公共应用组件调用目录」). + let body = [run_p("封面"), run_p("公共应用组件调用目录"), sdt, run_p("正文开始")].concat(); + let document = format!(r#"{body}"#); + let bytes = docx_parts(&[("word/document.xml", &document)]); + let keep = crate::to_text_bytes_with(&bytes, crate::Format::Docx, opts(true, true, true)).unwrap(); + assert!(keep.contains("第一章 总则 1") && keep.contains("目录"), "got:\n{keep}"); + let drop = crate::to_text_bytes_with(&bytes, crate::Format::Docx, opts(true, false, true)).unwrap(); + assert!(!drop.contains("总则") && !drop.contains("目录"), "got:\n{drop}"); + assert!(keep.contains("公共应用组件调用目录"), "got:\n{keep}"); + assert!(drop.contains("封面") && drop.contains("正文开始"), "got:\n{drop}"); + } + + #[test] + fn bare_toc_field_paragraphs_and_title_are_dropped() { + // TOC field begins in the first entry paragraph and ends in the last + // one (Word's bare-field form, not wrapped in an SDT). + let fld = |kind: &str| format!(r#""#); + let first = format!( + r#"{} TOC \o "1-3" \h \z \u {}第1章 概况 3"#, + fld("begin"), + fld("separate") + ); + let middle = run_p("第2章 需求 9"); + let last = format!(r#"第3章 方案 20{}"#, fld("end")); + // A blank paragraph between title and field (bullet_problem). + let blank = "".to_string(); + let body = [run_p("目 录"), blank, first, middle, last, run_p("第1章 概况正文")].concat(); + let document = format!(r#"{body}"#); + let bytes = docx_parts(&[("word/document.xml", &document)]); + let keep = crate::to_markdown_bytes_with(&bytes, crate::Format::Docx, opts(true, true, true)).unwrap(); + assert!(keep.contains("第2章 需求 9") && keep.contains("目 录"), "got:\n{keep}"); + let drop = crate::to_markdown_bytes_with(&bytes, crate::Format::Docx, opts(true, false, true)).unwrap(); + for gone in ["目 录", "概况 3", "需求 9", "方案 20"] { + assert!(!drop.contains(gone), "{gone} survived:\n{drop}"); + } + assert!(drop.contains("第1章 概况正文"), "body lost:\n{drop}"); + } + + fn merged_table_docx() -> Vec { + // 3×3: row 0 「甲」 spans two columns; column 0 「乙」 spans rows 1–2. + let tc = |pr: &str, text: &str| format!(r#"{pr}{}"#, run_p(text)); + let rows = [ + [tc(r#""#, "甲"), tc("", "丙")].concat(), + [tc(r#""#, "乙"), tc("", "a"), tc("", "b")].concat(), + [tc("", ""), tc("", "c"), tc("", "d")].concat(), + ] + .map(|r| format!("{r}")) + .concat(); + let grid = r#""#; + let document = format!(r#"{grid}{rows}"#); + docx_parts(&[("word/document.xml", &document)]) + } + + #[test] + fn dedup_cells_false_repeats_merged_content() { + let bytes = merged_table_docx(); + let count = |s: &str, w: &str| s.matches(w).count(); + // Markdown, HTML form: spans by default, repeated cells without dedup. + let html = crate::to_markdown_bytes_with(&bytes, crate::Format::Docx, opts(true, true, true)).unwrap(); + assert!(html.contains("colspan") && html.contains("rowspan"), "got:\n{html}"); + assert_eq!((count(&html, "甲"), count(&html, "乙")), (1, 1), "got:\n{html}"); + let html = crate::to_markdown_bytes_with(&bytes, crate::Format::Docx, opts(false, true, true)).unwrap(); + assert!(!html.contains("colspan") && !html.contains("rowspan"), "got:\n{html}"); + assert_eq!((count(&html, "甲"), count(&html, "乙")), (2, 2), "got:\n{html}"); + // GFM keeps three columns either way. + let gfm = crate::to_markdown_bytes_with(&bytes, crate::Format::Docx, opts(false, true, false)).unwrap(); + assert_eq!((count(&gfm, "甲"), count(&gfm, "乙")), (2, 2), "got:\n{gfm}"); + assert!(gfm.lines().all(|l| !l.starts_with('|') || l.matches('|').count() == 4), "got:\n{gfm}"); + // Plain text. + let txt = crate::to_text_bytes_with(&bytes, crate::Format::Docx, opts(true, true, true)).unwrap(); + assert_eq!((count(&txt, "甲"), count(&txt, "乙")), (1, 1), "got:\n{txt}"); + let txt = crate::to_text_bytes_with(&bytes, crate::Format::Docx, opts(false, true, true)).unwrap(); + assert_eq!((count(&txt, "甲"), count(&txt, "乙")), (2, 2), "got:\n{txt}"); + assert!(txt.contains("甲\t甲\t丙") && txt.contains("乙\tc\td"), "got:\n{txt}"); + } } diff --git a/src/formats/docx/numbering.rs b/src/formats/docx/numbering.rs index a6334008..a42b35eb 100644 --- a/src/formats/docx/numbering.rs +++ b/src/formats/docx/numbering.rs @@ -6,7 +6,7 @@ use crate::error::ConvertError; use crate::package::xml::{Element, ns}; use crate::shared::list::MarkerKind; use crate::shared::numbering::{NumberPattern, composite_label, parse_percent_pattern}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; pub const LEVELS: usize = 9; @@ -36,6 +36,8 @@ impl Default for LevelDef { #[derive(Debug, Clone)] struct AbstractNum { + /// `w:abstractNumId`: the identity numbering counters are shared under. + id: String, levels: [LevelDef; LEVELS], /// Paragraph style each level binds to via `w:lvl > w:pStyle`. pstyles: [Option; LEVELS], @@ -46,6 +48,16 @@ struct AbstractNum { pub struct Instance { pub levels: [LevelDef; LEVELS], pstyles: [Option; LEVELS], + /// Fork (sznb): id of the resolved abstract (after `numStyleLink` + /// indirection). Word/LibreOffice share one counter across every `numId` + /// pointing at the same abstractNum -- a `numId` is only a list-instance + /// handle -- so counters are keyed by this, not by `numId`. + counter_key: String, + /// Fork (sznb): `w:lvlOverride > w:startOverride` per level. It restarts + /// the shared counter at this value the first time this `numId` is used at + /// that level; later paragraphs (of any `numId` on the abstract) continue + /// from there (LibreOffice: 1 -> 5 (override) -> 6). + start_overrides: [Option; LEVELS], } impl Instance { @@ -89,7 +101,7 @@ pub fn parse( } let num_style_link = abs.find(ns::W, "numStyleLink").and_then(|e| e.attr(ns::W, "val")).map(str::to_string); - abstracts.insert(id, AbstractNum { levels, pstyles, num_style_link }); + abstracts.insert(id, AbstractNum { id: id.to_string(), levels, pstyles, num_style_link }); } // First pass: direct abstract references per numId. @@ -112,6 +124,7 @@ pub fn parse( }; let mut levels = abs.levels.clone(); let mut pstyles = abs.pstyles.clone(); + let mut start_overrides: [Option; LEVELS] = [None; LEVELS]; for over in num_elem.find_all(ns::W, "lvlOverride") { let ilvl: usize = over.attr(ns::W, "ilvl").and_then(|v| v.parse().ok()).unwrap_or(0); if ilvl >= LEVELS { @@ -129,9 +142,13 @@ pub fn parse( .and_then(parse_start) { levels[ilvl].start = start; + start_overrides[ilvl] = Some(start); } } - numbering.instances.insert(num_id, Instance { levels, pstyles }); + numbering.instances.insert( + num_id, + Instance { levels, pstyles, counter_key: abs.id.clone(), start_overrides }, + ); } Ok(numbering) } @@ -189,6 +206,14 @@ fn parse_level(lvl: &Element) -> LevelDef { "upperLetter" => Some(MarkerKind::UpperAlpha), "lowerRoman" => Some(MarkerKind::LowerRoman), "upperRoman" => Some(MarkerKind::UpperRoman), + // Fork (sznb): CJK counting families render 一、二、三 / 壹、贰、叁 + // instead of degrading to 1、2、3. + "chineseCounting" | "chineseCountingThousand" | "ideographDigital" + | "japaneseCounting" | "japaneseDigitalTenThousand" | "taiwaneseCounting" + | "taiwaneseCountingThousand" | "taiwaneseDigital" => Some(MarkerKind::ChineseCounting), + "chineseLegalSimplified" | "ideographLegalTraditional" | "japaneseLegal" => { + Some(MarkerKind::ChineseLegal) + } _ => Some(MarkerKind::Decimal), }; let start = lvl @@ -214,11 +239,15 @@ fn parse_level(lvl: &Element) -> LevelDef { LevelDef { marker, start, restart, pattern: NumberPattern { text, legal } } } -/// Document-order numbering state: one counter array per instance, shared -/// across interruptions so continuation works, with `lvlRestart` semantics. +/// Document-order numbering state: one counter array per abstract numbering +/// definition (shared by all `numId`s referencing it, as Word/LibreOffice do), +/// kept across interruptions so continuation works, with `lvlRestart` +/// semantics. #[derive(Default)] pub struct Counters { - state: HashMap, + state: HashMap, + /// `(numId, ilvl)` pairs whose `startOverride` restart has been applied. + applied_overrides: HashSet<(u64, usize)>, } #[derive(Default)] @@ -234,15 +263,32 @@ impl Counters { /// reproducible from the marker kind alone, the composite label. pub fn next(&mut self, num_id: u64, ilvl: usize, instance: &Instance) -> (u64, Option) { let ilvl = ilvl.min(LEVELS - 1); - let state = self.state.entry(num_id).or_default(); + let override_start = instance.start_overrides[ilvl] + .filter(|_| self.applied_overrides.insert((num_id, ilvl))); + let state = self.state.entry(instance.counter_key.clone()).or_default(); let def = &instance.levels[ilvl]; - if !state.initialized[ilvl] || state.restart_pending[ilvl] { + if let Some(start) = override_start { + state.value[ilvl] = start; + state.initialized[ilvl] = true; + state.restart_pending[ilvl] = false; + } else if !state.initialized[ilvl] || state.restart_pending[ilvl] { state.value[ilvl] = def.start; state.initialized[ilvl] = true; state.restart_pending[ilvl] = false; } else { state.value[ilvl] = state.value[ilvl].saturating_add(1); } + // Fork (sznb): an orphan deeper item consumes each unadvanced upper + // level at its start value (Word/LibreOffice: ilvl1, ilvl1, ilvl0, + // ilvl1 → 1.1 / 1.2 / 2 / 2.1). Upstream only *rendered* the start + // without recording it, so the first real upper item restarted at 1. + for upper in 0..ilvl { + if !state.initialized[upper] || state.restart_pending[upper] { + state.value[upper] = instance.levels[upper].start; + state.initialized[upper] = true; + state.restart_pending[upper] = false; + } + } // Using level `ilvl` schedules restarts for deeper levels according // to their lvlRestart settings. for (deeper, level_def) in instance.levels.iter().enumerate().skip(ilvl + 1) { diff --git a/src/formats/docx/ole.rs b/src/formats/docx/ole.rs new file mode 100644 index 00000000..b4677b7e --- /dev/null +++ b/src/formats/docx/ole.rs @@ -0,0 +1,601 @@ +//! Fork (sznb): embedded OLE workbook extraction (`excel.sheet.8`). +//! +//! Old-style Word OLE charts embed a complete Excel workbook as a BIFF8 +//! `Workbook` stream inside an OLE compound file (`embeddings/oleObject*.bin`). +//! The stock parser only emitted an "Embedded object" marker line — the data +//! behind every such chart in a document was lost. This module reads the CFB +//! container, extracts the `Workbook` stream, and lifts each worksheet into a +//! [`Table`] (values only, merged ranges included). + +use crate::model::{Block, Cell, CellSlot, Inline, Style, Table, TableKind}; + +// ---------- OLE compound file (CFB) minimal reader ---------- + +const SIG: [u8; 8] = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]; +const FREESECT: u32 = 0xFFFFFFFF; +const ENDCHECK: u32 = 0xFFFFFFFE; + +struct Cfb { + data: Vec, + ss: usize, // sector size + mss: usize, // mini sector size + fat: Vec, + mini_fat: Vec, + ministream: Vec, + entries: Vec<(String, u32, u64)>, // name, start sector, size +} + +/// Bounds-checked reads: record bodies may be truncated at the stream end, +/// and untrusted data must never panic the parser. +fn u16at(d: &[u8], o: usize) -> u16 { + if o + 2 <= d.len() { + u16::from_le_bytes([d[o], d[o + 1]]) + } else { + 0 + } +} +fn u32at(d: &[u8], o: usize) -> u32 { + if o + 4 <= d.len() { + u32::from_le_bytes([d[o], d[o + 1], d[o + 2], d[o + 3]]) + } else { + 0 + } +} + +impl Cfb { + fn open(data: &[u8]) -> Option { + if data.len() < 512 || data[..8] != SIG { + return None; + } + let ss = 1usize << u16at(data, 30) as usize; + let mss = 1usize << u16at(data, 32) as usize; + let num_fat = u32at(data, 44) as usize; + let first_dir = u32at(data, 48); + let cutoff = u32at(data, 56) as u64; + let first_mini = u32at(data, 60); + let first_difat = u32at(data, 68); + let _num_difat = u32at(data, 72) as usize; + let sec = |n: u32| -> usize { 512 + n as usize * ss }; + + // DIFAT: 109 entries in the header, then a chain of DIFAT sectors. + let mut difat: Vec = Vec::with_capacity(num_fat); + for i in 0..109 { + let v = u32at(data, 76 + i * 4); + if v != FREESECT { + difat.push(v); + } + } + let mut cur = first_difat; + let mut guard = 0; + while cur != FREESECT && cur != ENDCHECK && guard < 1 << 16 { + let base = sec(cur); + for i in 0..(ss / 4 - 1) { + let v = u32at(data, base + i * 4); + if v != FREESECT { + difat.push(v); + } + } + cur = u32at(data, base + ss - 4); + guard += 1; + } + difat.truncate(num_fat.max(difat.len().min(num_fat))); + + // FAT sectors hold the sector chains. + let mut fat: Vec = Vec::new(); + for fs in difat.iter().take(num_fat) { + let base = sec(*fs); + if base + ss > data.len() { + return None; + } + for i in 0..(ss / 4) { + fat.push(u32at(data, base + i * 4)); + } + } + + let chain = |fat: &[u32], mut s: u32| -> Vec { + let mut out = Vec::new(); + let mut guard = 0; + while (s as usize) < fat.len() && fat[s as usize] != ENDCHECK && guard < 1 << 22 { + out.push(s); + s = fat[s as usize]; + guard += 1; + } + out + }; + + // Directory stream. + let dir_secs = chain(&fat, first_dir); + let mut dir = Vec::with_capacity(dir_secs.len() * ss); + for s in &dir_secs { + let b = sec(*s); + if b + ss > data.len() { + return None; + } + dir.extend_from_slice(&data[b..b + ss]); + } + + let mut entries = Vec::new(); + let mut root: Option<(u32, u64)> = None; + for e in dir.chunks(128) { + if e.len() < 128 { + break; + } + let nlen = u16at(e, 64) as usize; + if nlen < 2 || nlen > 64 { + continue; + } + // Directory entry names are UTF-16LE (nlen includes the NUL). + let name = { + let units: Vec = e[..nlen - 2] + .chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]])) + .collect(); + String::from_utf16_lossy(&units) + }; + let kind = e[66]; + let start = u32at(e, 116); + let size = u64::from(u32at(e, 120)) | (u64::from(u32at(e, 124)) << 32); + if kind == 5 && name == "Root Entry" { + root = Some((start, size)); + } + if kind == 2 { + entries.push((name, start, size)); + } + } + + // Mini stream + mini FAT. + let ministream = root + .and_then(|(start, size)| { + let secs = chain(&fat, start); + let mut buf = Vec::with_capacity(size as usize); + for s in secs { + let b = sec(s); + if b + ss > data.len() { + return None; + } + buf.extend_from_slice(&data[b..b + ss]); + } + buf.truncate(size as usize); + Some(buf) + }) + .unwrap_or_default(); + let mut mini_fat = Vec::new(); + for s in chain(&fat, first_mini) { + let b = sec(s); + if b + ss > data.len() { + return None; + } + for i in 0..(ss / 4) { + mini_fat.push(u32at(data, b + i * 4)); + } + } + + let _ = cutoff; + Some(Cfb { data: data.to_vec(), ss, mss, fat, mini_fat, ministream, entries }) + } + + fn stream(&self, want: &str) -> Option> { + let (name, start, size) = self.entries.iter().find(|(n, _, _)| n == want)?; + let _ = name; + let size = *size as usize; + if size >= 4096 { + let mut out = Vec::with_capacity(size); + let mut s = *start; + let mut guard = 0; + while (s as usize) < self.fat.len() && self.fat[s as usize] != ENDCHECK && guard < 1 << 22 { + let b = 512 + s as usize * self.ss; + if b + self.ss > self.data.len() { + return None; + } + out.extend_from_slice(&self.data[b..b + self.ss]); + s = self.fat[s as usize]; + guard += 1; + } + out.truncate(size); + Some(out) + } else { + let mut out = Vec::with_capacity(size); + let mut s = *start; + let mut guard = 0; + while (s as usize) < self.mini_fat.len() + && self.mini_fat[s as usize] != ENDCHECK + && guard < 1 << 22 + { + let b = s as usize * self.mss; + if b + self.mss > self.ministream.len() { + return None; + } + out.extend_from_slice(&self.ministream[b..b + self.mss]); + s = self.mini_fat[s as usize]; + guard += 1; + } + out.truncate(size); + Some(out) + } + } +} + +// ---------- BIFF8 minimal extraction ---------- + +struct Segs { + parts: Vec>, + seg: usize, + pos: usize, +} + +impl Segs { + fn u8(&mut self) -> Option { + loop { + let p = self.parts.get(self.seg)?; + if self.pos < p.len() { + let b = p[self.pos]; + self.pos += 1; + return Some(b); + } + self.seg += 1; + self.pos = 0; + } + } + fn u16(&mut self) -> Option { + let lo = self.u8()?; + let hi = self.u8()?; + Some(u16::from_le_bytes([lo, hi])) + } + fn u32(&mut self) -> Option { + let a = self.u16()?; + let b = self.u16()?; + Some(u32::from_le_bytes([a.to_le_bytes()[0], a.to_le_bytes()[1], + b.to_le_bytes()[0], b.to_le_bytes()[1]])) + } + /// Read `chars` characters; at a segment boundary the next byte selects + /// 8/16-bit for the remainder (CONTINUE semantics). + fn xl_string(&mut self, chars: usize, wide: bool) -> Option { + let mut out = String::new(); + let mut wide = wide; + let mut done = 0; + while done < chars { + // Advance past exhausted segments (get() keeps this in bounds). + while self.parts.get(self.seg).is_some_and(|p| self.pos >= p.len()) { + self.seg += 1; + self.pos = 0; + } + if self.parts.get(self.seg).is_none() { + return Some(out); + } + // A segment boundary mid-string begins with a fresh flags byte. + if self.pos == 0 && self.seg > 0 { + let Some(flags) = self.u8() else { return Some(out) }; + wide = flags & 1 == 1; + } + let ch = if wide { + let Some(c) = self.u16() else { return Some(out) }; + char::from_u32(u32::from(c)).unwrap_or('\u{FFFD}') + } else { + let Some(c) = self.u8() else { return Some(out) }; + char::from_u32(u32::from(c)).unwrap_or('\u{FFFD}') + }; + out.push(ch); + done += 1; + } + Some(out) + } +} + +/// Short XLUnicodeString in a record body: cch u16, grbit u16. +fn short_string(rec: &[u8]) -> String { + if rec.len() < 4 { + return String::new(); + } + let cch = u16at(rec, 0) as usize; + let wide = rec[2] & 1 == 1; + let bytes = if wide { rec[4..].to_vec() } else { rec[3..].to_vec() }; + let mut segs = Segs { parts: vec![bytes], seg: 0, pos: 0 }; + segs.xl_string(cch, wide).unwrap_or_default() +} + +fn rk_value(v: u32) -> f64 { + if v & 2 != 0 { + let n = (v >> 2) as i32; + let mut f = n as f64; + if v & 1 != 0 { + f /= 100.0; + } + f + } else { + let bits = (u64::from(v & 0xFFFFFFFC) << 32); + let mut f = f64::from_bits(bits); + if v & 1 != 0 { + f /= 100.0; + } + f + } +} + +fn num_text(f: f64) -> String { + if f == f.trunc() && f.abs() < 1e15 { + format!("{}", f as i64) + } else { + format!("{f}") + } +} + +struct SheetBuilder { + cells: Vec>>, // [row][col] + merged: Vec<(usize, usize, usize, usize)>, // r1, r2, c1, c2 inclusive +} + +impl SheetBuilder { + fn put(&mut self, r: usize, c: usize, v: String) { + if self.cells.len() <= r { + self.cells.resize(r + 1, Vec::new()); + } + if self.cells[r].len() <= c { + self.cells[r].resize(c + 1, None); + } + self.cells[r][c] = Some(v); + } + + fn build(self) -> Option
{ + let rows = self.cells.len(); + let cols = self.cells.iter().map(|r| r.len()).max()?; // cols == 0 -> None + // Trim trailing all-empty rows/cols. + let last_row = (0..rows).rev().find(|&r| self.cells[r].iter().any(|c| c.is_some()))?; + let last_col = (0..cols).rev().find(|&c| + self.cells.iter().any(|r| r.get(c).is_some_and(|v| v.is_some())))?; + let covered = |r: usize, c: usize| { + self.merged.iter().any(|&(r1, r2, c1, c2)| { + r >= r1 && r <= r2 && c >= c1 && c <= c2 && !(r == r1 && c == c1) + }) + }; + let span_of = |r: usize, c: usize| { + self.merged.iter().find_map(|&(r1, r2, c1, c2)| { + (r == r1 && c == c1).then_some((c2 - c1 + 1, r2 - r1 + 1)) + }) + }; + let mut grid: Vec> = Vec::with_capacity(last_row + 1); + let mut any = false; + for r in 0..=last_row { + let mut row = Vec::with_capacity(last_col + 1); + for c in 0..=last_col { + if covered(r, c) { + // Find its origin for the back-reference. + let org = self.merged.iter().find(|&&(r1, r2, c1, c2)| { + r >= r1 && r <= r2 && c >= c1 && c <= c2 + }); + let (or, oc) = org.map(|&(r1, _r2, c1, _c2)| (r1, c1)).unwrap_or((r, c)); + row.push(CellSlot::Covered { origin_row: or, origin_col: oc }); + } else { + let text = self.cells[r].get(c).and_then(|v| v.clone()).unwrap_or_default(); + if !text.is_empty() { + any = true; + } + let (col_span, row_span) = span_of(r, c).unwrap_or((1, 1)); + row.push(CellSlot::Origin(Cell { + blocks: text_to_block(&text), + col_span: col_span as u32, + row_span: row_span as u32, + })); + } + } + grid.push(row); + } + any.then(|| Table { grid, header_rows: 0, kind: TableKind::Data, keep_empty_rows: false }) + } +} + +fn text_to_block(text: &str) -> Vec { + if text.is_empty() { + return Vec::new(); + } + vec![Block::Paragraph(vec![Inline::Text { text: text.to_string(), style: Style::PLAIN }])] +} + +fn biff_tables(wb: &[u8]) -> Vec
{ + // Pass 1 (workbook globals): SST + BOUNDSHEET positions. + let mut sst: Vec = Vec::new(); + let mut sheets: Vec = Vec::new(); + let mut pos = 0usize; + let mut sst_pending: Vec> = Vec::new(); + while pos + 4 <= wb.len() { + let t = u16at(wb, pos); + let len = u16at(wb, pos + 2) as usize; + let body = &wb[pos + 4..(pos + 4 + len).min(wb.len())]; + match t { + 0x0809 if pos == 0 => {} // workbook globals BOF + 0x0085 => { + // BOUNDSHEET: lbPlyPos u32 | state u16 | sheetType u8 | name + if body.len() >= 7 { + sheets.push(u32at(body, 0) as usize); + } + } + 0x00FC => { + sst_pending.clear(); + sst_pending.push(body.to_vec()); + } + 0x003C if !sst_pending.is_empty() => { + sst_pending.push(body.to_vec()); + } + 0x000A => { + if !sst_pending.is_empty() { + sst = parse_sst(&sst_pending); + sst_pending.clear(); + } + if !sheets.is_empty() || pos > 0 { + break; // end of globals + } + } + _ => {} + } + pos += 4 + len; + } + + let mut out = Vec::new(); + for start in sheets { + if let Some(tb) = parse_sheet(wb, start, &sst) { + out.push(tb); + } + } + out +} + +fn parse_sst(parts: &[Vec]) -> Vec { + let first = &parts[0]; + if first.len() < 8 { + return Vec::new(); + } + let unique = u32at(first, 4) as usize; + // String data starts at offset 8 of the first part. + let mut segs = Segs { + parts: { + let mut v: Vec> = parts.to_vec(); + v[0] = first[8..].to_vec(); + v + }, + seg: 0, + pos: 0, + }; + let mut out = Vec::with_capacity(unique.min(1 << 20)); + for _ in 0..unique { + let Some(cch) = segs.u16() else { break }; + // One flags byte, not the spec's two: WPS-ET-authored BIFF writes + // the SST grbit as a single byte (verified byte-level on the + // government-corpus files — a 2-byte read shifts every string). + let Some(flags) = segs.u8() else { break }; + let wide = flags & 1 == 1; + let rich = flags & 8 != 0; + let far_east = flags & 4 != 0; + let c_run = if rich { segs.u16().unwrap_or(0) as usize } else { 0 }; + let cb_ext = if far_east { segs.u32().unwrap_or(0) as usize } else { 0 }; + let s = segs.xl_string(cch as usize, wide).unwrap_or_default(); + // 4 bytes per rich run, then the Far-East string blob. + for _ in 0..(4 * c_run + cb_ext) { + if segs.u8().is_none() { + break; + } + } + out.push(s); + } + out +} + +fn parse_sheet(wb: &[u8], start: usize, sst: &[String]) -> Option
{ + let mut b = SheetBuilder { cells: Vec::new(), merged: Vec::new() }; + // A FORMULA whose result is a string is followed by a STRING record + // carrying the value for that (row, col). + let mut pending_string_formula: Option<(usize, usize)> = None; + let mut pos = start; + while pos + 4 <= wb.len() { + let t = u16at(wb, pos); + let len = u16at(wb, pos + 2) as usize; + let body = &wb[pos + 4..(pos + 4 + len).min(wb.len())]; + match t { + 0x00FD => if body.len() >= 10 { // LABELSST: row, col, ixfe, isst + let r = u16at(body, 0) as usize; + let c = u16at(body, 2) as usize; + let idx = u32at(body, 6) as usize; + if let Some(s) = sst.get(idx) { + b.put(r, c, s.clone()); + } + }, + 0x0203 => if body.len() >= 14 { // NUMBER + let r = u16at(body, 0) as usize; + let c = u16at(body, 2) as usize; + let mut b8 = [0u8; 8]; + b8.copy_from_slice(&body[6..14]); + b.put(r, c, num_text(f64::from_le_bytes(b8))); + }, + 0x027E => if body.len() >= 10 { // RK + let r = u16at(body, 0) as usize; + let c = u16at(body, 2) as usize; + b.put(r, c, num_text(rk_value(u32at(body, 6)))); + }, + 0x00BD => { // MULRK. Standard: row rowLast colFirst {IXFE,RK}* colLast + // (len = N*6+8). WPS-ET variant omits colFirst entirely + // (len = N*6+6); its first column is colLast-N+1 — verified + // byte-level on the government corpus (row 1: cols 1..3, + // 56.67 / -14.4 / 0.4). + let blen = body.len(); + if blen >= 12 && blen % 6 == 0 { + let n = (blen - 6) / 6; + let r = u16at(body, 0) as usize; + let c2 = u16at(body, blen - 2) as usize; + if c2 + 1 >= n { + let c1 = c2 + 1 - n; + for i in 0..n { + let off = 4 + i * 6; + b.put(r, c1 + i, num_text(rk_value(u32at(body, off + 2)))); + } + } + } else if blen >= 8 { + let r = u16at(body, 0) as usize; + let c1 = u16at(body, 4) as usize; + let c2 = u16at(body, blen - 2) as usize; + for c in c1..=c2 { + let off = 6 + (c - c1) * 6; + if off + 6 > blen { + break; + } + b.put(r, c, num_text(rk_value(u32at(body, off + 2)))); + } + } + }, + 0x0006 => if body.len() >= 14 { // FORMULA + let r = u16at(body, 0) as usize; + let c = u16at(body, 2) as usize; + let mut b8 = [0u8; 8]; + b8.copy_from_slice(&body[6..14]); + let f = f64::from_le_bytes(b8); + if f.is_finite() { + b.put(r, c, num_text(f)); + } else { + // Non-numeric result: the value follows in a STRING record. + pending_string_formula = Some((r, c)); + } + }, + 0x0207 => { // STRING: value of the preceding string-result FORMULA + if let Some((r, c)) = pending_string_formula.take() + && body.len() >= 3 + { + let cch = u16at(body, 0) as usize; + let wide = body[2] & 1 == 1; + let data = &body[3..]; + let mut segs = Segs { parts: vec![data.to_vec()], seg: 0, pos: 0 }; + b.put(r, c, segs.xl_string(cch, wide).unwrap_or_default()); + } + }, + 0x0204 | 0x00D6 => if body.len() >= 6 { // LABEL / RSTRING + let r = u16at(body, 0) as usize; + let c = u16at(body, 2) as usize; + b.put(r, c, short_string(&body[6..])); + }, + 0x00E5 => { // MERGEDCELLS + if body.len() >= 2 { + let n = u16at(body, 0) as usize; + for i in 0..n { + let o = 2 + i * 8; + if o + 8 <= body.len() { + b.merged.push(( + u16at(body, o) as usize, + u16at(body, o + 2) as usize, + u16at(body, o + 4) as usize, + u16at(body, o + 6) as usize, + )); + } + } + } + } + 0x000A => break, // EOF: sheet done + _ => {} + } + pos += 4 + len; + } + b.build() +} + +/// Extract worksheet tables from an `excel.sheet.8` OLE payload. +pub fn workbook_tables(bytes: &[u8]) -> Option> { + let cfb = Cfb::open(bytes)?; + let wb = cfb.stream("Workbook")?; + let tables = biff_tables(&wb); + (!tables.is_empty()).then_some(tables) +} diff --git a/src/formats/mod.rs b/src/formats/mod.rs index d570b264..6d95e7da 100644 --- a/src/formats/mod.rs +++ b/src/formats/mod.rs @@ -17,17 +17,24 @@ use crate::error::ConvertError; use crate::model::Document; pub fn parse(bytes: &[u8], format: Format) -> Result { + parse_with(bytes, format, true) +} + +/// Fork (sznb): parse with the table of contents kept (`include_toc = true`, +/// the historical behaviour) or dropped. Only the Word frontends (docx, doc / +/// wps) carry a TOC construct; the flag is inert for every other format. +pub fn parse_with(bytes: &[u8], format: Format, include_toc: bool) -> Result { match format { Format::Excel => sheet::parse(bytes), Format::Csv => csv::parse(bytes), - Format::Docx => docx::parse(bytes), + Format::Docx => docx::parse_with(bytes, include_toc), Format::Odt | Format::Ods | Format::Odp => odf::parse(bytes), Format::Pptx => pptx::parse(bytes), Format::Epub => epub::parse(bytes), Format::Rtf => rtf::parse(bytes), // RTF files wearing a .doc extension are common in the wild. Format::Doc if bytes.starts_with(b"{\\rtf") => rtf::parse(bytes), - Format::Doc => doc::parse(bytes), + Format::Doc => doc::parse_with(bytes, include_toc), Format::Ppt => ppt::parse(bytes), // pdf-inspector produces Markdown directly; there is no document // model for PDFs. `to_markdown_bytes` routes them to `pdf`. diff --git a/src/formats/ppt/mod.rs b/src/formats/ppt/mod.rs index 0b84ede2..edf722c7 100644 --- a/src/formats/ppt/mod.rs +++ b/src/formats/ppt/mod.rs @@ -9,10 +9,11 @@ mod styletext; use crate::error::ConvertError; -use crate::model::{Block, Document, Inline, Style, inlines_are_empty}; +use crate::model::{Block, Cell, Document, GridBuilder, Inline, Style, TableKind, inlines_are_empty}; use crate::package::limits; use crate::shared::binary::{get_u32, read_ole_stream, utf16le_units}; use crate::shared::delta::{StyleDelta, rebase_emphasis}; +use crate::shared::header::resolve_header_rows; use crate::shared::list::{ListEntry, ListKey, MarkerKind, flush_list}; use crate::shared::officeart::record_at; use crate::shared::text::clean_text; @@ -414,6 +415,14 @@ impl Extractor { 0x03F0 | 0x03F8 | 0x0FC9 => {} // Only instance 0 of SlideListWithText holds slide text here. 0x0FF0 if ver_inst >> 4 != 0 => {} + // Fork (sznb): a table is a group shape flagged by the + // tableProperties property; rebuild its grid from the cell + // rectangles instead of emitting every cell as a paragraph. + 0xF003 if is_table_group(body) => { + if let Some(table) = self.table_from_group(body)? { + self.current.push(table); + } + } _ => { if stack.len() >= limits::MAX_RECORD_DEPTH { return Err(ConvertError::ResourceLimit { @@ -445,6 +454,109 @@ impl Extractor { Ok(()) } + /// Fork (sznb): text of one shape container as blocks of its own, with + /// the slide-level text state set aside so nothing leaks across. + fn capture(&mut self, body: &[u8]) -> Result, ConvertError> { + self.flush_shape(); + flush_list(&mut self.current, &mut self.list_run); + let saved_current = std::mem::take(&mut self.current); + let saved_list = std::mem::take(&mut self.list_run); + let walked = self.walk(body); + self.flush_shape(); + flush_list(&mut self.current, &mut self.list_run); + let blocks = std::mem::replace(&mut self.current, saved_current); + self.list_run = saved_list; + walked?; + Ok(blocks) + } + + /// Fork (sznb): rebuild a table group. PowerPoint 97-2003 stores a table + /// as a group of cell rectangles plus border lines; column / row + /// boundaries come from the cell anchors, and a rectangle crossing + /// boundaries is a merged cell with the matching spans. Without anchored + /// cells the group's text is emitted in order and `None` returned. + fn table_from_group(&mut self, group: &[u8]) -> Result, ConvertError> { + let mut cells: Vec<(Rect, &[u8])> = Vec::new(); + let mut loose: Vec<&[u8]> = Vec::new(); + // The first SpContainer is the group shape itself. + for (_, rec_type, body) in children(group).skip(1) { + self.charge_record()?; + if rec_type != 0xF004 { + loose.push(body); + continue; + } + let shape_type = + children(body).find(|&(_, t, _)| t == 0xF00A).map(|(vi, ..)| vi >> 4).unwrap_or(0); + // msosptLine: cell borders. + if shape_type == 20 { + continue; + } + match anchor_of(body) { + Some(r) if r.right > r.left && r.bottom > r.top => cells.push((r, body)), + _ => loose.push(body), + } + } + let xs = boundaries(cells.iter().flat_map(|(r, _)| [r.left, r.right])); + let ys = boundaries(cells.iter().flat_map(|(r, _)| [r.top, r.bottom])); + let (ncols, nrows) = (xs.len().saturating_sub(1), ys.len().saturating_sub(1)); + if cells.is_empty() + || ncols == 0 + || nrows == 0 + || (ncols as u64) * (nrows as u64) > limits::MAX_EXPANSION + { + for (_, body) in cells { + loose.push(body); + } + for body in loose { + let blocks = self.capture(body)?; + self.current.extend(blocks); + } + return Ok(None); + } + // Claim each cell's grid area; an origin landing on an owned position + // (overlapping rectangles) is dropped rather than double-placed. + let mut owned = vec![vec![false; ncols]; nrows]; + let mut placed: Vec<((usize, usize), u32, u32, &[u8])> = Vec::new(); + for (rect, body) in &cells { + let (c0, c1) = (snap(rect.left, &xs), snap(rect.right, &xs)); + let (r0, r1) = (snap(rect.top, &ys), snap(rect.bottom, &ys)); + if c1 <= c0 || r1 <= r0 || owned[r0][c0] { + continue; + } + for row in owned.iter_mut().take(r1).skip(r0) { + for slot in row.iter_mut().take(c1).skip(c0) { + *slot = true; + } + } + placed.push(((r0, c0), (c1 - c0) as u32, (r1 - r0) as u32, *body)); + } + let mut origins: HashMap<(usize, usize), Cell> = HashMap::new(); + for (pos, col_span, row_span, body) in placed { + let blocks = self.capture(body)?; + origins.insert(pos, Cell::spanning(blocks, col_span, row_span)); + } + let mut builder = GridBuilder::new(); + // Authored slide tables keep blank tail rows (as Word sources do). + builder.keep_empty_rows(); + for (r, row) in owned.iter().enumerate() { + builder.next_row(); + for (c, &is_owned) in row.iter().enumerate() { + if let Some(cell) = origins.remove(&(r, c)) { + builder.place(cell)?; + } else if !is_owned { + builder.place(Cell::default())?; + } + // Positions a span covers materialize inside the builder. + } + } + let mut table = builder.finish(TableKind::Data); + if table.grid.is_empty() { + return Ok(None); + } + table.header_rows = resolve_header_rows(&table, 0); + Ok(Some(Block::Table(table))) + } + fn charge_record(&mut self) -> Result<(), ConvertError> { self.records += 1; if self.records > limits::MAX_RECORDS { @@ -634,3 +746,200 @@ impl Extractor { } } } + +/// Fork (sznb): a shape rectangle in its parent's coordinate space. +#[derive(Debug, Clone, Copy)] +struct Rect { + left: i32, + top: i32, + right: i32, + bottom: i32, +} + +fn get_i32(b: &[u8], off: usize) -> Option { + get_u32(b, off).map(|v| v as i32) +} + +fn get_i16(b: &[u8], off: usize) -> Option { + crate::shared::binary::get_u16(b, off).map(|v| v as i16 as i32) +} + +/// Fork (sznb): a group shape is a table when its own shape (the group's +/// first SpContainer) carries tableProperties (0x039F, low bit set) or +/// tableRowProperties (0x03A0) in its primary or tertiary property table. +fn is_table_group(group: &[u8]) -> bool { + let Some((_, 0xF004, own)) = record_at(group, 0) else { + return false; + }; + children(own).filter(|&(_, t, _)| t == 0xF00B || t == 0xF122).any(|(vi, _, opt)| { + let count = (vi >> 4) as usize; + (0..count).any(|k| { + let Some(head) = crate::shared::binary::get_u16(opt, 6 * k) else { + return false; + }; + let value = get_u32(opt, 6 * k + 2).unwrap_or(0); + match head & 0x3FFF { + 0x039F => value & 1 != 0, + 0x03A0 => true, + _ => false, + } + }) + }) +} + +/// Fork (sznb): a shape's anchor - OfficeArtChildAnchor (0xF00F, +/// left/top/right/bottom i32) inside groups, or the slide-level client +/// anchor (0xF010: top/left/right/bottom, i16 or i32). +fn anchor_of(shape: &[u8]) -> Option { + for (_, rec_type, body) in children(shape) { + match (rec_type, body.len()) { + (0xF00F, 16..) => { + return Some(Rect { + left: get_i32(body, 0)?, + top: get_i32(body, 4)?, + right: get_i32(body, 8)?, + bottom: get_i32(body, 12)?, + }); + } + (0xF010, 8) => { + return Some(Rect { + top: get_i16(body, 0)?, + left: get_i16(body, 2)?, + right: get_i16(body, 4)?, + bottom: get_i16(body, 6)?, + }); + } + (0xF010, 16..) => { + return Some(Rect { + top: get_i32(body, 0)?, + left: get_i32(body, 4)?, + right: get_i32(body, 8)?, + bottom: get_i32(body, 12)?, + }); + } + _ => {} + } + } + None +} + +/// Edges closer than this merge into one boundary (master units, 576 per +/// inch): writers round each cell's edges independently. +const EDGE_TOLERANCE: i32 = 8; + +/// Fork (sznb): sorted distinct edges, merging edges within the tolerance. +fn boundaries(edges: impl Iterator) -> Vec { + let mut v: Vec = edges.collect(); + v.sort_unstable(); + let mut out: Vec = Vec::new(); + for e in v { + if out.last().is_none_or(|&last| e - last > EDGE_TOLERANCE) { + out.push(e); + } + } + out +} + +/// Index of the boundary nearest to `edge`. +fn snap(edge: i32, bounds: &[i32]) -> usize { + bounds + .iter() + .enumerate() + .min_by_key(|&(_, &b)| (b - edge).abs()) + .map(|(i, _)| i) + .unwrap_or(0) +} + +#[cfg(test)] +mod table_tests { + use super::*; + use crate::model::CellSlot; + + #[test] + fn boundaries_merge_close_edges() { + assert_eq!(boundaries([0, 100, 103, 200, 0].into_iter()), vec![0, 100, 200]); + assert_eq!(snap(104, &[0, 100, 200]), 1); + assert_eq!(snap(199, &[0, 100, 200]), 2); + } + + /// One record: header (ver_inst, type, len) + body. + fn rec(ver_inst: u16, rec_type: u16, body: &[u8]) -> Vec { + let mut v = Vec::new(); + v.extend_from_slice(&ver_inst.to_le_bytes()); + v.extend_from_slice(&rec_type.to_le_bytes()); + v.extend_from_slice(&(body.len() as u32).to_le_bytes()); + v.extend_from_slice(body); + v + } + + fn shape(shape_type: u16, rect: [i32; 4], text: Option<&str>) -> Vec { + let mut body = rec((shape_type << 4) | 2, 0xF00A, &[0; 8]); + let anchor: Vec = rect.iter().flat_map(|v| v.to_le_bytes()).collect(); + body.extend(rec(0, 0xF00F, &anchor)); + if let Some(t) = text { + let mut tb = rec(0, 0x0F9F, &[4, 0, 0, 0]); + let units: Vec = t.encode_utf16().flat_map(|u| u.to_le_bytes()).collect(); + tb.extend(rec(0, 0x0FA0, &units)); + body.extend(rec(0xF, 0xF00D, &tb)); + } + rec(0xF, 0xF004, &body) + } + + fn group(table: bool) -> Vec { + let mut own = rec(0x1, 0xF009, &[0; 16]); + if table { + let mut opt = Vec::new(); + opt.extend_from_slice(&0x039Fu16.to_le_bytes()); + opt.extend_from_slice(&1u32.to_le_bytes()); + own.extend(rec((1 << 4) | 3, 0xF122, &opt)); + } + let mut g = rec(0xF, 0xF004, &own); + // 2 cols x 3 rows; column 0 rows 1-2 merged; edges off by 1 snap. + g.extend(shape(1, [0, 0, 100, 50], Some("A"))); + g.extend(shape(1, [100, 0, 200, 50], Some("B"))); + g.extend(shape(1, [0, 50, 100, 150], Some("M"))); + g.extend(shape(1, [100, 50, 200, 100], Some("C"))); + g.extend(shape(1, [101, 100, 200, 150], None)); + g.extend(shape(20, [0, 50, 200, 50], None)); + g + } + + fn text_of(slot: &CellSlot) -> String { + match slot { + CellSlot::Origin(c) => c + .blocks + .iter() + .map(|b| match b { + Block::Paragraph(i) => crate::model::inlines_to_plain_text(i), + _ => String::new(), + }) + .collect(), + CellSlot::Covered { .. } => "".into(), + } + } + + #[test] + fn table_group_rebuilds_grid_with_spans() { + let g = group(true); + assert!(is_table_group(&g)); + let mut ex = Extractor::default(); + let Some(Block::Table(t)) = ex.table_from_group(&g).unwrap() else { + panic!("expected a table") + }; + assert_eq!(t.grid.len(), 3); + assert!(t.grid.iter().all(|r| r.len() == 2)); + let texts: Vec> = + t.grid.iter().map(|r| r.iter().map(text_of).collect()).collect(); + assert_eq!(texts, vec![vec!["A", "B"], vec!["M", "C"], vec!["", ""]]); + match &t.grid[1][0] { + CellSlot::Origin(c) => assert_eq!((c.col_span, c.row_span), (1, 2)), + _ => panic!("merged origin missing"), + } + assert!(ex.current.is_empty(), "cell text must not leak into the slide"); + } + + #[test] + fn plain_group_is_not_a_table() { + assert!(!is_table_group(&group(false))); + } +} diff --git a/src/formats/rtf/table.rs b/src/formats/rtf/table.rs index a3e65fd7..248ac7ad 100644 --- a/src/formats/rtf/table.rs +++ b/src/formats/rtf/table.rs @@ -175,8 +175,8 @@ impl TableState { } let t = std::mem::take(&mut self.tables[depth - 1]); let rows = - t.rows.into_iter().map(|r| GridRow { cells: r.cells, header: r.header }).collect(); - build_edge_table(rows) + t.rows.into_iter().map(|r| GridRow { cells: r.cells, header: r.header, left: None }).collect(); + build_edge_table(rows, false) } /// Build a finished table at `depth` into cell blocks one level up. diff --git a/src/lib.rs b/src/lib.rs index 4ca94c58..47123fcf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,10 +13,13 @@ mod formats; mod package; mod render; mod shared; +mod transform; pub use error::ConvertError; +pub use transform::images::{ExtractedImage, ImageOptions, ImageReport, extract_images}; -use render::markdown::document_to_markdown; +use render::markdown::document_to_markdown_with; +use render::text::document_to_text_with; use std::path::Path; @@ -25,7 +28,8 @@ use std::path::Path; /// [`Format::from_extension`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Format { - /// Binary Word 97-2003 (`.doc`). + /// Binary Word 97-2003 (`.doc`), and the OLE2-based Kingsoft WPS Writer + /// / Microsoft Works wrapper that shares its `WordDocument` stream (`.wps`). Doc, /// WordprocessingML (`.docx`, `.docm`), both Transitional and Strict. Docx, @@ -46,8 +50,9 @@ pub enum Format { Rtf, /// EPUB 2 and 3 (`.epub`). Epub, - /// Excel workbooks: `.xlsx`, `.xlsm`, binary `.xlsb`, and legacy - /// OLE-based `.xls`. + /// Excel workbooks in every container calamine reads: `.xlsx`, `.xlsm`, + /// `.xlsb`, binary `.xls`, and the OLE2-based Kingsoft WPS Spreadsheet + /// wrapper that shares its `Workbook`/`Book` stream (`.et`). Excel, /// OpenDocument Spreadsheet (`.ods`). Ods, @@ -72,7 +77,9 @@ impl Format { /// case-insensitively. `None` for anything unrecognized. pub fn from_extension(ext: &str) -> Option { Some(match ext.to_ascii_lowercase().as_str() { - "doc" => Format::Doc, + // `.wps` is the Kingsoft WPS Writer / Microsoft Works wrapper, + // an OLE2 compound file with a `WordDocument` stream like `.doc`. + "doc" | "wps" => Format::Doc, "docx" | "docm" => Format::Docx, "odt" => Format::Odt, "pdf" => Format::Pdf, @@ -80,7 +87,9 @@ impl Format { "ppt" | "pps" | "pot" => Format::Ppt, "rtf" => Format::Rtf, "epub" => Format::Epub, - "xlsx" | "xlsm" | "xlsb" | "xls" => Format::Excel, + // `.et` is the Kingsoft WPS Spreadsheet wrapper; modern files are + // an OLE2 compound file with a `Workbook`/`Book` stream like `.xls`. + "xlsx" | "xlsm" | "xlsb" | "xls" | "et" => Format::Excel, "ods" => Format::Ods, "odp" => Format::Odp, "csv" => Format::Csv, @@ -116,14 +125,125 @@ pub fn to_markdown(path: impl AsRef) -> Result { pub fn to_markdown_bytes( bytes: &[u8], format: impl Into>, +) -> Result { + to_markdown_bytes_opts(bytes, format, true) +} + +/// Fork (sznb): like [`to_markdown_bytes`], with the table form selected by +/// `html_tables` — `true` renders HTML `
` carrying merged-cell spans, +/// `false` renders GFM pipe tables. +pub fn to_markdown_bytes_opts( + bytes: &[u8], + format: impl Into>, + html_tables: bool, +) -> Result { + to_markdown_bytes_with(bytes, format, ConvertOptions { html_tables, ..Default::default() }) +} + +/// Fork (sznb): output-shaping options — the conversion service's +/// `table_as_html` / `dedup_table_cells` / `content` parameters. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ConvertOptions { + /// Markdown tables as HTML `
` carrying merged-cell spans (true) + /// or GFM pipe tables (false). Plain text ignores it. + pub html_tables: bool, + /// Merged cells show their content once, on the origin (true), or repeat + /// it in every position the merge covers (false; spans collapse). + pub dedup_cells: bool, + /// Keep the table of contents (true) or drop it (false): Word `Table of + /// Contents` SDTs and bare `TOC` field results, with the 目录 title just + /// before. Only the Word frontends (docx, doc / wps) carry a TOC. + pub include_toc: bool, +} + +impl Default for ConvertOptions { + fn default() -> Self { + ConvertOptions { html_tables: true, dedup_cells: true, include_toc: true } + } +} + +/// Fork (sznb): [`to_markdown_bytes`] with every output-shaping option. +pub fn to_markdown_bytes_with( + bytes: &[u8], + format: impl Into>, + opts: ConvertOptions, ) -> Result { let format = resolve_format(bytes, format.into())?; // PDFs convert to Markdown directly (pdf-inspector) without passing - // through the document model. + // through the document model; the shaping options do not apply. if format == Format::Pdf { return formats::pdf::to_markdown(bytes); } - Ok(document_to_markdown(&to_document(bytes, format)?)) + let doc = formats::parse_with(bytes, format, opts.include_toc)?; + Ok(document_to_markdown_with(&doc, opts.html_tables, opts.dedup_cells)) +} + +/// Fork (sznb): convert an in-memory document to plain text (no Markdown +/// syntax; tables render as tab-joined rows carrying merged content once). +/// +/// Unsupported for [`Format::Pdf`]: PDF conversion produces Markdown +/// directly and has no document-model form; use [`to_markdown_bytes`]. +pub fn to_text_bytes( + bytes: &[u8], + format: impl Into>, +) -> Result { + to_text_bytes_with(bytes, format, ConvertOptions::default()) +} + +/// Fork (sznb): [`to_text_bytes`] with `dedup_cells` / `include_toc` +/// (`html_tables` does not apply to plain text). +pub fn to_text_bytes_with( + bytes: &[u8], + format: impl Into>, + opts: ConvertOptions, +) -> Result { + let format = resolve_format(bytes, format.into())?; + if format == Format::Pdf { + return Err(ConvertError::Unsupported( + "pdf has no document-model form; use to_markdown_bytes".into(), + )); + } + let doc = formats::parse_with(bytes, format, opts.include_toc)?; + Ok(document_to_text_with(&doc, opts.dedup_cells)) +} + +/// Fork (sznb): [`to_markdown_bytes_with`] plus image extraction — embedded +/// images become placeholder tokens in the Markdown (small ones are dropped) +/// and their payloads come back in the [`ImageReport`]; see +/// [`extract_images`]. PDFs convert without the document model, so they +/// yield their Markdown and an empty report. +pub fn to_markdown_with_images( + bytes: &[u8], + format: impl Into>, + opts: ConvertOptions, + images: &ImageOptions, +) -> Result<(String, ImageReport), ConvertError> { + let format = resolve_format(bytes, format.into())?; + if format == Format::Pdf { + return Ok((formats::pdf::to_markdown(bytes)?, ImageReport::default())); + } + let mut doc = formats::parse_with(bytes, format, opts.include_toc)?; + let report = extract_images(&mut doc, images); + Ok((document_to_markdown_with(&doc, opts.html_tables, opts.dedup_cells), report)) +} + +/// Fork (sznb): [`to_text_bytes_with`] plus image extraction (as for +/// [`to_markdown_with_images`]). Unsupported for [`Format::Pdf`]. +pub fn to_text_with_images( + bytes: &[u8], + format: impl Into>, + opts: ConvertOptions, + images: &ImageOptions, +) -> Result<(String, ImageReport), ConvertError> { + let format = resolve_format(bytes, format.into())?; + if format == Format::Pdf { + return Err(ConvertError::Unsupported( + "pdf has no document-model form; use to_markdown_bytes".into(), + )); + } + let mut doc = formats::parse_with(bytes, format, opts.include_toc)?; + let report = extract_images(&mut doc, images); + Ok((document_to_text_with(&doc, opts.dedup_cells), report)) } /// Parse an in-memory document into the document model. Pass a [`Format`] to diff --git a/src/model/inline.rs b/src/model/inline.rs index 61d2cd3a..faa3a7ba 100644 --- a/src/model/inline.rs +++ b/src/model/inline.rs @@ -64,6 +64,7 @@ fn collect_plain_text(inlines: &[Inline], out: &mut String) { match inline { Inline::Text { text, .. } => out.push_str(text), Inline::Link { content, .. } => collect_plain_text(content, out), + Inline::Image { source: ImageSource::Placeholder(token), .. } => out.push_str(token), Inline::Image { alt, .. } => out.push_str(alt), Inline::Math(tex) => out.push_str(tex), Inline::Checkbox(checked) => out.push_str(checkbox_text(*checked)), diff --git a/src/model/link.rs b/src/model/link.rs index 19b70798..f70ad44b 100644 --- a/src/model/link.rs +++ b/src/model/link.rs @@ -37,4 +37,9 @@ pub enum ImageSource { /// No usable source: the image's part is missing or unreadable and it /// has no URL. Only the alt text remains. Unavailable, + /// Fork (sznb): the image was extracted by + /// [`extract_images`](crate::extract_images) and is replaced in place by + /// this literal token (e.g. `[IMG::img_0123abcd.png]`), which renderers + /// emit verbatim. + Placeholder(String), } diff --git a/src/model/list.rs b/src/model/list.rs index 170005fa..97571305 100644 --- a/src/model/list.rs +++ b/src/model/list.rs @@ -15,6 +15,13 @@ pub enum MarkerKind { LowerRoman, /// `I`, `II`, `III`. UpperRoman, + /// Fork (sznb): CJK counting — `一`, `二`, `三` (numFmt chineseCounting, + /// ideographDigital, chineseCountingThousand, ...). Ubiquitous in Chinese + /// official documents; without it these levels degrade to `1`, `2`, `3`. + ChineseCounting, + /// Fork (sznb): CJK legal — `壹`, `贰`, `叁` (numFmt chineseLegalSimplified, + /// ideographLegalTraditional, japaneseLegal). + ChineseLegal, } impl MarkerKind { @@ -24,10 +31,14 @@ impl MarkerKind { } /// The marker text for ordinal `n` (1-based), without trailing space: - /// `3.`, `c.`, `iv.`; bullets have no ordinal text. + /// `3.`, `c.`, `iv.`; CJK counting defaults to the ideographic comma + /// (`一、`) — the punctuation official-document lists actually use. pub fn label(self, n: u64) -> String { match self { MarkerKind::Bullet => "-".to_string(), + MarkerKind::ChineseCounting | MarkerKind::ChineseLegal => { + format!("{}、", self.ordinal(n)) + } _ => format!("{}.", self.ordinal(n)), } } @@ -41,6 +52,8 @@ impl MarkerKind { MarkerKind::UpperAlpha => alpha(n).to_ascii_uppercase(), MarkerKind::LowerRoman => roman(n), MarkerKind::UpperRoman => roman(n).to_ascii_uppercase(), + MarkerKind::ChineseCounting => cjk_numeral(n, false), + MarkerKind::ChineseLegal => cjk_numeral(n, true), } } } @@ -89,6 +102,77 @@ fn roman(mut n: u64) -> String { out } +/// Fork (sznb): CJK numeral for chineseCounting-family markers. +/// +/// Positional style with 万 grouping (`一万零一`, `十万零一千`), runs of +/// interior zeros collapsed to a single 零, and 10-19 reading `十` without a +/// leading 一 in plain style (Word's own rendering); the legal style keeps +/// the explicit 壹 (`壹拾`) as finance convention requires. Values beyond +/// 9999 9999 fall back to Arabic digits — list ordinals never reach them. +pub(crate) fn cjk_numeral(n: u64, legal: bool) -> String { + if n == 0 { + return "零".to_string(); + } + if n > 99_999_999 { + return n.to_string(); + } + let digits: [&str; 10] = if legal { + ["", "壹", "贰", "叁", "肆", "伍", "陆", "柒", "捌", "玖"] + } else { + ["", "一", "二", "三", "四", "五", "六", "七", "八", "九"] + }; + // One 1..=9999 group: positional digits with units, interior zeros + // collapsed, leading one before 十 dropped in plain style. + let group = |g: u64| -> String { + const UNITS: [&str; 4] = ["千", "百", "十", ""]; + let mut d = [0u64; 4]; + let mut v = g; + for i in (0..4).rev() { + d[i] = v % 10; + v /= 10; + } + let mut out = String::new(); + let mut started = false; + let mut pending_zero = false; + for i in 0..4 { + if d[i] == 0 { + if started { + pending_zero = true; + } + continue; + } + if pending_zero { + out.push('零'); + pending_zero = false; + } + let leading_one = d[i] == 1 && UNITS[i] == "十" && !started && !legal; + if !leading_one { + out.push_str(digits[d[i] as usize]); + } + out.push_str(UNITS[i]); + started = true; + } + out + }; + let wan = n / 10_000; + let rest = n % 10_000; + if wan == 0 { + return group(n); + } + let mut out = group(wan); + out.push_str("万"); + if rest > 0 { + // A zero at the boundary (the 万 group's ones digit, or the ones + // group's thousands digit) reads as a single 零: 一万零一 (10001), + // 十万零一千 (101000); 一万一千 (11000) has none. + if wan % 10 == 0 || rest < 1000 { + out.push('零'); + } + out.push_str(&group(rest)); + } + out +} + /// A fully resolved list: numbering identity and marker resolution happen in /// the frontends (`shared::list`), which split runs whenever the list /// instance or marker kind changes. @@ -134,5 +218,33 @@ mod tests { assert_eq!(MarkerKind::LowerRoman.label(4), "iv."); assert_eq!(MarkerKind::LowerRoman.label(1994), "mcmxciv."); assert_eq!(MarkerKind::UpperRoman.label(9), "IX."); + assert_eq!(MarkerKind::ChineseCounting.label(1), "一、"); + assert_eq!(MarkerKind::ChineseCounting.label(10), "十、"); + assert_eq!(MarkerKind::ChineseLegal.label(11), "壹拾壹、"); + } + + #[test] + fn cjk_numerals_read_like_word() { + assert_eq!(cjk_numeral(1, false), "一"); + assert_eq!(cjk_numeral(9, false), "九"); + assert_eq!(cjk_numeral(10, false), "十"); + assert_eq!(cjk_numeral(11, false), "十一"); + assert_eq!(cjk_numeral(20, false), "二十"); + assert_eq!(cjk_numeral(21, false), "二十一"); + assert_eq!(cjk_numeral(100, false), "一百"); + assert_eq!(cjk_numeral(101, false), "一百零一"); + assert_eq!(cjk_numeral(110, false), "一百一十"); + assert_eq!(cjk_numeral(120, false), "一百二十"); + assert_eq!(cjk_numeral(1000, false), "一千"); + assert_eq!(cjk_numeral(1001, false), "一千零一"); + assert_eq!(cjk_numeral(1100, false), "一千一百"); + assert_eq!(cjk_numeral(10000, false), "一万"); + assert_eq!(cjk_numeral(10001, false), "一万零一"); + assert_eq!(cjk_numeral(11000, false), "一万一千"); + assert_eq!(cjk_numeral(101000, false), "十万零一千"); + assert_eq!(cjk_numeral(12345, false), "一万二千三百四十五"); + assert_eq!(cjk_numeral(10, true), "壹拾"); + assert_eq!(cjk_numeral(11, true), "壹拾壹"); + assert_eq!(cjk_numeral(110, true), "壹佰壹拾"); } } diff --git a/src/model/table.rs b/src/model/table.rs index 94941580..932466c3 100644 --- a/src/model/table.rs +++ b/src/model/table.rs @@ -16,6 +16,12 @@ pub struct Table { pub header_rows: usize, /// Whether the source used this table for data or for layout. pub kind: TableKind, + /// Fork (sznb): keep trailing all-empty rows instead of eliding them as + /// filler. Set by the Word frontends (docx, doc / wps): an authored blank + /// row is real structure there (forms leave rows to fill in), and the + /// service's V1 converter keeps them too. Spreadsheets keep the trim — + /// their used range routinely drags in empty tails. + pub keep_empty_rows: bool, } /// What a table is for. @@ -100,6 +106,44 @@ impl Table { table } + /// Fork (sznb): the service's `dedup_table_cells=false` form — every + /// position a span covers repeats its origin's content and all spans + /// collapse to 1, so each grid position renders on its own (GFM, HTML and + /// plain text alike). Nested tables stay on the origin only: copying them + /// would duplicate whole tables. The result is span-less, so the + /// exactly-once invariant holds trivially. + pub fn expanded(&self) -> Table { + let grid = self + .grid + .iter() + .map(|row| { + row.iter() + .map(|slot| match slot { + CellSlot::Origin(c) => CellSlot::Origin(Cell::new(c.blocks.clone())), + CellSlot::Covered { origin_row, origin_col } => { + let blocks = match self.grid.get(*origin_row).and_then(|r| r.get(*origin_col)) { + Some(CellSlot::Origin(o)) => o + .blocks + .iter() + .filter(|b| !matches!(b, Block::Table(_))) + .cloned() + .collect(), + _ => Vec::new(), + }; + CellSlot::Origin(Cell::new(blocks)) + } + }) + .collect() + }) + .collect(); + Table { grid, header_rows: self.header_rows, kind: self.kind, keep_empty_rows: self.keep_empty_rows } + } + + /// Fork (sznb): true when any position is covered by a span. + pub fn has_spans(&self) -> bool { + self.grid.iter().flatten().any(|s| matches!(s, CellSlot::Covered { .. })) + } + /// True when the table is a single origin cell (any covered padding aside). pub fn is_single_cell(&self) -> bool { self.grid.len() == 1 @@ -123,6 +167,8 @@ pub struct GridBuilder { /// Whether trailing rows holding only covered positions survive /// [`GridBuilder::finish`]; see [`GridBuilder::keep_covered_tail`]. keep_covered_tail: bool, + /// Fork (sznb): see [`Table::keep_empty_rows`]. + keep_empty_rows: bool, } impl GridBuilder { @@ -142,6 +188,12 @@ impl GridBuilder { self.keep_covered_tail = true; } + /// Fork (sznb): keep trailing all-empty rows (Word sources); the built + /// table carries the flag so renderers skip their own tail trim too. + pub fn keep_empty_rows(&mut self) { + self.keep_empty_rows = true; + } + fn row_index(&mut self) -> usize { if self.grid.is_empty() { self.grid.push(Vec::new()); @@ -255,8 +307,13 @@ impl GridBuilder { self.grid[row].push(CellSlot::Covered { origin_row, origin_col }); } } - // Drop trailing all-empty rows. - while self.grid.last().is_some_and(|r| { + // Drop trailing all-empty rows (unless the source keeps them; a + // fully blank table still collapses to nothing). + let all_blank = self.grid.iter().flatten().all(|s| match s { + CellSlot::Origin(c) => c.is_empty(), + CellSlot::Covered { .. } => true, + }); + while (!self.keep_empty_rows || all_blank) && self.grid.last().is_some_and(|r| { r.iter().all(|s| match s { CellSlot::Origin(c) => c.is_empty(), CellSlot::Covered { .. } => !self.keep_covered_tail, @@ -276,7 +333,7 @@ impl GridBuilder { } } } - Table { grid: self.grid, header_rows: 0, kind } + Table { grid: self.grid, header_rows: 0, kind, keep_empty_rows: self.keep_empty_rows } } } @@ -492,4 +549,32 @@ mod tests { let t = b.finish(TableKind::Data); assert_eq!(widths(&t), vec![1]); } + + #[test] + fn keep_empty_rows_preserves_trailing_blank_rows() { + // Fork (sznb): Word sources keep authored blank rows. + let mut b = GridBuilder::new(); + b.keep_empty_rows(); + b.next_row(); + b.place(text_cell("a")).unwrap(); + b.next_row(); + b.place(Cell::default()).unwrap(); + b.next_row(); + b.place(Cell::default()).unwrap(); + let t = b.finish(TableKind::Data); + assert_eq!(widths(&t), vec![1, 1, 1]); + assert!(t.keep_empty_rows); + } + + #[test] + fn keep_empty_rows_still_drops_wholly_blank_table() { + let mut b = GridBuilder::new(); + b.keep_empty_rows(); + b.next_row(); + b.place(Cell::default()).unwrap(); + b.next_row(); + b.place(Cell::default()).unwrap(); + let t = b.finish(TableKind::Data); + assert!(t.grid.is_empty()); + } } diff --git a/src/render/markdown/escape.rs b/src/render/markdown/escape.rs index 225f32e2..84e808d7 100644 --- a/src/render/markdown/escape.rs +++ b/src/render/markdown/escape.rs @@ -13,6 +13,7 @@ pub(crate) enum InlineContext { /// Fine-grained escaping context beyond [`InlineContext`]: where the run /// sits relative to its surroundings. #[derive(Clone, Copy, Default)] +#[allow(dead_code)] // fork: positional fields survive for upstream callers pub(crate) struct EscapeOpts { /// The run begins at the start of an output line, where block syntax /// (headings, list markers, setext underlines) could form. @@ -139,115 +140,40 @@ fn can_close(chars: &[char], j: usize, end: usize) -> bool { /// Escape Markdown syntax in document text. pub(crate) fn escape_text(text: &str, ctx: InlineContext, opts: EscapeOpts) -> String { - let EscapeOpts { - at_line_start, - styled, - trailing_active, - trailing_nonspace, - trailing_delims, - in_label, - } = opts; - let chars: Vec = text.chars().collect(); - // Last position of each delimiter that can pair; one with no later - // partner is inert. - let mut last: [Option; 6] = [None; 6]; // * _ ~ ` ] $ - let mut j = 0; - while j < chars.len() { - let end = run_end(&chars, j); - if let Some(slot) = partner_slot(&chars, j, end) { - last[slot] = Some(end - 1); - } - j = end; - } + // Fork (sznb): document text renders verbatim — no Markdown escaping — + // matching the original pipeline's raw-text output (`1.` list-looking + // lines, `12*8GB` formulas, `C:\path` backslashes, `#`-starting lines all + // stay exactly as written). Escaping survives only where a bare + // character would break structure the renderer itself emits: + // - a `|` inside a GFM table cell would split the cell; + // - a `]` inside a link label would terminate it; + // - pairing delimiters inside a styled span would close it early. + let styled = opts.styled; + let in_label = opts.in_label; let mut out = String::with_capacity(text.len() + 8); - let mut line_has_content = !(at_line_start && ctx == InlineContext::Block); - let mut i = 0; - while i < chars.len() { - let c = chars[i]; - if c == '\n' { - out.push('\n'); - if ctx == InlineContext::Block { - line_has_content = false; - } - i += 1; - continue; - } - let start_of_line = !line_has_content; - if !c.is_whitespace() { - line_has_content = true; - } - let next = chars.get(i + 1).copied(); - // At the run's end the next character is unknown; trailing_active assumes the worst. - let next_nonspace = - next.map_or(trailing_active || trailing_nonspace, |n| !n.is_whitespace()); - let paired = |slot: usize| { - trailing_active || trailing_delims.contains(slot) || last[slot].is_some_and(|j| j > i) - }; + for c in text.chars() { let escape = match c { - '\\' => true, - // Opens a math span only directly before non-space, and only - // when a `$` that can close follows. - '$' => next_nonspace && paired(5), - ']' if in_label => true, - '`' => styled || paired(3), - '*' => styled || start_of_line || (next_nonspace && paired(0)), - '_' => { - let prev_alnum = i > 0 && chars[i - 1].is_alphanumeric(); - let next_alnum = next.is_some_and(char::is_alphanumeric); - styled || (next_nonspace && !(prev_alnum && next_alnum) && paired(1)) - } - '~' => styled || (next_nonspace && paired(2)), - '[' => in_label || paired(4), - '<' => next.is_some_and(|n| n.is_ascii_alphabetic() || matches!(n, '/' | '!' | '?')), - '!' => next.is_none() && trailing_active, '|' if ctx == InlineContext::TableCell => true, - '&' if entity_ahead(&chars[i..]) => { - out.push_str("&"); - i += 1; - continue; - } - '#' if start_of_line => { - let j = (i..chars.len()).find(|&j| chars[j] != '#').unwrap_or(chars.len()); - chars.get(j).is_none_or(|n| n.is_whitespace()) - } - '-' if start_of_line => !next_nonspace || line_is_only(&chars[i..], '-'), - '+' if start_of_line => !next_nonspace, - '>' if start_of_line => true, - '=' if start_of_line => line_is_only(&chars[i..], '='), - '0'..='9' if start_of_line => { - let mut j = i; - while j < chars.len() && chars[j].is_ascii_digit() { - j += 1; - } - if j < chars.len() - && (chars[j] == '.' || chars[j] == ')') - && chars.get(j + 1).is_none_or(|n| n.is_whitespace()) - { - out.extend(&chars[i..j]); - out.push('\\'); - out.push(chars[j]); - i = j + 1; - continue; - } - false - } + ']' if in_label => true, + '*' | '~' | '`' if styled => true, _ => false, }; if escape { out.push('\\'); } out.push(c); - i += 1; } out } /// True when the rest of the current line is just `c`, spaces, and tabs /// (a setext underline or thematic break). +#[allow(dead_code)] // fork: kept for upstream parity fn line_is_only(chars: &[char], c: char) -> bool { chars.iter().take_while(|&&ch| ch != '\n').all(|&ch| ch == c || ch == ' ' || ch == '\t') } +#[allow(dead_code)] // fork: kept for upstream parity fn entity_ahead(chars: &[char]) -> bool { let mut i = 1; if i < chars.len() && chars[i] == '#' { @@ -290,6 +216,7 @@ pub(crate) fn format_url(url: &str) -> String { } } +#[allow(dead_code)] pub(crate) fn escape_url_as_text(url: &str, ctx: InlineContext) -> String { let cleaned: String = url.chars().map(|c| if c.is_control() { ' ' } else { c }).collect(); escape_text( diff --git a/src/render/markdown/inline.rs b/src/render/markdown/inline.rs index c3aadb79..066e8b5d 100644 --- a/src/render/markdown/inline.rs +++ b/src/render/markdown/inline.rs @@ -130,7 +130,11 @@ fn render_inlines_mode(inlines: &[Inline], ctx: InlineContext, in_label: bool, r } } Norm::LineBreak => match ctx { - InlineContext::Block => out.push_str("\\\n"), + // Fork (sznb): a plain newline, no `\` hard-break marker — + // matching the original pipeline's line-break form (LibreOffice + // emits a bare newline; V2's concatenation reads the same + // once Markdown renders the in-paragraph newline as a space). + InlineContext::Block => out.push('\n'), InlineContext::Heading => out.push(' '), InlineContext::TableCell => out.push('\n'), }, @@ -154,29 +158,19 @@ fn render_link( rc: &Ctx, out: &mut String, ) { - let label = render_inlines_mode(content, ctx, true, rc); - let url = match target { - LinkTarget::External(url) | LinkTarget::Relative(url) => url.clone(), - LinkTarget::Anchor(id) => match rc.anchors.fragment(id) { - Some(fragment) => format!("#{fragment}"), - None => { - // Target exists nowhere in the document: degrade to plain text. - log::debug!("unresolved internal link target: {id}"); - out.push_str(&render_inlines_mode(content, ctx, false, rc)); - return; - } - }, - }; - // Emptiness is tested on the trimmed label, but the rendered label keeps - // its source-significant edge spaces. + // Fork (sznb): every link renders as plain text — internal anchors and + // external http(s) alike. The conversion corpus is official documents + // whose TOC entries, cross references and titled hyperlinks read as plain + // text downstream (the original pipeline drops the href entirely); a + // label that is itself the URL keeps the URL visible as text. + let label = render_inlines_mode(content, ctx, false, rc); if label.trim().is_empty() { - if matches!(target, LinkTarget::Anchor(_)) { - return; + if let LinkTarget::External(url) | LinkTarget::Relative(url) = target { + out.push_str(url); } - let _ = write!(out, "[{}]({})", escape_url_as_text(&url, ctx), format_url(&url)); - } else { - let _ = write!(out, "[{}]({})", label, format_url(&url)); + return; } + out.push_str(&label); } fn render_image( @@ -195,6 +189,8 @@ fn render_image( // Embedded assets render as their alt text: Markdown cannot embed // bytes, and the bytes stay available in `Document::assets`. A // source-less image has only its alt text to offer. + // Fork (sznb): an extracted image's placeholder token, verbatim. + ImageSource::Placeholder(token) => out.push_str(token), ImageSource::Asset(_) | ImageSource::Unavailable => { if !alt.trim().is_empty() { out.push_str(&escape_text( @@ -270,6 +266,7 @@ fn delims_of(run: &Norm, rc: &Ctx) -> Delims { ImageSource::External(_) => {} // Sourceless images degrade to their alt as plain text. ImageSource::Asset(_) | ImageSource::Unavailable => delims.insert_closers(alt), + ImageSource::Placeholder(token) => delims.insert_closers(token), }, Norm::NoteRef(_) | Norm::Anchor(_) @@ -339,9 +336,8 @@ fn render_text_run( if style.strike { open.push_str("~~"); } - if style.bold { - open.push_str("**"); - } + // Fork (sznb): bold runs render without `**` delimiters — the + // conversion service downstream wants emphasis-free Markdown. if style.italic { open.push('*'); } @@ -407,3 +403,31 @@ pub(crate) fn push_code_span(text: &str, ctx: InlineContext, out: &mut String) { }; let _ = write!(out, "{fence}{pad}{text}{pad}{fence}"); } + +/// Fork (sznb): inline content as plain text — no Markdown syntax. Links +/// contribute their label text; images and anchors contribute nothing; note +/// references render as their `[^n]` number via the shared numbering. Used +/// by the HTML table renderer and the plain-text serializer. +pub(crate) fn inlines_to_plain( + inlines: &[Inline], + nums: &crate::render::markdown::NoteNumbers, + out: &mut String, +) { + for inline in inlines { + match inline { + Inline::Text { text, .. } => out.push_str(text), + Inline::Link { content, .. } => inlines_to_plain(content, nums, out), + // Fork (sznb): extracted images keep their placeholder token. + Inline::Image { source: ImageSource::Placeholder(token), .. } => out.push_str(token), + Inline::Image { .. } | Inline::Anchor(_) => {} + Inline::NoteRef(id) => { + if let Some(n) = nums.get(id) { + let _ = write!(out, "[^{n}]"); + } + } + Inline::LineBreak => out.push('\n'), + Inline::Math(tex) => out.push_str(tex), + Inline::Checkbox(checked) => out.push_str(if *checked { "[x] " } else { "[ ] " }), + } + } +} diff --git a/src/render/markdown/mod.rs b/src/render/markdown/mod.rs index 095c10e0..e145ed3c 100644 --- a/src/render/markdown/mod.rs +++ b/src/render/markdown/mod.rs @@ -2,7 +2,7 @@ mod anchors; mod escape; -mod inline; +pub(crate) mod inline; mod table; #[cfg(test)] @@ -29,17 +29,43 @@ pub(crate) fn escape_marker_label(label: &str, ctx: InlineContext) -> String { } /// Footnote id -> rendered number, shared by all render functions. -type NoteNumbers = HashMap; +pub(crate) type NoteNumbers = HashMap; /// Immutable render context threaded through every render function. pub(crate) struct Ctx { - nums: NoteNumbers, + pub(crate) nums: NoteNumbers, anchors: AnchorMap, + /// Fork (sznb): tables render as HTML `
` (true, default) or as + /// GFM pipe tables (false) — the conversion service's `table_as_html` + /// parameter reaches the fork through this flag. + pub(crate) html_tables: bool, + /// Fork (sznb): merged cells show their content once (true, default) or + /// repeat it in every covered position (false) — `dedup_table_cells`. + pub(crate) dedup_cells: bool, } +/// Test-only shim for upstream tests written against the pre-fork name +/// (the fork renamed it to `document_to_markdown_with`). +#[cfg(test)] pub fn document_to_markdown(doc: &Document) -> String { - let rc = Ctx { nums: number_notes(doc), anchors: resolve_anchors(doc) }; - let mut parts: Vec = doc.blocks.iter().filter_map(|b| render_block(b, &rc)).collect(); + document_to_markdown_with(doc, true, true) +} + +/// Fork (sznb): render with the table form (`html_tables`) and merged-cell +/// handling (`dedup_cells`: content once on the origin vs repeated in every +/// covered position) selected. +pub fn document_to_markdown_with(doc: &Document, html_tables: bool, dedup_cells: bool) -> String { + let rc = Ctx { + nums: number_notes(doc), + anchors: resolve_anchors(doc), + html_tables, + dedup_cells, + }; + let mut parts: Vec<(bool, String)> = doc + .blocks + .iter() + .filter_map(|b| render_block(b, &rc).map(|s| (block_needs_blank(b), s))) + .collect(); let mut rendered_defs: HashSet = HashSet::new(); let mut ordered: Vec<(&Note, usize)> = doc.notes.iter().filter_map(|n| rc.nums.get(&n.id).map(|&num| (n, num))).collect(); @@ -64,18 +90,59 @@ pub fn document_to_markdown(doc: &Document) -> String { s.push_str(line); } } - parts.push(s); + parts.push((true, s)); + } + let out = join_block_parts(&parts); + if out.is_empty() { + return out; } - let mut out = parts.join("\n\n"); - if !out.is_empty() { - out.push('\n'); + // Fork (sznb): trailing blank paragraphs collapse, one final newline. + let trimmed = out.trim_end_matches('\n'); + if trimmed.is_empty() { + return String::new(); + } + format!("{trimmed}\n") +} + +/// Fork (sznb): block separation follows the original pipeline's line +/// layout — paragraphs, headings, lists and quotes join with a single +/// newline (no blank line), while tables, code blocks, math and rules keep +/// a blank line around them so the emitted structures still parse. +fn block_needs_blank(b: &Block) -> bool { + matches!(b, Block::Table(_) | Block::CodeBlock { .. } | Block::Math(_) | Block::Rule) +} + +fn join_block_parts(parts: &[(bool, String)]) -> String { + let mut out = String::new(); + let mut prev_blank = true; + for (blank, s) in parts { + if s.is_empty() { + // Fork (sznb): a blank paragraph renders as one blank line; + // consecutive ones fold, leading ones drop. + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + prev_blank = false; + } + continue; + } + if !out.is_empty() { + // A blank paragraph just emitted a line break; the structural + // blank of a table/code block is then already satisfied. + if out.ends_with('\n') { + out.push('\n'); + } else { + out.push_str(if *blank || prev_blank { "\n\n" } else { "\n" }); + } + } + out.push_str(s); + prev_blank = *blank; } out } /// Number notes in first-reference order; unreferenced notes follow at the /// end. The first note wins a duplicated id. -fn number_notes(doc: &Document) -> NoteNumbers { +pub(crate) fn number_notes(doc: &Document) -> NoteNumbers { let mut valid: HashMap<&str, &Note> = HashMap::new(); for note in &doc.notes { if !note.blocks.iter().all(block_is_blank) { @@ -153,8 +220,11 @@ fn collect_note_refs( } fn render_blocks(blocks: &[Block], rc: &Ctx) -> String { - let parts: Vec = blocks.iter().filter_map(|b| render_block(b, rc)).collect(); - parts.join("\n\n") + let parts: Vec<(bool, String)> = blocks + .iter() + .filter_map(|b| render_block(b, rc).map(|s| (block_needs_blank(b), s))) + .collect(); + join_block_parts(&parts) } fn render_block(block: &Block, rc: &Ctx) -> Option { @@ -171,7 +241,9 @@ fn render_block(block: &Block, rc: &Ctx) -> Option { Block::Paragraph(inlines) => { let text = render_inlines(inlines, InlineContext::Block, rc); let trimmed = trim_paragraph(&text); - if trimmed.is_empty() { None } else { Some(trimmed) } + // Fork (sznb): an empty paragraph renders as a blank line (the + // original pipeline keeps the source's spacer paragraphs). + if trimmed.is_empty() { Some(String::new()) } else { Some(trimmed) } } Block::List(list) => render_list(list, rc), // Trivial layout tables are scaffolding; render their content directly. @@ -227,23 +299,31 @@ fn render_list(list: &List, rc: &Ctx) -> Option { let mut rendered_items: Vec = Vec::new(); let mut loose = false; for (i, item) in list.items.iter().enumerate() { - // GFM has decimal ordered lists only, so Roman/alphabetic levels - // render as bullets carrying the source marker as literal text - // (`- iv. …`) — the source marker semantics stay visible. Items with - // an explicit label (composite number text) render it the same way. - let marker = match (&item.marker_label, list.marker) { + // Fork (sznb): the source marker renders as-is, without stacking a + // `- ` bullet in front. GFM ordered lists only parse `1.` decimals, + // so `1、` / `iv.` / composite labels stay literal text lines that + // mirror the source — the Chinese-official-document corpus reads + // best without the doubled prefix (`- 1、 …`). + let (marker, gfm) = match (&item.marker_label, list.marker) { (Some(label), _) => { - format!("- {} ", escape_marker_label(label, InlineContext::Block)) + (format!("{} ", escape_marker_label(label, InlineContext::Block)), false) } - (None, MarkerKind::Bullet) => "- ".to_string(), - (None, MarkerKind::Decimal) => format!("{}. ", list.start.saturating_add(i as u64)), - (None, kind) => format!("- {} ", kind.label(list.start.saturating_add(i as u64))), + (None, MarkerKind::Bullet) => ("- ".to_string(), true), + (None, MarkerKind::Decimal) => { + (format!("{}. ", list.start.saturating_add(i as u64)), true) + } + (None, kind) => (format!("{} ", kind.label(list.start.saturating_add(i as u64))), false), }; let body = render_blocks(&item.blocks, rc); if item.blocks.len() > 1 { loose = true; } - let indent = " ".repeat(marker.chars().count()); + // Fork (sznb): only real GFM list items (`- ` / `N. `) indent their + // continuation lines to stay inside the item. A literal marker + // (`1.1.`, `一、`, `iv.`) is plain text, so indenting its nested + // content only adds spaces the source never shows -- and four or + // more would turn a deep line into an indented code block. + let indent = if gfm { " ".repeat(marker.chars().count()) } else { String::new() }; let mut lines = body.lines(); let first = lines.next().unwrap_or(""); let mut s = format!("{marker}{first}"); @@ -262,31 +342,21 @@ fn render_list(list: &List, rc: &Ctx) -> Option { Some(rendered_items.join(sep)) } -/// Trim paragraph lines, keeping hard-break backslashes intact. +/// Trim paragraph lines. Fork (sznb): in-paragraph line breaks render as +/// bare newlines (no `\` hard-break marker), so every line simply trims its +/// edges and the paragraph keeps its interior newlines. fn trim_paragraph(text: &str) -> String { let lines: Vec<&str> = text .lines() .map(|l| { - let t = l.trim_start(); - let t = if ends_with_hard_break(t) { t } else { t.trim_end() }; - if t.trim_end_matches('\\').trim().is_empty() { "" } else { t } + let t = l.trim(); + if t.is_empty() { "" } else { t } }) .collect(); let start = lines.iter().position(|l| !l.is_empty()); let end = lines.iter().rposition(|l| !l.is_empty()); match (start, end) { - (Some(s), Some(e)) => { - let mut out = lines[s..=e].join("\n"); - if ends_with_hard_break(&out) { - out.pop(); - out.truncate(out.trim_end().len()); - } - out - } + (Some(s), Some(e)) => lines[s..=e].join("\n"), _ => String::new(), } } - -fn ends_with_hard_break(line: &str) -> bool { - line.chars().rev().take_while(|&c| c == '\\').count() % 2 == 1 -} diff --git a/src/render/markdown/table.rs b/src/render/markdown/table.rs index 1c1601ff..28faf6fe 100644 --- a/src/render/markdown/table.rs +++ b/src/render/markdown/table.rs @@ -1,18 +1,190 @@ -//! Table rendering over the canonical grid. Covered positions render as -//! blank cells (GFM has no span syntax); the grid invariant means the -//! renderer never synthesizes or duplicates columns. +//! Table rendering over the canonical grid. Fork (sznb): two forms, selected +//! by the render context's `html_tables` flag (the conversion service's +//! `table_as_html` parameter): +//! +//! - **HTML** (default): `
` markup carrying the span structure on the +//! origin slots — `colspan`/`rowspan` preserve merged cells exactly as the +//! source had them; covered positions emit nothing. Cell text is +//! HTML-escaped plain text; nested tables render as independent tables +//! appended after the host table, recursively. +//! - **GFM**: pipe tables (the upstream renderer, kept verbatim apart from +//! the fork-wide "no `**`" policy). Covered positions render as blank +//! cells — GFM has no span syntax; the grid invariant means the renderer +//! never synthesizes or duplicates columns. use crate::model::{Block, Cell, CellSlot, Table}; use crate::render::markdown::Ctx; use crate::render::markdown::escape::InlineContext; -use crate::render::markdown::inline::{push_code_span, push_math_span, render_inlines}; +use crate::render::markdown::inline::{ + push_code_span, push_math_span, render_inlines, inlines_to_plain, +}; + +pub(crate) fn render_table(table: &Table, rc: &Ctx) -> Option { + // Fork (sznb): dedup_table_cells=false repeats merged content into every + // covered position (spans collapse), in both table forms. + if !rc.dedup_cells && table.has_spans() { + let expanded = table.expanded(); + return render_table_form(&expanded, rc); + } + render_table_form(table, rc) +} + +fn render_table_form(table: &Table, rc: &Ctx) -> Option { + if rc.html_tables { + render_table_html(table, rc) + } else { + render_table_gfm(table, rc) + } +} + +// ---------- HTML form ---------- + +/// HTML-escape a text run for cell content. Order matters: `&` first. +fn html_escape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + c => out.push(c), + } + } + out +} + +/// True when no block in the cell yields any non-whitespace text. A cell +/// holding only a nested table is NOT empty: the nested table renders after +/// the host, and treating it as blank once dropped a whole 1x1 host table +/// together with everything nested inside it (content loss on real +/// government documents that wrap report tables in a layout host). +fn cell_is_empty(cell: &Cell, rc: &Ctx) -> bool { + if cell.blocks.iter().any(|b| matches!(b, Block::Table(_))) { + return false; + } + let mut probe = Vec::new(); + cell_block_plain(&cell.blocks, rc, &mut probe, &mut Vec::new()); + probe.iter().all(|p| p.trim().is_empty()) +} + +fn render_table_html(table: &Table, rc: &Ctx) -> Option { + if table.grid.is_empty() { + return None; + } + // Trailing rows whose every slot is blank (covered, or an empty origin) + // are scaffolding; drop them, as the GFM renderer did. Fork (sznb): Word + // sources keep them (`Table::keep_empty_rows`) — only a wholly blank + // table still renders as nothing. + let last_meaningful = table + .grid + .iter() + .rposition(|row| { + row.iter().any(|slot| match slot { + CellSlot::Origin(cell) => !cell_is_empty(cell, rc), + CellSlot::Covered { .. } => false, + }) + })?; + let last_meaningful = if table.keep_empty_rows { table.grid.len() - 1 } else { last_meaningful }; + let mut lines: Vec = vec!["
".to_string()]; + let mut nested: Vec = Vec::new(); + for (ri, row) in table.grid.iter().take(last_meaningful + 1).enumerate() { + lines.push(" ".to_string()); + let tag = if ri < table.header_rows { "th" } else { "td" }; + for slot in row { + // Covered positions belong to an origin elsewhere; emitting them + // would duplicate merged content. + let CellSlot::Origin(cell) = slot else { continue }; + let mut attrs = String::new(); + if cell.col_span > 1 { + attrs.push_str(&format!(" colspan=\"{}\"", cell.col_span)); + } + if cell.row_span > 1 { + attrs.push_str(&format!(" rowspan=\"{}\"", cell.row_span)); + } + let text = render_cell_html(cell, rc, &mut nested); + lines.push(format!(" <{tag}{attrs}>{text}")); + } + lines.push(" ".to_string()); + } + lines.push("
".to_string()); + for tbl in nested { + lines.push(String::new()); + lines.push(tbl); + } + Some(lines.join("\n")) +} + +/// Cell content as escaped HTML: paragraph-level blocks joined with `
`, +/// and in-paragraph line breaks kept as `
` too (the source's own +/// line-break positions — stacked unit names, split titles — are layout +/// information worth preserving); nested tables are collected for +/// independent rendering after the host table. +fn render_cell_html(cell: &Cell, rc: &Ctx, nested: &mut Vec) -> String { + let mut parts: Vec = Vec::new(); + cell_block_plain(&cell.blocks, rc, &mut parts, nested); + parts + .iter() + .map(|p| html_escape(p).replace('\n', "
")) + .collect::>() + .join("
") +} + +/// Flatten cell blocks into plain-text parts (one per paragraph-ish block). +fn cell_block_plain(blocks: &[Block], rc: &Ctx, parts: &mut Vec, nested: &mut Vec) { + for block in blocks { + match block { + Block::Heading { content, .. } | Block::Paragraph(content) => { + let mut s = String::new(); + inlines_to_plain(content, &rc.nums, &mut s); + // Keep interior line breaks; render_cell_html turns them + // into `
`. Only edge whitespace is padding. + if !s.trim().is_empty() { + parts.push(s.trim().to_string()); + } + } + Block::List(list) => { + for (i, item) in list.items.iter().enumerate() { + let mut inner: Vec = Vec::new(); + cell_block_plain(&item.blocks, rc, &mut inner, nested); + if inner.is_empty() { + continue; + } + let marker = match (&item.marker_label, list.marker) { + (Some(label), _) => label.clone(), + (None, crate::model::MarkerKind::Bullet) => "-".to_string(), + (None, kind) => kind.label(list.start.saturating_add(i as u64)), + }; + parts.push(format!("{marker} {}", inner.join(" "))); + } + } + Block::Table(t) => { + // Nested table: rendered after the host table, host cell + // stays free of its content (mirrors the V2 convention). + if let Some(rendered) = render_table(t, rc) { + nested.push(rendered); + } + } + Block::BlockQuote(inner) => cell_block_plain(inner, rc, parts, nested), + Block::CodeBlock { text, .. } | Block::Math(text) => { + let t = text.trim(); + if !t.is_empty() { + parts.push(t.replace('\n', " ")); + } + } + Block::Rule => {} + } + } +} + +// ---------- GFM form (upstream renderer, fork: no `**` in cells) ---------- struct RenderedCell { text: String, covered_span: bool, } -pub(crate) fn render_table(table: &Table, rc: &Ctx) -> Option { +fn render_table_gfm(table: &Table, rc: &Ctx) -> Option { // Interior empty rows render as blank rows — they carry the source's // row coordinates; trailing blank rows are popped below. if table.grid.is_empty() { @@ -27,7 +199,7 @@ pub(crate) fn render_table(table: &Table, rc: &Ctx) -> Option { .iter() .map(|slot| match slot { CellSlot::Origin(cell) => { - RenderedCell { text: render_cell(cell, rc), covered_span: false } + RenderedCell { text: render_cell_gfm(cell, rc), covered_span: false } } CellSlot::Covered { .. } => { RenderedCell { text: String::new(), covered_span: true } @@ -38,7 +210,8 @@ pub(crate) fn render_table(table: &Table, rc: &Ctx) -> Option { cells }) .collect(); - while rendered.len() > 1 + while !table.keep_empty_rows + && rendered.len() > 1 && rendered.last().is_some_and(|r| r.iter().all(|c| c.text.is_empty() && !c.covered_span)) { rendered.pop(); @@ -56,9 +229,11 @@ pub(crate) fn render_table(table: &Table, rc: &Ctx) -> Option { } let mut out = String::new(); - // GFM tables always carry a delimiter row, so a table with no header row - // of its own renders an empty one above the data. - let header: Vec = if table.header_rows >= 1 && !rendered.is_empty() { + // Fork (sznb): GFM pipe tables have no headerless form, so the first row + // is the header — never a synthetic empty header row (the original + // pipeline does the same; an inserted `| | | |` line pushed the real + // header into the data rows). + let header: Vec = if !rendered.is_empty() { rendered.remove(0).into_iter().map(|c| c.text).collect() } else { vec![String::new(); width] @@ -83,21 +258,23 @@ fn format_row<'a>(cells: impl IntoIterator) -> String { s } -/// Flatten arbitrary block content into a single table-cell line. -pub(crate) fn render_cell(cell: &Cell, rc: &Ctx) -> String { +/// Flatten arbitrary block content into a single table-cell line (GFM form). +pub(crate) fn render_cell_gfm(cell: &Cell, rc: &Ctx) -> String { let mut parts: Vec = Vec::new(); for block in &cell.blocks { cell_block_text(block, rc, &mut parts); } - // A cell's edge whitespace is padding the table's own padding would + // Fork (sznb): pipe-table cells join their interior line breaks with a + // space (the original pipeline's form — `数据记录 （万条）`), never + // `
`; edge whitespace is padding the table's own padding would // swallow anyway, and spreadsheets carry it by the thousand. parts - .join("
") + .join(" ") .lines() .filter(|l| !l.trim().is_empty()) .map(str::trim) .collect::>() - .join("
") + .join(" ") } fn cell_block_text(block: &Block, rc: &Ctx, parts: &mut Vec) { @@ -105,7 +282,8 @@ fn cell_block_text(block: &Block, rc: &Ctx, parts: &mut Vec) { Block::Heading { content, .. } => { let t = render_inlines(content, InlineContext::TableCell, rc); if !t.trim().is_empty() { - parts.push(format!("**{}**", t.trim())); + // Fork (sznb): no `**` around heading cells. + parts.push(t.trim().to_string()); } } Block::Paragraph(inlines) => { @@ -148,7 +326,7 @@ fn cell_block_text(block: &Block, rc: &Ctx, parts: &mut Vec) { let cells: Vec = row .iter() .map(|slot| match slot { - CellSlot::Origin(cell) => render_cell(cell, rc), + CellSlot::Origin(cell) => render_cell_gfm(cell, rc), CellSlot::Covered { .. } => String::new(), }) .collect(); diff --git a/src/render/markdown/tests.rs b/src/render/markdown/tests.rs index 0a7b19a4..0770922b 100644 --- a/src/render/markdown/tests.rs +++ b/src/render/markdown/tests.rs @@ -48,7 +48,7 @@ fn math_renders_in_dollar_delimiters_and_text_dollars_are_escaped() { ); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn math_in_a_table_cell_escapes_pipes() { let cell = |inlines| Cell { blocks: vec![Block::Paragraph(inlines)], col_span: 1, row_span: 1 }; let md = doc(vec![table_from( @@ -133,7 +133,7 @@ fn line_start_lookalikes_unescaped() { assert_eq!(md, "\\---\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn negative_number_in_cell_unescaped() { let md = doc(vec![table_from( vec![vec![ @@ -223,19 +223,19 @@ fn escaped_backtick_in_later_run_still_pairs() { assert_eq!(md, "a `\\\n[ x](https://e.test)\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn bold_trailing_space_moved_out() { let md = doc(vec![Block::Paragraph(vec![styled("bold ", BOLD), Inline::plain("plain")])]); assert_eq!(md, "**bold** plain\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn adjacent_same_style_runs_merged() { let md = doc(vec![Block::Paragraph(vec![styled("bo", BOLD), styled("ld", BOLD)])]); assert_eq!(md, "**bold**\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn bold_italic_combo() { let md = doc(vec![Block::Paragraph(vec![styled( "both", @@ -244,7 +244,7 @@ fn bold_italic_combo() { assert_eq!(md, "***both***\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn whitespace_only_run_loses_styling() { let md = doc(vec![Block::Paragraph(vec![ styled("a", BOLD), @@ -322,13 +322,13 @@ fn layout_table_unwrapped() { assert_eq!(md, "# Boxed\n\nbody\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn data_table_1x1_not_unwrapped() { let md = doc(vec![table_from(vec![vec![Cell::from_inlines(vec![Inline::plain("x")])]], 0)]); assert_eq!(md, "| |\n| --- |\n| x |\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn trailing_empty_rows_and_columns_trimmed() { let empty = Cell::default; let filled = |s: &str| Cell::from_inlines(vec![Inline::plain(s)]); @@ -343,7 +343,7 @@ fn trailing_empty_rows_and_columns_trimmed() { assert_eq!(md, "| a | b |\n| --- | --- |\n| c | |\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn table_basic() { let md = doc(vec![table_from( vec![ @@ -361,7 +361,7 @@ fn table_basic() { assert_eq!(md, "| Name | Age |\n| --- | --- |\n| Ann \\| Bob | 30 |\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn table_headerless_and_ragged() { let md = doc(vec![table_from( vec![ @@ -376,7 +376,7 @@ fn table_headerless_and_ragged() { assert_eq!(md, "| | |\n| --- | --- |\n| a | |\n| b | c |\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn table_multiparagraph_cell() { let cell = Cell::new(vec![ Block::Paragraph(vec![Inline::plain("one")]), @@ -387,7 +387,7 @@ fn table_multiparagraph_cell() { assert_eq!(md, "| | |\n| --- | --- |\n| one
two | x |\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn merged_cells_render_blank_covered_positions() { let mut b = GridBuilder::new(); b.next_row(); @@ -403,7 +403,7 @@ fn merged_cells_render_blank_covered_positions() { assert_eq!(md, "| wide | | end |\n| --- | --- | --- |\n| a | b | c |\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn trailing_covered_columns_are_preserved() { let mut b = GridBuilder::new(); b.next_row(); @@ -414,7 +414,7 @@ fn trailing_covered_columns_are_preserved() { assert_eq!(md, "| wide | | |\n| --- | --- | --- |\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn url_pipes_cannot_split_table_cells() { let cell = Cell::from_inlines(vec![Inline::Link { content: Vec::new(), @@ -424,7 +424,7 @@ fn url_pipes_cannot_split_table_cells() { assert_eq!(md, "| |\n| --- |\n| [https://e.test/a\\|b](https://e.test/a%7Cb) |\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn code_span_pipes_cannot_split_table_cells() { let code = |t: &str| { Cell::from_inlines(vec![Inline::Text { @@ -481,6 +481,33 @@ fn nested_list() { assert_eq!(md, "- outer\n\n 3. inner\n"); } +#[test] +fn literal_label_items_do_not_indent_nested_content() { + // Fork (sznb): a composite label is plain text, not GFM list syntax, so + // its nested list stays flush left like the source. + let para = |t: &str| Block::Paragraph(vec![Inline::plain(t)]); + let md = doc(vec![Block::List(List { + marker: MarkerKind::Decimal, + start: 5, + items: vec![ListItem { + blocks: vec![ + para("发区数字"), + Block::List(List { + marker: MarkerKind::Decimal, + start: 1, + items: vec![ListItem { + blocks: vec![para("经济发展局")], + marker_label: Some("1.5.1.".into()), + }], + }), + ], + marker_label: Some("1.5.".into()), + }], + })]); + assert!(md.contains("1.5. 发区数字"), "{md:?}"); + assert!(md.contains("\n1.5.1. 经济发展局"), "nested line must not be indented: {md:?}"); +} + #[test] fn roman_and_alpha_markers_render_literally() { let item = |text: &str| ListItem { @@ -669,7 +696,7 @@ fn task_list() { assert_eq!(md, "- [x] done\n- [ ] todo\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn checkbox_in_a_table_cell() { let md = doc(vec![table_from( vec![vec![ @@ -755,7 +782,7 @@ fn duplicate_heading_slugs_deduped() { assert_eq!(md, "# Same\n\n# Same\n\n[second](#same-1)\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn code_block_in_cell_uses_code_span() { let cell = Cell::new(vec![Block::CodeBlock { lang: None, text: "let `x` = 1;".into() }]); let md = doc(vec![table_from(vec![vec![cell]], 0)]); @@ -768,7 +795,7 @@ fn code_block_keeps_its_language_hint() { assert_eq!(md, "```rust\nfn main() {}\n```\n"); } -#[test] +#[ignore = "fork(sznb): bold renders undelimited, tables render as HTML"] fn cell_edge_whitespace_is_trimmed() { // S10: cell padding from spreadsheets and CSV is layout, and the table's // own padding would swallow it anyway. diff --git a/src/render/mod.rs b/src/render/mod.rs index 163a4fba..135f43c4 100644 --- a/src/render/mod.rs +++ b/src/render/mod.rs @@ -1 +1,2 @@ pub mod markdown; +pub mod text; diff --git a/src/render/text.rs b/src/render/text.rs new file mode 100644 index 00000000..209af6f0 --- /dev/null +++ b/src/render/text.rs @@ -0,0 +1,206 @@ +//! Plain-text serializer for the document model. Fork (sznb): `to_text_bytes` +//! produces readable plain text directly — headings and paragraphs as text +//! lines, tables as tab-joined rows (merged content appears once, on its +//! origin slot; covered positions emit nothing), nested tables appended +//! after their host. + +use crate::model::{Block, Cell, CellSlot, Document, List, MarkerKind}; +use crate::render::markdown::NoteNumbers; +use crate::render::markdown::inline::inlines_to_plain; +use crate::render::markdown::number_notes; + +/// Fork (sznb): text render context — note numbers plus the merged-cell +/// handling (`dedup`: content once on the origin vs repeated in every +/// covered position). Derefs to the note numbers the inline helpers take. +pub(crate) struct TextCtx { + nums: NoteNumbers, + dedup: bool, +} + +impl std::ops::Deref for TextCtx { + type Target = NoteNumbers; + fn deref(&self) -> &NoteNumbers { + &self.nums + } +} + +/// Fork (sznb): plain text with `dedup_table_cells` selected (`dedup`: merged +/// content once on the origin, or repeated in every covered position). +pub fn document_to_text_with(doc: &Document, dedup: bool) -> String { + let nums = TextCtx { nums: number_notes(doc), dedup }; + let mut parts: Vec = Vec::new(); + for block in &doc.blocks { + if let Some(s) = render_block(block, &nums) { + parts.push(s); + } + } + if !doc.notes.is_empty() { + let mut section = String::from("── 脚注 ──"); + for note in &doc.notes { + if let Some(n) = nums.get(¬e.id) { + let body = render_blocks(¬e.blocks, &nums).replace('\n', " "); + let body = body.trim(); + if !body.is_empty() { + section.push_str(&format!("\n[{n}] {body}")); + } + } + } + if section.len() > "── 脚注 ──".len() { + parts.push(section); + } + } + let mut out = parts.join("\n\n"); + if !out.is_empty() { + out.push('\n'); + } + out +} + +fn render_blocks(blocks: &[Block], nums: &TextCtx) -> String { + let parts: Vec = blocks.iter().filter_map(|b| render_block(b, nums)).collect(); + parts.join("\n\n") +} + +fn render_block(block: &Block, nums: &TextCtx) -> Option { + match block { + Block::Heading { content, .. } | Block::Paragraph(content) => { + let mut s = String::new(); + inlines_to_plain(content, nums, &mut s); + // Fork (sznb): keep in-paragraph line breaks as bare newlines + // (matching the original pipeline); only edges trim. + let s = s.trim().to_string(); + if s.is_empty() { None } else { Some(s) } + } + Block::List(list) => render_list(list, nums), + Block::Table(t) => render_table_text(t, nums), + Block::BlockQuote(blocks) => { + let inner = render_blocks(blocks, nums); + if inner.is_empty() { None } else { Some(inner) } + } + Block::CodeBlock { text, .. } | Block::Math(text) => { + let t = text.trim(); + if t.is_empty() { None } else { Some(t.to_string()) } + } + Block::Rule => None, + } +} + +fn render_list(list: &List, nums: &TextCtx) -> Option { + if list.items.is_empty() { + return None; + } + let mut lines: Vec = Vec::new(); + for (i, item) in list.items.iter().enumerate() { + let body = render_blocks(&item.blocks, nums).trim().to_string(); + if body.is_empty() { + continue; + } + let marker = match (&item.marker_label, list.marker) { + (Some(label), _) => label.clone(), + (None, MarkerKind::Bullet) => "-".to_string(), + (None, kind) => kind.label(list.start.saturating_add(i as u64)), + }; + lines.push(format!("{marker} {body}")); + } + if lines.is_empty() { None } else { Some(lines.join("\n")) } +} + +fn render_table_text(table: &crate::model::Table, nums: &TextCtx) -> Option { + if table.grid.is_empty() { + return None; + } + // Fork (sznb): dedup_table_cells=false repeats merged content into every + // covered position. + if !nums.dedup && table.has_spans() { + return render_table_text(&table.expanded(), nums); + } + let mut rows: Vec = Vec::new(); + let mut nested: Vec = Vec::new(); + for row in &table.grid { + // Fork (sznb): every grid position emits (origin -> text, covered -> + // empty). Plain text has no span syntax, so the per-column tab + // alignment is the only column structure that survives; skipping + // blank/covered slots collapses neighbouring columns together. + let mut cells: Vec = Vec::new(); + let mut has_content = false; + for slot in row { + match slot { + CellSlot::Origin(cell) => { + let parts = cell_plain_parts(cell, nums, &mut nested); + if parts.is_empty() { + cells.push(String::new()); + } else { + has_content = true; + cells.push(parts.join(" / ")); + } + } + CellSlot::Covered { .. } => cells.push(String::new()), + } + } + if has_content { + rows.push(cells.join("\t")); + } + } + for tbl in nested { + rows.push(String::new()); + rows.push(tbl); + } + if rows.is_empty() { None } else { Some(rows.join("\n")) } +} + +/// Cell content flattened to plain parts (one per paragraph-ish block); +/// nested tables render into `nested` for independent appending. +fn cell_plain_parts(cell: &Cell, nums: &TextCtx, nested: &mut Vec) -> Vec { + let mut parts: Vec = Vec::new(); + fn walk(blocks: &[Block], nums: &TextCtx, parts: &mut Vec, nested: &mut Vec) { + for block in blocks { + match block { + Block::Heading { content, .. } | Block::Paragraph(content) => { + let mut s = String::new(); + inlines_to_plain(content, nums, &mut s); + let s = s.replace('\n', " ").trim().to_string(); + if !s.is_empty() { + parts.push(s); + } + } + Block::List(list) => { + // Fork (sznb): list items inside a table cell join with + // newlines (one item per line reads far better than a + // single run-on line in plain text). + let mut items: Vec = Vec::new(); + for (i, item) in list.items.iter().enumerate() { + let mut inner: Vec = Vec::new(); + walk(&item.blocks, nums, &mut inner, nested); + if inner.is_empty() { + continue; + } + let marker = match (&item.marker_label, list.marker) { + (Some(label), _) => label.clone(), + (None, MarkerKind::Bullet) => "-".to_string(), + (None, kind) => kind.label(list.start.saturating_add(i as u64)), + }; + items.push(format!("{marker} {}", inner.join(" "))); + } + if !items.is_empty() { + parts.push(items.join("\n")); + } + } + Block::Table(t) => { + if let Some(rendered) = render_table_text(t, nums) { + nested.push(rendered); + } + } + Block::BlockQuote(inner) => walk(inner, nums, parts, nested), + Block::CodeBlock { text, .. } | Block::Math(text) => { + let t = text.trim().replace('\n', " "); + if !t.is_empty() { + parts.push(t); + } + } + Block::Rule => {} + } + } + } + walk(&cell.blocks, nums, &mut parts, nested); + parts +} diff --git a/src/shared/fields.rs b/src/shared/fields.rs index b05c2ff1..bb65d48e 100644 --- a/src/shared/fields.rs +++ b/src/shared/fields.rs @@ -22,6 +22,120 @@ pub fn field_result(instr: &str, content: Vec) -> Vec { } } +/// Fork (sznb): complex-field nesting across paragraphs, to find the +/// paragraphs a `TOC` field's result covers (the service's `content=false` +/// drops them). Feed field events in document order, then call +/// [`TocTracker::finish_paragraph`] at each paragraph end. +#[derive(Default)] +pub struct TocTracker { + /// Open fields (begin without its end yet). + depth: usize, + /// Nesting depth the open TOC field began at. + toc_at: Option, + /// Instruction text of the innermost field still before its separator. + instr: Option<(usize, String)>, + /// The current paragraph lies (partly) inside a TOC field. + touched: bool, + /// A TOC field began in the current paragraph. + started: bool, +} + +impl TocTracker { + pub fn begin(&mut self) { + self.depth += 1; + self.instr = Some((self.depth, String::new())); + } + + pub fn instr_text(&mut self, text: &str) { + if let Some((_, buf)) = self.instr.as_mut() { + buf.push_str(text); + } + } + + pub fn separate(&mut self) { + self.close_instr(); + } + + pub fn end(&mut self) { + self.close_instr(); + if self.toc_at == Some(self.depth) { + self.toc_at = None; + self.touched = true; + } + self.depth = self.depth.saturating_sub(1); + } + + /// A self-contained field (`w:fldSimple`) with this instruction. + pub fn simple(&mut self, instr: &str) { + if is_toc_instr(instr) { + self.touched = true; + self.started = true; + } + } + + /// True while the instruction of the innermost field is being read. + pub fn reading_instr(&self) -> bool { + self.instr.is_some() + } + + fn close_instr(&mut self) { + if let Some((depth, instr)) = self.instr.take() + && self.toc_at.is_none() + && is_toc_instr(&instr) + { + self.toc_at = Some(depth); + self.touched = true; + self.started = true; + } + } + + /// Paragraph end: `(in_toc, toc_started_here)`; resets for the next one. + pub fn finish_paragraph(&mut self) -> (bool, bool) { + let out = (self.touched || self.toc_at.is_some(), self.started); + self.touched = self.toc_at.is_some(); + self.started = false; + out + } +} + +/// A `TOC` field instruction (`TOC \o "1-3" \h \z \u`), any case. +pub fn is_toc_instr(instr: &str) -> bool { + let t = instr.trim_start(); + t.get(..3).is_some_and(|head| head.eq_ignore_ascii_case("TOC")) + && t[3..].chars().next().is_none_or(|c| c.is_whitespace() || c == '\\') +} + +/// The title paragraph Word / WPS put right before a TOC (目录 / 目 录 / +/// Contents / Table of Contents, or a short custom title ending in 目录 such +/// as 「公共应用组件调用目录」). Only ever asked of the block right before a +/// TOC, so the suffix rule cannot reach body text. +pub fn is_toc_title(block: &crate::model::Block) -> bool { + use crate::model::Block; + let (Block::Paragraph(inlines) | Block::Heading { content: inlines, .. }) = block else { + return false; + }; + let text: String = crate::model::inlines_to_plain_text(inlines) + .chars() + .filter(|c| !c.is_whitespace()) + .collect::() + .to_lowercase(); + matches!(text.as_str(), "目录" | "目錄" | "contents" | "tableofcontents") + || ((text.ends_with("目录") || text.ends_with("目錄")) && text.chars().count() <= 30) +} + +/// Fork (sznb): drop the TOC title — the last non-blank block, when it is +/// one ([`is_toc_title`]). Blank paragraphs between title and TOC (vertical +/// spacing) are looked past and left in place. +pub fn drop_toc_title(blocks: &mut Vec) { + use crate::model::Block; + let blank = |b: &Block| matches!(b, Block::Paragraph(inlines) if crate::model::inlines_are_empty(inlines)); + if let Some(i) = blocks.iter().rposition(|b| !blank(b)) + && is_toc_title(&blocks[i]) + { + blocks.remove(i); + } +} + #[derive(Debug, PartialEq)] enum Token { Word(String), diff --git a/src/shared/grid.rs b/src/shared/grid.rs index 9b351380..97a16d46 100644 --- a/src/shared/grid.rs +++ b/src/shared/grid.rs @@ -31,10 +31,16 @@ pub struct CellProp { pub struct GridRow { pub cells: Vec<(Vec, CellProp)>, pub header: bool, + /// Fork (sznb): the row's left boundary (first cell's left edge) in + /// twips, when the producer declares one (DOC `rgdxaCenter[0]`). A row + /// indented past the table's leftmost edge starts at the matching grid + /// column, with empty cells filling the gap (Word `gridBefore` + /// semantics); `None` keeps the row anchored at column 0. + pub left: Option, } /// Assemble logical rows into the canonical grid. -pub fn build_edge_table(rows: Vec) -> Result, ConvertError> { +pub fn build_edge_table(rows: Vec, keep_empty_rows: bool) -> Result, ConvertError> { struct Origin { blocks: Vec, /// Half-open global column range. @@ -51,6 +57,11 @@ pub fn build_edge_table(rows: Vec) -> Result, ConvertErro // Normalize each row's right edges to be strictly increasing (cells past // the declared boundaries get synthetic edges), then cluster the union // into the global column-edge list. + // Fork (sznb): rows' left edges. The leftmost one is the table origin; + // any row starting further right contributes its left edge as a column + // boundary so its cells land in the columns they actually cover. + let lefts: Vec> = rows.iter().map(|r| r.left).collect(); + let origin_left = lefts.iter().flatten().copied().min(); let rows_edged: Vec, CellProp)>> = rows .into_iter() .map(|row| { @@ -68,6 +79,9 @@ pub fn build_edge_table(rows: Vec) -> Result, ConvertErro }) .collect(); let mut edges: Vec = rows_edged.iter().flatten().map(|(_, prop)| prop.right).collect(); + if let Some(o) = origin_left { + edges.extend(lefts.iter().flatten().copied().filter(|&l| l - o > EDGE_TOLERANCE)); + } edges.sort_unstable(); let mut clusters: Vec = Vec::new(); for e in edges { @@ -78,9 +92,16 @@ pub fn build_edge_table(rows: Vec) -> Result, ConvertErro let col_of = |x: i64| -> usize { clusters.partition_point(|&c| c < x - EDGE_TOLERANCE) }; let mut rows: Vec> = Vec::new(); - for row in rows_edged { + let mut leads: Vec = Vec::new(); + for (row, left) in rows_edged.into_iter().zip(lefts) { let mut out: Vec = Vec::new(); - let mut col = 0usize; + // Indented row: column k spans (clusters[k-1], clusters[k]], so a row + // whose left edge sits on clusters[j] starts at column j + 1. + let mut col = match (left, origin_left) { + (Some(l), Some(o)) if l - o > EDGE_TOLERANCE => col_of(l) + 1, + _ => 0, + }; + let lead = col; let mut cells = row.into_iter().peekable(); while let Some((blocks, prop)) = cells.next() { let mut merged_blocks = blocks; @@ -103,6 +124,7 @@ pub fn build_edge_table(rows: Vec) -> Result, ConvertErro }); col = col_r; } + leads.push(lead); rows.push(out); } @@ -129,8 +151,14 @@ pub fn build_edge_table(rows: Vec) -> Result, ConvertErro } let mut builder = GridBuilder::new(); - for row in rows { + if keep_empty_rows { + builder.keep_empty_rows(); + } + for (row, lead) in rows.into_iter().zip(leads) { builder.next_row(); + for _ in 0..lead { + builder.place(Cell::default())?; + } for origin in row { let span = (origin.col_r - origin.col_l) as u32; if origin.covered { @@ -149,3 +177,72 @@ pub fn build_edge_table(rows: Vec) -> Result, ConvertErro table.header_rows = resolve_header_rows(&table, header_rows); Ok(Some(Block::Table(table))) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{CellSlot, Inline}; + + fn cell(text: &str, right: i64) -> (Vec, CellProp) { + let blocks = if text.is_empty() { + Vec::new() + } else { + vec![Block::Paragraph(vec![Inline::plain(text)])] + }; + (blocks, CellProp { right, ..Default::default() }) + } + + fn texts(block: &Block) -> Vec> { + let Block::Table(t) = block else { panic!("not a table") }; + t.grid + .iter() + .map(|r| { + r.iter() + .map(|s| match s { + CellSlot::Origin(c) => c + .blocks + .iter() + .flat_map(|b| match b { + Block::Paragraph(inl) => inl.clone(), + _ => Vec::new(), + }) + .map(|i| match i { + Inline::Text { text, .. } => text, + _ => String::new(), + }) + .collect(), + CellSlot::Covered { .. } => "".into(), + }) + .collect() + }) + .collect() + } + + #[test] + fn indented_row_starts_at_its_left_column() { + let rows = vec![ + GridRow { cells: vec![cell("title", 5000)], header: false, left: Some(0) }, + GridRow { + cells: vec![cell("a", 2000), cell("b", 3500), cell("c", 5000)], + header: false, + left: Some(1000), + }, + ]; + let t = build_edge_table(rows, true).unwrap().unwrap(); + let g = texts(&t); + assert_eq!(g[1], vec!["", "a", "b", "c"]); + let Block::Table(tbl) = &t else { unreachable!() }; + let CellSlot::Origin(c) = &tbl.grid[0][0] else { panic!() }; + assert_eq!(c.col_span, 4); + } + + #[test] + fn missing_left_keeps_rows_at_column_zero() { + let rows = vec![ + GridRow { cells: vec![cell("x", 1000), cell("y", 2000)], header: false, left: None }, + GridRow { cells: vec![cell("z", 2000)], header: false, left: None }, + ]; + let g = texts(&build_edge_table(rows, false).unwrap().unwrap()); + assert_eq!(g[1][0], "z"); + } +} diff --git a/src/shared/image_meta.rs b/src/shared/image_meta.rs new file mode 100644 index 00000000..4a7b1f1f --- /dev/null +++ b/src/shared/image_meta.rs @@ -0,0 +1,406 @@ +//! Fork (sznb): header-only image metadata — pixel dimensions and a file +//! extension, read from the first bytes of the payload without decoding. +//! +//! Covers what office documents embed: PNG, JPEG, GIF, BMP, WEBP, TIFF and +//! the Windows metafiles (EMF, placeable WMF). Anything unreadable yields +//! `None`, which callers treat as "size unknown" (the image is kept). + +fn u16_le(b: &[u8], at: usize) -> Option { + b.get(at..at + 2).map(|s| u16::from_le_bytes([s[0], s[1]]) as u32) +} + +fn u16_be(b: &[u8], at: usize) -> Option { + b.get(at..at + 2).map(|s| u16::from_be_bytes([s[0], s[1]]) as u32) +} + +fn u32_le(b: &[u8], at: usize) -> Option { + b.get(at..at + 4).map(|s| u32::from_le_bytes([s[0], s[1], s[2], s[3]])) +} + +fn u32_be(b: &[u8], at: usize) -> Option { + b.get(at..at + 4).map(|s| u32::from_be_bytes([s[0], s[1], s[2], s[3]])) +} + +fn i32_le(b: &[u8], at: usize) -> Option { + u32_le(b, at).map(|v| v as i32 as i64) +} + +fn i16_le(b: &[u8], at: usize) -> Option { + u16_le(b, at).map(|v| v as u16 as i16 as i64) +} + +fn u24_le(b: &[u8], at: usize) -> Option { + b.get(at..at + 3).map(|s| s[0] as u32 | (s[1] as u32) << 8 | (s[2] as u32) << 16) +} + +/// The image kinds the header sniffer recognizes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ImageKind { + /// Portable Network Graphics. + Png, + /// JPEG / JFIF / EXIF. + Jpeg, + /// GIF 87a / 89a. + Gif, + /// Windows bitmap. + Bmp, + /// WebP (lossy, lossless or extended). + Webp, + /// TIFF, either byte order. + Tiff, + /// Enhanced metafile. + Emf, + /// Windows metafile (placeable or bare). + Wmf, +} + +impl ImageKind { + /// The conventional file extension, with its leading dot. + pub fn ext(self) -> &'static str { + match self { + ImageKind::Png => ".png", + ImageKind::Jpeg => ".jpg", + ImageKind::Gif => ".gif", + ImageKind::Bmp => ".bmp", + ImageKind::Webp => ".webp", + ImageKind::Tiff => ".tif", + ImageKind::Emf => ".emf", + ImageKind::Wmf => ".wmf", + } + } + + /// The canonical MIME type. + pub fn media_type(self) -> &'static str { + match self { + ImageKind::Png => "image/png", + ImageKind::Jpeg => "image/jpeg", + ImageKind::Gif => "image/gif", + ImageKind::Bmp => "image/bmp", + ImageKind::Webp => "image/webp", + ImageKind::Tiff => "image/tiff", + ImageKind::Emf => "image/x-emf", + ImageKind::Wmf => "image/x-wmf", + } + } +} + +const WMF_PLACEABLE: u32 = 0x9AC6_CDD7; + +/// Identify the image kind from its signature. +pub fn sniff(b: &[u8]) -> Option { + if b.starts_with(b"\x89PNG\r\n\x1a\n") { + Some(ImageKind::Png) + } else if b.starts_with(&[0xFF, 0xD8, 0xFF]) { + Some(ImageKind::Jpeg) + } else if b.starts_with(b"GIF87a") || b.starts_with(b"GIF89a") { + Some(ImageKind::Gif) + } else if b.starts_with(b"BM") && b.len() >= 26 { + Some(ImageKind::Bmp) + } else if b.starts_with(b"RIFF") && b.get(8..12) == Some(b"WEBP") { + Some(ImageKind::Webp) + } else if b.starts_with(b"II*\0") || b.starts_with(b"MM\0*") { + Some(ImageKind::Tiff) + } else if u32_le(b, 0) == Some(1) && b.get(40..44) == Some(b" EMF") { + Some(ImageKind::Emf) + } else if u32_le(b, 0) == Some(WMF_PLACEABLE) + || (matches!(u16_le(b, 0), Some(1 | 2)) && u16_le(b, 2) == Some(9)) + { + Some(ImageKind::Wmf) + } else { + None + } +} + +/// Pixel dimensions `(width, height)` read from the header, or `None` when +/// the format is unrecognized or the header is truncated / implausible. +/// Metafiles report their bounds in device pixels (EMF) or at 96 dpi +/// (placeable WMF); bare WMF carries no extent. +pub fn dimensions(b: &[u8]) -> Option<(u32, u32)> { + let dims = match sniff(b)? { + ImageKind::Png => (u32_be(b, 16)?, u32_be(b, 20)?), + ImageKind::Jpeg => jpeg(b)?, + ImageKind::Gif => (u16_le(b, 6)?, u16_le(b, 8)?), + ImageKind::Bmp => { + if u32_le(b, 14)? == 12 { + (u16_le(b, 18)?, u16_le(b, 20)?) + } else { + (i32_le(b, 18)?.unsigned_abs() as u32, i32_le(b, 22)?.unsigned_abs() as u32) + } + } + ImageKind::Webp => webp(b)?, + ImageKind::Tiff => tiff(b)?, + ImageKind::Emf => emf(b)?, + ImageKind::Wmf => wmf(b)?, + }; + (dims.0 > 0 && dims.1 > 0).then_some(dims) +} + +fn jpeg(b: &[u8]) -> Option<(u32, u32)> { + let mut i = 2; + while i + 3 < b.len() { + if b[i] != 0xFF { + // Not at a marker: the stream is corrupt for our purposes. + return None; + } + // Fill bytes. + while i < b.len() && b[i] == 0xFF { + i += 1; + } + let marker = *b.get(i)?; + i += 1; + // Standalone markers carry no length. + if marker == 0x01 || (0xD0..=0xD9).contains(&marker) { + continue; + } + let len = u16_be(b, i)? as usize; + if len < 2 { + return None; + } + // SOFn (baseline, progressive, lossless, ...) but not DHT/JPG/DAC. + if (0xC0..=0xCF).contains(&marker) && !matches!(marker, 0xC4 | 0xC8 | 0xCC) { + let h = u16_be(b, i + 3)?; + let w = u16_be(b, i + 5)?; + return Some((w, h)); + } + if marker == 0xDA { + // Start of scan before any frame header. + return None; + } + i += len; + } + None +} + +fn webp(b: &[u8]) -> Option<(u32, u32)> { + match b.get(12..16)? { + b"VP8 " => { + if b.get(23..26)? != [0x9D, 0x01, 0x2A] { + return None; + } + Some((u16_le(b, 26)? & 0x3FFF, u16_le(b, 28)? & 0x3FFF)) + } + b"VP8L" => { + if *b.get(20)? != 0x2F { + return None; + } + let bits = u32_le(b, 21)?; + Some(((bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1)) + } + b"VP8X" => Some((u24_le(b, 24)? + 1, u24_le(b, 27)? + 1)), + _ => None, + } +} + +fn tiff(b: &[u8]) -> Option<(u32, u32)> { + let le = b.starts_with(b"II"); + let r16 = |at: usize| if le { u16_le(b, at) } else { u16_be(b, at) }; + let r32 = |at: usize| if le { u32_le(b, at) } else { u32_be(b, at) }; + let ifd = r32(4)? as usize; + let count = r16(ifd)? as usize; + let (mut w, mut h) = (None, None); + for n in 0..count.min(512) { + let e = ifd + 2 + n * 12; + let tag = r16(e)?; + if tag != 256 && tag != 257 { + continue; + } + let value = match r16(e + 2)? { + 3 => r16(e + 8)?, + 4 => r32(e + 8)?, + _ => continue, + }; + if tag == 256 { + w = Some(value); + } else { + h = Some(value); + } + if w.is_some() && h.is_some() { + break; + } + } + Some((w?, h?)) +} + +fn emf(b: &[u8]) -> Option<(u32, u32)> { + // EMR_HEADER.rclBounds: inclusive device-pixel rectangle. + let (l, t, r, btm) = (i32_le(b, 8)?, i32_le(b, 12)?, i32_le(b, 16)?, i32_le(b, 20)?); + let (w, h) = (r - l + 1, btm - t + 1); + if w > 1 && h > 1 { + return Some((w as u32, h as u32)); + } + // Empty bounds: fall back to rclFrame (0.01 mm) scaled by the reference + // device's pixels-per-millimetre. + let (fl, ft, fr, fb) = (i32_le(b, 24)?, i32_le(b, 28)?, i32_le(b, 32)?, i32_le(b, 36)?); + let (dev_w, dev_h) = (i32_le(b, 72)?, i32_le(b, 76)?); + let (mm_w, mm_h) = (i32_le(b, 80)?, i32_le(b, 84)?); + if mm_w <= 0 || mm_h <= 0 { + return None; + } + let w = (fr - fl) * dev_w / (mm_w * 100); + let h = (fb - ft) * dev_h / (mm_h * 100); + (w > 0 && h > 0).then(|| (w as u32, h as u32)) +} + +fn wmf(b: &[u8]) -> Option<(u32, u32)> { + if u32_le(b, 0)? != WMF_PLACEABLE { + // A bare WMF has no extent in its header. + return None; + } + let (l, t, r, btm) = (i16_le(b, 6)?, i16_le(b, 8)?, i16_le(b, 10)?, i16_le(b, 12)?); + let inch = u16_le(b, 14)? as i64; + if inch == 0 { + return None; + } + let w = (r - l).abs() * 96 / inch; + let h = (btm - t).abs() * 96 / inch; + Some((w as u32, h as u32)) +} + +/// The file extension (with its dot) for an image payload: sniffed from the +/// bytes first, then the declared MIME type, then the source part's own +/// extension, finally `.bin`. +pub fn ext_for(bytes: &[u8], media_type: &str, origin_part: &str) -> String { + if let Some(kind) = sniff(bytes) { + return kind.ext().to_string(); + } + let from_mime = match media_type.to_ascii_lowercase().as_str() { + "image/png" => Some(".png"), + "image/jpeg" | "image/jpg" | "image/pjpeg" => Some(".jpg"), + "image/gif" => Some(".gif"), + "image/bmp" | "image/x-bmp" | "image/x-ms-bmp" => Some(".bmp"), + "image/webp" => Some(".webp"), + "image/tiff" | "image/tif" => Some(".tif"), + "image/x-emf" | "image/emf" => Some(".emf"), + "image/x-wmf" | "image/wmf" => Some(".wmf"), + "image/svg+xml" => Some(".svg"), + _ => None, + }; + if let Some(ext) = from_mime { + return ext.to_string(); + } + let tail = origin_part.rsplit(['/', '\\']).next().unwrap_or(""); + if let Some((_, ext)) = tail.rsplit_once('.') { + if !ext.is_empty() && ext.len() <= 5 && ext.chars().all(|c| c.is_ascii_alphanumeric()) { + return format!(".{}", ext.to_ascii_lowercase()); + } + } + ".bin".to_string() +} + +/// The MIME type for an image payload: sniffed, else the declared one. +pub fn media_type_for(bytes: &[u8], declared: &str) -> String { + match sniff(bytes) { + Some(kind) => kind.media_type().to_string(), + None => declared.to_string(), + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + /// A minimal PNG header (signature + IHDR) of the given size. + pub(crate) fn png(w: u32, h: u32) -> Vec { + let mut b = b"\x89PNG\r\n\x1a\n".to_vec(); + b.extend_from_slice(&13u32.to_be_bytes()); + b.extend_from_slice(b"IHDR"); + b.extend_from_slice(&w.to_be_bytes()); + b.extend_from_slice(&h.to_be_bytes()); + b.extend_from_slice(&[8, 6, 0, 0, 0, 0, 0, 0, 0]); + b + } + + #[test] + fn png_header() { + assert_eq!(dimensions(&png(640, 480)), Some((640, 480))); + assert_eq!(ext_for(&png(1, 1), "", ""), ".png"); + } + + #[test] + fn jpeg_skips_app_segments_to_the_frame_header() { + let mut b = vec![0xFF, 0xD8]; + // APP0 with 14 payload bytes. + b.extend_from_slice(&[0xFF, 0xE0, 0x00, 0x10]); + b.extend_from_slice(b"JFIF\0\x01\x01\0\0\x01\0\x01\0\0"); + // SOF2 (progressive): len, precision, height 300, width 200. + b.extend_from_slice(&[0xFF, 0xC2, 0x00, 0x11, 0x08, 0x01, 0x2C, 0x00, 0xC8, 0x03]); + b.extend_from_slice(&[0; 9]); + assert_eq!(dimensions(&b), Some((200, 300))); + assert_eq!(ext_for(&b, "image/png", "x.png"), ".jpg"); + } + + #[test] + fn jpeg_dht_is_not_a_frame_header() { + let mut b = vec![0xFF, 0xD8, 0xFF, 0xC4, 0x00, 0x04, 0xAA, 0xBB]; + b.extend_from_slice(&[0xFF, 0xC0, 0x00, 0x11, 0x08, 0x00, 0x20, 0x00, 0x40, 0x03]); + b.extend_from_slice(&[0; 9]); + assert_eq!(dimensions(&b), Some((64, 32))); + } + + #[test] + fn gif_bmp_webp_tiff() { + let mut gif = b"GIF89a".to_vec(); + gif.extend_from_slice(&[10, 0, 20, 0, 0, 0, 0]); + assert_eq!(dimensions(&gif), Some((10, 20))); + + let mut bmp = b"BM".to_vec(); + bmp.extend_from_slice(&[0; 12]); + bmp.extend_from_slice(&40u32.to_le_bytes()); + bmp.extend_from_slice(&120i32.to_le_bytes()); + bmp.extend_from_slice(&(-80i32).to_le_bytes()); // top-down + assert_eq!(dimensions(&bmp), Some((120, 80))); + + let mut vp8x = b"RIFF\0\0\0\0WEBPVP8X".to_vec(); + vp8x.extend_from_slice(&[0; 8]); + vp8x.extend_from_slice(&[99, 0, 0, 49, 0, 0]); + assert_eq!(dimensions(&vp8x), Some((100, 50))); + + let mut vp8l = b"RIFF\0\0\0\0WEBPVP8L\0\0\0\0\x2f".to_vec(); + let bits: u32 = (15 - 1) | ((25 - 1) << 14); + vp8l.extend_from_slice(&bits.to_le_bytes()); + assert_eq!(dimensions(&vp8l), Some((15, 25))); + + // Little-endian TIFF: IFD at 8, two entries (SHORT width, LONG height). + let mut tif = b"II*\0".to_vec(); + tif.extend_from_slice(&8u32.to_le_bytes()); + tif.extend_from_slice(&2u16.to_le_bytes()); + tif.extend_from_slice(&256u16.to_le_bytes()); + tif.extend_from_slice(&3u16.to_le_bytes()); + tif.extend_from_slice(&1u32.to_le_bytes()); + tif.extend_from_slice(&[0x20, 0x03, 0, 0]); // 800 + tif.extend_from_slice(&257u16.to_le_bytes()); + tif.extend_from_slice(&4u16.to_le_bytes()); + tif.extend_from_slice(&1u32.to_le_bytes()); + tif.extend_from_slice(&600u32.to_le_bytes()); + assert_eq!(dimensions(&tif), Some((800, 600))); + } + + #[test] + fn metafiles() { + let mut emf = vec![0u8; 88]; + emf[0] = 1; + emf[8..12].copy_from_slice(&0i32.to_le_bytes()); + emf[12..16].copy_from_slice(&0i32.to_le_bytes()); + emf[16..20].copy_from_slice(&299i32.to_le_bytes()); + emf[20..24].copy_from_slice(&149i32.to_le_bytes()); + emf[40..44].copy_from_slice(b" EMF"); + assert_eq!(dimensions(&emf), Some((300, 150))); + assert_eq!(ext_for(&emf, "", ""), ".emf"); + + let mut wmf = vec![0u8; 22]; + wmf[0..4].copy_from_slice(&WMF_PLACEABLE.to_le_bytes()); + wmf[10..12].copy_from_slice(&1440i16.to_le_bytes()); + wmf[12..14].copy_from_slice(&720i16.to_le_bytes()); + wmf[14..16].copy_from_slice(&1440u16.to_le_bytes()); + assert_eq!(dimensions(&wmf), Some((96, 48))); + } + + #[test] + fn unknown_or_truncated_is_none() { + assert_eq!(dimensions(b"not an image"), None); + assert_eq!(dimensions(&png(0, 10)), None); + assert_eq!(dimensions(&b"\x89PNG\r\n\x1a\n"[..]), None); + assert_eq!(ext_for(b"???", "application/x-foo", "word/media/image3.jpeg"), ".jpeg"); + assert_eq!(ext_for(b"???", "", ""), ".bin"); + } +} diff --git a/src/shared/list.rs b/src/shared/list.rs index 4523ace9..d76628ae 100644 --- a/src/shared/list.rs +++ b/src/shared/list.rs @@ -92,17 +92,15 @@ fn build_lists(entries: Vec) -> Vec { if sublists.is_empty() { continue; } - if current.is_none() { - // Sub-level content with no parent item yet: host it in an - // anonymous item so nesting is preserved. - let key = ListKey { instance: u64::MAX, marker: MarkerKind::Bullet }; - current = Some(( - List { marker: MarkerKind::Bullet, start: 1, items: Vec::new() }, - key, - 0, - )); - } - let (list, _, _) = current.as_mut().unwrap(); + // Fork (sznb): sub-level content with no parent item yet (a run + // that opens below its shallowest level, e.g. `1.1.` items ahead + // of the first `六、`) is hoisted as sibling blocks. Upstream + // hosted it in an anonymous bullet item, which invented a `- ` + // marker and an indent the source never shows. + let Some((list, _, _)) = current.as_mut() else { + out.extend(sublists); + continue; + }; if list.items.is_empty() { list.items.push(ListItem::default()); } @@ -193,4 +191,22 @@ mod tests { assert_eq!(l.items.len(), 2); assert!(matches!(l.items[0].blocks.last(), Some(Block::List(sub)) if sub.ordered())); } + + #[test] + fn orphan_sub_level_is_hoisted_without_an_anonymous_bullet() { + // Fork (sznb): deeper entries ahead of the first shallow entry become + // sibling lists, not children of an invented `- ` item. + let out = lists(vec![ + entry(1, 1, MarkerKind::Decimal, 1, "deep1"), + entry(1, 1, MarkerKind::Decimal, 2, "deep2"), + entry(0, 2, MarkerKind::Decimal, 6, "shallow"), + ]); + assert_eq!(out.len(), 2); + let Block::List(first) = &out[0] else { panic!() }; + assert!(first.ordered()); + assert_eq!((first.start, first.items.len()), (1, 2)); + assert!(!first.items.iter().any(|i| matches!(i.blocks.first(), Some(Block::List(_))))); + let Block::List(second) = &out[1] else { panic!() }; + assert_eq!(second.start, 6); + } } diff --git a/src/shared/mod.rs b/src/shared/mod.rs index 8e422408..9b54161a 100644 --- a/src/shared/mod.rs +++ b/src/shared/mod.rs @@ -13,6 +13,7 @@ pub mod fields; pub mod grid; pub mod header; pub mod html; +pub mod image_meta; pub mod list; pub mod math; pub mod mc; diff --git a/src/transform/images.rs b/src/transform/images.rs new file mode 100644 index 00000000..140c628a --- /dev/null +++ b/src/transform/images.rs @@ -0,0 +1,357 @@ +//! Fork (sznb): image extraction for mixed text/image output. +//! +//! [`extract_images`] walks a parsed [`Document`], measures every embedded +//! image (header-only, see `shared::image_meta`), drops images smaller than +//! the threshold, and replaces the rest in place by a placeholder token +//! (`[IMG::]` by default). The extracted payloads come back in an +//! [`ImageReport`] so a caller can store them (e.g. upload to object storage) +//! under the very names the placeholders carry. Everything stays in memory. +//! +//! Names are content-addressed (`{prefix}{sha256[..16]}{ext}`): the same +//! picture referenced twice, or embedded twice under different parts, maps to +//! one image and one name, and re-uploading it is idempotent. + +use std::collections::HashMap; + +use sha2::{Digest, Sha256}; + +use crate::model::{AssetId, Block, CellSlot, Document, ImageSource, Inline}; +use crate::shared::image_meta; + +/// What to extract and how to name it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImageOptions { + /// Images narrower than this many pixels are dropped (0 disables). + pub min_width: u32, + /// Images shorter than this many pixels are dropped (0 disables). + pub min_height: u32, + /// Placeholder template; `{name}` is replaced by the image's file name. + pub placeholder: String, + /// Prefix of every generated file name. + pub name_prefix: String, +} + +impl Default for ImageOptions { + fn default() -> Self { + ImageOptions { + min_width: 50, + min_height: 50, + placeholder: "[IMG::{name}]".to_string(), + name_prefix: "img_".to_string(), + } + } +} + +/// One extracted image. +#[derive(Debug, Clone)] +pub struct ExtractedImage { + /// Content-addressed file name, e.g. `img_3f2a9c0d1b2e4f56.png`. + pub name: String, + /// The token that stands in for it in the rendered output. + pub placeholder: String, + /// MIME type (sniffed from the bytes when recognizable). + pub media_type: String, + /// Pixel width, when the header gives one. + pub width: Option, + /// Pixel height, when the header gives one. + pub height: Option, + /// The payload, exactly as embedded. + pub bytes: Vec, + /// How many places in the document reference it. + pub occurrences: usize, + /// Package part or stream the first copy came from. + pub origin_part: String, +} + +/// The result of [`extract_images`]. +#[derive(Debug, Clone, Default)] +pub struct ImageReport { + /// Extracted images, in order of first appearance (body, then notes). + pub images: Vec, + /// Image references dropped for being under the size threshold. + pub skipped: usize, +} + +#[derive(Clone, Copy)] +enum Decision { + /// Keep: index into `ImageReport::images`. + Keep(usize), + /// Under the size threshold. + Drop, + /// Not extractable (asset missing / empty): leave the inline untouched. + Leave, +} + +struct Extractor<'a> { + opts: &'a ImageOptions, + assets: &'a [crate::model::Asset], + decided: HashMap, + by_name: HashMap, + report: ImageReport, +} + +/// Extract the embedded images of `doc` in place: small ones are removed, +/// the rest are replaced by [`ImageSource::Placeholder`] tokens. External and +/// unavailable images are left as they are. +pub fn extract_images(doc: &mut Document, opts: &ImageOptions) -> ImageReport { + let assets = std::mem::take(&mut doc.assets); + let mut ex = Extractor { + opts, + assets: &assets, + decided: HashMap::new(), + by_name: HashMap::new(), + report: ImageReport::default(), + }; + ex.blocks(&mut doc.blocks); + for note in &mut doc.notes { + ex.blocks(&mut note.blocks); + } + let report = ex.report; + doc.assets = assets; + report +} + +impl Extractor<'_> { + fn blocks(&mut self, blocks: &mut [Block]) { + for block in blocks { + match block { + Block::Heading { content, .. } | Block::Paragraph(content) => self.inlines(content), + Block::List(list) => { + for item in &mut list.items { + self.blocks(&mut item.blocks); + } + } + Block::Table(table) => { + for row in &mut table.grid { + for slot in row { + if let CellSlot::Origin(cell) = slot { + self.blocks(&mut cell.blocks); + } + } + } + } + Block::BlockQuote(inner) => self.blocks(inner), + Block::CodeBlock { .. } | Block::Rule | Block::Math(_) => {} + } + } + } + + fn inlines(&mut self, inlines: &mut Vec) { + inlines.retain_mut(|inline| match inline { + Inline::Link { content, .. } => { + self.inlines(content); + true + } + Inline::Image { source, .. } => { + let ImageSource::Asset(id) = *source else { return true }; + match self.decide(id) { + Decision::Keep(idx) => { + let image = &mut self.report.images[idx]; + image.occurrences += 1; + *source = ImageSource::Placeholder(image.placeholder.clone()); + true + } + Decision::Drop => { + self.report.skipped += 1; + false + } + Decision::Leave => true, + } + } + _ => true, + }); + } + + fn decide(&mut self, id: AssetId) -> Decision { + if let Some(&decision) = self.decided.get(&id) { + return decision; + } + let decision = self.classify(id); + self.decided.insert(id, decision); + decision + } + + fn classify(&mut self, id: AssetId) -> Decision { + let Some(asset) = self.assets.get(id.0) else { return Decision::Leave }; + if asset.bytes.is_empty() { + return Decision::Leave; + } + // Only pictures: an embedded OLE payload (Word / PowerPoint object + // shown through an image inline) keeps its "Embedded object" alt text, + // as the python pipeline only ever extracts `media/image*` parts. + if image_meta::sniff(&asset.bytes).is_none() + && !asset.media_type.to_ascii_lowercase().starts_with("image/") + { + return Decision::Leave; + } + let dims = image_meta::dimensions(&asset.bytes); + // Unknown size keeps the image (same rule as the python pipeline). + if let Some((w, h)) = dims { + if w < self.opts.min_width || h < self.opts.min_height { + return Decision::Drop; + } + } + let digest = Sha256::digest(&asset.bytes); + let hex: String = digest.iter().take(8).map(|b| format!("{b:02x}")).collect(); + let ext = image_meta::ext_for(&asset.bytes, &asset.media_type, &asset.origin_part); + let name = format!("{}{}{}", self.opts.name_prefix, hex, ext); + if let Some(&idx) = self.by_name.get(&name) { + return Decision::Keep(idx); + } + let idx = self.report.images.len(); + self.report.images.push(ExtractedImage { + placeholder: self.opts.placeholder.replace("{name}", &name), + name: name.clone(), + media_type: image_meta::media_type_for(&asset.bytes, &asset.media_type), + width: dims.map(|d| d.0), + height: dims.map(|d| d.1), + bytes: asset.bytes.clone(), + occurrences: 0, + origin_part: asset.origin_part.clone(), + }); + self.by_name.insert(name, idx); + Decision::Keep(idx) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{Asset, Cell, Table, TableKind}; + use crate::render::markdown::document_to_markdown_with; + use crate::render::text::document_to_text_with; + use crate::shared::image_meta::tests::png; + + fn asset(id: usize, bytes: Vec) -> Asset { + Asset { + id: AssetId(id), + media_type: "image/png".into(), + origin_part: format!("word/media/image{id}.png"), + bytes, + } + } + + fn img(id: usize) -> Inline { + Inline::Image { alt: "图".into(), source: ImageSource::Asset(AssetId(id)) } + } + + fn sample() -> Document { + let big = png(200, 100); + let mut doc = Document::default(); + // 0: big; 1: 10x10 icon; 2: the big image again under another part. + doc.assets = vec![asset(0, big.clone()), asset(1, png(10, 10)), asset(2, big)]; + doc.blocks = vec![ + Block::Paragraph(vec![Inline::plain("前文"), img(0), Inline::plain("后文")]), + Block::Paragraph(vec![img(1)]), + Block::Table(Table::from_rows( + vec![vec![ + Cell::from_inlines(vec![Inline::plain("单元格"), img(2)]), + Cell::from_inlines(vec![img(1), Inline::plain("x")]), + ]], + 0, + TableKind::Data, + )), + ]; + doc + } + + #[test] + fn extracts_filters_and_dedups() { + let mut doc = sample(); + let report = extract_images(&mut doc, &ImageOptions::default()); + assert_eq!(report.images.len(), 1, "same bytes under two parts -> one image"); + assert_eq!(report.skipped, 2, "the 10x10 icon is referenced twice"); + let image = &report.images[0]; + assert_eq!(image.occurrences, 2); + assert_eq!((image.width, image.height), (Some(200), Some(100))); + assert!(image.name.starts_with("img_") && image.name.ends_with(".png")); + assert_eq!(image.name.len(), "img_".len() + 16 + ".png".len()); + assert_eq!(image.placeholder, format!("[IMG::{}]", image.name)); + // Assets survive the walk. + assert_eq!(doc.assets.len(), 3); + + let md = document_to_markdown_with(&doc, true, true); + let token = &image.placeholder; + assert!(md.contains(&format!("前文{token}后文")), "{md}"); + assert_eq!(md.matches(token.as_str()).count(), 2, "{md}"); + assert!(!md.contains('图'), "alt text is replaced, not appended: {md}"); + + let gfm = document_to_markdown_with(&doc, false, true); + assert_eq!(gfm.matches(token.as_str()).count(), 2, "{gfm}"); + + let text = document_to_text_with(&doc, true); + assert!(text.contains(&format!("前文{token}后文")), "{text}"); + assert!(text.contains(&format!("单元格{token}")), "{text}"); + // The image-only paragraph of the small icon disappears entirely. + assert!(!text.contains("\n\n\n"), "{text:?}"); + } + + #[test] + fn thresholds_and_template_are_configurable() { + let mut doc = sample(); + let opts = ImageOptions { + min_width: 0, + min_height: 0, + placeholder: "<<{name}>>".into(), + name_prefix: "p_".into(), + }; + let report = extract_images(&mut doc, &opts); + assert_eq!(report.images.len(), 2); + assert_eq!(report.skipped, 0); + assert!(report.images.iter().all(|i| i.name.starts_with("p_"))); + assert_eq!(report.images[1].placeholder, format!("<<{}>>", report.images[1].name)); + } + + #[test] + fn unknown_size_is_kept_and_external_untouched() { + let mut doc = Document::default(); + doc.assets = vec![Asset { + id: AssetId(0), + media_type: "image/x-wmf".into(), + origin_part: "word/media/image1.wmf".into(), + bytes: vec![1, 0, 9, 0, 0, 3, 0, 0], // bare WMF: no extent + }]; + doc.blocks = vec![Block::Paragraph(vec![ + img(0), + Inline::Image { alt: "外链".into(), source: ImageSource::External("https://e.com/a.png".into()) }, + ])]; + let report = extract_images(&mut doc, &ImageOptions::default()); + assert_eq!(report.images.len(), 1); + assert_eq!(report.images[0].width, None); + assert!(report.images[0].name.ends_with(".wmf")); + let md = document_to_markdown_with(&doc, true, true); + assert!(md.contains("![外链](https://e.com/a.png)"), "{md}"); + } + + #[test] + fn ole_payloads_keep_their_alt_text() { + let mut doc = Document::default(); + doc.assets = vec![Asset { + id: AssetId(0), + media_type: "application/vnd.ms-ole-object".into(), + origin_part: "word/embeddings/oleObject1.bin".into(), + bytes: vec![0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1], + }]; + doc.blocks = vec![Block::Paragraph(vec![Inline::Image { + alt: "Embedded object: Word.Document.12".into(), + source: ImageSource::Asset(AssetId(0)), + }])]; + let report = extract_images(&mut doc, &ImageOptions::default()); + assert!(report.images.is_empty() && report.skipped == 0); + let md = document_to_markdown_with(&doc, true, true); + assert!(md.contains("Embedded object: Word.Document.12"), "{md}"); + } + + #[test] + fn external_only_document_is_untouched() { + let mut doc = Document::default(); + doc.blocks = vec![Block::Paragraph(vec![Inline::Image { + alt: "外链".into(), + source: ImageSource::External("https://e.com/a.png".into()), + }])]; + let report = extract_images(&mut doc, &ImageOptions::default()); + assert!(report.images.is_empty()); + let md = document_to_markdown_with(&doc, true, true); + assert!(md.contains("![外链](https://e.com/a.png)"), "{md}"); + } +} diff --git a/src/transform/mod.rs b/src/transform/mod.rs new file mode 100644 index 00000000..df4f7741 --- /dev/null +++ b/src/transform/mod.rs @@ -0,0 +1,4 @@ +//! Fork (sznb): document-model transforms applied between parsing and +//! rendering. + +pub mod images;