diff --git a/docs/HEADERLESS_TABLES.md b/docs/HEADERLESS_TABLES.md new file mode 100644 index 0000000..c2687fe --- /dev/null +++ b/docs/HEADERLESS_TABLES.md @@ -0,0 +1,43 @@ +# Header-neutral table conversion + +`convert_pdf_to_markdown` accepts the explicit, default-off +`allow_headerless_tables: true` option. The selected renderer profile is +`1.22.0`; ordinary calls and explicit `false` retain profile `1.21.0` and +the previous complete result envelope. + +This addresses a narrow representational limit: GFM assigns header semantics +to its first row, so the default converter refuses a grid whose source does +not distinguish a header. The opt-in profile can preserve that grid as an +HTML table containing only escaped `` data cells. It creates no ``, +spans, inherited values or semantic blank values. Header roles remain +unestablished and a typed conversion gap remains visible. + +The fallback applies only to complete source-covered ruled-rectangle cell +grids. Every cell must have rectangle evidence, every relevant source line +and item must be accounted for, and each text advance box must fit its one +cell. Missing/competing rectangles, overlapping items, cross-cell text, +truncated evidence, suspect text and ordinary aligned prose do not qualify. +This does not broaden the captioned/solid-mask, segmented-rule, or unruled +table paths. Those and existing GFM tables keep their prior rules. Source +text is HTML-escaped, never interpreted as markup or an attribute. + +Example tool arguments: + +```json +{ + "pdf_path": "/absolute/local/document.pdf", + "allow_headerless_tables": true +} +``` + +The option composes with `compact` and `emit_table_proposals`. Table cells +are excluded from compact normalizations; if proposal emission is requested, +the unresolved-header region remains source-bound and available for review. +No OCR, model call or automatic verification is introduced. Empty cells +represent retained text coverage, not proof that the source's data is blank. + +The earlier full corpus score belongs to the default text-only profile and +has not been improved, replaced or rescored by this option. Synthetic table +success does not establish general table accuracy, scan quality, unique +semantic topology or competitive benchmark performance. The broader +equation, scanned-document and uncertain-topology gaps remain separate work. diff --git a/docs/MCP_CONTRACT.md b/docs/MCP_CONTRACT.md index 344d4c0..cdc6fa5 100644 --- a/docs/MCP_CONTRACT.md +++ b/docs/MCP_CONTRACT.md @@ -91,6 +91,12 @@ means persistence; each result retains its own verification or uncertainty statu `review_extraction_workspace` is a read-only, source- and generation-bound viewer entrypoint with bounded value, history, citation, and page-coverage views. +`convert_pdf_to_markdown` has a default-off `allow_headerless_tables` option +for narrowly qualified complete ruled-rectangle grids. The opted-in renderer +profile `1.22.0` emits escaped HTML data cells without assigning header roles; +typed uncertainty remains. Omitted or false retains the unchanged default +profile `1.21.0`. See [Header-neutral tables](HEADERLESS_TABLES.md). + `get_pdf_identity` provides parser-independent artifact binding for planning and provenance. It streams at most 250 MiB from one read-only descriptor, returns the canonical local path, exact byte length, and SHA-256, and rejects diff --git a/pdf-toolkit-mcp-share/server/index.js b/pdf-toolkit-mcp-share/server/index.js index a8d891d..a26eb4b 100755 --- a/pdf-toolkit-mcp-share/server/index.js +++ b/pdf-toolkit-mcp-share/server/index.js @@ -3772,7 +3772,7 @@ async function listTools(request) { }, { name: "convert_pdf_to_markdown", - description: "Convert a bounded page range of a local PDF into deterministic Markdown from the source-validated PDF Tools Extraction IR. Headings and lists require geometry or literal marker evidence. A table requires a complete recurring text grid, a clean ruled-rectangle grid, or a complete closed painted grid, with every item in one cell and real header evidence. A link requires a source-validated external http or https annotation target covering one contiguous text run. Unsupported tables and link targets, including merged cells, internal destinations, actions, and other schemes, stay escaped text with typed gaps. Compact extreme-margin page numbers and provenance furniture, plus same-band text repeated across selected pages, are removed with typed counts by default; detected headings and all source-evidenced table-region lines are preserved, and removal can be disabled. No OCR or external model. Optionally enable compact mode for counted dot-leader and spaced-hyphen normalizations. Optionally saves transactional UTF-8 Markdown. Use absolute or ~/ local paths, not Claude container paths (/mnt/...).", + description: "Convert a bounded page range of a local PDF into deterministic Markdown from the source-validated PDF Tools Extraction IR. Headings and lists require geometry or literal marker evidence. By default, a table requires a complete recurring text grid, a clean ruled-rectangle grid, or a complete closed painted grid, with every item in one cell and real header evidence. A link requires a source-validated external http or https annotation target covering one contiguous text run. Unsupported tables and link targets, including merged cells, internal destinations, actions, and other schemes, stay escaped text with typed gaps. Compact extreme-margin page numbers and provenance furniture, plus same-band text repeated across selected pages, are removed with typed counts by default; detected headings and all source-evidenced table-region lines are preserved, and removal can be disabled. No OCR or external model. Optionally enable compact mode for counted dot-leader and spaced-hyphen normalizations. Optionally saves transactional UTF-8 Markdown. Use absolute or ~/ local paths, not Claude container paths (/mnt/...).", inputSchema: { type: "object", additionalProperties: false, @@ -3788,6 +3788,7 @@ async function listTools(request) { compact: { type: "boolean", description: "Opt into counted dot-leader, isolated page-number, and Unicode-letter spaced-hyphen normalizations. Default: false." }, remove_page_furniture: { type: "boolean", description: "Remove only source-evidenced extreme-margin page furniture and report every removed line by typed gap and counts. Set false to preserve all source lines. Default: true." }, emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, + allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.22.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." }, output_path: { type: "string", description: "Optional absolute .md path, or ~/ path. The file is written only after complete bytes are staged and verified." }, overwrite: { type: "boolean", description: "Replace an existing output_path only when its exact expected_output_identity is also supplied. Default: false." }, expected_output_identity: EXPECTED_MARKDOWN_OUTPUT_IDENTITY_INPUT_SCHEMA @@ -5867,6 +5868,7 @@ async function handleToolCall(request) { "compact", "remove_page_furniture", "emit_table_proposals", + "allow_headerless_tables", "output_path", "overwrite", "expected_output_identity", @@ -5884,6 +5886,7 @@ async function handleToolCall(request) { true, ); const emitTableProposals = optionalBooleanArgument(markdownArgs.emit_table_proposals, "emit_table_proposals", false); + const allowHeaderlessTables = optionalBooleanArgument(markdownArgs.allow_headerless_tables, "allow_headerless_tables", false); const overwrite = optionalBooleanArgument(markdownArgs.overwrite, "overwrite", false); const expectedOutputIdentity = normalizeExpectedOutputIdentity( markdownArgs.expected_output_identity, @@ -5944,6 +5947,7 @@ async function handleToolCall(request) { compact, removePageFurniture, emitTableProposals, + allowHeaderlessTables, }); const pagesNeedingVision = deriveMarkdownVisionRouting(layout); diff --git a/pdf-toolkit-mcp-share/server/markdown-conversion.js b/pdf-toolkit-mcp-share/server/markdown-conversion.js index d659bb8..bf6c9ee 100644 --- a/pdf-toolkit-mcp-share/server/markdown-conversion.js +++ b/pdf-toolkit-mcp-share/server/markdown-conversion.js @@ -8,6 +8,7 @@ const RENDERER = Object.freeze({ name: "pdf-tools.layout-markdown-renderer", version: "1.21.0", }); +const HEADERLESS_RENDERER = Object.freeze({ ...RENDERER, version: "1.22.0" }); const SUPPORTED_LAYOUT_IR_VERSION = "1.6.0"; // Bounded geometric table inference. A run of adjacent lines is treated as a @@ -97,6 +98,14 @@ const LIMITATIONS = Object.freeze([ "Unsafe control characters and malformed UTF-16 surrogates are replaced with the Unicode replacement character and reported as conversion gaps.", ]); +function headerlessLimitations() { + return [ + ...LIMITATIONS.map(value => value.startsWith("Tables are reconstructed") + ? `Default GFM table rules: ${value}` : value), + "The explicit header-neutral profile may instead emit a complete, fully source-covered ruled-rectangle cell grid as escaped HTML table data cells when header evidence is absent. It assigns no header roles, spans, inherited values or semantic blank values. A typed header-role gap remains, and partial, ambiguous or cross-cell evidence stays reading-order text. This does not perform OCR or interpret cell artwork.", + ]; +} + function assertion(condition, message) { if (!condition) throw new Error(`Invalid Markdown conversion semantics: ${message}`); } @@ -2009,7 +2018,7 @@ function hasFirstRowBandEvidence(bands) { && Math.abs(rectBottom(band.rect) - rectBottom(firstRow.rect)) <= RECT_SNAP_TOLERANCE); } -function tryBuildRectGrid(page, run, clusters, itemById) { +function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables = false) { const runLineIds = new Set(run.map(row => row.line.id)); const candidates = []; for (const cluster of clusters) { @@ -2178,10 +2187,46 @@ function tryBuildRectGrid(page, run, clusters, itemById) { const candidateRows = lineRecords .sort((left, right) => left.line.y - right.line.y) .map(record => ({ line: record.line, cells: record.structuralInsideNonWhitespace })); - if (!hasHeaderEvidence(candidateRows) && !hasFirstRowBandEvidence(bands)) { + const headerless = !hasHeaderEvidence(candidateRows) && !hasFirstRowBandEvidence(bands); + if (headerless && !allowHeaderlessTables) { return { reason: "header" }; } if (candidateRows.some(row => !runLineIds.has(row.line.id))) return { reason: "topology" }; + if (headerless) { + // Header-neutral emission is narrower than the established GFM path. + // A partial rectangle census or a center-only cell assignment cannot + // establish the complete grid this profile promises to preserve. + const candidateLineIds = new Set(candidateRows.map(row => row.line.id)); + if (candidateLineIds.size !== runLineIds.size + || [...runLineIds].some(id => !candidateLineIds.has(id))) { + return { reason: "topology" }; + } + // Vector paint makes an otherwise fully observed text page "partial" + // in the IR. Test evidence completeness, not that modality label. + if (page.truncation?.truncated || page.text_layer_status !== "present" + || page.image_detection_status === "failed" || page.errors.length !== 0 + || page.link_annotations.status !== "available" || page.link_annotations.truncated + || page.painted_rectangles.status !== "available" || page.painted_rectangles.truncated + || page.ruling_segments.status !== "available" || page.ruling_segments.truncated + || page.text_integrity?.status === "suspect" || bands.length !== 0 + || rowCount * columnCount > RULED_TABLE_MAX_CELLS + || filledCells !== totalCells || occupiedCells.size !== totalCells + || page.raw_items.some(item => !item.geometry_valid || (item.text?.trim() && !itemBox(item)))) { + return { reason: "topology" }; + } + for (let row = 0; row < rowCount; row += 1) { + for (let column = 0; column < columnCount; column += 1) { + if (cells[row][column].some(({ item }) => ( + item.bbox.x < xEdges[column] - RECT_CELL_ASSIGNMENT_SLACK + || item.bbox.x + item.bbox.width > xEdges[column + 1] + RECT_CELL_ASSIGNMENT_SLACK + || item.bbox.y < yEdges[row] - RECT_CELL_ASSIGNMENT_SLACK + || item.bbox.y + item.bbox.height > yEdges[row + 1] + RECT_CELL_ASSIGNMENT_SLACK + ))) return { reason: "topology" }; + } + } + // Keep empty outer slots rather than trimming source grid columns. + return { kind: "table", grid, rows: candidateRows, headerless: true }; + } return { kind: "table", grid: trimmedGrid, rows: candidateRows }; } return null; @@ -2598,7 +2643,7 @@ function segmentedRuleTable(page, rows) { coveredLineIds: selected, insertionIndex: indices[0] }; } -function segmentTextRows(page, rows, itemById, ruledClusters) { +function segmentTextRows(page, rows, itemById, ruledClusters, allowHeaderlessTables = false) { const pageItems = paintedItems(page); const segments = []; const regions = []; @@ -2622,10 +2667,14 @@ function segmentTextRows(page, rows, itemById, ruledClusters) { // Ordinary prose that happens to share a left margin must not be // reported as an unreconstructed table. const ruling = ruledClusters.length > 0 - ? tryBuildRectGrid(page, run, ruledClusters, itemById) + ? tryBuildRectGrid(page, run, ruledClusters, itemById, allowHeaderlessTables) : null; if (ruling?.kind === "table") { - segments.push({ kind: "table", rows: run, grid: ruling.grid }); + segments.push({ kind: "table", rows: run, grid: ruling.grid, ...(ruling.headerless ? { headerless: true } : {}) }); + if (ruling.headerless) { + tableReason = "header"; + regions.push({ run, reason: "header" }); + } } else { // Preserve the reason from the rect-evidence attempt. The text path // can independently see a header/topology failure, but it must not @@ -2661,7 +2710,7 @@ function segmentTextRows(page, rows, itemById, ruledClusters) { * that cannot be reconstructed are returned as text with tableReason set, so * the caller can report typed partial coverage instead of silently flattening. */ -function segmentPageLines(page) { +function segmentPageLines(page, allowHeaderlessTables = false) { const itemById = new Map(page.raw_items.map(item => [item.id, item])); const rows = page.lines.map(line => ({ line, cells: lineCells(line, itemById) })); const ruledRects = preprocessRuledRects(page); @@ -2669,7 +2718,7 @@ function segmentPageLines(page) { const ruled = ruledGridSegment(page); if (!ruled.segment) ruled.segment = segmentedRuleTable(page, rows); if (!ruled.segment) { - const text = segmentTextRows(page, rows, itemById, ruledClusters); + const text = segmentTextRows(page, rows, itemById, ruledClusters, allowHeaderlessTables); return { segments: text.segments, tableReason: ruled.tableReason ?? text.tableReason, @@ -2678,8 +2727,8 @@ function segmentPageLines(page) { } const remaining = rows.map((row, index) => ({ ...row, sourceIndex: index })) .filter(row => !ruled.segment.coveredLineIds.has(row.line.id)); - const before = segmentTextRows(page, remaining.filter(row => row.sourceIndex < ruled.segment.insertionIndex), itemById, ruledClusters); - const after = segmentTextRows(page, remaining.filter(row => row.sourceIndex >= ruled.segment.insertionIndex), itemById, ruledClusters); + const before = segmentTextRows(page, remaining.filter(row => row.sourceIndex < ruled.segment.insertionIndex), itemById, ruledClusters, allowHeaderlessTables); + const after = segmentTextRows(page, remaining.filter(row => row.sourceIndex >= ruled.segment.insertionIndex), itemById, ruledClusters, allowHeaderlessTables); return { segments: [...before.segments, ruled.segment, ...after.segments], tableReason: before.tableReason ?? after.tableReason, @@ -2862,6 +2911,18 @@ function renderTable(grid) { return lines; } +function renderHeaderlessTable(grid) { + const escape = value => sanitizeUnsafeText(String(value)).replace(/&/gu, "&") + .replace(//gu, ">") + .replace(/"/gu, """).replace(/'/gu, "'") + .replace(/https?:/giu, match => `${match.slice(0, -1)}:`) + .replace(/\b(?:[\p{L}\p{N}-]+\.)+[\p{L}]{2,}\b/giu, domain => domain.replace(".", ".")) + .replace(/@(?=[\p{L}\p{N}.-]+.[\p{L}]{2,})/giu, "@"); + return ["", ...grid.map(row => ( + `${row.map(value => ``).join("")}` + )), "
${escape(value)}
"]; +} + /** * Project a verifier-accepted rectangular table through the same GFM escaping * used by deterministic conversion. GFM has no span syntax, so structured @@ -3078,7 +3139,12 @@ function pageGaps(page, analysis, linkState, { add("TABLE_TOPOLOGY_UNKNOWN", "Table-like content was detected but its column topology could not be reconstructed, so it remains reading-order text."); } if (analysis?.tableReason === "header") { - add("TABLE_TOPOLOGY_UNKNOWN", "A column grid was detected but no source evidence distinguishes a header row, and a Markdown table would impose one, so it remains reading-order text."); + add("TABLE_TOPOLOGY_UNKNOWN", analysis.segments.some(segment => segment.headerless) + ? "A complete source-backed rectangle grid is emitted as header-neutral HTML data cells. Header roles are not established; other unsupported regions remain reading-order text. Empty slots describe retained text coverage, not proof of blank data values." + : "A column grid was detected but no source evidence distinguishes a header row, and a Markdown table would impose one, so it remains reading-order text."); + } + if (analysis?.tableReason !== "header" && analysis?.segments.some(segment => segment.headerless)) { + add("TABLE_TOPOLOGY_UNKNOWN", "A complete source-backed rectangle grid is emitted as header-neutral HTML data cells. Header roles and blank data values are not established."); } if (analysis?.tableReason === "ruling_unsupported") { add("TABLE_RULING_UNSUPPORTED", "Ruled rectangle evidence described a grid-shaped region, but its table topology could not be reconstructed, so it remains reading-order text."); @@ -3208,13 +3274,14 @@ function renderPage(page, { pageBoundaryBefore = false, pageBoundaryAfter = false, collectTableProposals = false, + allowHeaderlessTables = false, pageFurniture = [], } = {}) { const furnitureLineIds = new Set(pageFurniture.map(entry => entry.lineId)); const pageItems = paintedItems(page); const headings = headingLevels(page); const headingContinuations = headingContinuationIds(page, headings); - const analysis = segmentPageLines(page); + const analysis = segmentPageLines(page, allowHeaderlessTables); const linkState = analyzePageLinks(page, analysis, headings, furnitureLineIds); const unsafePage = analysis.tableReason !== null || linkState.unavailable @@ -3227,7 +3294,7 @@ function renderPage(page, { }); const records = analysis.segments.flatMap(segment => { if (segment.kind === "table") { - return renderTable(segment.grid).map(text => ({ + return (segment.headerless ? renderHeaderlessTable(segment.grid) : renderTable(segment.grid)).map(text => ({ text, sourceText: text, normalizable: false, @@ -3331,7 +3398,7 @@ function renderPage(page, { return rendered; } -function renderDocumentMarkdown(renderedPages, includePageBoundaries, gaps) { +function renderDocumentMarkdown(renderedPages, includePageBoundaries, gaps, limitations = LIMITATIONS) { const sections = renderedPages.map(({ page, markdown }) => ( includePageBoundaries ? `\n\n${markdown}` : markdown )); @@ -3347,7 +3414,7 @@ function renderDocumentMarkdown(renderedPages, includePageBoundaries, gaps) { parts.push([ "## Conversion limitations", "", - ...LIMITATIONS.map(limitation => `- ${escapePlainMarkdown(limitation)}`), + ...limitations.map(limitation => `- ${escapePlainMarkdown(limitation)}`), ].join("\n")); return `${parts.filter(Boolean).join("\n\n")}\n`; } @@ -3437,7 +3504,10 @@ function validateNormalizations(normalizations) { * source evidence. Callers must source-validate the layout IR before rendering. */ export function validateMarkdownConversionSemantics(result, { layout = null } = {}) { - assertion(sameJson(result.renderer, RENDERER), "renderer identity mismatch"); + const allowHeaderlessTables = result.options?.allow_headerless_tables === true; + assertion(!Object.hasOwn(result.options ?? {}, "allow_headerless_tables") || allowHeaderlessTables, + "allow_headerless_tables must be absent or true"); + assertion(sameJson(result.renderer, allowHeaderlessTables ? HEADERLESS_RENDERER : RENDERER), "renderer identity mismatch"); assertion(result.options?.include_page_boundaries === true || result.options?.include_page_boundaries === false, "include_page_boundaries must be boolean"); assertion(result.options?.compact === true || result.options?.compact === false, @@ -3476,7 +3546,8 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = const flattenedGaps = result.pages.flatMap(page => page.gaps); assertion(sameJson(result.gaps, flattenedGaps), "document gaps are not the exact flattened page gaps"); assertion(result.conversion_status === documentStatus(result.pages), "document conversion status mismatch"); - assertion(sameJson(result.limitations, LIMITATIONS), "renderer limitations mismatch"); + const limitations = allowHeaderlessTables ? headerlessLimitations() : LIMITATIONS; + assertion(sameJson(result.limitations, limitations), "renderer limitations mismatch"); if (layout !== null) { validatePdfLayoutSemantics(layout, { enforceOutputBudget: false }); @@ -3490,6 +3561,7 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = for (let index = 0; index < layout.pages.length; index += 1) { const expected = renderPage(layout.pages[index], { compact: result.options.compact, + allowHeaderlessTables, pageBoundaryBefore: index > 0, pageBoundaryAfter: true, pageFurniture: furniturePlans.get(layout.pages[index].page), @@ -3505,6 +3577,7 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = assertion(sameJson(result.normalizations, aggregateNormalizations( layout.pages.map((page, index) => renderPage(page, { compact: result.options.compact, + allowHeaderlessTables, pageBoundaryBefore: index > 0, pageBoundaryAfter: true, pageFurniture: furniturePlans.get(page.page), @@ -3513,12 +3586,14 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = const expectedMarkdown = renderDocumentMarkdown( layout.pages.map((page, index) => renderPage(page, { compact: result.options.compact, + allowHeaderlessTables, pageBoundaryBefore: index > 0, pageBoundaryAfter: true, pageFurniture: furniturePlans.get(page.page), })), result.options.include_page_boundaries, flattenedGaps, + limitations, ); assertion(result.markdown === expectedMarkdown, "markdown does not match the bound layout IR"); } @@ -3780,6 +3855,7 @@ export function renderPdfLayoutToMarkdown(layout, { compact = false, removePageFurniture = true, emitTableProposals = false, + allowHeaderlessTables = false, } = {}) { validatePdfLayoutSemantics(layout, { enforceOutputBudget: false }); validateSupportedLayoutIdentity(layout); @@ -3795,6 +3871,9 @@ export function renderPdfLayoutToMarkdown(layout, { if (typeof emitTableProposals !== "boolean") { throw new TypeError("emitTableProposals must be a boolean."); } + if (typeof allowHeaderlessTables !== "boolean") { + throw new TypeError("allowHeaderlessTables must be a boolean."); + } if (!Number.isSafeInteger(maxMarkdownBytes) || maxMarkdownBytes < 1 || maxMarkdownBytes > MAX_MARKDOWN_BYTES_LIMIT) { @@ -3810,10 +3889,12 @@ export function renderPdfLayoutToMarkdown(layout, { // number from a single-page selection. pageBoundaryAfter: true, collectTableProposals: emitTableProposals, + allowHeaderlessTables, pageFurniture: furniturePlans.get(page.page), })); const gaps = renderedPages.flatMap(page => page.gaps); - const markdown = renderDocumentMarkdown(renderedPages, includePageBoundaries, gaps); + const limitations = allowHeaderlessTables ? headerlessLimitations() : LIMITATIONS; + const markdown = renderDocumentMarkdown(renderedPages, includePageBoundaries, gaps, limitations); const markdownBytes = utf8Bytes(markdown); if (markdownBytes > maxMarkdownBytes) { // Deliberately throws rather than emitting a gap: truncating would cut a @@ -3826,7 +3907,7 @@ export function renderPdfLayoutToMarkdown(layout, { } const result = { - renderer: { ...RENDERER }, + renderer: { ...(allowHeaderlessTables ? HEADERLESS_RENDERER : RENDERER) }, conversion_status: documentStatus(renderedPages), markdown, markdown_sha256: sha256(markdown), @@ -3835,6 +3916,7 @@ export function renderPdfLayoutToMarkdown(layout, { include_page_boundaries: includePageBoundaries, compact, remove_page_furniture: removePageFurniture, + ...(allowHeaderlessTables ? { allow_headerless_tables: true } : {}), }, limits: { max_markdown_bytes: maxMarkdownBytes }, pages: renderedPages.map(({ @@ -3844,7 +3926,7 @@ export function renderPdfLayoutToMarkdown(layout, { ...page }) => page), gaps, - limitations: [...LIMITATIONS], + limitations: [...limitations], normalizations: aggregateNormalizations(renderedPages), provenance: provenanceFromLayout(layout), }; diff --git a/pdf-toolkit-mcp-share/server/output-schemas.js b/pdf-toolkit-mcp-share/server/output-schemas.js index c400d6e..57b04ff 100644 --- a/pdf-toolkit-mcp-share/server/output-schemas.js +++ b/pdf-toolkit-mcp-share/server/output-schemas.js @@ -1503,7 +1503,7 @@ export const TOOL_SUCCESS_OUTPUT_SCHEMAS = Object.freeze({ { renderer: object({ name: { const: "pdf-tools.layout-markdown-renderer" }, - version: { const: "1.21.0" }, + version: enumString(["1.21.0", "1.22.0"]), }), conversion_status: enumString(["complete", "partial", "failed"]), markdown: string, @@ -1513,7 +1513,8 @@ export const TOOL_SUCCESS_OUTPUT_SCHEMAS = Object.freeze({ include_page_boundaries: boolean, compact: boolean, remove_page_furniture: boolean, - }), + allow_headerless_tables: { const: true }, + }, ["include_page_boundaries", "compact", "remove_page_furniture"]), limits: object({ max_markdown_bytes: { type: "integer", minimum: 1, maximum: 200000 } }), pages: arrayOf(markdownPage), pages_needing_vision: arrayOf(visionRoutingPage), diff --git a/scripts/macos-claude-installed-shannon.mjs b/scripts/macos-claude-installed-shannon.mjs index 14386d4..c996e93 100755 --- a/scripts/macos-claude-installed-shannon.mjs +++ b/scripts/macos-claude-installed-shannon.mjs @@ -23,7 +23,7 @@ if (!process.argv[2] || !process.argv[3]) { // const EXPECTED_SOURCE_SHA256 = "6e4e3411984f3edf99dbfe8b941cb5e8a321379ff0cae6ae5c1f592ad8882ca8"; const EXPECTED_MARKDOWN_SHA256 = "b9bd7ca7db26fe807f294565c39de8d4fe9337908afa74cd7e2896a800c92991"; -const EXPECTED_TOOL_CONTRACT_SHA256 = "639189a0677085df0ef8d8ee92d23c5a5c1c3121a779e926ebe36f266c3de84c"; +const EXPECTED_TOOL_CONTRACT_SHA256 = "225dccb33c6168690d56ab0e40ef3fb364f10d4f0af8fdfa4577938f3566a235"; const EXPECTED_PAGE_COUNT = 55; const EXPECTED_GAP_COUNT = 26; const EXPECTED_REPLACEMENT_CHARACTER_COUNT = 6; diff --git a/scripts/macos-claude-installed-smoke.mjs b/scripts/macos-claude-installed-smoke.mjs index ffa174b..662acaa 100755 --- a/scripts/macos-claude-installed-smoke.mjs +++ b/scripts/macos-claude-installed-smoke.mjs @@ -33,7 +33,7 @@ const textFixture = path.join(fixtureDirectory, "synthetic-text-two-page.pdf"); const rasterFixture = path.join(fixtureDirectory, "synthetic-raster-only.pdf"); const mutationDirectory = path.join(fixtureDirectory, "mutation-output"); const toolNames = []; -const EXPECTED_TOOL_CONTRACT_SHA256 = "639189a0677085df0ef8d8ee92d23c5a5c1c3121a779e926ebe36f266c3de84c"; +const EXPECTED_TOOL_CONTRACT_SHA256 = "225dccb33c6168690d56ab0e40ef3fb364f10d4f0af8fdfa4577938f3566a235"; const ACCESSIBILITY_CONCLUSION_KEYS = Object.freeze([ "certification", "document_accessibility", diff --git a/server/index.js b/server/index.js index a8d891d..a26eb4b 100755 --- a/server/index.js +++ b/server/index.js @@ -3772,7 +3772,7 @@ async function listTools(request) { }, { name: "convert_pdf_to_markdown", - description: "Convert a bounded page range of a local PDF into deterministic Markdown from the source-validated PDF Tools Extraction IR. Headings and lists require geometry or literal marker evidence. A table requires a complete recurring text grid, a clean ruled-rectangle grid, or a complete closed painted grid, with every item in one cell and real header evidence. A link requires a source-validated external http or https annotation target covering one contiguous text run. Unsupported tables and link targets, including merged cells, internal destinations, actions, and other schemes, stay escaped text with typed gaps. Compact extreme-margin page numbers and provenance furniture, plus same-band text repeated across selected pages, are removed with typed counts by default; detected headings and all source-evidenced table-region lines are preserved, and removal can be disabled. No OCR or external model. Optionally enable compact mode for counted dot-leader and spaced-hyphen normalizations. Optionally saves transactional UTF-8 Markdown. Use absolute or ~/ local paths, not Claude container paths (/mnt/...).", + description: "Convert a bounded page range of a local PDF into deterministic Markdown from the source-validated PDF Tools Extraction IR. Headings and lists require geometry or literal marker evidence. By default, a table requires a complete recurring text grid, a clean ruled-rectangle grid, or a complete closed painted grid, with every item in one cell and real header evidence. A link requires a source-validated external http or https annotation target covering one contiguous text run. Unsupported tables and link targets, including merged cells, internal destinations, actions, and other schemes, stay escaped text with typed gaps. Compact extreme-margin page numbers and provenance furniture, plus same-band text repeated across selected pages, are removed with typed counts by default; detected headings and all source-evidenced table-region lines are preserved, and removal can be disabled. No OCR or external model. Optionally enable compact mode for counted dot-leader and spaced-hyphen normalizations. Optionally saves transactional UTF-8 Markdown. Use absolute or ~/ local paths, not Claude container paths (/mnt/...).", inputSchema: { type: "object", additionalProperties: false, @@ -3788,6 +3788,7 @@ async function listTools(request) { compact: { type: "boolean", description: "Opt into counted dot-leader, isolated page-number, and Unicode-letter spaced-hyphen normalizations. Default: false." }, remove_page_furniture: { type: "boolean", description: "Remove only source-evidenced extreme-margin page furniture and report every removed line by typed gap and counts. Set false to preserve all source lines. Default: true." }, emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, + allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.22.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." }, output_path: { type: "string", description: "Optional absolute .md path, or ~/ path. The file is written only after complete bytes are staged and verified." }, overwrite: { type: "boolean", description: "Replace an existing output_path only when its exact expected_output_identity is also supplied. Default: false." }, expected_output_identity: EXPECTED_MARKDOWN_OUTPUT_IDENTITY_INPUT_SCHEMA @@ -5867,6 +5868,7 @@ async function handleToolCall(request) { "compact", "remove_page_furniture", "emit_table_proposals", + "allow_headerless_tables", "output_path", "overwrite", "expected_output_identity", @@ -5884,6 +5886,7 @@ async function handleToolCall(request) { true, ); const emitTableProposals = optionalBooleanArgument(markdownArgs.emit_table_proposals, "emit_table_proposals", false); + const allowHeaderlessTables = optionalBooleanArgument(markdownArgs.allow_headerless_tables, "allow_headerless_tables", false); const overwrite = optionalBooleanArgument(markdownArgs.overwrite, "overwrite", false); const expectedOutputIdentity = normalizeExpectedOutputIdentity( markdownArgs.expected_output_identity, @@ -5944,6 +5947,7 @@ async function handleToolCall(request) { compact, removePageFurniture, emitTableProposals, + allowHeaderlessTables, }); const pagesNeedingVision = deriveMarkdownVisionRouting(layout); diff --git a/server/markdown-conversion.js b/server/markdown-conversion.js index d659bb8..bf6c9ee 100644 --- a/server/markdown-conversion.js +++ b/server/markdown-conversion.js @@ -8,6 +8,7 @@ const RENDERER = Object.freeze({ name: "pdf-tools.layout-markdown-renderer", version: "1.21.0", }); +const HEADERLESS_RENDERER = Object.freeze({ ...RENDERER, version: "1.22.0" }); const SUPPORTED_LAYOUT_IR_VERSION = "1.6.0"; // Bounded geometric table inference. A run of adjacent lines is treated as a @@ -97,6 +98,14 @@ const LIMITATIONS = Object.freeze([ "Unsafe control characters and malformed UTF-16 surrogates are replaced with the Unicode replacement character and reported as conversion gaps.", ]); +function headerlessLimitations() { + return [ + ...LIMITATIONS.map(value => value.startsWith("Tables are reconstructed") + ? `Default GFM table rules: ${value}` : value), + "The explicit header-neutral profile may instead emit a complete, fully source-covered ruled-rectangle cell grid as escaped HTML table data cells when header evidence is absent. It assigns no header roles, spans, inherited values or semantic blank values. A typed header-role gap remains, and partial, ambiguous or cross-cell evidence stays reading-order text. This does not perform OCR or interpret cell artwork.", + ]; +} + function assertion(condition, message) { if (!condition) throw new Error(`Invalid Markdown conversion semantics: ${message}`); } @@ -2009,7 +2018,7 @@ function hasFirstRowBandEvidence(bands) { && Math.abs(rectBottom(band.rect) - rectBottom(firstRow.rect)) <= RECT_SNAP_TOLERANCE); } -function tryBuildRectGrid(page, run, clusters, itemById) { +function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables = false) { const runLineIds = new Set(run.map(row => row.line.id)); const candidates = []; for (const cluster of clusters) { @@ -2178,10 +2187,46 @@ function tryBuildRectGrid(page, run, clusters, itemById) { const candidateRows = lineRecords .sort((left, right) => left.line.y - right.line.y) .map(record => ({ line: record.line, cells: record.structuralInsideNonWhitespace })); - if (!hasHeaderEvidence(candidateRows) && !hasFirstRowBandEvidence(bands)) { + const headerless = !hasHeaderEvidence(candidateRows) && !hasFirstRowBandEvidence(bands); + if (headerless && !allowHeaderlessTables) { return { reason: "header" }; } if (candidateRows.some(row => !runLineIds.has(row.line.id))) return { reason: "topology" }; + if (headerless) { + // Header-neutral emission is narrower than the established GFM path. + // A partial rectangle census or a center-only cell assignment cannot + // establish the complete grid this profile promises to preserve. + const candidateLineIds = new Set(candidateRows.map(row => row.line.id)); + if (candidateLineIds.size !== runLineIds.size + || [...runLineIds].some(id => !candidateLineIds.has(id))) { + return { reason: "topology" }; + } + // Vector paint makes an otherwise fully observed text page "partial" + // in the IR. Test evidence completeness, not that modality label. + if (page.truncation?.truncated || page.text_layer_status !== "present" + || page.image_detection_status === "failed" || page.errors.length !== 0 + || page.link_annotations.status !== "available" || page.link_annotations.truncated + || page.painted_rectangles.status !== "available" || page.painted_rectangles.truncated + || page.ruling_segments.status !== "available" || page.ruling_segments.truncated + || page.text_integrity?.status === "suspect" || bands.length !== 0 + || rowCount * columnCount > RULED_TABLE_MAX_CELLS + || filledCells !== totalCells || occupiedCells.size !== totalCells + || page.raw_items.some(item => !item.geometry_valid || (item.text?.trim() && !itemBox(item)))) { + return { reason: "topology" }; + } + for (let row = 0; row < rowCount; row += 1) { + for (let column = 0; column < columnCount; column += 1) { + if (cells[row][column].some(({ item }) => ( + item.bbox.x < xEdges[column] - RECT_CELL_ASSIGNMENT_SLACK + || item.bbox.x + item.bbox.width > xEdges[column + 1] + RECT_CELL_ASSIGNMENT_SLACK + || item.bbox.y < yEdges[row] - RECT_CELL_ASSIGNMENT_SLACK + || item.bbox.y + item.bbox.height > yEdges[row + 1] + RECT_CELL_ASSIGNMENT_SLACK + ))) return { reason: "topology" }; + } + } + // Keep empty outer slots rather than trimming source grid columns. + return { kind: "table", grid, rows: candidateRows, headerless: true }; + } return { kind: "table", grid: trimmedGrid, rows: candidateRows }; } return null; @@ -2598,7 +2643,7 @@ function segmentedRuleTable(page, rows) { coveredLineIds: selected, insertionIndex: indices[0] }; } -function segmentTextRows(page, rows, itemById, ruledClusters) { +function segmentTextRows(page, rows, itemById, ruledClusters, allowHeaderlessTables = false) { const pageItems = paintedItems(page); const segments = []; const regions = []; @@ -2622,10 +2667,14 @@ function segmentTextRows(page, rows, itemById, ruledClusters) { // Ordinary prose that happens to share a left margin must not be // reported as an unreconstructed table. const ruling = ruledClusters.length > 0 - ? tryBuildRectGrid(page, run, ruledClusters, itemById) + ? tryBuildRectGrid(page, run, ruledClusters, itemById, allowHeaderlessTables) : null; if (ruling?.kind === "table") { - segments.push({ kind: "table", rows: run, grid: ruling.grid }); + segments.push({ kind: "table", rows: run, grid: ruling.grid, ...(ruling.headerless ? { headerless: true } : {}) }); + if (ruling.headerless) { + tableReason = "header"; + regions.push({ run, reason: "header" }); + } } else { // Preserve the reason from the rect-evidence attempt. The text path // can independently see a header/topology failure, but it must not @@ -2661,7 +2710,7 @@ function segmentTextRows(page, rows, itemById, ruledClusters) { * that cannot be reconstructed are returned as text with tableReason set, so * the caller can report typed partial coverage instead of silently flattening. */ -function segmentPageLines(page) { +function segmentPageLines(page, allowHeaderlessTables = false) { const itemById = new Map(page.raw_items.map(item => [item.id, item])); const rows = page.lines.map(line => ({ line, cells: lineCells(line, itemById) })); const ruledRects = preprocessRuledRects(page); @@ -2669,7 +2718,7 @@ function segmentPageLines(page) { const ruled = ruledGridSegment(page); if (!ruled.segment) ruled.segment = segmentedRuleTable(page, rows); if (!ruled.segment) { - const text = segmentTextRows(page, rows, itemById, ruledClusters); + const text = segmentTextRows(page, rows, itemById, ruledClusters, allowHeaderlessTables); return { segments: text.segments, tableReason: ruled.tableReason ?? text.tableReason, @@ -2678,8 +2727,8 @@ function segmentPageLines(page) { } const remaining = rows.map((row, index) => ({ ...row, sourceIndex: index })) .filter(row => !ruled.segment.coveredLineIds.has(row.line.id)); - const before = segmentTextRows(page, remaining.filter(row => row.sourceIndex < ruled.segment.insertionIndex), itemById, ruledClusters); - const after = segmentTextRows(page, remaining.filter(row => row.sourceIndex >= ruled.segment.insertionIndex), itemById, ruledClusters); + const before = segmentTextRows(page, remaining.filter(row => row.sourceIndex < ruled.segment.insertionIndex), itemById, ruledClusters, allowHeaderlessTables); + const after = segmentTextRows(page, remaining.filter(row => row.sourceIndex >= ruled.segment.insertionIndex), itemById, ruledClusters, allowHeaderlessTables); return { segments: [...before.segments, ruled.segment, ...after.segments], tableReason: before.tableReason ?? after.tableReason, @@ -2862,6 +2911,18 @@ function renderTable(grid) { return lines; } +function renderHeaderlessTable(grid) { + const escape = value => sanitizeUnsafeText(String(value)).replace(/&/gu, "&") + .replace(//gu, ">") + .replace(/"/gu, """).replace(/'/gu, "'") + .replace(/https?:/giu, match => `${match.slice(0, -1)}:`) + .replace(/\b(?:[\p{L}\p{N}-]+\.)+[\p{L}]{2,}\b/giu, domain => domain.replace(".", ".")) + .replace(/@(?=[\p{L}\p{N}.-]+.[\p{L}]{2,})/giu, "@"); + return ["", ...grid.map(row => ( + `${row.map(value => ``).join("")}` + )), "
${escape(value)}
"]; +} + /** * Project a verifier-accepted rectangular table through the same GFM escaping * used by deterministic conversion. GFM has no span syntax, so structured @@ -3078,7 +3139,12 @@ function pageGaps(page, analysis, linkState, { add("TABLE_TOPOLOGY_UNKNOWN", "Table-like content was detected but its column topology could not be reconstructed, so it remains reading-order text."); } if (analysis?.tableReason === "header") { - add("TABLE_TOPOLOGY_UNKNOWN", "A column grid was detected but no source evidence distinguishes a header row, and a Markdown table would impose one, so it remains reading-order text."); + add("TABLE_TOPOLOGY_UNKNOWN", analysis.segments.some(segment => segment.headerless) + ? "A complete source-backed rectangle grid is emitted as header-neutral HTML data cells. Header roles are not established; other unsupported regions remain reading-order text. Empty slots describe retained text coverage, not proof of blank data values." + : "A column grid was detected but no source evidence distinguishes a header row, and a Markdown table would impose one, so it remains reading-order text."); + } + if (analysis?.tableReason !== "header" && analysis?.segments.some(segment => segment.headerless)) { + add("TABLE_TOPOLOGY_UNKNOWN", "A complete source-backed rectangle grid is emitted as header-neutral HTML data cells. Header roles and blank data values are not established."); } if (analysis?.tableReason === "ruling_unsupported") { add("TABLE_RULING_UNSUPPORTED", "Ruled rectangle evidence described a grid-shaped region, but its table topology could not be reconstructed, so it remains reading-order text."); @@ -3208,13 +3274,14 @@ function renderPage(page, { pageBoundaryBefore = false, pageBoundaryAfter = false, collectTableProposals = false, + allowHeaderlessTables = false, pageFurniture = [], } = {}) { const furnitureLineIds = new Set(pageFurniture.map(entry => entry.lineId)); const pageItems = paintedItems(page); const headings = headingLevels(page); const headingContinuations = headingContinuationIds(page, headings); - const analysis = segmentPageLines(page); + const analysis = segmentPageLines(page, allowHeaderlessTables); const linkState = analyzePageLinks(page, analysis, headings, furnitureLineIds); const unsafePage = analysis.tableReason !== null || linkState.unavailable @@ -3227,7 +3294,7 @@ function renderPage(page, { }); const records = analysis.segments.flatMap(segment => { if (segment.kind === "table") { - return renderTable(segment.grid).map(text => ({ + return (segment.headerless ? renderHeaderlessTable(segment.grid) : renderTable(segment.grid)).map(text => ({ text, sourceText: text, normalizable: false, @@ -3331,7 +3398,7 @@ function renderPage(page, { return rendered; } -function renderDocumentMarkdown(renderedPages, includePageBoundaries, gaps) { +function renderDocumentMarkdown(renderedPages, includePageBoundaries, gaps, limitations = LIMITATIONS) { const sections = renderedPages.map(({ page, markdown }) => ( includePageBoundaries ? `\n\n${markdown}` : markdown )); @@ -3347,7 +3414,7 @@ function renderDocumentMarkdown(renderedPages, includePageBoundaries, gaps) { parts.push([ "## Conversion limitations", "", - ...LIMITATIONS.map(limitation => `- ${escapePlainMarkdown(limitation)}`), + ...limitations.map(limitation => `- ${escapePlainMarkdown(limitation)}`), ].join("\n")); return `${parts.filter(Boolean).join("\n\n")}\n`; } @@ -3437,7 +3504,10 @@ function validateNormalizations(normalizations) { * source evidence. Callers must source-validate the layout IR before rendering. */ export function validateMarkdownConversionSemantics(result, { layout = null } = {}) { - assertion(sameJson(result.renderer, RENDERER), "renderer identity mismatch"); + const allowHeaderlessTables = result.options?.allow_headerless_tables === true; + assertion(!Object.hasOwn(result.options ?? {}, "allow_headerless_tables") || allowHeaderlessTables, + "allow_headerless_tables must be absent or true"); + assertion(sameJson(result.renderer, allowHeaderlessTables ? HEADERLESS_RENDERER : RENDERER), "renderer identity mismatch"); assertion(result.options?.include_page_boundaries === true || result.options?.include_page_boundaries === false, "include_page_boundaries must be boolean"); assertion(result.options?.compact === true || result.options?.compact === false, @@ -3476,7 +3546,8 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = const flattenedGaps = result.pages.flatMap(page => page.gaps); assertion(sameJson(result.gaps, flattenedGaps), "document gaps are not the exact flattened page gaps"); assertion(result.conversion_status === documentStatus(result.pages), "document conversion status mismatch"); - assertion(sameJson(result.limitations, LIMITATIONS), "renderer limitations mismatch"); + const limitations = allowHeaderlessTables ? headerlessLimitations() : LIMITATIONS; + assertion(sameJson(result.limitations, limitations), "renderer limitations mismatch"); if (layout !== null) { validatePdfLayoutSemantics(layout, { enforceOutputBudget: false }); @@ -3490,6 +3561,7 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = for (let index = 0; index < layout.pages.length; index += 1) { const expected = renderPage(layout.pages[index], { compact: result.options.compact, + allowHeaderlessTables, pageBoundaryBefore: index > 0, pageBoundaryAfter: true, pageFurniture: furniturePlans.get(layout.pages[index].page), @@ -3505,6 +3577,7 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = assertion(sameJson(result.normalizations, aggregateNormalizations( layout.pages.map((page, index) => renderPage(page, { compact: result.options.compact, + allowHeaderlessTables, pageBoundaryBefore: index > 0, pageBoundaryAfter: true, pageFurniture: furniturePlans.get(page.page), @@ -3513,12 +3586,14 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = const expectedMarkdown = renderDocumentMarkdown( layout.pages.map((page, index) => renderPage(page, { compact: result.options.compact, + allowHeaderlessTables, pageBoundaryBefore: index > 0, pageBoundaryAfter: true, pageFurniture: furniturePlans.get(page.page), })), result.options.include_page_boundaries, flattenedGaps, + limitations, ); assertion(result.markdown === expectedMarkdown, "markdown does not match the bound layout IR"); } @@ -3780,6 +3855,7 @@ export function renderPdfLayoutToMarkdown(layout, { compact = false, removePageFurniture = true, emitTableProposals = false, + allowHeaderlessTables = false, } = {}) { validatePdfLayoutSemantics(layout, { enforceOutputBudget: false }); validateSupportedLayoutIdentity(layout); @@ -3795,6 +3871,9 @@ export function renderPdfLayoutToMarkdown(layout, { if (typeof emitTableProposals !== "boolean") { throw new TypeError("emitTableProposals must be a boolean."); } + if (typeof allowHeaderlessTables !== "boolean") { + throw new TypeError("allowHeaderlessTables must be a boolean."); + } if (!Number.isSafeInteger(maxMarkdownBytes) || maxMarkdownBytes < 1 || maxMarkdownBytes > MAX_MARKDOWN_BYTES_LIMIT) { @@ -3810,10 +3889,12 @@ export function renderPdfLayoutToMarkdown(layout, { // number from a single-page selection. pageBoundaryAfter: true, collectTableProposals: emitTableProposals, + allowHeaderlessTables, pageFurniture: furniturePlans.get(page.page), })); const gaps = renderedPages.flatMap(page => page.gaps); - const markdown = renderDocumentMarkdown(renderedPages, includePageBoundaries, gaps); + const limitations = allowHeaderlessTables ? headerlessLimitations() : LIMITATIONS; + const markdown = renderDocumentMarkdown(renderedPages, includePageBoundaries, gaps, limitations); const markdownBytes = utf8Bytes(markdown); if (markdownBytes > maxMarkdownBytes) { // Deliberately throws rather than emitting a gap: truncating would cut a @@ -3826,7 +3907,7 @@ export function renderPdfLayoutToMarkdown(layout, { } const result = { - renderer: { ...RENDERER }, + renderer: { ...(allowHeaderlessTables ? HEADERLESS_RENDERER : RENDERER) }, conversion_status: documentStatus(renderedPages), markdown, markdown_sha256: sha256(markdown), @@ -3835,6 +3916,7 @@ export function renderPdfLayoutToMarkdown(layout, { include_page_boundaries: includePageBoundaries, compact, remove_page_furniture: removePageFurniture, + ...(allowHeaderlessTables ? { allow_headerless_tables: true } : {}), }, limits: { max_markdown_bytes: maxMarkdownBytes }, pages: renderedPages.map(({ @@ -3844,7 +3926,7 @@ export function renderPdfLayoutToMarkdown(layout, { ...page }) => page), gaps, - limitations: [...LIMITATIONS], + limitations: [...limitations], normalizations: aggregateNormalizations(renderedPages), provenance: provenanceFromLayout(layout), }; diff --git a/server/output-schemas.js b/server/output-schemas.js index c400d6e..57b04ff 100644 --- a/server/output-schemas.js +++ b/server/output-schemas.js @@ -1503,7 +1503,7 @@ export const TOOL_SUCCESS_OUTPUT_SCHEMAS = Object.freeze({ { renderer: object({ name: { const: "pdf-tools.layout-markdown-renderer" }, - version: { const: "1.21.0" }, + version: enumString(["1.21.0", "1.22.0"]), }), conversion_status: enumString(["complete", "partial", "failed"]), markdown: string, @@ -1513,7 +1513,8 @@ export const TOOL_SUCCESS_OUTPUT_SCHEMAS = Object.freeze({ include_page_boundaries: boolean, compact: boolean, remove_page_furniture: boolean, - }), + allow_headerless_tables: { const: true }, + }, ["include_page_boundaries", "compact", "remove_page_furniture"]), limits: object({ max_markdown_bytes: { type: "integer", minimum: 1, maximum: 200000 } }), pages: arrayOf(markdownPage), pages_needing_vision: arrayOf(visionRoutingPage), diff --git a/test/convert-pdf-to-markdown.test.js b/test/convert-pdf-to-markdown.test.js index 078f13f..5c1b800 100644 --- a/test/convert-pdf-to-markdown.test.js +++ b/test/convert-pdf-to-markdown.test.js @@ -5,7 +5,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; -import { PDFDocument, PDFName, PDFNumber, PDFString, StandardFonts, degrees } from "pdf-lib"; +import { PDFDocument, PDFName, PDFNumber, PDFString, StandardFonts, degrees, rgb } from "pdf-lib"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { TOOL_OUTPUT_SCHEMAS } from "../server/output-schemas.js"; import { projectMarkdownTable, scoreTable } from "./eval/extraction-bakeoff-scorer.js"; @@ -67,6 +67,20 @@ const makeUniformGridFixture = target => drawGrid(target, { headerSize: 11, sepa // Identical size, but the header uses a second embed of the same visible font. const makeDoubleEmbedGridFixture = target => drawGrid(target, { headerSize: 11, separateHeaderResource: true }); +async function makeHeaderlessRectGridFixture(targetPath) { + const document = await PDFDocument.create(); + const page = document.addPage([612, 792]); + const font = await document.embedFont(StandardFonts.Helvetica); + const rows = [["Region", "Q1", "Q2"], ["&|'", "1200", "1450"], + ["South", "980", "1020"], ["West", "1500", "1380"]]; + rows.forEach((row, r) => row.forEach((value, c) => { + page.drawRectangle({ x: 72 + c * 150, y: 680 - (r + 1) * 48, + width: 150, height: 48, borderWidth: 1, borderColor: rgb(0, 0, 0) }); + page.drawText(value, { x: 82 + c * 150, y: 650 - r * 48, size: 12, font }); + })); + await fs.writeFile(targetPath, await document.save({ useObjectStreams: false })); +} + async function makeDelimiterTableFixture(targetPath) { const document = await PDFDocument.create(); const page = document.addPage([612, 792]); @@ -235,6 +249,7 @@ describe("convert_pdf_to_markdown MCP tool", () => { let linkFixture; let rotatedLinkFixture; let hostileLinkFixture; + let headerlessRectFixture; beforeAll(async () => { temporaryRoot = await fs.mkdtemp(path.join(os.tmpdir(), "pdf-tools-markdown-")); @@ -249,6 +264,7 @@ describe("convert_pdf_to_markdown MCP tool", () => { linkFixture = path.join(temporaryRoot, "link.pdf"); rotatedLinkFixture = path.join(temporaryRoot, "link-rotated.pdf"); hostileLinkFixture = path.join(temporaryRoot, "link-hostile.pdf"); + headerlessRectFixture = path.join(temporaryRoot, "headerless-rect.pdf"); await makeStructureFixture(structureFixture); await makeDenseFixture(denseFixture); await makeGeometryFixture(geometryFixture); @@ -260,6 +276,7 @@ describe("convert_pdf_to_markdown MCP tool", () => { await makeLinkFixture(linkFixture); await makeLinkFixture(rotatedLinkFixture, { rotation: 90, crop: true }); await makeLinkFixture(hostileLinkFixture, { url: "https://example.com/a(b)c" }); + await makeHeaderlessRectGridFixture(headerlessRectFixture); transport = new StdioClientTransport({ command: process.execPath, args: [path.join(REPO_ROOT, "server/index.js")], @@ -405,6 +422,30 @@ describe("convert_pdf_to_markdown MCP tool", () => { expect(result.structuredContent.conversion_status).toBe("partial"); }, 30_000); + it("round-trips a real headerless PDF grid through explicit HTML mode and both optional flags", async () => { + const before = await fs.readFile(headerlessRectFixture); + const args = { pdf_path: headerlessRectFixture, max_markdown_bytes: 100000 }; + const baseline = await client.callTool({ name: "convert_pdf_to_markdown", arguments: args }); + const disabled = await client.callTool({ name: "convert_pdf_to_markdown", + arguments: { ...args, allow_headerless_tables: false } }); + expect(disabled).toEqual(baseline); + expect(baseline.isError).not.toBe(true); + expect(baseline.structuredContent.markdown).not.toContain(""); + const opted = await client.callTool({ name: "convert_pdf_to_markdown", + arguments: { ...args, allow_headerless_tables: true, emit_table_proposals: true, compact: true } }); + expect(opted.isError).not.toBe(true); + expect(opted.structuredContent.renderer.version).toBe("1.22.0"); + expect(opted.structuredContent.options.allow_headerless_tables).toBe(true); + expect(opted.structuredContent.markdown).toContain(""); + expect(opted.structuredContent.markdown).not.toContain("
<img>&|'12001450
"); + expect(opted.structuredContent.conversion_status).toBe("partial"); + expect(opted.structuredContent.table_proposals).toHaveLength(1); + expect(await fs.readFile(headerlessRectFixture)).toEqual(before); + const rejected = await client.callTool({ name: "convert_pdf_to_markdown", + arguments: { ...args, allow_headerless_tables: "true" } }); + expect(rejected.isError).toBe(true); + }, 30_000); + it("escapes a table cell delimiter exactly once", async () => { const result = await client.callTool({ name: "convert_pdf_to_markdown", diff --git a/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json b/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json index 2b28923..01394fc 100644 --- a/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json +++ b/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json @@ -79,14 +79,14 @@ { "role": "markdown_conversion_module", "path": "server/markdown-conversion.js", - "bytes": 180929, - "sha256": "c4e321ac552d6d72540f7833c1b53a6f2247a7206ed4647899f714e827826593" + "bytes": 186425, + "sha256": "cd8da7d747101cf746a2ac72336627758cba4feaab47b7fc9672cf2631e31b9c" }, { "role": "output_schemas_module", "path": "server/output-schemas.js", - "bytes": 84250, - "sha256": "4373f1a18a55556097dafe3e19e168e28aa1ec8d7acbeddbaa7195edf3773b27" + "bytes": 84378, + "sha256": "4a522decf2d5338ca48f8383c188a36719c9ee88a702791c3f6c1c8f29844f77" }, { "role": "package_json", @@ -131,7 +131,7 @@ "sha256": "ef40501b2afbe4cd2adfa80480344783bbec1a00e8e9850086a5d044231d1f53" } ], - "validator_source_set_sha256": "838314e04ab44bc0704e95931d1cc9045a3aebe6f0291203205eff19f2940b5f", + "validator_source_set_sha256": "a67e5beb7c30a8b52c90fe7dc388e628b004b5833539cf1faa4aa8e7b450baad", "pdfjs_version": "5.4.624", "generation_contract": { "source_path": "source.pdf", diff --git a/test/fixtures/eval/trajectories/tool-contracts.v3.json b/test/fixtures/eval/trajectories/tool-contracts.v3.json index 0ed98c8..5366232 100644 --- a/test/fixtures/eval/trajectories/tool-contracts.v3.json +++ b/test/fixtures/eval/trajectories/tool-contracts.v3.json @@ -5,7 +5,7 @@ "name": "pdf-tools", "version": "0.14.0" }, - "tools_sha256": "d78d5d5621685adddd30f4db45bc4c0d49530acd75db9f138c897e2a6abb578c", + "tools_sha256": "09cb0675a8be40e01e9249a6a6f39a5e2fdf7503fab12722e9db564b96424db8", "tools": [ { "name": "add_signature_field", @@ -590,6 +590,10 @@ "type": "boolean", "description": "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, + "allow_headerless_tables": { + "type": "boolean", + "description": "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.22.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." + }, "output_path": { "type": "string", "description": "Optional absolute .md path, or ~/ path. The file is written only after complete bytes are staged and verified." diff --git a/test/markdown-conversion.test.js b/test/markdown-conversion.test.js index ab3c6e3..e1c3a53 100644 --- a/test/markdown-conversion.test.js +++ b/test/markdown-conversion.test.js @@ -62,6 +62,12 @@ function ruledGridItems({ headerSize = 13 } = {}) { }))); } +function headerlessGridRects() { + return Array.from({ length: 4 }, (_, row) => Array.from({ length: 3 }, (_, column) => ({ + x: 72 + column * 150, y: 112 + row * 48, width: 150, height: 48, verb: "stroke", + }))).flat(); +} + function attachRuledRects(layout, items, status = "available") { const page = layout.pages[0]; page.ruled_rects = { @@ -422,6 +428,147 @@ describe("layout Markdown renderer", () => { expect(first.conversion_status).toBe("complete"); }); + it("keeps the complete default envelope unchanged when header-neutral mode is off", async () => { + const layout = attachRuledRects(await validatedSyntheticLayout([{ + items: ruledGridItems({ headerSize: 12 }), + }]), headerlessGridRects()); + const baseline = renderPdfLayoutToMarkdown(layout); + expect(renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: false })).toEqual(baseline); + expect(baseline.options).not.toHaveProperty("allow_headerless_tables"); + expect(baseline.renderer.version).toBe("1.21.0"); + expect(baseline.markdown).not.toContain(""); + expect(baseline.gaps.some(gap => gap.message.includes("no source evidence distinguishes a header"))).toBe(true); + }); + + it("emits only inert data cells for an opted-in complete source-covered headerless grid", async () => { + const layout = attachRuledRects(await validatedSyntheticLayout([{ + items: ruledGridItems({ headerSize: 12 }), + }]), headerlessGridRects()); + const result = renderPdfLayoutToMarkdown(layout, { + allowHeaderlessTables: true, emitTableProposals: true, compact: true, + }); + expect(result.renderer.version).toBe("1.22.0"); + expect(result.options.allow_headerless_tables).toBe(true); + expect(result.markdown).toContain(""); + expect(result.markdown).not.toContain(""); + expect(result.markdown).not.toContain(""); + }); + + it.each(["\u0000", "\u0001", "\u001b", "\ud800"])( + "sanitizes an unsafe code unit in an otherwise admissible header-neutral cell: %j", async unsafe => { + const items = ruledGridItems({ headerSize: 12 }).map(item => item.str === "North" + ? { ...item, str: `No${unsafe}rth` } : item); + const layout = attachRuledRects(await validatedSyntheticLayout([{ items }]), headerlessGridRects()); + const result = renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: true }); + expect(result.markdown).toContain(""); + expect(result.markdown).not.toContain(unsafe); + expect(result.gaps.some(gap => gap.code === "CONTROL_CHARACTERS_SANITIZED")).toBe(true); + expect(() => validateMarkdownConversionSemantics(result, { layout })).not.toThrow(); + }, + ); + + it("preserves a multi-column source row outside the selected header-neutral grid", async () => { + const extraValues = ["OutsideRow", "2100", "2300"]; + const items = [...ruledGridItems({ headerSize: 12 }), ...extraValues.map((value, column) => textItem(value, { + top: 322, left: 82 + column * 150, fontSize: 12, eol: column === 2, + }))]; + const layout = attachRuledRects(await validatedSyntheticLayout([{ items }]), headerlessGridRects()); + const result = renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: true }); + expect(result.markdown).not.toContain("
North12001450
"); + expect(result.markdown).not.toMatch(/rowspan=|colspan=/u); + expect(result.conversion_status).toBe("partial"); + expect(result.gaps.some(gap => gap.message.includes("Header roles are not established"))).toBe(true); + expect(result.table_proposal_regions).toHaveLength(1); + expect(() => validateMarkdownConversionSemantics(result, { layout })).not.toThrow(); + }); + + it("escapes source markup without interpreting pipe characters or creating HTML attributes", async () => { + const items = ruledGridItems({ headerSize: 12 }).map(item => item.str === "North" + ? textItem("&|\"'", { top: 178, left: 82, fontSize: 12, eol: false }) : item); + const layout = attachRuledRects(await validatedSyntheticLayout([{ items }]), headerlessGridRects()); + const result = renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: true }); + expect(result.markdown).toContain("<img>&|"'No\ufffdrth
"); + for (const value of ["Region", "North", "South", "West", ...extraValues]) { + expect(result.markdown.split(value)).toHaveLength(2); + } + expect(() => validateMarkdownConversionSemantics(result, { layout })).not.toThrow(); + }); + + it("preserves both disjoint grids when they share one contiguous multi-column run", async () => { + const secondValues = [["Second", "Q3", "Q4"], ["East", "4100", "4300"], + ["Upland", "5100", "5300"], ["Lowland", "6100", "6300"]]; + const items = [...ruledGridItems({ headerSize: 12 }), ...secondValues.flatMap((row, rowIndex) => ( + row.map((value, column) => textItem(value, { + top: 322 + rowIndex * 48, left: 82 + column * 150, fontSize: 12, eol: column === 2, + })) + ))]; + const rects = [...headerlessGridRects(), ...secondValues.flatMap((row, rowIndex) => row.map((_, column) => ({ + x: 72 + column * 150, y: 310 + rowIndex * 48, width: 150, height: 48, verb: "stroke", + })))]; + const layout = attachRuledRects(await validatedSyntheticLayout([{ items }]), rects); + const result = renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: true }); + expect(result.markdown).not.toContain("
"); + for (const value of ["Region", "North", "South", "West", ...secondValues.flat()]) { + expect(result.markdown.split(value)).toHaveLength(2); + } + expect(() => validateMarkdownConversionSemantics(result, { layout })).not.toThrow(); + }); + + it("keeps fully covered multiline cell text without dropping the additional source item", async () => { + const items = [...ruledGridItems({ headerSize: 12 }), + textItem("AdditionalCellText", { top: 288, left: 82, fontSize: 12, eol: false }), + textItem("TailCell", { top: 288, left: 232, fontSize: 12 })]; + const layout = attachRuledRects(await validatedSyntheticLayout([{ items }]), headerlessGridRects()); + const result = renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: true }); + expect(result.markdown).toContain("
"); + expect(result.markdown.split("AdditionalCellText")).toHaveLength(2); + expect(result.markdown.split("TailCell")).toHaveLength(2); + expect(() => validateMarkdownConversionSemantics(result, { layout })).not.toThrow(); + }); + + it.each(["http://a.co", "https://a.co", "a.co", "a@b.co"])( + "neutralizes URL-looking header-neutral cell text: %s", async value => { + const items = ruledGridItems({ headerSize: 12 }).map(item => item.str === "North" + ? { ...item, str: value, width: value.length * 6 } : item); + const layout = attachRuledRects(await validatedSyntheticLayout([{ items }]), headerlessGridRects()); + const result = renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: true }); + expect(result.markdown).toContain("
"); + expect(result.markdown).not.toContain(``); + expect(result.markdown).not.toContain(" validateMarkdownConversionSemantics(result, { layout })).not.toThrow(); + }, + ); + + it.each(["missing_rect", "cross_cell", "ambiguous_boundary", "overlap", "truncated", "annotation_failure"])( + "refuses header-neutral emission with %s evidence", async attack => { + let items = ruledGridItems({ headerSize: 12 }); + let rects = headerlessGridRects(); + if (attack === "missing_rect") rects = rects.filter(rect => !(rect.x === 222 && rect.y === 160)); + if (attack === "cross_cell") items = items.map(item => item.str === "North" + ? { ...item, width: 160 } : item); + if (attack === "ambiguous_boundary") items = items.map(item => item.str === "Q1" + ? textItem("Q1", { top: 130, left: 210, fontSize: 12, eol: false }) : item); + if (attack === "overlap") items.push(textItem("Twin", { top: 130, left: 84, fontSize: 12, eol: false })); + const layout = attachRuledRects(await validatedSyntheticLayout([{ items, + ...(attack === "truncated" ? { operations: [1] } : {}), + ...(attack === "annotation_failure" ? { annotationError: new Error("Synthetic annotation failure") } : {}), + }]), rects, + attack === "truncated" ? "truncated" : "available"); + const result = renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: true }); + expect(result.markdown).not.toContain("
${value}
"); + }, + ); + + it("does not promote aligned prose without cell rectangles and rejects a forged profile", async () => { + const layout = await validatedSyntheticLayout([{ items: ruledGridItems({ headerSize: 12 }) }]); + const result = renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: true }); + expect(result.markdown).not.toContain("
"); + expect(() => renderPdfLayoutToMarkdown(layout, { allowHeaderlessTables: "true" })).toThrow(/boolean/); + const forged = structuredClone(result); + delete forged.options.allow_headerless_tables; + expect(() => validateMarkdownConversionSemantics(forged)).toThrow(/renderer identity/); + }); + it("abstains on a ruled merged span with TABLE_TOPOLOGY_UNKNOWN", async () => { const mergedRects = ruledGridRects().filter(rect => rect.y !== 112); mergedRects.push({ x: 72, y: 112, width: 150, height: 48, verb: "stroke" }); diff --git a/test/mcp-contract.test.js b/test/mcp-contract.test.js index b66cc68..c8469d2 100644 --- a/test/mcp-contract.test.js +++ b/test/mcp-contract.test.js @@ -236,7 +236,9 @@ const EXAMPLE_PDF = path.join(REPO_ROOT, "example-fw9.pdf"); // three app-only and 56 model-visible; no existing input authority changes. // 2026-10-12: add batch verification and source-bound Evidence review; CSV // export is explicitly new-file-only. Optional OCR remains absent by default. -const TOOL_CONTRACT_SHA256 = "639189a0677085df0ef8d8ee92d23c5a5c1c3121a779e926ebe36f266c3de84c"; +// 2026-10-13: default-off header-neutral HTML table option and its coupled +// renderer-profile/output-schema contract. Default conversion bytes unchanged. +const TOOL_CONTRACT_SHA256 = "225dccb33c6168690d56ab0e40ef3fb364f10d4f0af8fdfa4577938f3566a235"; const CLOSED_READ = Object.freeze({ readOnlyHint: true,