diff --git a/docs/MAINTAINERS.md b/docs/MAINTAINERS.md index 0c97446..f2c2014 100644 --- a/docs/MAINTAINERS.md +++ b/docs/MAINTAINERS.md @@ -313,6 +313,28 @@ regions suppress header hints. The same source region is used by document maps and fresh proposal verification. Ordinary Markdown and conversion gaps do not change, and matching source/region tokens do not excuse obsolete assignments. +With `emit_table_proposals: true`, qualifying three-rule regions can also carry +`review_candidate`: a proposed arrangement with `status: needs_review`, exact +source-item assignments and line fragments, and inert HTML data cells. The +same preview and review notice appear in the text response. It groups the top +band provisionally and derives columns from whitespace corridors recurring +across every nonoverlapping body line. All region items must be assigned once, +every cell must contain source text, and the complete maximal partition must +recur on every body line. Ragged, wrapped, straddling or over-budget layouts +retain the evidence packet without a draft. Drafts are capped at 100 rows, +12 columns, 400 source items, 20,000 HTML bytes and 40,000 canonical JSON bytes. + +Rows, columns and header roles remain unverified; no header tags, spans, blank +values or new source text are inferred. Candidate digests bind algorithm, +PDF SHA, region, assignments, fragments, uncertainty and presentation. When +layout is supplied, semantic validation reconstructs the entire packet from +it. Standalone output validation reconstructs the draft from its retained text +and geometry as well, but cannot authenticate that packet to the original PDF; +it does not replace fresh source replay. Document maps carry no candidate cells. +Ordinary Markdown, its gap list and strict `verify_table_proposal` acceptance +remain unchanged. Review the draft beside the source PDF before using its +grouping; a visible draft is not an accepted table or verified extraction. + Filesystem operations and rasterization happen locally, and PDF Tools does not upload files to a separate PDF service. Text, images, and metadata returned through MCP may still be processed under the selected host or model provider's diff --git a/pdf-toolkit-mcp-share/server/index.js b/pdf-toolkit-mcp-share/server/index.js index b6ff513..5d08a04 100755 --- a/pdf-toolkit-mcp-share/server/index.js +++ b/pdf-toolkit-mcp-share/server/index.js @@ -3852,7 +3852,7 @@ async function listTools(request) { include_page_boundaries: { type: "boolean", description: "Include deterministic HTML comments marking page boundaries. Default: true." }, compact: { type: "boolean", description: "Opt into counted dot-leader, isolated page-number, and Unicode-letter spaced-hyphen normalizations. Default: false." }, remove_page_furniture: { type: "boolean", description: "Remove only source-evidenced extreme-margin page furniture and report every removed line by typed gap and counts. Set false to preserve all source lines. Default: true." }, - emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, + emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. A narrowly qualifying captioned three-rule table may also include a draft arrangement and header-neutral HTML preview marked needs_review; its row/column grouping and header roles are unverified. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.23.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." }, output_path: { type: "string", description: "Optional absolute .md path, or ~/ path. The file is written only after complete bytes are staged and verified." }, overwrite: { type: "boolean", description: "Replace an existing output_path only when its exact expected_output_identity is also supplied. Default: false." }, @@ -6082,6 +6082,7 @@ async function handleToolCall(request) { saved_output: savedOutput, ...(tableProposals !== undefined ? { table_proposals: tableProposals } : {}), }; + const reviewCandidates = tableProposals?.filter(region => region.review_candidate) ?? []; const summary = [ `Converted pages ${layout.page_range.start_page}-${layout.page_range.end_page} of ${fileName} to deterministic Markdown.`, `Status: ${rendered.conversion_status}. UTF-8 bytes: ${rendered.markdown_bytes}.`, @@ -6091,8 +6092,14 @@ async function handleToolCall(request) { ...(pagesNeedingVision.length > 0 ? [`Vision routing: use render_pdf_page for pages ${pagesNeedingVision.map(entry => entry.page).join(", ")}.`] : []), - "No OCR, external model, hidden link-target recovery, or table inference beyond complete source-bound text, ruled, or painted grids was performed.", + reviewCandidates.length > 0 + ? "The optional table drafts use whitespace alignment and need review. No OCR, external model or hidden link-target recovery was performed." + : "No OCR, external model, hidden link-target recovery, or table inference beyond complete source-bound text, ruled, or painted grids was performed.", rendered.markdown, + ...reviewCandidates.map(region => [ + `Page ${region.page}, ${region.region_id}: ${region.review_candidate.review_notice}`, + region.review_candidate.html, + ].join("\n\n")), ].join("\n\n"); return { content: [{ type: "text", text: summary }], diff --git a/pdf-toolkit-mcp-share/server/markdown-conversion.js b/pdf-toolkit-mcp-share/server/markdown-conversion.js index 324092d..4df92b2 100644 --- a/pdf-toolkit-mcp-share/server/markdown-conversion.js +++ b/pdf-toolkit-mcp-share/server/markdown-conversion.js @@ -60,6 +60,7 @@ export const MAX_TABLE_PROPOSAL_TEXT_ITEMS = 400; const MAX_TABLE_PROPOSAL_RULED_RECTS = 400; const MAX_TABLE_PROPOSAL_RULING_SEGMENTS = 400; const MAX_TABLE_PROPOSAL_PAINTED_RECTS = 400; +export const TABLE_REVIEW_NOTICE = "Proposed arrangement, needs review. Check the row and column grouping against the PDF before using it; headers and cell structure are unverified."; const MAX_MARKDOWN_BYTES_LIMIT = 200_000; const GAP_CODES = new Set([ @@ -2815,6 +2816,24 @@ function tableProposalHeaderHints(run) { }; } +function joinedThinFillBands(rectangles) { + const tolerance = 0.05; + const bars = rectangles.filter(rect => ["fill", "eoFill"].includes(rect.verb) + && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5) + .sort((a, b) => a.y - b.y || a.x - b.x); + const bands = []; + for (const bar of bars) { + const prior = bands.at(-1); + if (prior && Math.abs(prior.y - bar.y) <= tolerance + && Math.abs(prior.height - bar.height) <= tolerance + && Math.abs(prior.x + prior.width - bar.x) <= tolerance) { + prior.width = bar.x + bar.width - prior.x; + prior.parts.push(bar); + } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] }); + } + return bands; +} + function threeRuleProposalEnvelope(page, run) { // This refines an already abandoned proposal, never reconstructs a table. // Three full-width paint bands and a nearby caption can bound the source @@ -2832,19 +2851,7 @@ function threeRuleProposalEnvelope(page, run) { const text = page.raw_items.filter(item => item.text_kind === "non_whitespace"); if (text.some(item => !item.geometry_valid || !item.bbox)) return null; const tolerance = 0.05; - const bars = evidence.items.filter(rect => ["fill", "eoFill"].includes(rect.verb) - && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5) - .sort((a, b) => a.y - b.y || a.x - b.x); - const bands = []; - for (const bar of bars) { - const prior = bands.at(-1); - if (prior && Math.abs(prior.y - bar.y) <= tolerance - && Math.abs(prior.height - bar.height) <= tolerance - && Math.abs(prior.x + prior.width - bar.x) <= tolerance) { - prior.width = bar.x + bar.width - prior.x; - prior.parts.push(bar); - } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] }); - } + const bands = joinedThinFillBands(evidence.items); const contains = (outer, inner) => inner.x >= outer.x && inner.y >= outer.y && inner.x + inner.width <= outer.x + outer.width && inner.y + inner.height <= outer.y + outer.height; @@ -2888,17 +2895,109 @@ function threeRuleProposalEnvelope(page, run) { || !lines.some(line => line.y + line.height <= middle.y) || !lines.some(line => line.y >= middle.y + middle.height)) continue; const itemById = new Map(page.raw_items.map(item => [item.id, item])); - candidates.push({ bbox, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) }); + candidates.push({ bbox, middle, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) }); } return candidates.length === 1 ? candidates[0] : null; } +function canonicalReviewCandidate(value) { + if (Array.isArray(value)) return `[${value.map(canonicalReviewCandidate).join(",")}]`; + if (value && typeof value === "object") return `{${Object.keys(value).sort() + .map(key => `${JSON.stringify(key)}:${canonicalReviewCandidate(value[key])}`).join(",")}}`; + return JSON.stringify(value); +} + +function threeRuleReviewCandidate(region, envelope, sourceSha256) { + if (!/^[a-f0-9]{64}$/.test(sourceSha256 ?? "") + || Object.values(region.truncation).some(value => value !== "complete")) return null; + const items = region.text_items; + if (items.length === 0 || new Set(items.map(item => item.id)).size !== items.length) return null; + const { middle } = envelope; + const upper = items.filter(item => item.bbox.y + item.bbox.height <= middle.y); + const lower = items.filter(item => item.bbox.y >= middle.y + middle.height); + if (upper.length === 0 || upper.length + lower.length !== items.length) return null; + const byLine = new Map(); + for (const item of lower) { + if (!byLine.has(item.line_id)) byLine.set(item.line_id, []); + byLine.get(item.line_id).push(item); + } + const rows = [...byLine.values()].map(row => ({ + items: row.sort((a, b) => a.bbox.x - b.bbox.x || a.reading_order_index - b.reading_order_index), + top: Math.min(...row.map(item => item.bbox.y)), + bottom: Math.max(...row.map(item => item.bbox.y + item.bbox.height)), + })).sort((a, b) => a.top - b.top); + if (rows.length < 2 || rows.length > 99 || rows.some((row, index) => ( + index > 0 && row.top <= rows[index - 1].bottom + ))) return null; + // A maximal set of whitespace corridors shared by every body line supplies + // one deterministic proposed partition. Whitespace does not prove topology. + const minGap = Math.max(5, ...lower.map(item => item.bbox.height)); + let corridors = null; + let gapCount = null; + for (const row of rows) { + const gaps = []; + let right = row.items[0].bbox.x + row.items[0].bbox.width; + for (const item of row.items.slice(1)) { + if (item.bbox.x - right >= minGap) gaps.push([right, item.bbox.x]); + right = Math.max(right, item.bbox.x + item.bbox.width); + } + if (gapCount !== null && gaps.length !== gapCount) return null; + gapCount = gaps.length; + corridors = corridors === null ? gaps : corridors.flatMap(([left, end]) => gaps + .map(([start, rightEdge]) => [Math.max(left, start), Math.min(end, rightEdge)]) + .filter(([start, rightEdge]) => rightEdge - start >= minGap)); + } + if (corridors.length < 1 || corridors.length > 11 || corridors.length !== gapCount) return null; + const cuts = corridors.map(([left, right]) => (left + right) / 2); + const sourceRows = [upper, ...rows.map(row => row.items)]; + const cells = []; + for (const [row, sourceItems] of sourceRows.entries()) { + const columns = Array.from({ length: cuts.length + 1 }, () => []); + for (const item of sourceItems) { + const column = cuts.filter(cut => item.bbox.x >= cut).length; + if (column < cuts.length && item.bbox.x + item.bbox.width > cuts[column]) return null; + columns[column].push(item); + } + if (columns.some(column => column.length === 0)) return null; + for (const [column, assigned] of columns.entries()) { + assigned.sort((a, b) => a.reading_order_index - b.reading_order_index); + const lines = []; + for (const item of assigned) { + if (lines.at(-1)?.line_id !== item.line_id) { + if (lines.some(line => line.line_id === item.line_id)) return null; + lines.push({ line_id: item.line_id, item_ids: [], text: "" }); + } + lines.at(-1).item_ids.push(item.id); + lines.at(-1).text += item.text; + } + cells.push({ row, column, item_ids: assigned.map(item => item.id), lines }); + } + } + const assigned = cells.flatMap(cell => cell.item_ids); + if (new Set(assigned).size !== items.length || assigned.length !== items.length) return null; + const grid = sourceRows.map((_, row) => cells.filter(cell => cell.row === row) + .map(cell => cell.lines.map(line => escapeHeaderNeutralText(line.text)).join("
"))); + const html = ["", ...grid.map(row => `${row.map(text => ``).join("")}`), "
${text}
"].join("\n"); + const candidate = { + algorithm: { name: "pdf-tools.three-rule-review-candidate", version: "0.1.0" }, + status: "needs_review", topology_verified: false, header_roles_verified: false, + source_sha256: sourceSha256, region_id: region.region_id, page: region.page, + content_origin: "source_text_items", structure_origin: "whitespace_alignment_proposal", + row_count: sourceRows.length, column_count: cuts.length + 1, + source_item_count: items.length, assigned_item_count: assigned.length, unassigned_item_count: 0, + cells, html, html_format: "header_neutral_html", html_bytes: utf8Bytes(html), + review_notice: TABLE_REVIEW_NOTICE, + }; + if (candidate.html_bytes > 20_000 || utf8Bytes(canonicalReviewCandidate(candidate)) > 40_000) return null; + return { ...candidate, candidate_sha256: sha256(canonicalReviewCandidate(candidate)) }; +} + /** * Build a bounded proposal from an abandoned run. A uniquely captioned * three-rule envelope can refine source membership but supplies no topology * or header authority. Pure over source-validated IR; no token, time or I/O. */ -export function buildTableProposalRegion(page, run, reason, ordinal) { +export function buildTableProposalRegion(page, run, reason, ordinal, { sourceSha256 = null } = {}) { const envelope = threeRuleProposalEnvelope(page, run); if (envelope) run = envelope.run; const itemById = new Map(page.raw_items.map(item => [item.id, item])); @@ -2962,7 +3061,7 @@ export function buildTableProposalRegion(page, run, reason, ordinal) { bbox: painted.bbox, })); - return { + const proposal = { region_id: `p${page.page}-t${ordinal}`, page: page.page, reason: regionGapCode(reason), @@ -2983,6 +3082,11 @@ export function buildTableProposalRegion(page, run, reason, ordinal) { painted_rectangles: paintedTruncated ? "truncated" : "complete", }, }; + if (envelope) { + const candidate = threeRuleReviewCandidate(proposal, envelope, sourceSha256); + if (candidate) proposal.review_candidate = candidate; + } + return proposal; } // escapePlainMarkdown already escapes "|" (and backslashes before it), so a @@ -3007,15 +3111,18 @@ function renderTable(grid) { return lines; } -function renderHeaderlessTable(grid) { - const escape = value => sanitizeUnsafeText(String(value)).replace(/&/gu, "&") +function escapeHeaderNeutralText(value) { + return sanitizeUnsafeText(String(value)).replace(/&/gu, "&") .replace(//gu, ">") .replace(/"/gu, """).replace(/'/gu, "'") .replace(/https?:/giu, match => `${match.slice(0, -1)}:`) .replace(/\b(?:[\p{L}\p{N}-]+\.)+[\p{L}]{2,}\b/giu, domain => domain.replace(".", ".")) .replace(/@(?=[\p{L}\p{N}.-]+.[\p{L}]{2,})/giu, "@"); +} + +function renderHeaderlessTable(grid) { return ["", ...grid.map(row => ( - `${row.map(value => ``).join("")}` + `${row.map(value => ``).join("")}` )), "
${escape(value)}
${escapeHeaderNeutralText(value)}
"]; } @@ -3370,6 +3477,7 @@ function renderPage(page, { pageBoundaryBefore = false, pageBoundaryAfter = false, collectTableProposals = false, + sourceSha256 = null, allowHeaderlessTables = false, pageFurniture = [], } = {}) { @@ -3488,7 +3596,7 @@ function renderPage(page, { // markdown, gaps, or byte counts. It is only present when the opt-in flag // is set, so the default page shape is byte-identical. rendered.tableProposalRegions = analysis.regions.map( - (region, index) => buildTableProposalRegion(page, region.run, region.reason, index + 1), + (region, index) => buildTableProposalRegion(page, region.run, region.reason, index + 1, { sourceSha256 }), ); } return rendered; @@ -3706,6 +3814,28 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = if (proposals !== undefined) { validateTableProposals(proposals, { requireToken: payloadProposals !== undefined }); validateTableProposalDocumentTruncation(result.table_proposals_truncation, proposals); + if (layout !== null) { + const expected = boundTableProposalRegions(layout.pages.flatMap(page => ( + segmentPageLines(page, allowHeaderlessTables).regions.map((region, index) => ( + buildTableProposalRegion(page, region.run, region.reason, index + 1, + { sourceSha256: layout.source.sha256 }) + )) + ))); + const actual = proposals.map(({ proposal_token: _token, ...proposal }) => proposal); + assertion(sameJson(actual, expected.regions), "table proposals do not match the bound layout IR"); + assertion(sameJson(result.table_proposals_truncation, expected.truncation), + "table proposal truncation does not match the bound layout IR"); + } + for (const proposal of proposals) { + if (proposal.review_candidate) { + const { candidate_sha256: digest, ...candidate } = proposal.review_candidate; + assertion(candidate.source_sha256 === result.provenance.source.sha256 + && candidate.region_id === proposal.region_id && candidate.page === proposal.page, + "table review candidate source binding mismatch"); + assertion(digest === sha256(canonicalReviewCandidate(candidate)), "table review candidate digest mismatch"); + validateTableReviewCandidate(proposal, candidate); + } + } } else { assertion(result.table_proposals_truncation === undefined, "table proposal truncation cannot exist without proposals"); @@ -3713,6 +3843,20 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = return result; } +function validateTableReviewCandidate(region, candidate) { + const bands = joinedThinFillBands(region.ruled_rects); + assertion(bands.length === 3 && bands.every(band => Math.abs(band.x - region.bbox.x) <= 0.05 + && Math.abs(band.width - region.bbox.width) <= 0.05) + && Math.abs(bands[0].y - region.bbox.y) <= 0.05 + && Math.abs(bands[2].y + bands[2].height - region.bbox.y - region.bbox.height) <= 0.05, + "table review candidate rule envelope mismatch"); + const replay = threeRuleReviewCandidate(region, { middle: bands[1] }, candidate.source_sha256); + assertion(replay !== null, "table review candidate geometry is unsupported"); + const { candidate_sha256: _digest, ...replayed } = replay; + assertion(canonicalReviewCandidate(candidate) === canonicalReviewCandidate(replayed), + "table review candidate does not match its retained source geometry"); +} + function isFiniteNumber(value) { return typeof value === "number" && Number.isFinite(value); } @@ -3985,6 +4129,7 @@ export function renderPdfLayoutToMarkdown(layout, { // number from a single-page selection. pageBoundaryAfter: true, collectTableProposals: emitTableProposals, + sourceSha256: layout.source.sha256, allowHeaderlessTables, pageFurniture: furniturePlans.get(page.page), })); diff --git a/pdf-toolkit-mcp-share/server/output-schemas.js b/pdf-toolkit-mcp-share/server/output-schemas.js index 8d11e92..3c984ec 100644 --- a/pdf-toolkit-mcp-share/server/output-schemas.js +++ b/pdf-toolkit-mcp-share/server/output-schemas.js @@ -1,6 +1,6 @@ import { AjvJsonSchemaValidator } from "@modelcontextprotocol/server/validators/ajv"; import { validatePdfLayoutSemantics } from "./layout-extraction.js"; -import { validateMarkdownConversionSemantics } from "./markdown-conversion.js"; +import { TABLE_REVIEW_NOTICE, validateMarkdownConversionSemantics } from "./markdown-conversion.js"; import { validatePdfObservationSemantics, validateRenderObservationSemantics, @@ -1172,6 +1172,27 @@ const tableProposal = object({ truncation: tableProposalTruncation, proposal_token: sha256Digest, }); +// Additive only on opt-in proposal packets. Strict table acceptance is separate. +tableProposal.properties.review_candidate = object({ + algorithm: object({ name: { const: "pdf-tools.three-rule-review-candidate" }, version: { const: "0.1.0" } }), + status: { const: "needs_review" }, topology_verified: { const: false }, header_roles_verified: { const: false }, + source_sha256: sha256Digest, region_id: string, page: { type: "integer", minimum: 1 }, + content_origin: { const: "source_text_items" }, structure_origin: { const: "whitespace_alignment_proposal" }, + row_count: { type: "integer", minimum: 3, maximum: 100 }, + column_count: { type: "integer", minimum: 2, maximum: 12 }, + source_item_count: { type: "integer", minimum: 1, maximum: 400 }, + assigned_item_count: { type: "integer", minimum: 1, maximum: 400 }, unassigned_item_count: { const: 0 }, + cells: { type: "array", minItems: 6, maxItems: 400, items: object({ + row: { type: "integer", minimum: 0, maximum: 99 }, column: { type: "integer", minimum: 0, maximum: 11 }, + item_ids: { type: "array", minItems: 1, maxItems: 400, items: string }, + lines: { type: "array", minItems: 1, maxItems: 400, items: object({ + line_id: string, item_ids: { type: "array", minItems: 1, maxItems: 400, items: string }, text: string, + }) }, + }) }, + html: string, html_format: { const: "header_neutral_html" }, + html_bytes: { type: "integer", minimum: 1, maximum: 20000 }, + review_notice: { const: TABLE_REVIEW_NOTICE }, candidate_sha256: sha256Digest, +}); const tableProposalCheckStatus = enumString(["passed", "failed", "not_run"]); const verifiedTableCell = object({ row: { type: "integer", minimum: 0 }, diff --git a/scripts/macos-claude-installed-shannon.mjs b/scripts/macos-claude-installed-shannon.mjs index 7ee10e6..184a213 100755 --- a/scripts/macos-claude-installed-shannon.mjs +++ b/scripts/macos-claude-installed-shannon.mjs @@ -23,7 +23,7 @@ if (!process.argv[2] || !process.argv[3]) { // const EXPECTED_SOURCE_SHA256 = "6e4e3411984f3edf99dbfe8b941cb5e8a321379ff0cae6ae5c1f592ad8882ca8"; const EXPECTED_MARKDOWN_SHA256 = "b9bd7ca7db26fe807f294565c39de8d4fe9337908afa74cd7e2896a800c92991"; -const EXPECTED_TOOL_CONTRACT_SHA256 = "17e2067e2c014bfc77e234432af45f7711494104f2a14d5f652e42e917251128"; +const EXPECTED_TOOL_CONTRACT_SHA256 = "2239a74dcdb71387f855c49bb26bc01e52ca9a14e773be13a32d19d3af9ec2d4"; const EXPECTED_PAGE_COUNT = 55; const EXPECTED_GAP_COUNT = 26; const EXPECTED_REPLACEMENT_CHARACTER_COUNT = 6; diff --git a/scripts/macos-claude-installed-smoke.mjs b/scripts/macos-claude-installed-smoke.mjs index e9a2d73..9460f72 100755 --- a/scripts/macos-claude-installed-smoke.mjs +++ b/scripts/macos-claude-installed-smoke.mjs @@ -33,7 +33,7 @@ const textFixture = path.join(fixtureDirectory, "synthetic-text-two-page.pdf"); const rasterFixture = path.join(fixtureDirectory, "synthetic-raster-only.pdf"); const mutationDirectory = path.join(fixtureDirectory, "mutation-output"); const toolNames = []; -const EXPECTED_TOOL_CONTRACT_SHA256 = "17e2067e2c014bfc77e234432af45f7711494104f2a14d5f652e42e917251128"; +const EXPECTED_TOOL_CONTRACT_SHA256 = "2239a74dcdb71387f855c49bb26bc01e52ca9a14e773be13a32d19d3af9ec2d4"; const ACCESSIBILITY_CONCLUSION_KEYS = Object.freeze([ "certification", "document_accessibility", diff --git a/server/index.js b/server/index.js index b6ff513..5d08a04 100755 --- a/server/index.js +++ b/server/index.js @@ -3852,7 +3852,7 @@ async function listTools(request) { include_page_boundaries: { type: "boolean", description: "Include deterministic HTML comments marking page boundaries. Default: true." }, compact: { type: "boolean", description: "Opt into counted dot-leader, isolated page-number, and Unicode-letter spaced-hyphen normalizations. Default: false." }, remove_page_furniture: { type: "boolean", description: "Remove only source-evidenced extreme-margin page furniture and report every removed line by typed gap and counts. Set false to preserve all source lines. Default: true." }, - emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, + emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. A narrowly qualifying captioned three-rule table may also include a draft arrangement and header-neutral HTML preview marked needs_review; its row/column grouping and header roles are unverified. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.23.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." }, output_path: { type: "string", description: "Optional absolute .md path, or ~/ path. The file is written only after complete bytes are staged and verified." }, overwrite: { type: "boolean", description: "Replace an existing output_path only when its exact expected_output_identity is also supplied. Default: false." }, @@ -6082,6 +6082,7 @@ async function handleToolCall(request) { saved_output: savedOutput, ...(tableProposals !== undefined ? { table_proposals: tableProposals } : {}), }; + const reviewCandidates = tableProposals?.filter(region => region.review_candidate) ?? []; const summary = [ `Converted pages ${layout.page_range.start_page}-${layout.page_range.end_page} of ${fileName} to deterministic Markdown.`, `Status: ${rendered.conversion_status}. UTF-8 bytes: ${rendered.markdown_bytes}.`, @@ -6091,8 +6092,14 @@ async function handleToolCall(request) { ...(pagesNeedingVision.length > 0 ? [`Vision routing: use render_pdf_page for pages ${pagesNeedingVision.map(entry => entry.page).join(", ")}.`] : []), - "No OCR, external model, hidden link-target recovery, or table inference beyond complete source-bound text, ruled, or painted grids was performed.", + reviewCandidates.length > 0 + ? "The optional table drafts use whitespace alignment and need review. No OCR, external model or hidden link-target recovery was performed." + : "No OCR, external model, hidden link-target recovery, or table inference beyond complete source-bound text, ruled, or painted grids was performed.", rendered.markdown, + ...reviewCandidates.map(region => [ + `Page ${region.page}, ${region.region_id}: ${region.review_candidate.review_notice}`, + region.review_candidate.html, + ].join("\n\n")), ].join("\n\n"); return { content: [{ type: "text", text: summary }], diff --git a/server/markdown-conversion.js b/server/markdown-conversion.js index 324092d..4df92b2 100644 --- a/server/markdown-conversion.js +++ b/server/markdown-conversion.js @@ -60,6 +60,7 @@ export const MAX_TABLE_PROPOSAL_TEXT_ITEMS = 400; const MAX_TABLE_PROPOSAL_RULED_RECTS = 400; const MAX_TABLE_PROPOSAL_RULING_SEGMENTS = 400; const MAX_TABLE_PROPOSAL_PAINTED_RECTS = 400; +export const TABLE_REVIEW_NOTICE = "Proposed arrangement, needs review. Check the row and column grouping against the PDF before using it; headers and cell structure are unverified."; const MAX_MARKDOWN_BYTES_LIMIT = 200_000; const GAP_CODES = new Set([ @@ -2815,6 +2816,24 @@ function tableProposalHeaderHints(run) { }; } +function joinedThinFillBands(rectangles) { + const tolerance = 0.05; + const bars = rectangles.filter(rect => ["fill", "eoFill"].includes(rect.verb) + && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5) + .sort((a, b) => a.y - b.y || a.x - b.x); + const bands = []; + for (const bar of bars) { + const prior = bands.at(-1); + if (prior && Math.abs(prior.y - bar.y) <= tolerance + && Math.abs(prior.height - bar.height) <= tolerance + && Math.abs(prior.x + prior.width - bar.x) <= tolerance) { + prior.width = bar.x + bar.width - prior.x; + prior.parts.push(bar); + } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] }); + } + return bands; +} + function threeRuleProposalEnvelope(page, run) { // This refines an already abandoned proposal, never reconstructs a table. // Three full-width paint bands and a nearby caption can bound the source @@ -2832,19 +2851,7 @@ function threeRuleProposalEnvelope(page, run) { const text = page.raw_items.filter(item => item.text_kind === "non_whitespace"); if (text.some(item => !item.geometry_valid || !item.bbox)) return null; const tolerance = 0.05; - const bars = evidence.items.filter(rect => ["fill", "eoFill"].includes(rect.verb) - && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5) - .sort((a, b) => a.y - b.y || a.x - b.x); - const bands = []; - for (const bar of bars) { - const prior = bands.at(-1); - if (prior && Math.abs(prior.y - bar.y) <= tolerance - && Math.abs(prior.height - bar.height) <= tolerance - && Math.abs(prior.x + prior.width - bar.x) <= tolerance) { - prior.width = bar.x + bar.width - prior.x; - prior.parts.push(bar); - } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] }); - } + const bands = joinedThinFillBands(evidence.items); const contains = (outer, inner) => inner.x >= outer.x && inner.y >= outer.y && inner.x + inner.width <= outer.x + outer.width && inner.y + inner.height <= outer.y + outer.height; @@ -2888,17 +2895,109 @@ function threeRuleProposalEnvelope(page, run) { || !lines.some(line => line.y + line.height <= middle.y) || !lines.some(line => line.y >= middle.y + middle.height)) continue; const itemById = new Map(page.raw_items.map(item => [item.id, item])); - candidates.push({ bbox, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) }); + candidates.push({ bbox, middle, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) }); } return candidates.length === 1 ? candidates[0] : null; } +function canonicalReviewCandidate(value) { + if (Array.isArray(value)) return `[${value.map(canonicalReviewCandidate).join(",")}]`; + if (value && typeof value === "object") return `{${Object.keys(value).sort() + .map(key => `${JSON.stringify(key)}:${canonicalReviewCandidate(value[key])}`).join(",")}}`; + return JSON.stringify(value); +} + +function threeRuleReviewCandidate(region, envelope, sourceSha256) { + if (!/^[a-f0-9]{64}$/.test(sourceSha256 ?? "") + || Object.values(region.truncation).some(value => value !== "complete")) return null; + const items = region.text_items; + if (items.length === 0 || new Set(items.map(item => item.id)).size !== items.length) return null; + const { middle } = envelope; + const upper = items.filter(item => item.bbox.y + item.bbox.height <= middle.y); + const lower = items.filter(item => item.bbox.y >= middle.y + middle.height); + if (upper.length === 0 || upper.length + lower.length !== items.length) return null; + const byLine = new Map(); + for (const item of lower) { + if (!byLine.has(item.line_id)) byLine.set(item.line_id, []); + byLine.get(item.line_id).push(item); + } + const rows = [...byLine.values()].map(row => ({ + items: row.sort((a, b) => a.bbox.x - b.bbox.x || a.reading_order_index - b.reading_order_index), + top: Math.min(...row.map(item => item.bbox.y)), + bottom: Math.max(...row.map(item => item.bbox.y + item.bbox.height)), + })).sort((a, b) => a.top - b.top); + if (rows.length < 2 || rows.length > 99 || rows.some((row, index) => ( + index > 0 && row.top <= rows[index - 1].bottom + ))) return null; + // A maximal set of whitespace corridors shared by every body line supplies + // one deterministic proposed partition. Whitespace does not prove topology. + const minGap = Math.max(5, ...lower.map(item => item.bbox.height)); + let corridors = null; + let gapCount = null; + for (const row of rows) { + const gaps = []; + let right = row.items[0].bbox.x + row.items[0].bbox.width; + for (const item of row.items.slice(1)) { + if (item.bbox.x - right >= minGap) gaps.push([right, item.bbox.x]); + right = Math.max(right, item.bbox.x + item.bbox.width); + } + if (gapCount !== null && gaps.length !== gapCount) return null; + gapCount = gaps.length; + corridors = corridors === null ? gaps : corridors.flatMap(([left, end]) => gaps + .map(([start, rightEdge]) => [Math.max(left, start), Math.min(end, rightEdge)]) + .filter(([start, rightEdge]) => rightEdge - start >= minGap)); + } + if (corridors.length < 1 || corridors.length > 11 || corridors.length !== gapCount) return null; + const cuts = corridors.map(([left, right]) => (left + right) / 2); + const sourceRows = [upper, ...rows.map(row => row.items)]; + const cells = []; + for (const [row, sourceItems] of sourceRows.entries()) { + const columns = Array.from({ length: cuts.length + 1 }, () => []); + for (const item of sourceItems) { + const column = cuts.filter(cut => item.bbox.x >= cut).length; + if (column < cuts.length && item.bbox.x + item.bbox.width > cuts[column]) return null; + columns[column].push(item); + } + if (columns.some(column => column.length === 0)) return null; + for (const [column, assigned] of columns.entries()) { + assigned.sort((a, b) => a.reading_order_index - b.reading_order_index); + const lines = []; + for (const item of assigned) { + if (lines.at(-1)?.line_id !== item.line_id) { + if (lines.some(line => line.line_id === item.line_id)) return null; + lines.push({ line_id: item.line_id, item_ids: [], text: "" }); + } + lines.at(-1).item_ids.push(item.id); + lines.at(-1).text += item.text; + } + cells.push({ row, column, item_ids: assigned.map(item => item.id), lines }); + } + } + const assigned = cells.flatMap(cell => cell.item_ids); + if (new Set(assigned).size !== items.length || assigned.length !== items.length) return null; + const grid = sourceRows.map((_, row) => cells.filter(cell => cell.row === row) + .map(cell => cell.lines.map(line => escapeHeaderNeutralText(line.text)).join("
"))); + const html = ["", ...grid.map(row => `${row.map(text => ``).join("")}`), "
${text}
"].join("\n"); + const candidate = { + algorithm: { name: "pdf-tools.three-rule-review-candidate", version: "0.1.0" }, + status: "needs_review", topology_verified: false, header_roles_verified: false, + source_sha256: sourceSha256, region_id: region.region_id, page: region.page, + content_origin: "source_text_items", structure_origin: "whitespace_alignment_proposal", + row_count: sourceRows.length, column_count: cuts.length + 1, + source_item_count: items.length, assigned_item_count: assigned.length, unassigned_item_count: 0, + cells, html, html_format: "header_neutral_html", html_bytes: utf8Bytes(html), + review_notice: TABLE_REVIEW_NOTICE, + }; + if (candidate.html_bytes > 20_000 || utf8Bytes(canonicalReviewCandidate(candidate)) > 40_000) return null; + return { ...candidate, candidate_sha256: sha256(canonicalReviewCandidate(candidate)) }; +} + /** * Build a bounded proposal from an abandoned run. A uniquely captioned * three-rule envelope can refine source membership but supplies no topology * or header authority. Pure over source-validated IR; no token, time or I/O. */ -export function buildTableProposalRegion(page, run, reason, ordinal) { +export function buildTableProposalRegion(page, run, reason, ordinal, { sourceSha256 = null } = {}) { const envelope = threeRuleProposalEnvelope(page, run); if (envelope) run = envelope.run; const itemById = new Map(page.raw_items.map(item => [item.id, item])); @@ -2962,7 +3061,7 @@ export function buildTableProposalRegion(page, run, reason, ordinal) { bbox: painted.bbox, })); - return { + const proposal = { region_id: `p${page.page}-t${ordinal}`, page: page.page, reason: regionGapCode(reason), @@ -2983,6 +3082,11 @@ export function buildTableProposalRegion(page, run, reason, ordinal) { painted_rectangles: paintedTruncated ? "truncated" : "complete", }, }; + if (envelope) { + const candidate = threeRuleReviewCandidate(proposal, envelope, sourceSha256); + if (candidate) proposal.review_candidate = candidate; + } + return proposal; } // escapePlainMarkdown already escapes "|" (and backslashes before it), so a @@ -3007,15 +3111,18 @@ function renderTable(grid) { return lines; } -function renderHeaderlessTable(grid) { - const escape = value => sanitizeUnsafeText(String(value)).replace(/&/gu, "&") +function escapeHeaderNeutralText(value) { + return sanitizeUnsafeText(String(value)).replace(/&/gu, "&") .replace(//gu, ">") .replace(/"/gu, """).replace(/'/gu, "'") .replace(/https?:/giu, match => `${match.slice(0, -1)}:`) .replace(/\b(?:[\p{L}\p{N}-]+\.)+[\p{L}]{2,}\b/giu, domain => domain.replace(".", ".")) .replace(/@(?=[\p{L}\p{N}.-]+.[\p{L}]{2,})/giu, "@"); +} + +function renderHeaderlessTable(grid) { return ["", ...grid.map(row => ( - `${row.map(value => ``).join("")}` + `${row.map(value => ``).join("")}` )), "
${escape(value)}
${escapeHeaderNeutralText(value)}
"]; } @@ -3370,6 +3477,7 @@ function renderPage(page, { pageBoundaryBefore = false, pageBoundaryAfter = false, collectTableProposals = false, + sourceSha256 = null, allowHeaderlessTables = false, pageFurniture = [], } = {}) { @@ -3488,7 +3596,7 @@ function renderPage(page, { // markdown, gaps, or byte counts. It is only present when the opt-in flag // is set, so the default page shape is byte-identical. rendered.tableProposalRegions = analysis.regions.map( - (region, index) => buildTableProposalRegion(page, region.run, region.reason, index + 1), + (region, index) => buildTableProposalRegion(page, region.run, region.reason, index + 1, { sourceSha256 }), ); } return rendered; @@ -3706,6 +3814,28 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = if (proposals !== undefined) { validateTableProposals(proposals, { requireToken: payloadProposals !== undefined }); validateTableProposalDocumentTruncation(result.table_proposals_truncation, proposals); + if (layout !== null) { + const expected = boundTableProposalRegions(layout.pages.flatMap(page => ( + segmentPageLines(page, allowHeaderlessTables).regions.map((region, index) => ( + buildTableProposalRegion(page, region.run, region.reason, index + 1, + { sourceSha256: layout.source.sha256 }) + )) + ))); + const actual = proposals.map(({ proposal_token: _token, ...proposal }) => proposal); + assertion(sameJson(actual, expected.regions), "table proposals do not match the bound layout IR"); + assertion(sameJson(result.table_proposals_truncation, expected.truncation), + "table proposal truncation does not match the bound layout IR"); + } + for (const proposal of proposals) { + if (proposal.review_candidate) { + const { candidate_sha256: digest, ...candidate } = proposal.review_candidate; + assertion(candidate.source_sha256 === result.provenance.source.sha256 + && candidate.region_id === proposal.region_id && candidate.page === proposal.page, + "table review candidate source binding mismatch"); + assertion(digest === sha256(canonicalReviewCandidate(candidate)), "table review candidate digest mismatch"); + validateTableReviewCandidate(proposal, candidate); + } + } } else { assertion(result.table_proposals_truncation === undefined, "table proposal truncation cannot exist without proposals"); @@ -3713,6 +3843,20 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = return result; } +function validateTableReviewCandidate(region, candidate) { + const bands = joinedThinFillBands(region.ruled_rects); + assertion(bands.length === 3 && bands.every(band => Math.abs(band.x - region.bbox.x) <= 0.05 + && Math.abs(band.width - region.bbox.width) <= 0.05) + && Math.abs(bands[0].y - region.bbox.y) <= 0.05 + && Math.abs(bands[2].y + bands[2].height - region.bbox.y - region.bbox.height) <= 0.05, + "table review candidate rule envelope mismatch"); + const replay = threeRuleReviewCandidate(region, { middle: bands[1] }, candidate.source_sha256); + assertion(replay !== null, "table review candidate geometry is unsupported"); + const { candidate_sha256: _digest, ...replayed } = replay; + assertion(canonicalReviewCandidate(candidate) === canonicalReviewCandidate(replayed), + "table review candidate does not match its retained source geometry"); +} + function isFiniteNumber(value) { return typeof value === "number" && Number.isFinite(value); } @@ -3985,6 +4129,7 @@ export function renderPdfLayoutToMarkdown(layout, { // number from a single-page selection. pageBoundaryAfter: true, collectTableProposals: emitTableProposals, + sourceSha256: layout.source.sha256, allowHeaderlessTables, pageFurniture: furniturePlans.get(page.page), })); diff --git a/server/output-schemas.js b/server/output-schemas.js index 8d11e92..3c984ec 100644 --- a/server/output-schemas.js +++ b/server/output-schemas.js @@ -1,6 +1,6 @@ import { AjvJsonSchemaValidator } from "@modelcontextprotocol/server/validators/ajv"; import { validatePdfLayoutSemantics } from "./layout-extraction.js"; -import { validateMarkdownConversionSemantics } from "./markdown-conversion.js"; +import { TABLE_REVIEW_NOTICE, validateMarkdownConversionSemantics } from "./markdown-conversion.js"; import { validatePdfObservationSemantics, validateRenderObservationSemantics, @@ -1172,6 +1172,27 @@ const tableProposal = object({ truncation: tableProposalTruncation, proposal_token: sha256Digest, }); +// Additive only on opt-in proposal packets. Strict table acceptance is separate. +tableProposal.properties.review_candidate = object({ + algorithm: object({ name: { const: "pdf-tools.three-rule-review-candidate" }, version: { const: "0.1.0" } }), + status: { const: "needs_review" }, topology_verified: { const: false }, header_roles_verified: { const: false }, + source_sha256: sha256Digest, region_id: string, page: { type: "integer", minimum: 1 }, + content_origin: { const: "source_text_items" }, structure_origin: { const: "whitespace_alignment_proposal" }, + row_count: { type: "integer", minimum: 3, maximum: 100 }, + column_count: { type: "integer", minimum: 2, maximum: 12 }, + source_item_count: { type: "integer", minimum: 1, maximum: 400 }, + assigned_item_count: { type: "integer", minimum: 1, maximum: 400 }, unassigned_item_count: { const: 0 }, + cells: { type: "array", minItems: 6, maxItems: 400, items: object({ + row: { type: "integer", minimum: 0, maximum: 99 }, column: { type: "integer", minimum: 0, maximum: 11 }, + item_ids: { type: "array", minItems: 1, maxItems: 400, items: string }, + lines: { type: "array", minItems: 1, maxItems: 400, items: object({ + line_id: string, item_ids: { type: "array", minItems: 1, maxItems: 400, items: string }, text: string, + }) }, + }) }, + html: string, html_format: { const: "header_neutral_html" }, + html_bytes: { type: "integer", minimum: 1, maximum: 20000 }, + review_notice: { const: TABLE_REVIEW_NOTICE }, candidate_sha256: sha256Digest, +}); const tableProposalCheckStatus = enumString(["passed", "failed", "not_run"]); const verifiedTableCell = object({ row: { type: "integer", minimum: 0 }, diff --git a/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json b/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json index 10b81b3..2105fe9 100644 --- a/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json +++ b/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json @@ -79,14 +79,14 @@ { "role": "markdown_conversion_module", "path": "server/markdown-conversion.js", - "bytes": 192276, - "sha256": "838f9b1adb38faf5eb3b31d7be9479eb8521dff0fd7013dcd3c89fc3cbcb95d1" + "bytes": 200207, + "sha256": "988a3b3b4c9ebb27b66a63cd2703dd002519935c8dd8660a8af78f53621b257a" }, { "role": "output_schemas_module", "path": "server/output-schemas.js", - "bytes": 84427, - "sha256": "beb9255b64456319bf2b747e643c32d3d175c29f51feddfdf5f419da19f9fc3c" + "bytes": 85973, + "sha256": "f26eea7b94ccda2faca95880afe33bb3186511b9cf7ac32b799b895ee84b226d" }, { "role": "package_json", @@ -131,7 +131,7 @@ "sha256": "ef40501b2afbe4cd2adfa80480344783bbec1a00e8e9850086a5d044231d1f53" } ], - "validator_source_set_sha256": "6e875209853d97288e5eae1464c0c3a333f37b31ad3467839954aa4edbf657c4", + "validator_source_set_sha256": "1903def9a7de315d775ec2a4f0494778cefad449c88fdf8e9283fc2749eabf01", "pdfjs_version": "5.4.624", "generation_contract": { "source_path": "source.pdf", diff --git a/test/fixtures/eval/trajectories/tool-contracts.v3.json b/test/fixtures/eval/trajectories/tool-contracts.v3.json index ff68473..9c7cfd0 100644 --- a/test/fixtures/eval/trajectories/tool-contracts.v3.json +++ b/test/fixtures/eval/trajectories/tool-contracts.v3.json @@ -5,7 +5,7 @@ "name": "pdf-tools", "version": "0.14.0" }, - "tools_sha256": "93202e8e6e04b8914f81003624692177e9dbe02c6fc93aa877ebf7e82b2556a7", + "tools_sha256": "cbd4f2614576b123130ad430542f58cf07ca3730026c59545e7fe4b447dce4bf", "tools": [ { "name": "add_signature_field", @@ -554,7 +554,7 @@ }, "emit_table_proposals": { "type": "boolean", - "description": "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." + "description": "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. A narrowly qualifying captioned three-rule table may also include a draft arrangement and header-neutral HTML preview marked needs_review; its row/column grouping and header roles are unverified. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, "allow_headerless_tables": { "type": "boolean", diff --git a/test/markdown-conversion.test.js b/test/markdown-conversion.test.js index c1a397b..c84789f 100644 --- a/test/markdown-conversion.test.js +++ b/test/markdown-conversion.test.js @@ -15,6 +15,13 @@ import { validateMarkdownConversionSemantics, } from "../server/markdown-conversion.js"; +function canonicalCandidate(value) { + if (Array.isArray(value)) return `[${value.map(canonicalCandidate).join(",")}]`; + if (value && typeof value === "object") return `{${Object.keys(value).sort() + .map(key => `${JSON.stringify(key)}:${canonicalCandidate(value[key])}`).join(",")}}`; + return JSON.stringify(value); +} + function multiply(left, right) { return [ left[0] * right[0] + left[2] * right[1], @@ -471,12 +478,12 @@ describe("layout Markdown renderer", () => { textItem("Table 1. Example", { top: 88, left: 50, fontSize: 10 }), textItem("Concentration", { top: 113, left: 60, fontSize: 9 }), textItem("level", { top: 124, left: 60, fontSize: 9 }), - textItem("A", { top: 120, left: 140, fontSize: 9, eol: false }), - textItem("B", { top: 120, left: 220, fontSize: 9 }), + textItem("A", { top: 120, left: 150, fontSize: 9, eol: false }), + textItem("B", { top: 120, left: 230, fontSize: 9 }), ...["Control", "Low", "High"].flatMap((label, index) => [ textItem(label, { top: 145 + 15 * index, left: 60, fontSize: 9, eol: false }), - textItem(String(index), { top: 145 + 15 * index, left: 140, fontSize: 9, eol: false }), - textItem(String(index + 1), { top: 145 + 15 * index, left: 220, fontSize: 9 }), + textItem(String(index), { top: 145 + 15 * index, left: 150, fontSize: 9, eol: false }), + textItem(String(index + 1), { top: 145 + 15 * index, left: 230, fontSize: 9 }), ]), textItem("3. Results", { top: 205, left: 50, fontSize: 10 }), ]; @@ -496,10 +503,39 @@ describe("layout Markdown renderer", () => { "Concentration", "level", "A", "B", "Control", "0", "1", "Low", "1", "2", "High", "2", "3", ]); expect(region.header_hints.status).toBe("unavailable"); + expect(region.review_candidate).toMatchObject({ status: "needs_review", row_count: 4, column_count: 3 }); const mapped = analyzeValidatedPdfPagesForDocumentMap(layout.pages); - expect(mapped[0].table_regions).toEqual(result.table_proposal_regions); + expect(mapped[0].table_regions).toEqual(result.table_proposal_regions.map(({ review_candidate: _candidate, ...item }) => item)); expect(analyzeValidatedPdfPagesForDocumentMap(layout.pages)).toEqual(mapped); expect(() => validateMarkdownConversionSemantics(result, { layout })).not.toThrow(); + for (const mutate of [ + candidate => { candidate.topology_verified = true; }, + candidate => { candidate.cells[0].lines[0].text = "invented"; }, + candidate => { candidate.cells[0].item_ids.pop(); }, + candidate => { candidate.cells[0].item_ids.push(candidate.cells[1].item_ids[0]); }, + candidate => { candidate.html += ""; }, + candidate => { candidate.source_sha256 = "f".repeat(64); }, + candidate => { candidate.cells[0].column = 1; }, + ]) { + const changed = structuredClone(result); + const candidate = changed.table_proposal_regions.find(value => value.review_candidate).review_candidate; + mutate(candidate); + const { candidate_sha256: _digest, ...body } = candidate; + candidate.candidate_sha256 = createHash("sha256").update(canonicalCandidate(body)).digest("hex"); + expect(() => validateMarkdownConversionSemantics(changed)).toThrow(); + expect(() => validateMarkdownConversionSemantics(changed, { layout })).toThrow(); + } + // Even a content-consistent, freshly rehashed regrouping must fail the + // packet-only geometry replay as well as the complete bound-IR replay. + const regrouped = structuredClone(result); + const draft = regrouped.table_proposal_regions.find(value => value.review_candidate).review_candidate; + [draft.cells[1].item_ids, draft.cells[2].item_ids] = [draft.cells[2].item_ids, draft.cells[1].item_ids]; + [draft.cells[1].lines, draft.cells[2].lines] = [draft.cells[2].lines, draft.cells[1].lines]; + draft.html = draft.html.replace("AB", "BA"); + const { candidate_sha256: _digest, ...body } = draft; + draft.candidate_sha256 = createHash("sha256").update(canonicalCandidate(body)).digest("hex"); + expect(() => validateMarkdownConversionSemantics(regrouped)).toThrow(/retained source geometry/); + expect(() => validateMarkdownConversionSemantics(regrouped, { layout })).toThrow(/bound layout IR/); }); it("emits only inert data cells for an opted-in complete source-covered headerless grid", async () => { diff --git a/test/mcp-contract.test.js b/test/mcp-contract.test.js index 9e8b58e..09385f5 100644 --- a/test/mcp-contract.test.js +++ b/test/mcp-contract.test.js @@ -242,7 +242,8 @@ const EXAMPLE_PDF = path.join(REPO_ROOT, "example-fw9.pdf"); // neutrally described and no longer exposes arbitrary credential headers. // 2026-10-20: version the optional tight-grid/caption repair as renderer 1.23.0. // 2026-10-20: private local password entry replaces public credential inputs. -const TOOL_CONTRACT_SHA256 = "17e2067e2c014bfc77e234432af45f7711494104f2a14d5f652e42e917251128"; +// 2026-10-21: optional table packets describe draft arrangements that require review. +const TOOL_CONTRACT_SHA256 = "2239a74dcdb71387f855c49bb26bc01e52ca9a14e773be13a32d19d3af9ec2d4"; const CLOSED_READ = Object.freeze({ readOnlyHint: true, diff --git a/test/table-review-candidate.test.js b/test/table-review-candidate.test.js new file mode 100644 index 0000000..56b7ba5 --- /dev/null +++ b/test/table-review-candidate.test.js @@ -0,0 +1,162 @@ +import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { PDFDocument, StandardFonts } from "pdf-lib"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { buildTableProposalRegion } from "../server/markdown-conversion.js"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const sourceSha256 = "a".repeat(64); +function fixture() { + const entries = [ + ["Table 1. Synthetic measurements", 50, 88, 160, "caption"], + ["Concentration", 60, 114, 55, "top-1"], + ["level", 70, 126, 20, "top-2"], + ["A", 145, 120, 10, "top-3"], ["B", 225, 120, 10, "top-3"], + ...["Control", "Low", "High"].flatMap((text, index) => [ + [text, 60, 148 + index * 15, 40, `body-${index}`], + [String(index), 140, 148 + index * 15, 8, `body-${index}`], + [String(index + 1), 220, 148 + index * 15, 8, `body-${index}`], + ]), + ]; + const raw_items = entries.map(([text, x, y, width, line_id], index) => ({ + id: `item-${index}`, text, bbox: { x, y, width, height: 8 }, line_id, + reading_order_index: index, geometry_valid: true, text_kind: "non_whitespace", + })); + const lines = [...new Set(entries.map(row => row[4]))].map(id => { + const items = raw_items.filter(item => item.line_id === id); + const x = Math.min(...items.map(item => item.bbox.x)); + const y = Math.min(...items.map(item => item.bbox.y)); + return { id, x, y, width: Math.max(...items.map(item => item.bbox.x + item.bbox.width)) - x, + height: 8, text: items.map(item => item.text).join(" "), item_ids: items.map(item => item.id) }; + }); + const bars = [110, 140, 195].flatMap(y => [ + { x: 50, y, width: 120, height: 1, verb: "fill" }, + { x: 170, y, width: 100, height: 1, verb: "fill" }, + ]); + return { page: { page: 1, raw_items, lines, truncation: { truncated: false }, + ruled_rects: { status: "available", observed_count: 6, returned_count: 6, items: bars }, + ruling_segments: { status: "available", truncated: false, observed_count: 0, returned_count: 0, items: [] }, + painted_rectangles: { status: "available", truncated: false, observed_count: 0, returned_count: 0, items: [] } }, + run: lines.slice(3).map(line => ({ line, cells: raw_items.filter(item => item.line_id === line.id) })) }; +} +function packet(input, sha = sourceSha256) { + return buildTableProposalRegion(input.page, input.run, "topology", 1, { sourceSha256: sha }); +} + +describe("three-rule table arrangement for review", () => { + it("produces one deterministic complete draft with multiline labels and no header claim", () => { + const input = fixture(); + const before = structuredClone(input); + const result = packet(input); + const candidate = result.review_candidate; + expect(candidate).toMatchObject({ status: "needs_review", topology_verified: false, + header_roles_verified: false, row_count: 4, column_count: 3, + source_item_count: 13, assigned_item_count: 13, unassigned_item_count: 0, source_sha256: sourceSha256 }); + expect(candidate.cells[0].lines.map(line => line.text)).toEqual(["Concentration", "level"]); + expect(candidate.html).toContain("Concentration
level"); + expect(candidate.html).not.toContain(" cell.item_ids).sort()) + .toEqual(result.text_items.map(item => item.id).sort()); + expect(packet(input)).toEqual(result); + expect(input).toEqual(before); + expect(packet(input, "b".repeat(64)).review_candidate.candidate_sha256).not.toBe(candidate.candidate_sha256); + expect(buildTableProposalRegion(input.page, input.run, "topology", 1)).not.toHaveProperty("review_candidate"); + }); + + const attacks = { + missing_caption: input => { input.page.lines[0].text = "Ordinary paragraph"; }, + competing_paint: input => { + input.page.ruled_rects.items.push({ x: 50, y: 160, width: 220, height: 1, verb: "fill" }); + input.page.ruled_rects.observed_count++; input.page.ruled_rects.returned_count++; + }, + truncated_text: input => { input.page.truncation.truncated = true; }, + truncated_geometry: input => { input.page.ruled_rects.status = "truncated"; }, + overlapping_body_rows: input => { + for (const item of input.page.raw_items.filter(item => item.line_id === "body-1")) item.bbox.y = 152; + }, + wrapped_body_cell: input => { input.page.raw_items.find(item => item.text === "Low").line_id = "wrapped-body"; }, + missing_cell: input => { + const item = input.page.raw_items.find(item => item.line_id === "body-1" && item.bbox.x === 220); + item.bbox.x = 142; + }, + spanning_top_cell: input => { input.page.raw_items[1].bbox.width = 120; }, + no_common_corridor: input => { + const item = input.page.raw_items.find(item => item.line_id === "body-1" && item.bbox.x === 140); + item.bbox.x = 90; item.bbox.width = 120; + }, + oversized_preview: input => { input.page.raw_items[1].text = "X".repeat(40_001); }, + }; + it.each(Object.keys(attacks))("keeps evidence without a draft for %s", name => { + const input = fixture(); attacks[name](input); + expect(packet(input)).not.toHaveProperty("review_candidate"); + }); + + it("escapes PDF markup and preserves exact source fragments", () => { + const input = fixture(); + input.page.raw_items[1].text = ' & https://evil.test'; + const result = packet(input); + const candidate = result.review_candidate; + expect(candidate.cells[0].lines[0].text).toBe(input.page.raw_items[1].text); + expect(candidate.html).toContain("<img"); + expect(candidate.html).toContain("https://evil.test"); + expect(candidate.html).not.toContain(" { + let directory, pdfPath, client, transport; + beforeAll(async () => { + directory = await mkdtemp(path.join(os.tmpdir(), "pdf-tools-table-review-")); + pdfPath = path.join(directory, "synthetic.pdf"); + const pdf = await PDFDocument.create(); + const page = pdf.addPage([612, 792]); + const font = await pdf.embedFont(StandardFonts.Helvetica); + const draw = (text, x, top) => page.drawText(text, { x, y: 792 - top - 9, size: 9, font }); + draw("Table 1. Synthetic measurements", 50, 88); + draw("Concentration", 60, 114); draw("level", 70, 126); + draw("A", 150, 120); draw("B", 230, 120); + for (const [index, label] of ["Control", "Low", "High"].entries()) { + draw(label, 60, 148 + index * 15); draw(String(index), 150, 148 + index * 15); + draw(String(index + 1), 230, 148 + index * 15); + } + draw("3. Results", 50, 210); + for (const y of [110, 140, 195]) page.drawRectangle({ x: 50, y: 792 - y - 1, width: 220, height: 1 }); + await writeFile(pdfPath, await pdf.save(), { flag: "wx" }); + transport = new StdioClientTransport({ command: process.execPath, + args: [path.join(root, "server/index.js")], cwd: root, + env: { ALLOWED_DIRECTORIES: directory }, stderr: "ignore" }); + client = new Client({ name: "table-review-test", version: "1.0.0" }); + await client.connect(transport); + }, 30000); + afterAll(async () => { await client?.close(); await transport?.close(); await rm(directory, { recursive: true, force: true }); }); + + it("exposes the same draft and warning in text and structure while strict verification rejects", async () => { + const baseline = await client.callTool({ name: "convert_pdf_to_markdown", arguments: { pdf_path: pdfPath } }); + const result = await client.callTool({ name: "convert_pdf_to_markdown", arguments: { + pdf_path: pdfPath, emit_table_proposals: true, + } }); + expect(result.isError).not.toBe(true); + const region = result.structuredContent.table_proposals.find(item => item.review_candidate); + expect(region).toBeDefined(); + const candidate = region.review_candidate; + expect(candidate).toMatchObject({ row_count: 4, column_count: 3, topology_verified: false }); + expect(result.content[0].text).toContain(candidate.review_notice); + expect(result.content[0].text).toContain(candidate.html); + expect(result.structuredContent.markdown).toBe(baseline.structuredContent.markdown); + expect(result.structuredContent.gaps).toEqual(baseline.structuredContent.gaps); + const verification = await client.callTool({ name: "verify_table_proposal", arguments: { + pdf_path: pdfPath, region_id: region.region_id, proposal_token: region.proposal_token, + cells: candidate.cells.map(cell => ({ row: cell.row, column: cell.column, + rowspan: 1, colspan: 1, item_ids: cell.item_ids })), + } }); + expect(verification.isError).not.toBe(true); + expect(verification.structuredContent.status).toBe("rejected"); + expect(verification.structuredContent.table).toBeNull(); + expect(verification.structuredContent.reason_codes).toContain("TABLE_PROPOSAL_TOPOLOGY_AMBIGUOUS"); + expect(verification.structuredContent).not.toHaveProperty("review_candidate"); + }, 30000); +});