From d4c4aaed21772451bc7e290144d0100eecad0232 Mon Sep 17 00:00:00 2001 From: silverstein Date: Wed, 21 Oct 2026 13:00:00 +0000 Subject: [PATCH 1/6] Offer source-accounted three-rule table drafts for review --- pdf-toolkit-mcp-share/server/index.js | 5 + .../server/markdown-conversion.js | 182 +++++++++++++++++- .../server/output-schemas.js | 23 ++- server/index.js | 5 + server/markdown-conversion.js | 182 +++++++++++++++++- server/output-schemas.js | 23 ++- test/markdown-conversion.test.js | 16 +- test/table-review-candidate.test.js | 163 ++++++++++++++++ 8 files changed, 582 insertions(+), 17 deletions(-) create mode 100644 test/table-review-candidate.test.js diff --git a/pdf-toolkit-mcp-share/server/index.js b/pdf-toolkit-mcp-share/server/index.js index b6ff513..2bcc831 100755 --- a/pdf-toolkit-mcp-share/server/index.js +++ b/pdf-toolkit-mcp-share/server/index.js @@ -6082,6 +6082,7 @@ async function handleToolCall(request) { saved_output: savedOutput, ...(tableProposals !== undefined ? { table_proposals: tableProposals } : {}), }; + const reviewCandidates = tableProposals?.filter(region => region.review_candidate) ?? []; const summary = [ `Converted pages ${layout.page_range.start_page}-${layout.page_range.end_page} of ${fileName} to deterministic Markdown.`, `Status: ${rendered.conversion_status}. UTF-8 bytes: ${rendered.markdown_bytes}.`, @@ -6093,6 +6094,10 @@ async function handleToolCall(request) { : []), "No OCR, external model, hidden link-target recovery, or table inference beyond complete source-bound text, ruled, or painted grids was performed.", rendered.markdown, + ...reviewCandidates.map(region => [ + `Page ${region.page}, ${region.region_id}: ${region.review_candidate.review_notice}`, + region.review_candidate.html, + ].join("\n\n")), ].join("\n\n"); return { content: [{ type: "text", text: summary }], diff --git a/pdf-toolkit-mcp-share/server/markdown-conversion.js b/pdf-toolkit-mcp-share/server/markdown-conversion.js index 324092d..7855dfa 100644 --- a/pdf-toolkit-mcp-share/server/markdown-conversion.js +++ b/pdf-toolkit-mcp-share/server/markdown-conversion.js @@ -60,6 +60,7 @@ export const MAX_TABLE_PROPOSAL_TEXT_ITEMS = 400; const MAX_TABLE_PROPOSAL_RULED_RECTS = 400; const MAX_TABLE_PROPOSAL_RULING_SEGMENTS = 400; const MAX_TABLE_PROPOSAL_PAINTED_RECTS = 400; +export const TABLE_REVIEW_NOTICE = "Proposed arrangement, needs review. Check the row and column grouping against the PDF before using it; headers and cell structure are unverified."; const MAX_MARKDOWN_BYTES_LIMIT = 200_000; const GAP_CODES = new Set([ @@ -2888,17 +2889,106 @@ function threeRuleProposalEnvelope(page, run) { || !lines.some(line => line.y + line.height <= middle.y) || !lines.some(line => line.y >= middle.y + middle.height)) continue; const itemById = new Map(page.raw_items.map(item => [item.id, item])); - candidates.push({ bbox, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) }); + candidates.push({ bbox, middle, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) }); } return candidates.length === 1 ? candidates[0] : null; } +function canonicalReviewCandidate(value) { + if (Array.isArray(value)) return `[${value.map(canonicalReviewCandidate).join(",")}]`; + if (value && typeof value === "object") return `{${Object.keys(value).sort() + .map(key => `${JSON.stringify(key)}:${canonicalReviewCandidate(value[key])}`).join(",")}}`; + return JSON.stringify(value); +} + +function threeRuleReviewCandidate(region, envelope, sourceSha256) { + if (!/^[a-f0-9]{64}$/.test(sourceSha256 ?? "") + || Object.values(region.truncation).some(value => value !== "complete")) return null; + const items = region.text_items; + if (items.length === 0 || new Set(items.map(item => item.id)).size !== items.length) return null; + const { middle } = envelope; + const upper = items.filter(item => item.bbox.y + item.bbox.height <= middle.y); + const lower = items.filter(item => item.bbox.y >= middle.y + middle.height); + if (upper.length === 0 || upper.length + lower.length !== items.length) return null; + const byLine = new Map(); + for (const item of lower) { + if (!byLine.has(item.line_id)) byLine.set(item.line_id, []); + byLine.get(item.line_id).push(item); + } + const rows = [...byLine.values()].map(row => ({ + items: row.sort((a, b) => a.bbox.x - b.bbox.x || a.reading_order_index - b.reading_order_index), + top: Math.min(...row.map(item => item.bbox.y)), + bottom: Math.max(...row.map(item => item.bbox.y + item.bbox.height)), + })).sort((a, b) => a.top - b.top); + if (rows.length < 2 || rows.length > 99 || rows.some((row, index) => ( + index > 0 && row.top <= rows[index - 1].bottom + ))) return null; + // A maximal set of whitespace corridors shared by every body line supplies + // one deterministic proposed partition. Whitespace does not prove topology. + const minGap = Math.max(5, ...lower.map(item => item.bbox.height)); + let corridors = null; + for (const row of rows) { + const gaps = []; + let right = row.items[0].bbox.x + row.items[0].bbox.width; + for (const item of row.items.slice(1)) { + if (item.bbox.x - right >= minGap) gaps.push([right, item.bbox.x]); + right = Math.max(right, item.bbox.x + item.bbox.width); + } + corridors = corridors === null ? gaps : corridors.flatMap(([left, end]) => gaps + .map(([start, rightEdge]) => [Math.max(left, start), Math.min(end, rightEdge)]) + .filter(([start, rightEdge]) => rightEdge - start >= minGap)); + } + if (corridors.length < 1 || corridors.length > 11) return null; + const cuts = corridors.map(([left, right]) => (left + right) / 2); + const sourceRows = [upper, ...rows.map(row => row.items)]; + const cells = []; + for (const [row, sourceItems] of sourceRows.entries()) { + const columns = Array.from({ length: cuts.length + 1 }, () => []); + for (const item of sourceItems) { + const column = cuts.filter(cut => item.bbox.x >= cut).length; + if (column < cuts.length && item.bbox.x + item.bbox.width > cuts[column]) return null; + columns[column].push(item); + } + if (columns.some(column => column.length === 0)) return null; + for (const [column, assigned] of columns.entries()) { + assigned.sort((a, b) => a.reading_order_index - b.reading_order_index); + const lines = []; + for (const item of assigned) { + if (lines.at(-1)?.line_id !== item.line_id) { + if (lines.some(line => line.line_id === item.line_id)) return null; + lines.push({ line_id: item.line_id, item_ids: [], text: "" }); + } + lines.at(-1).item_ids.push(item.id); + lines.at(-1).text += item.text; + } + cells.push({ row, column, item_ids: assigned.map(item => item.id), lines }); + } + } + const assigned = cells.flatMap(cell => cell.item_ids); + if (new Set(assigned).size !== items.length || assigned.length !== items.length) return null; + const grid = sourceRows.map((_, row) => cells.filter(cell => cell.row === row) + .map(cell => cell.lines.map(line => escapeHeaderNeutralText(line.text)).join("
"))); + const html = ["", ...grid.map(row => `${row.map(text => ``).join("")}`), "
${text}
"].join("\n"); + const candidate = { + algorithm: { name: "pdf-tools.three-rule-review-candidate", version: "0.1.0" }, + status: "needs_review", topology_verified: false, header_roles_verified: false, + source_sha256: sourceSha256, region_id: region.region_id, page: region.page, + content_origin: "source_text_items", structure_origin: "whitespace_alignment_proposal", + row_count: sourceRows.length, column_count: cuts.length + 1, + source_item_count: items.length, assigned_item_count: assigned.length, unassigned_item_count: 0, + cells, html, html_format: "header_neutral_html", html_bytes: utf8Bytes(html), + review_notice: TABLE_REVIEW_NOTICE, + }; + if (candidate.html_bytes > 20_000 || utf8Bytes(canonicalReviewCandidate(candidate)) > 40_000) return null; + return { ...candidate, candidate_sha256: sha256(canonicalReviewCandidate(candidate)) }; +} + /** * Build a bounded proposal from an abandoned run. A uniquely captioned * three-rule envelope can refine source membership but supplies no topology * or header authority. Pure over source-validated IR; no token, time or I/O. */ -export function buildTableProposalRegion(page, run, reason, ordinal) { +export function buildTableProposalRegion(page, run, reason, ordinal, { sourceSha256 = null } = {}) { const envelope = threeRuleProposalEnvelope(page, run); if (envelope) run = envelope.run; const itemById = new Map(page.raw_items.map(item => [item.id, item])); @@ -2962,7 +3052,7 @@ export function buildTableProposalRegion(page, run, reason, ordinal) { bbox: painted.bbox, })); - return { + const proposal = { region_id: `p${page.page}-t${ordinal}`, page: page.page, reason: regionGapCode(reason), @@ -2983,6 +3073,11 @@ export function buildTableProposalRegion(page, run, reason, ordinal) { painted_rectangles: paintedTruncated ? "truncated" : "complete", }, }; + if (envelope) { + const candidate = threeRuleReviewCandidate(proposal, envelope, sourceSha256); + if (candidate) proposal.review_candidate = candidate; + } + return proposal; } // escapePlainMarkdown already escapes "|" (and backslashes before it), so a @@ -3007,15 +3102,18 @@ function renderTable(grid) { return lines; } -function renderHeaderlessTable(grid) { - const escape = value => sanitizeUnsafeText(String(value)).replace(/&/gu, "&") +function escapeHeaderNeutralText(value) { + return sanitizeUnsafeText(String(value)).replace(/&/gu, "&") .replace(//gu, ">") .replace(/"/gu, """).replace(/'/gu, "'") .replace(/https?:/giu, match => `${match.slice(0, -1)}:`) .replace(/\b(?:[\p{L}\p{N}-]+\.)+[\p{L}]{2,}\b/giu, domain => domain.replace(".", ".")) .replace(/@(?=[\p{L}\p{N}.-]+.[\p{L}]{2,})/giu, "@"); +} + +function renderHeaderlessTable(grid) { return ["", ...grid.map(row => ( - `${row.map(value => ``).join("")}` + `${row.map(value => ``).join("")}` )), "
${escape(value)}
${escapeHeaderNeutralText(value)}
"]; } @@ -3370,6 +3468,7 @@ function renderPage(page, { pageBoundaryBefore = false, pageBoundaryAfter = false, collectTableProposals = false, + sourceSha256 = null, allowHeaderlessTables = false, pageFurniture = [], } = {}) { @@ -3488,7 +3587,7 @@ function renderPage(page, { // markdown, gaps, or byte counts. It is only present when the opt-in flag // is set, so the default page shape is byte-identical. rendered.tableProposalRegions = analysis.regions.map( - (region, index) => buildTableProposalRegion(page, region.run, region.reason, index + 1), + (region, index) => buildTableProposalRegion(page, region.run, region.reason, index + 1, { sourceSha256 }), ); } return rendered; @@ -3706,6 +3805,28 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = if (proposals !== undefined) { validateTableProposals(proposals, { requireToken: payloadProposals !== undefined }); validateTableProposalDocumentTruncation(result.table_proposals_truncation, proposals); + if (layout !== null) { + const expected = boundTableProposalRegions(layout.pages.flatMap(page => ( + segmentPageLines(page, allowHeaderlessTables).regions.map((region, index) => ( + buildTableProposalRegion(page, region.run, region.reason, index + 1, + { sourceSha256: layout.source.sha256 }) + )) + ))); + const actual = proposals.map(({ proposal_token: _token, ...proposal }) => proposal); + assertion(sameJson(actual, expected.regions), "table proposals do not match the bound layout IR"); + assertion(sameJson(result.table_proposals_truncation, expected.truncation), + "table proposal truncation does not match the bound layout IR"); + } + for (const proposal of proposals) { + if (proposal.review_candidate) { + const { candidate_sha256: digest, ...candidate } = proposal.review_candidate; + assertion(candidate.source_sha256 === result.provenance.source.sha256 + && candidate.region_id === proposal.region_id && candidate.page === proposal.page, + "table review candidate source binding mismatch"); + assertion(digest === sha256(canonicalReviewCandidate(candidate)), "table review candidate digest mismatch"); + validateTableReviewCandidate(proposal, candidate); + } + } } else { assertion(result.table_proposals_truncation === undefined, "table proposal truncation cannot exist without proposals"); @@ -3713,6 +3834,52 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = return result; } +function validateTableReviewCandidate(region, candidate) { + assertion(candidate.algorithm?.name === "pdf-tools.three-rule-review-candidate" + && candidate.algorithm.version === "0.1.0" && candidate.status === "needs_review" + && candidate.topology_verified === false && candidate.header_roles_verified === false + && candidate.content_origin === "source_text_items" + && candidate.structure_origin === "whitespace_alignment_proposal" + && candidate.review_notice === TABLE_REVIEW_NOTICE, + "table review candidate claim boundary mismatch"); + assertion(Object.values(region.truncation).every(value => value === "complete"), + "table review candidate cannot use truncated evidence"); + assertion(Number.isSafeInteger(candidate.row_count) && candidate.row_count >= 3 && candidate.row_count <= 100 + && Number.isSafeInteger(candidate.column_count) && candidate.column_count >= 2 && candidate.column_count <= 12 + && candidate.cells?.length === candidate.row_count * candidate.column_count, + "table review candidate dimensions mismatch"); + const itemById = new Map(region.text_items.map(item => [item.id, item])); + const assigned = []; + for (const [index, cell] of candidate.cells.entries()) { + assertion(cell.row === Math.floor(index / candidate.column_count) + && cell.column === index % candidate.column_count && cell.item_ids.length > 0, + "table review candidate cell order mismatch"); + const lines = []; + for (const id of cell.item_ids) { + const item = itemById.get(id); + assertion(item !== undefined, "table review candidate unknown source item"); + assigned.push(id); + if (lines.at(-1)?.line_id !== item.line_id) { + lines.push({ line_id: item.line_id, item_ids: [], text: "" }); + } + lines.at(-1).item_ids.push(id); + lines.at(-1).text += item.text; + } + assertion(sameJson(cell.lines, lines), "table review candidate content mismatch"); + } + assertion(assigned.length === itemById.size && new Set(assigned).size === itemById.size + && candidate.source_item_count === itemById.size && candidate.assigned_item_count === assigned.length + && candidate.unassigned_item_count === 0, "table review candidate source coverage mismatch"); + const grid = Array.from({ length: candidate.row_count }, (_, row) => candidate.cells + .filter(cell => cell.row === row).map(cell => cell.lines + .map(line => escapeHeaderNeutralText(line.text)).join("
"))); + const html = ["", ...grid.map(row => `${row.map(text => ``).join("")}`), "
${text}
"].join("\n"); + assertion(candidate.html === html && candidate.html_format === "header_neutral_html" + && candidate.html_bytes === utf8Bytes(html) && candidate.html_bytes <= 20_000 + && utf8Bytes(canonicalReviewCandidate(candidate)) <= 40_000, + "table review candidate presentation mismatch"); +} + function isFiniteNumber(value) { return typeof value === "number" && Number.isFinite(value); } @@ -3985,6 +4152,7 @@ export function renderPdfLayoutToMarkdown(layout, { // number from a single-page selection. pageBoundaryAfter: true, collectTableProposals: emitTableProposals, + sourceSha256: layout.source.sha256, allowHeaderlessTables, pageFurniture: furniturePlans.get(page.page), })); diff --git a/pdf-toolkit-mcp-share/server/output-schemas.js b/pdf-toolkit-mcp-share/server/output-schemas.js index 8d11e92..3c984ec 100644 --- a/pdf-toolkit-mcp-share/server/output-schemas.js +++ b/pdf-toolkit-mcp-share/server/output-schemas.js @@ -1,6 +1,6 @@ import { AjvJsonSchemaValidator } from "@modelcontextprotocol/server/validators/ajv"; import { validatePdfLayoutSemantics } from "./layout-extraction.js"; -import { validateMarkdownConversionSemantics } from "./markdown-conversion.js"; +import { TABLE_REVIEW_NOTICE, validateMarkdownConversionSemantics } from "./markdown-conversion.js"; import { validatePdfObservationSemantics, validateRenderObservationSemantics, @@ -1172,6 +1172,27 @@ const tableProposal = object({ truncation: tableProposalTruncation, proposal_token: sha256Digest, }); +// Additive only on opt-in proposal packets. Strict table acceptance is separate. +tableProposal.properties.review_candidate = object({ + algorithm: object({ name: { const: "pdf-tools.three-rule-review-candidate" }, version: { const: "0.1.0" } }), + status: { const: "needs_review" }, topology_verified: { const: false }, header_roles_verified: { const: false }, + source_sha256: sha256Digest, region_id: string, page: { type: "integer", minimum: 1 }, + content_origin: { const: "source_text_items" }, structure_origin: { const: "whitespace_alignment_proposal" }, + row_count: { type: "integer", minimum: 3, maximum: 100 }, + column_count: { type: "integer", minimum: 2, maximum: 12 }, + source_item_count: { type: "integer", minimum: 1, maximum: 400 }, + assigned_item_count: { type: "integer", minimum: 1, maximum: 400 }, unassigned_item_count: { const: 0 }, + cells: { type: "array", minItems: 6, maxItems: 400, items: object({ + row: { type: "integer", minimum: 0, maximum: 99 }, column: { type: "integer", minimum: 0, maximum: 11 }, + item_ids: { type: "array", minItems: 1, maxItems: 400, items: string }, + lines: { type: "array", minItems: 1, maxItems: 400, items: object({ + line_id: string, item_ids: { type: "array", minItems: 1, maxItems: 400, items: string }, text: string, + }) }, + }) }, + html: string, html_format: { const: "header_neutral_html" }, + html_bytes: { type: "integer", minimum: 1, maximum: 20000 }, + review_notice: { const: TABLE_REVIEW_NOTICE }, candidate_sha256: sha256Digest, +}); const tableProposalCheckStatus = enumString(["passed", "failed", "not_run"]); const verifiedTableCell = object({ row: { type: "integer", minimum: 0 }, diff --git a/server/index.js b/server/index.js index b6ff513..2bcc831 100755 --- a/server/index.js +++ b/server/index.js @@ -6082,6 +6082,7 @@ async function handleToolCall(request) { saved_output: savedOutput, ...(tableProposals !== undefined ? { table_proposals: tableProposals } : {}), }; + const reviewCandidates = tableProposals?.filter(region => region.review_candidate) ?? []; const summary = [ `Converted pages ${layout.page_range.start_page}-${layout.page_range.end_page} of ${fileName} to deterministic Markdown.`, `Status: ${rendered.conversion_status}. UTF-8 bytes: ${rendered.markdown_bytes}.`, @@ -6093,6 +6094,10 @@ async function handleToolCall(request) { : []), "No OCR, external model, hidden link-target recovery, or table inference beyond complete source-bound text, ruled, or painted grids was performed.", rendered.markdown, + ...reviewCandidates.map(region => [ + `Page ${region.page}, ${region.region_id}: ${region.review_candidate.review_notice}`, + region.review_candidate.html, + ].join("\n\n")), ].join("\n\n"); return { content: [{ type: "text", text: summary }], diff --git a/server/markdown-conversion.js b/server/markdown-conversion.js index 324092d..7855dfa 100644 --- a/server/markdown-conversion.js +++ b/server/markdown-conversion.js @@ -60,6 +60,7 @@ export const MAX_TABLE_PROPOSAL_TEXT_ITEMS = 400; const MAX_TABLE_PROPOSAL_RULED_RECTS = 400; const MAX_TABLE_PROPOSAL_RULING_SEGMENTS = 400; const MAX_TABLE_PROPOSAL_PAINTED_RECTS = 400; +export const TABLE_REVIEW_NOTICE = "Proposed arrangement, needs review. Check the row and column grouping against the PDF before using it; headers and cell structure are unverified."; const MAX_MARKDOWN_BYTES_LIMIT = 200_000; const GAP_CODES = new Set([ @@ -2888,17 +2889,106 @@ function threeRuleProposalEnvelope(page, run) { || !lines.some(line => line.y + line.height <= middle.y) || !lines.some(line => line.y >= middle.y + middle.height)) continue; const itemById = new Map(page.raw_items.map(item => [item.id, item])); - candidates.push({ bbox, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) }); + candidates.push({ bbox, middle, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) }); } return candidates.length === 1 ? candidates[0] : null; } +function canonicalReviewCandidate(value) { + if (Array.isArray(value)) return `[${value.map(canonicalReviewCandidate).join(",")}]`; + if (value && typeof value === "object") return `{${Object.keys(value).sort() + .map(key => `${JSON.stringify(key)}:${canonicalReviewCandidate(value[key])}`).join(",")}}`; + return JSON.stringify(value); +} + +function threeRuleReviewCandidate(region, envelope, sourceSha256) { + if (!/^[a-f0-9]{64}$/.test(sourceSha256 ?? "") + || Object.values(region.truncation).some(value => value !== "complete")) return null; + const items = region.text_items; + if (items.length === 0 || new Set(items.map(item => item.id)).size !== items.length) return null; + const { middle } = envelope; + const upper = items.filter(item => item.bbox.y + item.bbox.height <= middle.y); + const lower = items.filter(item => item.bbox.y >= middle.y + middle.height); + if (upper.length === 0 || upper.length + lower.length !== items.length) return null; + const byLine = new Map(); + for (const item of lower) { + if (!byLine.has(item.line_id)) byLine.set(item.line_id, []); + byLine.get(item.line_id).push(item); + } + const rows = [...byLine.values()].map(row => ({ + items: row.sort((a, b) => a.bbox.x - b.bbox.x || a.reading_order_index - b.reading_order_index), + top: Math.min(...row.map(item => item.bbox.y)), + bottom: Math.max(...row.map(item => item.bbox.y + item.bbox.height)), + })).sort((a, b) => a.top - b.top); + if (rows.length < 2 || rows.length > 99 || rows.some((row, index) => ( + index > 0 && row.top <= rows[index - 1].bottom + ))) return null; + // A maximal set of whitespace corridors shared by every body line supplies + // one deterministic proposed partition. Whitespace does not prove topology. + const minGap = Math.max(5, ...lower.map(item => item.bbox.height)); + let corridors = null; + for (const row of rows) { + const gaps = []; + let right = row.items[0].bbox.x + row.items[0].bbox.width; + for (const item of row.items.slice(1)) { + if (item.bbox.x - right >= minGap) gaps.push([right, item.bbox.x]); + right = Math.max(right, item.bbox.x + item.bbox.width); + } + corridors = corridors === null ? gaps : corridors.flatMap(([left, end]) => gaps + .map(([start, rightEdge]) => [Math.max(left, start), Math.min(end, rightEdge)]) + .filter(([start, rightEdge]) => rightEdge - start >= minGap)); + } + if (corridors.length < 1 || corridors.length > 11) return null; + const cuts = corridors.map(([left, right]) => (left + right) / 2); + const sourceRows = [upper, ...rows.map(row => row.items)]; + const cells = []; + for (const [row, sourceItems] of sourceRows.entries()) { + const columns = Array.from({ length: cuts.length + 1 }, () => []); + for (const item of sourceItems) { + const column = cuts.filter(cut => item.bbox.x >= cut).length; + if (column < cuts.length && item.bbox.x + item.bbox.width > cuts[column]) return null; + columns[column].push(item); + } + if (columns.some(column => column.length === 0)) return null; + for (const [column, assigned] of columns.entries()) { + assigned.sort((a, b) => a.reading_order_index - b.reading_order_index); + const lines = []; + for (const item of assigned) { + if (lines.at(-1)?.line_id !== item.line_id) { + if (lines.some(line => line.line_id === item.line_id)) return null; + lines.push({ line_id: item.line_id, item_ids: [], text: "" }); + } + lines.at(-1).item_ids.push(item.id); + lines.at(-1).text += item.text; + } + cells.push({ row, column, item_ids: assigned.map(item => item.id), lines }); + } + } + const assigned = cells.flatMap(cell => cell.item_ids); + if (new Set(assigned).size !== items.length || assigned.length !== items.length) return null; + const grid = sourceRows.map((_, row) => cells.filter(cell => cell.row === row) + .map(cell => cell.lines.map(line => escapeHeaderNeutralText(line.text)).join("
"))); + const html = ["", ...grid.map(row => `${row.map(text => ``).join("")}`), "
${text}
"].join("\n"); + const candidate = { + algorithm: { name: "pdf-tools.three-rule-review-candidate", version: "0.1.0" }, + status: "needs_review", topology_verified: false, header_roles_verified: false, + source_sha256: sourceSha256, region_id: region.region_id, page: region.page, + content_origin: "source_text_items", structure_origin: "whitespace_alignment_proposal", + row_count: sourceRows.length, column_count: cuts.length + 1, + source_item_count: items.length, assigned_item_count: assigned.length, unassigned_item_count: 0, + cells, html, html_format: "header_neutral_html", html_bytes: utf8Bytes(html), + review_notice: TABLE_REVIEW_NOTICE, + }; + if (candidate.html_bytes > 20_000 || utf8Bytes(canonicalReviewCandidate(candidate)) > 40_000) return null; + return { ...candidate, candidate_sha256: sha256(canonicalReviewCandidate(candidate)) }; +} + /** * Build a bounded proposal from an abandoned run. A uniquely captioned * three-rule envelope can refine source membership but supplies no topology * or header authority. Pure over source-validated IR; no token, time or I/O. */ -export function buildTableProposalRegion(page, run, reason, ordinal) { +export function buildTableProposalRegion(page, run, reason, ordinal, { sourceSha256 = null } = {}) { const envelope = threeRuleProposalEnvelope(page, run); if (envelope) run = envelope.run; const itemById = new Map(page.raw_items.map(item => [item.id, item])); @@ -2962,7 +3052,7 @@ export function buildTableProposalRegion(page, run, reason, ordinal) { bbox: painted.bbox, })); - return { + const proposal = { region_id: `p${page.page}-t${ordinal}`, page: page.page, reason: regionGapCode(reason), @@ -2983,6 +3073,11 @@ export function buildTableProposalRegion(page, run, reason, ordinal) { painted_rectangles: paintedTruncated ? "truncated" : "complete", }, }; + if (envelope) { + const candidate = threeRuleReviewCandidate(proposal, envelope, sourceSha256); + if (candidate) proposal.review_candidate = candidate; + } + return proposal; } // escapePlainMarkdown already escapes "|" (and backslashes before it), so a @@ -3007,15 +3102,18 @@ function renderTable(grid) { return lines; } -function renderHeaderlessTable(grid) { - const escape = value => sanitizeUnsafeText(String(value)).replace(/&/gu, "&") +function escapeHeaderNeutralText(value) { + return sanitizeUnsafeText(String(value)).replace(/&/gu, "&") .replace(//gu, ">") .replace(/"/gu, """).replace(/'/gu, "'") .replace(/https?:/giu, match => `${match.slice(0, -1)}:`) .replace(/\b(?:[\p{L}\p{N}-]+\.)+[\p{L}]{2,}\b/giu, domain => domain.replace(".", ".")) .replace(/@(?=[\p{L}\p{N}.-]+.[\p{L}]{2,})/giu, "@"); +} + +function renderHeaderlessTable(grid) { return ["", ...grid.map(row => ( - `${row.map(value => ``).join("")}` + `${row.map(value => ``).join("")}` )), "
${escape(value)}
${escapeHeaderNeutralText(value)}
"]; } @@ -3370,6 +3468,7 @@ function renderPage(page, { pageBoundaryBefore = false, pageBoundaryAfter = false, collectTableProposals = false, + sourceSha256 = null, allowHeaderlessTables = false, pageFurniture = [], } = {}) { @@ -3488,7 +3587,7 @@ function renderPage(page, { // markdown, gaps, or byte counts. It is only present when the opt-in flag // is set, so the default page shape is byte-identical. rendered.tableProposalRegions = analysis.regions.map( - (region, index) => buildTableProposalRegion(page, region.run, region.reason, index + 1), + (region, index) => buildTableProposalRegion(page, region.run, region.reason, index + 1, { sourceSha256 }), ); } return rendered; @@ -3706,6 +3805,28 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = if (proposals !== undefined) { validateTableProposals(proposals, { requireToken: payloadProposals !== undefined }); validateTableProposalDocumentTruncation(result.table_proposals_truncation, proposals); + if (layout !== null) { + const expected = boundTableProposalRegions(layout.pages.flatMap(page => ( + segmentPageLines(page, allowHeaderlessTables).regions.map((region, index) => ( + buildTableProposalRegion(page, region.run, region.reason, index + 1, + { sourceSha256: layout.source.sha256 }) + )) + ))); + const actual = proposals.map(({ proposal_token: _token, ...proposal }) => proposal); + assertion(sameJson(actual, expected.regions), "table proposals do not match the bound layout IR"); + assertion(sameJson(result.table_proposals_truncation, expected.truncation), + "table proposal truncation does not match the bound layout IR"); + } + for (const proposal of proposals) { + if (proposal.review_candidate) { + const { candidate_sha256: digest, ...candidate } = proposal.review_candidate; + assertion(candidate.source_sha256 === result.provenance.source.sha256 + && candidate.region_id === proposal.region_id && candidate.page === proposal.page, + "table review candidate source binding mismatch"); + assertion(digest === sha256(canonicalReviewCandidate(candidate)), "table review candidate digest mismatch"); + validateTableReviewCandidate(proposal, candidate); + } + } } else { assertion(result.table_proposals_truncation === undefined, "table proposal truncation cannot exist without proposals"); @@ -3713,6 +3834,52 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = return result; } +function validateTableReviewCandidate(region, candidate) { + assertion(candidate.algorithm?.name === "pdf-tools.three-rule-review-candidate" + && candidate.algorithm.version === "0.1.0" && candidate.status === "needs_review" + && candidate.topology_verified === false && candidate.header_roles_verified === false + && candidate.content_origin === "source_text_items" + && candidate.structure_origin === "whitespace_alignment_proposal" + && candidate.review_notice === TABLE_REVIEW_NOTICE, + "table review candidate claim boundary mismatch"); + assertion(Object.values(region.truncation).every(value => value === "complete"), + "table review candidate cannot use truncated evidence"); + assertion(Number.isSafeInteger(candidate.row_count) && candidate.row_count >= 3 && candidate.row_count <= 100 + && Number.isSafeInteger(candidate.column_count) && candidate.column_count >= 2 && candidate.column_count <= 12 + && candidate.cells?.length === candidate.row_count * candidate.column_count, + "table review candidate dimensions mismatch"); + const itemById = new Map(region.text_items.map(item => [item.id, item])); + const assigned = []; + for (const [index, cell] of candidate.cells.entries()) { + assertion(cell.row === Math.floor(index / candidate.column_count) + && cell.column === index % candidate.column_count && cell.item_ids.length > 0, + "table review candidate cell order mismatch"); + const lines = []; + for (const id of cell.item_ids) { + const item = itemById.get(id); + assertion(item !== undefined, "table review candidate unknown source item"); + assigned.push(id); + if (lines.at(-1)?.line_id !== item.line_id) { + lines.push({ line_id: item.line_id, item_ids: [], text: "" }); + } + lines.at(-1).item_ids.push(id); + lines.at(-1).text += item.text; + } + assertion(sameJson(cell.lines, lines), "table review candidate content mismatch"); + } + assertion(assigned.length === itemById.size && new Set(assigned).size === itemById.size + && candidate.source_item_count === itemById.size && candidate.assigned_item_count === assigned.length + && candidate.unassigned_item_count === 0, "table review candidate source coverage mismatch"); + const grid = Array.from({ length: candidate.row_count }, (_, row) => candidate.cells + .filter(cell => cell.row === row).map(cell => cell.lines + .map(line => escapeHeaderNeutralText(line.text)).join("
"))); + const html = ["", ...grid.map(row => `${row.map(text => ``).join("")}`), "
${text}
"].join("\n"); + assertion(candidate.html === html && candidate.html_format === "header_neutral_html" + && candidate.html_bytes === utf8Bytes(html) && candidate.html_bytes <= 20_000 + && utf8Bytes(canonicalReviewCandidate(candidate)) <= 40_000, + "table review candidate presentation mismatch"); +} + function isFiniteNumber(value) { return typeof value === "number" && Number.isFinite(value); } @@ -3985,6 +4152,7 @@ export function renderPdfLayoutToMarkdown(layout, { // number from a single-page selection. pageBoundaryAfter: true, collectTableProposals: emitTableProposals, + sourceSha256: layout.source.sha256, allowHeaderlessTables, pageFurniture: furniturePlans.get(page.page), })); diff --git a/server/output-schemas.js b/server/output-schemas.js index 8d11e92..3c984ec 100644 --- a/server/output-schemas.js +++ b/server/output-schemas.js @@ -1,6 +1,6 @@ import { AjvJsonSchemaValidator } from "@modelcontextprotocol/server/validators/ajv"; import { validatePdfLayoutSemantics } from "./layout-extraction.js"; -import { validateMarkdownConversionSemantics } from "./markdown-conversion.js"; +import { TABLE_REVIEW_NOTICE, validateMarkdownConversionSemantics } from "./markdown-conversion.js"; import { validatePdfObservationSemantics, validateRenderObservationSemantics, @@ -1172,6 +1172,27 @@ const tableProposal = object({ truncation: tableProposalTruncation, proposal_token: sha256Digest, }); +// Additive only on opt-in proposal packets. Strict table acceptance is separate. +tableProposal.properties.review_candidate = object({ + algorithm: object({ name: { const: "pdf-tools.three-rule-review-candidate" }, version: { const: "0.1.0" } }), + status: { const: "needs_review" }, topology_verified: { const: false }, header_roles_verified: { const: false }, + source_sha256: sha256Digest, region_id: string, page: { type: "integer", minimum: 1 }, + content_origin: { const: "source_text_items" }, structure_origin: { const: "whitespace_alignment_proposal" }, + row_count: { type: "integer", minimum: 3, maximum: 100 }, + column_count: { type: "integer", minimum: 2, maximum: 12 }, + source_item_count: { type: "integer", minimum: 1, maximum: 400 }, + assigned_item_count: { type: "integer", minimum: 1, maximum: 400 }, unassigned_item_count: { const: 0 }, + cells: { type: "array", minItems: 6, maxItems: 400, items: object({ + row: { type: "integer", minimum: 0, maximum: 99 }, column: { type: "integer", minimum: 0, maximum: 11 }, + item_ids: { type: "array", minItems: 1, maxItems: 400, items: string }, + lines: { type: "array", minItems: 1, maxItems: 400, items: object({ + line_id: string, item_ids: { type: "array", minItems: 1, maxItems: 400, items: string }, text: string, + }) }, + }) }, + html: string, html_format: { const: "header_neutral_html" }, + html_bytes: { type: "integer", minimum: 1, maximum: 20000 }, + review_notice: { const: TABLE_REVIEW_NOTICE }, candidate_sha256: sha256Digest, +}); const tableProposalCheckStatus = enumString(["passed", "failed", "not_run"]); const verifiedTableCell = object({ row: { type: "integer", minimum: 0 }, diff --git a/test/markdown-conversion.test.js b/test/markdown-conversion.test.js index c1a397b..cdd1e23 100644 --- a/test/markdown-conversion.test.js +++ b/test/markdown-conversion.test.js @@ -496,10 +496,24 @@ describe("layout Markdown renderer", () => { "Concentration", "level", "A", "B", "Control", "0", "1", "Low", "1", "2", "High", "2", "3", ]); expect(region.header_hints.status).toBe("unavailable"); + expect(region.review_candidate).toMatchObject({ status: "needs_review", row_count: 4, column_count: 3 }); const mapped = analyzeValidatedPdfPagesForDocumentMap(layout.pages); - expect(mapped[0].table_regions).toEqual(result.table_proposal_regions); + expect(mapped[0].table_regions).toEqual(result.table_proposal_regions.map(({ review_candidate: _candidate, ...item }) => item)); expect(analyzeValidatedPdfPagesForDocumentMap(layout.pages)).toEqual(mapped); expect(() => validateMarkdownConversionSemantics(result, { layout })).not.toThrow(); + for (const mutate of [ + candidate => { candidate.topology_verified = true; }, + candidate => { candidate.cells[0].lines[0].text = "invented"; }, + candidate => { candidate.cells[0].item_ids.pop(); }, + candidate => { candidate.cells[0].item_ids.push(candidate.cells[1].item_ids[0]); }, + candidate => { candidate.html += ""; }, + candidate => { candidate.source_sha256 = "f".repeat(64); }, + candidate => { candidate.cells[0].column = 1; }, + ]) { + const changed = structuredClone(result); + mutate(changed.table_proposal_regions.find(value => value.review_candidate).review_candidate); + expect(() => validateMarkdownConversionSemantics(changed, { layout })).toThrow(); + } }); it("emits only inert data cells for an opted-in complete source-covered headerless grid", async () => { diff --git a/test/table-review-candidate.test.js b/test/table-review-candidate.test.js new file mode 100644 index 0000000..18ede87 --- /dev/null +++ b/test/table-review-candidate.test.js @@ -0,0 +1,163 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { PDFDocument, StandardFonts } from "pdf-lib"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { buildTableProposalRegion } from "../server/markdown-conversion.js"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const sourceSha256 = "a".repeat(64); +function fixture() { + const entries = [ + ["Table 1. Synthetic measurements", 50, 88, 160, "caption"], + ["Concentration", 60, 114, 55, "top-1"], + ["level", 70, 126, 20, "top-2"], + ["A", 145, 120, 10, "top-3"], ["B", 225, 120, 10, "top-3"], + ...["Control", "Low", "High"].flatMap((text, index) => [ + [text, 60, 148 + index * 15, 40, `body-${index}`], + [String(index), 140, 148 + index * 15, 8, `body-${index}`], + [String(index + 1), 220, 148 + index * 15, 8, `body-${index}`], + ]), + ]; + const raw_items = entries.map(([text, x, y, width, line_id], index) => ({ + id: `item-${index}`, text, bbox: { x, y, width, height: 8 }, line_id, + reading_order_index: index, geometry_valid: true, text_kind: "non_whitespace", + })); + const lines = [...new Set(entries.map(row => row[4]))].map(id => { + const items = raw_items.filter(item => item.line_id === id); + const x = Math.min(...items.map(item => item.bbox.x)); + const y = Math.min(...items.map(item => item.bbox.y)); + return { id, x, y, width: Math.max(...items.map(item => item.bbox.x + item.bbox.width)) - x, + height: 8, text: items.map(item => item.text).join(" "), item_ids: items.map(item => item.id) }; + }); + const bars = [110, 140, 195].flatMap(y => [ + { x: 50, y, width: 120, height: 1, verb: "fill" }, + { x: 170, y, width: 100, height: 1, verb: "fill" }, + ]); + return { page: { page: 1, raw_items, lines, truncation: { truncated: false }, + ruled_rects: { status: "available", observed_count: 6, returned_count: 6, items: bars }, + ruling_segments: { status: "available", truncated: false, observed_count: 0, returned_count: 0, items: [] }, + painted_rectangles: { status: "available", truncated: false, observed_count: 0, returned_count: 0, items: [] } }, + run: lines.slice(3).map(line => ({ line, cells: raw_items.filter(item => item.line_id === line.id) })) }; +} +function packet(input, sha = sourceSha256) { + return buildTableProposalRegion(input.page, input.run, "topology", 1, { sourceSha256: sha }); +} + +describe("three-rule table arrangement for review", () => { + it("produces one deterministic complete draft with multiline labels and no header claim", () => { + const input = fixture(); + const before = structuredClone(input); + const result = packet(input); + const candidate = result.review_candidate; + expect(candidate).toMatchObject({ status: "needs_review", topology_verified: false, + header_roles_verified: false, row_count: 4, column_count: 3, + source_item_count: 13, assigned_item_count: 13, unassigned_item_count: 0, source_sha256: sourceSha256 }); + expect(candidate.cells[0].lines.map(line => line.text)).toEqual(["Concentration", "level"]); + expect(candidate.html).toContain("Concentration
level"); + expect(candidate.html).not.toContain(" cell.item_ids).sort()) + .toEqual(result.text_items.map(item => item.id).sort()); + expect(packet(input)).toEqual(result); + expect(input).toEqual(before); + expect(packet(input, "b".repeat(64)).review_candidate.candidate_sha256).not.toBe(candidate.candidate_sha256); + expect(buildTableProposalRegion(input.page, input.run, "topology", 1)).not.toHaveProperty("review_candidate"); + }); + + const attacks = { + missing_caption: input => { input.page.lines[0].text = "Ordinary paragraph"; }, + competing_paint: input => { + input.page.ruled_rects.items.push({ x: 50, y: 160, width: 220, height: 1, verb: "fill" }); + input.page.ruled_rects.observed_count++; input.page.ruled_rects.returned_count++; + }, + truncated_text: input => { input.page.truncation.truncated = true; }, + truncated_geometry: input => { input.page.ruled_rects.status = "truncated"; }, + overlapping_body_rows: input => { + for (const item of input.page.raw_items.filter(item => item.line_id === "body-1")) item.bbox.y = 152; + }, + wrapped_body_cell: input => { input.page.raw_items.find(item => item.text === "Low").line_id = "wrapped-body"; }, + missing_cell: input => { + const item = input.page.raw_items.find(item => item.line_id === "body-1" && item.bbox.x === 220); + item.bbox.x = 142; + }, + spanning_top_cell: input => { input.page.raw_items[1].bbox.width = 120; }, + no_common_corridor: input => { + const item = input.page.raw_items.find(item => item.line_id === "body-1" && item.bbox.x === 140); + item.bbox.x = 90; item.bbox.width = 120; + }, + oversized_preview: input => { input.page.raw_items[1].text = "X".repeat(40_001); }, + }; + it.each(Object.keys(attacks))("keeps evidence without a draft for %s", name => { + const input = fixture(); attacks[name](input); + expect(packet(input)).not.toHaveProperty("review_candidate"); + }); + + it("escapes PDF markup and preserves exact source fragments", () => { + const input = fixture(); + input.page.raw_items[1].text = ' & https://evil.test'; + const result = packet(input); + const candidate = result.review_candidate; + expect(candidate.cells[0].lines[0].text).toBe(input.page.raw_items[1].text); + expect(candidate.html).toContain("<img"); + expect(candidate.html).toContain("https://evil.test"); + expect(candidate.html).not.toContain(" { + let directory, pdfPath, client, transport; + beforeAll(async () => { + directory = await mkdtemp(path.join(os.tmpdir(), "pdf-tools-table-review-")); + pdfPath = path.join(directory, "synthetic.pdf"); + const pdf = await PDFDocument.create(); + const page = pdf.addPage([612, 792]); + const font = await pdf.embedFont(StandardFonts.Helvetica); + const draw = (text, x, top) => page.drawText(text, { x, y: 792 - top - 9, size: 9, font }); + draw("Table 1. Synthetic measurements", 50, 88); + draw("Concentration", 60, 114); draw("level", 70, 126); + draw("A", 140, 120); draw("B", 220, 120); + for (const [index, label] of ["Control", "Low", "High"].entries()) { + draw(label, 60, 148 + index * 15); draw(String(index), 140, 148 + index * 15); + draw(String(index + 1), 220, 148 + index * 15); + } + draw("3. Results", 50, 210); + for (const y of [110, 140, 195]) page.drawRectangle({ x: 50, y: 792 - y - 1, width: 220, height: 1 }); + await writeFile(pdfPath, await pdf.save(), { flag: "wx" }); + transport = new StdioClientTransport({ command: process.execPath, + args: [path.join(root, "server/index.js")], cwd: root, + env: { ALLOWED_DIRECTORIES: directory }, stderr: "ignore" }); + client = new Client({ name: "table-review-test", version: "1.0.0" }); + await client.connect(transport); + }, 30000); + afterAll(async () => { await client?.close(); await transport?.close(); await rm(directory, { recursive: true, force: true }); }); + + it("exposes the same draft and warning in text and structure while strict verification rejects", async () => { + const baseline = await client.callTool({ name: "convert_pdf_to_markdown", arguments: { pdf_path: pdfPath } }); + const result = await client.callTool({ name: "convert_pdf_to_markdown", arguments: { + pdf_path: pdfPath, emit_table_proposals: true, + } }); + expect(result.isError).not.toBe(true); + const region = result.structuredContent.table_proposals.find(item => item.review_candidate); + expect(region).toBeDefined(); + const candidate = region.review_candidate; + expect(candidate).toMatchObject({ row_count: 4, column_count: 3, topology_verified: false }); + expect(result.content[0].text).toContain(candidate.review_notice); + expect(result.content[0].text).toContain(candidate.html); + expect(result.structuredContent.markdown).toBe(baseline.structuredContent.markdown); + expect(result.structuredContent.gaps).toEqual(baseline.structuredContent.gaps); + const verification = await client.callTool({ name: "verify_table_proposal", arguments: { + pdf_path: pdfPath, region_id: region.region_id, proposal_token: region.proposal_token, + cells: candidate.cells.map(cell => ({ row: cell.row, column: cell.column, + rowspan: 1, colspan: 1, item_ids: cell.item_ids })), + } }); + expect(verification.isError).not.toBe(true); + expect(verification.structuredContent.status).toBe("rejected"); + expect(verification.structuredContent.table).toBeNull(); + expect(verification.structuredContent.reason_codes).toContain("TABLE_PROPOSAL_TOPOLOGY_AMBIGUOUS"); + expect(verification.structuredContent).not.toHaveProperty("review_candidate"); + }, 30000); +}); From 506285cc3bfe162db795746e5f50cfc374c68ece Mon Sep 17 00:00:00 2001 From: silverstein Date: Wed, 21 Oct 2026 13:01:07 +0000 Subject: [PATCH 2/6] Decline inconsistent whitespace partitions and qualify whole-item fixtures --- pdf-toolkit-mcp-share/server/markdown-conversion.js | 5 ++++- server/markdown-conversion.js | 5 ++++- test/markdown-conversion.test.js | 8 ++++---- test/table-review-candidate.test.js | 7 +++---- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/pdf-toolkit-mcp-share/server/markdown-conversion.js b/pdf-toolkit-mcp-share/server/markdown-conversion.js index 7855dfa..c3aca82 100644 --- a/pdf-toolkit-mcp-share/server/markdown-conversion.js +++ b/pdf-toolkit-mcp-share/server/markdown-conversion.js @@ -2927,6 +2927,7 @@ function threeRuleReviewCandidate(region, envelope, sourceSha256) { // one deterministic proposed partition. Whitespace does not prove topology. const minGap = Math.max(5, ...lower.map(item => item.bbox.height)); let corridors = null; + let gapCount = null; for (const row of rows) { const gaps = []; let right = row.items[0].bbox.x + row.items[0].bbox.width; @@ -2934,11 +2935,13 @@ function threeRuleReviewCandidate(region, envelope, sourceSha256) { if (item.bbox.x - right >= minGap) gaps.push([right, item.bbox.x]); right = Math.max(right, item.bbox.x + item.bbox.width); } + if (gapCount !== null && gaps.length !== gapCount) return null; + gapCount = gaps.length; corridors = corridors === null ? gaps : corridors.flatMap(([left, end]) => gaps .map(([start, rightEdge]) => [Math.max(left, start), Math.min(end, rightEdge)]) .filter(([start, rightEdge]) => rightEdge - start >= minGap)); } - if (corridors.length < 1 || corridors.length > 11) return null; + if (corridors.length < 1 || corridors.length > 11 || corridors.length !== gapCount) return null; const cuts = corridors.map(([left, right]) => (left + right) / 2); const sourceRows = [upper, ...rows.map(row => row.items)]; const cells = []; diff --git a/server/markdown-conversion.js b/server/markdown-conversion.js index 7855dfa..c3aca82 100644 --- a/server/markdown-conversion.js +++ b/server/markdown-conversion.js @@ -2927,6 +2927,7 @@ function threeRuleReviewCandidate(region, envelope, sourceSha256) { // one deterministic proposed partition. Whitespace does not prove topology. const minGap = Math.max(5, ...lower.map(item => item.bbox.height)); let corridors = null; + let gapCount = null; for (const row of rows) { const gaps = []; let right = row.items[0].bbox.x + row.items[0].bbox.width; @@ -2934,11 +2935,13 @@ function threeRuleReviewCandidate(region, envelope, sourceSha256) { if (item.bbox.x - right >= minGap) gaps.push([right, item.bbox.x]); right = Math.max(right, item.bbox.x + item.bbox.width); } + if (gapCount !== null && gaps.length !== gapCount) return null; + gapCount = gaps.length; corridors = corridors === null ? gaps : corridors.flatMap(([left, end]) => gaps .map(([start, rightEdge]) => [Math.max(left, start), Math.min(end, rightEdge)]) .filter(([start, rightEdge]) => rightEdge - start >= minGap)); } - if (corridors.length < 1 || corridors.length > 11) return null; + if (corridors.length < 1 || corridors.length > 11 || corridors.length !== gapCount) return null; const cuts = corridors.map(([left, right]) => (left + right) / 2); const sourceRows = [upper, ...rows.map(row => row.items)]; const cells = []; diff --git a/test/markdown-conversion.test.js b/test/markdown-conversion.test.js index cdd1e23..fc02d12 100644 --- a/test/markdown-conversion.test.js +++ b/test/markdown-conversion.test.js @@ -471,12 +471,12 @@ describe("layout Markdown renderer", () => { textItem("Table 1. Example", { top: 88, left: 50, fontSize: 10 }), textItem("Concentration", { top: 113, left: 60, fontSize: 9 }), textItem("level", { top: 124, left: 60, fontSize: 9 }), - textItem("A", { top: 120, left: 140, fontSize: 9, eol: false }), - textItem("B", { top: 120, left: 220, fontSize: 9 }), + textItem("A", { top: 120, left: 150, fontSize: 9, eol: false }), + textItem("B", { top: 120, left: 230, fontSize: 9 }), ...["Control", "Low", "High"].flatMap((label, index) => [ textItem(label, { top: 145 + 15 * index, left: 60, fontSize: 9, eol: false }), - textItem(String(index), { top: 145 + 15 * index, left: 140, fontSize: 9, eol: false }), - textItem(String(index + 1), { top: 145 + 15 * index, left: 220, fontSize: 9 }), + textItem(String(index), { top: 145 + 15 * index, left: 150, fontSize: 9, eol: false }), + textItem(String(index + 1), { top: 145 + 15 * index, left: 230, fontSize: 9 }), ]), textItem("3. Results", { top: 205, left: 50, fontSize: 10 }), ]; diff --git a/test/table-review-candidate.test.js b/test/table-review-candidate.test.js index 18ede87..56b7ba5 100644 --- a/test/table-review-candidate.test.js +++ b/test/table-review-candidate.test.js @@ -1,4 +1,3 @@ -import { createHash } from "node:crypto"; import { mkdtemp, writeFile, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -119,10 +118,10 @@ describe("actual MCP table draft presentation", () => { const draw = (text, x, top) => page.drawText(text, { x, y: 792 - top - 9, size: 9, font }); draw("Table 1. Synthetic measurements", 50, 88); draw("Concentration", 60, 114); draw("level", 70, 126); - draw("A", 140, 120); draw("B", 220, 120); + draw("A", 150, 120); draw("B", 230, 120); for (const [index, label] of ["Control", "Low", "High"].entries()) { - draw(label, 60, 148 + index * 15); draw(String(index), 140, 148 + index * 15); - draw(String(index + 1), 220, 148 + index * 15); + draw(label, 60, 148 + index * 15); draw(String(index), 150, 148 + index * 15); + draw(String(index + 1), 230, 148 + index * 15); } draw("3. Results", 50, 210); for (const y of [110, 140, 195]) page.drawRectangle({ x: 50, y: 792 - y - 1, width: 220, height: 1 }); From e48e7381fb9806c5d98c2b0f177a7b0ae869b4cb Mon Sep 17 00:00:00 2001 From: silverstein Date: Wed, 21 Oct 2026 13:03:02 +0000 Subject: [PATCH 3/6] Document review-only table drafts and test rehashed substitutions --- docs/MAINTAINERS.md | 21 +++++++++++++++++++++ pdf-toolkit-mcp-share/server/index.js | 2 +- server/index.js | 2 +- test/markdown-conversion.test.js | 24 +++++++++++++++++++++++- 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/docs/MAINTAINERS.md b/docs/MAINTAINERS.md index 0c97446..3aa2f19 100644 --- a/docs/MAINTAINERS.md +++ b/docs/MAINTAINERS.md @@ -313,6 +313,27 @@ regions suppress header hints. The same source region is used by document maps and fresh proposal verification. Ordinary Markdown and conversion gaps do not change, and matching source/region tokens do not excuse obsolete assignments. +With `emit_table_proposals: true`, qualifying three-rule regions can also carry +`review_candidate`: a proposed arrangement with `status: needs_review`, exact +source-item assignments and line fragments, and inert HTML data cells. The +same preview and review notice appear in the text response. It groups the top +band provisionally and derives columns from whitespace corridors recurring +across every nonoverlapping body line. All region items must be assigned once, +every cell must contain source text, and the complete maximal partition must +recur on every body line. Ragged, wrapped, straddling or over-budget layouts +retain the evidence packet without a draft. Drafts are capped at 100 rows, +12 columns, 400 source items, 20,000 HTML bytes and 40,000 canonical JSON bytes. + +Rows, columns and header roles remain unverified; no header tags, spans, blank +values or new source text are inferred. Candidate digests bind algorithm, +PDF SHA, region, assignments, fragments, uncertainty and presentation. When +layout is supplied, semantic validation reconstructs the entire packet from +it. Standalone output validation checks content/digest consistency only and +does not replace fresh source replay. Document maps carry no candidate cells. +Ordinary Markdown, its gap list and strict `verify_table_proposal` acceptance +remain unchanged. Review the draft beside the source PDF before using its +grouping; a visible draft is not an accepted table or verified extraction. + Filesystem operations and rasterization happen locally, and PDF Tools does not upload files to a separate PDF service. Text, images, and metadata returned through MCP may still be processed under the selected host or model provider's diff --git a/pdf-toolkit-mcp-share/server/index.js b/pdf-toolkit-mcp-share/server/index.js index 2bcc831..13203b0 100755 --- a/pdf-toolkit-mcp-share/server/index.js +++ b/pdf-toolkit-mcp-share/server/index.js @@ -3852,7 +3852,7 @@ async function listTools(request) { include_page_boundaries: { type: "boolean", description: "Include deterministic HTML comments marking page boundaries. Default: true." }, compact: { type: "boolean", description: "Opt into counted dot-leader, isolated page-number, and Unicode-letter spaced-hyphen normalizations. Default: false." }, remove_page_furniture: { type: "boolean", description: "Remove only source-evidenced extreme-margin page furniture and report every removed line by typed gap and counts. Set false to preserve all source lines. Default: true." }, - emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, + emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. A narrowly qualifying captioned three-rule table may also include a draft arrangement and header-neutral HTML preview marked needs_review; its row/column grouping and header roles are unverified. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.23.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." }, output_path: { type: "string", description: "Optional absolute .md path, or ~/ path. The file is written only after complete bytes are staged and verified." }, overwrite: { type: "boolean", description: "Replace an existing output_path only when its exact expected_output_identity is also supplied. Default: false." }, diff --git a/server/index.js b/server/index.js index 2bcc831..13203b0 100755 --- a/server/index.js +++ b/server/index.js @@ -3852,7 +3852,7 @@ async function listTools(request) { include_page_boundaries: { type: "boolean", description: "Include deterministic HTML comments marking page boundaries. Default: true." }, compact: { type: "boolean", description: "Opt into counted dot-leader, isolated page-number, and Unicode-letter spaced-hyphen normalizations. Default: false." }, remove_page_furniture: { type: "boolean", description: "Remove only source-evidenced extreme-margin page furniture and report every removed line by typed gap and counts. Set false to preserve all source lines. Default: true." }, - emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, + emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. A narrowly qualifying captioned three-rule table may also include a draft arrangement and header-neutral HTML preview marked needs_review; its row/column grouping and header roles are unverified. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.23.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." }, output_path: { type: "string", description: "Optional absolute .md path, or ~/ path. The file is written only after complete bytes are staged and verified." }, overwrite: { type: "boolean", description: "Replace an existing output_path only when its exact expected_output_identity is also supplied. Default: false." }, diff --git a/test/markdown-conversion.test.js b/test/markdown-conversion.test.js index fc02d12..360b1bc 100644 --- a/test/markdown-conversion.test.js +++ b/test/markdown-conversion.test.js @@ -15,6 +15,13 @@ import { validateMarkdownConversionSemantics, } from "../server/markdown-conversion.js"; +function canonicalCandidate(value) { + if (Array.isArray(value)) return `[${value.map(canonicalCandidate).join(",")}]`; + if (value && typeof value === "object") return `{${Object.keys(value).sort() + .map(key => `${JSON.stringify(key)}:${canonicalCandidate(value[key])}`).join(",")}}`; + return JSON.stringify(value); +} + function multiply(left, right) { return [ left[0] * right[0] + left[2] * right[1], @@ -511,9 +518,24 @@ describe("layout Markdown renderer", () => { candidate => { candidate.cells[0].column = 1; }, ]) { const changed = structuredClone(result); - mutate(changed.table_proposal_regions.find(value => value.review_candidate).review_candidate); + const candidate = changed.table_proposal_regions.find(value => value.review_candidate).review_candidate; + mutate(candidate); + const { candidate_sha256: _digest, ...body } = candidate; + candidate.candidate_sha256 = createHash("sha256").update(canonicalCandidate(body)).digest("hex"); + expect(() => validateMarkdownConversionSemantics(changed)).toThrow(); expect(() => validateMarkdownConversionSemantics(changed, { layout })).toThrow(); } + // Swapping complete assignments and their source fragments can be internally + // consistent; only a reconstruction from the bound IR proves this algorithm's output. + const regrouped = structuredClone(result); + const draft = regrouped.table_proposal_regions.find(value => value.review_candidate).review_candidate; + [draft.cells[1].item_ids, draft.cells[2].item_ids] = [draft.cells[2].item_ids, draft.cells[1].item_ids]; + [draft.cells[1].lines, draft.cells[2].lines] = [draft.cells[2].lines, draft.cells[1].lines]; + draft.html = draft.html.replace("AB", "BA"); + const { candidate_sha256: _digest, ...body } = draft; + draft.candidate_sha256 = createHash("sha256").update(canonicalCandidate(body)).digest("hex"); + expect(() => validateMarkdownConversionSemantics(regrouped)).not.toThrow(); + expect(() => validateMarkdownConversionSemantics(regrouped, { layout })).toThrow(/bound layout IR/); }); it("emits only inert data cells for an opted-in complete source-covered headerless grid", async () => { From 91e85ab987c14228b0c067ab96623f81e0e1515c Mon Sep 17 00:00:00 2001 From: silverstein Date: Wed, 21 Oct 2026 13:03:59 +0000 Subject: [PATCH 4/6] Replay draft assignments against packet geometry before returning them --- docs/MAINTAINERS.md | 5 ++- pdf-toolkit-mcp-share/server/index.js | 4 +- .../server/markdown-conversion.js | 43 +++++++++++++------ server/index.js | 4 +- server/markdown-conversion.js | 43 +++++++++++++------ test/markdown-conversion.test.js | 6 +-- 6 files changed, 72 insertions(+), 33 deletions(-) diff --git a/docs/MAINTAINERS.md b/docs/MAINTAINERS.md index 3aa2f19..f2c2014 100644 --- a/docs/MAINTAINERS.md +++ b/docs/MAINTAINERS.md @@ -328,8 +328,9 @@ Rows, columns and header roles remain unverified; no header tags, spans, blank values or new source text are inferred. Candidate digests bind algorithm, PDF SHA, region, assignments, fragments, uncertainty and presentation. When layout is supplied, semantic validation reconstructs the entire packet from -it. Standalone output validation checks content/digest consistency only and -does not replace fresh source replay. Document maps carry no candidate cells. +it. Standalone output validation reconstructs the draft from its retained text +and geometry as well, but cannot authenticate that packet to the original PDF; +it does not replace fresh source replay. Document maps carry no candidate cells. Ordinary Markdown, its gap list and strict `verify_table_proposal` acceptance remain unchanged. Review the draft beside the source PDF before using its grouping; a visible draft is not an accepted table or verified extraction. diff --git a/pdf-toolkit-mcp-share/server/index.js b/pdf-toolkit-mcp-share/server/index.js index 13203b0..5d08a04 100755 --- a/pdf-toolkit-mcp-share/server/index.js +++ b/pdf-toolkit-mcp-share/server/index.js @@ -6092,7 +6092,9 @@ async function handleToolCall(request) { ...(pagesNeedingVision.length > 0 ? [`Vision routing: use render_pdf_page for pages ${pagesNeedingVision.map(entry => entry.page).join(", ")}.`] : []), - "No OCR, external model, hidden link-target recovery, or table inference beyond complete source-bound text, ruled, or painted grids was performed.", + reviewCandidates.length > 0 + ? "The optional table drafts use whitespace alignment and need review. No OCR, external model or hidden link-target recovery was performed." + : "No OCR, external model, hidden link-target recovery, or table inference beyond complete source-bound text, ruled, or painted grids was performed.", rendered.markdown, ...reviewCandidates.map(region => [ `Page ${region.page}, ${region.region_id}: ${region.review_candidate.review_notice}`, diff --git a/pdf-toolkit-mcp-share/server/markdown-conversion.js b/pdf-toolkit-mcp-share/server/markdown-conversion.js index c3aca82..1e0ef9b 100644 --- a/pdf-toolkit-mcp-share/server/markdown-conversion.js +++ b/pdf-toolkit-mcp-share/server/markdown-conversion.js @@ -2816,6 +2816,24 @@ function tableProposalHeaderHints(run) { }; } +function joinedThinFillBands(rectangles) { + const tolerance = 0.05; + const bars = rectangles.filter(rect => ["fill", "eoFill"].includes(rect.verb) + && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5) + .sort((a, b) => a.y - b.y || a.x - b.x); + const bands = []; + for (const bar of bars) { + const prior = bands.at(-1); + if (prior && Math.abs(prior.y - bar.y) <= tolerance + && Math.abs(prior.height - bar.height) <= tolerance + && Math.abs(prior.x + prior.width - bar.x) <= tolerance) { + prior.width = bar.x + bar.width - prior.x; + prior.parts.push(bar); + } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] }); + } + return bands; +} + function threeRuleProposalEnvelope(page, run) { // This refines an already abandoned proposal, never reconstructs a table. // Three full-width paint bands and a nearby caption can bound the source @@ -2833,19 +2851,7 @@ function threeRuleProposalEnvelope(page, run) { const text = page.raw_items.filter(item => item.text_kind === "non_whitespace"); if (text.some(item => !item.geometry_valid || !item.bbox)) return null; const tolerance = 0.05; - const bars = evidence.items.filter(rect => ["fill", "eoFill"].includes(rect.verb) - && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5) - .sort((a, b) => a.y - b.y || a.x - b.x); - const bands = []; - for (const bar of bars) { - const prior = bands.at(-1); - if (prior && Math.abs(prior.y - bar.y) <= tolerance - && Math.abs(prior.height - bar.height) <= tolerance - && Math.abs(prior.x + prior.width - bar.x) <= tolerance) { - prior.width = bar.x + bar.width - prior.x; - prior.parts.push(bar); - } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] }); - } + const bands = joinedThinFillBands(evidence.items); const contains = (outer, inner) => inner.x >= outer.x && inner.y >= outer.y && inner.x + inner.width <= outer.x + outer.width && inner.y + inner.height <= outer.y + outer.height; @@ -3838,6 +3844,17 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = } function validateTableReviewCandidate(region, candidate) { + const bands = joinedThinFillBands(region.ruled_rects); + assertion(bands.length === 3 && bands.every(band => Math.abs(band.x - region.bbox.x) <= 0.05 + && Math.abs(band.width - region.bbox.width) <= 0.05) + && Math.abs(bands[0].y - region.bbox.y) <= 0.05 + && Math.abs(bands[2].y + bands[2].height - region.bbox.y - region.bbox.height) <= 0.05, + "table review candidate rule envelope mismatch"); + const replay = threeRuleReviewCandidate(region, { middle: bands[1] }, candidate.source_sha256); + assertion(replay !== null, "table review candidate geometry is unsupported"); + const { candidate_sha256: _digest, ...replayed } = replay; + assertion(canonicalReviewCandidate(candidate) === canonicalReviewCandidate(replayed), + "table review candidate does not match its retained source geometry"); assertion(candidate.algorithm?.name === "pdf-tools.three-rule-review-candidate" && candidate.algorithm.version === "0.1.0" && candidate.status === "needs_review" && candidate.topology_verified === false && candidate.header_roles_verified === false diff --git a/server/index.js b/server/index.js index 13203b0..5d08a04 100755 --- a/server/index.js +++ b/server/index.js @@ -6092,7 +6092,9 @@ async function handleToolCall(request) { ...(pagesNeedingVision.length > 0 ? [`Vision routing: use render_pdf_page for pages ${pagesNeedingVision.map(entry => entry.page).join(", ")}.`] : []), - "No OCR, external model, hidden link-target recovery, or table inference beyond complete source-bound text, ruled, or painted grids was performed.", + reviewCandidates.length > 0 + ? "The optional table drafts use whitespace alignment and need review. No OCR, external model or hidden link-target recovery was performed." + : "No OCR, external model, hidden link-target recovery, or table inference beyond complete source-bound text, ruled, or painted grids was performed.", rendered.markdown, ...reviewCandidates.map(region => [ `Page ${region.page}, ${region.region_id}: ${region.review_candidate.review_notice}`, diff --git a/server/markdown-conversion.js b/server/markdown-conversion.js index c3aca82..1e0ef9b 100644 --- a/server/markdown-conversion.js +++ b/server/markdown-conversion.js @@ -2816,6 +2816,24 @@ function tableProposalHeaderHints(run) { }; } +function joinedThinFillBands(rectangles) { + const tolerance = 0.05; + const bars = rectangles.filter(rect => ["fill", "eoFill"].includes(rect.verb) + && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5) + .sort((a, b) => a.y - b.y || a.x - b.x); + const bands = []; + for (const bar of bars) { + const prior = bands.at(-1); + if (prior && Math.abs(prior.y - bar.y) <= tolerance + && Math.abs(prior.height - bar.height) <= tolerance + && Math.abs(prior.x + prior.width - bar.x) <= tolerance) { + prior.width = bar.x + bar.width - prior.x; + prior.parts.push(bar); + } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] }); + } + return bands; +} + function threeRuleProposalEnvelope(page, run) { // This refines an already abandoned proposal, never reconstructs a table. // Three full-width paint bands and a nearby caption can bound the source @@ -2833,19 +2851,7 @@ function threeRuleProposalEnvelope(page, run) { const text = page.raw_items.filter(item => item.text_kind === "non_whitespace"); if (text.some(item => !item.geometry_valid || !item.bbox)) return null; const tolerance = 0.05; - const bars = evidence.items.filter(rect => ["fill", "eoFill"].includes(rect.verb) - && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5) - .sort((a, b) => a.y - b.y || a.x - b.x); - const bands = []; - for (const bar of bars) { - const prior = bands.at(-1); - if (prior && Math.abs(prior.y - bar.y) <= tolerance - && Math.abs(prior.height - bar.height) <= tolerance - && Math.abs(prior.x + prior.width - bar.x) <= tolerance) { - prior.width = bar.x + bar.width - prior.x; - prior.parts.push(bar); - } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] }); - } + const bands = joinedThinFillBands(evidence.items); const contains = (outer, inner) => inner.x >= outer.x && inner.y >= outer.y && inner.x + inner.width <= outer.x + outer.width && inner.y + inner.height <= outer.y + outer.height; @@ -3838,6 +3844,17 @@ export function validateMarkdownConversionSemantics(result, { layout = null } = } function validateTableReviewCandidate(region, candidate) { + const bands = joinedThinFillBands(region.ruled_rects); + assertion(bands.length === 3 && bands.every(band => Math.abs(band.x - region.bbox.x) <= 0.05 + && Math.abs(band.width - region.bbox.width) <= 0.05) + && Math.abs(bands[0].y - region.bbox.y) <= 0.05 + && Math.abs(bands[2].y + bands[2].height - region.bbox.y - region.bbox.height) <= 0.05, + "table review candidate rule envelope mismatch"); + const replay = threeRuleReviewCandidate(region, { middle: bands[1] }, candidate.source_sha256); + assertion(replay !== null, "table review candidate geometry is unsupported"); + const { candidate_sha256: _digest, ...replayed } = replay; + assertion(canonicalReviewCandidate(candidate) === canonicalReviewCandidate(replayed), + "table review candidate does not match its retained source geometry"); assertion(candidate.algorithm?.name === "pdf-tools.three-rule-review-candidate" && candidate.algorithm.version === "0.1.0" && candidate.status === "needs_review" && candidate.topology_verified === false && candidate.header_roles_verified === false diff --git a/test/markdown-conversion.test.js b/test/markdown-conversion.test.js index 360b1bc..c84789f 100644 --- a/test/markdown-conversion.test.js +++ b/test/markdown-conversion.test.js @@ -525,8 +525,8 @@ describe("layout Markdown renderer", () => { expect(() => validateMarkdownConversionSemantics(changed)).toThrow(); expect(() => validateMarkdownConversionSemantics(changed, { layout })).toThrow(); } - // Swapping complete assignments and their source fragments can be internally - // consistent; only a reconstruction from the bound IR proves this algorithm's output. + // Even a content-consistent, freshly rehashed regrouping must fail the + // packet-only geometry replay as well as the complete bound-IR replay. const regrouped = structuredClone(result); const draft = regrouped.table_proposal_regions.find(value => value.review_candidate).review_candidate; [draft.cells[1].item_ids, draft.cells[2].item_ids] = [draft.cells[2].item_ids, draft.cells[1].item_ids]; @@ -534,7 +534,7 @@ describe("layout Markdown renderer", () => { draft.html = draft.html.replace("AB", "BA"); const { candidate_sha256: _digest, ...body } = draft; draft.candidate_sha256 = createHash("sha256").update(canonicalCandidate(body)).digest("hex"); - expect(() => validateMarkdownConversionSemantics(regrouped)).not.toThrow(); + expect(() => validateMarkdownConversionSemantics(regrouped)).toThrow(/retained source geometry/); expect(() => validateMarkdownConversionSemantics(regrouped, { layout })).toThrow(/bound layout IR/); }); From 7abbf0a63ffb1d1d1f892cfa5fcf2713d8a25178 Mon Sep 17 00:00:00 2001 From: silverstein Date: Wed, 21 Oct 2026 13:04:43 +0000 Subject: [PATCH 5/6] Use complete algorithm replay as the candidate validation check --- .../server/markdown-conversion.js | 43 ------------------- server/markdown-conversion.js | 43 ------------------- 2 files changed, 86 deletions(-) diff --git a/pdf-toolkit-mcp-share/server/markdown-conversion.js b/pdf-toolkit-mcp-share/server/markdown-conversion.js index 1e0ef9b..4df92b2 100644 --- a/pdf-toolkit-mcp-share/server/markdown-conversion.js +++ b/pdf-toolkit-mcp-share/server/markdown-conversion.js @@ -3855,49 +3855,6 @@ function validateTableReviewCandidate(region, candidate) { const { candidate_sha256: _digest, ...replayed } = replay; assertion(canonicalReviewCandidate(candidate) === canonicalReviewCandidate(replayed), "table review candidate does not match its retained source geometry"); - assertion(candidate.algorithm?.name === "pdf-tools.three-rule-review-candidate" - && candidate.algorithm.version === "0.1.0" && candidate.status === "needs_review" - && candidate.topology_verified === false && candidate.header_roles_verified === false - && candidate.content_origin === "source_text_items" - && candidate.structure_origin === "whitespace_alignment_proposal" - && candidate.review_notice === TABLE_REVIEW_NOTICE, - "table review candidate claim boundary mismatch"); - assertion(Object.values(region.truncation).every(value => value === "complete"), - "table review candidate cannot use truncated evidence"); - assertion(Number.isSafeInteger(candidate.row_count) && candidate.row_count >= 3 && candidate.row_count <= 100 - && Number.isSafeInteger(candidate.column_count) && candidate.column_count >= 2 && candidate.column_count <= 12 - && candidate.cells?.length === candidate.row_count * candidate.column_count, - "table review candidate dimensions mismatch"); - const itemById = new Map(region.text_items.map(item => [item.id, item])); - const assigned = []; - for (const [index, cell] of candidate.cells.entries()) { - assertion(cell.row === Math.floor(index / candidate.column_count) - && cell.column === index % candidate.column_count && cell.item_ids.length > 0, - "table review candidate cell order mismatch"); - const lines = []; - for (const id of cell.item_ids) { - const item = itemById.get(id); - assertion(item !== undefined, "table review candidate unknown source item"); - assigned.push(id); - if (lines.at(-1)?.line_id !== item.line_id) { - lines.push({ line_id: item.line_id, item_ids: [], text: "" }); - } - lines.at(-1).item_ids.push(id); - lines.at(-1).text += item.text; - } - assertion(sameJson(cell.lines, lines), "table review candidate content mismatch"); - } - assertion(assigned.length === itemById.size && new Set(assigned).size === itemById.size - && candidate.source_item_count === itemById.size && candidate.assigned_item_count === assigned.length - && candidate.unassigned_item_count === 0, "table review candidate source coverage mismatch"); - const grid = Array.from({ length: candidate.row_count }, (_, row) => candidate.cells - .filter(cell => cell.row === row).map(cell => cell.lines - .map(line => escapeHeaderNeutralText(line.text)).join("
"))); - const html = ["", ...grid.map(row => `${row.map(text => ``).join("")}`), "
${text}
"].join("\n"); - assertion(candidate.html === html && candidate.html_format === "header_neutral_html" - && candidate.html_bytes === utf8Bytes(html) && candidate.html_bytes <= 20_000 - && utf8Bytes(canonicalReviewCandidate(candidate)) <= 40_000, - "table review candidate presentation mismatch"); } function isFiniteNumber(value) { diff --git a/server/markdown-conversion.js b/server/markdown-conversion.js index 1e0ef9b..4df92b2 100644 --- a/server/markdown-conversion.js +++ b/server/markdown-conversion.js @@ -3855,49 +3855,6 @@ function validateTableReviewCandidate(region, candidate) { const { candidate_sha256: _digest, ...replayed } = replay; assertion(canonicalReviewCandidate(candidate) === canonicalReviewCandidate(replayed), "table review candidate does not match its retained source geometry"); - assertion(candidate.algorithm?.name === "pdf-tools.three-rule-review-candidate" - && candidate.algorithm.version === "0.1.0" && candidate.status === "needs_review" - && candidate.topology_verified === false && candidate.header_roles_verified === false - && candidate.content_origin === "source_text_items" - && candidate.structure_origin === "whitespace_alignment_proposal" - && candidate.review_notice === TABLE_REVIEW_NOTICE, - "table review candidate claim boundary mismatch"); - assertion(Object.values(region.truncation).every(value => value === "complete"), - "table review candidate cannot use truncated evidence"); - assertion(Number.isSafeInteger(candidate.row_count) && candidate.row_count >= 3 && candidate.row_count <= 100 - && Number.isSafeInteger(candidate.column_count) && candidate.column_count >= 2 && candidate.column_count <= 12 - && candidate.cells?.length === candidate.row_count * candidate.column_count, - "table review candidate dimensions mismatch"); - const itemById = new Map(region.text_items.map(item => [item.id, item])); - const assigned = []; - for (const [index, cell] of candidate.cells.entries()) { - assertion(cell.row === Math.floor(index / candidate.column_count) - && cell.column === index % candidate.column_count && cell.item_ids.length > 0, - "table review candidate cell order mismatch"); - const lines = []; - for (const id of cell.item_ids) { - const item = itemById.get(id); - assertion(item !== undefined, "table review candidate unknown source item"); - assigned.push(id); - if (lines.at(-1)?.line_id !== item.line_id) { - lines.push({ line_id: item.line_id, item_ids: [], text: "" }); - } - lines.at(-1).item_ids.push(id); - lines.at(-1).text += item.text; - } - assertion(sameJson(cell.lines, lines), "table review candidate content mismatch"); - } - assertion(assigned.length === itemById.size && new Set(assigned).size === itemById.size - && candidate.source_item_count === itemById.size && candidate.assigned_item_count === assigned.length - && candidate.unassigned_item_count === 0, "table review candidate source coverage mismatch"); - const grid = Array.from({ length: candidate.row_count }, (_, row) => candidate.cells - .filter(cell => cell.row === row).map(cell => cell.lines - .map(line => escapeHeaderNeutralText(line.text)).join("
"))); - const html = ["", ...grid.map(row => `${row.map(text => ``).join("")}`), "
${text}
"].join("\n"); - assertion(candidate.html === html && candidate.html_format === "header_neutral_html" - && candidate.html_bytes === utf8Bytes(html) && candidate.html_bytes <= 20_000 - && utf8Bytes(canonicalReviewCandidate(candidate)) <= 40_000, - "table review candidate presentation mismatch"); } function isFiniteNumber(value) { From e36657dadaa49939b013dd4640d7232f53fe4219 Mon Sep 17 00:00:00 2001 From: silverstein Date: Wed, 21 Oct 2026 13:07:12 +0000 Subject: [PATCH 6/6] Refresh captured tool contracts and source provenance for table drafts --- scripts/macos-claude-installed-shannon.mjs | 2 +- scripts/macos-claude-installed-smoke.mjs | 2 +- .../extraction/phase1/layout-occurrence-oracle.v1.json | 10 +++++----- test/fixtures/eval/trajectories/tool-contracts.v3.json | 4 ++-- test/mcp-contract.test.js | 3 ++- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/macos-claude-installed-shannon.mjs b/scripts/macos-claude-installed-shannon.mjs index 7ee10e6..184a213 100755 --- a/scripts/macos-claude-installed-shannon.mjs +++ b/scripts/macos-claude-installed-shannon.mjs @@ -23,7 +23,7 @@ if (!process.argv[2] || !process.argv[3]) { // const EXPECTED_SOURCE_SHA256 = "6e4e3411984f3edf99dbfe8b941cb5e8a321379ff0cae6ae5c1f592ad8882ca8"; const EXPECTED_MARKDOWN_SHA256 = "b9bd7ca7db26fe807f294565c39de8d4fe9337908afa74cd7e2896a800c92991"; -const EXPECTED_TOOL_CONTRACT_SHA256 = "17e2067e2c014bfc77e234432af45f7711494104f2a14d5f652e42e917251128"; +const EXPECTED_TOOL_CONTRACT_SHA256 = "2239a74dcdb71387f855c49bb26bc01e52ca9a14e773be13a32d19d3af9ec2d4"; const EXPECTED_PAGE_COUNT = 55; const EXPECTED_GAP_COUNT = 26; const EXPECTED_REPLACEMENT_CHARACTER_COUNT = 6; diff --git a/scripts/macos-claude-installed-smoke.mjs b/scripts/macos-claude-installed-smoke.mjs index e9a2d73..9460f72 100755 --- a/scripts/macos-claude-installed-smoke.mjs +++ b/scripts/macos-claude-installed-smoke.mjs @@ -33,7 +33,7 @@ const textFixture = path.join(fixtureDirectory, "synthetic-text-two-page.pdf"); const rasterFixture = path.join(fixtureDirectory, "synthetic-raster-only.pdf"); const mutationDirectory = path.join(fixtureDirectory, "mutation-output"); const toolNames = []; -const EXPECTED_TOOL_CONTRACT_SHA256 = "17e2067e2c014bfc77e234432af45f7711494104f2a14d5f652e42e917251128"; +const EXPECTED_TOOL_CONTRACT_SHA256 = "2239a74dcdb71387f855c49bb26bc01e52ca9a14e773be13a32d19d3af9ec2d4"; const ACCESSIBILITY_CONCLUSION_KEYS = Object.freeze([ "certification", "document_accessibility", diff --git a/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json b/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json index 10b81b3..2105fe9 100644 --- a/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json +++ b/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json @@ -79,14 +79,14 @@ { "role": "markdown_conversion_module", "path": "server/markdown-conversion.js", - "bytes": 192276, - "sha256": "838f9b1adb38faf5eb3b31d7be9479eb8521dff0fd7013dcd3c89fc3cbcb95d1" + "bytes": 200207, + "sha256": "988a3b3b4c9ebb27b66a63cd2703dd002519935c8dd8660a8af78f53621b257a" }, { "role": "output_schemas_module", "path": "server/output-schemas.js", - "bytes": 84427, - "sha256": "beb9255b64456319bf2b747e643c32d3d175c29f51feddfdf5f419da19f9fc3c" + "bytes": 85973, + "sha256": "f26eea7b94ccda2faca95880afe33bb3186511b9cf7ac32b799b895ee84b226d" }, { "role": "package_json", @@ -131,7 +131,7 @@ "sha256": "ef40501b2afbe4cd2adfa80480344783bbec1a00e8e9850086a5d044231d1f53" } ], - "validator_source_set_sha256": "6e875209853d97288e5eae1464c0c3a333f37b31ad3467839954aa4edbf657c4", + "validator_source_set_sha256": "1903def9a7de315d775ec2a4f0494778cefad449c88fdf8e9283fc2749eabf01", "pdfjs_version": "5.4.624", "generation_contract": { "source_path": "source.pdf", diff --git a/test/fixtures/eval/trajectories/tool-contracts.v3.json b/test/fixtures/eval/trajectories/tool-contracts.v3.json index ff68473..9c7cfd0 100644 --- a/test/fixtures/eval/trajectories/tool-contracts.v3.json +++ b/test/fixtures/eval/trajectories/tool-contracts.v3.json @@ -5,7 +5,7 @@ "name": "pdf-tools", "version": "0.14.0" }, - "tools_sha256": "93202e8e6e04b8914f81003624692177e9dbe02c6fc93aa877ebf7e82b2556a7", + "tools_sha256": "cbd4f2614576b123130ad430542f58cf07ca3730026c59545e7fe4b447dce4bf", "tools": [ { "name": "add_signature_field", @@ -554,7 +554,7 @@ }, "emit_table_proposals": { "type": "boolean", - "description": "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." + "description": "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. A narrowly qualifying captioned three-rule table may also include a draft arrangement and header-neutral HTML preview marked needs_review; its row/column grouping and header roles are unverified. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." }, "allow_headerless_tables": { "type": "boolean", diff --git a/test/mcp-contract.test.js b/test/mcp-contract.test.js index 9e8b58e..09385f5 100644 --- a/test/mcp-contract.test.js +++ b/test/mcp-contract.test.js @@ -242,7 +242,8 @@ const EXAMPLE_PDF = path.join(REPO_ROOT, "example-fw9.pdf"); // neutrally described and no longer exposes arbitrary credential headers. // 2026-10-20: version the optional tight-grid/caption repair as renderer 1.23.0. // 2026-10-20: private local password entry replaces public credential inputs. -const TOOL_CONTRACT_SHA256 = "17e2067e2c014bfc77e234432af45f7711494104f2a14d5f652e42e917251128"; +// 2026-10-21: optional table packets describe draft arrangements that require review. +const TOOL_CONTRACT_SHA256 = "2239a74dcdb71387f855c49bb26bc01e52ca9a14e773be13a32d19d3af9ec2d4"; const CLOSED_READ = Object.freeze({ readOnlyHint: true,