diff --git a/docs/MAINTAINERS.md b/docs/MAINTAINERS.md
index 0c97446..f2c2014 100644
--- a/docs/MAINTAINERS.md
+++ b/docs/MAINTAINERS.md
@@ -313,6 +313,28 @@ regions suppress header hints. The same source region is used by document maps
and fresh proposal verification. Ordinary Markdown and conversion gaps do not
change, and matching source/region tokens do not excuse obsolete assignments.
+With `emit_table_proposals: true`, qualifying three-rule regions can also carry
+`review_candidate`: a proposed arrangement with `status: needs_review`, exact
+source-item assignments and line fragments, and inert HTML data cells. The
+same preview and review notice appear in the text response. It groups the top
+band provisionally and derives columns from whitespace corridors recurring
+across every nonoverlapping body line. All region items must be assigned once,
+every cell must contain source text, and the complete maximal partition must
+recur on every body line. Ragged, wrapped, straddling or over-budget layouts
+retain the evidence packet without a draft. Drafts are capped at 100 rows,
+12 columns, 400 source items, 20,000 HTML bytes and 40,000 canonical JSON bytes.
+
+Rows, columns and header roles remain unverified; no header tags, spans, blank
+values or new source text are inferred. Candidate digests bind algorithm,
+PDF SHA, region, assignments, fragments, uncertainty and presentation. When
+layout is supplied, semantic validation reconstructs the entire packet from
+it. Standalone output validation reconstructs the draft from its retained text
+and geometry as well, but cannot authenticate that packet to the original PDF;
+it does not replace fresh source replay. Document maps carry no candidate cells.
+Ordinary Markdown, its gap list and strict `verify_table_proposal` acceptance
+remain unchanged. Review the draft beside the source PDF before using its
+grouping; a visible draft is not an accepted table or verified extraction.
+
Filesystem operations and rasterization happen locally, and PDF Tools does not
upload files to a separate PDF service. Text, images, and metadata returned
through MCP may still be processed under the selected host or model provider's
diff --git a/pdf-toolkit-mcp-share/server/index.js b/pdf-toolkit-mcp-share/server/index.js
index b6ff513..5d08a04 100755
--- a/pdf-toolkit-mcp-share/server/index.js
+++ b/pdf-toolkit-mcp-share/server/index.js
@@ -3852,7 +3852,7 @@ async function listTools(request) {
include_page_boundaries: { type: "boolean", description: "Include deterministic HTML comments marking page boundaries. Default: true." },
compact: { type: "boolean", description: "Opt into counted dot-leader, isolated page-number, and Unicode-letter spaced-hyphen normalizations. Default: false." },
remove_page_furniture: { type: "boolean", description: "Remove only source-evidenced extreme-margin page furniture and report every removed line by typed gap and counts. Set false to preserve all source lines. Default: true." },
- emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." },
+ emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. A narrowly qualifying captioned three-rule table may also include a draft arrangement and header-neutral HTML preview marked needs_review; its row/column grouping and header roles are unverified. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." },
allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.23.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." },
output_path: { type: "string", description: "Optional absolute .md path, or ~/ path. The file is written only after complete bytes are staged and verified." },
overwrite: { type: "boolean", description: "Replace an existing output_path only when its exact expected_output_identity is also supplied. Default: false." },
@@ -6082,6 +6082,7 @@ async function handleToolCall(request) {
saved_output: savedOutput,
...(tableProposals !== undefined ? { table_proposals: tableProposals } : {}),
};
+ const reviewCandidates = tableProposals?.filter(region => region.review_candidate) ?? [];
const summary = [
`Converted pages ${layout.page_range.start_page}-${layout.page_range.end_page} of ${fileName} to deterministic Markdown.`,
`Status: ${rendered.conversion_status}. UTF-8 bytes: ${rendered.markdown_bytes}.`,
@@ -6091,8 +6092,14 @@ async function handleToolCall(request) {
...(pagesNeedingVision.length > 0
? [`Vision routing: use render_pdf_page for pages ${pagesNeedingVision.map(entry => entry.page).join(", ")}.`]
: []),
- "No OCR, external model, hidden link-target recovery, or table inference beyond complete source-bound text, ruled, or painted grids was performed.",
+ reviewCandidates.length > 0
+ ? "The optional table drafts use whitespace alignment and need review. No OCR, external model or hidden link-target recovery was performed."
+ : "No OCR, external model, hidden link-target recovery, or table inference beyond complete source-bound text, ruled, or painted grids was performed.",
rendered.markdown,
+ ...reviewCandidates.map(region => [
+ `Page ${region.page}, ${region.region_id}: ${region.review_candidate.review_notice}`,
+ region.review_candidate.html,
+ ].join("\n\n")),
].join("\n\n");
return {
content: [{ type: "text", text: summary }],
diff --git a/pdf-toolkit-mcp-share/server/markdown-conversion.js b/pdf-toolkit-mcp-share/server/markdown-conversion.js
index 324092d..4df92b2 100644
--- a/pdf-toolkit-mcp-share/server/markdown-conversion.js
+++ b/pdf-toolkit-mcp-share/server/markdown-conversion.js
@@ -60,6 +60,7 @@ export const MAX_TABLE_PROPOSAL_TEXT_ITEMS = 400;
const MAX_TABLE_PROPOSAL_RULED_RECTS = 400;
const MAX_TABLE_PROPOSAL_RULING_SEGMENTS = 400;
const MAX_TABLE_PROPOSAL_PAINTED_RECTS = 400;
+export const TABLE_REVIEW_NOTICE = "Proposed arrangement, needs review. Check the row and column grouping against the PDF before using it; headers and cell structure are unverified.";
const MAX_MARKDOWN_BYTES_LIMIT = 200_000;
const GAP_CODES = new Set([
@@ -2815,6 +2816,24 @@ function tableProposalHeaderHints(run) {
};
}
+function joinedThinFillBands(rectangles) {
+ const tolerance = 0.05;
+ const bars = rectangles.filter(rect => ["fill", "eoFill"].includes(rect.verb)
+ && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5)
+ .sort((a, b) => a.y - b.y || a.x - b.x);
+ const bands = [];
+ for (const bar of bars) {
+ const prior = bands.at(-1);
+ if (prior && Math.abs(prior.y - bar.y) <= tolerance
+ && Math.abs(prior.height - bar.height) <= tolerance
+ && Math.abs(prior.x + prior.width - bar.x) <= tolerance) {
+ prior.width = bar.x + bar.width - prior.x;
+ prior.parts.push(bar);
+ } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] });
+ }
+ return bands;
+}
+
function threeRuleProposalEnvelope(page, run) {
// This refines an already abandoned proposal, never reconstructs a table.
// Three full-width paint bands and a nearby caption can bound the source
@@ -2832,19 +2851,7 @@ function threeRuleProposalEnvelope(page, run) {
const text = page.raw_items.filter(item => item.text_kind === "non_whitespace");
if (text.some(item => !item.geometry_valid || !item.bbox)) return null;
const tolerance = 0.05;
- const bars = evidence.items.filter(rect => ["fill", "eoFill"].includes(rect.verb)
- && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5)
- .sort((a, b) => a.y - b.y || a.x - b.x);
- const bands = [];
- for (const bar of bars) {
- const prior = bands.at(-1);
- if (prior && Math.abs(prior.y - bar.y) <= tolerance
- && Math.abs(prior.height - bar.height) <= tolerance
- && Math.abs(prior.x + prior.width - bar.x) <= tolerance) {
- prior.width = bar.x + bar.width - prior.x;
- prior.parts.push(bar);
- } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] });
- }
+ const bands = joinedThinFillBands(evidence.items);
const contains = (outer, inner) => inner.x >= outer.x && inner.y >= outer.y
&& inner.x + inner.width <= outer.x + outer.width
&& inner.y + inner.height <= outer.y + outer.height;
@@ -2888,17 +2895,109 @@ function threeRuleProposalEnvelope(page, run) {
|| !lines.some(line => line.y + line.height <= middle.y)
|| !lines.some(line => line.y >= middle.y + middle.height)) continue;
const itemById = new Map(page.raw_items.map(item => [item.id, item]));
- candidates.push({ bbox, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) });
+ candidates.push({ bbox, middle, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) });
}
return candidates.length === 1 ? candidates[0] : null;
}
+function canonicalReviewCandidate(value) {
+ if (Array.isArray(value)) return `[${value.map(canonicalReviewCandidate).join(",")}]`;
+ if (value && typeof value === "object") return `{${Object.keys(value).sort()
+ .map(key => `${JSON.stringify(key)}:${canonicalReviewCandidate(value[key])}`).join(",")}}`;
+ return JSON.stringify(value);
+}
+
+function threeRuleReviewCandidate(region, envelope, sourceSha256) {
+ if (!/^[a-f0-9]{64}$/.test(sourceSha256 ?? "")
+ || Object.values(region.truncation).some(value => value !== "complete")) return null;
+ const items = region.text_items;
+ if (items.length === 0 || new Set(items.map(item => item.id)).size !== items.length) return null;
+ const { middle } = envelope;
+ const upper = items.filter(item => item.bbox.y + item.bbox.height <= middle.y);
+ const lower = items.filter(item => item.bbox.y >= middle.y + middle.height);
+ if (upper.length === 0 || upper.length + lower.length !== items.length) return null;
+ const byLine = new Map();
+ for (const item of lower) {
+ if (!byLine.has(item.line_id)) byLine.set(item.line_id, []);
+ byLine.get(item.line_id).push(item);
+ }
+ const rows = [...byLine.values()].map(row => ({
+ items: row.sort((a, b) => a.bbox.x - b.bbox.x || a.reading_order_index - b.reading_order_index),
+ top: Math.min(...row.map(item => item.bbox.y)),
+ bottom: Math.max(...row.map(item => item.bbox.y + item.bbox.height)),
+ })).sort((a, b) => a.top - b.top);
+ if (rows.length < 2 || rows.length > 99 || rows.some((row, index) => (
+ index > 0 && row.top <= rows[index - 1].bottom
+ ))) return null;
+ // A maximal set of whitespace corridors shared by every body line supplies
+ // one deterministic proposed partition. Whitespace does not prove topology.
+ const minGap = Math.max(5, ...lower.map(item => item.bbox.height));
+ let corridors = null;
+ let gapCount = null;
+ for (const row of rows) {
+ const gaps = [];
+ let right = row.items[0].bbox.x + row.items[0].bbox.width;
+ for (const item of row.items.slice(1)) {
+ if (item.bbox.x - right >= minGap) gaps.push([right, item.bbox.x]);
+ right = Math.max(right, item.bbox.x + item.bbox.width);
+ }
+ if (gapCount !== null && gaps.length !== gapCount) return null;
+ gapCount = gaps.length;
+ corridors = corridors === null ? gaps : corridors.flatMap(([left, end]) => gaps
+ .map(([start, rightEdge]) => [Math.max(left, start), Math.min(end, rightEdge)])
+ .filter(([start, rightEdge]) => rightEdge - start >= minGap));
+ }
+ if (corridors.length < 1 || corridors.length > 11 || corridors.length !== gapCount) return null;
+ const cuts = corridors.map(([left, right]) => (left + right) / 2);
+ const sourceRows = [upper, ...rows.map(row => row.items)];
+ const cells = [];
+ for (const [row, sourceItems] of sourceRows.entries()) {
+ const columns = Array.from({ length: cuts.length + 1 }, () => []);
+ for (const item of sourceItems) {
+ const column = cuts.filter(cut => item.bbox.x >= cut).length;
+ if (column < cuts.length && item.bbox.x + item.bbox.width > cuts[column]) return null;
+ columns[column].push(item);
+ }
+ if (columns.some(column => column.length === 0)) return null;
+ for (const [column, assigned] of columns.entries()) {
+ assigned.sort((a, b) => a.reading_order_index - b.reading_order_index);
+ const lines = [];
+ for (const item of assigned) {
+ if (lines.at(-1)?.line_id !== item.line_id) {
+ if (lines.some(line => line.line_id === item.line_id)) return null;
+ lines.push({ line_id: item.line_id, item_ids: [], text: "" });
+ }
+ lines.at(-1).item_ids.push(item.id);
+ lines.at(-1).text += item.text;
+ }
+ cells.push({ row, column, item_ids: assigned.map(item => item.id), lines });
+ }
+ }
+ const assigned = cells.flatMap(cell => cell.item_ids);
+ if (new Set(assigned).size !== items.length || assigned.length !== items.length) return null;
+ const grid = sourceRows.map((_, row) => cells.filter(cell => cell.row === row)
+ .map(cell => cell.lines.map(line => escapeHeaderNeutralText(line.text)).join("
")));
+ const html = ["
", ...grid.map(row => `${row.map(text => `| ${text} | `).join("")}
`), "
"].join("\n");
+ const candidate = {
+ algorithm: { name: "pdf-tools.three-rule-review-candidate", version: "0.1.0" },
+ status: "needs_review", topology_verified: false, header_roles_verified: false,
+ source_sha256: sourceSha256, region_id: region.region_id, page: region.page,
+ content_origin: "source_text_items", structure_origin: "whitespace_alignment_proposal",
+ row_count: sourceRows.length, column_count: cuts.length + 1,
+ source_item_count: items.length, assigned_item_count: assigned.length, unassigned_item_count: 0,
+ cells, html, html_format: "header_neutral_html", html_bytes: utf8Bytes(html),
+ review_notice: TABLE_REVIEW_NOTICE,
+ };
+ if (candidate.html_bytes > 20_000 || utf8Bytes(canonicalReviewCandidate(candidate)) > 40_000) return null;
+ return { ...candidate, candidate_sha256: sha256(canonicalReviewCandidate(candidate)) };
+}
+
/**
* Build a bounded proposal from an abandoned run. A uniquely captioned
* three-rule envelope can refine source membership but supplies no topology
* or header authority. Pure over source-validated IR; no token, time or I/O.
*/
-export function buildTableProposalRegion(page, run, reason, ordinal) {
+export function buildTableProposalRegion(page, run, reason, ordinal, { sourceSha256 = null } = {}) {
const envelope = threeRuleProposalEnvelope(page, run);
if (envelope) run = envelope.run;
const itemById = new Map(page.raw_items.map(item => [item.id, item]));
@@ -2962,7 +3061,7 @@ export function buildTableProposalRegion(page, run, reason, ordinal) {
bbox: painted.bbox,
}));
- return {
+ const proposal = {
region_id: `p${page.page}-t${ordinal}`,
page: page.page,
reason: regionGapCode(reason),
@@ -2983,6 +3082,11 @@ export function buildTableProposalRegion(page, run, reason, ordinal) {
painted_rectangles: paintedTruncated ? "truncated" : "complete",
},
};
+ if (envelope) {
+ const candidate = threeRuleReviewCandidate(proposal, envelope, sourceSha256);
+ if (candidate) proposal.review_candidate = candidate;
+ }
+ return proposal;
}
// escapePlainMarkdown already escapes "|" (and backslashes before it), so a
@@ -3007,15 +3111,18 @@ function renderTable(grid) {
return lines;
}
-function renderHeaderlessTable(grid) {
- const escape = value => sanitizeUnsafeText(String(value)).replace(/&/gu, "&")
+function escapeHeaderNeutralText(value) {
+ return sanitizeUnsafeText(String(value)).replace(/&/gu, "&")
.replace(//gu, ">")
.replace(/"/gu, """).replace(/'/gu, "'")
.replace(/https?:/giu, match => `${match.slice(0, -1)}:`)
.replace(/\b(?:[\p{L}\p{N}-]+\.)+[\p{L}]{2,}\b/giu, domain => domain.replace(".", "."))
.replace(/@(?=[\p{L}\p{N}.-]+.[\p{L}]{2,})/giu, "@");
+}
+
+function renderHeaderlessTable(grid) {
return ["", ...grid.map(row => (
- `${row.map(value => `| ${escape(value)} | `).join("")}
`
+ `${row.map(value => `| ${escapeHeaderNeutralText(value)} | `).join("")}
`
)), "
"];
}
@@ -3370,6 +3477,7 @@ function renderPage(page, {
pageBoundaryBefore = false,
pageBoundaryAfter = false,
collectTableProposals = false,
+ sourceSha256 = null,
allowHeaderlessTables = false,
pageFurniture = [],
} = {}) {
@@ -3488,7 +3596,7 @@ function renderPage(page, {
// markdown, gaps, or byte counts. It is only present when the opt-in flag
// is set, so the default page shape is byte-identical.
rendered.tableProposalRegions = analysis.regions.map(
- (region, index) => buildTableProposalRegion(page, region.run, region.reason, index + 1),
+ (region, index) => buildTableProposalRegion(page, region.run, region.reason, index + 1, { sourceSha256 }),
);
}
return rendered;
@@ -3706,6 +3814,28 @@ export function validateMarkdownConversionSemantics(result, { layout = null } =
if (proposals !== undefined) {
validateTableProposals(proposals, { requireToken: payloadProposals !== undefined });
validateTableProposalDocumentTruncation(result.table_proposals_truncation, proposals);
+ if (layout !== null) {
+ const expected = boundTableProposalRegions(layout.pages.flatMap(page => (
+ segmentPageLines(page, allowHeaderlessTables).regions.map((region, index) => (
+ buildTableProposalRegion(page, region.run, region.reason, index + 1,
+ { sourceSha256: layout.source.sha256 })
+ ))
+ )));
+ const actual = proposals.map(({ proposal_token: _token, ...proposal }) => proposal);
+ assertion(sameJson(actual, expected.regions), "table proposals do not match the bound layout IR");
+ assertion(sameJson(result.table_proposals_truncation, expected.truncation),
+ "table proposal truncation does not match the bound layout IR");
+ }
+ for (const proposal of proposals) {
+ if (proposal.review_candidate) {
+ const { candidate_sha256: digest, ...candidate } = proposal.review_candidate;
+ assertion(candidate.source_sha256 === result.provenance.source.sha256
+ && candidate.region_id === proposal.region_id && candidate.page === proposal.page,
+ "table review candidate source binding mismatch");
+ assertion(digest === sha256(canonicalReviewCandidate(candidate)), "table review candidate digest mismatch");
+ validateTableReviewCandidate(proposal, candidate);
+ }
+ }
} else {
assertion(result.table_proposals_truncation === undefined,
"table proposal truncation cannot exist without proposals");
@@ -3713,6 +3843,20 @@ export function validateMarkdownConversionSemantics(result, { layout = null } =
return result;
}
+function validateTableReviewCandidate(region, candidate) {
+ const bands = joinedThinFillBands(region.ruled_rects);
+ assertion(bands.length === 3 && bands.every(band => Math.abs(band.x - region.bbox.x) <= 0.05
+ && Math.abs(band.width - region.bbox.width) <= 0.05)
+ && Math.abs(bands[0].y - region.bbox.y) <= 0.05
+ && Math.abs(bands[2].y + bands[2].height - region.bbox.y - region.bbox.height) <= 0.05,
+ "table review candidate rule envelope mismatch");
+ const replay = threeRuleReviewCandidate(region, { middle: bands[1] }, candidate.source_sha256);
+ assertion(replay !== null, "table review candidate geometry is unsupported");
+ const { candidate_sha256: _digest, ...replayed } = replay;
+ assertion(canonicalReviewCandidate(candidate) === canonicalReviewCandidate(replayed),
+ "table review candidate does not match its retained source geometry");
+}
+
function isFiniteNumber(value) {
return typeof value === "number" && Number.isFinite(value);
}
@@ -3985,6 +4129,7 @@ export function renderPdfLayoutToMarkdown(layout, {
// number from a single-page selection.
pageBoundaryAfter: true,
collectTableProposals: emitTableProposals,
+ sourceSha256: layout.source.sha256,
allowHeaderlessTables,
pageFurniture: furniturePlans.get(page.page),
}));
diff --git a/pdf-toolkit-mcp-share/server/output-schemas.js b/pdf-toolkit-mcp-share/server/output-schemas.js
index 8d11e92..3c984ec 100644
--- a/pdf-toolkit-mcp-share/server/output-schemas.js
+++ b/pdf-toolkit-mcp-share/server/output-schemas.js
@@ -1,6 +1,6 @@
import { AjvJsonSchemaValidator } from "@modelcontextprotocol/server/validators/ajv";
import { validatePdfLayoutSemantics } from "./layout-extraction.js";
-import { validateMarkdownConversionSemantics } from "./markdown-conversion.js";
+import { TABLE_REVIEW_NOTICE, validateMarkdownConversionSemantics } from "./markdown-conversion.js";
import {
validatePdfObservationSemantics,
validateRenderObservationSemantics,
@@ -1172,6 +1172,27 @@ const tableProposal = object({
truncation: tableProposalTruncation,
proposal_token: sha256Digest,
});
+// Additive only on opt-in proposal packets. Strict table acceptance is separate.
+tableProposal.properties.review_candidate = object({
+ algorithm: object({ name: { const: "pdf-tools.three-rule-review-candidate" }, version: { const: "0.1.0" } }),
+ status: { const: "needs_review" }, topology_verified: { const: false }, header_roles_verified: { const: false },
+ source_sha256: sha256Digest, region_id: string, page: { type: "integer", minimum: 1 },
+ content_origin: { const: "source_text_items" }, structure_origin: { const: "whitespace_alignment_proposal" },
+ row_count: { type: "integer", minimum: 3, maximum: 100 },
+ column_count: { type: "integer", minimum: 2, maximum: 12 },
+ source_item_count: { type: "integer", minimum: 1, maximum: 400 },
+ assigned_item_count: { type: "integer", minimum: 1, maximum: 400 }, unassigned_item_count: { const: 0 },
+ cells: { type: "array", minItems: 6, maxItems: 400, items: object({
+ row: { type: "integer", minimum: 0, maximum: 99 }, column: { type: "integer", minimum: 0, maximum: 11 },
+ item_ids: { type: "array", minItems: 1, maxItems: 400, items: string },
+ lines: { type: "array", minItems: 1, maxItems: 400, items: object({
+ line_id: string, item_ids: { type: "array", minItems: 1, maxItems: 400, items: string }, text: string,
+ }) },
+ }) },
+ html: string, html_format: { const: "header_neutral_html" },
+ html_bytes: { type: "integer", minimum: 1, maximum: 20000 },
+ review_notice: { const: TABLE_REVIEW_NOTICE }, candidate_sha256: sha256Digest,
+});
const tableProposalCheckStatus = enumString(["passed", "failed", "not_run"]);
const verifiedTableCell = object({
row: { type: "integer", minimum: 0 },
diff --git a/scripts/macos-claude-installed-shannon.mjs b/scripts/macos-claude-installed-shannon.mjs
index 7ee10e6..184a213 100755
--- a/scripts/macos-claude-installed-shannon.mjs
+++ b/scripts/macos-claude-installed-shannon.mjs
@@ -23,7 +23,7 @@ if (!process.argv[2] || !process.argv[3]) {
//
const EXPECTED_SOURCE_SHA256 = "6e4e3411984f3edf99dbfe8b941cb5e8a321379ff0cae6ae5c1f592ad8882ca8";
const EXPECTED_MARKDOWN_SHA256 = "b9bd7ca7db26fe807f294565c39de8d4fe9337908afa74cd7e2896a800c92991";
-const EXPECTED_TOOL_CONTRACT_SHA256 = "17e2067e2c014bfc77e234432af45f7711494104f2a14d5f652e42e917251128";
+const EXPECTED_TOOL_CONTRACT_SHA256 = "2239a74dcdb71387f855c49bb26bc01e52ca9a14e773be13a32d19d3af9ec2d4";
const EXPECTED_PAGE_COUNT = 55;
const EXPECTED_GAP_COUNT = 26;
const EXPECTED_REPLACEMENT_CHARACTER_COUNT = 6;
diff --git a/scripts/macos-claude-installed-smoke.mjs b/scripts/macos-claude-installed-smoke.mjs
index e9a2d73..9460f72 100755
--- a/scripts/macos-claude-installed-smoke.mjs
+++ b/scripts/macos-claude-installed-smoke.mjs
@@ -33,7 +33,7 @@ const textFixture = path.join(fixtureDirectory, "synthetic-text-two-page.pdf");
const rasterFixture = path.join(fixtureDirectory, "synthetic-raster-only.pdf");
const mutationDirectory = path.join(fixtureDirectory, "mutation-output");
const toolNames = [];
-const EXPECTED_TOOL_CONTRACT_SHA256 = "17e2067e2c014bfc77e234432af45f7711494104f2a14d5f652e42e917251128";
+const EXPECTED_TOOL_CONTRACT_SHA256 = "2239a74dcdb71387f855c49bb26bc01e52ca9a14e773be13a32d19d3af9ec2d4";
const ACCESSIBILITY_CONCLUSION_KEYS = Object.freeze([
"certification",
"document_accessibility",
diff --git a/server/index.js b/server/index.js
index b6ff513..5d08a04 100755
--- a/server/index.js
+++ b/server/index.js
@@ -3852,7 +3852,7 @@ async function listTools(request) {
include_page_boundaries: { type: "boolean", description: "Include deterministic HTML comments marking page boundaries. Default: true." },
compact: { type: "boolean", description: "Opt into counted dot-leader, isolated page-number, and Unicode-letter spaced-hyphen normalizations. Default: false." },
remove_page_furniture: { type: "boolean", description: "Remove only source-evidenced extreme-margin page furniture and report every removed line by typed gap and counts. Set false to preserve all source lines. Default: true." },
- emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." },
+ emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. A narrowly qualifying captioned three-rule table may also include a draft arrangement and header-neutral HTML preview marked needs_review; its row/column grouping and header roles are unverified. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." },
allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.23.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." },
output_path: { type: "string", description: "Optional absolute .md path, or ~/ path. The file is written only after complete bytes are staged and verified." },
overwrite: { type: "boolean", description: "Replace an existing output_path only when its exact expected_output_identity is also supplied. Default: false." },
@@ -6082,6 +6082,7 @@ async function handleToolCall(request) {
saved_output: savedOutput,
...(tableProposals !== undefined ? { table_proposals: tableProposals } : {}),
};
+ const reviewCandidates = tableProposals?.filter(region => region.review_candidate) ?? [];
const summary = [
`Converted pages ${layout.page_range.start_page}-${layout.page_range.end_page} of ${fileName} to deterministic Markdown.`,
`Status: ${rendered.conversion_status}. UTF-8 bytes: ${rendered.markdown_bytes}.`,
@@ -6091,8 +6092,14 @@ async function handleToolCall(request) {
...(pagesNeedingVision.length > 0
? [`Vision routing: use render_pdf_page for pages ${pagesNeedingVision.map(entry => entry.page).join(", ")}.`]
: []),
- "No OCR, external model, hidden link-target recovery, or table inference beyond complete source-bound text, ruled, or painted grids was performed.",
+ reviewCandidates.length > 0
+ ? "The optional table drafts use whitespace alignment and need review. No OCR, external model or hidden link-target recovery was performed."
+ : "No OCR, external model, hidden link-target recovery, or table inference beyond complete source-bound text, ruled, or painted grids was performed.",
rendered.markdown,
+ ...reviewCandidates.map(region => [
+ `Page ${region.page}, ${region.region_id}: ${region.review_candidate.review_notice}`,
+ region.review_candidate.html,
+ ].join("\n\n")),
].join("\n\n");
return {
content: [{ type: "text", text: summary }],
diff --git a/server/markdown-conversion.js b/server/markdown-conversion.js
index 324092d..4df92b2 100644
--- a/server/markdown-conversion.js
+++ b/server/markdown-conversion.js
@@ -60,6 +60,7 @@ export const MAX_TABLE_PROPOSAL_TEXT_ITEMS = 400;
const MAX_TABLE_PROPOSAL_RULED_RECTS = 400;
const MAX_TABLE_PROPOSAL_RULING_SEGMENTS = 400;
const MAX_TABLE_PROPOSAL_PAINTED_RECTS = 400;
+export const TABLE_REVIEW_NOTICE = "Proposed arrangement, needs review. Check the row and column grouping against the PDF before using it; headers and cell structure are unverified.";
const MAX_MARKDOWN_BYTES_LIMIT = 200_000;
const GAP_CODES = new Set([
@@ -2815,6 +2816,24 @@ function tableProposalHeaderHints(run) {
};
}
+function joinedThinFillBands(rectangles) {
+ const tolerance = 0.05;
+ const bars = rectangles.filter(rect => ["fill", "eoFill"].includes(rect.verb)
+ && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5)
+ .sort((a, b) => a.y - b.y || a.x - b.x);
+ const bands = [];
+ for (const bar of bars) {
+ const prior = bands.at(-1);
+ if (prior && Math.abs(prior.y - bar.y) <= tolerance
+ && Math.abs(prior.height - bar.height) <= tolerance
+ && Math.abs(prior.x + prior.width - bar.x) <= tolerance) {
+ prior.width = bar.x + bar.width - prior.x;
+ prior.parts.push(bar);
+ } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] });
+ }
+ return bands;
+}
+
function threeRuleProposalEnvelope(page, run) {
// This refines an already abandoned proposal, never reconstructs a table.
// Three full-width paint bands and a nearby caption can bound the source
@@ -2832,19 +2851,7 @@ function threeRuleProposalEnvelope(page, run) {
const text = page.raw_items.filter(item => item.text_kind === "non_whitespace");
if (text.some(item => !item.geometry_valid || !item.bbox)) return null;
const tolerance = 0.05;
- const bars = evidence.items.filter(rect => ["fill", "eoFill"].includes(rect.verb)
- && rect.height >= 0.05 && rect.height <= 2 && rect.width >= 5)
- .sort((a, b) => a.y - b.y || a.x - b.x);
- const bands = [];
- for (const bar of bars) {
- const prior = bands.at(-1);
- if (prior && Math.abs(prior.y - bar.y) <= tolerance
- && Math.abs(prior.height - bar.height) <= tolerance
- && Math.abs(prior.x + prior.width - bar.x) <= tolerance) {
- prior.width = bar.x + bar.width - prior.x;
- prior.parts.push(bar);
- } else bands.push({ x: bar.x, y: bar.y, width: bar.width, height: bar.height, parts: [bar] });
- }
+ const bands = joinedThinFillBands(evidence.items);
const contains = (outer, inner) => inner.x >= outer.x && inner.y >= outer.y
&& inner.x + inner.width <= outer.x + outer.width
&& inner.y + inner.height <= outer.y + outer.height;
@@ -2888,17 +2895,109 @@ function threeRuleProposalEnvelope(page, run) {
|| !lines.some(line => line.y + line.height <= middle.y)
|| !lines.some(line => line.y >= middle.y + middle.height)) continue;
const itemById = new Map(page.raw_items.map(item => [item.id, item]));
- candidates.push({ bbox, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) });
+ candidates.push({ bbox, middle, run: lines.map(line => ({ line, cells: lineCells(line, itemById) })) });
}
return candidates.length === 1 ? candidates[0] : null;
}
+function canonicalReviewCandidate(value) {
+ if (Array.isArray(value)) return `[${value.map(canonicalReviewCandidate).join(",")}]`;
+ if (value && typeof value === "object") return `{${Object.keys(value).sort()
+ .map(key => `${JSON.stringify(key)}:${canonicalReviewCandidate(value[key])}`).join(",")}}`;
+ return JSON.stringify(value);
+}
+
+function threeRuleReviewCandidate(region, envelope, sourceSha256) {
+ if (!/^[a-f0-9]{64}$/.test(sourceSha256 ?? "")
+ || Object.values(region.truncation).some(value => value !== "complete")) return null;
+ const items = region.text_items;
+ if (items.length === 0 || new Set(items.map(item => item.id)).size !== items.length) return null;
+ const { middle } = envelope;
+ const upper = items.filter(item => item.bbox.y + item.bbox.height <= middle.y);
+ const lower = items.filter(item => item.bbox.y >= middle.y + middle.height);
+ if (upper.length === 0 || upper.length + lower.length !== items.length) return null;
+ const byLine = new Map();
+ for (const item of lower) {
+ if (!byLine.has(item.line_id)) byLine.set(item.line_id, []);
+ byLine.get(item.line_id).push(item);
+ }
+ const rows = [...byLine.values()].map(row => ({
+ items: row.sort((a, b) => a.bbox.x - b.bbox.x || a.reading_order_index - b.reading_order_index),
+ top: Math.min(...row.map(item => item.bbox.y)),
+ bottom: Math.max(...row.map(item => item.bbox.y + item.bbox.height)),
+ })).sort((a, b) => a.top - b.top);
+ if (rows.length < 2 || rows.length > 99 || rows.some((row, index) => (
+ index > 0 && row.top <= rows[index - 1].bottom
+ ))) return null;
+ // A maximal set of whitespace corridors shared by every body line supplies
+ // one deterministic proposed partition. Whitespace does not prove topology.
+ const minGap = Math.max(5, ...lower.map(item => item.bbox.height));
+ let corridors = null;
+ let gapCount = null;
+ for (const row of rows) {
+ const gaps = [];
+ let right = row.items[0].bbox.x + row.items[0].bbox.width;
+ for (const item of row.items.slice(1)) {
+ if (item.bbox.x - right >= minGap) gaps.push([right, item.bbox.x]);
+ right = Math.max(right, item.bbox.x + item.bbox.width);
+ }
+ if (gapCount !== null && gaps.length !== gapCount) return null;
+ gapCount = gaps.length;
+ corridors = corridors === null ? gaps : corridors.flatMap(([left, end]) => gaps
+ .map(([start, rightEdge]) => [Math.max(left, start), Math.min(end, rightEdge)])
+ .filter(([start, rightEdge]) => rightEdge - start >= minGap));
+ }
+ if (corridors.length < 1 || corridors.length > 11 || corridors.length !== gapCount) return null;
+ const cuts = corridors.map(([left, right]) => (left + right) / 2);
+ const sourceRows = [upper, ...rows.map(row => row.items)];
+ const cells = [];
+ for (const [row, sourceItems] of sourceRows.entries()) {
+ const columns = Array.from({ length: cuts.length + 1 }, () => []);
+ for (const item of sourceItems) {
+ const column = cuts.filter(cut => item.bbox.x >= cut).length;
+ if (column < cuts.length && item.bbox.x + item.bbox.width > cuts[column]) return null;
+ columns[column].push(item);
+ }
+ if (columns.some(column => column.length === 0)) return null;
+ for (const [column, assigned] of columns.entries()) {
+ assigned.sort((a, b) => a.reading_order_index - b.reading_order_index);
+ const lines = [];
+ for (const item of assigned) {
+ if (lines.at(-1)?.line_id !== item.line_id) {
+ if (lines.some(line => line.line_id === item.line_id)) return null;
+ lines.push({ line_id: item.line_id, item_ids: [], text: "" });
+ }
+ lines.at(-1).item_ids.push(item.id);
+ lines.at(-1).text += item.text;
+ }
+ cells.push({ row, column, item_ids: assigned.map(item => item.id), lines });
+ }
+ }
+ const assigned = cells.flatMap(cell => cell.item_ids);
+ if (new Set(assigned).size !== items.length || assigned.length !== items.length) return null;
+ const grid = sourceRows.map((_, row) => cells.filter(cell => cell.row === row)
+ .map(cell => cell.lines.map(line => escapeHeaderNeutralText(line.text)).join("
")));
+ const html = ["", ...grid.map(row => `${row.map(text => `| ${text} | `).join("")}
`), "
"].join("\n");
+ const candidate = {
+ algorithm: { name: "pdf-tools.three-rule-review-candidate", version: "0.1.0" },
+ status: "needs_review", topology_verified: false, header_roles_verified: false,
+ source_sha256: sourceSha256, region_id: region.region_id, page: region.page,
+ content_origin: "source_text_items", structure_origin: "whitespace_alignment_proposal",
+ row_count: sourceRows.length, column_count: cuts.length + 1,
+ source_item_count: items.length, assigned_item_count: assigned.length, unassigned_item_count: 0,
+ cells, html, html_format: "header_neutral_html", html_bytes: utf8Bytes(html),
+ review_notice: TABLE_REVIEW_NOTICE,
+ };
+ if (candidate.html_bytes > 20_000 || utf8Bytes(canonicalReviewCandidate(candidate)) > 40_000) return null;
+ return { ...candidate, candidate_sha256: sha256(canonicalReviewCandidate(candidate)) };
+}
+
/**
* Build a bounded proposal from an abandoned run. A uniquely captioned
* three-rule envelope can refine source membership but supplies no topology
* or header authority. Pure over source-validated IR; no token, time or I/O.
*/
-export function buildTableProposalRegion(page, run, reason, ordinal) {
+export function buildTableProposalRegion(page, run, reason, ordinal, { sourceSha256 = null } = {}) {
const envelope = threeRuleProposalEnvelope(page, run);
if (envelope) run = envelope.run;
const itemById = new Map(page.raw_items.map(item => [item.id, item]));
@@ -2962,7 +3061,7 @@ export function buildTableProposalRegion(page, run, reason, ordinal) {
bbox: painted.bbox,
}));
- return {
+ const proposal = {
region_id: `p${page.page}-t${ordinal}`,
page: page.page,
reason: regionGapCode(reason),
@@ -2983,6 +3082,11 @@ export function buildTableProposalRegion(page, run, reason, ordinal) {
painted_rectangles: paintedTruncated ? "truncated" : "complete",
},
};
+ if (envelope) {
+ const candidate = threeRuleReviewCandidate(proposal, envelope, sourceSha256);
+ if (candidate) proposal.review_candidate = candidate;
+ }
+ return proposal;
}
// escapePlainMarkdown already escapes "|" (and backslashes before it), so a
@@ -3007,15 +3111,18 @@ function renderTable(grid) {
return lines;
}
-function renderHeaderlessTable(grid) {
- const escape = value => sanitizeUnsafeText(String(value)).replace(/&/gu, "&")
+function escapeHeaderNeutralText(value) {
+ return sanitizeUnsafeText(String(value)).replace(/&/gu, "&")
.replace(//gu, ">")
.replace(/"/gu, """).replace(/'/gu, "'")
.replace(/https?:/giu, match => `${match.slice(0, -1)}:`)
.replace(/\b(?:[\p{L}\p{N}-]+\.)+[\p{L}]{2,}\b/giu, domain => domain.replace(".", "."))
.replace(/@(?=[\p{L}\p{N}.-]+.[\p{L}]{2,})/giu, "@");
+}
+
+function renderHeaderlessTable(grid) {
return ["", ...grid.map(row => (
- `${row.map(value => `| ${escape(value)} | `).join("")}
`
+ `${row.map(value => `| ${escapeHeaderNeutralText(value)} | `).join("")}
`
)), "
"];
}
@@ -3370,6 +3477,7 @@ function renderPage(page, {
pageBoundaryBefore = false,
pageBoundaryAfter = false,
collectTableProposals = false,
+ sourceSha256 = null,
allowHeaderlessTables = false,
pageFurniture = [],
} = {}) {
@@ -3488,7 +3596,7 @@ function renderPage(page, {
// markdown, gaps, or byte counts. It is only present when the opt-in flag
// is set, so the default page shape is byte-identical.
rendered.tableProposalRegions = analysis.regions.map(
- (region, index) => buildTableProposalRegion(page, region.run, region.reason, index + 1),
+ (region, index) => buildTableProposalRegion(page, region.run, region.reason, index + 1, { sourceSha256 }),
);
}
return rendered;
@@ -3706,6 +3814,28 @@ export function validateMarkdownConversionSemantics(result, { layout = null } =
if (proposals !== undefined) {
validateTableProposals(proposals, { requireToken: payloadProposals !== undefined });
validateTableProposalDocumentTruncation(result.table_proposals_truncation, proposals);
+ if (layout !== null) {
+ const expected = boundTableProposalRegions(layout.pages.flatMap(page => (
+ segmentPageLines(page, allowHeaderlessTables).regions.map((region, index) => (
+ buildTableProposalRegion(page, region.run, region.reason, index + 1,
+ { sourceSha256: layout.source.sha256 })
+ ))
+ )));
+ const actual = proposals.map(({ proposal_token: _token, ...proposal }) => proposal);
+ assertion(sameJson(actual, expected.regions), "table proposals do not match the bound layout IR");
+ assertion(sameJson(result.table_proposals_truncation, expected.truncation),
+ "table proposal truncation does not match the bound layout IR");
+ }
+ for (const proposal of proposals) {
+ if (proposal.review_candidate) {
+ const { candidate_sha256: digest, ...candidate } = proposal.review_candidate;
+ assertion(candidate.source_sha256 === result.provenance.source.sha256
+ && candidate.region_id === proposal.region_id && candidate.page === proposal.page,
+ "table review candidate source binding mismatch");
+ assertion(digest === sha256(canonicalReviewCandidate(candidate)), "table review candidate digest mismatch");
+ validateTableReviewCandidate(proposal, candidate);
+ }
+ }
} else {
assertion(result.table_proposals_truncation === undefined,
"table proposal truncation cannot exist without proposals");
@@ -3713,6 +3843,20 @@ export function validateMarkdownConversionSemantics(result, { layout = null } =
return result;
}
+function validateTableReviewCandidate(region, candidate) {
+ const bands = joinedThinFillBands(region.ruled_rects);
+ assertion(bands.length === 3 && bands.every(band => Math.abs(band.x - region.bbox.x) <= 0.05
+ && Math.abs(band.width - region.bbox.width) <= 0.05)
+ && Math.abs(bands[0].y - region.bbox.y) <= 0.05
+ && Math.abs(bands[2].y + bands[2].height - region.bbox.y - region.bbox.height) <= 0.05,
+ "table review candidate rule envelope mismatch");
+ const replay = threeRuleReviewCandidate(region, { middle: bands[1] }, candidate.source_sha256);
+ assertion(replay !== null, "table review candidate geometry is unsupported");
+ const { candidate_sha256: _digest, ...replayed } = replay;
+ assertion(canonicalReviewCandidate(candidate) === canonicalReviewCandidate(replayed),
+ "table review candidate does not match its retained source geometry");
+}
+
function isFiniteNumber(value) {
return typeof value === "number" && Number.isFinite(value);
}
@@ -3985,6 +4129,7 @@ export function renderPdfLayoutToMarkdown(layout, {
// number from a single-page selection.
pageBoundaryAfter: true,
collectTableProposals: emitTableProposals,
+ sourceSha256: layout.source.sha256,
allowHeaderlessTables,
pageFurniture: furniturePlans.get(page.page),
}));
diff --git a/server/output-schemas.js b/server/output-schemas.js
index 8d11e92..3c984ec 100644
--- a/server/output-schemas.js
+++ b/server/output-schemas.js
@@ -1,6 +1,6 @@
import { AjvJsonSchemaValidator } from "@modelcontextprotocol/server/validators/ajv";
import { validatePdfLayoutSemantics } from "./layout-extraction.js";
-import { validateMarkdownConversionSemantics } from "./markdown-conversion.js";
+import { TABLE_REVIEW_NOTICE, validateMarkdownConversionSemantics } from "./markdown-conversion.js";
import {
validatePdfObservationSemantics,
validateRenderObservationSemantics,
@@ -1172,6 +1172,27 @@ const tableProposal = object({
truncation: tableProposalTruncation,
proposal_token: sha256Digest,
});
+// Additive only on opt-in proposal packets. Strict table acceptance is separate.
+tableProposal.properties.review_candidate = object({
+ algorithm: object({ name: { const: "pdf-tools.three-rule-review-candidate" }, version: { const: "0.1.0" } }),
+ status: { const: "needs_review" }, topology_verified: { const: false }, header_roles_verified: { const: false },
+ source_sha256: sha256Digest, region_id: string, page: { type: "integer", minimum: 1 },
+ content_origin: { const: "source_text_items" }, structure_origin: { const: "whitespace_alignment_proposal" },
+ row_count: { type: "integer", minimum: 3, maximum: 100 },
+ column_count: { type: "integer", minimum: 2, maximum: 12 },
+ source_item_count: { type: "integer", minimum: 1, maximum: 400 },
+ assigned_item_count: { type: "integer", minimum: 1, maximum: 400 }, unassigned_item_count: { const: 0 },
+ cells: { type: "array", minItems: 6, maxItems: 400, items: object({
+ row: { type: "integer", minimum: 0, maximum: 99 }, column: { type: "integer", minimum: 0, maximum: 11 },
+ item_ids: { type: "array", minItems: 1, maxItems: 400, items: string },
+ lines: { type: "array", minItems: 1, maxItems: 400, items: object({
+ line_id: string, item_ids: { type: "array", minItems: 1, maxItems: 400, items: string }, text: string,
+ }) },
+ }) },
+ html: string, html_format: { const: "header_neutral_html" },
+ html_bytes: { type: "integer", minimum: 1, maximum: 20000 },
+ review_notice: { const: TABLE_REVIEW_NOTICE }, candidate_sha256: sha256Digest,
+});
const tableProposalCheckStatus = enumString(["passed", "failed", "not_run"]);
const verifiedTableCell = object({
row: { type: "integer", minimum: 0 },
diff --git a/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json b/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json
index 10b81b3..2105fe9 100644
--- a/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json
+++ b/test/fixtures/eval/extraction/phase1/layout-occurrence-oracle.v1.json
@@ -79,14 +79,14 @@
{
"role": "markdown_conversion_module",
"path": "server/markdown-conversion.js",
- "bytes": 192276,
- "sha256": "838f9b1adb38faf5eb3b31d7be9479eb8521dff0fd7013dcd3c89fc3cbcb95d1"
+ "bytes": 200207,
+ "sha256": "988a3b3b4c9ebb27b66a63cd2703dd002519935c8dd8660a8af78f53621b257a"
},
{
"role": "output_schemas_module",
"path": "server/output-schemas.js",
- "bytes": 84427,
- "sha256": "beb9255b64456319bf2b747e643c32d3d175c29f51feddfdf5f419da19f9fc3c"
+ "bytes": 85973,
+ "sha256": "f26eea7b94ccda2faca95880afe33bb3186511b9cf7ac32b799b895ee84b226d"
},
{
"role": "package_json",
@@ -131,7 +131,7 @@
"sha256": "ef40501b2afbe4cd2adfa80480344783bbec1a00e8e9850086a5d044231d1f53"
}
],
- "validator_source_set_sha256": "6e875209853d97288e5eae1464c0c3a333f37b31ad3467839954aa4edbf657c4",
+ "validator_source_set_sha256": "1903def9a7de315d775ec2a4f0494778cefad449c88fdf8e9283fc2749eabf01",
"pdfjs_version": "5.4.624",
"generation_contract": {
"source_path": "source.pdf",
diff --git a/test/fixtures/eval/trajectories/tool-contracts.v3.json b/test/fixtures/eval/trajectories/tool-contracts.v3.json
index ff68473..9c7cfd0 100644
--- a/test/fixtures/eval/trajectories/tool-contracts.v3.json
+++ b/test/fixtures/eval/trajectories/tool-contracts.v3.json
@@ -5,7 +5,7 @@
"name": "pdf-tools",
"version": "0.14.0"
},
- "tools_sha256": "93202e8e6e04b8914f81003624692177e9dbe02c6fc93aa877ebf7e82b2556a7",
+ "tools_sha256": "cbd4f2614576b123130ad430542f58cf07ca3730026c59545e7fe4b447dce4bf",
"tools": [
{
"name": "add_signature_field",
@@ -554,7 +554,7 @@
},
"emit_table_proposals": {
"type": "boolean",
- "description": "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false."
+ "description": "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. A narrowly qualifying captioned three-rule table may also include a draft arrangement and header-neutral HTML preview marked needs_review; its row/column grouping and header roles are unverified. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false."
},
"allow_headerless_tables": {
"type": "boolean",
diff --git a/test/markdown-conversion.test.js b/test/markdown-conversion.test.js
index c1a397b..c84789f 100644
--- a/test/markdown-conversion.test.js
+++ b/test/markdown-conversion.test.js
@@ -15,6 +15,13 @@ import {
validateMarkdownConversionSemantics,
} from "../server/markdown-conversion.js";
+function canonicalCandidate(value) {
+ if (Array.isArray(value)) return `[${value.map(canonicalCandidate).join(",")}]`;
+ if (value && typeof value === "object") return `{${Object.keys(value).sort()
+ .map(key => `${JSON.stringify(key)}:${canonicalCandidate(value[key])}`).join(",")}}`;
+ return JSON.stringify(value);
+}
+
function multiply(left, right) {
return [
left[0] * right[0] + left[2] * right[1],
@@ -471,12 +478,12 @@ describe("layout Markdown renderer", () => {
textItem("Table 1. Example", { top: 88, left: 50, fontSize: 10 }),
textItem("Concentration", { top: 113, left: 60, fontSize: 9 }),
textItem("level", { top: 124, left: 60, fontSize: 9 }),
- textItem("A", { top: 120, left: 140, fontSize: 9, eol: false }),
- textItem("B", { top: 120, left: 220, fontSize: 9 }),
+ textItem("A", { top: 120, left: 150, fontSize: 9, eol: false }),
+ textItem("B", { top: 120, left: 230, fontSize: 9 }),
...["Control", "Low", "High"].flatMap((label, index) => [
textItem(label, { top: 145 + 15 * index, left: 60, fontSize: 9, eol: false }),
- textItem(String(index), { top: 145 + 15 * index, left: 140, fontSize: 9, eol: false }),
- textItem(String(index + 1), { top: 145 + 15 * index, left: 220, fontSize: 9 }),
+ textItem(String(index), { top: 145 + 15 * index, left: 150, fontSize: 9, eol: false }),
+ textItem(String(index + 1), { top: 145 + 15 * index, left: 230, fontSize: 9 }),
]),
textItem("3. Results", { top: 205, left: 50, fontSize: 10 }),
];
@@ -496,10 +503,39 @@ describe("layout Markdown renderer", () => {
"Concentration", "level", "A", "B", "Control", "0", "1", "Low", "1", "2", "High", "2", "3",
]);
expect(region.header_hints.status).toBe("unavailable");
+ expect(region.review_candidate).toMatchObject({ status: "needs_review", row_count: 4, column_count: 3 });
const mapped = analyzeValidatedPdfPagesForDocumentMap(layout.pages);
- expect(mapped[0].table_regions).toEqual(result.table_proposal_regions);
+ expect(mapped[0].table_regions).toEqual(result.table_proposal_regions.map(({ review_candidate: _candidate, ...item }) => item));
expect(analyzeValidatedPdfPagesForDocumentMap(layout.pages)).toEqual(mapped);
expect(() => validateMarkdownConversionSemantics(result, { layout })).not.toThrow();
+ for (const mutate of [
+ candidate => { candidate.topology_verified = true; },
+ candidate => { candidate.cells[0].lines[0].text = "invented"; },
+ candidate => { candidate.cells[0].item_ids.pop(); },
+ candidate => { candidate.cells[0].item_ids.push(candidate.cells[1].item_ids[0]); },
+ candidate => { candidate.html += ""; },
+ candidate => { candidate.source_sha256 = "f".repeat(64); },
+ candidate => { candidate.cells[0].column = 1; },
+ ]) {
+ const changed = structuredClone(result);
+ const candidate = changed.table_proposal_regions.find(value => value.review_candidate).review_candidate;
+ mutate(candidate);
+ const { candidate_sha256: _digest, ...body } = candidate;
+ candidate.candidate_sha256 = createHash("sha256").update(canonicalCandidate(body)).digest("hex");
+ expect(() => validateMarkdownConversionSemantics(changed)).toThrow();
+ expect(() => validateMarkdownConversionSemantics(changed, { layout })).toThrow();
+ }
+ // Even a content-consistent, freshly rehashed regrouping must fail the
+ // packet-only geometry replay as well as the complete bound-IR replay.
+ const regrouped = structuredClone(result);
+ const draft = regrouped.table_proposal_regions.find(value => value.review_candidate).review_candidate;
+ [draft.cells[1].item_ids, draft.cells[2].item_ids] = [draft.cells[2].item_ids, draft.cells[1].item_ids];
+ [draft.cells[1].lines, draft.cells[2].lines] = [draft.cells[2].lines, draft.cells[1].lines];
+ draft.html = draft.html.replace("A | B | ", "B | A | ");
+ const { candidate_sha256: _digest, ...body } = draft;
+ draft.candidate_sha256 = createHash("sha256").update(canonicalCandidate(body)).digest("hex");
+ expect(() => validateMarkdownConversionSemantics(regrouped)).toThrow(/retained source geometry/);
+ expect(() => validateMarkdownConversionSemantics(regrouped, { layout })).toThrow(/bound layout IR/);
});
it("emits only inert data cells for an opted-in complete source-covered headerless grid", async () => {
diff --git a/test/mcp-contract.test.js b/test/mcp-contract.test.js
index 9e8b58e..09385f5 100644
--- a/test/mcp-contract.test.js
+++ b/test/mcp-contract.test.js
@@ -242,7 +242,8 @@ const EXAMPLE_PDF = path.join(REPO_ROOT, "example-fw9.pdf");
// neutrally described and no longer exposes arbitrary credential headers.
// 2026-10-20: version the optional tight-grid/caption repair as renderer 1.23.0.
// 2026-10-20: private local password entry replaces public credential inputs.
-const TOOL_CONTRACT_SHA256 = "17e2067e2c014bfc77e234432af45f7711494104f2a14d5f652e42e917251128";
+// 2026-10-21: optional table packets describe draft arrangements that require review.
+const TOOL_CONTRACT_SHA256 = "2239a74dcdb71387f855c49bb26bc01e52ca9a14e773be13a32d19d3af9ec2d4";
const CLOSED_READ = Object.freeze({
readOnlyHint: true,
diff --git a/test/table-review-candidate.test.js b/test/table-review-candidate.test.js
new file mode 100644
index 0000000..56b7ba5
--- /dev/null
+++ b/test/table-review-candidate.test.js
@@ -0,0 +1,162 @@
+import { mkdtemp, writeFile, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { PDFDocument, StandardFonts } from "pdf-lib";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { buildTableProposalRegion } from "../server/markdown-conversion.js";
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const sourceSha256 = "a".repeat(64);
+function fixture() {
+ const entries = [
+ ["Table 1. Synthetic measurements", 50, 88, 160, "caption"],
+ ["Concentration", 60, 114, 55, "top-1"],
+ ["level", 70, 126, 20, "top-2"],
+ ["A", 145, 120, 10, "top-3"], ["B", 225, 120, 10, "top-3"],
+ ...["Control", "Low", "High"].flatMap((text, index) => [
+ [text, 60, 148 + index * 15, 40, `body-${index}`],
+ [String(index), 140, 148 + index * 15, 8, `body-${index}`],
+ [String(index + 1), 220, 148 + index * 15, 8, `body-${index}`],
+ ]),
+ ];
+ const raw_items = entries.map(([text, x, y, width, line_id], index) => ({
+ id: `item-${index}`, text, bbox: { x, y, width, height: 8 }, line_id,
+ reading_order_index: index, geometry_valid: true, text_kind: "non_whitespace",
+ }));
+ const lines = [...new Set(entries.map(row => row[4]))].map(id => {
+ const items = raw_items.filter(item => item.line_id === id);
+ const x = Math.min(...items.map(item => item.bbox.x));
+ const y = Math.min(...items.map(item => item.bbox.y));
+ return { id, x, y, width: Math.max(...items.map(item => item.bbox.x + item.bbox.width)) - x,
+ height: 8, text: items.map(item => item.text).join(" "), item_ids: items.map(item => item.id) };
+ });
+ const bars = [110, 140, 195].flatMap(y => [
+ { x: 50, y, width: 120, height: 1, verb: "fill" },
+ { x: 170, y, width: 100, height: 1, verb: "fill" },
+ ]);
+ return { page: { page: 1, raw_items, lines, truncation: { truncated: false },
+ ruled_rects: { status: "available", observed_count: 6, returned_count: 6, items: bars },
+ ruling_segments: { status: "available", truncated: false, observed_count: 0, returned_count: 0, items: [] },
+ painted_rectangles: { status: "available", truncated: false, observed_count: 0, returned_count: 0, items: [] } },
+ run: lines.slice(3).map(line => ({ line, cells: raw_items.filter(item => item.line_id === line.id) })) };
+}
+function packet(input, sha = sourceSha256) {
+ return buildTableProposalRegion(input.page, input.run, "topology", 1, { sourceSha256: sha });
+}
+
+describe("three-rule table arrangement for review", () => {
+ it("produces one deterministic complete draft with multiline labels and no header claim", () => {
+ const input = fixture();
+ const before = structuredClone(input);
+ const result = packet(input);
+ const candidate = result.review_candidate;
+ expect(candidate).toMatchObject({ status: "needs_review", topology_verified: false,
+ header_roles_verified: false, row_count: 4, column_count: 3,
+ source_item_count: 13, assigned_item_count: 13, unassigned_item_count: 0, source_sha256: sourceSha256 });
+ expect(candidate.cells[0].lines.map(line => line.text)).toEqual(["Concentration", "level"]);
+ expect(candidate.html).toContain("Concentration level | ");
+ expect(candidate.html).not.toContain(" cell.item_ids).sort())
+ .toEqual(result.text_items.map(item => item.id).sort());
+ expect(packet(input)).toEqual(result);
+ expect(input).toEqual(before);
+ expect(packet(input, "b".repeat(64)).review_candidate.candidate_sha256).not.toBe(candidate.candidate_sha256);
+ expect(buildTableProposalRegion(input.page, input.run, "topology", 1)).not.toHaveProperty("review_candidate");
+ });
+
+ const attacks = {
+ missing_caption: input => { input.page.lines[0].text = "Ordinary paragraph"; },
+ competing_paint: input => {
+ input.page.ruled_rects.items.push({ x: 50, y: 160, width: 220, height: 1, verb: "fill" });
+ input.page.ruled_rects.observed_count++; input.page.ruled_rects.returned_count++;
+ },
+ truncated_text: input => { input.page.truncation.truncated = true; },
+ truncated_geometry: input => { input.page.ruled_rects.status = "truncated"; },
+ overlapping_body_rows: input => {
+ for (const item of input.page.raw_items.filter(item => item.line_id === "body-1")) item.bbox.y = 152;
+ },
+ wrapped_body_cell: input => { input.page.raw_items.find(item => item.text === "Low").line_id = "wrapped-body"; },
+ missing_cell: input => {
+ const item = input.page.raw_items.find(item => item.line_id === "body-1" && item.bbox.x === 220);
+ item.bbox.x = 142;
+ },
+ spanning_top_cell: input => { input.page.raw_items[1].bbox.width = 120; },
+ no_common_corridor: input => {
+ const item = input.page.raw_items.find(item => item.line_id === "body-1" && item.bbox.x === 140);
+ item.bbox.x = 90; item.bbox.width = 120;
+ },
+ oversized_preview: input => { input.page.raw_items[1].text = "X".repeat(40_001); },
+ };
+ it.each(Object.keys(attacks))("keeps evidence without a draft for %s", name => {
+ const input = fixture(); attacks[name](input);
+ expect(packet(input)).not.toHaveProperty("review_candidate");
+ });
+
+ it("escapes PDF markup and preserves exact source fragments", () => {
+ const input = fixture();
+ input.page.raw_items[1].text = ' & https://evil.test';
+ const result = packet(input);
+ const candidate = result.review_candidate;
+ expect(candidate.cells[0].lines[0].text).toBe(input.page.raw_items[1].text);
+ expect(candidate.html).toContain("<img");
+ expect(candidate.html).toContain("https://evil.test");
+ expect(candidate.html).not.toContain(" {
+ let directory, pdfPath, client, transport;
+ beforeAll(async () => {
+ directory = await mkdtemp(path.join(os.tmpdir(), "pdf-tools-table-review-"));
+ pdfPath = path.join(directory, "synthetic.pdf");
+ const pdf = await PDFDocument.create();
+ const page = pdf.addPage([612, 792]);
+ const font = await pdf.embedFont(StandardFonts.Helvetica);
+ const draw = (text, x, top) => page.drawText(text, { x, y: 792 - top - 9, size: 9, font });
+ draw("Table 1. Synthetic measurements", 50, 88);
+ draw("Concentration", 60, 114); draw("level", 70, 126);
+ draw("A", 150, 120); draw("B", 230, 120);
+ for (const [index, label] of ["Control", "Low", "High"].entries()) {
+ draw(label, 60, 148 + index * 15); draw(String(index), 150, 148 + index * 15);
+ draw(String(index + 1), 230, 148 + index * 15);
+ }
+ draw("3. Results", 50, 210);
+ for (const y of [110, 140, 195]) page.drawRectangle({ x: 50, y: 792 - y - 1, width: 220, height: 1 });
+ await writeFile(pdfPath, await pdf.save(), { flag: "wx" });
+ transport = new StdioClientTransport({ command: process.execPath,
+ args: [path.join(root, "server/index.js")], cwd: root,
+ env: { ALLOWED_DIRECTORIES: directory }, stderr: "ignore" });
+ client = new Client({ name: "table-review-test", version: "1.0.0" });
+ await client.connect(transport);
+ }, 30000);
+ afterAll(async () => { await client?.close(); await transport?.close(); await rm(directory, { recursive: true, force: true }); });
+
+ it("exposes the same draft and warning in text and structure while strict verification rejects", async () => {
+ const baseline = await client.callTool({ name: "convert_pdf_to_markdown", arguments: { pdf_path: pdfPath } });
+ const result = await client.callTool({ name: "convert_pdf_to_markdown", arguments: {
+ pdf_path: pdfPath, emit_table_proposals: true,
+ } });
+ expect(result.isError).not.toBe(true);
+ const region = result.structuredContent.table_proposals.find(item => item.review_candidate);
+ expect(region).toBeDefined();
+ const candidate = region.review_candidate;
+ expect(candidate).toMatchObject({ row_count: 4, column_count: 3, topology_verified: false });
+ expect(result.content[0].text).toContain(candidate.review_notice);
+ expect(result.content[0].text).toContain(candidate.html);
+ expect(result.structuredContent.markdown).toBe(baseline.structuredContent.markdown);
+ expect(result.structuredContent.gaps).toEqual(baseline.structuredContent.gaps);
+ const verification = await client.callTool({ name: "verify_table_proposal", arguments: {
+ pdf_path: pdfPath, region_id: region.region_id, proposal_token: region.proposal_token,
+ cells: candidate.cells.map(cell => ({ row: cell.row, column: cell.column,
+ rowspan: 1, colspan: 1, item_ids: cell.item_ids })),
+ } });
+ expect(verification.isError).not.toBe(true);
+ expect(verification.structuredContent.status).toBe("rejected");
+ expect(verification.structuredContent.table).toBeNull();
+ expect(verification.structuredContent.reason_codes).toContain("TABLE_PROPOSAL_TOPOLOGY_AMBIGUOUS");
+ expect(verification.structuredContent).not.toHaveProperty("review_candidate");
+ }, 30000);
+});
|