Preserve complete tight table grids with their captions

docs/HEADERLESS_TABLES.md

@@ -2,7 +2,7 @@
 
 `convert_pdf_to_markdown` accepts the explicit, default-off
 `allow_headerless_tables: true` option. The selected renderer profile is
-`1.22.0`; ordinary calls and explicit `false` retain profile `1.21.0` and
+`1.23.0`; ordinary calls and explicit `false` retain profile `1.21.0` and
 the previous complete result envelope.
 
 This addresses a narrow representational limit: GFM assigns header semantics
@@ -21,6 +21,15 @@ This does not broaden the captioned/solid-mask, segmented-rule, or unruled
 table paths. Those and existing GFM tables keep their prior rules. Source
 text is HTML-escaped, never interpreted as markup or an attribute.
 
+Profile `1.23.0` uses each complete text advance box when choosing its row,
+so tightly spaced rows do not become ambiguous merely because the top of
+one box is within the preceding row's tolerance. A single explicit `Table`
+caption may precede the complete grid in the same text run when its whole
+line is outside the grid. The caption stays ordinary text and is excluded
+from the cell proposal. Other extra rows, overlapping captions and text
+crossing cells still prevent conversion. The earlier `1.22.0` profile could
+reject these complete captioned grids; no header role is added by this repair.
+
 Example tool arguments:
 
 ```json

docs/MCP_CONTRACT.md

@@ -93,7 +93,7 @@ entrypoint with bounded value, history, citation, and page-coverage views.
 
 `convert_pdf_to_markdown` has a default-off `allow_headerless_tables` option
 for narrowly qualified complete ruled-rectangle grids. The opted-in renderer
-profile `1.22.0` emits escaped HTML data cells without assigning header roles;
+profile `1.23.0` emits escaped HTML data cells without assigning header roles;
 typed uncertainty remains. Omitted or false retains the unchanged default
 profile `1.21.0`. See [Header-neutral tables](HEADERLESS_TABLES.md).
 

pdf-toolkit-mcp-share/server/index.js

@@ -3788,7 +3788,7 @@ async function listTools(request) {
             compact: { type: "boolean", description: "Opt into counted dot-leader, isolated page-number, and Unicode-letter spaced-hyphen normalizations. Default: false." },
             remove_page_furniture: { type: "boolean", description: "Remove only source-evidenced extreme-margin page furniture and report every removed line by typed gap and counts. Set false to preserve all source lines. Default: true." },
             emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." },
-            allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.22.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." },
+            allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.23.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." },
             output_path: { type: "string", description: "Optional absolute .md path, or ~/ path. The file is written only after complete bytes are staged and verified." },
             overwrite: { type: "boolean", description: "Replace an existing output_path only when its exact expected_output_identity is also supplied. Default: false." },
             expected_output_identity: EXPECTED_MARKDOWN_OUTPUT_IDENTITY_INPUT_SCHEMA

pdf-toolkit-mcp-share/server/markdown-conversion.js

@@ -8,7 +8,7 @@ const RENDERER = Object.freeze({
   name: "pdf-tools.layout-markdown-renderer",
   version: "1.21.0",
 });
-const HEADERLESS_RENDERER = Object.freeze({ ...RENDERER, version: "1.22.0" });
+const HEADERLESS_RENDERER = Object.freeze({ ...RENDERER, version: "1.23.0" });
 const SUPPORTED_LAYOUT_IR_VERSION = "1.6.0";
 
 // Bounded geometric table inference. A run of adjacent lines is treated as a
@@ -2111,6 +2111,10 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables =
     }
     if (filledCells / totalCells < RECT_MIN_FILL_RATIO) return { reason: "ruling_unsupported" };
 
+    const candidateRows = lineRecords
+      .toSorted((left, right) => left.line.y - right.line.y)
+      .map(record => ({ line: record.line, cells: record.structuralInsideNonWhitespace }));
+    const headerless = !hasHeaderEvidence(candidateRows) && !hasFirstRowBandEvidence(bands);
     const cells = Array.from({ length: rowCount }, () => (
       Array.from({ length: columnCount }, () => [])
     ));
@@ -2129,7 +2133,8 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables =
             if (centerX >= xEdges[column] - RECT_CELL_ASSIGNMENT_SLACK
               && centerX <= xEdges[column + 1] + RECT_CELL_ASSIGNMENT_SLACK
               && box.y >= yEdges[row] - RECT_CELL_ASSIGNMENT_SLACK
-              && box.y <= yEdges[row + 1] + RECT_CELL_ASSIGNMENT_SLACK) {
+              && (allowHeaderlessTables && headerless ? rectBottom(box) : box.y)
+                <= yEdges[row + 1] + RECT_CELL_ASSIGNMENT_SLACK) {
               matches.push({ row, column });
             }
           }
@@ -2184,10 +2189,6 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables =
     if (emptyColumns.slice(firstColumn, lastColumn + 1).some(Boolean)) return { reason: "ruling_unsupported" };
     if (firstColumn > lastColumn) return { reason: "ruling_unsupported" };
     const trimmedGrid = grid.map(row => row.slice(firstColumn, lastColumn + 1));
-    const candidateRows = lineRecords
-      .sort((left, right) => left.line.y - right.line.y)
-      .map(record => ({ line: record.line, cells: record.structuralInsideNonWhitespace }));
-    const headerless = !hasHeaderEvidence(candidateRows) && !hasFirstRowBandEvidence(bands);
     if (headerless && !allowHeaderlessTables) {
       return { reason: "header" };
     }
@@ -2197,8 +2198,18 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables =
       // A partial rectangle census or a center-only cell assignment cannot
       // establish the complete grid this profile promises to preserve.
       const candidateLineIds = new Set(candidateRows.map(row => row.line.id));
-      if (candidateLineIds.size !== runLineIds.size
-        || [...runLineIds].some(id => !candidateLineIds.has(id))) {
+      const first = run.findIndex(row => candidateLineIds.has(row.line.id));
+      const last = run.findLastIndex(row => candidateLineIds.has(row.line.id));
+      const gridBox = { x: bounds.x, y: bounds.y,
+        width: bounds.right - bounds.x, height: bounds.bottom - bounds.y };
+      // One explicit caption can share the run with the grid. Keep it as
+      // text only when it precedes the complete contiguous grid and lies
+      // wholly outside its envelope. Other extra rows remain ambiguous.
+      if (first < 0 || last - first + 1 !== candidateLineIds.size
+        || run.slice(first, last + 1).some(row => !candidateLineIds.has(row.line.id))
+        || last !== run.length - 1 || first > 1
+        || (first === 1 && (!/^table\s+(?:\d+|[ivxlcdm]+)(?:[:.]|\s|$)/iu.test(run[0].line.text.trim())
+          || boxesOverlap(run[0].line, gridBox)))) {
         return { reason: "topology" };
       }
       // Vector paint makes an otherwise fully observed text page "partial"
@@ -2225,7 +2236,8 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables =
         }
       }
       // Keep empty outer slots rather than trimming source grid columns.
-      return { kind: "table", grid, rows: candidateRows, headerless: true };
+      return { kind: "table", grid, rows: candidateRows, headerless: true,
+        leadingRows: run.slice(0, first) };
     }
     return { kind: "table", grid: trimmedGrid, rows: candidateRows };
   }
@@ -2670,10 +2682,12 @@ function segmentTextRows(page, rows, itemById, ruledClusters, allowHeaderlessTab
         ? tryBuildRectGrid(page, run, ruledClusters, itemById, allowHeaderlessTables)
         : null;
       if (ruling?.kind === "table") {
-        segments.push({ kind: "table", rows: run, grid: ruling.grid, ...(ruling.headerless ? { headerless: true } : {}) });
+        if (ruling.leadingRows?.length) segments.push({ kind: "text", rows: ruling.leadingRows });
+        segments.push({ kind: "table", rows: ruling.headerless ? ruling.rows : run,
+          grid: ruling.grid, ...(ruling.headerless ? { headerless: true } : {}) });
         if (ruling.headerless) {
           tableReason = "header";
-          regions.push({ run, reason: "header" });
+          regions.push({ run: ruling.rows, reason: "header" });
         }
       } else {
         // Preserve the reason from the rect-evidence attempt. The text path

pdf-toolkit-mcp-share/server/output-schemas.js

@@ -1503,7 +1503,7 @@ export const TOOL_SUCCESS_OUTPUT_SCHEMAS = Object.freeze({
     {
       renderer: object({
         name: { const: "pdf-tools.layout-markdown-renderer" },
-        version: enumString(["1.21.0", "1.22.0"]),
+        version: enumString(["1.21.0", "1.23.0"]),
       }),
       conversion_status: enumString(["complete", "partial", "failed"]),
       markdown: string,

scripts/macos-claude-installed-shannon.mjs

@@ -23,7 +23,7 @@ if (!process.argv[2] || !process.argv[3]) {
 //
 const EXPECTED_SOURCE_SHA256 = "6e4e3411984f3edf99dbfe8b941cb5e8a321379ff0cae6ae5c1f592ad8882ca8";
 const EXPECTED_MARKDOWN_SHA256 = "b9bd7ca7db26fe807f294565c39de8d4fe9337908afa74cd7e2896a800c92991";
-const EXPECTED_TOOL_CONTRACT_SHA256 = "fccd8d1a5387816d4407868b8bebe26865b7e81318b5f20fb6ebb6209fc56386";
+const EXPECTED_TOOL_CONTRACT_SHA256 = "d5e40605f3f8d0fa656a798ed38400b3f12981250d8fcc9b05ff094a861dc17f";
 const EXPECTED_PAGE_COUNT = 55;
 const EXPECTED_GAP_COUNT = 26;
 const EXPECTED_REPLACEMENT_CHARACTER_COUNT = 6;

scripts/macos-claude-installed-smoke.mjs

@@ -33,7 +33,7 @@ const textFixture = path.join(fixtureDirectory, "synthetic-text-two-page.pdf");
 const rasterFixture = path.join(fixtureDirectory, "synthetic-raster-only.pdf");
 const mutationDirectory = path.join(fixtureDirectory, "mutation-output");
 const toolNames = [];
-const EXPECTED_TOOL_CONTRACT_SHA256 = "fccd8d1a5387816d4407868b8bebe26865b7e81318b5f20fb6ebb6209fc56386";
+const EXPECTED_TOOL_CONTRACT_SHA256 = "d5e40605f3f8d0fa656a798ed38400b3f12981250d8fcc9b05ff094a861dc17f";
 const ACCESSIBILITY_CONCLUSION_KEYS = Object.freeze([
   "certification",
   "document_accessibility",

server/index.js

@@ -3788,7 +3788,7 @@ async function listTools(request) {
             compact: { type: "boolean", description: "Opt into counted dot-leader, isolated page-number, and Unicode-letter spaced-hyphen normalizations. Default: false." },
             remove_page_furniture: { type: "boolean", description: "Remove only source-evidenced extreme-margin page furniture and report every removed line by typed gap and counts. Set false to preserve all source lines. Default: true." },
             emit_table_proposals: { type: "boolean", description: "Opt into emitting one bounded, deterministic table_proposals packet per abandoned table region (TABLE_TOPOLOGY_UNKNOWN or TABLE_RULING_UNSUPPORTED), carrying the region's text items, ruled and painted evidence, header hints, page, bbox, coordinate_space, and a source-and-IR-bound proposal_token so a host model can propose a structure for later read-only verification. Additive: the abstention gap is unchanged and default output stays byte-identical. Default: false." },
-            allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.22.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." },
+            allow_headerless_tables: { type: "boolean", description: "Opt into header-neutral HTML data cells for a complete source-covered ruled-rectangle cell grid without header evidence. No header roles, spans or blank data values are inferred; incomplete, ambiguous and cross-cell geometry remains text with typed gaps. Uses renderer profile 1.23.0; default false keeps profile 1.21.0 and unchanged output. No OCR or model." },
             output_path: { type: "string", description: "Optional absolute .md path, or ~/ path. The file is written only after complete bytes are staged and verified." },
             overwrite: { type: "boolean", description: "Replace an existing output_path only when its exact expected_output_identity is also supplied. Default: false." },
             expected_output_identity: EXPECTED_MARKDOWN_OUTPUT_IDENTITY_INPUT_SCHEMA

server/markdown-conversion.js

@@ -8,7 +8,7 @@ const RENDERER = Object.freeze({
   name: "pdf-tools.layout-markdown-renderer",
   version: "1.21.0",
 });
-const HEADERLESS_RENDERER = Object.freeze({ ...RENDERER, version: "1.22.0" });
+const HEADERLESS_RENDERER = Object.freeze({ ...RENDERER, version: "1.23.0" });
 const SUPPORTED_LAYOUT_IR_VERSION = "1.6.0";
 
 // Bounded geometric table inference. A run of adjacent lines is treated as a
@@ -2111,6 +2111,10 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables =
     }
     if (filledCells / totalCells < RECT_MIN_FILL_RATIO) return { reason: "ruling_unsupported" };
 
+    const candidateRows = lineRecords
+      .toSorted((left, right) => left.line.y - right.line.y)
+      .map(record => ({ line: record.line, cells: record.structuralInsideNonWhitespace }));
+    const headerless = !hasHeaderEvidence(candidateRows) && !hasFirstRowBandEvidence(bands);
     const cells = Array.from({ length: rowCount }, () => (
       Array.from({ length: columnCount }, () => [])
     ));
@@ -2129,7 +2133,8 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables =
             if (centerX >= xEdges[column] - RECT_CELL_ASSIGNMENT_SLACK
               && centerX <= xEdges[column + 1] + RECT_CELL_ASSIGNMENT_SLACK
               && box.y >= yEdges[row] - RECT_CELL_ASSIGNMENT_SLACK
-              && box.y <= yEdges[row + 1] + RECT_CELL_ASSIGNMENT_SLACK) {
+              && (allowHeaderlessTables && headerless ? rectBottom(box) : box.y)
+                <= yEdges[row + 1] + RECT_CELL_ASSIGNMENT_SLACK) {
               matches.push({ row, column });
             }
           }
@@ -2184,10 +2189,6 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables =
     if (emptyColumns.slice(firstColumn, lastColumn + 1).some(Boolean)) return { reason: "ruling_unsupported" };
     if (firstColumn > lastColumn) return { reason: "ruling_unsupported" };
     const trimmedGrid = grid.map(row => row.slice(firstColumn, lastColumn + 1));
-    const candidateRows = lineRecords
-      .sort((left, right) => left.line.y - right.line.y)
-      .map(record => ({ line: record.line, cells: record.structuralInsideNonWhitespace }));
-    const headerless = !hasHeaderEvidence(candidateRows) && !hasFirstRowBandEvidence(bands);
     if (headerless && !allowHeaderlessTables) {
       return { reason: "header" };
     }
@@ -2197,8 +2198,18 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables =
       // A partial rectangle census or a center-only cell assignment cannot
       // establish the complete grid this profile promises to preserve.
       const candidateLineIds = new Set(candidateRows.map(row => row.line.id));
-      if (candidateLineIds.size !== runLineIds.size
-        || [...runLineIds].some(id => !candidateLineIds.has(id))) {
+      const first = run.findIndex(row => candidateLineIds.has(row.line.id));
+      const last = run.findLastIndex(row => candidateLineIds.has(row.line.id));
+      const gridBox = { x: bounds.x, y: bounds.y,
+        width: bounds.right - bounds.x, height: bounds.bottom - bounds.y };
+      // One explicit caption can share the run with the grid. Keep it as
+      // text only when it precedes the complete contiguous grid and lies
+      // wholly outside its envelope. Other extra rows remain ambiguous.
+      if (first < 0 || last - first + 1 !== candidateLineIds.size
+        || run.slice(first, last + 1).some(row => !candidateLineIds.has(row.line.id))
+        || last !== run.length - 1 || first > 1
+        || (first === 1 && (!/^table\s+(?:\d+|[ivxlcdm]+)(?:[:.]|\s|$)/iu.test(run[0].line.text.trim())
+          || boxesOverlap(run[0].line, gridBox)))) {
         return { reason: "topology" };
       }
       // Vector paint makes an otherwise fully observed text page "partial"
@@ -2225,7 +2236,8 @@ function tryBuildRectGrid(page, run, clusters, itemById, allowHeaderlessTables =
         }
       }
       // Keep empty outer slots rather than trimming source grid columns.
-      return { kind: "table", grid, rows: candidateRows, headerless: true };
+      return { kind: "table", grid, rows: candidateRows, headerless: true,
+        leadingRows: run.slice(0, first) };
     }
     return { kind: "table", grid: trimmedGrid, rows: candidateRows };
   }
@@ -2670,10 +2682,12 @@ function segmentTextRows(page, rows, itemById, ruledClusters, allowHeaderlessTab
         ? tryBuildRectGrid(page, run, ruledClusters, itemById, allowHeaderlessTables)
         : null;
       if (ruling?.kind === "table") {
-        segments.push({ kind: "table", rows: run, grid: ruling.grid, ...(ruling.headerless ? { headerless: true } : {}) });
+        if (ruling.leadingRows?.length) segments.push({ kind: "text", rows: ruling.leadingRows });
+        segments.push({ kind: "table", rows: ruling.headerless ? ruling.rows : run,
+          grid: ruling.grid, ...(ruling.headerless ? { headerless: true } : {}) });
         if (ruling.headerless) {
           tableReason = "header";
-          regions.push({ run, reason: "header" });
+          regions.push({ run: ruling.rows, reason: "header" });
         }
       } else {
         // Preserve the reason from the rect-evidence attempt. The text path

server/output-schemas.js

@@ -1503,7 +1503,7 @@ export const TOOL_SUCCESS_OUTPUT_SCHEMAS = Object.freeze({
     {
       renderer: object({
         name: { const: "pdf-tools.layout-markdown-renderer" },
-        version: enumString(["1.21.0", "1.22.0"]),
+        version: enumString(["1.21.0", "1.23.0"]),
       }),
       conversion_status: enumString(["complete", "partial", "failed"]),
       markdown: string,